diff --git a/openaddr/conform.py b/openaddr/conform.py index 111486be..80f73e7a 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -716,18 +716,26 @@ def csv_source_to_csv(source_config, source_path, dest_path): def geojson_source_to_csv(source_config, source_path, dest_path): ''' ''' + # Not every feature shares the same set of properties, so make a first + # pass to collect the union of every feature's property keys (in + # first-seen order) before opening the CSV writer. + out_fieldnames = [] + seen_fieldnames = set() + with open(source_path) as file: + for feature in stream_geojson(file): + for key in feature['properties'].keys(): + if key not in seen_fieldnames: + seen_fieldnames.add(key) + out_fieldnames.append(key) + out_fieldnames.append(GEOM_FIELDNAME) + # For every row in the source GeoJSON with open(source_path) as file: # Write the extracted CSV file with open(dest_path, 'w', encoding='utf-8') as dest_fp: - writer = None + writer = csv.DictWriter(dest_fp, out_fieldnames) + writer.writeheader() for (row_number, feature) in enumerate(stream_geojson(file)): - if writer is None: - out_fieldnames = list(feature['properties'].keys()) - out_fieldnames.append(GEOM_FIELDNAME) - writer = csv.DictWriter(dest_fp, out_fieldnames) - writer.writeheader() - try: row = feature['properties'] if feature['geometry'] is None: diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index f35ef831..bec50520 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -2212,6 +2212,47 @@ def test_geojson_source_to_csv(self): self.assertEqual(row[GEOM_FIELDNAME], 'POINT (-74.9833483425103 40.05498715)') self.assertEqual(row['PARCEL_NUM'], '02-022-003') + def test_geojson_source_to_csv_non_uniform_properties(self): + ''' Features with different property keys should not crash the writer. + ''' + c = SourceConfig(dict({ + "schema": 2, + "layers": { + "addresses": [{ + "name": "default", + "conform": { } + }] + } + }), "addresses", "default") + + geojson_path = os.path.join(self.testdir, 'non-uniform.geojson') + with open(geojson_path, 'w', encoding='utf8') as file: + json.dump({ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {"pid": "1"}, + "geometry": {"type": "Point", "coordinates": [-121.2, 39.3]} + }, + { + "type": "Feature", + "properties": {"pid": "2", "address": "123 Maple St"}, + "geometry": {"type": "Point", "coordinates": [-121.3, 39.4]} + } + ] + }, file) + + csv_path = os.path.join(self.testdir, 'non-uniform-conformed.csv') + geojson_source_to_csv(c, geojson_path, csv_path) + + with open(csv_path, encoding='utf8') as file: + rows = list(csv.DictReader(file)) + self.assertEqual(rows[0]['pid'], '1') + self.assertEqual(rows[0]['address'], '') + self.assertEqual(rows[1]['pid'], '2') + self.assertEqual(rows[1]['address'], '123 Maple St') + class TestConformCsv(unittest.TestCase): "Fixture to create real files to test csv_source_to_csv()"