Add KML support to conform pipeline

openaddr/conform.py

@@ -427,6 +427,28 @@ def find_source_path(data_source, source_paths):
                     return c
             _L.warning("Source names file %s but could not find it", source_file_name)
             return None
+    elif format_string == "kml":
+        candidates = []
+        for fn in source_paths:
+            basename, ext = os.path.splitext(fn)
+            if ext.lower() == ".kml":
+                candidates.append(fn)
+        if len(candidates) == 0:
+            _L.warning("No KML found in %s", source_paths)
+            return None
+        elif len(candidates) == 1:
+            _L.debug("Selected %s for source", candidates[0])
+            return candidates[0]
+        else:
+            if "file" not in conform:
+                _L.warning("Multiple KML files found, but source has no file attribute.")
+                return None
+            source_file_name = conform["file"]
+            for c in candidates:
+                if source_file_name == os.path.basename(c):
+                    return c
+            _L.warning("Source names file %s but could not find it", source_file_name)
+            return None
     elif format_string == "xml":
         # Return file if it's specified, else return the first .gml file we find
         if "file" in conform:
@@ -1197,7 +1219,7 @@ def extract_to_source_csv(source_config, source_path, extract_path):
     format_string = source_config.data_source["conform"]['format']
     protocol_string = source_config.data_source['protocol']
 
-    if format_string in ("shapefile", "xml", "gdb", "gpkg"):
+    if format_string in ("shapefile", "xml", "gdb", "gpkg", "kml"):
         ogr_source_path = normalize_ogr_filename_case(source_path)
         ogr_source_to_csv(source_config, ogr_source_path, extract_path)
     elif format_string == "csv":
@@ -1240,7 +1262,7 @@ def conform_cli(source_config, source_path, dest_path):
 
     format_string = source_config.data_source["conform"].get('format')
 
-    if not format_string in ["shapefile", "geojson", "csv", "xml", "gdb", "gpkg"]:
+    if not format_string in ["shapefile", "geojson", "csv", "xml", "gdb", "gpkg", "kml"]:
         _L.warning("Skipping file with unknown conform: %s", source_path)
         return 1
 

openaddr/tests/conform.py

@@ -1806,6 +1806,36 @@ def test_lake_man_gpkg(self):
             self.assertEqual(rows[5]['properties']['number'], '5115')
             self.assertEqual(rows[5]['properties']['street'], 'OLD MILL RD')
 
+    def test_lake_man_kml(self):
+        with open(os.path.join(self.conforms_dir, "lake-man-kml.json")) as file:
+            source_config = SourceConfig(json.load(file), "addresses", "default")
+        source_path = os.path.join(self.conforms_dir, "lake-man.kml")
+        dest_path = os.path.join(self.testdir, 'lake-man-kml-conformed.csv')
+
+        rc = conform_cli(source_config, source_path, dest_path)
+        self.assertEqual(0, rc)
+
+        with open(dest_path) as fp:
+            rows = list(map(json.loads, list(fp)))
+
+            self.assertEqual('Point', rows[0]['geometry']['type'])
+            self.assertAlmostEqual(-122.2592497, rows[0]['geometry']['coordinates'][0], places=4)
+            self.assertAlmostEqual(37.8026126, rows[0]['geometry']['coordinates'][1], places=4)
+
+            self.assertEqual(6, len(rows))
+            self.assertEqual(rows[0]['properties']['number'], '5115')
+            self.assertEqual(rows[0]['properties']['street'], 'FRUITED PLAINS LN')
+            self.assertEqual(rows[1]['properties']['number'], '5121')
+            self.assertEqual(rows[1]['properties']['street'], 'FRUITED PLAINS LN')
+            self.assertEqual(rows[2]['properties']['number'], '5133')
+            self.assertEqual(rows[2]['properties']['street'], 'FRUITED PLAINS LN')
+            self.assertEqual(rows[3]['properties']['number'], '5126')
+            self.assertEqual(rows[3]['properties']['street'], 'FRUITED PLAINS LN')
+            self.assertEqual(rows[4]['properties']['number'], '5120')
+            self.assertEqual(rows[4]['properties']['street'], 'FRUITED PLAINS LN')
+            self.assertEqual(rows[5]['properties']['number'], '5115')
+            self.assertEqual(rows[5]['properties']['street'], 'OLD MILL RD')
+
     def test_lake_man_split(self):
         rc, dest_path = self._run_conform_on_source('lake-man-split', 'shp')
         self.assertEqual(0, rc)

openaddr/tests/conforms/lake-man-kml.json

@@ -0,0 +1,18 @@
+{
+    "schema": 2,
+    "layers": {
+        "addresses": [{
+            "name": "default",
+            "data": "http://fake-web/lake-man.kml",
+            "cache": "http://fake-cache/lake-man.kml",
+            "protocol": "http",
+            "conform": {
+                "lon": "X",
+                "lat": "Y",
+                "number": "NUMBER",
+                "street": "STRNAME",
+                "format": "kml"
+            }
+        }]
+    }
+}

openaddr/tests/conforms/lake-man.kml

@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<kml xmlns="http://www.opengis.net/kml/2.2">
+<Document>
+  <name>lake-man</name>
+  <Placemark>
+    <name>5115 FRUITED PLAINS LN</name>
+    <ExtendedData>
+      <Data name="NUMBER"><value>5115</value></Data>
+      <Data name="STRNAME"><value>FRUITED PLAINS LN</value></Data>
+    </ExtendedData>
+    <Point><coordinates>-122.259249687195,37.8026126376074</coordinates></Point>
+  </Placemark>
+  <Placemark>
+    <name>5121 FRUITED PLAINS LN</name>
+    <ExtendedData>
+      <Data name="NUMBER"><value>5121</value></Data>
+      <Data name="STRNAME"><value>FRUITED PLAINS LN</value></Data>
+    </ExtendedData>
+    <Point><coordinates>-122.256717681885,37.8025278661215</coordinates></Point>
+  </Placemark>
+  <Placemark>
+    <name>5133 FRUITED PLAINS LN</name>
+    <ExtendedData>
+      <Data name="NUMBER"><value>5133</value></Data>
+      <Data name="STRNAME"><value>FRUITED PLAINS LN</value></Data>
+    </ExtendedData>
+    <Point><coordinates>-122.257940769196,37.802968676786</coordinates></Point>
+  </Placemark>
+  <Placemark>
+    <name>5126 FRUITED PLAINS LN</name>
+    <ExtendedData>
+      <Data name="NUMBER"><value>5126</value></Data>
+      <Data name="STRNAME"><value>FRUITED PLAINS LN</value></Data>
+    </ExtendedData>
+    <Point><coordinates>-122.258970737457,37.8007476424409</coordinates></Point>
+  </Placemark>
+  <Placemark>
+    <name>5120 FRUITED PLAINS LN</name>
+    <ExtendedData>
+      <Data name="NUMBER"><value>5120</value></Data>
+      <Data name="STRNAME"><value>FRUITED PLAINS LN</value></Data>
+    </ExtendedData>
+    <Point><coordinates>-122.256953716278,37.800713733002</coordinates></Point>
+  </Placemark>
+  <Placemark>
+    <name>5115 OLD MILL RD</name>
+    <ExtendedData>
+      <Data name="NUMBER"><value>5115</value></Data>
+      <Data name="STRNAME"><value>OLD MILL RD</value></Data>
+    </ExtendedData>
+    <Point><coordinates>-122.257640361786,37.8043589085714</coordinates></Point>
+  </Placemark>
+</Document>
+</kml>