diff --git a/backend/config.py b/backend/config.py index d621a2a5..515318aa 100644 --- a/backend/config.py +++ b/backend/config.py @@ -26,6 +26,11 @@ NMBGMRWaterLevelSource, NMBGMRAnalyteSource, ) +from .connectors.ocotillo.source import ( + OcotilloSiteSource, + OcotilloWaterLevelSource, + OcotilloAnalyteSource, +) from .connectors.bor.source import BORSiteSource, BORAnalyteSource from .connectors.nmenv.source import DWBSiteSource, DWBAnalyteSource from .connectors.nmose.source import NMOSEPODSiteSource @@ -76,24 +81,24 @@ PARAMETER_SOURCE_MAP = { - WATERLEVELS: {"agencies": ["bernco", "cabq", "ebid", "nmbgmr_amp", "nmose_isc_seven_rivers", "nmose_roswell", "nwis", "pvacd", "wqp"]}, - CARBONATE: {"agencies": ["nmbgmr_amp", "wqp"]}, - ARSENIC: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "wqp"]}, - URANIUM: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "wqp"]}, - SPECIFIC_CONDUCTANCE: {"agencies": ["nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "wqp"]}, + WATERLEVELS: {"agencies": ["bernco", "cabq", "ebid", "nmbgmr_amp", "nmose_isc_seven_rivers", "nmose_roswell", "nwis", "ocotillo", "pvacd", "wqp"]}, + CARBONATE: {"agencies": ["nmbgmr_amp", "ocotillo", "wqp"]}, + ARSENIC: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "ocotillo", "wqp"]}, + URANIUM: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "ocotillo", "wqp"]}, + SPECIFIC_CONDUCTANCE: {"agencies": ["nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "ocotillo", "wqp"]}, CONDUCTIVITY: {"agencies": ["bor", "nmose_isc_seven_rivers", "wqp"]}, - BICARBONATE: {"agencies": ["nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "wqp"]}, - CALCIUM: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "wqp"]}, - CHLORIDE: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "wqp"]}, - FLUORIDE: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "wqp"]}, - MAGNESIUM: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "wqp"]}, - NITRATE: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "wqp"]}, - PH: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "wqp"]}, - POTASSIUM: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "wqp"]}, - SILICA: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "wqp"]}, - SODIUM: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "wqp"]}, - SULFATE: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "wqp"]}, - TDS: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "wqp"]}, + BICARBONATE: {"agencies": ["nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "ocotillo", "wqp"]}, + CALCIUM: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "ocotillo", "wqp"]}, + CHLORIDE: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "ocotillo", "wqp"]}, + FLUORIDE: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "ocotillo", "wqp"]}, + MAGNESIUM: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "ocotillo", "wqp"]}, + NITRATE: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "ocotillo", "wqp"]}, + PH: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "ocotillo", "wqp"]}, + POTASSIUM: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "ocotillo", "wqp"]}, + SILICA: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "ocotillo", "wqp"]}, + SODIUM: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "ocotillo", "wqp"]}, + SULFATE: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "ocotillo", "wqp"]}, + TDS: {"agencies": ["bor", "nmbgmr_amp", "nmed_dwb", "nmose_isc_seven_rivers", "ocotillo", "wqp"]}, } SOURCE_DICT = { @@ -107,6 +112,7 @@ "nmose_pod": NMOSEPODSiteSource, "nmose_roswell": NMOSERoswellSiteSource, "nwis": NWISSiteSource, + "ocotillo": OcotilloSiteSource, "pvacd": PVACDSiteSource, "wqp": WQPSiteSource, } @@ -123,6 +129,7 @@ "nmose_isc_seven_rivers": (ISCSevenRiversSiteSource, ISCSevenRiversAnalyteSource), "nmbgmr_amp": (NMBGMRSiteSource, NMBGMRAnalyteSource), "nmed_dwb": (DWBSiteSource, DWBAnalyteSource), + "ocotillo": (OcotilloSiteSource, OcotilloAnalyteSource), } WATERLEVEL_SOURCE_PAIRS = { @@ -134,6 +141,7 @@ "bernco": (BernCoSiteSource, BernCoWaterLevelSource), "ebid": (EBIDSiteSource, EBIDWaterLevelSource), "cabq": (CABQSiteSource, CABQWaterLevelSource), + "ocotillo": (OcotilloSiteSource, OcotilloWaterLevelSource), "wqp": (WQPSiteSource, WQPWaterLevelSource), } @@ -180,6 +188,7 @@ class Config: use_source_nmose_pod: bool = True use_source_nmose_roswell: bool = True use_source_nwis: bool = True + use_source_ocotillo: bool = True use_source_pvacd: bool = True use_source_wqp: bool = True diff --git a/backend/connectors/ocotillo/__init__.py b/backend/connectors/ocotillo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/connectors/ocotillo/mappings.py b/backend/connectors/ocotillo/mappings.py new file mode 100644 index 00000000..0ba4ef6d --- /dev/null +++ b/backend/connectors/ocotillo/mappings.py @@ -0,0 +1,79 @@ +# =============================================================================== +# Copyright 2024 Jake Ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Ocotillo OGC API - Features (pygeoapi/PostGIS) connector mappings. + +The Ocotillo API (ocotillo-api.newmexicowaterdata.org/ogcapi) is intended as a +replacement for the NMBGMR AMP API. Unlike AMP, it does NOT expose raw +per-observation time series. It only publishes "latest" / "summary" snapshot +collections. Therefore this connector supports SUMMARY output only; time series +output is unsupported (see OcotilloWaterLevelSource/OcotilloAnalyteSource). + +Because the API pre-aggregates, several DIG summary columns cannot be populated +and are intentionally left null (per project decision): mean, earliest_* for all +parameters; nrecords/min/max for chemistry and TDS (only water levels expose +count/min/max via water_well_summary). +""" +from backend.constants import ( + ARSENIC, + BICARBONATE, + CALCIUM, + CARBONATE, + CHLORIDE, + FLUORIDE, + MAGNESIUM, + NITRATE, + PH, + POTASSIUM, + SILICA, + SODIUM, + SPECIFIC_CONDUCTANCE, + SULFATE, + TDS, + URANIUM, +) + +# Collection ids on the Ocotillo OGC API +SITE_COLLECTION = "water_wells" +WATERLEVEL_SUMMARY_COLLECTION = "water_well_summary" +MAJOR_CHEMISTRY_COLLECTION = "major_chemistry_results" +MINOR_CHEMISTRY_COLLECTION = "minor_chemistry_wells" +TDS_COLLECTION = "latest_tds_wells" + +# DIG analyte -> (Ocotillo collection, property/column holding the latest value). +# Major-ion / field chemistry live in major_chemistry_results as static analyte +# columns; trace metals live in minor_chemistry_wells; TDS has its own +# latest_tds_wells collection with bespoke field names (handled in the source). +OCOTILLO_ANALYTE_MAPPING: dict = { + CALCIUM: (MAJOR_CHEMISTRY_COLLECTION, "calcium"), + MAGNESIUM: (MAJOR_CHEMISTRY_COLLECTION, "magnesium"), + SODIUM: (MAJOR_CHEMISTRY_COLLECTION, "sodium"), + POTASSIUM: (MAJOR_CHEMISTRY_COLLECTION, "potassium"), + BICARBONATE: (MAJOR_CHEMISTRY_COLLECTION, "bicarbonate"), + CARBONATE: (MAJOR_CHEMISTRY_COLLECTION, "carbonate"), + SULFATE: (MAJOR_CHEMISTRY_COLLECTION, "sulfate"), + CHLORIDE: (MAJOR_CHEMISTRY_COLLECTION, "chloride"), + NITRATE: (MAJOR_CHEMISTRY_COLLECTION, "nitrate"), + FLUORIDE: (MAJOR_CHEMISTRY_COLLECTION, "fluoride"), + SILICA: (MAJOR_CHEMISTRY_COLLECTION, "silica"), + PH: (MAJOR_CHEMISTRY_COLLECTION, "ph"), + SPECIFIC_CONDUCTANCE: (MAJOR_CHEMISTRY_COLLECTION, "specific_conductance"), + ARSENIC: (MINOR_CHEMISTRY_COLLECTION, "arsenic"), + URANIUM: (MINOR_CHEMISTRY_COLLECTION, "uranium"), + TDS: (TDS_COLLECTION, "latest_tds_value"), +} + +# ============= EOF ============================================= diff --git a/backend/connectors/ocotillo/source.py b/backend/connectors/ocotillo/source.py new file mode 100644 index 00000000..fa9bd1b6 --- /dev/null +++ b/backend/connectors/ocotillo/source.py @@ -0,0 +1,178 @@ +# =============================================================================== +# Copyright 2024 Jake Ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Ocotillo OGC API - Features connector. + +Ocotillo (ocotillo-api.newmexicowaterdata.org/ogcapi) is a pygeoapi/PostGIS +service intended to replace the NMBGMR AMP API. It publishes only pre-aggregated +"latest"/"summary" feature collections and exposes NO raw observation time +series. Consequently this connector produces SUMMARY output only; requesting +time series output logs a warning and yields nothing. + +Sites are seeded from the ``water_wells`` collection. Water-level summaries come +from ``water_well_summary`` (count/min/max/latest); chemistry and TDS come from +their respective latest-value collections. See mappings.py for details and for +the list of summary columns that are intentionally null. +""" +import os + +from backend.connectors import NM_STATE_BOUNDING_POLYGON +from backend.connectors.ocotillo.mappings import ( + OCOTILLO_ANALYTE_MAPPING, + SITE_COLLECTION, + WATERLEVEL_SUMMARY_COLLECTION, +) +from backend.connectors.ocotillo.transformer import ( + OcotilloSiteTransformer, + OcotilloWaterLevelTransformer, + OcotilloAnalyteTransformer, +) +from backend.source import ( + BaseSiteSource, + BaseWaterLevelSource, + BaseAnalyteSource, +) + +TIMEOUT = 15 * 60 +DEFAULT_URL = "https://ocotillo-api.newmexicowaterdata.org/ogcapi" + + +def _base_url(): + return os.getenv("OCOTILLO_URL", DEFAULT_URL).rstrip("/") + + +def _bbox_params(config): + """OGC API bbox filter (minx,miny,maxx,maxy) from the configured bounds. + Rectangular only; finer WKT/county filtering happens in the transformer's + geographic filter.""" + params = {} + if config.has_bounds(): + x1, y1, x2, y2 = config.bbox_bounding_points() + params["bbox"] = f"{x1},{y1},{x2},{y2}" + return params + + +def _fetch_all_features(source, collection, params): + """Page through an OGC API - Features collection, returning all features.""" + url = f"{_base_url()}/collections/{collection}/items" + limit = 1000 + offset = 0 + features = [] + while True: + page_params = {"f": "json", "limit": limit, "offset": offset} + page_params.update(params) + fc = source._execute_json_request(url, page_params, timeout=TIMEOUT) + page = fc.get("features", []) if isinstance(fc, dict) else [] + features.extend(page) + if len(page) < limit: + break + offset += limit + return features + + +class OcotilloSiteSource(BaseSiteSource): + chunk_size = 100 + bounding_polygon = NM_STATE_BOUNDING_POLYGON + + def __init__(self): + super().__init__(transformer=OcotilloSiteTransformer()) + + def __repr__(self): + return "OcotilloSiteSource" + + def health(self): + try: + url = f"{_base_url()}/collections/{SITE_COLLECTION}/items" + resp = self._execute_json_request(url, {"f": "json", "limit": 1}) + return bool(resp) + except Exception: + return False + + def get_records(self): + return _fetch_all_features(self, SITE_COLLECTION, _bbox_params(self.config)) + + +class _OcotilloSummaryParameterSource: + """Shared behavior for Ocotillo parameter sources. + + The whole (bbox-filtered) collection is fetched once and cached keyed by + well name; per-site extraction is a dict lookup. Summarization is a direct + passthrough to the transformer because the API already aggregated the data. + Time series output is unsupported. + """ + + def _get_collection(self): + raise NotImplementedError + + def get_records(self, site_record): + cache = getattr(self, "_feature_cache", None) + if cache is None: + cache = {} + for feature in _fetch_all_features( + self, self._get_collection(), _bbox_params(self.config) + ): + name = feature.get("properties", {}).get("name") + if name is not None: + cache[name] = feature + self._feature_cache = cache + return cache + + def _extract_site_records(self, records, site_record): + feature = records.get(site_record.id) + return [feature] if feature is not None else [] + + def _summarize_records(self, site, cleaned): + # cleaned is the single pre-aggregated feature for this site. + return self.transformer.do_transform(cleaned[0], site) + + def read_timeseries(self, site_record): + self.warn( + "Ocotillo source supports summary output only; the API exposes no " + "raw time series. Skipping timeseries output." + ) + return None + + +class OcotilloWaterLevelSource(_OcotilloSummaryParameterSource, BaseWaterLevelSource): + def __init__(self): + super().__init__(transformer=OcotilloWaterLevelTransformer()) + + def __repr__(self): + return "OcotilloWaterLevelSource" + + def _get_collection(self): + return WATERLEVEL_SUMMARY_COLLECTION + + +class OcotilloAnalyteSource(_OcotilloSummaryParameterSource, BaseAnalyteSource): + def __init__(self): + super().__init__(transformer=OcotilloAnalyteTransformer()) + + def __repr__(self): + return "OcotilloAnalyteSource" + + def _get_collection(self): + try: + collection, _column = OCOTILLO_ANALYTE_MAPPING[self.config.parameter] + except KeyError: + raise ValueError( + f"Ocotillo source does not provide parameter " + f"{self.config.parameter!r}. Valid: {sorted(OCOTILLO_ANALYTE_MAPPING)}" + ) + return collection + + +# ============= EOF ============================================= diff --git a/backend/connectors/ocotillo/transformer.py b/backend/connectors/ocotillo/transformer.py new file mode 100644 index 00000000..e65bb6d7 --- /dev/null +++ b/backend/connectors/ocotillo/transformer.py @@ -0,0 +1,188 @@ +# =============================================================================== +# Copyright 2024 Jake Ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +from backend.constants import DTW, FEET, TDS +from backend.transformer import ( + SiteTransformer, + WaterLevelTransformer, + AnalyteTransformer, + standardize_datetime, +) +from backend.connectors.ocotillo.mappings import OCOTILLO_ANALYTE_MAPPING + +SOURCE_TAG = "NMBGMR-Ocotillo" + + +class OcotilloSiteTransformer(SiteTransformer): + def _transform(self, record): + props = record["properties"] + lon, lat = record["geometry"]["coordinates"][:2] + rec = { + "source": SOURCE_TAG, + "id": props["name"], + "name": props["name"], + "latitude": lat, + "longitude": lon, + # water_wells carries no elevation, datum, or vertical datum. + # Coordinates are published in WGS84 (GeoJSON default). + "elevation": None, + "elevation_units": "", + "horizontal_datum": "WGS84", + "vertical_datum": "", + "usgs_site_id": "", + "alternate_site_id": props.get("nma_pk_welldata") or "", + "formation": props.get("nma_formation_zone") or "", + "well_depth": props.get("well_depth"), + "well_depth_units": FEET, + } + return rec + + +class _OcotilloSummaryTransformer: + """Mixin: build a SummaryRecord dict directly from a pre-aggregated Ocotillo + feature. Ocotillo publishes no raw observations, so the usual summarize path + (which needs the full record set and both earliest+latest) does not apply. + Only the latest value (and, for water levels, count/min/max) are available; + every other summary column is left null.""" + + source_tag = SOURCE_TAG + + def _summary_rec( + self, + site_record, + parameter, + out_units, + source_name, + latest_value, + source_units, + latest_datetime, + nrecords=None, + min_value=None, + max_value=None, + ): + def conv(v): + if v is None: + return None + value, _factor, warning = self.converter.convert( + v, + source_units or out_units, + out_units, + source_name, + parameter, + latest_datetime, + ) + if warning: + self.warn(f"{warning} for {site_record.id}") + return None + return value + + latest = conv(latest_value) + if latest is None: + # No usable latest value -> no summary for this site. + return None + + if latest_datetime: + latest_date, latest_time = standardize_datetime( + latest_datetime, site_record.id + ) + else: + latest_date, latest_time = None, None + + return { + "source": self.source_tag, + "id": site_record.id, + "name": site_record.name, + "usgs_site_id": site_record.usgs_site_id, + "alternate_site_id": site_record.alternate_site_id, + "latitude": site_record.latitude, + "longitude": site_record.longitude, + "horizontal_datum": site_record.horizontal_datum, + "elevation": site_record.elevation, + "elevation_units": site_record.elevation_units, + "well_depth": site_record.well_depth, + "well_depth_units": site_record.well_depth_units, + "parameter_name": parameter, + "parameter_units": out_units, + "nrecords": nrecords, + "min": conv(min_value), + "max": conv(max_value), + # Ocotillo does not expose mean or the earliest observation. + "mean": None, + "earliest_date": None, + "earliest_time": None, + "earliest_value": None, + "earliest_units": None, + "latest_date": latest_date, + "latest_time": latest_time, + "latest_value": latest, + "latest_units": out_units, + } + + +class OcotilloWaterLevelTransformer(_OcotilloSummaryTransformer, WaterLevelTransformer): + source_tag = SOURCE_TAG + + def _transform(self, record, site_record): + props = record["properties"] + _parameter, out_units = self._get_parameter_name_and_units() + return self._summary_rec( + site_record, + parameter=DTW, + out_units=out_units, + source_name="depth_to_water_bgs", + latest_value=props.get("last_water_level"), + source_units=FEET, + latest_datetime=props.get("last_water_level_datetime"), + nrecords=props.get("total_water_levels"), + min_value=props.get("min_water_level"), + max_value=props.get("max_water_level"), + ) + + +class OcotilloAnalyteTransformer(_OcotilloSummaryTransformer, AnalyteTransformer): + source_tag = SOURCE_TAG + + def _transform(self, record, site_record): + props = record["properties"] + parameter, out_units = self._get_parameter_name_and_units() + + if parameter == TDS: + value = props.get("latest_tds_value") + source_units = props.get("latest_tds_units") + latest_datetime = props.get("latest_tds_observation_date") + source_name = "tds" + else: + _collection, column = OCOTILLO_ANALYTE_MAPPING[parameter] + value = props.get(column) + source_units = props.get(f"{column}_units") + latest_datetime = props.get("latest_chemistry_date") + source_name = column + + return self._summary_rec( + site_record, + parameter=parameter, + out_units=out_units, + source_name=source_name, + latest_value=value, + source_units=source_units, + latest_datetime=latest_datetime, + # Chemistry/TDS collections expose only a single latest value. + nrecords=None, + min_value=None, + max_value=None, + ) + + +# ============= EOF ============================================= diff --git a/frontend/cli.py b/frontend/cli.py index be226cfb..b1bed071 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -104,6 +104,13 @@ def cli(): show_default=True, help="Exclude NWIS data. Default is to include", ), + click.option( + "--no-ocotillo", + is_flag=True, + default=False, + show_default=True, + help="Exclude Ocotillo (NMBGMR OGC API) data. Default is to include", + ), click.option( "--no-pvacd", is_flag=True, @@ -258,6 +265,7 @@ def weave( no_nmose_pod, no_nmose_roswell, no_nwis, + no_ocotillo, no_pvacd, no_wqp, site_limit, @@ -364,6 +372,7 @@ def sites( no_nmose_pod, no_nmose_roswell, no_nwis, + no_ocotillo, no_pvacd, no_wqp, site_limit, @@ -392,6 +401,7 @@ def sites( "nmose_isc_seven_rivers", "nmose_roswell", "nwis", + "ocotillo", "pvacd", "wqp", "nmose_pod", diff --git a/tests/test_sources/test_ocotillo.py b/tests/test_sources/test_ocotillo.py new file mode 100644 index 00000000..7fe68963 --- /dev/null +++ b/tests/test_sources/test_ocotillo.py @@ -0,0 +1,123 @@ +""" +Live integration tests for the Ocotillo OGC API - Features connector. + +Ocotillo is SUMMARY-ONLY: the API publishes pre-aggregated "latest"/"summary" +collections and exposes no raw observation time series. It therefore does not +fit the shared BaseSourceTestClass (which exercises time series output), so the +supported surface is tested directly here: + + * health check + * water-level summary (count/min/max/latest populated; mean/earliest null) + * analyte summary (latest value only; count/min/max/mean/earliest null) + * time series requests return nothing (graceful no-op) + +A small bounding box is used to keep the requests fast and deterministic. +""" +from pathlib import Path + +import pytest + +from backend.config import Config, SOURCE_KEYS +from backend.constants import WATERLEVELS, CALCIUM, MILLIGRAMS_PER_LITER, FEET +from backend.logger import setup_logging +from backend.record import SummaryRecord +from backend.unifier import unify_analytes, unify_waterlevels +from tests import recursively_clean_directory + +# Chaves/Eddy county area with several NMBGMR monitoring wells. +BBOX = "-104.15 32.55,-103.9 32.7" +SUMMARY_HEADERS = list(SummaryRecord.keys) + + +def _make_config(parameter): + config = Config() + for agency in SOURCE_KEYS: + setattr(config, f"use_source_{agency}", False) + config.use_source_ocotillo = True + config.parameter = parameter + config.bbox = BBOX + config.output_summary = True + config.finalize() + setup_logging(path=config.output_path) + return config + + +@pytest.fixture +def waterlevel_config(): + config = _make_config(WATERLEVELS) + yield config + recursively_clean_directory(Path(config.output_path)) + + +@pytest.fixture +def analyte_config(): + config = _make_config(CALCIUM) + yield config + recursively_clean_directory(Path(config.output_path)) + + +def _read_summary_rows(config): + summary_file = Path(config.output_path) / "summary.csv" + assert summary_file.exists() + with open(summary_file) as f: + lines = [ln.strip() for ln in f.readlines()] + headers = lines[0].split(",") + assert headers == SUMMARY_HEADERS + return [dict(zip(headers, ln.split(","))) for ln in lines[1:] if ln] + + +def test_health(): + config = _make_config(WATERLEVELS) + try: + source = config.all_site_sources()[0][0] + assert source.health() + finally: + recursively_clean_directory(Path(config.output_path)) + + +def test_waterlevel_summary(waterlevel_config): + unify_waterlevels(waterlevel_config) + rows = _read_summary_rows(waterlevel_config) + + assert rows, "expected at least one water-level summary row" + for row in rows: + assert row["source"] == "NMBGMR-Ocotillo" + assert row["parameter_name"] == "depth_to_water_below_ground_surface" + # latest is always populated; count/min/max come from water_well_summary + assert row["latest_value"] != "" + assert row["nrecords"] != "" + # Ocotillo exposes neither mean nor the earliest observation. + assert row["mean"] == "" + assert row["earliest_date"] == "" + assert row["earliest_value"] == "" + + +def test_analyte_summary(analyte_config): + unify_analytes(analyte_config) + rows = _read_summary_rows(analyte_config) + + assert rows, "expected at least one analyte summary row" + for row in rows: + assert row["source"] == "NMBGMR-Ocotillo" + assert row["parameter_name"] == CALCIUM + assert row["parameter_units"] == MILLIGRAMS_PER_LITER + assert row["latest_value"] != "" + # Chemistry collections carry only the latest value. + assert row["min"] == "" + assert row["max"] == "" + assert row["mean"] == "" + assert row["earliest_value"] == "" + + +def test_timeseries_unsupported(): + """Time series output is unsupported and must be a graceful no-op, not an + error or partial file.""" + config = _make_config(WATERLEVELS) + config.output_summary = False + config.output_timeseries_unified = True + try: + unify_waterlevels(config) + # No timeseries file is produced because the source yields no records. + assert not (Path(config.output_path) / "timeseries_unified.csv").exists() + finally: + recursively_clean_directory(Path(config.output_path))