diff --git a/CLAUDE.md b/CLAUDE.md index 5e5d14a06..a5f9358c7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -204,6 +204,17 @@ ArcGIS Pro cannot send a bearer token at all and neither desktop client can refresh an Authentik token. Read **`docs/internal-ogc-desktop-gis.md`** before changing the credential paths. +### OGC field descriptions + +Per-column `title`/`description`/unit for every collection lives in +`core/ogc-field-descriptions.yml`, keyed by backing relation, and is published +on `/schema` and `/queryables` through `core/feature_provider.py` and a wrapper +over pygeoapi's queryables handler. The feature leans on unpinned behaviour of +the pinned pygeoapi version — most sharply, `BaseProvider.fields` returns +`self._fields` and never calls `get_fields()`. Read +**`docs/ogc-field-descriptions.md`** before changing field metadata or +upgrading pygeoapi. + ### Database Configuration The application supports two database modes (configured via `DB_DRIVER` in `.env`): diff --git a/cli/generate_chemistry_field_descriptions.py b/cli/generate_chemistry_field_descriptions.py new file mode 100644 index 000000000..6ad1811dd --- /dev/null +++ b/cli/generate_chemistry_field_descriptions.py @@ -0,0 +1,269 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Emit the chemistry blocks of ``core/ogc-field-descriptions.yml``. + +``ogc_major_chemistry_results`` and ``ogc_minor_chemistry_wells`` publish one +column per analyte plus a paired units column -- 190 columns between them. +Hand-writing that is error-prone, so this script generates it and the output is +reviewed and committed. Run it again when the analyte lists change: + + uv run python -m cli.generate_chemistry_field_descriptions > /tmp/chem.yml + +Source of truth is the analyte lists in the migration that builds the two +views, which are the column names themselves. (``core/parameter.json`` holds +only two field parameters, so the lexicon cannot supply this.) + +Analytes needing more than a one-line gloss are spelled out in ANALYTE_PROSE; +anything absent falls back to a generated title and a stock description. Prose +here loses to a hand-written entry in the YAML, which wins on merge. +""" + +import importlib.util +import sys +import textwrap +from pathlib import Path + +MIGRATION = ( + Path(__file__).resolve().parents[1] + / "alembic/versions/f4a5b6c7d8e9_apply_public_release_status_filter_to_ogc_views.py" +) + +# Analyte key -> (title, description). Everything else gets a generated title +# and the stock "dissolved concentration" line. +ANALYTE_PROSE = { + "tds": ( + "Total dissolved solids", + "Total mass of dissolved mineral matter in the water -- in plain terms, " + "how salty it is. Drinking-water guidance sits around 500 mg/L.", + ), + "ph": ( + "pH", + "Acidity of the water on the 0-14 scale, where 7 is neutral. Unitless. " + "Most New Mexico groundwater falls between 7 and 8.5.", + ), + "specific_conductance": ( + "Specific conductance", + "How well the water conducts electricity, which rises with dissolved " + "mineral content. Used as a fast field proxy for total dissolved solids.", + ), + "hardness": ( + "Hardness", + "Combined calcium and magnesium content, reported as an equivalent mass " + "of calcium carbonate. What determines whether water is 'hard'.", + ), + "alkalinity": ( + "Alkalinity", + "The water's capacity to neutralise acid, reported as an equivalent mass " + "of calcium carbonate. Mostly supplied by bicarbonate and carbonate.", + ), + "ion_balance": ( + "Ion balance", + "Percentage difference between the total positive and total negative " + "charge in the analysis. Charge must balance in reality, so a figure far " + "from zero means the analysis is incomplete or in error.", + ), + "total_cations": ( + "Total cations", + "Sum of the positively charged dissolved constituents in the analysis.", + ), + "total_anions": ( + "Total anions", + "Sum of the negatively charged dissolved constituents in the analysis.", + ), + "sodium_plus_potassium": ( + "Sodium plus potassium", + "Combined sodium and potassium concentration, reported together where the " + "laboratory did not separate them.", + ), + "nitrate": ( + "Nitrate", + "Dissolved nitrate concentration, usually from fertiliser, septic systems, " + "or livestock. The drinking-water limit is 10 mg/L as nitrogen.", + ), + "nitrate_as_n": ( + "Nitrate as nitrogen", + "Nitrate concentration expressed as the mass of nitrogen alone, which is " + "how the 10 mg/L drinking-water limit is written. Roughly a quarter of the " + "same sample reported as nitrate.", + ), + "nitrite": ( + "Nitrite", + "Dissolved nitrite concentration, an intermediate stage in the breakdown of " + "nitrogen compounds.", + ), + "silica": ( + "Silica", + "Dissolved silica concentration, weathered out of silicate rock. Useful for " + "estimating the temperature water last equilibrated at.", + ), + "arsenic": ( + "Arsenic", + "Dissolved arsenic concentration. Naturally elevated in parts of New Mexico " + "and regulated in drinking water at 0.010 mg/L.", + ), + "uranium": ( + "Uranium", + "Dissolved uranium concentration. Naturally present near uranium-bearing " + "rock and regulated in drinking water at 0.030 mg/L.", + ), + "fluoride": ( + "Fluoride", + "Dissolved fluoride concentration. Beneficial in small amounts; the " + "drinking-water limit is 4 mg/L.", + ), + "h2r": ( + "Deuterium ratio", + "Ratio of heavy to ordinary hydrogen in the water, reported as per-mil " + "difference from ocean water. Fingerprints where the water fell as " + "precipitation.", + ), + "o18r": ( + "Oxygen-18 ratio", + "Ratio of heavy to ordinary oxygen in the water, reported as per-mil " + "difference from ocean water. Read with the deuterium ratio to trace the " + "water's origin and evaporation history.", + ), + "c13r": ( + "Carbon-13 ratio", + "Ratio of carbon-13 to carbon-12 in the water's dissolved carbon, reported " + "as per-mil difference from a standard. Helps identify where the carbon " + "came from, which is needed to correct a carbon-14 age.", + ), + "c14": ( + "Carbon-14", + "Carbon-14 remaining in the water's dissolved carbon, as a percentage of " + "the modern atmospheric level. The basis for dating groundwater up to " + "roughly 40,000 years old.", + ), + "c14_years": ( + "Carbon-14 age", + "Apparent age of the water in years, calculated from its carbon-14 content. " + "Uncorrected for carbon picked up from rock, so treat it as an upper bound.", + ), + "bromide": ( + "Bromide", + "Dissolved bromide concentration. Read against chloride, it distinguishes " + "seawater-derived salinity from dissolved halite.", + ), +} + +# Elements whose column name is not the plain element name. +ELEMENT_NAMES = { + "silicon": "silicon", + "molybdenum": "molybdenum", + "strontium": "strontium", +} + +STOCK_DESCRIPTION = ( + "Dissolved {name} concentration in the most recent sample analysed for it." +) +TOTAL_DESCRIPTION = ( + "Total {name} concentration -- the unfiltered determination, which counts " + "{name} bound to suspended particles as well as the dissolved fraction." +) +UNITS_DESCRIPTION = ( + "Units the {title_lower} value is reported in, as the laboratory recorded them." +) + + +def _load_analyte_lists(): + """Import the migration module by path and read its analyte column lists.""" + spec = importlib.util.spec_from_file_location("_ogc_filter_migration", MIGRATION) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return ( + [key for key, _ in module.STATIC_ANALYTE_COLUMNS_MAJOR], + [key for key, _ in module.STATIC_ANALYTE_COLUMNS_MINOR], + ) + + +def _entry(analyte_key: str): + if analyte_key in ANALYTE_PROSE: + return ANALYTE_PROSE[analyte_key] + + if analyte_key.endswith("_total"): + base = analyte_key[: -len("_total")] + name = ELEMENT_NAMES.get(base, base).replace("_", " ") + title = f"{name.capitalize()} (total)" + return title, TOTAL_DESCRIPTION.format(name=name) + + name = ELEMENT_NAMES.get(analyte_key, analyte_key).replace("_", " ") + return name.capitalize(), STOCK_DESCRIPTION.format(name=name) + + +def _yaml_block(field: str, title: str, description: str) -> str: + body = textwrap.fill( + description, + width=74, + initial_indent=" " * 6, + subsequent_indent=" " * 6, + break_on_hyphens=False, + break_long_words=False, + ) + return f" {field}:\n title: {title}\n description: >-\n{body}\n" + + +def render(table: str, analyte_keys) -> str: + lines = [f"{table}:"] + lines.append( + _yaml_block( + "location_id", + "Location ID", + "Identifier of the location record the well's coordinates came from.", + ) + ) + lines.append( + _yaml_block( + "analyte_count", + "Analyte count", + "Number of distinct analytes with a value in this row. A low count " + "means the well has only been analysed for part of the suite.", + ) + ) + lines.append( + _yaml_block( + "latest_chemistry_date", + "Latest analysis date", + "Date of the most recent result in this row. Analytes are carried " + "forward independently, so an individual value may be older than " + "this date.", + ) + ) + for key in analyte_keys: + title, description = _entry(key) + lines.append(_yaml_block(key, title, description)) + lines.append( + _yaml_block( + f"{key}_units", + f"{title} units", + UNITS_DESCRIPTION.format(title_lower=title.lower()), + ) + ) + return "\n".join(lines) + + +def main() -> int: + major, minor = _load_analyte_lists() + print( + "# Generated by cli/generate_chemistry_field_descriptions.py -- review before committing." + ) + print(render("major_chemistry_results", major)) + print(render("minor_chemistry_wells", minor)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/core/edr_provider.py b/core/edr_provider.py index 35815d089..eba54880d 100644 --- a/core/edr_provider.py +++ b/core/edr_provider.py @@ -46,6 +46,8 @@ ) from pygeoapi.provider.base_edr import BaseEDRProvider +from core.ogc_field_metadata import describe_fields, table_entries + LOGGER = logging.getLogger(__name__) GEOGRAPHIC_CRS = { @@ -144,9 +146,16 @@ def _has_column(self, column): # -------------------------------------------------------------- fields def get_fields(self): - """Return the parameter-name fields present in the backing view.""" + """Return the parameter-name fields present in the backing view. + + Each call hands back fresh per-field dicts. pygeoapi's + get_collection_schema mutates what a provider returns in place -- + popping ``format``, assigning ``x-ogc-role`` -- so returning the + cached dicts themselves would let one request's edits accumulate on + the next one's response. + """ if self._fields: - return self._fields + return {name: dict(field) for name, field in self._fields.items()} try: rows = self._fetch( f"SELECT DISTINCT parameter_name, unit " # noqa: S608 (trusted table) @@ -161,7 +170,11 @@ def get_fields(self): "title": row["parameter_name"], "x-ogc-unit": row["unit"], } - return self._fields + # Same prose source as the feature collections, keyed by parameter + # name rather than column name. Parameter names are read out of the + # data, so an undocumented analyte keeps its generated title. + self._fields = describe_fields(self.table, self._fields) + return {name: dict(field) for name, field in self._fields.items()} @property def fields(self): @@ -344,6 +357,10 @@ def _read( ) # ------------------------------------------------------- coveragejson + def _parameter_documentation(self, parameter_name): + """Documented title/description for one EDR parameter, or ``{}``.""" + return table_entries(self.table).get(parameter_name, {}) + def _coverage_collection(self, rows): if not rows: raise ProviderNoDataError("No data found") @@ -355,10 +372,17 @@ def _coverage_collection(self, rows): stations.setdefault(row["thing_id"], []).append(row) name = row["parameter_name"] if name not in parameters: + # A CoverageJSON client reads observedProperty.label for the + # display name and description for the explanation; both were + # the raw parameter name before the field metadata existed. + entry = self._parameter_documentation(name) parameters[name] = { "type": "Parameter", - "description": {"en": name}, - "observedProperty": {"id": name, "label": {"en": name}}, + "description": {"en": entry.get("description", name)}, + "observedProperty": { + "id": name, + "label": {"en": entry.get("title", name)}, + }, "unit": {"symbol": row["unit"], "label": {"en": row["unit"]}}, } diff --git a/core/feature_provider.py b/core/feature_provider.py new file mode 100644 index 000000000..e13751a38 --- /dev/null +++ b/core/feature_provider.py @@ -0,0 +1,58 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Feature provider that publishes field-level prose alongside the columns. + +pygeoapi's PostgreSQL provider reflects a table and reports each column's +JSON Schema type and format. It does not read column comments, and there is +no hook for documentation, so /collections/{id}/schema publishes bare column +names. This subclass annotates the reflected fields from +core/ogc-field-descriptions.yml on the way out. + +Read docs/ogc-field-descriptions.md before changing this. +""" + +import logging + +from pygeoapi.provider.sql import PostgreSQLProvider + +from core.ogc_field_metadata import describe_fields + +LOGGER = logging.getLogger(__name__) + + +class DescribedPostgreSQLProvider(PostgreSQLProvider): + """PostgreSQLProvider that annotates reflected columns with prose.""" + + def get_fields(self): + """Reflect the table, then annotate the result. + + The write back into ``self._fields`` is the point of this method, not + an optimisation. ``BaseProvider.fields`` -- which is what + ``get_collection_schema`` and ``get_collection_queryables`` actually + read -- returns ``self._fields`` directly and never calls + ``get_fields()``. A subclass that only returned an annotated copy + would be silently ignored, since ``GenericSQLProvider.__init__`` + populates ``_fields`` with the raw reflection at construction. + """ + fields = super().get_fields() + if fields and not getattr(self, "_fields_described", False): + self._fields = describe_fields(self.table, fields) + # super().get_fields() short-circuits on a populated _fields, so + # without this flag a later call would re-describe the annotated + # dict. Harmless today (describe_fields is idempotent) but it + # would quietly depend on that staying true. + self._fields_described = True + return self._fields diff --git a/core/ogc-field-descriptions.yml b/core/ogc-field-descriptions.yml new file mode 100644 index 000000000..2c83681fe --- /dev/null +++ b/core/ogc-field-descriptions.yml @@ -0,0 +1,2068 @@ +# Per-field documentation for the OGC collections. +# +# Keyed by backing relation with the ogc_/ogc_internal_ prefix stripped, so the +# public and internal mounts share one entry per view. `_defaults` applies to +# every table; a per-table entry wins over it. +# +# Allowed keys per field: title, description, x-ogc-unit, x-ogc-unitLang, +# x-ogc-propertySeq. Types and formats come from provider reflection, never +# from here. +# +# Say what the value means and what its datum or convention is -- not how the +# view is assembled. That belongs in the collection description. +# +# See docs/ogc-field-descriptions.md. + +_defaults: + id: + title: Feature ID + description: >- + Stable identifier for this feature within the collection. Unique inside + the collection, not across collections. + name: + title: Name + description: >- + Name or identifier the monitoring point is known by, as recorded by the + Bureau. + thing_type: + title: Feature type + description: >- + Controlled-vocabulary type of the monitoring point, such as water well, + spring, or meteorological station. + enum-lexicon: thing_type + release_status: + title: Release status + description: >- + Publication state of the record. Only records marked public appear on + the public /ogcapi mount; the authenticated internal mount also carries + private and draft records. + enum-lexicon: release_status + first_visit_date: + title: First visit date + description: Date of the earliest Bureau visit on record for this feature. + nma_pk_welldata: + title: Legacy NM_Aquifer well key + description: >- + Primary key of this feature's record in the legacy NM_Aquifer WellData + table, kept so migrated rows can be traced back to their source. + elevation: + title: Ground-surface elevation + description: >- + Surveyed elevation of the ground surface at the feature, in metres above + the NAVD 88 vertical datum. + x-ogc-unit: https://qudt.org/vocab/unit/M + x-ogc-unitLang: QUDT + well_depth: + title: Well depth + description: >- + Total depth of the finished well, from ground surface to the bottom of + the well. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + hole_depth: + title: Borehole depth + description: >- + Depth of the drilled hole, from ground surface to the bottom of the + borehole. Usually deeper than the finished well. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + well_casing_diameter: + title: Casing diameter + description: Inside diameter of the well casing. + x-ogc-unit: https://qudt.org/vocab/unit/IN + x-ogc-unitLang: QUDT + well_casing_depth: + title: Casing depth + description: >- + Depth from ground surface to the bottom of the well casing. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + well_completion_date: + title: Completion date + description: Date the well was finished, where it is known. + well_driller_name: + title: Driller + description: Name of the driller or drilling company that constructed the well. + well_construction_method: + title: Construction method + description: >- + How the well was constructed, such as air rotary, cable tool, or dug, + from a controlled vocabulary. + enum-lexicon: well_construction_method + well_pump_type: + title: Pump type + description: >- + Type of pump installed in the well, such as submersible or windmill, + from a controlled vocabulary. + enum-lexicon: well_pump_type + well_pump_depth: + title: Pump intake depth + description: Depth from ground surface to the pump intake. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + formation_completion_code: + title: Completion formation + description: >- + Geologic formation the well is completed in -- the formation it draws + from, not the full sequence of rock it passes through. + nma_formation_zone: + title: Legacy formation zone + description: >- + Formation zone exactly as recorded in the legacy NM_Aquifer WellData + table, kept unedited alongside the controlled-vocabulary value. + county: + title: County + description: New Mexico county the feature falls in. + state: + title: State + description: State the feature falls in. + api: + title: API number + description: >- + American Petroleum Institute well number, the standard unique identifier + for a drilled well in the United States. + well_name: + title: Well name + description: Name the well is recorded under in the legacy NM_Wells database. + well_num: + title: Well number + description: Operator's number for the well within its lease or unit. + well_data_id: + title: Legacy NM_Wells well key + description: >- + Identifier of the well's record in the legacy NM_Wells database, kept so + rows can be traced back to their source. + total_depth: + title: Total depth + description: Total drilled depth of the well, as reported on its record. + source_id: + title: Source ID + description: >- + Identifier of the publication or data submission the record came from, + in the legacy NM_Wells source register. + entry_date: + title: Record entry date + description: Date the record was entered into the legacy NM_Wells database. + lat_dd83: + title: Latitude (NAD 83) + description: Latitude in decimal degrees on the NAD 83 datum. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + long_dd83: + title: Longitude (NAD 83) + description: Longitude in decimal degrees on the NAD 83 datum. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + lat_dd27: + title: Latitude (NAD 27) + description: >- + Latitude in decimal degrees on the older NAD 27 datum, as originally + recorded. Positions differ from NAD 83 by roughly 100 metres. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + long_dd27: + title: Longitude (NAD 27) + description: >- + Longitude in decimal degrees on the older NAD 27 datum, as originally + recorded. Positions differ from NAD 83 by roughly 100 metres. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + elev_gl: + title: Ground-level elevation + description: Elevation of the ground surface at the well head. + elev_kb: + title: Kelly bushing elevation + description: >- + Elevation of the kelly bushing, the point on the drilling rig that + drilled depths were measured from. Typically several metres above ground + level. + elev_unspc: + title: Elevation (unspecified datum) + description: >- + Elevation recorded without a stated reference point, so it may be ground + level or a drilling datum. + depth_unit: + title: Depth unit + description: Unit the depths on this record are reported in. + temp_unit: + title: Temperature unit + description: Unit the temperatures on this record are reported in. + +locations: + nma_pk_location: + title: Legacy NM_Aquifer location key + description: >- + Primary key of this site's record in the legacy NM_Aquifer Location + table, kept so migrated rows can be traced back to their source. + description: + title: Site description + description: Free-text description of the site. + quad_name: + title: USGS quadrangle + description: Name of the USGS 7.5-minute topographic quadrangle the site falls in. + nma_location_notes: + title: Location notes + description: >- + Notes about the site carried over from NM_Aquifer, typically covering + access and how to find it on the ground. + nma_coordinate_notes: + title: Coordinate notes + description: >- + Notes on how the coordinates were obtained -- GPS, digitised from a map, + or derived from a legal description. + nma_data_reliability: + title: Data reliability + description: >- + Legacy rating of how much confidence to place in the site's recorded + position. + nma_date_created: + title: Legacy record created + description: Date the site record was created in NM_Aquifer. + nma_site_date: + title: Site date + description: Date associated with the site itself in NM_Aquifer, where one was recorded. + +project_areas: + name: + title: Project name + description: Name of the Bureau project the area belongs to. + description: + title: Project description + description: Free-text description of the project. + group_type: + title: Group type + description: >- + Kind of grouping the record represents, such as a project or a + geographic area. + +water_well_summary: + elevation_method: + title: Elevation method + description: >- + How the ground-surface elevation was determined, such as GPS survey or + read from a digital elevation model. Governs how much precision the + elevation deserves. + enum-lexicon: collection_method + formation_zone: + title: Formation zone + description: Geologic formation the well draws from, as recorded for the well. + total_water_levels: + title: Water-level measurement count + description: >- + Number of manual groundwater-level measurements behind this row's + statistics. Small counts make the range and trend unreliable. + last_water_level: + title: Latest water level + description: >- + Most recent groundwater-level measurement, as a depth below ground + surface. Reported in the units of the source reading, which is feet for + almost the whole record; unlike water_elevation_wells this layer does + not convert metric readings. + last_water_level_datetime: + title: Latest measurement time + description: Date and time of the most recent groundwater-level measurement. + min_water_level: + title: Shallowest water level + description: >- + Smallest depth below ground surface on record -- the high-water mark, + since a smaller depth means water nearer the surface. + max_water_level: + title: Deepest water level + description: >- + Largest depth below ground surface on record -- the low-water mark, + since a larger depth means water further down. + water_level_trend_ft_per_year: + title: Water-level trend + description: >- + Slope of a straight line fitted through the well's depth-to-water + measurements over time, in feet per year. Positive means depth is + increasing, so the water table is falling; negative means it is rising. + x-ogc-unit: https://qudt.org/vocab/unit/FT-PER-YR + x-ogc-unitLang: QUDT + +actively_monitored_wells: + elevation_method: + title: Elevation method + description: >- + How the ground-surface elevation was determined, such as GPS survey or + read from a digital elevation model. + enum-lexicon: collection_method + formation_zone: + title: Formation zone + description: Geologic formation the well draws from, as recorded for the well. + total_water_levels: + title: Water-level measurement count + description: Number of manual groundwater-level measurements on record for the well. + last_water_level: + title: Latest water level + description: >- + Most recent groundwater-level measurement, as a depth below ground + surface, in the units of the source reading. + last_water_level_datetime: + title: Latest measurement time + description: Date and time of the most recent groundwater-level measurement. + min_water_level: + title: Shallowest water level + description: Smallest depth below ground surface on record for the well. + max_water_level: + title: Deepest water level + description: Largest depth below ground surface on record for the well. + water_level_trend_ft_per_year: + title: Water-level trend + description: >- + Slope of a straight line fitted through the well's depth-to-water + measurements over time, in feet per year. Positive means the water table + is falling. + x-ogc-unit: https://qudt.org/vocab/unit/FT-PER-YR + x-ogc-unitLang: QUDT + group_id: + title: Network ID + description: Identifier of the monitoring network the well belongs to. + group_name: + title: Network name + description: >- + Name of the monitoring network the well belongs to. Always the Water + Level Network in this collection. + group_type: + title: Network type + description: Kind of grouping the network record represents. + +depth_to_water_trend_wells: + record_count: + title: Measurement count + description: >- + Number of groundwater-level measurements the trend was fitted to. Below + 10 measurements -- or below 4 spanning less than two years -- the trend + is reported as not enough data. + first_observation_datetime: + title: First measurement time + description: Date and time of the earliest measurement used in the fit. + last_observation_datetime: + title: Latest measurement time + description: Date and time of the most recent measurement used in the fit. + span_years: + title: Record span + description: >- + Years between the first and last measurement used in the fit. A steep + slope over a short span is weak evidence of a real trend. + x-ogc-unit: https://qudt.org/vocab/unit/YR + x-ogc-unitLang: QUDT + slope_ft_per_year: + title: Trend slope + description: >- + Slope of a straight line fitted through depth to water below ground + surface over time, in feet per year. Positive means depth is increasing, + so the water table is falling. + x-ogc-unit: https://qudt.org/vocab/unit/FT-PER-YR + x-ogc-unitLang: QUDT + trend_category: + title: Trend category + description: >- + Plain-language reading of the slope: increasing (water table falling + faster than 0.25 ft/yr), decreasing (rising faster than 0.25 ft/yr), + stable, or not enough data. + enum: [increasing, decreasing, stable, not enough data] + +water_elevation_wells: + observation_id: + title: Measurement ID + description: Identifier of the groundwater-level measurement this row was calculated from. + observation_datetime: + title: Measurement time + description: Date and time the groundwater level was measured. + elevation_m: + title: Ground-surface elevation + description: >- + Surveyed elevation of the ground surface at the well, in metres above + the NAVD 88 vertical datum. + x-ogc-unit: https://qudt.org/vocab/unit/M + x-ogc-unitLang: QUDT + depth_to_water_below_ground_surface_ft: + title: Depth to water + description: >- + Distance from ground surface down to the water table at the time of + measurement. Metric readings are converted to feet, and a reading with + no recorded measuring-point height is treated as taken at ground level. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + water_elevation_ft: + title: Water-table elevation + description: >- + Height of the water table above sea level: ground-surface elevation + converted to feet, minus the depth to water. Comparable between wells + standing at different ground elevations. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + +latest_depth_to_water_wells: + observation_id: + title: Measurement ID + description: Identifier of the groundwater-level measurement this row reports. + observation_datetime: + title: Measurement time + description: Date and time the groundwater level was measured. + depth_to_water_reference: + title: Depth to water from reference point + description: >- + Depth to water as read in the field, measured down from the measuring + point rather than from the ground. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + measuring_point_height: + title: Measuring-point height + description: >- + Height of the measuring point -- usually the top of the well casing -- + above ground surface. Subtracted from the field reading to give a depth + below ground surface; a reading with no recorded height is treated as + taken at ground level. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + depth_to_water_bgs: + title: Depth to water below ground surface + description: >- + Distance from ground surface down to the water table, after subtracting + the measuring-point height from the field reading. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + +latest_tds_wells: + major_chemistry_id: + title: Analysis ID + description: Identifier of the laboratory result this row reports. + latest_tds_observation_date: + title: Analysis date + description: >- + Date the reported TDS result was analysed, or the date the sample was + collected where no analysis date was recorded. + latest_tds_value: + title: Total dissolved solids + description: >- + Most recent measured concentration of dissolved mineral matter in the + water. Higher values mean saltier water; drinking-water guidance sits + around 500 mg/L. + latest_tds_units: + title: TDS units + description: >- + Units the TDS value is reported in, as the laboratory recorded them -- + usually milligrams per litre. + +avg_tds_wells: + tds_observation_count: + title: Analysis count + description: >- + Number of TDS results the average was taken over. Across the catalog + this averages about 1.9, so most rows average one or two samples. + avg_tds_value: + title: Average total dissolved solids + description: >- + Arithmetic mean of every TDS result on record for the well, without + weighting by date. Read alongside the analysis count before relying on + it. + first_tds_observation_date: + title: First analysis date + description: Date of the earliest TDS result included in the average. + last_tds_observation_date: + title: Latest analysis date + description: Date of the most recent TDS result included in the average. + +geothermal_wells_bht: + bht_count: + title: Reading count + description: Number of bottom-hole temperature readings recorded for the well. + max_bht: + title: Highest bottom-hole temperature (as recorded) + description: >- + Highest bottom-hole temperature on record for the well, in the units the + source recorded it in. Use max_bht_c for a comparable value. + min_bht: + title: Lowest bottom-hole temperature (as recorded) + description: >- + Lowest bottom-hole temperature on record for the well, in the units the + source recorded it in. Use min_bht_c for a comparable value. + max_bht_c: + title: Highest bottom-hole temperature + description: >- + Highest bottom-hole temperature on record for the well, converted to + degrees Celsius. + x-ogc-unit: https://qudt.org/vocab/unit/DEG_C + x-ogc-unitLang: QUDT + min_bht_c: + title: Lowest bottom-hole temperature + description: >- + Lowest bottom-hole temperature on record for the well, converted to + degrees Celsius. + x-ogc-unit: https://qudt.org/vocab/unit/DEG_C + x-ogc-unitLang: QUDT + max_bht_depth: + title: Deepest reading depth + description: Depth of the deepest bottom-hole temperature reading for the well. + temp_unit: + title: Temperature unit + description: >- + Unit the converted temperatures are reported in. Always Celsius; see + temp_unit_source for what the readings arrived as. + enum: [C] + temp_unit_source: + title: Source temperature units + description: >- + Units the underlying readings were recorded in, comma-separated where + the well's readings did not agree. + temp_unit_mixed: + title: Mixed source units + description: >- + True when the well's readings arrived in more than one temperature unit, + which is a sign the source record needs review. + unconvertible_count: + title: Unconvertible reading count + description: >- + Number of readings that could not be converted to Celsius because their + unit was missing or unrecognised. These are excluded from the minimum + and maximum. + +geothermal_wells_temperature_profile: + reading_count: + title: Reading count + description: Number of temperature-versus-depth readings logged in the well. + min_depth: + title: Shallowest reading depth + description: Depth of the shallowest temperature reading in the profile. + max_depth: + title: Deepest reading depth + description: Depth of the deepest temperature reading in the profile. + min_temp: + title: Lowest temperature (as recorded) + description: >- + Lowest temperature in the profile, in the units the source recorded it + in. Use min_temp_c for a comparable value. + max_temp: + title: Highest temperature (as recorded) + description: >- + Highest temperature in the profile, in the units the source recorded it + in. Use max_temp_c for a comparable value. + min_temp_c: + title: Lowest temperature + description: Lowest temperature in the profile, converted to degrees Celsius. + x-ogc-unit: https://qudt.org/vocab/unit/DEG_C + x-ogc-unitLang: QUDT + max_temp_c: + title: Highest temperature + description: Highest temperature in the profile, converted to degrees Celsius. + x-ogc-unit: https://qudt.org/vocab/unit/DEG_C + x-ogc-unitLang: QUDT + temp_unit: + title: Temperature unit + description: >- + Unit the converted temperatures are reported in. Always Celsius; see + temp_unit_source for what the readings arrived as. + enum: [C] + temp_unit_source: + title: Source temperature units + description: >- + Units the underlying readings were recorded in, comma-separated where + the well's readings did not agree. + temp_unit_mixed: + title: Mixed source units + description: >- + True when the well's readings arrived in more than one temperature unit, + which is a sign the source record needs review. + unconvertible_count: + title: Unconvertible reading count + description: >- + Number of readings that could not be converted to Celsius because their + unit was missing or unrecognised. + series: + title: Temperature-depth profile + description: >- + The whole profile as a list of readings, each carrying a depth, the + temperature as recorded, and the temperature converted to Celsius. + +bht_measurements: + operator: + title: Operator + description: Company operating the well when the record was made. + well_type: + title: Well type + description: Purpose the well was drilled for, such as oil, gas, or observation. + well_tvd: + title: True vertical depth + description: >- + Vertical depth of the well, which is shorter than the drilled length for + a deviated hole. + completion_date: + title: Completion date + description: Date the well was finished. + current_status: + title: Current status + description: Latest recorded status of the well, such as producing or plugged. + cuttings: + title: Cuttings held + description: >- + Whether rock cuttings from the well are held in the Bureau's subsurface + library. + core_exists: + title: Core held + description: Whether a rock core from the well is held in the Bureau's subsurface library. + bht_depth: + title: Reading depth + description: Depth the bottom-hole temperature was measured at. + bht: + title: Bottom-hole temperature + description: >- + Temperature measured at the bottom of the hole, in the units recorded by + the source. + hours_since_circulation: + title: Hours since circulation + description: >- + Time between the last circulation of drilling fluid and the reading. + Drilling fluid cools the rock, so a reading taken soon after circulation + is too low; this figure decides whether it can be corrected. + x-ogc-unit: https://qudt.org/vocab/unit/HR + x-ogc-unitLang: QUDT + date_measured: + title: Measurement date + description: Date the temperature was measured. + +temp_depth_measurements: + sample_fm: + title: Formation + description: Geologic formation at the depth the reading was taken. + loc_acc_val: + title: Location accuracy + description: Recorded accuracy of the well's coordinates. + entered_by: + title: Entered by + description: Person who entered the record into the legacy NM_Wells database. + depth: + title: Reading depth + description: Depth below the reference datum the temperature was measured at. + temp: + title: Temperature + description: Temperature measured at this depth, in the units recorded by the source. + sample_date: + title: Measurement date + description: Date the temperature was measured. + +heat_flow: + elevation_m: + title: Elevation + description: Well elevation converted to metres. + x-ogc-unit: https://qudt.org/vocab/unit/M + x-ogc-unitLang: QUDT + depth_units: + title: Depth units + description: Units the depths on this record were originally reported in. + total_depth_m: + title: Total depth (metres) + description: Total drilled depth of the well converted to metres. + x-ogc-unit: https://qudt.org/vocab/unit/M + x-ogc-unitLang: QUDT + from_depth: + title: Interval top + description: Depth to the top of the interval the determination covers. + to_depth: + title: Interval base + description: Depth to the bottom of the interval the determination covers. + therml_cond: + title: Thermal conductivity (as published) + description: >- + How readily the rock conducts heat, in the units it was published in. + Use tc_si for a comparable value. + tcond_range: + title: Thermal conductivity range + description: Published spread of conductivity values for the interval. + tcond_error: + title: Thermal conductivity error + description: Published uncertainty on the conductivity value. + tcond_unit: + title: Thermal conductivity unit + description: >- + Unit the published conductivity is in. TCU denotes the older thermal + conductivity unit, mcal/cm-s-degC. + tc_si: + title: Thermal conductivity + description: >- + Thermal conductivity converted to SI units, watts per metre-kelvin. + Typical rock sits between 1 and 5. + x-ogc-unit: https://qudt.org/vocab/unit/W-PER-M-K + x-ogc-unitLang: QUDT + sample_type: + title: Sample type + description: What the conductivity was measured on, such as core or cuttings. + num_samples: + title: Sample count + description: Number of samples the conductivity value was measured from. + therml_grad: + title: Thermal gradient + description: >- + How fast temperature rises with depth over the interval, in the units it + was published in. Continental crust averages roughly 25 degrees Celsius + per kilometre. + tgrad_range: + title: Thermal gradient range + description: Published spread of gradient values for the interval. + tg_error: + title: Thermal gradient error + description: Published uncertainty on the gradient value. + grad_unit: + title: Thermal gradient unit + description: Unit the published gradient is in. + heat_flow: + title: Heat flow (as published) + description: >- + Rate at which heat escapes through the ground over this interval, in the + units it was published in. Use heat_flow_si for a comparable value. + ht_flow_unit: + title: Heat flow unit + description: >- + Unit the published heat flow is in. HFU denotes the older heat flow + unit, equal to 41.84 milliwatts per square metre. + heat_flow_si: + title: Heat flow + description: >- + Heat flow converted to SI units, milliwatts per square metre. Continental + averages sit near 65; values well above that mark geothermal interest. + x-ogc-unit: https://qudt.org/vocab/unit/MilliW-PER-M2 + x-ogc-unitLang: QUDT + ht_flow_est: + title: Estimated heat flow (as published) + description: >- + Heat flow the author estimated rather than measured, in the units it was + published in. + ht_flow_est_si: + title: Estimated heat flow + description: >- + Author-estimated heat flow converted to milliwatts per square metre. + x-ogc-unit: https://qudt.org/vocab/unit/MilliW-PER-M2 + x-ogc-unitLang: QUDT + quality: + title: Quality rating + description: >- + The publication's own assessment of how much confidence the + determination deserves. + first_auth: + title: First author + description: First author of the publication the determination came from. + pub_year: + title: Publication year + description: Year the determination was published. + title: + title: Publication title + description: Title of the publication the determination came from. + journal: + title: Journal + description: Journal or report series the determination was published in. + volume: + title: Volume + description: Volume of the journal or report series. + page_no: + title: Pages + description: Page range of the publication. + +dst: + dst_name: + title: Test name + description: Name recorded for the drill stem test. + dst_operator: + title: Testing contractor + description: Company that ran the drill stem test. + dst_number: + title: Test number + description: Sequence number of this test within the well. + dst_date: + title: Test date + description: Date the drill stem test was run. + from_depth: + title: Interval top + description: Depth to the top of the tested interval. + to_depth: + title: Interval base + description: Depth to the bottom of the tested interval. + target_fm: + title: Target formation + description: Geologic formation the test was aimed at. + packer_from: + title: Upper packer depth + description: >- + Depth of the upper packer, the seal that isolates the tested interval + from the rest of the hole. + packer_to: + title: Lower packer depth + description: Depth of the lower packer sealing the bottom of the tested interval. + srf_choke_sz: + title: Surface choke size + description: >- + Size of the choke at surface, which limits how fast fluid is allowed to + flow during the test. + bot_choke_sz: + title: Bottom choke size + description: Size of the choke at the bottom of the string. + prs_gage_dpt: + title: Pressure gauge depth + description: Depth the pressure gauge was set at. + pipe_dia: + title: Pipe diameter + description: Diameter of the drill pipe used for the test. + pipe_length: + title: Pipe length + description: Length of drill pipe run for the test. + flow_history: + title: Flow history + description: >- + The operations logged during the test, in order -- opening the tool, + flow periods, and shut-in periods. + init_flow: + title: Initial flow pressure + description: Pressure recorded at the start of the first flow period. + flw_prs_in_min: + title: Initial flow duration + description: Length of the first flow period, in minutes. + x-ogc-unit: https://qudt.org/vocab/unit/MIN + x-ogc-unitLang: QUDT + fin_flow: + title: Final flow pressure + description: Pressure recorded at the end of the last flow period. + flw_prs_fin_min: + title: Final flow duration + description: Length of the last flow period, in minutes. + x-ogc-unit: https://qudt.org/vocab/unit/MIN + x-ogc-unitLang: QUDT + prs_init_clsd_in: + title: Initial shut-in pressure + description: >- + Pressure built up during the first shut-in period, after the tool was + closed and fluid stopped flowing. + in_sht_in_min: + title: Initial shut-in duration + description: Length of the first shut-in period, in minutes. + x-ogc-unit: https://qudt.org/vocab/unit/MIN + x-ogc-unitLang: QUDT + fin_shut_in: + title: Final shut-in pressure + description: >- + Pressure built up during the last shut-in period. Usually the closest + available estimate of true formation pressure. + fn_sht_in_min: + title: Final shut-in duration + description: Length of the last shut-in period, in minutes. + x-ogc-unit: https://qudt.org/vocab/unit/MIN + x-ogc-unitLang: QUDT + hydrost_prs_in: + title: Initial hydrostatic pressure + description: >- + Pressure of the fluid column in the hole before the test, used as a + reference for the flowing pressures. + hyd_st_prs_fl: + title: Final hydrostatic pressure + description: Pressure of the fluid column in the hole at the end of the test. + press_units: + title: Pressure units + description: Units the pressures on this record are reported in. + blanked_off: + title: Blanked off + description: Whether the tested interval was blanked off during the test. + fm_temp: + title: Formation temperature + description: Temperature recorded for the formation during the test. + +# --------------------------------------------------------------------------- +# Chemistry analyte columns below are generated by +# cli/generate_chemistry_field_descriptions.py and reviewed by hand. Re-run it +# when the analyte lists in the ogc_* view migrations change. +# --------------------------------------------------------------------------- +major_chemistry_results: + location_id: + title: Location ID + description: >- + Identifier of the location record the well's coordinates came from. + + analyte_count: + title: Analyte count + description: >- + Number of distinct analytes with a value in this row. A low count + means the well has only been analysed for part of the suite. + + latest_chemistry_date: + title: Latest analysis date + description: >- + Date of the most recent result in this row. Analytes are carried + forward independently, so an individual value may be older than this + date. + + tds: + title: Total dissolved solids + description: >- + Total mass of dissolved mineral matter in the water -- in plain + terms, how salty it is. Drinking-water guidance sits around 500 + mg/L. + + tds_units: + title: Total dissolved solids units + description: >- + Units the total dissolved solids value is reported in, as the + laboratory recorded them. + + calcium: + title: Calcium + description: >- + Dissolved calcium concentration in the most recent sample analysed + for it. + + calcium_units: + title: Calcium units + description: >- + Units the calcium value is reported in, as the laboratory recorded + them. + + calcium_total: + title: Calcium (total) + description: >- + Total calcium concentration -- the unfiltered determination, which + counts calcium bound to suspended particles as well as the dissolved + fraction. + + calcium_total_units: + title: Calcium (total) units + description: >- + Units the calcium (total) value is reported in, as the laboratory + recorded them. + + magnesium: + title: Magnesium + description: >- + Dissolved magnesium concentration in the most recent sample analysed + for it. + + magnesium_units: + title: Magnesium units + description: >- + Units the magnesium value is reported in, as the laboratory recorded + them. + + magnesium_total: + title: Magnesium (total) + description: >- + Total magnesium concentration -- the unfiltered determination, which + counts magnesium bound to suspended particles as well as the + dissolved fraction. + + magnesium_total_units: + title: Magnesium (total) units + description: >- + Units the magnesium (total) value is reported in, as the laboratory + recorded them. + + sodium: + title: Sodium + description: >- + Dissolved sodium concentration in the most recent sample analysed + for it. + + sodium_units: + title: Sodium units + description: >- + Units the sodium value is reported in, as the laboratory recorded + them. + + sodium_total: + title: Sodium (total) + description: >- + Total sodium concentration -- the unfiltered determination, which + counts sodium bound to suspended particles as well as the dissolved + fraction. + + sodium_total_units: + title: Sodium (total) units + description: >- + Units the sodium (total) value is reported in, as the laboratory + recorded them. + + potassium: + title: Potassium + description: >- + Dissolved potassium concentration in the most recent sample analysed + for it. + + potassium_units: + title: Potassium units + description: >- + Units the potassium value is reported in, as the laboratory recorded + them. + + potassium_total: + title: Potassium (total) + description: >- + Total potassium concentration -- the unfiltered determination, which + counts potassium bound to suspended particles as well as the + dissolved fraction. + + potassium_total_units: + title: Potassium (total) units + description: >- + Units the potassium (total) value is reported in, as the laboratory + recorded them. + + sodium_plus_potassium: + title: Sodium plus potassium + description: >- + Combined sodium and potassium concentration, reported together where + the laboratory did not separate them. + + sodium_plus_potassium_units: + title: Sodium plus potassium units + description: >- + Units the sodium plus potassium value is reported in, as the + laboratory recorded them. + + bicarbonate: + title: Bicarbonate + description: >- + Dissolved bicarbonate concentration in the most recent sample + analysed for it. + + bicarbonate_units: + title: Bicarbonate units + description: >- + Units the bicarbonate value is reported in, as the laboratory + recorded them. + + carbonate: + title: Carbonate + description: >- + Dissolved carbonate concentration in the most recent sample analysed + for it. + + carbonate_units: + title: Carbonate units + description: >- + Units the carbonate value is reported in, as the laboratory recorded + them. + + sulfate: + title: Sulfate + description: >- + Dissolved sulfate concentration in the most recent sample analysed + for it. + + sulfate_units: + title: Sulfate units + description: >- + Units the sulfate value is reported in, as the laboratory recorded + them. + + chloride: + title: Chloride + description: >- + Dissolved chloride concentration in the most recent sample analysed + for it. + + chloride_units: + title: Chloride units + description: >- + Units the chloride value is reported in, as the laboratory recorded + them. + + ion_balance: + title: Ion balance + description: >- + Percentage difference between the total positive and total negative + charge in the analysis. Charge must balance in reality, so a figure + far from zero means the analysis is incomplete or in error. + + ion_balance_units: + title: Ion balance units + description: >- + Units the ion balance value is reported in, as the laboratory + recorded them. + + total_anions: + title: Total anions + description: >- + Sum of the negatively charged dissolved constituents in the + analysis. + + total_anions_units: + title: Total anions units + description: >- + Units the total anions value is reported in, as the laboratory + recorded them. + + total_cations: + title: Total cations + description: >- + Sum of the positively charged dissolved constituents in the + analysis. + + total_cations_units: + title: Total cations units + description: >- + Units the total cations value is reported in, as the laboratory + recorded them. + + alkalinity: + title: Alkalinity + description: >- + The water's capacity to neutralise acid, reported as an equivalent + mass of calcium carbonate. Mostly supplied by bicarbonate and + carbonate. + + alkalinity_units: + title: Alkalinity units + description: >- + Units the alkalinity value is reported in, as the laboratory + recorded them. + + hardness: + title: Hardness + description: >- + Combined calcium and magnesium content, reported as an equivalent + mass of calcium carbonate. What determines whether water is 'hard'. + + hardness_units: + title: Hardness units + description: >- + Units the hardness value is reported in, as the laboratory recorded + them. + + specific_conductance: + title: Specific conductance + description: >- + How well the water conducts electricity, which rises with dissolved + mineral content. Used as a fast field proxy for total dissolved + solids. + + specific_conductance_units: + title: Specific conductance units + description: >- + Units the specific conductance value is reported in, as the + laboratory recorded them. + + ph: + title: pH + description: >- + Acidity of the water on the 0-14 scale, where 7 is neutral. + Unitless. Most New Mexico groundwater falls between 7 and 8.5. + + ph_units: + title: pH units + description: >- + Units the ph value is reported in, as the laboratory recorded them. + + nitrate: + title: Nitrate + description: >- + Dissolved nitrate concentration, usually from fertiliser, septic + systems, or livestock. The drinking-water limit is 10 mg/L as + nitrogen. + + nitrate_units: + title: Nitrate units + description: >- + Units the nitrate value is reported in, as the laboratory recorded + them. + + fluoride: + title: Fluoride + description: >- + Dissolved fluoride concentration. Beneficial in small amounts; the + drinking-water limit is 4 mg/L. + + fluoride_units: + title: Fluoride units + description: >- + Units the fluoride value is reported in, as the laboratory recorded + them. + + silica: + title: Silica + description: >- + Dissolved silica concentration, weathered out of silicate rock. + Useful for estimating the temperature water last equilibrated at. + + silica_units: + title: Silica units + description: >- + Units the silica value is reported in, as the laboratory recorded + them. + +minor_chemistry_wells: + location_id: + title: Location ID + description: >- + Identifier of the location record the well's coordinates came from. + + analyte_count: + title: Analyte count + description: >- + Number of distinct analytes with a value in this row. A low count + means the well has only been analysed for part of the suite. + + latest_chemistry_date: + title: Latest analysis date + description: >- + Date of the most recent result in this row. Analytes are carried + forward independently, so an individual value may be older than this + date. + + h2r: + title: Deuterium ratio + description: >- + Ratio of heavy to ordinary hydrogen in the water, reported as + per-mil difference from ocean water. Fingerprints where the water + fell as precipitation. + + h2r_units: + title: Deuterium ratio units + description: >- + Units the deuterium ratio value is reported in, as the laboratory + recorded them. + + o18r: + title: Oxygen-18 ratio + description: >- + Ratio of heavy to ordinary oxygen in the water, reported as per-mil + difference from ocean water. Read with the deuterium ratio to trace + the water's origin and evaporation history. + + o18r_units: + title: Oxygen-18 ratio units + description: >- + Units the oxygen-18 ratio value is reported in, as the laboratory + recorded them. + + c13r: + title: Carbon-13 ratio + description: >- + Ratio of carbon-13 to carbon-12 in the water's dissolved carbon, + reported as per-mil difference from a standard. Helps identify where + the carbon came from, which is needed to correct a carbon-14 age. + + c13r_units: + title: Carbon-13 ratio units + description: >- + Units the carbon-13 ratio value is reported in, as the laboratory + recorded them. + + c14: + title: Carbon-14 + description: >- + Carbon-14 remaining in the water's dissolved carbon, as a percentage + of the modern atmospheric level. The basis for dating groundwater up + to roughly 40,000 years old. + + c14_units: + title: Carbon-14 units + description: >- + Units the carbon-14 value is reported in, as the laboratory recorded + them. + + c14_years: + title: Carbon-14 age + description: >- + Apparent age of the water in years, calculated from its carbon-14 + content. Uncorrected for carbon picked up from rock, so treat it as + an upper bound. + + c14_years_units: + title: Carbon-14 age units + description: >- + Units the carbon-14 age value is reported in, as the laboratory + recorded them. + + fluoride: + title: Fluoride + description: >- + Dissolved fluoride concentration. Beneficial in small amounts; the + drinking-water limit is 4 mg/L. + + fluoride_units: + title: Fluoride units + description: >- + Units the fluoride value is reported in, as the laboratory recorded + them. + + barium: + title: Barium + description: >- + Dissolved barium concentration in the most recent sample analysed + for it. + + barium_units: + title: Barium units + description: >- + Units the barium value is reported in, as the laboratory recorded + them. + + barium_total: + title: Barium (total) + description: >- + Total barium concentration -- the unfiltered determination, which + counts barium bound to suspended particles as well as the dissolved + fraction. + + barium_total_units: + title: Barium (total) units + description: >- + Units the barium (total) value is reported in, as the laboratory + recorded them. + + copper: + title: Copper + description: >- + Dissolved copper concentration in the most recent sample analysed + for it. + + copper_units: + title: Copper units + description: >- + Units the copper value is reported in, as the laboratory recorded + them. + + copper_total: + title: Copper (total) + description: >- + Total copper concentration -- the unfiltered determination, which + counts copper bound to suspended particles as well as the dissolved + fraction. + + copper_total_units: + title: Copper (total) units + description: >- + Units the copper (total) value is reported in, as the laboratory + recorded them. + + zinc: + title: Zinc + description: >- + Dissolved zinc concentration in the most recent sample analysed for + it. + + zinc_units: + title: Zinc units + description: >- + Units the zinc value is reported in, as the laboratory recorded + them. + + zinc_total: + title: Zinc (total) + description: >- + Total zinc concentration -- the unfiltered determination, which + counts zinc bound to suspended particles as well as the dissolved + fraction. + + zinc_total_units: + title: Zinc (total) units + description: >- + Units the zinc (total) value is reported in, as the laboratory + recorded them. + + molybdenum: + title: Molybdenum + description: >- + Dissolved molybdenum concentration in the most recent sample + analysed for it. + + molybdenum_units: + title: Molybdenum units + description: >- + Units the molybdenum value is reported in, as the laboratory + recorded them. + + molybdenum_total: + title: Molybdenum (total) + description: >- + Total molybdenum concentration -- the unfiltered determination, + which counts molybdenum bound to suspended particles as well as the + dissolved fraction. + + molybdenum_total_units: + title: Molybdenum (total) units + description: >- + Units the molybdenum (total) value is reported in, as the laboratory + recorded them. + + silica: + title: Silica + description: >- + Dissolved silica concentration, weathered out of silicate rock. + Useful for estimating the temperature water last equilibrated at. + + silica_units: + title: Silica units + description: >- + Units the silica value is reported in, as the laboratory recorded + them. + + silicon: + title: Silicon + description: >- + Dissolved silicon concentration in the most recent sample analysed + for it. + + silicon_units: + title: Silicon units + description: >- + Units the silicon value is reported in, as the laboratory recorded + them. + + silicon_total: + title: Silicon (total) + description: >- + Total silicon concentration -- the unfiltered determination, which + counts silicon bound to suspended particles as well as the dissolved + fraction. + + silicon_total_units: + title: Silicon (total) units + description: >- + Units the silicon (total) value is reported in, as the laboratory + recorded them. + + manganese: + title: Manganese + description: >- + Dissolved manganese concentration in the most recent sample analysed + for it. + + manganese_units: + title: Manganese units + description: >- + Units the manganese value is reported in, as the laboratory recorded + them. + + manganese_total: + title: Manganese (total) + description: >- + Total manganese concentration -- the unfiltered determination, which + counts manganese bound to suspended particles as well as the + dissolved fraction. + + manganese_total_units: + title: Manganese (total) units + description: >- + Units the manganese (total) value is reported in, as the laboratory + recorded them. + + iron: + title: Iron + description: >- + Dissolved iron concentration in the most recent sample analysed for + it. + + iron_units: + title: Iron units + description: >- + Units the iron value is reported in, as the laboratory recorded + them. + + iron_total: + title: Iron (total) + description: >- + Total iron concentration -- the unfiltered determination, which + counts iron bound to suspended particles as well as the dissolved + fraction. + + iron_total_units: + title: Iron (total) units + description: >- + Units the iron (total) value is reported in, as the laboratory + recorded them. + + strontium: + title: Strontium + description: >- + Dissolved strontium concentration in the most recent sample analysed + for it. + + strontium_units: + title: Strontium units + description: >- + Units the strontium value is reported in, as the laboratory recorded + them. + + strontium_total: + title: Strontium (total) + description: >- + Total strontium concentration -- the unfiltered determination, which + counts strontium bound to suspended particles as well as the + dissolved fraction. + + strontium_total_units: + title: Strontium (total) units + description: >- + Units the strontium (total) value is reported in, as the laboratory + recorded them. + + chromium: + title: Chromium + description: >- + Dissolved chromium concentration in the most recent sample analysed + for it. + + chromium_units: + title: Chromium units + description: >- + Units the chromium value is reported in, as the laboratory recorded + them. + + chromium_total: + title: Chromium (total) + description: >- + Total chromium concentration -- the unfiltered determination, which + counts chromium bound to suspended particles as well as the + dissolved fraction. + + chromium_total_units: + title: Chromium (total) units + description: >- + Units the chromium (total) value is reported in, as the laboratory + recorded them. + + boron: + title: Boron + description: >- + Dissolved boron concentration in the most recent sample analysed for + it. + + boron_units: + title: Boron units + description: >- + Units the boron value is reported in, as the laboratory recorded + them. + + boron_total: + title: Boron (total) + description: >- + Total boron concentration -- the unfiltered determination, which + counts boron bound to suspended particles as well as the dissolved + fraction. + + boron_total_units: + title: Boron (total) units + description: >- + Units the boron (total) value is reported in, as the laboratory + recorded them. + + uranium: + title: Uranium + description: >- + Dissolved uranium concentration. Naturally present near + uranium-bearing rock and regulated in drinking water at 0.030 mg/L. + + uranium_units: + title: Uranium units + description: >- + Units the uranium value is reported in, as the laboratory recorded + them. + + uranium_total: + title: Uranium (total) + description: >- + Total uranium concentration -- the unfiltered determination, which + counts uranium bound to suspended particles as well as the dissolved + fraction. + + uranium_total_units: + title: Uranium (total) units + description: >- + Units the uranium (total) value is reported in, as the laboratory + recorded them. + + lithium: + title: Lithium + description: >- + Dissolved lithium concentration in the most recent sample analysed + for it. + + lithium_units: + title: Lithium units + description: >- + Units the lithium value is reported in, as the laboratory recorded + them. + + lithium_total: + title: Lithium (total) + description: >- + Total lithium concentration -- the unfiltered determination, which + counts lithium bound to suspended particles as well as the dissolved + fraction. + + lithium_total_units: + title: Lithium (total) units + description: >- + Units the lithium (total) value is reported in, as the laboratory + recorded them. + + silver: + title: Silver + description: >- + Dissolved silver concentration in the most recent sample analysed + for it. + + silver_units: + title: Silver units + description: >- + Units the silver value is reported in, as the laboratory recorded + them. + + silver_total: + title: Silver (total) + description: >- + Total silver concentration -- the unfiltered determination, which + counts silver bound to suspended particles as well as the dissolved + fraction. + + silver_total_units: + title: Silver (total) units + description: >- + Units the silver (total) value is reported in, as the laboratory + recorded them. + + antimony: + title: Antimony + description: >- + Dissolved antimony concentration in the most recent sample analysed + for it. + + antimony_units: + title: Antimony units + description: >- + Units the antimony value is reported in, as the laboratory recorded + them. + + antimony_total: + title: Antimony (total) + description: >- + Total antimony concentration -- the unfiltered determination, which + counts antimony bound to suspended particles as well as the + dissolved fraction. + + antimony_total_units: + title: Antimony (total) units + description: >- + Units the antimony (total) value is reported in, as the laboratory + recorded them. + + beryllium: + title: Beryllium + description: >- + Dissolved beryllium concentration in the most recent sample analysed + for it. + + beryllium_units: + title: Beryllium units + description: >- + Units the beryllium value is reported in, as the laboratory recorded + them. + + beryllium_total: + title: Beryllium (total) + description: >- + Total beryllium concentration -- the unfiltered determination, which + counts beryllium bound to suspended particles as well as the + dissolved fraction. + + beryllium_total_units: + title: Beryllium (total) units + description: >- + Units the beryllium (total) value is reported in, as the laboratory + recorded them. + + lead: + title: Lead + description: >- + Dissolved lead concentration in the most recent sample analysed for + it. + + lead_units: + title: Lead units + description: >- + Units the lead value is reported in, as the laboratory recorded + them. + + lead_total: + title: Lead (total) + description: >- + Total lead concentration -- the unfiltered determination, which + counts lead bound to suspended particles as well as the dissolved + fraction. + + lead_total_units: + title: Lead (total) units + description: >- + Units the lead (total) value is reported in, as the laboratory + recorded them. + + thallium: + title: Thallium + description: >- + Dissolved thallium concentration in the most recent sample analysed + for it. + + thallium_units: + title: Thallium units + description: >- + Units the thallium value is reported in, as the laboratory recorded + them. + + thallium_total: + title: Thallium (total) + description: >- + Total thallium concentration -- the unfiltered determination, which + counts thallium bound to suspended particles as well as the + dissolved fraction. + + thallium_total_units: + title: Thallium (total) units + description: >- + Units the thallium (total) value is reported in, as the laboratory + recorded them. + + bromide: + title: Bromide + description: >- + Dissolved bromide concentration. Read against chloride, it + distinguishes seawater-derived salinity from dissolved halite. + + bromide_units: + title: Bromide units + description: >- + Units the bromide value is reported in, as the laboratory recorded + them. + + selenium: + title: Selenium + description: >- + Dissolved selenium concentration in the most recent sample analysed + for it. + + selenium_units: + title: Selenium units + description: >- + Units the selenium value is reported in, as the laboratory recorded + them. + + selenium_total: + title: Selenium (total) + description: >- + Total selenium concentration -- the unfiltered determination, which + counts selenium bound to suspended particles as well as the + dissolved fraction. + + selenium_total_units: + title: Selenium (total) units + description: >- + Units the selenium (total) value is reported in, as the laboratory + recorded them. + + vanadium: + title: Vanadium + description: >- + Dissolved vanadium concentration in the most recent sample analysed + for it. + + vanadium_units: + title: Vanadium units + description: >- + Units the vanadium value is reported in, as the laboratory recorded + them. + + vanadium_total: + title: Vanadium (total) + description: >- + Total vanadium concentration -- the unfiltered determination, which + counts vanadium bound to suspended particles as well as the + dissolved fraction. + + vanadium_total_units: + title: Vanadium (total) units + description: >- + Units the vanadium (total) value is reported in, as the laboratory + recorded them. + + aluminum: + title: Aluminum + description: >- + Dissolved aluminum concentration in the most recent sample analysed + for it. + + aluminum_units: + title: Aluminum units + description: >- + Units the aluminum value is reported in, as the laboratory recorded + them. + + aluminum_total: + title: Aluminum (total) + description: >- + Total aluminum concentration -- the unfiltered determination, which + counts aluminum bound to suspended particles as well as the + dissolved fraction. + + aluminum_total_units: + title: Aluminum (total) units + description: >- + Units the aluminum (total) value is reported in, as the laboratory + recorded them. + + arsenic: + title: Arsenic + description: >- + Dissolved arsenic concentration. Naturally elevated in parts of New + Mexico and regulated in drinking water at 0.010 mg/L. + + arsenic_units: + title: Arsenic units + description: >- + Units the arsenic value is reported in, as the laboratory recorded + them. + + arsenic_total: + title: Arsenic (total) + description: >- + Total arsenic concentration -- the unfiltered determination, which + counts arsenic bound to suspended particles as well as the dissolved + fraction. + + arsenic_total_units: + title: Arsenic (total) units + description: >- + Units the arsenic (total) value is reported in, as the laboratory + recorded them. + + nickel: + title: Nickel + description: >- + Dissolved nickel concentration in the most recent sample analysed + for it. + + nickel_units: + title: Nickel units + description: >- + Units the nickel value is reported in, as the laboratory recorded + them. + + nickel_total: + title: Nickel (total) + description: >- + Total nickel concentration -- the unfiltered determination, which + counts nickel bound to suspended particles as well as the dissolved + fraction. + + nickel_total_units: + title: Nickel (total) units + description: >- + Units the nickel (total) value is reported in, as the laboratory + recorded them. + + cadmium: + title: Cadmium + description: >- + Dissolved cadmium concentration in the most recent sample analysed + for it. + + cadmium_units: + title: Cadmium units + description: >- + Units the cadmium value is reported in, as the laboratory recorded + them. + + cadmium_total: + title: Cadmium (total) + description: >- + Total cadmium concentration -- the unfiltered determination, which + counts cadmium bound to suspended particles as well as the dissolved + fraction. + + cadmium_total_units: + title: Cadmium (total) units + description: >- + Units the cadmium (total) value is reported in, as the laboratory + recorded them. + + cobalt: + title: Cobalt + description: >- + Dissolved cobalt concentration in the most recent sample analysed + for it. + + cobalt_units: + title: Cobalt units + description: >- + Units the cobalt value is reported in, as the laboratory recorded + them. + + cobalt_total: + title: Cobalt (total) + description: >- + Total cobalt concentration -- the unfiltered determination, which + counts cobalt bound to suspended particles as well as the dissolved + fraction. + + cobalt_total_units: + title: Cobalt (total) units + description: >- + Units the cobalt (total) value is reported in, as the laboratory + recorded them. + + phosphate: + title: Phosphate + description: >- + Dissolved phosphate concentration in the most recent sample analysed + for it. + + phosphate_units: + title: Phosphate units + description: >- + Units the phosphate value is reported in, as the laboratory recorded + them. + + nitrite: + title: Nitrite + description: >- + Dissolved nitrite concentration, an intermediate stage in the + breakdown of nitrogen compounds. + + nitrite_units: + title: Nitrite units + description: >- + Units the nitrite value is reported in, as the laboratory recorded + them. + + nitrate: + title: Nitrate + description: >- + Dissolved nitrate concentration, usually from fertiliser, septic + systems, or livestock. The drinking-water limit is 10 mg/L as + nitrogen. + + nitrate_units: + title: Nitrate units + description: >- + Units the nitrate value is reported in, as the laboratory recorded + them. + + nitrate_as_n: + title: Nitrate as nitrogen + description: >- + Nitrate concentration expressed as the mass of nitrogen alone, which + is how the 10 mg/L drinking-water limit is written. Roughly a + quarter of the same sample reported as nitrate. + + nitrate_as_n_units: + title: Nitrate as nitrogen units + description: >- + Units the nitrate as nitrogen value is reported in, as the + laboratory recorded them. + + thorium: + title: Thorium + description: >- + Dissolved thorium concentration in the most recent sample analysed + for it. + + thorium_units: + title: Thorium units + description: >- + Units the thorium value is reported in, as the laboratory recorded + them. + + thorium_total: + title: Thorium (total) + description: >- + Total thorium concentration -- the unfiltered determination, which + counts thorium bound to suspended particles as well as the dissolved + fraction. + + thorium_total_units: + title: Thorium (total) units + description: >- + Units the thorium (total) value is reported in, as the laboratory + recorded them. + + tin: + title: Tin + description: >- + Dissolved tin concentration in the most recent sample analysed for + it. + + tin_units: + title: Tin units + description: >- + Units the tin value is reported in, as the laboratory recorded them. + + tin_total: + title: Tin (total) + description: >- + Total tin concentration -- the unfiltered determination, which + counts tin bound to suspended particles as well as the dissolved + fraction. + + tin_total_units: + title: Tin (total) units + description: >- + Units the tin (total) value is reported in, as the laboratory + recorded them. + + mercury: + title: Mercury + description: >- + Dissolved mercury concentration in the most recent sample analysed + for it. + + mercury_units: + title: Mercury units + description: >- + Units the mercury value is reported in, as the laboratory recorded + them. + + mercury_total: + title: Mercury (total) + description: >- + Total mercury concentration -- the unfiltered determination, which + counts mercury bound to suspended particles as well as the dissolved + fraction. + + mercury_total_units: + title: Mercury (total) units + description: >- + Units the mercury (total) value is reported in, as the laboratory + recorded them. + + titanium: + title: Titanium + description: >- + Dissolved titanium concentration in the most recent sample analysed + for it. + + titanium_units: + title: Titanium units + description: >- + Units the titanium value is reported in, as the laboratory recorded + them. + + titanium_total: + title: Titanium (total) + description: >- + Total titanium concentration -- the unfiltered determination, which + counts titanium bound to suspended particles as well as the + dissolved fraction. + + titanium_total_units: + title: Titanium (total) units + description: >- + Units the titanium (total) value is reported in, as the laboratory + recorded them. + + +# EDR collections. Keys here are parameter names read out of the data, not +# column names -- ogc_waterlevels stamps a single literal, while +# ogc_water_chemistry carries the analyte text exactly as the laboratory +# recorded it, so most chemistry parameters take a generated title. +waterlevels: + groundwater level: + title: Groundwater level + description: >- + Depth from the measuring point down to the water table, as measured by + hand during a site visit or logged automatically by a pressure + transducer left in the well. Larger values mean the water table is + further below the surface. diff --git a/core/ogc_field_metadata.py b/core/ogc_field_metadata.py new file mode 100644 index 000000000..7befabcec --- /dev/null +++ b/core/ogc_field_metadata.py @@ -0,0 +1,204 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Per-field prose for the OGC collections. + +Collection-level ``title``/``description``/``keywords`` live in +``core/pygeoapi.py`` and the two pygeoapi config templates. This module is the +level below: what an individual column means, and what unit it is in. + +The copy lives in ``core/ogc-field-descriptions.yml``, keyed by backing +relation name with the ``ogc_``/``ogc_internal_`` prefix stripped, so the +public and internal mounts share one entry per view. + +Read ``docs/ogc-field-descriptions.md`` before changing the shape of the YAML +or upgrading pygeoapi. +""" + +import json +import logging +from pathlib import Path + +import yaml + +LOGGER = logging.getLogger(__name__) + +# The prefixes _thing_collections_block and _edr_collections_block prepend to a +# collection id to reach its backing relation. Longest first: "ogc_internal_" +# also starts with "ogc_". +TABLE_PREFIXES = ("ogc_internal_", "ogc_") + +# Entries carry documentation, not schema. Types and formats stay with the +# provider's own reflection. +ALLOWED_KEYS = frozenset( + { + "title", + "description", + "x-ogc-unit", + "x-ogc-unitLang", + "x-ogc-propertySeq", + # JSON Schema's own keyword. pygeoapi's HTML renders it as the schema + # table's "Values" column, and its queryables handler emits it too. + "enum", + # Names a category in core/lexicon.json, expanded to `enum` on the way + # out so a controlled vocabulary is not duplicated here. + "enum-lexicon", + } +) + +DEFAULTS_KEY = "_defaults" + +LEXICON_KEY = "enum-lexicon" + +_CACHE = None +_LEXICON_CACHE = None + + +def _metadata_path() -> Path: + return Path(__file__).resolve().parent / "ogc-field-descriptions.yml" + + +def _lexicon_path() -> Path: + return Path(__file__).resolve().parent / "lexicon.json" + + +def lexicon_terms(category: str) -> list: + """Terms in one core/lexicon.json category, in file order. + + The lexicon file seeds the database's controlled vocabularies, so reading + it here keeps one source of truth for an enumerated column's valid values + -- and keeps this module free of any database dependency. + """ + global _LEXICON_CACHE + if _LEXICON_CACHE is None: + raw = json.loads(_lexicon_path().read_text(encoding="utf-8")) + by_category: dict[str, list] = {} + for term in raw.get("terms", []): + for name in term.get("categories", []): + by_category.setdefault(name, []).append(term["term"]) + _LEXICON_CACHE = by_category + return list(_LEXICON_CACHE.get(category, [])) + + +def _validate(raw: dict, path: Path) -> dict: + if not isinstance(raw, dict): + raise ValueError(f"{path} must contain a mapping of table -> fields.") + + for table, fields in raw.items(): + if not isinstance(fields, dict): + raise ValueError(f"{path}: {table} must be a mapping of field -> entry.") + for field, entry in fields.items(): + if not isinstance(entry, dict): + raise ValueError( + f"{path}: {table}.{field} must be a mapping, got {type(entry).__name__}." + ) + if not entry.get("title"): + raise ValueError(f"{path}: {table}.{field} is missing a title.") + unknown = set(entry) - ALLOWED_KEYS + if unknown: + raise ValueError( + f"{path}: {table}.{field} has unsupported keys " + f"{sorted(unknown)}; allowed keys are {sorted(ALLOWED_KEYS)}." + ) + values = entry.get("enum") + if values is not None and (not isinstance(values, list) or not values): + raise ValueError( + f"{path}: {table}.{field} enum must be a non-empty list." + ) + category = entry.get(LEXICON_KEY) + if category is not None and not lexicon_terms(category): + raise ValueError( + f"{path}: {table}.{field} names lexicon category " + f"{category!r}, which has no terms in core/lexicon.json." + ) + return raw + + +def load_field_metadata(refresh: bool = False) -> dict: + """Return the parsed YAML, read once per process. + + Deliberately free of any database dependency: this is called during + OpenAPI generation, which runs before the backing views need to exist. + """ + global _CACHE + if _CACHE is None or refresh: + path = _metadata_path() + raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + _CACHE = _validate(raw, path) + return _CACHE + + +def strip_table_prefix(table: str) -> str: + """Reduce ``ogc_water_wells``/``ogc_internal_water_wells`` to ``water_wells``.""" + for prefix in TABLE_PREFIXES: + if table.startswith(prefix): + return table[len(prefix) :] + return table + + +def default_title(column_name: str) -> str: + """Fallback title for a column with no entry: ``well_depth`` -> ``Well Depth``.""" + return column_name.replace("_", " ").strip().title() + + +def table_entries(table: str) -> dict: + """Documentation entries in force for ``table``, defaults included.""" + metadata = load_field_metadata() + entries = dict(metadata.get(DEFAULTS_KEY, {})) + entries.update(metadata.get(strip_table_prefix(table), {})) + return entries + + +def describe_fields(table: str, fields: dict) -> dict: + """Annotate a provider's reflected ``fields`` with prose from the YAML. + + Returns a new dict of new per-field dicts. That is not tidiness: + ``pygeoapi.api.get_collection_schema`` assigns the provider's own field + dict into the response and then mutates it in place (pops ``format``, + assigns ``x-ogc-role``), so handing out references into the cached YAML + would let one request's mutations leak into the next one's. + """ + entries = table_entries(table) + described = {} + undocumented = [] + + for name, field in (fields or {}).items(): + annotated = dict(field) + entry = entries.get(name) + if entry: + for key, value in entry.items(): + if key == LEXICON_KEY: + # Expanded here rather than stored, so the vocabulary stays + # defined in one place. An entry may still pin a literal + # `enum` instead when the column's values are set by the + # view's own SQL rather than by the lexicon. + annotated.setdefault("enum", lexicon_terms(value)) + continue + annotated[key] = value + else: + annotated.setdefault("title", default_title(name)) + undocumented.append(name) + described[name] = annotated + + if undocumented: + # Not fatal: a response with a generated title beats a 500. The drift + # guard in tests/test_ogc_field_descriptions.py is what fails the build. + LOGGER.warning( + "No field description for %s.%s; falling back to a generated title.", + table, + ", ".join(sorted(undocumented)), + ) + + return described diff --git a/core/pygeoapi-config-internal.yml b/core/pygeoapi-config-internal.yml index f2ddb5001..578bc8daa 100644 --- a/core/pygeoapi-config-internal.yml +++ b/core/pygeoapi-config-internal.yml @@ -50,15 +50,22 @@ resources: locations: type: collection title: Locations - description: Geographic locations and site coordinates used by Ocotillo features. - keywords: [locations] + description: >- + The raw geographic location records that every monitoring point hangs + off -- one feature per surveyed site, with its elevation, county, + quadrangle, and the notes recorded about how the coordinates were + obtained and how reliable they are. Most consumers want a feature-type + layer such as water_wells instead, which pairs the same coordinates + with what is actually monitored there; this layer is kept for staff + work that needs the location record itself. + keywords: [locations, sites, coordinates, elevation, county, data-reliability] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -73,15 +80,22 @@ resources: latest_depth_to_water_wells: type: collection title: Latest Depth to Water (Water Wells) - description: Most recent depth-to-water below ground surface observation for each water well. - keywords: [water-wells, groundwater-level, depth-to-water-bgs, latest] + description: >- + The most recent depth-to-water reading for each well, measured below + ground surface -- the measured depth minus the height of the measuring + point above ground, with readings that have no recorded + measuring-point height treated as taken at ground level. + water_well_summary publishes the same latest reading alongside the + count, range and trend of the whole record, so this layer is kept only + for staff clients that already depend on its narrower shape. + keywords: [water-wells, groundwater-level, depth-to-water, latest-value, below-ground-surface] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -96,15 +110,24 @@ resources: avg_tds_wells: type: collection title: Average TDS (Water Wells) - description: Average total dissolved solids (TDS) from major chemistry results for each water well. - keywords: [water-wells, chemistry, tds, total-dissolved-solids, average] + description: >- + The arithmetic mean of all total dissolved solids (TDS) results on + record for each water well. Treat with care: across the catalog the + average rests on about 1.9 analyses per well, so for many wells it is + a mean of one or two samples taken years apart and is not a reliable + summary of the well's water quality. Prefer latest_tds_wells, which + reports a single dated result. + keywords: [ + water-wells, chemistry, tds, total-dissolved-solids, average, + low-sample-count, use-with-caution + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -119,15 +142,26 @@ resources: latest_tds_wells: type: collection title: Latest TDS (Water Wells) - description: Most recent total dissolved solids (TDS) result from major chemistry for each water well. - keywords: [water-wells, chemistry, tds, total-dissolved-solids, latest] + description: >- + Total dissolved solids (TDS) measures how much mineral matter is + dissolved in the water -- in plain terms, how salty it is. This layer + reads every laboratory major-chemistry analysis on record for each + water well, keeps only the TDS results, and publishes the single most + recent one per well, dated by its analysis date or, where that is + missing, by the date the sample was collected. Use it for a current + statewide picture of groundwater salinity without working through each + well's full analysis history. + keywords: [ + water-wells, water-quality, chemistry, tds, total-dissolved-solids, + salinity, latest-value, groundwater + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -142,15 +176,30 @@ resources: depth_to_water_trend_wells: type: collection title: Depth to Water Trend (Water Wells) - description: Trend classification for depth to water based on slope in feet per year. - keywords: [water-wells, groundwater-level, depth-to-water, trend, slope] + description: >- + Shows whether the water table beneath each well has been falling, + rising, or holding steady. Every manual groundwater-level measurement + for the well is converted to a depth below ground surface -- the + measured depth minus the height of the measuring point above ground, + with readings that have no recorded measuring-point height treated as + taken at ground level -- and a straight line is fitted through those + depths over time. The slope of that line in feet per year is reported + as increasing (water table falling faster than 0.25 ft/yr), decreasing + (rising faster than 0.25 ft/yr), or stable. Wells with fewer than 10 + measurements, or fewer than 4 spanning less than two years, are + labelled "not enough data" rather than given a trend the record cannot + support. + keywords: [ + water-wells, groundwater-level, depth-to-water, trend, slope, + feet-per-year, declining-water-levels, aquifer-condition + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -165,15 +214,27 @@ resources: water_elevation_wells: type: collection title: Water Elevation (Water Wells) - description: Most recent water elevation per well calculated as elevation minus depth to water below ground surface. - keywords: [water-wells, groundwater-level, water-elevation, depth-to-water] + description: >- + Gives the height of the water table above sea level at each well, so + that levels can be compared between wells standing at different ground + elevations. The most recent groundwater-level measurement is converted + to feet, the height of the measuring point above ground is subtracted + to give the depth below ground surface (readings with no recorded + measuring-point height are treated as taken at ground level), and that + depth is subtracted from the surveyed ground-surface elevation at the + well. Use it to map the shape of the water table or to work out which + way groundwater is flowing. + keywords: [ + water-wells, groundwater-level, water-table-elevation, water-elevation, + depth-to-water, above-sea-level, groundwater-flow + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -188,15 +249,30 @@ resources: water_well_summary: type: collection title: Water Well Summary - description: Summary metrics per water well, including latest, min/max, and trend for water levels. - keywords: [water-wells, summary, groundwater-level, trend] + description: >- + One row per water well, condensing that well's entire manual + groundwater-level record into a few numbers: how many measurements + exist, the most recent one and its date, the shallowest and deepest + ever recorded, and the long-term trend as a straight-line slope in + feet per year. Depths are below ground surface -- the measured depth + minus the height of the measuring point above ground, with readings + that have no recorded measuring-point height treated as taken at + ground level. Each row also carries the well's depth, its surveyed + ground elevation and how that elevation was determined, and the + geologic zone the well is completed in. Wells with no water-level + measurements at all are left out. Use it as the at-a-glance record for + a well before digging into individual readings. + keywords: [ + water-wells, summary, groundwater-level, water-level-history, trend, + well-depth, elevation, at-a-glance + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -211,15 +287,29 @@ resources: major_chemistry_results: type: collection title: Major Chemistry (Water Wells) - description: Latest major chemistry analyte values for water wells, represented as static analyte columns. - keywords: [water-wells, chemistry, analytes, major-chemistry] + description: >- + The major dissolved constituents that make up most of the chemistry of + groundwater -- calcium, magnesium, sodium, potassium, bicarbonate, + carbonate, sulfate and chloride -- alongside TDS, pH, hardness, + alkalinity and specific conductance. Laboratory records name the same + analyte in many different ways, so this layer first maps those names + and symbols onto one canonical set, then keeps the most recent result + for each analyte at each well and lays the values out as fixed + columns, each with its own units column. Analytes at one well may come + from different sampling dates; the reported chemistry date is the most + recent among them. Use it to compare water chemistry between wells or + to screen against drinking-water standards. + keywords: [ + water-wells, water-quality, chemistry, major-ions, analytes, calcium, + sodium, chloride, sulfate, ph, hardness + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -234,15 +324,27 @@ resources: minor_chemistry_wells: type: collection title: Minor Chemistry (Water Wells) - description: Latest minor/trace chemistry analyte values for water wells, represented as static analyte columns. - keywords: [water-wells, chemistry, analytes, minor-chemistry, trace-chemistry] + description: >- + Trace elements and isotopes measured in groundwater -- arsenic, + uranium, lead, iron, manganese, boron, lithium and dozens more, plus + the stable isotopes and carbon-14 used to work out how long water has + been underground. Built the same way as the major chemistry layer: + legacy laboratory records are mapped onto one canonical analyte set, + the most recent result for each analyte at each well is kept, and the + values are laid out as fixed columns each with its own units column. + Use it for contaminant screening and for questions about the age and + origin of groundwater. + keywords: [ + water-wells, water-quality, chemistry, trace-elements, minor-chemistry, + isotopes, arsenic, uranium, carbon-14, contaminants + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -257,15 +359,25 @@ resources: actively_monitored_wells: type: collection title: Actively Monitored Wells - description: Wells in the collaborative network currently flagged as actively monitored. - keywords: [water-wells, monitoring, collaborative-network, actively-monitored] + description: >- + The wells being measured today, rather than every well ever recorded. + A well appears here only if it belongs to the Water Level Network + group and its most recent monitoring-status entry reads "Currently + monitored"; the summary statistics attached to each one are the same + water-level figures published in water_well_summary. Use it to see the + live monitoring network -- where measurements are still being + collected, and where coverage is thin. + keywords: [ + water-wells, monitoring, water-level-network, actively-monitored, + monitoring-network, groundwater-level + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -280,15 +392,19 @@ resources: project_areas: type: collection title: Project Areas - description: Project groups with polygon project-area boundaries. - keywords: [project-areas, groups, boundaries] + description: >- + The study-area boundaries of Bureau projects, as polygons. Any project + group that has a mapped boundary is published here with its name and + description. Use it to see which part of New Mexico a project covers, + or to clip the other layers to a project's footprint. + keywords: [project-areas, study-areas, boundaries, polygons, projects, groups] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} diff --git a/core/pygeoapi-config.yml b/core/pygeoapi-config.yml index c82fcea2a..4dbf0c972 100644 --- a/core/pygeoapi-config.yml +++ b/core/pygeoapi-config.yml @@ -52,15 +52,26 @@ resources: latest_tds_wells: type: collection title: Latest TDS (Water Wells) - description: Most recent total dissolved solids (TDS) result from major chemistry for each water well. - keywords: [water-wells, chemistry, tds, total-dissolved-solids, latest] + description: >- + Total dissolved solids (TDS) measures how much mineral matter is + dissolved in the water -- in plain terms, how salty it is. This layer + reads every laboratory major-chemistry analysis on record for each + water well, keeps only the TDS results, and publishes the single most + recent one per well, dated by its analysis date or, where that is + missing, by the date the sample was collected. Use it for a current + statewide picture of groundwater salinity without working through each + well's full analysis history. + keywords: [ + water-wells, water-quality, chemistry, tds, total-dissolved-solids, + salinity, latest-value, groundwater + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -75,15 +86,30 @@ resources: depth_to_water_trend_wells: type: collection title: Depth to Water Trend (Water Wells) - description: Trend classification for depth to water based on slope in feet per year. - keywords: [water-wells, groundwater-level, depth-to-water, trend, slope] + description: >- + Shows whether the water table beneath each well has been falling, + rising, or holding steady. Every manual groundwater-level measurement + for the well is converted to a depth below ground surface -- the + measured depth minus the height of the measuring point above ground, + with readings that have no recorded measuring-point height treated as + taken at ground level -- and a straight line is fitted through those + depths over time. The slope of that line in feet per year is reported + as increasing (water table falling faster than 0.25 ft/yr), decreasing + (rising faster than 0.25 ft/yr), or stable. Wells with fewer than 10 + measurements, or fewer than 4 spanning less than two years, are + labelled "not enough data" rather than given a trend the record cannot + support. + keywords: [ + water-wells, groundwater-level, depth-to-water, trend, slope, + feet-per-year, declining-water-levels, aquifer-condition + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -98,15 +124,27 @@ resources: water_elevation_wells: type: collection title: Water Elevation (Water Wells) - description: Most recent water elevation per well calculated as elevation minus depth to water below ground surface. - keywords: [water-wells, groundwater-level, water-elevation, depth-to-water] + description: >- + Gives the height of the water table above sea level at each well, so + that levels can be compared between wells standing at different ground + elevations. The most recent groundwater-level measurement is converted + to feet, the height of the measuring point above ground is subtracted + to give the depth below ground surface (readings with no recorded + measuring-point height are treated as taken at ground level), and that + depth is subtracted from the surveyed ground-surface elevation at the + well. Use it to map the shape of the water table or to work out which + way groundwater is flowing. + keywords: [ + water-wells, groundwater-level, water-table-elevation, water-elevation, + depth-to-water, above-sea-level, groundwater-flow + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -121,15 +159,30 @@ resources: water_well_summary: type: collection title: Water Well Summary - description: Summary metrics per water well, including latest, min/max, and trend for water levels. - keywords: [water-wells, summary, groundwater-level, trend] + description: >- + One row per water well, condensing that well's entire manual + groundwater-level record into a few numbers: how many measurements + exist, the most recent one and its date, the shallowest and deepest + ever recorded, and the long-term trend as a straight-line slope in + feet per year. Depths are below ground surface -- the measured depth + minus the height of the measuring point above ground, with readings + that have no recorded measuring-point height treated as taken at + ground level. Each row also carries the well's depth, its surveyed + ground elevation and how that elevation was determined, and the + geologic zone the well is completed in. Wells with no water-level + measurements at all are left out. Use it as the at-a-glance record for + a well before digging into individual readings. + keywords: [ + water-wells, summary, groundwater-level, water-level-history, trend, + well-depth, elevation, at-a-glance + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -144,15 +197,29 @@ resources: major_chemistry_results: type: collection title: Major Chemistry (Water Wells) - description: Latest major chemistry analyte values for water wells, represented as static analyte columns. - keywords: [water-wells, chemistry, analytes, major-chemistry] + description: >- + The major dissolved constituents that make up most of the chemistry of + groundwater -- calcium, magnesium, sodium, potassium, bicarbonate, + carbonate, sulfate and chloride -- alongside TDS, pH, hardness, + alkalinity and specific conductance. Laboratory records name the same + analyte in many different ways, so this layer first maps those names + and symbols onto one canonical set, then keeps the most recent result + for each analyte at each well and lays the values out as fixed + columns, each with its own units column. Analytes at one well may come + from different sampling dates; the reported chemistry date is the most + recent among them. Use it to compare water chemistry between wells or + to screen against drinking-water standards. + keywords: [ + water-wells, water-quality, chemistry, major-ions, analytes, calcium, + sodium, chloride, sulfate, ph, hardness + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -167,15 +234,27 @@ resources: minor_chemistry_wells: type: collection title: Minor Chemistry (Water Wells) - description: Latest minor/trace chemistry analyte values for water wells, represented as static analyte columns. - keywords: [water-wells, chemistry, analytes, minor-chemistry, trace-chemistry] + description: >- + Trace elements and isotopes measured in groundwater -- arsenic, + uranium, lead, iron, manganese, boron, lithium and dozens more, plus + the stable isotopes and carbon-14 used to work out how long water has + been underground. Built the same way as the major chemistry layer: + legacy laboratory records are mapped onto one canonical analyte set, + the most recent result for each analyte at each well is kept, and the + values are laid out as fixed columns each with its own units column. + Use it for contaminant screening and for questions about the age and + origin of groundwater. + keywords: [ + water-wells, water-quality, chemistry, trace-elements, minor-chemistry, + isotopes, arsenic, uranium, carbon-14, contaminants + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -190,15 +269,25 @@ resources: actively_monitored_wells: type: collection title: Actively Monitored Wells - description: Wells in the collaborative network currently flagged as actively monitored. - keywords: [water-wells, monitoring, collaborative-network, actively-monitored] + description: >- + The wells being measured today, rather than every well ever recorded. + A well appears here only if it belongs to the Water Level Network + group and its most recent monitoring-status entry reads "Currently + monitored"; the summary statistics attached to each one are the same + water-level figures published in water_well_summary. Use it to see the + live monitoring network -- where measurements are still being + collected, and where coverage is thin. + keywords: [ + water-wells, monitoring, water-level-network, actively-monitored, + monitoring-network, groundwater-level + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -213,15 +302,19 @@ resources: project_areas: type: collection title: Project Areas - description: Project groups with polygon project-area boundaries. - keywords: [project-areas, groups, boundaries] + description: >- + The study-area boundaries of Bureau projects, as polygons. Any project + group that has a mapped boundary is published here with its name and + description. Use it to see which part of New Mexico a project covers, + or to clip the other layers to a project's footprint. + keywords: [project-areas, study-areas, boundaries, polygons, projects, groups] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -238,15 +331,28 @@ resources: geothermal_wells_bht: type: collection title: Geothermal Wells — Bottom-Hole Temperature - description: Geothermal wells with bottom-hole temperature (BHT) measurements from the NM_Wells database. - keywords: [geothermal, wells, bottom-hole-temperature, bht] + description: >- + Bottom-hole temperature (BHT) is the temperature at the deepest point + of a borehole, usually recorded while drilling, and is the cheapest + broad indicator of how hot the subsurface is. This layer rolls every + BHT reading for a well in the legacy NM_Wells oil, gas and geothermal + records up into a single point: how many readings exist, the hottest + and coolest, the depth of the deepest, and those temperatures + converted to degrees Celsius. Source records mix Fahrenheit and + Celsius, so each well also carries a flag when its readings arrived in + mixed units and a count of any that could not be converted. Use it to + find warm areas worth closer investigation. + keywords: [ + geothermal, bottom-hole-temperature, bht, subsurface-temperature, wells, + nm-wells, celsius, heat-resource + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -261,15 +367,27 @@ resources: geothermal_wells_temperature_profile: type: collection title: Geothermal Wells — Temperature-Depth Profile - description: Geothermal wells with downhole temperature-vs-depth series from the NM_Wells database. - keywords: [geothermal, wells, temperature, depth, profile] + description: >- + How temperature changes with depth down a borehole, summarised as one + point per well. Every temperature-versus-depth reading logged for the + well is gathered into a single record: the number of readings, the + depth range they cover, the coolest and hottest values in degrees + Celsius, and the whole profile as a list of depth/temperature pairs. + Mixed source temperature units are flagged as they are for bottom-hole + temperatures. Use it to estimate the geothermal gradient -- how + quickly the ground warms with depth -- without pulling every + individual reading. + keywords: [ + geothermal, temperature-profile, temperature-depth, geothermal-gradient, + wells, nm-wells, celsius + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -284,15 +402,27 @@ resources: bht_measurements: type: collection title: BHT Measurements - description: Individual bottom-hole temperature measurements with well header and location data from the NM_Wells database. - keywords: [geothermal, bht, bottom-hole-temperature, measurements] + description: >- + Every individual bottom-hole temperature reading, one feature per + measurement, for consumers who need the raw record rather than the + per-well roll-up in geothermal_wells_bht. Each row carries the + temperature, the depth it was taken at, the date, and the hours since + drilling fluid was last circulated -- readings taken soon after + circulation are cooler than the rock itself, so that figure decides + whether a reading can be corrected. Well header details (operator, + well type, total depth, completion date, current status) and the + county are carried along for context. + keywords: [ + geothermal, bht, bottom-hole-temperature, measurements, raw-readings, + drilling, nm-wells + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -307,15 +437,24 @@ resources: temp_depth_measurements: type: collection title: Temperature-Depth Measurements - description: Individual downhole temperature readings with well header, location, and elevation data from the NM_Wells database. - keywords: [geothermal, temperature, depth, measurements] + description: >- + Every individual downhole temperature reading, one feature per + measurement, for consumers who need the raw record rather than the + per-well roll-up in geothermal_wells_temperature_profile. Each row + gives the temperature, the depth it was recorded at, the well it came + from and that well's elevation datum, so gradients can be recomputed + from scratch or checked against the summarised profile. + keywords: [ + geothermal, temperature, temperature-depth, downhole, measurements, + raw-readings, nm-wells + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -330,15 +469,27 @@ resources: heat_flow: type: collection title: Heat Flow - description: Summary heat-flow records with thermal conductivity, gradient, and publication attribution from the NM_Wells database. - keywords: [geothermal, heat-flow, thermal-conductivity, gradient] + description: >- + Heat flow is the rate at which the Earth's internal heat escapes + through the ground surface, and is the standard measure of geothermal + potential. Each row is one published determination over one depth + interval in one well, obtained by multiplying the temperature gradient + measured in the hole by the thermal conductivity of the rock. Values + recorded in the older heat-flow and conductivity units are republished + alongside SI equivalents (milliwatts per square metre, watts per + metre-kelvin), and every record carries the quality rating and the + literature citation it was published with. + keywords: [ + geothermal, heat-flow, thermal-conductivity, thermal-gradient, + geothermal-potential, nm-wells, publications + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -353,15 +504,24 @@ resources: dst: type: collection title: Drill Stem Tests - description: Drill stem test intervals with pressure, flow history, and well header data from the NM_Wells database. - keywords: [geothermal, dst, drill-stem-test, pressure, formation] + description: >- + A drill stem test is a temporary completion run while a well is still + being drilled: the drill pipe is opened against a chosen depth + interval so that formation fluid can flow in, and the pressures and + flow behaviour are recorded. Each row here is one tested interval -- + its depth range, target formation, packer settings, choke sizes and + gauge depth -- with the sequence of operations logged during the test + joined together in order as its flow history. Use it for formation + pressure and fluid evidence in wells that were never completed for + production. + keywords: [drill-stem-test, dst, formation-pressure, flow-history, reservoir, wells, nm-wells] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} diff --git a/core/pygeoapi.py b/core/pygeoapi.py index 849f4f381..392c2224a 100644 --- a/core/pygeoapi.py +++ b/core/pygeoapi.py @@ -10,6 +10,8 @@ import yaml from fastapi import FastAPI +from core.pygeoapi_patches import apply_queryables_patch + # Consumed by pygeoapi at import time only; see _load_pygeoapi_app. _PYGEOAPI_ENV_KEYS = ("PYGEOAPI_CONFIG", "PYGEOAPI_OPENAPI") @@ -19,61 +21,144 @@ "title": "Water Wells", "thing_type": "water well", "description": ( - "Groundwater wells used for monitoring, production, and " - "hydrogeologic investigations." + "Groundwater wells: drilled or dug access points into an aquifer, " + "used for monitoring, production, and hydrogeologic investigation. " + "Each feature is one well from the monitoring-point register, placed " + "at the most recent location recorded for it, and carries the " + "construction details held for it -- total and hole depth, casing " + "diameter and depth, completion date, driller, construction method, " + "pump type and depth, and the geologic formation it is completed in. " + "This is the starting point for groundwater work: the water-level and " + "chemistry layers are all derived from these same wells." ), - "keywords": ["well", "groundwater", "water-well"], + "keywords": [ + "water-wells", + "wells", + "groundwater", + "aquifer", + "monitoring-points", + "well-construction", + ], }, { "id": "springs", "title": "Springs", "thing_type": "spring", "description": ( - "Natural spring features and associated spring monitoring points." + "Springs: places where groundwater reaches the land surface under its " + "own pressure, without pumping. Each feature is one spring from the " + "monitoring-point register, placed at the most recent location " + "recorded for it. Use it to map natural groundwater discharge, the " + "groundwater contribution to streamflow, and the water sources that " + "support desert ecosystems." ), - "keywords": ["springs", "groundwater-discharge"], + "keywords": [ + "springs", + "groundwater-discharge", + "monitoring-points", + "surface-water", + "seeps", + ], }, { "id": "diversions_surface_water", "title": "Surface Water Diversions", "thing_type": "diversion of surface water, etc.", "description": ( - "Diversion structures such as ditches, canals, and intake points." + "Surface-water diversions: structures that take water out of a " + "stream, river, or canal -- ditches, acequias, headgates and intakes. " + "Each feature is one diversion from the monitoring-point register, " + "placed at the most recent location recorded for it. Use it to see " + "where surface water is withdrawn and to pair those points with " + "downstream flow records." ), - "keywords": ["surface-water", "diversion"], + "keywords": [ + "surface-water", + "diversion", + "ditches", + "acequias", + "headgates", + "monitoring-points", + ], }, { "id": "ephemeral_streams", "title": "Ephemeral Streams", "thing_type": "ephemeral stream", "description": ( - "Stream reaches that flow only in direct response to " - "precipitation events." + "Ephemeral stream reaches: channels that carry water only in direct " + "response to rain or snowmelt and are dry the rest of the year. Each " + "feature is one monitored reach from the register, placed at the most " + "recent location recorded for it. Use it for flash-flow and " + "storm-response work, and to distinguish these channels from reaches " + "that flow year-round." ), - "keywords": ["ephemeral-stream", "surface-water"], + "keywords": [ + "ephemeral-stream", + "surface-water", + "intermittent-flow", + "storm-response", + "monitoring-points", + ], }, { "id": "lakes_ponds_reservoirs", "title": "Lakes, Ponds, and Reservoirs", "thing_type": "lake, pond or reservoir", - "description": "Surface-water bodies monitored as feature locations.", - "keywords": ["lake", "pond", "reservoir", "surface-water"], + "description": ( + "Standing bodies of surface water monitored as sites -- natural " + "lakes, ponds, and built reservoirs. Each feature is one water body " + "from the monitoring-point register, placed at the most recent " + "location recorded for it. Use it for storage and surface-water " + "quality work, and as context for nearby groundwater levels." + ), + "keywords": [ + "lake", + "pond", + "reservoir", + "surface-water", + "storage", + "monitoring-points", + ], }, { "id": "meteorological_stations", "title": "Meteorological Stations", "thing_type": "meteorological station", - "description": "Weather and climate monitoring station locations.", - "keywords": ["meteorological-station", "weather"], + "description": ( + "Weather and climate stations: sites that record conditions such as " + "precipitation, temperature, and evaporation. Each feature is one " + "station from the monitoring-point register, placed at the most " + "recent location recorded for it. Use it to relate groundwater and " + "streamflow behaviour to the weather that drives it." + ), + "keywords": [ + "meteorological-station", + "weather", + "climate", + "precipitation", + "monitoring-points", + ], }, { "id": "other_things", "title": "Other Thing Types", "thing_type": "other", "description": ( - "Feature records that do not match another defined thing type." + "Monitoring points that do not fall into any of the defined feature " + "types. Each feature is one such point from the register, placed at " + "the most recent location recorded for it. The set is small and " + "mixed, with no shared meaning between its members, so it is " + "published only on the internal mount for staff triage -- typically " + "to find records that need reclassifying." ), - "keywords": ["other"], + "keywords": [ + "other", + "unclassified", + "monitoring-points", + "internal", + "triage", + ], # "Thing" is internal data-model vocabulary and "other" names no # recognisable feature class, so this layer is not published on the # public mount (BDMS-979). Staff GIS clients still reach it through @@ -84,31 +169,79 @@ "id": "outfalls_wastewater_return_flow", "title": "Outfalls and Return Flow", "thing_type": "outfall of wastewater or return flow", - "description": "Outfall and return-flow monitoring points.", - "keywords": ["outfall", "return-flow", "surface-water"], + "description": ( + "Outfalls and return flow: points where treated wastewater or unused " + "irrigation water re-enters a stream or channel. Each feature is one " + "outfall from the monitoring-point register, placed at the most " + "recent location recorded for it. Use it in water-quality work, where " + "these points mark deliberate inputs to a watercourse." + ), + "keywords": [ + "outfall", + "return-flow", + "wastewater", + "surface-water", + "water-quality", + "monitoring-points", + ], }, { "id": "perennial_streams", "title": "Perennial Streams", "thing_type": "perennial stream", - "description": ("Stream reaches with continuous or near-continuous flow."), - "keywords": ["perennial-stream", "surface-water"], + "description": ( + "Perennial stream reaches: channels that flow year-round in most " + "years, sustained between storms by groundwater discharge. Each " + "feature is one monitored reach from the register, placed at the most " + "recent location recorded for it. Use it for base-flow and " + "surface-water/groundwater interaction work." + ), + "keywords": [ + "perennial-stream", + "surface-water", + "base-flow", + "streamflow", + "monitoring-points", + ], }, { "id": "rock_sample_locations", "title": "Rock Sample Locations", "thing_type": "rock sample location", - "description": ("Locations where rock samples were collected or documented."), - "keywords": ["rock-sample"], + "description": ( + "Places where rock samples were collected or outcrop geology was " + "documented. Each feature is one sample location from the " + "monitoring-point register, placed at the most recent location " + "recorded for it. Use it to find where physical samples backing " + "geologic mapping and laboratory analysis came from." + ), + "keywords": [ + "rock-sample", + "geology", + "sample-location", + "outcrop", + "monitoring-points", + ], }, { "id": "soil_gas_sample_locations", "title": "Soil Gas Sample Locations", "thing_type": "soil gas sample location", "description": ( - "Locations where soil gas measurements or samples were collected." + "Places where gas held in the pore space of soil was sampled. Each " + "feature is one sample location from the monitoring-point register, " + "placed at the most recent location recorded for it. Soil gas is used " + "to detect vapours rising from buried contamination or from geologic " + "sources, so these points usually mark contamination or " + "resource-exploration surveys." ), - "keywords": ["soil-gas", "sample-location"], + "keywords": [ + "soil-gas", + "sample-location", + "vapour-survey", + "contamination", + "monitoring-points", + ], }, ] @@ -120,11 +253,25 @@ "id": "waterlevels", "title": "Water Levels", "description": ( - "Depth-to-water observations (manual readings and continuous " - "transducer time series) served as OGC API - EDR coverages. " - "Each transducer deployment is exposed as an EDR instance." + "Depth-to-water through time at each well, served as time series " + "rather than as one point per well. Two kinds of record are combined: " + "manual measurements taken by field staff during a visit, and " + "continuous records from pressure transducers left down the well, " + "which log automatically at a fixed interval. Each transducer " + "deployment is exposed as its own EDR instance, so a well's record " + "can be read deployment by deployment or as a whole. Use it to plot " + "hydrographs and to see how water levels respond to pumping, " + "recharge, and drought." ), - "keywords": ["groundwater", "water-level", "depth-to-water", "edr"], + "keywords": [ + "groundwater", + "water-level", + "depth-to-water", + "time-series", + "hydrograph", + "transducer", + "edr", + ], "table": "ogc_waterlevels", "instance_field": "deployment_id", }, @@ -132,10 +279,22 @@ "id": "water_chemistry", "title": "Water Chemistry", "description": ( - "Water-chemistry analyses keyed by analyte, served as OGC API - " - "EDR coverages." + "Water-chemistry analyses through time, one record per analyte per " + "sample, served as time series. The layer draws together the major, " + "minor and trace, and field-parameter analysis records from the " + "legacy chemistry tables, keyed by the analyte name as the laboratory " + "recorded it. Use it to follow one constituent at one site over time " + "-- the chemistry feature layers, by contrast, give the latest value " + "for every analyte at once." ), - "keywords": ["water-chemistry", "analyte", "edr"], + "keywords": [ + "water-chemistry", + "water-quality", + "analyte", + "time-series", + "laboratory-results", + "edr", + ], "table": "ogc_water_chemistry", "instance_field": None, }, @@ -272,7 +431,7 @@ def _thing_collections_block( "providers": [ { "type": "feature", - "name": "PostgreSQL", + "name": "core.feature_provider.DescribedPostgreSQLProvider", "data": { "host": host, "port": port, @@ -494,6 +653,11 @@ def _load_pygeoapi_app(instance: str, config_path: Path, openapi_path: Path): # handlers of the app already built for the first one -- both mounts end # up serving whichever config was loaded last. Give each mount its own # module object so the two sets of globals can never alias. + # Before the module is executed, so the mount's handlers resolve the + # patched queryables function. Idempotent and process-wide by nature: + # pygeoapi.api.itemtypes is one module object shared by both mounts. + apply_queryables_patch() + module_name = "pygeoapi.starlette_app" spec = find_spec(module_name) if spec is None or spec.loader is None: diff --git a/core/pygeoapi_patches.py b/core/pygeoapi_patches.py new file mode 100644 index 000000000..355137627 --- /dev/null +++ b/core/pygeoapi_patches.py @@ -0,0 +1,130 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Runtime patches over pygeoapi. + +Only one, and only because pygeoapi leaves no hook for it. +``get_collection_schema`` copies a provider's field entries into its response +wholesale, so the documentation ``DescribedPostgreSQLProvider`` attaches +reaches the client untouched. ``get_collection_queryables`` instead builds a +fresh dict per property and hardcodes ``'title': k`` -- the raw column name -- +dropping every description on the floor. + +Rather than fork the 130-line handler, this wraps it and merges the +provider's own ``title``/``description`` into the JSON it returned. The cost +is a JSON round trip on a low-traffic endpoint; the benefit is that the rest +of pygeoapi's logic (property filtering, domains, enums, roles) stays theirs. + +Read docs/ogc-field-descriptions.md before changing this, and re-check it on +any pygeoapi upgrade. +""" + +import json +import logging + +LOGGER = logging.getLogger(__name__) + +# Keys taken from the provider's field entry when the handler dropped them. +DOCUMENTATION_KEYS = ("title", "description", "x-ogc-unit", "x-ogc-unitLang", "enum") + +# Keys the handler may have filled in itself, which we must not overwrite -- +# ?profile=actual-domain asks for the live values in the data, and those beat +# our authored vocabulary. +PRESERVED_KEYS = frozenset({"enum"}) + +_QUERYABLES_PATCHED = False + + +def _documented_fields(api, dataset): + """The provider's annotated fields for ``dataset``, or ``{}``. + + Never raises: queryables must keep working for a collection whose backing + view is missing, exactly as it did before this patch. + """ + try: + from pygeoapi.plugin import load_plugin + from pygeoapi.provider import get_provider_by_type + + providers = api.config["resources"][dataset]["providers"] + # Builds a second provider for the request: the handler's own instance + # is local to it. That costs one table reflection on an endpoint that + # is queried rarely and cached downstream. + provider = load_plugin("provider", get_provider_by_type(providers, "feature")) + return provider.fields or {} + except Exception as err: # noqa: BLE001 - documentation is never fatal + LOGGER.debug("No documented fields available for %s: %s", dataset, err) + return {} + + +def _merge_documentation(payload: str, fields: dict) -> str: + document = json.loads(payload) + properties = document.get("properties") + if not isinstance(properties, dict): + return payload + + for name, prop in properties.items(): + field = fields.get(name) + if not isinstance(field, dict): + continue + for key in DOCUMENTATION_KEYS: + value = field.get(key) + if value is None: + continue + if key in PRESERVED_KEYS and prop.get(key): + continue + prop[key] = value + + return json.dumps(document, indent=4) + + +def apply_queryables_patch() -> None: + """Make /collections/{id}/queryables carry the provider's field prose. + + Idempotent, and deliberately not config-dependent: pygeoapi.api.itemtypes + is a single module object shared by both mounts, and starlette_app + resolves the handler off it per request, so patching once before either + mount is built covers both. + """ + global _QUERYABLES_PATCHED + if _QUERYABLES_PATCHED: + return + + import pygeoapi.api.itemtypes as itemtypes + + original = itemtypes.get_collection_queryables + + def get_collection_queryables(api, request, dataset=None): + headers, status, content = original(api, request, dataset) + + # Leave HTML rendering, errors, and anything unparseable alone. + if status != 200 or not isinstance(content, str): + return headers, status, content + if not headers.get("Content-Type", "").startswith("application/schema+json"): + return headers, status, content + + fields = _documented_fields(api, dataset) + if not fields: + return headers, status, content + + try: + return headers, status, _merge_documentation(content, fields) + except (ValueError, TypeError) as err: + LOGGER.warning("Could not annotate queryables for %s: %s", dataset, err) + return headers, status, content + + get_collection_queryables.__wrapped__ = original + itemtypes.get_collection_queryables = get_collection_queryables + _QUERYABLES_PATCHED = True + LOGGER.debug("Patched pygeoapi get_collection_queryables for field descriptions.") diff --git a/docs/ogc-field-descriptions.md b/docs/ogc-field-descriptions.md new file mode 100644 index 000000000..54f134b25 --- /dev/null +++ b/docs/ogc-field-descriptions.md @@ -0,0 +1,155 @@ +# OGC field descriptions + +Collection-level prose — `title`, `description`, `keywords` — lives in +`core/pygeoapi.py` (`THING_COLLECTIONS`, `EDR_COLLECTIONS`) and the two +pygeoapi config templates. This document covers the level below it: what an +individual **column** means, and what unit it is in. + +Published through the standard endpoints: + +| Endpoint | Standard | Carries | +| --- | --- | --- | +| `GET /collections/{id}/schema` | OGC API - Features Part 5 (draft), Common Part 3 | `title`, `description`, `x-ogc-unit`, `x-ogc-unitLang`, plus pygeoapi's own `x-ogc-role` | +| `GET /collections/{id}/queryables` | OGC API - Features Part 3 | the same `title` and `description` | + +Both mounts serve both endpoints. EDR collections additionally carry the +documentation in their CoverageJSON `parameters` block +(`observedProperty.label` and `description`). + +## Where the copy lives + +`core/ogc-field-descriptions.yml`, keyed by **backing relation** with the +`ogc_` / `ogc_internal_` prefix stripped: + +```yaml +_defaults: + id: + title: Feature ID + description: >- + Stable identifier for this feature within the collection. + +water_well_summary: + water_level_trend_ft_per_year: + title: Water-level trend + description: >- + Slope of a straight line fitted through the well's depth-to-water + measurements over time, in feet per year. Positive means the water + table is falling. + x-ogc-unit: https://qudt.org/vocab/unit/FT-PER-YR + x-ogc-unitLang: QUDT +``` + +Keying by relation rather than collection id means the public and internal +mounts share one entry per view, and the provider looks itself up from +`self.table` with no extra plumbing. `_defaults` applies everywhere and a +per-table entry wins over it — which matters for a name like `description`, +whose meaning differs between `locations` and `project_areas`. + +Allowed keys: `title`, `description`, `x-ogc-unit`, `x-ogc-unitLang`, +`x-ogc-propertySeq`, `enum`, `enum-lexicon`. Anything else fails validation at +load. Types and formats come from the provider's reflection and must never be +set here. + +### Enumerated values + +`enum` is what pygeoapi's HTML schema view renders as its **Values** column, +and nothing else fills it in — the SQL provider reports only `type` and +`format`, and implements no `get_domains()`, so `?profile=actual-domain` has +nothing to offer either. + +Two ways to set it, and the distinction matters: + +```yaml +trend_category: + enum: [increasing, decreasing, stable, not enough data] # set by the view's SQL + +well_pump_type: + enum-lexicon: well_pump_type # a controlled vocabulary +``` + +Use a literal `enum` only when the view's own SQL decides the values — a `CASE` +expression or a stamped literal. Use `enum-lexicon` for anything the lexicon +governs; it names a category in `core/lexicon.json` and is expanded on the way +out, so the vocabulary is never copied. A category with no terms fails +validation at load rather than publishing an empty column. + +`enum` is a JSON Schema constraint, not a sample: it says these are the *valid* +values. Do not populate it from `SELECT DISTINCT` — a value absent from today's +data is not thereby invalid. + +On `/queryables` an `enum` pygeoapi produced itself wins over the authored one, +so `?profile=actual-domain` still reports the live domain where a provider +supports it. + +### Adding a field + +1. Add the entry under the table's block, or under `_defaults` if the column + means the same thing in every view that has it. +2. Say what the value means and what its datum or convention is — not how the + view is built. That belongs in the collection description. +3. Run `uv run pytest tests/test_ogc_field_descriptions.py`. + +The 190 chemistry analyte columns are generated: + +```bash +uv run python -m cli.generate_chemistry_field_descriptions +``` + +Review the output and paste it into the YAML. The generator reads the analyte +lists out of the view migration — `core/parameter.json` holds only two field +parameters, so the lexicon cannot supply this. A hand-written entry in the YAML +wins over the generated one. + +### Why not `COMMENT ON COLUMN` + +It would put the prose next to the data, but every wording fix would need an +Alembic revision and a materialized-view rebuild, and pygeoapi's reflection does +not read column comments, so a catalog query would be needed anyway. This was +considered and rejected; please don't relitigate it without a new argument. + +## How it reaches the client + +`core/feature_provider.py` — `DescribedPostgreSQLProvider` — annotates the +reflected fields. Every feature collection in both config templates and in +`_thing_collections_block` names it as its provider. + +`core/pygeoapi_patches.py` wraps `get_collection_queryables`. + +`core/edr_provider.py` routes its parameter fields through the same YAML. + +## What this depends on inside pygeoapi (0.24.0) + +These are unpinned behaviours of a pinned version. **Re-check every one of them +when bumping pygeoapi**; `tests/test_ogc_field_descriptions.py` guards the first. + +1. **`pygeoapi/api/__init__.py::get_collection_schema`** (~line 1082) assigns the + provider's field entry into the response wholesale + (`schema['properties'][k] = v`), so anything the provider attaches passes + through. This is why `/schema` needs no patch. +2. It then **mutates that same dict in place** — `v.pop('format', None)`, and + assigns `x-ogc-role`. `describe_fields()` therefore returns fresh dicts; + handing out references into the cached YAML would let one request's + mutations leak into the next. +3. **`pygeoapi/api/itemtypes.py::get_collection_queryables`** (~line 198) builds + a fresh dict per property and hardcodes `'title': k`. Hence the patch. +4. **`pygeoapi/provider/base.py::BaseProvider.fields`** (~line 107) returns + `self._fields` directly and **never calls `get_fields()`**, while + `GenericSQLProvider.__init__` (~line 143) calls `get_fields()` once at + construction. A `get_fields()` override that only returns an annotated copy + is silently discarded — it must write back into `self._fields`. +5. **`pygeoapi/starlette_app.py`** imports `pygeoapi.api.itemtypes` as a module + and resolves handlers off it per request, so rebinding the module attribute + reaches both mounts even though each gets its own `starlette_app` module + object. + +## Deliberate gaps + +- **`geometry`** carries no title. pygeoapi injects it after the provider's + fields with only a `format` and `x-ogc-role`, so it is excluded from the + "every property is documented" assertions. +- **`ogc_water_chemistry` parameters** are the analyte text exactly as the + laboratory recorded it — open-ended, alias-ridden, and only knowable from the + data. Undocumented analytes get a generated title from the parameter name. +- **A missing entry never fails a request.** It yields a generated title + (`depth_to_water_bgs` → `Depth To Water Bgs`) and a logged warning. The drift + guard in the tests is what fails the build. diff --git a/tests/test_ogc_field_descriptions.py b/tests/test_ogc_field_descriptions.py new file mode 100644 index 000000000..6779a9461 --- /dev/null +++ b/tests/test_ogc_field_descriptions.py @@ -0,0 +1,410 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Field-level documentation on /schema and /queryables, both mounts. + +`geometry` is excluded from the "every property is documented" assertions +throughout: pygeoapi injects it itself, after the provider's fields, with only +a format and an x-ogc-role. +""" + +import pytest +from fastapi.testclient import TestClient + +from core.factory import create_api_app +from core.dependencies import ( + admin_function, + amp_admin_function, + amp_editor_function, + amp_viewer_function, + editor_function, + viewer_function, +) +from core.ogc_field_metadata import table_entries +from tests import override_authentication + +GEOMETRY_PROPERTY = "geometry" + + +@pytest.fixture(scope="module") +def ogc_client(): + app = create_api_app() + for dependency in ( + admin_function, + editor_function, + amp_admin_function, + amp_editor_function, + ): + app.dependency_overrides[dependency] = override_authentication( + default={"name": "foobar", "sub": "1234567890"} + ) + for dependency in (viewer_function, amp_viewer_function): + app.dependency_overrides[dependency] = override_authentication() + + with TestClient(app) as client: + yield client + + app.dependency_overrides = {} + + +def _documented_properties(payload): + return { + name: prop + for name, prop in payload["properties"].items() + if name != GEOMETRY_PROPERTY + } + + +# --------------------------------------------------------------------- schema + + +def test_schema_documents_every_property(ogc_client): + response = ogc_client.get("/ogcapi/collections/water_wells/schema") + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("application/schema+json") + for name, prop in _documented_properties(response.json()).items(): + assert prop.get("title"), f"{name} has no title" + + +def test_schema_carries_the_authored_prose(ogc_client): + properties = ogc_client.get("/ogcapi/collections/water_wells/schema").json()[ + "properties" + ] + + assert properties["well_depth"]["title"] == "Well depth" + assert properties["well_depth"]["description"].startswith( + "Total depth of the finished well" + ) + assert properties["well_depth"]["x-ogc-unit"] == "https://qudt.org/vocab/unit/FT" + assert properties["well_depth"]["x-ogc-unitLang"] == "QUDT" + assert properties["nma_formation_zone"]["title"] == "Legacy formation zone" + + +def test_schema_keeps_pygeoapi_roles(ogc_client): + # The annotation must not displace the roles pygeoapi assigns after the + # provider hands its fields over. + properties = ogc_client.get("/ogcapi/collections/water_wells/schema").json()[ + "properties" + ] + + assert properties["id"]["x-ogc-role"] == "id" + assert properties[GEOMETRY_PROPERTY]["x-ogc-role"] == "primary-geometry" + assert properties["first_visit_date"]["x-ogc-role"] == "primary-instant" + + +def test_schema_roles_do_not_leak_between_requests(ogc_client): + # describe_fields hands out fresh dicts precisely so that pygeoapi's + # in-place mutation of one response cannot reach the next one. + first = ogc_client.get("/ogcapi/collections/water_wells/schema").json() + second = ogc_client.get("/ogcapi/collections/water_wells/schema").json() + + assert first["properties"] == second["properties"] + assert "x-ogc-role" not in second["properties"]["well_depth"] + + +def test_derived_collections_document_their_calculated_columns(ogc_client): + properties = ogc_client.get( + "/ogcapi/collections/depth_to_water_trend_wells/schema" + ).json()["properties"] + + assert properties["slope_ft_per_year"]["title"] == "Trend slope" + assert "water table is falling" in properties["slope_ft_per_year"]["description"] + assert properties["trend_category"]["description"].startswith( + "Plain-language reading of the slope" + ) + + +def test_chemistry_analyte_columns_are_documented(ogc_client): + properties = ogc_client.get( + "/ogcapi/collections/major_chemistry_results/schema" + ).json()["properties"] + + assert properties["tds"]["title"] == "Total dissolved solids" + assert properties["tds_units"]["title"] == "Total dissolved solids units" + assert properties["ion_balance"]["description"].startswith("Percentage difference") + + +# ---------------------------------------------------------------- both mounts + + +@pytest.mark.parametrize("mount", ["/ogcapi", "/ogcapi-internal"]) +@pytest.mark.parametrize("endpoint", ["schema", "queryables"]) +def test_both_mounts_document_water_wells(ogc_client, mount, endpoint): + response = ogc_client.get(f"{mount}/collections/water_wells/{endpoint}") + + assert response.status_code == 200 + properties = _documented_properties(response.json()) + assert properties["well_depth"]["title"] == "Well depth" + for name, prop in properties.items(): + assert prop.get("title"), f"{mount} {endpoint}: {name} has no title" + + +def test_internal_only_collection_is_documented(ogc_client): + # other_things is published on the internal mount only (BDMS-979); its + # backing view is ogc_internal_other_things, so the "ogc_internal_" + # prefix has to be stripped for the lookup to land. + response = ogc_client.get("/ogcapi-internal/collections/other_things/schema") + + assert response.status_code == 200 + properties = _documented_properties(response.json()) + assert properties["well_depth"]["title"] == "Well depth" + assert properties["release_status"]["description"] + + +# ----------------------------------------------------------------- rendering + + +@pytest.mark.parametrize("endpoint", ["schema", "queryables"]) +@pytest.mark.parametrize("fmt", ["json", "html"]) +def test_endpoints_render_in_both_formats(ogc_client, endpoint, fmt): + response = ogc_client.get( + f"/ogcapi/collections/water_wells/{endpoint}", params={"f": fmt} + ) + + assert response.status_code == 200 + + +# --------------------------------------------------------------- drift guard + + +def _feature_collection_tables(client, mount): + payload = client.get(f"{mount}/collections").json() + return [collection["id"] for collection in payload["collections"]] + + +def test_every_published_column_has_an_entry(ogc_client): + """A renamed or added matview column must fail here, not degrade the API. + + EDR collections are excluded on purpose: their fields are analyte names + read out of the data, not the backing view's columns. + """ + undocumented = {} + + for mount in ("/ogcapi", "/ogcapi-internal"): + for collection_id in _feature_collection_tables(ogc_client, mount): + response = ogc_client.get(f"{mount}/collections/{collection_id}/schema") + if response.status_code != 200: + continue + payload = response.json() + if payload.get("type") != "object": + continue + gaps = [ + name + for name, prop in _documented_properties(payload).items() + if not prop.get("description") + ] + if gaps: + undocumented[f"{mount}/{collection_id}"] = sorted(gaps) + + assert not undocumented, f"columns with no YAML entry: {undocumented}" + + +def test_fallback_title_for_an_undocumented_column(): + # The fallback path itself, without needing a real undocumented column in + # the database. + from core.ogc_field_metadata import describe_fields + + described = describe_fields( + "ogc_water_wells", {"brand_new_column": {"type": "string"}} + ) + + assert described["brand_new_column"]["title"] == "Brand New Column" + assert "description" not in described["brand_new_column"] + + +def test_defaults_cover_the_shared_thing_columns(): + # All 11 thing-type views share one column signature, so a gap in + # _defaults would hit every one of them at once. + entries = table_entries("ogc_springs") + + for column in ( + "id", + "name", + "first_visit_date", + "well_depth", + "release_status", + "elevation", + ): + assert entries[column]["description"], f"{column} lost its default entry" + + +# ------------------------------------------------------------- upgrade guard + + +def test_pygeoapi_still_passes_provider_fields_through_to_schema(): + """Guard on the pygeoapi behaviour this whole feature rests on. + + `get_collection_schema` copies each provider field entry into the response + wholesale, which is why documentation set by the provider reaches the + client. A pygeoapi bump that rebuilds the dict instead -- the way + `get_collection_queryables` already does -- would silently drop every + description. Fail loudly here instead. + """ + import inspect + + from pygeoapi.api import get_collection_schema + + source = inspect.getsource(get_collection_schema) + + assert "schema['properties'][k] = v" in source, ( + "pygeoapi no longer assigns the provider's field entry into the schema " + "response; /schema descriptions need re-checking against the new " + "implementation (see docs/ogc-field-descriptions.md)." + ) + + +# ----------------------------------------------------------------------- EDR + + +def test_edr_schema_documents_its_parameter(ogc_client): + response = ogc_client.get("/ogcapi/collections/waterlevels/schema") + + assert response.status_code == 200 + properties = response.json()["properties"] + if "groundwater level" not in properties: + # EDR fields are read out of the data, not reflected from columns, so + # this assertion only has something to bite on when the suite has left + # water-level rows behind. The CoverageJSON test below covers the same + # lookup without needing any. + pytest.skip("no water-level rows in ogc_waterlevels for this database state") + parameter = properties["groundwater level"] + assert parameter["title"] == "Groundwater level" + assert parameter["description"].startswith("Depth from the measuring point") + + +def test_edr_coveragejson_carries_the_parameter_description(): + # Exercises the CoverageJSON parameters block without a database: the + # provider's __init__ opens a connection, which this does not need. + from datetime import datetime + + from core.edr_provider import WaterEDRProvider + + provider = object.__new__(WaterEDRProvider) + provider.table = "ogc_waterlevels" + + coverage = provider._coverage_collection( + [ + { + "thing_id": 1, + "station_name": "Test well", + "longitude": -106.0, + "latitude": 34.0, + "datetime": datetime(2024, 1, 1), + "value": 42.0, + "unit": "ft", + "parameter_name": "groundwater level", + } + ] + ) + + parameter = coverage["parameters"]["groundwater level"] + assert parameter["observedProperty"]["label"]["en"] == "Groundwater level" + assert parameter["description"]["en"].startswith("Depth from the measuring point") + + +def test_edr_falls_back_for_an_undocumented_analyte(): + from datetime import datetime + + from core.edr_provider import WaterEDRProvider + + provider = object.__new__(WaterEDRProvider) + provider.table = "ogc_water_chemistry" + + coverage = provider._coverage_collection( + [ + { + "thing_id": 1, + "station_name": "Test well", + "longitude": -106.0, + "latitude": 34.0, + "datetime": datetime(2024, 1, 1), + "value": 1.0, + "unit": "mg/L", + "parameter_name": "Some Unmapped Analyte", + } + ] + ) + + parameter = coverage["parameters"]["Some Unmapped Analyte"] + assert parameter["observedProperty"]["label"]["en"] == "Some Unmapped Analyte" + assert parameter["description"]["en"] == "Some Unmapped Analyte" + + +# ---------------------------------------------------------- enumerated values + + +def test_schema_publishes_enumerated_values(ogc_client): + # pygeoapi's HTML schema view renders `enum` as its "Values" column, and + # nothing fills it in: the SQL provider reports only type and format, and + # implements no get_domains(). These come from the YAML. + properties = ogc_client.get( + "/ogcapi/collections/depth_to_water_trend_wells/schema" + ).json()["properties"] + + assert properties["trend_category"]["enum"] == [ + "increasing", + "decreasing", + "stable", + "not enough data", + ] + + +def test_queryables_publishes_enumerated_values(ogc_client): + properties = ogc_client.get( + "/ogcapi/collections/depth_to_water_trend_wells/queryables" + ).json()["properties"] + + assert "not enough data" in properties["trend_category"]["enum"] + + +def test_lexicon_backed_enums_come_from_the_lexicon(ogc_client): + from core.ogc_field_metadata import lexicon_terms + + properties = ogc_client.get("/ogcapi/collections/water_wells/schema").json()[ + "properties" + ] + + assert properties["well_pump_type"]["enum"] == lexicon_terms("well_pump_type") + assert properties["well_construction_method"]["enum"] == lexicon_terms( + "well_construction_method" + ) + assert properties["release_status"]["enum"] == lexicon_terms("release_status") + + +def test_enum_lexicon_key_is_expanded_not_published(ogc_client): + # The YAML shorthand must not reach the client. + properties = ogc_client.get("/ogcapi/collections/water_wells/schema").json()[ + "properties" + ] + + assert "enum-lexicon" not in properties["well_pump_type"] + + +def test_enum_entries_validate_against_the_lexicon(): + # A category with no terms is a typo, and it must fail at load rather than + # publish an empty Values column. + import pytest as _pytest + + from core.ogc_field_metadata import _validate + + with _pytest.raises(ValueError, match="no terms"): + _validate( + {"water_wells": {"well_pump_type": {"title": "x", "enum-lexicon": "nope"}}}, + "test.yml", + ) + + with _pytest.raises(ValueError, match="non-empty list"): + _validate({"water_wells": {"a_column": {"title": "x", "enum": []}}}, "test.yml") diff --git a/tests/test_ogc_field_metadata.py b/tests/test_ogc_field_metadata.py new file mode 100644 index 000000000..494825e71 --- /dev/null +++ b/tests/test_ogc_field_metadata.py @@ -0,0 +1,123 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Unit tests for core/ogc_field_metadata.py. No database, no pygeoapi.""" + +import pytest + +from core.ogc_field_metadata import ( + ALLOWED_KEYS, + default_title, + describe_fields, + load_field_metadata, + strip_table_prefix, + table_entries, +) + + +def test_metadata_file_loads_and_validates(): + metadata = load_field_metadata() + + assert "_defaults" in metadata + for table, fields in metadata.items(): + for field, entry in fields.items(): + assert entry["title"], f"{table}.{field} has no title" + assert not set(entry) - ALLOWED_KEYS + + +@pytest.mark.parametrize( + "table,expected", + [ + ("ogc_water_wells", "water_wells"), + ("ogc_internal_water_wells", "water_wells"), + ("water_wells", "water_wells"), + # "ogc_internal_" has to be stripped before "ogc_", or the internal + # tables would look up an "internal_water_wells" block that has no + # entries and every field would fall back. + ("ogc_internal_other_things", "other_things"), + ], +) +def test_strip_table_prefix(table, expected): + assert strip_table_prefix(table) == expected + + +def test_default_title(): + assert default_title("depth_to_water_bgs") == "Depth To Water Bgs" + assert default_title("id") == "Id" + + +def test_table_entries_merge_defaults_under_the_table_block(): + entries = table_entries("ogc_water_well_summary") + + # From _defaults. + assert entries["id"]["title"] == "Feature ID" + # From the table block. + assert entries["total_water_levels"]["title"] == "Water-level measurement count" + + +def test_table_entries_prefer_the_table_block_over_defaults(): + # locations.description is the site description, not any default. + assert table_entries("ogc_locations")["description"]["title"] == "Site description" + assert table_entries("ogc_project_areas")["description"]["title"] == ( + "Project description" + ) + + +def test_describe_fields_annotates_documented_columns(): + described = describe_fields( + "ogc_internal_water_wells", + {"well_depth": {"type": "number", "format": None}}, + ) + + field = described["well_depth"] + assert field["type"] == "number" + assert field["format"] is None + assert field["title"] == "Well depth" + assert field["description"].startswith("Total depth of the finished well") + assert field["x-ogc-unit"] == "https://qudt.org/vocab/unit/FT" + assert field["x-ogc-unitLang"] == "QUDT" + + +def test_describe_fields_falls_back_without_raising(caplog): + described = describe_fields( + "ogc_water_wells", {"not_documented": {"type": "string"}} + ) + + assert described["not_documented"] == { + "type": "string", + "title": "Not Documented", + } + assert "not_documented" in caplog.text + + +def test_describe_fields_returns_fresh_dicts(): + # pygeoapi's get_collection_schema assigns the provider's field dict into + # its response and then mutates it in place. Handing out references into + # the cached YAML would let one request's mutations leak into the next. + fields = {"well_depth": {"type": "number"}} + + first = describe_fields("ogc_water_wells", fields) + first["well_depth"]["x-ogc-role"] = "id" + first["well_depth"].pop("description") + + second = describe_fields("ogc_water_wells", fields) + assert "x-ogc-role" not in second["well_depth"] + assert second["well_depth"]["description"] + assert fields["well_depth"] == {"type": "number"} + + +def test_describe_fields_tolerates_empty_input(): + assert describe_fields("ogc_water_wells", {}) == {} + assert describe_fields("ogc_water_wells", None) == {} diff --git a/tests/test_pygeoapi_mount.py b/tests/test_pygeoapi_mount.py index ee9b11df8..e267d1803 100644 --- a/tests/test_pygeoapi_mount.py +++ b/tests/test_pygeoapi_mount.py @@ -14,6 +14,7 @@ """ import os +import re import sys from core import pygeoapi @@ -134,3 +135,39 @@ def test_hidden_layers_are_internal_only(): # The thing-type layers that stay public are on both mounts; the two # catalogs otherwise differ (the geothermal layers are public-only). assert {"water_wells", "springs"}.issubset(public_ids & internal_ids) + + +# Wording that says nothing to a consumer reading the catalog cold: internal +# data-model vocabulary, or a description that only restates the layer name. +PLACEHOLDER_TERMS = ("todo", "tbd", "xxx", "placeholder", "example.com", "lorem") + + +def test_every_collection_description_explains_the_layer(): + # A description has to tell a non-specialist how the layer was derived and + # what it is for -- not repeat the title. Short entries are the failure + # mode this guards: they are what the catalog shipped with before. + for module in _load_both(): + for name, resource in module.api_.config["resources"].items(): + description = resource.get("description", "") + lowered = description.lower() + assert len(description) >= 200, f"{name} description is too thin" + assert not any( + term in lowered for term in PLACEHOLDER_TERMS + ), f"{name} description contains placeholder wording" + assert description.rstrip().endswith( + "." + ), f"{name} description is not a complete sentence" + # YAML folds a line break into a space, so a hyphenated word split + # across lines ("measuring-\npoint") reaches consumers as + # "measuring- point". Wrap on whitespace only. + assert not re.search( + r"\w- \w", description + ), f"{name} description has a hyphenated word split across lines" + + keywords = resource.get("keywords", []) + assert len(keywords) >= 4, f"{name} has too few keywords" + assert len(set(keywords)) == len(keywords), f"{name} repeats a keyword" + for keyword in keywords: + assert re.fullmatch( + r"[a-z0-9]+(?:-[a-z0-9]+)*", keyword + ), f"{name} keyword {keyword!r} is not a lowercase hyphenated token"