chore(ogc): hide four layers from the public catalog

README.md

@@ -39,22 +39,21 @@ hits to `/_ah/warmup`.
 curl http://localhost:8000/ogcapi
 curl http://localhost:8000/ogcapi/conformance
 curl http://localhost:8000/ogcapi/collections
-curl http://localhost:8000/ogcapi/collections/locations
+curl http://localhost:8000/ogcapi/collections/water_wells
 ```
 
 ### Items (GeoJSON)
 
 ```bash
-curl "http://localhost:8000/ogcapi/collections/locations/items?limit=10&offset=0"
-curl "http://localhost:8000/ogcapi/collections/water_wells/items?limit=5"
+curl "http://localhost:8000/ogcapi/collections/water_wells/items?limit=10&offset=0"
 curl "http://localhost:8000/ogcapi/collections/springs/items?limit=5"
-curl "http://localhost:8000/ogcapi/collections/locations/items/123"
+curl "http://localhost:8000/ogcapi/collections/water_wells/items/123"
 ```
 
 ### BBOX + datetime filters
 
 ```bash
-curl "http://localhost:8000/ogcapi/collections/locations/items?bbox=-107.9,33.8,-107.8,33.9"
+curl "http://localhost:8000/ogcapi/collections/water_wells/items?bbox=-107.9,33.8,-107.8,33.9"
 curl "http://localhost:8000/ogcapi/collections/water_wells/items?datetime=2020-01-01/2024-01-01"
 ```
 
@@ -63,7 +62,7 @@ curl "http://localhost:8000/ogcapi/collections/water_wells/items?datetime=2020-0
 Use `filter` + `filter-lang=cql2-text` with `WITHIN(...)`:
 
 ```bash
-curl "http://localhost:8000/ogcapi/collections/locations/items?filter=WITHIN(geometry,POLYGON((-107.9 33.8,-107.8 33.8,-107.8 33.9,-107.9 33.9,-107.9 33.8)))&filter-lang=cql2-text"
+curl "http://localhost:8000/ogcapi/collections/water_wells/items?filter=WITHIN(geometry,POLYGON((-107.9 33.8,-107.8 33.8,-107.8 33.9,-107.9 33.9,-107.9 33.8)))&filter-lang=cql2-text"
 ```
 
 ### OpenAPI UI

core/pygeoapi-config.yml

@@ -49,75 +49,6 @@ metadata:
     # publishes "pointOfContact" as the service's hours of operation.
 
 resources:
-  locations:
-    type: collection
-    title: Locations
-    description: Geographic locations and site coordinates used by Ocotillo features.
-    keywords: [locations]
-    extents:
-      spatial:
-        bbox: [-109.05, 31.33, -103.00, 37.00]
-        crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84
-    providers:
-      - type: feature
-        name: PostgreSQL
-        data:
-          host: {postgres_host}
-          port: {postgres_port}
-          dbname: {postgres_db}
-          user: {postgres_user}
-          password: {postgres_password_env}
-          search_path: [public]
-        id_field: id
-        table: ogc_locations
-        geom_field: point
-
-  latest_depth_to_water_wells:
-    type: collection
-    title: Latest Depth to Water (Water Wells)
-    description: Most recent depth-to-water below ground surface observation for each water well.
-    keywords: [water-wells, groundwater-level, depth-to-water-bgs, latest]
-    extents:
-      spatial:
-        bbox: [-109.05, 31.33, -103.00, 37.00]
-        crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84
-    providers:
-      - type: feature
-        name: PostgreSQL
-        data:
-          host: {postgres_host}
-          port: {postgres_port}
-          dbname: {postgres_db}
-          user: {postgres_user}
-          password: {postgres_password_env}
-          search_path: [public]
-        id_field: id
-        table: ogc_latest_depth_to_water_wells
-        geom_field: point
-
-  avg_tds_wells:
-    type: collection
-    title: Average TDS (Water Wells)
-    description: Average total dissolved solids (TDS) from major chemistry results for each water well.
-    keywords: [water-wells, chemistry, tds, total-dissolved-solids, average]
-    extents:
-      spatial:
-        bbox: [-109.05, 31.33, -103.00, 37.00]
-        crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84
-    providers:
-      - type: feature
-        name: PostgreSQL
-        data:
-          host: {postgres_host}
-          port: {postgres_port}
-          dbname: {postgres_db}
-          user: {postgres_user}
-          password: {postgres_password_env}
-          search_path: [public]
-        id_field: id
-        table: ogc_avg_tds_wells
-        geom_field: point
-
   latest_tds_wells:
     type: collection
     title: Latest TDS (Water Wells)

core/pygeoapi.py

@@ -74,6 +74,11 @@
             "Feature records that do not match another defined thing type."
         ),
         "keywords": ["other"],
+        # "Thing" is internal data-model vocabulary and "other" names no
+        # recognisable feature class, so this layer is not published on the
+        # public mount (BDMS-979). Staff GIS clients still reach it through
+        # /ogcapi-internal, and ogc_other_things is retained either way.
+        "internal_only": True,
     },
     {
         "id": "outfalls_wastewater_return_flow",
@@ -247,9 +252,12 @@ def _thing_collections_block(
     user: str,
     password_placeholder: str,
     table_prefix: str = "ogc_",
+    include_internal_only: bool = False,
 ) -> str:
     resources: dict[str, dict] = {}
     for collection in THING_COLLECTIONS:
+        if collection.get("internal_only") and not include_internal_only:
+            continue
         resources[collection["id"]] = {
             "type": "collection",
             "title": collection["title"],
@@ -384,6 +392,7 @@ def _write_config(
     table_prefix: str = "ogc_",
     template_path: Path | None = None,
     include_edr: bool = False,
+    include_internal_only: bool = False,
 ) -> None:
     host, port, dbname, user, password_placeholder = _pygeoapi_db_settings()
     template = (template_path or _template_path()).read_text(encoding="utf-8")
@@ -394,6 +403,7 @@ def _write_config(
         user=user,
         password_placeholder=password_placeholder,
         table_prefix=table_prefix,
+        include_internal_only=include_internal_only,
     )
     if include_edr:
         # EDR collections (core/edr_provider.py), backed by
@@ -564,6 +574,7 @@ def mount_pygeoapi_internal(app: FastAPI) -> None:
         table_prefix="ogc_internal_",
         template_path=_internal_template_path(),
         include_edr=True,
+        include_internal_only=True,
     )
     _generate_openapi(config_path, openapi_path)
     _assert_server_settings_match(_pygeoapi_dir() / "pygeoapi-config.yml", config_path)

tests/features/ogc-cleanup-sprint1.feature

@@ -51,23 +51,21 @@ Feature: OGC Feature Layer Cleanup — Sprint 1
       | meteorological_stations       |
       | diversions_surface_water      |
       | lakes_ponds_reservoirs        |
-      | other_things                  |
       | water_well_summary            |
       | depth_to_water_trend_wells    |
       | water_elevation_wells         |
       | major_chemistry_results       |
       | minor_chemistry_wells         |
       | latest_tds_wells              |
       | actively_monitored_wells      |
-      | avg_tds_wells                 |
-      | latest_depth_to_water_wells   |
-      | locations                     |
       | project_areas                 |
     Then each response contains only records where release_status is "public"
     And no response contains a record where release_status is "private"
     And no response contains a record where release_status is "draft"
-    # other_things above: A1 must apply the filter to its view, but A18 removes
-    # other_things from the catalog — run this scenario before A18 is applied
+    # other_things, avg_tds_wells, latest_depth_to_water_wells and locations
+    # are not listed above: A16/A17/A18 took them off the public catalog, so a
+    # public client can no longer request their items. A1's filter still
+    # applies to their views, which the SQL-level scenarios above cover.
 
   @backend @ogc-exposure @sprint-1 @high-priority @A1 @production
   Scenario: project_areas returns 56 rows after all records are updated to public
@@ -237,10 +235,9 @@ Feature: OGC Feature Layer Cleanup — Sprint 1
       | lakes_ponds_reservoirs            |
       | soil_gas_sample_locations         |
       | outfalls_wastewater_return_flow   |
-      | other_things                      |
     Then each feature includes a last_observation_date property
-    # other_things above: included in Group A view template, but A18 removes it
-    # from the catalog — run this scenario before A18 is applied
+    # other_things is not listed: it is in the Group A view template, but A18
+    # took it off the public catalog — it is only reachable on /ogcapi-internal.
 
   @backend @ogc-data-currency @sprint-1 @medium-priority @A13
   Scenario: last_observation_date is NULL for things with no associated observations
@@ -256,11 +253,10 @@ Feature: OGC Feature Layer Cleanup — Sprint 1
       | lakes_ponds_reservoirs            |
       | soil_gas_sample_locations         |
       | outfalls_wastewater_return_flow   |
-      | other_things                      |
     When a client requests those features
     Then each feature's last_observation_date property is null
-    # other_things above: included in Group A view template, but A18 removes it
-    # from the catalog — run this scenario before A18 is applied
+    # other_things is not listed: it is in the Group A view template, but A18
+    # took it off the public catalog — it is only reachable on /ogcapi-internal.
 
   @backend @ogc-data-currency @sprint-1 @medium-priority @A13
   Scenario: Consumers can filter Group A layers by last_observation_date
@@ -276,30 +272,29 @@ Feature: OGC Feature Layer Cleanup — Sprint 1
       | lakes_ponds_reservoirs          |
       | soil_gas_sample_locations       |
       | outfalls_wastewater_return_flow |
-      | other_things                    |
     When a client requests items from each of those layers with filter
       """
       last_observation_date > '2021-01-01'
       """
     Then only features with a last_observation_date of "2023-06-01" are returned from each layer
-    # other_things above: included in Group A view template, but A18 removes it
-    # from the catalog — run this scenario before A18 is applied
+    # other_things is not listed: it is in the Group A view template, but A18
+    # took it off the public catalog — it is only reachable on /ogcapi-internal.
 
   # ---------------------------------------------------------------------------
   # A16 — Hide avg_tds_wells and latest_depth_to_water_wells from public catalog
   # ---------------------------------------------------------------------------
 
-  @backend @ogc-data-currency @sprint-1 @medium-priority @A16
+  @backend @ogc-data-currency @sprint-1 @medium-priority @A16 @production
   Scenario: avg_tds_wells is absent from the public collections catalog
     When a client requests /ogcapi/collections
     Then the response does not include a collection with id avg_tds_wells
 
-  @backend @ogc-data-currency @sprint-1 @medium-priority @A16
+  @backend @ogc-data-currency @sprint-1 @medium-priority @A16 @production
   Scenario: latest_depth_to_water_wells is absent from the public collections catalog
     When a client requests /ogcapi/collections
     Then the response does not include a collection with id latest_depth_to_water_wells
 
-  @backend @ogc-data-currency @sprint-1 @medium-priority @A16
+  @backend @ogc-data-currency @sprint-1 @medium-priority @A16 @production
   Scenario: Backing matviews for hidden layers are retained in the database
     Given avg_tds_wells and latest_depth_to_water_wells have been removed from the service catalog
     When the database schema is inspected
@@ -310,12 +305,12 @@ Feature: OGC Feature Layer Cleanup — Sprint 1
   # A17 — Hide locations layer from the public catalog
   # ---------------------------------------------------------------------------
 
-  @backend @ogc-data-currency @sprint-1 @medium-priority @A17
+  @backend @ogc-data-currency @sprint-1 @medium-priority @A17 @production
   Scenario: locations is absent from the public collections catalog
     When a client requests /ogcapi/collections
     Then the response does not include a collection with id locations
 
-  @backend @ogc-data-currency @sprint-1 @medium-priority @A17
+  @backend @ogc-data-currency @sprint-1 @medium-priority @A17 @production
   Scenario: Underlying locations table is retained in the database after catalog removal
     Given the locations entry has been removed from the service configuration
     When the database schema is inspected
@@ -325,22 +320,21 @@ Feature: OGC Feature Layer Cleanup — Sprint 1
   # A18 — Remove other_things from the public catalog
   # ---------------------------------------------------------------------------
 
-  @backend @ogc-naming @sprint-1 @medium-priority @A18
+  @backend @ogc-naming @sprint-1 @medium-priority @A18 @production
   Scenario: other_things is absent from the public collections catalog
     When a client requests /ogcapi/collections
     Then the response does not include a collection with id other_things
 
-  @backend @ogc-naming @sprint-1 @medium-priority @A18
-  Scenario: other_things backing view is dropped when no internal usage exists
-    Given the other_things view has zero references in the application codebase
-    When the cleanup is applied
-    Then the other_things backing view does not exist in the database schema
-
-  @backend @ogc-naming @sprint-1 @medium-priority @A18
-  Scenario: other_things backing view is retained when internal usage exists
+  # The A18 review found internal usage: /ogcapi-internal still publishes the
+  # layer to staff GIS clients off ogc_internal_other_things, and the public
+  # ogc_other_things view is still built by the shared Group A view template.
+  # Both views are therefore retained.
+  @backend @ogc-naming @sprint-1 @medium-priority @A18 @production
+  Scenario: other_things backing views are retained because the internal mount uses them
     Given the other_things view has at least one reference in the application codebase
-    When the cleanup is applied
+    When the database schema is inspected
     Then the other_things backing view still exists in the database schema
+    And the internal other_things backing view still exists in the database schema
 
   # ---------------------------------------------------------------------------
   # A22 — Verify NULL measuring_point_height assumption for water level layers

tests/features/steps/ogc-cleanup-sprint1.py

@@ -857,6 +857,78 @@ def step_then_no_collection_id_prefixed(context, prefix):
     assert not offending, f"found collections with id prefixed {prefix!r}: {offending}"
 
 
+# ---------------------------------------------------------------------------
+# A16/A17/A18 -- Layers hidden from the public catalog, backing relations kept
+# ---------------------------------------------------------------------------
+
+
+@given(
+    "avg_tds_wells and latest_depth_to_water_wells have been removed from the "
+    "service catalog"
+)
+@given("the locations entry has been removed from the service configuration")
+@given("the other_things view has at least one reference in the application codebase")
+def step_given_layer_hidden_from_public_catalog(context):
+    # No-op marker: the catalog is core/pygeoapi-config.yml and
+    # core.pygeoapi.THING_COLLECTIONS, both artifacts under test rather than
+    # runtime state to arrange. Same treatment as the A1/A2/A11 givens.
+    pass
+
+
+@then("the response does not include a collection with id {collection_id}")
+def step_then_response_excludes_collection_id(context, collection_id):
+    payload = context.response.json()
+    ids = {collection["id"] for collection in payload["collections"]}
+    assert collection_id not in ids, (
+        f"{collection_id} is still published on the public catalog; "
+        f"collections: {sorted(ids)}"
+    )
+
+
+@then("the materialized view for {layer_id} exists in the database schema")
+def step_then_matview_for_layer_exists(context, layer_id):
+    relation = f"ogc_{layer_id}"
+    assert relation in context.schema_relations, (
+        f"{relation} is missing -- A16 hides the layer from the catalog but "
+        "keeps its materialized view for internal use"
+    )
+
+
+@then("the locations table still exists")
+def step_then_locations_table_still_exists(context):
+    # context.schema_relations covers views and materialized views only, so
+    # the base table needs its own lookup.
+    with session_ctx() as session:
+        relkind = session.execute(
+            text(
+                "SELECT c.relkind FROM pg_class c "
+                "JOIN pg_namespace n ON n.oid = c.relnamespace "
+                "WHERE n.nspname = 'public' AND c.relname = 'location'"
+            )
+        ).scalar_one_or_none()
+    assert relkind == "r", (
+        "the location table is missing -- A17 hides the layer from the "
+        f"catalog but keeps the underlying table (relkind={relkind!r})"
+    )
+
+
+def _assert_other_things_view_exists(context, relation):
+    assert relation in context.schema_relations, (
+        f"{relation} is missing -- A18 hides other_things from the public "
+        "catalog but /ogcapi-internal still serves the layer"
+    )
+
+
+@then("the other_things backing view still exists in the database schema")
+def step_then_other_things_view_still_exists(context):
+    _assert_other_things_view_exists(context, "ogc_other_things")
+
+
+@then("the internal other_things backing view still exists in the database schema")
+def step_then_internal_other_things_view_still_exists(context):
+    _assert_other_things_view_exists(context, "ogc_internal_other_things")
+
+
 # ---------------------------------------------------------------------------
 # A2 -- Replace OGC server metadata placeholders in pygeoapi-config.yml
 # ---------------------------------------------------------------------------