diff --git a/.dockerignore b/.dockerignore index b694934fb..95b01de27 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,45 @@ -.venv \ No newline at end of file +# Keep the build context to what the image actually runs. +# +# The tracked tree is ~13 MB; the context was ~316 MB, almost entirely .git. +# CI clones full history, and every Dagster+ deploy was transferring it before +# the first layer could build. + +# Version control. Nothing in the image reads git metadata. +.git +.github +.gitignore + +# Python build and cache artifacts. Stale .pyc from a different interpreter is +# worse than useless in an image built on a pinned base. +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.coverage +htmlcov/ + +# Virtualenvs and local tooling state. +.venv +.dg +*.tfstate +*.tfstate.* +.terraform/ + +# Local-only data from legacy transfer runs. Untracked, machine-specific, and +# the largest thing on a developer checkout by an order of magnitude -- a local +# `docker build` would otherwise ship ~900 MB of CSV cache. +transfers/data/ +transfers/logs/ +transfers/metrics/ + +# Test fixtures and BDD features. The image runs the code location, not the +# suite; CI runs the suite outside the image. +tests/ +features/ + +# Editor and OS noise. +.DS_Store +.idea/ +.vscode/ diff --git a/.env.example b/.env.example index dfdc98844..54c576b32 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,26 @@ POSTGRES_PORT=5432 PYGEOAPI_POSTGRES_PASSWORD=your_password PYGEOAPI_POSTGRES_USER=your_username +# PYGEOAPI internal mount (/ogcapi-internal) -- authenticated, unfiltered +# (private/draft-inclusive) mirror of /ogcapi. Shares PYGEOAPI_POSTGRES_* +# above; only the mount path, runtime dir, and advertised server URL differ. +PYGEOAPI_INTERNAL_MOUNT_PATH=/ogcapi-internal +PYGEOAPI_INTERNAL_RUNTIME_DIR=/tmp/pygeoapi-internal +# Leave blank to derive from PYGEOAPI_SERVER_URL's application root. Only set +# this when the internal mount is served from a different host than /ogcapi. +PYGEOAPI_INTERNAL_SERVER_URL= + +# Static API keys for /ogcapi-internal, for desktop GIS clients that cannot +# refresh an Authentik access token (ArcGIS Pro, QGIS). Comma- or +# whitespace-separated `label:sha256hex` entries; the label is bookkeeping +# only. Blank means bearer-JWT access only. Mint one with: +# python -c "import secrets,hashlib;k=secrets.token_urlsafe(32);print(k,hashlib.sha256(k.encode()).hexdigest())" +# Give the first value to the user, put `label:` here. +# Deployed environments source this from the Secret Manager secret +# `internal-ogc-api-keys`, not from a GitHub secret. +# See docs/internal-ogc-desktop-gis.md. +INTERNAL_OGC_API_KEYS= + # Connection pool configuration for parallel transfers # pool_size: number of persistent connections to maintain # max_overflow: additional connections allowed during peak usage @@ -73,19 +93,28 @@ MODE=development # ENABLE_PG_CRON=0 # disable authentication (for development only) +# +# Honored ONLY when MODE=development. With any other MODE (including unset or +# "staging"), the app refuses to start -- core.permissions.assert_auth_configuration() +# raises AuthConfigurationError rather than serving every endpoint anonymously. AUTHENTIK_DISABLE_AUTHENTICATION=1 # erase and rebuild the database for step tests REBUILD_DB=1 # authentik +# AUTHENTIK_URL is both the JWKS base and the expected `iss` claim; trailing +# slash optional, both spellings are accepted. AUTHENTIK_URL= AUTHENTIK_CLIENT_ID= AUTHENTIK_AUTHORIZE_URL= AUTHENTIK_TOKEN_URL= -# middleware -SESSION_SECRET_KEY=your_secret_key_here +# How long a fetched JWKS document is trusted, in seconds (default 3600). +# An unrecognized `kid` forces one immediate refresh regardless, so this only +# bounds how long a revoked key stays usable. +# AUTHENTIK_JWKS_TTL_SECONDS=3600 + # feedback endpoint (POST /feedback) — bug reports and feature requests JIRA_BASE_URL=https://nmbgmr.atlassian.net diff --git a/.github/app.template.yaml b/.github/app.template.yaml index 6a1a52fb4..a0f67c7b9 100644 --- a/.github/app.template.yaml +++ b/.github/app.template.yaml @@ -34,6 +34,13 @@ env_variables: PYGEOAPI_POSTGRES_PASSWORD: |- ${PYGEOAPI_POSTGRES_PASSWORD} PYGEOAPI_SERVER_URL: "${PYGEOAPI_SERVER_URL}" + # Hashed static API keys for the authenticated /ogcapi-internal mount, as + # `label:sha256hex` entries, sourced from the Secret Manager secret + # `internal-ogc-api-keys`. Needed because ArcGIS Pro and QGIS cannot refresh + # an Authentik access token; see core/internal_ogc_auth.py and + # docs/internal-ogc-desktop-gis.md. Unset means bearer-JWT access only. + INTERNAL_OGC_API_KEYS: |- + ${INTERNAL_OGC_API_KEYS} CLOUD_SQL_IAM_AUTH: "${CLOUD_SQL_IAM_AUTH}" GCS_SERVICE_ACCOUNT_KEY: |- ${GCS_SERVICE_ACCOUNT_KEY} @@ -42,8 +49,6 @@ env_variables: AUTHENTIK_CLIENT_ID: "${AUTHENTIK_CLIENT_ID}" AUTHENTIK_AUTHORIZE_URL: "${AUTHENTIK_AUTHORIZE_URL}" AUTHENTIK_TOKEN_URL: "${AUTHENTIK_TOKEN_URL}" - SESSION_SECRET_KEY: |- - ${SESSION_SECRET_KEY} APITALLY_CLIENT_ID: "${APITALLY_CLIENT_ID}" JIRA_BASE_URL: "${JIRA_BASE_URL}" JIRA_EMAIL: "${JIRA_EMAIL}" diff --git a/.github/skills/code-review/SKILL.md b/.github/skills/code-review/SKILL.md new file mode 100644 index 000000000..723969783 --- /dev/null +++ b/.github/skills/code-review/SKILL.md @@ -0,0 +1,84 @@ +--- +name: code-review +description: Repository-specific review rules for OcotilloAPI pull requests. Use this when reviewing a pull request in this repository, so review comments account for the authorization, schema, domain-layer, and migration conventions that are easy to violate silently. +--- + +# Reviewing OcotilloAPI pull requests + +OcotilloAPI is a FastAPI + PostgreSQL/PostGIS geospatial service for the New Mexico +Bureau of Geology and Mineral Resources. Read `CLAUDE.md` at the repository root for +the full architecture; this skill lists the mistakes worth flagging in review because +they fail silently rather than breaking a test. + +Use the GitHub MCP server tools (`list_workflow_runs`, `summarize_job_log_failures`, +`get_job_logs`) to check whether the `Test Suite` workflow is failing before commenting +on behavior — a failing `unit-tests` job often explains the diff better than the diff does. + +## Authorization is opt-in, so omissions are invisible + +Authorization is applied per endpoint via a parameter in the route signature, not by a +router-level `dependencies=[...]`. Two failure modes to flag: + +1. A new route with no `user: _dependency` parameter is fully public and raises no + error. If the pull request adds a route, check whether it belongs in the anonymous-route + allowlist in `tests/test_authorization.py`. If it does not, it needs a role dependency. +2. The dependency must be a **type annotation** (`user: viewer_dependency`), never a default + value (`user=viewer_dependency`). The latter silently disables the dependency, and FastAPI + reinterprets it as a query parameter. Flag this every time. + +Role families are orthogonal: general `Admin` confers nothing in the `AMP*` or `Lexicon*` +families. Only tiers within one family nest. A diff that treats `Admin` as a superset of +`AMPEditor` is wrong. + +`@in_public_schema` controls anonymous OpenAPI visibility only. It grants no access and +removes no dependency; flag any use that appears to be standing in for authorization. + +`/ogcapi-internal` is a raw Starlette Mount and is gated at the ASGI layer in +`core/internal_ogc_auth.py`, outside `Depends()`. Changes to its credential paths should +cite `docs/internal-ogc-desktop-gis.md`. + +The development auth bypass (`AUTHENTIK_DISABLE_AUTHENTICATION=1`) is honored only when +`MODE=development`. Any change that widens that condition is a security finding. + +## Model changes are a five-step workflow + +A pull request that edits a model in `db/` is incomplete unless it also covers the matching +Pydantic schemas in `schemas/`, an Alembic migration, test fixtures and payloads in `tests/`, +and the field mappings in `transfers/` when the field is populated from the legacy AMPAPI +data. Flag whichever step is missing. + +Schema conventions: `Create` schemas use `` for non-nullable and ` | None = None` +for nullable; `Update` schemas make every field optional with a `None` default; `Response` +schemas use `` for non-nullable and ` | None` for nullable. + +Validation split: input validation belongs in Pydantic validators and produces 422s. Database +constraint checks are manual in the endpoint and produce 409s. Custom exceptions should use +`PydanticStyleException` from `services/exceptions_helper.py` so error bodies stay consistent. + +## Layer boundaries + +`domain/` holds business rules as plain functions over plain values. Modules there must not +import from `api/`, `db/`, `schemas/`, or `services/`, and must not import `fastapi`, +`sqlalchemy`, `pydantic`, or `httpx`. Flag any new import that breaks this — it is what keeps +the rules testable without a database. Domain errors subclass `ValueError` because the CSV +importers treat a `ValueError` on a row as a per-row validation failure; an exception type +that does not subclass `ValueError` will escape that handling. See `ADR4.md`. + +`services/` is the layer that loads data, calls the domain rule, and persists the result. + +## Spatial and query specifics + +All geometries are WGS84 (SRID 4326). Legacy transfer scripts convert from UTM (SRID 26913); +a missing transformation puts points in the wrong hemisphere rather than raising. + +List filters arrive from the Refine UI as repeated `filter` query parameters containing JSON. +Association-backed columns are virtual and map to EXISTS subqueries in +`services/query_helper.py`, not to `ILIKE` on an ORM proxy. Sorting by monitoring status or +well status must use SQL subqueries on `StatusHistory`, because `ORDER BY` cannot see a Python +`@property`. See `docs/refine-json-filters-and-virtual-fields.md`. + +## Migrations + +Alembic schema migrations run automatically in the deployment pipeline. Registered *data* +migrations do not — they sit unapplied until someone runs them by hand. If a pull request adds +a data migration, ask how and when it will be run. diff --git a/.github/workflows/CD_dagster_branch.yml b/.github/workflows/CD_dagster_branch.yml new file mode 100644 index 000000000..040f47372 --- /dev/null +++ b/.github/workflows/CD_dagster_branch.yml @@ -0,0 +1,270 @@ +# Creates a Dagster+ branch deployment for a pull request, so ingestion changes +# can be materialized against an isolated deployment before they reach prod. +# +# Path-filtered: most PRs in this repository touch only the API and should not +# create a Dagster+ deployment at all. +# +# ## Two deploy paths +# +# The default path builds no container image at all: it packages the +# dependencies and the source into two PEX files and uploads them, so a source +# change reuses the previously published deps.pex instead of rebuilding and +# pushing several hundred megabytes to ECR. See +# https://dagster.io/blog/fast-deploys-with-pex-and-docker. The deps.pex cache +# is keyed per repository, not per deployment, so a PR that changes no +# dependency reuses the one the last prod deploy published. +# +# Setting ENABLE_FAST_DEPLOYS to 'false' below switches the whole workflow back +# to the Docker image build in the `dagster-branch-docker-deploy` job. Keep both +# paths working, and keep this file's setting in step with CD_dagster_prod.yml -- +# the two share the deps.pex cache, and a PR built the other way just misses it. +name: CD (Dagster+ branch deployment) + +on: + pull_request: + types: [opened, synchronize, reopened, closed] + paths: + - "automated_ingestion/**" + # The code location imports db/ models and domain/ rules in-process, + # so a change to either alters what this image runs even when no + # ingestion file moves. Without these, a domain fix merged to + # production would leave the pipeline running the old rule against + # the live database. The cost is that ordinary API changes to these + # directories also trigger a build; a stale code location is worse. + - "db/**" + - "domain/**" + - "dagster_cloud.yaml" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/CD_dagster_branch.yml" + +permissions: + contents: read + pull-requests: write + +# One deployment per PR; a force-push supersedes the run it interrupts. +concurrency: + group: dagster-branch-deploy-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + # The PEX path targets `$DAGSTER_CLOUD_URL/`, so this is the + # organization URL with no deployment path appended. The Docker path takes the + # organization id as an action input instead and ignores this. + DAGSTER_CLOUD_URL: https://${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }}.dagster.cloud + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + # Read by actions/utils/prerun, which reports `pex-deploy` or `docker-deploy` + # and so decides which of the two jobs below runs. + ENABLE_FAST_DEPLOYS: "true" + # The PEX files are resolved for this interpreter only, and requires-python is + # >= 3.13. Anything lower makes the deps resolve fail with "no matching + # distribution", which reads like a broken requirements file rather than a + # version mismatch. + PYTHON_VERSION: "3.13" + DAGSTER_CLOUD_FILE: dagster_cloud.yaml + +jobs: + dagster-branch-deploy: + runs-on: ubuntu-latest + # Forks cannot read the Dagster+ secrets, and a branch deployment from an + # untrusted fork would run our code against our infrastructure regardless. + if: github.event.pull_request.head.repo.full_name == github.repository + + # The notify steps post build status as a PR comment and read the token from + # the workflow environment -- `env.GITHUB_TOKEN`, not the `secrets` context. + # Without this the run dies on an empty-token assertion before it ever + # reaches Dagster+, which reads as an auth failure but is not one. + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + outputs: + # Set only on the Docker path. The fallback job keys off it being empty. + build_info: ${{ steps.parse.outputs.build_info }} + + steps: + # Two jobs in one step. On a `closed` event it runs `ci + # branch-deployment`, which marks the branch deployment closed so stale + # deployments do not accumulate, and reports `skip` -- both deploy paths + # then do nothing. Otherwise it reads ENABLE_FAST_DEPLOYS and reports + # `pex-deploy` or `docker-deploy`. + # + # Its own checkout goes to prerun_checkout_dir, so it does not disturb the + # working tree either path sets up below. + - name: Prerun checks + id: prerun + uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v1.13.19 + + # parse_workspace performs its own `actions/checkout`, which cleans the + # working tree -- fine here because the Docker path runs in a separate job + # that checks out again. Only its `build_info` output crosses the boundary. + - name: Parse dagster_cloud.yaml + if: steps.prerun.outputs.result == 'docker-deploy' + id: parse + uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v1.13.19 + with: + dagster_cloud_file: ${{ env.DAGSTER_CLOUD_FILE }} + + # Checked out under a subdirectory because build_deploy_python_executable + # takes an absolute path to the location file and does not check out + # anything itself, so nothing later can clobber the generated + # requirements.txt. + - name: Check out source repository + if: steps.prerun.outputs.result == 'pex-deploy' + uses: actions/checkout@v7.0.1 + with: + ref: ${{ github.head_ref }} + path: project-repo + + - name: Install uv + if: steps.prerun.outputs.result == 'pex-deploy' + uses: astral-sh/setup-uv@v10.0.1 + with: + version: "latest" + + # `--group ingestion` adds dagster and dlt on top of the runtime + # dependencies; the runtime ones are needed too, because the loader + # imports `db/` and `domain/`. + # + # `--no-hashes` is required on this path and only on this path. The PEX + # builder unions requirements.txt with `[project].dependencies` from + # pyproject.toml, and those pins carry no hashes -- a hashed + # requirements.txt would put pip in --require-hashes mode, where every + # unhashed line is an error. Both sources resolve from the same uv.lock, + # so the duplicate pins agree and drop out. + - name: Generate requirements.txt + if: steps.prerun.outputs.result == 'pex-deploy' + working-directory: project-repo + run: | + uv export \ + --format requirements-txt \ + --no-emit-project \ + --no-dev \ + --no-hashes \ + --group ingestion \ + --output-file requirements.txt + + # Builds deps.pex and source.pex and publishes them, then creates or + # updates the branch deployment for this PR -- the action derives the + # deployment name from the pull_request event, so there is no `deployment` + # input to set here. + # + # On an ubuntu-24.04 runner deps.pex is built inside a python:3.13-slim + # container so the wheels match the serverless base image; source.pex is + # always built on the runner. That container only spins up when the + # dependency hash changes. + - name: Deploy to Dagster+ branch deployment + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v1.13.19 + with: + dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/${{ env.DAGSTER_CLOUD_FILE }}" + build_output_dir: "$GITHUB_WORKSPACE/build" + python_version: ${{ env.PYTHON_VERSION }} + + # Materializing the heartbeat is the run-time half of the check. The steps + # above prove the agent can load the code location; the loader is not the + # process that executes a step, and the two do not necessarily agree about + # sys.path, so loading cleanly does not prove a step can import `db` or + # `domain`. This does. The asset touches no database, no network and no + # GCS, so a failure here is a packaging problem and nothing else -- which + # is worth one Dagster+ run on a PR that already changed how the code + # location is packaged. + # + # The deployment name has to be asked for rather than assumed: it is + # derived from the branch, and the deploy action does not report it. This + # is the same `ci branch-deployment` call the deploy makes internally, and + # it is idempotent -- it returns the existing deployment for this PR. + - name: Resolve branch deployment name + if: steps.prerun.outputs.result == 'pex-deploy' + id: branch_deployment + run: | + name=$(uvx --from "dagster-cloud-cli==1.13.18" \ + dagster-cloud ci branch-deployment project-repo) + echo "name=$name" >> "$GITHUB_OUTPUT" + echo "Branch deployment: $name" + + # Deliberately not the vendor's `launch_job` action, and the success + # assertion is deliberately our own. + # + # `launch_job` documents `wait: true` as "the action will wait for the run + # to finish and fail if the run fails". It does wait, but it cannot fail: + # its run.sh captures the CLI output in a command substitution and never + # checks the exit code, deciding success by whether it can regex a run id + # out of the text. Underneath, `dagster-cloud job launch --wait` reports a + # failed run with `ui.error(...)` -- and `ui.error` only *returns* an + # exception rather than raising it, so the CLI exits 0 too. A failed + # materialization would have produced a green check on both counts. + # + # So call the CLI directly and require the success line. Matching on output + # is not lovely, but it is the only signal either layer actually emits. + - name: Materialize ingestion_heartbeat + if: steps.prerun.outputs.result == 'pex-deploy' + env: + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + DEPLOYMENT: ${{ steps.branch_deployment.outputs.name }} + run: | + set -uo pipefail + out=$(uvx --from "dagster-cloud-cli==1.13.18" \ + dagster-cloud job launch \ + --url "$DAGSTER_CLOUD_URL" \ + --deployment "$DEPLOYMENT" \ + --location ocotillo-automated-ingestion \ + --job ingestion_heartbeat_check \ + --wait --interval 10 2>&1 | tee /dev/stderr) + case "$out" in + *"finished successfully"*) ;; + *) echo "::error title=Heartbeat failed::ingestion_heartbeat did not finish successfully on $DEPLOYMENT" + exit 1 ;; + esac + + # Fallback path, reached only when ENABLE_FAST_DEPLOYS is 'false' above. This + # is the pre-PEX workflow unchanged, including the post-install hook in + # dagster_cloud_post_install.sh that the PEX path replaces with its own + # `uv pip install --no-deps .` of the repository. + dagster-branch-docker-deploy: + runs-on: ubuntu-latest + needs: dagster-branch-deploy + if: needs.dagster-branch-deploy.outputs.build_info + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + strategy: + fail-fast: false + matrix: + location: ${{ fromJSON(needs.dagster-branch-deploy.outputs.build_info) }} + + steps: + - name: Check out source repository + uses: actions/checkout@v7.0.1 + with: + ref: ${{ github.head_ref }} + + - name: Install uv in container + uses: astral-sh/setup-uv@v10.0.1 + with: + version: "latest" + + # Dagster+ builds from a requirements.txt. Hashes are kept here: this path + # feeds the file straight to `pip install -r`, with nothing unhashed mixed + # in. + - name: Generate requirements.txt + run: | + uv export \ + --format requirements-txt \ + --no-emit-project \ + --no-dev \ + --group ingestion \ + --output-file requirements.txt + + # checkout_repo is false because requirements.txt is generated above and + # a second checkout would discard it. + - name: Deploy to Dagster+ branch deployment + uses: dagster-io/dagster-cloud-action/actions/serverless_branch_deploy@v1.13.19 + with: + organization_id: ${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }} + dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + location: ${{ toJson(matrix.location) }} + checkout_repo: false + # The action defaults to python:3.8-slim, which cannot install a + # lockfile resolved for requires-python >= 3.13 -- pip reports the + # pins as having no matching distribution rather than as a version + # conflict, which reads like a broken requirements file. + base_image: python:${{ env.PYTHON_VERSION }}-slim diff --git a/.github/workflows/CD_dagster_prod.yml b/.github/workflows/CD_dagster_prod.yml new file mode 100644 index 000000000..833332486 --- /dev/null +++ b/.github/workflows/CD_dagster_prod.yml @@ -0,0 +1,218 @@ +# Deploys the `ocotillo-automated-ingestion` code location to the Dagster+ prod +# deployment. +# +# "prod" here names the Dagster+ deployment, not the API's `production` branch. +# It tracks `staging`, which is this repository's integration branch and the +# only place `automated_ingestion/` currently exists -- `production` is still on +# 1.2.2 and would never fire. This mirrors how `ocotillo-api-staging` follows +# staging, and keeps the code location current with the work. +# +# Move this to `production` in the same change that first points the pipeline at +# the production database. Until then a deploy here publishes code, not data: +# the location has no database or vendor credentials, so the worst it can do is +# fail to materialize. +# +# Path-filtered so an ordinary API change does not spend a Dagster+ build. The +# filter includes pyproject.toml and uv.lock because the location's dependency +# set is exported from them, so a lockfile bump changes the built image even +# when no ingestion source file does. +# +# ## Two deploy paths +# +# The default path builds no container image at all: it packages the +# dependencies and the source into two PEX files and uploads them, so a source +# change reuses the previously published deps.pex instead of rebuilding and +# pushing several hundred megabytes to ECR. See +# https://dagster.io/blog/fast-deploys-with-pex-and-docker. +# +# Setting ENABLE_FAST_DEPLOYS to 'false' below switches the whole workflow back +# to the Docker image build in the `dagster-prod-docker-deploy` job. That job is +# the escape hatch for anything the PEX path cannot express -- a dependency with +# no Linux wheel that also fails to build from source, or a system package that +# has to be installed with apt. Keep both paths working. +name: CD (Dagster+ prod) + +on: + push: + branches: [staging] + paths: + - "automated_ingestion/**" + # The code location imports db/ models and domain/ rules in-process, + # so a change to either alters what this image runs even when no + # ingestion file moves. Without these, a domain fix merged to + # production would leave the pipeline running the old rule against + # the live database. The cost is that ordinary API changes to these + # directories also trigger a build; a stale code location is worse. + - "db/**" + - "domain/**" + - "dagster_cloud.yaml" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/CD_dagster_prod.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: dagster-prod-deploy + cancel-in-progress: false + +env: + # The PEX path targets `$DAGSTER_CLOUD_URL/`, so this is the + # organization URL with no deployment path appended. The Docker path takes the + # organization id as an action input instead and ignores this. + DAGSTER_CLOUD_URL: https://${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }}.dagster.cloud + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + # Read by actions/utils/prerun, which reports `pex-deploy` or `docker-deploy` + # and so decides which of the two jobs below runs. + ENABLE_FAST_DEPLOYS: "true" + # The PEX files are resolved for this interpreter only, and requires-python is + # >= 3.13. Anything lower makes the deps resolve fail with "no matching + # distribution", which reads like a broken requirements file rather than a + # version mismatch. + PYTHON_VERSION: "3.13" + DAGSTER_CLOUD_FILE: dagster_cloud.yaml + +jobs: + dagster-prod-deploy: + runs-on: ubuntu-latest + # Deliberately not bound to the `production` GitHub environment. This job + # reads only repository-level DAGSTER_CLOUD_API_TOKEN and + # DAGSTER_CLOUD_ORGANIZATION_ID -- none of that environment's secrets -- and + # it runs on every push to `staging` that touches the code location. Binding + # it would put an approval gate on routine merges once `production` requires + # reviewers, which is a gate on the wrong thing: this publishes code, not + # data. + outputs: + # Set only on the Docker path. The fallback job keys off it being empty. + build_info: ${{ steps.parse.outputs.build_info }} + + steps: + # Reads ENABLE_FAST_DEPLOYS and emits `pex-deploy` or `docker-deploy`. + # Its own checkout goes to prerun_checkout_dir, so it does not disturb the + # working tree either path sets up below. + - name: Prerun checks + id: prerun + uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v1.13.19 + + # parse_workspace performs its own `actions/checkout`, which cleans the + # working tree -- fine here because the Docker path runs in a separate job + # that checks out again. Only its `build_info` output crosses the boundary. + - name: Parse dagster_cloud.yaml + if: steps.prerun.outputs.result == 'docker-deploy' + id: parse + uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v1.13.19 + with: + dagster_cloud_file: ${{ env.DAGSTER_CLOUD_FILE }} + + # Checked out under a subdirectory because build_deploy_python_executable + # takes an absolute path to the location file and does not check out + # anything itself, so nothing later can clobber the generated + # requirements.txt. + - name: Check out source repository + if: steps.prerun.outputs.result == 'pex-deploy' + uses: actions/checkout@v7.0.1 + with: + ref: ${{ github.sha }} + path: project-repo + + - name: Install uv + if: steps.prerun.outputs.result == 'pex-deploy' + uses: astral-sh/setup-uv@v10.0.1 + with: + version: "latest" + + # `--group ingestion` adds dagster and dlt on top of the runtime + # dependencies; the runtime ones are needed too, because the loader + # imports `db/` and `domain/`. + # + # `--no-hashes` is required on this path and only on this path. The PEX + # builder unions requirements.txt with `[project].dependencies` from + # pyproject.toml, and those pins carry no hashes -- a hashed + # requirements.txt would put pip in --require-hashes mode, where every + # unhashed line is an error. Both sources resolve from the same uv.lock, + # so the duplicate pins agree and drop out. + - name: Generate requirements.txt + if: steps.prerun.outputs.result == 'pex-deploy' + working-directory: project-repo + run: | + uv export \ + --format requirements-txt \ + --no-emit-project \ + --no-dev \ + --no-hashes \ + --group ingestion \ + --output-file requirements.txt + + # Builds deps.pex and source.pex and publishes them. deps.pex is keyed by + # a hash of the resolved requirements and cached per repository, so a run + # that changes only ingestion source skips the dependency build entirely. + # + # On an ubuntu-24.04 runner the action builds deps.pex inside a + # python:3.13-slim container so the wheels match the serverless base + # image; source.pex is always built on the runner. That container only + # spins up when the dependency hash changes. + - name: Deploy to Dagster+ prod + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v1.13.19 + with: + dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/${{ env.DAGSTER_CLOUD_FILE }}" + build_output_dir: "$GITHUB_WORKSPACE/build" + python_version: ${{ env.PYTHON_VERSION }} + deployment: prod + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Fallback path, reached only when ENABLE_FAST_DEPLOYS is 'false' above. This + # is the pre-PEX workflow unchanged, including the post-install hook in + # dagster_cloud_post_install.sh that the PEX path replaces with its own + # `uv pip install --no-deps .` of the repository. + dagster-prod-docker-deploy: + runs-on: ubuntu-latest + needs: dagster-prod-deploy + if: needs.dagster-prod-deploy.outputs.build_info + strategy: + fail-fast: false + matrix: + location: ${{ fromJSON(needs.dagster-prod-deploy.outputs.build_info) }} + + steps: + - name: Check out source repository + uses: actions/checkout@v7.0.1 + with: + ref: ${{ github.sha }} + + - name: Install uv in container + uses: astral-sh/setup-uv@v10.0.1 + with: + version: "latest" + + # Dagster+ builds from a requirements.txt. Hashes are kept here: this path + # feeds the file straight to `pip install -r`, with nothing unhashed mixed + # in. + - name: Generate requirements.txt + run: | + uv export \ + --format requirements-txt \ + --no-emit-project \ + --no-dev \ + --group ingestion \ + --output-file requirements.txt + + # checkout_repo is false because requirements.txt is generated above and + # a second checkout would discard it. + - name: Deploy to Dagster+ prod + uses: dagster-io/dagster-cloud-action/actions/serverless_prod_deploy@v1.13.19 + with: + organization_id: ${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }} + dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + location: ${{ toJson(matrix.location) }} + checkout_repo: false + # The action defaults to python:3.8-slim, which cannot install a + # lockfile resolved for requires-python >= 3.13 -- pip reports the + # pins as having no matching distribution rather than as a version + # conflict, which reads like a broken requirements file. + base_image: python:${{ env.PYTHON_VERSION }}-slim + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CD_production.yml b/.github/workflows/CD_production.yml index 0017e25a5..8c65c029b 100644 --- a/.github/workflows/CD_production.yml +++ b/.github/workflows/CD_production.yml @@ -46,7 +46,7 @@ jobs: fi - name: Check out source repository - uses: actions/checkout@v7.0.0 + uses: actions/checkout@v7.0.1 with: fetch-depth: 0 # Fully-qualified tag ref avoids ambiguity if a branch is ever @@ -54,7 +54,7 @@ jobs: ref: refs/tags/${{ env.DEPLOY_TAG }} - name: Install uv in container - uses: astral-sh/setup-uv@v8.3.2 + uses: astral-sh/setup-uv@v10.0.1 with: version: "latest" @@ -71,11 +71,14 @@ jobs: with: credentials_json: ${{ secrets.CLOUD_DEPLOY_SERVICE_ACCOUNT_KEY }} - # Feedback endpoint credentials live in Google Secret Manager, not - # GitHub secrets. The deploy service account needs - # roles/secretmanager.secretAccessor on these secrets. - - name: Fetch feedback secrets from Secret Manager - id: feedback-secrets + # Application credentials live in Google Secret Manager, not GitHub + # secrets. The deploy service account needs + # roles/secretmanager.secretAccessor on these secrets. Every secret + # listed here must already exist in the target project or the deploy + # fails -- see docs/internal-ogc-desktop-gis.md for the placeholder + # value to seed internal-ogc-api-keys with. + - name: Fetch application secrets from Secret Manager + id: app-secrets uses: 'google-github-actions/get-secretmanager-secrets@v3' with: secrets: |- @@ -83,6 +86,7 @@ jobs: jira_api_token:${{ vars.GCP_PROJECT_ID }}/jira-api-token slack_feedback_webhook_url:${{ vars.GCP_PROJECT_ID }}/slack-feedback-webhook-url slack_edits_webhook_url:${{ vars.GCP_PROJECT_ID }}/slack-edits-webhook-url + internal_ogc_api_keys:${{ vars.GCP_PROJECT_ID }}/internal-ogc-api-keys - name: Run Alembic migrations on production database env: @@ -119,6 +123,7 @@ jobs: PYGEOAPI_POSTGRES_PORT: "${{ vars.PYGEOAPI_POSTGRES_PORT || '5432' }}" PYGEOAPI_POSTGRES_PASSWORD: "${{ secrets.PYGEOAPI_POSTGRES_PASSWORD }}" PYGEOAPI_SERVER_URL: "${{ vars.PYGEOAPI_SERVER_URL }}" + INTERNAL_OGC_API_KEYS: "${{ steps.app-secrets.outputs.internal_ogc_api_keys }}" CLOUD_SQL_IAM_AUTH: "true" GCS_SERVICE_ACCOUNT_KEY: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" GCS_BUCKET_NAME: "${{ vars.GCS_BUCKET_NAME }}" @@ -126,14 +131,13 @@ jobs: AUTHENTIK_CLIENT_ID: "${{ vars.AUTHENTIK_CLIENT_ID }}" AUTHENTIK_AUTHORIZE_URL: "${{ vars.AUTHENTIK_AUTHORIZE_URL }}" AUTHENTIK_TOKEN_URL: "${{ vars.AUTHENTIK_TOKEN_URL }}" - SESSION_SECRET_KEY: "${{ secrets.SESSION_SECRET_KEY }}" APITALLY_CLIENT_ID: "${{ vars.APITALLY_CLIENT_ID }}" JIRA_BASE_URL: "${{ vars.JIRA_BASE_URL || 'https://nmbgmr.atlassian.net' }}" - JIRA_EMAIL: "${{ steps.feedback-secrets.outputs.jira_email }}" - JIRA_API_TOKEN: "${{ steps.feedback-secrets.outputs.jira_api_token }}" + JIRA_EMAIL: "${{ steps.app-secrets.outputs.jira_email }}" + JIRA_API_TOKEN: "${{ steps.app-secrets.outputs.jira_api_token }}" JIRA_DEFAULT_PROJECT: "${{ vars.JIRA_DEFAULT_PROJECT || 'BDMS' }}" - SLACK_FEEDBACK_WEBHOOK_URL: "${{ steps.feedback-secrets.outputs.slack_feedback_webhook_url }}" - SLACK_EDITS_WEBHOOK_URL: "${{ steps.feedback-secrets.outputs.slack_edits_webhook_url }}" + SLACK_FEEDBACK_WEBHOOK_URL: "${{ steps.app-secrets.outputs.slack_feedback_webhook_url }}" + SLACK_EDITS_WEBHOOK_URL: "${{ steps.app-secrets.outputs.slack_edits_webhook_url }}" OCOTILLO_UI_BASE_URL: "${{ vars.OCOTILLO_UI_BASE_URL || 'https://ocotillo.newmexicowaterdata.org' }}" run: | export MAX_INSTANCES="10" diff --git a/.github/workflows/CD_staging.yml b/.github/workflows/CD_staging.yml index 232981071..b4af038bd 100644 --- a/.github/workflows/CD_staging.yml +++ b/.github/workflows/CD_staging.yml @@ -14,12 +14,12 @@ jobs: steps: - name: Check out source repository - uses: actions/checkout@v7.0.0 + uses: actions/checkout@v7.0.1 with: fetch-depth: 0 - name: Install uv in container - uses: astral-sh/setup-uv@v8.3.2 + uses: astral-sh/setup-uv@v10.0.1 with: version: "latest" @@ -36,11 +36,14 @@ jobs: with: credentials_json: ${{ secrets.CLOUD_DEPLOY_SERVICE_ACCOUNT_KEY }} - # Feedback endpoint credentials live in Google Secret Manager, not - # GitHub secrets. The deploy service account needs - # roles/secretmanager.secretAccessor on these secrets. - - name: Fetch feedback secrets from Secret Manager - id: feedback-secrets + # Application credentials live in Google Secret Manager, not GitHub + # secrets. The deploy service account needs + # roles/secretmanager.secretAccessor on these secrets. Every secret + # listed here must already exist in the target project or the deploy + # fails -- see docs/internal-ogc-desktop-gis.md for the placeholder + # value to seed internal-ogc-api-keys with. + - name: Fetch application secrets from Secret Manager + id: app-secrets uses: 'google-github-actions/get-secretmanager-secrets@v3' with: secrets: |- @@ -48,6 +51,7 @@ jobs: jira_api_token:${{ vars.GCP_PROJECT_ID }}/jira-api-token slack_feedback_webhook_url:${{ vars.GCP_PROJECT_ID }}/slack-feedback-webhook-url slack_edits_webhook_url:${{ vars.GCP_PROJECT_ID }}/slack-edits-webhook-url + internal_ogc_api_keys:${{ vars.GCP_PROJECT_ID }}/internal-ogc-api-keys - name: Run Alembic migrations on staging database env: @@ -59,6 +63,22 @@ jobs: run: | uv run --no-dev alembic upgrade head + # Data migrations are deliberately not applied here -- they change content + # rather than structure, are often irreversible, and applying one is a + # decision made in the Data Migrations workflow. This only reports, so a + # merged migration cannot sit unnoticed. It never fails the deploy: the + # deploy worked, and a pipeline that goes red for something else is a + # pipeline people learn to ignore. + - name: Report pending data migrations + env: + DB_DRIVER: "cloudsql" + CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}" + CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}" + CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}" + CLOUD_SQL_IAM_AUTH: true + run: | + uv run --no-dev python -m scripts.report_pending_data_migrations + - name: Ensure envsubst is available run: | if ! command -v envsubst >/dev/null 2>&1; then @@ -79,6 +99,7 @@ jobs: PYGEOAPI_POSTGRES_PORT: "${{ vars.PYGEOAPI_POSTGRES_PORT || '5432' }}" PYGEOAPI_POSTGRES_PASSWORD: "${{ secrets.PYGEOAPI_POSTGRES_PASSWORD }}" PYGEOAPI_SERVER_URL: "${{ vars.PYGEOAPI_SERVER_URL }}" + INTERNAL_OGC_API_KEYS: "${{ steps.app-secrets.outputs.internal_ogc_api_keys }}" CLOUD_SQL_IAM_AUTH: "true" GCS_SERVICE_ACCOUNT_KEY: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" GCS_BUCKET_NAME: "${{ vars.GCS_BUCKET_NAME }}" @@ -86,14 +107,13 @@ jobs: AUTHENTIK_CLIENT_ID: "${{ vars.AUTHENTIK_CLIENT_ID }}" AUTHENTIK_AUTHORIZE_URL: "${{ vars.AUTHENTIK_AUTHORIZE_URL }}" AUTHENTIK_TOKEN_URL: "${{ vars.AUTHENTIK_TOKEN_URL }}" - SESSION_SECRET_KEY: "${{ secrets.SESSION_SECRET_KEY }}" APITALLY_CLIENT_ID: "${{ vars.APITALLY_CLIENT_ID }}" JIRA_BASE_URL: "${{ vars.JIRA_BASE_URL || 'https://nmbgmr.atlassian.net' }}" - JIRA_EMAIL: "${{ steps.feedback-secrets.outputs.jira_email }}" - JIRA_API_TOKEN: "${{ steps.feedback-secrets.outputs.jira_api_token }}" + JIRA_EMAIL: "${{ steps.app-secrets.outputs.jira_email }}" + JIRA_API_TOKEN: "${{ steps.app-secrets.outputs.jira_api_token }}" JIRA_DEFAULT_PROJECT: "${{ vars.JIRA_DEFAULT_PROJECT || 'BDMS' }}" - SLACK_FEEDBACK_WEBHOOK_URL: "${{ steps.feedback-secrets.outputs.slack_feedback_webhook_url }}" - SLACK_EDITS_WEBHOOK_URL: "${{ steps.feedback-secrets.outputs.slack_edits_webhook_url }}" + SLACK_FEEDBACK_WEBHOOK_URL: "${{ steps.app-secrets.outputs.slack_feedback_webhook_url }}" + SLACK_EDITS_WEBHOOK_URL: "${{ steps.app-secrets.outputs.slack_edits_webhook_url }}" OCOTILLO_UI_BASE_URL: "${{ vars.OCOTILLO_UI_BASE_URL || 'https://ocotillo-staging.newmexicowaterdata.org' }}" run: | export MAX_INSTANCES="10" diff --git a/.github/workflows/CD_testing.yml b/.github/workflows/CD_testing.yml index f9c0ac890..1323f6b0f 100644 --- a/.github/workflows/CD_testing.yml +++ b/.github/workflows/CD_testing.yml @@ -14,12 +14,12 @@ jobs: steps: - name: Check out source repository - uses: actions/checkout@v7.0.0 + uses: actions/checkout@v7.0.1 with: fetch-depth: 0 - name: Install uv in container - uses: astral-sh/setup-uv@v8.3.2 + uses: astral-sh/setup-uv@v10.0.1 with: version: "latest" @@ -36,11 +36,14 @@ jobs: with: credentials_json: ${{ secrets.CLOUD_DEPLOY_SERVICE_ACCOUNT_KEY }} - # Feedback endpoint credentials live in Google Secret Manager, not - # GitHub secrets. The deploy service account needs - # roles/secretmanager.secretAccessor on these secrets. - - name: Fetch feedback secrets from Secret Manager - id: feedback-secrets + # Application credentials live in Google Secret Manager, not GitHub + # secrets. The deploy service account needs + # roles/secretmanager.secretAccessor on these secrets. Every secret + # listed here must already exist in the target project or the deploy + # fails -- see docs/internal-ogc-desktop-gis.md for the placeholder + # value to seed internal-ogc-api-keys with. + - name: Fetch application secrets from Secret Manager + id: app-secrets uses: 'google-github-actions/get-secretmanager-secrets@v3' with: secrets: |- @@ -48,6 +51,7 @@ jobs: jira_api_token:${{ vars.GCP_PROJECT_ID }}/jira-api-token slack_feedback_webhook_url:${{ vars.GCP_PROJECT_ID }}/slack-feedback-webhook-url slack_edits_webhook_url:${{ vars.GCP_PROJECT_ID }}/slack-edits-webhook-url + internal_ogc_api_keys:${{ vars.GCP_PROJECT_ID }}/internal-ogc-api-keys - name: Run Alembic migrations on staging database env: @@ -79,6 +83,7 @@ jobs: PYGEOAPI_POSTGRES_PORT: "${{ vars.PYGEOAPI_POSTGRES_PORT || '5432' }}" PYGEOAPI_POSTGRES_PASSWORD: "${{ secrets.PYGEOAPI_POSTGRES_PASSWORD }}" PYGEOAPI_SERVER_URL: "${{ vars.PYGEOAPI_SERVER_URL }}" + INTERNAL_OGC_API_KEYS: "${{ steps.app-secrets.outputs.internal_ogc_api_keys }}" CLOUD_SQL_IAM_AUTH: "true" GCS_SERVICE_ACCOUNT_KEY: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" GCS_BUCKET_NAME: "${{ vars.GCS_BUCKET_NAME }}" @@ -86,14 +91,13 @@ jobs: AUTHENTIK_CLIENT_ID: "${{ vars.AUTHENTIK_CLIENT_ID }}" AUTHENTIK_AUTHORIZE_URL: "${{ vars.AUTHENTIK_AUTHORIZE_URL }}" AUTHENTIK_TOKEN_URL: "${{ vars.AUTHENTIK_TOKEN_URL }}" - SESSION_SECRET_KEY: "${{ secrets.SESSION_SECRET_KEY }}" APITALLY_CLIENT_ID: "${{ vars.APITALLY_CLIENT_ID }}" JIRA_BASE_URL: "${{ vars.JIRA_BASE_URL || 'https://nmbgmr.atlassian.net' }}" - JIRA_EMAIL: "${{ steps.feedback-secrets.outputs.jira_email }}" - JIRA_API_TOKEN: "${{ steps.feedback-secrets.outputs.jira_api_token }}" + JIRA_EMAIL: "${{ steps.app-secrets.outputs.jira_email }}" + JIRA_API_TOKEN: "${{ steps.app-secrets.outputs.jira_api_token }}" JIRA_DEFAULT_PROJECT: "${{ vars.JIRA_DEFAULT_PROJECT || 'BDMS' }}" - SLACK_FEEDBACK_WEBHOOK_URL: "${{ steps.feedback-secrets.outputs.slack_feedback_webhook_url }}" - SLACK_EDITS_WEBHOOK_URL: "${{ steps.feedback-secrets.outputs.slack_edits_webhook_url }}" + SLACK_FEEDBACK_WEBHOOK_URL: "${{ steps.app-secrets.outputs.slack_feedback_webhook_url }}" + SLACK_EDITS_WEBHOOK_URL: "${{ steps.app-secrets.outputs.slack_edits_webhook_url }}" OCOTILLO_UI_BASE_URL: "${{ vars.OCOTILLO_UI_BASE_URL || 'https://ocotillo-staging.newmexicowaterdata.org' }}" run: | export MAX_INSTANCES="10" diff --git a/.github/workflows/data_migrations.yml b/.github/workflows/data_migrations.yml new file mode 100644 index 000000000..32def6fd7 --- /dev/null +++ b/.github/workflows/data_migrations.yml @@ -0,0 +1,139 @@ +name: Data Migrations + +# Data migrations (data_migrations/migrations/) are not part of CD. The deploy +# workflows only run `alembic upgrade head`, so a registered data migration sits +# unapplied until someone runs it by hand against a live database -- which needs +# Cloud SQL credentials most people do not have locally. This workflow runs them +# with the same environment secrets the deploys already use. +# +# Start with action = status. It prints what is registered, what has been +# applied, and when. It applies no migration, though it is not strictly +# read-only: get_status() calls ensure_history_table(), so a database that has +# never run one gets an empty data_migration_history table created. + +on: + workflow_dispatch: + inputs: + environment: + description: "Target environment" + type: choice + options: + - staging + - production + default: staging + action: + description: "status (applies nothing) | run-all | run (single migration)" + type: choice + options: + - status + - run-all + - run + default: status + migration_id: + description: "Migration id -- required when action = run" + type: string + default: "" + include_repeatable: + description: "Include repeatable migrations (action = run-all)" + type: boolean + default: false + force: + description: "Re-run migrations already recorded as applied" + type: boolean + default: false + +permissions: + contents: read + +# One run at a time per environment: these write to a live database, and two +# concurrent runs could both pass the "already applied" check. +concurrency: + group: data-migrations-${{ inputs.environment }} + cancel-in-progress: false + +jobs: + data-migrations: + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + + steps: + - name: Validate inputs + run: | + if [ "${{ inputs.action }}" = "run" ] && [ -z "${{ inputs.migration_id }}" ]; then + echo "::error::action = run requires migration_id." + echo "::error::Run this workflow with action = status to list the registered ids." + exit 1 + fi + if [ "${{ inputs.action }}" = "status" ] && [ "${{ inputs.force }}" = "true" ]; then + echo "::warning::force has no effect on a status check." + fi + + - name: Check out source repository + uses: actions/checkout@v7.0.1 + + - name: Install uv in container + uses: astral-sh/setup-uv@v10.0.1 + with: + version: "latest" + + - name: Authenticate to Google Cloud + uses: "google-github-actions/auth@v3" + with: + credentials_json: ${{ secrets.CLOUD_DEPLOY_SERVICE_ACCOUNT_KEY }} + + - name: Report target + run: | + echo "Environment: ${{ inputs.environment }}" + echo "Database: ${{ vars.CLOUD_SQL_DATABASE }}" + echo "Action: ${{ inputs.action }}" + + # Runs before the action so the log shows the before/after pair on a + # single run, and so a status-only run needs no second step. + - name: Status before + env: + DB_DRIVER: "cloudsql" + CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}" + CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}" + CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}" + CLOUD_SQL_IAM_AUTH: true + run: uv run --no-dev oco data-migrations status + + - name: Apply migrations + if: inputs.action != 'status' + env: + DB_DRIVER: "cloudsql" + CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}" + CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}" + CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}" + CLOUD_SQL_IAM_AUTH: true + run: | + set -euo pipefail + + args=() + if [ "${{ inputs.action }}" = "run" ]; then + args=(run "${{ inputs.migration_id }}") + if [ "${{ inputs.force }}" = "true" ]; then + args+=(--force) + fi + else + args=(run-all) + if [ "${{ inputs.include_repeatable }}" = "true" ]; then + args+=(--include-repeatable) + fi + if [ "${{ inputs.force }}" = "true" ]; then + args+=(--force) + fi + fi + + echo "oco data-migrations ${args[*]}" + uv run --no-dev oco data-migrations "${args[@]}" + + - name: Status after + if: inputs.action != 'status' + env: + DB_DRIVER: "cloudsql" + CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}" + CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}" + CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}" + CLOUD_SQL_IAM_AUTH: true + run: uv run --no-dev oco data-migrations status diff --git a/.github/workflows/format_code.yml b/.github/workflows/format_code.yml index 6eb001bed..7e9797129 100644 --- a/.github/workflows/format_code.yml +++ b/.github/workflows/format_code.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out source repository - uses: actions/checkout@v7.0.0 + uses: actions/checkout@v7.0.1 - name: Set up Python environment - 3.12 uses: actions/setup-python@v7.0.0 with: @@ -34,7 +34,7 @@ jobs: contents: write pull-requests: write steps: - - uses: actions/checkout@v7.0.0 + - uses: actions/checkout@v7.0.1 with: ref: ${{ github.head_ref }} - uses: psf/black@stable diff --git a/.github/workflows/forward-merge.yml b/.github/workflows/forward-merge.yml index c84a540c9..8b36722d2 100644 --- a/.github/workflows/forward-merge.yml +++ b/.github/workflows/forward-merge.yml @@ -54,7 +54,7 @@ jobs: GH_TOKEN: ${{ secrets.FORWARD_MERGE_TOKEN || github.token }} TAG: ${{ inputs.tag_name }} steps: - - uses: actions/checkout@v7.0.0 + - uses: actions/checkout@v7.0.1 with: fetch-depth: 0 token: ${{ secrets.FORWARD_MERGE_TOKEN || github.token }} @@ -103,7 +103,7 @@ jobs: # the lockfile is re-locked (see commit 27751110). Idempotent: no # lockfile change -> no commit. - name: Install uv - uses: astral-sh/setup-uv@v8.3.2 + uses: astral-sh/setup-uv@v10.0.1 with: enable-cache: true cache-dependency-glob: uv.lock @@ -149,7 +149,7 @@ jobs: TAG: ${{ inputs.tag_name }} SOURCE: ${{ inputs.source_branch }} steps: - - uses: actions/checkout@v7.0.0 + - uses: actions/checkout@v7.0.1 with: fetch-depth: 0 ref: ${{ inputs.source_branch }} @@ -166,7 +166,7 @@ jobs: # push. Plain push (not force) so an out-of-date checkout fails loudly # instead of clobbering newer hotfix commits. - name: Install uv - uses: astral-sh/setup-uv@v8.3.2 + uses: astral-sh/setup-uv@v10.0.1 with: enable-cache: true cache-dependency-glob: uv.lock diff --git a/.github/workflows/hotfix-start.yml b/.github/workflows/hotfix-start.yml index 6f1a81e4c..bec2fcbcc 100644 --- a/.github/workflows/hotfix-start.yml +++ b/.github/workflows/hotfix-start.yml @@ -24,7 +24,7 @@ jobs: create-hotfix-branch: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7.0.0 + - uses: actions/checkout@v7.0.1 with: fetch-depth: 0 diff --git a/.github/workflows/jira_codex_pr.yml b/.github/workflows/jira_codex_pr.yml index ba9f368da..f352b7410 100644 --- a/.github/workflows/jira_codex_pr.yml +++ b/.github/workflows/jira_codex_pr.yml @@ -41,7 +41,7 @@ jobs: timeout-minutes: 60 steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4 with: fetch-depth: 0 @@ -59,7 +59,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Set up uv (with cache) - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v4 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v4 with: enable-cache: true diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index bac76bf49..3b32a2cd9 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -37,7 +37,7 @@ jobs: # configs set include-v-in-tag: true, so the tag is `v`. Prefer # the action's own output if it is ever non-empty. - if: ${{ steps.release.outputs.release_created == 'true' }} - uses: actions/checkout@v7.0.0 + uses: actions/checkout@v7.0.1 - id: resolve_tag if: ${{ steps.release.outputs.release_created == 'true' }} env: diff --git a/.github/workflows/smoke_dagster_location.yml b/.github/workflows/smoke_dagster_location.yml new file mode 100644 index 000000000..d38ab9ad8 --- /dev/null +++ b/.github/workflows/smoke_dagster_location.yml @@ -0,0 +1,74 @@ +# Materializes the heartbeat asset on a Dagster+ deployment, on demand. +# +# This is the run-time half of the deploy check. CD_dagster_*.yml prove the +# agent can load the code location; the loader is not the process that executes +# a step, and the two do not necessarily agree about sys.path, so loading +# cleanly does not prove a step can import `db` or `domain`. Materializing +# `ingestion_heartbeat` does. It touches no database, no network and no GCS, so a +# failure here is a packaging or deployment problem and nothing else. +# +# Dispatch-only, and mainly for `prod`: CD_dagster_branch.yml already runs this +# same job against a branch deployment as the last step of every PEX deploy, so +# on a PR the check happens without anyone asking. This workflow is the way to +# ask for it anywhere else -- after a prod deploy, or against a branch +# deployment that was deployed before this check existed. +# +# Note a GitHub constraint: `workflow_dispatch` is only offered for workflows +# present on the default branch, so this is not runnable from a feature branch. +# +# `deployment` takes a branch deployment id (the hex name in the Dagster+ URL, +# which CD_dagster_branch.yml prints as "Deploying to branch deployment: ...") +# or `prod`. Run it against the ref whose deployment you are testing: the job +# must exist in the deployed code location, not just on the branch. +name: Smoke test (Dagster+ code location) + +on: + workflow_dispatch: + inputs: + deployment: + description: "Dagster+ deployment: a branch deployment id, or 'prod'" + required: true + default: "prod" + +permissions: + contents: read + +concurrency: + group: dagster-smoke-${{ github.event.inputs.deployment }} + cancel-in-progress: false + +env: + DAGSTER_CLOUD_URL: https://${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }}.dagster.cloud + +jobs: + heartbeat: + runs-on: ubuntu-latest + steps: + - name: Install uv + uses: astral-sh/setup-uv@v10.0.1 + with: + version: "latest" + + # The vendor's `launch_job` action is not used here, for the reason spelled + # out in CD_dagster_branch.yml: neither it nor `dagster-cloud job launch` + # exits nonzero on a failed run, so `wait: true` waits without gating. + # Requiring the success line is what makes this workflow's result mean + # something. + - name: Materialize ingestion_heartbeat + env: + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + DEPLOYMENT: ${{ github.event.inputs.deployment }} + run: | + set -uo pipefail + out=$(uvx --from "dagster-cloud-cli==1.13.18" \ + dagster-cloud job launch \ + --url "$DAGSTER_CLOUD_URL" \ + --deployment "$DEPLOYMENT" \ + --location ocotillo-automated-ingestion \ + --job ingestion_heartbeat_check \ + --wait --interval 10 2>&1 | tee /dev/stderr) + case "$out" in + *"finished successfully"*) ;; + *) echo "::error title=Heartbeat failed::ingestion_heartbeat did not finish successfully on $DEPLOYMENT" + exit 1 ;; + esac diff --git a/.github/workflows/stale-prs.yml b/.github/workflows/stale-prs.yml index 42dc48246..9579c2ccb 100644 --- a/.github/workflows/stale-prs.yml +++ b/.github/workflows/stale-prs.yml @@ -13,7 +13,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v10 + - uses: actions/stale@v11 with: days-before-pr-stale: 14 days-before-pr-close: 0 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a82d54d46..632b9feac 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,7 +14,12 @@ jobs: unit-tests: runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + env: + COVERAGE_FAIL_UNDER: "75" MODE: development POSTGRES_HOST: localhost POSTGRES_PORT: 5432 @@ -28,7 +33,6 @@ jobs: PYGEOAPI_POSTGRES_DB: ocotilloapi_test DB_DRIVER: postgres BASE_URL: http://localhost:8000 - SESSION_SECRET_KEY: supersecretkeyforunittests AUTHENTIK_DISABLE_AUTHENTICATION: 1 services: @@ -49,7 +53,7 @@ jobs: steps: - name: Check out source repository - uses: actions/checkout@v7.0.0 + uses: actions/checkout@v7.0.1 - name: Wait for database readiness run: | @@ -63,7 +67,7 @@ jobs: exit 1 - name: Install uv - uses: astral-sh/setup-uv@v8.3.2 + uses: astral-sh/setup-uv@v10.0.1 with: enable-cache: true cache-dependency-glob: uv.lock @@ -82,7 +86,7 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('uv.lock') }} - name: Install the project - run: uv sync --locked --all-extras --dev --group cli + run: uv sync --locked --all-extras --dev --group cli --group ingestion - name: Show Alembic heads run: uv run alembic heads @@ -94,13 +98,53 @@ jobs: PGPASSWORD=postgres psql -h localhost -p 5432 -U postgres -d ocotilloapi_test -c "CREATE EXTENSION IF NOT EXISTS postgis" - name: Run tests - run: uv run pytest -vv --durations=20 --cov --cov-report=xml --junitxml=junit.xml --ignore=tests/transfers - - - name: Upload results to Codecov - uses: codecov/codecov-action@v6 + # --cov-fail-under is set here rather than in pyproject so that running a + # single test file locally does not fail on the whole-project total. + # --ignore=tests/transfers excludes the deprecated NM_Aquifer / NM_Wells + # transfer tests; those scripts are frozen and run by hand against SQL + # Server, so they must not gate a pull request. See transfers/README.md. + run: uv run pytest -vv --durations=20 --cov --cov-report=xml --cov-report=html --cov-report=term-missing --cov-fail-under="$COVERAGE_FAIL_UNDER" --junitxml=junit.xml --ignore=tests/transfers + + - name: Write coverage summary + # Runs even when the coverage gate above fails, so the job summary shows + # which modules dropped rather than only the failing total. + if: ${{ !cancelled() }} + run: | + { + echo "## Coverage" + echo + echo '```' + uv run coverage report + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Comment coverage summary on the pull request + # A comment failure must not red the build, and the GITHUB_TOKEN is + # read-only for pull requests opened from a fork. + if: ${{ !cancelled() }} + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + gh pr diff "$PR_NUMBER" --name-only > changed-files.txt + uv run python scripts/coverage_pr_comment.py \ + --changed-files changed-files.txt \ + --fail-under "$COVERAGE_FAIL_UNDER" > coverage-comment.md + gh pr comment "$PR_NUMBER" \ + --body-file coverage-comment.md \ + --edit-last --create-if-none + + - name: Upload coverage reports + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 with: - report_type: test_results - token: ${{ secrets.CODECOV_TOKEN }} + name: coverage-${{ github.run_id }} + path: | + coverage.xml + htmlcov/ + junit.xml + retention-days: 14 bdd-tests: runs-on: ubuntu-latest @@ -119,7 +163,6 @@ jobs: PYGEOAPI_POSTGRES_DB: ocotilloapi_test DB_DRIVER: postgres BASE_URL: http://localhost:8000 - SESSION_SECRET_KEY: supersecretkeyforunittests AUTHENTIK_DISABLE_AUTHENTICATION: 1 DROP_AND_REBUILD_DB: 1 @@ -141,7 +184,7 @@ jobs: steps: - name: Check out source repository - uses: actions/checkout@v7.0.0 + uses: actions/checkout@v7.0.1 - name: Wait for database readiness run: | @@ -155,7 +198,7 @@ jobs: exit 1 - name: Install uv - uses: astral-sh/setup-uv@v8.3.2 + uses: astral-sh/setup-uv@v10.0.1 with: enable-cache: true cache-dependency-glob: uv.lock @@ -174,7 +217,7 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('uv.lock') }} - name: Install the project - run: uv sync --locked --all-extras --dev --group cli + run: uv sync --locked --all-extras --dev --group cli --group ingestion - name: Show Alembic heads run: uv run alembic heads diff --git a/.gitignore b/.gitignore index b001e6f5e..eb6f7c340 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ wheels/ .coverage.* htmlcov/ coverage.xml +junit.xml # Virtual environments .venv diff --git a/.release-please-manifest.staging.json b/.release-please-manifest.staging.json index fa8324b78..63ae49803 100644 --- a/.release-please-manifest.staging.json +++ b/.release-please-manifest.staging.json @@ -1,3 +1,3 @@ { - ".": "1.2.0-rc.1" + ".": "1.3.0-rc" } diff --git a/ADR4.md b/ADR4.md new file mode 100644 index 000000000..00bd92cdb --- /dev/null +++ b/ADR4.md @@ -0,0 +1,80 @@ +# ADR4: A Domain Layer for Import Rules + +## Status + +Accepted, partially applied. The `domain/` package exists and the two CSV +importers use it. The rest of `services/` is untouched and stays that way until +someone has a reason to open those files. + +## Context + +`services/` is documented as "business logic and database interactions", and it +does both in the same functions. The clearest example is +`services/well_inventory_csv.py`: a single call to `_add_csv_row` mixed unit +conversion, cross-column validation, note formatting, and `session.add(...)`. + +Three consequences: + +1. **Rules could not be tested without a database.** Verifying that a + measuring point height conflict is rejected meant standing up PostGIS, + building a `Thing`, and running an import. +2. **Rules drifted between callers.** The groundwater-level sample name was + written out three times across two files. The foot/meter conversion was + duplicated until BDMS-284 consolidated it. Field staff contact lookup had + two different WHERE clauses, one of which was wrong (see below). +3. **There was no obvious home for a new rule.** `services/util.py` had quietly + become one — it holds the unit conversions — but nothing named it as such, so + the next rule went wherever it was first needed. + +## Decision + +Add a `domain/` package holding business rules as plain functions over plain +values. Modules there import nothing from `api/`, `db/`, `schemas/`, or +`services/`, and no `fastapi`, `sqlalchemy`, `pydantic`, or `httpx`. + +`services/` keeps its orchestration role: load rows, call the rule, persist the +result, translate errors into the transport's shape. + +Domain errors subclass `ValueError`, because the importers already treat a +`ValueError` raised while handling a row as a per-row validation failure rather +than an aborted run. + +### What we did *not* decide + +This is not an adoption of hexagonal architecture or DDD. There are no entities, +repositories, aggregates, or mapping layers, and `services/` still talks to +SQLAlchemy models directly. The cost of a full restructure is not justified at +this size, and a half-applied one — domain objects that quietly hold a session — +is worse than none. + +Extraction is opportunistic: when you open an importer to change a rule, move +the rule. There is no migration plan for the remaining service modules. + +## Consequences + +**Good.** The extracted rules have 67 tests that need no database and run in +seconds. `services/util.py` no longer has to be imported to convert feet to +meters, which previously dragged in `httpx`, `pyproj`, and SQLAlchemy. + +**Cost.** One more package, and a rule now lives one call away from where it is +used. For a rule with a single caller this is pure overhead; extract when a rule +is shared, subtle, or expensive to test in place, not by default. + +**Watch for.** `services/util.py` re-exports the unit conversions for backwards +compatibility. That re-export is a transition aid, not a pattern — new code +should import from `domain.units`. + +## Notes + +Aligning the two field-staff contact lookups surfaced a real defect. +`services/water_level_csv.py` looked contacts up on `(name, organization)` with a +comment explaining that `Contact` enforces uniqueness on exactly that pair, while +`services/well_inventory_csv.py` also filtered on `contact_type`. The second form +misses an existing contact created with a different type and then fails on the +duplicate insert. Both now use the `(name, organization)` key. + +Two remaining copies of the enum-unwrapping idiom in +`services/well_inventory_csv.py` (`groundwater_level_reason`, `nma_data_quality`) +were left alone: each treats a falsy non-enum value slightly differently from +`domain.values.enum_value`, and reconciling them is a behavior change that wants +its own ticket. diff --git a/CHANGELOG-rc.md b/CHANGELOG-rc.md index ffaf3b394..ad8001036 100644 --- a/CHANGELOG-rc.md +++ b/CHANGELOG-rc.md @@ -1,5 +1,124 @@ # Changelog +## [1.3.0-rc](https://github.com/DataIntegrationGroup/OcotilloAPI/compare/v1.2.1...v1.3.0-rc) (2026-08-24) + + +### Features + +* **chemistry:** report which legacy table a result came from ([67b522c](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/67b522ca6bc26b8995a0e2d2d7e24b3dac8bbba8)) +* **chemistry:** serve legacy water chemistry over REST ([dcb0cb1](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/dcb0cb198b9faa1e8d41c203c8fff3c5b831580a)) +* **chemistry:** serve legacy water chemistry over REST ([97d7f9f](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/97d7f9fbff533dd19c933148decf18c9acb3cf45)) +* **data-migrations:** publish existing project_areas ([70c688f](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/70c688f49528542c6eded1974a9f4cfaae8eb2cd)) +* **geothermal:** add /thing/geothermal-well endpoints ([c60faf8](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/c60faf8b56504e07428bc344c43d2e635483a64b)) +* **geothermal:** add /thing/geothermal-well endpoints ([83e6604](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/83e66040a2977e95a429360c0ba11b8ba6eb3327)) +* **geothermal:** free-text search on the well list endpoint ([7e0a259](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/7e0a2597cd63fedea065cda845d13fb7306358cb)) +* **geothermal:** free-text search on the well list endpoint ([e5748a5](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/e5748a540aec737136ac13795c2f9f917f198c37)) +* **geothermal:** normalize OGC view temperatures to Celsius ([c6cfb7e](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/c6cfb7e1d22536b6e0b74ba53499c7d75c76f6b6)) +* **geothermal:** normalize OGC view temperatures to Celsius ([013fd75](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/013fd7554702f9e4c13ebdf006cd726f4b957c8e)) +* **gis:** generate shareable QGIS and ArcGIS Pro artifacts ([04fafce](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/04fafcea29ff5278ac224eff4f9dc7181e5000e0)) +* **gis:** serve the artifact catalogue as JSON for frontend clients ([3aa6611](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/3aa6611ce96bf87722143fe304aa5d7f66bd1fd3)) +* **ingestion:** add automated ingestion pipeline foundations ([055b51e](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/055b51ea9a26fb9b5df6d536ff18cc4d6f4b58eb)) +* **ingestion:** add raw-zone infrastructure and database connectivity ([475841d](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/475841da0b0cbb3e7f4fdd91974318de34fc445e)) +* **ingestion:** add the Diver-HUB client and correct the source mapping ([e01fd8c](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/e01fd8c34f3eccc7f65000e54118042033d547c3)) +* **ingestion:** add the shared backfill primitives ([279ff6e](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/279ff6e0ea33b89db5f04d83a34653bafd0cc415)) +* **ingestion:** add the shared backfill primitives ([6d1ced3](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/6d1ced36e8965ed313f7b10b162cb229716adc87)) +* **ingestion:** add the transducer unique constraint and upsert loader ([6f3232d](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/6f3232d8ba350bb0228bf5c6f8568cf1411b2a83)) +* **ingestion:** add the transducer unique constraint and upsert loader ([31e0644](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/31e06448988f0edff6a6c19b98caa61c4a385645)) +* **ingestion:** add the Van Essen domain rules and adapter ([34c4b1f](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/34c4b1f281d8c36ae3eee55f7d3640c36c3beedf)) +* **ingestion:** add the Van Essen domain rules and adapter ([91f4504](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/91f450436bab030a9e834271169c68c6bfd25cae)) +* **ingestion:** derive the watermark from Postgres ([87b8e9c](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/87b8e9c24324e6abde5cbd0c415b4ea87e7c0bf3)) +* **ingestion:** derive the watermark from Postgres ([a0af312](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/a0af3127d52d97c2baa0f795b0b203c362cbd891)) +* **ingestion:** land San Acacia locations and readings in the raw zone ([6c9ab70](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/6c9ab7097b234d37e15fb71e29f42a0cd69422cb)) +* **ingestion:** reconcile San Acacia points against Ocotillo wells ([fecbae8](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/fecbae8f4c77618cde0a1d8b48ccfa055643e3d1)) +* **ingestion:** reconcile San Acacia points against Ocotillo wells ([c20e2e3](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/c20e2e39795f161d6ae3405df3cc9f30e506e131)) +* **ingestion:** resolve the datum enum and the source unit ([d10b2b4](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/d10b2b457b21aa36d135eef3245fbd3c49d822d0)) +* **ingestion:** scaffold the automated_ingestion Dagster code location ([0a2109e](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/0a2109e8a514c618d794cc317665b33777135ac0)) +* **ingestion:** schedule the San Acacia ingest weekly ([8c02977](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/8c029773cb6a0d4e2d3276e37b63c5a3ddd80da1)) +* **ingestion:** schedule the San Acacia ingest weekly ([e4b1880](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/e4b1880c89d5c93369b3ed2b6469614376c9965c)) +* **ingestion:** wire the loader end to end ([3e38614](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/3e38614bebab7d5e69707f1bb62ababb859c3c86)) +* **ingestion:** wire the loader end to end ([113fdd6](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/113fdd6f4d8ef1f786c9bc8dfd27650d2b2f5ac2)) +* **lexicon:** add new organizations to lexicon ([b30bea8](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/b30bea8b58242c805f9c65f41bf7b758b29ae9f7)) +* **lexicon:** add organization and sort organization category alphabetically ([1057c86](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/1057c86397902ac94089bf1f7a8fa229049f08ea)) +* **lexicon:** add organization and sort organization category alphabetically ([21090b5](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/21090b50e90eea9d52295cc68b18c2926e9dd9a5)) +* **ogc:** add authenticated internal OGC mount (BDMS-985 A11) ([ce45b91](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/ce45b919db9004aeb1bcac065b315044986eb2c9)) +* **ogc:** add public data disclaimer page ([72f26a5](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/72f26a56c349d59e16a3110eb54cec035eb4ee54)) +* **ogc:** add the field-description source of truth ([832b659](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/832b659b18b3c186bfa7b186df6fa44b0b616276)) +* **ogc:** carry field descriptions onto /queryables ([1d41b56](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/1d41b5654aca4a27ed9b11a37d43f7db3829066e)) +* **ogc:** document EDR parameters and cover the lot with tests ([ac0513d](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/ac0513d36431bb3bcb64185dc98a0a551f1ef15e)) +* **ogc:** expose last_observation_date on the Group A layers ([7aa1746](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/7aa174631546ae1673f7c293baf3d92039ae08f9)) +* **ogc:** filter ogc_* views to public records ([dbc760b](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/dbc760bf91bb072b8402a8f52190e1f1411e432f)) +* **ogc:** make /ogcapi-internal usable from ArcGIS Pro and QGIS ([cfdf146](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/cfdf1461d7f60b4ad31c1791a647a5c5a8910035)) +* **ogc:** make /ogcapi-internal usable from ArcGIS Pro and QGIS ([4c0099e](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/4c0099eb2f745312d5a43f5a9bb2ed25c32578b5)) +* **ogc:** mirror EDR views in internal OGC mount ([cc5f72c](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/cc5f72cfec7318b9200cc2fd9c013a972ee4aa23)) +* **ogc:** ogc A2 replace server metadata placeholders (BDMS-972) ([fd4c446](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/fd4c446ba18eb7f87f1103c082ba8fdf3dc00de3)) +* **ogc:** populate the schema view's Values column ([9fc6966](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/9fc69668792cb1eb9534ae40cea874c1b4178355)) +* **ogc:** publish a well water-column layer ([1a9361a](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/1a9361a7c0f7b03e5a3729ec4fa0d8dae2b50e03)) +* **ogc:** publish a well water-column layer ([8a0c3ce](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/8a0c3cef8e18758f7c019f37f3869c5d6f4b48a6)) +* **ogc:** replace server metadata placeholders ([93206ab](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/93206abee8a68aa4cfeba58669eb48b47e0fadc7)) +* **ogc:** serve field descriptions from /schema ([f6af982](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/f6af982633c31c36977f80ffbf31ca58c1e2cc5c)) +* **scripts:** seed the test DB with real NMA legacy chemistry ([8e55d1d](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/8e55d1d44d98df7e54df5becc32d29a4bab439c4)) +* **transducer:** add data_maturity to observations ([09926b2](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/09926b239d2e78b8a29b6f3f3198220ac2d15aad)) +* **transducer:** add data_maturity to observations ([9dff7fb](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/9dff7fb5a9282f14a182500d51320e606683ba44)) +* **transducer:** backfill data_maturity on acoustic (Wellntel) observations ([60ffcf4](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/60ffcf44efd609a30c91ea02b3b02ee390d7112a)) +* **transducer:** backfill data_maturity on acoustic observations ([7a3915f](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/7a3915f41a7eaa3da7f6886e0a93d3caa7cbdfe0)) +* **transducer:** publish and range-delete for corrected hydrographs ([cfd9243](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/cfd9243baa1fb46bfe0801802e5773d71b8a7628)) +* **transducer:** publish and range-delete for corrected hydrographs ([63bf502](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/63bf5023e1cd4e538671350b698acbd4107d5562)) + + +### Bug Fixes + +* **api/asset:** update access from admin to editor ([c73d866](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/c73d8668afd46f2c00831780d5eb034e535eb6c5)) +* **build:** package data_migrations with the app ([32c6462](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/32c6462afb05f433863acf96d35ad4147052a673)) +* **ci:** assert the heartbeat run succeeded ([f95ad29](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/f95ad29ddee91318b7076f4271fc1e1f66fd9a29)) +* correct water_wells collection name in README examples and Added time_field(BDMS-973) ([#823](https://github.com/DataIntegrationGroup/OcotilloAPI/issues/823)) ([395a63c](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/395a63c95712d241d93960442ff9c94981503a90)) +* **db:** repair EDR water views skipped by a stamped revision ([0cc0d93](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/0cc0d93af78814608d0c8c3c2a38f6b9c3b6e308)) +* **db:** repair EDR water views skipped by a stamped revision ([e448741](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/e448741fe6a733a72f2e994b239c915dc4318ca0)) +* drop support for Python 3.7-3.9 (<a ([b779661](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/b7796616bf7e8a3adac0e425f48f688013d763b2)) +* **edr:** expose thing_type and materialize the chemistry coverages ([aac3d87](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/aac3d8714512c840b6930d640d54e0fbd4e0a586)) +* **edr:** implement pygeoapi's instance contract ([02f0fe5](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/02f0fe50f4825c2b46580a2ff9dfe8acb4388d35)) +* **edr:** source water-chemistry EDR from the legacy NMA tables ([ad37f78](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/ad37f783e034b92e2fcbd27cba525fded5f5ea85)) +* **edr:** source water-chemistry EDR from the legacy NMA tables ([8863430](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/88634304f44b8a445f60b8e909e356f5c1f73080)) +* expand actively_monitored_wells to include wells from all groups(BDMS-974/1178) ([#866](https://github.com/DataIntegrationGroup/OcotilloAPI/issues/866)) ([550fc18](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/550fc18331be4f7f647a3d0fe87b999cf89fa096)) +* **gis:** document the content type the artifact routes actually send ([70b21b8](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/70b21b893e7e9f3f071e2b37a757018ffd6c8bae)) +* **ingestion:** do not match on external ids by default ([d102cb7](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/d102cb7b4b760bdbc4beaa50ee75101995a88931)) +* **ingestion:** do not overwrite approved observations by default ([b522052](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/b522052b26c2828bd0424e72e2d69982867a4fc4)) +* **ingestion:** do not overwrite approved observations by default ([5a50381](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/5a50381b243569e64348ab1b5f95db3509ba8e64)) +* **ingestion:** grant bucket read, and name the pipeline after the bucket ([b127f52](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/b127f5230dded63db0274fc3ef277646c5cc57b1)) +* **ingestion:** grant bucket read, and name the pipeline after the bucket ([c390073](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/c3900738408544a8614f347e11dde3bdf24aba1b)) +* **ingestion:** import ThingIdLink from where it actually lives ([50963f4](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/50963f4e42f275dd9aafd4b041a6cc8701aafcec)) +* **ingestion:** install the repository into the Dagster+ image ([619cdf3](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/619cdf3686d4b676ecd41d797cbe4dea68a6f2d0)) +* **ingestion:** install the repository into the Dagster+ image ([7795959](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/7795959b6d3fd4f8511f54426329d5fd0ac8b603)) +* **ingestion:** make db and domain importable in the deployed image ([a425954](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/a4259545fd47344f4e8262178198e85463c9b0bb)) +* **ingestion:** make the IAM database path internally consistent ([f246d61](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/f246d61107f376a23add2856968eda6c5085c70f)) +* **ingestion:** make the role grants runnable and drop the CREATE ROLE ([0bd2213](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/0bd2213a30061da949434ee0b29c8dd15739b15f)) +* **ingestion:** make the role grants runnable and drop the CREATE ROLE ([33b541c](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/33b541c866331092a29cf0e3af3bef30d3ce05b3)) +* **ingestion:** raise the ingestion floor to 2024 ([ac7b5e8](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/ac7b5e86a1182316bd65ac3e79a420ce0eb972fe)) +* **ingestion:** reject a bare Cloud SQL instance name ([a178f90](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/a178f90132cacabbc09ccd4cb77055defc48d839)) +* **ingestion:** repair two CI failures the first PR run exposed ([fd6e869](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/fd6e86920a27ef373be7782c8e3c2736fe1ec7c1)) +* **ingestion:** report the import environment from the heartbeat asset ([5ccb0d6](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/5ccb0d67602b844bb79182647f20a7c496f7869f)) +* **ingestion:** set code location env vars at a scope that reaches the container ([773de45](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/773de45af40ef94be64e38d2ce496852afdffa6e)) +* **ingestion:** set code location env vars at a scope that reaches the container ([affa35c](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/affa35c3e0f5504cf4d85588f73e91a48bc21d50)) +* **ingestion:** set PYTHONPATH and report the import environment ([1df7d8c](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/1df7d8c63a85ce7a48146fd9ba31159a2c32e06c)) +* **ingestion:** stop duplicate instants reaching one INSERT ([ef38ef2](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/ef38ef236149f99b1b7b9a2e2c075a7ff01e9ff7)) +* **ingestion:** stop duplicate instants reaching one INSERT ([1405ce9](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/1405ce9c05c75df7c0e4c6cf288249d02c3b58a8)) +* **ingestion:** supply GCP credentials in a runtime that has none ([8c1c32d](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/8c1c32d8ac2f5ff7d2aaae7e84df9583c5608b05)) +* **ingestion:** supply GCP credentials in a runtime that has none ([2799f91](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/2799f9188f20770cc8d14540a0443a70dca11632)) +* **ingestion:** write the raw zone as parquet ([9aa0eac](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/9aa0eac8e1cf6ae5ca9b6d6c9caac78b1efb0851)) +* **ingestion:** write the raw zone as parquet ([78bc921](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/78bc921627fbad2ad546c84aa7a1a375d66cba4e)) +* **ogc:** drop duplicate collections step definition ([3882a5d](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/3882a5df77138a46e34fd53711284892e01a2408)) +* **ogc:** gate ogc_waterlevels on the well's release status ([c47f481](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/c47f4813a7a4e47aad9b149a70631ec727ee8b1c)) +* **ogc:** isolate public and internal pygeoapi module globals ([5bb2aae](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/5bb2aae7cf967a967e88610365df6dfe216b6b0f)) +* **ogc:** isolate public and internal pygeoapi module globals ([860a3f4](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/860a3f45ed7ab0ca352b69d006324c8f9fb7a289)) +* **ogc:** re-point migration to new staging head ([8ae9fe1](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/8ae9fe1873f3c6e7bbd62ea9e8e08fa76bdae1db)) +* **ogc:** stop EDR field dicts leaking between requests ([598f1ac](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/598f1acc7168743aa742667ec6e5d6769e8434c7)) +* **seed:** load reference data even when migrations seeded a term ([cdb4246](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/cdb4246d1cde5474b44f293988144f6642088794)) +* **seed:** load reference data even when migrations seeded a term ([fb64f68](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/fb64f68300631b871dc12b569b1f75214802ce00)) +* **tests:** remove hardcoded group id assumption ([2c17ce6](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/2c17ce6301f11db4108bb6d977597b281a3af4b6)) +* **thing:** a well with no location no longer 500s the listing ([bff7faa](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/bff7faa91dbc14f5b3f8635082a1d32b9e3a77ed)) +* **transducer:** backfill data_maturity from the legacy QC flag ([95b9b78](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/95b9b78b8dbd74b5ee08859bb48d57ff3192ec75)) +* **transducer:** serialize series writes and scope the publish parameter ([4701979](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/4701979f8b4240e1bd1db53e820626476344e8df)) +* **transducer:** spell the block time-order constraint correctly ([ba73c6b](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/ba73c6b012c9c780cb3dc5c3d0bd67e559882867)) + ## [1.2.0-rc.1](https://github.com/DataIntegrationGroup/OcotilloAPI/compare/v1.2.0-rc...v1.2.0-rc.1) (2026-07-21) diff --git a/CLAUDE.md b/CLAUDE.md index d193e6a7c..a5f9358c7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -77,9 +77,15 @@ POSTGRES_PASSWORD= ``` ### Data Migration +Both legacy transfer drivers are **deprecated** (see `transfers/README.md`); they +raise `DeprecationWarning` and take no new migrations, but stay runnable for +backfills. ```bash -# Transfer data from legacy AMPAPI (NM_Aquifer) to new schema +# NM_Aquifer (AMPAPI) -> new schema. Deprecated. python -m transfers.transfer + +# NM_Wells (geothermal) Phase-1 staging mirror. Deprecated. +python -m transfers.transfer_geothermal ``` ## Architecture @@ -117,8 +123,9 @@ Location (geographic point) ├── db/ # SQLAlchemy models (one file per table/resource) │ ├── engine.py # Database connection configuration │ └── ... +├── domain/ # Business rules as plain functions (no DB, no HTTP) ├── schemas/ # Pydantic schemas (validation, serialization) -├── services/ # Business logic and database interactions +├── services/ # Orchestration: load, call domain rules, persist ├── tests/ # Pytest test suite │ ├── conftest.py # Shared fixtures (test data setup) │ └── __init__.py # Sets test database (ocotilloapi_test) @@ -129,6 +136,22 @@ Location (geographic point) └── main.py # Application entry point ``` +### Domain Rules + +`domain/` holds business rules as plain functions over plain values -- unit +conversion, cross-column validation, deterministic naming. Modules there import +nothing from `api/`, `db/`, `schemas/`, or `services/`, and no `fastapi`, +`sqlalchemy`, `pydantic`, or `httpx`, so the rules are testable without a +database. + +`services/` loads the data, calls the rule, and persists the result. Domain +errors subclass `ValueError` because the CSV importers treat a `ValueError` +raised on a row as a per-row validation failure. + +Extraction is opportunistic, not a migration: move a rule into `domain/` when +you are already editing it and it is shared, subtle, or awkward to test in +place. Read **`ADR4.md`** before extending the layer. + ### Authentication & Authorization The system uses **Authentik** for OAuth2 authentication with role-based access control: @@ -138,8 +161,60 @@ The system uses **Authentik** for OAuth2 authentication with role-based access c - **Editor**: Can modify existing records (includes Viewer permissions) - **Admin**: Can create new records (includes Editor + Viewer permissions) +The hierarchy is enforced in code, via `authenticated(any_of=[...])` group lists — +`Admin` satisfies an editor- or viewer-gated route without needing all three +Authentik groups granted. + **AMP-Specific Roles**: `AMPAdmin`, `AMPEditor`, `AMPViewer` for legacy AMPAPI integration +**Role families are orthogonal**: general `Admin` confers nothing in the AMP or +Lexicon families. Only tiers *within* a family nest. + +**`AMP.Staging`** is a standalone group, not a fourth AMP tier — `AMPAdmin` +does not satisfy it. It gates the hydrograph corrector's publish and range-delete +routes while the workbench is being validated against real logger files, so they +ship dark. Read **`docs/hydrograph-correction-publish.md`** before changing +them. + +**Authorization is opt-in per endpoint** — a `user: _dependency` parameter +in the signature, not a router-level `dependencies=[...]`. Omitting it produces a +fully public endpoint with no error. `tests/test_authorization.py` holds the +allowlist of intentionally anonymous routes and fails on anything else. Note the +annotation must be a *type annotation* (`user: viewer_dependency`), never a +default value (`user=viewer_dependency`) — the latter silently disables the +dependency and FastAPI treats it as a query parameter. + +**Development bypass**: `AUTHENTIK_DISABLE_AUTHENTICATION=1` is honored only when +`MODE=development`. Any other `MODE` (including unset) makes +`assert_auth_configuration()` abort startup. + +**`@in_public_schema`** (`core/app.py`) controls anonymous OpenAPI visibility +only — it grants no access and removes no dependency. Apply it only to routes +that genuinely have none. + +**`/ogcapi-internal` is gated outside `Depends()`.** It is a raw Starlette +Mount, so `core/internal_ogc_auth.py` gates it at the ASGI layer instead. It +accepts a bearer Authentik JWT carrying `OGCInternal`, **or** a static API key +presented as a bearer token, as the Basic password, or as `?token=`. Only the +key digests are stored, as `label:sha256hex` entries in `INTERNAL_OGC_API_KEYS` +— sourced in deployed environments from the Secret Manager secret +`internal-ogc-api-keys` at deploy time, so revoking a key needs a redeploy. +Never a GitHub secret. The static keys exist because +ArcGIS Pro cannot send a bearer token at all and neither desktop client can +refresh an Authentik token. Read **`docs/internal-ogc-desktop-gis.md`** before +changing the credential paths. + +### OGC field descriptions + +Per-column `title`/`description`/unit for every collection lives in +`core/ogc-field-descriptions.yml`, keyed by backing relation, and is published +on `/schema` and `/queryables` through `core/feature_provider.py` and a wrapper +over pygeoapi's queryables handler. The feature leans on unpinned behaviour of +the pinned pygeoapi version — most sharply, `BaseProvider.fields` returns +`self._fields` and never calls `get_fields()`. Read +**`docs/ogc-field-descriptions.md`** before changing field metadata or +upgrading pygeoapi. + ### Database Configuration The application supports two database modes (configured via `DB_DRIVER` in `.env`): @@ -233,6 +308,24 @@ GitHub Actions workflows (`.github/workflows/`): ## Legacy System Migration +**Deprecated.** Both legacy drivers are frozen -- `transfers/transfer.py` +(NM_Aquifer/AMPAPI) and `transfers/transfer_geothermal.py` (NM_Wells, with +`nmw_mirror_transfer.py`, `nmw_sql_dump.py`, `export_nmw_csvs.py`). Entry points +raise `DeprecationWarning`. Do not add new migrations to either. They remain +runnable because live API routes still read the `NMA_*` and `NMW_*` tables. +Read **`transfers/README.md`** before touching this layer. + +Their tests live in `tests/transfers/` and **do not gate CI** -- +`.github/workflows/tests.yml` runs pytest with `--ignore=tests/transfers`, and +`transfers/*` is omitted from coverage in `pyproject.toml`. Run them by hand: +`uv run pytest tests/transfers`. Tests for the `NMA_*`/`NMW_*` ORM models +(`db/nma_legacy.py`, `db/nmw_legacy.py`) stay in `tests/` proper and still gate +CI, since live routes depend on those models. + +Still live, *not* deprecated: `services/scoped_transfer.py` and the +`oco scoped-transfer` command, which import the individual NM_Aquifer +transferers directly. + **Source**: AMPAPI (SQL Server, `NM_Aquifer` schema) **Target**: OcotilloAPI (PostgreSQL + PostGIS) diff --git a/Procfile b/Procfile deleted file mode 100644 index 2486669cb..000000000 --- a/Procfile +++ /dev/null @@ -1 +0,0 @@ -web: python3 -m transfers.transfer diff --git a/README.md b/README.md index 90ca4bc99..5b5df58b8 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,6 @@ [![Dependabot Updates](https://github.com/DataIntegrationGroup/NMSampleLocations/actions/workflows/dependabot/dependabot-updates/badge.svg)](https://github.com/DataIntegrationGroup/NMSampleLocations/actions/workflows/dependabot/dependabot-updates) [![Sentry Release](https://github.com/DataIntegrationGroup/NMSampleLocations/actions/workflows/release.yml/badge.svg)](https://github.com/DataIntegrationGroup/NMSampleLocations/actions/workflows/release.yml) [![Tests](https://github.com/DataIntegrationGroup/NMSampleLocations/actions/workflows/tests.yml/badge.svg)](https://github.com/DataIntegrationGroup/NMSampleLocations/actions/workflows/tests.yml) -[![codecov](https://codecov.io/gh/DataIntegrationGroup/NMSampleLocations/graph/badge.svg?token=Y20QB357OO)](https://codecov.io/gh/DataIntegrationGroup/NMSampleLocations) **Geospatial Sample Data Management System** _New Mexico Bureau of Geology and Mineral Resources_ @@ -29,7 +28,7 @@ supports research, field operations, and public data delivery for the Bureau of ## 🗺️ OGC API - Features The API exposes OGC API - Features endpoints under `/ogcapi` using `pygeoapi`. -In App Engine deployments, `/admin` and `/ogcapi` are served from the same +In App Engine deployments, `/ogcapi` is served from the same application as the primary API. The service is intended to scale to zero outside business hours and be kept warm during the workday with Cloud Scheduler hits to `/_ah/warmup`. @@ -40,23 +39,22 @@ hits to `/_ah/warmup`. curl http://localhost:8000/ogcapi curl http://localhost:8000/ogcapi/conformance curl http://localhost:8000/ogcapi/collections -curl http://localhost:8000/ogcapi/collections/locations +curl http://localhost:8000/ogcapi/collections/water_wells ``` ### Items (GeoJSON) ```bash -curl "http://localhost:8000/ogcapi/collections/locations/items?limit=10&offset=0" -curl "http://localhost:8000/ogcapi/collections/wells/items?limit=5" +curl "http://localhost:8000/ogcapi/collections/water_wells/items?limit=10&offset=0" curl "http://localhost:8000/ogcapi/collections/springs/items?limit=5" -curl "http://localhost:8000/ogcapi/collections/locations/items/123" +curl "http://localhost:8000/ogcapi/collections/water_wells/items/123" ``` ### BBOX + datetime filters ```bash -curl "http://localhost:8000/ogcapi/collections/locations/items?bbox=-107.9,33.8,-107.8,33.9" -curl "http://localhost:8000/ogcapi/collections/wells/items?datetime=2020-01-01/2024-01-01" +curl "http://localhost:8000/ogcapi/collections/water_wells/items?bbox=-107.9,33.8,-107.8,33.9" +curl "http://localhost:8000/ogcapi/collections/water_wells/items?datetime=2020-01-01/2024-01-01" ``` ### Polygon filter (CQL2 text) @@ -64,7 +62,7 @@ curl "http://localhost:8000/ogcapi/collections/wells/items?datetime=2020-01-01/2 Use `filter` + `filter-lang=cql2-text` with `WITHIN(...)`: ```bash -curl "http://localhost:8000/ogcapi/collections/locations/items?filter=WITHIN(geometry,POLYGON((-107.9 33.8,-107.8 33.8,-107.8 33.9,-107.9 33.9,-107.9 33.8)))&filter-lang=cql2-text" +curl "http://localhost:8000/ogcapi/collections/water_wells/items?filter=WITHIN(geometry,POLYGON((-107.9 33.8,-107.8 33.8,-107.8 33.9,-107.9 33.9,-107.9 33.8)))&filter-lang=cql2-text" ``` ### OpenAPI UI @@ -152,7 +150,6 @@ Minimum vars to set in `.env` for local development: * `POSTGRES_HOST` (`localhost` for local psql/pytest against mapped Docker port) * `POSTGRES_PORT` (`5432`) * `MODE` (`development` recommended locally) -* `SESSION_SECRET_KEY` (required if you want to use `/admin`) Auth-related vars (required when auth is enabled, optional when `AUTHENTIK_DISABLE_AUTHENTICATION=1`): * `AUTHENTIK_DISABLE_AUTHENTICATION` @@ -206,7 +203,7 @@ Notes: * Requires Docker Desktop. * By default, spins up two containers: * `db` for PostGIS/PostgreSQL - * `app` for the primary API, admin UI, and OGC API on `http://localhost:8000` + * `app` for the primary API and OGC API on `http://localhost:8000` * `db` initializes both application databases in the same Postgres service: * `ocotilloapi_dev` * `ocotilloapi_test` @@ -216,7 +213,6 @@ Notes: * test: `ocotilloapi_test` (created by init SQL in `docker/db/init/01-create-test-db.sql`) * The database listens on port `5432` both inside the container and on your host. Ensure `POSTGRES_PORT=5432` and `POSTGRES_DB=ocotilloapi_dev` in your `.env` to run local commands against the Docker dev DB (e.g., `uv run pytest`, `uv run python -m transfers.transfer`). * To restore a local or GCS-backed SQL dump into your local target DB, run `source .venv/bin/activate && python -m cli.cli restore-local-db path/to/dump.sql` or `source .venv/bin/activate && python -m cli.cli restore-local-db gs://ocotillo/sql-exports/latest.sql.gz`. -* `SESSION_SECRET_KEY` only needs to be set in `.env` if you plan to use `/admin`; without it, the API and `/ogcapi` still boot, but `/admin` will be unavailable. #### Staging Data diff --git a/SPEC.md b/SPEC.md index 3731fa02e..727c3ade7 100644 --- a/SPEC.md +++ b/SPEC.md @@ -61,7 +61,7 @@ T8|x|export_nmw_csvs.py pymssql export|I.export T9|x|transfer_geothermal.py orchestrator|I.cli T10|x|6 OGC collections in pygeoapi-config.yml|V6,I.ogc T11|x|FK enforced via migration op.create_foreign_key; model index-only (resolved)|V2,V10 -T12|x|add NMW_* mirror/loader/migration/OGC tests (tests/test_nmw_mirror.py, 19 tests); found+fixed CAST-unwrap bug B1|V1,V2,V3,V5,V6,V10,V11 +T12|x|add NMW_* mirror/loader/migration/OGC tests (tests/transfers/test_nmw_mirror.py, 19 tests); found+fixed CAST-unwrap bug B1|V1,V2,V3,V5,V6,V10,V11 T13|.|verify alembic down path drops all views+tables (V3) on real db|V3 T14|.|run end-to-end load vs real dump, capture row counts per table|V2,V4 T15|.|finish PR #738 body (truncated at "- I ") + reviewer notes|- diff --git a/admin/auth.py b/admin/auth.py deleted file mode 100644 index 903068ab7..000000000 --- a/admin/auth.py +++ /dev/null @@ -1,298 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -Admin authentication provider integrating with existing Authentik OIDC auth. - -This module provides a Starlette Admin AuthProvider that integrates with the -existing Authentik-based authentication system used by the OcotilloAPI API. -""" - -import base64 -import hashlib -import os -import secrets -from core.permissions import _get_token_payload, verify_token -from dataclasses import dataclass -from starlette.requests import Request -from starlette.responses import RedirectResponse -from starlette_admin.auth import AdminUser, AuthProvider -from starlette_admin.exceptions import LoginFailed -from typing import List -from typing import Optional -from urllib.parse import urlencode - - -@dataclass -class AdminUserWithRoles(AdminUser): - """Extended AdminUser with roles for RBAC.""" - - roles: List[str] = None - - def __post_init__(self): - if self.roles is None: - self.roles = [] - - -class NMSampleLocationsAuthProvider(AuthProvider): - """ - Custom auth provider that integrates with existing Authentik OIDC authentication. - - Reuses the existing authentication infrastructure from core.permissions module. - - For MS Access users: This replaces Access file-level security with user-level - authentication. Each user logs in with their Authentik credentials and gets - assigned roles (Admin, Editor, Viewer) which control what they can do in the - admin interface. - """ - - async def is_authenticated(self, request: Request) -> bool: - """ - Check if user is authenticated by verifying their JWT token. - - This method is called on every admin page request to determine if the - user should be allowed access. - - Returns: - bool: True if user has a valid JWT token, False otherwise - """ - # Check if authentication is disabled (development mode only) - if int(os.environ.get("AUTHENTIK_DISABLE_AUTHENTICATION", 0)): - from core.settings import settings - - if settings.mode != "production": - # Allow unauthenticated access in development mode - request.state.user = AdminUserWithRoles( - username="dev_user", roles=["admin"] - ) - return True - - try: - # Try to get token from Authorization header - authorization = request.headers.get("Authorization") - if not authorization: - # Try to get token from session/cookie - token = request.session.get("token") - if not token: - return False - else: - # Extract token from "Bearer " format - token = ( - authorization.split(" ")[1] - if " " in authorization - else authorization - ) - - # Verify token using existing authentication system - is_valid = verify_token(token, scope=None, permissions=None) - - if is_valid: - # Store user in request state for later access - request.state.user = self._create_admin_user_from_token(token) - - return is_valid - except Exception: - return False - - def _create_admin_user_from_token(self, token: str) -> Optional[AdminUser]: - """ - Extract user information from JWT token and create AdminUser instance. - - Args: - token: JWT access token from Authentik - - Returns: - AdminUser instance with username and roles, or None if token invalid - """ - try: - # Decode JWT payload - payload = _get_token_payload(token) - - # Extract user information from JWT claims - username = ( - payload.get("preferred_username") - or payload.get("email") - or payload.get("sub") - ) - email = payload.get("email") - groups = payload.get("groups", []) - - # Map Authentik groups to admin roles - roles = [] - - # Standard roles - if "Admin" in groups: - roles.append("admin") - if "Editor" in groups: - roles.append("editor") - if "Viewer" in groups: - roles.append("viewer") - - # AMP-specific roles (for AMPAPI-related data) - if "AMPAdmin" in groups: - roles.append("amp_admin") - if "AMPEditor" in groups: - roles.append("amp_editor") - if "AMPViewer" in groups: - roles.append("amp_viewer") - - # Lexicon-specific roles - if "LexiconAdmin" in groups: - roles.append("lexicon_admin") - if "LexiconEditor" in groups: - roles.append("lexicon_editor") - - return AdminUserWithRoles( - username=username, - photo_url=None, # Could add user avatar URL from OIDC if available - roles=roles, - ) - except Exception: - return None - - def get_admin_user(self, request: Request) -> Optional[AdminUser]: - """ - Get the current admin user from the request. - - This method is called by Starlette Admin to get user information for - display in the UI and permission checks. - - Returns: - AdminUser instance with username and roles, or None if not authenticated - """ - # Check if user is already stored in request state - if hasattr(request.state, "user"): - return request.state.user - - try: - # Get token from request - authorization = request.headers.get("Authorization") - if not authorization: - token = request.session.get("token") - if not token: - return None - else: - token = ( - authorization.split(" ")[1] - if " " in authorization - else authorization - ) - - # Create AdminUser from token - admin_user = self._create_admin_user_from_token(token) - - # Store in request state for future calls - if admin_user: - request.state.user = admin_user - - return admin_user - except Exception: - return None - - async def login(self, *args, **kwargs) -> RedirectResponse: - """ - Redirect to Authentik OIDC login page. - - Note: Starlette Admin will show a login form, but we ignore the username/password - and redirect to Authentik OAuth flow instead. - - Args: - request: Starlette request object (extracted from args/kwargs) - *args/**kwargs: Ignored, kept for compatibility with different - Starlette Admin login call signatures - - Returns: - RedirectResponse to Authentik authorization endpoint - """ - # Starlette Admin has changed the AuthProvider.login signature across versions. - # Accept *args/**kwargs and extract the Request to stay compatible whether - # it calls login(request, data, ...) or login(username, password, remember_me, request). - request: Optional[Request] = kwargs.get("request") - if request is None: - for arg in args: - if isinstance(arg, Request): - request = arg - break - - if request is None: - raise LoginFailed("Unable to determine login request context.") - - authentik_authorize_url = os.environ.get("AUTHENTIK_AUTHORIZE_URL") - authentik_client_id = os.environ.get("AUTHENTIK_CLIENT_ID") - if not authentik_authorize_url or not authentik_client_id: - raise LoginFailed( - "Authentik authentication is not configured. Please set AUTHENTIK_AUTHORIZE_URL and AUTHENTIK_CLIENT_ID environment variables." - ) - - # Store original URL to redirect back after login - original_url = str(request.url_for("admin:index")) - request.session["auth_redirect"] = original_url - redirect_uri = str(request.url_for("admin_auth_callback")) - - # PKCE for public clients - code_verifier = secrets.token_urlsafe(64) - digest = hashlib.sha256(code_verifier.encode("ascii")).digest() - code_challenge = base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") - - state = secrets.token_urlsafe(32) - request.session["auth_state"] = state - request.session["auth_code_verifier"] = code_verifier - - params = { - "response_type": "code", - "client_id": authentik_client_id, - "redirect_uri": redirect_uri, - "scope": "openid profile email", - "state": state, - "code_challenge": code_challenge, - "code_challenge_method": "S256", - } - - authorize_url = f"{authentik_authorize_url}?{urlencode(params)}" - return RedirectResponse(url=authorize_url, status_code=302) - - async def logout(self, *args, **kwargs) -> RedirectResponse: - """ - Handle logout by clearing session and redirecting. - - Args: - request: Starlette request object (extracted from args/kwargs) - *args/**kwargs: Ignored, kept for compatibility with different - Starlette Admin logout call signatures - - Returns: - RedirectResponse to home page - """ - request: Optional[Request] = kwargs.get("request") - if request is None: - for arg in args: - if isinstance(arg, Request): - request = arg - break - - if request is None: - raise LoginFailed("Unable to determine logout request context.") - - # Clear session tokens - request.session.pop("token", None) - request.session.pop("auth_redirect", None) - - # Clear user from request state - if hasattr(request.state, "user"): - delattr(request.state, "user") - - # Redirect to home page - # TODO: Consider redirecting to Authentik logout endpoint to fully log out - return RedirectResponse(url="/", status_code=302) diff --git a/admin/auth_routes.py b/admin/auth_routes.py deleted file mode 100644 index 9db20669e..000000000 --- a/admin/auth_routes.py +++ /dev/null @@ -1,78 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -Admin authentication callback routes. -""" - -import os - -import httpx -from fastapi import APIRouter, Request -from starlette.responses import RedirectResponse -from starlette_admin.exceptions import LoginFailed - -router = APIRouter() - - -@router.get("/admin/auth/callback", name="admin_auth_callback", include_in_schema=False) -async def admin_auth_callback(request: Request): - code = request.query_params.get("code") - state = request.query_params.get("state") - expected_state = request.session.get("auth_state") - - if not code or not state or state != expected_state: - raise LoginFailed("Invalid authentication response.") - - token_url = os.environ.get("AUTHENTIK_TOKEN_URL") - client_id = os.environ.get("AUTHENTIK_CLIENT_ID") - if not token_url or not client_id: - raise LoginFailed( - "Authentik authentication is not configured. Please set AUTHENTIK_TOKEN_URL and AUTHENTIK_CLIENT_ID." - ) - - redirect_uri = str(request.url_for("admin_auth_callback")) - code_verifier = request.session.get("auth_code_verifier") - - data = { - "grant_type": "authorization_code", - "client_id": client_id, - "code": code, - "redirect_uri": redirect_uri, - } - - if code_verifier: - data["code_verifier"] = code_verifier - - client_secret = os.environ.get("AUTHENTIK_CLIENT_SECRET") - if client_secret: - data["client_secret"] = client_secret - - async with httpx.AsyncClient(timeout=15.0) as client: - resp = await client.post(token_url, data=data) - if resp.status_code >= 400: - raise LoginFailed("Failed to exchange token from Authentik.") - token_payload = resp.json() - - access_token = token_payload.get("access_token") - if not access_token: - raise LoginFailed("Authentik did not return an access token.") - - request.session["token"] = access_token - request.session.pop("auth_state", None) - request.session.pop("auth_code_verifier", None) - - redirect_to = request.session.pop("auth_redirect", "/admin") - return RedirectResponse(url=redirect_to, status_code=302) diff --git a/admin/config.py b/admin/config.py deleted file mode 100644 index e559fef92..000000000 --- a/admin/config.py +++ /dev/null @@ -1,218 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -Starlette Admin configuration and initialization. - -This module creates and configures the admin interface for OcotilloAPI. -""" - -from admin.auth import NMSampleLocationsAuthProvider -from admin.views import ( - AquiferSystemAdmin, - AquiferTypeAdmin, - AssetAdmin, - AssociatedDataAdmin, - ChemistrySampleInfoAdmin, - ContactAdmin, - DataProvenanceAdmin, - DeploymentAdmin, - FieldActivityAdmin, - FieldEventAdmin, - GeologicFormationAdmin, - GroupAdmin, - HydraulicsDataAdmin, - LexiconCategoryAdmin, - LexiconTermAdmin, - LocationAdmin, - MajorChemistryAdmin, - MinorTraceChemistryAdmin, - NotesAdmin, - ObservationAdmin, - ParameterAdmin, - RadionuclidesAdmin, - SampleAdmin, - SensorAdmin, - SoilRockResultsAdmin, - StratigraphyAdmin, - SurfaceWaterDataAdmin, - SurfaceWaterPhotosAdmin, - ThingAdmin, - TransducerObservationAdmin, - WaterLevelsContinuousPressureDailyAdmin, - WeatherPhotosAdmin, - WeatherDataAdmin, - FieldParametersAdmin, -) -from db import NMA_FieldParameters -from db.aquifer_system import AquiferSystem -from db.aquifer_type import AquiferType -from db.asset import Asset -from db.contact import Contact -from db.data_provenance import DataProvenance -from db.deployment import Deployment -from db.engine import engine -from db.field import FieldActivity, FieldEvent -from db.geologic_formation import GeologicFormation -from db.group import Group -from db.lexicon import LexiconCategory, LexiconTerm -from db.location import Location -from db.nma_legacy import ( - NMA_AssociatedData, - NMA_Chemistry_SampleInfo, - NMA_MajorChemistry, - NMA_MinorTraceChemistry, - NMA_Radionuclides, - NMA_HydraulicsData, - NMA_Soil_Rock_Results, - NMA_Stratigraphy, - NMA_SurfaceWaterData, - NMA_WaterLevelsContinuous_Pressure_Daily, - NMA_WeatherPhotos, - NMA_SurfaceWaterPhotos, - NMA_WeatherData, -) -from db.notes import Notes -from db.observation import Observation -from db.parameter import Parameter -from db.sample import Sample -from db.sensor import Sensor -from db.thing import Thing -from db.transducer import TransducerObservation -from starlette_admin.contrib.sqla import Admin - - -def create_admin(app): - """ - Create and configure Starlette Admin instance. - - This function sets up the admin interface and mounts it to the FastAPI app - at the /admin route. - - For MS Access users: This replaces the Access database file with a web-based - admin interface. Instead of opening a .accdb file, staff will navigate to - https://your-domain.com/admin in their web browser. - - Args: - app: FastAPI application instance - - Returns: - Admin: Configured Starlette Admin instance - """ - # Create admin instance - admin = Admin( - engine=engine, - title="Ocotillod Admin", - base_url="/admin", - logo_url=None, # TODO: Add NMBGMR logo - auth_provider=NMSampleLocationsAuthProvider(), - middlewares=[], # Add custom middlewares here if needed - ) - - # Register model views - # Assets - admin.add_view(AssetAdmin(Asset)) - - # Aquifer - admin.add_view(AquiferSystemAdmin(AquiferSystem)) - admin.add_view(AquiferTypeAdmin(AquiferType)) - - # Contacts - admin.add_view(ContactAdmin(Contact)) - - # Data provenance - admin.add_view(DataProvenanceAdmin(DataProvenance)) - - # Deployment / Equipment - admin.add_view(DeploymentAdmin(Deployment)) - admin.add_view(SensorAdmin(Sensor)) - - # Field - admin.add_view(FieldActivityAdmin(FieldActivity)) - admin.add_view(FieldEventAdmin(FieldEvent)) - - # Geology - admin.add_view(GeologicFormationAdmin(GeologicFormation)) - - # Geography - admin.add_view(LocationAdmin(Location)) - # Associated data - admin.add_view(AssociatedDataAdmin(NMA_AssociatedData)) - - # Aquifer - admin.add_view(AquiferSystemAdmin(AquiferSystem)) - admin.add_view(AquiferTypeAdmin(AquiferType)) - - # Groups - admin.add_view(GroupAdmin(Group)) - - # Hydraulics - admin.add_view(HydraulicsDataAdmin(NMA_HydraulicsData)) - admin.add_view(MinorTraceChemistryAdmin(NMA_MinorTraceChemistry)) - admin.add_view(RadionuclidesAdmin(NMA_Radionuclides)) - admin.add_view(MajorChemistryAdmin(NMA_MajorChemistry)) - - # Lexicon - admin.add_view(LexiconCategoryAdmin(LexiconCategory)) - admin.add_view(LexiconTermAdmin(LexiconTerm)) - - # Notes - admin.add_view(NotesAdmin(Notes)) - - # Observations - admin.add_view(ObservationAdmin(Observation)) - - # Parameters - admin.add_view(ParameterAdmin(Parameter)) - admin.add_view(FieldParametersAdmin(NMA_FieldParameters)) - - # Samples - admin.add_view(ChemistrySampleInfoAdmin(NMA_Chemistry_SampleInfo)) - admin.add_view(SampleAdmin(Sample)) - admin.add_view(SurfaceWaterDataAdmin(NMA_SurfaceWaterData)) - - # Soil & Stratigraphy - admin.add_view(SoilRockResultsAdmin(NMA_Soil_Rock_Results)) - admin.add_view(StratigraphyAdmin(NMA_Stratigraphy)) - - # Things (Wells, Springs, etc.) - admin.add_view(ThingAdmin(Thing)) - - # Transducer observations - admin.add_view(TransducerObservationAdmin(TransducerObservation)) - - # Water Levels - Continuous (legacy) - admin.add_view( - WaterLevelsContinuousPressureDailyAdmin( - NMA_WaterLevelsContinuous_Pressure_Daily - ) - ) - - # Weather - admin.add_view(WeatherPhotosAdmin(NMA_WeatherPhotos)) - - # Surface Water Photos - admin.add_view(SurfaceWaterPhotosAdmin(NMA_SurfaceWaterPhotos)) - # Weather - admin.add_view(WeatherDataAdmin(NMA_WeatherData)) - - # Future: Add more views here as they are implemented - # admin.add_view(SampleAdmin) - # admin.add_view(GroupAdmin) - - # Mount admin to app - admin.mount_to(app) - - return admin diff --git a/admin/fields.py b/admin/fields.py deleted file mode 100644 index 9da16f9e9..000000000 --- a/admin/fields.py +++ /dev/null @@ -1,141 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -Custom fields for Starlette Admin. - -Provides field handlers for complex data types like PostGIS geometry. -""" - -from typing import Any - -from geoalchemy2 import WKTElement -from geoalchemy2.shape import to_shape -from starlette.requests import Request -from starlette_admin import StringField - -from core.constants import SRID_WGS84 - - -class WKTField(StringField): - """ - Custom field for GeoAlchemy2 Geometry columns. - - This field converts between PostGIS geometry (WKBElement) and human-readable - WKT (Well-Known Text) format for display and editing in the admin interface. - - For MS Access users: Instead of entering Easting/Northing/UTM Zone in separate - fields, you'll enter coordinates in WKT format, for example: - POINT(-106.123 35.456) - - Note: Longitude comes first, then latitude (POINT(lon lat), not POINT(lat lon)) - """ - - async def serialize_value(self, request: Request, value: Any, action: str) -> str: - """ - Convert WKBElement (PostGIS geometry) to WKT string for display in form. - - This is called when rendering the edit/create form to show the current - value in a text input. - - Args: - request: Starlette request object - value: WKBElement from database (PostGIS geometry) - action: 'list', 'detail', 'edit', or 'create' - - Returns: - WKT string representation of geometry (e.g., "POINT(-106.123 35.456)") - """ - if value is None: - return "" - - try: - # Convert WKBElement to Shapely geometry, then to WKT - shape = to_shape(value) - return shape.wkt - except Exception: - # If conversion fails, return string representation - return str(value) - - async def parse_form_data( - self, request: Request, form_data: dict, action: str - ) -> Any: - """ - Convert WKT string from form input to WKTElement for database storage. - - This is called when saving the form to convert the user's input into - a format that can be stored in the PostGIS database. - - Args: - request: Starlette request object - form_data: Dictionary of form data - action: 'edit' or 'create' - - Returns: - WKTElement with SRID for PostGIS storage, or None if empty - - Raises: - ValueError: If WKT string is invalid - """ - wkt_string = form_data.get(self.name) - - if not wkt_string or wkt_string.strip() == "": - return None - - try: - # Parse and validate WKT string - from shapely.wkt import loads as wkt_loads - - shape = wkt_loads(wkt_string.strip()) - - # Convert to WKTElement with SRID (spatial reference identifier) - return WKTElement(shape.wkt, srid=SRID_WGS84) - except Exception as e: - raise ValueError( - f"Invalid WKT geometry: {e}. " - f"Expected format: POINT(longitude latitude), e.g., POINT(-106.123 35.456). " - f"Note: Longitude comes first, then latitude." - ) - - -class CoordinateHelpField(WKTField): - """ - Extended WKT field with detailed help text for coordinate entry. - - This version includes comprehensive help text for users transitioning - from MS Access UTM coordinate entry to WKT format. - """ - - def __init__(self, *args, **kwargs): - # Add detailed help text if not provided - if "help_text" not in kwargs: - kwargs["help_text"] = ( - "Enter coordinates in WKT (Well-Known Text) format.\n\n" - "Format: POINT(longitude latitude)\n" - "Example: POINT(-106.65082 35.08352)\n\n" - "Important:\n" - " * Longitude comes FIRST (negative for western hemisphere)\n" - " * Latitude comes SECOND\n" - " * No comma between values\n" - " * Use decimal degrees (not degrees-minutes-seconds)\n" - " * Coordinate system: WGS84 (SRID 4326)\n\n" - "If you have UTM coordinates:\n" - " 1. Use an online converter (e.g., https://www.latlong.net/utm-to-lat-long)\n" - " 2. Enter your Easting, Northing, and UTM Zone\n" - " 3. Convert to WGS84 lat/lon\n" - " 4. Enter here as POINT(lon lat)" - ) - - super().__init__(*args, **kwargs) diff --git a/admin/views/__init__.py b/admin/views/__init__.py deleted file mode 100644 index c8d0f5ad2..000000000 --- a/admin/views/__init__.py +++ /dev/null @@ -1,97 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -Admin views package for OcotilloAPI. - -Provides MS Access-like interface for CRUD operations on database models. -""" - -from admin.views.aquifer_system import AquiferSystemAdmin -from admin.views.aquifer_type import AquiferTypeAdmin -from admin.views.asset import AssetAdmin -from admin.views.associated_data import AssociatedDataAdmin -from admin.views.chemistry_sampleinfo import ChemistrySampleInfoAdmin -from admin.views.contact import ContactAdmin -from admin.views.data_provenance import DataProvenanceAdmin -from admin.views.deployment import DeploymentAdmin -from admin.views.field import ( - FieldActivityAdmin, - FieldEventAdmin, - FieldEventParticipantAdmin, -) -from admin.views.field_parameters import FieldParametersAdmin -from admin.views.geologic_formation import GeologicFormationAdmin -from admin.views.group import GroupAdmin -from admin.views.hydraulicsdata import HydraulicsDataAdmin -from admin.views.lexicon import LexiconCategoryAdmin, LexiconTermAdmin -from admin.views.location import LocationAdmin -from admin.views.major_chemistry import MajorChemistryAdmin -from admin.views.minor_trace_chemistry import MinorTraceChemistryAdmin -from admin.views.notes import NotesAdmin -from admin.views.observation import ObservationAdmin -from admin.views.parameter import ParameterAdmin -from admin.views.radionuclides import RadionuclidesAdmin -from admin.views.sample import SampleAdmin -from admin.views.sensor import SensorAdmin -from admin.views.soil_rock_results import SoilRockResultsAdmin -from admin.views.stratigraphy import StratigraphyAdmin -from admin.views.surface_water import SurfaceWaterDataAdmin -from admin.views.surface_water_photos import SurfaceWaterPhotosAdmin -from admin.views.thing import ThingAdmin -from admin.views.transducer_observation import TransducerObservationAdmin -from admin.views.waterlevelscontinuous_pressure_daily import ( - WaterLevelsContinuousPressureDailyAdmin, -) -from admin.views.weather_data import WeatherDataAdmin -from admin.views.weather_photos import WeatherPhotosAdmin - -__all__ = [ - "AssetAdmin", - "AssociatedDataAdmin", - "AquiferSystemAdmin", - "AquiferTypeAdmin", - "ChemistrySampleInfoAdmin", - "ContactAdmin", - "DataProvenanceAdmin", - "DeploymentAdmin", - "FieldActivityAdmin", - "FieldEventAdmin", - "FieldEventParticipantAdmin", - "FieldParametersAdmin", - "GeologicFormationAdmin", - "GroupAdmin", - "HydraulicsDataAdmin", - "LexiconCategoryAdmin", - "LexiconTermAdmin", - "LocationAdmin", - "MajorChemistryAdmin", - "MinorTraceChemistryAdmin", - "NotesAdmin", - "ObservationAdmin", - "ParameterAdmin", - "RadionuclidesAdmin", - "SampleAdmin", - "SensorAdmin", - "SoilRockResultsAdmin", - "StratigraphyAdmin", - "SurfaceWaterDataAdmin", - "SurfaceWaterPhotosAdmin", - "ThingAdmin", - "TransducerObservationAdmin", - "WaterLevelsContinuousPressureDailyAdmin", - "WeatherPhotosAdmin", - "WeatherDataAdmin", -] diff --git a/admin/views/aquifer_system.py b/admin/views/aquifer_system.py deleted file mode 100644 index 9b384e098..000000000 --- a/admin/views/aquifer_system.py +++ /dev/null @@ -1,94 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -AquiferSystemAdmin view for OcotilloAPI. -""" - -from admin.fields import WKTField -from admin.views.base import OcotilloModelView - - -class AquiferSystemAdmin(OcotilloModelView): - """ - Admin view for AquiferSystem model. - """ - - # ========== Basic Configuration ========== - - name = "Aquifer Systems" - label = "Aquifer Systems" - icon = "fa fa-globe" - - # ========== List View ========== - - sortable_fields = [ - "id", - "name", - "primary_aquifer_type", - "geographic_scale", - "release_status", - "created_at", - ] - - fields_default_sort = [("name", False)] - - searchable_fields = [ - "name", - "description", - "primary_aquifer_type", - "geographic_scale", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View ========== - - fields = [ - "id", - "name", - "description", - "primary_aquifer_type", - "geographic_scale", - WKTField("boundary", label="Boundary (WKT)"), - "release_status", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - ] - - -# ============= EOF ============================================= diff --git a/admin/views/aquifer_type.py b/admin/views/aquifer_type.py deleted file mode 100644 index ad319b6d3..000000000 --- a/admin/views/aquifer_type.py +++ /dev/null @@ -1,86 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -AquiferTypeAdmin view for OcotilloAPI. -""" - -from admin.views.base import OcotilloModelView - - -class AquiferTypeAdmin(OcotilloModelView): - """ - Admin view for AquiferType model. - """ - - # ========== Basic Configuration ========== - - name = "Aquifer Types" - label = "Aquifer Types" - icon = "fa fa-tint" - - # ========== List View ========== - - sortable_fields = [ - "id", - "thing_aquifer_association_id", - "aquifer_type", - "release_status", - "created_at", - ] - - fields_default_sort = [("created_at", True)] - - searchable_fields = [ - "aquifer_type", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View ========== - - fields = [ - "id", - "thing_aquifer_association_id", - "aquifer_type", - "release_status", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - ] - - -# ============= EOF ============================================= diff --git a/admin/views/asset.py b/admin/views/asset.py deleted file mode 100644 index acec3bb80..000000000 --- a/admin/views/asset.py +++ /dev/null @@ -1,100 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -AssetAdmin view for OcotilloAPI. - -Provides MS Access-like interface for CRUD operations on Asset model. -""" - -from admin.views.base import OcotilloModelView - - -class AssetAdmin(OcotilloModelView): - """ - Admin view for Asset model. - """ - - # ========== Basic Configuration ========== - - name = "Assets" - label = "Assets" - icon = "fa fa-file" - - # ========== List View ========== - - sortable_fields = [ - "id", - "name", - "mime_type", - "storage_service", - "size", - "release_status", - "created_at", - ] - - fields_default_sort = [("created_at", True)] - - searchable_fields = [ - "name", - "label", - "mime_type", - "storage_service", - "storage_path", - "uri", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View ========== - - fields = [ - "id", - "name", - "label", - "storage_service", - "storage_path", - "mime_type", - "size", - "uri", - "release_status", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - ] - - -# ============= EOF ============================================= diff --git a/admin/views/associated_data.py b/admin/views/associated_data.py deleted file mode 100644 index f58dcd628..000000000 --- a/admin/views/associated_data.py +++ /dev/null @@ -1,113 +0,0 @@ -# =============================================================================== -# Copyright 2026 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -AssociatedDataAdmin view for legacy NMA_AssociatedData. - -Updated for Integer PK schema: -- id: Integer PK (autoincrement) -- nma_assoc_id: Legacy UUID PK (AssocID), UNIQUE for audit -- nma_location_id: Legacy LocationId UUID, UNIQUE -- nma_point_id: Legacy PointID string -- nma_object_id: Legacy OBJECTID, UNIQUE -""" - -from starlette.requests import Request - -from admin.views.base import OcotilloModelView - - -class AssociatedDataAdmin(OcotilloModelView): - """ - Admin view for legacy AssociatedData model (NMA_AssociatedData). - Read-only, MS Access-like listing/details. - """ - - # ========== Basic Configuration ========== - name = "NMA Associated Data" - label = "NMA Associated Data" - icon = "fa fa-link" - - # Integer PK - pk_attr = "id" - pk_type = int - - def can_create(self, request: Request) -> bool: - return False - - def can_edit(self, request: Request) -> bool: - return False - - def can_delete(self, request: Request) -> bool: - return False - - # ========== List View ========== - - list_fields = [ - "id", - "nma_assoc_id", - "nma_location_id", - "nma_point_id", - "nma_object_id", - "notes", - "formation", - "thing_id", - ] - - sortable_fields = [ - "id", - "nma_assoc_id", - "nma_object_id", - "nma_point_id", - ] - - fields_default_sort = [("nma_point_id", False), ("nma_object_id", False)] - - searchable_fields = [ - "nma_point_id", - "nma_assoc_id", - "notes", - "formation", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View ========== - - fields = [ - "id", - "nma_assoc_id", - "nma_location_id", - "nma_point_id", - "nma_object_id", - "notes", - "formation", - "thing_id", - ] - - field_labels = { - "id": "ID", - "nma_assoc_id": "NMA AssocID (Legacy)", - "nma_location_id": "NMA LocationId (Legacy)", - "nma_point_id": "NMA PointID (Legacy)", - "nma_object_id": "NMA OBJECTID (Legacy)", - "notes": "Notes", - "formation": "Formation", - "thing_id": "Thing ID", - } - - -# ============= EOF ============================================= diff --git a/admin/views/base.py b/admin/views/base.py deleted file mode 100644 index a44f51c53..000000000 --- a/admin/views/base.py +++ /dev/null @@ -1,142 +0,0 @@ -# =============================================================================== -# Copyright 2026 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -from __future__ import annotations - -from typing import Any, Iterable, Sequence - -from sqlalchemy import select, update -from starlette.requests import Request -from starlette.responses import Response -from starlette_admin import ExportType, action -from starlette_admin.contrib.sqla import ModelView - -from db.engine import session_ctx - - -class OcotilloModelView(ModelView): - """ - Shared admin behaviors for Ocotillo data models. - - - RBAC: admin can create/edit/delete; editor can edit; any authenticated user can view. - - Data visibility: non-admin/editor users only see published rows when a release field exists. - - Publish/Unpublish actions: toggle release status when enabled and a release field is present. - """ - - release_field = "release_status" - draft_value = "draft" - published_value = "published" - enable_publish_actions: bool = True - export_types: Sequence[ExportType] = (ExportType.CSV, ExportType.EXCEL) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # ========= Permissions (RBAC) ========= - def _get_user(self, request: Request) -> Any: - return getattr(request.state, "user", None) - - def _roles(self, request: Request) -> list[str]: - user = self._get_user(request) - return getattr(user, "roles", []) if user else [] - - def _has_role(self, request: Request, roles: Iterable[str]) -> bool: - return bool(set(self._roles(request)) & set(roles)) - - def can_create(self, request: Request) -> bool: - return self._has_role(request, {"admin"}) - - def can_edit(self, request: Request) -> bool: - return self._has_role(request, {"admin", "editor"}) - - def can_delete(self, request: Request) -> bool: - return self._has_role(request, {"admin"}) - - def can_view_details(self, request: Request) -> bool: - return self._get_user(request) is not None - - # ========= Data Visibility ========= - def get_list_query(self, request: Request): - query = select(self.model) - user = self._get_user(request) - if user is None: - # Return an empty result set for anonymous users - return query.where(self.model.id == -1) - - if not hasattr(self.model, self.release_field): - return query - - if self._has_role(request, {"admin", "editor"}): - return query - return query.where( - getattr(self.model, self.release_field) == self.published_value - ) - - # ========= Actions (Publish / Unpublish) ========= - def _ensure_release_field(self) -> bool: - return self.enable_publish_actions and hasattr(self.model, self.release_field) - - @action( - name="publish_selected", - text="Publish Selected", - confirmation="Are you sure you want to publish the selected records?", - submit_btn_text="Yes, publish", - submit_btn_class="btn btn-success", - ) - async def publish_selected(self, request: Request, pks: list[int]) -> Response: - if not self._has_role(request, {"admin"}): - return Response("Only admins can publish", status_code=403) - if not self._ensure_release_field(): - return Response( - "Publish action not available for this model", status_code=400 - ) - - with session_ctx() as session: - result = session.execute( - update(self.model) - .where(self.model.id.in_(pks)) - .values({self.release_field: self.published_value}) - ) - session.commit() - updated_count = result.rowcount - return Response(f"Published {updated_count} record(s)", status_code=200) - - @action( - name="unpublish_selected", - text="Unpublish Selected (set to draft)", - confirmation="Are you sure you want to unpublish the selected records?", - submit_btn_text="Yes, unpublish", - submit_btn_class="btn btn-warning", - ) - async def unpublish_selected(self, request: Request, pks: list[int]) -> Response: - if not self._has_role(request, {"admin"}): - return Response("Only admins can unpublish", status_code=403) - if not self._ensure_release_field(): - return Response( - "Unpublish action not available for this model", status_code=400 - ) - - with session_ctx() as session: - result = session.execute( - update(self.model) - .where(self.model.id.in_(pks)) - .values({self.release_field: self.draft_value}) - ) - session.commit() - updated_count = result.rowcount - return Response(f"Unpublished {updated_count} record(s)", status_code=200) - - -# ============= EOF ============================================= diff --git a/admin/views/chemistry_sampleinfo.py b/admin/views/chemistry_sampleinfo.py deleted file mode 100644 index b588da038..000000000 --- a/admin/views/chemistry_sampleinfo.py +++ /dev/null @@ -1,175 +0,0 @@ -# =============================================================================== -# Copyright 2026 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -ChemistrySampleInfoAdmin view for legacy Chemistry_SampleInfo. - -Updated for Integer PK schema: -- id: Integer PK (autoincrement) -- nma_sample_pt_id: Legacy UUID PK (SamplePtID), UNIQUE for audit -- nma_wclab_id: Legacy WCLab_ID -- nma_sample_point_id: Legacy SamplePointID -- nma_object_id: Legacy OBJECTID, UNIQUE -- nma_location_id: Legacy LocationId UUID (for audit trail) - -FK Change (2026-01): -- thing_id: Integer FK to Thing.id -""" - -from starlette.requests import Request -from starlette_admin.fields import HasOne - -from admin.views.base import OcotilloModelView - - -class ChemistrySampleInfoAdmin(OcotilloModelView): - """ - Admin view for ChemistrySampleInfo model. - """ - - # ========== Basic Configuration ========== - - name = "NMA Chemistry Sample Info" - label = "NMA Chemistry Sample Info" - icon = "fa fa-flask" - - # Integer PK - pk_attr = "id" - pk_type = int - - def can_create(self, request: Request) -> bool: - return False - - def can_edit(self, request: Request) -> bool: - return False - - def can_delete(self, request: Request) -> bool: - return False - - # ========== List View ========== - - list_fields = [ - "id", - "nma_sample_pt_id", - "nma_wclab_id", - "nma_sample_point_id", - "nma_object_id", - "nma_location_id", - "thing_id", - HasOne("thing", identity="thing"), - "collection_date", - "collection_method", - "collected_by", - "analyses_agency", - "sample_type", - "sample_material_not_h2o", - "water_type", - "study_sample", - "data_source", - "data_quality", - "public_release", - "added_day_to_date", - "added_month_day_to_date", - "sample_notes", - ] - - sortable_fields = [ - "id", - "nma_sample_pt_id", - "nma_wclab_id", - "nma_sample_point_id", - "nma_object_id", - "collection_date", - "sample_type", - "data_source", - "data_quality", - "public_release", - ] - - fields_default_sort = [("collection_date", True)] - - searchable_fields = [ - "nma_sample_pt_id", - "nma_wclab_id", - "nma_sample_point_id", - "collection_date", - "collected_by", - "analyses_agency", - "sample_type", - "sample_material_not_h2o", - "water_type", - "study_sample", - "data_source", - "data_quality", - "public_release", - "sample_notes", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View ========== - - fields = [ - "id", - "nma_sample_pt_id", - "nma_wclab_id", - "nma_sample_point_id", - "nma_object_id", - "nma_location_id", - "thing_id", - HasOne("thing", identity="thing"), - "collection_date", - "collection_method", - "collected_by", - "analyses_agency", - "sample_type", - "sample_material_not_h2o", - "water_type", - "study_sample", - "data_source", - "data_quality", - "public_release", - "added_day_to_date", - "added_month_day_to_date", - "sample_notes", - ] - - field_labels = { - "id": "ID", - "nma_sample_pt_id": "NMA SamplePtID (Legacy)", - "nma_wclab_id": "NMA WCLab_ID (Legacy)", - "nma_sample_point_id": "NMA SamplePointID (Legacy)", - "nma_object_id": "NMA OBJECTID (Legacy)", - "nma_location_id": "NMA LocationId (Legacy)", - "thing_id": "Thing ID", - "collection_date": "Collection Date", - "collection_method": "Collection Method", - "collected_by": "Collected By", - "analyses_agency": "Analyses Agency", - "sample_type": "Sample Type", - "sample_material_not_h2o": "Sample Material Not H2O", - "water_type": "Water Type", - "study_sample": "Study Sample", - "data_source": "Data Source", - "data_quality": "Data Quality", - "public_release": "Public Release", - "added_day_to_date": "Added Day to Date", - "added_month_day_to_date": "Added Month/Day to Date", - "sample_notes": "Sample Notes", - } - - -# ============= EOF ============================================= diff --git a/admin/views/contact.py b/admin/views/contact.py deleted file mode 100644 index 36bea8ee4..000000000 --- a/admin/views/contact.py +++ /dev/null @@ -1,129 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -ContactAdmin view for OcotilloAPI. - -Provides MS Access-like interface for CRUD operations on Contact (Owners) model. -""" - -from admin.views.base import OcotilloModelView - - -class ContactAdmin(OcotilloModelView): - """ - Admin view for Contact model (Well Owners/Managers). - - Designed to replicate MS Access "Owners Data Entry Form" and "Owners Datasheet View". - - Permission Model: - - Admin: Can create, edit, delete all contacts - - Editor: Can create and edit, cannot delete - - Viewer: Can only view published contacts (read-only) - """ - - # ========== Basic Configuration ========== - - name = "Contacts" - label = "Contacts (Owners)" - icon = "fa fa-users" - - # ========== List View (MS Access Datasheet View Equivalent) ========== - - sortable_fields = [ - "id", - "name", - "organization", - "role", - "contact_type", - "release_status", - "created_at", - ] - - fields_default_sort = [("name", False)] # Alphabetical by name - - searchable_fields = [ - "name", - "organization", - "role", - "contact_type", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View (MS Access Form View Equivalent) ========== - - fields = [ - "id", - # Contact Information - "name", - "organization", - "role", - "contact_type", - # Release Status - "release_status", - # Audit Fields - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - # Legacy Migration Fields - "nma_pk_owners", - "nma_pk_waterlevels", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - "nma_pk_owners", - "nma_pk_waterlevels", - # Exclude complex relationships (manage separately) - "phones", - "emails", - "addresses", - "incomplete_nma_phones", - "permissions", - "author_associations", - "thing_associations", - "field_event_participants", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "nma_pk_owners", - "nma_pk_waterlevels", - # Exclude complex relationships (manage separately) - "phones", - "emails", - "addresses", - "incomplete_nma_phones", - "permissions", - "author_associations", - "thing_associations", - "field_event_participants", - ] - - # ========== Field Labels and Help Text ========== diff --git a/admin/views/data_provenance.py b/admin/views/data_provenance.py deleted file mode 100644 index c1a91551f..000000000 --- a/admin/views/data_provenance.py +++ /dev/null @@ -1,94 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -DataProvenanceAdmin view for OcotilloAPI. -""" - -from admin.views.base import OcotilloModelView - - -class DataProvenanceAdmin(OcotilloModelView): - """ - Admin view for DataProvenance model. - """ - - name = "Data Provenance" - label = "Data Provenance" - icon = "fa fa-history" - - sortable_fields = [ - "id", - "target_table", - "target_id", - "field_name", - "origin_type", - "collection_method", - "release_status", - "created_at", - ] - - fields_default_sort = [("created_at", True)] - - searchable_fields = [ - "target_table", - "field_name", - "origin_source", - "origin_type", - "collection_method", - "accuracy_unit", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - fields = [ - "id", - "target_table", - "target_id", - "field_name", - "origin_type", - "origin_source", - "collection_method", - "accuracy_value", - "accuracy_unit", - "release_status", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - ] - - -# ============= EOF ============================================= diff --git a/admin/views/deployment.py b/admin/views/deployment.py deleted file mode 100644 index ccdf535da..000000000 --- a/admin/views/deployment.py +++ /dev/null @@ -1,138 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -DeploymentAdmin view for OcotilloAPI. - -Provides MS Access-like interface for CRUD operations on Deployment model. -""" - -from admin.views.base import OcotilloModelView - - -class DeploymentAdmin(OcotilloModelView): - """ - Admin view for Deployment model (Equipment Installation Log). - - Designed to replicate MS Access "Equipment Deployment Form" and "Deployment Datasheet View". - - Permission Model: - - Admin: Can create, edit, delete all deployments - - Editor: Can create and edit, cannot delete - - Viewer: Can only view published deployments (read-only) - """ - - # ========== Basic Configuration ========== - - name = "Deployments" - label = "Deployments (Equipment Installations)" - icon = "fa fa-plug" - - # ========== List View (MS Access Datasheet View Equivalent) ========== - - sortable_fields = [ - "id", - "thing_id", - "sensor_id", - "installation_date", - "removal_date", - "recording_interval", - "release_status", - "created_at", - "nma_WI_Duration", - "nma_WI_EndFrequency", - "nma_WI_Magnitude", - "nma_WI_MicGain", - "nma_WI_MinSoundDepth", - "nma_WI_StartFrequency", - ] - - fields_default_sort = [ - ("installation_date", True) - ] # True = descending (newest first) - - searchable_fields = [ - "hanging_point_description", - "notes", - "installation_date", - "removal_date", - "recording_interval_units", - "release_status", - "created_at", - "nma_WI_Duration", - "nma_WI_EndFrequency", - "nma_WI_Magnitude", - "nma_WI_MicGain", - "nma_WI_MinSoundDepth", - "nma_WI_StartFrequency", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View (MS Access Form View Equivalent) ========== - - fields = [ - "id", - # Deployment Information - "thing_id", - "sensor_id", - "installation_date", - "removal_date", - "recording_interval", - "recording_interval_units", - "hanging_cable_length", - "hanging_point_height", - "hanging_point_description", - "notes", - "nma_WI_Duration", - "nma_WI_EndFrequency", - "nma_WI_Magnitude", - "nma_WI_MicGain", - "nma_WI_MinSoundDepth", - "nma_WI_StartFrequency", - # Release Status - "release_status", - # Audit Fields - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - # Exclude relationship objects (use IDs instead) - "thing", - "sensor", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - # Exclude relationship objects (use IDs instead) - "thing", - "sensor", - ] - - # ========== Field Labels and Help Text ========== diff --git a/admin/views/field.py b/admin/views/field.py deleted file mode 100644 index 43a7b2cb5..000000000 --- a/admin/views/field.py +++ /dev/null @@ -1,199 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -Field admin views for OcotilloAPI. -""" - -from admin.views.base import OcotilloModelView - - -class FieldEventAdmin(OcotilloModelView): - """ - Admin view for FieldEvent model. - """ - - name = "Field Events" - label = "Field Events" - icon = "fa fa-calendar" - - sortable_fields = [ - "id", - "thing_id", - "event_date", - "release_status", - "created_at", - ] - - fields_default_sort = [("event_date", True)] - - searchable_fields = [ - "notes", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - fields = [ - "id", - "thing_id", - "event_date", - "notes", - "release_status", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - ] - - -class FieldActivityAdmin(OcotilloModelView): - """ - Admin view for FieldActivity model. - """ - - name = "Field Activities" - label = "Field Activities" - icon = "fa fa-tasks" - - sortable_fields = [ - "id", - "field_event_id", - "activity_type", - "release_status", - "created_at", - ] - - fields_default_sort = [("created_at", True)] - - searchable_fields = [ - "notes", - "activity_type", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - fields = [ - "id", - "field_event_id", - "activity_type", - "notes", - "release_status", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - ] - - -class FieldEventParticipantAdmin(OcotilloModelView): - """ - Admin view for FieldEventParticipant model. - """ - - name = "Field Event Participants" - label = "Field Event Participants" - icon = "fa fa-users" - - sortable_fields = [ - "id", - "field_event_id", - "contact_id", - "participant_role", - "release_status", - "created_at", - ] - - fields_default_sort = [("created_at", True)] - - searchable_fields = [ - "participant_role", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - fields = [ - "id", - "field_event_id", - "contact_id", - "participant_role", - "release_status", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - ] - - -# ============= EOF ============================================= diff --git a/admin/views/field_parameters.py b/admin/views/field_parameters.py deleted file mode 100644 index 5638370cc..000000000 --- a/admin/views/field_parameters.py +++ /dev/null @@ -1,139 +0,0 @@ -# =============================================================================== -# Copyright 2026 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -FieldParametersAdmin view for legacy NMA_FieldParameters. - -Updated for Integer PK schema: -- id: Integer PK (autoincrement) -- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit -- chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id -- nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit -- nma_sample_point_id: Legacy SamplePointID string -- nma_object_id: Legacy OBJECTID -- nma_wclab_id: Legacy WCLab_ID -""" - -from starlette.requests import Request - -from admin.views.base import OcotilloModelView - - -class FieldParametersAdmin(OcotilloModelView): - """ - Admin view for FieldParameters model. - """ - - # ========== Basic Configuration ========== - - name = "NMA Field Parameters" - label = "NMA Field Parameters" - icon = "fa fa-tachometer" - - # Integer PK - pk_attr = "id" - pk_type = int - - def can_create(self, request: Request) -> bool: - return False - - def can_edit(self, request: Request) -> bool: - return False - - def can_delete(self, request: Request) -> bool: - return False - - # ========== List View ========== - - list_fields = [ - "id", - "nma_global_id", - "chemistry_sample_info_id", - "nma_sample_pt_id", - "nma_sample_point_id", - "field_parameter", - "sample_value", - "units", - "notes", - "analyses_agency", - "nma_wclab_id", - "nma_object_id", - ] - - sortable_fields = [ - "id", - "nma_global_id", - "chemistry_sample_info_id", - "nma_sample_pt_id", - "nma_sample_point_id", - "field_parameter", - "sample_value", - "units", - "notes", - "analyses_agency", - "nma_wclab_id", - "nma_object_id", - ] - - fields_default_sort = [("nma_sample_point_id", True)] - - searchable_fields = [ - "nma_global_id", - "nma_sample_pt_id", - "nma_sample_point_id", - "field_parameter", - "units", - "notes", - "analyses_agency", - "nma_wclab_id", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View ========== - - fields = [ - "id", - "nma_global_id", - "chemistry_sample_info_id", - "nma_sample_pt_id", - "nma_sample_point_id", - "field_parameter", - "sample_value", - "units", - "notes", - "nma_object_id", - "analyses_agency", - "nma_wclab_id", - ] - - field_labels = { - "id": "ID", - "nma_global_id": "NMA GlobalID (Legacy)", - "chemistry_sample_info_id": "Chemistry Sample Info ID", - "nma_sample_pt_id": "NMA SamplePtID (Legacy)", - "nma_sample_point_id": "NMA SamplePointID (Legacy)", - "field_parameter": "FieldParameter", - "sample_value": "SampleValue", - "units": "Units", - "notes": "Notes", - "nma_object_id": "NMA OBJECTID (Legacy)", - "analyses_agency": "AnalysesAgency", - "nma_wclab_id": "NMA WCLab_ID (Legacy)", - } - - -# ============= EOF ============================================= diff --git a/admin/views/geologic_formation.py b/admin/views/geologic_formation.py deleted file mode 100644 index bb6212026..000000000 --- a/admin/views/geologic_formation.py +++ /dev/null @@ -1,85 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -GeologicFormationAdmin view for OcotilloAPI. -""" - -from admin.fields import WKTField -from admin.views.base import OcotilloModelView - - -class GeologicFormationAdmin(OcotilloModelView): - """ - Admin view for GeologicFormation model. - """ - - name = "Geologic Formations" - label = "Geologic Formations" - icon = "fa fa-layer-group" - - sortable_fields = [ - "id", - "formation_code", - "lithology", - "release_status", - "created_at", - ] - - fields_default_sort = [("formation_code", False)] - - searchable_fields = [ - "formation_code", - "description", - "lithology", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - fields = [ - "id", - "formation_code", - "description", - "lithology", - WKTField("boundary", label="Boundary (WKT)"), - "release_status", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - ] - - -# ============= EOF ============================================= diff --git a/admin/views/group.py b/admin/views/group.py deleted file mode 100644 index f06a9ab76..000000000 --- a/admin/views/group.py +++ /dev/null @@ -1,93 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -GroupAdmin view for OcotilloAPI. -""" - -from admin.fields import WKTField -from admin.views.base import OcotilloModelView - - -class GroupAdmin(OcotilloModelView): - """ - Admin view for Group model. - """ - - # ========== Basic Configuration ========== - - name = "Groups" - label = "Groups" - icon = "fa fa-object-group" - - # ========== List View ========== - - sortable_fields = [ - "id", - "name", - "group_type", - "parent_group_id", - "release_status", - "created_at", - ] - - fields_default_sort = [("name", False)] - - searchable_fields = [ - "name", - "description", - "group_type", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View ========== - - fields = [ - "id", - "name", - "description", - "group_type", - "parent_group_id", - WKTField("project_area", label="Project Area (WKT)"), - "release_status", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - ] - - -# ============= EOF ============================================= diff --git a/admin/views/hydraulicsdata.py b/admin/views/hydraulicsdata.py deleted file mode 100644 index 9723cbb38..000000000 --- a/admin/views/hydraulicsdata.py +++ /dev/null @@ -1,149 +0,0 @@ -# =============================================================================== -# Copyright 2026 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -HydraulicsDataAdmin view for legacy NMA_HydraulicsData. - -Updated for Integer PK schema: -- id: Integer PK (autoincrement) -- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit -- nma_well_id: Legacy WellID UUID -- nma_point_id: Legacy PointID string -- nma_object_id: Legacy OBJECTID, UNIQUE -""" - -from admin.views.base import OcotilloModelView - - -class HydraulicsDataAdmin(OcotilloModelView): - """ - Admin view for NMA_HydraulicsData model. - """ - - # ========== Basic Configuration ========== - - name = "Hydraulics Data" - label = "Hydraulics Data" - icon = "fa fa-tint" - - # Integer PK - pk_attr = "id" - pk_type = int - - can_create = False - can_edit = False - can_delete = False - - # ========== List View ========== - - list_fields = [ - "id", - "nma_global_id", - "nma_well_id", - "nma_point_id", - "thing_id", - "hydraulic_unit", - "hydraulic_unit_type", - "test_top", - "test_bottom", - "t_ft2_d", - "k_darcy", - "data_source", - "nma_object_id", - ] - - sortable_fields = [ - "id", - "nma_global_id", - "nma_well_id", - "nma_point_id", - "thing_id", - "hydraulic_unit", - "hydraulic_unit_type", - "test_top", - "test_bottom", - "t_ft2_d", - "k_darcy", - "data_source", - "nma_object_id", - ] - - searchable_fields = [ - "nma_global_id", - "nma_point_id", - "hydraulic_unit", - "hydraulic_remarks", - "data_source", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View ========== - - fields = [ - "id", - "nma_global_id", - "nma_well_id", - "nma_point_id", - "thing_id", - "hydraulic_unit", - "hydraulic_unit_type", - "hydraulic_remarks", - "test_top", - "test_bottom", - "t_ft2_d", - "s_dimensionless", - "ss_ft_1", - "sy_decimalfractn", - "kh_ft_d", - "kv_ft_d", - "hl_day_1", - "hd_ft2_d", - "cs_gal_d_ft", - "p_decimal_fraction", - "k_darcy", - "data_source", - "nma_object_id", - ] - - field_labels = { - "id": "ID", - "nma_global_id": "NMA GlobalID (Legacy)", - "nma_well_id": "NMA WellID (Legacy)", - "nma_point_id": "NMA PointID (Legacy)", - "thing_id": "Thing ID", - "hydraulic_unit": "HydraulicUnit", - "hydraulic_unit_type": "HydraulicUnitType", - "hydraulic_remarks": "Hydraulic Remarks", - "test_top": "TestTop", - "test_bottom": "TestBottom", - "t_ft2_d": "T (ft2/d)", - "s_dimensionless": "S (dimensionless)", - "ss_ft_1": "Ss (ft-1)", - "sy_decimalfractn": "Sy (decimalfractn)", - "kh_ft_d": "KH (ft/d)", - "kv_ft_d": "KV (ft/d)", - "hl_day_1": "HL (day-1)", - "hd_ft2_d": "HD (ft2/d)", - "cs_gal_d_ft": "Cs (gal/d/ft)", - "p_decimal_fraction": "P (decimal fraction)", - "k_darcy": "k (darcy)", - "data_source": "Data Source", - "nma_object_id": "NMA OBJECTID (Legacy)", - } - - -# ============= EOF ============================================= diff --git a/admin/views/lexicon.py b/admin/views/lexicon.py deleted file mode 100644 index 57cafa6a5..000000000 --- a/admin/views/lexicon.py +++ /dev/null @@ -1,97 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -Lexicon admin views for OcotilloAPI. -""" - -from admin.views.base import OcotilloModelView - - -class LexiconTermAdmin(OcotilloModelView): - """ - Admin view for LexiconTerm model. - """ - - name = "Lexicon Terms" - label = "Lexicon Terms" - icon = "fa fa-book" - enable_publish_actions = False - - sortable_fields = [ - "id", - "term", - ] - - fields_default_sort = [("term", False)] - - searchable_fields = [ - "term", - "definition", - ] - - fields = [ - "id", - "term", - "definition", - ] - - exclude_fields_from_create = [ - "id", - ] - - exclude_fields_from_edit = [ - "id", - ] - - -class LexiconCategoryAdmin(OcotilloModelView): - """ - Admin view for LexiconCategory model. - """ - - name = "Lexicon Categories" - label = "Lexicon Categories" - icon = "fa fa-tags" - enable_publish_actions = False - - sortable_fields = [ - "id", - "name", - ] - - fields_default_sort = [("name", False)] - - searchable_fields = [ - "name", - "description", - ] - - fields = [ - "id", - "name", - "description", - ] - - exclude_fields_from_create = [ - "id", - ] - - exclude_fields_from_edit = [ - "id", - ] - - -# ============= EOF ============================================= diff --git a/admin/views/location.py b/admin/views/location.py deleted file mode 100644 index 2ec2f2616..000000000 --- a/admin/views/location.py +++ /dev/null @@ -1,122 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -LocationAdmin view for OcotilloAPI. - -Provides MS Access-like interface for CRUD operations on Location model. -""" - -from admin.fields import CoordinateHelpField -from admin.views.base import OcotilloModelView - - -class LocationAdmin(OcotilloModelView): - """ - Admin view for Location model. - - Designed to replicate MS Access "Location Entry Form" and "Location Datasheet View". - - Permission Model: - - Admin: Can create, edit, delete all locations - - Editor: Can create and edit, cannot delete - - Viewer: Can only view published locations (read-only) - """ - - # ========== Basic Configuration ========== - - name = "Locations" - label = "Locations" - icon = "fa fa-map-marker" - - # ========== List View (MS Access Datasheet View Equivalent) ========== - - sortable_fields = [ - "id", - "description", - "elevation", - "county", - "state", - "quad_name", - "release_status", - "created_at", - ] - - fields_default_sort = [("created_at", True)] # True = descending - - searchable_fields = [ - "description", - "county", - "state", - "quad_name", - "release_status", - "elevation", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View (MS Access Form View Equivalent) ========== - - fields = [ - "id", - "description", - CoordinateHelpField( - "point", - label="Coordinates (WKT)", - required=True, - ), - "elevation", - "county", - "state", - "quad_name", - "nma_location_notes", - "nma_coordinate_notes", - "nma_data_reliability", - "release_status", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - "nma_pk_location", - "nma_date_created", - "nma_site_date", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - "nma_pk_location", - "nma_date_created", - "nma_site_date", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "nma_pk_location", - "nma_date_created", - "nma_site_date", - ] - - # ========== Field Labels and Help Text ========== diff --git a/admin/views/major_chemistry.py b/admin/views/major_chemistry.py deleted file mode 100644 index 9578f60d1..000000000 --- a/admin/views/major_chemistry.py +++ /dev/null @@ -1,169 +0,0 @@ -# =============================================================================== -# Copyright 2026 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -MajorChemistryAdmin view for legacy NMA_MajorChemistry. - -Updated for Integer PK schema: -- id: Integer PK (autoincrement) -- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit -- chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id -- nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit -- nma_sample_point_id: Legacy SamplePointID string -- nma_object_id: Legacy OBJECTID -- nma_wclab_id: Legacy WCLab_ID -""" - -from starlette.requests import Request -from starlette_admin.fields import HasOne - -from admin.views.base import OcotilloModelView - - -class MajorChemistryAdmin(OcotilloModelView): - """ - Admin view for NMA_MajorChemistry model. - """ - - # ========== Basic Configuration ========== - - identity = "n-m-a_-major-chemistry" - name = "NMA Major Chemistry" - label = "NMA Major Chemistry" - icon = "fa fa-flask" - - # Integer PK - pk_attr = "id" - pk_type = int - - def can_create(self, request: Request) -> bool: - return False - - def can_edit(self, request: Request) -> bool: - return False - - def can_delete(self, request: Request) -> bool: - return False - - # ========== List View ========== - - list_fields = [ - "id", - "nma_global_id", - "chemistry_sample_info_id", - "nma_sample_pt_id", - "nma_sample_point_id", - HasOne("chemistry_sample_info", identity="n-m-a_-chemistry_-sample-info"), - "analyte", - "symbol", - "sample_value", - "units", - "uncertainty", - "analysis_method", - "analysis_date", - "notes", - "volume", - "volume_unit", - "nma_object_id", - "analyses_agency", - "nma_wclab_id", - ] - - sortable_fields = [ - "id", - "nma_global_id", - "chemistry_sample_info_id", - "nma_sample_pt_id", - "nma_sample_point_id", - "analyte", - "symbol", - "sample_value", - "units", - "uncertainty", - "analysis_method", - "analysis_date", - "notes", - "volume", - "volume_unit", - "nma_object_id", - "analyses_agency", - "nma_wclab_id", - ] - - fields_default_sort = [("analysis_date", True)] - - searchable_fields = [ - "nma_global_id", - "nma_sample_pt_id", - "nma_sample_point_id", - "analyte", - "symbol", - "analysis_method", - "notes", - "analyses_agency", - "nma_wclab_id", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View ========== - - fields = [ - "id", - "nma_global_id", - "chemistry_sample_info_id", - "nma_sample_pt_id", - "nma_sample_point_id", - HasOne("chemistry_sample_info", identity="n-m-a_-chemistry_-sample-info"), - "analyte", - "symbol", - "sample_value", - "units", - "uncertainty", - "analysis_method", - "analysis_date", - "notes", - "volume", - "volume_unit", - "nma_object_id", - "analyses_agency", - "nma_wclab_id", - ] - - field_labels = { - "id": "ID", - "nma_global_id": "NMA GlobalID (Legacy)", - "chemistry_sample_info_id": "Chemistry Sample Info ID", - "nma_sample_pt_id": "NMA SamplePtID (Legacy)", - "nma_sample_point_id": "NMA SamplePointID (Legacy)", - "chemistry_sample_info": "Chemistry Sample Info", - "analyte": "Analyte", - "symbol": "Symbol", - "sample_value": "Sample Value", - "units": "Units", - "uncertainty": "Uncertainty", - "analysis_method": "Analysis Method", - "analysis_date": "Analysis Date", - "notes": "Notes", - "volume": "Volume", - "volume_unit": "Volume Unit", - "nma_object_id": "NMA OBJECTID (Legacy)", - "analyses_agency": "Analyses Agency", - "nma_wclab_id": "NMA WCLab_ID (Legacy)", - } - - -# ============= EOF ============================================= diff --git a/admin/views/minor_trace_chemistry.py b/admin/views/minor_trace_chemistry.py deleted file mode 100644 index 0c51e609e..000000000 --- a/admin/views/minor_trace_chemistry.py +++ /dev/null @@ -1,138 +0,0 @@ -# =============================================================================== -# Copyright 2026 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -MinorTraceChemistryAdmin view for legacy NMA_MinorTraceChemistry. - -Updated for Integer PK schema: -- id: Integer PK (autoincrement) -- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit -- chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id -- nma_chemistry_sample_info_uuid: Legacy UUID FK for audit -""" - -from starlette.requests import Request -from starlette_admin.fields import HasOne - -from admin.views.base import OcotilloModelView - - -class MinorTraceChemistryAdmin(OcotilloModelView): - """ - Admin view for NMA_MinorTraceChemistry model. - """ - - # ========== Basic Configuration ========== - - identity = "n-m-a_-minor-trace-chemistry" - name = "Minor Trace Chemistry" - label = "Minor Trace Chemistry" - icon = "fa fa-flask" - - # Integer PK - pk_attr = "id" - pk_type = int - - def can_create(self, request: Request) -> bool: - return False - - def can_edit(self, request: Request) -> bool: - return False - - def can_delete(self, request: Request) -> bool: - return False - - # ========== List View ========== - - list_fields = [ - "id", - "nma_global_id", - HasOne("chemistry_sample_info", identity="n-m-a_-chemistry_-sample-info"), - "nma_chemistry_sample_info_uuid", - "analyte", - "sample_value", - "units", - "symbol", - "analysis_date", - "analyses_agency", - ] - - sortable_fields = [ - "id", - "nma_global_id", - "chemistry_sample_info_id", - "analyte", - "sample_value", - "units", - "symbol", - "analysis_date", - "analyses_agency", - ] - - fields_default_sort = [("analysis_date", True)] - - searchable_fields = [ - "nma_global_id", - "analyte", - "symbol", - "analysis_method", - "notes", - "analyses_agency", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View ========== - - fields = [ - "id", - "nma_global_id", - HasOne("chemistry_sample_info", identity="n-m-a_-chemistry_-sample-info"), - "nma_chemistry_sample_info_uuid", - "analyte", - "symbol", - "sample_value", - "units", - "uncertainty", - "analysis_method", - "analysis_date", - "notes", - "volume", - "volume_unit", - "analyses_agency", - ] - - field_labels = { - "id": "ID", - "nma_global_id": "NMA GlobalID (Legacy)", - "chemistry_sample_info": "Chemistry Sample Info", - "chemistry_sample_info_id": "Chemistry Sample Info ID", - "nma_chemistry_sample_info_uuid": "NMA Chemistry Sample Info UUID (Legacy)", - "analyte": "Analyte", - "symbol": "Symbol", - "sample_value": "Sample Value", - "units": "Units", - "uncertainty": "Uncertainty", - "analysis_method": "Analysis Method", - "analysis_date": "Analysis Date", - "notes": "Notes", - "volume": "Volume", - "volume_unit": "Volume Unit", - "analyses_agency": "Analyses Agency", - } - - -# ============= EOF ============================================= diff --git a/admin/views/notes.py b/admin/views/notes.py deleted file mode 100644 index 6be42f912..000000000 --- a/admin/views/notes.py +++ /dev/null @@ -1,91 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -NotesAdmin view for OcotilloAPI. -""" - -from admin.views.base import OcotilloModelView - - -class NotesAdmin(OcotilloModelView): - """ - Admin view for Notes model. - """ - - # ========== Basic Configuration ========== - - name = "Notes" - label = "Notes" - icon = "fa fa-sticky-note" - - # ========== List View ========== - - sortable_fields = [ - "id", - "target_table", - "target_id", - "note_type", - "release_status", - "created_at", - ] - - fields_default_sort = [("created_at", True)] - - searchable_fields = [ - "target_table", - "note_type", - "content", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View ========== - - fields = [ - "id", - "target_table", - "target_id", - "note_type", - "content", - "release_status", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - ] - - -# ============= EOF ============================================= diff --git a/admin/views/observation.py b/admin/views/observation.py deleted file mode 100644 index d2e206e36..000000000 --- a/admin/views/observation.py +++ /dev/null @@ -1,128 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -ObservationAdmin view for OcotilloAPI. - -Provides MS Access-like interface for CRUD operations on Observation (Water Levels) model. -""" - -from admin.views.base import OcotilloModelView - - -class ObservationAdmin(OcotilloModelView): - """ - Admin view for Observation model (Water Levels). - - Designed to replicate MS Access "Water Level Entry Form" and "Water Level Datasheet View". - - Permission Model: - - Admin: Can create, edit, delete all observations - - Editor: Can create and edit, cannot delete - - Viewer: Can only view published observations (read-only) - """ - - # ========== Basic Configuration ========== - - name = "Observations" - label = "Observations (Water Levels)" - icon = "fa fa-line-chart" - - # ========== List View (MS Access Datasheet View Equivalent) ========== - - sortable_fields = [ - "id", - "observation_datetime", - "value", - "unit", - "measuring_point_height", - "release_status", - "created_at", - ] - - fields_default_sort = [ - ("observation_datetime", True) - ] # True = descending (newest first) - - searchable_fields = [ - "groundwater_level_reason", - "notes", - "observation_datetime", - "unit", - "groundwater_level_reason", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200, 500] - - # ========== Form View (MS Access Form View Equivalent) ========== - - fields = [ - "id", - # Core measurement data - "observation_datetime", - "value", - "unit", - "measuring_point_height", - "groundwater_level_reason", - "notes", - # Relationships (display as selects) - "sample_id", - "sensor_id", - "parameter_id", - "analysis_method_id", - # Release Status - "release_status", - # Audit Fields - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - # Legacy Migration Fields - "nma_pk_waterlevels", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - "nma_pk_waterlevels", - # Exclude relationship objects (use IDs instead) - "sample", - "sensor", - "parameter", - "analysis_method", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "nma_pk_waterlevels", - # Exclude relationship objects (use IDs instead) - "sample", - "sensor", - "parameter", - "analysis_method", - ] - - # ========== Field Labels and Help Text ========== diff --git a/admin/views/parameter.py b/admin/views/parameter.py deleted file mode 100644 index 50eb674a8..000000000 --- a/admin/views/parameter.py +++ /dev/null @@ -1,90 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -ParameterAdmin view for OcotilloAPI. -""" - -from admin.views.base import OcotilloModelView - - -class ParameterAdmin(OcotilloModelView): - """ - Admin view for Parameter model. - """ - - name = "Parameters" - label = "Parameters" - icon = "fa fa-flask" - - sortable_fields = [ - "id", - "parameter_name", - "matrix", - "parameter_type", - "cas_number", - "default_unit", - "release_status", - "created_at", - ] - - fields_default_sort = [("parameter_name", False)] - - searchable_fields = [ - "parameter_name", - "cas_number", - "matrix", - "parameter_type", - "default_unit", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - fields = [ - "id", - "parameter_name", - "matrix", - "parameter_type", - "cas_number", - "default_unit", - "release_status", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - ] - - -# ============= EOF ============================================= diff --git a/admin/views/radionuclides.py b/admin/views/radionuclides.py deleted file mode 100644 index 27c240aea..000000000 --- a/admin/views/radionuclides.py +++ /dev/null @@ -1,165 +0,0 @@ -# =============================================================================== -# Copyright 2026 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -RadionuclidesAdmin view for legacy NMA_Radionuclides. - -Updated for Integer PK schema: -- id: Integer PK (autoincrement) -- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit -- chemistry_sample_info_id: Integer FK to NMA_Chemistry_SampleInfo.id -- nma_sample_pt_id: Legacy UUID FK (SamplePtID) for audit -- nma_sample_point_id: Legacy SamplePointID string -- nma_object_id: Legacy OBJECTID, UNIQUE -- nma_wclab_id: Legacy WCLab_ID -""" - -from starlette.requests import Request - -from admin.views.base import OcotilloModelView - - -class RadionuclidesAdmin(OcotilloModelView): - """ - Admin view for NMA_Radionuclides model. - """ - - # ========== Basic Configuration ========== - - name = "NMA Radionuclides" - label = "NMA Radionuclides" - icon = "fa fa-radiation" - - # Integer PK - pk_attr = "id" - pk_type = int - - def can_create(self, request: Request) -> bool: - return False - - def can_edit(self, request: Request) -> bool: - return False - - def can_delete(self, request: Request) -> bool: - return False - - # ========== List View ========== - - list_fields = [ - "id", - "nma_global_id", - "chemistry_sample_info_id", - "nma_sample_pt_id", - "nma_sample_point_id", - "analyte", - "symbol", - "sample_value", - "units", - "uncertainty", - "analysis_method", - "analysis_date", - "notes", - "volume", - "volume_unit", - "nma_object_id", - "analyses_agency", - "nma_wclab_id", - ] - - sortable_fields = [ - "id", - "nma_global_id", - "chemistry_sample_info_id", - "nma_sample_pt_id", - "nma_sample_point_id", - "analyte", - "symbol", - "sample_value", - "units", - "uncertainty", - "analysis_method", - "analysis_date", - "notes", - "volume", - "volume_unit", - "nma_object_id", - "analyses_agency", - "nma_wclab_id", - ] - - fields_default_sort = [("analysis_date", True)] - - searchable_fields = [ - "nma_global_id", - "nma_sample_pt_id", - "nma_sample_point_id", - "analyte", - "symbol", - "analysis_method", - "analysis_date", - "notes", - "analyses_agency", - "nma_wclab_id", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View ========== - - fields = [ - "id", - "nma_global_id", - "chemistry_sample_info_id", - "nma_sample_pt_id", - "nma_sample_point_id", - "analyte", - "symbol", - "sample_value", - "units", - "uncertainty", - "analysis_method", - "analysis_date", - "notes", - "volume", - "volume_unit", - "nma_object_id", - "analyses_agency", - "nma_wclab_id", - ] - - field_labels = { - "id": "ID", - "nma_global_id": "NMA GlobalID (Legacy)", - "chemistry_sample_info_id": "Chemistry Sample Info ID", - "nma_sample_pt_id": "NMA SamplePtID (Legacy)", - "nma_sample_point_id": "NMA SamplePointID (Legacy)", - "analyte": "Analyte", - "symbol": "Symbol", - "sample_value": "Sample Value", - "units": "Units", - "uncertainty": "Uncertainty", - "analysis_method": "Analysis Method", - "analysis_date": "Analysis Date", - "notes": "Notes", - "volume": "Volume", - "volume_unit": "Volume Unit", - "nma_object_id": "NMA OBJECTID (Legacy)", - "analyses_agency": "Analyses Agency", - "nma_wclab_id": "NMA WCLab_ID (Legacy)", - } - - -# ============= EOF ============================================= diff --git a/admin/views/sample.py b/admin/views/sample.py deleted file mode 100644 index b5247a913..000000000 --- a/admin/views/sample.py +++ /dev/null @@ -1,103 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -SampleAdmin view for OcotilloAPI. -""" - -from admin.views.base import OcotilloModelView - - -class SampleAdmin(OcotilloModelView): - """ - Admin view for Sample model. - """ - - # ========== Basic Configuration ========== - - name = "Samples" - label = "Samples" - icon = "fa fa-flask" - - # ========== List View ========== - - sortable_fields = [ - "id", - "sample_name", - "sample_date", - "sample_matrix", - "sample_method", - "qc_type", - "release_status", - "created_at", - ] - - fields_default_sort = [("sample_date", True)] - - searchable_fields = [ - "sample_name", - "notes", - "nma_pk_waterlevels", - "sample_matrix", - "sample_method", - "qc_type", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View ========== - - fields = [ - "id", - "field_activity_id", - "field_event_participant_id", - "sample_date", - "sample_name", - "sample_matrix", - "sample_method", - "qc_type", - "depth_top", - "depth_bottom", - "notes", - "nma_pk_waterlevels", - "release_status", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - ] - - -# ============= EOF ============================================= diff --git a/admin/views/sensor.py b/admin/views/sensor.py deleted file mode 100644 index 28d41e44e..000000000 --- a/admin/views/sensor.py +++ /dev/null @@ -1,123 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -SensorAdmin view for OcotilloAPI. - -Provides MS Access-like interface for CRUD operations on Sensor (Equipment) model. -""" - -from admin.views.base import OcotilloModelView - - -class SensorAdmin(OcotilloModelView): - """ - Admin view for Sensor model (Equipment). - - Designed to replicate MS Access "Equipment Entry Form" and "Equipment Datasheet View". - - Permission Model: - - Admin: Can create, edit, delete all sensors - - Editor: Can create and edit, cannot delete - - Viewer: Can only view published sensors (read-only) - """ - - # ========== Basic Configuration ========== - - name = "Sensors" - label = "Sensors (Equipment)" - icon = "fa fa-microchip" - - # ========== List View (MS Access Datasheet View Equivalent) ========== - - sortable_fields = [ - "id", - "name", - "sensor_type", - "model", - "serial_no", - "owner_agency", - "sensor_status", - "release_status", - "created_at", - ] - - fields_default_sort = [("created_at", True)] # True = descending - - searchable_fields = [ - "name", - "serial_no", - "model", - "pcn_number", - "sensor_type", - "owner_agency", - "sensor_status", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View (MS Access Form View Equivalent) ========== - - fields = [ - "id", - # Equipment Information - "name", - "sensor_type", - "model", - "serial_no", - "pcn_number", - "owner_agency", - "sensor_status", - "notes", - # Release Status - "release_status", - # Audit Fields - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - # Legacy Migration Fields - "nma_pk_equipment", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - "nma_pk_equipment", - # Exclude complex relationships - "observations", - "deployments", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "nma_pk_equipment", - # Exclude complex relationships - "observations", - "deployments", - ] - - # ========== Field Labels and Help Text ========== diff --git a/admin/views/soil_rock_results.py b/admin/views/soil_rock_results.py deleted file mode 100644 index 947804980..000000000 --- a/admin/views/soil_rock_results.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -SoilRockResultsAdmin view for legacy NMA_Soil_Rock_Results. - -Already has Integer PK. Updated for legacy column rename: -- point_id -> nma_point_id -""" - -from admin.views.base import OcotilloModelView - - -class SoilRockResultsAdmin(OcotilloModelView): - """ - Read-only admin view for SoilRockResults legacy model. - """ - - # ========== Basic Configuration ========== - name = "NMA Soil Rock Results" - label = "NMA Soil Rock Results" - icon = "fa fa-mountain" - - # Integer PK (already correct) - pk_attr = "id" - pk_type = int - - # Pagination - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== List View ========== - list_fields = [ - "id", - "nma_point_id", - "sample_type", - "date_sampled", - "d13c", - "d18o", - "sampled_by", - "thing_id", - ] - - sortable_fields = [ - "id", - "nma_point_id", - ] - - searchable_fields = [ - "nma_point_id", - "sample_type", - "date_sampled", - "sampled_by", - ] - - fields_default_sort = [("id", True)] - - # ========== Detail View ========== - fields = [ - "id", - "nma_point_id", - "sample_type", - "date_sampled", - "d13c", - "d18o", - "sampled_by", - "thing_id", - ] - - # ========== Legacy Field Labels ========== - field_labels = { - "id": "ID", - "nma_point_id": "NMA Point_ID (Legacy)", - "sample_type": "Sample Type", - "date_sampled": "Date Sampled", - "d13c": "d13C", - "d18o": "d18O", - "sampled_by": "Sampled by", - "thing_id": "ThingID", - } diff --git a/admin/views/stratigraphy.py b/admin/views/stratigraphy.py deleted file mode 100644 index 0bbd32231..000000000 --- a/admin/views/stratigraphy.py +++ /dev/null @@ -1,100 +0,0 @@ -""" -StratigraphyAdmin view for legacy stratigraphy. - -Updated for Integer PK schema: -- id: Integer PK (autoincrement) -- nma_global_id: Legacy UUID PK (GlobalID), UNIQUE for audit -- nma_well_id: Legacy WellID UUID -- nma_point_id: Legacy PointID string -- nma_object_id: Legacy OBJECTID, UNIQUE -""" - -from admin.views.base import OcotilloModelView - - -class StratigraphyAdmin(OcotilloModelView): - """ - Read-only admin view for Stratigraphy legacy model. - """ - - # ========== Basic Configuration ========== - name = "NMA Stratigraphy" - label = "NMA Stratigraphy" - icon = "fa fa-layer-group" - - # Integer PK - pk_attr = "id" - pk_type = int - - # Pagination - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== List View ========== - - sortable_fields = [ - "id", - "nma_global_id", - "nma_object_id", - "nma_point_id", - ] - - fields_default_sort = [("nma_point_id", False), ("strat_top", False)] - - searchable_fields = [ - "nma_point_id", - "nma_global_id", - "unit_identifier", - "lithology", - "lithologic_modifier", - "contributing_unit", - "strat_source", - "strat_notes", - ] - - # ========== Form View ========== - - fields = [ - "id", - "nma_global_id", - "nma_well_id", - "nma_point_id", - "thing_id", - "strat_top", - "strat_bottom", - "unit_identifier", - "lithology", - "lithologic_modifier", - "contributing_unit", - "strat_source", - "strat_notes", - "nma_object_id", - ] - - exclude_fields_from_create = [ - "id", - "nma_object_id", - ] - - exclude_fields_from_edit = [ - "id", - "nma_object_id", - ] - - # ========== Legacy Field Labels ========== - field_labels = { - "id": "ID", - "nma_global_id": "NMA GlobalID (Legacy)", - "nma_well_id": "NMA WellID (Legacy)", - "nma_point_id": "NMA PointID (Legacy)", - "thing_id": "ThingID", - "strat_top": "StratTop", - "strat_bottom": "StratBottom", - "unit_identifier": "UnitIdentifier", - "lithology": "Lithology", - "lithologic_modifier": "LithologicModifier", - "contributing_unit": "ContributingUnit", - "strat_source": "StratSource", - "strat_notes": "StratNotes", - "nma_object_id": "NMA OBJECTID (Legacy)", - } diff --git a/admin/views/surface_water.py b/admin/views/surface_water.py deleted file mode 100644 index be6da860d..000000000 --- a/admin/views/surface_water.py +++ /dev/null @@ -1,96 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -SurfaceWaterDataAdmin view for OcotilloAPI. -""" - -from admin.views.base import OcotilloModelView - - -class SurfaceWaterDataAdmin(OcotilloModelView): - """ - Admin view for SurfaceWaterData legacy model. - """ - - name = "NMA Surface Water Data" - label = "NMA Surface Water Data" - icon = "fa fa-water" - enable_publish_actions = False - - sortable_fields = [ - "surface_id", - "point_id", - "date_measured", - "discharge", - "discharge_units", - "discharge_method", - "discharge_source", - "formation_zone", - "aq_class", - ] - - fields_default_sort = [("date_measured", True)] - - searchable_fields = [ - "point_id", - "discharge", - "formation_zone", - "aq_class", - "data_source", - "discharge_units", - "discharge_method", - "discharge_source", - "formation_zone", - "aq_class", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - fields = [ - "surface_id", - "point_id", - "object_id", - "date_measured", - "discharge", - "discharge_rate", - "discharge_units", - "discharge_method", - "discharge_source", - "formation_zone", - "aq_class", - "site_notes", - "field_method_notes", - "source_notes", - "data_source", - ] - - # ========== READ ONLY ========== - enable_publish_actions = ( - False # hides publish/unpublish actions inherited from base - ) - - def can_create(self, request) -> bool: - return False - - def can_edit(self, request) -> bool: - return False - - def can_delete(self, request) -> bool: - return False - - -# ============= EOF ============================================= diff --git a/admin/views/surface_water_photos.py b/admin/views/surface_water_photos.py deleted file mode 100644 index 2d2b73299..000000000 --- a/admin/views/surface_water_photos.py +++ /dev/null @@ -1,71 +0,0 @@ -from admin.views.base import OcotilloModelView - - -class SurfaceWaterPhotosAdmin(OcotilloModelView): - """ - Admin view for legacy SurfaceWaterPhotos model (NMA_SurfaceWaterPhotos). - """ - - # ========== Basic Configuration ========== - name = "NMA Surface Water Photos" - label = "NMA Surface Water Photos" - icon = "fa fa-water" - - # Pagination - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== List View ========== - list_fields = [ - "surface_id", - "point_id", - "ole_path", - "object_id", - "global_id", - ] - - sortable_fields = [ - "global_id", - "object_id", - "point_id", - ] - - fields_default_sort = [("point_id", False), ("object_id", False)] - - searchable_fields = [ - "point_id", - "global_id", - "ole_path", - ] - - # ========== Detail View ========== - fields = [ - "surface_id", - "point_id", - "ole_path", - "object_id", - "global_id", - ] - - # ========== Legacy Field Labels ========== - field_labels = { - "surface_id": "SurfaceID", - "point_id": "PointID", - "ole_path": "OLEPath", - "object_id": "OBJECTID", - "global_id": "GlobalID", - } - - # ========== READ ONLY ========== - enable_publish_actions = ( - False # hides publish/unpublish actions inherited from base - ) - - def can_create(self, request) -> bool: - return False - - def can_edit(self, request) -> bool: - return False - - def can_delete(self, request) -> bool: - return False diff --git a/admin/views/thing.py b/admin/views/thing.py deleted file mode 100644 index da6d7acbb..000000000 --- a/admin/views/thing.py +++ /dev/null @@ -1,161 +0,0 @@ -# =============================================================================== -# Copyright 2025 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -ThingAdmin view for OcotilloAPI. - -Provides MS Access-like interface for CRUD operations on Thing (Wells/Springs) model. -""" - -from admin.views.base import OcotilloModelView - - -class ThingAdmin(OcotilloModelView): - """ - Admin view for Thing model (Wells, Springs, etc.). - - Designed to replicate MS Access "Well Data Entry Form" and "Well Datasheet View". - - Permission Model: - - Admin: Can create, edit, delete all things - - Editor: Can create and edit, cannot delete - - Viewer: Can only view published things (read-only) - """ - - # ========== Basic Configuration ========== - - identity = "thing" - name = "Things" - label = "Things (Wells/Springs)" - icon = "fa fa-tint" - - # ========== List View (MS Access Datasheet View Equivalent) ========== - - sortable_fields = [ - "id", - "name", - "thing_type", - "well_depth", - "hole_depth", - "first_visit_date", - "release_status", - "created_at", - ] - - fields_default_sort = [("created_at", True)] # True = descending - - searchable_fields = [ - "name", - "thing_type", - "well_driller_name", - "well_depth", - "first_visit_date", - "release_status", - "created_at", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View (MS Access Form View Equivalent) ========== - - fields = [ - "id", - # Basic Information - "name", - "thing_type", - "first_visit_date", - # Well Construction - "well_depth", - "hole_depth", - "well_casing_diameter", - "well_casing_depth", - "well_completion_date", - "well_driller_name", - "well_construction_method", - "well_pump_type", - "well_pump_depth", - "formation_completion_code", - # Spring-specific - "spring_type", - # Release Status - "release_status", - # Audit Fields - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - # Legacy Migration Fields - "nma_pk_welldata", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - "nma_pk_welldata", - # Exclude complex relationships from create form - "location_associations", - "contact_associations", - "asset_associations", - "field_events", - "deployments", - "group_associations", - "screens", - "well_purposes", - "well_casing_materials", - "links", - "measuring_points", - "monitoring_frequencies", - "aquifer_associations", - "formation_associations", - "status_history", - "permission_history", - "data_provenance", - "notes", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "nma_pk_welldata", - # Exclude complex relationships from edit form (manage separately) - "location_associations", - "contact_associations", - "asset_associations", - "field_events", - "deployments", - "group_associations", - "screens", - "well_purposes", - "well_casing_materials", - "links", - "measuring_points", - "monitoring_frequencies", - "aquifer_associations", - "formation_associations", - "status_history", - "permission_history", - "data_provenance", - "notes", - ] - - # ========== Field Labels and Help Text ========== diff --git a/admin/views/transducer_observation.py b/admin/views/transducer_observation.py deleted file mode 100644 index d9318d0e8..000000000 --- a/admin/views/transducer_observation.py +++ /dev/null @@ -1,205 +0,0 @@ -# =============================================================================== -# Copyright 2026 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -TransducerObservationAdmin view for transducer observations. -""" - -from admin.views.base import OcotilloModelView - - -class TransducerObservationAdmin(OcotilloModelView): - """ - Admin view for TransducerObservation model. - """ - - # ========== Basic Configuration ========== - - name = "Transducer Observations" - label = "Transducer Observations" - icon = "fa fa-tachometer-alt" - - # ========== List View ========== - - sortable_fields = [ - "id", - "observation_datetime", - "value", - "parameter_id", - "deployment_id", - "release_status", - ] - - fields_default_sort = [("observation_datetime", True)] - - searchable_fields = [ - "observation_datetime", - "parameter_id", - "deployment_id", - "release_status", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Form View ========== - - fields = [ - "id", - "observation_datetime", - "value", - "parameter_id", - "deployment_id", - "release_status", - "nma_waterlevelscontinuous_pressure_conddl_ms_cm", - "nma_waterlevelscontinuous_pressure_checked_by", - "nma_waterlevelscontinuous_pressure_created", - "nma_waterlevelscontinuous_pressure_data_source", - "nma_waterlevelscontinuous_pressure_global_id", - "nma_waterlevelscontinuous_pressure_measurement_method", - "nma_waterlevelscontinuous_pressure_measuring_agency", - "nma_waterlevelscontinuous_pressure_notes", - "nma_waterlevelscontinuous_pressure_processed_by", - "nma_waterlevelscontinuous_pressure_qced", - "nma_waterlevelscontinuous_pressure_temperature_water", - "nma_waterlevelscontinuous_pressure_updated", - "nma_waterlevelscontinuous_pressure_water_head", - "nma_waterlevelscontinuous_pressure_water_head_adjusted", - "nma_waterlevelscontinuous_acoustic_created", - "nma_waterlevelscontinuous_acoustic_data_source", - "nma_waterlevelscontinuous_acoustic_global_id", - "nma_waterlevelscontinuous_acoustic_measurement_method", - "nma_waterlevelscontinuous_acoustic_measuring_agency", - "nma_waterlevelscontinuous_acoustic_notes", - "nma_waterlevelscontinuous_acoustic_point_id", - "nma_waterlevelscontinuous_acoustic_pre_process_data_field", - "nma_waterlevelscontinuous_acoustic_public_release", - "nma_waterlevelscontinuous_acoustic_sensor_hgt_above_mp", - "nma_waterlevelscontinuous_acoustic_serial_no", - "nma_waterlevelscontinuous_acoustic_server_receipt_date", - "nma_waterlevelscontinuous_acoustic_speaker_to_mic_length", - "nma_waterlevelscontinuous_acoustic_temperature_air", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - ] - - exclude_fields_from_create = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "updated_by_id", - "updated_by_name", - "nma_waterlevelscontinuous_pressure_conddl_ms_cm", - "nma_waterlevelscontinuous_pressure_checked_by", - "nma_waterlevelscontinuous_pressure_created", - "nma_waterlevelscontinuous_pressure_data_source", - "nma_waterlevelscontinuous_pressure_global_id", - "nma_waterlevelscontinuous_pressure_measurement_method", - "nma_waterlevelscontinuous_pressure_measuring_agency", - "nma_waterlevelscontinuous_pressure_notes", - "nma_waterlevelscontinuous_pressure_processed_by", - "nma_waterlevelscontinuous_pressure_qced", - "nma_waterlevelscontinuous_pressure_temperature_water", - "nma_waterlevelscontinuous_pressure_updated", - "nma_waterlevelscontinuous_pressure_water_head", - "nma_waterlevelscontinuous_pressure_water_head_adjusted", - "nma_waterlevelscontinuous_acoustic_created", - "nma_waterlevelscontinuous_acoustic_data_source", - "nma_waterlevelscontinuous_acoustic_global_id", - "nma_waterlevelscontinuous_acoustic_measurement_method", - "nma_waterlevelscontinuous_acoustic_measuring_agency", - "nma_waterlevelscontinuous_acoustic_notes", - "nma_waterlevelscontinuous_acoustic_point_id", - "nma_waterlevelscontinuous_acoustic_pre_process_data_field", - "nma_waterlevelscontinuous_acoustic_public_release", - "nma_waterlevelscontinuous_acoustic_sensor_hgt_above_mp", - "nma_waterlevelscontinuous_acoustic_serial_no", - "nma_waterlevelscontinuous_acoustic_server_receipt_date", - "nma_waterlevelscontinuous_acoustic_speaker_to_mic_length", - "nma_waterlevelscontinuous_acoustic_temperature_air", - ] - - exclude_fields_from_edit = [ - "id", - "created_at", - "created_by_id", - "created_by_name", - "nma_waterlevelscontinuous_pressure_conddl_ms_cm", - "nma_waterlevelscontinuous_pressure_checked_by", - "nma_waterlevelscontinuous_pressure_created", - "nma_waterlevelscontinuous_pressure_data_source", - "nma_waterlevelscontinuous_pressure_global_id", - "nma_waterlevelscontinuous_pressure_measurement_method", - "nma_waterlevelscontinuous_pressure_measuring_agency", - "nma_waterlevelscontinuous_pressure_notes", - "nma_waterlevelscontinuous_pressure_processed_by", - "nma_waterlevelscontinuous_pressure_qced", - "nma_waterlevelscontinuous_pressure_temperature_water", - "nma_waterlevelscontinuous_pressure_updated", - "nma_waterlevelscontinuous_pressure_water_head", - "nma_waterlevelscontinuous_pressure_water_head_adjusted", - "nma_waterlevelscontinuous_acoustic_created", - "nma_waterlevelscontinuous_acoustic_data_source", - "nma_waterlevelscontinuous_acoustic_global_id", - "nma_waterlevelscontinuous_acoustic_measurement_method", - "nma_waterlevelscontinuous_acoustic_measuring_agency", - "nma_waterlevelscontinuous_acoustic_notes", - "nma_waterlevelscontinuous_acoustic_point_id", - "nma_waterlevelscontinuous_acoustic_pre_process_data_field", - "nma_waterlevelscontinuous_acoustic_public_release", - "nma_waterlevelscontinuous_acoustic_sensor_hgt_above_mp", - "nma_waterlevelscontinuous_acoustic_serial_no", - "nma_waterlevelscontinuous_acoustic_server_receipt_date", - "nma_waterlevelscontinuous_acoustic_speaker_to_mic_length", - "nma_waterlevelscontinuous_acoustic_temperature_air", - ] - - readonly_fields = [ - "nma_waterlevelscontinuous_pressure_conddl_ms_cm", - "nma_waterlevelscontinuous_pressure_checked_by", - "nma_waterlevelscontinuous_pressure_created", - "nma_waterlevelscontinuous_pressure_data_source", - "nma_waterlevelscontinuous_pressure_global_id", - "nma_waterlevelscontinuous_pressure_measurement_method", - "nma_waterlevelscontinuous_pressure_measuring_agency", - "nma_waterlevelscontinuous_pressure_notes", - "nma_waterlevelscontinuous_pressure_processed_by", - "nma_waterlevelscontinuous_pressure_qced", - "nma_waterlevelscontinuous_pressure_temperature_water", - "nma_waterlevelscontinuous_pressure_updated", - "nma_waterlevelscontinuous_pressure_water_head", - "nma_waterlevelscontinuous_pressure_water_head_adjusted", - "nma_waterlevelscontinuous_acoustic_created", - "nma_waterlevelscontinuous_acoustic_data_source", - "nma_waterlevelscontinuous_acoustic_global_id", - "nma_waterlevelscontinuous_acoustic_measurement_method", - "nma_waterlevelscontinuous_acoustic_measuring_agency", - "nma_waterlevelscontinuous_acoustic_notes", - "nma_waterlevelscontinuous_acoustic_point_id", - "nma_waterlevelscontinuous_acoustic_pre_process_data_field", - "nma_waterlevelscontinuous_acoustic_public_release", - "nma_waterlevelscontinuous_acoustic_sensor_hgt_above_mp", - "nma_waterlevelscontinuous_acoustic_serial_no", - "nma_waterlevelscontinuous_acoustic_server_receipt_date", - "nma_waterlevelscontinuous_acoustic_speaker_to_mic_length", - "nma_waterlevelscontinuous_acoustic_temperature_air", - ] - - -# ============= EOF ============================================= diff --git a/admin/views/waterlevelscontinuous_pressure_daily.py b/admin/views/waterlevelscontinuous_pressure_daily.py deleted file mode 100644 index ac2afb020..000000000 --- a/admin/views/waterlevelscontinuous_pressure_daily.py +++ /dev/null @@ -1,148 +0,0 @@ -# =============================================================================== -# Copyright 2026 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -""" -WaterLevelsContinuousPressureDailyAdmin view for legacy NMA_WaterLevelsContinuous_Pressure_Daily. -""" - -from starlette.requests import Request - -from admin.views.base import OcotilloModelView - - -class WaterLevelsContinuousPressureDailyAdmin(OcotilloModelView): - """ - Admin view for NMA_WaterLevelsContinuous_Pressure_Daily model. - """ - - # ========== Basic Configuration ========== - name = "NMA Water Levels Continuous Pressure Daily" - label = "NMA Water Levels Continuous Pressure Daily" - icon = "fa fa-tachometer-alt" - - def can_create(self, request: Request) -> bool: - return False - - def can_edit(self, request: Request) -> bool: - return False - - def can_delete(self, request: Request) -> bool: - return False - - # ========== List View ========== - list_fields = [ - "global_id", - "object_id", - "well_id", - "point_id", - "date_measured", - "temperature_water", - "water_head", - "water_head_adjusted", - "depth_to_water_bgs", - "measurement_method", - "data_source", - "measuring_agency", - "qced", - "notes", - "created", - "updated", - "processed_by", - "checked_by", - "cond_dl_ms_cm", - ] - - sortable_fields = [ - "global_id", - "object_id", - "well_id", - "point_id", - "date_measured", - "water_head", - "depth_to_water_bgs", - "measurement_method", - "data_source", - "measuring_agency", - "qced", - "created", - "updated", - "processed_by", - "checked_by", - "cond_dl_ms_cm", - ] - - fields_default_sort = [("date_measured", True)] - - searchable_fields = [ - "global_id", - "well_id", - "point_id", - "date_measured", - "measurement_method", - "data_source", - "measuring_agency", - "notes", - ] - - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== Detail View ========== - fields = [ - "global_id", - "object_id", - "well_id", - "point_id", - "date_measured", - "temperature_water", - "water_head", - "water_head_adjusted", - "depth_to_water_bgs", - "measurement_method", - "data_source", - "measuring_agency", - "qced", - "notes", - "created", - "updated", - "processed_by", - "checked_by", - "cond_dl_ms_cm", - ] - - field_labels = { - "global_id": "GlobalID", - "object_id": "OBJECTID", - "well_id": "WellID", - "point_id": "PointID", - "date_measured": "Date Measured", - "temperature_water": "Temperature Water", - "water_head": "Water Head", - "water_head_adjusted": "Water Head Adjusted", - "depth_to_water_bgs": "Depth To Water (BGS)", - "measurement_method": "Measurement Method", - "data_source": "Data Source", - "measuring_agency": "Measuring Agency", - "qced": "QCed", - "notes": "Notes", - "created": "Created", - "updated": "Updated", - "processed_by": "Processed By", - "checked_by": "Checked By", - "cond_dl_ms_cm": "CONDDL (mS/cm)", - } - - -# ============= EOF ============================================= diff --git a/admin/views/weather_data.py b/admin/views/weather_data.py deleted file mode 100644 index 662721c3a..000000000 --- a/admin/views/weather_data.py +++ /dev/null @@ -1,66 +0,0 @@ -from admin.views.base import OcotilloModelView - - -class WeatherDataAdmin(OcotilloModelView): - """ - Admin view for legacy WeatherData model (NMA_WeatherData). - """ - - # ========== Basic Configuration ========== - name = "NMA Weather Data" - label = "NMA Weather Data" - icon = "fa fa-cloud-sun" - - # Pagination - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== List View ========== - list_fields = [ - "location_id", - "point_id", - "weather_id", - "object_id", - ] - - sortable_fields = [ - "object_id", - "point_id", - ] - - fields_default_sort = [("point_id", False), ("object_id", False)] - - searchable_fields = [ - "point_id", - "weather_id", - ] - - # ========== Detail View ========== - fields = [ - "location_id", - "point_id", - "weather_id", - "object_id", - ] - - # ========== Legacy Field Labels ========== - field_labels = { - "location_id": "LocationId", - "point_id": "PointID", - "weather_id": "WeatherID", - "object_id": "OBJECTID", - } - - # ========== READ ONLY ========== - enable_publish_actions = ( - False # hides publish/unpublish actions inherited from base - ) - - def can_create(self, request) -> bool: - return False - - def can_edit(self, request) -> bool: - return False - - def can_delete(self, request) -> bool: - return False diff --git a/admin/views/weather_photos.py b/admin/views/weather_photos.py deleted file mode 100644 index 006d1b10a..000000000 --- a/admin/views/weather_photos.py +++ /dev/null @@ -1,70 +0,0 @@ -from admin.views.base import OcotilloModelView - - -class WeatherPhotosAdmin(OcotilloModelView): - """ - Admin view for legacy WeatherPhotos model (NMA_WeatherPhotos). - """ - - # ========== Basic Configuration ========== - name = "NMA Weather Photos" - label = "NMA Weather Photos" - icon = "fa fa-cloud" - - # Pagination - page_size = 50 - page_size_options = [25, 50, 100, 200] - - # ========== List View ========== - list_fields = [ - "weather_id", - "point_id", - "ole_path", - "object_id", - "global_id", - ] - - sortable_fields = [ - "global_id", - "object_id", - "point_id", - ] - - fields_default_sort = [("point_id", False), ("object_id", False)] - - searchable_fields = [ - "point_id", - "ole_path", - ] - - # ========== Detail View ========== - fields = [ - "weather_id", - "point_id", - "ole_path", - "object_id", - "global_id", - ] - - # ========== Legacy Field Labels ========== - field_labels = { - "weather_id": "WeatherID", - "point_id": "PointID", - "ole_path": "OLEPath", - "object_id": "OBJECTID", - "global_id": "GlobalID", - } - - # ========== READ ONLY ========== - enable_publish_actions = ( - False # hides publish/unpublish actions inherited from base - ) - - def can_create(self, request) -> bool: - return False - - def can_edit(self, request) -> bool: - return False - - def can_delete(self, request) -> bool: - return False diff --git a/alembic/versions/2d3c3a268652_create_internal_ogc_views.py b/alembic/versions/2d3c3a268652_create_internal_ogc_views.py new file mode 100644 index 000000000..a7e7c5835 --- /dev/null +++ b/alembic/versions/2d3c3a268652_create_internal_ogc_views.py @@ -0,0 +1,1432 @@ +"""create internal ogc views + +Companion migration to f4a5b6c7d8e9 (public release_status filter on ogc_* +views): creates a second, unfiltered copy of the same 22 relations, named +ogc_internal_, backing the authenticated /ogcapi-internal mount +(core/pygeoapi.py::mount_pygeoapi_internal). Full parity with the public +set, per ticket A11 -- not a subset. + +Also mirrors ogc_waterlevels/ogc_water_chemistry from z9a0b1c2d3e4 (added to +staging after this migration's original 22-relation scope was written, per +ADR3's EDR feature) as ogc_internal_waterlevels/ogc_internal_water_chemistry, +bringing the total to 24. Unfiltered in all three places the public views +predicate on release_status: the manual-readings and chemistry selects' +`o.release_status = 'public'`, and the transducer union's +`tobs.release_status = 'public'`. + +The major/minor chemistry analyte-mapping CASE blocks and +STATIC_ANALYTE_COLUMNS lists below are intentionally character-for-character +identical (modulo view name) to their counterparts in f4a5b6c7d8e9 -- this +codebase keeps migrations self-contained with no cross-migration imports, so +the logic is duplicated here rather than shared. tests/test_migration_view_ +parity.py enforces the two stay in sync: if you fix an analyte mapping in +one file, apply the same fix to the other. + +ogc_internal_locations has no release_status predicate at all (unlike +ogc_locations, which is always public-only) -- the internal mount is +unfiltered by design, and ogc_internal_locations never existed before this +migration in any form. + +ogc_internal_actively_monitored_wells gets no predicate of its own -- like +its public counterpart, it inherits whichever rows ogc_internal_water_well_ +summary exposes (here, all of them) transitively via a direct JOIN. Because +of that JOIN, it must be dropped before ogc_internal_water_well_summary and +recreated after (same ordering constraint as the public side). + +All 24 relations here are newly created by this migration -- none of them +existed in any form beforehand -- so downgrade() simply drops them rather +than recreating a prior state. + +Revision ID: 2d3c3a268652 +Revises: f4a5b6c7d8e9 +Create Date: 2026-07-16 00:00:00.000000 +""" + +import re +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "2d3c3a268652" +down_revision: Union[str, Sequence[str], None] = "f4a5b6c7d8e9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +REQUIRED_TABLES = { + "thing", + "location", + "location_thing_association", + "group", + "group_thing_association", + "status_history", + "observation", + "sample", + "field_activity", + "field_event", + "data_provenance", + "NMA_MajorChemistry", + "NMA_Chemistry_SampleInfo", + "NMA_MinorTraceChemistry", + # For the ogc_internal_waterlevels/ogc_internal_water_chemistry EDR + # mirrors (see z9a0b1c2d3e4_add_edr_water_views.py). + "transducer_observation", + "deployment", + "parameter", +} + +LATEST_LOCATION_CTE = """ +SELECT DISTINCT ON (lta.thing_id) + lta.thing_id, + lta.location_id, + lta.effective_start +FROM location_thing_association AS lta +WHERE lta.effective_end IS NULL +ORDER BY lta.thing_id, lta.effective_start DESC +""".strip() + +# Same 11 thing-type views as f4a5b6c7d8e9's THING_VIEWS. +THING_VIEWS = [ + ("water_wells", "water well"), + ("springs", "spring"), + ("diversions_surface_water", "diversion of surface water, etc."), + ("ephemeral_streams", "ephemeral stream"), + ("lakes_ponds_reservoirs", "lake, pond or reservoir"), + ("meteorological_stations", "meteorological station"), + ("other_things", "other"), + ("outfalls_wastewater_return_flow", "outfall of wastewater or return flow"), + ("perennial_streams", "perennial stream"), + ("rock_sample_locations", "rock sample location"), + ("soil_gas_sample_locations", "soil gas sample location"), +] + + +def _safe_view_id(view_id: str) -> str: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", view_id): + raise ValueError(f"Unsafe view id: {view_id!r}") + return view_id + + +def _drop_view_or_materialized_view(view_name: str) -> None: + # DROP VIEW IF EXISTS / DROP MATERIALIZED VIEW IF EXISTS only suppress + # "relation does not exist" -- Postgres still raises WrongObjectType if + # the relation exists as the other kind (e.g. DROP VIEW against an + # existing materialized view), so the relation's actual kind must be + # checked first rather than trying both blindly. + bind = op.get_bind() + relkind = bind.execute( + text("SELECT relkind FROM pg_class WHERE oid = to_regclass(:name)"), + {"name": view_name}, + ).scalar() + if relkind == "m": + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {view_name}")) + elif relkind == "v": + op.execute(text(f"DROP VIEW IF EXISTS {view_name}")) + + +def _check_required_tables() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing_tables = set(inspector.get_table_names(schema="public")) + missing = REQUIRED_TABLES - existing_tables + if missing: + raise RuntimeError( + "Cannot create internal OGC views. " + f"Missing required tables: {', '.join(sorted(missing))}" + ) + + +def _create_thing_view(view_id: str, thing_type: str, public_only: bool) -> str: + safe_view_id = _safe_view_id(view_id) + escaped_thing_type = thing_type.replace("'", "''") + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE VIEW ogc_internal_{safe_view_id} AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ) + SELECT + t.id, + t.name, + t.first_visit_date, + t.nma_pk_welldata, + t.well_depth, + t.hole_depth, + t.well_casing_diameter, + t.well_casing_depth, + t.well_completion_date, + t.well_driller_name, + t.well_construction_method, + t.well_pump_type, + t.well_pump_depth, + t.formation_completion_code, + t.nma_formation_zone, + t.release_status, + l.elevation, + l.point + FROM thing AS t + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE t.thing_type = '{escaped_thing_type}'{release_filter} + """ + + +def _create_latest_depth_view(public_only: bool) -> str: + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE MATERIALIZED VIEW ogc_internal_latest_depth_to_water_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + ranked_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + o.value, + o.measuring_point_height, + -- Treat NULL measuring_point_height as 0 when computing + -- depth_to_water_bgs. + ( + o.value - COALESCE(o.measuring_point_height, 0) + ) AS depth_to_water_bgs, + ROW_NUMBER() OVER ( + PARTITION BY fe.thing_id + ORDER BY o.observation_datetime DESC, o.id DESC + ) AS rn + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL{release_filter} + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + ro.observation_id, + ro.observation_datetime, + ro.value AS depth_to_water_reference, + ro.measuring_point_height, + ro.depth_to_water_bgs, + l.point + FROM ranked_obs AS ro + JOIN thing AS t ON t.id = ro.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE ro.rn = 1 + """ + + +def _create_avg_tds_view(public_only: bool) -> str: + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE MATERIALIZED VIEW ogc_internal_avg_tds_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + tds_obs AS ( + SELECT + csi.thing_id, + mc.id AS major_chemistry_id, + COALESCE(mc."AnalysisDate", csi."CollectionDate")::date AS observation_date, + mc."SampleValue" AS sample_value, + mc."Units" AS units + FROM "NMA_MajorChemistry" AS mc + JOIN "NMA_Chemistry_SampleInfo" AS csi + ON csi.id = mc.chemistry_sample_info_id + JOIN thing AS t ON t.id = csi.thing_id + WHERE + t.thing_type = 'water well' + AND mc."SampleValue" IS NOT NULL + AND ( + lower(coalesce(mc."Analyte", '')) IN ( + 'tds', + 'total dissolved solids' + ) + OR lower(coalesce(mc."Symbol", '')) = 'tds' + ){release_filter} + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + COUNT(to2.major_chemistry_id)::integer AS tds_observation_count, + AVG(to2.sample_value)::double precision AS avg_tds_value, + MIN(to2.observation_date) AS first_tds_observation_date, + MAX(to2.observation_date) AS last_tds_observation_date, + l.point + FROM tds_obs AS to2 + JOIN thing AS t ON t.id = to2.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + GROUP BY t.id, t.name, t.thing_type, l.point + """ + + +def _create_latest_tds_view(public_only: bool) -> str: + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE VIEW ogc_internal_latest_tds_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + tds_obs AS ( + SELECT + csi.thing_id, + mc.id AS major_chemistry_id, + COALESCE(mc."AnalysisDate", csi."CollectionDate") AS observation_datetime, + mc."SampleValue" AS sample_value, + mc."Units" AS units + FROM "NMA_MajorChemistry" AS mc + JOIN "NMA_Chemistry_SampleInfo" AS csi + ON csi.id = mc.chemistry_sample_info_id + JOIN thing AS t ON t.id = csi.thing_id + WHERE + t.thing_type = 'water well' + AND mc."SampleValue" IS NOT NULL + AND ( + lower(coalesce(mc."Analyte", '')) IN ( + 'tds', + 'total dissolved solids' + ) + OR lower(coalesce(mc."Symbol", '')) = 'tds' + ){release_filter} + ), + ranked_tds AS ( + SELECT + to2.thing_id, + to2.major_chemistry_id, + to2.observation_datetime, + to2.sample_value, + to2.units, + ROW_NUMBER() OVER ( + PARTITION BY to2.thing_id + ORDER BY to2.observation_datetime DESC NULLS LAST, to2.major_chemistry_id DESC + ) AS rn + FROM tds_obs AS to2 + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + rt.major_chemistry_id, + rt.observation_datetime::date AS latest_tds_observation_date, + rt.sample_value AS latest_tds_value, + rt.units AS latest_tds_units, + l.point + FROM ranked_tds AS rt + JOIN thing AS t ON t.id = rt.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE rt.rn = 1 + """ + + +def _create_depth_to_water_trend_view(public_only: bool) -> str: + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE MATERIALIZED VIEW ogc_internal_depth_to_water_trend_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + obs AS ( + SELECT + fe.thing_id, + o.observation_datetime, + (o.value - COALESCE(o.measuring_point_height, 0)) AS depth_to_water_bgs + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL{release_filter} + ), + agg AS ( + SELECT + ob.thing_id, + COUNT(*)::integer AS record_count, + MIN(ob.observation_datetime) AS first_observation_datetime, + MAX(ob.observation_datetime) AS last_observation_datetime, + EXTRACT(EPOCH FROM (MAX(ob.observation_datetime) - MIN(ob.observation_datetime))) + / 31557600.0 AS span_years, + REGR_SLOPE( + ob.depth_to_water_bgs, + EXTRACT(EPOCH FROM ob.observation_datetime) + ) * 31557600.0 AS slope_ft_per_year + FROM obs AS ob + GROUP BY ob.thing_id + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + a.record_count, + a.first_observation_datetime, + a.last_observation_datetime, + a.span_years, + a.slope_ft_per_year, + CASE + WHEN a.record_count >= 10 OR (a.record_count >= 4 AND a.span_years >= 2.0) THEN + CASE + WHEN a.slope_ft_per_year IS NULL THEN 'not enough data' + WHEN a.slope_ft_per_year > 0.25 THEN 'increasing' + WHEN a.slope_ft_per_year < -0.25 THEN 'decreasing' + ELSE 'stable' + END + ELSE 'not enough data' + END AS trend_category, + l.point + FROM agg AS a + JOIN thing AS t ON t.id = a.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + """ + + +def _create_water_well_summary_view(public_only: bool) -> str: + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE MATERIALIZED VIEW ogc_internal_water_well_summary AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + wl_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + (o.value - COALESCE(o.measuring_point_height, 0)) AS water_level + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL{release_filter} + ), + wl_agg AS ( + SELECT + w.thing_id, + COUNT(*)::integer AS total_water_levels, + MIN(w.water_level) AS min_water_level, + MAX(w.water_level) AS max_water_level, + REGR_SLOPE( + w.water_level, + EXTRACT(EPOCH FROM w.observation_datetime) + ) * 31557600.0 AS water_level_trend_ft_per_year + FROM wl_obs AS w + GROUP BY w.thing_id + ), + wl_last AS ( + SELECT + ranked.thing_id, + ranked.water_level AS last_water_level, + ranked.observation_datetime AS last_water_level_datetime + FROM ( + SELECT + w.thing_id, + w.water_level, + w.observation_datetime, + ROW_NUMBER() OVER ( + PARTITION BY w.thing_id + ORDER BY w.observation_datetime DESC, w.observation_id DESC + ) AS rn + FROM wl_obs AS w + ) AS ranked + WHERE ranked.rn = 1 + ) + SELECT + t.id AS id, + t.name, + t.well_depth, + l.elevation, + dpl.collection_method AS elevation_method, + t.nma_formation_zone AS formation_zone, + wa.total_water_levels, + wl.last_water_level, + wl.last_water_level_datetime, + wa.min_water_level, + wa.max_water_level, + wa.water_level_trend_ft_per_year, + l.point + FROM thing AS t + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + JOIN wl_agg AS wa ON wa.thing_id = t.id + LEFT JOIN wl_last AS wl ON wl.thing_id = t.id + LEFT JOIN LATERAL ( + SELECT dp.collection_method + FROM data_provenance AS dp + WHERE + dp.target_table = 'location' + AND dp.target_id = l.id + AND dp.field_name = 'elevation' + ORDER BY dp.id DESC + LIMIT 1 + ) AS dpl ON true + WHERE t.thing_type = 'water well' + AND wa.total_water_levels > 0 + """ + + +# Static analyte columns for major chemistry pivots. +# Includes aliases observed in current DB values (e.g., Ca(total), IONBAL, TAn, TCat, Na+K). +# Kept character-for-character identical to f4a5b6c7d8e9's copy -- see +# tests/test_migration_view_parity.py. +STATIC_ANALYTE_COLUMNS_MAJOR: list[tuple[str, str]] = [ + ("tds", "tds"), + ("calcium", "calcium"), + ("calcium_total", "calcium_total"), + ("magnesium", "magnesium"), + ("magnesium_total", "magnesium_total"), + ("sodium", "sodium"), + ("sodium_total", "sodium_total"), + ("potassium", "potassium"), + ("potassium_total", "potassium_total"), + ("sodium_plus_potassium", "sodium_plus_potassium"), + ("bicarbonate", "bicarbonate"), + ("carbonate", "carbonate"), + ("sulfate", "sulfate"), + ("chloride", "chloride"), + ("ion_balance", "ion_balance"), + ("total_anions", "total_anions"), + ("total_cations", "total_cations"), + ("alkalinity", "alkalinity"), + ("hardness", "hardness"), + ("specific_conductance", "specific_conductance"), + ("ph", "ph"), + ("nitrate", "nitrate"), + ("fluoride", "fluoride"), + ("silica", "silica"), +] + + +def _major_chemistry_select_columns() -> str: + return ",\n".join( + [ + ( + " MAX(lr.sample_value) FILTER " + f"(WHERE lr.analyte_key = '{analyte_key}') AS {column_name}" + ) + for analyte_key, column_name in STATIC_ANALYTE_COLUMNS_MAJOR + ] + ) + + +def _major_chemistry_unit_columns() -> str: + return ",\n".join( + [ + ( + " MAX(lr.units) FILTER " + f"(WHERE lr.analyte_key = '{analyte_key}') AS {column_name}_units" + ) + for analyte_key, column_name in STATIC_ANALYTE_COLUMNS_MAJOR + ] + ) + + +def _create_major_chemistry_results_view(public_only: bool) -> str: + static_columns = _major_chemistry_select_columns() + static_unit_columns = _major_chemistry_unit_columns() + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE MATERIALIZED VIEW ogc_internal_major_chemistry_results AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + chemistry_rows AS ( + SELECT + csi.thing_id, + mc.id AS result_id, + COALESCE(mc."AnalysisDate", csi."CollectionDate") AS observation_datetime, + trim(mc."Analyte") AS analyte_name, + trim(mc."Symbol") AS symbol_name, + mc."SampleValue"::double precision AS sample_value, + mc."Units" AS units + FROM "NMA_MajorChemistry" AS mc + JOIN "NMA_Chemistry_SampleInfo" AS csi + ON csi.id = mc.chemistry_sample_info_id + JOIN thing AS t + ON t.id = csi.thing_id + WHERE mc."SampleValue" IS NOT NULL + AND t.thing_type = 'water well'{release_filter} + ), + normalized_rows AS ( + SELECT + cr.thing_id, + cr.result_id, + cr.observation_datetime, + NULLIF( + regexp_replace( + lower(trim(coalesce(cr.analyte_name, ''))), + '[^a-z0-9]+', + '', + 'g' + ), + '' + ) AS analyte_token, + NULLIF( + regexp_replace( + lower(trim(coalesce(cr.symbol_name, ''))), + '[^a-z0-9]+', + '', + 'g' + ), + '' + ) AS symbol_token, + cr.sample_value, + cr.units + FROM chemistry_rows AS cr + ), + mapped_rows AS ( + SELECT + nr.thing_id, + nr.result_id, + nr.observation_datetime, + CASE + WHEN coalesce(nr.symbol_token, '') = 'tds' + OR coalesce(nr.analyte_token, '') IN ('tds', 'totaldissolvedsolids') + THEN 'tds' + + WHEN coalesce(nr.symbol_token, '') = 'ca' + OR coalesce(nr.analyte_token, '') = 'ca' + THEN 'calcium' + WHEN coalesce(nr.analyte_token, '') = 'catotal' + THEN 'calcium_total' + + WHEN coalesce(nr.symbol_token, '') = 'mg' + OR coalesce(nr.analyte_token, '') = 'mg' + THEN 'magnesium' + WHEN coalesce(nr.analyte_token, '') = 'mgtotal' + THEN 'magnesium_total' + + WHEN coalesce(nr.symbol_token, '') = 'na' + OR coalesce(nr.analyte_token, '') = 'na' + THEN 'sodium' + WHEN coalesce(nr.analyte_token, '') = 'natotal' + THEN 'sodium_total' + + WHEN coalesce(nr.symbol_token, '') = 'k' + OR coalesce(nr.analyte_token, '') = 'k' + THEN 'potassium' + WHEN coalesce(nr.analyte_token, '') = 'ktotal' + THEN 'potassium_total' + + WHEN coalesce(nr.analyte_token, '') = 'nak' + THEN 'sodium_plus_potassium' + + WHEN coalesce(nr.symbol_token, '') = 'hco3' + OR coalesce(nr.analyte_token, '') = 'hco3' + THEN 'bicarbonate' + WHEN coalesce(nr.symbol_token, '') = 'co3' + OR coalesce(nr.analyte_token, '') = 'co3' + THEN 'carbonate' + WHEN coalesce(nr.symbol_token, '') = 'so4' + OR coalesce(nr.analyte_token, '') = 'so4' + THEN 'sulfate' + WHEN coalesce(nr.symbol_token, '') = 'cl' + OR coalesce(nr.analyte_token, '') = 'cl' + THEN 'chloride' + + WHEN coalesce(nr.analyte_token, '') = 'ionbal' + THEN 'ion_balance' + WHEN coalesce(nr.analyte_token, '') = 'tan' + THEN 'total_anions' + WHEN coalesce(nr.analyte_token, '') = 'tcat' + THEN 'total_cations' + + WHEN coalesce(nr.analyte_token, '') IN ('alk', 'alkalinity') + THEN 'alkalinity' + WHEN coalesce(nr.analyte_token, '') IN ('hrd', 'hardness') + THEN 'hardness' + WHEN coalesce(nr.analyte_token, '') IN ( + 'condlab', + 'specificconductance', + 'specificconductivity', + 'conductivity' + ) + THEN 'specific_conductance' + WHEN coalesce(nr.symbol_token, '') = 'ph' + OR coalesce(nr.analyte_token, '') IN ('ph', 'phl') + THEN 'ph' + + WHEN coalesce(nr.symbol_token, '') = 'no3' + OR coalesce(nr.analyte_token, '') IN ('no3', 'nitrate') + THEN 'nitrate' + WHEN coalesce(nr.symbol_token, '') = 'f' + OR coalesce(nr.analyte_token, '') IN ('f', 'fluoride') + THEN 'fluoride' + WHEN coalesce(nr.symbol_token, '') = 'sio2' + OR coalesce(nr.analyte_token, '') IN ('sio2', 'silica') + THEN 'silica' + + ELSE NULL + END AS analyte_key, + nr.sample_value, + nr.units + FROM normalized_rows AS nr + ), + latest_results AS ( + SELECT + mr.thing_id, + mr.analyte_key, + mr.sample_value, + mr.units, + mr.observation_datetime, + ROW_NUMBER() OVER ( + PARTITION BY mr.thing_id, mr.analyte_key + ORDER BY mr.observation_datetime DESC NULLS LAST, mr.result_id DESC + ) AS rn + FROM mapped_rows AS mr + WHERE mr.analyte_key IS NOT NULL + ) + SELECT + t.id AS id, + ll.location_id, + t.name, + t.thing_type, + COUNT(*)::integer AS analyte_count, + MAX(lr.observation_datetime::date) AS latest_chemistry_date, +{static_columns}, +{static_unit_columns}, + l.point + FROM latest_results AS lr + JOIN thing AS t ON t.id = lr.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE lr.rn = 1 + GROUP BY t.id, ll.location_id, t.name, t.thing_type, l.point + """ + + +# Kept character-for-character identical to f4a5b6c7d8e9's copy -- see +# tests/test_migration_view_parity.py. +STATIC_ANALYTE_COLUMNS_MINOR: list[tuple[str, str]] = [ + ("h2r", "h2r"), + ("o18r", "o18r"), + ("c13r", "c13r"), + ("c14", "c14"), + ("c14_years", "c14_years"), + ("fluoride", "fluoride"), + ("barium", "barium"), + ("barium_total", "barium_total"), + ("copper", "copper"), + ("copper_total", "copper_total"), + ("zinc", "zinc"), + ("zinc_total", "zinc_total"), + ("molybdenum", "molybdenum"), + ("molybdenum_total", "molybdenum_total"), + ("silica", "silica"), + ("silicon", "silicon"), + ("silicon_total", "silicon_total"), + ("manganese", "manganese"), + ("manganese_total", "manganese_total"), + ("iron", "iron"), + ("iron_total", "iron_total"), + ("strontium", "strontium"), + ("strontium_total", "strontium_total"), + ("chromium", "chromium"), + ("chromium_total", "chromium_total"), + ("boron", "boron"), + ("boron_total", "boron_total"), + ("uranium", "uranium"), + ("uranium_total", "uranium_total"), + ("lithium", "lithium"), + ("lithium_total", "lithium_total"), + ("silver", "silver"), + ("silver_total", "silver_total"), + ("antimony", "antimony"), + ("antimony_total", "antimony_total"), + ("beryllium", "beryllium"), + ("beryllium_total", "beryllium_total"), + ("lead", "lead"), + ("lead_total", "lead_total"), + ("thallium", "thallium"), + ("thallium_total", "thallium_total"), + ("bromide", "bromide"), + ("selenium", "selenium"), + ("selenium_total", "selenium_total"), + ("vanadium", "vanadium"), + ("vanadium_total", "vanadium_total"), + ("aluminum", "aluminum"), + ("aluminum_total", "aluminum_total"), + ("arsenic", "arsenic"), + ("arsenic_total", "arsenic_total"), + ("nickel", "nickel"), + ("nickel_total", "nickel_total"), + ("cadmium", "cadmium"), + ("cadmium_total", "cadmium_total"), + ("cobalt", "cobalt"), + ("cobalt_total", "cobalt_total"), + ("phosphate", "phosphate"), + ("nitrite", "nitrite"), + ("nitrate", "nitrate"), + ("nitrate_as_n", "nitrate_as_n"), + ("thorium", "thorium"), + ("thorium_total", "thorium_total"), + ("tin", "tin"), + ("tin_total", "tin_total"), + ("mercury", "mercury"), + ("mercury_total", "mercury_total"), + ("titanium", "titanium"), + ("titanium_total", "titanium_total"), +] + + +def _minor_chemistry_value_columns() -> str: + return ",\n".join( + [ + ( + " MAX(lr.sample_value) FILTER " + f"(WHERE lr.analyte_key = '{analyte_key}') AS {column_name}" + ) + for analyte_key, column_name in STATIC_ANALYTE_COLUMNS_MINOR + ] + ) + + +def _minor_chemistry_unit_columns() -> str: + return ",\n".join( + [ + ( + " MAX(lr.units) FILTER " + f"(WHERE lr.analyte_key = '{analyte_key}') AS {column_name}_units" + ) + for analyte_key, column_name in STATIC_ANALYTE_COLUMNS_MINOR + ] + ) + + +def _create_minor_chemistry_wells_view(public_only: bool) -> str: + value_columns = _minor_chemistry_value_columns() + unit_columns = _minor_chemistry_unit_columns() + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE MATERIALIZED VIEW ogc_internal_minor_chemistry_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + chemistry_rows AS ( + SELECT + csi.thing_id, + mtc.id AS result_id, + COALESCE(mtc.analysis_date::timestamp, csi."CollectionDate") AS observation_datetime, + trim(mtc.analyte) AS analyte_name, + mtc.sample_value::double precision AS sample_value, + mtc.units AS units + FROM "NMA_MinorTraceChemistry" AS mtc + JOIN "NMA_Chemistry_SampleInfo" AS csi + ON csi.id = mtc.chemistry_sample_info_id + JOIN thing AS t ON t.id = csi.thing_id + WHERE + mtc.sample_value IS NOT NULL + AND t.thing_type = 'water well'{release_filter} + ), + normalized_rows AS ( + SELECT + cr.thing_id, + cr.result_id, + cr.observation_datetime, + NULLIF( + regexp_replace( + lower(trim(coalesce(cr.analyte_name, ''))), + '[^a-z0-9]+', + '', + 'g' + ), + '' + ) AS analyte_token, + cr.sample_value, + cr.units + FROM chemistry_rows AS cr + ), + mapped_rows AS ( + SELECT + nr.thing_id, + nr.result_id, + nr.observation_datetime, + CASE + WHEN coalesce(nr.analyte_token, '') = 'h2r' THEN 'h2r' + WHEN coalesce(nr.analyte_token, '') = 'o18r' THEN 'o18r' + WHEN coalesce(nr.analyte_token, '') = 'c13r' THEN 'c13r' + WHEN coalesce(nr.analyte_token, '') = 'c14' THEN 'c14' + WHEN coalesce(nr.analyte_token, '') = 'c14years' THEN 'c14_years' + + WHEN coalesce(nr.analyte_token, '') = 'f' THEN 'fluoride' + WHEN coalesce(nr.analyte_token, '') = 'ba' THEN 'barium' + WHEN coalesce(nr.analyte_token, '') = 'batotal' THEN 'barium_total' + WHEN coalesce(nr.analyte_token, '') = 'cu' THEN 'copper' + WHEN coalesce(nr.analyte_token, '') = 'cutotal' THEN 'copper_total' + WHEN coalesce(nr.analyte_token, '') = 'zn' THEN 'zinc' + WHEN coalesce(nr.analyte_token, '') = 'zntotal' THEN 'zinc_total' + WHEN coalesce(nr.analyte_token, '') = 'mo' THEN 'molybdenum' + WHEN coalesce(nr.analyte_token, '') = 'mototal' THEN 'molybdenum_total' + WHEN coalesce(nr.analyte_token, '') = 'sio2' THEN 'silica' + WHEN coalesce(nr.analyte_token, '') = 'si' THEN 'silicon' + WHEN coalesce(nr.analyte_token, '') = 'sitotal' THEN 'silicon_total' + WHEN coalesce(nr.analyte_token, '') = 'mn' THEN 'manganese' + WHEN coalesce(nr.analyte_token, '') = 'mntotal' THEN 'manganese_total' + WHEN coalesce(nr.analyte_token, '') = 'fe' THEN 'iron' + WHEN coalesce(nr.analyte_token, '') = 'fetotal' THEN 'iron_total' + WHEN coalesce(nr.analyte_token, '') = 'sr' THEN 'strontium' + WHEN coalesce(nr.analyte_token, '') = 'srtotal' THEN 'strontium_total' + WHEN coalesce(nr.analyte_token, '') = 'cr' THEN 'chromium' + WHEN coalesce(nr.analyte_token, '') = 'crtotal' THEN 'chromium_total' + WHEN coalesce(nr.analyte_token, '') = 'b' THEN 'boron' + WHEN coalesce(nr.analyte_token, '') = 'btotal' THEN 'boron_total' + WHEN coalesce(nr.analyte_token, '') = 'u' THEN 'uranium' + WHEN coalesce(nr.analyte_token, '') = 'utotal' THEN 'uranium_total' + WHEN coalesce(nr.analyte_token, '') = 'li' THEN 'lithium' + WHEN coalesce(nr.analyte_token, '') = 'litotal' THEN 'lithium_total' + WHEN coalesce(nr.analyte_token, '') = 'ag' THEN 'silver' + WHEN coalesce(nr.analyte_token, '') = 'agtotal' THEN 'silver_total' + WHEN coalesce(nr.analyte_token, '') = 'sb' THEN 'antimony' + WHEN coalesce(nr.analyte_token, '') = 'sbtotal' THEN 'antimony_total' + WHEN coalesce(nr.analyte_token, '') = 'be' THEN 'beryllium' + WHEN coalesce(nr.analyte_token, '') = 'betotal' THEN 'beryllium_total' + WHEN coalesce(nr.analyte_token, '') = 'pb' THEN 'lead' + WHEN coalesce(nr.analyte_token, '') = 'pbtotal' THEN 'lead_total' + WHEN coalesce(nr.analyte_token, '') = 'tl' THEN 'thallium' + WHEN coalesce(nr.analyte_token, '') = 'tltotal' THEN 'thallium_total' + WHEN coalesce(nr.analyte_token, '') = 'br' THEN 'bromide' + WHEN coalesce(nr.analyte_token, '') = 'se' THEN 'selenium' + WHEN coalesce(nr.analyte_token, '') = 'setotal' THEN 'selenium_total' + WHEN coalesce(nr.analyte_token, '') = 'v' THEN 'vanadium' + WHEN coalesce(nr.analyte_token, '') = 'vtotal' THEN 'vanadium_total' + WHEN coalesce(nr.analyte_token, '') = 'al' THEN 'aluminum' + WHEN coalesce(nr.analyte_token, '') = 'altotal' THEN 'aluminum_total' + WHEN coalesce(nr.analyte_token, '') = 'as' THEN 'arsenic' + WHEN coalesce(nr.analyte_token, '') = 'astotal' THEN 'arsenic_total' + WHEN coalesce(nr.analyte_token, '') = 'ni' THEN 'nickel' + WHEN coalesce(nr.analyte_token, '') = 'nitotal' THEN 'nickel_total' + WHEN coalesce(nr.analyte_token, '') = 'cd' THEN 'cadmium' + WHEN coalesce(nr.analyte_token, '') = 'cdtotal' THEN 'cadmium_total' + WHEN coalesce(nr.analyte_token, '') = 'co' THEN 'cobalt' + WHEN coalesce(nr.analyte_token, '') = 'cototal' THEN 'cobalt_total' + WHEN coalesce(nr.analyte_token, '') = 'po4' THEN 'phosphate' + WHEN coalesce(nr.analyte_token, '') = 'no2' THEN 'nitrite' + WHEN coalesce(nr.analyte_token, '') = 'no3' THEN 'nitrate' + WHEN coalesce(nr.analyte_token, '') = 'no3n' THEN 'nitrate_as_n' + WHEN coalesce(nr.analyte_token, '') = 'th' THEN 'thorium' + WHEN coalesce(nr.analyte_token, '') = 'thtotal' THEN 'thorium_total' + WHEN coalesce(nr.analyte_token, '') = 'sn' THEN 'tin' + WHEN coalesce(nr.analyte_token, '') = 'sntotal' THEN 'tin_total' + WHEN coalesce(nr.analyte_token, '') = 'hg' THEN 'mercury' + WHEN coalesce(nr.analyte_token, '') = 'hgtotal' THEN 'mercury_total' + WHEN coalesce(nr.analyte_token, '') = 'ti' THEN 'titanium' + WHEN coalesce(nr.analyte_token, '') = 'titotal' THEN 'titanium_total' + ELSE NULL + END AS analyte_key, + nr.sample_value, + nr.units + FROM normalized_rows AS nr + ), + latest_results AS ( + SELECT + mr.thing_id, + mr.analyte_key, + mr.sample_value, + mr.units, + mr.observation_datetime, + ROW_NUMBER() OVER ( + PARTITION BY mr.thing_id, mr.analyte_key + ORDER BY mr.observation_datetime DESC NULLS LAST, mr.result_id DESC + ) AS rn + FROM mapped_rows AS mr + WHERE mr.analyte_key IS NOT NULL + ) + SELECT + t.id AS id, + ll.location_id, + t.name, + t.thing_type, + COUNT(*)::integer AS analyte_count, + MAX(lr.observation_datetime::date) AS latest_chemistry_date, +{value_columns}, +{unit_columns}, + l.point + FROM latest_results AS lr + JOIN thing AS t ON t.id = lr.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE lr.rn = 1 + AND t.thing_type = 'water well' + GROUP BY t.id, ll.location_id, t.name, t.thing_type, l.point + """ + + +METERS_TO_FEET = 3.28084 + + +def _create_water_elevation_view(public_only: bool) -> str: + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE MATERIALIZED VIEW ogc_internal_water_elevation_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + ranked_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + CASE + WHEN lower(trim(o.unit)) IN ('m', 'meter', 'meters', 'metre', 'metres') THEN + (o.value * {METERS_TO_FEET}) - COALESCE(o.measuring_point_height, 0) + WHEN lower(trim(o.unit)) IN ('ft', 'foot', 'feet') THEN + o.value - COALESCE(o.measuring_point_height, 0) + ELSE + NULL + END AS depth_to_water_below_ground_surface + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL + AND lower(trim(o.unit)) IN ( + 'm', + 'meter', + 'meters', + 'metre', + 'metres', + 'ft', + 'foot', + 'feet' + ){release_filter} + ), + latest_obs AS ( + SELECT + ro.*, + ROW_NUMBER() OVER ( + PARTITION BY ro.thing_id + ORDER BY ro.observation_datetime DESC, ro.observation_id DESC + ) AS rn + FROM ranked_obs AS ro + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + lo.observation_id, + lo.observation_datetime, + l.elevation AS elevation_m, + lo.depth_to_water_below_ground_surface AS depth_to_water_below_ground_surface_ft, + ((l.elevation * {METERS_TO_FEET}) - lo.depth_to_water_below_ground_surface) + AS water_elevation_ft, + l.point + FROM latest_obs AS lo + JOIN thing AS t ON t.id = lo.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE lo.rn = 1 + """ + + +def _create_actively_monitored_wells_view() -> str: + # No predicate of its own -- inherits whatever rows + # ogc_internal_water_well_summary exposes (here, all of them) + # transitively via the JOIN below. Mirrors the public side's + # ogc_actively_monitored_wells, which likewise never filters on + # status_history.release_status directly (see + # w1x2y3z4a5b6_drop_child_release_filters_from_ngwmn_views.py for why). + return """ + CREATE VIEW ogc_internal_actively_monitored_wells AS + WITH latest_monitoring_status AS ( + SELECT DISTINCT ON (sh.target_id) + sh.target_id AS thing_id, + sh.status_value + FROM status_history AS sh + WHERE + sh.target_table = 'thing' + AND sh.status_type = 'Monitoring Status' + ORDER BY sh.target_id, sh.start_date DESC, sh.id DESC + ) + SELECT + wws.id, + wws.name, + 'water well'::text AS thing_type, + wws.well_depth, + wws.elevation, + wws.elevation_method, + wws.formation_zone, + wws.total_water_levels, + wws.last_water_level, + wws.last_water_level_datetime, + wws.min_water_level, + wws.max_water_level, + wws.water_level_trend_ft_per_year, + g.id AS group_id, + g.name AS group_name, + g.group_type, + wws.point + FROM "group" AS g + JOIN group_thing_association AS gta ON gta.group_id = g.id + JOIN ogc_internal_water_well_summary AS wws ON wws.id = gta.thing_id + JOIN latest_monitoring_status AS lms ON lms.thing_id = wws.id + WHERE lower(trim(g.name)) = 'water level network' + AND lms.status_value = 'Currently monitored' + """ + + +def _create_project_areas_view(public_only: bool) -> str: + release_filter = " AND g.release_status = 'public'" if public_only else "" + return f""" + CREATE VIEW ogc_internal_project_areas AS + SELECT + g.id, + g.name, + g.description, + g.group_type, + g.release_status, + g.project_area + FROM "group" AS g + WHERE g.project_area IS NOT NULL{release_filter} + """ + + +def _create_locations_view() -> str: + # Unlike ogc_locations (always public-only, even on the public side's + # downgrade path), ogc_internal_locations has no release_status + # predicate at all -- the internal mount is unfiltered by design, and + # this relation never existed in any form before this migration. + # Column list matches ogc_locations exactly; see + # f4a5b6c7d8e9_apply_public_release_status_filter_to_ogc_views.py's + # _create_locations_view() for the db/location.py column verification. + return """ + CREATE VIEW ogc_internal_locations AS + SELECT + l.id, + l.nma_pk_location, + l.description, + l.county, + l.state, + l.quad_name, + l.nma_location_notes, + l.nma_coordinate_notes, + l.nma_data_reliability, + l.nma_date_created, + l.nma_site_date, + l.release_status, + l.elevation, + l.point + FROM location AS l + """ + + +# Shared join from a thing to its current location point -- same shape as +# z9a0b1c2d3e4's _LOCATION_JOIN. +_EDR_LOCATION_JOIN = """ + JOIN location_thing_association lta + ON lta.thing_id = t.id AND lta.effective_end IS NULL + JOIN location l ON l.id = lta.location_id +""" + + +def _create_internal_waterlevels_view() -> str: + # Mirrors z9a0b1c2d3e4's ogc_waterlevels with both release_status + # predicates dropped (manual readings: o.release_status; transducer + # readings: tobs.release_status). release_status itself is still + # selected as a column, same as the public view. + return f""" + CREATE VIEW ogc_internal_waterlevels AS + -- manual water-level readings + SELECT + 'm-' || o.id AS id, + t.id AS thing_id, + t.name AS station_name, + ST_X(l.point) AS longitude, + ST_Y(l.point) AS latitude, + o.observation_datetime AS datetime, + o.value AS value, + o.unit AS unit, + 'groundwater level' AS parameter_name, + 'manual' AS source, + NULL::integer AS deployment_id, + o.release_status AS release_status + FROM observation o + JOIN parameter p + ON p.id = o.parameter_id AND p.parameter_name = 'groundwater level' + JOIN sample sm ON sm.id = o.sample_id + JOIN field_activity fa ON fa.id = sm.field_activity_id + JOIN field_event fe ON fe.id = fa.field_event_id + JOIN thing t ON t.id = fe.thing_id + {_EDR_LOCATION_JOIN} + WHERE o.value IS NOT NULL + + UNION ALL + + -- transducer (instrument) water-level readings + SELECT + 't-' || tobs.id AS id, + t.id AS thing_id, + t.name AS station_name, + ST_X(l.point) AS longitude, + ST_Y(l.point) AS latitude, + tobs.observation_datetime AS datetime, + tobs.value AS value, + p.default_unit AS unit, + 'groundwater level' AS parameter_name, + 'transducer' AS source, + tobs.deployment_id AS deployment_id, + tobs.release_status AS release_status + FROM transducer_observation tobs + JOIN parameter p + ON p.id = tobs.parameter_id AND p.parameter_name = 'groundwater level' + JOIN deployment d ON d.id = tobs.deployment_id + JOIN thing t ON t.id = d.thing_id + {_EDR_LOCATION_JOIN} + WHERE tobs.value IS NOT NULL + """ + + +def _create_internal_water_chemistry_view() -> str: + # Mirrors z9a0b1c2d3e4's ogc_water_chemistry with its release_status + # predicate (o.release_status) dropped. + return f""" + CREATE VIEW ogc_internal_water_chemistry AS + SELECT + 'c-' || o.id AS id, + t.id AS thing_id, + t.name AS station_name, + ST_X(l.point) AS longitude, + ST_Y(l.point) AS latitude, + o.observation_datetime AS datetime, + o.value AS value, + o.unit AS unit, + p.parameter_name AS parameter_name, + o.sample_id AS sample_id, + o.release_status AS release_status + FROM observation o + JOIN parameter p + ON p.id = o.parameter_id AND p.parameter_name <> 'groundwater level' + JOIN sample sm ON sm.id = o.sample_id + JOIN field_activity fa ON fa.id = sm.field_activity_id + JOIN field_event fe ON fe.id = fa.field_event_id + JOIN thing t ON t.id = fe.thing_id + {_EDR_LOCATION_JOIN} + WHERE o.value IS NOT NULL + """ + + +def _recreate_all_internal_views() -> None: + # ogc_internal_actively_monitored_wells depends on + # ogc_internal_water_well_summary via a direct JOIN; Postgres refuses to + # drop a materialized view while a dependent view exists, so it must go + # first and come back last -- same ordering constraint as the public side. + _drop_view_or_materialized_view("ogc_internal_actively_monitored_wells") + + for view_id, thing_type in THING_VIEWS: + _drop_view_or_materialized_view(f"ogc_internal_{_safe_view_id(view_id)}") + op.execute(text(_create_thing_view(view_id, thing_type, public_only=False))) + + _drop_view_or_materialized_view("ogc_internal_latest_depth_to_water_wells") + op.execute(text(_create_latest_depth_view(public_only=False))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_internal_latest_depth_to_water_wells IS " + "'Unfiltered latest depth-to-water per well view for the internal pygeoapi mount.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_internal_latest_depth_to_water_wells_id " + "ON ogc_internal_latest_depth_to_water_wells (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_internal_avg_tds_wells") + op.execute(text(_create_avg_tds_view(public_only=False))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_internal_avg_tds_wells IS " + "'Unfiltered average TDS per well from major chemistry results for the internal pygeoapi mount.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_internal_avg_tds_wells_id " + "ON ogc_internal_avg_tds_wells (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_internal_latest_tds_wells") + op.execute(text(_create_latest_tds_view(public_only=False))) + op.execute( + text( + "COMMENT ON VIEW ogc_internal_latest_tds_wells IS " + "'Unfiltered latest TDS per well from major chemistry results for the internal pygeoapi mount.'" + ) + ) + + _drop_view_or_materialized_view("ogc_internal_depth_to_water_trend_wells") + op.execute(text(_create_depth_to_water_trend_view(public_only=False))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_internal_depth_to_water_trend_wells IS " + "'Unfiltered depth-to-water trend classification for water wells, for the internal pygeoapi mount.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_internal_depth_to_water_trend_wells_id " + "ON ogc_internal_depth_to_water_trend_wells (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_internal_water_well_summary") + op.execute(text(_create_water_well_summary_view(public_only=False))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_internal_water_well_summary IS " + "'Unfiltered summary statistics for water wells including water-level trend, for the internal pygeoapi mount.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_internal_water_well_summary_id " + "ON ogc_internal_water_well_summary (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_internal_major_chemistry_results") + op.execute(text(_create_major_chemistry_results_view(public_only=False))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_internal_major_chemistry_results IS " + "'Unfiltered latest major-chemistry analyte values per location, pivoted into static analyte columns, for the internal pygeoapi mount.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_internal_major_chemistry_results_id " + "ON ogc_internal_major_chemistry_results (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_internal_minor_chemistry_wells") + op.execute(text(_create_minor_chemistry_wells_view(public_only=False))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_internal_minor_chemistry_wells IS " + "'Unfiltered latest minor/trace chemistry analyte values for water wells, pivoted into static analyte columns, for the internal pygeoapi mount.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_internal_minor_chemistry_wells_id " + "ON ogc_internal_minor_chemistry_wells (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_internal_water_elevation_wells") + op.execute(text(_create_water_elevation_view(public_only=False))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_internal_water_elevation_wells IS " + "'Unfiltered latest water elevation per well with explicit units: " + "elevation_m, depth_to_water_below_ground_surface_ft, water_elevation_ft, for the internal pygeoapi mount.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_internal_water_elevation_wells_id " + "ON ogc_internal_water_elevation_wells (id)" + ) + ) + + # Recreate now that ogc_internal_water_well_summary exists again. + op.execute(text(_create_actively_monitored_wells_view())) + op.execute( + text( + "COMMENT ON VIEW ogc_internal_actively_monitored_wells IS " + "'Unfiltered wells in the Water Level Network group, for the internal pygeoapi mount.'" + ) + ) + + _drop_view_or_materialized_view("ogc_internal_project_areas") + op.execute(text(_create_project_areas_view(public_only=False))) + op.execute( + text( + "COMMENT ON VIEW ogc_internal_project_areas IS " + "'Unfiltered project areas for groups with polygon boundaries, for the internal pygeoapi mount.'" + ) + ) + + _drop_view_or_materialized_view("ogc_internal_locations") + op.execute(text(_create_locations_view())) + op.execute( + text( + "COMMENT ON VIEW ogc_internal_locations IS " + "'Unfiltered locations for the internal pygeoapi mount.'" + ) + ) + + _drop_view_or_materialized_view("ogc_internal_waterlevels") + op.execute(text(_create_internal_waterlevels_view())) + op.execute( + text( + "COMMENT ON VIEW ogc_internal_waterlevels IS " + "'Unfiltered depth-to-water readings (manual + transducer) for the internal pygeoapi mount.'" + ) + ) + + _drop_view_or_materialized_view("ogc_internal_water_chemistry") + op.execute(text(_create_internal_water_chemistry_view())) + op.execute( + text( + "COMMENT ON VIEW ogc_internal_water_chemistry IS " + "'Unfiltered water-chemistry analyses (by analyte) for the internal pygeoapi mount.'" + ) + ) + + +# All 24 relations this migration creates, in an order safe for DROP (the +# dependent view first, mirroring _recreate_all_internal_views's ordering). +ALL_INTERNAL_RELATIONS = [ + "ogc_internal_actively_monitored_wells", + *[f"ogc_internal_{view_id}" for view_id, _ in THING_VIEWS], + "ogc_internal_latest_depth_to_water_wells", + "ogc_internal_avg_tds_wells", + "ogc_internal_latest_tds_wells", + "ogc_internal_depth_to_water_trend_wells", + "ogc_internal_water_well_summary", + "ogc_internal_major_chemistry_results", + "ogc_internal_minor_chemistry_wells", + "ogc_internal_water_elevation_wells", + "ogc_internal_project_areas", + "ogc_internal_locations", + "ogc_internal_waterlevels", + "ogc_internal_water_chemistry", +] + + +def upgrade() -> None: + _check_required_tables() + _recreate_all_internal_views() + + +def downgrade() -> None: + # None of these 24 relations existed before this migration -- unlike + # f4a5b6c7d8e9's downgrade (which recreates the prior unfiltered public + # views), there is no prior state to restore, so downgrade just drops + # everything this migration created. + for relation in ALL_INTERNAL_RELATIONS: + _drop_view_or_materialized_view(relation) diff --git a/alembic/versions/986e0eb85ab3_expand_actively_monitored_wells_to_all_.py b/alembic/versions/986e0eb85ab3_expand_actively_monitored_wells_to_all_.py new file mode 100644 index 000000000..0b71d362d --- /dev/null +++ b/alembic/versions/986e0eb85ab3_expand_actively_monitored_wells_to_all_.py @@ -0,0 +1,288 @@ +"""expand actively_monitored_wells to all groups + +Drops the "WHERE group name = 'water level network'" restriction so the view +covers currently-monitored wells in any group, not just one. Public view +adds a group release_status = 'public' check instead, so draft/private +groups don't leak through now that any group can show up. A well in +multiple groups is aggregated into one row (group_ids/group_names/group_types +as arrays) rather than one row per group, so `id` stays unique -- pygeoapi's +id_field: id assumes exactly one row per id for /items/{id} lookups. + +Revision ID: 986e0eb85ab3 +Revises: c3d4e5f6a7b8 +Create Date: 2026-08-20 10:55:25.697907 + +""" + +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import text + +# revision identifiers, used by Alembic. +revision: str = "986e0eb85ab3" +down_revision: Union[str, Sequence[str], None] = "c3d4e5f6a7b8" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _drop_view_or_materialized_view(view_name: str) -> None: + # DROP VIEW IF EXISTS / DROP MATERIALIZED VIEW IF EXISTS only suppress + # "relation does not exist" -- Postgres still raises WrongObjectType if + # the relation exists as the other kind, so the relation's actual kind + # must be checked first rather than trying both blindly. + bind = op.get_bind() + relkind = bind.execute( + text("SELECT relkind FROM pg_class WHERE oid = to_regclass(:name)"), + {"name": view_name}, + ).scalar() + if relkind == "m": + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {view_name}")) + elif relkind == "v": + op.execute(text(f"DROP VIEW IF EXISTS {view_name}")) + + +def _create_actively_monitored_wells_view(all_groups: bool) -> str: + if all_groups: + # Aggregated: one row per well, group_ids/group_names/group_types as + # arrays, so `id` stays unique even when a well belongs to several + # groups. release_status = 'public' is checked on the group row + # itself (mirrors _create_project_areas_view's public_only handling) + # since any group can appear here now, not just one hardcoded one. + # group_thing_association has no unique constraint on + # (group_id, thing_id), so distinct_memberships de-dupes before + # aggregating; all three arrays are ordered by the same group_id key + # so they stay index-aligned with each other (ordering each array by + # its own column, e.g. names alphabetically, would desync them). + return """ + CREATE VIEW ogc_actively_monitored_wells AS + WITH latest_monitoring_status AS ( + SELECT DISTINCT ON (sh.target_id) + sh.target_id AS thing_id, + sh.status_value + FROM status_history AS sh + WHERE + sh.target_table = 'thing' + AND sh.status_type = 'Monitoring Status' + ORDER BY sh.target_id, sh.start_date DESC, sh.id DESC + ), + distinct_memberships AS ( + SELECT DISTINCT + gta.thing_id, + g.id AS group_id, + g.name AS group_name, + g.group_type + FROM group_thing_association AS gta + JOIN "group" AS g ON g.id = gta.group_id + WHERE g.release_status = 'public' + ) + SELECT + wws.id, + wws.name, + 'water well'::text AS thing_type, + wws.well_depth, + wws.elevation, + wws.elevation_method, + wws.formation_zone, + wws.total_water_levels, + wws.last_water_level, + wws.last_water_level_datetime, + wws.min_water_level, + wws.max_water_level, + wws.water_level_trend_ft_per_year, + array_agg(dm.group_id ORDER BY dm.group_id) AS group_ids, + array_agg(dm.group_name ORDER BY dm.group_id) AS group_names, + array_agg(dm.group_type ORDER BY dm.group_id) AS group_types, + wws.point + FROM ogc_water_well_summary AS wws + JOIN latest_monitoring_status AS lms ON lms.thing_id = wws.id + JOIN distinct_memberships AS dm ON dm.thing_id = wws.id + WHERE lms.status_value = 'Currently monitored' + GROUP BY + wws.id, wws.name, wws.well_depth, wws.elevation, + wws.elevation_method, wws.formation_zone, + wws.total_water_levels, wws.last_water_level, + wws.last_water_level_datetime, wws.min_water_level, + wws.max_water_level, wws.water_level_trend_ft_per_year, + wws.point + """ + # Historical (downgrade target): byte-for-byte the pre-fix view, single + # group_id/group_name/group_type columns, scoped to one hardcoded group. + return """ + CREATE VIEW ogc_actively_monitored_wells AS + WITH latest_monitoring_status AS ( + SELECT DISTINCT ON (sh.target_id) + sh.target_id AS thing_id, + sh.status_value + FROM status_history AS sh + WHERE + sh.target_table = 'thing' + AND sh.status_type = 'Monitoring Status' + ORDER BY sh.target_id, sh.start_date DESC, sh.id DESC + ) + SELECT + wws.id, + wws.name, + 'water well'::text AS thing_type, + wws.well_depth, + wws.elevation, + wws.elevation_method, + wws.formation_zone, + wws.total_water_levels, + wws.last_water_level, + wws.last_water_level_datetime, + wws.min_water_level, + wws.max_water_level, + wws.water_level_trend_ft_per_year, + g.id AS group_id, + g.name AS group_name, + g.group_type, + wws.point + FROM "group" AS g + JOIN group_thing_association AS gta ON gta.group_id = g.id + JOIN ogc_water_well_summary AS wws ON wws.id = gta.thing_id + JOIN latest_monitoring_status AS lms ON lms.thing_id = wws.id + WHERE lower(trim(g.name)) = 'water level network' + AND lms.status_value = 'Currently monitored' + """ + + +def _create_internal_actively_monitored_wells_view(all_groups: bool) -> str: + if all_groups: + # Aggregated, same shape as the public view's all_groups branch, but + # no release_status filter -- the internal mount is unfiltered by + # design, same as its sibling views. See the public branch's comment + # for why distinct_memberships + a shared ORDER BY key is needed. + return """ + CREATE VIEW ogc_internal_actively_monitored_wells AS + WITH latest_monitoring_status AS ( + SELECT DISTINCT ON (sh.target_id) + sh.target_id AS thing_id, + sh.status_value + FROM status_history AS sh + WHERE + sh.target_table = 'thing' + AND sh.status_type = 'Monitoring Status' + ORDER BY sh.target_id, sh.start_date DESC, sh.id DESC + ), + distinct_memberships AS ( + SELECT DISTINCT + gta.thing_id, + g.id AS group_id, + g.name AS group_name, + g.group_type + FROM group_thing_association AS gta + JOIN "group" AS g ON g.id = gta.group_id + ) + SELECT + wws.id, + wws.name, + 'water well'::text AS thing_type, + wws.well_depth, + wws.elevation, + wws.elevation_method, + wws.formation_zone, + wws.total_water_levels, + wws.last_water_level, + wws.last_water_level_datetime, + wws.min_water_level, + wws.max_water_level, + wws.water_level_trend_ft_per_year, + array_agg(dm.group_id ORDER BY dm.group_id) AS group_ids, + array_agg(dm.group_name ORDER BY dm.group_id) AS group_names, + array_agg(dm.group_type ORDER BY dm.group_id) AS group_types, + wws.point + FROM ogc_internal_water_well_summary AS wws + JOIN latest_monitoring_status AS lms ON lms.thing_id = wws.id + JOIN distinct_memberships AS dm ON dm.thing_id = wws.id + WHERE lms.status_value = 'Currently monitored' + GROUP BY + wws.id, wws.name, wws.well_depth, wws.elevation, + wws.elevation_method, wws.formation_zone, + wws.total_water_levels, wws.last_water_level, + wws.last_water_level_datetime, wws.min_water_level, + wws.max_water_level, wws.water_level_trend_ft_per_year, + wws.point + """ + # Historical (downgrade target): byte-for-byte the pre-fix view. + return """ + CREATE VIEW ogc_internal_actively_monitored_wells AS + WITH latest_monitoring_status AS ( + SELECT DISTINCT ON (sh.target_id) + sh.target_id AS thing_id, + sh.status_value + FROM status_history AS sh + WHERE + sh.target_table = 'thing' + AND sh.status_type = 'Monitoring Status' + ORDER BY sh.target_id, sh.start_date DESC, sh.id DESC + ) + SELECT + wws.id, + wws.name, + 'water well'::text AS thing_type, + wws.well_depth, + wws.elevation, + wws.elevation_method, + wws.formation_zone, + wws.total_water_levels, + wws.last_water_level, + wws.last_water_level_datetime, + wws.min_water_level, + wws.max_water_level, + wws.water_level_trend_ft_per_year, + g.id AS group_id, + g.name AS group_name, + g.group_type, + wws.point + FROM "group" AS g + JOIN group_thing_association AS gta ON gta.group_id = g.id + JOIN ogc_internal_water_well_summary AS wws ON wws.id = gta.thing_id + JOIN latest_monitoring_status AS lms ON lms.thing_id = wws.id + WHERE lower(trim(g.name)) = 'water level network' + AND lms.status_value = 'Currently monitored' + """ + + +def upgrade() -> None: + """Upgrade schema.""" + _drop_view_or_materialized_view("ogc_actively_monitored_wells") + op.execute(text(_create_actively_monitored_wells_view(all_groups=True))) + op.execute( + text( + "COMMENT ON VIEW ogc_actively_monitored_wells IS " + "'Actively (currently) monitored wells across all groups for pygeoapi.'" + ) + ) + + _drop_view_or_materialized_view("ogc_internal_actively_monitored_wells") + op.execute(text(_create_internal_actively_monitored_wells_view(all_groups=True))) + op.execute( + text( + "COMMENT ON VIEW ogc_internal_actively_monitored_wells IS " + "'Actively (currently) monitored wells across all groups, " + "for the internal pygeoapi mount.'" + ) + ) + + +def downgrade() -> None: + """Downgrade schema.""" + _drop_view_or_materialized_view("ogc_actively_monitored_wells") + op.execute(text(_create_actively_monitored_wells_view(all_groups=False))) + op.execute( + text( + "COMMENT ON VIEW ogc_actively_monitored_wells IS " + "'Wells in the Water Level Network group for pygeoapi.'" + ) + ) + + _drop_view_or_materialized_view("ogc_internal_actively_monitored_wells") + op.execute(text(_create_internal_actively_monitored_wells_view(all_groups=False))) + op.execute( + text( + "COMMENT ON VIEW ogc_internal_actively_monitored_wells IS " + "'Unfiltered wells in the Water Level Network group, " + "for the internal pygeoapi mount.'" + ) + ) diff --git a/alembic/versions/a1b2c3d4e5f6_unique_transducer_observation.py b/alembic/versions/a1b2c3d4e5f6_unique_transducer_observation.py new file mode 100644 index 000000000..8b70eb7bc --- /dev/null +++ b/alembic/versions/a1b2c3d4e5f6_unique_transducer_observation.py @@ -0,0 +1,48 @@ +"""unique constraint on transducer_observation + +Revision ID: a1b2c3d4e5f6 +Revises: d9e0f1a2b3c4 +Create Date: 2026-08-19 + +The table had only an index on (deployment_id, parameter_id, +observation_datetime), so nothing prevented the same reading being inserted +twice. That absence is what forces a delete-then-repost load strategy: without a +constraint to conflict on, a re-run can only avoid duplicates by removing what +is already there first, which leaves a window where the data is missing. + +With this constraint the loader can use ON CONFLICT DO UPDATE and a re-run +becomes idempotent, so a backfill overlapping existing data is safe. + +Note the constraint is on `deployment_id`, not `thing_id` -- the plan named a +column this table does not have. A deployment is a thing/sensor pairing, so two +sensors on the same well may legitimately report the same instant; scoping +uniqueness to the deployment allows that while still catching a re-inserted row. + +**Run automated_ingestion/sql/find_duplicate_observations.sql first.** This +migration fails on a table that already violates the constraint, and it is +better to know that before starting than halfway through. +""" + +from alembic import op + +revision = "a1b2c3d4e5f6" +down_revision = "d9e0f1a2b3c4" +branch_labels = None +depends_on = None + +CONSTRAINT_NAME = "uq_transducer_observation_deployment_parameter_datetime" +INDEX_NAME = "ix_transducer_observation_deployment_parameter_datetime" +COLUMNS = ["deployment_id", "parameter_id", "observation_datetime"] + + +def upgrade() -> None: + # The unique constraint creates its own index on the same columns, so the + # existing one would be redundant -- two indexes maintained on every insert + # into the largest table in the schema. + op.drop_index(INDEX_NAME, table_name="transducer_observation") + op.create_unique_constraint(CONSTRAINT_NAME, "transducer_observation", COLUMNS) + + +def downgrade() -> None: + op.drop_constraint(CONSTRAINT_NAME, "transducer_observation", type_="unique") + op.create_index(INDEX_NAME, "transducer_observation", COLUMNS) diff --git a/alembic/versions/b2c3d4e5f6a7_transducer_data_maturity.py b/alembic/versions/b2c3d4e5f6a7_transducer_data_maturity.py new file mode 100644 index 000000000..3037e3cd0 --- /dev/null +++ b/alembic/versions/b2c3d4e5f6a7_transducer_data_maturity.py @@ -0,0 +1,122 @@ +"""data_maturity on transducer_observation + +Revision ID: b2c3d4e5f6a7 +Revises: a1b2c3d4e5f6 +Create Date: 2026-08-19 + +`release_status` is one column whose lexicon lists `public` and `provisional` as +siblings, so a reading cannot be both visible and marked unreviewed. Those are +orthogonal: visibility is who may see it, maturity is how much it should be +trusted. This adds the second axis. + +Terms follow USGS usage. `provisional` and `approved` are what USGS publishes +against -- "provisional data subject to revision" is the standard caveat on +unapproved records. `in review` is the intermediate state from the Aquarius +approval levels USGS uses for continuous time series (Working / In Review / +Approved); Aquarius' `Working` is folded into `provisional` because the two are +indistinguishable to a consumer. + +Existing rows are backfilled from the legacy AMPAPI QC flag, +`nma_waterlevelscontinuous_pressure_qced`, which records exactly this: whether a +reading has been quality controlled. True becomes `approved`, false becomes +`provisional`. + +Rows where that flag is NULL stay NULL. Those did not come from the NMA +transducer tables, so there is no evidence either way, and NULL reads as "not +stated" -- which is true, where guessing would not be. +""" + +import sqlalchemy as sa +from alembic import op + +revision = "b2c3d4e5f6a7" +down_revision = "a1b2c3d4e5f6" +branch_labels = None +depends_on = None + +CATEGORY = "data_maturity" +TERMS = ("provisional", "in review", "approved") + + +def upgrade() -> None: + connection = op.get_bind() + + # `lexicon_term.term` is globally unique and categories share terms through + # an association table, so `provisional` and `approved` already exist from + # `release_status` and `review_status`. Only the intermediate state is new. + connection.execute( + sa.text( + "INSERT INTO lexicon_term (term, definition) VALUES (:term, :definition) " + "ON CONFLICT (term) DO NOTHING" + ), + { + "term": "in review", + "definition": ( + "Under review and not yet approved. Intermediate state from the " + "USGS Aquarius approval levels used for continuous records." + ), + }, + ) + connection.execute( + sa.text( + "INSERT INTO lexicon_category (name) VALUES (:name) " + "ON CONFLICT (name) DO NOTHING" + ), + {"name": CATEGORY}, + ) + connection.execute( + sa.text(""" + INSERT INTO lexicon_term_category_association (term_id, category_id) + SELECT t.id, c.id + FROM lexicon_term t, lexicon_category c + WHERE t.term = ANY(:terms) AND c.name = :category + ON CONFLICT DO NOTHING + """), + {"terms": list(TERMS), "category": CATEGORY}, + ) + + op.add_column( + "transducer_observation", + sa.Column( + "data_maturity", + sa.String(length=100), + nullable=True, + comment=( + "How far through review this reading is. Orthogonal to " + "release_status, which controls visibility. NULL means not stated." + ), + ), + ) + op.create_foreign_key( + "fk_transducer_observation_data_maturity", + "transducer_observation", + "lexicon_term", + ["data_maturity"], + ["term"], + onupdate="CASCADE", + ) + + # The legacy QC flag answers this question directly, so the maturity of + # historical rows is a lookup rather than a guess. Done after the foreign + # key so a bad value here would fail loudly rather than persist. + connection.execute(sa.text(""" + UPDATE transducer_observation + SET data_maturity = CASE + WHEN nma_waterlevelscontinuous_pressure_qced THEN 'approved' + ELSE 'provisional' + END + WHERE nma_waterlevelscontinuous_pressure_qced IS NOT NULL + """)) + + +def downgrade() -> None: + op.drop_constraint( + "fk_transducer_observation_data_maturity", + "transducer_observation", + type_="foreignkey", + ) + op.drop_column("transducer_observation", "data_maturity") + + # The terms are left in place. They may have been adopted elsewhere by the + # time this is reversed, and an unused lexicon term is harmless where a + # missing one breaks a foreign key. diff --git a/alembic/versions/b7c8d9e0f1a2_repair_missing_edr_water_views.py b/alembic/versions/b7c8d9e0f1a2_repair_missing_edr_water_views.py new file mode 100644 index 000000000..572c14b89 --- /dev/null +++ b/alembic/versions/b7c8d9e0f1a2_repair_missing_edr_water_views.py @@ -0,0 +1,126 @@ +"""repair missing EDR water views + +Recreates ogc_waterlevels / ogc_water_chemistry on any database whose +alembic_version claims z9a0b1c2d3e4 was applied while the views are in fact +absent. + +Why this is needed: the CD run that first carried z9a0b1c2d3e4 to staging +failed in the Alembic step with "Multiple head revisions are present for given +argument 'head'". The revision graph was then repaired in-tree (eb89d046, +8ae9fe18), but the staging database came out the other side stamped past +z9a0b1c2d3e4 without its DDL ever having executed. Downstream revisions applied +normally, so nothing surfaced until an EDR query hit the missing relation: + + psycopg2.errors.UndefinedTable: relation "ogc_waterlevels" does not exist + +Re-running z9a0b1c2d3e4 is not an option -- alembic_version already lists it, +and downgrading to it would tear out every revision since. This revision closes +the hole from the front of the chain instead. + +The view SQL is imported from z9a0b1c2d3e4 rather than copied so the repaired +definition cannot drift from the definition of record. + +Idempotent and safe on healthy databases: a view that is already present is +left untouched, so this is a no-op everywhere except the environments that +actually skipped the original revision. + +Revision ID: b7c8d9e0f1a2 +Revises: f3a1c2b4d5e6 +Create Date: 2026-08-13 13:20:00.000000 +""" + +import importlib.util +from pathlib import Path +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "b7c8d9e0f1a2" +down_revision: Union[str, Sequence[str], None] = "f3a1c2b4d5e6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_SOURCE_REVISION = "z9a0b1c2d3e4_add_edr_water_views.py" + + +def _load_source_revision(): + # The view definitions live in z9a0b1c2d3e4. Importing them keeps this + # repair honest: whatever that revision creates is exactly what a database + # that skipped it gets back. + path = Path(__file__).with_name(_SOURCE_REVISION) + if not path.exists(): + raise RuntimeError( + f"Cannot repair the EDR water views: {_SOURCE_REVISION} is missing " + "from alembic/versions, so the view definitions of record are " + "unavailable." + ) + spec = importlib.util.spec_from_file_location("_edr_water_views", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +VIEW_COMMENTS = { + "ogc_waterlevels": ( + "Public depth-to-water readings (manual + transducer) for EDR." + ), + "ogc_water_chemistry": "Public water-chemistry analyses (by analyte) for EDR.", +} + + +def _relkind(view_name: str) -> str | None: + bind = op.get_bind() + return bind.execute( + text("SELECT relkind FROM pg_class WHERE oid = to_regclass(:name)"), + {"name": view_name}, + ).scalar() + + +def _check_required_tables(required_tables: set[str]) -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing = set(inspector.get_table_names(schema="public")) + missing = required_tables - existing + if missing: + raise RuntimeError( + "Cannot repair the EDR water views. Missing required tables: " + f"{sorted(missing)}" + ) + + +def _repair_view(view_name: str, create_sql: str) -> None: + relkind = _relkind(view_name) + if relkind == "v": + # Already present and the right kind -- the database applied + # z9a0b1c2d3e4 for real. Leave it alone rather than churning DDL that + # other objects may depend on. + return + if relkind is not None: + # Present as something other than a plain view (materialized view, + # table). That is not a state z9a0b1c2d3e4 or its downstream revisions + # produce, so fail loudly instead of silently replacing it. + raise RuntimeError( + f"Cannot repair {view_name}: it already exists with relkind " + f"{relkind!r}, not a plain view. Inspect it by hand before " + "re-running this migration." + ) + + op.execute(text(create_sql)) + op.execute(text(f"COMMENT ON VIEW {view_name} IS '{VIEW_COMMENTS[view_name]}'")) + + +def upgrade() -> None: + source = _load_source_revision() + _check_required_tables(set(source.REQUIRED_TABLES)) + + _repair_view("ogc_waterlevels", source._create_waterlevels_view()) + _repair_view("ogc_water_chemistry", source._create_water_chemistry_view()) + + +def downgrade() -> None: + # Deliberately a no-op. These views belong to z9a0b1c2d3e4; dropping them + # here would break EDR on every database that applied that revision + # correctly. Downgrading past z9a0b1c2d3e4 removes them. + pass diff --git a/alembic/versions/b8c9d0e1f2a3_add_last_observation_date_to_thing_views.py b/alembic/versions/b8c9d0e1f2a3_add_last_observation_date_to_thing_views.py new file mode 100644 index 000000000..0c04fb2b2 --- /dev/null +++ b/alembic/versions/b8c9d0e1f2a3_add_last_observation_date_to_thing_views.py @@ -0,0 +1,239 @@ +"""add last_observation_date to the Group A thing views + +Ticket A13. The 11 thing-type layers (Group A) carry construction and location +detail but no signal of data recency: a consumer could not tell a well measured +last month from one last visited in 1994 without querying a second layer. + +This adds `last_observation_date` to the shared thing-view template -- the date +of the most recent observation recorded against the thing, or NULL where the +thing has no observations at all. All 11 public views and their 11 +`ogc_internal_` counterparts are rebuilt from the same template here, so the +two mounts stay column-for-column identical. + +Scope of "observation": rows in the `observation` table, reached through the +sample -> field_activity -> field_event chain that every other observation- +backed view in this schema uses. Continuous transducer readings +(`transducer_observation`) are deliberately *not* folded in: they live on a +different chain (deployment -> thing), they exist for a handful of instrumented +water wells rather than for Group A generally, and a max() over the largest +table in the schema would need its own index on +(deployment_id, observation_datetime) to stay cheap. Wells with logger data are +served by ogc_actively_monitored_wells and the water-elevation layers. If +Group A currency should later include instrument readings, that is a separate +ticket and a separate index. + +The date is the UTC calendar date of the observation timestamp -- same +convention as transducer_daily_data (v0w1x2y3z4a5) -- rather than a +session-timezone cast, so the value does not depend on who is querying. + +Public views count only observations with release_status='public', matching how +the public mount filters everything else; the internal views count all of them. +A public well whose only observations are private therefore reads NULL on +/ogcapi and carries a date on /ogcapi-internal. + +Per-thing lookup is a LEFT JOIN LATERAL rather than a grouped CTE so that a +paginated or single-feature request touches only the observations of the rows +it returns. That path had no indexes at all (Postgres does not index foreign +keys on its own), so the four it needs are created here. + +The view bodies below are otherwise character-for-character the templates from +f4a5b6c7d8e9 (public) and 2d3c3a268652 (internal); downgrade() restores them. + +Revision ID: b8c9d0e1f2a3 +Revises: 986e0eb85ab3 +Create Date: 2026-08-24 00:00:00.000000 +""" + +import re +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "b8c9d0e1f2a3" +down_revision: Union[str, Sequence[str], None] = "baba91fe5e83" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +REQUIRED_TABLES = { + "thing", + "location", + "location_thing_association", + "observation", + "sample", + "field_activity", + "field_event", +} + +LATEST_LOCATION_CTE = """ +SELECT DISTINCT ON (lta.thing_id) + lta.thing_id, + lta.location_id, + lta.effective_start +FROM location_thing_association AS lta +WHERE lta.effective_end IS NULL +ORDER BY lta.thing_id, lta.effective_start DESC +""".strip() + +# Same 11 thing-type views as f4a5b6c7d8e9's THING_VIEWS. +THING_VIEWS = [ + ("water_wells", "water well"), + ("springs", "spring"), + ("diversions_surface_water", "diversion of surface water, etc."), + ("ephemeral_streams", "ephemeral stream"), + ("lakes_ponds_reservoirs", "lake, pond or reservoir"), + ("meteorological_stations", "meteorological station"), + ("other_things", "other"), + ("outfalls_wastewater_return_flow", "outfall of wastewater or return flow"), + ("perennial_streams", "perennial stream"), + ("rock_sample_locations", "rock sample location"), + ("soil_gas_sample_locations", "soil gas sample location"), +] + +# (name, table, columns) for the observation chain the lateral walks +# thing -> field_event -> field_activity -> sample -> observation. +SUPPORTING_INDEXES = [ + ("ix_field_event_thing_id", "field_event", "thing_id"), + ("ix_field_activity_field_event_id", "field_activity", "field_event_id"), + ("ix_sample_field_activity_id", "sample", "field_activity_id"), + ( + "ix_observation_sample_id_observation_datetime", + "observation", + "sample_id, observation_datetime", + ), +] + + +def _safe_view_id(view_id: str) -> str: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", view_id): + raise ValueError(f"Unsafe view id: {view_id!r}") + return view_id + + +def _check_required_tables() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing_tables = set(inspector.get_table_names(schema="public")) + missing = REQUIRED_TABLES - existing_tables + if missing: + raise RuntimeError( + "Cannot add last_observation_date to the OGC thing views. " + f"Missing required tables: {', '.join(sorted(missing))}" + ) + + +def _create_thing_view( + view_id: str, thing_type: str, public_only: bool, table_prefix: str +) -> str: + """The Group A view template, with last_observation_date.""" + safe_view_id = _safe_view_id(f"{table_prefix}{view_id}") + escaped_thing_type = thing_type.replace("'", "''") + release_filter = " AND t.release_status = 'public'" if public_only else "" + observation_release_filter = ( + "\n AND o.release_status = 'public'" if public_only else "" + ) + return f""" + CREATE VIEW {safe_view_id} AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ) + SELECT + t.id, + t.name, + t.first_visit_date, + ( + last_obs.last_observation_datetime AT TIME ZONE 'UTC' + )::date AS last_observation_date, + t.nma_pk_welldata, + t.well_depth, + t.hole_depth, + t.well_casing_diameter, + t.well_casing_depth, + t.well_completion_date, + t.well_driller_name, + t.well_construction_method, + t.well_pump_type, + t.well_pump_depth, + t.formation_completion_code, + t.nma_formation_zone, + t.release_status, + l.elevation, + l.point + FROM thing AS t + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + LEFT JOIN LATERAL ( + SELECT MAX(o.observation_datetime) AS last_observation_datetime + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + WHERE fe.thing_id = t.id{observation_release_filter} + ) AS last_obs ON TRUE + WHERE t.thing_type = '{escaped_thing_type}'{release_filter} + """ + + +def _create_thing_view_pre_a13( + view_id: str, thing_type: str, public_only: bool, table_prefix: str +) -> str: + """The template as it stood in f4a5b6c7d8e9/2d3c3a268652, for downgrade.""" + safe_view_id = _safe_view_id(f"{table_prefix}{view_id}") + escaped_thing_type = thing_type.replace("'", "''") + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE VIEW {safe_view_id} AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ) + SELECT + t.id, + t.name, + t.first_visit_date, + t.nma_pk_welldata, + t.well_depth, + t.hole_depth, + t.well_casing_diameter, + t.well_casing_depth, + t.well_completion_date, + t.well_driller_name, + t.well_construction_method, + t.well_pump_type, + t.well_pump_depth, + t.formation_completion_code, + t.nma_formation_zone, + t.release_status, + l.elevation, + l.point + FROM thing AS t + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE t.thing_type = '{escaped_thing_type}'{release_filter} + """ + + +def _rebuild_thing_views(builder) -> None: + for table_prefix, public_only in (("ogc_", True), ("ogc_internal_", False)): + for view_id, thing_type in THING_VIEWS: + view_name = _safe_view_id(f"{table_prefix}{view_id}") + op.execute(text(f"DROP VIEW IF EXISTS {view_name}")) + op.execute(text(builder(view_id, thing_type, public_only, table_prefix))) + + +def upgrade() -> None: + _check_required_tables() + + for index_name, table_name, columns in SUPPORTING_INDEXES: + op.execute( + text(f"CREATE INDEX IF NOT EXISTS {index_name} ON {table_name} ({columns})") + ) + + _rebuild_thing_views(_create_thing_view) + + +def downgrade() -> None: + _rebuild_thing_views(_create_thing_view_pre_a13) + + for index_name, _table_name, _columns in SUPPORTING_INDEXES: + op.execute(text(f"DROP INDEX IF EXISTS {index_name}")) diff --git a/alembic/versions/baba91fe5e83_gate_ogc_waterlevels_on_thing_release.py b/alembic/versions/baba91fe5e83_gate_ogc_waterlevels_on_thing_release.py new file mode 100644 index 000000000..2eee5c2a5 --- /dev/null +++ b/alembic/versions/baba91fe5e83_gate_ogc_waterlevels_on_thing_release.py @@ -0,0 +1,122 @@ +"""gate ogc_waterlevels on the thing's release status + +ogc_waterlevels filtered on the reading's own release_status only, so a well +whose release_status is 'draft' or 'private' still published its public +readings through OGC API - EDR -- with the well's name and coordinates +attached. ogc_water_chemistry (d9e0f1a2b3c4) already required the parent thing +to be public; this brings water levels onto the same rule. + +The internal mirror, ogc_internal_waterlevels, is deliberately left alone: it +carries non-public records by design for authenticated staff clients, the same +way ogc_internal_water_chemistry does. + +Revision ID: baba91fe5e83 +Revises: 986e0eb85ab3 +Create Date: 2026-08-22 18:35:00.000000 + +""" + +import importlib.util +from pathlib import Path +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import text + +# revision identifiers, used by Alembic. +revision: str = "baba91fe5e83" +down_revision: Union[str, Sequence[str], None] = "986e0eb85ab3" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_ORIGINAL_REVISION = "z9a0b1c2d3e4_add_edr_water_views.py" + +# Shared join from a thing to its current location point. Mirrors the join in +# z9a0b1c2d3e4, which is the definition of record for this view. +_LOCATION_JOIN = """ + JOIN location_thing_association lta + ON lta.thing_id = t.id AND lta.effective_end IS NULL + JOIN location l ON l.id = lta.location_id +""" + + +def _load_original_module(): + """Import z9a0b1c2d3e4 so downgrade restores its SQL rather than a copy.""" + path = Path(__file__).with_name(_ORIGINAL_REVISION) + if not path.exists(): + raise RuntimeError( + "Cannot restore the previous ogc_waterlevels definition: " + f"{_ORIGINAL_REVISION} is missing from alembic/versions." + ) + spec = importlib.util.spec_from_file_location("_edr_water_views", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _create_waterlevels_view() -> str: + return f""" + CREATE VIEW ogc_waterlevels AS + -- manual water-level readings + SELECT + 'm-' || o.id AS id, + t.id AS thing_id, + t.name AS station_name, + ST_X(l.point) AS longitude, + ST_Y(l.point) AS latitude, + o.observation_datetime AS datetime, + o.value AS value, + o.unit AS unit, + 'groundwater level' AS parameter_name, + 'manual' AS source, + NULL::integer AS deployment_id, + o.release_status AS release_status + FROM observation o + JOIN parameter p + ON p.id = o.parameter_id AND p.parameter_name = 'groundwater level' + JOIN sample sm ON sm.id = o.sample_id + JOIN field_activity fa ON fa.id = sm.field_activity_id + JOIN field_event fe ON fe.id = fa.field_event_id + JOIN thing t ON t.id = fe.thing_id + {_LOCATION_JOIN} + WHERE o.release_status = 'public' + AND t.release_status = 'public' + AND o.value IS NOT NULL + + UNION ALL + + -- transducer (instrument) water-level readings + SELECT + 't-' || tobs.id AS id, + t.id AS thing_id, + t.name AS station_name, + ST_X(l.point) AS longitude, + ST_Y(l.point) AS latitude, + tobs.observation_datetime AS datetime, + tobs.value AS value, + p.default_unit AS unit, + 'groundwater level' AS parameter_name, + 'transducer' AS source, + tobs.deployment_id AS deployment_id, + tobs.release_status AS release_status + FROM transducer_observation tobs + JOIN parameter p + ON p.id = tobs.parameter_id AND p.parameter_name = 'groundwater level' + JOIN deployment d ON d.id = tobs.deployment_id + JOIN thing t ON t.id = d.thing_id + {_LOCATION_JOIN} + WHERE tobs.release_status = 'public' + AND t.release_status = 'public' + AND tobs.value IS NOT NULL + """ + + +def upgrade() -> None: + op.execute(text("DROP VIEW IF EXISTS ogc_waterlevels")) + op.execute(text(_create_waterlevels_view())) + + +def downgrade() -> None: + original = _load_original_module() + op.execute(text("DROP VIEW IF EXISTS ogc_waterlevels")) + op.execute(text(original._create_waterlevels_view())) diff --git a/alembic/versions/c3d4e5f6a7b8_hydrograph_correction_publish.py b/alembic/versions/c3d4e5f6a7b8_hydrograph_correction_publish.py new file mode 100644 index 000000000..48e6bebdf --- /dev/null +++ b/alembic/versions/c3d4e5f6a7b8_hydrograph_correction_publish.py @@ -0,0 +1,130 @@ +"""publish provenance for corrected transducer blocks + +Revision ID: c3d4e5f6a7b8 +Revises: b2c3d4e5f6a7 +Create Date: 2026-08-19 + +The hydrograph corrector publishes a *derived* series: water head converted to +depth below ground surface against manual anchors, then shifted, snapped, and +drift-corrected. None of those numbers are what the instrument recorded, so the +database has to carry enough to tell a reviewer what happened to them. + +Three columns on the block cover the batch: the file it came from, whether that +file held water head or depth to water, and the ordered list of corrections +applied. `comment` already exists and takes the publisher's free-text note. + +One column on the observation covers the row: `note`, set only on readings a +correction actually moved. NULL therefore means "as measured", which is the +distinction review needs. The legacy `nma_waterlevelscontinuous_*_notes` +columns cannot serve -- each is scoped to one legacy source table. + +The block time-order check is relaxed from `>` to `>=`. A block spanning a +single instant is legitimate: a published file with one reading, or a block +narrowed by a range delete until one observation survives. The block reader +matches observations inclusively on both bounds, so a zero-width block still +covers its reading. Loosening a check constraint cannot invalidate existing +rows. + +That check also gets its name spelled right on the way through. It was created +as `check_transuder_block_time_order` -- no `c` -- and since Postgres cannot +alter a check in place, the drop-and-recreate this migration already performs +is the free moment to fix it. The old name is dropped and the new one created; +no separate RENAME is needed. +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "c3d4e5f6a7b8" +down_revision = "b2c3d4e5f6a7" +branch_labels = None +depends_on = None + +# The name as created by the initial migration, misspelled. Drops and +# downgrades have to use it verbatim: `op.drop_constraint` matches on the name +# in the live database, so correcting the spelling anywhere it is used to +# *find* the constraint would make the statement a no-op target and fail. +LEGACY_TIME_ORDER_CONSTRAINT = "check_transuder_block_time_order" + +# What it is called from this migration forward. +TIME_ORDER_CONSTRAINT = "check_transducer_block_time_order" + + +def upgrade() -> None: + op.add_column( + "transducer_observation_block", + sa.Column( + "source_file", + sa.String(length=255), + nullable=True, + comment="Name of the logger file the corrected series was derived from", + ), + ) + op.add_column( + "transducer_observation_block", + sa.Column( + "source_kind", + sa.String(length=50), + nullable=True, + comment="What the source file measured: water_head or depth_to_water", + ), + ) + op.add_column( + "transducer_observation_block", + sa.Column( + "corrections", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="Corrections applied to the source series, in applied order", + ), + ) + op.add_column( + "transducer_observation", + sa.Column( + "note", + sa.Text(), + nullable=True, + comment=( + "Per-reading correction annotation; NULL means the value is as " + "measured" + ), + ), + ) + + # Dropped under the old name, recreated under the new one: the rename and + # the relaxation are the same statement pair, so there is no window where + # the table is unconstrained beyond the one this already needs. + op.drop_constraint( + LEGACY_TIME_ORDER_CONSTRAINT, "transducer_observation_block", type_="check" + ) + op.create_check_constraint( + TIME_ORDER_CONSTRAINT, + "transducer_observation_block", + "end_datetime >= start_datetime", + ) + + +def downgrade() -> None: + # Zero-width blocks may have been created while the loosened constraint was + # in force, so widen them by a second rather than let the stricter + # constraint fail to validate. A one-second span on a block that covered an + # instant is a smaller lie than a failed downgrade. + op.execute( + "UPDATE transducer_observation_block " + "SET end_datetime = start_datetime + interval '1 second' " + "WHERE end_datetime = start_datetime" + ) + op.drop_constraint( + TIME_ORDER_CONSTRAINT, "transducer_observation_block", type_="check" + ) + op.create_check_constraint( + LEGACY_TIME_ORDER_CONSTRAINT, + "transducer_observation_block", + "end_datetime > start_datetime", + ) + + op.drop_column("transducer_observation", "note") + op.drop_column("transducer_observation_block", "corrections") + op.drop_column("transducer_observation_block", "source_kind") + op.drop_column("transducer_observation_block", "source_file") diff --git a/alembic/versions/c9d0e1f2a3b4_add_well_water_column_ogc_views.py b/alembic/versions/c9d0e1f2a3b4_add_well_water_column_ogc_views.py new file mode 100644 index 000000000..315146f31 --- /dev/null +++ b/alembic/versions/c9d0e1f2a3b4_add_well_water_column_ogc_views.py @@ -0,0 +1,212 @@ +"""add the well water-column OGC layer + +A water well's construction record says how deep the hole goes; its +groundwater-level record says how far down the water sits. The difference -- +the standing column of water inside the well -- is the number that says whether +a well still has usable water in it, and nothing in the catalogue published it. + +This creates ogc_well_water_column (public) and ogc_internal_well_water_column +(unfiltered), one row per water well, carrying the same well and location +fields the water_wells layer publishes plus four derived depths, all in feet: + + water_column_latest well depth minus the most recent depth to water + water_column_average well depth minus the mean depth to water + water_column_maximum well depth minus the shallowest depth to water + water_column_minimum well depth minus the deepest depth to water + +Shallowest water gives the largest column and deepest water the smallest, hence +the maximum/minimum naming: these are the extremes of the water column itself, +not of the readings behind them. + +Readings are manual groundwater-level observations, taken below ground surface +as (value - measuring_point_height) with a missing height treated as ground +level -- the same convention as ogc_water_well_summary and +ogc_latest_depth_to_water_wells, so the three layers cannot disagree about what +a depth to water is. Continuous transducer readings are not included. + +Negative results are clamped to zero. A reading deeper than the recorded well +depth is a contradiction between two records rather than a well holding +negative water, and the clamp keeps consumers from having to special-case it; +the contradiction itself stays visible in water_well_summary, which publishes +the raw shallowest and deepest readings next to the well depth. + +Rows are restricted to wells that have both a well depth and at least one +usable reading -- without either, all four columns would be NULL and the row +would say nothing. + +Materialized, because every column but the latest one aggregates a well's +entire reading history. The nightly pg_cron job (b6c7d8e9f0a1) refreshes every +matview in the public schema by name, so these two are picked up with no change +to the schedule. Both carry a unique index on id so the refresh can also be run +CONCURRENTLY by hand (`oco refresh-matview --concurrently`). + +Revision ID: c9d0e1f2a3b4 +Revises: b8c9d0e1f2a3 +Create Date: 2026-08-24 00:00:00.000000 +""" + +import re +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "c9d0e1f2a3b4" +down_revision: Union[str, Sequence[str], None] = "b8c9d0e1f2a3" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +REQUIRED_TABLES = { + "thing", + "location", + "location_thing_association", + "observation", + "sample", + "field_activity", + "field_event", +} + +LATEST_LOCATION_CTE = """ +SELECT DISTINCT ON (lta.thing_id) + lta.thing_id, + lta.location_id, + lta.effective_start +FROM location_thing_association AS lta +WHERE lta.effective_end IS NULL +ORDER BY lta.thing_id, lta.effective_start DESC +""".strip() + +VIEWS = [ + ("ogc_well_water_column", True), + ("ogc_internal_well_water_column", False), +] + + +def _safe_relation_name(name: str) -> str: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name): + raise ValueError(f"Unsafe relation name: {name!r}") + return name + + +def _check_required_tables() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing_tables = set(inspector.get_table_names(schema="public")) + missing = REQUIRED_TABLES - existing_tables + if missing: + raise RuntimeError( + "Cannot create the well water-column views. " + f"Missing required tables: {', '.join(sorted(missing))}" + ) + + +def _create_well_water_column_view(view_name: str, public_only: bool) -> str: + safe_view_name = _safe_relation_name(view_name) + release_filter = " AND t.release_status = 'public'" if public_only else "" + observation_release_filter = ( + "\n AND o.release_status = 'public'" if public_only else "" + ) + return f""" + CREATE MATERIALIZED VIEW {safe_view_name} AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + wl_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + (o.value - COALESCE(o.measuring_point_height, 0)) AS water_level + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL{observation_release_filter} + ), + wl_agg AS ( + SELECT + w.thing_id, + AVG(w.water_level) AS avg_water_level, + MIN(w.water_level) AS min_water_level, + MAX(w.water_level) AS max_water_level + FROM wl_obs AS w + GROUP BY w.thing_id + ), + wl_last AS ( + SELECT + ranked.thing_id, + ranked.water_level AS last_water_level + FROM ( + SELECT + w.thing_id, + w.water_level, + ROW_NUMBER() OVER ( + PARTITION BY w.thing_id + ORDER BY w.observation_datetime DESC, w.observation_id DESC + ) AS rn + FROM wl_obs AS w + ) AS ranked + WHERE ranked.rn = 1 + ) + SELECT + t.id AS id, + t.name, + t.first_visit_date, + t.nma_pk_welldata, + t.well_depth, + t.hole_depth, + t.well_casing_diameter, + t.well_casing_depth, + t.well_completion_date, + t.well_driller_name, + t.well_construction_method, + t.well_pump_type, + t.well_pump_depth, + t.formation_completion_code, + t.nma_formation_zone, + t.release_status, + GREATEST(t.well_depth - wl.last_water_level, 0) AS water_column_latest, + GREATEST(t.well_depth - wa.avg_water_level, 0) AS water_column_average, + -- The shallowest reading leaves the most water in the well, the + -- deepest the least, so min/max swap sides here. + GREATEST(t.well_depth - wa.min_water_level, 0) AS water_column_maximum, + GREATEST(t.well_depth - wa.max_water_level, 0) AS water_column_minimum, + l.elevation, + l.point + FROM thing AS t + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + JOIN wl_agg AS wa ON wa.thing_id = t.id + JOIN wl_last AS wl ON wl.thing_id = t.id + WHERE + t.thing_type = 'water well' + AND t.well_depth IS NOT NULL{release_filter} + """ + + +def upgrade() -> None: + _check_required_tables() + + for view_name, public_only in VIEWS: + safe_view_name = _safe_relation_name(view_name) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {safe_view_name}")) + op.execute(text(_create_well_water_column_view(view_name, public_only))) + # Unique index required for REFRESH MATERIALIZED VIEW CONCURRENTLY. + op.execute( + text( + f"CREATE UNIQUE INDEX ix_{safe_view_name}_id " + f"ON {safe_view_name} (id)" + ) + ) + + +def downgrade() -> None: + for view_name, _public_only in VIEWS: + safe_view_name = _safe_relation_name(view_name) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {safe_view_name}")) diff --git a/alembic/versions/d9e0f1a2b3c4_edr_water_chemistry_from_legacy_tables.py b/alembic/versions/d9e0f1a2b3c4_edr_water_chemistry_from_legacy_tables.py new file mode 100644 index 000000000..1cd253665 --- /dev/null +++ b/alembic/versions/d9e0f1a2b3c4_edr_water_chemistry_from_legacy_tables.py @@ -0,0 +1,330 @@ +"""rebuild the EDR water-chemistry views on the legacy NMA chemistry tables + +ogc_water_chemistry (z9a0b1c2d3e4) and its internal mirror (2d3c3a268652) read +the normalized chain -- observation -> sample -> field_activity -> field_event +-> thing. Nothing populates that chain with analyte data: per +docs/chemistry-ingestion-runbook.md, the live ingestion path +(services/chemistry_lims.py, services/chemistry_drive.py, `oco water-chemistry +bulk-upload`) writes only to the legacy NMA_* tables. So the EDR collection is +advertised in /ogcapi/collections and returns an empty FeatureCollection, while +ogc_major_chemistry_results and ogc_minor_chemistry_wells -- both built on the +same legacy tables -- serve thousands of rows. + +This revision repoints both EDR chemistry views at the legacy tables, at the +per-result grain EDR needs (one row per analyte measurement, not the per-well +summary the pivot views produce). Four families are unioned, all sharing the +same shape via NMA_Chemistry_SampleInfo: + + NMA_MajorChemistry "Analyte"/"Symbol", "SampleValue", "Units" + NMA_MinorTraceChemistry analyte/symbol, sample_value, units + NMA_Radionuclides "Analyte"/"Symbol", "SampleValue", "Units" + NMA_FieldParameters "FieldParameter", "SampleValue", "Units" + +This is interim. When chemistry lands in the normalized Sample/Observation +model, the views move back and the EDR contract does not change -- consumers +see the same collection, parameter-names, and CoverageJSON either way. + +Three deliberate differences from the pivot views, each of which would +otherwise be a silent surprise: + +* No thing_type filter. ogc_major_chemistry_results restricts to + thing_type = 'water well' because it is a wells layer; this is a chemistry + collection, so chemistry collected at a spring belongs in it. thing_type is + carried as a column instead, so a consumer can tell a well from a spring + rather than having the distinction silently dropped -- the EDR provider + surfaces it on /locations features when the backing view has the column. +* Publication is gated on thing.release_status = 'public' (the convention + f4a5b6c7d8e9 established for the legacy-backed views) AND on + NMA_Chemistry_SampleInfo."PublicRelease" not being explicitly false. The + pivot views ignore PublicRelease; honouring it here errs toward + withholding, and NULL is treated as "not suppressed" so the two layers stay + consistent on the rows that carry no opinion. +* parameter_name is the raw trimmed legacy analyte text, falling back to the + symbol. The pivot views canonicalize analytes through long CASE blocks, but + those cover only the subset they expose as columns. Raw text keeps every + analyte reachable at the cost of aliases appearing as separate + parameter-names ("Ca" and "Calcium" both surface). That is ADR3's open + "chemistry parameter cardinality" question; canonicalizing is follow-up work + and changes only the parameter-name vocabulary, not this plumbing. + +Rows without a usable timestamp are dropped: EDR needs a time axis, and +COALESCE(analysis date, collection date) is the best available. Field +parameters carry no analysis date of their own, so they ride on the sample's +CollectionDate. + +Both are MATERIALIZED views, matching ogc_major_chemistry_results and +ogc_minor_chemistry_wells. A plain view would be re-planned on every request +across a four-way UNION of the full legacy result tables, and the provider's +get_fields() runs SELECT DISTINCT parameter_name, unit at provider +construction -- a full scan per request, against tables that already hold far +more than the pivot views' per-well row counts suggest. Indexes cover the +provider's three filter columns (thing_id, datetime, parameter_name), and the +unique index on id is what allows CONCURRENTLY refreshes. + +The cost is staleness: the nightly pg_cron job discovers every materialized +view from the catalog (x2y3z4a5b6c7), so these refresh with the rest, and +services/materialized_views.py lists them for `oco refresh-materialized-views` +after an ad-hoc chemistry ingestion. That is the same freshness contract the +existing chemistry layers already have. + +Revision ID: d9e0f1a2b3c4 +Revises: b7c8d9e0f1a2 +Create Date: 2026-08-13 15:40:00.000000 +""" + +import importlib.util +from pathlib import Path +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "d9e0f1a2b3c4" +down_revision: Union[str, Sequence[str], None] = "b7c8d9e0f1a2" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +REQUIRED_TABLES = { + "NMA_Chemistry_SampleInfo", + "NMA_MajorChemistry", + "NMA_MinorTraceChemistry", + "NMA_Radionuclides", + "NMA_FieldParameters", + "thing", + "location", + "location_thing_association", +} + +PUBLIC_VIEW = "ogc_water_chemistry" +INTERNAL_VIEW = "ogc_internal_water_chemistry" + +VIEW_COMMENTS = { + PUBLIC_VIEW: ( + "Public water-chemistry analyses (by analyte) for EDR, sourced from " + "the legacy NMA chemistry tables." + ), + INTERNAL_VIEW: ( + "All water-chemistry analyses (by analyte) for internal EDR, sourced " + "from the legacy NMA chemistry tables." + ), +} + +# Same latest-location shape the other ogc_* views use (d5e6f7a8b9c0). +_LATEST_LOCATION_CTE = """ + SELECT DISTINCT ON (lta.thing_id) + lta.thing_id, + lta.location_id, + lta.effective_start + FROM location_thing_association AS lta + WHERE lta.effective_end IS NULL + ORDER BY lta.thing_id, lta.effective_start DESC +""" + + +def _result_family( + *, + id_prefix: str, + table: str, + analyte_column: str, + value_column: str, + unit_column: str, + date_column: str | None, +) -> str: + """One SELECT over a legacy chemistry table, normalized to a common shape. + + ``date_column`` is None for NMA_FieldParameters, which has no analysis + date of its own and falls back to the sample's CollectionDate. + """ + observed_at = ( + f'COALESCE(r.{date_column}, csi."CollectionDate")' + if date_column + else 'csi."CollectionDate"' + ) + return f""" + SELECT + '{id_prefix}-' || r.id AS id, + csi.id AS sample_id, + csi.thing_id AS thing_id, + csi."PublicRelease" AS sample_public_release, + {observed_at} AS datetime, + r.{value_column}::double precision AS value, + r.{unit_column} AS unit, + NULLIF(trim({analyte_column}), '') AS parameter_name + FROM "{table}" AS r + JOIN "NMA_Chemistry_SampleInfo" AS csi + ON csi.id = r.chemistry_sample_info_id + WHERE r.{value_column} IS NOT NULL + """ + + +def _result_families() -> str: + families = [ + _result_family( + id_prefix="maj", + table="NMA_MajorChemistry", + analyte_column='COALESCE(r."Analyte", r."Symbol")', + value_column='"SampleValue"', + unit_column='"Units"', + date_column='"AnalysisDate"', + ), + _result_family( + id_prefix="min", + table="NMA_MinorTraceChemistry", + analyte_column="COALESCE(r.analyte, r.symbol)", + value_column="sample_value", + unit_column="units", + date_column="analysis_date", + ), + _result_family( + id_prefix="rad", + table="NMA_Radionuclides", + analyte_column='COALESCE(r."Analyte", r."Symbol")', + value_column='"SampleValue"', + unit_column='"Units"', + date_column='"AnalysisDate"', + ), + _result_family( + id_prefix="fld", + table="NMA_FieldParameters", + analyte_column='r."FieldParameter"', + value_column='"SampleValue"', + unit_column='"Units"', + date_column=None, + ), + ] + return "\n UNION ALL\n".join(families) + + +def _create_water_chemistry_view(view_name: str, public_only: bool) -> str: + release_filter = ( + """ + AND t.release_status = 'public' + AND results.sample_public_release IS NOT FALSE""" + if public_only + else "" + ) + return f""" + CREATE MATERIALIZED VIEW {view_name} AS + WITH latest_location AS ( + {_LATEST_LOCATION_CTE} + ), + results AS ( + {_result_families()} + ) + SELECT + results.id AS id, + t.id AS thing_id, + t.name AS station_name, + t.thing_type AS thing_type, + ST_X(l.point) AS longitude, + ST_Y(l.point) AS latitude, + results.datetime AS datetime, + results.value AS value, + results.unit AS unit, + results.parameter_name AS parameter_name, + results.sample_id AS sample_id, + t.release_status AS release_status + FROM results + JOIN thing AS t ON t.id = results.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE results.parameter_name IS NOT NULL + AND results.datetime IS NOT NULL{release_filter} + """ + + +def _load_revision_module(filename: str, module_name: str): + path = Path(__file__).with_name(filename) + if not path.exists(): + raise RuntimeError( + f"Cannot restore the previous EDR chemistry views: {filename} is " + "missing from alembic/versions." + ) + spec = importlib.util.spec_from_file_location(module_name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _drop_view_or_materialized_view(view_name: str) -> None: + # DROP VIEW IF EXISTS only suppresses "relation does not exist" -- Postgres + # still raises WrongObjectType if the relation is a materialized view, so + # check the actual kind first. + bind = op.get_bind() + relkind = bind.execute( + text("SELECT relkind FROM pg_class WHERE oid = to_regclass(:name)"), + {"name": view_name}, + ).scalar() + if relkind == "m": + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {view_name}")) + elif relkind == "v": + op.execute(text(f"DROP VIEW IF EXISTS {view_name}")) + + +def _check_required_tables() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing = set(inspector.get_table_names(schema="public")) + missing = REQUIRED_TABLES - existing + if missing: + raise RuntimeError( + "Cannot rebuild the EDR water-chemistry views. Missing required " + f"tables: {sorted(missing)}" + ) + + +def _create_indexes(view_name: str) -> None: + # The unique index is what lets REFRESH MATERIALIZED VIEW CONCURRENTLY run + # (`oco refresh-materialized-views --concurrently`); Postgres refuses + # without one. id is unique by construction -- each family prefixes its own + # primary key. + op.execute(text(f"CREATE UNIQUE INDEX ux_{view_name}_id ON {view_name} (id)")) + # The provider filters on thing_id (locations / position), datetime + # (interval), and parameter_name (parameter-name), so each gets an index. + op.execute(text(f"CREATE INDEX ix_{view_name}_thing_id ON {view_name} (thing_id)")) + op.execute(text(f"CREATE INDEX ix_{view_name}_datetime ON {view_name} (datetime)")) + op.execute( + text( + f"CREATE INDEX ix_{view_name}_parameter_name " + f"ON {view_name} (parameter_name)" + ) + ) + + +def upgrade() -> None: + _check_required_tables() + + for view_name, public_only in ((PUBLIC_VIEW, True), (INTERNAL_VIEW, False)): + _drop_view_or_materialized_view(view_name) + op.execute(text(_create_water_chemistry_view(view_name, public_only))) + _create_indexes(view_name) + op.execute( + text( + f"COMMENT ON MATERIALIZED VIEW {view_name} IS " + f"'{VIEW_COMMENTS[view_name]}'" + ) + ) + + +def downgrade() -> None: + # Restore the normalized-model definitions from the revisions that own + # them, rather than a copy that could drift from those files. + edr = _load_revision_module( + "z9a0b1c2d3e4_add_edr_water_views.py", "_edr_water_views" + ) + internal = _load_revision_module( + "2d3c3a268652_create_internal_ogc_views.py", "_internal_ogc_views" + ) + + _drop_view_or_materialized_view(PUBLIC_VIEW) + op.execute(text(edr._create_water_chemistry_view())) + op.execute( + text( + "COMMENT ON VIEW ogc_water_chemistry IS " + "'Public water-chemistry analyses (by analyte) for EDR.'" + ) + ) + + _drop_view_or_materialized_view(INTERNAL_VIEW) + op.execute(text(internal._create_internal_water_chemistry_view())) diff --git a/alembic/versions/f3a1c2b4d5e6_normalize_geothermal_temperature_units.py b/alembic/versions/f3a1c2b4d5e6_normalize_geothermal_temperature_units.py new file mode 100644 index 000000000..4eb0ee6ed --- /dev/null +++ b/alembic/versions/f3a1c2b4d5e6_normalize_geothermal_temperature_units.py @@ -0,0 +1,314 @@ +"""Normalize geothermal OGC view temperatures to Celsius + +Revision ID: f3a1c2b4d5e6 +Revises: 2d3c3a268652 +Create Date: 2026-08-06 + +The geothermal per-well views created in d1e2f3a4b5c6 passed legacy +temperatures through unconverted and labelled them with ``max("TempUnit")``. +That is wrong two ways: + + 1. ``max()`` over a mixed-unit well picks a unit lexically ('F' > 'C'), + so a well holding both C and F readings was labelled 'F' while the + values stayed mixed. + 2. ``min("Temp")`` / ``max("Temp")`` aggregate across those mixed units, + so 100 F sorts above 40 C and the extremes are meaningless. + +This revision adds two helper functions and rebuilds the two temperature +views so every temperature is also published in Celsius: + + nmw_temp_unit_code(text) -> text + Canonicalizes a legacy unit string to 'C', 'F', 'K', or NULL when + unrecognized. NMW_GtTempDepths."TempUnit" is String(1) while + NMW_GtBhtData."TempUnit" is String(5), so both single-letter codes + and spelled-out forms are accepted. + + nmw_temp_to_c(double precision, text) -> double precision + Converts a value to Celsius using that code. Returns NULL when the + unit is unrecognized rather than assuming a default, so unconvertible + readings are visible instead of silently wrong. + +Changes to ogc_geothermal_wells_bht and +ogc_geothermal_wells_temperature_profile: + + * new ``*_c`` columns (min_bht_c/max_bht_c, min_temp_c/max_temp_c) + aggregated over normalized values -- these are the ones to chart. + * ``temp_unit`` is now the constant 'C', describing the ``*_c`` columns. + * new ``temp_unit_source`` lists the distinct source units actually + present for the well ('C', 'F', 'C,F', 'UNKNOWN', ...), and + ``temp_unit_mixed`` flags wells that mix units. + * new ``unconvertible_count`` counts readings whose unit was not + recognized (present in the raw columns, NULL in the ``*_c`` columns). + * pre-existing raw columns (min_bht/max_bht, min_temp/max_temp, and the + profile ``series`` 'temp' key) are kept unchanged for compatibility. + They remain mixed-unit; consumers should move to the ``*_c`` columns. + * profile ``series`` objects gain 'temp_c' and 'temp_unit_source'. + +Heat-flow units (HtFlowUnit, GradUnit, TCondUnit, Q_unit, Kpr_unit, Ka_unit) +and depth units are NOT normalized here -- the summary and interval heat-flow +views are untouched. + +Rebuilding ogc_geothermal_wells_temperature_profile drops and recreates the +materialized view, which repopulates it WITH DATA. Expect the usual matview +build cost against the ~370k-row NMW_GtTempDepths source. +""" + +from alembic import op +from sqlalchemy import text + +revision = "f3a1c2b4d5e6" +down_revision = "2d3c3a268652" +branch_labels = None +depends_on = None + +_BHT_VIEW = "ogc_geothermal_wells_bht" +_PROFILE_VIEW = "ogc_geothermal_wells_temperature_profile" + +_LOC_CTE = """ + WITH loc AS ( + SELECT DISTINCT ON ("WellDataID") + "WellDataID", "Lat_dd83", "Long_dd83" + FROM "NMW_WellLocations" + WHERE "Lat_dd83" IS NOT NULL + AND "Long_dd83" IS NOT NULL + ORDER BY "WellDataID", "OBJECTID" + ) +""" + + +def upgrade() -> None: + op.execute(text(""" + CREATE OR REPLACE FUNCTION public.nmw_temp_unit_code(unit text) + RETURNS text + LANGUAGE sql + IMMUTABLE + AS $$ + SELECT CASE upper(regexp_replace(coalesce(unit, ''), '[^A-Za-z]', '', 'g')) + WHEN 'C' THEN 'C' + WHEN 'DEGC' THEN 'C' + WHEN 'DEGREESC' THEN 'C' + WHEN 'CELSIUS' THEN 'C' + WHEN 'CENTIGRADE' THEN 'C' + WHEN 'F' THEN 'F' + WHEN 'DEGF' THEN 'F' + WHEN 'DEGREESF' THEN 'F' + WHEN 'FAHRENHEIT' THEN 'F' + WHEN 'K' THEN 'K' + WHEN 'DEGK' THEN 'K' + WHEN 'KELVIN' THEN 'K' + ELSE NULL + END + $$ + """)) + + op.execute(text(""" + CREATE OR REPLACE FUNCTION public.nmw_temp_to_c(val double precision, unit text) + RETURNS double precision + LANGUAGE sql + IMMUTABLE + AS $$ + SELECT CASE public.nmw_temp_unit_code(unit) + WHEN 'C' THEN val + WHEN 'F' THEN (val - 32.0) * 5.0 / 9.0 + WHEN 'K' THEN val - 273.15 + ELSE NULL + END + $$ + """)) + + # ogc_geothermal_wells_bht + op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_VIEW}"')) + op.execute(text(f""" + CREATE VIEW "{_BHT_VIEW}" AS + {_LOC_CTE} + SELECT + row_number() OVER () AS id, + r."WellDataID"::text AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + hdr."TotalDepth" AS total_depth, + count(d.*) AS bht_count, + max(d."BHT") AS max_bht, + min(d."BHT") AS min_bht, + max(public.nmw_temp_to_c(d."BHT", d."TempUnit")) AS max_bht_c, + min(public.nmw_temp_to_c(d."BHT", d."TempUnit")) AS min_bht_c, + max(d."Depth") AS max_bht_depth, + 'C'::text AS temp_unit, + string_agg( + DISTINCT coalesce(public.nmw_temp_unit_code(d."TempUnit"), 'UNKNOWN'), + ',' + ORDER BY coalesce(public.nmw_temp_unit_code(d."TempUnit"), 'UNKNOWN') + ) AS temp_unit_source, + count(DISTINCT coalesce(public.nmw_temp_unit_code(d."TempUnit"), 'UNKNOWN')) > 1 + AS temp_unit_mixed, + count(*) FILTER ( + WHERE d."BHT" IS NOT NULL + AND public.nmw_temp_to_c(d."BHT", d."TempUnit") IS NULL + ) AS unconvertible_count, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtBhtData" AS d + JOIN "NMW_GtBhtHeaders" AS h ON h."BHTGUID" = d."BHTGUID" + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = h."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API", + hdr."TotalDepth" + """)) + + # ogc_geothermal_wells_temperature_profile (materialized) + op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_PROFILE_VIEW}"')) + op.execute(text(f""" + CREATE MATERIALIZED VIEW "{_PROFILE_VIEW}" AS + {_LOC_CTE} + SELECT + row_number() OVER () AS id, + r."WellDataID"::text AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + count(td.*) AS reading_count, + min(td."Depth") AS min_depth, + max(td."Depth") AS max_depth, + min(td."Temp") AS min_temp, + max(td."Temp") AS max_temp, + min(public.nmw_temp_to_c(td."Temp", td."TempUnit")) AS min_temp_c, + max(public.nmw_temp_to_c(td."Temp", td."TempUnit")) AS max_temp_c, + 'C'::text AS temp_unit, + string_agg( + DISTINCT coalesce(public.nmw_temp_unit_code(td."TempUnit"), 'UNKNOWN'), + ',' + ORDER BY coalesce(public.nmw_temp_unit_code(td."TempUnit"), 'UNKNOWN') + ) AS temp_unit_source, + count(DISTINCT coalesce(public.nmw_temp_unit_code(td."TempUnit"), 'UNKNOWN')) > 1 + AS temp_unit_mixed, + count(*) FILTER ( + WHERE public.nmw_temp_to_c(td."Temp", td."TempUnit") IS NULL + ) AS unconvertible_count, + json_agg( + json_build_object( + 'depth', td."Depth", + 'temp', td."Temp", + 'temp_c', public.nmw_temp_to_c(td."Temp", td."TempUnit"), + 'temp_unit_source', + coalesce(public.nmw_temp_unit_code(td."TempUnit"), 'UNKNOWN') + ) + ORDER BY td."Depth" + ) AS series, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtTempDepths" AS td + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE td."Depth" IS NOT NULL + AND td."Temp" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API" + """)) + op.execute( + text(f'CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_id ON "{_PROFILE_VIEW}" (id)') + ) + op.execute( + text( + f'CREATE INDEX ix_{_PROFILE_VIEW}_geom ON "{_PROFILE_VIEW}" USING GIST (geom)' + ) + ) + + +def downgrade() -> None: + # Restore the d1e2f3a4b5c6 definitions verbatim, then drop the helpers. + op.execute(text(f'DROP VIEW IF EXISTS "{_BHT_VIEW}"')) + op.execute(text(f""" + CREATE VIEW "{_BHT_VIEW}" AS + {_LOC_CTE} + SELECT + row_number() OVER () AS id, + r."WellDataID"::text AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + hdr."TotalDepth" AS total_depth, + count(d.*) AS bht_count, + max(d."BHT") AS max_bht, + min(d."BHT") AS min_bht, + max(d."Depth") AS max_bht_depth, + max(d."TempUnit") AS temp_unit, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtBhtData" AS d + JOIN "NMW_GtBhtHeaders" AS h ON h."BHTGUID" = d."BHTGUID" + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = h."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API", + hdr."TotalDepth" + """)) + + op.execute(text(f'DROP MATERIALIZED VIEW IF EXISTS "{_PROFILE_VIEW}"')) + op.execute(text(f""" + CREATE MATERIALIZED VIEW "{_PROFILE_VIEW}" AS + {_LOC_CTE} + SELECT + row_number() OVER () AS id, + r."WellDataID"::text AS well_data_id, + hdr."CurWellNam" AS well_name, + hdr."API" AS api, + count(td.*) AS reading_count, + min(td."Depth") AS min_depth, + max(td."Depth") AS max_depth, + min(td."Temp") AS min_temp, + max(td."Temp") AS max_temp, + max(td."TempUnit") AS temp_unit, + json_agg( + json_build_object('depth', td."Depth", 'temp', td."Temp") + ORDER BY td."Depth" + ) AS series, + ST_SetSRID( + ST_MakePoint(loc."Long_dd83", loc."Lat_dd83"), 4326 + ) AS geom + FROM "NMW_GtTempDepths" AS td + JOIN "NMW_WellSamples" AS s ON s."SamplSetID" = td."SamplSetID" + JOIN "NMW_WellRecords" AS r ON r."RecrdSetID" = s."RecrdsetID" + JOIN loc ON loc."WellDataID" = r."WellDataID" + LEFT JOIN "NMW_WellHeaders" AS hdr ON hdr."WellDataID" = r."WellDataID" + WHERE td."Depth" IS NOT NULL + AND td."Temp" IS NOT NULL + GROUP BY + r."WellDataID", + loc."Lat_dd83", + loc."Long_dd83", + hdr."CurWellNam", + hdr."API" + """)) + op.execute( + text(f'CREATE UNIQUE INDEX ux_{_PROFILE_VIEW}_id ON "{_PROFILE_VIEW}" (id)') + ) + op.execute( + text( + f'CREATE INDEX ix_{_PROFILE_VIEW}_geom ON "{_PROFILE_VIEW}" USING GIST (geom)' + ) + ) + + op.execute( + text("DROP FUNCTION IF EXISTS public.nmw_temp_to_c(double precision, text)") + ) + op.execute(text("DROP FUNCTION IF EXISTS public.nmw_temp_unit_code(text)")) diff --git a/alembic/versions/f4a5b6c7d8e9_apply_public_release_status_filter_to_ogc_views.py b/alembic/versions/f4a5b6c7d8e9_apply_public_release_status_filter_to_ogc_views.py new file mode 100644 index 000000000..82659be52 --- /dev/null +++ b/alembic/versions/f4a5b6c7d8e9_apply_public_release_status_filter_to_ogc_views.py @@ -0,0 +1,1290 @@ +"""apply public release_status filter to ogc views + +Restricts every ogc_* view/materialized view to release_status = 'public' +so that published, unauthenticated OGC endpoints (/ogcapi) never expose +private or draft records. Reversible: downgrade() recreates the same 22 +relations with the byte-identical unfiltered SQL that was in production +before this migration, so "no predicate" is restored exactly rather than +approximated. + +ogc_actively_monitored_wells gets no predicate of its own -- it inherits +public-only rows transitively once ogc_water_well_summary is filtered (see +alembic/versions/w1x2y3z4a5b6_drop_child_release_filters_from_ngwmn_views.py +for why an extra child-table release_status filter here would be wrong). +Because it depends on ogc_water_well_summary via a direct JOIN, it must be +dropped before ogc_water_well_summary and recreated after. + +ogc_locations does not exist before this migration -- core/pygeoapi-config.yml +points the locations collection directly at the raw location table. This +migration creates ogc_locations for the first time (explicit column list, +no SELECT *, matching every other view in this file) and a separate change +repoints that one config line at it. Since ogc_locations never existed +unfiltered in production, downgrade() drops it rather than recreating an +unfiltered copy. + +Revision ID: f4a5b6c7d8e9 +Revises: b6c7d8e9f0a1 +Create Date: 2026-07-14 00:00:00.000000 + +Re-pointed from the original y3z4a5b6c7d8 on 2026-08-03: three colleague +migrations (z9a0b1c2d3e4, a5b6c7d8e9f0, b6c7d8e9f0a1) landed on staging off +that same revision while this branch was still open, forking the history. +Neither touches anything this migration's SQL depends on -- the two new EDR +views (z9a0b1c2d3e4) already filter on release_status = 'public' themselves, +and the other two only touch the unrelated NMW measurement views and +pg_cron scheduling. +""" + +import re +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "f4a5b6c7d8e9" +down_revision: Union[str, Sequence[str], None] = "b6c7d8e9f0a1" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +REQUIRED_TABLES = { + "thing", + "location", + "location_thing_association", + "group", + "group_thing_association", + "status_history", + "observation", + "sample", + "field_activity", + "field_event", + "data_provenance", + "NMA_MajorChemistry", + "NMA_Chemistry_SampleInfo", + "NMA_MinorTraceChemistry", +} + +LATEST_LOCATION_CTE = """ +SELECT DISTINCT ON (lta.thing_id) + lta.thing_id, + lta.location_id, + lta.effective_start +FROM location_thing_association AS lta +WHERE lta.effective_end IS NULL +ORDER BY lta.thing_id, lta.effective_start DESC +""".strip() + +# The 11 thing-type views still in scope after +# s4t5u6v7w8x9_drop_unused_well_type_ogc_views.py removed the well-subtype +# variants (abandoned_wells, artesian_wells, dry_holes, dug_wells, +# exploration_wells, injection_wells, monitoring_wells, observation_wells, +# piezometers, production_wells, test_wells). +THING_VIEWS = [ + ("water_wells", "water well"), + ("springs", "spring"), + ("diversions_surface_water", "diversion of surface water, etc."), + ("ephemeral_streams", "ephemeral stream"), + ("lakes_ponds_reservoirs", "lake, pond or reservoir"), + ("meteorological_stations", "meteorological station"), + ("other_things", "other"), + ("outfalls_wastewater_return_flow", "outfall of wastewater or return flow"), + ("perennial_streams", "perennial stream"), + ("rock_sample_locations", "rock sample location"), + ("soil_gas_sample_locations", "soil gas sample location"), +] + + +def _safe_view_id(view_id: str) -> str: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", view_id): + raise ValueError(f"Unsafe view id: {view_id!r}") + return view_id + + +def _drop_view_or_materialized_view(view_name: str) -> None: + # DROP VIEW IF EXISTS / DROP MATERIALIZED VIEW IF EXISTS only suppress + # "relation does not exist" -- Postgres still raises WrongObjectType if + # the relation exists as the other kind (e.g. DROP VIEW against an + # existing materialized view), so the relation's actual kind must be + # checked first rather than trying both blindly. + bind = op.get_bind() + relkind = bind.execute( + text("SELECT relkind FROM pg_class WHERE oid = to_regclass(:name)"), + {"name": view_name}, + ).scalar() + if relkind == "m": + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {view_name}")) + elif relkind == "v": + op.execute(text(f"DROP VIEW IF EXISTS {view_name}")) + + +def _check_required_tables() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing_tables = set(inspector.get_table_names(schema="public")) + missing = REQUIRED_TABLES - existing_tables + if missing: + raise RuntimeError( + "Cannot apply public release_status filter to OGC views. " + f"Missing required tables: {', '.join(sorted(missing))}" + ) + + +def _create_thing_view(view_id: str, thing_type: str, public_only: bool) -> str: + safe_view_id = _safe_view_id(view_id) + escaped_thing_type = thing_type.replace("'", "''") + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE VIEW ogc_{safe_view_id} AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ) + SELECT + t.id, + t.name, + t.first_visit_date, + t.nma_pk_welldata, + t.well_depth, + t.hole_depth, + t.well_casing_diameter, + t.well_casing_depth, + t.well_completion_date, + t.well_driller_name, + t.well_construction_method, + t.well_pump_type, + t.well_pump_depth, + t.formation_completion_code, + t.nma_formation_zone, + t.release_status, + l.elevation, + l.point + FROM thing AS t + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE t.thing_type = '{escaped_thing_type}'{release_filter} + """ + + +def _create_latest_depth_view(public_only: bool) -> str: + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE MATERIALIZED VIEW ogc_latest_depth_to_water_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + ranked_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + o.value, + o.measuring_point_height, + -- Treat NULL measuring_point_height as 0 when computing + -- depth_to_water_bgs. + ( + o.value - COALESCE(o.measuring_point_height, 0) + ) AS depth_to_water_bgs, + ROW_NUMBER() OVER ( + PARTITION BY fe.thing_id + ORDER BY o.observation_datetime DESC, o.id DESC + ) AS rn + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL{release_filter} + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + ro.observation_id, + ro.observation_datetime, + ro.value AS depth_to_water_reference, + ro.measuring_point_height, + ro.depth_to_water_bgs, + l.point + FROM ranked_obs AS ro + JOIN thing AS t ON t.id = ro.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE ro.rn = 1 + """ + + +def _create_avg_tds_view(public_only: bool) -> str: + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE MATERIALIZED VIEW ogc_avg_tds_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + tds_obs AS ( + SELECT + csi.thing_id, + mc.id AS major_chemistry_id, + COALESCE(mc."AnalysisDate", csi."CollectionDate")::date AS observation_date, + mc."SampleValue" AS sample_value, + mc."Units" AS units + FROM "NMA_MajorChemistry" AS mc + JOIN "NMA_Chemistry_SampleInfo" AS csi + ON csi.id = mc.chemistry_sample_info_id + JOIN thing AS t ON t.id = csi.thing_id + WHERE + t.thing_type = 'water well' + AND mc."SampleValue" IS NOT NULL + AND ( + lower(coalesce(mc."Analyte", '')) IN ( + 'tds', + 'total dissolved solids' + ) + OR lower(coalesce(mc."Symbol", '')) = 'tds' + ){release_filter} + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + COUNT(to2.major_chemistry_id)::integer AS tds_observation_count, + AVG(to2.sample_value)::double precision AS avg_tds_value, + MIN(to2.observation_date) AS first_tds_observation_date, + MAX(to2.observation_date) AS last_tds_observation_date, + l.point + FROM tds_obs AS to2 + JOIN thing AS t ON t.id = to2.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + GROUP BY t.id, t.name, t.thing_type, l.point + """ + + +def _create_latest_tds_view(public_only: bool) -> str: + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE VIEW ogc_latest_tds_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + tds_obs AS ( + SELECT + csi.thing_id, + mc.id AS major_chemistry_id, + COALESCE(mc."AnalysisDate", csi."CollectionDate") AS observation_datetime, + mc."SampleValue" AS sample_value, + mc."Units" AS units + FROM "NMA_MajorChemistry" AS mc + JOIN "NMA_Chemistry_SampleInfo" AS csi + ON csi.id = mc.chemistry_sample_info_id + JOIN thing AS t ON t.id = csi.thing_id + WHERE + t.thing_type = 'water well' + AND mc."SampleValue" IS NOT NULL + AND ( + lower(coalesce(mc."Analyte", '')) IN ( + 'tds', + 'total dissolved solids' + ) + OR lower(coalesce(mc."Symbol", '')) = 'tds' + ){release_filter} + ), + ranked_tds AS ( + SELECT + to2.thing_id, + to2.major_chemistry_id, + to2.observation_datetime, + to2.sample_value, + to2.units, + ROW_NUMBER() OVER ( + PARTITION BY to2.thing_id + ORDER BY to2.observation_datetime DESC NULLS LAST, to2.major_chemistry_id DESC + ) AS rn + FROM tds_obs AS to2 + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + rt.major_chemistry_id, + rt.observation_datetime::date AS latest_tds_observation_date, + rt.sample_value AS latest_tds_value, + rt.units AS latest_tds_units, + l.point + FROM ranked_tds AS rt + JOIN thing AS t ON t.id = rt.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE rt.rn = 1 + """ + + +def _create_depth_to_water_trend_view(public_only: bool) -> str: + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE MATERIALIZED VIEW ogc_depth_to_water_trend_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + obs AS ( + SELECT + fe.thing_id, + o.observation_datetime, + (o.value - COALESCE(o.measuring_point_height, 0)) AS depth_to_water_bgs + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL{release_filter} + ), + agg AS ( + SELECT + ob.thing_id, + COUNT(*)::integer AS record_count, + MIN(ob.observation_datetime) AS first_observation_datetime, + MAX(ob.observation_datetime) AS last_observation_datetime, + EXTRACT(EPOCH FROM (MAX(ob.observation_datetime) - MIN(ob.observation_datetime))) + / 31557600.0 AS span_years, + REGR_SLOPE( + ob.depth_to_water_bgs, + EXTRACT(EPOCH FROM ob.observation_datetime) + ) * 31557600.0 AS slope_ft_per_year + FROM obs AS ob + GROUP BY ob.thing_id + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + a.record_count, + a.first_observation_datetime, + a.last_observation_datetime, + a.span_years, + a.slope_ft_per_year, + CASE + WHEN a.record_count >= 10 OR (a.record_count >= 4 AND a.span_years >= 2.0) THEN + CASE + WHEN a.slope_ft_per_year IS NULL THEN 'not enough data' + WHEN a.slope_ft_per_year > 0.25 THEN 'increasing' + WHEN a.slope_ft_per_year < -0.25 THEN 'decreasing' + ELSE 'stable' + END + ELSE 'not enough data' + END AS trend_category, + l.point + FROM agg AS a + JOIN thing AS t ON t.id = a.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + """ + + +def _create_water_well_summary_view(public_only: bool) -> str: + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE MATERIALIZED VIEW ogc_water_well_summary AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + wl_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + (o.value - COALESCE(o.measuring_point_height, 0)) AS water_level + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL{release_filter} + ), + wl_agg AS ( + SELECT + w.thing_id, + COUNT(*)::integer AS total_water_levels, + MIN(w.water_level) AS min_water_level, + MAX(w.water_level) AS max_water_level, + REGR_SLOPE( + w.water_level, + EXTRACT(EPOCH FROM w.observation_datetime) + ) * 31557600.0 AS water_level_trend_ft_per_year + FROM wl_obs AS w + GROUP BY w.thing_id + ), + wl_last AS ( + SELECT + ranked.thing_id, + ranked.water_level AS last_water_level, + ranked.observation_datetime AS last_water_level_datetime + FROM ( + SELECT + w.thing_id, + w.water_level, + w.observation_datetime, + ROW_NUMBER() OVER ( + PARTITION BY w.thing_id + ORDER BY w.observation_datetime DESC, w.observation_id DESC + ) AS rn + FROM wl_obs AS w + ) AS ranked + WHERE ranked.rn = 1 + ) + SELECT + t.id AS id, + t.name, + t.well_depth, + l.elevation, + dpl.collection_method AS elevation_method, + t.nma_formation_zone AS formation_zone, + wa.total_water_levels, + wl.last_water_level, + wl.last_water_level_datetime, + wa.min_water_level, + wa.max_water_level, + wa.water_level_trend_ft_per_year, + l.point + FROM thing AS t + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + JOIN wl_agg AS wa ON wa.thing_id = t.id + LEFT JOIN wl_last AS wl ON wl.thing_id = t.id + LEFT JOIN LATERAL ( + SELECT dp.collection_method + FROM data_provenance AS dp + WHERE + dp.target_table = 'location' + AND dp.target_id = l.id + AND dp.field_name = 'elevation' + ORDER BY dp.id DESC + LIMIT 1 + ) AS dpl ON true + WHERE t.thing_type = 'water well' + AND wa.total_water_levels > 0 + """ + + +# Static analyte columns for major chemistry pivots. +# Includes aliases observed in current DB values (e.g., Ca(total), IONBAL, TAn, TCat, Na+K). +# Mirrored character-for-character (modulo view name) in +# 2d3c3a268652_create_internal_ogc_views.py; tests/test_migration_view_parity.py +# enforces the two stay in sync. +STATIC_ANALYTE_COLUMNS_MAJOR: list[tuple[str, str]] = [ + ("tds", "tds"), + ("calcium", "calcium"), + ("calcium_total", "calcium_total"), + ("magnesium", "magnesium"), + ("magnesium_total", "magnesium_total"), + ("sodium", "sodium"), + ("sodium_total", "sodium_total"), + ("potassium", "potassium"), + ("potassium_total", "potassium_total"), + ("sodium_plus_potassium", "sodium_plus_potassium"), + ("bicarbonate", "bicarbonate"), + ("carbonate", "carbonate"), + ("sulfate", "sulfate"), + ("chloride", "chloride"), + ("ion_balance", "ion_balance"), + ("total_anions", "total_anions"), + ("total_cations", "total_cations"), + ("alkalinity", "alkalinity"), + ("hardness", "hardness"), + ("specific_conductance", "specific_conductance"), + ("ph", "ph"), + ("nitrate", "nitrate"), + ("fluoride", "fluoride"), + ("silica", "silica"), +] + + +def _major_chemistry_select_columns() -> str: + return ",\n".join( + [ + ( + " MAX(lr.sample_value) FILTER " + f"(WHERE lr.analyte_key = '{analyte_key}') AS {column_name}" + ) + for analyte_key, column_name in STATIC_ANALYTE_COLUMNS_MAJOR + ] + ) + + +def _major_chemistry_unit_columns() -> str: + return ",\n".join( + [ + ( + " MAX(lr.units) FILTER " + f"(WHERE lr.analyte_key = '{analyte_key}') AS {column_name}_units" + ) + for analyte_key, column_name in STATIC_ANALYTE_COLUMNS_MAJOR + ] + ) + + +def _create_major_chemistry_results_view(public_only: bool) -> str: + static_columns = _major_chemistry_select_columns() + static_unit_columns = _major_chemistry_unit_columns() + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE MATERIALIZED VIEW ogc_major_chemistry_results AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + chemistry_rows AS ( + SELECT + csi.thing_id, + mc.id AS result_id, + COALESCE(mc."AnalysisDate", csi."CollectionDate") AS observation_datetime, + trim(mc."Analyte") AS analyte_name, + trim(mc."Symbol") AS symbol_name, + mc."SampleValue"::double precision AS sample_value, + mc."Units" AS units + FROM "NMA_MajorChemistry" AS mc + JOIN "NMA_Chemistry_SampleInfo" AS csi + ON csi.id = mc.chemistry_sample_info_id + JOIN thing AS t + ON t.id = csi.thing_id + WHERE mc."SampleValue" IS NOT NULL + AND t.thing_type = 'water well'{release_filter} + ), + normalized_rows AS ( + SELECT + cr.thing_id, + cr.result_id, + cr.observation_datetime, + NULLIF( + regexp_replace( + lower(trim(coalesce(cr.analyte_name, ''))), + '[^a-z0-9]+', + '', + 'g' + ), + '' + ) AS analyte_token, + NULLIF( + regexp_replace( + lower(trim(coalesce(cr.symbol_name, ''))), + '[^a-z0-9]+', + '', + 'g' + ), + '' + ) AS symbol_token, + cr.sample_value, + cr.units + FROM chemistry_rows AS cr + ), + mapped_rows AS ( + SELECT + nr.thing_id, + nr.result_id, + nr.observation_datetime, + CASE + WHEN coalesce(nr.symbol_token, '') = 'tds' + OR coalesce(nr.analyte_token, '') IN ('tds', 'totaldissolvedsolids') + THEN 'tds' + + WHEN coalesce(nr.symbol_token, '') = 'ca' + OR coalesce(nr.analyte_token, '') = 'ca' + THEN 'calcium' + WHEN coalesce(nr.analyte_token, '') = 'catotal' + THEN 'calcium_total' + + WHEN coalesce(nr.symbol_token, '') = 'mg' + OR coalesce(nr.analyte_token, '') = 'mg' + THEN 'magnesium' + WHEN coalesce(nr.analyte_token, '') = 'mgtotal' + THEN 'magnesium_total' + + WHEN coalesce(nr.symbol_token, '') = 'na' + OR coalesce(nr.analyte_token, '') = 'na' + THEN 'sodium' + WHEN coalesce(nr.analyte_token, '') = 'natotal' + THEN 'sodium_total' + + WHEN coalesce(nr.symbol_token, '') = 'k' + OR coalesce(nr.analyte_token, '') = 'k' + THEN 'potassium' + WHEN coalesce(nr.analyte_token, '') = 'ktotal' + THEN 'potassium_total' + + WHEN coalesce(nr.analyte_token, '') = 'nak' + THEN 'sodium_plus_potassium' + + WHEN coalesce(nr.symbol_token, '') = 'hco3' + OR coalesce(nr.analyte_token, '') = 'hco3' + THEN 'bicarbonate' + WHEN coalesce(nr.symbol_token, '') = 'co3' + OR coalesce(nr.analyte_token, '') = 'co3' + THEN 'carbonate' + WHEN coalesce(nr.symbol_token, '') = 'so4' + OR coalesce(nr.analyte_token, '') = 'so4' + THEN 'sulfate' + WHEN coalesce(nr.symbol_token, '') = 'cl' + OR coalesce(nr.analyte_token, '') = 'cl' + THEN 'chloride' + + WHEN coalesce(nr.analyte_token, '') = 'ionbal' + THEN 'ion_balance' + WHEN coalesce(nr.analyte_token, '') = 'tan' + THEN 'total_anions' + WHEN coalesce(nr.analyte_token, '') = 'tcat' + THEN 'total_cations' + + WHEN coalesce(nr.analyte_token, '') IN ('alk', 'alkalinity') + THEN 'alkalinity' + WHEN coalesce(nr.analyte_token, '') IN ('hrd', 'hardness') + THEN 'hardness' + WHEN coalesce(nr.analyte_token, '') IN ( + 'condlab', + 'specificconductance', + 'specificconductivity', + 'conductivity' + ) + THEN 'specific_conductance' + WHEN coalesce(nr.symbol_token, '') = 'ph' + OR coalesce(nr.analyte_token, '') IN ('ph', 'phl') + THEN 'ph' + + WHEN coalesce(nr.symbol_token, '') = 'no3' + OR coalesce(nr.analyte_token, '') IN ('no3', 'nitrate') + THEN 'nitrate' + WHEN coalesce(nr.symbol_token, '') = 'f' + OR coalesce(nr.analyte_token, '') IN ('f', 'fluoride') + THEN 'fluoride' + WHEN coalesce(nr.symbol_token, '') = 'sio2' + OR coalesce(nr.analyte_token, '') IN ('sio2', 'silica') + THEN 'silica' + + ELSE NULL + END AS analyte_key, + nr.sample_value, + nr.units + FROM normalized_rows AS nr + ), + latest_results AS ( + SELECT + mr.thing_id, + mr.analyte_key, + mr.sample_value, + mr.units, + mr.observation_datetime, + ROW_NUMBER() OVER ( + PARTITION BY mr.thing_id, mr.analyte_key + ORDER BY mr.observation_datetime DESC NULLS LAST, mr.result_id DESC + ) AS rn + FROM mapped_rows AS mr + WHERE mr.analyte_key IS NOT NULL + ) + SELECT + t.id AS id, + ll.location_id, + t.name, + t.thing_type, + COUNT(*)::integer AS analyte_count, + MAX(lr.observation_datetime::date) AS latest_chemistry_date, +{static_columns}, +{static_unit_columns}, + l.point + FROM latest_results AS lr + JOIN thing AS t ON t.id = lr.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE lr.rn = 1 + GROUP BY t.id, ll.location_id, t.name, t.thing_type, l.point + """ + + +# Mirrored character-for-character (modulo view name) in +# 2d3c3a268652_create_internal_ogc_views.py; tests/test_migration_view_parity.py +# enforces the two stay in sync. +STATIC_ANALYTE_COLUMNS_MINOR: list[tuple[str, str]] = [ + ("h2r", "h2r"), + ("o18r", "o18r"), + ("c13r", "c13r"), + ("c14", "c14"), + ("c14_years", "c14_years"), + ("fluoride", "fluoride"), + ("barium", "barium"), + ("barium_total", "barium_total"), + ("copper", "copper"), + ("copper_total", "copper_total"), + ("zinc", "zinc"), + ("zinc_total", "zinc_total"), + ("molybdenum", "molybdenum"), + ("molybdenum_total", "molybdenum_total"), + ("silica", "silica"), + ("silicon", "silicon"), + ("silicon_total", "silicon_total"), + ("manganese", "manganese"), + ("manganese_total", "manganese_total"), + ("iron", "iron"), + ("iron_total", "iron_total"), + ("strontium", "strontium"), + ("strontium_total", "strontium_total"), + ("chromium", "chromium"), + ("chromium_total", "chromium_total"), + ("boron", "boron"), + ("boron_total", "boron_total"), + ("uranium", "uranium"), + ("uranium_total", "uranium_total"), + ("lithium", "lithium"), + ("lithium_total", "lithium_total"), + ("silver", "silver"), + ("silver_total", "silver_total"), + ("antimony", "antimony"), + ("antimony_total", "antimony_total"), + ("beryllium", "beryllium"), + ("beryllium_total", "beryllium_total"), + ("lead", "lead"), + ("lead_total", "lead_total"), + ("thallium", "thallium"), + ("thallium_total", "thallium_total"), + ("bromide", "bromide"), + ("selenium", "selenium"), + ("selenium_total", "selenium_total"), + ("vanadium", "vanadium"), + ("vanadium_total", "vanadium_total"), + ("aluminum", "aluminum"), + ("aluminum_total", "aluminum_total"), + ("arsenic", "arsenic"), + ("arsenic_total", "arsenic_total"), + ("nickel", "nickel"), + ("nickel_total", "nickel_total"), + ("cadmium", "cadmium"), + ("cadmium_total", "cadmium_total"), + ("cobalt", "cobalt"), + ("cobalt_total", "cobalt_total"), + ("phosphate", "phosphate"), + ("nitrite", "nitrite"), + ("nitrate", "nitrate"), + ("nitrate_as_n", "nitrate_as_n"), + ("thorium", "thorium"), + ("thorium_total", "thorium_total"), + ("tin", "tin"), + ("tin_total", "tin_total"), + ("mercury", "mercury"), + ("mercury_total", "mercury_total"), + ("titanium", "titanium"), + ("titanium_total", "titanium_total"), +] + + +def _minor_chemistry_value_columns() -> str: + return ",\n".join( + [ + ( + " MAX(lr.sample_value) FILTER " + f"(WHERE lr.analyte_key = '{analyte_key}') AS {column_name}" + ) + for analyte_key, column_name in STATIC_ANALYTE_COLUMNS_MINOR + ] + ) + + +def _minor_chemistry_unit_columns() -> str: + return ",\n".join( + [ + ( + " MAX(lr.units) FILTER " + f"(WHERE lr.analyte_key = '{analyte_key}') AS {column_name}_units" + ) + for analyte_key, column_name in STATIC_ANALYTE_COLUMNS_MINOR + ] + ) + + +def _create_minor_chemistry_wells_view(public_only: bool) -> str: + value_columns = _minor_chemistry_value_columns() + unit_columns = _minor_chemistry_unit_columns() + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE MATERIALIZED VIEW ogc_minor_chemistry_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + chemistry_rows AS ( + SELECT + csi.thing_id, + mtc.id AS result_id, + COALESCE(mtc.analysis_date::timestamp, csi."CollectionDate") AS observation_datetime, + trim(mtc.analyte) AS analyte_name, + mtc.sample_value::double precision AS sample_value, + mtc.units AS units + FROM "NMA_MinorTraceChemistry" AS mtc + JOIN "NMA_Chemistry_SampleInfo" AS csi + ON csi.id = mtc.chemistry_sample_info_id + JOIN thing AS t ON t.id = csi.thing_id + WHERE + mtc.sample_value IS NOT NULL + AND t.thing_type = 'water well'{release_filter} + ), + normalized_rows AS ( + SELECT + cr.thing_id, + cr.result_id, + cr.observation_datetime, + NULLIF( + regexp_replace( + lower(trim(coalesce(cr.analyte_name, ''))), + '[^a-z0-9]+', + '', + 'g' + ), + '' + ) AS analyte_token, + cr.sample_value, + cr.units + FROM chemistry_rows AS cr + ), + mapped_rows AS ( + SELECT + nr.thing_id, + nr.result_id, + nr.observation_datetime, + CASE + WHEN coalesce(nr.analyte_token, '') = 'h2r' THEN 'h2r' + WHEN coalesce(nr.analyte_token, '') = 'o18r' THEN 'o18r' + WHEN coalesce(nr.analyte_token, '') = 'c13r' THEN 'c13r' + WHEN coalesce(nr.analyte_token, '') = 'c14' THEN 'c14' + WHEN coalesce(nr.analyte_token, '') = 'c14years' THEN 'c14_years' + + WHEN coalesce(nr.analyte_token, '') = 'f' THEN 'fluoride' + WHEN coalesce(nr.analyte_token, '') = 'ba' THEN 'barium' + WHEN coalesce(nr.analyte_token, '') = 'batotal' THEN 'barium_total' + WHEN coalesce(nr.analyte_token, '') = 'cu' THEN 'copper' + WHEN coalesce(nr.analyte_token, '') = 'cutotal' THEN 'copper_total' + WHEN coalesce(nr.analyte_token, '') = 'zn' THEN 'zinc' + WHEN coalesce(nr.analyte_token, '') = 'zntotal' THEN 'zinc_total' + WHEN coalesce(nr.analyte_token, '') = 'mo' THEN 'molybdenum' + WHEN coalesce(nr.analyte_token, '') = 'mototal' THEN 'molybdenum_total' + WHEN coalesce(nr.analyte_token, '') = 'sio2' THEN 'silica' + WHEN coalesce(nr.analyte_token, '') = 'si' THEN 'silicon' + WHEN coalesce(nr.analyte_token, '') = 'sitotal' THEN 'silicon_total' + WHEN coalesce(nr.analyte_token, '') = 'mn' THEN 'manganese' + WHEN coalesce(nr.analyte_token, '') = 'mntotal' THEN 'manganese_total' + WHEN coalesce(nr.analyte_token, '') = 'fe' THEN 'iron' + WHEN coalesce(nr.analyte_token, '') = 'fetotal' THEN 'iron_total' + WHEN coalesce(nr.analyte_token, '') = 'sr' THEN 'strontium' + WHEN coalesce(nr.analyte_token, '') = 'srtotal' THEN 'strontium_total' + WHEN coalesce(nr.analyte_token, '') = 'cr' THEN 'chromium' + WHEN coalesce(nr.analyte_token, '') = 'crtotal' THEN 'chromium_total' + WHEN coalesce(nr.analyte_token, '') = 'b' THEN 'boron' + WHEN coalesce(nr.analyte_token, '') = 'btotal' THEN 'boron_total' + WHEN coalesce(nr.analyte_token, '') = 'u' THEN 'uranium' + WHEN coalesce(nr.analyte_token, '') = 'utotal' THEN 'uranium_total' + WHEN coalesce(nr.analyte_token, '') = 'li' THEN 'lithium' + WHEN coalesce(nr.analyte_token, '') = 'litotal' THEN 'lithium_total' + WHEN coalesce(nr.analyte_token, '') = 'ag' THEN 'silver' + WHEN coalesce(nr.analyte_token, '') = 'agtotal' THEN 'silver_total' + WHEN coalesce(nr.analyte_token, '') = 'sb' THEN 'antimony' + WHEN coalesce(nr.analyte_token, '') = 'sbtotal' THEN 'antimony_total' + WHEN coalesce(nr.analyte_token, '') = 'be' THEN 'beryllium' + WHEN coalesce(nr.analyte_token, '') = 'betotal' THEN 'beryllium_total' + WHEN coalesce(nr.analyte_token, '') = 'pb' THEN 'lead' + WHEN coalesce(nr.analyte_token, '') = 'pbtotal' THEN 'lead_total' + WHEN coalesce(nr.analyte_token, '') = 'tl' THEN 'thallium' + WHEN coalesce(nr.analyte_token, '') = 'tltotal' THEN 'thallium_total' + WHEN coalesce(nr.analyte_token, '') = 'br' THEN 'bromide' + WHEN coalesce(nr.analyte_token, '') = 'se' THEN 'selenium' + WHEN coalesce(nr.analyte_token, '') = 'setotal' THEN 'selenium_total' + WHEN coalesce(nr.analyte_token, '') = 'v' THEN 'vanadium' + WHEN coalesce(nr.analyte_token, '') = 'vtotal' THEN 'vanadium_total' + WHEN coalesce(nr.analyte_token, '') = 'al' THEN 'aluminum' + WHEN coalesce(nr.analyte_token, '') = 'altotal' THEN 'aluminum_total' + WHEN coalesce(nr.analyte_token, '') = 'as' THEN 'arsenic' + WHEN coalesce(nr.analyte_token, '') = 'astotal' THEN 'arsenic_total' + WHEN coalesce(nr.analyte_token, '') = 'ni' THEN 'nickel' + WHEN coalesce(nr.analyte_token, '') = 'nitotal' THEN 'nickel_total' + WHEN coalesce(nr.analyte_token, '') = 'cd' THEN 'cadmium' + WHEN coalesce(nr.analyte_token, '') = 'cdtotal' THEN 'cadmium_total' + WHEN coalesce(nr.analyte_token, '') = 'co' THEN 'cobalt' + WHEN coalesce(nr.analyte_token, '') = 'cototal' THEN 'cobalt_total' + WHEN coalesce(nr.analyte_token, '') = 'po4' THEN 'phosphate' + WHEN coalesce(nr.analyte_token, '') = 'no2' THEN 'nitrite' + WHEN coalesce(nr.analyte_token, '') = 'no3' THEN 'nitrate' + WHEN coalesce(nr.analyte_token, '') = 'no3n' THEN 'nitrate_as_n' + WHEN coalesce(nr.analyte_token, '') = 'th' THEN 'thorium' + WHEN coalesce(nr.analyte_token, '') = 'thtotal' THEN 'thorium_total' + WHEN coalesce(nr.analyte_token, '') = 'sn' THEN 'tin' + WHEN coalesce(nr.analyte_token, '') = 'sntotal' THEN 'tin_total' + WHEN coalesce(nr.analyte_token, '') = 'hg' THEN 'mercury' + WHEN coalesce(nr.analyte_token, '') = 'hgtotal' THEN 'mercury_total' + WHEN coalesce(nr.analyte_token, '') = 'ti' THEN 'titanium' + WHEN coalesce(nr.analyte_token, '') = 'titotal' THEN 'titanium_total' + ELSE NULL + END AS analyte_key, + nr.sample_value, + nr.units + FROM normalized_rows AS nr + ), + latest_results AS ( + SELECT + mr.thing_id, + mr.analyte_key, + mr.sample_value, + mr.units, + mr.observation_datetime, + ROW_NUMBER() OVER ( + PARTITION BY mr.thing_id, mr.analyte_key + ORDER BY mr.observation_datetime DESC NULLS LAST, mr.result_id DESC + ) AS rn + FROM mapped_rows AS mr + WHERE mr.analyte_key IS NOT NULL + ) + SELECT + t.id AS id, + ll.location_id, + t.name, + t.thing_type, + COUNT(*)::integer AS analyte_count, + MAX(lr.observation_datetime::date) AS latest_chemistry_date, +{value_columns}, +{unit_columns}, + l.point + FROM latest_results AS lr + JOIN thing AS t ON t.id = lr.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE lr.rn = 1 + AND t.thing_type = 'water well' + GROUP BY t.id, ll.location_id, t.name, t.thing_type, l.point + """ + + +METERS_TO_FEET = 3.28084 + + +def _create_water_elevation_view(public_only: bool) -> str: + release_filter = " AND t.release_status = 'public'" if public_only else "" + return f""" + CREATE MATERIALIZED VIEW ogc_water_elevation_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + ranked_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + CASE + WHEN lower(trim(o.unit)) IN ('m', 'meter', 'meters', 'metre', 'metres') THEN + (o.value * {METERS_TO_FEET}) - COALESCE(o.measuring_point_height, 0) + WHEN lower(trim(o.unit)) IN ('ft', 'foot', 'feet') THEN + o.value - COALESCE(o.measuring_point_height, 0) + ELSE + NULL + END AS depth_to_water_below_ground_surface + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL + AND lower(trim(o.unit)) IN ( + 'm', + 'meter', + 'meters', + 'metre', + 'metres', + 'ft', + 'foot', + 'feet' + ){release_filter} + ), + latest_obs AS ( + SELECT + ro.*, + ROW_NUMBER() OVER ( + PARTITION BY ro.thing_id + ORDER BY ro.observation_datetime DESC, ro.observation_id DESC + ) AS rn + FROM ranked_obs AS ro + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + lo.observation_id, + lo.observation_datetime, + l.elevation AS elevation_m, + lo.depth_to_water_below_ground_surface AS depth_to_water_below_ground_surface_ft, + ((l.elevation * {METERS_TO_FEET}) - lo.depth_to_water_below_ground_surface) + AS water_elevation_ft, + l.point + FROM latest_obs AS lo + JOIN thing AS t ON t.id = lo.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE lo.rn = 1 + """ + + +def _create_actively_monitored_wells_view() -> str: + # No predicate of its own -- inherits public-only rows transitively via + # the JOIN to ogc_water_well_summary, which is itself filtered. Adding a + # filter on status_history.release_status here would repeat the mistake + # reverted in w1x2y3z4a5b6 (that column is never actually populated for + # this table). + return """ + CREATE VIEW ogc_actively_monitored_wells AS + WITH latest_monitoring_status AS ( + SELECT DISTINCT ON (sh.target_id) + sh.target_id AS thing_id, + sh.status_value + FROM status_history AS sh + WHERE + sh.target_table = 'thing' + AND sh.status_type = 'Monitoring Status' + ORDER BY sh.target_id, sh.start_date DESC, sh.id DESC + ) + SELECT + wws.id, + wws.name, + 'water well'::text AS thing_type, + wws.well_depth, + wws.elevation, + wws.elevation_method, + wws.formation_zone, + wws.total_water_levels, + wws.last_water_level, + wws.last_water_level_datetime, + wws.min_water_level, + wws.max_water_level, + wws.water_level_trend_ft_per_year, + g.id AS group_id, + g.name AS group_name, + g.group_type, + wws.point + FROM "group" AS g + JOIN group_thing_association AS gta ON gta.group_id = g.id + JOIN ogc_water_well_summary AS wws ON wws.id = gta.thing_id + JOIN latest_monitoring_status AS lms ON lms.thing_id = wws.id + WHERE lower(trim(g.name)) = 'water level network' + AND lms.status_value = 'Currently monitored' + """ + + +def _create_project_areas_view(public_only: bool) -> str: + release_filter = " AND g.release_status = 'public'" if public_only else "" + return f""" + CREATE VIEW ogc_project_areas AS + SELECT + g.id, + g.name, + g.description, + g.group_type, + g.release_status, + g.project_area + FROM "group" AS g + WHERE g.project_area IS NOT NULL{release_filter} + """ + + +def _create_locations_view() -> str: + # Explicit column list verified against db/location.py and its mixins + # (AutoBaseMixin/AuditMixin, ReleaseMixin, NotesMixin, DataProvenanceMixin). + # NotesMixin/DataProvenanceMixin add only polymorphic relationships, no + # real columns. Audit columns (created_at, created_by_*, updated_by_*) + # are deliberately excluded, matching every other view in this file -- + # none of them expose those columns either, even for Thing's otherwise + # thorough column list. + return """ + CREATE VIEW ogc_locations AS + SELECT + l.id, + l.nma_pk_location, + l.description, + l.county, + l.state, + l.quad_name, + l.nma_location_notes, + l.nma_coordinate_notes, + l.nma_data_reliability, + l.nma_date_created, + l.nma_site_date, + l.release_status, + l.elevation, + l.point + FROM location AS l + WHERE l.release_status = 'public' + """ + + +def _recreate_governed_views(public_only: bool) -> None: + # ogc_actively_monitored_wells depends on ogc_water_well_summary via a + # direct JOIN; Postgres refuses to drop a materialized view while a + # dependent view exists, so it must go first and come back last. + _drop_view_or_materialized_view("ogc_actively_monitored_wells") + + for view_id, thing_type in THING_VIEWS: + _drop_view_or_materialized_view(f"ogc_{_safe_view_id(view_id)}") + op.execute(text(_create_thing_view(view_id, thing_type, public_only))) + + _drop_view_or_materialized_view("ogc_latest_depth_to_water_wells") + op.execute(text(_create_latest_depth_view(public_only))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_latest_depth_to_water_wells IS " + "'Latest depth-to-water per well view for pygeoapi.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_latest_depth_to_water_wells_id " + "ON ogc_latest_depth_to_water_wells (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_avg_tds_wells") + op.execute(text(_create_avg_tds_view(public_only))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_avg_tds_wells IS " + "'Average TDS per well from major chemistry results for pygeoapi.'" + ) + ) + op.execute( + text("CREATE UNIQUE INDEX ux_ogc_avg_tds_wells_id " "ON ogc_avg_tds_wells (id)") + ) + + _drop_view_or_materialized_view("ogc_latest_tds_wells") + op.execute(text(_create_latest_tds_view(public_only))) + op.execute( + text( + "COMMENT ON VIEW ogc_latest_tds_wells IS " + "'Latest TDS per well from major chemistry results for pygeoapi.'" + ) + ) + + _drop_view_or_materialized_view("ogc_depth_to_water_trend_wells") + op.execute(text(_create_depth_to_water_trend_view(public_only))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_depth_to_water_trend_wells IS " + "'Depth-to-water trend classification for water wells.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_depth_to_water_trend_wells_id " + "ON ogc_depth_to_water_trend_wells (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_water_well_summary") + op.execute(text(_create_water_well_summary_view(public_only))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_water_well_summary IS " + "'Summary statistics for water wells including water-level trend.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_water_well_summary_id " + "ON ogc_water_well_summary (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_major_chemistry_results") + op.execute(text(_create_major_chemistry_results_view(public_only))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_major_chemistry_results IS " + "'Latest major-chemistry analyte values per location, pivoted into static analyte columns.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_major_chemistry_results_id " + "ON ogc_major_chemistry_results (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_minor_chemistry_wells") + op.execute(text(_create_minor_chemistry_wells_view(public_only))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_minor_chemistry_wells IS " + "'Latest minor/trace chemistry analyte values for water wells, pivoted into static analyte columns.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_minor_chemistry_wells_id " + "ON ogc_minor_chemistry_wells (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_water_elevation_wells") + op.execute(text(_create_water_elevation_view(public_only))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_water_elevation_wells IS " + "'Latest water elevation per well with explicit units: " + "elevation_m, depth_to_water_below_ground_surface_ft, water_elevation_ft.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_water_elevation_wells_id " + "ON ogc_water_elevation_wells (id)" + ) + ) + + # Recreate now that ogc_water_well_summary exists again. + op.execute(text(_create_actively_monitored_wells_view())) + op.execute( + text( + "COMMENT ON VIEW ogc_actively_monitored_wells IS " + "'Wells in the Water Level Network group for pygeoapi.'" + ) + ) + + _drop_view_or_materialized_view("ogc_project_areas") + op.execute(text(_create_project_areas_view(public_only))) + op.execute( + text( + "COMMENT ON VIEW ogc_project_areas IS " + "'Project areas for groups with polygon boundaries for pygeoapi.'" + ) + ) + + +def upgrade() -> None: + _check_required_tables() + _recreate_governed_views(public_only=True) + + # ogc_locations does not exist before this migration -- see module + # docstring. Only ever created in its filtered form. + _drop_view_or_materialized_view("ogc_locations") + op.execute(text(_create_locations_view())) + op.execute( + text( + "COMMENT ON VIEW ogc_locations IS " + "'Public locations for pygeoapi, replacing the raw location table provider.'" + ) + ) + + +def downgrade() -> None: + _recreate_governed_views(public_only=False) + + # ogc_locations never existed unfiltered in production; downgrading + # drops it rather than recreating an unfiltered copy. + _drop_view_or_materialized_view("ogc_locations") diff --git a/api/chemisty.py b/api/chemisty.py index 5519c3f02..0fe0ed150 100644 --- a/api/chemisty.py +++ b/api/chemisty.py @@ -13,7 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +from datetime import datetime + from fastapi import APIRouter +from fastapi_pagination.ext.sqlalchemy import paginate +from sqlalchemy import asc, desc, select + +from api.pagination import CustomPage +from core.dependencies import amp_viewer_dependency, session_dependency +from db.chemistry_views import WaterChemistryResultsView +from schemas.chemistry import WaterChemistryResultResponse +from services.legacy_chemistry import canonical_parameter_name, result_kind # from services.validation.chemistry import validate_analyte @@ -25,6 +35,83 @@ ) +# Only columns that mean something to a client of this endpoint. A whitelist +# rather than getattr on the view: the latter would expose every column, +# including the ones carrying release state, as a public sort key. +_RESULT_SORT_COLUMNS = { + "observation_datetime": WaterChemistryResultsView.observation_datetime, + "parameter_name": WaterChemistryResultsView.parameter_name, + "value": WaterChemistryResultsView.value, + "id": WaterChemistryResultsView.id, +} + + +@router.get("/results", summary="Get water chemistry results", tags=["chemistry"]) +def get_water_chemistry_results( + session: session_dependency, + user: amp_viewer_dependency, + thing_id: int | None = None, + start_time: datetime | None = None, + end_time: datetime | None = None, + sort: str | None = None, + order: str | None = None, +) -> CustomPage[WaterChemistryResultResponse]: + """ + Retrieve water chemistry results, one row per analyte. + + Reads the legacy NMA chemistry tables, which is where the water chemistry + actually is -- the refactored `observation` table holds none of it. Rows + come from the public view, so an unreleased thing or a sample flagged + `PublicRelease = false` is not served here regardless of who is asking. + + `start_time` is inclusive and `end_time` exclusive, so a calendar year is + `start_time=YYYY-01-01&end_time=YYYY+1-01-01` with no risk of picking up a + result recorded at midnight on New Year's Day of the following year. + + `sort` accepts `observation_datetime`, `parameter_name`, `value`, or `id`; + `order` accepts `asc` or `desc`. The default is newest first, so a client + that wants a well's most recent analysis can ask for size 1. + """ + query = select(WaterChemistryResultsView) + + if thing_id is not None: + query = query.where(WaterChemistryResultsView.thing_id == thing_id) + + if start_time is not None: + query = query.where( + WaterChemistryResultsView.observation_datetime >= start_time + ) + + if end_time is not None: + query = query.where(WaterChemistryResultsView.observation_datetime < end_time) + + sort_column = _RESULT_SORT_COLUMNS.get( + sort or "observation_datetime", + WaterChemistryResultsView.observation_datetime, + ) + direction = asc if (order or "desc").lower() == "asc" else desc + + # id is the tiebreaker so paging is stable: without it two analytes sharing + # a timestamp can swap pages between requests and be served twice or never. + query = query.order_by(direction(sort_column), WaterChemistryResultsView.id) + + def transformer(rows): + # Analytes come out of the legacy tables as symbols; the response + # speaks the lexicon's names so a consumer can match a result to a + # drinking water standard without knowing the legacy vocabulary. + return [ + WaterChemistryResultResponse.model_validate(row).model_copy( + update={ + "parameter_name": canonical_parameter_name(row.parameter_name), + "result_kind": result_kind(row.id), + } + ) + for row in rows + ] + + return paginate(query=query, conn=session, transformer=transformer) + + # @router.get( # "/analysis_set", # response_model=CustomPage[WaterChemistryAnalysisSetResponse], diff --git a/api/disclaimer.py b/api/disclaimer.py new file mode 100644 index 000000000..b0ec7e836 --- /dev/null +++ b/api/disclaimer.py @@ -0,0 +1,115 @@ +# =============================================================================== +# Copyright 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Public data disclaimer page. + +Both pygeoapi mounts advertise this URL as +`metadata.identification.terms_of_service`, so it is deliberately +unauthenticated -- an OGC client following the advertised link has no +credentials to present. + +HTML is the default because the pygeoapi landing page renders +terms_of_service as a link a human clicks; JSON is offered for catalog +harvesters that want the text as data rather than markup. +""" + +import html +from typing import Annotated + +from fastapi import APIRouter, Query, Request +from fastapi.responses import HTMLResponse, JSONResponse + +from core.app import in_public_schema +from core.disclaimer import ( + DISCLAIMER_CONTACT_EMAIL, + DISCLAIMER_PARAGRAPHS, + DISCLAIMER_TITLE, +) + +router = APIRouter(tags=["disclaimer"]) + +_STYLE = ( + "max-width:44rem;margin:3rem auto;padding:0 1.25rem;" + "font-family:system-ui,-apple-system,'Segoe UI',sans-serif;" + "line-height:1.6;color:#1a1a1a" +) + + +def _wants_json(request: Request, f: str | None) -> bool: + # An explicit ?f= wins over content negotiation, matching pygeoapi's own + # precedence so the two surfaces behave the same way. + if f is not None: + return f.lower() == "json" + accept = request.headers.get("accept", "") + return "application/json" in accept and "text/html" not in accept + + +def _render_html() -> str: + paragraphs = [] + for paragraph in DISCLAIMER_PARAGRAPHS: + escaped = html.escape(paragraph) + escaped = escaped.replace( + DISCLAIMER_CONTACT_EMAIL, + f'' + f"{DISCLAIMER_CONTACT_EMAIL}", + ) + paragraphs.append(f"

{escaped}

") + body = "\n".join(paragraphs) + title = html.escape(DISCLAIMER_TITLE) + return ( + "\n" + '\n' + " \n" + ' \n' + ' \n' + f" {title} | Ocotillo\n" + " \n" + f' \n' + f"

{title}

\n" + f"{body}\n" + " \n" + "\n" + ) + + +@in_public_schema +@router.get( + "/disclaimer", + response_class=HTMLResponse, + summary="Data disclaimer and terms of service", + responses={ + 200: { + "content": {"text/html": {}, "application/json": {}}, + "description": "The disclaimer as HTML (default) or JSON (?f=json).", + } + }, +) +def get_disclaimer( + request: Request, + f: Annotated[ + str | None, + Query(description="Response format. Use 'json' for the text as data."), + ] = None, +): + if _wants_json(request, f): + return JSONResponse( + { + "title": DISCLAIMER_TITLE, + "paragraphs": list(DISCLAIMER_PARAGRAPHS), + "contact": DISCLAIMER_CONTACT_EMAIL, + } + ) + return HTMLResponse(_render_html()) diff --git a/api/feedback.py b/api/feedback.py index 68f632b2f..ce3d3473a 100644 --- a/api/feedback.py +++ b/api/feedback.py @@ -225,7 +225,7 @@ def _build_slack_payload(payload: FeedbackCreate, jira_key: str, jira_url: str) @router.post("", response_model=FeedbackResponse) async def create_feedback( payload: FeedbackCreate, - _user=viewer_dependency, + _user: viewer_dependency, ): jira_base = os.environ["JIRA_BASE_URL"] jira_email = os.environ["JIRA_EMAIL"] diff --git a/api/geothermal.py b/api/geothermal.py index 7d6f96395..8efd63094 100644 --- a/api/geothermal.py +++ b/api/geothermal.py @@ -13,126 +13,96 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -from fastapi import APIRouter +"""Geothermal well endpoints. -# -# from db.geothermal import ( -# GeothermalTemperatureProfile, -# GeothermalTemperatureProfileObservation, -# GeothermalBottomHoleTemperature, -# GeothermalWellInterval, -# GeothermalHeatFlow, -# GeothermalThermalConductivity, -# GeothermalSampleSet, -# GeothermalBottomHoleTemperatureHeader, -# ) +TEMPORARY BACKING: routes read from the legacy NM_Wells staging mirror +(``db/nmw_legacy.py``) via ``services/geothermal_helper.py``. Once the +NM_Wells -> Ocotillo transform lands these will be backed by the ``thing`` +table and ``thing_id`` will be populated on the response. The route path lives +under ``/thing`` so the URL is stable across that swap. +""" -router = APIRouter(prefix="/geothermal", tags=["geothermal"]) +from typing import Optional +from uuid import UUID +from fastapi import APIRouter, Query +from fastapi_pagination.ext.sqlalchemy import paginate +from starlette.status import HTTP_200_OK, HTTP_404_NOT_FOUND + +from api.pagination import CustomPage +from core.dependencies import session_dependency, viewer_dependency +from schemas.geothermal import GeothermalWellResponse +from services.exceptions_helper import PydanticStyleException +from services.geothermal_helper import ( + geothermal_wells_transformer, + get_geothermal_well_by_id, + get_geothermal_wells_query, +) + +router = APIRouter(prefix="/thing", tags=["geothermal"]) + + +@router.get( + "/geothermal-well", + summary="Get all geothermal wells", + status_code=HTTP_200_OK, +) +def get_geothermal_wells( + user: viewer_dependency, + session: session_dependency, + county: Optional[str] = None, + name_contains: Optional[str] = None, + q: Optional[str] = Query( + None, + description=( + "Free-text search across well name, API, well number, operator and " + "county. Whitespace-separated words are ANDed, so each word added " + "narrows the result. Case-insensitive substring match." + ), + ), +) -> CustomPage[GeothermalWellResponse]: + """List geothermal wells. + + NOTE: sourced from the legacy NM_Wells mirror (NMW_WellHeaders where + GthrmExist is set). Will be re-pointed at the thing table post-transform. + + ``q`` is what the UI well picker uses: the catalogue is far too large to + choose from by scrolling, so the term is matched server-side and the total + reported by the pagination envelope is the size of the match set. + """ + sql = get_geothermal_wells_query(county=county, name_contains=name_contains, q=q) + return paginate(query=sql, conn=session, transformer=geothermal_wells_transformer) + + +@router.get( + "/geothermal-well/{well_data_id}", + summary="Get geothermal well by legacy WellDataID", + status_code=HTTP_200_OK, +) +def get_geothermal_well( + user: viewer_dependency, + well_data_id: UUID, + session: session_dependency, +) -> GeothermalWellResponse: + """Get a single geothermal well by its legacy NMW WellDataID (GUID). + + NOTE: keyed by the legacy GUID because these rows are not yet in the thing + table. Post-transform this becomes an integer thing_id lookup. + """ + well = get_geothermal_well_by_id(session, well_data_id) + if well is None: + raise PydanticStyleException( + status_code=HTTP_404_NOT_FOUND, + detail=[ + { + "loc": ["path", "well_data_id"], + "msg": f"Geothermal well with WellDataID {well_data_id} not found.", + "type": "value_error", + "input": {"well_data_id": str(well_data_id)}, + } + ], + ) + return well -# @router.post("/sample_set", status_code=status.HTTP_201_CREATED) -# async def add_geothermal_sample_set( -# sample_set_data: CreateGeothermalSampleSet, # Replace with appropriate schema -# session: session_dependency -# ): -# """ -# Add a new geothermal sample set. -# """ -# # Assuming you have a model for GeothermalSampleSet -# return adder(session, GeothermalSampleSet, sample_set_data) -# -# -# @router.post("/bottom_hole_temperature_header", status_code=status.HTTP_201_CREATED) -# async def add_bottom_hole_temperature_header( -# bottom_hole_temperature_header_data: CreateBottomHoleTemperatureHeader, -# session: session_dependency -# ): -# """ -# Add a new bottom hole temperature header. -# """ -# # Assuming you have a model for GeothermalBottomHoleTemperatureHeader -# return adder( -# session, -# GeothermalBottomHoleTemperatureHeader, -# bottom_hole_temperature_header_data, -# ) -# -# -# @router.post("/temperature_profile", status_code=status.HTTP_201_CREATED) -# async def add_temperature_profile( -# temperature_profile_data: CreateTemperatureProfile, -# session: session_dependency -# ): -# """ -# Add a new temperature profile. -# """ -# return adder(session, GeothermalTemperatureProfile, temperature_profile_data) -# -# -# @router.post("/temperature_profile_observation", status_code=status.HTTP_201_CREATED) -# async def add_temperature_profile_observation( -# temperature_profile_observation_data: CreateTemperatureProfileObservation, -# session: session_dependency -# ): -# """ -# Add a new temperature profile observation. -# """ -# return adder( -# session, -# GeothermalTemperatureProfileObservation, -# temperature_profile_observation_data, -# ) -# -# -# @router.post("/bottom_hole_temperature", status_code=status.HTTP_201_CREATED) -# async def add_bottom_hole_temperature( -# bottom_hole_temperature_data: CreateBottomHoleTemperature, -# session: session_dependency -# ): -# """ -# Add a new bottom hole temperature. -# """ -# return adder( -# session, -# GeothermalBottomHoleTemperature, # Assuming this is the correct model -# bottom_hole_temperature_data, -# ) -# -# -# @router.post("/interval", status_code=status.HTTP_201_CREATED) -# async def add_geothermal_interval( -# interval_data: CreateGeothermalInterval, # Replace with appropriate schema -# session: session_dependency -# ): -# """ -# Add a new geothermal interval. -# """ -# # Assuming you have a model for GeothermalInterval -# return adder(session, GeothermalWellInterval, interval_data) -# -# -# @router.post("/thermal_conductivity", status_code=status.HTTP_201_CREATED) -# async def add_thermal_conductivity( -# thermal_conductivity_data: CreateThermalConductivity, # Replace with appropriate schema -# session: session_dependency -# ): -# """ -# Add a new geothermal thermal conductivity. -# """ -# # Assuming you have a model for GeothermalThermalConductivity -# return adder(session, GeothermalThermalConductivity, thermal_conductivity_data) -# -# -# @router.post("/heat_flow", status_code=status.HTTP_201_CREATED) -# async def add_heat_flow( -# heat_flow_data: CreateHeatFlow, -# session: session_dependency -# ): -# """ -# Add a new geothermal heat flow. -# """ -# # Assuming you have a model for GeothermalHeatFlow -# return adder(session, GeothermalHeatFlow, heat_flow_data) -# # ============= EOF ============================================= diff --git a/api/gis_artifacts.py b/api/gis_artifacts.py new file mode 100644 index 000000000..d815f0566 --- /dev/null +++ b/api/gis_artifacts.py @@ -0,0 +1,266 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Downloadable QGIS and ArcGIS Pro artifacts for the OGC API mounts. + +The public routes are deliberately anonymous: they describe the public +`/ogcapi` mount, which is itself anonymous, and a desktop GIS user fetching a +connection file has no credential to present. Nothing they return is +sensitive -- the URLs are already advertised in the pygeoapi landing page, and +no credential is ever embedded (see services/gis_artifacts). + +The internal connection file is gated, not because the file is secret, but +because the internal mount's existence is not something to advertise to +anonymous callers. Holding it still gets you nothing without an `OGCInternal` +API key. + +Read docs/ogc-desktop-gis-artifacts.md before changing what is emitted. +""" + +from typing import Annotated + +from fastapi import APIRouter, HTTPException, Query, Request +from fastapi.responses import HTMLResponse, JSONResponse, Response + +from core.app import in_public_schema +from core.dependencies import session_dependency, viewer_dependency +from core.pygeoapi import _app_base_url, _internal_server_url, _server_url +from services.gis_artifacts import ( + Connection, + arcgis_layer_file, + collection_fields, + find_curated_layer, + load_curated_layers, + qgis_connections_xml, + qgis_layer_definition, +) + +router = APIRouter(prefix="/gis", tags=["desktop gis"]) + +PUBLIC_CONNECTION_NAME = "NMBGMR Ocotillo" +INTERNAL_CONNECTION_NAME = "NMBGMR Ocotillo (internal)" + + +class XmlAttachment(Response): + """An XML download. The media type lives here so the OpenAPI schema and + the response itself cannot disagree: FastAPI reads `media_type` off the + `response_class` to document the operation, and `_attachment` returns an + instance of that same class rather than restating the string.""" + + media_type = "text/xml" + + +class JsonAttachment(Response): + """A JSON download. Not JSONResponse: the body is already serialised, and + re-encoding it would escape the CIM document into a JSON string.""" + + media_type = "application/json" + + +def _attachment(response_class: type[Response], body: str, filename: str) -> Response: + return response_class( + content=body, + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +def _wants_json(request: Request, f: str | None) -> bool: + # Same precedence as api/disclaimer.py and pygeoapi itself: an explicit + # ?f= beats the Accept header, so the surfaces behave alike. + if f is not None: + return f.lower() == "json" + accept = request.headers.get("accept", "") + return "application/json" in accept and "text/html" not in accept + + +def _index_payload() -> dict: + """Machine-readable catalogue of every artifact this router serves. + + Absolute hrefs, built from _app_base_url() rather than the request, so a + browser app on another origin can use them unchanged and a proxy that + rewrites Host cannot send the caller somewhere else. + """ + root = _app_base_url() + service = _public_base() + return { + "service_url": service, + "connections": [ + { + "client": "qgis", + "href": f"{root}/gis/qgis/connections.xml", + "media_type": XmlAttachment.media_type, + "filename": "ocotillo-ogcapi-connections.xml", + } + ], + "layers": [ + { + "id": layer.id, + "title": layer.title, + "abstract": layer.abstract, + "collection": layer.collection, + "collection_url": f"{service}/collections/{layer.collection}", + "geometry": layer.geometry, + "renderer": layer.renderer.get("type"), + "downloads": [ + { + "client": "qgis", + "href": f"{root}/gis/qgis/layers/{layer.id}.qlr", + "media_type": XmlAttachment.media_type, + "filename": f"{layer.id}.qlr", + }, + { + "client": "arcgis", + "href": f"{root}/gis/arcgis/layers/{layer.id}.lyrx", + "media_type": JsonAttachment.media_type, + "filename": f"{layer.id}.lyrx", + }, + ], + } + for layer in load_curated_layers() + ], + } + + +def _public_base() -> str: + # _server_url() is what pygeoapi stamps into its own `self`/`next` links. + # Deriving the artifact's URL from the same place means a client that + # imports the connection and then pages through `items` never crosses + # hosts -- the failure mode that PYGEOAPI_INTERNAL_SERVER_URL was added to + # fix (see the comment in core/pygeoapi._internal_server_url). + return _server_url() + + +@router.get("/qgis/connections.xml", response_class=XmlAttachment) +@in_public_schema +def qgis_connections() -> Response: + """QGIS connections file registering the public OGC API - Features mount. + + Import through **Browser panel > right-click "WFS / OGC API - Features" > + Load Connections**. + """ + body = qgis_connections_xml([Connection(PUBLIC_CONNECTION_NAME, _public_base())]) + return _attachment(XmlAttachment, body, "ocotillo-ogcapi-connections.xml") + + +@router.get("/qgis/connections-internal.xml", response_class=XmlAttachment) +def qgis_connections_internal(user: viewer_dependency) -> Response: + """QGIS connections file covering the public and internal mounts. + + Carries no credential. The internal entry only resolves for a client that + attaches its own `OGCInternal` API key -- see + docs/internal-ogc-desktop-gis.md for how one is issued and attached. + """ + body = qgis_connections_xml( + [ + Connection(PUBLIC_CONNECTION_NAME, _public_base()), + Connection(INTERNAL_CONNECTION_NAME, _internal_server_url()), + ] + ) + return _attachment(XmlAttachment, body, "ocotillo-ogcapi-connections-internal.xml") + + +@router.get( + "/qgis/layers/{layer_id}.qlr", + response_class=XmlAttachment, + responses={404: {"description": "No curated layer with that id."}}, +) +@in_public_schema +def qgis_layer(layer_id: str, session: session_dependency) -> Response: + """A styled QGIS layer definition for one curated layer.""" + layer = find_curated_layer(layer_id) + if layer is None: + raise HTTPException(status_code=404, detail=f"No curated layer {layer_id!r}.") + fields = collection_fields(session, layer.collection) + body = qgis_layer_definition(layer, _public_base(), fields) + return _attachment(XmlAttachment, body, f"{layer_id}.qlr") + + +@router.get( + "/arcgis/layers/{layer_id}.lyrx", + response_class=JsonAttachment, + responses={404: {"description": "No curated layer with that id."}}, +) +@in_public_schema +def arcgis_layer(layer_id: str, session: session_dependency) -> Response: + """A styled ArcGIS Pro layer file for one curated layer.""" + layer = find_curated_layer(layer_id) + if layer is None: + raise HTTPException(status_code=404, detail=f"No curated layer {layer_id!r}.") + fields = collection_fields(session, layer.collection) + body = arcgis_layer_file(layer, _public_base(), fields) + return _attachment(JsonAttachment, body, f"{layer_id}.lyrx") + + +_PAGE_STYLE = ( + "max-width:52rem;margin:3rem auto;padding:0 1.25rem;" + "font-family:system-ui,-apple-system,'Segoe UI',sans-serif;" + "line-height:1.6;color:#1a1a1a" +) + + +@router.get("", response_class=HTMLResponse) +@in_public_schema +def gis_index(request: Request, f: Annotated[str | None, Query()] = None) -> Response: + """Landing page listing every downloadable artifact. + + HTML by default for a human following the link; `?f=json` (or an + Accept: application/json header) returns the same catalogue as data, so a + frontend can enumerate the layers instead of hardcoding their ids. + """ + if _wants_json(request, f): + return JSONResponse(_index_payload()) + base = _public_base() + rows = "".join( + f"{layer.title}
" + f"{layer.abstract}" + f'.qlr' + f'.lyrx' + for layer in load_curated_layers() + ) + return HTMLResponse(f""" +Desktop GIS downloads + +

Using our OGC layers in QGIS and ArcGIS Pro

+

Service URL: {base}

+ +

Everything at once

+

QGIS connections file — +in QGIS, open the Browser panel, right-click +WFS / OGC API - Features, choose Load Connections, and pick +this file. Every collection then appears in the Browser panel.

+

ArcGIS Pro — Pro writes its own .ogc +connection file and we cannot generate one for you. Add the connection once: +Insert > Connections > Server > New OGC API Server, and paste +the service URL above. Pro saves a .ogc file into your project +folder that you can then share with colleagues.

+ +

One layer at a time

+

Styled, with field aliases already applied. Drag the file into QGIS, or add +the .lyrx to a map in Pro.

+ + +{rows} +
LayerQGISArcGIS Pro
+ +

Time series

+

Water levels and water chemistry are also published as +OGC API - EDR time series at {base}/collections/waterlevels and +{base}/collections/water_chemistry. Neither QGIS nor ArcGIS Pro +can read EDR, so the layers above carry the same measurements summarised per +site instead.

+""") + + +# ============= EOF ============================================= diff --git a/api/ngwmn.py b/api/ngwmn.py index 7fc2e1d51..c954fe788 100644 --- a/api/ngwmn.py +++ b/api/ngwmn.py @@ -16,6 +16,7 @@ from fastapi import APIRouter from starlette.responses import Response +from core.app import in_public_schema from core.dependencies import session_dependency from services.ngwmn_helper import ( make_waterlevels_response, @@ -25,7 +26,13 @@ router = APIRouter(prefix="/ngwmn", tags=["NGWMN"]) +# These three routes are intentionally anonymous: the federal NGWMN harvester +# polls them without credentials. @in_public_schema documents that (and lists +# them in /openapi.json) so tests/test_authorization.py can tell an intentional +# public route from an endpoint that simply forgot its `user:` dependency. + +@in_public_schema @router.get( "/waterlevels/{pointid}", summary="Get waterlevels for a given pointid in the NGWMN format", @@ -35,6 +42,7 @@ def read_ngwmn_waterlevels(pointid: str, db: session_dependency): return Response(content=data, media_type="application/xml") +@in_public_schema @router.get( "/wellconstruction/{pointid}", summary="Get wellconstruction for a given pointid in the NGWMN format", @@ -44,6 +52,7 @@ def read_ngwmn_wellconstruction(pointid: str, db: session_dependency): return Response(content=data, media_type="application/xml") +@in_public_schema @router.get( "/lithology/{pointid}", summary="Get lithology for a given pointid in the NGWMN format", diff --git a/api/observation.py b/api/observation.py index d4c7fff78..fb11ca9d7 100644 --- a/api/observation.py +++ b/api/observation.py @@ -28,6 +28,7 @@ session_dependency, amp_admin_dependency, amp_editor_dependency, + amp_staging_dependency, amp_viewer_dependency, ) from db import Observation, Parameter @@ -40,7 +41,12 @@ UpdateGroundwaterLevelObservation, UpdateWaterChemistryObservation, ) -from schemas.transducer import TransducerObservationWithBlockResponse +from schemas.transducer import ( + DeletedTransducerObservationsResponse, + PublishedTransducerBlockResponse, + PublishTransducerBlock, + TransducerObservationWithBlockResponse, +) from schemas.water_level_csv import WaterLevelBulkUploadResponse from services.crud_helper import model_deleter, model_adder from services.observation_helper import ( @@ -50,10 +56,30 @@ get_transducer_observations, ) from services.query_helper import simple_get_by_id +from services.transducer_helper import ( + delete_transducer_observations, + publish_transducer_block, +) from services.water_level_csv import bulk_upload_water_levels router = APIRouter(prefix="/observation", tags=["observation"]) + +def _groundwater_level_parameter_id(session) -> int: + """ + The lexicon id the transducer routes work in. + + Looked up rather than configured so the publish, read, and delete routes + cannot drift onto different parameters. + """ + return ( + session.query(Parameter) + .filter(Parameter.parameter_name == "groundwater level") + .one() + .id + ) + + """ TODO @@ -88,6 +114,37 @@ def add_water_chemistry_observation( return model_adder(session, Observation, obs_data, user=user) +@router.post( + "/transducer-groundwater-level/block", + status_code=HTTP_201_CREATED, + summary="Publish a corrected transducer series as one block", +) +def publish_transducer_groundwater_level_block( + payload: PublishTransducerBlock, + session: session_dependency, + user: amp_staging_dependency, + replace_overlapping: bool = False, +) -> PublishedTransducerBlockResponse: + """ + Publish one corrected logger file as a single observation block. + + The block's time span is derived from the measurements; the client does not + send it. Overlapping an existing block is a 409 listing the collisions -- + pass `replace_overlapping=true` to supersede them, which deletes those + blocks and their readings in the same transaction. + + Written by the hydrograph corrector in OcotilloUI. See + `docs/hydrograph-correction-publish.md`. + """ + return publish_transducer_block( + session, + payload, + parameter_id=_groundwater_level_parameter_id(session), + user=user, + replace_overlapping=replace_overlapping, + ) + + @router.post( "/groundwater-level/bulk-upload", response_model=WaterLevelBulkUploadResponse, @@ -155,17 +212,30 @@ def get_transducer_groundwater_level_observations( thing_id: int | None = None, start_time: datetime | None = None, end_time: datetime | None = None, + sort: str | None = None, + order: str | None = None, ) -> CustomPage[TransducerObservationWithBlockResponse]: + """ + Retrieve transducer groundwater level observations paired with the block + that covers them. - groundwater_parameter_id = ( - session.query(Parameter) - .filter(Parameter.parameter_name == "groundwater level") - .one() - .id - ) - + `sort` accepts `observation_datetime`, `value`, or `id`; `order` accepts + `asc` or `desc`. The default is newest first, so a client that wants the + latest stored reading for a well can ask for size 1. + """ + # Keyword arguments deliberately: the helper's fourth positional parameter + # is `sensor_id`, so the previous positional call passed `start_time` as a + # sensor id (unused, silently dropped), `end_time` as `start_time`, and + # nothing as `end_time` -- an upper bound the caller asked for was ignored + # and the lower bound came from the wrong argument. return get_transducer_observations( - session, thing_id, groundwater_parameter_id, start_time, end_time + session, + thing_id=thing_id, + parameter_id=_groundwater_level_parameter_id(session), + start_time=start_time, + end_time=end_time, + sort=sort, + order=order, ) @@ -302,6 +372,40 @@ def get_observation_by_id( # DELETE ======================================================================= +@router.delete( + "/transducer-groundwater-level", + status_code=HTTP_200_OK, + summary="Delete transducer groundwater level observations in a time range", +) +def delete_transducer_groundwater_level_observations( + session: session_dependency, + user: amp_staging_dependency, + thing_id: int, + start_time: datetime, + end_time: datetime, +) -> DeletedTransducerObservationsResponse: + """ + Delete every transducer groundwater level reading for a well inside a + closed time range, and reconcile the blocks that covered them: a block left + with no readings is deleted, one left with some has its span narrowed to + the survivors. + + All three parameters are required -- there is deliberately no form of this + request that deletes everything for a well. Scoped exactly like the `GET` + on this path, so the set previewed there is the set removed here. + + Irreversible, and it leaves the `transducer_daily_data` materialized view + stale until its next refresh. + """ + return delete_transducer_observations( + session, + thing_id=thing_id, + parameter_id=_groundwater_level_parameter_id(session), + start_time=start_time, + end_time=end_time, + ) + + @router.delete( "/{observation_id}", summary="Delete an observation", diff --git a/api/thing.py b/api/thing.py index baeed59e7..b1176071d 100644 --- a/api/thing.py +++ b/api/thing.py @@ -27,7 +27,6 @@ ) from api.pagination import CustomPage -from core.app import public_route from core.dependencies import ( session_dependency, admin_dependency, @@ -349,7 +348,6 @@ def get_thing_id_links( return paginate(query=sql, conn=session) -@public_route @router.get("/id-link/{link_id}", summary="Get thing links by link ID") def get_thing_id_links( user: viewer_dependency, @@ -362,7 +360,6 @@ def get_thing_id_links( return simple_get_by_id(session, ThingIdLink, link_id) -@public_route @router.get("", summary="Get all things", status_code=HTTP_200_OK) def get_things( user: viewer_dependency, diff --git a/automated_ingestion/__init__.py b/automated_ingestion/__init__.py new file mode 100644 index 000000000..401409576 --- /dev/null +++ b/automated_ingestion/__init__.py @@ -0,0 +1,41 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Automated ingestion: scheduled pipelines that land external monitoring data in +Ocotillo without anyone hand-carrying a file. + +This package is deployed as its own Dagster+ code location, separate from the +API process, but it lives in this repository so the loader can import ``db/`` +models and ``domain/`` rules directly instead of maintaining a second copy of +the Ocotillo schema elsewhere. + +Shape of a source: a dlt pipeline extracts the vendor API into a GCS raw zone, +an adapter maps raw records onto Ocotillo structures, and a loader writes them +to Postgres over a direct connection. San Acacia Reach (Van Essen divers) is +the first source; ``shared/`` holds what later sources reuse. + +See ``docs/automated-ingestion-pipeline-plan.md``. + +The image installs this repository as a package (see +``dagster_cloud_post_install.sh``), so ``db``, ``domain``, and the rest resolve +from site-packages rather than from whatever happens to be on ``sys.path``. That +matters because the process that loads the code location and the process that +executes a step do not agree about the path, and the loader's imports run in the +second one. Locally an editable install produces the same result, which is why +the difference is invisible until deployment. +""" + +# ============= EOF ============================================= diff --git a/admin/__init__.py b/automated_ingestion/defs/__init__.py similarity index 77% rename from admin/__init__.py rename to automated_ingestion/defs/__init__.py index 2816d3891..4bfea2869 100644 --- a/admin/__init__.py +++ b/automated_ingestion/defs/__init__.py @@ -1,5 +1,5 @@ # =============================================================================== -# Copyright 2025 +# Copyright 2026 ross # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,12 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -""" -Starlette Admin package for OcotilloAPI. +"""Dagster definitions: the code location's assets, jobs, and schedules.""" -Provides web-based administrative interface for managing database records. -""" - -from admin.config import create_admin - -__all__ = ["create_admin"] +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/assets/__init__.py b/automated_ingestion/defs/assets/__init__.py new file mode 100644 index 000000000..4e7a0494c --- /dev/null +++ b/automated_ingestion/defs/assets/__init__.py @@ -0,0 +1,45 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Asset collection for the code location. + +Per-source assets are declared in their own modules and gathered here so +``definitions.py`` never has to know which sources exist. +""" + +from dagster import AssetsDefinition + +from automated_ingestion.defs.assets.connectivity import database_connectivity +from automated_ingestion.defs.assets.heartbeat import ingestion_heartbeat +from automated_ingestion.sources.san_acacia.ingest import ( + raw_san_acacia_locations, + raw_san_acacia_readings, + san_acacia_observations, +) + + +def all_assets() -> list[AssetsDefinition]: + """Every asset the code location exposes.""" + return [ + ingestion_heartbeat, + database_connectivity, + raw_san_acacia_locations, + raw_san_acacia_readings, + san_acacia_observations, + ] + + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/assets/connectivity.py b/automated_ingestion/defs/assets/connectivity.py new file mode 100644 index 000000000..c8c83fcbc --- /dev/null +++ b/automated_ingestion/defs/assets/connectivity.py @@ -0,0 +1,62 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Proves the Dagster+ runtime can reach Ocotillo Postgres. + +Dagster+ Serverless runs outside the VPC, so Cloud SQL's private IP is +unreachable from it -- the connection has to go through the Cloud SQL connector +instead. That is the single riskiest assumption in the foundations task, and it +fails at run time rather than at deploy time. This asset makes it fail loudly, +on its own, in an asset whose only job is to fail there. + +It reads and never writes: connectivity and permission are separable problems, +and a write here would leave test rows in a real table. +""" + +from dagster import AssetExecutionContext, MetadataValue, Output, asset + +from automated_ingestion.defs.resources import OcotilloDatabase + + +@asset( + group_name="operations", + description="Reads from Ocotillo Postgres to prove the runtime can connect.", +) +def database_connectivity( + context: AssetExecutionContext, database: OcotilloDatabase +) -> Output[int]: + """Count transducer observations, returning the count as metadata.""" + from sqlalchemy import func, select + + from db.transducer import TransducerObservation + + with database.session() as session: + count = session.scalar(select(func.count()).select_from(TransducerObservation)) + + count = int(count or 0) + context.log.info("connected to Ocotillo; transducer_observation rows: %s", count) + return Output( + count, + metadata={ + "transducer_observation_rows": MetadataValue.int(count), + "note": MetadataValue.text( + "Read-only. A failure here is connectivity or grants, not data." + ), + }, + ) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/assets/heartbeat.py b/automated_ingestion/defs/assets/heartbeat.py new file mode 100644 index 000000000..8e78157d8 --- /dev/null +++ b/automated_ingestion/defs/assets/heartbeat.py @@ -0,0 +1,71 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +A trivial asset that proves the code location deploys and materializes. + +It touches nothing -- no database, no network, no GCS -- so a failure here is +unambiguously a packaging or deployment problem rather than a credential or +connectivity one. The Postgres connectivity check that BDMS task 1.4 calls for +is a separate asset, added when the least-privilege role exists. +""" + +from datetime import datetime, timezone + +from dagster import AssetExecutionContext, MetadataValue, Output, asset + + +@asset( + group_name="operations", + description="Static heartbeat proving the code location loaded and can run.", +) +def ingestion_heartbeat(context: AssetExecutionContext) -> Output[str]: + """Return the materialization timestamp, with the import environment. + + The environment metadata is here because a step process is not the process + that loaded the code location, and the two do not necessarily agree about + sys.path. When an import that works at load time fails at execution, this is + the asset that says why -- it runs without credentials, so it reports even + when everything else is broken. + """ + import os + import sys + from importlib.util import find_spec + + stamp = datetime.now(timezone.utc).isoformat() + context.log.info("automated_ingestion code location alive at %s", stamp) + + app_root = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + ) + try: + siblings = sorted(os.listdir(app_root)) + except OSError as exc: + siblings = [f""] + + return Output( + stamp, + metadata={ + "cwd": MetadataValue.text(os.getcwd()), + "app_root": MetadataValue.text(app_root), + "app_root_contents": MetadataValue.text(", ".join(siblings)), + "db_on_path": MetadataValue.bool(find_spec("db") is not None), + "domain_on_path": MetadataValue.bool(find_spec("domain") is not None), + "sys_path": MetadataValue.json(sys.path), + }, + ) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/definitions.py b/automated_ingestion/defs/definitions.py new file mode 100644 index 000000000..1bf2f7947 --- /dev/null +++ b/automated_ingestion/defs/definitions.py @@ -0,0 +1,41 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Entry point for the ``ocotillo-automated-ingestion`` Dagster+ code location. + +``[tool.dagster] module_name`` in ``pyproject.toml`` points here, so this is +what ``dagster dev`` and the Dagster+ agent import. Keep it thin: it collects +definitions declared elsewhere in the package rather than declaring them here. +""" + +from dagster import Definitions + +from automated_ingestion.defs.assets import all_assets +from automated_ingestion.defs.jobs.heartbeat import heartbeat_job +from automated_ingestion.defs.jobs.san_acacia import ( + san_acacia_job, + san_acacia_weekly_schedule, +) +from automated_ingestion.defs.resources import OcotilloDatabase + +defs = Definitions( + assets=all_assets(), + jobs=[heartbeat_job, san_acacia_job], + schedules=[san_acacia_weekly_schedule], + resources={"database": OcotilloDatabase()}, +) + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/jobs/__init__.py b/automated_ingestion/defs/jobs/__init__.py new file mode 100644 index 000000000..a33f53655 --- /dev/null +++ b/automated_ingestion/defs/jobs/__init__.py @@ -0,0 +1,18 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Jobs: backfill and any other non-schedule-driven runs.""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/jobs/backfill.py b/automated_ingestion/defs/jobs/backfill.py new file mode 100644 index 000000000..18236a0b2 --- /dev/null +++ b/automated_ingestion/defs/jobs/backfill.py @@ -0,0 +1,30 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Backfill job factory. + +Two modes are planned, both filled in under BDMS task 4: + +- **Mode A (refetch)** re-pulls a window from the vendor API when a gap is real + data we never collected. +- **Mode B (replay)** reprocesses parquet already in the GCS raw zone through + the current adapter, with no API calls, when the bug was in our mapping. + +Both chunk the window, checkpoint per chunk so an interrupted run resumes, and +default to ``dry_run=True``. +""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/jobs/heartbeat.py b/automated_ingestion/defs/jobs/heartbeat.py new file mode 100644 index 000000000..d95f80b66 --- /dev/null +++ b/automated_ingestion/defs/jobs/heartbeat.py @@ -0,0 +1,49 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +A named job wrapping the heartbeat asset, so a deploy can be smoke-tested +without the Dagster+ UI. + +The asset alone is not enough for that: `dagster-cloud-action`'s `launch_job` +identifies what to run by job name and exposes no asset selection, so an asset +reachable only through the implicit `__ASSET_JOB` cannot be launched from CI. +`.github/workflows/smoke_dagster_location.yml` runs this one. + +Worth having because a successful deploy is weaker evidence than a successful +run. The agent loading the code location proves the *loader* process can import +the package; it says nothing about the process that executes a step, which is a +different process with a different sys.path. See the note in +`assets/heartbeat.py` -- that gap is the whole reason the asset exists, and this +job is how CI closes it. +""" + +from dagster import AssetSelection, define_asset_job + +from automated_ingestion.defs.assets.heartbeat import ingestion_heartbeat + +heartbeat_job = define_asset_job( + name="ingestion_heartbeat_check", + selection=AssetSelection.assets(ingestion_heartbeat), + description=( + "Materialize the heartbeat asset only. Touches no database, no network, " + "and no GCS, so a failure is a packaging or deployment problem." + ), + # No retry policy. A retry would mask exactly the failure this job exists to + # surface: an import that works at load time and fails at execution does so + # deterministically. +) + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/jobs/san_acacia.py b/automated_ingestion/defs/jobs/san_acacia.py new file mode 100644 index 000000000..d38c1c35c --- /dev/null +++ b/automated_ingestion/defs/jobs/san_acacia.py @@ -0,0 +1,80 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +The scheduled run for San Acacia Reach. + +One job over the whole `san_acacia` asset group, so the three steps stay in +order: land the point roster, land the readings, then map and load them. Dagster +resolves that from the asset dependencies rather than from anything declared +here, which is why the selection is by group -- a fourth asset added to the +group joins the schedule without this file changing. + +Weekly rather than daily. These are five-minute diver readings and nobody is +watching them in real time; the vendor's endpoint answers 500 when pushed, and a +weekly cadence keeps each run's windows comfortably inside what it serves. The +watermark makes the interval a matter of freshness rather than correctness: a +run fetches from wherever the last one finished, so a missed week is picked up +by the next run rather than lost. +""" + +from dagster import ( + AssetSelection, + DefaultScheduleStatus, + RetryPolicy, + ScheduleDefinition, + define_asset_job, +) + +SAN_ACACIA_GROUP = "san_acacia" + +san_acacia_job = define_asset_job( + name="san_acacia_ingest", + selection=AssetSelection.groups(SAN_ACACIA_GROUP), + description=( + "Land the San Acacia point roster and readings in the raw zone, then " + "map and load them into Ocotillo." + ), + # A retry covers the vendor dropping a request or a token expiring mid-run. + # Two attempts, not more: a persistent 500 means the window is wrong or the + # endpoint is unwell, and hammering it makes both worse. + op_retry_policy=RetryPolicy(max_retries=2, delay=60), +) + +san_acacia_weekly_schedule = ScheduleDefinition( + name="san_acacia_weekly", + job=san_acacia_job, + # Mondays at 05:00 America/Denver -- after midnight so a run covers whole + # days, and early enough that a failure is visible at the start of the week + # rather than discovered the following Monday. + cron_schedule="0 5 * * 1", + execution_timezone="America/Denver", + # Local time rather than UTC deliberately: the wells, the people who read + # the data, and the working day are all in one timezone, so a schedule that + # shifts by an hour twice a year would be the surprising choice. + # + # Stopped by default. Turning it on starts writing to Ocotillo, and the + # first run for the 24 wells without history fetches back to the + # `INITIAL_START` floor. That should be somebody's decision, taken once, + # rather than a consequence of a merge. + default_status=DefaultScheduleStatus.STOPPED, + description=( + "Weekly San Acacia ingest. Each run resumes from each series' " + "watermark, so a missed week is caught up rather than lost." + ), +) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/resources.py b/automated_ingestion/defs/resources.py new file mode 100644 index 000000000..25172bfa1 --- /dev/null +++ b/automated_ingestion/defs/resources.py @@ -0,0 +1,61 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Dagster resources: the pipeline's handles on the outside world. + +The database resource deliberately delegates to ``db/engine.py`` rather than +building its own engine. Connection setup for Cloud SQL -- the connector, IAM +auth, the IP-type choice -- is intricate and already solved there; a second +implementation would be a second thing to get wrong, and would drift. +""" + +from collections.abc import Iterator +from contextlib import contextmanager + +from dagster import ConfigurableResource + + +class OcotilloDatabase(ConfigurableResource): + """A session against the Ocotillo database. + + Configured entirely through the environment that ``db/engine.py`` reads + (``DB_DRIVER``, ``CLOUD_SQL_*``), so the Dagster+ code location is + configured the same way the API is, with different credentials. + """ + + @contextmanager + def session(self) -> Iterator[object]: + """Yield a SQLAlchemy session, rolled back and closed on the way out.""" + # Credentials first: db.engine builds its Cloud SQL connector at import + # time, and the connector resolves Application Default Credentials right + # then. Serverless has none until they are written to disk, so doing this + # afterwards would be too late. + from automated_ingestion.shared.credentials import ( + ensure_application_default_credentials, + ) + + ensure_application_default_credentials() + + # Imported lazily: importing db.engine builds an engine from the + # environment at import time, which should happen when a run asks for a + # session, not when Dagster loads the code location to list assets. + from db.engine import session_ctx + + with session_ctx() as session: + yield session + + +# ============= EOF ============================================= diff --git a/automated_ingestion/iac/.gitignore b/automated_ingestion/iac/.gitignore new file mode 100644 index 000000000..d55808b94 --- /dev/null +++ b/automated_ingestion/iac/.gitignore @@ -0,0 +1,9 @@ +.terraform/ +.terraform.lock.hcl +# Written while a plan or apply holds the state lock, and left behind if the +# run is interrupted. Machine-specific and never useful to another checkout. +.terraform.tfstate.lock.info +terraform.tfstate +terraform.tfstate.* +terraform.tfvars +*.tfplan diff --git a/automated_ingestion/iac/main.tf b/automated_ingestion/iac/main.tf new file mode 100644 index 000000000..295cf3a35 --- /dev/null +++ b/automated_ingestion/iac/main.tf @@ -0,0 +1,163 @@ +# Raw-zone storage for the automated ingestion pipeline. +# +# Two buckets, one per environment, plus the service account the Dagster+ code +# location uses to write to them. Deliberately narrow: this configuration owns +# ingestion storage and nothing else, so a mistake here cannot affect the API's +# uploads bucket or any other project resource. +# +# Not applied by CI. Run it by hand, review the plan, and record the applied +# state -- see README.md. + +terraform { + required_version = ">= 1.5" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 6.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} + +locals { + environments = toset(["production", "staging"]) +} + +resource "google_storage_bucket" "ingestion_raw" { + for_each = local.environments + + name = "ocotillo-ingestion-${each.key}" + project = var.project_id + + # Bucket location is immutable: changing it replaces the bucket. That is + # tolerable only while the raw zone is empty. Once a backfill has landed, + # moving regions means copying objects across and re-pointing the pipeline, + # not editing this line. + location = var.bucket_location + + # The raw zone is the replay source for Mode B backfill: reprocessing a + # mapping bug must not depend on the vendor still serving that window. + # Deleting an object here is therefore a data-loss event, not a cleanup. + force_destroy = false + uniform_bucket_level_access = true + public_access_prevention = "enforced" + + versioning { + enabled = true + } + + # Raw payloads are read constantly for the first month (recent-window + # replays), then almost never. Age-out to colder classes rather than + # deleting: an old window is exactly what a historical replay needs. + lifecycle_rule { + condition { + age = 30 + } + action { + type = "SetStorageClass" + storage_class = "NEARLINE" + } + } + + lifecycle_rule { + condition { + age = 365 + } + action { + type = "SetStorageClass" + storage_class = "COLDLINE" + } + } + + # Bucket versioning would otherwise retain every superseded object forever. + lifecycle_rule { + condition { + num_newer_versions = 3 + with_state = "ARCHIVED" + } + action { + type = "Delete" + } + } + + labels = { + component = "automated-ingestion" + env = each.key + } +} + +resource "google_service_account" "ingestion" { + account_id = "ocotillo-ingestion" + display_name = "Ocotillo automated ingestion" + description = "Writes raw vendor payloads to the ingestion buckets from the Dagster+ code location." + project = var.project_id +} + +# Scoped to the two buckets, not granted at project level. objectAdmin rather +# than objectCreator because a replay overwrite rewrites an existing object. +resource "google_storage_bucket_iam_member" "ingestion_object_admin" { + for_each = google_storage_bucket.ingestion_raw + + bucket = each.value.name + role = "roles/storage.objectAdmin" + member = "serviceAccount:${google_service_account.ingestion.email}" +} + + +# objectAdmin covers objects and says nothing about the bucket itself, so it +# does not include storage.buckets.get. gcsfs checks a bucket exists before +# writing to it, that check is denied, and GCS reports a denial as absence -- +# so the pipeline fails with "Bucket does not exist" for a bucket that plainly +# does. +# +# legacyBucketReader adds buckets.get and objects.list and nothing else. It is +# the narrowest standard role that makes the existence check succeed; the +# alternative, storage.admin, would also grant deletion of the bucket. +resource "google_storage_bucket_iam_member" "ingestion_bucket_reader" { + for_each = google_storage_bucket.ingestion_raw + + bucket = each.value.name + role = "roles/storage.legacyBucketReader" + member = "serviceAccount:${google_service_account.ingestion.email}" +} + +# Database access for the ingestion service account. +# +# Only created when `cloud_sql_instance` is set, so the storage half of this +# configuration can be applied before the database half is decided. +# +# These grants are what make IAM database authentication work. Without them the +# Postgres role in automated_ingestion/sql/ingestion_role.sql exists but cannot +# be reached: the connector fails while acquiring a token, which surfaces as an +# authentication error and reads like a missing GRANT. +resource "google_project_iam_member" "ingestion_cloudsql_client" { + count = var.cloud_sql_instance == null ? 0 : 1 + + project = var.project_id + role = "roles/cloudsql.client" + member = "serviceAccount:${google_service_account.ingestion.email}" +} + +resource "google_project_iam_member" "ingestion_cloudsql_instance_user" { + count = var.cloud_sql_instance == null ? 0 : 1 + + project = var.project_id + role = "roles/cloudsql.instanceUser" + member = "serviceAccount:${google_service_account.ingestion.email}" +} + +# Registers the service account as a database user. The Postgres role itself, +# and its grants, come from ingestion_role.sql -- this only makes the login +# possible. +resource "google_sql_user" "ingestion" { + count = var.cloud_sql_instance == null ? 0 : 1 + + name = trimsuffix(google_service_account.ingestion.email, ".gserviceaccount.com") + instance = var.cloud_sql_instance + project = var.project_id + type = "CLOUD_IAM_SERVICE_ACCOUNT" +} diff --git a/automated_ingestion/iac/outputs.tf b/automated_ingestion/iac/outputs.tf new file mode 100644 index 000000000..77b57e6fc --- /dev/null +++ b/automated_ingestion/iac/outputs.tf @@ -0,0 +1,9 @@ +output "bucket_names" { + description = "Raw-zone bucket per environment. The matching value goes into INGESTION_GCS_BUCKET on the Dagster+ code location." + value = { for k, b in google_storage_bucket.ingestion_raw : k => b.name } +} + +output "service_account_email" { + description = "Ingestion service account. Grant nothing else to it without revisiting the least-privilege rationale in README.md." + value = google_service_account.ingestion.email +} diff --git a/automated_ingestion/iac/terraform.tfvars.example b/automated_ingestion/iac/terraform.tfvars.example new file mode 100644 index 000000000..6e1830113 --- /dev/null +++ b/automated_ingestion/iac/terraform.tfvars.example @@ -0,0 +1 @@ +project_id = "waterdatainitiative-271000" diff --git a/automated_ingestion/iac/variables.tf b/automated_ingestion/iac/variables.tf new file mode 100644 index 000000000..2fae9003a --- /dev/null +++ b/automated_ingestion/iac/variables.tf @@ -0,0 +1,22 @@ +variable "project_id" { + type = string + description = "GCP project that owns the ingestion buckets and service account." +} + +variable "region" { + type = string + description = "Default provider region." + default = "us-central1" +} + +variable "bucket_location" { + type = string + description = "Bucket location. Must match the Cloud SQL region so replay reads do not cross regions and pay egress. The dataservices instance is in us-west4." + default = "US-WEST4" +} + +variable "cloud_sql_instance" { + type = string + description = "Cloud SQL instance name for the IAM database user. Leave null to skip the database grants entirely -- useful before the instance is known, or when using password authentication instead." + default = null +} diff --git a/automated_ingestion/ocotillo/__init__.py b/automated_ingestion/ocotillo/__init__.py new file mode 100644 index 000000000..1b346fc5d --- /dev/null +++ b/automated_ingestion/ocotillo/__init__.py @@ -0,0 +1,24 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +The Ocotillo-facing half of ingestion: adapters and the structures they emit. + +Source packages know their vendor's payload shape; this package knows +Ocotillo's. An adapter is the seam between them, so adding a source means +writing an adapter rather than touching the loader. +""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/ocotillo/adapter.py b/automated_ingestion/ocotillo/adapter.py new file mode 100644 index 000000000..7ba38f2f6 --- /dev/null +++ b/automated_ingestion/ocotillo/adapter.py @@ -0,0 +1,46 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Base class for source adapters. + +An adapter converts one source's raw records into Ocotillo structures. It is +the only place a vendor's vocabulary appears alongside Ocotillo's, which keeps +vendor quirks out of ``domain/`` and out of the loader. +""" + +from abc import ABC, abstractmethod +from collections.abc import Iterable, Iterator +from typing import Any + +from automated_ingestion.ocotillo.structs import ObservationRecord + + +class SourceAdapter(ABC): + """Maps one source's raw records onto Ocotillo structures.""" + + @property + @abstractmethod + def source_key(self) -> str: + """Registry key of the source this adapter serves.""" + + @abstractmethod + def to_observations( + self, records: Iterable[dict[str, Any]] + ) -> Iterator[ObservationRecord]: + """Convert raw vendor records into observation records.""" + + +# ============= EOF ============================================= diff --git a/automated_ingestion/ocotillo/loader.py b/automated_ingestion/ocotillo/loader.py new file mode 100644 index 000000000..cabafccf5 --- /dev/null +++ b/automated_ingestion/ocotillo/loader.py @@ -0,0 +1,228 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Idempotent loading of observations into Ocotillo. + +Every write is an upsert against the unique constraint on +``(deployment_id, parameter_id, observation_datetime)``. That makes a re-run a +no-op rather than a duplication, which is what lets a backfill overlap existing +data safely. + +The alternative -- delete the window, then insert -- is what Aqueduct does +against FROST, because there is no constraint there to conflict on. It leaves a +window during which the data is simply missing, and a failure mid-way leaves it +missing permanently. Upserting has no such window. + +Rows are written with SQLAlchemy Core rather than ORM objects. ``AGENTS.md`` +is explicit about this for high-volume tables: instantiating a mapped class per +observation is what turns a backfill into an hour-long run. +""" + +from collections.abc import Iterable, Iterator +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +DEFAULT_BATCH_SIZE = 5_000 +"""Rows per statement. + +Large enough that a month of five-minute readings is a handful of round trips, +small enough that one batch's parameters do not approach Postgres' limit. Each +batch commits on its own, so an interrupted load keeps what it had already +written -- with an upsert, resuming simply rewrites those rows. +""" + + +@dataclass +class LoadResult: + """What a load did, for reporting as asset metadata.""" + + rows_seen: int = 0 + rows_written: int = 0 + batches: int = 0 + blocks_touched: list[int] = field(default_factory=list) + + @property + def rows_skipped(self) -> int: + return self.rows_seen - self.rows_written + + +def _batched(records: Iterable[Any], size: int) -> Iterator[list[Any]]: + batch: list[Any] = [] + for record in records: + batch.append(record) + if len(batch) >= size: + yield batch + batch = [] + if batch: + yield batch + + +DEFAULT_DATA_MATURITY = "provisional" +"""Maturity for a freshly ingested reading. + +USGS publishes unapproved records as provisional -- "provisional data subject to +revision" -- and that is what a diver reading is until somebody reviews it. +Orthogonal to ``release_status``: San Acacia data is public *and* provisional, +which is why this is a second column rather than another value in the first. +""" + + +def load_observations( + session: Any, + records: Iterable[Any], + deployment_id: int, + parameter_id: int, + release_status: str, + batch_size: int = DEFAULT_BATCH_SIZE, + data_maturity: str = DEFAULT_DATA_MATURITY, + overwrite_approved: bool = False, +) -> LoadResult: + """Upsert observations, committing per batch. + + ``records`` are ``ObservationRecord`` values from an adapter; resolving a + source's point identifier to a deployment belongs to reference-data + bootstrapping, not here, so the caller supplies the ids. + + ``overwrite_approved`` guards data somebody has already reviewed. By default + a row whose ``data_maturity`` is ``approved`` is left alone: the upsert exists + so a vendor correction can revise *our* provisional readings, not so a + re-fetch can quietly replace Bureau-approved history with a vendor's numbers + and downgrade it to provisional. + + This is not hypothetical. Fourteen of the thirty-eight San Acacia wells + already hold 542,161 approved observations from the AMPAPI transfer, running + to August 2022. A Mode A backfill over that window would have overwritten + every one of them. + + Setting it to True is a deliberate act: it says the incoming data is better + than what was reviewed, which is a judgement a person should make. + """ + from sqlalchemy.dialects.postgresql import insert + + from db.transducer import TransducerObservation + + result = LoadResult() + table = TransducerObservation.__table__ + + for batch in _batched(records, batch_size): + result.rows_seen += len(batch) + + # One row per instant within a statement. Postgres refuses an + # ON CONFLICT DO UPDATE that would touch the same row twice in one + # command, and a source can repeat a reading -- overlapping fetch + # windows, or a vendor logging the same instant twice. Keeping the last + # occurrence matches the upsert's own rule: a later value wins. + deduplicated = {record.observation_datetime: record for record in batch} + rows = [ + { + "deployment_id": deployment_id, + "parameter_id": parameter_id, + "observation_datetime": record.observation_datetime, + "value": record.value, + "release_status": release_status, + "data_maturity": data_maturity, + } + for record in deduplicated.values() + ] + if not rows: + continue + + statement = insert(table).values(rows) + # DO UPDATE rather than DO NOTHING: a vendor may correct a reading, and + # a correction arriving as a no-op would leave the old value in place + # while the run reported success. + conflict_kwargs: dict[str, Any] = { + "index_elements": [ + "deployment_id", + "parameter_id", + "observation_datetime", + ], + "set_": { + "value": statement.excluded.value, + "data_maturity": statement.excluded.data_maturity, + }, + } + if not overwrite_approved: + # IS DISTINCT FROM rather than != so NULL maturity still updates: + # a row with no recorded status has not been reviewed, and treating + # unknown as approved would freeze 394,086 legacy rows against every + # future correction. + conflict_kwargs["where"] = table.c.data_maturity.is_distinct_from( + "approved" + ) + statement = statement.on_conflict_do_update(**conflict_kwargs) + session.execute(statement) + session.commit() + + result.rows_written += len(rows) + result.batches += 1 + + return result + + +def ensure_block( + session: Any, + thing_id: int, + parameter_id: int, + start: datetime, + end: datetime, + release_status: str, + review_status: str = "not reviewed", +) -> int: + """Create or widen the QC block covering a loaded window. + + ``review_status`` defaults to ``not reviewed`` and callers should leave it + there. In Ocotillo ``approved`` asserts that a Bureau human reviewed the + data and carries a ``reviewer_id``; the vendor's own approval flag is a + different claim and is preserved separately. + + An existing block is widened rather than duplicated, so re-running a window + does not accumulate blocks. + """ + from sqlalchemy import select + + from db.transducer import TransducerObservationBlock + + existing = session.scalars( + select(TransducerObservationBlock) + .where(TransducerObservationBlock.thing_id == thing_id) + .where(TransducerObservationBlock.parameter_id == parameter_id) + .where(TransducerObservationBlock.review_status == review_status) + .where(TransducerObservationBlock.start_datetime <= end) + .where(TransducerObservationBlock.end_datetime >= start) + ).first() + + if existing is not None: + existing.start_datetime = min(existing.start_datetime, start) + existing.end_datetime = max(existing.end_datetime, end) + session.commit() + return existing.id + + block = TransducerObservationBlock( + thing_id=thing_id, + parameter_id=parameter_id, + review_status=review_status, + start_datetime=start, + end_datetime=end, + release_status=release_status, + ) + session.add(block) + session.commit() + return block.id + + +# ============= EOF ============================================= diff --git a/automated_ingestion/ocotillo/structs.py b/automated_ingestion/ocotillo/structs.py new file mode 100644 index 000000000..90cc29868 --- /dev/null +++ b/automated_ingestion/ocotillo/structs.py @@ -0,0 +1,44 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Plain structures passed from an adapter to the loader. + +These are deliberately not SQLAlchemy models. An adapter is pure and testable +without a database session; turning these into rows is the loader's job. +""" + +from dataclasses import dataclass +from datetime import datetime + + +@dataclass(frozen=True) +class ObservationRecord: + """One timestamped reading, already in Ocotillo's units and datum.""" + + external_point_id: str + """The vendor's identifier for the monitoring point.""" + + observation_datetime: datetime + """Timezone-aware instant of the reading.""" + + value: float + """Measurement in ``units``, on the datum the source's mapping fixes.""" + + units: str + """Unit symbol as it appears in the Ocotillo lexicon.""" + + +# ============= EOF ============================================= diff --git a/automated_ingestion/scripts/__init__.py b/automated_ingestion/scripts/__init__.py new file mode 100644 index 000000000..94919dc13 --- /dev/null +++ b/automated_ingestion/scripts/__init__.py @@ -0,0 +1,18 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""One-off instruments. Nothing here is imported by the pipeline.""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/scripts/compare_datum.py b/automated_ingestion/scripts/compare_datum.py new file mode 100644 index 000000000..b513ffdf5 --- /dev/null +++ b/automated_ingestion/scripts/compare_datum.py @@ -0,0 +1,217 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Check that ingested readings agree with the observations Ocotillo already holds. + +Fourteen San Acacia wells carry AMPAPI transducer data through August 2022, +loaded under a datum nobody has verified. This pipeline reads Diver-HUB with +``reference=3`` and converts centimetres to feet. If those disagree, the same +series ends up holding two datums -- and the numbers look plausible either way, +which is the failure this source is most prone to. + +Magnitude alone cannot settle it: ``reference=1`` (top of casing) differs from +``reference=3`` (ground surface) by a fixed 45.456 cm -- about 1.49 ft -- which +is well inside the natural range of these wells. Only values at the *same +instant* separate them, so this compares timestamp by timestamp. + +It fetches all four references rather than just the one in use, so the output +also independently confirms which reference Ocotillo's existing data was loaded +against. + + export DIVERHUB_USERNAME=... DIVERHUB_PASSWORD=... + uv run --group ingestion python -m \\ + automated_ingestion.scripts.compare_datum --well SO-0125 + +Read-only on both sides. +""" + +import argparse +import statistics +import sys +from datetime import timedelta + +REFERENCES = (0, 1, 2, 3) + + +def _existing(cursor, well: str, limit: int): + cursor.execute( + """ + SELECT o.observation_datetime, o.value + FROM transducer_observation o + JOIN deployment d ON d.id = o.deployment_id + JOIN thing t ON t.id = d.thing_id + WHERE t.name = %s + ORDER BY o.observation_datetime DESC + LIMIT %s + """, + (well, limit), + ) + return cursor.fetchall() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--well", default="SO-0125", help="Ocotillo PointID") + parser.add_argument("--point-id", type=int, help="Diver-HUB monitoring point id") + parser.add_argument( + "--instance", default="waterdatainitiative-271000:us-west4:dataservices" + ) + parser.add_argument("--database", default="ocotillo-staging") + parser.add_argument("--samples", type=int, default=200) + parser.add_argument( + "--tolerance-minutes", + type=int, + default=30, + help=( + "How far apart two readings may be and still count as the same " + "instant. Exact equality is too strict: the existing rows are on the " + "hour and the vendor logs at 15-minute offsets." + ), + ) + args = parser.parse_args() + + import requests + from google.cloud.sql.connector import Connector + + from automated_ingestion.sources.san_acacia.client import DiverHubClient + from automated_ingestion.sources.san_acacia.dlt_pipeline import PROJECT_ID + from domain.units import convert_cm_to_ft + from domain.van_essen import parse_reading_timestamp + + client = DiverHubClient(requests.Session()) + + point_id = args.point_id + if point_id is None: + matches = [ + p for p in client.monitoring_points(PROJECT_ID) if p["name"] == args.well + ] + if not matches: + print(f"{args.well} is not a Diver-HUB monitoring point.", file=sys.stderr) + return 2 + point_id = matches[0]["id"] + + connector = Connector() + conn = connector.connect( + args.instance, + "pg8000", + user=_account(), + db=args.database, + enable_iam_auth=True, + ) + try: + rows = _existing(conn.cursor(), args.well, args.samples) + finally: + conn.close() + connector.close() + + if not rows: + print(f"No existing observations for {args.well}.", file=sys.stderr) + return 1 + + existing = {stamp.replace(tzinfo=None): value for stamp, value in rows} + start = min(existing) - timedelta(days=1) + end = max(existing) + timedelta(days=1) + print(f"{args.well} (Diver-HUB point {point_id})") + print( + f" {len(existing)} existing observations, {min(existing)} -> {max(existing)}" + ) + print( + f" Ocotillo values: {min(existing.values()):.2f} .. {max(existing.values()):.2f} ft\n" + ) + + tolerance = timedelta(minutes=args.tolerance_minutes) + print( + f" {'reference':<12}{'vendor rows':>12}{'matched':>9}" + f"{'mean diff ft':>15}{'max diff ft':>14}" + ) + best = None + for reference in REFERENCES: + vendor = {} + for row in client.water_levels( + point_id, + int(start.timestamp()), + int(end.timestamp()), + reference=reference, + ): + if row.get("level") is None: + continue + stamp = parse_reading_timestamp(row["dateAndTime"]).replace(tzinfo=None) + vendor[stamp] = convert_cm_to_ft(row["level"]) + + # Nearest within tolerance rather than exact equality. A reading logged + # at :45 against one recorded on the hour is the same measurement to + # anyone comparing datums; insisting on identical timestamps finds + # nothing and says nothing. + stamps = sorted(vendor) + diffs = [] + for stamp, value in existing.items(): + near = min(stamps, key=lambda s: abs(s - stamp)) if stamps else None + if near is not None and abs(near - stamp) <= tolerance: + diffs.append(abs(value - vendor[near])) + + if not diffs: + print(f" reference={reference:<4}{len(vendor):>10}{'none':>11}") + continue + + mean, worst = statistics.mean(diffs), max(diffs) + print( + f" reference={reference:<4}{len(vendor):>10}{len(diffs):>9}" + f"{mean:>15.3f}{worst:>14.3f}" + ) + if best is None or mean < best[1]: + best = (reference, mean) + + if best is None: + print("\n Nothing to compare.") + print( + " If vendor rows is 0, Diver-HUB does not retain this window for " + "this point -- try a well whose data runs later, or widen --tolerance-minutes." + ) + return 1 + + reference, mean = best + print(f"\n Closest: reference={reference}, mean difference {mean:.3f} ft") + if mean < 0.05: + verdict = ( + f"Ocotillo's existing data matches reference={reference}." + if reference == 3 + else f"Ocotillo's existing data was loaded on reference={reference}, NOT 3." + ) + else: + verdict = ( + "No reference matches closely. The existing data may use a different " + "unit, datum or correction than any raw Diver-HUB series." + ) + print(f" {verdict}") + return 0 + + +def _account() -> str: + import subprocess + + return subprocess.run( + ["gcloud", "config", "get-value", "account"], + capture_output=True, + text=True, + timeout=30, + ).stdout.strip() + + +if __name__ == "__main__": + raise SystemExit(main()) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/scripts/diverhub_retention.py b/automated_ingestion/scripts/diverhub_retention.py new file mode 100644 index 000000000..0f23541de --- /dev/null +++ b/automated_ingestion/scripts/diverhub_retention.py @@ -0,0 +1,121 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Find how far back Diver-HUB actually serves each monitoring point. + +This matters for two reasons. + +``INITIAL_START`` is 2015-01-01, a floor chosen before anyone knew what the +vendor retains. A first run for a well with no history walks from there in +windows, and every window before the vendor's earliest reading is a request that +returns nothing -- against an endpoint that answers 500 when pushed. + +And the fourteen wells that already hold AMPAPI data stop in August 2022, while +the vendor appears to start much later. If so the two datasets never overlap, +which is why the datum comparison found nothing to compare: there is a gap +between them, not a seam. + +Binary search on presence, roughly ten requests per point rather than a walk. + + export DIVERHUB_USERNAME=... DIVERHUB_PASSWORD=... + uv run --group ingestion python -m \\ + automated_ingestion.scripts.diverhub_retention --limit 6 +""" + +import argparse +from datetime import datetime, timedelta, timezone + +PROBE_WINDOW = timedelta(days=30) + + +def _has_data(client, point_id: int, when: datetime, reference: int) -> bool: + """Is there any reading in the month starting at ``when``?""" + rows = client.water_levels( + point_id, + int(when.timestamp()), + int((when + PROBE_WINDOW).timestamp()), + reference=reference, + span=int(PROBE_WINDOW.total_seconds()), + ) + return any(True for _ in rows) + + +def earliest_reading( + client, point_id: int, reference: int, floor: datetime +) -> datetime | None: + """Approximate the first month that holds data, by bisection.""" + now = datetime.now(tz=timezone.utc) + if not _has_data(client, point_id, now - PROBE_WINDOW, reference): + # Nothing recent; the point may be retired. Fall back to a wide check. + if not _has_data(client, point_id, floor, reference): + pass # keep searching regardless -- absence now proves nothing + + low, high = floor, now + if _has_data(client, point_id, low, reference): + return low + + # Invariant: no data at `low`, data somewhere at or before `high`. + for _ in range(12): + if (high - low) <= PROBE_WINDOW: + break + middle = low + (high - low) / 2 + if _has_data(client, point_id, middle, reference): + high = middle + else: + low = middle + return high if _has_data(client, point_id, high - PROBE_WINDOW, reference) else high + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--limit", type=int, default=6, help="How many points to probe") + parser.add_argument("--floor", default="2015-01-01T00:00:00+00:00") + args = parser.parse_args() + + import requests + + from automated_ingestion.sources.san_acacia.client import ( + GROUND_SURFACE_REFERENCE, + DiverHubClient, + ) + from automated_ingestion.sources.san_acacia.dlt_pipeline import PROJECT_ID + from domain.van_essen import parse_reading_timestamp + + client = DiverHubClient(requests.Session()) + floor = parse_reading_timestamp(args.floor) + points = client.monitoring_points(PROJECT_ID)[: args.limit] + + print(f"Probing {len(points)} of {PROJECT_ID}'s monitoring points") + print(f" {'point':<12}{'earliest data (approx)':>26}") + for point in points: + found = earliest_reading(client, point["id"], GROUND_SURFACE_REFERENCE, floor) + shown = found.date().isoformat() if found else "none found" + print(f" {point['name']:<12}{shown:>26}") + + print( + "\nIf these cluster well after August 2022, the vendor and the existing\n" + "AMPAPI records do not overlap, and INITIAL_START can be raised to the\n" + "earliest date actually served -- saving a decade of empty requests on\n" + "every first run." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/scripts/probe_diverhub.py b/automated_ingestion/scripts/probe_diverhub.py new file mode 100644 index 000000000..b71cca638 --- /dev/null +++ b/automated_ingestion/scripts/probe_diverhub.py @@ -0,0 +1,302 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Answer the open questions in BDMS task 2.1 against the live Diver-HUB API. + +Run once, with credentials, and fold the output into +``docs/sources/san_acacia.md``. It reads and never writes. + + export DIVERHUB_USERNAME=... DIVERHUB_PASSWORD=... + uv run --group ingestion python -m automated_ingestion.scripts.probe_diverhub + +What it settles: + +* Which project holds San Acacia Reach, and whether it really has 33 points. +* **Which ``reference`` value is ground surface.** The swagger declares the enum + as ``[0, 1, 2, 3]`` and says nothing else, so this prints a sample from each + side by side. The ground-surface series is recognisable by magnitude and sign + against a well whose depth to water is roughly known -- a judgement a person + has to make, which is why this script prints rather than decides. +* The window ceiling. Three months is known good; this widens until the API + answers 500, so the production span is measured rather than guessed. + +Nothing here is imported by the pipeline. It is a one-off instrument. +""" + +import sys +from datetime import datetime, timezone + +from automated_ingestion.shared.windows import DAY +from automated_ingestion.sources.san_acacia.client import ( + DiverHubClient, + DiverHubError, +) + +REFERENCE_VALUES = (0, 1, 2, 3) + + +def _session(): + import requests + + return requests.Session() + + +def _iso(unix: int) -> str: + return datetime.fromtimestamp(unix, tz=timezone.utc).isoformat() + + +def probe_projects(client: DiverHubClient) -> list[dict]: + print("== Projects visible to these credentials ==") + projects = client.projects() + for project in projects: + print(f" {project['id']:>6} {project['name']}") + return projects + + +def probe_points(client: DiverHubClient, project_id: int) -> list[dict]: + print(f"\n== Monitoring points in project {project_id} ==") + points = client.monitoring_points(project_id) + print(f" {len(points)} points (the plan expects 33)") + if len(points) != 33: + print( + " ^ count differs from the plan; listing all so the extras\n can be identified before anything is reconciled." + ) + for point in points: + print(f" {point['id']:>6} {point['name']}") + return points + for point in points[:5]: + print(f" {point['id']:>6} {point['name']}") + if len(points) > 5: + print(f" ... and {len(points) - 5} more") + return points + + +def probe_reference_values( + client: DiverHubClient, points: list[dict], end: int +) -> None: + """Sample each reference value so a human can tell which is ground surface. + + Searches over a year, and moves on to another point if the first has gone + quiet -- a diver that stopped reporting months ago tells us nothing about + what the enum means. + """ + print("\n== WaterLevelReference values ==") + print(" Ground surface reads as depth below ground: positive, and") + print(" plausible as feet below surface. An elevation is a much larger") + print(" number. A vrd/TOC series looks like ground surface but is offset") + print(" by the stickup, so compare against a well you know.\n") + + start = end - 365 * DAY + for point in points[:6]: + point_id, name = point["id"], point["name"] + found = False + for reference in REFERENCE_VALUES: + try: + rows = list( + client.water_levels(point_id, start, end, reference=reference) + ) + except DiverHubError as exc: + print(f" {name} reference={reference}: error -- {exc}") + continue + if not rows: + print(f" {name} reference={reference}: no rows in 365d") + continue + found = True + levels = [r["level"] for r in rows if r.get("level") is not None] + print( + f" {name} reference={reference}: {len(rows):>5} rows, " + f"min={min(levels):>10.3f} max={max(levels):>10.3f} " + f"first={rows[0].get('dateAndTime')} last={rows[-1].get('dateAndTime')}" + ) + if found: + print(f"\n ^ compare these four for {name} and pick the datum.") + return + print(" No point returned water levels in the last year.") + + +def probe_window_ceiling(client: DiverHubClient, point_id: int, end: int) -> None: + """Find what actually triggers a 500. + + Widening from the present tests span. Sliding a fixed narrow window back + through time tests whether the failure is instead about *when* -- a range + that predates the point's data. The two look identical from the status + code, so both are worth separating here. + """ + print(f"\n== Window behaviour for point {point_id} ==") + print(" Widening back from now (tests span):") + for days in (90, 180, 365, 545, 730): + start = end - days * DAY + try: + rows = list( + client.water_levels( + point_id, + start, + end, + reference=REFERENCE_VALUES[0], + span=days * DAY, + ) + ) + print(f" {days:>5}d: ok, {len(rows)} rows") + except DiverHubError: + print(f" {days:>5}d: 500 even at the one-day floor") + + print(" Fixed 30-day window slid backwards (tests age, not span):") + for years_back in (0, 1, 2, 3): + window_end = end - years_back * 365 * DAY + window_start = window_end - 30 * DAY + label = f"{years_back}y ago" + try: + rows = list( + client.water_levels( + point_id, + window_start, + window_end, + reference=REFERENCE_VALUES[0], + span=30 * DAY, + ) + ) + print(f" {label:>8}: ok, {len(rows)} rows") + except DiverHubError: + print(f" {label:>8}: 500 at the floor") + + +def probe_datum_relationships( + client: DiverHubClient, point_id: int, name: str, start: int, end: int +) -> None: + """Settle what the four reference values mean, using the API against itself. + + Two questions the min/max summary cannot answer: + + 1. **Is any of them an elevation rather than a depth?** An elevation moves + opposite to a depth, so ``elevation + depth`` is constant while + ``depth - depth`` is constant. Comparing aligned rows distinguishes them; + comparing ranges does not, because both look like the same spread. + 2. **Which is ground surface?** ``ManualMeasurements`` reports + ``waterLevelToc`` -- explicitly top of casing. Whichever reference tracks + it *is* the TOC series, and ground surface is then the one shallower than + it by the casing stickup. + """ + print(f"\n== Datum relationships for {name} ==") + series: dict[int, dict[str, float]] = {} + for reference in REFERENCE_VALUES: + rows = list(client.water_levels(point_id, start, end, reference=reference)) + series[reference] = { + r["dateAndTime"]: r["level"] for r in rows if r.get("level") is not None + } + + shared = set.intersection(*(set(v) for v in series.values())) if series else set() + stamps = sorted(shared)[:3] + if not stamps: + print(" No overlapping timestamps across references.") + return + + print(" Aligned samples:") + print(f" {'timestamp':<22}" + "".join(f"ref{r:<14}" for r in REFERENCE_VALUES)) + for stamp in stamps: + cells = "".join(f"{series[r][stamp]:<17.3f}" for r in REFERENCE_VALUES) + print(f" {stamp:<22}{cells}") + + base = REFERENCE_VALUES[0] + print(f"\n Relationship to ref={base} across those samples:") + for reference in REFERENCE_VALUES[1:]: + diffs = {round(series[reference][t] - series[base][t], 3) for t in stamps} + sums = {round(series[reference][t] + series[base][t], 3) for t in stamps} + if len(diffs) == 1: + print( + f" ref={reference}: constant OFFSET {diffs.pop():+.3f} " + "-- same direction, so also a depth" + ) + elif len(sums) == 1: + print( + f" ref={reference}: constant SUM {sums.pop():.3f} " + "-- INVERTED, so this one is an elevation" + ) + else: + print(f" ref={reference}: neither constant; not a simple datum shift") + + print("\n Manual measurements (waterLevelToc = top of casing):") + try: + # Sparse by nature -- a few per year at best -- so search the whole + # record rather than the window used for the logged series. + manual = client.manual_measurements(point_id, end - 3650 * DAY, end) + except DiverHubError as exc: + print(f" unavailable -- {exc}") + return + if not manual: + print(" none in this window; try a wider one.") + return + for record in manual[:3]: + stamp = record.get("dateAndTime") + toc = record.get("waterLevelToc") + print(f" {stamp} toc={toc}") + nearest = min(stamps, key=lambda t: abs(_epoch(t) - _epoch(stamp))) + print(f" nearest logged sample {nearest}:") + for reference in REFERENCE_VALUES: + delta = series[reference][nearest] - toc if toc is not None else None + if delta is not None: + print( + f" ref={reference}: {series[reference][nearest]:.3f} " + f"(toc{delta:+.3f})" + ) + print("\n The reference nearest zero against toc IS the TOC series.") + print(" Ground surface is shallower than TOC by the casing stickup.") + + +def _epoch(stamp: str) -> float: + from datetime import datetime, timezone + + parsed = datetime.fromisoformat(stamp.replace("Z", "+00:00")) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.timestamp() + + +def main() -> int: + try: + client = DiverHubClient(_session()) + except DiverHubError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + end = int(datetime.now(tz=timezone.utc).timestamp()) + + projects = probe_projects(client) + if not projects: + print("No projects visible; nothing further to probe.", file=sys.stderr) + return 1 + + project_id = projects[0]["id"] + if len(projects) > 1: + print(f"\n(using project {project_id}; pass another by editing this script)") + + points = probe_points(client, project_id) + if not points: + return 1 + + point_id = points[0]["id"] + probe_reference_values(client, points, end) + probe_window_ceiling(client, point_id, end) + probe_datum_relationships(client, point_id, points[0]["name"], end - 730 * DAY, end) + + print("\nRecord the findings in docs/sources/san_acacia.md and set") + print("GROUND_SURFACE_REFERENCE in sources/san_acacia/client.py.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + +# ============= EOF ============================================= diff --git a/automated_ingestion/scripts/reconcile_san_acacia.py b/automated_ingestion/scripts/reconcile_san_acacia.py new file mode 100644 index 000000000..973cafbd6 --- /dev/null +++ b/automated_ingestion/scripts/reconcile_san_acacia.py @@ -0,0 +1,120 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Produce the San Acacia reconciliation report. + +Task 3.2 calls for this **before** anything is written: for each monitoring +point Diver-HUB returns, whether a matching Ocotillo well exists. Read-only on +both sides -- it fetches the vendor's point list and queries `thing`, and +changes nothing. + + export DIVERHUB_USERNAME=... DIVERHUB_PASSWORD=... + uv run --group ingestion python -m \\ + automated_ingestion.scripts.reconcile_san_acacia + +Exits non-zero when any point needs a human, so it can gate a later step +without anyone having to read the output carefully. +""" + +import sys + +from automated_ingestion.sources.san_acacia.reconcile import ( + ThingCandidate, + VendorPoint, + format_report, + reconcile, +) + + +def _vendor_points() -> list[VendorPoint]: + import requests + + from automated_ingestion.sources.san_acacia.client import DiverHubClient + from automated_ingestion.sources.san_acacia.dlt_pipeline import PROJECT_ID + + client = DiverHubClient(requests.Session()) + return [ + VendorPoint(monitoring_point_id=p["id"], name=p["name"]) + for p in client.monitoring_points(PROJECT_ID) + ] + + +def _candidates(prefix: str) -> list[ThingCandidate]: + """Wells that could plausibly be San Acacia points. + + Narrowed by name prefix rather than loading every well: the point ids are + `SO-####`, and comparing 38 names against the whole inventory would surface + coincidental matches from other prefixes without adding a real one. + """ + from sqlalchemy import select + + from db.engine import session_ctx + from db.thing import Thing, ThingIdLink + + with session_ctx() as session: + things = session.execute( + select(Thing.id, Thing.name).where(Thing.name.ilike(f"{prefix}%")) + ).all() + links = session.execute( + select(ThingIdLink.thing_id, ThingIdLink.alternate_id) + ).all() + + by_thing: dict[int, list[str]] = {} + for thing_id, alternate_id in links: + if alternate_id: + by_thing.setdefault(thing_id, []).append(alternate_id) + + return [ + ThingCandidate( + thing_id=thing_id, + name=name, + external_ids=tuple(by_thing.get(thing_id, ())), + ) + for thing_id, name in things + ] + + +def main() -> int: + import argparse + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--prefix", + default="SO-", + help="Well name prefix to consider as candidates (default: SO-).", + ) + args = parser.parse_args() + + try: + points = _vendor_points() + except Exception as exc: # noqa: BLE001 - the message is the useful part + print(f"Could not list monitoring points: {exc}", file=sys.stderr) + return 2 + + candidates = _candidates(args.prefix) + print(f"Vendor points from Diver-HUB : {len(points)}") + print(f"Ocotillo wells named {args.prefix}* : {len(candidates)}\n") + + report = reconcile(points, candidates) + print(format_report(report)) + return 0 if report.ready else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/scripts/report_duplicate_observations.py b/automated_ingestion/scripts/report_duplicate_observations.py new file mode 100644 index 000000000..96ffbba94 --- /dev/null +++ b/automated_ingestion/scripts/report_duplicate_observations.py @@ -0,0 +1,153 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Report duplicate transducer observations before the unique constraint migration. + +Does what ``sql/find_duplicate_observations.sql`` does, without needing ``psql`` +or a password: it connects through the Cloud SQL connector with IAM +authentication, so the credential is your own gcloud login and nothing is +stored. + + gcloud auth application-default login + uv run --group ingestion python -m \\ + automated_ingestion.scripts.report_duplicate_observations \\ + --instance waterdatainitiative-271000:us-west4:dataservices \\ + --database ocotillo-staging + +You need a database login. Being a project owner is not enough -- Cloud SQL +requires the principal to exist as a database user: + + gcloud sql users create YOUR_EMAIL --instance=dataservices \\ + --type=cloud_iam_user --project=waterdatainitiative-271000 + +Read-only. It counts and reports; deciding what to do about duplicates is a +judgement about the data, not something a script should make. +""" + +import argparse +import sys + +DUPLICATE_GROUPS = """ +SELECT deployment_id, parameter_id, observation_datetime, + count(*) AS copies, count(DISTINCT value) AS distinct_values +FROM transducer_observation +GROUP BY deployment_id, parameter_id, observation_datetime +HAVING count(*) > 1 +ORDER BY count(*) DESC, observation_datetime +LIMIT 20 +""" + +TOTALS = """ +SELECT count(*) AS duplicate_groups, + coalesce(sum(copies) - count(*), 0) AS rows_above_the_first, + count(*) FILTER (WHERE distinct_values > 1) AS groups_that_disagree +FROM ( + SELECT count(*) AS copies, count(DISTINCT value) AS distinct_values + FROM transducer_observation + GROUP BY deployment_id, parameter_id, observation_datetime + HAVING count(*) > 1 +) g +""" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--instance", required=True, help="PROJECT:REGION:INSTANCE") + parser.add_argument("--database", required=True, help="e.g. ocotillo-staging") + parser.add_argument("--user", help="IAM principal; defaults to your gcloud account") + args = parser.parse_args() + + user = args.user or _current_account() + if not user: + print("Could not determine your gcloud account; pass --user.", file=sys.stderr) + return 2 + + from google.cloud.sql.connector import Connector + + connector = Connector() + try: + conn = connector.connect( + args.instance, + "pg8000", + user=user, + db=args.database, + enable_iam_auth=True, + ) + except Exception as exc: # noqa: BLE001 - the message is the useful part + print(f"Could not connect as {user}: {exc}", file=sys.stderr) + print( + "\nIf this is a permissions error, the principal probably has no " + "database user:\n" + f" gcloud sql users create {user} --instance=" + f"{args.instance.split(':')[-1]} --type=cloud_iam_user", + file=sys.stderr, + ) + return 1 + + try: + cursor = conn.cursor() + cursor.execute(TOTALS) + groups, extra_rows, disagreeing = cursor.fetchone() + + print(f"Database: {args.database}") + print(f" duplicate groups : {groups}") + print(f" rows above the first : {extra_rows}") + print(f" groups that disagree : {disagreeing}") + + if not groups: + print("\nNo duplicates. The unique constraint migration is safe to run.") + return 0 + + print( + "\nThe migration will FAIL until these are resolved.\n" + "Groups that disagree are the ones to look at first: those rows hold " + "different values for the same instant, so they are conflicting " + "measurements rather than redundant copies, and collapsing them " + "discards a reading somebody recorded." + ) + cursor.execute(DUPLICATE_GROUPS) + print("\n deployment parameter observed copies values") + for dep, param, observed, copies, values in cursor.fetchall(): + print( + f" {dep:>10} {param:>9} {str(observed):<24} {copies:>6} {values:>6}" + ) + return 1 + finally: + conn.close() + connector.close() + + +def _current_account() -> str | None: + import subprocess + + try: + result = subprocess.run( + ["gcloud", "config", "get-value", "account"], + capture_output=True, + text=True, + timeout=30, + ) + except Exception: # noqa: BLE001 + return None + account = result.stdout.strip() + return account or None + + +if __name__ == "__main__": + raise SystemExit(main()) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/scripts/set_code_location_env.sh b/automated_ingestion/scripts/set_code_location_env.sh new file mode 100755 index 000000000..c6cd1de70 --- /dev/null +++ b/automated_ingestion/scripts/set_code_location_env.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Set every environment variable the ocotillo-automated-ingestion code location +# needs, in one pass. +# +# Requires a Dagster+ *user* token -- an agent token authenticates but is not +# authorized for these mutations, and dg reports that as an unhelpful KeyError: +# dg plus config set --api-token 'user:...' +# +# Secrets are never passed as arguments. `--from-local-env` reads them from this +# shell, so nothing sensitive reaches the command line, your shell history, or +# the Dagster+ audit log's argument capture. Export them first: +# +# read -rs "DIVERHUB_USERNAME?Diver-HUB username: "; echo +# read -rs "DIVERHUB_PASSWORD?Diver-HUB password: "; echo +# export DIVERHUB_USERNAME DIVERHUB_PASSWORD +# +# Variables are set at deployment scope. See the comment on set_var for why +# location scoping through dg does not work, and what to do instead if these +# values must not be visible to the other code locations in this deployment. +# +# Usage: +# ./automated_ingestion/scripts/set_code_location_env.sh storage +# ./automated_ingestion/scripts/set_code_location_env.sh credentials +# ./automated_ingestion/scripts/set_code_location_env.sh vendor +# ./automated_ingestion/scripts/set_code_location_env.sh database +# +# The phases are separate on purpose. `database` should wait until +# automated_ingestion/sql/ingestion_role.sql has been run: setting CLOUD_SQL_* +# against a role that does not exist yet makes database_connectivity fail in a +# way that looks like the serverless-to-Cloud-SQL problem it is meant to test. +set -euo pipefail + +DG="uv run --with dagster-dg-cli dg" +PHASE="${1:-}" + +# --global sets the variable at deployment level. That is broader than ideal -- +# this deployment also hosts aqueduct_dagster_defs_definitions and +# die-orchestration, which can then read these values -- but it is the scope +# that actually reaches the container. +# +# Location scoping through dg does not work here: dg names the location from the +# project (`OcotilloAPI`), not from `location_name` in dagster_cloud.yaml +# (`ocotillo-automated-ingestion`), and `code_location_name` in [tool.dg.project] +# is ignored for this command. Dagster+ accepts the unknown name without +# complaint, so the variable shows as set in the UI and is absent in the +# container -- which costs an afternoon to work out from a +# DefaultCredentialsError. +# +# To scope properly, set the variable in the Dagster+ UI against +# `ocotillo-automated-ingestion` instead. +set_var() { echo " $1"; $DG plus create env "$@" --global -y >/dev/null; } + +case "$PHASE" in +storage) + # The image copies the repository to /opt/dagster/app but never installs it, + # so db/ and domain/ are importable only if that directory is on the path. + # The process that loads the code location has it; the process that executes a + # step does not reliably, which shows up as ModuleNotFoundError for db at + # execution while the location itself loads fine. Setting PYTHONPATH removes + # the guesswork instead of depending on how each process was launched. + echo "Import path:" + set_var PYTHONPATH /opt/dagster/app + + echo "Raw-zone buckets (different value per scope):" + set_var INGESTION_GCS_BUCKET ocotillo-ingestion-production --scope full + set_var INGESTION_GCS_BUCKET ocotillo-ingestion-staging --scope branch + ;; +credentials) + : "${INGESTION_GCP_CREDENTIALS_JSON:?export the service account key JSON, not a path}" + # Serverless runs outside GCP, so there is no metadata server and nothing + # supplies Application Default Credentials. Both the Cloud SQL connector and + # gcsfs need them. Mint the key with: + # gcloud iam service-accounts keys create /dev/stdout \ + # --iam-account ocotillo-ingestion@waterdatainitiative-271000.iam.gserviceaccount.com + echo "GCP credentials (key JSON read from this shell, not echoed):" + set_var INGESTION_GCP_CREDENTIALS_JSON --from-local-env + ;; +vendor) + : "${DIVERHUB_USERNAME:?export it first, see the header}" + : "${DIVERHUB_PASSWORD:?export it first, see the header}" + echo "Diver-HUB credentials (values read from this shell, not echoed):" + set_var DIVERHUB_USERNAME --from-local-env + set_var DIVERHUB_PASSWORD --from-local-env + ;; +database) + : "${CLOUD_SQL_INSTANCE_NAME:?export it first, as PROJECT:REGION:INSTANCE}" + : "${CLOUD_SQL_DATABASE:?export it first}" + + # The connector wants the full connection name, not the instance name. A bare + # name is accepted by everything up to the point of connecting and then fails + # with a ValueError from deep inside the driver, several layers below anything + # this project wrote. Catch it here instead. + # gcloud sql instances list --format='value(name,connectionName)' + case "$CLOUD_SQL_INSTANCE_NAME" in + *:*:*) ;; + *) + echo "error: CLOUD_SQL_INSTANCE_NAME must be PROJECT:REGION:INSTANCE," >&2 + echo " got '${CLOUD_SQL_INSTANCE_NAME}'." >&2 + exit 65 + ;; + esac + echo "Cloud SQL connection:" + set_var DB_DRIVER cloudsql + set_var CLOUD_SQL_IP_TYPE public + set_var CLOUD_SQL_INSTANCE_NAME --from-local-env + set_var CLOUD_SQL_DATABASE --from-local-env + + # CLOUD_SQL_USER means different things in the two auth modes, and db/engine.py + # passes it straight to the connector either way. Under IAM auth it must be the + # service account with the .gserviceaccount.com suffix stripped; a plain + # Postgres role name there fails as an authentication error that reads like a + # missing grant. Deriving it here keeps the two settings from contradicting + # each other. + if [ -n "${CLOUD_SQL_PASSWORD:-}" ]; then + echo " (password auth)" + set_var CLOUD_SQL_IAM_AUTH 0 + set_var CLOUD_SQL_USER ocotillo_ingestion + set_var CLOUD_SQL_PASSWORD --from-local-env + else + IAM_SA="${INGESTION_SERVICE_ACCOUNT:-ocotillo-ingestion@waterdatainitiative-271000.iam.gserviceaccount.com}" + IAM_USER="${IAM_SA%.gserviceaccount.com}" + echo " (IAM auth as ${IAM_USER})" + set_var CLOUD_SQL_IAM_AUTH 1 + set_var CLOUD_SQL_USER "$IAM_USER" + fi + ;; +*) + echo "usage: $0 {storage|credentials|vendor|database}" >&2 + exit 64 + ;; +esac + +echo "Done. Verify in Dagster+ under Deployment -> Environment variables." diff --git a/automated_ingestion/shared/__init__.py b/automated_ingestion/shared/__init__.py new file mode 100644 index 000000000..809f48e4e --- /dev/null +++ b/automated_ingestion/shared/__init__.py @@ -0,0 +1,18 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Source-agnostic machinery reused by every ingestion source.""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/shared/backfill.py b/automated_ingestion/shared/backfill.py new file mode 100644 index 000000000..6799bdad5 --- /dev/null +++ b/automated_ingestion/shared/backfill.py @@ -0,0 +1,249 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Primitives shared by every backfill, in either mode. + +Ported from Aqueduct's ``shared/backfill.py`` rather than imported: the two +repositories deploy separately and are allowed to diverge. Where behaviour +differs from the original it is called out on the function, so the two can be +diffed later by someone who has both open. + +**Changed from Aqueduct.** ``ChunkResult`` counts ``rows_upserted`` where the +original counted ``observations_posted`` and ``observations_deleted``. That is +not a rename: Aqueduct deletes a window and re-posts it because FROST has no +constraint to conflict on, so it has two numbers and a window during which the +data is missing. Ocotillo upserts, so there is one number and no window. + +Everything here is pure except the checkpoint store, which is why the store is +an interface with an in-memory implementation -- a backfill's chunking and +resumption logic can then be tested without touching object storage. +""" + +import re +from calendar import monthrange +from collections.abc import Iterable, Iterator +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Protocol + + +@dataclass(frozen=True) +class Chunk: + """One calendar month of a backfill window, half-open at the end.""" + + start: datetime + end: datetime + + @property + def key(self) -> str: + """Stable identifier, used for checkpointing.""" + return self.start.strftime("%Y-%m") + + +@dataclass +class ChunkResult: + """What one chunk did. + + ``rows_upserted`` replaces Aqueduct's posted/deleted pair -- see the module + docstring. ``failures`` counts records the adapter refused, which are + per-record and never fatal to the chunk. + """ + + chunk_key: str + rows_ingested: int = 0 + rows_upserted: int = 0 + failures: int = 0 + + @property + def rows_refused(self) -> int: + return self.rows_ingested - self.rows_upserted + + +@dataclass +class BackfillTotals: + """Sum across chunks, for run-level metadata.""" + + chunks: int = 0 + rows_ingested: int = 0 + rows_upserted: int = 0 + failures: int = 0 + chunk_keys: list[str] = field(default_factory=list) + + +def month_chunks(start: datetime, end: datetime) -> Iterator[Chunk]: + """Split a window into calendar months. + + Calendar months rather than fixed-length windows because that is how a human + describes a gap ("we lost March"), and because it makes a chunk key legible + in a checkpoint file. The first and last chunks are clipped to the requested + range rather than widened to whole months -- widening would fetch data the + operator did not ask for. + """ + validate_date_order(start, end) + + cursor = start + while cursor < end: + _, last_day = monthrange(cursor.year, cursor.month) + month_end = cursor.replace( + day=last_day, hour=23, minute=59, second=59, microsecond=999999 + ) + chunk_end = min(month_end, end) + yield Chunk(start=cursor, end=chunk_end) + + if chunk_end >= end: + return + year = cursor.year + (1 if cursor.month == 12 else 0) + month = 1 if cursor.month == 12 else cursor.month + 1 + cursor = cursor.replace( + year=year, month=month, day=1, hour=0, minute=0, second=0, microsecond=0 + ) + + +def sum_chunk_results(results: Iterable[ChunkResult]) -> BackfillTotals: + """Aggregate chunk results for reporting.""" + totals = BackfillTotals() + for result in results: + totals.chunks += 1 + totals.rows_ingested += result.rows_ingested + totals.rows_upserted += result.rows_upserted + totals.failures += result.failures + totals.chunk_keys.append(result.chunk_key) + return totals + + +def parse_backfill_date(value: str) -> datetime: + """Parse an operator-supplied date into a timezone-aware UTC datetime. + + A bare date means midnight UTC. Accepting a naive value and treating it as + local time would make the same run config mean different windows on + different machines. + """ + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"Backfill date is missing or blank: {value!r}") + try: + parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError(f"Backfill date {value!r} is not ISO-8601.") from exc + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def validate_date_order(start: datetime, end: datetime) -> None: + """Reject a reversed or empty window. + + An empty window is rejected rather than treated as a no-op: a backfill that + reports success having done nothing is indistinguishable from one that + worked, and the operator would not learn they typed the dates backwards. + """ + if end <= start: + raise ValueError( + f"Backfill end {end.isoformat()} must be after start {start.isoformat()}." + ) + + +_UNSAFE_RUN_KEY = re.compile(r"[^A-Za-z0-9._-]+") + + +def sanitize_run_key(value: str) -> str: + """Reduce an operator-supplied run key to something safe as a path segment. + + Run keys end up in object storage paths. An unsanitized one containing a + slash would silently write checkpoints into a directory of its own, and a + resumed run would not find them. + """ + cleaned = _UNSAFE_RUN_KEY.sub("-", (value or "").strip()).strip("-") + if not cleaned: + raise ValueError(f"Run key {value!r} contains nothing usable.") + return cleaned + + +def attach_run_timestamp(run_key: str, now: datetime | None = None) -> str: + """Append a UTC timestamp, so two runs with the same key stay distinct. + + Only for keys that are *not* meant to resume. Resumption depends on the key + being stable, so the caller decides; this never applies it silently. + """ + stamp = (now or datetime.now(tz=timezone.utc)).strftime("%Y%m%dT%H%M%SZ") + return f"{sanitize_run_key(run_key)}-{stamp}" + + +def chunk_key(run_key: str, chunk: Chunk) -> str: + """Checkpoint identifier for one chunk of one run.""" + return f"{sanitize_run_key(run_key)}/{chunk.key}" + + +def resolve_location_ids( + requested: Iterable[Any], available: Iterable[Any] +) -> list[Any]: + """Validate requested locations against what the source offers. + + An empty request means every available location. An unknown id fails the + run, naming the bad ids -- Aqueduct's behaviour, and worth keeping: silently + backfilling nothing looks identical to backfilling successfully, and the + operator finds out weeks later that the gap is still there. + """ + available_list = list(available) + requested_list = [r for r in requested] if requested is not None else [] + if not requested_list: + return available_list + + known = set(available_list) + unknown = [r for r in requested_list if r not in known] + if unknown: + raise ValueError( + "Unknown location ids: " + + ", ".join(str(u) for u in sorted(unknown, key=str)) + + ". Nothing was backfilled." + ) + return [r for r in requested_list] + + +class CheckpointStore(Protocol): + """Which chunks of a run have completed.""" + + def completed(self, run_key: str) -> set[str]: ... + + def mark_complete(self, run_key: str, chunk: Chunk) -> None: ... + + +class InMemoryCheckpointStore: + """For tests, and for a dry run that must not persist anything.""" + + def __init__(self) -> None: + self._done: dict[str, set[str]] = {} + + def completed(self, run_key: str) -> set[str]: + return set(self._done.get(sanitize_run_key(run_key), set())) + + def mark_complete(self, run_key: str, chunk: Chunk) -> None: + self._done.setdefault(sanitize_run_key(run_key), set()).add(chunk.key) + + +def pending_chunks( + store: CheckpointStore, run_key: str, chunks: Iterable[Chunk] +) -> list[Chunk]: + """Chunks of this run that have not completed yet. + + A chunk is checkpointed only after ingest, transform, and load have all + succeeded, so anything not marked is safe to redo -- the load is an upsert, + and redoing a partially loaded chunk rewrites the same rows. + """ + done = store.completed(run_key) + return [chunk for chunk in chunks if chunk.key not in done] + + +# ============= EOF ============================================= diff --git a/automated_ingestion/shared/credentials.py b/automated_ingestion/shared/credentials.py new file mode 100644 index 000000000..cc108ba84 --- /dev/null +++ b/automated_ingestion/shared/credentials.py @@ -0,0 +1,83 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Application Default Credentials for a runtime that has none. + +Dagster+ Serverless runs outside GCP, so there is no metadata server to supply +credentials. Anything reaching Google -- the Cloud SQL connector for the loader, +gcsfs for the raw zone -- calls ``google.auth.default()`` and fails with +``DefaultCredentialsError`` unless something has put credentials on disk first. + +The service account key therefore travels as a Dagster+ secret and is written to +a file here, because ``GOOGLE_APPLICATION_CREDENTIALS`` names a path rather than +holding a value. The file lands in the process's temporary directory, which the +container discards when the run ends. +""" + +import json +import os +import tempfile + +CREDENTIALS_ENV_VAR = "INGESTION_GCP_CREDENTIALS_JSON" +"""Service account key JSON, as a Dagster+ secret. Never committed.""" + +_ADC_ENV_VAR = "GOOGLE_APPLICATION_CREDENTIALS" + +_written_path: str | None = None + + +def ensure_application_default_credentials() -> str | None: + """Materialize ADC from the environment, returning the path if written. + + Idempotent, and does nothing when credentials already exist -- locally that + means a developer's gcloud login is used as-is rather than being shadowed. + """ + global _written_path + + existing = os.environ.get(_ADC_ENV_VAR, "").strip() + if existing: + return existing + if _written_path is not None: + return _written_path + + raw = os.environ.get(CREDENTIALS_ENV_VAR, "").strip() + if not raw: + # No key configured. Leave google.auth to its own discovery, which + # succeeds on a developer machine and fails loudly in Serverless -- the + # right outcome in both cases. + return None + + try: + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"{CREDENTIALS_ENV_VAR} is set but is not valid JSON. It must hold the " + "service account key itself, not a path to one." + ) from exc + + handle = tempfile.NamedTemporaryFile( + mode="w", suffix=".json", prefix="ingestion-adc-", delete=False + ) + with handle as fh: + json.dump(parsed, fh) + os.chmod(handle.name, 0o600) + + os.environ[_ADC_ENV_VAR] = handle.name + _written_path = handle.name + return handle.name + + +# ============= EOF ============================================= diff --git a/automated_ingestion/shared/gcs.py b/automated_ingestion/shared/gcs.py new file mode 100644 index 000000000..b98b8836f --- /dev/null +++ b/automated_ingestion/shared/gcs.py @@ -0,0 +1,60 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +GCS raw-zone conventions. + +Every source writes date-partitioned parquet under one bucket per environment, +so a replay backfill can select an exact window by prefix without reading the +files. + +``services/gcs_helper.py`` serves user uploads from ``GCS_BUCKET_NAME``. +Ingestion deliberately reads a different variable: sharing it would let a +misconfigured deployment write raw vendor payloads into the uploads bucket. +""" + +BUCKET_ENV_VAR = "INGESTION_GCS_BUCKET" +"""Environment variable naming the raw-zone bucket. Never ``GCS_BUCKET_NAME``.""" + +RAW_LAYOUT = "{table_name}/year={YYYY}/month={MM}/day={DD}/{load_id}.{file_id}.{ext}" +"""dlt filesystem layout for the raw zone.""" + + +def raw_zone_bucket() -> str: + """Name of the raw-zone bucket for this environment. + + Raises rather than defaulting. A wrong bucket name is not a condition worth + guessing through: the failure would be a run that reports success while + writing nowhere useful, or worse, into a bucket that belongs to something + else. + """ + import os + + bucket = os.environ.get(BUCKET_ENV_VAR, "").strip() + if not bucket: + raise RuntimeError( + f"{BUCKET_ENV_VAR} is not set. The ingestion raw zone has no default; " + "set it on the Dagster+ code location to the bucket Terraform " + "created (see automated_ingestion/iac)." + ) + if bucket == os.environ.get("GCS_BUCKET_NAME", "").strip(): + raise RuntimeError( + f"{BUCKET_ENV_VAR} points at GCS_BUCKET_NAME, the API's user-upload " + "bucket. Raw vendor payloads must not be written there." + ) + return bucket + + +# ============= EOF ============================================= diff --git a/automated_ingestion/shared/http.py b/automated_ingestion/shared/http.py new file mode 100644 index 000000000..32adeb87d --- /dev/null +++ b/automated_ingestion/shared/http.py @@ -0,0 +1,24 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +HTTP session construction for vendor APIs. + +Centralized so every source inherits the same timeout, retry, and backoff +posture, and so one source's flaky endpoint cannot hang a run indefinitely. +Filled in alongside the first live extraction under BDMS task 2. +""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/shared/source_registry.py b/automated_ingestion/shared/source_registry.py new file mode 100644 index 000000000..a5ef76ab6 --- /dev/null +++ b/automated_ingestion/shared/source_registry.py @@ -0,0 +1,64 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Registry of ingestion sources. + +Each source declares itself once here so jobs, schedules, and the backfill +factory can enumerate sources without importing each one by name. +""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SourceDefinition: + """Static description of one ingestion source.""" + + key: str + """Stable identifier, used in asset keys and GCS prefixes.""" + + display_name: str + """Human-readable name for logs and the Dagster UI.""" + + dataset_name: str + """dlt dataset name; becomes the top-level GCS prefix.""" + + +_SOURCES: dict[str, SourceDefinition] = {} + + +def register(source: SourceDefinition) -> SourceDefinition: + """Add a source to the registry, rejecting duplicate keys.""" + if source.key in _SOURCES: + raise ValueError(f"Source {source.key!r} is already registered.") + _SOURCES[source.key] = source + return source + + +def get_source(key: str) -> SourceDefinition: + """Look up a registered source by key.""" + try: + return _SOURCES[key] + except KeyError: + raise KeyError(f"No ingestion source registered under {key!r}.") from None + + +def all_sources() -> tuple[SourceDefinition, ...]: + """Every registered source, ordered by key.""" + return tuple(_SOURCES[k] for k in sorted(_SOURCES)) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/shared/watermark.py b/automated_ingestion/shared/watermark.py new file mode 100644 index 000000000..3dfcdfebb --- /dev/null +++ b/automated_ingestion/shared/watermark.py @@ -0,0 +1,108 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Where a series got to, asked of the data rather than of a sidecar. + +Aqueduct keeps watermarks in a GCS object beside the raw zone, because its +destination is FROST and cannot be queried cheaply for a maximum. Ocotillo's +destination is Postgres, so the watermark is simply +``MAX(observation_datetime)`` for the series. + +**This is a deliberate divergence, not an oversight.** A stored watermark is a +second source of truth about what was loaded, and the two drift: a load that +half-succeeds, or a sidecar write that fails after the rows commit, leaves the +watermark claiming more or less than the data holds. Deriving it means the +answer cannot disagree with reality — and it makes a backfill safe by +construction, since re-loading an old window cannot move a maximum forward. + +**Keyed by thing, not by deployment.** Observations carry ``deployment_id``, but +a series outlives its hardware: replacing a diver creates a new deployment for +the same well, and a watermark keyed to the deployment would report nothing for +the new one and re-fetch the entire history. The query joins through +``deployment`` to ask the question the pipeline actually has -- how far along is +this well's depth-to-water record. +""" + +from datetime import datetime +from typing import Any, Protocol + + +class WatermarkStore(Protocol): + """Where a series has been loaded up to.""" + + def get(self, thing_id: int, parameter_id: int) -> datetime | None: + """Latest observation for the series, or ``None`` if never loaded.""" + ... + + +class PostgresWatermarkStore: + """Reads the watermark from the observations themselves. + + Takes the session the loader is using, so the watermark reflects that + session's committed state rather than a separate connection's snapshot. + """ + + def __init__(self, session: Any) -> None: + self._session = session + + def get(self, thing_id: int, parameter_id: int) -> datetime | None: + from sqlalchemy import func, select + + from db.deployment import Deployment + from db.transducer import TransducerObservation + + return self._session.scalar( + select(func.max(TransducerObservation.observation_datetime)) + .join( + Deployment, + Deployment.id == TransducerObservation.deployment_id, + ) + .where(Deployment.thing_id == thing_id) + .where(TransducerObservation.parameter_id == parameter_id) + ) + + +class InMemoryWatermarkStore: + """For tests, and for reasoning about a run without a database.""" + + def __init__(self, watermarks: dict[tuple[int, int], datetime] | None = None): + self._watermarks = dict(watermarks or {}) + + def get(self, thing_id: int, parameter_id: int) -> datetime | None: + return self._watermarks.get((thing_id, parameter_id)) + + def set(self, thing_id: int, parameter_id: int, value: datetime) -> None: + self._watermarks[(thing_id, parameter_id)] = value + + +def resolve_start( + store: WatermarkStore, + thing_id: int, + parameter_id: int, + floor: datetime, +) -> datetime: + """Where the next fetch should begin. + + ``floor`` applies only to a series that has never been loaded. It is not a + backfill lever: lowering it will not re-fetch history for a series whose + watermark has already advanced past it, because the watermark wins whenever + one exists. Re-fetching history is what the backfill jobs are for. + """ + watermark = store.get(thing_id, parameter_id) + return watermark if watermark is not None else floor + + +# ============= EOF ============================================= diff --git a/automated_ingestion/shared/windows.py b/automated_ingestion/shared/windows.py new file mode 100644 index 000000000..039eff232 --- /dev/null +++ b/automated_ingestion/shared/windows.py @@ -0,0 +1,90 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Time-window arithmetic for sources that cannot be asked for an open range. + +Diver-HUB answers `DiverData` and `WaterLevels` for an explicit +``startTime``/``endTime`` in Unix seconds, and returns HTTP 500 -- not a +pagination cursor, not a 413 -- when the span is too wide. So a "fetch this +series" operation is always a sequence of bounded windows, and the useful +response to a 500 is to ask for less rather than to give up. + +Pure arithmetic, no HTTP: the retry policy that uses it is in the client, and +the point of separating them is that the tricky part is testable without a +network. +""" + +from collections.abc import Iterator +from dataclasses import dataclass + +DAY = 86_400 + +DEFAULT_SPAN = 90 * DAY +"""Starting window width. Three months is confirmed to work; the ceiling is +not yet measured, so this is the largest span known to be safe rather than the +largest span that is.""" + +MINIMUM_SPAN = DAY +"""Floor for bisection. A 500 on a single day is a real failure -- something +other than volume -- and must surface rather than shrink forever.""" + + +@dataclass(frozen=True) +class Window: + """A half-open interval in Unix seconds, ``start`` inclusive.""" + + start: int + end: int + + def __post_init__(self) -> None: + if self.end < self.start: + raise ValueError(f"Window end {self.end} precedes start {self.start}.") + + @property + def span(self) -> int: + return self.end - self.start + + def bisect(self) -> tuple["Window", "Window"]: + """Split in two. Raises at the floor rather than shrinking forever.""" + if self.span <= MINIMUM_SPAN: + raise ValueError( + f"Refusing to split a {self.span}s window below the {MINIMUM_SPAN}s " + "floor. A failure this narrow is not a volume problem." + ) + midpoint = self.start + self.span // 2 + return Window(self.start, midpoint), Window(midpoint, self.end) + + +def iter_windows(start: int, end: int, span: int = DEFAULT_SPAN) -> Iterator[Window]: + """Walk ``[start, end]`` in windows of at most ``span`` seconds.""" + if span <= 0: + raise ValueError(f"Window span must be positive, got {span}.") + if end < start: + raise ValueError(f"End {end} precedes start {start}.") + # Windows must not share a boundary. Diver-HUB's ranges are inclusive at + # both ends -- "from start time up to and including end time" -- so + # [0, span] and [span, 2*span] both return the reading logged exactly at + # `span`. That duplicate reaches the loader in one batch and Postgres + # rejects the statement: "ON CONFLICT DO UPDATE command cannot affect row a + # second time". + cursor = start + while cursor < end: + chunk_end = min(cursor + span, end) + yield Window(cursor, chunk_end) + cursor = chunk_end + 1 + + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/__init__.py b/automated_ingestion/sources/__init__.py new file mode 100644 index 000000000..4fbe9cf77 --- /dev/null +++ b/automated_ingestion/sources/__init__.py @@ -0,0 +1,18 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""One subpackage per ingestion source.""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/__init__.py b/automated_ingestion/sources/san_acacia/__init__.py new file mode 100644 index 000000000..28d879b9d --- /dev/null +++ b/automated_ingestion/sources/san_acacia/__init__.py @@ -0,0 +1,35 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +San Acacia Reach -- 33 Van Essen divers, one depth-to-groundwater series each. + +The pilot source: small and already mapped, so it exercises the whole path end +to end without a large or unfamiliar dataset complicating the first build. + +Readings come from the private Diver-HUB API, which shapes the extraction in +two ways. Requests carry a JWT good for one hour, so anything long-running +refreshes mid-run rather than authenticating once at the start. And +``DiverData/ByMonitoringPoint/{id}`` returns HTTP 500 when asked for too wide a +span instead of paginating, so reads are always bounded windows in Unix +seconds -- roughly three months is known to work. + +Readings land on the **ground-surface** datum (Van Essen's ``gs`` +arrays, never ``vrd``), public but provisional, and always ``not reviewed`` -- +the vendor's own approval flag records what the vendor approved, not a Bureau +review. +""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/adapter.py b/automated_ingestion/sources/san_acacia/adapter.py new file mode 100644 index 000000000..d02bedecf --- /dev/null +++ b/automated_ingestion/sources/san_acacia/adapter.py @@ -0,0 +1,104 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Van Essen mapping rules for the San Acacia source. + +The adapter is the only place the vendor's vocabulary meets Ocotillo's. It is +pure enough to test without a database: it takes raw records and returns +structures, and the loader turns those into rows. + +Per-record failure isolation matches the rest of the pipeline. One unparseable +reading costs that reading, not the series -- a diver that logs one bad row +should not lose a month of good ones. +""" + +from collections.abc import Iterable, Iterator +from typing import Any + +from domain.van_essen import ( + GROUND_SURFACE_REFERENCE, + MEASUREMENT_UNIT, + VanEssenMappingError, + depth_to_water_ft, + external_point_key, + parse_reading_timestamp, +) + +from automated_ingestion.ocotillo.adapter import SourceAdapter +from automated_ingestion.ocotillo.structs import ObservationRecord + + +class SanAcaciaAdapter(SourceAdapter): + """Maps Diver-HUB water levels onto Ocotillo observations.""" + + def __init__(self) -> None: + self.failures: list[dict[str, Any]] = [] + + @property + def source_key(self) -> str: + return "san_acacia" + + def to_observations( + self, records: Iterable[dict[str, Any]] + ) -> Iterator[ObservationRecord]: + """Convert raw rows, collecting per-record failures rather than raising. + + Rows whose ``reference`` is not ground surface are refused outright. The + datum is chosen at request time and cannot be recovered from the row, so + accepting one would mean storing a number whose meaning is unknown -- + the single failure this pipeline must not produce quietly. + """ + for record in records: + try: + yield self._to_observation(record) + except VanEssenMappingError as exc: + self.failures.append({"record": _identify(record), "error": str(exc)}) + + def _to_observation(self, record: dict[str, Any]) -> ObservationRecord: + reference = record.get("reference") + if reference != GROUND_SURFACE_REFERENCE: + raise VanEssenMappingError( + f"Reading was fetched with reference={reference!r}, not " + f"{GROUND_SURFACE_REFERENCE} (ground surface). Its datum is not " + "recoverable from the row." + ) + + unit = record.get("unit") + if unit != "cm": + raise VanEssenMappingError( + f"Reading unit is {unit!r}, expected 'cm'. Converting a value " + "whose unit is not what it claims would be wrong by a factor." + ) + + point_id = record.get("monitoring_point_id") + value = depth_to_water_ft(record.get("level")) + if value is None: + raise VanEssenMappingError("Reading has no level; nothing to store.") + + return ObservationRecord( + external_point_id=external_point_key(point_id), + observation_datetime=parse_reading_timestamp(record.get("dateAndTime")), + value=value, + units=MEASUREMENT_UNIT, + ) + + +def _identify(record: dict[str, Any]) -> str: + """A short handle for a failed record, for logs and metadata.""" + return f"{record.get('monitoring_point_id')}@{record.get('dateAndTime')}" + + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/backfill.py b/automated_ingestion/sources/san_acacia/backfill.py new file mode 100644 index 000000000..eb3cd127e --- /dev/null +++ b/automated_ingestion/sources/san_acacia/backfill.py @@ -0,0 +1,18 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""San Acacia backfill wiring. Built under BDMS task 4.""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/client.py b/automated_ingestion/sources/san_acacia/client.py new file mode 100644 index 000000000..8e3b269e5 --- /dev/null +++ b/automated_ingestion/sources/san_acacia/client.py @@ -0,0 +1,291 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Client for the private Diver-HUB API. + +Three things about this API shape the code, and all three differ from what the +retired FROST pipeline suggested: + +1. **Bearer JWT with a real expiry.** ``POST /Accounts/Login`` returns a token + and a ``validTo`` timestamp. The token is refreshed against that timestamp + rather than against an assumed lifetime, and once more on a 401 -- a clock + difference between us and the server should not end a backfill. +2. **Bounded windows.** Readings endpoints take ``startTime``/``endTime`` in + Unix seconds and answer HTTP 500 when the span is too wide, so a fetch walks + windows and narrows on failure. +3. **Datum is a request parameter, not a response field.** ``WaterLevels`` + returns ``{dateAndTime, level}``; which datum that level is on depends on the + ``reference`` value sent. See ``GROUND_SURFACE_REFERENCE``. + +Credentials come from the environment and are never logged. The token is not +logged either: it is a bearer credential for the whole account. +""" + +import os +import time +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Any, Protocol + +from automated_ingestion.shared.windows import DEFAULT_SPAN, Window, iter_windows + +BASE_URL = "https://diver-hub.com/private/api/v1" + +USERNAME_ENV_VAR = "DIVERHUB_USERNAME" +PASSWORD_ENV_VAR = "DIVERHUB_PASSWORD" + +EXPIRY_SKEW_SECONDS = 60 +"""Refresh this long before ``validTo``, so a request in flight at the boundary +does not arrive expired.""" + +GROUND_SURFACE_REFERENCE = 3 +"""Which ``WaterLevelReference`` value means depth below ground surface. + +The swagger declares the enum as ``[0, 1, 2, 3]`` with no names, so this was +determined by measurement rather than read off the specification. Probing +SO-0125 showed all four values return the same rows at the same timestamps, +related by constants that held identically across two windows eighteen months +apart: + + ref1 + ref0 = 518.160 ref3 + ref0 = 472.704 ref2 - ref0 = 139001.296 + +``ref0`` and ``ref2`` rise with the water; ``ref1`` and ``ref3`` fall, so the +latter pair are depths. ``ref1`` is deeper than ``ref3`` by a fixed 45.456 cm +(1.49 ft) -- a casing stickup -- which makes ``ref1`` top of casing and ``ref3`` +ground surface. The reading checks out physically: ground surface lands at +1394.74 m (4576 ft), right for San Acacia, and depth to water runs 2.2-4.7 m, +right for a riparian piezometer. + +See ``docs/sources/san_acacia.md``. Do not change this without re-running +``scripts/probe_diverhub.py``: the wrong value returns plausible numbers on the +wrong datum rather than an error. +""" + +TOP_OF_CASING_REFERENCE = 1 +"""Depth below top of casing. Not ingested -- recorded so the value is not +mistaken for ground surface, which it resembles to within a stickup.""" + +ELEVATION_REFERENCE = 2 +"""Water-surface elevation above sea level. Not ingested.""" + +SOURCE_UNIT = "cm" +"""Diver-HUB reports centimeters; Ocotillo stores feet. Convert with +``domain.units.convert_cm_to_ft`` -- never store a raw value.""" + + +class Response(Protocol): + """The subset of a `requests` response this module uses.""" + + status_code: int + + def json(self) -> Any: ... + + +class Transport(Protocol): + """The subset of a `requests` session this module uses.""" + + def post(self, url: str, **kwargs: Any) -> Response: ... + + def get(self, url: str, **kwargs: Any) -> Response: ... + + +class DiverHubError(RuntimeError): + """The API refused a request in a way retrying will not fix.""" + + +@dataclass +class _Token: + value: str + valid_to: float + + def expired(self, now: float) -> bool: + return now >= self.valid_to - EXPIRY_SKEW_SECONDS + + +class DiverHubClient: + """Authenticated, window-aware access to Diver-HUB.""" + + def __init__( + self, + transport: Transport, + username: str | None = None, + password: str | None = None, + base_url: str = BASE_URL, + timeout: int = 60, + ) -> None: + self._transport = transport + self._base_url = base_url.rstrip("/") + self._timeout = timeout + self._username = username or os.environ.get(USERNAME_ENV_VAR, "") + self._password = password or os.environ.get(PASSWORD_ENV_VAR, "") + self._token: _Token | None = None + if not self._username or not self._password: + raise DiverHubError( + f"Diver-HUB credentials are not set. Provide {USERNAME_ENV_VAR} and " + f"{PASSWORD_ENV_VAR} in the environment." + ) + + # -- authentication ---------------------------------------------------- + + def _login(self) -> _Token: + response = self._transport.post( + f"{self._base_url}/Accounts/Login", + json={"username": self._username, "password": self._password}, + timeout=self._timeout, + ) + if response.status_code == 401: + raise DiverHubError("Diver-HUB rejected the credentials.") + if response.status_code != 200: + raise DiverHubError(f"Login failed with HTTP {response.status_code}.") + payload = response.json() + return _Token( + value=payload["token"], + valid_to=_parse_timestamp(payload["validTo"]), + ) + + def _authorization(self) -> dict[str, str]: + if self._token is None or self._token.expired(time.time()): + self._token = self._login() + return {"Authorization": f"Bearer {self._token.value}"} + + def _get(self, path: str, params: dict[str, Any] | None = None) -> Response: + """GET with one forced re-login if the token is rejected.""" + response = self._transport.get( + f"{self._base_url}/{path.lstrip('/')}", + headers=self._authorization(), + params=params, + timeout=self._timeout, + ) + if response.status_code == 401: + self._token = None + response = self._transport.get( + f"{self._base_url}/{path.lstrip('/')}", + headers=self._authorization(), + params=params, + timeout=self._timeout, + ) + return response + + # -- reference data ---------------------------------------------------- + + def projects(self) -> list[dict[str, Any]]: + """Projects visible to these credentials.""" + return _expect_ok(self._get("Projects"), "Projects").json() + + def monitoring_points(self, project_id: int) -> list[dict[str, Any]]: + """Monitoring points in a project. Returns ``{id, name}`` only -- + no coordinates and no construction detail, so geometry and depth have + to be resolved from Ocotillo rather than from here.""" + path = f"MonitoringPoints/ByProject/{project_id}" + return _expect_ok(self._get(path), path).json() + + def manual_measurements( + self, monitoring_point_id: int, start: int, end: int + ) -> list[dict[str, Any]]: + """Manual readings, reported against top of casing. + + Not ingested -- Ocotillo's manual-measurement path owns these. Fetched + only to identify which ``reference`` value is the TOC series, since the + swagger names the enum members not at all. + """ + path = f"ManualMeasurements/ByMonitoringPoint/{monitoring_point_id}" + response = self._get(path, {"startTime": start, "endTime": end}) + return _expect_ok(response, path).json() + + # -- series ------------------------------------------------------------ + + def water_levels( + self, + monitoring_point_id: int, + start: int, + end: int, + reference: int, + approved: bool | None = None, + span: int = DEFAULT_SPAN, + ) -> Iterator[dict[str, Any]]: + """Yield ``{dateAndTime, level}`` records across bounded windows. + + ``reference`` selects the datum and is required: there is no safe + default, because the wrong value produces plausible numbers rather than + an error. + """ + params: dict[str, Any] = {"reference": reference} + if approved is not None: + params["approved"] = approved + path = f"WaterLevels/ByMonitoringPoint/{monitoring_point_id}" + for window in iter_windows(start, end, span): + yield from self._fetch_window(path, window, params) + + def diver_data( + self, + monitoring_point_id: int, + start: int, + end: int, + span: int = DEFAULT_SPAN, + ) -> Iterator[dict[str, Any]]: + """Yield raw ``DataPoint`` records -- pressure, temperature, and the + rest. Not water level; see ``water_levels`` for that.""" + path = f"DiverData/ByMonitoringPoint/{monitoring_point_id}" + for window in iter_windows(start, end, span): + yield from self._fetch_window(path, window, {}) + + def _fetch_window( + self, path: str, window: Window, params: dict[str, Any] + ) -> Iterator[dict[str, Any]]: + """Fetch one window, halving it on a 500 until it succeeds or hits the + floor. A 500 here means "too much data", which is the API's way of + asking to be given a narrower range.""" + response = self._get( + path, {**params, "startTime": window.start, "endTime": window.end} + ) + if response.status_code == 500: + try: + left, right = window.bisect() + except ValueError as exc: + raise DiverHubError( + f"{path} returned HTTP 500 for {window.span}s starting " + f"{window.start}, which is already at the minimum window. " + "This is not a volume problem." + ) from exc + yield from self._fetch_window(path, left, params) + yield from self._fetch_window(path, right, params) + return + yield from _expect_ok(response, path).json() + + +def _expect_ok(response: Response, what: str) -> Response: + if response.status_code != 200: + raise DiverHubError(f"{what} returned HTTP {response.status_code}.") + return response + + +def _parse_timestamp(value: str) -> float: + """Parse an ISO-8601 instant into a Unix timestamp. + + The API reports UTC but does not always mark it, so a naive value is read + as UTC rather than as local time -- reading it as local would shift token + expiry by the machine's offset and, worse, shift every reading. + """ + from datetime import datetime, timezone + + text = value.replace("Z", "+00:00") + parsed = datetime.fromisoformat(text) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.timestamp() + + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/dlt_pipeline.py b/automated_ingestion/sources/san_acacia/dlt_pipeline.py new file mode 100644 index 000000000..5caa4c122 --- /dev/null +++ b/automated_ingestion/sources/san_acacia/dlt_pipeline.py @@ -0,0 +1,230 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +dlt resources landing San Acacia in the GCS raw zone. + +Two resources, with deliberately different dispositions: + +* ``vanessen_locations`` -- the monitoring point roster, ``replace``. It is a + snapshot of what the vendor currently lists, and a point disappearing is + information we want to see rather than accumulate. +* ``vanessen_readings`` -- the water level series, ``append``, incremental on + the reading timestamp. Appending is what makes Mode B replay possible: the + raw zone keeps what the vendor said at the time, not just what it says now. + +Nothing is transformed here. The raw zone stores the vendor's payload as it +arrived, in the vendor's units and on the vendor's datum, so a mapping bug is a +reprocess rather than a re-fetch. Conversion to Ocotillo's model happens in the +adapter, downstream. +""" + +from collections.abc import Iterator +from typing import Any + +import dlt + +from automated_ingestion.shared.gcs import RAW_LAYOUT, raw_zone_bucket +from automated_ingestion.shared.windows import DAY +from automated_ingestion.shared.source_registry import SourceDefinition, register +from automated_ingestion.sources.san_acacia.client import ( + GROUND_SURFACE_REFERENCE, + SOURCE_UNIT, + DiverHubClient, + DiverHubError, +) + +PROJECT_ID = 4317 +"""Diver-HUB project ``SanAcaciaReach``. Confirmed by probing, not assumed.""" + +READING_SPAN = 365 * DAY +"""Window width for this source, measured rather than assumed. + +``WaterLevels`` served 730 days and 18111 rows in a single request when probed, +so the generic 90-day default in ``shared/windows.py`` would quadruple the +request count for no benefit -- a first run for one point covers a decade. This +sits at half the largest span observed to work, leaving room for a denser point +than SO-0125. +""" + +LOADER_FILE_FORMAT = "parquet" +"""Raw-zone file format. + +dlt writes gzipped JSONL unless told otherwise, and the first live run landed +that way. Parquet is what Mode B replay assumes: replay reads the raw zone +filtered on event time, and a columnar format with real types lets that read a +window without decompressing and parsing every record. It also preserves the +distinction between a null and a missing field, which JSONL round-trips less +reliably. +""" + +INITIAL_START = "2024-01-01T00:00:00+00:00" +"""Floor for a point that has never been ingested. + +Diver-HUB serves nothing before late 2024. Probing six points put their earliest +reading at 2024-10-08 and 2024-11-10, which matches the deployments on these +wells being installed 2024-11-25 -- the vendor project was populated then. + +The floor sits at 2024-01-01 rather than at the earliest observed reading, +because only six of the thirty-eight points were probed and a well with slightly +earlier data should not be silently truncated. Nine months of margin costs one +extra empty window; guessing too late loses real readings. + +It was 2015-01-01, chosen before anyone knew what the vendor retains. At a +365-day span that made a first run walk about twelve windows per well, ten of +them guaranteed empty, against an endpoint that answers 500 when pushed. + +Still a floor, never a backfill lever: lowering it will not re-fetch history for +a series whose watermark has advanced past it (`shared/watermark.py`), and there +is no history before 2024 to fetch. + +**The record has a gap.** The fourteen wells carrying AMPAPI data stop in August +2022 and the vendor starts in late 2024, so roughly twenty-seven months are +missing and cannot be recovered from this source. +""" + +SOURCE = register( + SourceDefinition( + key="san_acacia", + display_name="San Acacia Reach", + dataset_name="raw_sanacaciareach", + ) +) + + +@dlt.resource(name="vanessen_locations", write_disposition="replace") +def vanessen_locations(client: DiverHubClient) -> Iterator[dict[str, Any]]: + """The monitoring point roster. + + One request, no pagination. The payload is ``{id, name}`` and nothing more + -- no coordinates, no construction detail -- so this cannot be the source + of a well's geometry. It exists to enumerate the points a reading fetch + walks, and to record what the vendor listed on a given day. + """ + for point in client.monitoring_points(PROJECT_ID): + yield { + "monitoring_point_id": point["id"], + "name": point["name"], + "project_id": PROJECT_ID, + } + + +@dlt.resource(name="vanessen_readings", write_disposition="append") +def vanessen_readings( + client: DiverHubClient, + monitoring_points: list[dict[str, Any]], + end: int, + failures: list[dict[str, Any]], + cursor: dlt.sources.incremental[str] = dlt.sources.incremental( + "dateAndTime", initial_value=INITIAL_START + ), +) -> Iterator[dict[str, Any]]: + """Water levels for every point, from each point's watermark to ``end``. + + Failure is isolated per point. One diver returning a 500 for its whole + history should cost that diver's data for this run, not the other + thirty-seven -- so exceptions are caught here and appended to ``failures`` + rather than raised. + + ``failures`` is supplied by the caller rather than stashed on the resource: + a dlt resource is a module-level object shared by every run, so recording + per-run state on it would have one run overwriting another's. + """ + from automated_ingestion.sources.san_acacia.client import _parse_timestamp + + start = int(_parse_timestamp(cursor.last_value)) + + for point in monitoring_points: + point_id = point["monitoring_point_id"] + try: + approved_at = _approved_timestamps(client, point_id, start, end) + for row in client.water_levels( + point_id, + start, + end, + reference=GROUND_SURFACE_REFERENCE, + span=READING_SPAN, + ): + yield { + "monitoring_point_id": point_id, + "name": point["name"], + "dateAndTime": row["dateAndTime"], + "level": row["level"], + "unit": SOURCE_UNIT, + "reference": GROUND_SURFACE_REFERENCE, + "vendor_approved": row["dateAndTime"] in approved_at, + } + except DiverHubError as exc: + failures.append({"monitoring_point_id": point_id, "error": str(exc)}) + + +def _approved_timestamps( + client: DiverHubClient, point_id: int, start: int, end: int +) -> set[str]: + """Timestamps the vendor has marked approved. + + ``approved`` is a request parameter rather than a response field, so the + flag has to be recovered by asking twice. We take the unfiltered series as + the authoritative row set and use this only to tag it -- fetching + ``approved=true`` and ``approved=false`` separately and concatenating would + duplicate every row if the two sets overlap, which is not yet known. + + A failure here is not fatal: an untagged reading is worth more than no + reading, and the vendor flag is not Ocotillo's review status anyway. + """ + try: + rows = client.water_levels( + point_id, + start, + end, + reference=GROUND_SURFACE_REFERENCE, + approved=True, + span=READING_SPAN, + ) + return {row["dateAndTime"] for row in rows} + except DiverHubError: + return set() + + +def build_pipeline() -> Any: + """A dlt pipeline writing parquet to the raw zone. + + The pipeline is named after the bucket it writes to rather than after a + separately supplied environment. Those were two sources of truth for one + fact, and they disagreed the first time this ran in Dagster+: a pipeline + called ``san_acacia_staging`` writing to the production bucket, because the + name came from a run tag that was absent and the bucket came from the + environment. Deriving one from the other makes that impossible. + """ + # gcsfs resolves Application Default Credentials the same way the Cloud SQL + # connector does, and Serverless supplies none of its own. + from automated_ingestion.shared.credentials import ( + ensure_application_default_credentials, + ) + + ensure_application_default_credentials() + + bucket = raw_zone_bucket() + return dlt.pipeline( + pipeline_name=f"{SOURCE.key}_{bucket}", + destination=dlt.destinations.filesystem( + bucket_url=f"gs://{bucket}", + layout=RAW_LAYOUT, + ), + dataset_name=SOURCE.dataset_name, + ) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/ingest.py b/automated_ingestion/sources/san_acacia/ingest.py new file mode 100644 index 000000000..52228c4da --- /dev/null +++ b/automated_ingestion/sources/san_acacia/ingest.py @@ -0,0 +1,312 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Dagster assets for San Acacia. + +Both assets land raw payloads in GCS and report what happened as metadata -- +row counts, and for readings the number of points that failed. A run that +silently ingests nothing looks identical to a run with nothing to ingest, and +the metadata is what separates them. +""" + +from datetime import datetime, timezone +from typing import Any + +from dagster import AssetExecutionContext, MetadataValue, Output, asset + +from automated_ingestion.defs.resources import OcotilloDatabase +from automated_ingestion.sources.san_acacia.client import DiverHubClient + + +def _client() -> DiverHubClient: + import requests + + return DiverHubClient(requests.Session()) + + +@asset( + group_name="san_acacia", + description="Monitoring point roster for the San Acacia project, landed raw.", +) +def raw_san_acacia_locations(context: AssetExecutionContext) -> Output[int]: + """Land the point roster in the raw zone.""" + from automated_ingestion.sources.san_acacia.dlt_pipeline import ( + LOADER_FILE_FORMAT, + PROJECT_ID, + build_pipeline, + vanessen_locations, + ) + + client = _client() + points = list(client.monitoring_points(PROJECT_ID)) + pipeline = build_pipeline() + pipeline.run(vanessen_locations(client), loader_file_format=LOADER_FILE_FORMAT) + + context.log.info("landed %s monitoring points", len(points)) + return Output( + len(points), + metadata={ + "monitoring_points": MetadataValue.int(len(points)), + "project_id": MetadataValue.int(PROJECT_ID), + "names": MetadataValue.text(", ".join(p["name"] for p in points[:10])), + }, + ) + + +@asset( + group_name="san_acacia", + deps=[raw_san_acacia_locations], + description="Water level readings for every San Acacia point, landed raw.", +) +def raw_san_acacia_readings(context: AssetExecutionContext) -> Output[int]: + """Land water levels for every point, isolating per-point failure.""" + from automated_ingestion.sources.san_acacia.dlt_pipeline import ( + LOADER_FILE_FORMAT, + PROJECT_ID, + build_pipeline, + vanessen_readings, + ) + + client = _client() + points = [ + {"monitoring_point_id": p["id"], "name": p["name"]} + for p in client.monitoring_points(PROJECT_ID) + ] + end = int(datetime.now(tz=timezone.utc).timestamp()) + + pipeline = build_pipeline() + failures: list[dict[str, Any]] = [] + info = pipeline.run( + vanessen_readings(client, points, end, failures), + loader_file_format=LOADER_FILE_FORMAT, + ) + rows = _row_count(info) + + if failures: + context.log.warning( + "%s of %s points failed: %s", + len(failures), + len(points), + ", ".join(str(f["monitoring_point_id"]) for f in failures), + ) + + return Output( + rows, + metadata={ + "rows_ingested": MetadataValue.int(rows), + "points_attempted": MetadataValue.int(len(points)), + "points_failed": MetadataValue.int(len(failures)), + "failures": MetadataValue.json(failures), + }, + ) + + +@asset( + group_name="san_acacia", + deps=[raw_san_acacia_readings], + description="Water levels mapped to the Ocotillo model and loaded to Postgres.", +) +def san_acacia_observations( + context: AssetExecutionContext, database: OcotilloDatabase +) -> Output[int]: + """Load San Acacia water levels into `transducer_observation`. + + Per well: match the vendor point to an Ocotillo well, choose the deployment + its transducer hangs from, ask the database where that series got to, fetch + forward from there, map, and upsert. + + A well that cannot be resolved is skipped and counted, never guessed at. + Ingestion does not create wells or pick between candidate deployments, so an + unresolved well is a question for a person -- and skipping it costs that + well's readings for this run, not the other thirty-seven's. + """ + from datetime import datetime, timezone + + from automated_ingestion.ocotillo.loader import ensure_block, load_observations + from automated_ingestion.shared.watermark import ( + PostgresWatermarkStore, + resolve_start, + ) + from automated_ingestion.sources.san_acacia.adapter import SanAcaciaAdapter + from automated_ingestion.sources.san_acacia.client import GROUND_SURFACE_REFERENCE + from automated_ingestion.sources.san_acacia.dlt_pipeline import ( + INITIAL_START, + PROJECT_ID, + READING_SPAN, + ) + from automated_ingestion.sources.san_acacia.reconcile import ( + VendorPoint, + reconcile, + ) + from automated_ingestion.sources.san_acacia.resolve import ( + PARAMETER_NAME, + resolve_deployment, + ) + from domain.van_essen import parse_reading_timestamp + + client = _client() + points = [ + VendorPoint(monitoring_point_id=p["id"], name=p["name"]) + for p in client.monitoring_points(PROJECT_ID) + ] + end = int(datetime.now(tz=timezone.utc).timestamp()) + floor = parse_reading_timestamp(INITIAL_START) + + rows_loaded = 0 + skipped: list[dict[str, Any]] = [] + adapter_failures = 0 + + with database.session() as session: + parameter_id = _parameter_id(session, PARAMETER_NAME) + report = reconcile(points, _well_candidates(session)) + watermarks = PostgresWatermarkStore(session) + + for match in report.matches: + if match.needs_a_human: + skipped.append({"point": match.point.name, "reason": match.kind.value}) + continue + + thing_id = match.thing_id + resolution = resolve_deployment(_deployments(session, thing_id)) + if resolution.needs_a_human: + skipped.append( + {"point": match.point.name, "reason": resolution.kind.value} + ) + continue + + start = resolve_start(watermarks, thing_id, parameter_id, floor) + adapter = SanAcaciaAdapter() + raw = ( + { + "monitoring_point_id": match.point.monitoring_point_id, + "dateAndTime": row["dateAndTime"], + "level": row["level"], + "unit": "cm", + "reference": GROUND_SURFACE_REFERENCE, + } + for row in client.water_levels( + match.point.monitoring_point_id, + int(start.timestamp()), + end, + reference=GROUND_SURFACE_REFERENCE, + span=READING_SPAN, + ) + ) + + observations = list(adapter.to_observations(raw)) + adapter_failures += len(adapter.failures) + if not observations: + continue + + result = load_observations( + session, + observations, + resolution.deployment_id, + parameter_id, + release_status="public", + ) + rows_loaded += result.rows_written + ensure_block( + session, + thing_id=thing_id, + parameter_id=parameter_id, + start=min(o.observation_datetime for o in observations), + end=max(o.observation_datetime for o in observations), + release_status="public", + ) + + if skipped: + context.log.warning( + "%s of %s wells skipped: %s", + len(skipped), + len(points), + ", ".join(f"{s['point']} ({s['reason']})" for s in skipped), + ) + + return Output( + rows_loaded, + metadata={ + "rows_loaded": MetadataValue.int(rows_loaded), + "wells_attempted": MetadataValue.int(len(points)), + "wells_skipped": MetadataValue.int(len(skipped)), + "adapter_failures": MetadataValue.int(adapter_failures), + "skipped": MetadataValue.json(skipped), + }, + ) + + +def _parameter_id(session: Any, name: str) -> int: + from sqlalchemy import select + + from db.parameter import Parameter + + parameter_id = session.scalar( + select(Parameter.id).where(Parameter.parameter_name == name) + ) + if parameter_id is None: + raise RuntimeError( + f"No parameter named {name!r}. Ingestion does not create parameters; " + "seed it before loading." + ) + return parameter_id + + +def _well_candidates(session: Any) -> list[Any]: + """Ocotillo wells the vendor points might be, narrowed by name prefix.""" + from sqlalchemy import select + + from db.thing import Thing + + from automated_ingestion.sources.san_acacia.reconcile import ThingCandidate + + rows = session.execute( + select(Thing.id, Thing.name).where(Thing.name.ilike("SO-%")) + ).all() + return [ThingCandidate(thing_id=i, name=n) for i, n in rows] + + +def _deployments(session: Any, thing_id: int) -> list[Any]: + from sqlalchemy import select + + from db.deployment import Deployment + from db.sensor import Sensor + + from automated_ingestion.sources.san_acacia.resolve import DeploymentCandidate + + rows = session.execute( + select(Deployment.id, Sensor.sensor_type, Deployment.removal_date) + .join(Sensor, Sensor.id == Deployment.sensor_id) + .where(Deployment.thing_id == thing_id) + ).all() + return [ + DeploymentCandidate(deployment_id=i, sensor_type=t, removal_date=r) + for i, t, r in rows + ] + + +def _row_count(load_info: Any) -> int: + """Rows dlt reports as loaded, or 0 when it reports nothing.""" + try: + return sum( + metrics.get("rows_count", 0) + for job in load_info.load_packages + for metrics in getattr(job, "jobs", {}).values() + ) + except Exception: # noqa: BLE001 - metadata must never fail a good load + return 0 + + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/reconcile.py b/automated_ingestion/sources/san_acacia/reconcile.py new file mode 100644 index 000000000..9a555c866 --- /dev/null +++ b/automated_ingestion/sources/san_acacia/reconcile.py @@ -0,0 +1,223 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Matching Diver-HUB monitoring points to Ocotillo wells. + +Ingestion never creates a well. A vendor point that matches nothing is a +question for a person, not a row to invent -- the duplicate Geographic Area +groups elsewhere in this database are the standing reminder that "looks like a +new record" is not proof. + +So this decides, per point, one of three things: exactly one candidate +(matched), more than one (ambiguous, escalate), or none (unmatched, escalate). +It never picks a winner among candidates. Choosing between two plausible wells +is precisely the judgement that should not be automated. + +**Matching is on identifiers only.** The plan called for coordinate proximity as +a third signal; the live ``MonitoringPoint`` payload is ``{id, name}`` and +carries no coordinates, so there is nothing to compare. That removes the one +fuzzy signal and leaves two exact ones, which is a better position to be in -- +every match here is defensible rather than probabilistic. + +The functions are pure: they take vendor points and candidate rows and return a +report. Loading the candidates is the caller's job, so the decision logic is +testable without a database. +""" + +from collections.abc import Iterable +from dataclasses import dataclass, field +from enum import Enum + + +class MatchKind(str, Enum): + """How a point was matched, or why it was not.""" + + NAME = "matched-by-name" + EXTERNAL_ID = "matched-by-external-id" + AMBIGUOUS = "ambiguous" + UNMATCHED = "unmatched" + + +@dataclass(frozen=True) +class VendorPoint: + """A monitoring point as Diver-HUB reports it.""" + + monitoring_point_id: int + name: str + + +@dataclass(frozen=True) +class ThingCandidate: + """An Ocotillo well that might be the same well.""" + + thing_id: int + name: str + external_ids: tuple[str, ...] = () + + +@dataclass(frozen=True) +class Match: + """What was decided about one vendor point.""" + + point: VendorPoint + kind: MatchKind + thing_id: int | None = None + candidates: tuple[int, ...] = () + + @property + def needs_a_human(self) -> bool: + return self.kind in (MatchKind.AMBIGUOUS, MatchKind.UNMATCHED) + + +@dataclass +class ReconciliationReport: + """The whole picture, for a person to read before anything is written.""" + + matches: list[Match] = field(default_factory=list) + + @property + def matched(self) -> list[Match]: + return [m for m in self.matches if not m.needs_a_human] + + @property + def ambiguous(self) -> list[Match]: + return [m for m in self.matches if m.kind is MatchKind.AMBIGUOUS] + + @property + def unmatched(self) -> list[Match]: + return [m for m in self.matches if m.kind is MatchKind.UNMATCHED] + + @property + def ready(self) -> bool: + """True when every point resolved to exactly one well. + + Deliberately strict. A partial run that ingests the wells it recognised + and quietly skips the rest produces a series that looks complete and is + not. + """ + return bool(self.matches) and not any(m.needs_a_human for m in self.matches) + + +def _normalize(value: str) -> str: + """Reduce a well identifier to its significant characters. + + Case, spacing and punctuation are dropped, so ``SO-0125``, ``so 0125`` and + ``SO0125`` compare equal -- one identifier written three ways. + + This is still exact matching, not similarity: every significant character + must agree, so ``SO-0126`` remains a different well. The distinction matters + because a fuzzy matcher here would eventually merge two real wells, and the + whole point of this module is that it never chooses between candidates. + """ + return "".join(c for c in (value or "") if c.isalnum()).upper() + + +def match_point( + point: VendorPoint, + candidates: Iterable[ThingCandidate], + use_external_ids: bool = False, +) -> Match: + """Decide one point against the wells it might be. + + ``use_external_ids`` is off by default, for a specific reason. + ``thing_id_link`` holds identifiers from several organizations that disagree + with each other. In staging, ``SO-0131`` carries NMBGMR ``BRN-E04B + (shallow)`` plus an unattributed ``BRN-E04A``, while ``SO-0132`` carries + NMBGMR ``BRN-E04A (deep)`` plus an unattributed ``BRN-E04B`` -- the two + sources swap which physical well is A and which is B. + + Matching ``BRN-E04A`` against that returns a single confident hit on + SO-0131, contradicting NMBGMR, because the parenthetical suffix stops the + collision registering as ambiguous. A wrong answer delivered confidently is + worse than no answer. + + It costs nothing today: all 38 Diver-HUB points match Ocotillo wells by name. + """ + target = _normalize(point.name) + + by_name = [c for c in candidates if _normalize(c.name) == target] + by_external = ( + [ + c + for c in candidates + if any(_normalize(x) == target for x in c.external_ids) and c not in by_name + ] + if use_external_ids + else [] + ) + + # Name first: it is the identifier the Bureau uses, and an external id link + # is a record of an association someone made, which may be older. + hits = by_name or by_external + kind = MatchKind.NAME if by_name else MatchKind.EXTERNAL_ID + + if len(hits) == 1: + return Match(point=point, kind=kind, thing_id=hits[0].thing_id) + if len(hits) > 1: + return Match( + point=point, + kind=MatchKind.AMBIGUOUS, + candidates=tuple(c.thing_id for c in hits), + ) + return Match(point=point, kind=MatchKind.UNMATCHED) + + +def reconcile( + points: Iterable[VendorPoint], + candidates: Iterable[ThingCandidate], + use_external_ids: bool = False, +) -> ReconciliationReport: + """Match every vendor point, reporting rather than resolving.""" + candidate_list = list(candidates) + report = ReconciliationReport() + for point in points: + report.matches.append( + match_point(point, candidate_list, use_external_ids=use_external_ids) + ) + return report + + +def format_report(report: ReconciliationReport) -> str: + """Human-readable summary. This is the deliverable of task 3.2.""" + lines = [ + f"Vendor points : {len(report.matches)}", + f" matched : {len(report.matched)}", + f" ambiguous : {len(report.ambiguous)}", + f" unmatched : {len(report.unmatched)}", + "", + ] + if report.ready: + lines.append("Every point resolved to exactly one well.") + return "\n".join(lines) + + if report.ambiguous: + lines.append("Ambiguous -- more than one well matches. Do not auto-merge:") + for match in report.ambiguous: + ids = ", ".join(str(c) for c in match.candidates) + lines.append(f" {match.point.name:<12} thing ids: {ids}") + lines.append("") + if report.unmatched: + lines.append("Unmatched -- no well found. Ingestion will not create one:") + for match in report.unmatched: + lines.append( + f" {match.point.name:<12} (vendor id {match.point.monitoring_point_id})" + ) + lines.append("") + lines.append("Resolve these before loading; a partial load looks complete.") + return "\n".join(lines) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/resolve.py b/automated_ingestion/sources/san_acacia/resolve.py new file mode 100644 index 000000000..68638976d --- /dev/null +++ b/automated_ingestion/sources/san_acacia/resolve.py @@ -0,0 +1,111 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Choosing which deployment a water level belongs to. + +A San Acacia well carries several open deployments at once, because a deployment +is a piece of equipment rather than a measured property. SO-0140 has three: + + DiverLink DN431-1ch telemetry + Pressure Transducer DI801 10m measures the water level + Diver Cable AS2006-6m the cable + +Only the pressure transducer produces the reading being ingested, so that is the +deployment an observation hangs from. Picking any of the others would attribute +a water level to a cable. + +Like the reconciler, this never chooses between equally good candidates. Two +open transducers on one well is a question about the equipment record, not +something to resolve by taking the lower id. +""" + +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import date +from enum import Enum + +WATER_LEVEL_SENSOR_TYPE = "Pressure Transducer" +"""The sensor type whose deployment carries a water level. + +Checked against staging: of the 38 San Acacia wells, 35 have exactly one open +deployment of this type, 2 have two, and 1 has none. The other types present are +`DiverLink`, `Diver Cable` and `Barometer`, none of which measure depth to +water. +""" + +PARAMETER_NAME = "groundwater level" +"""The Ocotillo parameter these readings are. Its `default_unit` is `ft`, which +is what the adapter emits -- the conversion from the vendor's centimetres +happens in `domain/van_essen.py`.""" + + +class ResolutionKind(str, Enum): + RESOLVED = "resolved" + AMBIGUOUS = "ambiguous" + MISSING = "missing" + + +@dataclass(frozen=True) +class DeploymentCandidate: + """A deployment on the well, with the bit needed to judge it.""" + + deployment_id: int + sensor_type: str + removal_date: date | None = None + + @property + def is_open(self) -> bool: + return self.removal_date is None + + +@dataclass(frozen=True) +class Resolution: + """Which deployment to load into, or why none was chosen.""" + + kind: ResolutionKind + deployment_id: int | None = None + candidates: tuple[int, ...] = () + + @property + def needs_a_human(self) -> bool: + return self.kind is not ResolutionKind.RESOLVED + + +def resolve_deployment(candidates: Iterable[DeploymentCandidate]) -> Resolution: + """Pick the open pressure-transducer deployment, or refuse. + + Closed deployments are excluded rather than preferred-against: a removed + transducer is not where today's readings belong, and treating it as a + fallback would quietly write current data against retired equipment. + """ + open_transducers = [ + c for c in candidates if c.is_open and c.sensor_type == WATER_LEVEL_SENSOR_TYPE + ] + + if len(open_transducers) == 1: + return Resolution( + kind=ResolutionKind.RESOLVED, + deployment_id=open_transducers[0].deployment_id, + ) + if len(open_transducers) > 1: + return Resolution( + kind=ResolutionKind.AMBIGUOUS, + candidates=tuple(c.deployment_id for c in open_transducers), + ) + return Resolution(kind=ResolutionKind.MISSING) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/transform.py b/automated_ingestion/sources/san_acacia/transform.py new file mode 100644 index 000000000..4630ae8a2 --- /dev/null +++ b/automated_ingestion/sources/san_acacia/transform.py @@ -0,0 +1,30 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Reshaping that precedes adaptation. + +Less is needed here than the plan first assumed. The retired FROST pipeline +suggested Van Essen returned parallel arrays that had to be zipped into +records; the live API returns ``[{dateAndTime, level}]`` already, and selects +datum and approval through query parameters rather than through which array a +value came from. + +What remains for this module is timestamp normalisation and whatever +per-record tidying the live responses turn out to need. Filled in under BDMS +task 3.1, once the probe has run. +""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/sql/find_duplicate_observations.sql b/automated_ingestion/sql/find_duplicate_observations.sql new file mode 100644 index 000000000..6b49727ce --- /dev/null +++ b/automated_ingestion/sql/find_duplicate_observations.sql @@ -0,0 +1,44 @@ +-- Find duplicate transducer observations before adding the unique constraint. +-- +-- The migration that adds UNIQUE (deployment_id, parameter_id, +-- observation_datetime) will fail on a table that already violates it, and +-- failing halfway through a production migration is worse than not starting. +-- Run this first, on every environment the migration will touch. +-- +-- psql "..." -f automated_ingestion/sql/find_duplicate_observations.sql +-- +-- No rows means the migration is safe to run. Rows mean a decision is needed +-- about which copy to keep, and that decision belongs to someone who knows the +-- data -- deleting the higher id is a guess, not a rule, because the rows may +-- differ in `value` rather than being true duplicates. + +\echo '== Duplicate groups ==' +SELECT + deployment_id, + parameter_id, + observation_datetime, + count(*) AS copies, + count(DISTINCT value) AS distinct_values, + min(id) AS lowest_id, + max(id) AS highest_id +FROM transducer_observation +GROUP BY deployment_id, parameter_id, observation_datetime +HAVING count(*) > 1 +ORDER BY copies DESC, observation_datetime +LIMIT 100; + +\echo '' +\echo '== Totals ==' +-- `distinct_values > 1` is the interesting case: those are not redundant copies +-- but disagreeing measurements, and collapsing them silently would discard a +-- reading somebody recorded. +SELECT + count(*) AS duplicate_groups, + sum(copies) - count(*) AS rows_above_the_first, + count(*) FILTER (WHERE distinct_values > 1) AS groups_that_disagree +FROM ( + SELECT count(*) AS copies, count(DISTINCT value) AS distinct_values + FROM transducer_observation + GROUP BY deployment_id, parameter_id, observation_datetime + HAVING count(*) > 1 +) g; diff --git a/automated_ingestion/sql/ingestion_role.sql b/automated_ingestion/sql/ingestion_role.sql new file mode 100644 index 000000000..25350ca9d --- /dev/null +++ b/automated_ingestion/sql/ingestion_role.sql @@ -0,0 +1,102 @@ +-- Least-privilege Postgres role for the automated ingestion pipeline. +-- +-- Run by hand against each environment as a superuser. Not an Alembic +-- migration: roles and grants are per-environment infrastructure, not schema, +-- and migrations run under this database's application role rather than a +-- superuser. +-- +-- The point of the role is blast radius. The pipeline writes observations and +-- the reference rows they hang from, and reads everything it must resolve +-- against. It cannot touch chemistry, contacts, assets, or the legacy NMA_* +-- and NMW_* tables, so a bug in an adapter cannot corrupt data no ingestion +-- path should ever reach. + +-- IAM authentication is the configured path, and the reason is that it removes +-- the credential rather than rotating it: Cloud SQL mints a short-lived token +-- from the service account, so there is no password to store anywhere. +-- +-- **The role already exists.** Registering the service account as a Cloud SQL +-- IAM user creates the Postgres role automatically -- Terraform does that via +-- google_sql_user.ingestion. Confirmed with: +-- +-- gcloud sql users list --instance=dataservices +-- ... +-- ocotillo-ingestion@waterdatainitiative-271000.iam CLOUD_IAM_SERVICE_ACCOUNT +-- +-- So this script only grants. Do not add a CREATE ROLE: it would fail, and +-- reaching for one is a sign the Terraform half has not been applied. +-- +-- Run it as a superuser, passing both names -- nothing is hardcoded, because +-- the instance hosts `ocotillo` and `ocotillo-staging` and running the wrong +-- one is silent: +-- +-- psql "host=... dbname=ocotillo user=postgres" \ +-- -v db_name=ocotillo \ +-- -v role_name=ocotillo-ingestion@waterdatainitiative-271000.iam \ +-- -f automated_ingestion/sql/ingestion_role.sql +-- +-- The role name has an @ and dots, so every reference below uses :"role_name", +-- which quotes it as an identifier. An unquoted one is a syntax error. +-- +-- Password authentication, if IAM is ever unavailable: create the role by hand, +-- store the password in Secret Manager, set CLOUD_SQL_IAM_AUTH=0, and pass +-- -v role_name=ocotillo_ingestion instead. + +\if :{?db_name} +\else +\echo 'ERROR: pass -v db_name=. The instance hosts more than one.' +\quit +\endif + +\if :{?role_name} +\else +\echo 'ERROR: pass -v role_name=. See the header for the IAM role name.' +\quit +\endif + +\echo 'Granting to' :"role_name" 'on' :"db_name" + +GRANT CONNECT ON DATABASE :"db_name" TO :"role_name"; +GRANT USAGE ON SCHEMA public TO :"role_name"; + +-- Written: the observations themselves and the rows a new series needs. +GRANT SELECT, INSERT, UPDATE ON + transducer_observation, + transducer_observation_block, + deployment, + sensor, + parameter +TO :"role_name"; + +-- `parameter` is versioned by sqlalchemy-continuum, so an insert there also +-- writes a version row and a transaction row. Without these two grants the +-- write fails at runtime with a permission error on a table the code never +-- names directly -- an unpleasant thing to debug. +GRANT SELECT, INSERT ON parameter_version, transaction TO :"role_name"; + +-- Read-only: resolved against, never written. `thing` and `location` are +-- deliberately not writable. Reconciling the 33 San Acacia wells means +-- matching them to rows that already exist; if reconciliation finds a well +-- missing, that is a decision for a human, not a row the pipeline invents. +GRANT SELECT ON + thing, + thing_id_link, + location, + lexicon_term, + lexicon_category, + lexicon_term_category_association +TO :"role_name"; + +-- Inserts need the sequences behind the autoincrement primary keys. +GRANT USAGE, SELECT ON SEQUENCE + transducer_observation_id_seq, + transducer_observation_block_id_seq, + deployment_id_seq, + sensor_id_seq, + parameter_id_seq, + transaction_id_seq +TO :"role_name"; + +-- No default privileges are granted. A table added later is invisible to this +-- role until someone grants it deliberately, which is the intended failure +-- mode: a new table reaching the pipeline should be a decision. diff --git a/automated_ingestion/tests/__init__.py b/automated_ingestion/tests/__init__.py new file mode 100644 index 000000000..6df237838 --- /dev/null +++ b/automated_ingestion/tests/__init__.py @@ -0,0 +1,18 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Tests for the automated ingestion code location.""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/conftest.py b/automated_ingestion/tests/conftest.py new file mode 100644 index 000000000..801f5d827 --- /dev/null +++ b/automated_ingestion/tests/conftest.py @@ -0,0 +1,29 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Skip this directory when the ingestion dependency group is not installed. + +dagster lives in the optional ``ingestion`` group, so a developer who ran a +plain ``uv sync`` has no dagster in the environment. Without this guard, +collecting these modules raises ``ImportError`` and takes the whole suite down +with it -- the API tests would fail for a package the API never imports. +""" + +from importlib.util import find_spec + +collect_ignore_glob = [] if find_spec("dagster") else ["*.py"] + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_backfill.py b/automated_ingestion/tests/test_backfill.py new file mode 100644 index 000000000..6b9a88808 --- /dev/null +++ b/automated_ingestion/tests/test_backfill.py @@ -0,0 +1,179 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Backfill primitives. Pure, so none of this needs a database or a network. +""" + +from datetime import datetime, timezone + +import pytest + +from automated_ingestion.shared.backfill import ( + Chunk, + ChunkResult, + InMemoryCheckpointStore, + attach_run_timestamp, + chunk_key, + month_chunks, + parse_backfill_date, + pending_chunks, + resolve_location_ids, + sanitize_run_key, + sum_chunk_results, + validate_date_order, +) + + +def _utc(y, m, d): + return datetime(y, m, d, tzinfo=timezone.utc) + + +class TestMonthChunks: + def test_window_is_split_on_calendar_months(self): + chunks = list(month_chunks(_utc(2026, 1, 15), _utc(2026, 4, 10))) + assert [c.key for c in chunks] == ["2026-01", "2026-02", "2026-03", "2026-04"] + + def test_edges_are_clipped_not_widened(self): + # Widening would fetch data the operator did not ask for. + chunks = list(month_chunks(_utc(2026, 1, 15), _utc(2026, 2, 10))) + assert chunks[0].start == _utc(2026, 1, 15) + assert chunks[-1].end == _utc(2026, 2, 10) + + def test_chunks_do_not_overlap(self): + chunks = list(month_chunks(_utc(2025, 11, 3), _utc(2026, 3, 20))) + for earlier, later in zip(chunks, chunks[1:]): + assert earlier.end < later.start + + def test_window_inside_one_month_is_a_single_chunk(self): + assert len(list(month_chunks(_utc(2026, 1, 5), _utc(2026, 1, 20)))) == 1 + + def test_year_boundary(self): + chunks = list(month_chunks(_utc(2025, 12, 20), _utc(2026, 1, 10))) + assert [c.key for c in chunks] == ["2025-12", "2026-01"] + + def test_reversed_window_is_rejected(self): + with pytest.raises(ValueError, match="must be after"): + list(month_chunks(_utc(2026, 5, 1), _utc(2026, 1, 1))) + + +class TestDates: + def test_bare_date_is_utc_midnight(self): + assert parse_backfill_date("2026-01-15") == _utc(2026, 1, 15) + + def test_naive_datetime_is_read_as_utc(self): + # The same run config must mean the same window on every machine. + assert parse_backfill_date("2026-01-15T00:00:00") == _utc(2026, 1, 15) + + def test_offset_is_normalized(self): + assert parse_backfill_date("2026-01-14T18:00:00-06:00") == _utc(2026, 1, 15) + + @pytest.mark.parametrize("value", ["", " ", "yesterday", None]) + def test_unusable_dates_are_rejected(self, value): + with pytest.raises(ValueError): + parse_backfill_date(value) + + def test_empty_window_is_rejected(self): + # A backfill that succeeds having done nothing is indistinguishable from + # one that worked, and the operator never learns they typed the dates + # backwards. + with pytest.raises(ValueError): + validate_date_order(_utc(2026, 1, 1), _utc(2026, 1, 1)) + + +class TestRunKeys: + def test_path_separators_are_removed(self): + # An unsanitized key with a slash writes checkpoints into a directory of + # its own, and a resumed run does not find them. + assert "/" not in sanitize_run_key("march/gap") + + def test_unusable_keys_are_rejected(self): + with pytest.raises(ValueError): + sanitize_run_key("///") + + def test_timestamp_is_appended_only_when_asked(self): + stamped = attach_run_timestamp("gap", now=_utc(2026, 1, 15)) + assert stamped == "gap-20260115T000000Z" + + def test_chunk_key_combines_run_and_month(self): + chunk = Chunk(start=_utc(2026, 3, 1), end=_utc(2026, 3, 31)) + assert chunk_key("march gap", chunk) == "march-gap/2026-03" + + +class TestLocationIds: + def test_empty_request_means_everything(self): + assert resolve_location_ids([], [39, 40, 41]) == [39, 40, 41] + + def test_unknown_ids_fail_the_run(self): + # Silently backfilling nothing looks identical to backfilling + # successfully, and the gap is still there weeks later. + with pytest.raises(ValueError, match="99"): + resolve_location_ids([39, 99], [39, 40]) + + def test_error_names_every_bad_id(self): + with pytest.raises(ValueError) as exc: + resolve_location_ids([98, 99], [39]) + assert "98" in str(exc.value) and "99" in str(exc.value) + + def test_requested_subset_is_preserved(self): + assert resolve_location_ids([41, 39], [39, 40, 41]) == [41, 39] + + +class TestCheckpoints: + def test_pending_excludes_completed(self): + store = InMemoryCheckpointStore() + chunks = list(month_chunks(_utc(2026, 1, 1), _utc(2026, 4, 1))) + store.mark_complete("gap", chunks[0]) + assert [c.key for c in pending_chunks(store, "gap", chunks)] == [ + "2026-02", + "2026-03", + ] + + def test_checkpoints_are_scoped_to_the_run(self): + store = InMemoryCheckpointStore() + chunks = list(month_chunks(_utc(2026, 1, 1), _utc(2026, 3, 1))) + store.mark_complete("gap", chunks[0]) + assert len(pending_chunks(store, "other", chunks)) == 2 + + def test_run_key_is_sanitized_consistently(self): + # Marking under one spelling and resuming under another must not lose + # the checkpoint. + store = InMemoryCheckpointStore() + chunk = Chunk(start=_utc(2026, 1, 1), end=_utc(2026, 1, 31)) + store.mark_complete("march gap", chunk) + assert store.completed("march-gap") == {"2026-01"} + + +class TestTotals: + def test_totals_sum_across_chunks(self): + totals = sum_chunk_results( + [ + ChunkResult("2026-01", rows_ingested=100, rows_upserted=98, failures=2), + ChunkResult("2026-02", rows_ingested=50, rows_upserted=50), + ] + ) + assert totals.chunks == 2 + assert totals.rows_ingested == 150 + assert totals.rows_upserted == 148 + assert totals.failures == 2 + assert totals.chunk_keys == ["2026-01", "2026-02"] + + def test_refused_rows_are_derived(self): + assert ( + ChunkResult("2026-01", rows_ingested=10, rows_upserted=7).rows_refused == 3 + ) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_connectivity.py b/automated_ingestion/tests/test_connectivity.py new file mode 100644 index 000000000..622f9f1e7 --- /dev/null +++ b/automated_ingestion/tests/test_connectivity.py @@ -0,0 +1,85 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +The connectivity asset is wired up, and does not reach the database until run. + +Loading the code location must not open a connection: Dagster lists assets far +more often than it runs them, and a code location that needs a database to load +is a code location that breaks whenever the database is briefly unreachable. +""" + +from dagster import AssetKey + +from automated_ingestion.defs.definitions import defs + + +def test_connectivity_asset_is_registered(): + assert AssetKey(["database_connectivity"]) in defs.resolve_all_asset_keys() + + +def test_database_resource_is_provided(): + assert "database" in defs.resources + + +def test_loading_definitions_does_not_import_db_engine(): + # db.engine builds its engine at import time, so listing assets must not + # reach it. Checking sys.modules in-process would only observe whichever + # test imported it first, so ask a clean interpreter instead. + import subprocess + import sys + + result = subprocess.run( + [ + sys.executable, + "-c", + "import automated_ingestion.defs.definitions as d; " + "import sys; " + "assert d.defs is not None; " + "print('db.engine' in sys.modules)", + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "False", result.stdout + + +def test_db_imports_from_an_unrelated_working_directory(): + # Reproduces the deployed condition: a process whose cwd is not the + # repository. The lazy imports in the resource and the connectivity asset + # run at step execution, not at load, so this is the path that broke. + # In the image this passes because the repository is installed; locally + # because the editable install has the same effect. + import subprocess + import sys + + result = subprocess.run( + [ + sys.executable, + "-c", + "import automated_ingestion; " + "from db.transducer import TransducerObservation; " + "print(TransducerObservation.__tablename__)", + ], + capture_output=True, + text=True, + cwd="/", + ) + assert result.returncode == 0, result.stderr + assert "transducer_observation" in result.stdout + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_credentials.py b/automated_ingestion/tests/test_credentials.py new file mode 100644 index 000000000..c7e788f88 --- /dev/null +++ b/automated_ingestion/tests/test_credentials.py @@ -0,0 +1,88 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Credential materialization for a runtime with no metadata server. + +The failure this prevents is not subtle -- DefaultCredentialsError -- but it +only appears in Serverless, so the tests stand in for a deployment. +""" + +import json +import os + +import pytest + +from automated_ingestion.shared import credentials +from automated_ingestion.shared.credentials import ( + CREDENTIALS_ENV_VAR, + ensure_application_default_credentials, +) + +KEY = {"type": "service_account", "project_id": "waterdatainitiative-271000"} + + +@pytest.fixture(autouse=True) +def _clean(monkeypatch): + monkeypatch.setattr(credentials, "_written_path", None) + monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) + monkeypatch.delenv(CREDENTIALS_ENV_VAR, raising=False) + + +def test_existing_credentials_are_left_alone(monkeypatch): + # A developer's gcloud login must not be shadowed by a key in the + # environment; whatever is already configured wins. + monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/existing/adc.json") + monkeypatch.setenv(CREDENTIALS_ENV_VAR, json.dumps(KEY)) + assert ensure_application_default_credentials() == "/existing/adc.json" + + +def test_no_key_configured_is_not_an_error(monkeypatch): + # Locally this is normal -- google.auth finds its own credentials. In + # Serverless it fails later, loudly, which is the correct outcome. + assert ensure_application_default_credentials() is None + assert "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ + + +def test_key_is_written_and_pointed_at(monkeypatch): + monkeypatch.setenv(CREDENTIALS_ENV_VAR, json.dumps(KEY)) + path = ensure_application_default_credentials() + assert path and os.path.exists(path) + assert os.environ["GOOGLE_APPLICATION_CREDENTIALS"] == path + with open(path) as fh: + assert json.load(fh) == KEY + + +def test_key_file_is_not_world_readable(monkeypatch): + monkeypatch.setenv(CREDENTIALS_ENV_VAR, json.dumps(KEY)) + path = ensure_application_default_credentials() + assert oct(os.stat(path).st_mode)[-3:] == "600" + + +def test_repeated_calls_write_once(monkeypatch): + monkeypatch.setenv(CREDENTIALS_ENV_VAR, json.dumps(KEY)) + first = ensure_application_default_credentials() + assert ensure_application_default_credentials() == first + + +def test_a_path_instead_of_a_key_is_rejected(monkeypatch): + # Setting the variable to a filename is the obvious mistake, and it would + # otherwise fail much later inside google.auth. + monkeypatch.setenv(CREDENTIALS_ENV_VAR, "/path/to/key.json") + with pytest.raises(RuntimeError, match="not valid JSON"): + ensure_application_default_credentials() + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_definitions.py b/automated_ingestion/tests/test_definitions.py new file mode 100644 index 000000000..b0745dcb0 --- /dev/null +++ b/automated_ingestion/tests/test_definitions.py @@ -0,0 +1,45 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +The code location loads. + +Cheap, but it is the check that catches the failure this package is most prone +to: a Dagster+ deploy that builds fine and then cannot import. +""" + +from dagster import AssetKey, Definitions + +from automated_ingestion.defs.definitions import defs + + +def test_definitions_object_is_loadable(): + assert isinstance(defs, Definitions) + + +def test_heartbeat_asset_is_registered(): + assert AssetKey(["ingestion_heartbeat"]) in defs.resolve_all_asset_keys() + + +def test_heartbeat_materializes_without_external_dependencies(): + from dagster import materialize + + from automated_ingestion.defs.assets.heartbeat import ingestion_heartbeat + + result = materialize([ingestion_heartbeat]) + assert result.success + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_diverhub_client.py b/automated_ingestion/tests/test_diverhub_client.py new file mode 100644 index 000000000..78ccd37b8 --- /dev/null +++ b/automated_ingestion/tests/test_diverhub_client.py @@ -0,0 +1,200 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Client behaviour that is easy to get wrong and expensive to get wrong: +token refresh, the 401 retry, and narrowing on a 500. + +No network. The transport is a stub that records what it was asked for. +""" + +import pytest + +from automated_ingestion.shared.windows import DAY +from automated_ingestion.sources.san_acacia.client import ( + DiverHubClient, + DiverHubError, +) + + +class FakeResponse: + def __init__(self, status_code=200, payload=None): + self.status_code = status_code + self._payload = payload if payload is not None else [] + + def json(self): + return self._payload + + +class FakeTransport: + """Records calls and replays queued responses.""" + + def __init__(self, get_responses=None, token_valid_for=3600): + self.posts = [] + self.gets = [] + self._get_responses = list(get_responses or []) + self._token_valid_for = token_valid_for + self.login_count = 0 + + def post(self, url, **kwargs): + self.posts.append((url, kwargs)) + self.login_count += 1 + from datetime import datetime, timedelta, timezone + + valid_to = datetime.now(tz=timezone.utc) + timedelta( + seconds=self._token_valid_for + ) + return FakeResponse( + 200, + {"token": f"token-{self.login_count}", "validTo": valid_to.isoformat()}, + ) + + def get(self, url, **kwargs): + self.gets.append((url, kwargs)) + if self._get_responses: + return self._get_responses.pop(0) + return FakeResponse(200, []) + + +def _client(transport): + return DiverHubClient(transport, username="u", password="p") + + +def test_missing_credentials_fail_fast(monkeypatch): + monkeypatch.delenv("DIVERHUB_USERNAME", raising=False) + monkeypatch.delenv("DIVERHUB_PASSWORD", raising=False) + with pytest.raises(DiverHubError, match="credentials"): + DiverHubClient(FakeTransport()) + + +def test_token_is_reused_across_calls(): + transport = FakeTransport() + client = _client(transport) + client.projects() + client.projects() + assert transport.login_count == 1 + + +def test_token_is_refreshed_once_expired(): + # validTo in the past means every call re-authenticates. + transport = FakeTransport(token_valid_for=-10) + client = _client(transport) + client.projects() + client.projects() + assert transport.login_count == 2 + + +def test_expiry_skew_refreshes_before_the_deadline(): + # A token valid for 30s is already inside the skew window, so it must not + # be used: a request in flight at the boundary would arrive expired. + transport = FakeTransport(token_valid_for=30) + client = _client(transport) + client.projects() + client.projects() + assert transport.login_count == 2 + + +def test_401_forces_one_reauthentication_and_retry(): + transport = FakeTransport( + get_responses=[FakeResponse(401), FakeResponse(200, [{"id": 1}])] + ) + client = _client(transport) + assert client.projects() == [{"id": 1}] + assert transport.login_count == 2 + assert len(transport.gets) == 2 + + +def test_500_narrows_the_window_and_stitches_the_halves(): + # First window 500s; each half then succeeds and both are returned. + transport = FakeTransport( + get_responses=[ + FakeResponse(500), + FakeResponse(200, [{"level": 1.0}]), + FakeResponse(200, [{"level": 2.0}]), + ] + ) + client = _client(transport) + rows = list(client.water_levels(40, 0, 100 * DAY, reference=0, span=100 * DAY)) + assert [r["level"] for r in rows] == [1.0, 2.0] + + +def test_persistent_500_at_the_floor_is_an_error_not_a_loop(): + transport = FakeTransport(get_responses=[FakeResponse(500)] * 50) + client = _client(transport) + with pytest.raises(DiverHubError, match="not a volume problem"): + list(client.water_levels(40, 0, DAY, reference=0, span=DAY)) + + +def test_water_levels_sends_reference_and_unix_seconds(): + transport = FakeTransport() + client = _client(transport) + list(client.water_levels(40, 0, DAY, reference=2, span=DAY)) + _, kwargs = transport.gets[0] + params = kwargs["params"] + assert params["reference"] == 2 + assert params["startTime"] == 0 + assert params["endTime"] == DAY + assert isinstance(params["startTime"], int) + + +def test_approved_is_omitted_unless_asked_for(): + transport = FakeTransport() + client = _client(transport) + list(client.water_levels(40, 0, DAY, reference=0, span=DAY)) + assert "approved" not in transport.gets[0][1]["params"] + + +def test_naive_valid_to_is_read_as_utc(): + # The API documents UTC but does not always mark it. Reading a naive + # timestamp as local time would shift expiry by the machine's offset. + from automated_ingestion.sources.san_acacia.client import _parse_timestamp + + naive = _parse_timestamp("2026-08-18T20:00:00") + aware = _parse_timestamp("2026-08-18T20:00:00Z") + assert naive == aware + + +def test_datum_constants_match_the_measured_relationships(): + # Determined by probing, not read from the spec: ref0/ref2 rise with the + # water and ref1/ref3 fall, so the depths are 1 and 3, and ref1 is deeper + # than ref3 by a fixed casing stickup. Getting this wrong does not raise -- + # it silently records every reading on the wrong datum. + from automated_ingestion.sources.san_acacia import client as module + + assert module.GROUND_SURFACE_REFERENCE == 3 + assert module.TOP_OF_CASING_REFERENCE == 1 + assert module.ELEVATION_REFERENCE == 2 + assert module.GROUND_SURFACE_REFERENCE != module.TOP_OF_CASING_REFERENCE + + +def test_source_unit_is_centimeters_not_feet(): + # The vendor reports cm and Ocotillo stores ft. A value passed through + # unconverted is wrong by a factor of 30.48 and still looks like a plausible + # depth, which is exactly the kind of error that survives review. + from domain.units import convert_cm_to_ft + from automated_ingestion.sources.san_acacia import client as module + + assert module.SOURCE_UNIT == "cm" + # SO-0125 on 2024-10-30: 471.518 cm below ground surface. + assert convert_cm_to_ft(471.518) == 15.469751 + + +def test_cm_conversion_passes_none_through(): + from domain.units import convert_cm_to_ft + + assert convert_cm_to_ft(None) is None + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_gcs.py b/automated_ingestion/tests/test_gcs.py new file mode 100644 index 000000000..ad219715f --- /dev/null +++ b/automated_ingestion/tests/test_gcs.py @@ -0,0 +1,77 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Bucket resolution refuses to guess. + +The uploads-bucket check is the one worth testing: `services/gcs_helper.py` +already uses GCS_BUCKET_NAME, and the two variables being confused is a +configuration mistake that would otherwise succeed quietly. +""" + +import pytest + +from automated_ingestion.shared.gcs import BUCKET_ENV_VAR, RAW_LAYOUT, raw_zone_bucket + + +def test_returns_the_configured_bucket(monkeypatch): + monkeypatch.setenv(BUCKET_ENV_VAR, "ocotillo-ingestion-staging") + monkeypatch.delenv("GCS_BUCKET_NAME", raising=False) + assert raw_zone_bucket() == "ocotillo-ingestion-staging" + + +def test_unset_bucket_raises(monkeypatch): + monkeypatch.delenv(BUCKET_ENV_VAR, raising=False) + with pytest.raises(RuntimeError, match=BUCKET_ENV_VAR): + raw_zone_bucket() + + +def test_blank_bucket_raises(monkeypatch): + monkeypatch.setenv(BUCKET_ENV_VAR, " ") + with pytest.raises(RuntimeError, match=BUCKET_ENV_VAR): + raw_zone_bucket() + + +def test_uploads_bucket_is_rejected(monkeypatch): + monkeypatch.setenv(BUCKET_ENV_VAR, "ocotillo-uploads") + monkeypatch.setenv("GCS_BUCKET_NAME", "ocotillo-uploads") + with pytest.raises(RuntimeError, match="user-upload"): + raw_zone_bucket() + + +def test_layout_partitions_by_date(): + # Mode B replay selects a window by prefix, which only works if the date + # is in the path rather than inside the file. + assert "year={YYYY}" in RAW_LAYOUT + assert "month={MM}" in RAW_LAYOUT + assert "day={DD}" in RAW_LAYOUT + + +def test_pipeline_name_follows_the_bucket(monkeypatch): + # The name and the destination must not be able to disagree. They did once: + # a pipeline called san_acacia_staging wrote to the production bucket, + # because the name came from an absent run tag and the bucket from the + # environment. + monkeypatch.setenv(BUCKET_ENV_VAR, "ocotillo-ingestion-production") + monkeypatch.delenv("GCS_BUCKET_NAME", raising=False) + monkeypatch.setenv("INGESTION_GCP_CREDENTIALS_JSON", "") + + from automated_ingestion.sources.san_acacia.dlt_pipeline import build_pipeline + + pipeline = build_pipeline() + assert "ocotillo-ingestion-production" in pipeline.pipeline_name + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_ingest_assets.py b/automated_ingestion/tests/test_ingest_assets.py new file mode 100644 index 000000000..7170fd9b3 --- /dev/null +++ b/automated_ingestion/tests/test_ingest_assets.py @@ -0,0 +1,328 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +The Dagster assets, which is where the pieces meet. + +Every part of `san_acacia_observations` is covered on its own -- matching, +resolving, watermarks, the adapter, the loader. What was not covered is the +orchestration between them: which wells get skipped, what the metadata says, and +whether one unresolvable well costs the others. + +Fakes stand in at the process boundaries -- the vendor client, the database +session, the dlt pipeline -- and nowhere else. The reconciler, resolver and +adapter run for real, so a change in their behaviour shows up here. +""" + +from contextlib import contextmanager +from datetime import date + +import pytest +from dagster import build_asset_context + +from automated_ingestion.ocotillo import loader as loader_module +from automated_ingestion.ocotillo.loader import LoadResult +from automated_ingestion.shared import watermark as watermark_module +from automated_ingestion.sources.san_acacia import ingest +from automated_ingestion.sources.san_acacia.reconcile import ThingCandidate +from automated_ingestion.sources.san_acacia.resolve import DeploymentCandidate + +READING = {"dateAndTime": "2026-04-15T22:45:00", "level": 471.518} + + +class FakeClient: + """The vendor, reduced to what the assets ask of it.""" + + def __init__(self, points, readings=None, approved=()): + self._points = points + self._readings = READING if readings is None else readings + self._approved = approved + self.water_level_calls = [] + + def monitoring_points(self, project_id): + return self._points + + def water_levels(self, point_id, start, end, reference, approved=None, span=None): + self.water_level_calls.append((point_id, start, end, approved)) + if approved: + return iter(self._approved) + return iter( + self._readings if isinstance(self._readings, list) else [self._readings] + ) + + +class FakeDatabase: + """Stands in for OcotilloDatabase. The session is never really used -- + every function that would touch it is replaced.""" + + @contextmanager + def session(self): + yield object() + + +class NoWatermark: + def __init__(self, session): + pass + + def get(self, thing_id, parameter_id): + return None + + +@pytest.fixture() +def wired(monkeypatch): + """Wire the asset to fakes, returning the recorded loads.""" + loaded = [] + + def fake_load(session, records, deployment_id, parameter_id, release_status, **kw): + records = list(records) + loaded.append( + { + "deployment_id": deployment_id, + "parameter_id": parameter_id, + "release_status": release_status, + "rows": len(records), + } + ) + return LoadResult(rows_seen=len(records), rows_written=len(records), batches=1) + + monkeypatch.setattr(loader_module, "load_observations", fake_load) + monkeypatch.setattr(loader_module, "ensure_block", lambda *a, **k: 1) + monkeypatch.setattr(watermark_module, "PostgresWatermarkStore", NoWatermark) + monkeypatch.setattr(ingest, "_parameter_id", lambda session, name: 1) + return loaded + + +def _run(monkeypatch, client, candidates, deployments): + monkeypatch.setattr(ingest, "_client", lambda: client) + monkeypatch.setattr(ingest, "_well_candidates", lambda session: candidates) + monkeypatch.setattr(ingest, "_deployments", lambda session, thing_id: deployments) + return ingest.san_acacia_observations(build_asset_context(), FakeDatabase()) + + +TRANSDUCER = DeploymentCandidate(437, "Pressure Transducer") + + +class TestObservationsAsset: + def test_a_resolvable_well_is_loaded(self, monkeypatch, wired): + output = _run( + monkeypatch, + FakeClient([{"id": 39, "name": "SO-0125"}]), + [ThingCandidate(2343, "SO-0125")], + [TRANSDUCER], + ) + assert output.value == 1 + assert wired[0]["deployment_id"] == 437 + assert wired[0]["release_status"] == "public" + assert output.metadata["wells_skipped"].value == 0 + + def test_an_unmatched_well_is_skipped_not_invented(self, monkeypatch, wired): + # No Ocotillo well by that name. Ingestion does not create wells. + output = _run( + monkeypatch, + FakeClient([{"id": 39, "name": "SO-9999"}]), + [ThingCandidate(2343, "SO-0125")], + [TRANSDUCER], + ) + assert output.value == 0 + assert wired == [] + assert output.metadata["wells_skipped"].value == 1 + assert "unmatched" in str(output.metadata["skipped"].data) + + def test_an_ambiguous_well_is_skipped(self, monkeypatch, wired): + # Two wells share the name -- picking one would be a silent guess. + output = _run( + monkeypatch, + FakeClient([{"id": 39, "name": "SO-0125"}]), + [ThingCandidate(1, "SO-0125"), ThingCandidate(2, "SO-0125")], + [TRANSDUCER], + ) + assert wired == [] + assert "ambiguous" in str(output.metadata["skipped"].data) + + def test_a_well_without_a_transducer_is_skipped(self, monkeypatch, wired): + # SO-0246 is in this state in production. + output = _run( + monkeypatch, + FakeClient([{"id": 39, "name": "SO-0125"}]), + [ThingCandidate(2343, "SO-0125")], + [DeploymentCandidate(436, "DiverLink")], + ) + assert wired == [] + assert "missing" in str(output.metadata["skipped"].data) + + def test_a_removed_transducer_does_not_qualify(self, monkeypatch, wired): + output = _run( + monkeypatch, + FakeClient([{"id": 39, "name": "SO-0125"}]), + [ThingCandidate(2343, "SO-0125")], + [ + DeploymentCandidate( + 437, "Pressure Transducer", removal_date=date(2024, 1, 1) + ) + ], + ) + assert wired == [] + + def test_one_bad_well_does_not_cost_the_others(self, monkeypatch, wired): + # The point of skipping rather than raising. + output = _run( + monkeypatch, + FakeClient( + [ + {"id": 39, "name": "SO-9999"}, + {"id": 40, "name": "SO-0125"}, + ] + ), + [ThingCandidate(2343, "SO-0125")], + [TRANSDUCER], + ) + assert output.value == 1 + assert output.metadata["wells_attempted"].value == 2 + assert output.metadata["wells_skipped"].value == 1 + + def test_a_reading_the_adapter_refuses_is_counted(self, monkeypatch, wired): + # A null level has nothing to store; it should surface, not vanish. + client = FakeClient( + [{"id": 39, "name": "SO-0125"}], + readings=[{"dateAndTime": "2026-04-15T22:45:00", "level": None}], + ) + output = _run( + monkeypatch, client, [ThingCandidate(2343, "SO-0125")], [TRANSDUCER] + ) + assert output.value == 0 + assert output.metadata["adapter_failures"].value == 1 + + def test_no_wells_at_all(self, monkeypatch, wired): + output = _run(monkeypatch, FakeClient([]), [], [TRANSDUCER]) + assert output.value == 0 + assert output.metadata["wells_attempted"].value == 0 + + +class TestParameterLookup: + def test_a_missing_parameter_is_a_clear_error(self): + # Ingestion does not create parameters, so the message has to say what + # to do instead of surfacing an integrity error later. + class Empty: + def scalar(self, *_): + return None + + with pytest.raises(RuntimeError, match="does not create parameters"): + ingest._parameter_id(Empty(), "groundwater level") + + def test_a_found_parameter_is_returned(self): + class Found: + def scalar(self, *_): + return 7 + + assert ingest._parameter_id(Found(), "groundwater level") == 7 + + +class TestRowCount: + def test_malformed_load_info_reports_zero(self): + # Metadata must never fail a load that worked. + assert ingest._row_count(object()) == 0 + + def test_none_reports_zero(self): + assert ingest._row_count(None) == 0 + + +class FakePipeline: + """Stands in for the dlt pipeline. Records what it was asked to run.""" + + def __init__(self): + self.runs = [] + + def run(self, resource, loader_file_format=None): + # Consume the resource so the generator body actually executes. + rows = list(resource) if hasattr(resource, "__iter__") else [] + self.runs.append({"format": loader_file_format, "rows": len(rows)}) + return object() + + +class TestRawAssets: + def test_locations_reports_what_it_landed(self, monkeypatch): + from automated_ingestion.sources.san_acacia import dlt_pipeline + + pipeline = FakePipeline() + client = FakeClient( + [{"id": 39, "name": "SO-0125"}, {"id": 40, "name": "SO-0131"}] + ) + monkeypatch.setattr(ingest, "_client", lambda: client) + monkeypatch.setattr(dlt_pipeline, "build_pipeline", lambda: pipeline) + + output = ingest.raw_san_acacia_locations(build_asset_context()) + + assert output.value == 2 + assert output.metadata["monitoring_points"].value == 2 + assert "SO-0125" in output.metadata["names"].value + + def test_locations_are_written_as_parquet(self, monkeypatch): + # dlt writes gzipped JSONL unless told otherwise, and Mode B replay + # assumes parquet. + from automated_ingestion.sources.san_acacia import dlt_pipeline + + pipeline = FakePipeline() + monkeypatch.setattr(ingest, "_client", lambda: FakeClient([])) + monkeypatch.setattr(dlt_pipeline, "build_pipeline", lambda: pipeline) + + ingest.raw_san_acacia_locations(build_asset_context()) + assert pipeline.runs[0]["format"] == "parquet" + + def test_readings_report_per_point_failures(self, monkeypatch): + from automated_ingestion.sources.san_acacia import dlt_pipeline + + pipeline = FakePipeline() + client = FakeClient([{"id": 39, "name": "SO-0125"}]) + monkeypatch.setattr(ingest, "_client", lambda: client) + monkeypatch.setattr(dlt_pipeline, "build_pipeline", lambda: pipeline) + + output = ingest.raw_san_acacia_readings(build_asset_context()) + + assert output.metadata["points_attempted"].value == 1 + assert output.metadata["points_failed"].value == 0 + + def test_readings_are_written_as_parquet(self, monkeypatch): + from automated_ingestion.sources.san_acacia import dlt_pipeline + + pipeline = FakePipeline() + monkeypatch.setattr(ingest, "_client", lambda: FakeClient([])) + monkeypatch.setattr(dlt_pipeline, "build_pipeline", lambda: pipeline) + + ingest.raw_san_acacia_readings(build_asset_context()) + assert pipeline.runs[0]["format"] == "parquet" + + def test_readings_count_a_point_the_vendor_refuses(self, monkeypatch): + # One diver failing must cost that diver, not the run. The count is how + # anyone finds out it happened. + from automated_ingestion.sources.san_acacia import dlt_pipeline + from automated_ingestion.sources.san_acacia.client import DiverHubError + + class Refusing(FakeClient): + def water_levels(self, *a, **kw): + raise DiverHubError("500 at the minimum window") + + pipeline = FakePipeline() + client = Refusing([{"id": 39, "name": "SO-0125"}]) + monkeypatch.setattr(ingest, "_client", lambda: client) + monkeypatch.setattr(dlt_pipeline, "build_pipeline", lambda: pipeline) + + output = ingest.raw_san_acacia_readings(build_asset_context()) + + assert output.metadata["points_failed"].value == 1 + assert output.metadata["points_attempted"].value == 1 + assert "500" in str(output.metadata["failures"].data) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_reconcile.py b/automated_ingestion/tests/test_reconcile.py new file mode 100644 index 000000000..3317d0bc8 --- /dev/null +++ b/automated_ingestion/tests/test_reconcile.py @@ -0,0 +1,168 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Reconciliation decisions. + +The rule that matters: never pick a winner among candidates. Ingestion does not +create wells and must not choose between two plausible ones. +""" + +from automated_ingestion.sources.san_acacia.reconcile import ( + MatchKind, + ThingCandidate, + VendorPoint, + format_report, + match_point, + reconcile, +) + +POINT = VendorPoint(monitoring_point_id=39, name="SO-0125") + + +def test_exact_name_match(): + match = match_point(POINT, [ThingCandidate(thing_id=7, name="SO-0125")]) + assert match.kind is MatchKind.NAME + assert match.thing_id == 7 + assert not match.needs_a_human + + +def test_name_match_ignores_case_spacing_and_punctuation(): + # One identifier written three ways. Still exact on significant characters. + for written in ("so 0125", "SO0125", " so-0125 "): + match = match_point(POINT, [ThingCandidate(thing_id=7, name=written)]) + assert match.thing_id == 7, written + + +def test_adjacent_identifier_is_not_a_match(): + # Normalization must not become fuzziness: SO-0126 is a different well. + match = match_point(POINT, [ThingCandidate(thing_id=7, name="SO-0126")]) + assert match.kind is MatchKind.UNMATCHED + + +def test_external_ids_are_ignored_by_default(): + match = match_point( + POINT, + [ThingCandidate(thing_id=9, name="Renamed Well", external_ids=("SO-0125",))], + ) + assert match.kind is MatchKind.UNMATCHED + + +def test_external_id_match_when_explicitly_enabled(): + match = match_point( + POINT, + [ThingCandidate(thing_id=9, name="Renamed Well", external_ids=("SO-0125",))], + use_external_ids=True, + ) + assert match.kind is MatchKind.EXTERNAL_ID + assert match.thing_id == 9 + + +def test_external_ids_can_produce_a_confident_wrong_answer(): + """Why external id matching is off by default. Real rows from staging. + + SO-0131 and SO-0132 swap which physical well is A and which is B between + NMBGMR and the unattributed source. Matching BRN-E04A returns SO-0131 with + no hint of trouble, while NMBGMR asserts SO-0132 is BRN-E04A -- the + parenthetical suffix stops the collision registering as ambiguous. + """ + candidates = [ + ThingCandidate(2369, "SO-0131", ("BRN-E04B (shallow)", "BRN-E04A")), + ThingCandidate(2373, "SO-0132", ("BRN-E04A (deep)", "BRN-E04B")), + ] + enabled = match_point( + VendorPoint(999, "BRN-E04A"), candidates, use_external_ids=True + ) + assert enabled.thing_id == 2369 # contradicts NMBGMR, and looks certain + + default = match_point(VendorPoint(999, "BRN-E04A"), candidates) + assert default.kind is MatchKind.UNMATCHED # escalates instead + + +def test_name_wins_over_external_id(): + # Only relevant when external ids are enabled. + # The name is the identifier the Bureau uses now; a link records an + # association someone made earlier, which may be stale. + match = match_point( + POINT, + [ + ThingCandidate(thing_id=7, name="SO-0125"), + ThingCandidate(thing_id=9, name="Other", external_ids=("SO-0125",)), + ], + use_external_ids=True, + ) + assert match.thing_id == 7 + + +def test_two_wells_with_the_same_name_are_ambiguous(): + # Duplicate rows exist in this database. Picking one is exactly the + # judgement that must not be automated. + match = match_point( + POINT, + [ + ThingCandidate(thing_id=7, name="SO-0125"), + ThingCandidate(thing_id=8, name="SO-0125"), + ], + ) + assert match.kind is MatchKind.AMBIGUOUS + assert match.thing_id is None + assert match.candidates == (7, 8) + assert match.needs_a_human + + +def test_no_candidate_is_unmatched_not_created(): + match = match_point(POINT, []) + assert match.kind is MatchKind.UNMATCHED + assert match.thing_id is None + + +class TestReport: + def _report(self): + points = [ + VendorPoint(39, "SO-0125"), + VendorPoint(40, "SO-0131"), + VendorPoint(41, "SO-0140"), + ] + candidates = [ + ThingCandidate(1, "SO-0125"), + ThingCandidate(2, "SO-0131"), + ThingCandidate(3, "SO-0131"), + ] + return reconcile(points, candidates) + + def test_counts_split_by_outcome(self): + report = self._report() + assert len(report.matched) == 1 + assert len(report.ambiguous) == 1 + assert len(report.unmatched) == 1 + + def test_not_ready_while_anything_needs_a_human(self): + # A partial load produces a series that looks complete and is not. + assert self._report().ready is False + + def test_ready_only_when_everything_resolves(self): + report = reconcile([VendorPoint(39, "SO-0125")], [ThingCandidate(1, "SO-0125")]) + assert report.ready is True + + def test_empty_input_is_not_ready(self): + # Nothing to reconcile is not the same as everything reconciled. + assert reconcile([], []).ready is False + + def test_report_names_the_points_needing_attention(self): + text = format_report(self._report()) + assert "SO-0131" in text and "SO-0140" in text + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_resolve.py b/automated_ingestion/tests/test_resolve.py new file mode 100644 index 000000000..f5330ba09 --- /dev/null +++ b/automated_ingestion/tests/test_resolve.py @@ -0,0 +1,93 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Choosing the deployment a water level belongs to. + +A well carries several open deployments because a deployment is equipment, not a +measured property. Picking the wrong one attributes a water level to a cable. +""" + +from datetime import date + +from automated_ingestion.sources.san_acacia.resolve import ( + DeploymentCandidate, + ResolutionKind, + resolve_deployment, +) + +# The real equipment on SO-0140 in staging. +DIVERLINK = DeploymentCandidate(436, "DiverLink") +TRANSDUCER = DeploymentCandidate(437, "Pressure Transducer") +CABLE = DeploymentCandidate(438, "Diver Cable") + + +def test_the_transducer_is_chosen_from_a_full_nest(): + resolution = resolve_deployment([DIVERLINK, TRANSDUCER, CABLE]) + assert resolution.kind is ResolutionKind.RESOLVED + assert resolution.deployment_id == 437 + + +def test_a_barometer_is_not_a_water_level(): + # Barometers are deployed on these wells too, and measure air pressure. + resolution = resolve_deployment([DeploymentCandidate(500, "Barometer"), TRANSDUCER]) + assert resolution.deployment_id == 437 + + +def test_two_open_transducers_are_ambiguous(): + # Two of the 38 wells are in this state. Taking the lower id would be a + # guess about equipment, made silently. + resolution = resolve_deployment( + [TRANSDUCER, DeploymentCandidate(600, "Pressure Transducer")] + ) + assert resolution.kind is ResolutionKind.AMBIGUOUS + assert resolution.deployment_id is None + assert resolution.candidates == (437, 600) + assert resolution.needs_a_human + + +def test_no_transducer_is_missing_not_invented(): + # SO-0246 has no open transducer deployment at all. + resolution = resolve_deployment([DIVERLINK, CABLE]) + assert resolution.kind is ResolutionKind.MISSING + assert resolution.deployment_id is None + + +def test_a_removed_transducer_is_not_a_fallback(): + # Writing today's readings against retired equipment would be worse than + # skipping the well, because it would look like it worked. + resolution = resolve_deployment( + [DeploymentCandidate(700, "Pressure Transducer", removal_date=date(2024, 1, 1))] + ) + assert resolution.kind is ResolutionKind.MISSING + + +def test_a_removed_transducer_does_not_make_a_live_one_ambiguous(): + resolution = resolve_deployment( + [ + DeploymentCandidate( + 700, "Pressure Transducer", removal_date=date(2024, 1, 1) + ), + TRANSDUCER, + ] + ) + assert resolution.deployment_id == 437 + + +def test_no_deployments_at_all(): + assert resolve_deployment([]).kind is ResolutionKind.MISSING + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_san_acacia_adapter.py b/automated_ingestion/tests/test_san_acacia_adapter.py new file mode 100644 index 000000000..a4e074a02 --- /dev/null +++ b/automated_ingestion/tests/test_san_acacia_adapter.py @@ -0,0 +1,71 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Adapter behaviour: what it refuses, and that one bad row costs only that row. +""" + +from automated_ingestion.sources.san_acacia.adapter import SanAcaciaAdapter + + +def _row(**overrides): + row = { + "monitoring_point_id": 40, + "dateAndTime": "2024-10-30T20:00:00", + "level": 471.518, + "unit": "cm", + "reference": 3, + } + row.update(overrides) + return row + + +def test_maps_a_good_row(): + [observation] = list(SanAcaciaAdapter().to_observations([_row()])) + assert observation.external_point_id == "sanacaciareach-40" + assert observation.value == 15.469751 + assert observation.units == "ft" + + +def test_wrong_datum_is_refused(): + # The datum is chosen at request time and cannot be recovered from the row, + # so a reading fetched against another reference has unknown meaning. + adapter = SanAcaciaAdapter() + assert list(adapter.to_observations([_row(reference=1)])) == [] + assert "not 3" in adapter.failures[0]["error"] + + +def test_unexpected_unit_is_refused(): + # Converting a value whose unit is not what it claims is wrong by a factor + # of 30.48 and still looks like a plausible depth. + adapter = SanAcaciaAdapter() + assert list(adapter.to_observations([_row(unit="ft")])) == [] + assert "expected 'cm'" in adapter.failures[0]["error"] + + +def test_one_bad_row_does_not_lose_the_others(): + adapter = SanAcaciaAdapter() + rows = [_row(), _row(dateAndTime="broken"), _row(dateAndTime="2024-10-30T21:00:00")] + assert len(list(adapter.to_observations(rows))) == 2 + assert len(adapter.failures) == 1 + + +def test_failures_identify_the_record(): + adapter = SanAcaciaAdapter() + list(adapter.to_observations([_row(level=None)])) + assert adapter.failures[0]["record"] == "40@2024-10-30T20:00:00" + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_san_acacia_resources.py b/automated_ingestion/tests/test_san_acacia_resources.py new file mode 100644 index 000000000..64fe21fbb --- /dev/null +++ b/automated_ingestion/tests/test_san_acacia_resources.py @@ -0,0 +1,164 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Resource behaviour: failure isolation, approval tagging, and the raw-zone +contract that nothing is converted on the way in. +""" + +from automated_ingestion.sources.san_acacia.client import ( + GROUND_SURFACE_REFERENCE, + DiverHubClient, +) +from automated_ingestion.sources.san_acacia.dlt_pipeline import ( + PROJECT_ID, + vanessen_locations, + vanessen_readings, +) +from automated_ingestion.tests.test_diverhub_client import FakeResponse, FakeTransport + + +class ScriptedTransport(FakeTransport): + """Answers per-path so a single point can be made to fail.""" + + def __init__(self, handler): + super().__init__() + self._handler = handler + + def get(self, url, **kwargs): + self.gets.append((url, kwargs)) + return self._handler(url, kwargs) + + +def _points_payload(): + return [{"id": 39, "name": "SO-0125"}, {"id": 40, "name": "SO-0131"}] + + +def test_locations_flatten_to_the_raw_shape(): + transport = ScriptedTransport(lambda url, kw: FakeResponse(200, _points_payload())) + client = DiverHubClient(transport, username="u", password="p") + rows = list(vanessen_locations(client)) + assert rows == [ + {"monitoring_point_id": 39, "name": "SO-0125", "project_id": PROJECT_ID}, + {"monitoring_point_id": 40, "name": "SO-0131", "project_id": PROJECT_ID}, + ] + + +READINGS = [ + {"dateAndTime": "2026-04-15T22:45:00", "level": 199.356}, + {"dateAndTime": "2026-04-15T23:00:00", "level": 200.0}, +] + + +def _within(rows, params): + """Return only rows inside the requested window, as the API does. + + A stub that ignores startTime/endTime returns its whole payload for every + window, which turns a decade-long fetch into fifty copies of the same rows + and hides whether the caller is windowing correctly at all. + """ + from automated_ingestion.sources.san_acacia.client import _parse_timestamp + + start, end = params["startTime"], params["endTime"] + return [r for r in rows if start <= _parse_timestamp(r["dateAndTime"]) <= end] + + +def _reading_handler(failing_point=None, approved_stamps=()): + def handler(url, kwargs): + if "WaterLevels" in url: + point_id = int(url.rstrip("/").split("/")[-1]) + if point_id == failing_point: + return FakeResponse(500) + params = kwargs.get("params", {}) + if params.get("approved"): + approved = [{"dateAndTime": s, "level": 1.0} for s in approved_stamps] + return FakeResponse(200, _within(approved, params)) + return FakeResponse(200, _within(READINGS, params)) + return FakeResponse(200, []) + + return handler + + +def _run_readings(handler, points=None, failures=None): + transport = ScriptedTransport(handler) + client = DiverHubClient(transport, username="u", password="p") + points = ( + points + if points is not None + else [ + {"monitoring_point_id": 39, "name": "SO-0125"}, + {"monitoring_point_id": 40, "name": "SO-0131"}, + ] + ) + collected = failures if failures is not None else [] + resource = vanessen_readings(client, points, 1_800_000_000, collected) + return list(resource), collected + + +def test_readings_carry_unit_and_reference_untransformed(): + # The raw zone stores what the vendor said, on the vendor's datum in the + # vendor's units. Converting here would make a mapping bug a re-fetch + # instead of a reprocess. + rows, _ = _run_readings(_reading_handler()) + assert rows[0]["level"] == 199.356 + assert rows[0]["unit"] == "cm" + assert rows[0]["reference"] == GROUND_SURFACE_REFERENCE + + +def test_one_failing_point_does_not_lose_the_others(): + rows, failures = _run_readings(_reading_handler(failing_point=39)) + assert [r["monitoring_point_id"] for r in rows] == [40, 40] + assert len(failures) == 1 + assert failures[0]["monitoring_point_id"] == 39 + + +def test_failures_are_recorded_for_the_caller_not_the_resource(): + # Per-run state on a module-level resource would have concurrent runs + # overwriting one another. + own = [] + _run_readings(_reading_handler(failing_point=39), failures=own) + assert len(own) == 1 + assert not hasattr(vanessen_readings, "failures") + + +def test_vendor_approval_tags_rows_without_duplicating_them(): + rows, _ = _run_readings( + _reading_handler(approved_stamps=["2026-04-15T22:45:00"]), + points=[{"monitoring_point_id": 39, "name": "SO-0125"}], + ) + # Two readings in, two readings out -- the approved fetch tags, never adds. + assert len(rows) == 2 + assert rows[0]["vendor_approved"] is True + assert rows[1]["vendor_approved"] is False + + +def test_unavailable_approval_flag_does_not_lose_readings(): + def handler(url, kwargs): + if "WaterLevels" in url: + params = kwargs.get("params", {}) + if params.get("approved"): + return FakeResponse(500) + return FakeResponse(200, _within(READINGS[:1], params)) + return FakeResponse(200, []) + + rows, failures = _run_readings( + handler, points=[{"monitoring_point_id": 39, "name": "SO-0125"}] + ) + assert len(rows) == 1 + assert rows[0]["vendor_approved"] is False + assert failures == [] + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_schedule.py b/automated_ingestion/tests/test_schedule.py new file mode 100644 index 000000000..4b2000e87 --- /dev/null +++ b/automated_ingestion/tests/test_schedule.py @@ -0,0 +1,74 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +The weekly schedule selects what it claims to. + +A group name is a string, so a typo yields a schedule that runs successfully and +ingests nothing -- which looks like everything is fine. +""" + +from dagster import DefaultScheduleStatus + +from automated_ingestion.defs.definitions import defs + +EXPECTED = { + "raw_san_acacia_locations", + "raw_san_acacia_readings", + "san_acacia_observations", +} + + +def _schedule(): + return next(s for s in defs.schedules if s.name == "san_acacia_weekly") + + +def test_the_schedule_is_registered(): + assert _schedule().job.name == "san_acacia_ingest" + + +def test_it_selects_every_san_acacia_asset_and_nothing_else(): + selected = { + key.to_user_string() + for key in _schedule().job.selection.resolve(list(defs.assets)) + } + assert selected == EXPECTED + + +def test_operations_assets_are_excluded(): + # ingestion_heartbeat and database_connectivity are diagnostics. Running + # them weekly would add noise and, for connectivity, a pointless query. + selected = { + key.to_user_string() + for key in _schedule().job.selection.resolve(list(defs.assets)) + } + assert "ingestion_heartbeat" not in selected + assert "database_connectivity" not in selected + + +def test_it_runs_weekly_in_local_time(): + schedule = _schedule() + assert schedule.cron_schedule == "0 5 * * 1" + assert schedule.execution_timezone == "America/Denver" + + +def test_it_is_stopped_until_somebody_starts_it(): + # Turning it on begins writing to Ocotillo, and the first run for the wells + # without history fetches back to the floor. That is a decision, not a + # consequence of a merge. + assert _schedule().default_status is DefaultScheduleStatus.STOPPED + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_source_registry.py b/automated_ingestion/tests/test_source_registry.py new file mode 100644 index 000000000..18b4e6efd --- /dev/null +++ b/automated_ingestion/tests/test_source_registry.py @@ -0,0 +1,59 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Registry behavior: duplicate keys are a bug, not a silent overwrite.""" + +import pytest + +from automated_ingestion.shared import source_registry +from automated_ingestion.shared.source_registry import SourceDefinition + + +@pytest.fixture(autouse=True) +def _isolated_registry(monkeypatch): + monkeypatch.setattr(source_registry, "_SOURCES", {}) + + +def _definition(key="san_acacia"): + return SourceDefinition( + key=key, + display_name="San Acacia Reach", + dataset_name="raw_sanacaciareach", + ) + + +def test_registered_source_is_retrievable(): + source_registry.register(_definition()) + assert source_registry.get_source("san_acacia").display_name == "San Acacia Reach" + + +def test_duplicate_key_is_rejected(): + source_registry.register(_definition()) + with pytest.raises(ValueError, match="already registered"): + source_registry.register(_definition()) + + +def test_unknown_key_raises(): + with pytest.raises(KeyError, match="san_acacia"): + source_registry.get_source("san_acacia") + + +def test_all_sources_is_sorted_by_key(): + source_registry.register(_definition("van_essen")) + source_registry.register(_definition("bernco")) + assert [s.key for s in source_registry.all_sources()] == ["bernco", "van_essen"] + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_windows.py b/automated_ingestion/tests/test_windows.py new file mode 100644 index 000000000..d4d9cf2ab --- /dev/null +++ b/automated_ingestion/tests/test_windows.py @@ -0,0 +1,83 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Window arithmetic, including the refusal to shrink past the floor.""" + +import pytest + +from automated_ingestion.shared.windows import ( + DAY, + MINIMUM_SPAN, + Window, + iter_windows, +) + + +def test_windows_do_not_share_a_boundary(): + # Diver-HUB ranges are inclusive at both ends, so touching windows both + # return the reading logged exactly on the boundary. That duplicate reaches + # the loader in one batch and Postgres rejects the statement: "ON CONFLICT + # DO UPDATE command cannot affect row a second time". + windows = list(iter_windows(0, 10 * DAY, span=3 * DAY)) + assert windows[0].start == 0 + assert windows[-1].end == 10 * DAY + for earlier, later in zip(windows, windows[1:]): + assert later.start == earlier.end + 1 + + +def test_windows_leave_no_second_uncovered(): + # The gap is exactly one second and timestamps are second-resolution, so + # nothing can fall between two windows. + windows = list(iter_windows(0, 10 * DAY, span=3 * DAY)) + for earlier, later in zip(windows, windows[1:]): + assert later.start - earlier.end == 1 + + +def test_final_window_is_truncated_not_overshot(): + # Overshooting would ask the API for a future range, which is at best waste + # and at worst a 400. + windows = list(iter_windows(0, 10 * DAY, span=3 * DAY)) + assert windows[-1].end == 10 * DAY + assert all(w.end <= 10 * DAY for w in windows) + + +def test_range_shorter_than_span_is_a_single_window(): + assert list(iter_windows(0, DAY, span=90 * DAY)) == [Window(0, DAY)] + + +def test_empty_range_yields_nothing(): + assert list(iter_windows(500, 500)) == [] + + +def test_reversed_range_is_rejected(): + with pytest.raises(ValueError, match="precedes"): + list(iter_windows(10, 5)) + + +def test_bisect_splits_in_half(): + left, right = Window(0, 100 * DAY).bisect() + assert left.start == 0 + assert left.end == right.start + assert right.end == 100 * DAY + + +def test_bisect_refuses_below_the_floor(): + # A 500 on one day is not a volume problem, and silently halving forever + # would turn one real failure into an unbounded pile of requests. + with pytest.raises(ValueError, match="floor"): + Window(0, MINIMUM_SPAN).bisect() + + +# ============= EOF ============================================= diff --git a/cli/generate_chemistry_field_descriptions.py b/cli/generate_chemistry_field_descriptions.py new file mode 100644 index 000000000..6ad1811dd --- /dev/null +++ b/cli/generate_chemistry_field_descriptions.py @@ -0,0 +1,269 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Emit the chemistry blocks of ``core/ogc-field-descriptions.yml``. + +``ogc_major_chemistry_results`` and ``ogc_minor_chemistry_wells`` publish one +column per analyte plus a paired units column -- 190 columns between them. +Hand-writing that is error-prone, so this script generates it and the output is +reviewed and committed. Run it again when the analyte lists change: + + uv run python -m cli.generate_chemistry_field_descriptions > /tmp/chem.yml + +Source of truth is the analyte lists in the migration that builds the two +views, which are the column names themselves. (``core/parameter.json`` holds +only two field parameters, so the lexicon cannot supply this.) + +Analytes needing more than a one-line gloss are spelled out in ANALYTE_PROSE; +anything absent falls back to a generated title and a stock description. Prose +here loses to a hand-written entry in the YAML, which wins on merge. +""" + +import importlib.util +import sys +import textwrap +from pathlib import Path + +MIGRATION = ( + Path(__file__).resolve().parents[1] + / "alembic/versions/f4a5b6c7d8e9_apply_public_release_status_filter_to_ogc_views.py" +) + +# Analyte key -> (title, description). Everything else gets a generated title +# and the stock "dissolved concentration" line. +ANALYTE_PROSE = { + "tds": ( + "Total dissolved solids", + "Total mass of dissolved mineral matter in the water -- in plain terms, " + "how salty it is. Drinking-water guidance sits around 500 mg/L.", + ), + "ph": ( + "pH", + "Acidity of the water on the 0-14 scale, where 7 is neutral. Unitless. " + "Most New Mexico groundwater falls between 7 and 8.5.", + ), + "specific_conductance": ( + "Specific conductance", + "How well the water conducts electricity, which rises with dissolved " + "mineral content. Used as a fast field proxy for total dissolved solids.", + ), + "hardness": ( + "Hardness", + "Combined calcium and magnesium content, reported as an equivalent mass " + "of calcium carbonate. What determines whether water is 'hard'.", + ), + "alkalinity": ( + "Alkalinity", + "The water's capacity to neutralise acid, reported as an equivalent mass " + "of calcium carbonate. Mostly supplied by bicarbonate and carbonate.", + ), + "ion_balance": ( + "Ion balance", + "Percentage difference between the total positive and total negative " + "charge in the analysis. Charge must balance in reality, so a figure far " + "from zero means the analysis is incomplete or in error.", + ), + "total_cations": ( + "Total cations", + "Sum of the positively charged dissolved constituents in the analysis.", + ), + "total_anions": ( + "Total anions", + "Sum of the negatively charged dissolved constituents in the analysis.", + ), + "sodium_plus_potassium": ( + "Sodium plus potassium", + "Combined sodium and potassium concentration, reported together where the " + "laboratory did not separate them.", + ), + "nitrate": ( + "Nitrate", + "Dissolved nitrate concentration, usually from fertiliser, septic systems, " + "or livestock. The drinking-water limit is 10 mg/L as nitrogen.", + ), + "nitrate_as_n": ( + "Nitrate as nitrogen", + "Nitrate concentration expressed as the mass of nitrogen alone, which is " + "how the 10 mg/L drinking-water limit is written. Roughly a quarter of the " + "same sample reported as nitrate.", + ), + "nitrite": ( + "Nitrite", + "Dissolved nitrite concentration, an intermediate stage in the breakdown of " + "nitrogen compounds.", + ), + "silica": ( + "Silica", + "Dissolved silica concentration, weathered out of silicate rock. Useful for " + "estimating the temperature water last equilibrated at.", + ), + "arsenic": ( + "Arsenic", + "Dissolved arsenic concentration. Naturally elevated in parts of New Mexico " + "and regulated in drinking water at 0.010 mg/L.", + ), + "uranium": ( + "Uranium", + "Dissolved uranium concentration. Naturally present near uranium-bearing " + "rock and regulated in drinking water at 0.030 mg/L.", + ), + "fluoride": ( + "Fluoride", + "Dissolved fluoride concentration. Beneficial in small amounts; the " + "drinking-water limit is 4 mg/L.", + ), + "h2r": ( + "Deuterium ratio", + "Ratio of heavy to ordinary hydrogen in the water, reported as per-mil " + "difference from ocean water. Fingerprints where the water fell as " + "precipitation.", + ), + "o18r": ( + "Oxygen-18 ratio", + "Ratio of heavy to ordinary oxygen in the water, reported as per-mil " + "difference from ocean water. Read with the deuterium ratio to trace the " + "water's origin and evaporation history.", + ), + "c13r": ( + "Carbon-13 ratio", + "Ratio of carbon-13 to carbon-12 in the water's dissolved carbon, reported " + "as per-mil difference from a standard. Helps identify where the carbon " + "came from, which is needed to correct a carbon-14 age.", + ), + "c14": ( + "Carbon-14", + "Carbon-14 remaining in the water's dissolved carbon, as a percentage of " + "the modern atmospheric level. The basis for dating groundwater up to " + "roughly 40,000 years old.", + ), + "c14_years": ( + "Carbon-14 age", + "Apparent age of the water in years, calculated from its carbon-14 content. " + "Uncorrected for carbon picked up from rock, so treat it as an upper bound.", + ), + "bromide": ( + "Bromide", + "Dissolved bromide concentration. Read against chloride, it distinguishes " + "seawater-derived salinity from dissolved halite.", + ), +} + +# Elements whose column name is not the plain element name. +ELEMENT_NAMES = { + "silicon": "silicon", + "molybdenum": "molybdenum", + "strontium": "strontium", +} + +STOCK_DESCRIPTION = ( + "Dissolved {name} concentration in the most recent sample analysed for it." +) +TOTAL_DESCRIPTION = ( + "Total {name} concentration -- the unfiltered determination, which counts " + "{name} bound to suspended particles as well as the dissolved fraction." +) +UNITS_DESCRIPTION = ( + "Units the {title_lower} value is reported in, as the laboratory recorded them." +) + + +def _load_analyte_lists(): + """Import the migration module by path and read its analyte column lists.""" + spec = importlib.util.spec_from_file_location("_ogc_filter_migration", MIGRATION) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return ( + [key for key, _ in module.STATIC_ANALYTE_COLUMNS_MAJOR], + [key for key, _ in module.STATIC_ANALYTE_COLUMNS_MINOR], + ) + + +def _entry(analyte_key: str): + if analyte_key in ANALYTE_PROSE: + return ANALYTE_PROSE[analyte_key] + + if analyte_key.endswith("_total"): + base = analyte_key[: -len("_total")] + name = ELEMENT_NAMES.get(base, base).replace("_", " ") + title = f"{name.capitalize()} (total)" + return title, TOTAL_DESCRIPTION.format(name=name) + + name = ELEMENT_NAMES.get(analyte_key, analyte_key).replace("_", " ") + return name.capitalize(), STOCK_DESCRIPTION.format(name=name) + + +def _yaml_block(field: str, title: str, description: str) -> str: + body = textwrap.fill( + description, + width=74, + initial_indent=" " * 6, + subsequent_indent=" " * 6, + break_on_hyphens=False, + break_long_words=False, + ) + return f" {field}:\n title: {title}\n description: >-\n{body}\n" + + +def render(table: str, analyte_keys) -> str: + lines = [f"{table}:"] + lines.append( + _yaml_block( + "location_id", + "Location ID", + "Identifier of the location record the well's coordinates came from.", + ) + ) + lines.append( + _yaml_block( + "analyte_count", + "Analyte count", + "Number of distinct analytes with a value in this row. A low count " + "means the well has only been analysed for part of the suite.", + ) + ) + lines.append( + _yaml_block( + "latest_chemistry_date", + "Latest analysis date", + "Date of the most recent result in this row. Analytes are carried " + "forward independently, so an individual value may be older than " + "this date.", + ) + ) + for key in analyte_keys: + title, description = _entry(key) + lines.append(_yaml_block(key, title, description)) + lines.append( + _yaml_block( + f"{key}_units", + f"{title} units", + UNITS_DESCRIPTION.format(title_lower=title.lower()), + ) + ) + return "\n".join(lines) + + +def main() -> int: + major, minor = _load_analyte_lists() + print( + "# Generated by cli/generate_chemistry_field_descriptions.py -- review before committing." + ) + print(render("major_chemistry_results", major)) + print(render("minor_chemistry_wells", minor)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/core/app.py b/core/app.py index d14ccecf3..dd392f08b 100644 --- a/core/app.py +++ b/core/app.py @@ -27,6 +27,7 @@ get_swagger_ui_oauth2_redirect_html, ) from fastapi.openapi.utils import get_openapi +from fastapi.routing import iter_route_contexts from sqlalchemy import text from sqlalchemy.orm import Session @@ -123,27 +124,32 @@ def public_openapi(): routes=app.routes, ) - # Keep only operations where the endpoint function is marked public. + # Collect the operations whose endpoint carries @in_public_schema. + # + # This walks iter_route_contexts() rather than app.routes. Routes added + # via app.include_router() are not flattened into app.routes -- they + # live inside opaque _IncludedRouter branches -- so the previous + # `next(r for r in app.routes if r.path == path)` lookup matched + # nothing but the few endpoints declared directly on `app`, and + # silently dropped every decorated router route from the public schema. + # iter_route_contexts() is the same helper get_openapi() itself walks, + # so prefixes resolve identically to the paths in `schema`. + public_operations = set() + for route_context in iter_route_contexts(app.routes): + if not getattr(route_context.endpoint, "_in_public_schema", False): + continue + route_path = route_context.path_format or route_context.path + for route_method in route_context.methods or (): + public_operations.add((route_path, route_method.lower())) + new_paths = {} for path, path_item in schema["paths"].items(): new_methods = {} for method, operation in path_item.items(): - route = next( - ( - r - for r in app.routes - if getattr(r, "path", None) == path - and method.upper() in getattr(r, "methods", set()) - ), - None, - ) - if not route: + if (path, method.lower()) not in public_operations: continue - - endpoint = getattr(route, "endpoint", None) - if getattr(endpoint, "_is_public", False): - operation["security"] = [] - new_methods[method] = operation + operation["security"] = [] + new_methods[method] = operation if new_methods: new_paths[path] = new_methods @@ -224,7 +230,7 @@ async def warmup(): return {"status": "ok"} @app.get("/health", tags=["meta"]) - @public_route + @in_public_schema def health(response: Response, session: Session = Depends(get_db_session)): # Ping the database so a 200 actually proves PostGIS is reachable, not # just that the process is up. Uptime monitors / status pages assert on @@ -248,9 +254,18 @@ def health(response: Response, session: Session = Depends(get_db_session)): return app -def public_route(func): - """Mark a route as public for OpenAPI filtering.""" - setattr(func, "_is_public", True) +def in_public_schema(func): + """Advertise a route in the anonymous OpenAPI schema (/openapi.json). + + Schema visibility only -- this grants no access and removes no dependency. + It was previously named `public_route`, which read like an authorization + decorator; two `/thing` endpoints carried it *and* a `viewer_dependency`, + so the public schema advertised operations that 401 for anonymous callers. + + Apply it only to routes that genuinely have no auth dependency. + tests/test_authorization.py asserts the two sets match exactly. + """ + setattr(func, "_in_public_schema", True) return func diff --git a/core/dependencies.py b/core/dependencies.py index eabcd009a..95d11f3c8 100644 --- a/core/dependencies.py +++ b/core/dependencies.py @@ -34,30 +34,62 @@ Admin, can do everything Editor and Viewer can do + create new objects +That hierarchy is enforced here, by `any_of=` group lists rather than by +Authentik group membership overlap: an Admin-only account satisfies an +editor- or viewer-gated route because "Admin" appears in those lists. Before +this was explicit, `authenticated(permissions=["Viewer"])` required the +literal Viewer group, so the hierarchy held only as long as whoever +provisioned the Authentik groups granted all three tiers to every admin. + +The three families below are deliberately orthogonal -- general `Admin` does +not confer `AMPAdmin` or `LexiconAdmin`. Only tiers *within* a family nest. """ # General Purpose Authentication/Permissions ----------------------------------- -admin_function = authenticated(permissions=["Admin"]) -editor_function = authenticated(permissions=["Editor"]) -viewer_function = authenticated(permissions=["Viewer"]) +admin_function = authenticated(any_of=["Admin"]) +editor_function = authenticated(any_of=["Admin", "Editor"]) +viewer_function = authenticated(any_of=["Admin", "Editor", "Viewer"]) # AMP-Specific Authentication/Permissions -------------------------------------- -amp_admin_function = authenticated(permissions=["AMPAdmin"]) -amp_editor_function = authenticated(permissions=["AMPEditor"]) -amp_viewer_function = authenticated(permissions=["AMPViewer"]) +amp_admin_function = authenticated(any_of=["AMPAdmin"]) +amp_editor_function = authenticated(any_of=["AMPAdmin", "AMPEditor"]) +amp_viewer_function = authenticated(any_of=["AMPAdmin", "AMPEditor", "AMPViewer"]) + + +# Hydrograph-Corrector Staging Permissions ------------------------------------- +# The hydrograph corrector's publish and range-delete routes write and destroy +# transducer records, and the workbench driving them is still being validated +# against real logger files. `AMP.Staging` is its own group with no tier below +# it and no AMP tier above it -- an AMPAdmin does not satisfy it. Nobody holds +# it until it is granted in Authentik, so the routes ship dark and reachable +# only by whoever is testing them. +# +# This is deliberately not a fourth rung on the AMP ladder. When the workbench +# is trusted, these routes move to `amp_admin_dependency` and the group goes +# away; leaving it as a tier would make that a schema change instead of a +# one-line edit. +amp_staging_function = authenticated(any_of=["AMP.Staging"]) # Lexicon-Specific Authentication/Permissions ---------------------------------- -lexicon_admin_function = authenticated(permissions=["LexiconAdmin"]) -lexicon_editor_function = authenticated(permissions=["LexiconEditor"]) +lexicon_admin_function = authenticated(any_of=["LexiconAdmin"]) +lexicon_editor_function = authenticated(any_of=["LexiconAdmin", "LexiconEditor"]) + + +# OGC-Internal Authentication/Permissions -------------------------------------- +# INTERNAL_OGC_GROUP ("OGCInternal") lives in core/permissions.py, not here -- +# it gates core/internal_ogc_auth.py's ASGI middleware in front of the +# /ogcapi-internal mount, which runs outside FastAPI's Depends() machinery. # Testing-Specific Authentication/Permissions ---------------------------------- -no_permission_function = authenticated(permissions=["NoPermission"]) +# A group nobody is ever granted, so this dependency always 403s. Used to +# assert that group enforcement is actually wired up. +no_permission_function = authenticated(any_of=["NoPermission"]) # Permissions Dependencies ----------------------------------------------------- @@ -72,5 +104,7 @@ amp_editor_dependency: TypeAlias = Annotated[dict, Depends(amp_editor_function)] amp_viewer_dependency: TypeAlias = Annotated[dict, Depends(amp_viewer_function)] +amp_staging_dependency: TypeAlias = Annotated[dict, Depends(amp_staging_function)] + no_permission_dependency: TypeAlias = Annotated[dict, Depends(no_permission_function)] # ============= EOF ============================================= diff --git a/core/disclaimer.py b/core/disclaimer.py new file mode 100644 index 000000000..3227074e6 --- /dev/null +++ b/core/disclaimer.py @@ -0,0 +1,47 @@ +# =============================================================================== +# Copyright 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Canonical text of the Ocotillo data disclaimer. + +The disclaimer is served at GET /disclaimer (api/disclaimer.py) and is the +target of `metadata.identification.terms_of_service` in both pygeoapi configs. +It lives here as plain constants rather than a template or a static file so +that the HTML and JSON renderings cannot drift apart, and so it ships with the +`core` package without any package-data wiring. +""" + +DISCLAIMER_TITLE = "Disclaimer" + +DISCLAIMER_CONTACT_EMAIL = "ocotillo-nmbg@nmt.edu" + +DISCLAIMER_PARAGRAPHS: tuple[str, ...] = ( + "These geospatial data are shared to help the public understand New " + "Mexico's geologic and water resources. All datasets have limitations, " + "particularly when combining data collected at different times, scales, " + "or for different purposes. Users should review the metadata for each " + "dataset and verify conditions on-site before making legal, regulatory, " + "or other high-consequence decisions. All geospatial datasets are " + "inherently scale-dependent.", + "The New Mexico Bureau of Geology and Mineral Resources (NMBGMR) provides " + "these data 'as-is' without warranties. NMBGMR does not guarantee the " + "accuracy, completeness, and timeliness of these data for any particular " + "purpose. Conditions may have changed since the data were collected. " + "Neither NMBGMR nor any partner agency providing data assumes liability " + "for any errors, omissions, or consequences arising from the use or " + "misuse of these data.", + "References to specific products or companies do not imply endorsement. " + "Proper citation of these data is appreciated. Questions or feedback: " + f"{DISCLAIMER_CONTACT_EMAIL}", +) diff --git a/core/edr_provider.py b/core/edr_provider.py index db377af5b..95af6a79d 100644 --- a/core/edr_provider.py +++ b/core/edr_provider.py @@ -46,6 +46,8 @@ ) from pygeoapi.provider.base_edr import BaseEDRProvider +from core.ogc_field_metadata import describe_fields, table_entries + LOGGER = logging.getLogger(__name__) GEOGRAPHIC_CRS = { @@ -95,6 +97,12 @@ def __init__(self, provider_def): self._fields = {} self.get_fields() + # Station metadata carried by some backing views but not others: the + # chemistry views (d9e0f1a2b3c4) span wells and springs and expose + # thing_type so a consumer can tell them apart. Detected rather than + # assumed, so a view without the column keeps working unchanged. + self._has_thing_type = self._has_column("thing_type") + # ------------------------------------------------------------------ db def _connect(self): try: @@ -117,11 +125,37 @@ def _fetch(self, sql, params=None): if conn is not None: conn.close() + def _has_column(self, column): + """Whether the backing relation exposes ``column``. + + Reads pg_attribute rather than information_schema.columns: the + chemistry collections are backed by materialized views, which + information_schema does not list at all. + """ + try: + rows = self._fetch( + "SELECT 1 FROM pg_attribute " + "WHERE attrelid = to_regclass(%s) AND attname = %s " + "AND attnum > 0 AND NOT attisdropped LIMIT 1", + [self.table, column], + ) + except ProviderConnectionError: + # View may not exist yet (e.g. OpenAPI generation before migrate). + return False + return bool(rows) + # -------------------------------------------------------------- fields def get_fields(self): - """Return the parameter-name fields present in the backing view.""" + """Return the parameter-name fields present in the backing view. + + Each call hands back fresh per-field dicts. pygeoapi's + get_collection_schema mutates what a provider returns in place -- + popping ``format``, assigning ``x-ogc-role`` -- so returning the + cached dicts themselves would let one request's edits accumulate on + the next one's response. + """ if self._fields: - return self._fields + return {name: dict(field) for name, field in self._fields.items()} try: rows = self._fetch( f"SELECT DISTINCT parameter_name, unit " # noqa: S608 (trusted table) @@ -136,15 +170,29 @@ def get_fields(self): "title": row["parameter_name"], "x-ogc-unit": row["unit"], } - return self._fields + # Same prose source as the feature collections, keyed by parameter + # name rather than column name. Parameter names are read out of the + # data, so an undocumented analyte keeps its generated title. + self._fields = describe_fields(self.table, self._fields) + return {name: dict(field) for name, field in self._fields.items()} @property def fields(self): return self.get_fields() # ----------------------------------------------------------- instances - def get_instances(self): - """List transducer-deployment instance identifiers.""" + def instances(self): + """List transducer-deployment instance identifiers. + + Named for pygeoapi's EDR contract, not ours: ``get_collection_edr_ + instances`` calls ``p.instances()`` and ``p.instance(id)``, and + ``BaseEDRProvider`` *returns* (rather than raises) a + ``NotImplementedError`` instance from both. A provider that spells + these ``get_instances``/``get_instance`` therefore does not override + anything -- /instances iterates the NotImplementedError object and + 500s, and /instances/{id}/... validates against a truthy object, so + any id at all is accepted. + """ if not self.instance_field: return [] rows = self._fetch( @@ -154,9 +202,9 @@ def get_instances(self): ) return [str(row["iid"]) for row in rows] - def get_instance(self, instance): + def instance(self, instance): """Validate an instance identifier.""" - return instance in set(self.get_instances()) + return str(instance) in set(self.instances()) # ------------------------------------------------------------ queries def locations( @@ -192,8 +240,11 @@ def locations( bbox=bbox, ) where = (" WHERE " + " AND ".join(clauses)) if clauses else "" + columns = "thing_id, station_name, longitude, latitude" + if self._has_thing_type: + columns += ", thing_type" rows = self._fetch( - f"SELECT DISTINCT thing_id, station_name, longitude, latitude " # noqa: S608 + f"SELECT DISTINCT {columns} " # noqa: S608 (trusted table/columns) f"FROM {self.table}{where} ORDER BY thing_id", params, ) @@ -207,12 +258,18 @@ def locations( "type": "Point", "coordinates": [row["longitude"], row["latitude"]], }, - "properties": {"name": row["station_name"]}, + "properties": self._station_properties(row), } for row in rows ], } + def _station_properties(self, row): + properties = {"name": row["station_name"]} + if self._has_thing_type: + properties["thing_type"] = row["thing_type"] + return properties + def area( self, wkt=None, select_properties=None, datetime_=None, instance=None, **kwargs ): @@ -310,6 +367,10 @@ def _read( ) # ------------------------------------------------------- coveragejson + def _parameter_documentation(self, parameter_name): + """Documented title/description for one EDR parameter, or ``{}``.""" + return table_entries(self.table).get(parameter_name, {}) + def _coverage_collection(self, rows): if not rows: raise ProviderNoDataError("No data found") @@ -321,10 +382,17 @@ def _coverage_collection(self, rows): stations.setdefault(row["thing_id"], []).append(row) name = row["parameter_name"] if name not in parameters: + # A CoverageJSON client reads observedProperty.label for the + # display name and description for the explanation; both were + # the raw parameter name before the field metadata existed. + entry = self._parameter_documentation(name) parameters[name] = { "type": "Parameter", - "description": {"en": name}, - "observedProperty": {"id": name, "label": {"en": name}}, + "description": {"en": entry.get("description", name)}, + "observedProperty": { + "id": name, + "label": {"en": entry.get("title", name)}, + }, "unit": {"symbol": row["unit"], "label": {"en": row["unit"]}}, } diff --git a/core/enums.py b/core/enums.py index 663f367ef..790272125 100644 --- a/core/enums.py +++ b/core/enums.py @@ -18,6 +18,7 @@ from services.lexicon_helper import build_enum_from_lexicon_category ActivityType: type[Enum] = build_enum_from_lexicon_category("activity_type") +DataMaturity: type[Enum] = build_enum_from_lexicon_category("data_maturity") AddressType: type[Enum] = build_enum_from_lexicon_category("address_type") AnalysisMethodType: type[Enum] = build_enum_from_lexicon_category( "analysis_method_type" diff --git a/core/factory.py b/core/factory.py index 69bcfba7e..79a347e73 100644 --- a/core/factory.py +++ b/core/factory.py @@ -6,8 +6,6 @@ from core.initializers import ( configure_apitally_middleware, configure_cors_middleware, - configure_lazy_admin, - configure_session_middleware, register_api_routes, ) @@ -42,14 +40,21 @@ def initialize_runtime() -> None: def create_api_app(): initialize_runtime() + + # After initialize_runtime()'s load_dotenv(), so MODE and + # AUTHENTIK_DISABLE_AUTHENTICATION are both resolved. Raises + # AuthConfigurationError -- boot fails loudly rather than serving every + # endpoint anonymously. + from core.permissions import assert_auth_configuration + + assert_auth_configuration() + app = create_base_app() register_api_routes(app) - from core.pygeoapi import mount_pygeoapi + from core.pygeoapi import mount_pygeoapi, mount_pygeoapi_internal mount_pygeoapi(app) - if os.environ.get("SESSION_SECRET_KEY"): - configure_session_middleware(app) + mount_pygeoapi_internal(app) configure_cors_middleware(app) configure_apitally_middleware(app) - configure_lazy_admin(app) return app diff --git a/core/feature_provider.py b/core/feature_provider.py new file mode 100644 index 000000000..e13751a38 --- /dev/null +++ b/core/feature_provider.py @@ -0,0 +1,58 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Feature provider that publishes field-level prose alongside the columns. + +pygeoapi's PostgreSQL provider reflects a table and reports each column's +JSON Schema type and format. It does not read column comments, and there is +no hook for documentation, so /collections/{id}/schema publishes bare column +names. This subclass annotates the reflected fields from +core/ogc-field-descriptions.yml on the way out. + +Read docs/ogc-field-descriptions.md before changing this. +""" + +import logging + +from pygeoapi.provider.sql import PostgreSQLProvider + +from core.ogc_field_metadata import describe_fields + +LOGGER = logging.getLogger(__name__) + + +class DescribedPostgreSQLProvider(PostgreSQLProvider): + """PostgreSQLProvider that annotates reflected columns with prose.""" + + def get_fields(self): + """Reflect the table, then annotate the result. + + The write back into ``self._fields`` is the point of this method, not + an optimisation. ``BaseProvider.fields`` -- which is what + ``get_collection_schema`` and ``get_collection_queryables`` actually + read -- returns ``self._fields`` directly and never calls + ``get_fields()``. A subclass that only returned an annotated copy + would be silently ignored, since ``GenericSQLProvider.__init__`` + populates ``_fields`` with the raw reflection at construction. + """ + fields = super().get_fields() + if fields and not getattr(self, "_fields_described", False): + self._fields = describe_fields(self.table, fields) + # super().get_fields() short-circuits on a populated _fields, so + # without this flag a later call would re-describe the annotated + # dict. Harmless today (describe_fields is idempotent) but it + # would quietly depend on that staying true. + self._fields_described = True + return self._fields diff --git a/core/gis-curated-layers.yml b/core/gis-curated-layers.yml new file mode 100644 index 000000000..372df17bb --- /dev/null +++ b/core/gis-curated-layers.yml @@ -0,0 +1,116 @@ +# Curated desktop-GIS layers. +# +# Each entry becomes one QGIS .qlr and one ArcGIS Pro .lyrx. These are the +# "I just want water levels" artifacts -- a small, opinionated set, not a +# mirror of the collection list. The connection files cover "give me +# everything"; anything a user can reach by browsing the connection does not +# need an entry here. +# +# `collection` must name a collection served by the OGC API - Features mount. +# The two EDR collections (waterlevels, water_chemistry) cannot appear here: +# neither QGIS nor ArcGIS Pro has an OGC API - EDR client, so a layer file +# pointing at one would not open. The feature collections below carry the same +# measurements summarised per site, which is what a GIS user wants on a map. +# +# Field aliases and value maps are NOT listed here. They are derived from +# core/ogc-field-descriptions.yml, the same file that feeds /schema and +# /queryables, so a renamed field cannot drift between the API and the shipped +# layer files. +# +# Colours are chosen to stay distinguishable for the common forms of colour +# blindness: the sequential ramps run light-to-dark so they survive being read +# by lightness alone, and the trend categories pair hue with a size difference. +# +# See docs/ogc-desktop-gis-artifacts.md. + +layers: + - id: water-wells + collection: water_wells + title: Water Wells + abstract: >- + Every groundwater well in the monitoring-point register, at its most + recent recorded location. + geometry: Point + renderer: + type: single + color: "31,119,180,255" + size: 2.2 + outline_color: "255,255,255,200" + + - id: depth-to-water + collection: water_elevation_wells + title: Depth to Water + abstract: >- + Depth to the water table at each well at its most recent measurement, in + feet below ground surface. Larger values mean a deeper water table. + geometry: Point + renderer: + type: graduated + field: depth_to_water_below_ground_surface_ft + size: 2.6 + classes: + - {lower: 0, upper: 25, label: "0 - 25 ft", color: "237,248,251,255"} + - {lower: 25, upper: 50, label: "25 - 50 ft", color: "179,205,227,255"} + - {lower: 50, upper: 100, label: "50 - 100 ft", color: "140,150,198,255"} + - {lower: 100, upper: 250, label: "100 - 250 ft", color: "136,86,167,255"} + - {lower: 250, upper: 100000, label: "over 250 ft", color: "129,15,124,255"} + + - id: water-level-trend + collection: depth_to_water_trend_wells + title: Water-Level Trend + abstract: >- + Direction of the fitted depth-to-water trend at each well. "Falling + water table" means depth below ground surface is increasing. + geometry: Point + renderer: + type: categorized + field: trend_category + size: 2.6 + categories: + - {value: "increasing", label: "Falling water table", color: "202,58,48,255", size: 3.2} + - {value: "decreasing", label: "Rising water table", color: "42,122,182,255", size: 3.2} + - {value: "stable", label: "Stable", color: "140,140,140,255", size: 2.2} + - {value: "not enough data", label: "Not enough data", color: "225,225,225,255", size: 1.8} + + - id: actively-monitored-wells + collection: actively_monitored_wells + title: Actively Monitored Wells + abstract: >- + Wells currently on a monitoring schedule, with their water-level record + summarised. + geometry: Point + renderer: + type: single + color: "44,140,80,255" + size: 2.8 + outline_color: "255,255,255,200" + + - id: springs + collection: springs + title: Springs + abstract: Natural groundwater discharge points in the register. + geometry: Point + renderer: + type: single + color: "23,150,140,255" + size: 2.6 + shape: triangle + outline_color: "255,255,255,200" + + - id: latest-tds + collection: latest_tds_wells + title: Latest Total Dissolved Solids + abstract: >- + Most recent total-dissolved-solids result at each well. 1000 mg/L is the + conventional fresh/brackish boundary. + geometry: Point + renderer: + type: graduated + field: latest_tds_value + size: 2.6 + classes: + - {lower: 0, upper: 500, label: "0 – 500 mg/L", color: "255,255,204,255"} + - {lower: 500, upper: 1000, label: "500 – 1000 mg/L", color: "161,218,180,255"} + - {lower: 1000, upper: 3000, label: "1000 – 3000 mg/L", color: "65,182,196,255"} + - {lower: 3000, upper: 10000, label: "3000 – 10000 mg/L", color: "44,127,184,255"} + - {lower: 10000, upper: 10000000, label: "over 10000 mg/L", color: "37,52,148,255"} diff --git a/core/initializers.py b/core/initializers.py index 356005d80..9f419caa2 100644 --- a/core/initializers.py +++ b/core/initializers.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -import asyncio import os from pathlib import Path @@ -21,7 +20,6 @@ from sqlalchemy import text, select from sqlalchemy.dialects.postgresql import insert from sqlalchemy.exc import DatabaseError -from starlette.responses import PlainTextResponse from db import Base from db.engine import session_ctx @@ -47,7 +45,15 @@ def init_parameter(path: str = None) -> None: default_parameter = json.load(f) with session_ctx() as session: + # A parameter is identified by name and matrix, so skip the ones already + # stored instead of letting every re-run trip the unique constraint. + existing = set( + session.execute(select(Parameter.parameter_name, Parameter.matrix)).all() + ) + for param in default_parameter: + if (param["parameter_name"], param["matrix"]) in existing: + continue try: parameter_obj = Parameter( parameter_name=param["parameter_name"], @@ -217,11 +223,18 @@ def register_api_routes(app): from api.geospatial import router as geospatial_router from api.ngwmn import router as ngwmn_router from api.feedback import router as feedback_router + from api.disclaimer import router as disclaimer_router + from api.geothermal import router as geothermal_router + from api.chemisty import router as chemistry_router + from api.gis_artifacts import router as gis_artifacts_router app.include_router(asset_router) + app.include_router(chemistry_router) app.include_router(author_router) app.include_router(contact_router) + app.include_router(disclaimer_router) app.include_router(geospatial_router) + app.include_router(gis_artifacts_router) app.include_router(group_router) app.include_router(lexicon_router) app.include_router(location_router) @@ -230,6 +243,9 @@ def register_api_routes(app): app.include_router(sample_router) app.include_router(sensor_router) app.include_router(search_router) + # geothermal shares the /thing prefix; register before thing_router so its + # explicit /thing/geothermal-well routes take precedence over /thing/{id} + app.include_router(geothermal_router) app.include_router(thing_router) app.include_router(ngwmn_router) app.include_router(feedback_router) @@ -237,17 +253,6 @@ def register_api_routes(app): app.state.api_routes_registered = True -def configure_session_middleware(app): - from starlette.middleware.sessions import SessionMiddleware - - if not getattr(app.state, "session_middleware_configured", False): - session_secret_key = os.environ.get("SESSION_SECRET_KEY") - if not session_secret_key: - raise ValueError("SESSION_SECRET_KEY environment variable is not set.") - app.add_middleware(SessionMiddleware, secret_key=session_secret_key) - app.state.session_middleware_configured = True - - def configure_cors_middleware(app): from starlette.middleware.cors import CORSMiddleware @@ -284,43 +289,8 @@ def configure_apitally_middleware(app): def configure_middleware(app): - configure_session_middleware(app) configure_cors_middleware(app) configure_apitally_middleware(app) -def configure_admin(app): - if getattr(app.state, "admin_configured", False): - return - - from admin import create_admin - from admin.auth_routes import router as admin_auth_router - - app.include_router(admin_auth_router) - create_admin(app) - app.state.admin_configured = True - - -def configure_lazy_admin(app): - if getattr(app.state, "lazy_admin_configured", False): - return - - app.state.admin_configure_lock = asyncio.Lock() - - @app.middleware("http") - async def ensure_admin_initialized(request, call_next): - if request.url.path.startswith("/admin"): - if not getattr(app.state, "session_middleware_configured", False): - return PlainTextResponse( - "Admin requires SESSION_SECRET_KEY to be configured.", - status_code=503, - ) - async with app.state.admin_configure_lock: - if not getattr(app.state, "admin_configured", False): - configure_admin(app) - return await call_next(request) - - app.state.lazy_admin_configured = True - - # ============= EOF ============================================= diff --git a/core/internal_ogc_auth.py b/core/internal_ogc_auth.py new file mode 100644 index 000000000..3d3631982 --- /dev/null +++ b/core/internal_ogc_auth.py @@ -0,0 +1,255 @@ +# =============================================================================== +# Copyright 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""ASGI auth gate for the authenticated internal OGC mount (/ogcapi-internal). + +pygeoapi is mounted via a raw Starlette Mount (core/pygeoapi.py), so FastAPI's +Depends() machinery never runs for it -- gating has to happen at the ASGI +layer, in front of the mount. This is a plain ASGI middleware class rather +than @app.middleware("http")/BaseHTTPMiddleware (used elsewhere in this +codebase): BaseHTTPMiddleware buffers the full response body and interferes +with client-disconnect propagation, which matters here since +/ogcapi-internal serves paginated GeoJSON up to `max_items: 10000`. On the +success path this calls straight through with zero buffering. + +Kept separate from core/permissions.py to avoid a circular import with +core/pygeoapi.py. + +Three credential transports are accepted, because the desktop GIS clients +this mount exists for cannot all carry an Authentik bearer token: + + * ``Authorization: Bearer `` -- QGIS's OAuth2 authentication method, + scripts, and anything that can talk to Authentik directly. + * ``Authorization: Basic `` -- the only scheme ArcGIS + Pro's "Add OGC API connection" dialog supports with saved credentials + (Authentication > Server Authentication). Esri does not support + token-secured OGC service connections at all. + * ``?token=`` -- ArcGIS Pro's "Custom request parameters", which the + client re-appends to every request it issues. Also a workaround for the + QGIS regression where OGC API - Features requests dropped the + Authorization header (qgis/QGIS#60473). + +The Basic and query-parameter transports carry a *static API key* (see +`api_key_label`) rather than a JWT, since neither ArcGIS nor QGIS can refresh +an Authentik access token before it expires. A bearer JWT is still accepted +and still checked for INTERNAL_OGC_GROUP membership; an API key is a +pre-authorized stand-in for that same group. + +The query-parameter transport puts the secret in the request URL, which App +Engine's request log records. Prefer Basic where the client supports it, and +treat keys handed out for ArcGIS as log-exposed when rotating. +""" + +import base64 +import binascii +import hashlib +import hmac +import json +import os +from urllib.parse import parse_qsl, urlencode + +from starlette.types import ASGIApp, Receive, Scope, Send + +from core import permissions +from core.settings import settings + +# Comma- or whitespace-separated `label:sha256hex` entries. The label is for +# operator bookkeeping (who holds this key) and never appears in a response. +API_KEYS_ENV = "INTERNAL_OGC_API_KEYS" + +# Query parameter carrying a credential. Stripped before the request reaches +# pygeoapi so it cannot trip pygeoapi's unknown-parameter handling or leak +# into a provider's filter parsing. +TOKEN_QUERY_PARAM = "token" + +# Sent on 401 so ArcGIS Pro and QGIS surface a credential prompt instead of a +# bare failure. +WWW_AUTHENTICATE = 'Basic realm="Ocotillo Internal OGC API", charset="UTF-8"' + + +def _configured_api_keys() -> dict[str, str]: + """Parse API_KEYS_ENV into {label: sha256hex}. + + Read fresh on every call for the same reason + permissions.authentication_disabled() is: an import-time snapshot diverges + from a value changed after import, and the two checks disagreeing is how + the earlier auth bugs in this codebase presented. + + Malformed entries are skipped rather than raising. A typo in one entry + must not take the whole mount down for every other key holder. + """ + raw = os.environ.get(API_KEYS_ENV) or "" + keys: dict[str, str] = {} + for entry in raw.replace(",", " ").split(): + label, sep, digest = entry.partition(":") + digest = digest.strip().lower() + if not sep or not label.strip() or len(digest) != 64: + continue + try: + int(digest, 16) + except ValueError: + continue + keys[label.strip()] = digest + return keys + + +def api_key_label(secret: str) -> str | None: + """Return the configured label for `secret`, or None if it matches none. + + Compared as SHA-256 hex with hmac.compare_digest so neither the stored + material nor the comparison timing reveals a valid key. + """ + configured = _configured_api_keys() + if not configured: + return None + presented = hashlib.sha256(secret.encode("utf-8")).hexdigest() + for label, expected in configured.items(): + if hmac.compare_digest(presented, expected): + return label + return None + + +def _extract_credential(scope: Scope) -> str | None: + """Pull a credential out of the Authorization header or ?token=. + + Header wins over query parameter, and within the header both Bearer and + Basic are accepted. For Basic, the password half carries the secret + (username ignored, conventionally "apikey"); a Basic credential with an + empty password falls back to the username so pasting a key into either + field of a connection dialog works. + """ + headers = dict(scope.get("headers") or []) + authorization = headers.get(b"authorization") + if authorization: + scheme, _, param = authorization.decode("latin-1").partition(" ") + scheme = scheme.lower() + param = param.strip() + if scheme == "bearer" and param: + return param + if scheme == "basic" and param: + try: + decoded = base64.b64decode(param, validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError, ValueError): + return None + username, sep, password = decoded.partition(":") + if not sep: + return None + return password or username or None + return None + + for key, value in parse_qsl( + (scope.get("query_string") or b"").decode("latin-1"), keep_blank_values=True + ): + if key == TOKEN_QUERY_PARAM and value: + return value + return None + + +def _strip_token_query_param(scope: Scope) -> Scope: + """Return `scope` with any ?token= removed, copied only if it was present. + + pygeoapi echoes the incoming query string into the `self` and `next` links + it emits; leaving the secret in place would publish it in every response + body as well as in the request log. + """ + query_string = scope.get("query_string") or b"" + if TOKEN_QUERY_PARAM.encode("latin-1") not in query_string: + return scope + pairs = parse_qsl(query_string.decode("latin-1"), keep_blank_values=True) + remaining = [(k, v) for k, v in pairs if k != TOKEN_QUERY_PARAM] + if len(remaining) == len(pairs): + return scope + scope = dict(scope) + scope["query_string"] = urlencode(remaining).encode("latin-1") + return scope + + +async def _send_json( + send: Send, status_code: int, detail: str, *, challenge: bool = False +) -> None: + body = json.dumps({"detail": detail}).encode("utf-8") + headers = [(b"content-type", b"application/json")] + if challenge: + headers.append((b"www-authenticate", WWW_AUTHENTICATE.encode("latin-1"))) + await send( + { + "type": "http.response.start", + "status": status_code, + "headers": headers, + } + ) + await send({"type": "http.response.body", "body": body}) + + +class InternalOGCAuthMiddleware: + """Gates every request under `mount_path` behind an API key or + INTERNAL_OGC_GROUP membership; requests to any other path pass straight + through untouched. + + Registered via app.add_middleware(), which wraps the whole app -- the + path check below is what keeps this scoped to the internal mount only. + """ + + def __init__(self, app: ASGIApp, mount_path: str) -> None: + self.app = app + self.mount_path = mount_path + + def _covers(self, path: str) -> bool: + # Segment-boundary match, not a bare startswith: with mount_path + # "/ogcapi" a plain prefix test would also swallow "/ogcapi-internal". + return path == self.mount_path or path.startswith(f"{self.mount_path}/") + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http" or not self._covers(scope["path"]): + await self.app(scope, receive, send) + return + + if permissions.authentication_disabled(): + if settings.mode != permissions.BYPASS_ALLOWED_MODE: + # HTTPException(424) (what core.permissions.authenticated() + # raises for this same misconfiguration) means nothing from + # raw ASGI code -- send the response directly so a + # misconfigured box degrades to "internal mount always 424s" + # rather than crashing the worker. + await _send_json( + send, 424, permissions.bypass_misconfiguration_detail() + ) + return + await self.app(_strip_token_query_param(scope), receive, send) + return + + secret = _extract_credential(scope) + if not secret: + await _send_json(send, 401, "Unauthorized", challenge=True) + return + + if api_key_label(secret) is None: + # Not a static key, so it has to be an Authentik access token. + try: + payload = permissions.decode_token_payload(secret) + except permissions.TokenInvalid: + await _send_json( + send, 401, "Could not validate credentials", challenge=True + ) + return + + if permissions.INTERNAL_OGC_GROUP not in payload.get("groups", []): + await _send_json(send, 403, "Forbidden") + return + + await self.app(_strip_token_query_param(scope), receive, send) + + +# ============= EOF ============================================= diff --git a/core/lexicon.json b/core/lexicon.json index 890ebc486..40291d696 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -243,12 +243,17 @@ { "name": "lithology", "description": null + }, + { + "name": "data_maturity", + "description": "How far through review a measurement is, on USGS terms. Orthogonal to release_status, which controls visibility rather than trust." } ], "terms": [ { "categories": [ - "review_status" + "review_status", + "data_maturity" ], "term": "approved", "definition": "approved" @@ -1762,7 +1767,8 @@ }, { "categories": [ - "release_status" + "release_status", + "data_maturity" ], "term": "provisional", "definition": "provisional version" @@ -2583,50 +2589,50 @@ "categories": [ "organization" ], - "term": "City of Aztec", - "definition": "City of Aztec" + "term": "A&T Pump & Well Service, LLC", + "definition": "A&T Pump & Well Service, LLC" }, { "categories": [ "organization" ], - "term": "Daybreak Investments", - "definition": "Daybreak Investments" + "term": "A. G. Wassenaar, Inc", + "definition": "A. G. Wassenaar, Inc" }, { "categories": [ "organization" ], - "term": "Vallecitos HOA", - "definition": "Vallecitos HOA" + "term": "Abeyta Engineering, Inc", + "definition": "Abeyta Engineering, Inc" }, { "categories": [ "organization" ], - "term": "SFC, Santa Fe Animal Shelter", - "definition": "Santa Fe County, Santa Fe Animal Shelter" + "term": "Adobe Ranch", + "definition": "Adobe Ranch" }, { "categories": [ "organization" ], - "term": "El Guicu Ditch Association", - "definition": "El Guicu Ditch Association" + "term": "Agua Fria Community Water Association", + "definition": "Agua Fria Community Water Association" }, { "categories": [ "organization" ], - "term": "Santa Fe Municipal Airport", - "definition": "Santa Fe Municipal Airport" + "term": "Agua Sana MWCD", + "definition": "Agua Sana MWCD" }, { "categories": [ "organization" ], - "term": "Uluru Development", - "definition": "Uluru Development" + "term": "Agua Sana WUA", + "definition": "Agua Sana Water Users Assn." }, { "categories": [ @@ -2639,456 +2645,484 @@ "categories": [ "organization" ], - "term": "Santa Fe Downs Resort", - "definition": "Santa Fe Downs Resort" + "term": "Alto Alps HOA", + "definition": "Alto Alps Homeowners Association" }, { "categories": [ "organization" ], - "term": "City of Truth or Consequences, WWTP", - "definition": "City of Truth or Consequences, WWTP" + "term": "AMEC", + "definition": "AMEC" }, { "categories": [ "organization" ], - "term": "Riverbend Hotsprings", - "definition": "Riverbend Hotsprings" + "term": "Anasazi Trails Water Co-op", + "definition": "Anasazi Trails Water Cooperative" }, { "categories": [ "organization" ], - "term": "Armendaris Ranch", - "definition": "Armendaris Ranch" + "term": "Apache Gap Ranch", + "definition": "Apache Gap Ranch" }, { "categories": [ "organization" ], - "term": "El Paso Water", - "definition": "El Paso Water" + "term": "Armendaris Ranch", + "definition": "Armendaris Ranch" }, { "categories": [ "organization" ], - "term": "BLM, Socorro Field Office", - "definition": "BLM, Socorro Field Office" + "term": "Aspendale Mountain Retreat", + "definition": "Aspendale Mountain Retreat" }, { "categories": [ "organization" ], - "term": "USFWS", - "definition": "US Fish & Wildlife Service" + "term": "Augustin Plains Ranch LLC", + "definition": "Augustin Plains Ranch LLC" }, { "categories": [ "organization" ], - "term": "Sile MDWCA", - "definition": "Sile Municipal Domestic Water Assn." + "term": "B & B Cattle Co", + "definition": "B & B Cattle Co" }, { "categories": [ "organization" ], - "term": "Pena Blanca Water & Sanitation District", - "definition": "Pena Blanca Water & Sanitation District" + "term": "Balleau Groundwater, Inc", + "definition": "Balleau Groundwater, Inc" }, { "categories": [ "organization" ], - "term": "Town of Questa", - "definition": "Town of Questa" + "term": "Bayard", + "definition": "Bayard Municipal Water" }, { "categories": [ "organization" ], - "term": "Town of Cerro", - "definition": "Town of Cerro" + "term": "Bernalillo County", + "definition": "Bernalillo County" }, { "categories": [ "organization" ], - "term": "Cerro MDWCA", - "definition": "Cerro MDWCA" + "term": "Berridge Distributing Company", + "definition": "Berridge Distributing Company" }, { "categories": [ "organization" ], - "term": "Farr Cattle Company", - "definition": "Farr Cattle Company (Farr Ranch)" + "term": "Bike Ranch", + "definition": "Bike Ranch" }, { "categories": [ "organization" ], - "term": "Carrizozo Orchard", - "definition": "Carrizozo Orchard" + "term": "Bishop's Lodge", + "definition": "Bishop's Lodge" }, { "categories": [ "organization" ], - "term": "White Oaks Pottery", - "definition": "White Oaks Pottery" + "term": "BLM", + "definition": "Bureau of Land Management" }, { "categories": [ "organization" ], - "term": "USFS, Kiowa Grasslands", - "definition": "USFS, Kiowa Grasslands" + "term": "BLM Taos Office", + "definition": "Bureau of Land Management Taos Office" }, { "categories": [ "organization" ], - "term": "Cloud Country West Subdivision", - "definition": "Cloud Country West Subdivision" + "term": "BLM, Roswell Office", + "definition": "BLM, Roswell Office" }, { "categories": [ "organization" ], - "term": "Chama West WUA", - "definition": "Chama West Water Users Assn." + "term": "BLM, Socorro Field Office", + "definition": "BLM, Socorro Field Office" }, { "categories": [ "organization" ], - "term": "El Rito Regional Water and Waste Water Association", - "definition": "El Rito Regional Water + Waste Water Association" + "term": "Bluewater Acres Domestic WUA", + "definition": "Bluewater Acres Domestic Water Users Assn." }, { "categories": [ "organization" ], - "term": "El Rito MDWCA", - "definition": "El Rito MDWCA" + "term": "Bluewater Lake MDWCA", + "definition": "Bluewater Lake MDWCA" }, { "categories": [ "organization" ], - "term": "West Rim MDWUA", - "definition": "West Rim MDWUA" + "term": "Bonanza Creek Ranch", + "definition": "Bonanza Creek Ranch" }, { "categories": [ "organization" ], - "term": "Village of Willard", - "definition": "Village of Willard" + "term": "Bourbon Grill at El Gancho", + "definition": "Bourbon Grill at El Gancho" }, { "categories": [ "organization" ], - "term": "Quemado Municipal Water & SWA", - "definition": "Quemado Mutual Water and Sewage Works Association" + "term": "Brazos MDWCA", + "definition": "Brazos Mutual Domestic Water Consumers Assn." }, { "categories": [ "organization" ], - "term": "Coyote Creek MDWUA", - "definition": "Coyote Creek MDWUA" + "term": "Bug Scuffle Water Association", + "definition": "Bug Scuffle Water Association" }, { "categories": [ "organization" ], - "term": "Lamy MDWCA", - "definition": "Lamy Mutual Domestic Water Assn." + "term": "Campbell Ranch", + "definition": "Campbell Ranch" }, { "categories": [ "organization" ], - "term": "La Joya CWDA", - "definition": "La Joya CWDA" + "term": "Canada Los Alamos MDWCA", + "definition": "Canada Los Alamos MDWCA" }, { "categories": [ "organization" ], - "term": "NM Firefighters Training Academy", - "definition": "NM Firefighters Training Academy" + "term": "Canjilon Mutual Domestic Water System", + "definition": "Canjilon Mutual Domestic Water System" }, { "categories": [ "organization" ], - "term": "Cebolleta Land Grant", - "definition": "Cebolleta Land Grant" + "term": "Canon MDWCA", + "definition": "Canon Mutual Domestic Water Consumer Assn." }, { "categories": [ "organization" ], - "term": "Madrid Water Co-op", - "definition": "Madrid Water Co-op" + "term": "Capitol Ford Santa Fe", + "definition": "Capitol Ford Santa Fe" }, { "categories": [ "organization" ], - "term": "Sun Valley Water and Sanitation", - "definition": "Sun Valley Water and Sanitation" + "term": "Carrizozo Municipal Water", + "definition": "Carrizozo Municipal Water" }, { "categories": [ "organization" ], - "term": "Bluewater Lake MDWCA", - "definition": "Bluewater Lake MDWCA" + "term": "Carrizozo Orchard", + "definition": "Carrizozo Orchard" }, { "categories": [ "organization" ], - "term": "Bluewater Acres Domestic WUA", - "definition": "Bluewater Acres Domestic Water Users Assn." + "term": "Casas Adobes MDWCA", + "definition": "Casas Adobes Mutual Domestic" }, { "categories": [ "organization" ], - "term": "Lybrook MDWCA", - "definition": "Lybrook Municipal" + "term": "CDM Smith", + "definition": "CDM Smith" }, { "categories": [ "organization" ], - "term": "New Mexico Museum of Natural History", - "definition": "New Mexico Museum of Natural History" + "term": "CDWR", + "definition": "Colorado Division of Water Resources" }, { "categories": [ "organization" ], - "term": "Hillsboro MDWCA", - "definition": "Hillsboro Mutual Domestic Water Consumer Assn." + "term": "Cebolla Mutual Domestic", + "definition": "Cebolla Mutual Domestic" }, { "categories": [ "organization" ], - "term": "Tyrone MDWCA", - "definition": "Tyrone Mutual Domestic Water Assn." + "term": "Cebolleta Land Grant", + "definition": "Cebolleta Land Grant" }, { "categories": [ "organization" ], - "term": "Santa Clara Water System", - "definition": "Santa Clara Water System" + "term": "Cemex, Inc", + "definition": "Cemex, Inc" }, { "categories": [ "organization" ], - "term": "Casas Adobes MDWCA", - "definition": "Casas Adobes Mutual Domestic" + "term": "Cerro Community Center", + "definition": "Cerro Community Center" }, { "categories": [ "organization" ], - "term": "Lake Roberts WUA", - "definition": "Lake Roberts Water Assn." + "term": "Cerro MDWCA", + "definition": "Cerro MDWCA" }, { "categories": [ "organization" ], - "term": "El Creston MDWCA", - "definition": "El Creston MDWCA" + "term": "CH2M Hill", + "definition": "CH2M Hill" }, { "categories": [ "organization" ], - "term": "Reserve Municipality Water Works", - "definition": "Reserve Municipality Water Works" + "term": "Chama West WUA", + "definition": "Chama West Water Users Assn." }, { "categories": [ "organization" ], - "term": "Town of Estancia", - "definition": "Town of Estancia" + "term": "Chamita MDWCA", + "definition": "Chamita Mutual Domestic Water Consumers Assn." }, { "categories": [ "organization" ], - "term": "Pie Town MDWCA", - "definition": "Pie Town MDWCA" + "term": "Chevron", + "definition": "Chevron" }, { "categories": [ "organization" ], - "term": "Roosevelt SWCD", - "definition": "Roosevelt Soil & Water Conservation District" + "term": "Chihuahuan Desert Rangeland Research Center (CDRRC)", + "definition": "Chihuahuan Desert Rangeland Research Center (CDRRC)" }, { "categories": [ "organization" ], - "term": "Otis MDWCA", - "definition": "Otis Mutual Domestic" + "term": "Chiricahua Desert Museum", + "definition": "Chiricahua Desert Museum" }, { "categories": [ "organization" ], - "term": "White Cliffs MDWUA", - "definition": "White Cliffs MDWUA" + "term": "Chupadero MDWCA", + "definition": "Chupadero MDWCA" }, { "categories": [ "organization" ], - "term": "Vista Linda Water Co-op", - "definition": "Vista Linda Water Co-op" + "term": "Cielo Lumbre HOA", + "definition": "Cielo Lumbre HOA" }, { "categories": [ "organization" ], - "term": "Anasazi Trails Water Co-op", - "definition": "Anasazi Trails Water Cooperative" + "term": "Circle Cross Ranch", + "definition": "Circle Cross Ranch" }, { "categories": [ "organization" ], - "term": "Canon MDWCA", - "definition": "Canon Mutual Domestic Water Consumer Assn." + "term": "City of Alamogordo", + "definition": "City of Alamogordo" }, { "categories": [ "organization" ], - "term": "Placitas Trails Water Co-op", - "definition": "Placitas Trails Water Coop" + "term": "City of Aztec", + "definition": "City of Aztec" }, { "categories": [ "organization" ], - "term": "BLM, Roswell Office", - "definition": "BLM, Roswell Office" + "term": "City of Portales, Public Works Dept.", + "definition": "City of Portales, Public Works Dept." }, { "categories": [ "organization" ], - "term": "Forked Lightning Ranch", - "definition": "Forked Lightning Ranch" + "term": "City of Santa Fe", + "definition": "City of Santa Fe" }, { "categories": [ "organization" ], - "term": "Cottonwood RWA", - "definition": "Cottonwood Rural Water Assn." + "term": "City of Santa Fe WWTP", + "definition": "City of Santa Fe WWTP" }, { "categories": [ "organization" ], - "term": "Pinon Ridge WUA", - "definition": "Pinon Ridge Water Users Association" + "term": "City of Santa Fe, Municipal Recreation Complex", + "definition": "City of Santa Fe, Municipal Recreation Complex" }, { "categories": [ "organization" ], - "term": "McSherry Farms", - "definition": "McSherry Farms" + "term": "City of Santa Fe, Sangre de Cristo Water Co.", + "definition": "City of Santa Fe, Sangre de Cristo Water Co." }, { "categories": [ "organization" ], - "term": "Agua Sana WUA", - "definition": "Agua Sana Water Users Assn." + "term": "City of Socorro", + "definition": "City of Socorro" }, { "categories": [ "organization" ], - "term": "Chamita MDWCA", - "definition": "Chamita Mutual Domestic Water Consumers Assn." + "term": "City of Truth or Consequences, WWTP", + "definition": "City of Truth or Consequences, WWTP" }, { "categories": [ "organization" ], - "term": "W Spear-bar Ranch", - "definition": "W Spear-bar Ranch" + "term": "Cloud Country West Subdivision", + "definition": "Cloud Country West Subdivision" }, { "categories": [ "organization" ], - "term": "Village of Capitan", - "definition": "Village of Capitan" + "term": "Commonwealth Conservancy", + "definition": "Commonwealth Conservancy" }, { "categories": [ "organization" ], - "term": "Brazos MDWCA", - "definition": "Brazos Mutual Domestic Water Consumers Assn." + "term": "Corbin Consulting, Inc", + "definition": "Corbin Consulting, Inc" }, { "categories": [ "organization" ], - "term": "Alto Alps HOA", - "definition": "Alto Alps Homeowners Association" + "term": "Costilla MDWCA", + "definition": "Costilla MDWCA" }, { "categories": [ "organization" ], - "term": "Chiricahua Desert Museum", - "definition": "Chiricahua Desert Museum" + "term": "Cottonwood RWA", + "definition": "Cottonwood Rural Water Assn." }, { "categories": [ "organization" ], - "term": "Bike Ranch", - "definition": "Bike Ranch" + "term": "Country Club Garden Mobile Home Park", + "definition": "Country Club Garden Mobile Home Park" }, { "categories": [ "organization" ], - "term": "Hachita MDWCA", - "definition": "Hachita MDWCA" + "term": "Coyote Creek MDWUA", + "definition": "Coyote Creek MDWUA" }, { "categories": [ "organization" ], - "term": "Carrizozo Municipal Water", - "definition": "Carrizozo Municipal Water" + "term": "Crossroads Cattle Co., Ltd", + "definition": "Crossroads Cattle Co., Ltd" + }, + { + "categories": [ + "organization" + ], + "term": "Daniel B. Stephens & Associates, Inc", + "definition": "Daniel B. Stephens & Associates, Inc" + }, + { + "categories": [ + "organization" + ], + "term": "Daybreak Investments", + "definition": "Daybreak Investments" + }, + { + "categories": [ + "organization" + ], + "term": "Desert Village RV & Mobile Home Park", + "definition": "Desert Village RV & Mobile Home Park" + }, + { + "categories": [ + "organization" + ], + "term": "Double H Ranch", + "definition": "Double H Ranch" }, { "categories": [ @@ -3101,1261 +3135,1261 @@ "categories": [ "organization" ], - "term": "Santa Fe Conservation Trust", - "definition": "Santa Fe Conservation Trust" + "term": "E.A. Meadows East", + "definition": "E.A. Meadows East" }, { "categories": [ "organization" ], - "term": "NMSU", - "definition": "New Mexico State University" + "term": "East Rio Arriba SWCD", + "definition": "East Rio Arriba SWCD" }, { "categories": [ "organization" ], - "term": "USGS", - "definition": "US Geological Survey" + "term": "El Camino Realty, Inc", + "definition": "El Camino Realty, Inc" }, { "categories": [ "organization" ], - "term": "TWDB", - "definition": "Texas Water Development Board" + "term": "El Creston MDWCA", + "definition": "El Creston MDWCA" }, { "categories": [ "organization" ], - "term": "NMED", - "definition": "New Mexico Environment Department" + "term": "El Guicu Ditch Association", + "definition": "El Guicu Ditch Association" }, { "categories": [ "organization" ], - "term": "NMOSE", - "definition": "New Mexico Office of the State Engineer" + "term": "El Paso Water", + "definition": "El Paso Water" }, { "categories": [ "organization" ], - "term": "NMBGMR", - "definition": "New Mexico Bureau of Geology and Mineral Resources" + "term": "El Prado HOA", + "definition": "El Prado HOA" }, { "categories": [ "organization" ], - "term": "Bernalillo County", - "definition": "Bernalillo County" + "term": "El Prado Municipal Water", + "definition": "El Prado Municipal Water" }, { "categories": [ "organization" ], - "term": "BLM", - "definition": "Bureau of Land Management" + "term": "El Rancho de las Golondrinas", + "definition": "El Rancho de las Golondrinas" }, { "categories": [ "organization" ], - "term": "BLM Taos Office", - "definition": "Bureau of Land Management Taos Office" + "term": "El Rito Canyon MDWCA", + "definition": "El Rito Canyon MDWCA" }, { "categories": [ "organization" ], - "term": "SFC", - "definition": "Santa Fe County" + "term": "El Rito MDWCA", + "definition": "El Rito MDWCA" }, { "categories": [ "organization" ], - "term": "SFC, Fire Facilities", - "definition": "Santa Fe County, Fire Facilities" + "term": "El Rito Regional Water and Waste Water Association", + "definition": "El Rito Regional Water + Waste Water Association" }, { "categories": [ "organization" ], - "term": "SFC, Utilities Dept.", - "definition": "Santa Fe County, Utilities Dept." + "term": "Eldorado Area Water & Sanitation District", + "definition": "Eldorado Area Water & Sanitation District" }, { "categories": [ "organization" ], - "term": "SFC, Valle Vista Water Utility, Inc.", - "definition": "Santa Fe County, Valle Vista Water Utility, Inc." + "term": "Encantado Enterprises", + "definition": "Encantado Enterprises" }, { "categories": [ "organization" ], - "term": "City of Santa Fe", - "definition": "City of Santa Fe" + "term": "EnecoTech", + "definition": "EnecoTech" }, { "categories": [ "organization" ], - "term": "City of Santa Fe WWTP", - "definition": "City of Santa Fe WWTP" + "term": "Estrella Concepts LLC", + "definition": "Estrella Concepts LLC" }, { "categories": [ "organization" ], - "term": "City of Santa Fe, Municipal Recreation Complex", - "definition": "City of Santa Fe, Municipal Recreation Complex" + "term": "Faith Engineering, Inc", + "definition": "Faith Engineering, Inc" }, { "categories": [ "organization" ], - "term": "City of Santa Fe, Sangre de Cristo Water Co.", - "definition": "City of Santa Fe, Sangre de Cristo Water Co." + "term": "Farr Cattle Company", + "definition": "Farr Cattle Company (Farr Ranch)" }, { "categories": [ "organization" ], - "term": "NMISC", - "definition": "New Mexico Interstate Stream Commission" + "term": "Fire Water Lodge", + "definition": "Fire Water Lodge" }, { "categories": [ "organization" ], - "term": "PVACD", - "definition": "Pecos Valley Artesian Conservancy District" + "term": "Ford County Land & Cattle Company, Inc", + "definition": "Ford County Land & Cattle Company, Inc" }, { "categories": [ "organization" ], - "term": "Bayard", - "definition": "Bayard Municipal Water" + "term": "Forked Lightning Ranch", + "definition": "Forked Lightning Ranch" }, { "categories": [ "organization" ], - "term": "SNL", - "definition": "Sandia National Laboratories" + "term": "Foster Well Service, Inc", + "definition": "Foster Well Service, Inc" }, { "categories": [ "organization" ], - "term": "USFS", - "definition": "United States Forest Service" + "term": "Friendly Construction, Inc", + "definition": "Friendly Construction, Inc" }, { "categories": [ "organization" ], - "term": "NMT", - "definition": "New Mexico Tech" + "term": "Glorieta Geoscience, Inc", + "definition": "Glorieta Geoscience, Inc" }, { "categories": [ "organization" ], - "term": "NPS", - "definition": "National Park Service" + "term": "Golder Associates, Inc", + "definition": "Golder Associates, Inc" }, { "categories": [ "organization" ], - "term": "NMRWA", - "definition": "New Mexico Rural Water Association" + "term": "Hachita MDWCA", + "definition": "Hachita MDWCA" }, { "categories": [ "organization" ], - "term": "NMDOT", - "definition": "New Mexico Department of Transportation" + "term": "Hachita Mutual Domestic", + "definition": "Hachita Mutual Domestic" }, { "categories": [ "organization" ], - "term": "Taos SWCD", - "definition": "Taos Soil and Water Conservation District" + "term": "Hacienda Del Cerezo", + "definition": "Hacienda Del Cerezo" }, { "categories": [ "organization" ], - "term": "Otero SWCD", - "definition": "Otero Soil and Water Conservation District" + "term": "Hathorn's Well Service, Inc", + "definition": "Hathorn's Well Service, Inc" }, { "categories": [ "organization" ], - "term": "Northeastern SWCD", - "definition": "Northeastern Soil and Water Conservation District" + "term": "Hefker Vega Ranch", + "definition": "Hefker Vega Ranch" }, { "categories": [ "organization" ], - "term": "CDWR", - "definition": "Colorado Division of Water Resources" + "term": "High Nogal Ranch", + "definition": "High Nogal Ranch" }, { "categories": [ "organization" ], - "term": "Pendaries Village", - "definition": "Pendaries Village" + "term": "Hillsboro MDWCA", + "definition": "Hillsboro Mutual Domestic Water Consumer Assn." }, { "categories": [ "organization" ], - "term": "A&T Pump & Well Service, LLC", - "definition": "A&T Pump & Well Service, LLC" + "term": "Holloman Air Force Base", + "definition": "Holloman Air Force Base" }, { "categories": [ "organization" ], - "term": "A. G. Wassenaar, Inc", - "definition": "A. G. Wassenaar, Inc" + "term": "Hyde Park Estates MDWCA", + "definition": "Hyde Park Estates MDWCA" }, { "categories": [ "organization" ], - "term": "AMEC", - "definition": "AMEC" + "term": "Hydroscience Associates, Inc", + "definition": "Hydroscience Associates, Inc" }, { "categories": [ "organization" ], - "term": "Balleau Groundwater, Inc", - "definition": "Balleau Groundwater, Inc" + "term": "IC Tech, Inc", + "definition": "IC Tech, Inc" }, { "categories": [ "organization" ], - "term": "CDM Smith", - "definition": "CDM Smith" + "term": "John Shomaker & Associates, Inc", + "definition": "John Shomaker & Associates, Inc" }, { "categories": [ "organization" ], - "term": "CH2M Hill", - "definition": "CH2M Hill" + "term": "Jornada Experimental Range (JER)", + "definition": "Jornada Experimental Range (JER)" }, { "categories": [ "organization" ], - "term": "Corbin Consulting, Inc", - "definition": "Corbin Consulting, Inc" + "term": "K. Schmitt Trust", + "definition": "K. Schmitt Trust" }, { "categories": [ "organization" ], - "term": "Chevron", - "definition": "Chevron" + "term": "Kuckleman Pump Service", + "definition": "Kuckleman Pump Service" }, { "categories": [ "organization" ], - "term": "Daniel B. Stephens & Associates, Inc", - "definition": "Daniel B. Stephens & Associates, Inc" + "term": "La Canada Way HOA", + "definition": "La Canada Way HOA" }, { "categories": [ "organization" ], - "term": "EnecoTech", - "definition": "EnecoTech" + "term": "La Cienega MDWCA", + "definition": "La Cienega MDWCA" }, { "categories": [ "organization" ], - "term": "Faith Engineering, Inc", - "definition": "Faith Engineering, Inc" + "term": "La Joya CWDA", + "definition": "La Joya CWDA" }, { "categories": [ "organization" ], - "term": "Foster Well Service, Inc", - "definition": "Foster Well Service, Inc" + "term": "La Vista HOA", + "definition": "La Vista HOA" }, { "categories": [ "organization" ], - "term": "Glorieta Geoscience, Inc", - "definition": "Glorieta Geoscience, Inc" + "term": "Lake Roberts WUA", + "definition": "Lake Roberts Water Assn." }, { "categories": [ "organization" ], - "term": "Golder Associates, Inc", - "definition": "Golder Associates, Inc" + "term": "Lamy MDWCA", + "definition": "Lamy Mutual Domestic Water Assn." }, { "categories": [ "organization" ], - "term": "Hathorn's Well Service, Inc", - "definition": "Hathorn's Well Service, Inc" + "term": "Land Ventures LLC", + "definition": "Land Ventures LLC" }, { "categories": [ "organization" ], - "term": "Hydroscience Associates, Inc", - "definition": "Hydroscience Associates, Inc" + "term": "Las Lagunitas", + "definition": "Las Lagunitas" }, { "categories": [ "organization" ], - "term": "IC Tech, Inc", - "definition": "IC Tech, Inc" + "term": "Las Lagunitas HOA", + "definition": "Las Lagunitas HOA" }, { "categories": [ "organization" ], - "term": "John Shomaker & Associates, Inc", - "definition": "John Shomaker & Associates, Inc" + "term": "Lightning Dock Zanskar", + "definition": "Lightning Dock Zanskar" }, { "categories": [ "organization" ], - "term": "Kuckleman Pump Service", - "definition": "Kuckleman Pump Service" + "term": "Living World Ministries", + "definition": "Living World Ministries" }, { "categories": [ "organization" ], - "term": "Los Golondrinas", - "definition": "Los Golondrinas" + "term": "Los Atrevidos, Inc", + "definition": "Los Atrevidos, Inc" }, { "categories": [ "organization" ], - "term": "Minton Engineers", - "definition": "Minton Engineers" + "term": "Los Golondrinas", + "definition": "Los Golondrinas" }, { "categories": [ "organization" ], - "term": "MJDarrconsult, Inc", - "definition": "MJDarrconsult, Inc" + "term": "Los Ojos Mutual Domestic", + "definition": "Los Ojos Mutual Domestic" }, { "categories": [ "organization" ], - "term": "Puerta del Canon Ranch", - "definition": "Puerta del Canon Ranch" + "term": "Los Prados HOA", + "definition": "Los Prados HOA" }, { "categories": [ "organization" ], - "term": "Rodgers & Company, Inc", - "definition": "Rodgers & Company, Inc" + "term": "Lower Rio Grande Public Water Works Authority", + "definition": "Lower Rio Grande Public Water Works Authority" }, { "categories": [ "organization" ], - "term": "San Pedro Creek Estates HOA", - "definition": "San Pedro Creek Estates HOA" + "term": "Lybrook MDWCA", + "definition": "Lybrook Municipal" }, { "categories": [ "organization" ], - "term": "Statewide Drilling, Inc", - "definition": "Statewide Drilling, Inc" + "term": "Madrid Water Co-op", + "definition": "Madrid Water Co-op" }, { "categories": [ "organization" ], - "term": "Tec Drilling Limited", - "definition": "Tec Drilling Limited" + "term": "Malaga MDWCA & SWA", + "definition": "Malaga MDWCA & SWA" }, { "categories": [ "organization" ], - "term": "Tetra Tech, Inc", - "definition": "Tetra Tech, Inc" + "term": "Mangas Outfitters", + "definition": "Mangas Outfitters" }, { "categories": [ "organization" ], - "term": "Thompson Drilling, Inc", - "definition": "Thompson Drilling, Inc" + "term": "McSherry Farms", + "definition": "McSherry Farms" }, { "categories": [ "organization" ], - "term": "Witcher & Associates", - "definition": "Witcher & Associates" + "term": "Medina Gravel Pit", + "definition": "Medina Gravel Pit" }, { "categories": [ "organization" ], - "term": "Zeigler Geologic Consulting, LLC", - "definition": "Zeigler Geologic Consulting, LLC" + "term": "Mendenhall Trading Co", + "definition": "Mendenhall Trading Co" }, { "categories": [ "organization" ], - "term": "Sandia Well Service, Inc", - "definition": "Sandia Well Service, Inc" + "term": "Mesa Verde Ranch", + "definition": "Mesa Verde Ranch" }, { "categories": [ "organization" ], - "term": "San Marcos Association", - "definition": "San Marcos Association" + "term": "Minton Engineers", + "definition": "Minton Engineers" }, { "categories": [ "organization" ], - "term": "URS", - "definition": "URS" + "term": "MJDarrconsult, Inc", + "definition": "MJDarrconsult, Inc" }, { "categories": [ "organization" ], - "term": "Vista del Oro", - "definition": "Vista del Oro" + "term": "Naiche Development", + "definition": "Naiche Development" }, { "categories": [ "organization" ], - "term": "Abeyta Engineering, Inc", - "definition": "Abeyta Engineering, Inc" + "term": "New Mexico Museum of Natural History", + "definition": "New Mexico Museum of Natural History" }, { "categories": [ "organization" ], - "term": "Adobe Ranch", - "definition": "Adobe Ranch" + "term": "NM Firefighters Training Academy", + "definition": "NM Firefighters Training Academy" }, { "categories": [ "organization" ], - "term": "Agua Fria Community Water Association", - "definition": "Agua Fria Community Water Association" + "term": "NMBGMR", + "definition": "New Mexico Bureau of Geology and Mineral Resources" }, { "categories": [ "organization" ], - "term": "Apache Gap Ranch", - "definition": "Apache Gap Ranch" + "term": "NMDGF", + "definition": "New Mexico Department of Game and Fish" }, { "categories": [ "organization" ], - "term": "Aspendale Mountain Retreat", - "definition": "Aspendale Mountain Retreat" + "term": "NMDOT", + "definition": "New Mexico Department of Transportation" }, { "categories": [ "organization" ], - "term": "Augustin Plains Ranch LLC", - "definition": "Augustin Plains Ranch LLC" + "term": "NMED", + "definition": "New Mexico Environment Department" }, { "categories": [ "organization" ], - "term": "B & B Cattle Co", - "definition": "B & B Cattle Co" + "term": "NMISC", + "definition": "New Mexico Interstate Stream Commission" }, { "categories": [ "organization" ], - "term": "Berridge Distributing Company", - "definition": "Berridge Distributing Company" + "term": "NMOSE", + "definition": "New Mexico Office of the State Engineer" }, { "categories": [ "organization" ], - "term": "Bishop's Lodge", - "definition": "Bishop's Lodge" + "term": "NMRWA", + "definition": "New Mexico Rural Water Association" }, { "categories": [ "organization" ], - "term": "Bonanza Creek Ranch", - "definition": "Bonanza Creek Ranch" + "term": "NMSA", + "definition": "New Mexico Spaceport Authority" }, { "categories": [ "organization" ], - "term": "Bug Scuffle Water Association", - "definition": "Bug Scuffle Water Association" + "term": "NMSU", + "definition": "New Mexico State University" }, { "categories": [ "organization" ], - "term": "Wehinahpay Mountain Camp", - "definition": "Wehinahpay Mountain Camp" + "term": "NMSU College of Agriculture", + "definition": "New Mexico State University College of Agriculture" }, { "categories": [ "organization" ], - "term": "Campbell Ranch", - "definition": "Campbell Ranch" + "term": "NMT", + "definition": "New Mexico Tech" }, { "categories": [ "organization" ], - "term": "Capitol Ford Santa Fe", - "definition": "Capitol Ford Santa Fe" + "term": "Nogal MDWCA", + "definition": "Nogal MDWCA" }, { "categories": [ "organization" ], - "term": "Cemex, Inc", - "definition": "Cemex, Inc" + "term": "Northeastern SWCD", + "definition": "Northeastern Soil and Water Conservation District" }, { "categories": [ "organization" ], - "term": "Cerro Community Center", - "definition": "Cerro Community Center" + "term": "NPS", + "definition": "National Park Service" }, { "categories": [ "organization" ], - "term": "Santa Fe Jewish Center", - "definition": "Santa Fe Jewish Center" + "term": "NRAO", + "definition": "National Radio Astronomy Observatory" }, { "categories": [ "organization" ], - "term": "Chupadero MDWCA", - "definition": "Chupadero MDWCA" + "term": "O Bar O Ranch", + "definition": "O Bar O Ranch" }, { "categories": [ "organization" ], - "term": "Cielo Lumbre HOA", - "definition": "Cielo Lumbre HOA" + "term": "Old Road Ranch Pardners Ltd", + "definition": "Old Road Ranch Pardners Ltd" }, { "categories": [ "organization" ], - "term": "Circle Cross Ranch", - "definition": "Circle Cross Ranch" + "term": "OMI Wastewater Treatment Plant", + "definition": "OMI Wastewater Treatment Plant" }, { "categories": [ "organization" ], - "term": "City of Alamogordo", - "definition": "City of Alamogordo" + "term": "Otero SWCD", + "definition": "Otero Soil and Water Conservation District" }, { "categories": [ "organization" ], - "term": "City of Portales, Public Works Dept.", - "definition": "City of Portales, Public Works Dept." + "term": "Otis MDWCA", + "definition": "Otis Mutual Domestic" }, { "categories": [ "organization" ], - "term": "City of Socorro", - "definition": "City of Socorro" + "term": "Our Lady of Guadalupe (OLG)", + "definition": "Our Lady of Guadalupe (OLG)" }, { "categories": [ "organization" ], - "term": "Commonwealth Conservancy", - "definition": "Commonwealth Conservancy" + "term": "Peace Tabernacle Church", + "definition": "Peace Tabernacle Church" }, { "categories": [ "organization" ], - "term": "Costilla MDWCA", - "definition": "Costilla MDWCA" + "term": "Pecos Trail Inn", + "definition": "Pecos Trail Inn" }, { "categories": [ "organization" ], - "term": "Country Club Garden Mobile Home Park", - "definition": "Country Club Garden Mobile Home Park" + "term": "Pelican Spa", + "definition": "Pelican Spa" }, { "categories": [ "organization" ], - "term": "Crossroads Cattle Co., Ltd", - "definition": "Crossroads Cattle Co., Ltd" + "term": "Pena Blanca Water & Sanitation District", + "definition": "Pena Blanca Water & Sanitation District" }, { "categories": [ "organization" ], - "term": "Double H Ranch", - "definition": "Double H Ranch" + "term": "Pendaries Village", + "definition": "Pendaries Village" }, { "categories": [ "organization" ], - "term": "E.A. Meadows East", - "definition": "E.A. Meadows East" + "term": "Pie Town MDWCA", + "definition": "Pie Town MDWCA" }, { "categories": [ "organization" ], - "term": "El Camino Realty, Inc", - "definition": "El Camino Realty, Inc" + "term": "Pinon Ridge WUA", + "definition": "Pinon Ridge Water Users Association" }, { "categories": [ "organization" ], - "term": "Eldorado Area Water & Sanitation District", - "definition": "Eldorado Area Water & Sanitation District" + "term": "Pistachio Tree Ranch", + "definition": "Pistachio Tree Ranch" }, { "categories": [ "organization" ], - "term": "Bourbon Grill at El Gancho", - "definition": "Bourbon Grill at El Gancho" + "term": "Placitas Trails Water Co-op", + "definition": "Placitas Trails Water Coop" }, { "categories": [ "organization" ], - "term": "El Prado HOA", - "definition": "El Prado HOA" + "term": "PLSS", + "definition": "Public Land Survey System" }, { "categories": [ "organization" ], - "term": "El Rancho de las Golondrinas", - "definition": "El Rancho de las Golondrinas" + "term": "PNM Service Center", + "definition": "PNM Service Center" }, { "categories": [ "organization" ], - "term": "El Rito Canyon MDWCA", - "definition": "El Rito Canyon MDWCA" + "term": "Puerta del Canon Ranch", + "definition": "Puerta del Canon Ranch" }, { "categories": [ "organization" ], - "term": "Encantado Enterprises", - "definition": "Encantado Enterprises" + "term": "PVACD", + "definition": "Pecos Valley Artesian Conservancy District" }, { "categories": [ "organization" ], - "term": "Estrella Concepts LLC", - "definition": "Estrella Concepts LLC" + "term": "Quemado Municipal Water & SWA", + "definition": "Quemado Mutual Water and Sewage Works Association" }, { "categories": [ "organization" ], - "term": "Sixteen Springs Fire Department", - "definition": "Sixteen Springs Fire Department" + "term": "Rancho Encantado", + "definition": "Rancho Encantado" }, { "categories": [ "organization" ], - "term": "Fire Water Lodge", - "definition": "Fire Water Lodge" + "term": "Rancho San Lucas", + "definition": "Rancho San Lucas" }, { "categories": [ "organization" ], - "term": "Ford County Land & Cattle Company, Inc", - "definition": "Ford County Land & Cattle Company, Inc" + "term": "Rancho San Marcos", + "definition": "Rancho San Marcos" }, { "categories": [ "organization" ], - "term": "Friendly Construction, Inc", - "definition": "Friendly Construction, Inc" + "term": "Rancho Viejo Partnership", + "definition": "Rancho Viejo Partnership" }, { "categories": [ "organization" ], - "term": "Hacienda Del Cerezo", - "definition": "Hacienda Del Cerezo" + "term": "Ranney Ranch", + "definition": "Ranney Ranch" }, { "categories": [ "organization" ], - "term": "Hefker Vega Ranch", - "definition": "Hefker Vega Ranch" + "term": "Reserve Municipality Water Works", + "definition": "Reserve Municipality Water Works" }, { "categories": [ "organization" ], - "term": "High Nogal Ranch", - "definition": "High Nogal Ranch" + "term": "Rio En Medio MDWCA", + "definition": "Rio En Medio MDWCA" }, { "categories": [ "organization" ], - "term": "Holloman Air Force Base", - "definition": "Holloman Air Force Base" + "term": "Riverbend Hotsprings", + "definition": "Riverbend Hotsprings" }, { "categories": [ "organization" ], - "term": "Hyde Park Estates MDWCA", - "definition": "Hyde Park Estates MDWCA" + "term": "Rodgers & Company, Inc", + "definition": "Rodgers & Company, Inc" }, { "categories": [ "organization" ], - "term": "Desert Village RV & Mobile Home Park", - "definition": "Desert Village RV & Mobile Home Park" + "term": "Roosevelt SWCD", + "definition": "Roosevelt Soil & Water Conservation District" }, { "categories": [ "organization" ], - "term": "K. Schmitt Trust", - "definition": "K. Schmitt Trust" + "term": "San Acacia MDWCA", + "definition": "San Acacia MDWCA" }, { "categories": [ "organization" ], - "term": "La Cienega MDWCA", - "definition": "La Cienega MDWCA" + "term": "San Juan Residences", + "definition": "San Juan Residences" }, { "categories": [ "organization" ], - "term": "La Vista HOA", - "definition": "La Vista HOA" + "term": "San Marcos Association", + "definition": "San Marcos Association" }, { "categories": [ "organization" ], - "term": "Land Ventures LLC", - "definition": "Land Ventures LLC" + "term": "San Pedro Creek Estates HOA", + "definition": "San Pedro Creek Estates HOA" }, { "categories": [ "organization" ], - "term": "Las Lagunitas", - "definition": "Las Lagunitas" + "term": "Sandia Well Service, Inc", + "definition": "Sandia Well Service, Inc" }, { "categories": [ "organization" ], - "term": "Las Lagunitas HOA", - "definition": "Las Lagunitas HOA" + "term": "Sangre de Cristo Center", + "definition": "Sangre de Cristo Center" }, { "categories": [ "organization" ], - "term": "Living World Ministries", - "definition": "Living World Ministries" + "term": "Sangre de Cristo Estates", + "definition": "Sangre de Cristo Estates" }, { "categories": [ "organization" ], - "term": "Los Atrevidos, Inc", - "definition": "Los Atrevidos, Inc" + "term": "Santa Ana Pueblo Department of Natural Resources", + "definition": "Santa Ana Pueblo Department of Natural Resources" }, { "categories": [ "organization" ], - "term": "Los Prados HOA", - "definition": "Los Prados HOA" + "term": "Santa Clara Water System", + "definition": "Santa Clara Water System" }, { "categories": [ "organization" ], - "term": "Malaga MDWCA & SWA", - "definition": "Malaga MDWCA & SWA" + "term": "Santa Fe Community College", + "definition": "Santa Fe Community College" }, { "categories": [ "organization" ], - "term": "Mangas Outfitters", - "definition": "Mangas Outfitters" + "term": "Santa Fe Conservation Trust", + "definition": "Santa Fe Conservation Trust" }, { "categories": [ "organization" ], - "term": "Medina Gravel Pit", - "definition": "Medina Gravel Pit" + "term": "Santa Fe Downs Resort", + "definition": "Santa Fe Downs Resort" }, { "categories": [ "organization" ], - "term": "Mendenhall Trading Co", - "definition": "Mendenhall Trading Co" + "term": "Santa Fe Horse Park", + "definition": "Santa Fe Horse Park" }, { "categories": [ "organization" ], - "term": "Mesa Verde Ranch", - "definition": "Mesa Verde Ranch" + "term": "Santa Fe Jewish Center", + "definition": "Santa Fe Jewish Center" }, { "categories": [ "organization" ], - "term": "NMDGF", - "definition": "New Mexico Department of Game and Fish" + "term": "Santa Fe Municipal Airport", + "definition": "Santa Fe Municipal Airport" }, { "categories": [ "organization" ], - "term": "NMSU College of Agriculture", - "definition": "New Mexico State University College of Agriculture" + "term": "Santa Fe Opera", + "definition": "Santa Fe Opera" }, { "categories": [ "organization" ], - "term": "Naiche Development", - "definition": "Naiche Development" + "term": "Santa Fe Waldorf School", + "definition": "Santa Fe Waldorf School" }, { "categories": [ "organization" ], - "term": "NRAO", - "definition": "National Radio Astronomy Observatory" + "term": "SFC", + "definition": "Santa Fe County" }, { "categories": [ "organization" ], - "term": "NMSA", - "definition": "New Mexico Spaceport Authority" + "term": "SFC, Fire Facilities", + "definition": "Santa Fe County, Fire Facilities" }, { "categories": [ "organization" ], - "term": "Nogal MDWCA", - "definition": "Nogal MDWCA" + "term": "SFC, Santa Fe Animal Shelter", + "definition": "Santa Fe County, Santa Fe Animal Shelter" }, { "categories": [ "organization" ], - "term": "O Bar O Ranch", - "definition": "O Bar O Ranch" + "term": "SFC, Utilities Dept.", + "definition": "Santa Fe County, Utilities Dept." }, { "categories": [ "organization" ], - "term": "OMI Wastewater Treatment Plant", - "definition": "OMI Wastewater Treatment Plant" + "term": "SFC, Valle Vista Water Utility, Inc.", + "definition": "Santa Fe County, Valle Vista Water Utility, Inc." }, { "categories": [ "organization" ], - "term": "Old Road Ranch Pardners Ltd", - "definition": "Old Road Ranch Pardners Ltd" + "term": "Shidoni Foundry and Gallery", + "definition": "Shidoni Foundry and Gallery" }, { "categories": [ "organization" ], - "term": "PNM Service Center", - "definition": "PNM Service Center" + "term": "Sierra Grande Lodge", + "definition": "Sierra Grande Lodge" }, { "categories": [ "organization" ], - "term": "Peace Tabernacle Church", - "definition": "Peace Tabernacle Church" + "term": "Sierra Vista Retirement Community", + "definition": "Sierra Vista Retirement Community" }, { "categories": [ "organization" ], - "term": "Pecos Trail Inn", - "definition": "Pecos Trail Inn" + "term": "Sile MDWCA", + "definition": "Sile Municipal Domestic Water Assn." }, { "categories": [ "organization" ], - "term": "Pelican Spa", - "definition": "Pelican Spa" + "term": "Sixteen Springs Fire Department", + "definition": "Sixteen Springs Fire Department" }, { "categories": [ "organization" ], - "term": "Pistachio Tree Ranch", - "definition": "Pistachio Tree Ranch" + "term": "Slash Triangle Ranch", + "definition": "Slash Triangle Ranch" }, { "categories": [ "organization" ], - "term": "Rancho Encantado", - "definition": "Rancho Encantado" + "term": "Smith Ranch LLC", + "definition": "Smith Ranch LLC" }, { "categories": [ "organization" ], - "term": "Rancho San Lucas", - "definition": "Rancho San Lucas" + "term": "SNL", + "definition": "Sandia National Laboratories" }, { "categories": [ "organization" ], - "term": "Rancho San Marcos", - "definition": "Rancho San Marcos" + "term": "Spanish Stirrup Rockshop", + "definition": "Spanish Stirrup Rockshop" }, { "categories": [ "organization" ], - "term": "Rancho Viejo Partnership", - "definition": "Rancho Viejo Partnership" + "term": "Sparrowhawk Farm", + "definition": "Sparrowhawk Farm" }, { "categories": [ "organization" ], - "term": "Ranney Ranch", - "definition": "Ranney Ranch" + "term": "Stagecoach Motel", + "definition": "Stagecoach Motel" }, { "categories": [ "organization" ], - "term": "Rio En Medio MDWCA", - "definition": "Rio En Medio MDWCA" + "term": "State of New Mexico", + "definition": "State of New Mexico" }, { "categories": [ "organization" ], - "term": "San Acacia MDWCA", - "definition": "San Acacia MDWCA" + "term": "Statewide Drilling, Inc", + "definition": "Statewide Drilling, Inc" }, { "categories": [ "organization" ], - "term": "San Juan Residences", - "definition": "San Juan Residences" + "term": "Stephenson Ranch", + "definition": "Stephenson Ranch" }, { "categories": [ "organization" ], - "term": "Sangre de Cristo Estates", - "definition": "Sangre de Cristo Estates" + "term": "Sun Broadcasting Network", + "definition": "Sun Broadcasting Network" }, { "categories": [ "organization" ], - "term": "Santa Fe Community College", - "definition": "Santa Fe Community College" + "term": "Sun Valley Water and Sanitation", + "definition": "Sun Valley Water and Sanitation" }, { "categories": [ "organization" ], - "term": "Sangre de Cristo Center", - "definition": "Sangre de Cristo Center" + "term": "Tano Rd LLC", + "definition": "Tano Rd LLC" }, { "categories": [ "organization" ], - "term": "Santa Fe Horse Park", - "definition": "Santa Fe Horse Park" + "term": "Taos SWCD", + "definition": "Taos Soil and Water Conservation District" }, { "categories": [ "organization" ], - "term": "Santa Fe Opera", - "definition": "Santa Fe Opera" + "term": "Tec Drilling Limited", + "definition": "Tec Drilling Limited" }, { "categories": [ "organization" ], - "term": "Santa Fe Waldorf School", - "definition": "Santa Fe Waldorf School" + "term": "Tee Pee Ranch/Tee Pee Subdivision", + "definition": "Tee Pee Ranch/Tee Pee Subdivision" }, { "categories": [ "organization" ], - "term": "Shidoni Foundry and Gallery", - "definition": "Shidoni Foundry and Gallery" + "term": "Tent Rock, Inc", + "definition": "Tent Rock, Inc" }, { "categories": [ "organization" ], - "term": "Sierra Grande Lodge", - "definition": "Sierra Grande Lodge" + "term": "Tesuque MDWCA", + "definition": "Tesuque MDWCA" }, { "categories": [ "organization" ], - "term": "Sierra Vista Retirement Community", - "definition": "Sierra Vista Retirement Community" + "term": "Tetra Tech, Inc", + "definition": "Tetra Tech, Inc" }, { "categories": [ "organization" ], - "term": "Slash Triangle Ranch", - "definition": "Slash Triangle Ranch" + "term": "The Great Cloud Zen Center", + "definition": "The Great Cloud Zen Center" }, { "categories": [ "organization" ], - "term": "Spanish Stirrup Rockshop", - "definition": "Spanish Stirrup Rockshop" + "term": "The Nature Conservancy (TNC)", + "definition": "The Nature Conservancy (TNC)" }, { "categories": [ "organization" ], - "term": "Stagecoach Motel", - "definition": "Stagecoach Motel" + "term": "Thompson Drilling, Inc", + "definition": "Thompson Drilling, Inc" }, { "categories": [ "organization" ], - "term": "State of New Mexico", - "definition": "State of New Mexico" + "term": "Three Rivers Ranch", + "definition": "Three Rivers Ranch" }, { "categories": [ "organization" ], - "term": "Stephenson Ranch", - "definition": "Stephenson Ranch" + "term": "Timberon Water and Sanitation District", + "definition": "Timberon Water and Sanitation District" }, { "categories": [ "organization" ], - "term": "Sun Broadcasting Network", - "definition": "Sun Broadcasting Network" + "term": "Town of Cerro", + "definition": "Town of Cerro" }, { "categories": [ "organization" ], - "term": "Tano Rd LLC", - "definition": "Tano Rd LLC" + "term": "Town of Estancia", + "definition": "Town of Estancia" }, { "categories": [ "organization" ], - "term": "UNM-Taos", - "definition": "UNM-Taos" + "term": "Town of Magdalena", + "definition": "Town of Magdalena" }, { "categories": [ "organization" ], - "term": "Tee Pee Ranch/Tee Pee Subdivision", - "definition": "Tee Pee Ranch/Tee Pee Subdivision" + "term": "Town of Questa", + "definition": "Town of Questa" }, { "categories": [ "organization" ], - "term": "Tent Rock, Inc", - "definition": "Tent Rock, Inc" + "term": "Town of Taos", + "definition": "Town of Taos" }, { "categories": [ "organization" ], - "term": "Tesuque MDWCA", - "definition": "Tesuque MDWCA" + "term": "Town of Taos, National Guard Armory", + "definition": "Town of Taos, National Guard Armory" }, { "categories": [ "organization" ], - "term": "The Great Cloud Zen Center", - "definition": "The Great Cloud Zen Center" + "term": "Trinity Ranch", + "definition": "Trinity Ranch" }, { "categories": [ "organization" ], - "term": "Three Rivers Ranch", - "definition": "Three Rivers Ranch" + "term": "Tularosa Basin National Desalination Research Facility", + "definition": "Tularosa Basin National Desalination Research Facility" }, { "categories": [ "organization" ], - "term": "Timberon Water and Sanitation District", - "definition": "Timberon Water and Sanitation District" + "term": "Turquoise Trail Charter School", + "definition": "Turquoise Trail Charter School" }, { "categories": [ "organization" ], - "term": "Town of Magdalena", - "definition": "Town of Magdalena" + "term": "TWDB", + "definition": "Texas Water Development Board" }, { "categories": [ "organization" ], - "term": "Town of Taos", - "definition": "Town of Taos" + "term": "Tyrone MDWCA", + "definition": "Tyrone Mutual Domestic Water Assn." }, { "categories": [ "organization" ], - "term": "Town of Taos, National Guard Armory", - "definition": "Town of Taos, National Guard Armory" + "term": "Uluru Development", + "definition": "Uluru Development" }, { "categories": [ "organization" ], - "term": "Trinity Ranch", - "definition": "Trinity Ranch" + "term": "UNM-Taos", + "definition": "UNM-Taos" }, { "categories": [ "organization" ], - "term": "Tularosa Basin National Desalination Research Facility", - "definition": "Tularosa Basin National Desalination Research Facility" + "term": "URS", + "definition": "URS" }, { "categories": [ "organization" ], - "term": "Turquoise Trail Charter School", - "definition": "Turquoise Trail Charter School" + "term": "US Bureau of Indian Affairs, Santa Fe Indian School", + "definition": "US Bureau of Indian Affairs, Santa Fe Indian School" }, { "categories": [ "organization" ], - "term": "US Bureau of Indian Affairs, Santa Fe Indian School", - "definition": "US Bureau of Indian Affairs, Santa Fe Indian School" + "term": "USFS", + "definition": "United States Forest Service" }, { "categories": [ @@ -4375,253 +4409,253 @@ "categories": [ "organization" ], - "term": "USFS, Santa Fe NF, Espanola Ranger District", - "definition": "USFS, Santa Fe NF, Espanola Ranger District" + "term": "USFS, Cibola NF, Supervisor's Office", + "definition": "USFS, Cibola NF, Supervisor's Office" }, { "categories": [ "organization" ], - "term": "Ute Mountain Farms", - "definition": "Ute Mountain Farms" + "term": "USFS, Kiowa Grasslands", + "definition": "USFS, Kiowa Grasslands" }, { "categories": [ "organization" ], - "term": "VA Hospital", - "definition": "VA Hospital" + "term": "USFS, Santa Fe NF, Espanola Ranger District", + "definition": "USFS, Santa Fe NF, Espanola Ranger District" }, { "categories": [ "organization" ], - "term": "Velte", - "definition": "Velte" + "term": "USFWS", + "definition": "US Fish & Wildlife Service" }, { "categories": [ "organization" ], - "term": "Vereda Serena Property", - "definition": "Vereda Serena Property" + "term": "USGS", + "definition": "US Geological Survey" }, { "categories": [ "organization" ], - "term": "Village of Corona", - "definition": "Village of Corona" + "term": "Ute Mountain Farms", + "definition": "Ute Mountain Farms" }, { "categories": [ "organization" ], - "term": "Village of Floyd", - "definition": "Village of Floyd" + "term": "VA Hospital", + "definition": "VA Hospital" }, { "categories": [ "organization" ], - "term": "Village of Melrose", - "definition": "Village of Melrose" + "term": "Vallecitos HOA", + "definition": "Vallecitos HOA" }, { "categories": [ "organization" ], - "term": "Village of Vaughn", - "definition": "Village of Vaughn" + "term": "Velte", + "definition": "Velte" }, { "categories": [ "organization" ], - "term": "Vista Land Company", - "definition": "Vista Land Company" + "term": "Vereda Serena Property", + "definition": "Vereda Serena Property" }, { "categories": [ "organization" ], - "term": "Vista Redonda MDWCA", - "definition": "Vista Redonda MDWCA" + "term": "Village of Capitan", + "definition": "Village of Capitan" }, { "categories": [ "organization" ], - "term": "Vista de Oro de Placitas Water Users Coop", - "definition": "Vista de Oro de Placitas Water Users Coop" + "term": "Village of Corona", + "definition": "Village of Corona" }, { "categories": [ "organization" ], - "term": "Walker Ranch", - "definition": "Walker Ranch" + "term": "Village of Floyd", + "definition": "Village of Floyd" }, { "categories": [ "organization" ], - "term": "Wild & Woolley Trailer Ranch", - "definition": "Wild & Woolley Trailer Ranch" + "term": "Village of Hope", + "definition": "Village of Hope" }, { "categories": [ "organization" ], - "term": "Winter Brothers", - "definition": "Winter Brothers" + "term": "Village of Melrose", + "definition": "Village of Melrose" }, { "categories": [ "organization" ], - "term": "Yates Petroleum Corporation", - "definition": "Yates Petroleum Corporation" + "term": "Village of Vaughn", + "definition": "Village of Vaughn" }, { "categories": [ "organization" ], - "term": "Zamora Accounting Services", - "definition": "Zamora Accounting Services" + "term": "Village of Willard", + "definition": "Village of Willard" }, { "categories": [ "organization" ], - "term": "Agua Sana MWCD", - "definition": "Agua Sana MWCD" + "term": "Vista de Oro de Placitas Water Users Coop", + "definition": "Vista de Oro de Placitas Water Users Coop" }, { "categories": [ "organization" ], - "term": "Canada Los Alamos MDWCA", - "definition": "Canada Los Alamos MDWCA" + "term": "Vista del Oro", + "definition": "Vista del Oro" }, { "categories": [ "organization" ], - "term": "Canjilon Mutual Domestic Water System", - "definition": "Canjilon Mutual Domestic Water System" + "term": "Vista Land Company", + "definition": "Vista Land Company" }, { "categories": [ "organization" ], - "term": "Cebolla Mutual Domestic", - "definition": "Cebolla Mutual Domestic" + "term": "Vista Linda Water Co-op", + "definition": "Vista Linda Water Co-op" }, { "categories": [ "organization" ], - "term": "Chihuahuan Desert Rangeland Research Center (CDRRC)", - "definition": "Chihuahuan Desert Rangeland Research Center (CDRRC)" + "term": "Vista Redonda MDWCA", + "definition": "Vista Redonda MDWCA" }, { "categories": [ "organization" ], - "term": "East Rio Arriba SWCD", - "definition": "East Rio Arriba SWCD" + "term": "W Spear-bar Ranch", + "definition": "W Spear-bar Ranch" }, { "categories": [ "organization" ], - "term": "El Prado Municipal Water", - "definition": "El Prado Municipal Water" + "term": "Walker Ranch", + "definition": "Walker Ranch" }, { "categories": [ "organization" ], - "term": "Hachita Mutual Domestic", - "definition": "Hachita Mutual Domestic" + "term": "Wehinahpay Mountain Camp", + "definition": "Wehinahpay Mountain Camp" }, { "categories": [ "organization" ], - "term": "Jornada Experimental Range (JER)", - "definition": "Jornada Experimental Range (JER)" + "term": "West Rim MDWUA", + "definition": "West Rim MDWUA" }, { "categories": [ "organization" ], - "term": "La Canada Way HOA", - "definition": "La Canada Way HOA" + "term": "White Cliffs MDWUA", + "definition": "White Cliffs MDWUA" }, { "categories": [ "organization" ], - "term": "Los Ojos Mutual Domestic", - "definition": "Los Ojos Mutual Domestic" + "term": "White Oaks Pottery", + "definition": "White Oaks Pottery" }, { "categories": [ "organization" ], - "term": "The Nature Conservancy (TNC)", - "definition": "The Nature Conservancy (TNC)" + "term": "Wild & Woolley Trailer Ranch", + "definition": "Wild & Woolley Trailer Ranch" }, { "categories": [ "organization" ], - "term": "Smith Ranch LLC", - "definition": "Smith Ranch LLC" + "term": "Winter Brothers", + "definition": "Winter Brothers" }, { "categories": [ "organization" ], - "term": "Santa Ana Pueblo Department of Natural Resources", - "definition": "Santa Ana Pueblo Department of Natural Resources" + "term": "Witcher & Associates", + "definition": "Witcher & Associates" }, { "categories": [ "organization" ], - "term": "Village of Hope", - "definition": "Village of Hope" + "term": "WSP", + "definition": "WSP" }, { "categories": [ "organization" ], - "term": "WSP", - "definition": "WSP" + "term": "Yates Petroleum Corporation", + "definition": "Yates Petroleum Corporation" }, { "categories": [ "organization" ], - "term": "Zia Pueblo", - "definition": "Zia Pueblo" + "term": "Zamora Accounting Services", + "definition": "Zamora Accounting Services" }, { "categories": [ "organization" ], - "term": "Our Lady of Guadalupe (OLG)", - "definition": "Our Lady of Guadalupe (OLG)" + "term": "Zeigler Geologic Consulting, LLC", + "definition": "Zeigler Geologic Consulting, LLC" }, { "categories": [ "organization" ], - "term": "PLSS", - "definition": "Public Land Survey System" + "term": "Zia Pueblo", + "definition": "Zia Pueblo" }, { "categories": [ @@ -8467,6 +8501,13 @@ ], "term": "Data not field checked, but considered reliable", "definition": "Data were not field checked but are considered reliable" + }, + { + "categories": [ + "data_maturity" + ], + "term": "in review", + "definition": "Under review and not yet approved. Intermediate state from the USGS Aquarius approval levels used for continuous records." } ] } \ No newline at end of file diff --git a/core/ogc-field-descriptions.yml b/core/ogc-field-descriptions.yml new file mode 100644 index 000000000..c462b6dd9 --- /dev/null +++ b/core/ogc-field-descriptions.yml @@ -0,0 +1,2113 @@ +# Per-field documentation for the OGC collections. +# +# Keyed by backing relation with the ogc_/ogc_internal_ prefix stripped, so the +# public and internal mounts share one entry per view. `_defaults` applies to +# every table; a per-table entry wins over it. +# +# Allowed keys per field: title, description, x-ogc-unit, x-ogc-unitLang, +# x-ogc-propertySeq. Types and formats come from provider reflection, never +# from here. +# +# Say what the value means and what its datum or convention is -- not how the +# view is assembled. That belongs in the collection description. +# +# See docs/ogc-field-descriptions.md. + +_defaults: + id: + title: Feature ID + description: >- + Stable identifier for this feature within the collection. Unique inside + the collection, not across collections. + name: + title: Name + description: >- + Name or identifier the monitoring point is known by, as recorded by the + Bureau. + thing_type: + title: Feature type + description: >- + Controlled-vocabulary type of the monitoring point, such as water well, + spring, or meteorological station. + enum-lexicon: thing_type + release_status: + title: Release status + description: >- + Publication state of the record. Only records marked public appear on + the public /ogcapi mount; the authenticated internal mount also carries + private and draft records. + enum-lexicon: release_status + first_visit_date: + title: First visit date + description: Date of the earliest Bureau visit on record for this feature. + last_observation_date: + title: Last observation date + description: >- + Date of the most recent measurement recorded against this feature, as a + UTC calendar date. Null where no measurement is on record for it. Counts + readings and laboratory results held in the observation record; continuous + instrument readings from a deployed logger are not included, so an + instrumented well can carry newer data than this date shows. On the public + mount only measurements released to the public are counted. + nma_pk_welldata: + title: Legacy NM_Aquifer well key + description: >- + Primary key of this feature's record in the legacy NM_Aquifer WellData + table, kept so migrated rows can be traced back to their source. + elevation: + title: Ground-surface elevation + description: >- + Surveyed elevation of the ground surface at the feature, in metres above + the NAVD 88 vertical datum. + x-ogc-unit: https://qudt.org/vocab/unit/M + x-ogc-unitLang: QUDT + well_depth: + title: Well depth + description: >- + Total depth of the finished well, from ground surface to the bottom of + the well. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + hole_depth: + title: Borehole depth + description: >- + Depth of the drilled hole, from ground surface to the bottom of the + borehole. Usually deeper than the finished well. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + well_casing_diameter: + title: Casing diameter + description: Inside diameter of the well casing. + x-ogc-unit: https://qudt.org/vocab/unit/IN + x-ogc-unitLang: QUDT + well_casing_depth: + title: Casing depth + description: >- + Depth from ground surface to the bottom of the well casing. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + well_completion_date: + title: Completion date + description: Date the well was finished, where it is known. + well_driller_name: + title: Driller + description: Name of the driller or drilling company that constructed the well. + well_construction_method: + title: Construction method + description: >- + How the well was constructed, such as air rotary, cable tool, or dug, + from a controlled vocabulary. + enum-lexicon: well_construction_method + well_pump_type: + title: Pump type + description: >- + Type of pump installed in the well, such as submersible or windmill, + from a controlled vocabulary. + enum-lexicon: well_pump_type + well_pump_depth: + title: Pump intake depth + description: Depth from ground surface to the pump intake. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + formation_completion_code: + title: Completion formation + description: >- + Geologic formation the well is completed in -- the formation it draws + from, not the full sequence of rock it passes through. + nma_formation_zone: + title: Legacy formation zone + description: >- + Formation zone exactly as recorded in the legacy NM_Aquifer WellData + table, kept unedited alongside the controlled-vocabulary value. + county: + title: County + description: New Mexico county the feature falls in. + state: + title: State + description: State the feature falls in. + api: + title: API number + description: >- + American Petroleum Institute well number, the standard unique identifier + for a drilled well in the United States. + well_name: + title: Well name + description: Name the well is recorded under in the legacy NM_Wells database. + well_num: + title: Well number + description: Operator's number for the well within its lease or unit. + well_data_id: + title: Legacy NM_Wells well key + description: >- + Identifier of the well's record in the legacy NM_Wells database, kept so + rows can be traced back to their source. + total_depth: + title: Total depth + description: Total drilled depth of the well, as reported on its record. + source_id: + title: Source ID + description: >- + Identifier of the publication or data submission the record came from, + in the legacy NM_Wells source register. + entry_date: + title: Record entry date + description: Date the record was entered into the legacy NM_Wells database. + lat_dd83: + title: Latitude (NAD 83) + description: Latitude in decimal degrees on the NAD 83 datum. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + long_dd83: + title: Longitude (NAD 83) + description: Longitude in decimal degrees on the NAD 83 datum. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + lat_dd27: + title: Latitude (NAD 27) + description: >- + Latitude in decimal degrees on the older NAD 27 datum, as originally + recorded. Positions differ from NAD 83 by roughly 100 metres. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + long_dd27: + title: Longitude (NAD 27) + description: >- + Longitude in decimal degrees on the older NAD 27 datum, as originally + recorded. Positions differ from NAD 83 by roughly 100 metres. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + elev_gl: + title: Ground-level elevation + description: Elevation of the ground surface at the well head. + elev_kb: + title: Kelly bushing elevation + description: >- + Elevation of the kelly bushing, the point on the drilling rig that + drilled depths were measured from. Typically several metres above ground + level. + elev_unspc: + title: Elevation (unspecified datum) + description: >- + Elevation recorded without a stated reference point, so it may be ground + level or a drilling datum. + depth_unit: + title: Depth unit + description: Unit the depths on this record are reported in. + temp_unit: + title: Temperature unit + description: Unit the temperatures on this record are reported in. + +locations: + nma_pk_location: + title: Legacy NM_Aquifer location key + description: >- + Primary key of this site's record in the legacy NM_Aquifer Location + table, kept so migrated rows can be traced back to their source. + description: + title: Site description + description: Free-text description of the site. + quad_name: + title: USGS quadrangle + description: Name of the USGS 7.5-minute topographic quadrangle the site falls in. + nma_location_notes: + title: Location notes + description: >- + Notes about the site carried over from NM_Aquifer, typically covering + access and how to find it on the ground. + nma_coordinate_notes: + title: Coordinate notes + description: >- + Notes on how the coordinates were obtained -- GPS, digitised from a map, + or derived from a legal description. + nma_data_reliability: + title: Data reliability + description: >- + Legacy rating of how much confidence to place in the site's recorded + position. + nma_date_created: + title: Legacy record created + description: Date the site record was created in NM_Aquifer. + nma_site_date: + title: Site date + description: Date associated with the site itself in NM_Aquifer, where one was recorded. + +project_areas: + name: + title: Project name + description: Name of the Bureau project the area belongs to. + description: + title: Project description + description: Free-text description of the project. + group_type: + title: Group type + description: >- + Kind of grouping the record represents, such as a project or a + geographic area. + +well_water_column: + water_column_latest: + title: Water column, latest reading + description: >- + Standing water in the well at the most recent measurement: the well's + depth less that reading's depth to water. Reported as zero where the + reading is deeper than the recorded well depth. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + water_column_average: + title: Water column, average reading + description: >- + Standing water the well holds on average: the well's depth less the mean + depth to water across every reading on record. Each reading counts once, + however unevenly spaced in time they are. Reported as zero where the mean + reading is deeper than the recorded well depth. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + water_column_maximum: + title: Water column, fullest on record + description: >- + The most standing water the well is known to have held: the well's depth + less the shallowest depth to water on record. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + water_column_minimum: + title: Water column, emptiest on record + description: >- + The least standing water the well is known to have held: the well's depth + less the deepest depth to water on record. Reported as zero where that + reading is deeper than the recorded well depth. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + +water_well_summary: + elevation_method: + title: Elevation method + description: >- + How the ground-surface elevation was determined, such as GPS survey or + read from a digital elevation model. Governs how much precision the + elevation deserves. + enum-lexicon: collection_method + formation_zone: + title: Formation zone + description: Geologic formation the well draws from, as recorded for the well. + total_water_levels: + title: Water-level measurement count + description: >- + Number of manual groundwater-level measurements behind this row's + statistics. Small counts make the range and trend unreliable. + last_water_level: + title: Latest water level + description: >- + Most recent groundwater-level measurement, as a depth below ground + surface. Reported in the units of the source reading, which is feet for + almost the whole record; unlike water_elevation_wells this layer does + not convert metric readings. + last_water_level_datetime: + title: Latest measurement time + description: Date and time of the most recent groundwater-level measurement. + min_water_level: + title: Shallowest water level + description: >- + Smallest depth below ground surface on record -- the high-water mark, + since a smaller depth means water nearer the surface. + max_water_level: + title: Deepest water level + description: >- + Largest depth below ground surface on record -- the low-water mark, + since a larger depth means water further down. + water_level_trend_ft_per_year: + title: Water-level trend + description: >- + Slope of a straight line fitted through the well's depth-to-water + measurements over time, in feet per year. Positive means depth is + increasing, so the water table is falling; negative means it is rising. + x-ogc-unit: https://qudt.org/vocab/unit/FT-PER-YR + x-ogc-unitLang: QUDT + +actively_monitored_wells: + elevation_method: + title: Elevation method + description: >- + How the ground-surface elevation was determined, such as GPS survey or + read from a digital elevation model. + enum-lexicon: collection_method + formation_zone: + title: Formation zone + description: Geologic formation the well draws from, as recorded for the well. + total_water_levels: + title: Water-level measurement count + description: Number of manual groundwater-level measurements on record for the well. + last_water_level: + title: Latest water level + description: >- + Most recent groundwater-level measurement, as a depth below ground + surface, in the units of the source reading. + last_water_level_datetime: + title: Latest measurement time + description: Date and time of the most recent groundwater-level measurement. + min_water_level: + title: Shallowest water level + description: Smallest depth below ground surface on record for the well. + max_water_level: + title: Deepest water level + description: Largest depth below ground surface on record for the well. + water_level_trend_ft_per_year: + title: Water-level trend + description: >- + Slope of a straight line fitted through the well's depth-to-water + measurements over time, in feet per year. Positive means the water table + is falling. + x-ogc-unit: https://qudt.org/vocab/unit/FT-PER-YR + x-ogc-unitLang: QUDT + group_ids: + title: Network IDs + description: Identifiers of every monitoring network the well belongs to. + group_names: + title: Network names + description: >- + Names of every monitoring network the well belongs to, in the same + order as group_ids. + group_types: + title: Network types + description: >- + Kind of grouping each network record represents, in the same order as + group_ids. + +depth_to_water_trend_wells: + record_count: + title: Measurement count + description: >- + Number of groundwater-level measurements the trend was fitted to. Below + 10 measurements -- or below 4 spanning less than two years -- the trend + is reported as not enough data. + first_observation_datetime: + title: First measurement time + description: Date and time of the earliest measurement used in the fit. + last_observation_datetime: + title: Latest measurement time + description: Date and time of the most recent measurement used in the fit. + span_years: + title: Record span + description: >- + Years between the first and last measurement used in the fit. A steep + slope over a short span is weak evidence of a real trend. + x-ogc-unit: https://qudt.org/vocab/unit/YR + x-ogc-unitLang: QUDT + slope_ft_per_year: + title: Trend slope + description: >- + Slope of a straight line fitted through depth to water below ground + surface over time, in feet per year. Positive means depth is increasing, + so the water table is falling. + x-ogc-unit: https://qudt.org/vocab/unit/FT-PER-YR + x-ogc-unitLang: QUDT + trend_category: + title: Trend category + description: >- + Plain-language reading of the slope: increasing (water table falling + faster than 0.25 ft/yr), decreasing (rising faster than 0.25 ft/yr), + stable, or not enough data. + enum: [increasing, decreasing, stable, not enough data] + +water_elevation_wells: + observation_id: + title: Measurement ID + description: Identifier of the groundwater-level measurement this row was calculated from. + observation_datetime: + title: Measurement time + description: Date and time the groundwater level was measured. + elevation_m: + title: Ground-surface elevation + description: >- + Surveyed elevation of the ground surface at the well, in metres above + the NAVD 88 vertical datum. + x-ogc-unit: https://qudt.org/vocab/unit/M + x-ogc-unitLang: QUDT + depth_to_water_below_ground_surface_ft: + title: Depth to water + description: >- + Distance from ground surface down to the water table at the time of + measurement. Metric readings are converted to feet, and a reading with + no recorded measuring-point height is treated as taken at ground level. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + water_elevation_ft: + title: Water-table elevation + description: >- + Height of the water table above sea level: ground-surface elevation + converted to feet, minus the depth to water. Comparable between wells + standing at different ground elevations. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + +latest_depth_to_water_wells: + observation_id: + title: Measurement ID + description: Identifier of the groundwater-level measurement this row reports. + observation_datetime: + title: Measurement time + description: Date and time the groundwater level was measured. + depth_to_water_reference: + title: Depth to water from reference point + description: >- + Depth to water as read in the field, measured down from the measuring + point rather than from the ground. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + measuring_point_height: + title: Measuring-point height + description: >- + Height of the measuring point -- usually the top of the well casing -- + above ground surface. Subtracted from the field reading to give a depth + below ground surface; a reading with no recorded height is treated as + taken at ground level. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + depth_to_water_bgs: + title: Depth to water below ground surface + description: >- + Distance from ground surface down to the water table, after subtracting + the measuring-point height from the field reading. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + +latest_tds_wells: + major_chemistry_id: + title: Analysis ID + description: Identifier of the laboratory result this row reports. + latest_tds_observation_date: + title: Analysis date + description: >- + Date the reported TDS result was analysed, or the date the sample was + collected where no analysis date was recorded. + latest_tds_value: + title: Total dissolved solids + description: >- + Most recent measured concentration of dissolved mineral matter in the + water. Higher values mean saltier water; drinking-water guidance sits + around 500 mg/L. + latest_tds_units: + title: TDS units + description: >- + Units the TDS value is reported in, as the laboratory recorded them -- + usually milligrams per litre. + +avg_tds_wells: + tds_observation_count: + title: Analysis count + description: >- + Number of TDS results the average was taken over. Across the catalog + this averages about 1.9, so most rows average one or two samples. + avg_tds_value: + title: Average total dissolved solids + description: >- + Arithmetic mean of every TDS result on record for the well, without + weighting by date. Read alongside the analysis count before relying on + it. + first_tds_observation_date: + title: First analysis date + description: Date of the earliest TDS result included in the average. + last_tds_observation_date: + title: Latest analysis date + description: Date of the most recent TDS result included in the average. + +geothermal_wells_bht: + bht_count: + title: Reading count + description: Number of bottom-hole temperature readings recorded for the well. + max_bht: + title: Highest bottom-hole temperature (as recorded) + description: >- + Highest bottom-hole temperature on record for the well, in the units the + source recorded it in. Use max_bht_c for a comparable value. + min_bht: + title: Lowest bottom-hole temperature (as recorded) + description: >- + Lowest bottom-hole temperature on record for the well, in the units the + source recorded it in. Use min_bht_c for a comparable value. + max_bht_c: + title: Highest bottom-hole temperature + description: >- + Highest bottom-hole temperature on record for the well, converted to + degrees Celsius. + x-ogc-unit: https://qudt.org/vocab/unit/DEG_C + x-ogc-unitLang: QUDT + min_bht_c: + title: Lowest bottom-hole temperature + description: >- + Lowest bottom-hole temperature on record for the well, converted to + degrees Celsius. + x-ogc-unit: https://qudt.org/vocab/unit/DEG_C + x-ogc-unitLang: QUDT + max_bht_depth: + title: Deepest reading depth + description: Depth of the deepest bottom-hole temperature reading for the well. + temp_unit: + title: Temperature unit + description: >- + Unit the converted temperatures are reported in. Always Celsius; see + temp_unit_source for what the readings arrived as. + enum: [C] + temp_unit_source: + title: Source temperature units + description: >- + Units the underlying readings were recorded in, comma-separated where + the well's readings did not agree. + temp_unit_mixed: + title: Mixed source units + description: >- + True when the well's readings arrived in more than one temperature unit, + which is a sign the source record needs review. + unconvertible_count: + title: Unconvertible reading count + description: >- + Number of readings that could not be converted to Celsius because their + unit was missing or unrecognised. These are excluded from the minimum + and maximum. + +geothermal_wells_temperature_profile: + reading_count: + title: Reading count + description: Number of temperature-versus-depth readings logged in the well. + min_depth: + title: Shallowest reading depth + description: Depth of the shallowest temperature reading in the profile. + max_depth: + title: Deepest reading depth + description: Depth of the deepest temperature reading in the profile. + min_temp: + title: Lowest temperature (as recorded) + description: >- + Lowest temperature in the profile, in the units the source recorded it + in. Use min_temp_c for a comparable value. + max_temp: + title: Highest temperature (as recorded) + description: >- + Highest temperature in the profile, in the units the source recorded it + in. Use max_temp_c for a comparable value. + min_temp_c: + title: Lowest temperature + description: Lowest temperature in the profile, converted to degrees Celsius. + x-ogc-unit: https://qudt.org/vocab/unit/DEG_C + x-ogc-unitLang: QUDT + max_temp_c: + title: Highest temperature + description: Highest temperature in the profile, converted to degrees Celsius. + x-ogc-unit: https://qudt.org/vocab/unit/DEG_C + x-ogc-unitLang: QUDT + temp_unit: + title: Temperature unit + description: >- + Unit the converted temperatures are reported in. Always Celsius; see + temp_unit_source for what the readings arrived as. + enum: [C] + temp_unit_source: + title: Source temperature units + description: >- + Units the underlying readings were recorded in, comma-separated where + the well's readings did not agree. + temp_unit_mixed: + title: Mixed source units + description: >- + True when the well's readings arrived in more than one temperature unit, + which is a sign the source record needs review. + unconvertible_count: + title: Unconvertible reading count + description: >- + Number of readings that could not be converted to Celsius because their + unit was missing or unrecognised. + series: + title: Temperature-depth profile + description: >- + The whole profile as a list of readings, each carrying a depth, the + temperature as recorded, and the temperature converted to Celsius. + +bht_measurements: + operator: + title: Operator + description: Company operating the well when the record was made. + well_type: + title: Well type + description: Purpose the well was drilled for, such as oil, gas, or observation. + well_tvd: + title: True vertical depth + description: >- + Vertical depth of the well, which is shorter than the drilled length for + a deviated hole. + completion_date: + title: Completion date + description: Date the well was finished. + current_status: + title: Current status + description: Latest recorded status of the well, such as producing or plugged. + cuttings: + title: Cuttings held + description: >- + Whether rock cuttings from the well are held in the Bureau's subsurface + library. + core_exists: + title: Core held + description: Whether a rock core from the well is held in the Bureau's subsurface library. + bht_depth: + title: Reading depth + description: Depth the bottom-hole temperature was measured at. + bht: + title: Bottom-hole temperature + description: >- + Temperature measured at the bottom of the hole, in the units recorded by + the source. + hours_since_circulation: + title: Hours since circulation + description: >- + Time between the last circulation of drilling fluid and the reading. + Drilling fluid cools the rock, so a reading taken soon after circulation + is too low; this figure decides whether it can be corrected. + x-ogc-unit: https://qudt.org/vocab/unit/HR + x-ogc-unitLang: QUDT + date_measured: + title: Measurement date + description: Date the temperature was measured. + +temp_depth_measurements: + sample_fm: + title: Formation + description: Geologic formation at the depth the reading was taken. + loc_acc_val: + title: Location accuracy + description: Recorded accuracy of the well's coordinates. + entered_by: + title: Entered by + description: Person who entered the record into the legacy NM_Wells database. + depth: + title: Reading depth + description: Depth below the reference datum the temperature was measured at. + temp: + title: Temperature + description: Temperature measured at this depth, in the units recorded by the source. + sample_date: + title: Measurement date + description: Date the temperature was measured. + +heat_flow: + elevation_m: + title: Elevation + description: Well elevation converted to metres. + x-ogc-unit: https://qudt.org/vocab/unit/M + x-ogc-unitLang: QUDT + depth_units: + title: Depth units + description: Units the depths on this record were originally reported in. + total_depth_m: + title: Total depth (metres) + description: Total drilled depth of the well converted to metres. + x-ogc-unit: https://qudt.org/vocab/unit/M + x-ogc-unitLang: QUDT + from_depth: + title: Interval top + description: Depth to the top of the interval the determination covers. + to_depth: + title: Interval base + description: Depth to the bottom of the interval the determination covers. + therml_cond: + title: Thermal conductivity (as published) + description: >- + How readily the rock conducts heat, in the units it was published in. + Use tc_si for a comparable value. + tcond_range: + title: Thermal conductivity range + description: Published spread of conductivity values for the interval. + tcond_error: + title: Thermal conductivity error + description: Published uncertainty on the conductivity value. + tcond_unit: + title: Thermal conductivity unit + description: >- + Unit the published conductivity is in. TCU denotes the older thermal + conductivity unit, mcal/cm-s-degC. + tc_si: + title: Thermal conductivity + description: >- + Thermal conductivity converted to SI units, watts per metre-kelvin. + Typical rock sits between 1 and 5. + x-ogc-unit: https://qudt.org/vocab/unit/W-PER-M-K + x-ogc-unitLang: QUDT + sample_type: + title: Sample type + description: What the conductivity was measured on, such as core or cuttings. + num_samples: + title: Sample count + description: Number of samples the conductivity value was measured from. + therml_grad: + title: Thermal gradient + description: >- + How fast temperature rises with depth over the interval, in the units it + was published in. Continental crust averages roughly 25 degrees Celsius + per kilometre. + tgrad_range: + title: Thermal gradient range + description: Published spread of gradient values for the interval. + tg_error: + title: Thermal gradient error + description: Published uncertainty on the gradient value. + grad_unit: + title: Thermal gradient unit + description: Unit the published gradient is in. + heat_flow: + title: Heat flow (as published) + description: >- + Rate at which heat escapes through the ground over this interval, in the + units it was published in. Use heat_flow_si for a comparable value. + ht_flow_unit: + title: Heat flow unit + description: >- + Unit the published heat flow is in. HFU denotes the older heat flow + unit, equal to 41.84 milliwatts per square metre. + heat_flow_si: + title: Heat flow + description: >- + Heat flow converted to SI units, milliwatts per square metre. Continental + averages sit near 65; values well above that mark geothermal interest. + x-ogc-unit: https://qudt.org/vocab/unit/MilliW-PER-M2 + x-ogc-unitLang: QUDT + ht_flow_est: + title: Estimated heat flow (as published) + description: >- + Heat flow the author estimated rather than measured, in the units it was + published in. + ht_flow_est_si: + title: Estimated heat flow + description: >- + Author-estimated heat flow converted to milliwatts per square metre. + x-ogc-unit: https://qudt.org/vocab/unit/MilliW-PER-M2 + x-ogc-unitLang: QUDT + quality: + title: Quality rating + description: >- + The publication's own assessment of how much confidence the + determination deserves. + first_auth: + title: First author + description: First author of the publication the determination came from. + pub_year: + title: Publication year + description: Year the determination was published. + title: + title: Publication title + description: Title of the publication the determination came from. + journal: + title: Journal + description: Journal or report series the determination was published in. + volume: + title: Volume + description: Volume of the journal or report series. + page_no: + title: Pages + description: Page range of the publication. + +dst: + dst_name: + title: Test name + description: Name recorded for the drill stem test. + dst_operator: + title: Testing contractor + description: Company that ran the drill stem test. + dst_number: + title: Test number + description: Sequence number of this test within the well. + dst_date: + title: Test date + description: Date the drill stem test was run. + from_depth: + title: Interval top + description: Depth to the top of the tested interval. + to_depth: + title: Interval base + description: Depth to the bottom of the tested interval. + target_fm: + title: Target formation + description: Geologic formation the test was aimed at. + packer_from: + title: Upper packer depth + description: >- + Depth of the upper packer, the seal that isolates the tested interval + from the rest of the hole. + packer_to: + title: Lower packer depth + description: Depth of the lower packer sealing the bottom of the tested interval. + srf_choke_sz: + title: Surface choke size + description: >- + Size of the choke at surface, which limits how fast fluid is allowed to + flow during the test. + bot_choke_sz: + title: Bottom choke size + description: Size of the choke at the bottom of the string. + prs_gage_dpt: + title: Pressure gauge depth + description: Depth the pressure gauge was set at. + pipe_dia: + title: Pipe diameter + description: Diameter of the drill pipe used for the test. + pipe_length: + title: Pipe length + description: Length of drill pipe run for the test. + flow_history: + title: Flow history + description: >- + The operations logged during the test, in order -- opening the tool, + flow periods, and shut-in periods. + init_flow: + title: Initial flow pressure + description: Pressure recorded at the start of the first flow period. + flw_prs_in_min: + title: Initial flow duration + description: Length of the first flow period, in minutes. + x-ogc-unit: https://qudt.org/vocab/unit/MIN + x-ogc-unitLang: QUDT + fin_flow: + title: Final flow pressure + description: Pressure recorded at the end of the last flow period. + flw_prs_fin_min: + title: Final flow duration + description: Length of the last flow period, in minutes. + x-ogc-unit: https://qudt.org/vocab/unit/MIN + x-ogc-unitLang: QUDT + prs_init_clsd_in: + title: Initial shut-in pressure + description: >- + Pressure built up during the first shut-in period, after the tool was + closed and fluid stopped flowing. + in_sht_in_min: + title: Initial shut-in duration + description: Length of the first shut-in period, in minutes. + x-ogc-unit: https://qudt.org/vocab/unit/MIN + x-ogc-unitLang: QUDT + fin_shut_in: + title: Final shut-in pressure + description: >- + Pressure built up during the last shut-in period. Usually the closest + available estimate of true formation pressure. + fn_sht_in_min: + title: Final shut-in duration + description: Length of the last shut-in period, in minutes. + x-ogc-unit: https://qudt.org/vocab/unit/MIN + x-ogc-unitLang: QUDT + hydrost_prs_in: + title: Initial hydrostatic pressure + description: >- + Pressure of the fluid column in the hole before the test, used as a + reference for the flowing pressures. + hyd_st_prs_fl: + title: Final hydrostatic pressure + description: Pressure of the fluid column in the hole at the end of the test. + press_units: + title: Pressure units + description: Units the pressures on this record are reported in. + blanked_off: + title: Blanked off + description: Whether the tested interval was blanked off during the test. + fm_temp: + title: Formation temperature + description: Temperature recorded for the formation during the test. + +# --------------------------------------------------------------------------- +# Chemistry analyte columns below are generated by +# cli/generate_chemistry_field_descriptions.py and reviewed by hand. Re-run it +# when the analyte lists in the ogc_* view migrations change. +# --------------------------------------------------------------------------- +major_chemistry_results: + location_id: + title: Location ID + description: >- + Identifier of the location record the well's coordinates came from. + + analyte_count: + title: Analyte count + description: >- + Number of distinct analytes with a value in this row. A low count + means the well has only been analysed for part of the suite. + + latest_chemistry_date: + title: Latest analysis date + description: >- + Date of the most recent result in this row. Analytes are carried + forward independently, so an individual value may be older than this + date. + + tds: + title: Total dissolved solids + description: >- + Total mass of dissolved mineral matter in the water -- in plain + terms, how salty it is. Drinking-water guidance sits around 500 + mg/L. + + tds_units: + title: Total dissolved solids units + description: >- + Units the total dissolved solids value is reported in, as the + laboratory recorded them. + + calcium: + title: Calcium + description: >- + Dissolved calcium concentration in the most recent sample analysed + for it. + + calcium_units: + title: Calcium units + description: >- + Units the calcium value is reported in, as the laboratory recorded + them. + + calcium_total: + title: Calcium (total) + description: >- + Total calcium concentration -- the unfiltered determination, which + counts calcium bound to suspended particles as well as the dissolved + fraction. + + calcium_total_units: + title: Calcium (total) units + description: >- + Units the calcium (total) value is reported in, as the laboratory + recorded them. + + magnesium: + title: Magnesium + description: >- + Dissolved magnesium concentration in the most recent sample analysed + for it. + + magnesium_units: + title: Magnesium units + description: >- + Units the magnesium value is reported in, as the laboratory recorded + them. + + magnesium_total: + title: Magnesium (total) + description: >- + Total magnesium concentration -- the unfiltered determination, which + counts magnesium bound to suspended particles as well as the + dissolved fraction. + + magnesium_total_units: + title: Magnesium (total) units + description: >- + Units the magnesium (total) value is reported in, as the laboratory + recorded them. + + sodium: + title: Sodium + description: >- + Dissolved sodium concentration in the most recent sample analysed + for it. + + sodium_units: + title: Sodium units + description: >- + Units the sodium value is reported in, as the laboratory recorded + them. + + sodium_total: + title: Sodium (total) + description: >- + Total sodium concentration -- the unfiltered determination, which + counts sodium bound to suspended particles as well as the dissolved + fraction. + + sodium_total_units: + title: Sodium (total) units + description: >- + Units the sodium (total) value is reported in, as the laboratory + recorded them. + + potassium: + title: Potassium + description: >- + Dissolved potassium concentration in the most recent sample analysed + for it. + + potassium_units: + title: Potassium units + description: >- + Units the potassium value is reported in, as the laboratory recorded + them. + + potassium_total: + title: Potassium (total) + description: >- + Total potassium concentration -- the unfiltered determination, which + counts potassium bound to suspended particles as well as the + dissolved fraction. + + potassium_total_units: + title: Potassium (total) units + description: >- + Units the potassium (total) value is reported in, as the laboratory + recorded them. + + sodium_plus_potassium: + title: Sodium plus potassium + description: >- + Combined sodium and potassium concentration, reported together where + the laboratory did not separate them. + + sodium_plus_potassium_units: + title: Sodium plus potassium units + description: >- + Units the sodium plus potassium value is reported in, as the + laboratory recorded them. + + bicarbonate: + title: Bicarbonate + description: >- + Dissolved bicarbonate concentration in the most recent sample + analysed for it. + + bicarbonate_units: + title: Bicarbonate units + description: >- + Units the bicarbonate value is reported in, as the laboratory + recorded them. + + carbonate: + title: Carbonate + description: >- + Dissolved carbonate concentration in the most recent sample analysed + for it. + + carbonate_units: + title: Carbonate units + description: >- + Units the carbonate value is reported in, as the laboratory recorded + them. + + sulfate: + title: Sulfate + description: >- + Dissolved sulfate concentration in the most recent sample analysed + for it. + + sulfate_units: + title: Sulfate units + description: >- + Units the sulfate value is reported in, as the laboratory recorded + them. + + chloride: + title: Chloride + description: >- + Dissolved chloride concentration in the most recent sample analysed + for it. + + chloride_units: + title: Chloride units + description: >- + Units the chloride value is reported in, as the laboratory recorded + them. + + ion_balance: + title: Ion balance + description: >- + Percentage difference between the total positive and total negative + charge in the analysis. Charge must balance in reality, so a figure + far from zero means the analysis is incomplete or in error. + + ion_balance_units: + title: Ion balance units + description: >- + Units the ion balance value is reported in, as the laboratory + recorded them. + + total_anions: + title: Total anions + description: >- + Sum of the negatively charged dissolved constituents in the + analysis. + + total_anions_units: + title: Total anions units + description: >- + Units the total anions value is reported in, as the laboratory + recorded them. + + total_cations: + title: Total cations + description: >- + Sum of the positively charged dissolved constituents in the + analysis. + + total_cations_units: + title: Total cations units + description: >- + Units the total cations value is reported in, as the laboratory + recorded them. + + alkalinity: + title: Alkalinity + description: >- + The water's capacity to neutralise acid, reported as an equivalent + mass of calcium carbonate. Mostly supplied by bicarbonate and + carbonate. + + alkalinity_units: + title: Alkalinity units + description: >- + Units the alkalinity value is reported in, as the laboratory + recorded them. + + hardness: + title: Hardness + description: >- + Combined calcium and magnesium content, reported as an equivalent + mass of calcium carbonate. What determines whether water is 'hard'. + + hardness_units: + title: Hardness units + description: >- + Units the hardness value is reported in, as the laboratory recorded + them. + + specific_conductance: + title: Specific conductance + description: >- + How well the water conducts electricity, which rises with dissolved + mineral content. Used as a fast field proxy for total dissolved + solids. + + specific_conductance_units: + title: Specific conductance units + description: >- + Units the specific conductance value is reported in, as the + laboratory recorded them. + + ph: + title: pH + description: >- + Acidity of the water on the 0-14 scale, where 7 is neutral. + Unitless. Most New Mexico groundwater falls between 7 and 8.5. + + ph_units: + title: pH units + description: >- + Units the ph value is reported in, as the laboratory recorded them. + + nitrate: + title: Nitrate + description: >- + Dissolved nitrate concentration, usually from fertiliser, septic + systems, or livestock. The drinking-water limit is 10 mg/L as + nitrogen. + + nitrate_units: + title: Nitrate units + description: >- + Units the nitrate value is reported in, as the laboratory recorded + them. + + fluoride: + title: Fluoride + description: >- + Dissolved fluoride concentration. Beneficial in small amounts; the + drinking-water limit is 4 mg/L. + + fluoride_units: + title: Fluoride units + description: >- + Units the fluoride value is reported in, as the laboratory recorded + them. + + silica: + title: Silica + description: >- + Dissolved silica concentration, weathered out of silicate rock. + Useful for estimating the temperature water last equilibrated at. + + silica_units: + title: Silica units + description: >- + Units the silica value is reported in, as the laboratory recorded + them. + +minor_chemistry_wells: + location_id: + title: Location ID + description: >- + Identifier of the location record the well's coordinates came from. + + analyte_count: + title: Analyte count + description: >- + Number of distinct analytes with a value in this row. A low count + means the well has only been analysed for part of the suite. + + latest_chemistry_date: + title: Latest analysis date + description: >- + Date of the most recent result in this row. Analytes are carried + forward independently, so an individual value may be older than this + date. + + h2r: + title: Deuterium ratio + description: >- + Ratio of heavy to ordinary hydrogen in the water, reported as + per-mil difference from ocean water. Fingerprints where the water + fell as precipitation. + + h2r_units: + title: Deuterium ratio units + description: >- + Units the deuterium ratio value is reported in, as the laboratory + recorded them. + + o18r: + title: Oxygen-18 ratio + description: >- + Ratio of heavy to ordinary oxygen in the water, reported as per-mil + difference from ocean water. Read with the deuterium ratio to trace + the water's origin and evaporation history. + + o18r_units: + title: Oxygen-18 ratio units + description: >- + Units the oxygen-18 ratio value is reported in, as the laboratory + recorded them. + + c13r: + title: Carbon-13 ratio + description: >- + Ratio of carbon-13 to carbon-12 in the water's dissolved carbon, + reported as per-mil difference from a standard. Helps identify where + the carbon came from, which is needed to correct a carbon-14 age. + + c13r_units: + title: Carbon-13 ratio units + description: >- + Units the carbon-13 ratio value is reported in, as the laboratory + recorded them. + + c14: + title: Carbon-14 + description: >- + Carbon-14 remaining in the water's dissolved carbon, as a percentage + of the modern atmospheric level. The basis for dating groundwater up + to roughly 40,000 years old. + + c14_units: + title: Carbon-14 units + description: >- + Units the carbon-14 value is reported in, as the laboratory recorded + them. + + c14_years: + title: Carbon-14 age + description: >- + Apparent age of the water in years, calculated from its carbon-14 + content. Uncorrected for carbon picked up from rock, so treat it as + an upper bound. + + c14_years_units: + title: Carbon-14 age units + description: >- + Units the carbon-14 age value is reported in, as the laboratory + recorded them. + + fluoride: + title: Fluoride + description: >- + Dissolved fluoride concentration. Beneficial in small amounts; the + drinking-water limit is 4 mg/L. + + fluoride_units: + title: Fluoride units + description: >- + Units the fluoride value is reported in, as the laboratory recorded + them. + + barium: + title: Barium + description: >- + Dissolved barium concentration in the most recent sample analysed + for it. + + barium_units: + title: Barium units + description: >- + Units the barium value is reported in, as the laboratory recorded + them. + + barium_total: + title: Barium (total) + description: >- + Total barium concentration -- the unfiltered determination, which + counts barium bound to suspended particles as well as the dissolved + fraction. + + barium_total_units: + title: Barium (total) units + description: >- + Units the barium (total) value is reported in, as the laboratory + recorded them. + + copper: + title: Copper + description: >- + Dissolved copper concentration in the most recent sample analysed + for it. + + copper_units: + title: Copper units + description: >- + Units the copper value is reported in, as the laboratory recorded + them. + + copper_total: + title: Copper (total) + description: >- + Total copper concentration -- the unfiltered determination, which + counts copper bound to suspended particles as well as the dissolved + fraction. + + copper_total_units: + title: Copper (total) units + description: >- + Units the copper (total) value is reported in, as the laboratory + recorded them. + + zinc: + title: Zinc + description: >- + Dissolved zinc concentration in the most recent sample analysed for + it. + + zinc_units: + title: Zinc units + description: >- + Units the zinc value is reported in, as the laboratory recorded + them. + + zinc_total: + title: Zinc (total) + description: >- + Total zinc concentration -- the unfiltered determination, which + counts zinc bound to suspended particles as well as the dissolved + fraction. + + zinc_total_units: + title: Zinc (total) units + description: >- + Units the zinc (total) value is reported in, as the laboratory + recorded them. + + molybdenum: + title: Molybdenum + description: >- + Dissolved molybdenum concentration in the most recent sample + analysed for it. + + molybdenum_units: + title: Molybdenum units + description: >- + Units the molybdenum value is reported in, as the laboratory + recorded them. + + molybdenum_total: + title: Molybdenum (total) + description: >- + Total molybdenum concentration -- the unfiltered determination, + which counts molybdenum bound to suspended particles as well as the + dissolved fraction. + + molybdenum_total_units: + title: Molybdenum (total) units + description: >- + Units the molybdenum (total) value is reported in, as the laboratory + recorded them. + + silica: + title: Silica + description: >- + Dissolved silica concentration, weathered out of silicate rock. + Useful for estimating the temperature water last equilibrated at. + + silica_units: + title: Silica units + description: >- + Units the silica value is reported in, as the laboratory recorded + them. + + silicon: + title: Silicon + description: >- + Dissolved silicon concentration in the most recent sample analysed + for it. + + silicon_units: + title: Silicon units + description: >- + Units the silicon value is reported in, as the laboratory recorded + them. + + silicon_total: + title: Silicon (total) + description: >- + Total silicon concentration -- the unfiltered determination, which + counts silicon bound to suspended particles as well as the dissolved + fraction. + + silicon_total_units: + title: Silicon (total) units + description: >- + Units the silicon (total) value is reported in, as the laboratory + recorded them. + + manganese: + title: Manganese + description: >- + Dissolved manganese concentration in the most recent sample analysed + for it. + + manganese_units: + title: Manganese units + description: >- + Units the manganese value is reported in, as the laboratory recorded + them. + + manganese_total: + title: Manganese (total) + description: >- + Total manganese concentration -- the unfiltered determination, which + counts manganese bound to suspended particles as well as the + dissolved fraction. + + manganese_total_units: + title: Manganese (total) units + description: >- + Units the manganese (total) value is reported in, as the laboratory + recorded them. + + iron: + title: Iron + description: >- + Dissolved iron concentration in the most recent sample analysed for + it. + + iron_units: + title: Iron units + description: >- + Units the iron value is reported in, as the laboratory recorded + them. + + iron_total: + title: Iron (total) + description: >- + Total iron concentration -- the unfiltered determination, which + counts iron bound to suspended particles as well as the dissolved + fraction. + + iron_total_units: + title: Iron (total) units + description: >- + Units the iron (total) value is reported in, as the laboratory + recorded them. + + strontium: + title: Strontium + description: >- + Dissolved strontium concentration in the most recent sample analysed + for it. + + strontium_units: + title: Strontium units + description: >- + Units the strontium value is reported in, as the laboratory recorded + them. + + strontium_total: + title: Strontium (total) + description: >- + Total strontium concentration -- the unfiltered determination, which + counts strontium bound to suspended particles as well as the + dissolved fraction. + + strontium_total_units: + title: Strontium (total) units + description: >- + Units the strontium (total) value is reported in, as the laboratory + recorded them. + + chromium: + title: Chromium + description: >- + Dissolved chromium concentration in the most recent sample analysed + for it. + + chromium_units: + title: Chromium units + description: >- + Units the chromium value is reported in, as the laboratory recorded + them. + + chromium_total: + title: Chromium (total) + description: >- + Total chromium concentration -- the unfiltered determination, which + counts chromium bound to suspended particles as well as the + dissolved fraction. + + chromium_total_units: + title: Chromium (total) units + description: >- + Units the chromium (total) value is reported in, as the laboratory + recorded them. + + boron: + title: Boron + description: >- + Dissolved boron concentration in the most recent sample analysed for + it. + + boron_units: + title: Boron units + description: >- + Units the boron value is reported in, as the laboratory recorded + them. + + boron_total: + title: Boron (total) + description: >- + Total boron concentration -- the unfiltered determination, which + counts boron bound to suspended particles as well as the dissolved + fraction. + + boron_total_units: + title: Boron (total) units + description: >- + Units the boron (total) value is reported in, as the laboratory + recorded them. + + uranium: + title: Uranium + description: >- + Dissolved uranium concentration. Naturally present near + uranium-bearing rock and regulated in drinking water at 0.030 mg/L. + + uranium_units: + title: Uranium units + description: >- + Units the uranium value is reported in, as the laboratory recorded + them. + + uranium_total: + title: Uranium (total) + description: >- + Total uranium concentration -- the unfiltered determination, which + counts uranium bound to suspended particles as well as the dissolved + fraction. + + uranium_total_units: + title: Uranium (total) units + description: >- + Units the uranium (total) value is reported in, as the laboratory + recorded them. + + lithium: + title: Lithium + description: >- + Dissolved lithium concentration in the most recent sample analysed + for it. + + lithium_units: + title: Lithium units + description: >- + Units the lithium value is reported in, as the laboratory recorded + them. + + lithium_total: + title: Lithium (total) + description: >- + Total lithium concentration -- the unfiltered determination, which + counts lithium bound to suspended particles as well as the dissolved + fraction. + + lithium_total_units: + title: Lithium (total) units + description: >- + Units the lithium (total) value is reported in, as the laboratory + recorded them. + + silver: + title: Silver + description: >- + Dissolved silver concentration in the most recent sample analysed + for it. + + silver_units: + title: Silver units + description: >- + Units the silver value is reported in, as the laboratory recorded + them. + + silver_total: + title: Silver (total) + description: >- + Total silver concentration -- the unfiltered determination, which + counts silver bound to suspended particles as well as the dissolved + fraction. + + silver_total_units: + title: Silver (total) units + description: >- + Units the silver (total) value is reported in, as the laboratory + recorded them. + + antimony: + title: Antimony + description: >- + Dissolved antimony concentration in the most recent sample analysed + for it. + + antimony_units: + title: Antimony units + description: >- + Units the antimony value is reported in, as the laboratory recorded + them. + + antimony_total: + title: Antimony (total) + description: >- + Total antimony concentration -- the unfiltered determination, which + counts antimony bound to suspended particles as well as the + dissolved fraction. + + antimony_total_units: + title: Antimony (total) units + description: >- + Units the antimony (total) value is reported in, as the laboratory + recorded them. + + beryllium: + title: Beryllium + description: >- + Dissolved beryllium concentration in the most recent sample analysed + for it. + + beryllium_units: + title: Beryllium units + description: >- + Units the beryllium value is reported in, as the laboratory recorded + them. + + beryllium_total: + title: Beryllium (total) + description: >- + Total beryllium concentration -- the unfiltered determination, which + counts beryllium bound to suspended particles as well as the + dissolved fraction. + + beryllium_total_units: + title: Beryllium (total) units + description: >- + Units the beryllium (total) value is reported in, as the laboratory + recorded them. + + lead: + title: Lead + description: >- + Dissolved lead concentration in the most recent sample analysed for + it. + + lead_units: + title: Lead units + description: >- + Units the lead value is reported in, as the laboratory recorded + them. + + lead_total: + title: Lead (total) + description: >- + Total lead concentration -- the unfiltered determination, which + counts lead bound to suspended particles as well as the dissolved + fraction. + + lead_total_units: + title: Lead (total) units + description: >- + Units the lead (total) value is reported in, as the laboratory + recorded them. + + thallium: + title: Thallium + description: >- + Dissolved thallium concentration in the most recent sample analysed + for it. + + thallium_units: + title: Thallium units + description: >- + Units the thallium value is reported in, as the laboratory recorded + them. + + thallium_total: + title: Thallium (total) + description: >- + Total thallium concentration -- the unfiltered determination, which + counts thallium bound to suspended particles as well as the + dissolved fraction. + + thallium_total_units: + title: Thallium (total) units + description: >- + Units the thallium (total) value is reported in, as the laboratory + recorded them. + + bromide: + title: Bromide + description: >- + Dissolved bromide concentration. Read against chloride, it + distinguishes seawater-derived salinity from dissolved halite. + + bromide_units: + title: Bromide units + description: >- + Units the bromide value is reported in, as the laboratory recorded + them. + + selenium: + title: Selenium + description: >- + Dissolved selenium concentration in the most recent sample analysed + for it. + + selenium_units: + title: Selenium units + description: >- + Units the selenium value is reported in, as the laboratory recorded + them. + + selenium_total: + title: Selenium (total) + description: >- + Total selenium concentration -- the unfiltered determination, which + counts selenium bound to suspended particles as well as the + dissolved fraction. + + selenium_total_units: + title: Selenium (total) units + description: >- + Units the selenium (total) value is reported in, as the laboratory + recorded them. + + vanadium: + title: Vanadium + description: >- + Dissolved vanadium concentration in the most recent sample analysed + for it. + + vanadium_units: + title: Vanadium units + description: >- + Units the vanadium value is reported in, as the laboratory recorded + them. + + vanadium_total: + title: Vanadium (total) + description: >- + Total vanadium concentration -- the unfiltered determination, which + counts vanadium bound to suspended particles as well as the + dissolved fraction. + + vanadium_total_units: + title: Vanadium (total) units + description: >- + Units the vanadium (total) value is reported in, as the laboratory + recorded them. + + aluminum: + title: Aluminum + description: >- + Dissolved aluminum concentration in the most recent sample analysed + for it. + + aluminum_units: + title: Aluminum units + description: >- + Units the aluminum value is reported in, as the laboratory recorded + them. + + aluminum_total: + title: Aluminum (total) + description: >- + Total aluminum concentration -- the unfiltered determination, which + counts aluminum bound to suspended particles as well as the + dissolved fraction. + + aluminum_total_units: + title: Aluminum (total) units + description: >- + Units the aluminum (total) value is reported in, as the laboratory + recorded them. + + arsenic: + title: Arsenic + description: >- + Dissolved arsenic concentration. Naturally elevated in parts of New + Mexico and regulated in drinking water at 0.010 mg/L. + + arsenic_units: + title: Arsenic units + description: >- + Units the arsenic value is reported in, as the laboratory recorded + them. + + arsenic_total: + title: Arsenic (total) + description: >- + Total arsenic concentration -- the unfiltered determination, which + counts arsenic bound to suspended particles as well as the dissolved + fraction. + + arsenic_total_units: + title: Arsenic (total) units + description: >- + Units the arsenic (total) value is reported in, as the laboratory + recorded them. + + nickel: + title: Nickel + description: >- + Dissolved nickel concentration in the most recent sample analysed + for it. + + nickel_units: + title: Nickel units + description: >- + Units the nickel value is reported in, as the laboratory recorded + them. + + nickel_total: + title: Nickel (total) + description: >- + Total nickel concentration -- the unfiltered determination, which + counts nickel bound to suspended particles as well as the dissolved + fraction. + + nickel_total_units: + title: Nickel (total) units + description: >- + Units the nickel (total) value is reported in, as the laboratory + recorded them. + + cadmium: + title: Cadmium + description: >- + Dissolved cadmium concentration in the most recent sample analysed + for it. + + cadmium_units: + title: Cadmium units + description: >- + Units the cadmium value is reported in, as the laboratory recorded + them. + + cadmium_total: + title: Cadmium (total) + description: >- + Total cadmium concentration -- the unfiltered determination, which + counts cadmium bound to suspended particles as well as the dissolved + fraction. + + cadmium_total_units: + title: Cadmium (total) units + description: >- + Units the cadmium (total) value is reported in, as the laboratory + recorded them. + + cobalt: + title: Cobalt + description: >- + Dissolved cobalt concentration in the most recent sample analysed + for it. + + cobalt_units: + title: Cobalt units + description: >- + Units the cobalt value is reported in, as the laboratory recorded + them. + + cobalt_total: + title: Cobalt (total) + description: >- + Total cobalt concentration -- the unfiltered determination, which + counts cobalt bound to suspended particles as well as the dissolved + fraction. + + cobalt_total_units: + title: Cobalt (total) units + description: >- + Units the cobalt (total) value is reported in, as the laboratory + recorded them. + + phosphate: + title: Phosphate + description: >- + Dissolved phosphate concentration in the most recent sample analysed + for it. + + phosphate_units: + title: Phosphate units + description: >- + Units the phosphate value is reported in, as the laboratory recorded + them. + + nitrite: + title: Nitrite + description: >- + Dissolved nitrite concentration, an intermediate stage in the + breakdown of nitrogen compounds. + + nitrite_units: + title: Nitrite units + description: >- + Units the nitrite value is reported in, as the laboratory recorded + them. + + nitrate: + title: Nitrate + description: >- + Dissolved nitrate concentration, usually from fertiliser, septic + systems, or livestock. The drinking-water limit is 10 mg/L as + nitrogen. + + nitrate_units: + title: Nitrate units + description: >- + Units the nitrate value is reported in, as the laboratory recorded + them. + + nitrate_as_n: + title: Nitrate as nitrogen + description: >- + Nitrate concentration expressed as the mass of nitrogen alone, which + is how the 10 mg/L drinking-water limit is written. Roughly a + quarter of the same sample reported as nitrate. + + nitrate_as_n_units: + title: Nitrate as nitrogen units + description: >- + Units the nitrate as nitrogen value is reported in, as the + laboratory recorded them. + + thorium: + title: Thorium + description: >- + Dissolved thorium concentration in the most recent sample analysed + for it. + + thorium_units: + title: Thorium units + description: >- + Units the thorium value is reported in, as the laboratory recorded + them. + + thorium_total: + title: Thorium (total) + description: >- + Total thorium concentration -- the unfiltered determination, which + counts thorium bound to suspended particles as well as the dissolved + fraction. + + thorium_total_units: + title: Thorium (total) units + description: >- + Units the thorium (total) value is reported in, as the laboratory + recorded them. + + tin: + title: Tin + description: >- + Dissolved tin concentration in the most recent sample analysed for + it. + + tin_units: + title: Tin units + description: >- + Units the tin value is reported in, as the laboratory recorded them. + + tin_total: + title: Tin (total) + description: >- + Total tin concentration -- the unfiltered determination, which + counts tin bound to suspended particles as well as the dissolved + fraction. + + tin_total_units: + title: Tin (total) units + description: >- + Units the tin (total) value is reported in, as the laboratory + recorded them. + + mercury: + title: Mercury + description: >- + Dissolved mercury concentration in the most recent sample analysed + for it. + + mercury_units: + title: Mercury units + description: >- + Units the mercury value is reported in, as the laboratory recorded + them. + + mercury_total: + title: Mercury (total) + description: >- + Total mercury concentration -- the unfiltered determination, which + counts mercury bound to suspended particles as well as the dissolved + fraction. + + mercury_total_units: + title: Mercury (total) units + description: >- + Units the mercury (total) value is reported in, as the laboratory + recorded them. + + titanium: + title: Titanium + description: >- + Dissolved titanium concentration in the most recent sample analysed + for it. + + titanium_units: + title: Titanium units + description: >- + Units the titanium value is reported in, as the laboratory recorded + them. + + titanium_total: + title: Titanium (total) + description: >- + Total titanium concentration -- the unfiltered determination, which + counts titanium bound to suspended particles as well as the + dissolved fraction. + + titanium_total_units: + title: Titanium (total) units + description: >- + Units the titanium (total) value is reported in, as the laboratory + recorded them. + + +# EDR collections. Keys here are parameter names read out of the data, not +# column names -- ogc_waterlevels stamps a single literal, while +# ogc_water_chemistry carries the analyte text exactly as the laboratory +# recorded it, so most chemistry parameters take a generated title. +waterlevels: + groundwater level: + title: Groundwater level + description: >- + Depth from the measuring point down to the water table, as measured by + hand during a site visit or logged automatically by a pressure + transducer left in the well. Larger values mean the water table is + further below the surface. diff --git a/core/ogc_field_metadata.py b/core/ogc_field_metadata.py new file mode 100644 index 000000000..7befabcec --- /dev/null +++ b/core/ogc_field_metadata.py @@ -0,0 +1,204 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Per-field prose for the OGC collections. + +Collection-level ``title``/``description``/``keywords`` live in +``core/pygeoapi.py`` and the two pygeoapi config templates. This module is the +level below: what an individual column means, and what unit it is in. + +The copy lives in ``core/ogc-field-descriptions.yml``, keyed by backing +relation name with the ``ogc_``/``ogc_internal_`` prefix stripped, so the +public and internal mounts share one entry per view. + +Read ``docs/ogc-field-descriptions.md`` before changing the shape of the YAML +or upgrading pygeoapi. +""" + +import json +import logging +from pathlib import Path + +import yaml + +LOGGER = logging.getLogger(__name__) + +# The prefixes _thing_collections_block and _edr_collections_block prepend to a +# collection id to reach its backing relation. Longest first: "ogc_internal_" +# also starts with "ogc_". +TABLE_PREFIXES = ("ogc_internal_", "ogc_") + +# Entries carry documentation, not schema. Types and formats stay with the +# provider's own reflection. +ALLOWED_KEYS = frozenset( + { + "title", + "description", + "x-ogc-unit", + "x-ogc-unitLang", + "x-ogc-propertySeq", + # JSON Schema's own keyword. pygeoapi's HTML renders it as the schema + # table's "Values" column, and its queryables handler emits it too. + "enum", + # Names a category in core/lexicon.json, expanded to `enum` on the way + # out so a controlled vocabulary is not duplicated here. + "enum-lexicon", + } +) + +DEFAULTS_KEY = "_defaults" + +LEXICON_KEY = "enum-lexicon" + +_CACHE = None +_LEXICON_CACHE = None + + +def _metadata_path() -> Path: + return Path(__file__).resolve().parent / "ogc-field-descriptions.yml" + + +def _lexicon_path() -> Path: + return Path(__file__).resolve().parent / "lexicon.json" + + +def lexicon_terms(category: str) -> list: + """Terms in one core/lexicon.json category, in file order. + + The lexicon file seeds the database's controlled vocabularies, so reading + it here keeps one source of truth for an enumerated column's valid values + -- and keeps this module free of any database dependency. + """ + global _LEXICON_CACHE + if _LEXICON_CACHE is None: + raw = json.loads(_lexicon_path().read_text(encoding="utf-8")) + by_category: dict[str, list] = {} + for term in raw.get("terms", []): + for name in term.get("categories", []): + by_category.setdefault(name, []).append(term["term"]) + _LEXICON_CACHE = by_category + return list(_LEXICON_CACHE.get(category, [])) + + +def _validate(raw: dict, path: Path) -> dict: + if not isinstance(raw, dict): + raise ValueError(f"{path} must contain a mapping of table -> fields.") + + for table, fields in raw.items(): + if not isinstance(fields, dict): + raise ValueError(f"{path}: {table} must be a mapping of field -> entry.") + for field, entry in fields.items(): + if not isinstance(entry, dict): + raise ValueError( + f"{path}: {table}.{field} must be a mapping, got {type(entry).__name__}." + ) + if not entry.get("title"): + raise ValueError(f"{path}: {table}.{field} is missing a title.") + unknown = set(entry) - ALLOWED_KEYS + if unknown: + raise ValueError( + f"{path}: {table}.{field} has unsupported keys " + f"{sorted(unknown)}; allowed keys are {sorted(ALLOWED_KEYS)}." + ) + values = entry.get("enum") + if values is not None and (not isinstance(values, list) or not values): + raise ValueError( + f"{path}: {table}.{field} enum must be a non-empty list." + ) + category = entry.get(LEXICON_KEY) + if category is not None and not lexicon_terms(category): + raise ValueError( + f"{path}: {table}.{field} names lexicon category " + f"{category!r}, which has no terms in core/lexicon.json." + ) + return raw + + +def load_field_metadata(refresh: bool = False) -> dict: + """Return the parsed YAML, read once per process. + + Deliberately free of any database dependency: this is called during + OpenAPI generation, which runs before the backing views need to exist. + """ + global _CACHE + if _CACHE is None or refresh: + path = _metadata_path() + raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + _CACHE = _validate(raw, path) + return _CACHE + + +def strip_table_prefix(table: str) -> str: + """Reduce ``ogc_water_wells``/``ogc_internal_water_wells`` to ``water_wells``.""" + for prefix in TABLE_PREFIXES: + if table.startswith(prefix): + return table[len(prefix) :] + return table + + +def default_title(column_name: str) -> str: + """Fallback title for a column with no entry: ``well_depth`` -> ``Well Depth``.""" + return column_name.replace("_", " ").strip().title() + + +def table_entries(table: str) -> dict: + """Documentation entries in force for ``table``, defaults included.""" + metadata = load_field_metadata() + entries = dict(metadata.get(DEFAULTS_KEY, {})) + entries.update(metadata.get(strip_table_prefix(table), {})) + return entries + + +def describe_fields(table: str, fields: dict) -> dict: + """Annotate a provider's reflected ``fields`` with prose from the YAML. + + Returns a new dict of new per-field dicts. That is not tidiness: + ``pygeoapi.api.get_collection_schema`` assigns the provider's own field + dict into the response and then mutates it in place (pops ``format``, + assigns ``x-ogc-role``), so handing out references into the cached YAML + would let one request's mutations leak into the next one's. + """ + entries = table_entries(table) + described = {} + undocumented = [] + + for name, field in (fields or {}).items(): + annotated = dict(field) + entry = entries.get(name) + if entry: + for key, value in entry.items(): + if key == LEXICON_KEY: + # Expanded here rather than stored, so the vocabulary stays + # defined in one place. An entry may still pin a literal + # `enum` instead when the column's values are set by the + # view's own SQL rather than by the lexicon. + annotated.setdefault("enum", lexicon_terms(value)) + continue + annotated[key] = value + else: + annotated.setdefault("title", default_title(name)) + undocumented.append(name) + described[name] = annotated + + if undocumented: + # Not fatal: a response with a generated title beats a 500. The drift + # guard in tests/test_ogc_field_descriptions.py is what fails the build. + LOGGER.warning( + "No field description for %s.%s; falling back to a generated title.", + table, + ", ".join(sorted(undocumented)), + ) + + return described diff --git a/core/permissions.py b/core/permissions.py index 952e844f4..ad3406f41 100644 --- a/core/permissions.py +++ b/core/permissions.py @@ -14,13 +14,13 @@ # limitations under the License. # =============================================================================== import os -from functools import lru_cache -from typing import Optional, List, Union, cast, Callable +import threading +import time +from typing import Optional, List, Sequence, Tuple, Union, cast, Callable import httpx from fastapi import Depends, HTTPException, status from fastapi.security import HTTPAuthorizationCredentials, OAuth2AuthorizationCodeBearer -from fastapi.security import OAuth2PasswordBearer from jose import jwt from jose.exceptions import JWTError from jwt.algorithms import RSAAlgorithm @@ -29,33 +29,133 @@ from core.settings import settings -AUTHENTIK_ISSUER = os.environ.get("AUTHENTIK_URL") ALGORITHMS = ["RS256"] -jwks = {} -auth_disabled = int(os.environ.get("AUTHENTIK_DISABLE_AUTHENTICATION", 0)) -if AUTHENTIK_ISSUER and not auth_disabled: - JWKS_URL = f"{AUTHENTIK_ISSUER}jwks/" +# The only MODE in which AUTHENTIK_DISABLE_AUTHENTICATION=1 is honored. +# assert_auth_configuration() refuses to boot anywhere else, so a box with an +# unset or mislabeled MODE can never come up with authentication switched off. +BYPASS_ALLOWED_MODE = "development" -@lru_cache(maxsize=1) -def get_jwks(): - if not AUTHENTIK_ISSUER or auth_disabled: +# How long a fetched JWKS document is trusted. Authentik rotates its signing +# keys; the cache used to be an unbounded lru_cache, so a rotation 401'd every +# request until the process was redeployed. get_public_key() also forces one +# refresh on an unrecognized `kid`, which covers rotations inside the window. +JWKS_TTL_SECONDS = int(os.environ.get("AUTHENTIK_JWKS_TTL_SECONDS", "3600")) + + +def _issuer() -> str: + """Authentik issuer URL, read lazily so it survives late load_dotenv().""" + return (os.environ.get("AUTHENTIK_URL") or "").strip() + + +def _accepted_issuers() -> Tuple[str, ...]: + """Issuer values accepted for the `iss` claim. + + Authentik's issuer is the provider URL, which operators configure with or + without a trailing slash depending on where they copied it from. Accept + both spellings rather than making token validation depend on that. + """ + issuer = _issuer() + if not issuer: + return () + return (issuer.rstrip("/"), issuer.rstrip("/") + "/") + + +def authentication_disabled() -> bool: + """Whether the development authentication bypass is switched on. + + Read fresh from the environment on every call. This used to be an + import-time snapshot (`auth_disabled`) that the per-request check in + authenticated() did not share: flipping the variable after import left + JWKS fetching disabled while token verification stayed live, so every + request failed with "Invalid signing key" instead of either enforcing or + bypassing cleanly. + """ + raw = (os.environ.get("AUTHENTIK_DISABLE_AUTHENTICATION") or "0").strip() + try: + return bool(int(raw)) + except ValueError: + return raw.lower() in {"true", "yes", "on"} + + +class AuthConfigurationError(RuntimeError): + """Raised at startup when the auth bypass is enabled outside development.""" + + +def bypass_misconfiguration_detail() -> str: + return ( + "AUTHENTIK_DISABLE_AUTHENTICATION is enabled but MODE is " + f"{settings.mode or ''!r}. The bypass is only permitted when " + f"MODE={BYPASS_ALLOWED_MODE!r}. Set " + "AUTHENTIK_DISABLE_AUTHENTICATION=0, or set MODE=development." + ) + + +def assert_auth_configuration() -> None: + """Fail fast when the app is configured to serve traffic without auth. + + Called from core.factory.create_api_app() after load_dotenv(). The old + guard was per-request and keyed on `settings.mode == "production"`, so a + deploy with MODE unset served every endpoint anonymously and logged + nothing. Two independent variables had to be right; now a wrong one stops + the process at boot. + """ + if authentication_disabled() and settings.mode != BYPASS_ALLOWED_MODE: + raise AuthConfigurationError(bypass_misconfiguration_detail()) + + +_jwks_lock = threading.Lock() +_jwks_cache: dict = {"payload": None, "fetched_at": 0.0} + + +def reset_jwks_cache() -> None: + """Drop the cached JWKS. Test hook and manual-invalidation escape hatch.""" + with _jwks_lock: + _jwks_cache["payload"] = None + _jwks_cache["fetched_at"] = 0.0 + + +def get_jwks(force_refresh: bool = False) -> dict: + if not _issuer() or authentication_disabled(): return {} - resp = httpx.get(JWKS_URL, timeout=10.0) + if not force_refresh: + with _jwks_lock: + cached = _jwks_cache["payload"] + age = time.monotonic() - _jwks_cache["fetched_at"] + if cached is not None and age < JWKS_TTL_SECONDS: + return cached + + resp = httpx.get(f"{_issuer().rstrip('/')}/jwks/", timeout=10.0) resp.raise_for_status() - return resp.json() + payload = resp.json() + with _jwks_lock: + _jwks_cache["payload"] = payload + _jwks_cache["fetched_at"] = time.monotonic() + return payload -def get_public_key(token): - unverified_header = jwt.get_unverified_header(token) - for key in get_jwks().get("keys", []): - if key["kid"] == unverified_header["kid"]: - return RSAAlgorithm.from_jwk(key) - raise HTTPException(status_code=401, detail="Invalid signing key") +def _find_signing_key(jwks: dict, kid: Optional[str]) -> Optional[dict]: + if not kid: + return None + for key in jwks.get("keys", []): + if key.get("kid") == kid: + return key + return None + + +def get_public_key(token): + kid = jwt.get_unverified_header(token).get("kid") -oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") + key = _find_signing_key(get_jwks(), kid) + if key is None: + # Unknown kid: Authentik may have rotated inside the TTL window. + # Refetch once before rejecting an otherwise valid token. + key = _find_signing_key(get_jwks(force_refresh=True), kid) + if key is None: + raise HTTPException(status_code=401, detail="Invalid signing key") + return RSAAlgorithm.from_jwk(key) TokenType = Union[str, HTTPAuthorizationCredentials] @@ -69,88 +169,123 @@ def get_public_key(token): ) +def authorize_groups( + payload: dict, + require_all: Optional[Sequence[str]] = None, + require_any: Optional[Sequence[str]] = None, +) -> bool: + """Check a decoded token's `groups` claim against a group requirement. + + `require_all` demands every listed group; `require_any` demands at least + one. Role tiers in core/dependencies.py use `require_any` so that an Admin + satisfies an Editor- or Viewer-gated route. The old check was all-of only, + which meant the documented Admin > Editor > Viewer hierarchy existed + nowhere in code -- it worked solely because operators happened to grant + overlapping groups in Authentik, and an Admin without the Viewer group got + a 403 on every read. + """ + groups = payload.get("groups") or [] + if require_all and not all(group in groups for group in require_all): + return False + if require_any and not any(group in groups for group in require_any): + return False + return True + + def authenticated( optional: bool = False, - scope: Optional[List[str]] = None, permissions: Optional[List[str]] = None, + any_of: Optional[List[str]] = None, ): + """Build a FastAPI dependency enforcing a bearer token and group membership. + + `permissions` requires every listed Authentik group, `any_of` requires at + least one. Returns the decoded token payload on success so endpoints can + read claims; returns True when the development bypass is active. + """ - def _authenicated( + def _authenticated( request: Request, response: Response, token: TokenType = Depends(cast(Callable, scheme)), ): - # def _authenicated(request: Request, response: Response): - # def _authenicated(): - """ - A placeholder for the authentication logic. - This function should check if the user is authenticated and has the required permissions. - If `optional` is True, it should allow unauthenticated access. - """ - - if int(os.environ.get("AUTHENTIK_DISABLE_AUTHENTICATION", 0)): - if settings.mode == "production": + if authentication_disabled(): + # assert_auth_configuration() already rejected this combination at + # startup; this is the belt for a variable flipped at runtime. + if settings.mode != BYPASS_ALLOWED_MODE: raise HTTPException( status_code=status.HTTP_424_FAILED_DEPENDENCY, - detail="Authentication is disabled in production mode. Set AUTHENTIK_DISABLE_AUTHENTICATION=0 to enable authentication.", + detail=bypass_misconfiguration_detail(), ) return True - if optional and not token: - return True - - # Here you would typically check the token against your authentication system - # and verify the user's permissions. - - if not token or not verify_token(token, scope, permissions): + if not token: + if optional: + return True raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Unauthorized" ) - # this is a placeholder for the actual authentication logic - return _get_token_payload(token) if token else None + # Decoded once and reused. The previous flow decoded the JWT twice per + # request: verify_token() decoded to read groups, then the caller + # decoded again to build the return value. + payload = _get_token_payload(token) - return _authenicated + if not authorize_groups(payload, require_all=permissions, require_any=any_of): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail="Forbidden" + ) + return payload -def verify_token( - token: TokenType, scope: Optional[List[str]], permissions: Optional[List[str]] -) -> bool: - """ - Placeholder function to verify the token. - This should contain the logic to check if the token is valid and has the required permissions. - """ - # Implement your token verification logic here + return _authenticated - payload = _get_token_payload(token) - # Optionally check scopes and permissions in payload - if scope: - if not all(s in payload.get("scope", []) for s in scope): - return False - if permissions: - if not all(p in payload.get("groups", []) for p in permissions): - return False - return True +def _decode(token: str) -> dict: + """Verify signature, audience, and issuer, returning the claims.""" + return jwt.decode( + token, + get_public_key(token), + algorithms=ALGORITHMS, + audience=os.environ.get("AUTHENTIK_CLIENT_ID"), # Authentik application + issuer=_accepted_issuers() or None, + ) -def _get_token_payload(token: str = Depends(oauth2_scheme)): +def _get_token_payload(token: str) -> dict: try: - public_key = get_public_key(token) - payload = jwt.decode( - token, - public_key, - algorithms=ALGORITHMS, - audience=os.environ.get( - "AUTHENTIK_CLIENT_ID" - ), # Must match Authentik application - ) - return payload - except JWTError as e: + return _decode(token) + except JWTError: raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Could not validate credentials", ) +class TokenInvalid(Exception): + """Raised by decode_token_payload() for any JWT verification failure. + + Not an HTTPException: this is called from raw ASGI middleware + (core/internal_ogc_auth.py), which has no FastAPI exception handler + watching, so raising HTTPException there would just be an unhandled + exception rather than the intended response. + """ + + +# Required Authentik group for the authenticated internal OGC mount +# (/ogcapi-internal). Not Depends()-shaped like the roles above -- see the +# cross-reference note in core/dependencies.py for why it still lives here. +INTERNAL_OGC_GROUP = "OGCInternal" + + +def decode_token_payload(token: str) -> dict: + """Same JWT verification as _get_token_payload (shared _decode helper), + but raises TokenInvalid instead of HTTPException(401). + """ + try: + return _decode(token) + except (JWTError, HTTPException) as e: + raise TokenInvalid(str(e)) from e + + # ============= EOF ============================================= diff --git a/core/pygeoapi-config-internal.yml b/core/pygeoapi-config-internal.yml new file mode 100644 index 000000000..e62723e0c --- /dev/null +++ b/core/pygeoapi-config-internal.yml @@ -0,0 +1,463 @@ +server: + bind: + host: 0.0.0.0 + port: 8000 + url: {server_url} + mimetype: application/json; charset=UTF-8 + encoding: utf-8 + language: en-US + limits: + default_items: 10 + max_items: 10000 + map: + url: https://tile.openstreetmap.org/{{z}}/{{x}}/{{y}}.png + attribution: "© OpenStreetMap contributors" + +logging: + level: INFO + +metadata: + identification: + title: Ocotillo OGC API (Internal) + description: >- + Authenticated internal OGC API - Features backed by PostGIS and + pygeoapi. Unlike the public /ogcapi mount, these collections are not + filtered by release_status and include private and draft records. + Provided without warranty - see the terms of service for data + limitations. + keywords: [features, ogcapi, postgis, pygeoapi, internal] + terms_of_service: {terms_of_service_url} + url: https://ocotillo.newmexicowaterdata.org + license: + name: CC-BY 4.0 + url: https://creativecommons.org/licenses/by/4.0/ + provider: + name: NMBGMR + url: https://geoinfo.nmt.edu + # pygeoapi builds OpenAPI info.contact from `provider`, not `contact` + # (pygeoapi/openapi.py gen_contact), so info.contact.email is empty + # without this line. + email: ocotillo-nmbg@nmt.edu + contact: + name: Ocotillo Support, NMBGMR + email: ocotillo-nmbg@nmt.edu + # No `role:` here. pygeoapi 0.23.5 writes contact.role into + # x-ogc-serviceContact.hoursOfService (pygeoapi/openapi.py, gen_contact -- + # the line is a copy-paste of the `hours` branch above it), so setting it + # publishes "pointOfContact" as the service's hours of operation. + +resources: + locations: + type: collection + title: Locations + description: >- + The raw geographic location records that every monitoring point hangs + off -- one feature per surveyed site, with its elevation, county, + quadrangle, and the notes recorded about how the coordinates were + obtained and how reliable they are. Most consumers want a feature-type + layer such as water_wells instead, which pairs the same coordinates + with what is actually monitored there; this layer is kept for staff + work that needs the location record itself. + keywords: [locations, sites, coordinates, elevation, county, data-reliability] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_locations + geom_field: point + + latest_depth_to_water_wells: + type: collection + title: Latest Depth to Water (Water Wells) + description: >- + The most recent depth-to-water reading for each well, measured below + ground surface -- the measured depth minus the height of the measuring + point above ground, with readings that have no recorded + measuring-point height treated as taken at ground level. + water_well_summary publishes the same latest reading alongside the + count, range and trend of the whole record, so this layer is kept only + for staff clients that already depend on its narrower shape. + keywords: [water-wells, groundwater-level, depth-to-water, latest-value, below-ground-surface] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_latest_depth_to_water_wells + geom_field: point + + avg_tds_wells: + type: collection + title: Average TDS (Water Wells) + description: >- + The arithmetic mean of all total dissolved solids (TDS) results on + record for each water well. Treat with care: across the catalog the + average rests on about 1.9 analyses per well, so for many wells it is + a mean of one or two samples taken years apart and is not a reliable + summary of the well's water quality. Prefer latest_tds_wells, which + reports a single dated result. + keywords: [ + water-wells, chemistry, tds, total-dissolved-solids, average, + low-sample-count, use-with-caution + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_avg_tds_wells + geom_field: point + + latest_tds_wells: + type: collection + title: Latest TDS (Water Wells) + description: >- + Total dissolved solids (TDS) measures how much mineral matter is + dissolved in the water -- in plain terms, how salty it is. This layer + reads every laboratory major-chemistry analysis on record for each + water well, keeps only the TDS results, and publishes the single most + recent one per well, dated by its analysis date or, where that is + missing, by the date the sample was collected. Use it for a current + statewide picture of groundwater salinity without working through each + well's full analysis history. + keywords: [ + water-wells, water-quality, chemistry, tds, total-dissolved-solids, + salinity, latest-value, groundwater + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_latest_tds_wells + geom_field: point + + depth_to_water_trend_wells: + type: collection + title: Depth to Water Trend (Water Wells) + description: >- + Shows whether the water table beneath each well has been falling, + rising, or holding steady. Every manual groundwater-level measurement + for the well is converted to a depth below ground surface -- the + measured depth minus the height of the measuring point above ground, + with readings that have no recorded measuring-point height treated as + taken at ground level -- and a straight line is fitted through those + depths over time. The slope of that line in feet per year is reported + as increasing (water table falling faster than 0.25 ft/yr), decreasing + (rising faster than 0.25 ft/yr), or stable. Wells with fewer than 10 + measurements, or fewer than 4 spanning less than two years, are + labelled "not enough data" rather than given a trend the record cannot + support. + keywords: [ + water-wells, groundwater-level, depth-to-water, trend, slope, + feet-per-year, declining-water-levels, aquifer-condition + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_depth_to_water_trend_wells + geom_field: point + + water_elevation_wells: + type: collection + title: Water Elevation (Water Wells) + description: >- + Gives the height of the water table above sea level at each well, so + that levels can be compared between wells standing at different ground + elevations. The most recent groundwater-level measurement is converted + to feet, the height of the measuring point above ground is subtracted + to give the depth below ground surface (readings with no recorded + measuring-point height are treated as taken at ground level), and that + depth is subtracted from the surveyed ground-surface elevation at the + well. Use it to map the shape of the water table or to work out which + way groundwater is flowing. + keywords: [ + water-wells, groundwater-level, water-table-elevation, water-elevation, + depth-to-water, above-sea-level, groundwater-flow + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_water_elevation_wells + geom_field: point + + water_well_summary: + type: collection + title: Water Well Summary + description: >- + One row per water well, condensing that well's entire manual + groundwater-level record into a few numbers: how many measurements + exist, the most recent one and its date, the shallowest and deepest + ever recorded, and the long-term trend as a straight-line slope in + feet per year. Depths are below ground surface -- the measured depth + minus the height of the measuring point above ground, with readings + that have no recorded measuring-point height treated as taken at + ground level. Each row also carries the well's depth, its surveyed + ground elevation and how that elevation was determined, and the + geologic zone the well is completed in. Wells with no water-level + measurements at all are left out. Use it as the at-a-glance record for + a well before digging into individual readings. + keywords: [ + water-wells, summary, groundwater-level, water-level-history, trend, + well-depth, elevation, at-a-glance + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_water_well_summary + geom_field: point + + well_water_column: + type: collection + title: Well Water Column (Water Wells) + description: >- + One row per water well, reporting how much standing water the well + holds: the well's depth minus its depth to water, in feet, worked out + four ways -- from the most recent reading, from the average of every + reading, from the shallowest water level on record (the fullest the + well has been) and from the deepest (the emptiest). Depths to water are + manual readings below ground surface -- the measured depth minus the + height of the measuring point above ground, with readings that have no + recorded measuring-point height treated as taken at ground level. + Continuous logger readings are not counted. A reading deeper than the + recorded well depth would give a negative column and is reported as + zero instead; water_well_summary publishes the raw shallowest and + deepest readings beside the well depth if you need to see that + contradiction. Wells with no depth on record, or no usable reading, are + left out. Each row also carries the well's construction record and + surveyed ground elevation. Use it to judge remaining water column and + how far it has swung over the well's history. + keywords: [ + water-wells, water-column, groundwater-level, well-depth, + depth-to-water, saturated-thickness, drawdown + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_well_water_column + geom_field: point + + major_chemistry_results: + type: collection + title: Major Chemistry (Water Wells) + description: >- + The major dissolved constituents that make up most of the chemistry of + groundwater -- calcium, magnesium, sodium, potassium, bicarbonate, + carbonate, sulfate and chloride -- alongside TDS, pH, hardness, + alkalinity and specific conductance. Laboratory records name the same + analyte in many different ways, so this layer first maps those names + and symbols onto one canonical set, then keeps the most recent result + for each analyte at each well and lays the values out as fixed + columns, each with its own units column. Analytes at one well may come + from different sampling dates; the reported chemistry date is the most + recent among them. Use it to compare water chemistry between wells or + to screen against drinking-water standards. + keywords: [ + water-wells, water-quality, chemistry, major-ions, analytes, calcium, + sodium, chloride, sulfate, ph, hardness + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_major_chemistry_results + geom_field: point + + minor_chemistry_wells: + type: collection + title: Minor Chemistry (Water Wells) + description: >- + Trace elements and isotopes measured in groundwater -- arsenic, + uranium, lead, iron, manganese, boron, lithium and dozens more, plus + the stable isotopes and carbon-14 used to work out how long water has + been underground. Built the same way as the major chemistry layer: + legacy laboratory records are mapped onto one canonical analyte set, + the most recent result for each analyte at each well is kept, and the + values are laid out as fixed columns each with its own units column. + Use it for contaminant screening and for questions about the age and + origin of groundwater. + keywords: [ + water-wells, water-quality, chemistry, trace-elements, minor-chemistry, + isotopes, arsenic, uranium, carbon-14, contaminants + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_minor_chemistry_wells + geom_field: point + + actively_monitored_wells: + type: collection + title: Actively Monitored Wells + description: >- + The wells being measured today, rather than every well ever recorded. + A well appears here when its most recent monitoring-status entry reads + "Currently monitored", whichever monitoring group it belongs to; the + summary statistics attached to each one are the same water-level + figures published in water_well_summary. A well belonging to several + groups still appears once, with every membership listed in group_ids, + group_names and group_types. Use it to see the live monitoring network + -- where measurements are still being collected, and where coverage is + thin. + keywords: [ + water-wells, monitoring, actively-monitored, monitoring-network, + monitoring-status, groundwater-level + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_actively_monitored_wells + geom_field: point + + project_areas: + type: collection + title: Project Areas + description: >- + The study-area boundaries of Bureau projects, as polygons. Any project + group that has a mapped boundary is published here with its name and + description. Use it to see which part of New Mexico a project covers, + or to clip the other layers to a project's footprint. + keywords: [project-areas, study-areas, boundaries, polygons, projects, groups] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_project_areas + geom_field: project_area + +{thing_collections_block} diff --git a/core/pygeoapi-config.yml b/core/pygeoapi-config.yml index 45f3bac17..0348aa048 100644 --- a/core/pygeoapi-config.yml +++ b/core/pygeoapi-config.yml @@ -19,79 +19,59 @@ logging: metadata: identification: title: Ocotillo OGC API - description: OGC API - Features backed by PostGIS and pygeoapi - keywords: [features, ogcapi, postgis, pygeoapi] - terms_of_service: https://example.com/terms - url: https://example.com + # The disclaimer pointer is repeated here because the JSON landing page + # carries only title/description/links -- terms_of_service below reaches + # the HTML landing page and the OpenAPI document, but not JSON clients. + description: >- + OGC API - Features service publishing New Mexico Bureau of Geology and + Mineral Resources groundwater, geochemistry, and monitoring-location + data. Provided without warranty - see the terms of service for data + limitations. + keywords: [features, ogcapi, postgis, pygeoapi, groundwater, new mexico] + terms_of_service: {terms_of_service_url} + url: https://ocotillo.newmexicowaterdata.org license: name: CC-BY 4.0 url: https://creativecommons.org/licenses/by/4.0/ provider: name: NMBGMR url: https://geoinfo.nmt.edu + # pygeoapi builds OpenAPI info.contact from `provider`, not `contact` + # (pygeoapi/openapi.py gen_contact), so info.contact.email is empty + # without this line. + email: ocotillo-nmbg@nmt.edu contact: - name: API Support - email: support@example.com + name: Ocotillo Support, NMBGMR + email: ocotillo-nmbg@nmt.edu + # No `role:` here. pygeoapi 0.23.5 writes contact.role into + # x-ogc-serviceContact.hoursOfService (pygeoapi/openapi.py, gen_contact -- + # the line is a copy-paste of the `hours` branch above it), so setting it + # publishes "pointOfContact" as the service's hours of operation. resources: - locations: - type: collection - title: Locations - description: Geographic locations and site coordinates used by Ocotillo features. - keywords: [locations] - extents: - spatial: - bbox: [-109.05, 31.33, -103.00, 37.00] - crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 - providers: - - type: feature - name: PostgreSQL - data: - host: {postgres_host} - port: {postgres_port} - dbname: {postgres_db} - user: {postgres_user} - password: {postgres_password_env} - search_path: [public] - id_field: id - table: location - geom_field: point - - latest_depth_to_water_wells: - type: collection - title: Latest Depth to Water (Water Wells) - description: Most recent depth-to-water below ground surface observation for each water well. - keywords: [water-wells, groundwater-level, depth-to-water-bgs, latest] - extents: - spatial: - bbox: [-109.05, 31.33, -103.00, 37.00] - crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 - providers: - - type: feature - name: PostgreSQL - data: - host: {postgres_host} - port: {postgres_port} - dbname: {postgres_db} - user: {postgres_user} - password: {postgres_password_env} - search_path: [public] - id_field: id - table: ogc_latest_depth_to_water_wells - geom_field: point - - avg_tds_wells: + latest_tds_wells: type: collection - title: Average TDS (Water Wells) - description: Average total dissolved solids (TDS) from major chemistry results for each water well. - keywords: [water-wells, chemistry, tds, total-dissolved-solids, average] + title: Latest TDS (Water Wells) + description: >- + Total dissolved solids (TDS) measures how much mineral matter is + dissolved in the water -- in plain terms, how salty it is. This layer + reads every laboratory major-chemistry analysis on record for each + water well, keeps only the TDS results, and publishes the single most + recent one per well, dated by its analysis date or, where that is + missing, by the date the sample was collected. Use it for a current + statewide picture of groundwater salinity without working through each + well's full analysis history. + keywords: [ + water-wells, water-quality, chemistry, tds, total-dissolved-solids, + salinity, latest-value, groundwater + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -100,21 +80,36 @@ resources: password: {postgres_password_env} search_path: [public] id_field: id - table: ogc_avg_tds_wells + table: ogc_latest_tds_wells geom_field: point - latest_tds_wells: + depth_to_water_trend_wells: type: collection - title: Latest TDS (Water Wells) - description: Most recent total dissolved solids (TDS) result from major chemistry for each water well. - keywords: [water-wells, chemistry, tds, total-dissolved-solids, latest] + title: Depth to Water Trend (Water Wells) + description: >- + Shows whether the water table beneath each well has been falling, + rising, or holding steady. Every manual groundwater-level measurement + for the well is converted to a depth below ground surface -- the + measured depth minus the height of the measuring point above ground, + with readings that have no recorded measuring-point height treated as + taken at ground level -- and a straight line is fitted through those + depths over time. The slope of that line in feet per year is reported + as increasing (water table falling faster than 0.25 ft/yr), decreasing + (rising faster than 0.25 ft/yr), or stable. Wells with fewer than 10 + measurements, or fewer than 4 spanning less than two years, are + labelled "not enough data" rather than given a trend the record cannot + support. + keywords: [ + water-wells, groundwater-level, depth-to-water, trend, slope, + feet-per-year, declining-water-levels, aquifer-condition + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -123,21 +118,33 @@ resources: password: {postgres_password_env} search_path: [public] id_field: id - table: ogc_latest_tds_wells + table: ogc_depth_to_water_trend_wells geom_field: point - depth_to_water_trend_wells: + water_elevation_wells: type: collection - title: Depth to Water Trend (Water Wells) - description: Trend classification for depth to water based on slope in feet per year. - keywords: [water-wells, groundwater-level, depth-to-water, trend, slope] + title: Water Elevation (Water Wells) + description: >- + Gives the height of the water table above sea level at each well, so + that levels can be compared between wells standing at different ground + elevations. The most recent groundwater-level measurement is converted + to feet, the height of the measuring point above ground is subtracted + to give the depth below ground surface (readings with no recorded + measuring-point height are treated as taken at ground level), and that + depth is subtracted from the surveyed ground-surface elevation at the + well. Use it to map the shape of the water table or to work out which + way groundwater is flowing. + keywords: [ + water-wells, groundwater-level, water-table-elevation, water-elevation, + depth-to-water, above-sea-level, groundwater-flow + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -146,21 +153,36 @@ resources: password: {postgres_password_env} search_path: [public] id_field: id - table: ogc_depth_to_water_trend_wells + table: ogc_water_elevation_wells geom_field: point - water_elevation_wells: + water_well_summary: type: collection - title: Water Elevation (Water Wells) - description: Most recent water elevation per well calculated as elevation minus depth to water below ground surface. - keywords: [water-wells, groundwater-level, water-elevation, depth-to-water] + title: Water Well Summary + description: >- + One row per water well, condensing that well's entire manual + groundwater-level record into a few numbers: how many measurements + exist, the most recent one and its date, the shallowest and deepest + ever recorded, and the long-term trend as a straight-line slope in + feet per year. Depths are below ground surface -- the measured depth + minus the height of the measuring point above ground, with readings + that have no recorded measuring-point height treated as taken at + ground level. Each row also carries the well's depth, its surveyed + ground elevation and how that elevation was determined, and the + geologic zone the well is completed in. Wells with no water-level + measurements at all are left out. Use it as the at-a-glance record for + a well before digging into individual readings. + keywords: [ + water-wells, summary, groundwater-level, water-level-history, trend, + well-depth, elevation, at-a-glance + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -169,21 +191,40 @@ resources: password: {postgres_password_env} search_path: [public] id_field: id - table: ogc_water_elevation_wells + table: ogc_water_well_summary geom_field: point - water_well_summary: + well_water_column: type: collection - title: Water Well Summary - description: Summary metrics per water well, including latest, min/max, and trend for water levels. - keywords: [water-wells, summary, groundwater-level, trend] + title: Well Water Column (Water Wells) + description: >- + One row per water well, reporting how much standing water the well + holds: the well's depth minus its depth to water, in feet, worked out + four ways -- from the most recent reading, from the average of every + reading, from the shallowest water level on record (the fullest the + well has been) and from the deepest (the emptiest). Depths to water are + manual readings below ground surface -- the measured depth minus the + height of the measuring point above ground, with readings that have no + recorded measuring-point height treated as taken at ground level. + Continuous logger readings are not counted. A reading deeper than the + recorded well depth would give a negative column and is reported as + zero instead; water_well_summary publishes the raw shallowest and + deepest readings beside the well depth if you need to see that + contradiction. Wells with no depth on record, or no usable reading, are + left out. Each row also carries the well's construction record and + surveyed ground elevation. Use it to judge remaining water column and + how far it has swung over the well's history. + keywords: [ + water-wells, water-column, groundwater-level, well-depth, + depth-to-water, saturated-thickness, drawdown + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -192,21 +233,35 @@ resources: password: {postgres_password_env} search_path: [public] id_field: id - table: ogc_water_well_summary + table: ogc_well_water_column geom_field: point major_chemistry_results: type: collection title: Major Chemistry (Water Wells) - description: Latest major chemistry analyte values for water wells, represented as static analyte columns. - keywords: [water-wells, chemistry, analytes, major-chemistry] + description: >- + The major dissolved constituents that make up most of the chemistry of + groundwater -- calcium, magnesium, sodium, potassium, bicarbonate, + carbonate, sulfate and chloride -- alongside TDS, pH, hardness, + alkalinity and specific conductance. Laboratory records name the same + analyte in many different ways, so this layer first maps those names + and symbols onto one canonical set, then keeps the most recent result + for each analyte at each well and lays the values out as fixed + columns, each with its own units column. Analytes at one well may come + from different sampling dates; the reported chemistry date is the most + recent among them. Use it to compare water chemistry between wells or + to screen against drinking-water standards. + keywords: [ + water-wells, water-quality, chemistry, major-ions, analytes, calcium, + sodium, chloride, sulfate, ph, hardness + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -221,15 +276,27 @@ resources: minor_chemistry_wells: type: collection title: Minor Chemistry (Water Wells) - description: Latest minor/trace chemistry analyte values for water wells, represented as static analyte columns. - keywords: [water-wells, chemistry, analytes, minor-chemistry, trace-chemistry] + description: >- + Trace elements and isotopes measured in groundwater -- arsenic, + uranium, lead, iron, manganese, boron, lithium and dozens more, plus + the stable isotopes and carbon-14 used to work out how long water has + been underground. Built the same way as the major chemistry layer: + legacy laboratory records are mapped onto one canonical analyte set, + the most recent result for each analyte at each well is kept, and the + values are laid out as fixed columns each with its own units column. + Use it for contaminant screening and for questions about the age and + origin of groundwater. + keywords: [ + water-wells, water-quality, chemistry, trace-elements, minor-chemistry, + isotopes, arsenic, uranium, carbon-14, contaminants + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -244,15 +311,27 @@ resources: actively_monitored_wells: type: collection title: Actively Monitored Wells - description: Wells in the collaborative network currently flagged as actively monitored. - keywords: [water-wells, monitoring, collaborative-network, actively-monitored] + description: >- + The wells being measured today, rather than every well ever recorded. + A well appears here when its most recent monitoring-status entry reads + "Currently monitored", whichever monitoring group it belongs to; the + summary statistics attached to each one are the same water-level + figures published in water_well_summary. A well belonging to several + groups still appears once, with every membership listed in group_ids, + group_names and group_types. Use it to see the live monitoring network + -- where measurements are still being collected, and where coverage is + thin. + keywords: [ + water-wells, monitoring, actively-monitored, monitoring-network, + monitoring-status, groundwater-level + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -267,15 +346,19 @@ resources: project_areas: type: collection title: Project Areas - description: Project groups with polygon project-area boundaries. - keywords: [project-areas, groups, boundaries] + description: >- + The study-area boundaries of Bureau projects, as polygons. Any project + group that has a mapped boundary is published here with its name and + description. Use it to see which part of New Mexico a project covers, + or to clip the other layers to a project's footprint. + keywords: [project-areas, study-areas, boundaries, polygons, projects, groups] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -292,15 +375,28 @@ resources: geothermal_wells_bht: type: collection title: Geothermal Wells — Bottom-Hole Temperature - description: Geothermal wells with bottom-hole temperature (BHT) measurements from the NM_Wells database. - keywords: [geothermal, wells, bottom-hole-temperature, bht] + description: >- + Bottom-hole temperature (BHT) is the temperature at the deepest point + of a borehole, usually recorded while drilling, and is the cheapest + broad indicator of how hot the subsurface is. This layer rolls every + BHT reading for a well in the legacy NM_Wells oil, gas and geothermal + records up into a single point: how many readings exist, the hottest + and coolest, the depth of the deepest, and those temperatures + converted to degrees Celsius. Source records mix Fahrenheit and + Celsius, so each well also carries a flag when its readings arrived in + mixed units and a count of any that could not be converted. Use it to + find warm areas worth closer investigation. + keywords: [ + geothermal, bottom-hole-temperature, bht, subsurface-temperature, wells, + nm-wells, celsius, heat-resource + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -315,15 +411,27 @@ resources: geothermal_wells_temperature_profile: type: collection title: Geothermal Wells — Temperature-Depth Profile - description: Geothermal wells with downhole temperature-vs-depth series from the NM_Wells database. - keywords: [geothermal, wells, temperature, depth, profile] + description: >- + How temperature changes with depth down a borehole, summarised as one + point per well. Every temperature-versus-depth reading logged for the + well is gathered into a single record: the number of readings, the + depth range they cover, the coolest and hottest values in degrees + Celsius, and the whole profile as a list of depth/temperature pairs. + Mixed source temperature units are flagged as they are for bottom-hole + temperatures. Use it to estimate the geothermal gradient -- how + quickly the ground warms with depth -- without pulling every + individual reading. + keywords: [ + geothermal, temperature-profile, temperature-depth, geothermal-gradient, + wells, nm-wells, celsius + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -338,15 +446,27 @@ resources: bht_measurements: type: collection title: BHT Measurements - description: Individual bottom-hole temperature measurements with well header and location data from the NM_Wells database. - keywords: [geothermal, bht, bottom-hole-temperature, measurements] + description: >- + Every individual bottom-hole temperature reading, one feature per + measurement, for consumers who need the raw record rather than the + per-well roll-up in geothermal_wells_bht. Each row carries the + temperature, the depth it was taken at, the date, and the hours since + drilling fluid was last circulated -- readings taken soon after + circulation are cooler than the rock itself, so that figure decides + whether a reading can be corrected. Well header details (operator, + well type, total depth, completion date, current status) and the + county are carried along for context. + keywords: [ + geothermal, bht, bottom-hole-temperature, measurements, raw-readings, + drilling, nm-wells + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -361,15 +481,24 @@ resources: temp_depth_measurements: type: collection title: Temperature-Depth Measurements - description: Individual downhole temperature readings with well header, location, and elevation data from the NM_Wells database. - keywords: [geothermal, temperature, depth, measurements] + description: >- + Every individual downhole temperature reading, one feature per + measurement, for consumers who need the raw record rather than the + per-well roll-up in geothermal_wells_temperature_profile. Each row + gives the temperature, the depth it was recorded at, the well it came + from and that well's elevation datum, so gradients can be recomputed + from scratch or checked against the summarised profile. + keywords: [ + geothermal, temperature, temperature-depth, downhole, measurements, + raw-readings, nm-wells + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -384,15 +513,27 @@ resources: heat_flow: type: collection title: Heat Flow - description: Summary heat-flow records with thermal conductivity, gradient, and publication attribution from the NM_Wells database. - keywords: [geothermal, heat-flow, thermal-conductivity, gradient] + description: >- + Heat flow is the rate at which the Earth's internal heat escapes + through the ground surface, and is the standard measure of geothermal + potential. Each row is one published determination over one depth + interval in one well, obtained by multiplying the temperature gradient + measured in the hole by the thermal conductivity of the rock. Values + recorded in the older heat-flow and conductivity units are republished + alongside SI equivalents (milliwatts per square metre, watts per + metre-kelvin), and every record carries the quality rating and the + literature citation it was published with. + keywords: [ + geothermal, heat-flow, thermal-conductivity, thermal-gradient, + geothermal-potential, nm-wells, publications + ] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} @@ -407,15 +548,24 @@ resources: dst: type: collection title: Drill Stem Tests - description: Drill stem test intervals with pressure, flow history, and well header data from the NM_Wells database. - keywords: [geothermal, dst, drill-stem-test, pressure, formation] + description: >- + A drill stem test is a temporary completion run while a well is still + being drilled: the drill pipe is opened against a chosen depth + interval so that formation fluid can flow in, and the pressures and + flow behaviour are recorded. Each row here is one tested interval -- + its depth range, target formation, packer settings, choke sizes and + gauge depth -- with the sequence of operations logged during the test + joined together in order as its flow history. Use it for formation + pressure and fluid evidence in wells that were never completed for + production. + keywords: [drill-stem-test, dst, formation-pressure, flow-history, reservoir, wells, nm-wells] extents: spatial: bbox: [-109.05, 31.33, -103.00, 37.00] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 providers: - type: feature - name: PostgreSQL + name: core.feature_provider.DescribedPostgreSQLProvider data: host: {postgres_host} port: {postgres_port} diff --git a/core/pygeoapi.py b/core/pygeoapi.py index 0cd69672a..392c2224a 100644 --- a/core/pygeoapi.py +++ b/core/pygeoapi.py @@ -1,105 +1,247 @@ -import importlib +import importlib.util import os import re import sys import textwrap from importlib.util import find_spec from pathlib import Path +from urllib.parse import urlparse import yaml from fastapi import FastAPI +from core.pygeoapi_patches import apply_queryables_patch + +# Consumed by pygeoapi at import time only; see _load_pygeoapi_app. +_PYGEOAPI_ENV_KEYS = ("PYGEOAPI_CONFIG", "PYGEOAPI_OPENAPI") + THING_COLLECTIONS = [ { "id": "water_wells", "title": "Water Wells", "thing_type": "water well", "description": ( - "Groundwater wells used for monitoring, production, and " - "hydrogeologic investigations." + "Groundwater wells: drilled or dug access points into an aquifer, " + "used for monitoring, production, and hydrogeologic investigation. " + "Each feature is one well from the monitoring-point register, placed " + "at the most recent location recorded for it, and carries the " + "construction details held for it -- total and hole depth, casing " + "diameter and depth, completion date, driller, construction method, " + "pump type and depth, and the geologic formation it is completed in. " + "This is the starting point for groundwater work: the water-level and " + "chemistry layers are all derived from these same wells." ), - "keywords": ["well", "groundwater", "water-well"], + "keywords": [ + "water-wells", + "wells", + "groundwater", + "aquifer", + "monitoring-points", + "well-construction", + ], }, { "id": "springs", "title": "Springs", "thing_type": "spring", "description": ( - "Natural spring features and associated spring monitoring points." + "Springs: places where groundwater reaches the land surface under its " + "own pressure, without pumping. Each feature is one spring from the " + "monitoring-point register, placed at the most recent location " + "recorded for it. Use it to map natural groundwater discharge, the " + "groundwater contribution to streamflow, and the water sources that " + "support desert ecosystems." ), - "keywords": ["springs", "groundwater-discharge"], + "keywords": [ + "springs", + "groundwater-discharge", + "monitoring-points", + "surface-water", + "seeps", + ], }, { "id": "diversions_surface_water", "title": "Surface Water Diversions", "thing_type": "diversion of surface water, etc.", "description": ( - "Diversion structures such as ditches, canals, and intake points." + "Surface-water diversions: structures that take water out of a " + "stream, river, or canal -- ditches, acequias, headgates and intakes. " + "Each feature is one diversion from the monitoring-point register, " + "placed at the most recent location recorded for it. Use it to see " + "where surface water is withdrawn and to pair those points with " + "downstream flow records." ), - "keywords": ["surface-water", "diversion"], + "keywords": [ + "surface-water", + "diversion", + "ditches", + "acequias", + "headgates", + "monitoring-points", + ], }, { "id": "ephemeral_streams", "title": "Ephemeral Streams", "thing_type": "ephemeral stream", "description": ( - "Stream reaches that flow only in direct response to " - "precipitation events." + "Ephemeral stream reaches: channels that carry water only in direct " + "response to rain or snowmelt and are dry the rest of the year. Each " + "feature is one monitored reach from the register, placed at the most " + "recent location recorded for it. Use it for flash-flow and " + "storm-response work, and to distinguish these channels from reaches " + "that flow year-round." ), - "keywords": ["ephemeral-stream", "surface-water"], + "keywords": [ + "ephemeral-stream", + "surface-water", + "intermittent-flow", + "storm-response", + "monitoring-points", + ], }, { "id": "lakes_ponds_reservoirs", "title": "Lakes, Ponds, and Reservoirs", "thing_type": "lake, pond or reservoir", - "description": "Surface-water bodies monitored as feature locations.", - "keywords": ["lake", "pond", "reservoir", "surface-water"], + "description": ( + "Standing bodies of surface water monitored as sites -- natural " + "lakes, ponds, and built reservoirs. Each feature is one water body " + "from the monitoring-point register, placed at the most recent " + "location recorded for it. Use it for storage and surface-water " + "quality work, and as context for nearby groundwater levels." + ), + "keywords": [ + "lake", + "pond", + "reservoir", + "surface-water", + "storage", + "monitoring-points", + ], }, { "id": "meteorological_stations", "title": "Meteorological Stations", "thing_type": "meteorological station", - "description": "Weather and climate monitoring station locations.", - "keywords": ["meteorological-station", "weather"], + "description": ( + "Weather and climate stations: sites that record conditions such as " + "precipitation, temperature, and evaporation. Each feature is one " + "station from the monitoring-point register, placed at the most " + "recent location recorded for it. Use it to relate groundwater and " + "streamflow behaviour to the weather that drives it." + ), + "keywords": [ + "meteorological-station", + "weather", + "climate", + "precipitation", + "monitoring-points", + ], }, { "id": "other_things", "title": "Other Thing Types", "thing_type": "other", "description": ( - "Feature records that do not match another defined thing type." + "Monitoring points that do not fall into any of the defined feature " + "types. Each feature is one such point from the register, placed at " + "the most recent location recorded for it. The set is small and " + "mixed, with no shared meaning between its members, so it is " + "published only on the internal mount for staff triage -- typically " + "to find records that need reclassifying." ), - "keywords": ["other"], + "keywords": [ + "other", + "unclassified", + "monitoring-points", + "internal", + "triage", + ], + # "Thing" is internal data-model vocabulary and "other" names no + # recognisable feature class, so this layer is not published on the + # public mount (BDMS-979). Staff GIS clients still reach it through + # /ogcapi-internal, and ogc_other_things is retained either way. + "internal_only": True, }, { "id": "outfalls_wastewater_return_flow", "title": "Outfalls and Return Flow", "thing_type": "outfall of wastewater or return flow", - "description": "Outfall and return-flow monitoring points.", - "keywords": ["outfall", "return-flow", "surface-water"], + "description": ( + "Outfalls and return flow: points where treated wastewater or unused " + "irrigation water re-enters a stream or channel. Each feature is one " + "outfall from the monitoring-point register, placed at the most " + "recent location recorded for it. Use it in water-quality work, where " + "these points mark deliberate inputs to a watercourse." + ), + "keywords": [ + "outfall", + "return-flow", + "wastewater", + "surface-water", + "water-quality", + "monitoring-points", + ], }, { "id": "perennial_streams", "title": "Perennial Streams", "thing_type": "perennial stream", - "description": ("Stream reaches with continuous or near-continuous flow."), - "keywords": ["perennial-stream", "surface-water"], + "description": ( + "Perennial stream reaches: channels that flow year-round in most " + "years, sustained between storms by groundwater discharge. Each " + "feature is one monitored reach from the register, placed at the most " + "recent location recorded for it. Use it for base-flow and " + "surface-water/groundwater interaction work." + ), + "keywords": [ + "perennial-stream", + "surface-water", + "base-flow", + "streamflow", + "monitoring-points", + ], }, { "id": "rock_sample_locations", "title": "Rock Sample Locations", "thing_type": "rock sample location", - "description": ("Locations where rock samples were collected or documented."), - "keywords": ["rock-sample"], + "description": ( + "Places where rock samples were collected or outcrop geology was " + "documented. Each feature is one sample location from the " + "monitoring-point register, placed at the most recent location " + "recorded for it. Use it to find where physical samples backing " + "geologic mapping and laboratory analysis came from." + ), + "keywords": [ + "rock-sample", + "geology", + "sample-location", + "outcrop", + "monitoring-points", + ], }, { "id": "soil_gas_sample_locations", "title": "Soil Gas Sample Locations", "thing_type": "soil gas sample location", "description": ( - "Locations where soil gas measurements or samples were collected." + "Places where gas held in the pore space of soil was sampled. Each " + "feature is one sample location from the monitoring-point register, " + "placed at the most recent location recorded for it. Soil gas is used " + "to detect vapours rising from buried contamination or from geologic " + "sources, so these points usually mark contamination or " + "resource-exploration surveys." ), - "keywords": ["soil-gas", "sample-location"], + "keywords": [ + "soil-gas", + "sample-location", + "vapour-survey", + "contamination", + "monitoring-points", + ], }, ] @@ -111,11 +253,25 @@ "id": "waterlevels", "title": "Water Levels", "description": ( - "Depth-to-water observations (manual readings and continuous " - "transducer time series) served as OGC API - EDR coverages. " - "Each transducer deployment is exposed as an EDR instance." + "Depth-to-water through time at each well, served as time series " + "rather than as one point per well. Two kinds of record are combined: " + "manual measurements taken by field staff during a visit, and " + "continuous records from pressure transducers left down the well, " + "which log automatically at a fixed interval. Each transducer " + "deployment is exposed as its own EDR instance, so a well's record " + "can be read deployment by deployment or as a whole. Use it to plot " + "hydrographs and to see how water levels respond to pumping, " + "recharge, and drought." ), - "keywords": ["groundwater", "water-level", "depth-to-water", "edr"], + "keywords": [ + "groundwater", + "water-level", + "depth-to-water", + "time-series", + "hydrograph", + "transducer", + "edr", + ], "table": "ogc_waterlevels", "instance_field": "deployment_id", }, @@ -123,10 +279,22 @@ "id": "water_chemistry", "title": "Water Chemistry", "description": ( - "Water-chemistry analyses keyed by analyte, served as OGC API - " - "EDR coverages." + "Water-chemistry analyses through time, one record per analyte per " + "sample, served as time series. The layer draws together the major, " + "minor and trace, and field-parameter analysis records from the " + "legacy chemistry tables, keyed by the analyte name as the laboratory " + "recorded it. Use it to follow one constituent at one site over time " + "-- the chemistry feature layers, by contrast, give the latest value " + "for every analyte at once." ), - "keywords": ["water-chemistry", "analyte", "edr"], + "keywords": [ + "water-chemistry", + "water-quality", + "analyte", + "time-series", + "laboratory-results", + "edr", + ], "table": "ogc_water_chemistry", "instance_field": None, }, @@ -137,13 +305,17 @@ def _template_path() -> Path: return Path(__file__).resolve().parent / "pygeoapi-config.yml" -def _mount_path() -> str: - # Read and sanitize the configured mount path, defaulting to "/ogcapi". - path = (os.environ.get("PYGEOAPI_MOUNT_PATH", "/ogcapi") or "").strip() +def _internal_template_path() -> Path: + return Path(__file__).resolve().parent / "pygeoapi-config-internal.yml" + + +def _sanitized_mount_path(env_var: str, default: str) -> str: + # Read and sanitize the configured mount path, falling back to `default`. + path = (os.environ.get(env_var, default) or "").strip() # Treat empty or root ("/") values as invalid and fall back to the default. if path in {"", "/"}: - path = "/ogcapi" + path = default # Ensure a single leading slash. if not path.startswith("/"): @@ -156,21 +328,27 @@ def _mount_path() -> str: # Disallow traversal/current-directory segments. segments = [segment for segment in path.split("/") if segment] if any(segment in {".", ".."} for segment in segments): - raise ValueError( - "Invalid PYGEOAPI_MOUNT_PATH: traversal segments are not allowed." - ) + raise ValueError(f"Invalid {env_var}: traversal segments are not allowed.") # Allow only slash-delimited segments of alphanumerics, underscore, # or hyphen. if not re.fullmatch(r"/[A-Za-z0-9_-]+(?:/[A-Za-z0-9_-]+)*", path): raise ValueError( - "Invalid PYGEOAPI_MOUNT_PATH: only letters, numbers, underscores, " + f"Invalid {env_var}: only letters, numbers, underscores, " "hyphens, and slashes are allowed." ) return path +def _mount_path() -> str: + return _sanitized_mount_path("PYGEOAPI_MOUNT_PATH", "/ogcapi") + + +def _internal_mount_path() -> str: + return _sanitized_mount_path("PYGEOAPI_INTERNAL_MOUNT_PATH", "/ogcapi-internal") + + def _server_url() -> str: configured = os.environ.get("PYGEOAPI_SERVER_URL") if configured: @@ -178,10 +356,50 @@ def _server_url() -> str: return f"http://localhost:8000{_mount_path()}" -def _pygeoapi_dir() -> Path: +def _internal_server_url() -> str: + configured = os.environ.get("PYGEOAPI_INTERNAL_SERVER_URL") + if configured: + return configured.rstrip("/") + # Derived from the application root rather than hardcoded to localhost. + # PYGEOAPI_INTERNAL_SERVER_URL is set in no deploy config -- not + # app.template.yaml, not any of the three CD workflows -- so every + # deployed environment fell into this branch and pygeoapi stamped + # "http://localhost:8000/ogcapi-internal" into the `self` and `next` links + # of every collection and items response. QGIS and ArcGIS Pro follow those + # links to page, so both walked off to localhost after the first page. + # _app_base_url() reads PYGEOAPI_SERVER_URL, which every deploy already + # sets, and still resolves to http://localhost:8000 for local development. + return f"{_app_base_url()}{_internal_mount_path()}" + + +def _app_base_url() -> str: + # Derived from PYGEOAPI_SERVER_URL rather than a dedicated env var: that + # variable is already set in app.template.yaml and all three CD workflows, + # and a second base-URL variable would be a fourth place to get a deploy + # wrong. PYGEOAPI_SERVER_URL points at the mount (".../ogcapi"), so strip + # the mount path back off to recover the application root. + server_url = _server_url() + mount_path = _mount_path() + if server_url.endswith(mount_path): + return server_url[: -len(mount_path)].rstrip("/") + # Deployment where the advertised OGC URL is not simply + # (a rewriting proxy, say). Scheme + netloc is the best root available. + parsed = urlparse(server_url) + if parsed.scheme and parsed.netloc: + return f"{parsed.scheme}://{parsed.netloc}" + return server_url.rstrip("/") + + +def _terms_of_service_url() -> str: + return f"{_app_base_url()}/disclaimer" + + +def _pygeoapi_dir( + runtime_dir_env: str = "PYGEOAPI_RUNTIME_DIR", default: str = "/tmp/pygeoapi" +) -> Path: # Use instance-local ephemeral storage by default (GAE-safe). - runtime_dir = (os.environ.get("PYGEOAPI_RUNTIME_DIR") or "").strip() - path = Path(runtime_dir) if runtime_dir else Path("/tmp/pygeoapi") + runtime_dir = (os.environ.get(runtime_dir_env) or "").strip() + path = Path(runtime_dir) if runtime_dir else Path(default) path.mkdir(parents=True, exist_ok=True) return path @@ -192,9 +410,13 @@ def _thing_collections_block( dbname: str, user: str, password_placeholder: str, + table_prefix: str = "ogc_", + include_internal_only: bool = False, ) -> str: resources: dict[str, dict] = {} for collection in THING_COLLECTIONS: + if collection.get("internal_only") and not include_internal_only: + continue resources[collection["id"]] = { "type": "collection", "title": collection["title"], @@ -209,7 +431,7 @@ def _thing_collections_block( "providers": [ { "type": "feature", - "name": "PostgreSQL", + "name": "core.feature_provider.DescribedPostgreSQLProvider", "data": { "host": host, "port": port, @@ -219,8 +441,9 @@ def _thing_collections_block( "search_path": ["public"], }, "id_field": "id", - "table": f"ogc_{collection['id']}", + "table": f"{table_prefix}{collection['id']}", "geom_field": "point", + "time_field": "first_visit_date", } ], } @@ -240,9 +463,15 @@ def _edr_collections_block( dbname: str, user: str, password_placeholder: str, + table_prefix: str = "ogc_", ) -> str: resources: dict[str, dict] = {} for collection in EDR_COLLECTIONS: + # EDR_COLLECTIONS' table values are hardcoded to the public + # ogc_waterlevels/ogc_water_chemistry names -- strip that literal + # "ogc_" so table_prefix (here, "ogc_internal_" for the internal + # mount) still applies, the same way _thing_collections_block does. + table_name = table_prefix + collection["table"].removeprefix("ogc_") provider = { "type": "edr", "name": "core.edr_provider.WaterEDRProvider", @@ -254,7 +483,7 @@ def _edr_collections_block( "password": password_placeholder, }, "id_field": "id", - "table": collection["table"], + "table": table_name, } if collection["instance_field"]: provider["instance_field"] = collection["instance_field"] @@ -315,41 +544,61 @@ def _pygeoapi_db_settings() -> tuple[str, str, str, str, str]: return host, port, dbname, user, "${PYGEOAPI_POSTGRES_PASSWORD}" -def _write_config(path: Path) -> None: +def _write_config( + path: Path, + *, + server_url: str, + table_prefix: str = "ogc_", + template_path: Path | None = None, + include_edr: bool = False, + include_internal_only: bool = False, +) -> None: host, port, dbname, user, password_placeholder = _pygeoapi_db_settings() - template = _template_path().read_text(encoding="utf-8") - config = template.format( - server_url=_server_url(), - postgres_host=host, - postgres_port=port, - postgres_db=dbname, - postgres_user=user, - postgres_password_env=password_placeholder, - thing_collections_block="\n".join( + template = (template_path or _template_path()).read_text(encoding="utf-8") + thing_collections_block = _thing_collections_block( + host=host, + port=port, + dbname=dbname, + user=user, + password_placeholder=password_placeholder, + table_prefix=table_prefix, + include_internal_only=include_internal_only, + ) + if include_edr: + # EDR collections (core/edr_provider.py), backed by + # ogc_waterlevels/ogc_water_chemistry (public) or + # ogc_internal_waterlevels/ogc_internal_water_chemistry (internal, + # see 2d3c3a268652_create_internal_ogc_views.py) depending on + # table_prefix. + thing_collections_block = "\n".join( [ - _thing_collections_block( - host=host, - port=port, - dbname=dbname, - user=user, - password_placeholder=password_placeholder, - ), + thing_collections_block, _edr_collections_block( host=host, port=port, dbname=dbname, user=user, password_placeholder=password_placeholder, + table_prefix=table_prefix, ), ] - ), + ) + config = template.format( + server_url=server_url, + terms_of_service_url=_terms_of_service_url(), + postgres_host=host, + postgres_port=port, + postgres_db=dbname, + postgres_user=user, + postgres_password_env=password_placeholder, + thing_collections_block=thing_collections_block, ) - # NOTE: The generated runtime config file at - # `${PYGEOAPI_RUNTIME_DIR}/pygeoapi-config.yml` (default: - # `/tmp/pygeoapi/pygeoapi-config.yml`) contains database connection details - # (host, port, dbname, user). Although the password is expected to be - # provided via environment variables at runtime by pygeoapi, this file - # should still be treated as sensitive configuration: + # NOTE: The generated runtime config file (default: + # `/tmp/pygeoapi/pygeoapi-config.yml` or + # `/tmp/pygeoapi-internal/pygeoapi-config.yml`) contains database + # connection details (host, port, dbname, user). Although the password is + # expected to be provided via environment variables at runtime by + # pygeoapi, this file should still be treated as sensitive configuration: # * Do not commit it to version control. # * Do not expose it in logs, error messages, or diagnostics. # * Ensure filesystem permissions restrict access appropriately. @@ -357,6 +606,34 @@ def _write_config(path: Path) -> None: path.chmod(0o600) +def _assert_server_settings_match( + public_config_path: Path, internal_config_path: Path +) -> None: + # pygeoapi.api.API.__init__ mutates process-wide, module-level globals + # (CHARSET, FORMAT_TYPES). Loading each mount from its own copy of + # pygeoapi.starlette_app does not help here, since both copies still + # share the one pygeoapi.api module -- whichever mount is constructed + # last wins for both. + # Inert as long as both configs agree on these settings; fail loudly at + # startup rather than let a future divergence silently corrupt responses + # on whichever mount lost the race. + public_server = yaml.safe_load(public_config_path.read_text(encoding="utf-8")).get( + "server", {} + ) + internal_server = yaml.safe_load( + internal_config_path.read_text(encoding="utf-8") + ).get("server", {}) + for key in ("encoding", "gzip"): + if public_server.get(key) != internal_server.get(key): + raise RuntimeError( + "pygeoapi public/internal config drift detected: " + f"server.{key} differs ({public_server.get(key)!r} vs " + f"{internal_server.get(key)!r}). Both configs must agree " + "here since pygeoapi.api.API.__init__ mutates shared " + "process-wide globals from these settings." + ) + + def _generate_openapi(config_path: Path, openapi_path: Path) -> None: from pygeoapi.openapi import generate_openapi_document @@ -368,12 +645,42 @@ def _generate_openapi(config_path: Path, openapi_path: Path) -> None: openapi_path.write_text(openapi, encoding="utf-8") -def _load_pygeoapi_app(): +def _load_pygeoapi_app(instance: str, config_path: Path, openapi_path: Path): + # pygeoapi.starlette_app resolves PYGEOAPI_CONFIG at import time into a + # module-level `api_`, and every route handler looks that name up in the + # module's globals at request time. importlib.reload() rebinds those + # globals *in place*, so reloading for the second mount retargets the + # handlers of the app already built for the first one -- both mounts end + # up serving whichever config was loaded last. Give each mount its own + # module object so the two sets of globals can never alias. + # Before the module is executed, so the mount's handlers resolve the + # patched queryables function. Idempotent and process-wide by nature: + # pygeoapi.api.itemtypes is one module object shared by both mounts. + apply_queryables_patch() + module_name = "pygeoapi.starlette_app" - if module_name in sys.modules: - module = importlib.reload(sys.modules[module_name]) - else: - module = importlib.import_module(module_name) + spec = find_spec(module_name) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to locate {module_name} for the {instance} mount.") + + module = importlib.util.module_from_spec(spec) + # Registered before exec_module so the module can survive importing itself. + sys.modules[f"{module_name}__ocotillo_{instance}"] = module + + previous = {key: os.environ.get(key) for key in _PYGEOAPI_ENV_KEYS} + os.environ["PYGEOAPI_CONFIG"] = str(config_path) + os.environ["PYGEOAPI_OPENAPI"] = str(openapi_path) + try: + spec.loader.exec_module(module) + finally: + # These are read only during import, so leaving the last mount's paths + # behind would silently decide the config for any later importer. + for key, value in previous.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + return module.APP @@ -389,14 +696,58 @@ def mount_pygeoapi(app: FastAPI) -> None: pygeoapi_dir = _pygeoapi_dir() config_path = pygeoapi_dir / "pygeoapi-config.yml" openapi_path = pygeoapi_dir / "pygeoapi-openapi.yml" - _write_config(config_path) + _write_config(config_path, server_url=_server_url(), include_edr=True) _generate_openapi(config_path, openapi_path) - os.environ["PYGEOAPI_CONFIG"] = str(config_path) - os.environ["PYGEOAPI_OPENAPI"] = str(openapi_path) - - pygeoapi_app = _load_pygeoapi_app() + pygeoapi_app = _load_pygeoapi_app("public", config_path, openapi_path) mount_path = _mount_path() app.mount(mount_path, pygeoapi_app) app.state.pygeoapi_mounted = True + + +def mount_pygeoapi_internal(app: FastAPI) -> None: + if getattr(app.state, "pygeoapi_internal_mounted", False): + return + if find_spec("pygeoapi") is None: + raise RuntimeError( + "pygeoapi is not installed. Rebuild/sync dependencies so " + "/ogcapi-internal can be mounted." + ) + + public_mount_path = _mount_path() + internal_mount_path = _internal_mount_path() + if internal_mount_path == public_mount_path: + # Starlette doesn't error on duplicate mount paths -- it registers + # both Mounts and matches whichever was registered first (the + # public mount), leaving the internal mount silently unreachable. + # Fail loudly at startup instead of that way blind. + raise RuntimeError( + "PYGEOAPI_MOUNT_PATH and PYGEOAPI_INTERNAL_MOUNT_PATH both " + f"resolve to {internal_mount_path!r}. They must be distinct." + ) + + internal_dir = _pygeoapi_dir( + "PYGEOAPI_INTERNAL_RUNTIME_DIR", "/tmp/pygeoapi-internal" + ) + config_path = internal_dir / "pygeoapi-config.yml" + openapi_path = internal_dir / "pygeoapi-openapi.yml" + _write_config( + config_path, + server_url=_internal_server_url(), + table_prefix="ogc_internal_", + template_path=_internal_template_path(), + include_edr=True, + include_internal_only=True, + ) + _generate_openapi(config_path, openapi_path) + _assert_server_settings_match(_pygeoapi_dir() / "pygeoapi-config.yml", config_path) + + pygeoapi_app = _load_pygeoapi_app("internal", config_path, openapi_path) + + from core.internal_ogc_auth import InternalOGCAuthMiddleware + + app.add_middleware(InternalOGCAuthMiddleware, mount_path=internal_mount_path) + app.mount(internal_mount_path, pygeoapi_app) + + app.state.pygeoapi_internal_mounted = True diff --git a/core/pygeoapi_patches.py b/core/pygeoapi_patches.py new file mode 100644 index 000000000..355137627 --- /dev/null +++ b/core/pygeoapi_patches.py @@ -0,0 +1,130 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Runtime patches over pygeoapi. + +Only one, and only because pygeoapi leaves no hook for it. +``get_collection_schema`` copies a provider's field entries into its response +wholesale, so the documentation ``DescribedPostgreSQLProvider`` attaches +reaches the client untouched. ``get_collection_queryables`` instead builds a +fresh dict per property and hardcodes ``'title': k`` -- the raw column name -- +dropping every description on the floor. + +Rather than fork the 130-line handler, this wraps it and merges the +provider's own ``title``/``description`` into the JSON it returned. The cost +is a JSON round trip on a low-traffic endpoint; the benefit is that the rest +of pygeoapi's logic (property filtering, domains, enums, roles) stays theirs. + +Read docs/ogc-field-descriptions.md before changing this, and re-check it on +any pygeoapi upgrade. +""" + +import json +import logging + +LOGGER = logging.getLogger(__name__) + +# Keys taken from the provider's field entry when the handler dropped them. +DOCUMENTATION_KEYS = ("title", "description", "x-ogc-unit", "x-ogc-unitLang", "enum") + +# Keys the handler may have filled in itself, which we must not overwrite -- +# ?profile=actual-domain asks for the live values in the data, and those beat +# our authored vocabulary. +PRESERVED_KEYS = frozenset({"enum"}) + +_QUERYABLES_PATCHED = False + + +def _documented_fields(api, dataset): + """The provider's annotated fields for ``dataset``, or ``{}``. + + Never raises: queryables must keep working for a collection whose backing + view is missing, exactly as it did before this patch. + """ + try: + from pygeoapi.plugin import load_plugin + from pygeoapi.provider import get_provider_by_type + + providers = api.config["resources"][dataset]["providers"] + # Builds a second provider for the request: the handler's own instance + # is local to it. That costs one table reflection on an endpoint that + # is queried rarely and cached downstream. + provider = load_plugin("provider", get_provider_by_type(providers, "feature")) + return provider.fields or {} + except Exception as err: # noqa: BLE001 - documentation is never fatal + LOGGER.debug("No documented fields available for %s: %s", dataset, err) + return {} + + +def _merge_documentation(payload: str, fields: dict) -> str: + document = json.loads(payload) + properties = document.get("properties") + if not isinstance(properties, dict): + return payload + + for name, prop in properties.items(): + field = fields.get(name) + if not isinstance(field, dict): + continue + for key in DOCUMENTATION_KEYS: + value = field.get(key) + if value is None: + continue + if key in PRESERVED_KEYS and prop.get(key): + continue + prop[key] = value + + return json.dumps(document, indent=4) + + +def apply_queryables_patch() -> None: + """Make /collections/{id}/queryables carry the provider's field prose. + + Idempotent, and deliberately not config-dependent: pygeoapi.api.itemtypes + is a single module object shared by both mounts, and starlette_app + resolves the handler off it per request, so patching once before either + mount is built covers both. + """ + global _QUERYABLES_PATCHED + if _QUERYABLES_PATCHED: + return + + import pygeoapi.api.itemtypes as itemtypes + + original = itemtypes.get_collection_queryables + + def get_collection_queryables(api, request, dataset=None): + headers, status, content = original(api, request, dataset) + + # Leave HTML rendering, errors, and anything unparseable alone. + if status != 200 or not isinstance(content, str): + return headers, status, content + if not headers.get("Content-Type", "").startswith("application/schema+json"): + return headers, status, content + + fields = _documented_fields(api, dataset) + if not fields: + return headers, status, content + + try: + return headers, status, _merge_documentation(content, fields) + except (ValueError, TypeError) as err: + LOGGER.warning("Could not annotate queryables for %s: %s", dataset, err) + return headers, status, content + + get_collection_queryables.__wrapped__ = original + itemtypes.get_collection_queryables = get_collection_queryables + _QUERYABLES_PATCHED = True + LOGGER.debug("Patched pygeoapi get_collection_queryables for field descriptions.") diff --git a/core/settings.py b/core/settings.py index 95ea93b68..c29c5719c 100644 --- a/core/settings.py +++ b/core/settings.py @@ -30,8 +30,18 @@ def _resolve_version() -> str: class Settings: version = _resolve_version() - def __init__(self): - self.mode = os.getenv("MODE", "") # Default mode + @property + def mode(self) -> str: + """Deployment mode, read fresh from the environment on every access. + + This used to be snapshotted in __init__. Settings() is instantiated + while core.app is imported, which happens before core.factory calls + load_dotenv() -- so whether MODE was visible depended on which module + happened to call load_dotenv() first. Reading it lazily makes the + value independent of import order, which matters because + core.permissions gates the authentication bypass on it. + """ + return os.getenv("MODE", "") def get_enum(self, name: str): if name == "MODE": diff --git a/dagster_cloud.yaml b/dagster_cloud.yaml new file mode 100644 index 000000000..342a9a4d3 --- /dev/null +++ b/dagster_cloud.yaml @@ -0,0 +1,16 @@ +# Dagster+ code locations for this repository. +# +# The API and the ingestion pipeline share a repo but not a runtime: this file +# describes only what Dagster+ builds and runs. `module_name` mirrors +# `[tool.dagster]` in pyproject.toml, so `dagster dev` locally and the Dagster+ +# agent load the same entry point. +# +# `directory` is the repository root rather than `automated_ingestion/` because +# the loader imports `db/` models and `domain/` rules -- the package is not +# self-contained by design (see docs/automated-ingestion-pipeline-plan.md). +locations: + - location_name: ocotillo-automated-ingestion + code_source: + module_name: automated_ingestion.defs.definitions + build: + directory: ./ diff --git a/dagster_cloud_post_install.sh b/dagster_cloud_post_install.sh new file mode 100755 index 000000000..09cbc6e9b --- /dev/null +++ b/dagster_cloud_post_install.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Runs inside the Dagster+ *image* build, after the repository has been copied to +# /opt/dagster/app and the pinned requirements installed. +# +# Only the Docker fallback path reaches this script. The default PEX path builds +# no image: `dagster_cloud_cli`'s source-pex builder runs its own +# `uv pip install --target ... --no-deps .` over this repository, which is the +# same install by a different route. Both CD_dagster_*.yml workflows document +# the switch between the two. +# +# Installs this repository as a package so `db`, `domain`, `services`, `core`, +# and `schemas` resolve from site-packages. Without it they are importable only +# while /opt/dagster/app happens to be on sys.path -- true for the process that +# loads the code location, not for the process that executes a step, which is +# where the loader's imports run. That difference is invisible locally, where an +# editable install puts the repository on the path unconditionally. +# +# --no-deps because the pinned, hashed requirements are already installed and +# this must not resolve anything on top of them. +set -euo pipefail +pip install --no-deps . diff --git a/data_migrations/migrations/20260714_0001_publish_project_areas.py b/data_migrations/migrations/20260714_0001_publish_project_areas.py new file mode 100644 index 000000000..244231667 --- /dev/null +++ b/data_migrations/migrations/20260714_0001_publish_project_areas.py @@ -0,0 +1,44 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +from sqlalchemy import update +from sqlalchemy.orm import Session + +from data_migrations.base import DataMigration +from db.group import Group + + +def run(session: Session) -> None: + session.execute( + update(Group) + .where(Group.project_area.isnot(None)) + .values(release_status="public") + ) + session.commit() + + +MIGRATION = DataMigration( + id="20260714_0001_publish_project_areas", + alembic_revision="f4a5b6c7d8e9", + name="Publish all project_areas records", + description=( + "Marks every group record backing the project_areas OGC layer " + "(project_area IS NOT NULL) as release_status='public'. Confirmed " + "via docs/ogc-layer-audit.md that all 56 current rows are " + "release_status='draft'." + ), + run=run, + is_repeatable=False, +) diff --git a/data_migrations/migrations/20260819_0001_drop_unknown_alternate_ids.py b/data_migrations/migrations/20260819_0001_drop_unknown_alternate_ids.py new file mode 100644 index 000000000..79b157479 --- /dev/null +++ b/data_migrations/migrations/20260819_0001_drop_unknown_alternate_ids.py @@ -0,0 +1,136 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Remove the unattributed `Unknown` alternate identifiers from the San Acacia +Reach wells that the automated ingestion pipeline reads. + +Each of these wells carries two identifier links: an `NMBGMR` one and an +`Unknown` one with no recorded provenance. For most wells they agree, and the +`Unknown` row is redundant. For some they contradict each other -- `SO-0131` +carries NMBGMR `BRN-E04B (shallow)` and Unknown `BRN-E04A`, while `SO-0132` has +them the other way round, so the two sources disagree about which physical well +is which (BDMS-1168). + +Removing the unattributed rows leaves NMBGMR as the single answer. That is the +point: an identifier nobody can source is worse than no identifier, because it +looks like corroboration. + +Scoped to the 38 wells this pipeline ingests, deliberately. The wider reach +network has 152 such links and every `SO-` well has 263 between them; widening +this is a separate decision, and 19 of the reach network's links are the +conflicting ones BDMS-1168 is tracking -- deleting those would remove the +evidence of the conflict along with the conflict. + +Wells are matched by name rather than by id so the migration means the same +thing in every environment. +""" + +from sqlalchemy import delete, select +from sqlalchemy.orm import Session + +from data_migrations.base import DataMigration +from db.thing import Thing, ThingIdLink + +UNATTRIBUTED = "Unknown" + +WELL_NAMES = ( + "SO-0125", + "SO-0131", + "SO-0140", + "SO-0142", + "SO-0144", + "SO-0145", + "SO-0146", + "SO-0148", + "SO-0160", + "SO-0163", + "SO-0165", + "SO-0166", + "SO-0167", + "SO-0170", + "SO-0175", + "SO-0177", + "SO-0189", + "SO-0190", + "SO-0191", + "SO-0194", + "SO-0200", + "SO-0204", + "SO-0213", + "SO-0215", + "SO-0219", + "SO-0221", + "SO-0223", + "SO-0224", + "SO-0226", + "SO-0234", + "SO-0236", + "SO-0238", + "SO-0245", + "SO-0246", + "SO-0247", + "SO-0249", + "SO-0250", + "SO-0261", +) + + +def run(session: Session) -> None: + """Delete the unattributed links, leaving every other organization alone.""" + thing_ids = session.scalars( + select(Thing.id).where(Thing.name.in_(WELL_NAMES)) + ).all() + + missing = len(WELL_NAMES) - len(thing_ids) + if missing: + # Not fatal -- a database without these wells is a database this + # migration has nothing to do in -- but silence would hide a rename. + print( + f" {missing} of {len(WELL_NAMES)} wells not found by name; " + "skipping those." + ) + + if not thing_ids: + return None + + result = session.execute( + delete(ThingIdLink).where( + ThingIdLink.thing_id.in_(thing_ids), + ThingIdLink.alternate_organization == UNATTRIBUTED, + ) + ) + print( + f" removed {result.rowcount} {UNATTRIBUTED!r} links from {len(thing_ids)} wells" + ) + return None + + +MIGRATION = DataMigration( + id="20260819_0001_drop_unknown_alternate_ids", + alembic_revision="b2c3d4e5f6a7", + name="Drop unattributed alternate IDs from San Acacia Reach wells", + description=( + "Each ingested San Acacia well carries an NMBGMR identifier and an " + "unattributed 'Unknown' one. They mostly duplicate, and sometimes " + "contradict -- SO-0131 and SO-0132 disagree about which is BRN-E04A " + "(BDMS-1168). Removing the unattributed rows leaves one answer." + ), + run=run, + is_repeatable=False, +) + + +# ============= EOF ============================================= diff --git a/data_migrations/migrations/20260820_0001_backfill_acoustic_data_maturity.py b/data_migrations/migrations/20260820_0001_backfill_acoustic_data_maturity.py new file mode 100644 index 000000000..7996e4b74 --- /dev/null +++ b/data_migrations/migrations/20260820_0001_backfill_acoustic_data_maturity.py @@ -0,0 +1,88 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Set `data_maturity` on the acoustic (Wellntel) transducer observations that +alembic revision `b2c3d4e5f6a7` left NULL. + +That revision backfilled maturity from `nma_waterlevelscontinuous_pressure_qced`, +the AMPAPI flag recording whether a reading was quality controlled. Acoustic +readings have no such flag -- AMPAPI's `WaterLevelsContinuous_Acoustic` table has +no `QCed` column at all -- so all 394,086 of them were skipped, which is the +entire acoustic record (BDMS-1169). + +`MATURITY` is a deliberate choice, not a derivation. There is no QC field in the +acoustic legacy schema to read, so nothing here computes the answer; the value +below is the one recorded for these readings, applied uniformly. The transfer's +`review_status='approved'` blocks are *not* evidence for it -- those come from +`PublicRelease`, which every acoustic source row carries and which describes +visibility rather than review. + +Rows are matched on `nma_waterlevelscontinuous_acoustic_global_id`, the AMPAPI +row identity. It is written by `WaterLevelsContinuousAcousticTransferer` on every +acoustic row and never by the pressure transferer, so it is the provenance +marker: 394,086 rows carry it, and they are exactly the rows with no +`pressure_qced`. + +Only rows where `data_maturity` is already NULL are touched. Re-running is +therefore a no-op, and a maturity set deliberately since -- by the hydrograph +corrector, or by a later migration once the acoustic QC history is known -- is +left alone rather than reset to the blanket value. +""" + +from sqlalchemy import update +from sqlalchemy.orm import Session + +from data_migrations.base import DataMigration +from db.transducer import TransducerObservation + +MATURITY = "approved" + + +def run(session: Session) -> None: + """Set the maturity on acoustic observations that have none.""" + result = session.execute( + update(TransducerObservation) + .where( + TransducerObservation.nma_waterlevelscontinuous_acoustic_global_id.isnot( + None + ), + TransducerObservation.data_maturity.is_(None), + ) + .values(data_maturity=MATURITY) + .execution_options(synchronize_session=False) + ) + print( + f" set data_maturity={MATURITY!r} on {result.rowcount} acoustic observations" + ) + return None + + +MIGRATION = DataMigration( + id="20260820_0001_backfill_acoustic_data_maturity", + alembic_revision="b2c3d4e5f6a7", + name="Backfill data_maturity on acoustic (Wellntel) observations", + description=( + "Revision b2c3d4e5f6a7 backfilled data_maturity from the pressure QC " + "flag, which acoustic readings do not have, leaving the entire 394,086 " + f"row Wellntel record NULL (BDMS-1169). Sets it to {MATURITY!r}. Only " + "touches rows whose maturity is still NULL." + ), + run=run, + is_repeatable=False, +) + + +# ============= EOF ============================================= diff --git a/db/chemistry_views.py b/db/chemistry_views.py new file mode 100644 index 000000000..925a75ee6 --- /dev/null +++ b/db/chemistry_views.py @@ -0,0 +1,77 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Read-only mappings over the legacy water-chemistry views. + +`ogc_water_chemistry` and `ogc_internal_water_chemistry` are materialized views +built in d9e0f1a2b3c4 by unioning the four legacy NMA chemistry tables +(NMA_MajorChemistry, NMA_MinorTraceChemistry, NMA_Radionuclides, +NMA_FieldParameters) into one analyte-per-row shape. They were added for the OGC +EDR mount; these mappings let the REST API serve the same rows, which is where +the chemistry data actually lives -- the refactored `observation` table holds no +water chemistry. + +Views only. Like db/ngwmn_views.py these use their own declarative base so +Alembic never tries to autogenerate a table for them, and the underlying +relations are refreshed by the migration that owns them, not from here. +""" + +from datetime import datetime + +from sqlalchemy import DateTime, Float, Integer, String +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + + +class ChemistryViewBase(DeclarativeBase): + """Declarative base for chemistry view mappings, excluded from Alembic.""" + + +class _WaterChemistryResultColumns: + """Columns shared by the public and internal chemistry views. + + `id` is a text key (``maj-1``, ``min-2``, ``rad-3``, ``fld-4``) rather than + an integer: a row's identity is which legacy table it came from plus that + table's own id, and the four id sequences overlap. + """ + + id: Mapped[str] = mapped_column("id", String, primary_key=True) + thing_id: Mapped[int] = mapped_column("thing_id", Integer) + station_name: Mapped[str | None] = mapped_column("station_name", String) + thing_type: Mapped[str | None] = mapped_column("thing_type", String) + sample_id: Mapped[int | None] = mapped_column("sample_id", Integer) + parameter_name: Mapped[str] = mapped_column("parameter_name", String) + value: Mapped[float | None] = mapped_column("value", Float) + unit: Mapped[str | None] = mapped_column("unit", String) + # Named `datetime` in the view; exposed under the name the observation + # endpoints already use so clients do not need a second field name. + observation_datetime: Mapped[datetime] = mapped_column("datetime", DateTime) + release_status: Mapped[str | None] = mapped_column("release_status", String) + + +class WaterChemistryResultsView(_WaterChemistryResultColumns, ChemistryViewBase): + """Public chemistry analyses: released things, released samples.""" + + __tablename__ = "ogc_water_chemistry" + + +class InternalWaterChemistryResultsView( + _WaterChemistryResultColumns, ChemistryViewBase +): + """Every chemistry analysis, including unreleased things and samples.""" + + __tablename__ = "ogc_internal_water_chemistry" + + +# ============= EOF ============================================= diff --git a/db/transducer.py b/db/transducer.py index 1670bb9fa..e129cfe60 100644 --- a/db/transducer.py +++ b/db/transducer.py @@ -28,6 +28,7 @@ Index, UniqueConstraint, ) +from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import mapped_column, Mapped, relationship from db import Base, AutoBaseMixin, ReleaseMixin, lexicon_term @@ -62,6 +63,32 @@ class TransducerObservationBlock(Base, AutoBaseMixin, ReleaseMixin): ) comment: Mapped[str] = mapped_column(Text, nullable=True) + + # Publish provenance. A corrected block is derived data -- the numbers in it + # are not what any instrument recorded -- so the file it came from and the + # operations applied to it are part of the record, not metadata about it. A + # reviewer who cannot see that a series was snapped to a manual measurement + # cannot review it. + source_file: Mapped[str] = mapped_column( + String(255), + nullable=True, + comment="Name of the logger file the corrected series was derived from", + ) + source_kind: Mapped[str] = mapped_column( + String(50), + nullable=True, + comment="What the source file measured: water_head or depth_to_water", + ) + # A list of strings in applied order rather than a modelled correction + # entity: the corrector's operation set is still moving, and freezing it + # into columns now would mean a migration per new operation. The strings + # are written by the workbench and read by humans. + corrections: Mapped[list] = mapped_column( + JSONB, + nullable=True, + comment="Corrections applied to the source series, in applied order", + ) + reviewer_id: Mapped[str] = mapped_column( ForeignKey("contact.id", ondelete="CASCADE"), nullable=True, @@ -81,8 +108,13 @@ class TransducerObservationBlock(Base, AutoBaseMixin, ReleaseMixin): "end_datetime", name="uq_transducer_block_thing_status_parameter_time", ), + # Non-strict: a block covering a single instant is legitimate -- a + # published file with one reading, or a block narrowed by a range + # delete until one observation survives. The block reader matches + # observations inclusively on both bounds, so a zero-width block still + # covers its reading. CheckConstraint( - "end_datetime > start_datetime", name="check_transuder_block_time_order" + "end_datetime >= start_datetime", name="check_transducer_block_time_order" ), Index( "ix_transducer_block_time", @@ -107,12 +139,20 @@ class TransducerObservation(Base, AutoBaseMixin, ReleaseMixin): """ __tablename__ = "transducer_observation" + # Unique rather than merely indexed: without a constraint to conflict on, a + # re-run can only avoid duplicates by deleting first, which leaves a window + # where the data is missing. With it the loader upserts and a repeated + # backfill is idempotent. + # + # Scoped to the deployment, not the thing: a deployment is a thing/sensor + # pairing, so two sensors on one well may legitimately report the same + # instant. __table_args__ = ( - Index( - "ix_transducer_observation_deployment_parameter_datetime", + UniqueConstraint( "deployment_id", "parameter_id", "observation_datetime", + name="uq_transducer_observation_deployment_parameter_datetime", ), ) @@ -128,6 +168,26 @@ class TransducerObservation(Base, AutoBaseMixin, ReleaseMixin): DateTime(timezone=True), nullable=False, index=True ) value: Mapped[float] = mapped_column(Float, nullable=False) + + # Why this reading differs from what the sensor recorded. Present only on + # readings a correction actually moved, so a NULL note means the value is + # as measured -- which is the distinction review needs and which the legacy + # `nma_waterlevelscontinuous_*_notes` columns cannot carry, being scoped to + # one legacy source each. + note: Mapped[str] = mapped_column( + Text, + nullable=True, + comment="Per-reading correction annotation; NULL means the value is as measured", + ) + + # How far through review this reading is, on USGS terms: provisional, + # in review, approved. Orthogonal to `release_status`, which says who may + # see it -- a reading can be public and provisional at once, which one + # column could not express because its lexicon lists those as siblings. + # + # Nullable because legacy rows predate it and nobody has established + # whether they are approved. NULL means not stated, which is honest. + data_maturity: Mapped[str] = lexicon_term(nullable=True) nma_waterlevelscontinuous_pressure_conddl_ms_cm: Mapped[float] = mapped_column( Float, nullable=True ) diff --git a/docker-compose.yml b/docker-compose.yml index 94991fb99..3cfdaffe6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,6 @@ services: - POSTGRES_PORT=5432 - MODE=${MODE} - AUTHENTIK_DISABLE_AUTHENTICATION=${AUTHENTIK_DISABLE_AUTHENTICATION} - - SESSION_SECRET_KEY=${SESSION_SECRET_KEY} - PYGEOAPI_POSTGRES_HOST=db - PYGEOAPI_POSTGRES_PORT=5432 - PYGEOAPI_POSTGRES_DB=ocotilloapi_dev diff --git a/docs/automated-ingestion-pipeline-plan.md b/docs/automated-ingestion-pipeline-plan.md new file mode 100644 index 000000000..5eaffdab5 --- /dev/null +++ b/docs/automated-ingestion-pipeline-plan.md @@ -0,0 +1,446 @@ +# Draft: Automated Ingestion Pipeline Epic (BDMS) + +1 new Epic → 4 Tasks → 17 Sub-tasks. **Nothing written to Jira yet.** + +## TL;DR + +Build the Bureau's first automated data ingestion pipeline, in the OcotilloAPI repo, so continuous depth-to-groundwater readings reach Ocotillo on a schedule instead of by hand. San Acacia Reach (38 Van Essen divers) is the pilot source; the structure it establishes is what every later source inherits. + +Stack: **Dagster+** code location → **dlt** extraction → **GCS** raw parquet → **`domain/`** mapping → direct **Postgres** load. Watermark and backfill mechanics are ported from Aqueduct, with two deliberate improvements a relational destination allows: the watermark is read from Postgres rather than a GCS sidecar, and an upsert replaces Aqueduct's delete-then-repost (removing its known window where data goes temporarily missing). + +**Decided** — four calls already made, so reviewers don't reopen them: + +- **Owned by OcotilloAPI, not Aqueduct.** The loader writes over a direct database connection, which wants the `db/` SQLAlchemy models and `domain/` rules in the same process. Running it as a third Aqueduct source would mean maintaining a copy of Ocotillo's schema in another repo. Aqueduct stays the FROST/SensorThings pipeline; shared code is **ported, not imported**, so the two can diverge without breaking each other. +- **Ground-surface datum.** `TransducerObservation.value` stores depth to water below ground surface, in feet. That picks Van Essen's `gs` arrays and drops `vrd` entirely. No measuring-point correction on ingest — `domain/water_levels.py`'s MP reconciliation belongs to the manual-measurement path, where a field crew measured the height on the day. Datum shifts are the Hydrograph Corrector's job, downstream. +- **Public + provisional.** Visible from the first run, and marked provisional so no consumer mistakes an uncorrected diver series for a reviewed one. This matches what the retired FROST pipeline asserted for this source (`is_provisional: true`) — adopted deliberately here rather than inherited silently, which was the open question left in Aqueduct's mapping doc. It needs a schema change: `release_status` is one column, and its lexicon lists `public` and `provisional` as siblings, so visibility and maturity — two orthogonal axes — currently collide. +- **Vendor approval flag ≠ Ocotillo review status.** Van Essen's `approvedWaterLevels*` records what *the vendor* approved. Ocotillo's `review_status` is `approved` / `not reviewed`, and `TransducerObservationBlock.reviewer_id` FKs a Bureau `Contact` — so `approved` asserts a Bureau human reviewed it. Mapping one onto the other would manufacture provenance that doesn't exist. All San Acacia blocks land `not reviewed`; the vendor flag is preserved as a separate per-row attribute. + +**Watch:** two schema changes — a unique constraint on `transducer_observation`, and a new field because `release_status` cannot hold "public" and "provisional" at once. The vendor blocker cleared on 2026-08-18: the readings endpoint works, but only through the private Diver-HUB API, only with a 1-hour JWT, and only in bounded time windows. + +**Sequencing:** Task 1 gates everything. Tasks 2 and 3 run largely in parallel after it. Nothing is vendor-blocked any more. + +## All tasks + +| # | Item | In one line | Blocked by | +|---|---|---|---| +| **T1** | **Foundations** | Package, Dagster+ code location, GCS, DB connectivity | — | +| 1.1 | Scaffold package + Dagster skeleton | `automated_ingestion/` layout, deps, loads in `dagster dev` | — | +| 1.2 | Register Dagster+ code location | `dagster_cloud.yaml` + prod/branch deploy workflows | 1.1 | +| 1.3 | GCS buckets + service account | `ocotillo-ingestion-{production,staging}`, date-partitioned layout | — | +| 1.4 | DB connectivity + least-privilege role | Cloud SQL connector from serverless; scoped Postgres role | 1.2 | +| **T2** | **Source extraction** | Van Essen API → GCS raw zone | T1 | +| 2.1 | Confirm endpoint + finalize mapping | **Unblocked.** Diver-HUB swagger, JWT login, measure the window ceiling | — | +| 2.2 | dlt resource: locations | 38 wells, `replace`, one call, no pagination | 1.3 | +| 2.3 | dlt resource: readings, incremental | Windowed per-point fetch, dlt cursor, `append`, token refresh, failure isolation | 2.1 | +| **T3** | **Domain mapping + load** | Van Essen records → Ocotillo Postgres | T1 | +| 3.1 | Domain layer | Pure functions: units, datum, timestamps, geometry, external keys | — | +| 3.2 | Bootstrap reference data | Reconcile 38 wells; seed parameter, sensor, deployments | 3.1 | +| 3.3 | Represent "public but provisional" | **Schema change.** `release_status` can't hold both axes | — | +| 3.4 | Unique constraint + upsert loader | **Schema change.** `ON CONFLICT DO UPDATE`; makes backfill idempotent | 3.2, 3.3 | +| 3.5 | Watermark from Postgres | `MAX(observation_datetime)` per series; no GCS sidecar | 3.4 | +| **T4** | **Backfill + operations** | Recover from gaps, bugs, and vendor corrections | T3 | +| 4.1 | Port shared backfill primitives | `month_chunks`, `BackfillCheckpointStore`, `ChunkResult` from Aqueduct | — | +| 4.2 | Mode A — refetch | Re-fetch from API for a window; `dry_run: true` default; chunked, resumable | 4.1, 2.3 | +| 4.3 | Mode B — replay | Reprocess GCS parquet through the current adapter; no API calls | 4.1, 3.4 | +| 4.4 | Schedule, observability, alerting | Daily schedule, log bridge, failure notification, run metadata | 4.2 | +| 4.5 | Documentation | Source mapping, storage conventions, backfill runbook, new-source checklist | 4.3 | + +--- + +# EPIC — Automated Ingestion Pipeline + +**Goal:** continuous depth-to-groundwater data lands in Ocotillo automatically, on a schedule, with no one hand-carrying files — starting with San Acacia Reach. + +The Hydrograph Corrector UI exists and works (BDMS-1137 done), but has no automatic supply of raw data. San Acacia Reach's 38 Van Essen divers historically flowed through the retired FROST/`st2` stack and now flow nowhere. This epic builds the supply. Correction, review, and publication workflows are **out of scope** and belong to their own epic. + +New top-level `automated_ingestion/` package in OcotilloAPI, deployed as its own Dagster+ code location in the existing `nmbgmr-data-services` org. dlt extracts the Van Essen API to a GCS raw zone; a `domain/` layer maps to the Ocotillo model; a loader writes to Ocotillo Postgres over a direct DB connection. Watermark and backfill mechanics come from Aqueduct. + +San Acacia first: 38 wells, one DTW series each, and already mapped in `Aqueduct/docs/sources/san_acacia.md`. It authenticates with a short-lived JWT and must be read in bounded time windows — both cheap enough here to establish the pattern before a harder source needs it. What it establishes — source registry, per-source dlt pipeline, adapter, backfill job factory — every later source inherits. + +**Ownership: OcotilloAPI.** Not a third Aqueduct source writing into Ocotillo. The loader writes over a direct database connection, which wants the `db/` SQLAlchemy models and `domain/` rules in-process rather than a duplicated schema in another repo. Aqueduct stays the FROST/SensorThings pipeline; this is Ocotillo's own. The two share code by porting (see below), not by importing. + +### Adopted from Aqueduct + +| Artifact | Adoption | +|---|---| +| `docs/BACKFILL_STRATEGY.md` | Wholesale: Mode A refetch / Mode B replay, per-source generated jobs, calendar-month chunking, `dry_run: true` default, `initial_start_date` as a floor only | +| `shared/backfill.py`, `shared/gcs.py` | Port near-verbatim — already destination-agnostic | +| `shared/source_registry.py` | Port the pattern; registry drives job + schedule generation | +| `canonical/base_adapter.py` | Adapt: same `extract`/`to_*`/`run` shape and per-record failure isolation, emitting Ocotillo structs | +| `loader/watermark_store.py` | **Adapt, not port** — see deviation 1 | +| `docs/STORAGE_CONVENTIONS.md` | Adopt, renamed for `ocotillo-ingestion-` | + +### Deviations from Aqueduct + +1. **Watermark in Postgres, not a GCS sidecar.** Aqueduct needs `_frost_watermarks.json` because FROST has no transactional read. Ocotillo's destination does: `MAX(observation_datetime)` per `(thing_id, parameter_id)`, read in the write transaction. No sidecar drift, no recovery path. +2. **Upsert replaces delete-then-repost.** `BACKFILL_STRATEGY.md` §4.4 accepts a temporary hole in FROST because observations have no dedup key there. Postgres does — unique constraint plus `ON CONFLICT DO UPDATE` makes load and backfill idempotent with no destructive delete. Resolves that doc's §6 open question. +3. **Target is `Thing → Deployment → TransducerObservation`**, not `FieldEvent → … → Observation`. 5-minute diver series are continuous, not field visits. + +### Data classification — decided + +- **Datum: ground surface.** `TransducerObservation.value` = depth to water below ground surface, feet. Ingest Van Essen's `gs` arrays, not `vrd`. No measuring-point correction on ingest — `domain/water_levels.py`'s MP reconciliation is the manual-measurement path. Datum shifts are the corrector's business. +- **Visibility public, maturity provisional.** Public from the first run, marked provisional so nobody mistakes an uncorrected diver series for a reviewed one. Matches what the old FROST pipeline asserted (`is_provisional: true`) — adopted deliberately, not inherited silently. +- **Schema cannot express this today.** `release_status` is one scalar column (`ReleaseMixin` → `lexicon_term.term`), and its lexicon category holds `public` *and* `provisional` as siblings. Visibility and maturity are orthogonal; the lexicon conflates them. Sub-task 3.3 resolves it. +- **Vendor approval ≠ Ocotillo review status.** `approvedWaterLevels*` records what the *vendor* approved. Ocotillo `review_status` is `approved` / `not reviewed`, and `TransducerObservationBlock.reviewer_id` FKs a Bureau `Contact` — `approved` means a Bureau human reviewed it. All San Acacia blocks land `not reviewed`; the vendor flag is kept as a separate per-row attribute. + +### Epic acceptance criteria + +- `automated_ingestion/` deploys as a Dagster+ code location on merge; jobs visible in the Dagster UI. +- Scheduled job runs end to end: Van Essen API → GCS parquet → domain mapping → Ocotillo Postgres. +- Re-running over an already-loaded window: zero duplicates, zero errors. +- Both backfill jobs exist, default `dry_run: true`, chunk by month, resume from last completed chunk. +- 38 wells resolve to `Thing` records — matched, never created, no duplicates. +- Readings are public, marked provisional, stored as DTW below ground surface in feet. +- Series render in the Hydrograph Corrector. +- Domain mapping unit-tested with no database, per `ADR4.md`. + +### Blocker — resolved 2026-08-18 + +The 500s were never a vendor outage. Two things were wrong on our side, both reported by Chase Martin: + +1. **Wrong API.** Readings come from the private Diver-HUB API — `GET https://diver-hub.com/private/api/v1/DiverData/ByMonitoringPoint/{id}` — not the doubled-segment `/api/api/monitoringPoint/{project}/{id}` path the earlier draft assumed. Swagger: `https://diver-hub.com/private/swagger/index.html`, which is now the authority over anything inferred from retired FROST data. +2. **Window too large.** The endpoint 500s rather than paginating or erroring cleanly when asked for too much. A confirmed-good request is a ~3-month window in **Unix seconds**: + + ``` + https://diver-hub.com/private/api/v1/DiverData/ByMonitoringPoint/40?startTime=1767225600&endTime=1775001600 + ``` + +**Auth:** POST to the login endpoint with the credentials Ethan circulated; it returns a **JWT valid for one hour**. This overturns the "unauthenticated" assumption in the earlier draft and has two consequences: the token is a secret needing the same handling as the DB credentials, and any run outliving an hour — every backfill — must refresh mid-run rather than acquire once at start. + +**Still open:** the actual window ceiling. Three months works; the limit is unmeasured. Until it is, chunk conservatively and treat a 500 as "too much data" rather than a hard failure. + +### Related + +BDMS-1137 (corrector zoom/selection, Done — the consumer of this data, not part of this epic) · BDMS-1090 (Wellpy Revival Discovery) · BDMS-362 (WellPy Ocotillo) · `DataIntegrationGroup/Aqueduct` · OcotilloAPI `ADR4.md`, `db/transducer.py`, `db/engine.py` + +--- + +# TASK 1 — Foundations: code location, GCS, DB connectivity + +Nothing in this repo runs on a schedule today. This task creates the package, gets it deploying to Dagster+, provisions GCS, and proves the Dagster runtime can reach Ocotillo Postgres. Carries the workstream's two infrastructure risks: build size and serverless→Cloud SQL connectivity. + +**Done when:** package loads in `dagster dev`; merge deploys to prod and PRs produce branch deployments; buckets exist with a least-privilege SA; a trivial asset reads Ocotillo Postgres from both deployments; pytest/ruff/mypy pass. + +### 1.1 — Scaffold `automated_ingestion/` and the Dagster skeleton + +``` +automated_ingestion/ +├── defs/ definitions.py (entry point), assets/, jobs/backfill.py +├── shared/ source_registry.py, backfill.py, gcs.py, http.py +├── ocotillo/ adapter base + Ocotillo structs +├── sources/san_acacia/ ingest / dlt_pipeline / adapter / transform / backfill +└── tests/ +``` + +- Layout above created; `automated_ingestion` added to `[tool.setuptools] packages` (same fix as `f33cd063` for `domain`). +- Deps added: dagster, dagster-cloud, `dlt[filesystem,gs]`, gcsfs, pyarrow. `[tool.dagster] module_name = "automated_ingestion.defs.definitions"`. +- `dagster dev` loads the location with no import errors; ruff/mypy cover it; pytest still green. + +Lives in this repo so the loader can import `db/` models and `domain/` rules rather than duplicate the schema. If the Dagster+ build proves too large, fall back to a `[project.optional-dependencies]` split. + +### 1.2 — Register as a Dagster+ code location with CI deploy + +Files written; nothing deployed yet — the secrets do not exist, so neither workflow has run. + +- ✅ `dagster_cloud.yaml` declaring `ocotillo-automated-ingestion` → `automated_ingestion.defs.definitions`, build directory `./`. The build directory is the repository root, not `automated_ingestion/`, because the loader imports `db/` and `domain/`. +- ✅ `CD_dagster_prod.yml` and `CD_dagster_branch.yml`, both on `dagster-io/dagster-cloud-action@v1.13.18` — pinned to the same version as the installed dagster. +- ✅ Path-filtered to `automated_ingestion/**`, `dagster_cloud.yaml`, `pyproject.toml`, and `uv.lock`. The last two matter: the location's dependency set is exported from them, so a lockfile bump changes the built image even when no ingestion file moves. +- ⬜ `DAGSTER_CLOUD_API_TOKEN` as a repository **secret**, `DAGSTER_CLOUD_ORGANIZATION_ID` as a repository **variable**. The token is a CI credential the action uses to reach Dagster+, so it belongs with `CLOUD_DEPLOY_SERVICE_ACCOUNT_KEY` rather than in Secret Manager -- reading it from Secret Manager would still require a GitHub secret to authenticate to GCP first, adding a hop without removing a trust root. The organization ID is not sensitive; it appears in the Dagster+ console URL. +- ⬜ Runtime secrets are a different question and are **not** GitHub's. The Diver-HUB login (2.1), the ingestion service account (1.3), and the Postgres role (1.4) are read by the pipeline while it runs, not by the deploy, so they belong in Secret Manager on the `internal-ogc-api-keys` precedent, reached from Dagster+ at runtime. +- ⬜ Test PR yields a working branch deployment; merge to `production` yields a working prod location. + +**Prod deploys from `production`, not `main`.** `main` was abandoned in July 2025 — it is 3,839 commits behind and is not part of the release flow (`docs/release-flow.md`). The `main` reference in the original draft was inherited from Aqueduct's layout without checking this repository's. + +**PEX vs Docker — answered: Docker.** `serverless_prod_deploy` and `serverless_branch_deploy` build with `docker/build-push-action` and a copied Dockerfile template; there is no PEX fast-deploy path in these actions. So build time is a full image build, and the dependency set matters: the image installs all 197 exported packages (the 135 runtime ones plus dagster, dlt, gcsfs, pyarrow). `pymssql` and `psycopg2-binary` are in that set and compile from source on some base images — the first real build is where that surfaces. + +**Ordering constraint, easy to break.** `utils/parse_workspace` runs its own `actions/checkout`, which cleans the working tree. It must run *before* `requirements.txt` is generated; putting the generation first silently deletes it, and the deploy fails on a missing file rather than on the real cause. + +Both workflows generate `requirements.txt` with `uv export --group ingestion`, since Dagster+ builds from a requirements file and the repository does not keep one under version control. + +### 1.3 — Provision GCS buckets and ingestion service account + +Terraform written in `automated_ingestion/iac/`; **not applied**. `terraform validate` and `fmt` pass, but no GCP credentials were available, so no resource exists yet. + +- ✅ `ocotillo-ingestion-production` and `-staging`, uniform bucket-level access, public access prevention enforced, `force_destroy = false`. +- ✅ Service account `ocotillo-ingestion` with `roles/storage.objectAdmin` bound **on the two buckets**, not at project level. `objectAdmin` rather than `objectCreator` because a Mode B replay overwrites an existing object. +- ✅ Lifecycle: NEARLINE at 30 days, COLDLINE at 365, and archived-version pruning past 3. Aged out rather than deleted — an old window is exactly what a historical replay reads. +- ✅ `INGESTION_GCS_BUCKET` resolved by `shared/gcs.raw_zone_bucket()`, which raises rather than defaulting and explicitly rejects a value equal to `GCS_BUCKET_NAME`. `services/gcs_helper.py` uses that variable for user uploads; the two being confused would write raw vendor payloads into the uploads bucket, and would otherwise do so silently. +- ⬜ `terraform apply`, then set `INGESTION_GCS_BUCKET` on the Dagster+ code location. + +The dlt layout `{table_name}/year={YYYY}/month={MM}/day={DD}/{load_id}.{file_id}.{ext}` is asserted by a test, because Mode B replay selects a window by prefix — the date has to be in the path, not inside the file. + +### 1.4 — DB connectivity from Dagster+ with a least-privilege role + +Dagster+ Serverless is outside the VPC, so Cloud SQL's private IP is unreachable from it. Code written; **nothing run against a database**. + +- ✅ `OcotilloDatabase` resource delegating to `db/engine.py`'s `DB_DRIVER=cloudsql` path rather than building a second engine. The import is lazy: `db.engine` builds its engine at import time, and a code location that needs a reachable database merely to *list* its assets breaks every time the database blips. A test asserts loading the definitions leaves `db.engine` unimported. +- ✅ `database_connectivity` asset, read-only. Connectivity and grants are separable problems, and a write here would leave test rows in a real table. +- ✅ Role DDL in `automated_ingestion/sql/ingestion_role.sql`, kept out of Alembic: roles and grants are per-environment infrastructure, not schema, and migrations do not run as a superuser. +- ⬜ Run the DDL per environment; set `DB_DRIVER`, `CLOUD_SQL_*` on the code location; materialize the asset from both a branch and prod deployment. + +**The grant list is narrower than the draft assumed, and one part of it is non-obvious.** Writable: `transducer_observation`, `transducer_observation_block`, `deployment`, `sensor`, `parameter`. Read-only: `thing`, `thing_id_link`, `location`, and the three `lexicon_*` tables — `thing` and `location` deliberately *not* writable, because reconciling the 38 wells means matching rows that already exist. A well found missing is a decision for a human, not a row the pipeline invents. + +`parameter` is versioned by sqlalchemy-continuum, so inserting one also writes to `parameter_version` and `transaction`. Without those two grants the write fails on a table the code never names — the kind of error that costs an afternoon. (`transducer_observation` itself is not versioned; only `aquifer_system`, `geologic_formation`, `location`, `observation`, `parameter`, `regulatory_limit`, and `thing` are.) Sequence `USAGE` is granted explicitly, and no default privileges are set: a table added later stays invisible until someone grants it deliberately. + +Fallback if the connector path fails: Hybrid agent in GCP. + +--- + +# TASK 2 — Source extraction: Van Essen → GCS raw zone + +Land locations and readings untransformed in GCS as date-partitioned parquet. Raw storage is what makes Mode B replay possible — a mapping bug becomes a reprocess, not a re-fetch. Carries the external blocker. + +**Done when:** both land at the documented paths; readings extraction is incremental; a per-entity failure doesn't abort the run; fixtures exist so downstream work needs no live API. + +### 2.1 — Confirm the readings endpoint; finalize the source mapping + +**The swagger is public** (`https://diver-hub.com/private/swagger/v1/swagger.json`) and reading it settled most of this without credentials. Full mapping in `docs/sources/san_acacia.md`; four corrections that invalidate parts of the original draft: + +- ✅ **No `/api/api/` segment, no `locations/sanacaciareach`.** Seven endpoints under `/api/v1/`. Reference data is `Projects` → `MonitoringPoints/ByProject/{id}`. +- ✅ **No `gs`/`vrd` arrays.** `WaterLevels/ByMonitoringPoint` returns a flat `[{dateAndTime, level}]`. Datum and approval are *query parameters* (`reference`, `approved`), not fields to pick out of parallel arrays. The reshaping `transform.py` was scaffolded for does not exist. +- ✅ **`DiverData` is not the series we want.** It returns `DataPoint` — pressure, temperature, conductivity, salinity — with no water level. It is what the known-good example URL fetches, which is why it looked like the readings endpoint. +- ✅ **`MonitoringPoint` is `{id, name}` only.** No coordinates, no `drillingDepth`. The planned centimetre conversion and geometry mapping have no source here; both must come from the Ocotillo rows the points reconcile against. + +Built, and testable without the network: + +- ✅ `sources/san_acacia/client.py` — JWT auth refreshed against `validTo` with a skew, one forced re-login on a 401, and windowed fetches that halve on a 500 and refuse to shrink past a one-day floor. +- ✅ `shared/windows.py` — the window arithmetic, kept pure so the tricky part is testable. +- ✅ `scripts/probe_diverhub.py` — a one-off instrument that answers the remaining questions against the live API. + +⬜ **Run the probe.** It needs the credentials Ethan circulated. Until then: + +**`WaterLevelReference` is `enum [0,1,2,3]` with no names in the spec, and this is the highest-risk unknown in the epic.** Which value means ground surface is not derivable, and choosing wrong does not fail — it returns plausible numbers on the wrong datum and silently poisons every reading. `GROUND_SURFACE_REFERENCE` is `None` in code and the client will not guess. The probe samples all four side by side so a person can identify it against a well whose depth to water is known. + +Also still open: the window ceiling (three months works, the limit is unmeasured), whether `approved=true`/`false` partition or overlap, whether `dateAndTime` is marked UTC, and whether `level` is feet. That last one gates correctness rather than completeness, same as the datum. + +### 2.2 — dlt resource: locations → GCS + +- ✅ `@dlt.resource(name="vanessen_locations")`, `write_disposition="replace"`, on `MonitoringPoints/ByProject/4317` — **not** the `locations/sanacaciareach` path in the original draft, which does not exist. One request, no pagination. +- ✅ Asset `raw_san_acacia_locations` emits the point count, project id, and a sample of names. Tested against a stub, no network. +- ✅ `replace` rather than `append`: this is a snapshot of what the vendor currently lists, and a point disappearing is information rather than something to accumulate. + +The payload is `{id, name}` only, so this cannot be a source of geometry or construction detail — it enumerates the points a reading fetch walks. **38 points.** Earlier drafts said 33; that figure came from Aqueduct's stale mapping doc, not from a Bureau record — see 3.2. + +### 2.3 — dlt resource: readings → GCS, incremental + +- ✅ `@dlt.resource(name="vanessen_readings")`, `write_disposition="append"`, dlt incremental cursor on `dateAndTime`, walking each point from its watermark. +- ✅ `INITIAL_START` (2015-01-01) documented as a floor for a point with no cursor, never a backfill lever. +- ✅ Per-point failure isolation: one diver failing costs that diver's data for the run, not the other thirty-seven. Failures are collected into a list **the caller owns** — a dlt resource is a module-level object shared by every run, so per-run state stashed on it would have concurrent runs overwriting each other. +- ✅ Asset `raw_san_acacia_readings` emits rows ingested, points attempted, points failed, and the failures themselves. +- ✅ Nothing is converted on the way in. The raw zone stores the vendor's `level` in the vendor's centimetres on the vendor's datum, with `unit` and `reference` recorded alongside, so a mapping bug is a reprocess rather than a re-fetch. + +**Vendor approval needs two requests.** `approved` is a query parameter, not a response field, so the flag cannot be read off a row. Fetching `approved=true` and `approved=false` separately and concatenating would duplicate every reading if the two sets overlap — which is still unknown (open question 4). Instead the unfiltered series is authoritative and a second `approved=true` fetch supplies a set of timestamps used only to tag it. A failure of that second fetch leaves rows tagged `false` rather than losing them: an untagged reading is worth more than no reading, and the vendor flag is not Ocotillo's review status regardless. + +**Window span is measured, not inherited.** `READING_SPAN` is 365 days for this source rather than the cautious 90-day default in `shared/windows.py`, because probing showed `WaterLevels` serving 730 days and 18111 rows in one request. At 90 days a first run for a single point would issue four times the requests for no benefit. It sits at half the largest span observed to work, leaving headroom for a denser point than SO-0125. + +--- + +# TASK 3 — Domain mapping and load into Ocotillo + +Where this stops resembling Aqueduct: the destination is a relational database with constraints and transactions, and mapping rules belong in `domain/` per `ADR4.md`. Three risks — matching 38 wells without duplicating them, representing "public but provisional" when the schema can't, and making the write idempotent so backfill is safe. + +**Done when:** mapping rules are pure functions tested without a database; 38 wells resolve with no duplicates; data is public and separately marked provisional; `transducer_observation` has a unique constraint and the loader upserts against it; loading the same window twice leaves the row count unchanged; the watermark comes from Postgres. + +### 3.1 — Domain layer: Van Essen record → Ocotillo model + +Built. `domain/van_essen.py` plus `sources/san_acacia/adapter.py`, 28 tests, no database and no network. + +**Scope is smaller than this section originally claimed.** The draft called for converting `drillingDepth` from centimetres and building a WGS84 point from `lat`/`lng`. The live `MonitoringPoint` payload is `{id, name}` — no depth, no coordinates — so those functions would have had no input. Well geometry and construction come from the Ocotillo records a point reconciles against, which is consistent with ingestion never creating wells. + +What the layer actually does: + +- ✅ Reading timestamp → timezone-aware UTC. A naive value is read as UTC, since the API documents UTC and does not always mark it; reading it as local would shift every observation by the machine's offset, and differently on a laptop than in a container. +- ✅ Centimetres → feet via `domain/units.convert_cm_to_ft`. +- ✅ Deterministic external keys, built from the vendor's **numeric** id rather than the name. `SO-0125` is a Bureau point id and can be corrected; the numeric id is what a re-run must resolve to the same record. The series key names the datum, because a point may later carry temperature or conductivity — both already in the vendor's raw payload. +- ✅ Errors subclass `ValueError`, matching the per-row contract the CSV importers expect. +- ✅ ADR4 layering verified by test rather than by inspection: importing `domain.van_essen` pulls in no `fastapi`, `sqlalchemy`, `pydantic`, `httpx`, `db`, `api`, `schemas`, or `services`. + +**The adapter refuses two things outright**, both because accepting them would produce plausible numbers rather than an error: + +- A row whose `reference` is not 3. The datum is chosen at request time and cannot be recovered from the row. +- A row whose `unit` is not `cm`. Converting a value whose unit is not what it claims is wrong by a factor of 30.48 and still reads as a plausible depth. + +Per-record failures are collected, not raised: one unparseable reading costs that reading, not the series. + +The module docstring lists every value the mapping **invents** rather than reads — the datum, the unit, and the timezone — since inventing is where a mapping goes quietly wrong. + +### 3.2 — Bootstrap reference data: reconcile wells, seed parameter, sensor, deployments + +Reconciliation report built — `sources/san_acacia/reconcile.py` and `scripts/reconcile_san_acacia.py`, 12 tests. The seeding half is not built. + +**The "33 wells" figure was wrong, and was never a blocker.** It came from Aqueduct's `docs/sources/san_acacia.md` — the same document that also supplied the doubled `/api/api/` path, the claim the source is unauthenticated, and the `gs`/`vrd` payload shape, all disproved against the live API. 38 is what the API returns. Whether all 38 are in scope is a question the per-well report answers concretely. + +**Coordinate proximity is not available.** This section called for matching on name, external id, *and* coordinate proximity. `MonitoringPoint` is `{id, name}` — no coordinates. That removes the only fuzzy signal and leaves two exact ones, which is a better position: every match is defensible rather than probabilistic. + +- ✅ Matching on name and on `thing_id_link.alternate_id`, normalized for case, spacing and punctuation so `SO-0125`, `so 0125` and `SO0125` compare equal. Still exact on significant characters — `SO-0126` stays a different well. +- ✅ Name beats external id when both hit. The name is the identifier the Bureau uses now; a link records an association someone made earlier. +- ✅ **Never picks a winner.** More than one candidate is `ambiguous` and escalates; none is `unmatched` and escalates. Ingestion does not create wells, and choosing between two plausible ones is the judgement that must not be automated. +- ✅ `report.ready` is false unless *every* point resolved, and false for empty input. A partial load produces a series that looks complete and is not. +- ✅ The script exits non-zero when anything needs a human, so it can gate a later step without relying on someone reading the output. +- ✅ **Run against staging: all 38 points match by name. Nothing ambiguous, nothing unmatched, `ready = True`.** The wells already exist — SO-0125 is thing 2343, SO-0131 is 2369, and so on through 277 `SO-` wells in that database. So the seeding half creates no wells; it only needs the parameter, sensor, deployments and external identifiers. +- ✅ **Production confirms it**: same 38 matches, same thing ids (SO-0125 is 2343 in both), `ready = True`. Staging is a clone of production for these tables, so the two agree by construction. + +**External-id matching is off by default, on evidence.** `thing_id_link` in staging holds 11,148 links from nine organization/relation pairs — NMBGMR (8,603), PLSS (7,052), an unattributed "Unknown" (4,825), NMOSE, USGS, NMED, TWDB — and they disagree with each other. `SO-0131` carries NMBGMR `BRN-E04B (shallow)` plus an unattributed `BRN-E04A`, while `SO-0132` carries NMBGMR `BRN-E04A (deep)` plus an unattributed `BRN-E04B`: the two sources swap which physical well is A and which is B. (`SO-0262`/`SO-0263` disagree more sharply still — NMBGMR calls them NRCS 3A/3B, the other source NRCS 2.) + +Matching `BRN-E04A` against that returns a single confident hit on `SO-0131`, contradicting NMBGMR, because the parenthetical suffix stops the collision registering as ambiguous. A wrong answer delivered confidently is worse than no answer, so the fallback is opt-in and a test pins the real rows. + +**This is production data, not a staging artifact.** The same contradictions are in both. They are worth someone's attention independently of this pipeline: `SO-0131`/`SO-0132` and `SO-0262`/`SO-0263` are paired shallow/deep piezometers whose A/B designations disagree between identifier sources, and a swap there means a shallow series attributed to a deep well. Ingestion is unaffected — the vendor names points `SO-####` and Ocotillo agrees on those — but anyone reasoning about these wells through the `BRN-`/`NRCS` names is working from two incompatible answers. +- ⬜ The seeding half: data migration creating missing `Location`/`Thing`, lexicon terms, DTW `Parameter`, `VanEssenDiver` `Sensor`, one `Deployment` per well, the vendor `uid` as external identifier, and `DataProvenance` for Van Essen-sourced attributes. + +### 3.2 seeding — nothing needed, measured 2026-08-19 + +The plan expected to create wells, a parameter, a sensor and deployments. Checked against staging: **all of it already exists.** + +- All 38 wells have deployments — 108 open ones between them, because a deployment is a piece of equipment rather than a measured property. SO-0140 carries three: a `DiverLink` (telemetry), a `Pressure Transducer` (the reading), and a `Diver Cable`. `Barometer` appears elsewhere. +- The parameter exists: id 1, `groundwater level`, `default_unit = ft` — which is what the adapter emits, so the centimetre conversion in `domain/van_essen.py` lands in the right unit. +- Existing observations for these wells already use that parameter. + +**So the series is chosen, not created.** `sources/san_acacia/resolve.py` picks the open `Pressure Transducer` deployment. Across the 38 wells that resolves cleanly for **35**; **2** have two open transducers and **1** (SO-0246) has none. Those three are skipped and reported rather than guessed at — taking the lower id would be a silent decision about equipment. + +A *removed* transducer is not used as a fallback. Writing current readings against retired equipment would look like success while being wrong. + +### 3.3 — Represent "public but provisional" + +Built. Migration `b2c3d4e5f6a7` adds `data_maturity` to `transducer_observation`. + +**Decided: a `data_maturity` lexicon term, not an `is_provisional` boolean.** A boolean can only say provisional or not, and review is a progression rather than a switch. + +**Terms follow USGS usage** — `provisional`, `in review`, `approved`. `provisional` and `approved` are what USGS publishes against ("provisional data subject to revision" is the standard caveat on unapproved records). `in review` is the intermediate state from the Aquarius approval levels USGS uses for continuous time series (Working / In Review / Approved); Aquarius' `Working` is folded into `provisional`, because to a consumer the two are indistinguishable. + +- ✅ `release_status` keeps meaning visibility; `data_maturity` means trust. A reading can be `public` **and** `provisional` at once, which one column could not express — its lexicon lists them as siblings. There is a test asserting exactly that pair. +- ✅ `DataMaturity` enum, built from `core/lexicon.json` like every other status enum. That file is the source of truth the enums read; the migration seeds the database to match. +- ✅ Exposed on `TransducerObservationResponse` and accepted on `CreateTransducerObservation`, both nullable. +- ✅ The loader defaults new readings to `provisional`, and an upsert refreshes maturity along with value — a corrected reading arriving as approved must not keep the older maturity. +- ✅ The column is a foreign key onto `lexicon_term`, so a typo is rejected by the database. Tested. +- ✅ Migration verified up and down against a database with 88,666 observations. + +**Existing rows are backfilled from the legacy QC flag.** `transducer_observation` already carries `nma_waterlevelscontinuous_pressure_qced`, the AMPAPI field recording whether a reading was quality controlled — the same question `data_maturity` asks. True becomes `approved`, false becomes `provisional`. All 88,666 rows in the development database are `qced = true`, so they land as `approved`. + +Rows where that flag is NULL stay NULL: they did not come from the NMA transducer tables, so there is no evidence either way. + +`provisional` and `approved` already existed as terms: `lexicon_term.term` is globally unique and categories share terms through an association table, so only `in review` is new. That means `approved` is now shared by `review_status` and `data_maturity`. They are asking different questions — `review_status` on the block records that a Bureau human reviewed it and carries a `reviewer_id`, while `data_maturity` describes the reading's revision state — and the shared vocabulary is how this lexicon is designed to work. + +⬜ Blast radius still to check: `services/ngwmn_helper.py` filters `Thing.release_status == "public"` for NGWMN publication. San Acacia data becoming public needs to be intended there too. + +### 3.4 — Unique constraint on `transducer_observation` + idempotent upsert loader + +Built. Migration `a1b2c3d4e5f6`, loader in `automated_ingestion/ocotillo/loader.py`, three tests against a real Postgres. + +**The constraint is on `deployment_id`, not `thing_id`.** This section named a column the table does not have — `TransducerObservation` carries `deployment_id`, and `thing_id` lives on `TransducerObservationBlock`. The existing index was already `(deployment_id, parameter_id, observation_datetime)`, so the constraint matches it. Semantically this is also the right scope: a deployment is a thing/sensor pairing, so two sensors on one well may legitimately report the same instant. + +- ✅ Migration drops the redundant index — the unique constraint creates its own on the same columns, and keeping both means two indexes maintained on every insert into the largest table in the schema. Verified up and down against a database with 88,666 observations. +- ✅ `automated_ingestion/sql/find_duplicate_observations.sql` reports violations **before** the migration runs, since it fails on a table that already violates it and failing halfway through a production migration is worse than not starting. It separates redundant copies from groups whose `value` disagrees — the latter are not duplicates but conflicting measurements, and collapsing them silently would discard a reading. +- ✅ Loader upserts with `ON CONFLICT DO UPDATE`, batching at 5,000 rows and committing per batch. **`DO UPDATE`, not `DO NOTHING`:** a vendor correction arriving as a no-op would leave the old value in place while the run reported success. +- ✅ SQLAlchemy Core, not ORM objects, per `AGENTS.md` — instantiating a mapped class per observation is what turns a backfill into an hour-long run. +- ✅ `ensure_block` widens an existing block rather than duplicating it, and defaults `review_status` to `not reviewed`. +- ✅ Test: loading the same window twice leaves the row count unchanged. That claim depends on Postgres enforcing the constraint, so it runs against the real database rather than a stub. + +⬜ Run the duplicate report against production and staging before applying the migration. The local development database was clean — 0 duplicate groups in 88,666 rows — which is encouraging and not evidence about production. + +### Existing San Acacia data — measured 2026-08-19 + +Ocotillo already holds transducer observations for **14 of the 38 wells**: 542,161 rows from the AMPAPI transfer, running 2016-07-08 to **2022-08-03**. They carry a real QC status, so `data_maturity` backfilled them as `approved`. + +Consequences, all of which the earlier plan assumed away: + +- **The watermark starts at 2022-08-03 for those 14**, not the 2015 floor, so a normal run fetches a four-year gap rather than a decade. The other 24 wells do start at the floor. +- **A backfill would have overwritten them.** The upsert's `DO UPDATE` was written for vendor corrections to our own provisional readings; against approved AMPAPI history it would have replaced 542,161 reviewed values with vendor numbers *and* downgraded them to provisional. `load_observations` now refuses to touch an `approved` row unless `overwrite_approved=True` is passed deliberately. +- **A datum comparison is still owed.** Those rows came from AMPAPI under whatever convention that pipeline used; ours are Diver-HUB ground-surface centimetres converted to feet. Before any window overlapping 2016–2022 is loaded, a few coinciding timestamps should be compared. Same failure shape as the `WaterLevelReference` question: plausible numbers, wrong meaning. + +Rows with `NULL` maturity still update. Unknown is not approved, and treating it as such would freeze the 394,086 legacy rows that have no QC record against every future correction. + +**The wider table**, for context: 2,180,989 approved, 7,351 provisional, 394,086 NULL. The NULL cohort is 176 deployments on a single parameter spanning 2016 to February 2025 with no AMPAPI provenance at all — a separate network, and **none of the 38 San Acacia wells are in it**. Worth identifying independently of this work. + +### 3.5 — Watermark from Postgres + +Built. `automated_ingestion/shared/watermark.py`, seven tests. + +- ✅ `PostgresWatermarkStore` returns `MAX(observation_datetime)` for the series, read through the loader's own session so it reflects that session's committed state rather than another connection's snapshot. +- ✅ `InMemoryWatermarkStore` for tests and for reasoning about a run without a database. +- ✅ `resolve_start` falls back to the `initial_start_date` floor only for a series that has never been loaded. A test asserts a floor *ahead* of the watermark does not win either — the floor is not a backfill lever in any direction. +- ✅ The divergence from Aqueduct is in the module docstring, so it reads as a decision rather than an oversight. + +**Keyed by thing, not deployment.** This section said `(thing_id, parameter_id)` and that turns out to be right for a reason worth stating: observations carry `deployment_id`, but a series outlives its hardware. Replacing a diver creates a new deployment for the same well, and a watermark keyed to the deployment would report nothing for the new one and re-fetch the entire history. The query joins through `deployment` to ask the question the pipeline actually has. + +**Why derive rather than store.** A stored watermark is a second source of truth about what was loaded, and the two drift — a half-succeeded load, or a sidecar write that fails after the rows commit, leaves it claiming more or less than the data holds. Aqueduct stores one because FROST cannot be queried cheaply for a maximum; Postgres can. + +The payoff is that "backfill never advances the normal watermark" stops being a rule to enforce and becomes a property that cannot be violated: re-loading a window behind the maximum cannot move a maximum forward. Asserted anyway, in two directions — older data, and the same window twice. + +--- + +# TASK 4 — Backfill and operations + +A forward-only pipeline isn't enough. `BACKFILL_STRATEGY.md` §3 lists twelve situations demanding backfill; most come from ongoing operation, not onboarding — outage gaps, vendor corrections, adapter bugs found later, newly mapped properties. + +**Done when:** both backfill jobs are registry-generated, unscheduled, default `dry_run: true`, chunk by month sequentially in one run, and resume from the last completed chunk; the daily pipeline is scheduled and alerts a human on failure; docs carry the runbook. + +### 4.1 — Port shared backfill primitives from Aqueduct + +Built. `automated_ingestion/shared/backfill.py`, 27 tests, all pure — no database, no network. + +**Written from the described shape, not copied.** The Aqueduct checkout available here was an empty directory skeleton, so these were rebuilt from this plan's description rather than ported line by line. Each docstring records provenance and what differs, so the two can still be diffed by someone with both open. + +- ✅ `month_chunks`, `Chunk`, `ChunkResult`, `sum_chunk_results`, `parse_backfill_date`, `validate_date_order`, `attach_run_timestamp`, `sanitize_run_key`, `chunk_key`, `resolve_location_ids`, `CheckpointStore` + `InMemoryCheckpointStore`, `pending_chunks`. +- ⬜ `atomic_write_json_with_retry()` and a GCS-backed checkpoint store — deferred until 4.2 needs persistence. The interface is in place so the logic is testable without object storage. + +**`ChunkResult` counts `rows_upserted`,** replacing Aqueduct's `observations_posted` / `observations_deleted`. Not a rename: Aqueduct deletes a window and re-posts it because FROST has no constraint to conflict on, so it has two numbers and a window during which the data is missing. With 3.4's constraint Ocotillo upserts — one number, no window. + +Behaviour worth knowing: + +- **Chunk edges are clipped, not widened.** A window starting mid-month yields a first chunk starting mid-month, because widening would fetch data the operator did not ask for. +- **An empty or reversed window is rejected.** A backfill that reports success having done nothing is indistinguishable from one that worked, and the operator would not learn they typed the dates backwards. +- **An unknown `location_id` fails the run, naming every bad id.** Kept from Aqueduct deliberately: silently backfilling nothing looks exactly like backfilling successfully, and the gap is still there weeks later. +- **Run keys are sanitized** before becoming path segments. One containing a slash would write checkpoints into a directory of its own, and a resumed run would not find them. Marking under `march gap` and resuming under `march-gap` finds the same checkpoint. +- **A naive date is read as UTC**, so the same run config means the same window on every machine. + +### 4.2 — Backfill Mode A (refetch) + +Covers data never ingested: onboarding, a late-added well, an outage gap beyond the retry budget, a vendor correction, extending history past the original floor (§3A). + +- `san_acacia_backfill_refetch`, generated from the registry via a factory so a second source needs a registry entry, not new wiring. No schedule; launched from the Launchpad. +- Run config: `location_ids` (empty = every location the API returns), `start_date`, `end_date`, `run_key`, `dry_run`. +- **`dry_run: true` default.** Logs the full plan — entities, range, chunk list, expected counts — making exactly one read-only API call to resolve and validate the entity list, writing nothing. +- An unknown `location_id` fails the run naming the bad IDs, rather than silently backfilling nothing. +- Calendar-month chunks, sequential within one Dagster run — one billed run regardless of chunk count. +- Ingest writes to `vanessen_backfill_readings` under isolated dlt pipeline state, so backfill can't roll back or race the scheduled cursor. +- A chunk checkpoints only after ingest + transform + load all succeed; same `run_key` resumes from the last completed chunk. +- Same idempotent upsert as normal load — no delete step, no window where data is missing. +- Metadata reports per-chunk and total rows ingested, rows upserted, adapter failures. + +### 4.3 — Backfill Mode B (replay) + +Covers raw already in GCS with only the mapping wrong: adapter or unit bug, newly mapped property, storage migration, upstream rename, Ocotillo-side loss with parquet intact (§3B). Aqueduct notes this is almost entirely generic — build it that way. + +- `san_acacia_backfill_replay` from the same factory. Never contacts the Van Essen API. +- Reads raw parquet for an explicit range, filtered on event time, re-running the source's *current* adapter — so fixing a domain bug and replaying picks it up automatically. +- Same chunking, checkpointing, `dry_run: true` default, and upsert load path as Mode A. +- Source-agnostic: a second source gets replay free once it has an adapter and a registry entry. Anything that can't be generic is called out in the docstring. +- Test: a deliberately wrong mapping, once corrected, is fully repaired by a replay over the affected window. + +### 4.4 — Schedule, observability, alerting + +Schedule built. `defs/jobs/san_acacia.py` — `san_acacia_ingest` over the whole `san_acacia` asset group, on `san_acacia_weekly`. + +- ✅ **Weekly, not daily.** These are five-minute diver readings nobody watches in real time, the vendor's endpoint answers 500 when pushed, and the watermark makes the interval a question of freshness rather than correctness — a missed week is caught up by the next run, not lost. +- ✅ Mondays 05:00 **America/Denver**, not UTC. The wells, the people reading the data and the working day are in one timezone; a schedule drifting an hour twice a year would be the surprising choice. After midnight so a run covers whole days, early enough that a failure is visible at the start of the week. +- ✅ Selected **by group**, so an asset added to `san_acacia` joins the schedule without touching the job. A test asserts the selection resolves to exactly the three ingest assets and excludes `ingestion_heartbeat` and `database_connectivity` — a group-name typo would otherwise produce a schedule that runs happily and ingests nothing. +- ✅ `RetryPolicy(max_retries=2, delay=60)` covers a dropped request or a token expiring mid-run. Two attempts, not more: a persistent 500 means the window is wrong or the endpoint is unwell, and hammering it makes both worse. +- ✅ **`DefaultScheduleStatus.STOPPED`.** Turning it on starts writing to Ocotillo, and the first run for the 24 wells without history fetches back to `INITIAL_START`. That should be a decision taken once, not a consequence of a merge. +- ⬜ Observability and alerting — log bridge, failure notification, run metadata. + +### 4.5 — Documentation + +- `docs/sources/san_acacia.md` — confirmed mapping. +- `docs/ingestion-storage-conventions.md` — bucket/dataset/table naming, date partitioning, control-file convention, checklist for adding a source or agency. +- `docs/ingestion-backfill.md` — Modes A and B, chunking, checkpoints, `dry_run` policy, and why Ocotillo upserts where Aqueduct deletes-then-reposts. +- `automated_ingestion/README.md` — architecture, local dev, deploy path. Runbook: launching each mode, reading a dry-run plan, recovering a failed run. +- `CLAUDE.md` section pointing at the above, in the style of the existing "Domain Rules" section. +- "Adding a new source" checklist usable without reading the San Acacia implementation. + +--- + +## Open questions + +1. **Provisional representation** (3.3) — new boolean, new lexicon category, or something else? Recommendation is in the sub-task. +2. **NGWMN** — `release_status = "public"` makes San Acacia wells eligible for NGWMN publication via `services/ngwmn_helper.py`. Intended? +3. **Epic name** — keep "Automated Ingestion Pipeline", or use "Hydrograph Corrector" as originally asked? diff --git a/docs/hydrograph-correction-publish.md b/docs/hydrograph-correction-publish.md new file mode 100644 index 000000000..3d8a59fbb --- /dev/null +++ b/docs/hydrograph-correction-publish.md @@ -0,0 +1,144 @@ +# Hydrograph correction — publish and range delete + +The hydrograph corrector in OcotilloUI (`/ocotillo/hydrograph-correction`) +ingests a raw logger file, converts water head to depth below ground surface +against manual measurements, applies corrections, and publishes the result +here. This document records what the API side actually does; the UI-side +proposal it was built from is +`OcotilloUI/docs/hydrograph-correction-upload-contract.md`. + +## Authorization + +Both write routes are gated on **`AMP.Staging`**, a standalone Authentik group. +It is not a fourth rung on the AMP ladder: `AMPAdmin` does not satisfy it, and +it satisfies nothing else. Nobody holds it until it is granted, so the routes +ship dark and are reachable only by whoever is validating the workbench against +real logger files. + +When the workbench is trusted, these routes move to `amp_admin_dependency` and +the group goes away. Leaving it as a tier would make that a schema change +instead of a one-line edit. + +The read route stays on `amp_viewer_dependency` — it was already public to +viewers and publishing does not change who may look. + +## `POST /observation/transducer-groundwater-level/block` + +One corrected logger file becomes one block plus all of its readings, in one +transaction. + +- **The span is derived, not sent.** `start_datetime`/`end_datetime` come from + the min/max measurement timestamp. A client-supplied span wider than the data + would make the block claim readings it does not contain, because nothing links + the observation table to the block table — the reader pairs them by time. +- **`deployment_id` is optional.** Omitted, it is resolved from the deployments + on the well whose installation period covers the span. A NULL installation date + reads as "always installed", a NULL removal date as "still installed". Zero or + more than one match is a 422 telling the client to send it explicitly, because + guessing attributes readings to hardware that did not record them. +- **`data_maturity` is derived from `review_status`**, not sent: a block + published as `not reviewed` is `provisional` on USGS terms. Sending both + separately would let a client store a contradiction. +- **Provenance is part of the record.** `source_file`, `source_kind`, and the + ordered `corrections` list live on the block; `provenance.notes` lands in the + block's existing `comment`. A reviewer who cannot see that a series was + snapped to a manual measurement cannot review it. +- **Per-reading `note`** is set only where a correction moved the value, so NULL + means "as measured" rather than "unknown". + +- **`parameter_id` is validated, not obeyed.** The client states it explicitly, + per the contract, but the route checks it against the parameter the route is + scoped to. The read and delete routes on this path resolve groundwater level + themselves, so a block accepted under any other parameter would be a 201 for + data neither of them could ever list or remove. + +### Concurrency + +Both write paths read state, decide, and then write based on what they read, so +each takes a transaction-scoped advisory lock on `(thing_id, parameter_id)` +first — `pg_advisory_xact_lock`. + +Without it, two publishes with different timestamps but overlapping spans each +see no existing block and both commit: the unique constraints only catch +identical spans and identical readings, and the inclusive reader then has two +blocks claiming the same instants. Two range deletes each compute survivors from +a snapshot the other is invalidating, and the later update can widen a block back +over readings the earlier one removed. + +An advisory lock rather than row locks because on publish there is no row to +lock — the conflict is with a block that does not exist yet — so what needs +guarding is the series, not a row. Both paths take the same key, so they +serialize against each other and cannot deadlock against one another. + +### Overlap + +An existing block for the same well and parameter whose span shares any instant +with the new one is a **409** listing the collisions in +`detail[0].input.overlapping_blocks`. `?replace_overlapping=true` deletes those +blocks **and their readings** in the same transaction and then publishes. + +The readings have to go with the block. Keeping them would leave rows the reader +cannot show — no block covers them — that still occupy the +deployment/parameter/instant the new series is about to claim, so a "replace" +that kept them would fail on the very insert it was asked to make room for. + +Overlap here is **inclusive on both bounds**, unlike +`TransducerObservationBlock.overlaps` on the model, which is half-open. The +reader matches a reading to a block with `start <= t <= end`, so two blocks +sharing an endpoint both claim any reading at that instant — exactly the +ambiguity this check exists to prevent. + +Readings can also survive a block deleted by hand. Those are caught separately +and reported as a 409 naming the earliest colliding timestamp, rather than +letting the insert abort the transaction with a constraint name. + +## `DELETE /observation/transducer-groundwater-level` + +`thing_id`, `start_time`, and `end_time` are all required. There is deliberately +no unbounded form of this request. The scope matches the `GET` on the same path +exactly, so the set a client previews is the set this removes. + +Blocks are reconciled afterwards: one left with no readings is deleted, one left +with some has its span narrowed to the survivors. A block narrowed to a single +reading becomes zero-width, which the `end_datetime >= start_datetime` check +constraint allows on purpose (migration `c3d4e5f6a7b8`) and which the inclusive +reader still covers. + +That same migration renames the constraint from `check_transuder_...` to +`check_transducer_...`. Postgres cannot alter a check in place, so the +drop-and-recreate the relaxation already required was the free moment to fix +the spelling. The downgrade puts the old name back, so anything reaching for +the constraint by name has to pick the spelling that matches the revision it is +running against. + +**This leaves the `transducer_daily_data` materialized view stale** until its +next scheduled refresh. Nothing here refreshes it — a full refresh on every +delete would cost far more than the correctness it buys between nightly runs. + +## Two things fixed in passing + +- The read route was calling `get_transducer_observations` positionally, and the + helper's fourth positional parameter is `sensor_id`. `start_time` was landing + in `sensor_id` (unused, silently dropped), `end_time` was landing in + `start_time`, and `end_time` was never set — so an upper bound a caller asked + for was ignored and the lower bound came from the wrong argument. The call is + keyword-only now. +- The read route honours `sort` (`observation_datetime`, `value`, `id`) and + `order` (`asc`/`desc`), defaulting to newest first. An unrecognised sort field + or order is a 422 rather than being ignored — silently returning a differently + ordered page reads as the data changing, not as a bad request. `order` matters + particularly here: anything other than `asc` used to fall through to + descending, so the near-miss `order=ascending` returned 200 with the rows in + exactly the opposite order to the one asked for. + +## Not built + +Everything in the contract's "Supporting endpoints for Wellntel ingestion" +section is deferred: the `GET /wellntel/readings` proxy and the `sensor_type` +filter on `GET /thing`. Both are blocked on open questions the contract itself +raises — where the Wellntel API key lives and where the wellname→PointID mapping +belongs. The UI already falls back to demo data when they are absent. + +Also open, and unchanged by this work: whether the raw water-head series should +be retained alongside the corrected one, and whether publishing as `provisional` +should feed a review queue. diff --git a/docs/internal-ogc-desktop-gis.md b/docs/internal-ogc-desktop-gis.md new file mode 100644 index 000000000..7efa66403 --- /dev/null +++ b/docs/internal-ogc-desktop-gis.md @@ -0,0 +1,139 @@ +# Connecting ArcGIS Pro and QGIS to `/ogcapi-internal` + +The internal OGC API mount serves the unfiltered (private- and draft-inclusive) +collections. It is gated by `core/internal_ogc_auth.py`, an ASGI middleware that +runs in front of the raw Starlette Mount — FastAPI's `Depends()` machinery never +sees these requests, so none of the `*_dependency` role parameters apply here. + +## Why there are static API keys at all + +The mount originally accepted only `Authorization: Bearer `. +Neither desktop client can sustain that: + +- **ArcGIS Pro** cannot send a bearer token to an OGC API connection. Its + connection dialog offers Basic ("Server Authentication"), Esri-portal OAuth, + and "Custom request parameters" (appended to the request URL). Esri + [does not support token-secured OGC service connections](https://pro.arcgis.com/en/pro-app/latest/help/data/services/add-ogc-api-services.htm). +- **QGIS** can send one via its OAuth2 or API Header authentication methods, but + shipped a regression where OGC API - Features requests dropped the + Authorization header entirely ([qgis/QGIS#60473](https://github.com/qgis/QGIS/issues/60473)). + +Neither client can refresh an Authentik access token before it expires, so even +a working bearer flow means re-pasting a token every hour. A static key issued +per user solves both problems. + +## Accepted credentials + +| Transport | Carries | Used by | +| --- | --- | --- | +| `Authorization: Bearer ` | Authentik JWT **or** API key | QGIS OAuth2 / API Header, scripts | +| `Authorization: Basic ` | API key (or JWT) as the password | ArcGIS Pro, QGIS Basic | +| `?token=` | API key (or JWT) | ArcGIS Pro custom request parameters | + +A JWT must additionally carry the `OGCInternal` group (`INTERNAL_OGC_GROUP` in +`core/permissions.py`); a valid JWT without it gets 403. An API key is a +pre-authorized stand-in for that group and carries no per-user claims. + +The `?token=` value is stripped from the query string before the request reaches +pygeoapi, so it never lands in the `self`/`next` links pygeoapi echoes into +response bodies. It is still recorded in App Engine's request log — prefer Basic +where the client supports it. + +## Where the keys live + +Only the **SHA-256 digests** are stored, never the keys themselves. The digest +list lives in a Google Secret Manager secret named `internal-ogc-api-keys`, one +per GCP project (production, staging, testing) — the same place the Jira and +Slack credentials live, not a GitHub secret. + +CD reads it at deploy time (`Fetch application secrets from Secret Manager` in +each `.github/workflows/CD_*.yml`) and `envsubst` renders it into `app.yaml` as +the `INTERNAL_OGC_API_KEYS` environment variable, which +`core/internal_ogc_auth.py` parses. The app makes no Secret Manager call at +runtime. + +Consequences worth knowing: + +- **The secret must exist before the next deploy of any environment.** + `get-secretmanager-secrets` fails the whole job on a missing secret. Seed each + project with a placeholder that parses to zero keys: + + ```bash + printf 'placeholder:none' | gcloud secrets create internal-ogc-api-keys --data-file=- --project + ``` + + The parser skips any entry whose digest is not 64 hex characters, so that + value is inert and means "bearer-JWT access only". + +- **Revoking a key requires a redeploy.** Adding a secret version does not + affect a running instance. If revocation ever needs to be immediate, that is + the point to switch to a runtime fetch with a TTL cache (same shape as the + JWKS cache in `core/permissions.py`) or to a keys table in Postgres. + +- The deploy service account needs `roles/secretmanager.secretAccessor` on + `internal-ogc-api-keys` in each project, alongside the four it already has. + +## Issuing a key + +```bash +python -c "import secrets,hashlib;k=secrets.token_urlsafe(32);print('key: ',k);print('digest:',hashlib.sha256(k.encode()).hexdigest())" +``` + +Give the **key** to the user over a secure channel and keep only the digest. +Append `