From a0fca892a3fd7d93aedcd6d615c38903cb62aba8 Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 20 Aug 2026 15:13:43 -0700 Subject: [PATCH 1/5] feat(cm): mirror the critical-minerals workbook into CM_legacy Phase-1 staging mirror of the Earth MRI critical-minerals chemistry workbook (McLemoreMasterChem, NMBGMR) into CM_* tables, plus a reconciliation report for the source-data owner. No transform into the Ocotillo data model yet. Seven mirror tables. The ChemicalData, GIS and QAQC sheets share a column set, so they land in one CM_ChemicalData behind a source_sheet discriminator: the first two are byte-identical in header text and order, and QAQC is those columns minus MapSymbol/Pd/Pt with latitude/longitude capitalized. Rows are keyed on (source_sheet, source_row) rather than SAMPLE, which repeats across 258 names. GIS is a stale hand-maintained fork of ChemicalData, not a location-enriched copy: it carries the same mixed datums and the same ~876 rows with no latitude, and 1704 of the 4848 shared rows disagree in both directions (GIS holds 533 Chem Lab File No., 485 Laboratory and 170 FeO/Fe2O3 values ChemicalData lacks; ChemicalData holds 633 Total, 184 Area and 18 appended Pearce (2020) samples GIS lacks). Neither sheet is authoritative, so both are mirrored in full and reconciliation is deferred to a per-column ruling by V.T. McLemore. The loader warns whenever the two row counts diverge. Every column is a String. The workbook stores censored analyte values as text (1154 '<' values in Au alone), carries '#VALUE!' errors, and mixes real dates with year-only text; parsing belongs to the transform. Column names are derived mechanically because sheet headers are not SQL identifiers, with analytes carrying the unit the workbook declares for them - which also keeps As and In off the Python and SQL keyword lists. Loading is idempotent per sheet and asserts the layout instead of guessing it: a moved header row or an unmapped column aborts the load rather than silently dropping cells. scripts/cm_reconciliation_report.py builds the decision workbook - the sheet-to-sheet drift plus 4276 integrity findings across 13 issue types, among them 724 values impossible for their declared unit (F = 27700 in a % column), 259 Au detection limits that are ppm in a ppb column, 300 non-numeric analyte tokens, and 64 of 84 analytes reported against more than three distinct detection limits. Verified by loading the delivered workbook: 10,100 rows, and 708 cells across 6 random rows compared against openpyxl with zero mismatches. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 17 + .../d4e5f6a7b8c9_cm_legacy_mirror_tables.py | 296 +++++ cli/cli.py | 34 + db/__init__.py | 1 + db/cm_legacy.py | 725 ++++++++++++ docs/critical-minerals-legacy-mirror.md | 193 ++++ scripts/cm_reconciliation_report.py | 1010 +++++++++++++++++ services/cm_legacy_mirror.py | 358 ++++++ tests/test_cm_legacy.py | 484 ++++++++ tests/test_cm_reconciliation_report.py | 305 +++++ 10 files changed, 3423 insertions(+) create mode 100644 alembic/versions/d4e5f6a7b8c9_cm_legacy_mirror_tables.py create mode 100644 db/cm_legacy.py create mode 100644 docs/critical-minerals-legacy-mirror.md create mode 100644 scripts/cm_reconciliation_report.py create mode 100644 services/cm_legacy_mirror.py create mode 100644 tests/test_cm_legacy.py create mode 100644 tests/test_cm_reconciliation_report.py diff --git a/CLAUDE.md b/CLAUDE.md index 5e5d14a06..e5196d0cf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -315,6 +315,23 @@ Still live, *not* deprecated: `services/scoped_transfer.py` and the `oco scoped-transfer` command, which import the individual NM_Aquifer transferers directly. +### Critical minerals mirror (`CM_legacy`) + +`db/cm_legacy.py` + `services/cm_legacy_mirror.py` mirror the McLemore Earth MRI +critical-minerals chemistry workbook into `CM_*` tables +(`oco load-critical-minerals-workbook --file ...`). Phase 1 staging only; no +transform into the Ocotillo model yet. Every column is a `String` because the +workbook stores censored analyte values as text (`<0.1`) and carries `#VALUE!` +errors. + +The workbook's `ChemicalData`, `GIS` and `QAQC` sheets share a column set and +mirror into one table behind a `source_sheet` discriminator. **`GIS` is a stale +fork of `ChemicalData`, not a location-enriched copy** — each sheet holds values +the other lacks, so reading one `source_sheet` alone silently drops data. +Reconciliation is deliberately deferred. Read +**`docs/critical-minerals-legacy-mirror.md`** before querying or extending this +layer. + **Source**: AMPAPI (SQL Server, `NM_Aquifer` schema) **Target**: OcotilloAPI (PostgreSQL + PostGIS) diff --git a/alembic/versions/d4e5f6a7b8c9_cm_legacy_mirror_tables.py b/alembic/versions/d4e5f6a7b8c9_cm_legacy_mirror_tables.py new file mode 100644 index 000000000..6addda549 --- /dev/null +++ b/alembic/versions/d4e5f6a7b8c9_cm_legacy_mirror_tables.py @@ -0,0 +1,296 @@ +"""CM_legacy staging mirror tables + +Revision ID: d4e5f6a7b8c9 +Revises: c3d4e5f6a7b8 +Create Date: 2026-08-20 + +1:1 staging mirror of the McLemore critical-minerals chemistry workbook +(Earth MRI, NMBGMR; see db/cm_legacy.py and +docs/critical-minerals-legacy-mirror.md). Faithful copies of the workbook +sheets; the transform into the Ocotillo data model is a later phase. + + ChemicalData / GIS / QAQC -> CM_ChemicalData (source_sheet discriminator) + DetectionLimits -> CM_DetectionLimits + References -> CM_References + MineralSystems -> CM_MineralSystems + world -> CM_WorldComparisons + world_ref -> CM_WorldReferences + General Information / MetaData / DefinitionOfFields + -> CM_WorkbookMetadata + +Every data column is a String: the workbook stores censored analyte values as +text ("<0.1"), carries Excel error text ("#VALUE!"), and mixes dates with free +text. Parsing belongs to the transform. + +The ChemicalData, GIS and QAQC sheets disagree with each other and none is +authoritative, so all three are mirrored and reconciliation is deferred. See +the module docstring in db/cm_legacy.py. +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "d4e5f6a7b8c9" +down_revision: Union[str, Sequence[str], None] = "c3d4e5f6a7b8" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "CM_ChemicalData", + sa.Column("id", sa.Integer(), nullable=False, autoincrement=True), + sa.Column("source_sheet", sa.String(), nullable=False), + sa.Column("source_row", sa.Integer(), nullable=False), + sa.Column("sample", sa.String(), nullable=True), + sa.Column("project", sa.String(), nullable=True), + sa.Column("area", sa.String(), nullable=True), + sa.Column("reference", sa.String(), nullable=True), + sa.Column("date_collected", sa.String(), nullable=True), + sa.Column("date_analyzed", sa.String(), nullable=True), + sa.Column("chem_lab_file_no", sa.String(), nullable=True), + sa.Column("laboratory", sa.String(), nullable=True), + sa.Column("latitude", sa.String(), nullable=True), + sa.Column("longitude", sa.String(), nullable=True), + sa.Column("coordinate_system", sa.String(), nullable=True), + sa.Column("county", sa.String(), nullable=True), + sa.Column("state", sa.String(), nullable=True), + sa.Column("lithology", sa.String(), nullable=True), + sa.Column("mineral_system", sa.String(), nullable=True), + sa.Column("deposit_types", sa.String(), nullable=True), + sa.Column("map_symbol", sa.String(), nullable=True), + sa.Column("method_collected", sa.String(), nullable=True), + sa.Column("sample_source", sa.String(), nullable=True), + sa.Column("mineralogy_deposit_type", sa.String(), nullable=True), + sa.Column("depth_legnth_ft", sa.String(), nullable=True), + sa.Column("mine_id", sa.String(), nullable=True), + sa.Column("location_notes", sa.String(), nullable=True), + sa.Column("comments", sa.String(), nullable=True), + sa.Column("paste_ph", sa.String(), nullable=True), + sa.Column("paste_conductivity", sa.String(), nullable=True), + sa.Column("tds_mg_l", sa.String(), nullable=True), + sa.Column("sio2_pct", sa.String(), nullable=True), + sa.Column("tio2_pct", sa.String(), nullable=True), + sa.Column("al2o3_pct", sa.String(), nullable=True), + sa.Column("fe2o3t_pct", sa.String(), nullable=True), + sa.Column("mno_pct", sa.String(), nullable=True), + sa.Column("mgo_pct", sa.String(), nullable=True), + sa.Column("cao_pct", sa.String(), nullable=True), + sa.Column("na2o_pct", sa.String(), nullable=True), + sa.Column("k2o_pct", sa.String(), nullable=True), + sa.Column("p2o5_pct", sa.String(), nullable=True), + sa.Column("loi_pct", sa.String(), nullable=True), + sa.Column("f_pct", sa.String(), nullable=True), + sa.Column("s_pct", sa.String(), nullable=True), + sa.Column("so3_pct", sa.String(), nullable=True), + sa.Column("so4_pct", sa.String(), nullable=True), + sa.Column("c_pct", sa.String(), nullable=True), + sa.Column("co2_pct", sa.String(), nullable=True), + sa.Column("total_pct", sa.String(), nullable=True), + sa.Column("feo_pct", sa.String(), nullable=True), + sa.Column("fe2o3_pct", sa.String(), nullable=True), + sa.Column("feo_star_pct", sa.String(), nullable=True), + sa.Column("h2o_plus_pct", sa.String(), nullable=True), + sa.Column("h2o_minus_pct", sa.String(), nullable=True), + sa.Column("au_ppb", sa.String(), nullable=True), + sa.Column("ag_ppm", sa.String(), nullable=True), + sa.Column("as_ppm", sa.String(), nullable=True), + sa.Column("b_ppm", sa.String(), nullable=True), + sa.Column("ba_ppm", sa.String(), nullable=True), + sa.Column("be_ppm", sa.String(), nullable=True), + sa.Column("bi_ppm", sa.String(), nullable=True), + sa.Column("br_ppm", sa.String(), nullable=True), + sa.Column("cd_ppm", sa.String(), nullable=True), + sa.Column("cl_ppm", sa.String(), nullable=True), + sa.Column("co_ppm", sa.String(), nullable=True), + sa.Column("cr_ppm", sa.String(), nullable=True), + sa.Column("cs_ppm", sa.String(), nullable=True), + sa.Column("cu_ppm", sa.String(), nullable=True), + sa.Column("ga_ppm", sa.String(), nullable=True), + sa.Column("ge_ppm", sa.String(), nullable=True), + sa.Column("hf_ppm", sa.String(), nullable=True), + sa.Column("hg_ppm", sa.String(), nullable=True), + sa.Column("in_ppm", sa.String(), nullable=True), + sa.Column("li_ppm", sa.String(), nullable=True), + sa.Column("mo_ppm", sa.String(), nullable=True), + sa.Column("nb_ppm", sa.String(), nullable=True), + sa.Column("ni_ppm", sa.String(), nullable=True), + sa.Column("pd_ppm", sa.String(), nullable=True), + sa.Column("pb_ppm", sa.String(), nullable=True), + sa.Column("pt_ppm", sa.String(), nullable=True), + sa.Column("rb_ppm", sa.String(), nullable=True), + sa.Column("re_ppm", sa.String(), nullable=True), + sa.Column("sb_ppm", sa.String(), nullable=True), + sa.Column("sc_ppm", sa.String(), nullable=True), + sa.Column("se_ppm", sa.String(), nullable=True), + sa.Column("sn_ppm", sa.String(), nullable=True), + sa.Column("sr_ppm", sa.String(), nullable=True), + sa.Column("ta_ppm", sa.String(), nullable=True), + sa.Column("te_ppm", sa.String(), nullable=True), + sa.Column("th_ppm", sa.String(), nullable=True), + sa.Column("tl_ppm", sa.String(), nullable=True), + sa.Column("u_ppm", sa.String(), nullable=True), + sa.Column("v_ppm", sa.String(), nullable=True), + sa.Column("w_ppm", sa.String(), nullable=True), + sa.Column("y_ppm", sa.String(), nullable=True), + sa.Column("zn_ppm", sa.String(), nullable=True), + sa.Column("zr_ppm", sa.String(), nullable=True), + sa.Column("la_ppm", sa.String(), nullable=True), + sa.Column("ce_ppm", sa.String(), nullable=True), + sa.Column("pr_ppm", sa.String(), nullable=True), + sa.Column("nd_ppm", sa.String(), nullable=True), + sa.Column("sm_ppm", sa.String(), nullable=True), + sa.Column("eu_ppm", sa.String(), nullable=True), + sa.Column("gd_ppm", sa.String(), nullable=True), + sa.Column("tb_ppm", sa.String(), nullable=True), + sa.Column("dy_ppm", sa.String(), nullable=True), + sa.Column("ho_ppm", sa.String(), nullable=True), + sa.Column("er_ppm", sa.String(), nullable=True), + sa.Column("tm_ppm", sa.String(), nullable=True), + sa.Column("yb_ppm", sa.String(), nullable=True), + sa.Column("lu_ppm", sa.String(), nullable=True), + sa.Column("tree_ppm", sa.String(), nullable=True), + sa.Column("mn_pct", sa.String(), nullable=True), + sa.Column("fe_pct", sa.String(), nullable=True), + sa.Column("al_pct", sa.String(), nullable=True), + sa.Column("ca_pct", sa.String(), nullable=True), + sa.Column("na_pct", sa.String(), nullable=True), + sa.Column("k_pct", sa.String(), nullable=True), + sa.Column("mg_pct", sa.String(), nullable=True), + sa.Column("p_pct", sa.String(), nullable=True), + sa.Column("si_pct", sa.String(), nullable=True), + sa.Column("ti_pct", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "source_sheet", "source_row", name="uq_cm_chemical_data_source_row" + ), + ) + op.create_index("ix_CM_ChemicalData_area", "CM_ChemicalData", ["area"]) + op.create_index("ix_CM_ChemicalData_sample", "CM_ChemicalData", ["sample"]) + op.create_index( + "ix_CM_ChemicalData_source_sheet", "CM_ChemicalData", ["source_sheet"] + ) + + op.create_table( + "CM_DetectionLimits", + sa.Column("id", sa.Integer(), nullable=False, autoincrement=True), + sa.Column("source_row", sa.Integer(), nullable=False), + sa.Column("method", sa.String(), nullable=True), + sa.Column("element", sa.String(), nullable=True), + sa.Column("lower_reporting_limit", sa.String(), nullable=True), + sa.Column("unit", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("source_row", name="uq_cm_detection_limits_source_row"), + ) + op.create_index("ix_CM_DetectionLimits_element", "CM_DetectionLimits", ["element"]) + + op.create_table( + "CM_References", + sa.Column("id", sa.Integer(), nullable=False, autoincrement=True), + sa.Column("source_row", sa.Integer(), nullable=False), + sa.Column("citation", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("source_row", name="uq_cm_references_source_row"), + ) + + op.create_table( + "CM_MineralSystems", + sa.Column("id", sa.Integer(), nullable=False, autoincrement=True), + sa.Column("source_row", sa.Integer(), nullable=False), + sa.Column("system_name", sa.String(), nullable=True), + sa.Column("synopsis", sa.String(), nullable=True), + sa.Column("deposit_types", sa.String(), nullable=True), + sa.Column("principal_commodities", sa.String(), nullable=True), + sa.Column("critical_minerals", sa.String(), nullable=True), + sa.Column("references", sa.String(), nullable=True), + sa.Column("phase_2", sa.String(), nullable=True), + sa.Column("phase_3", sa.String(), nullable=True), + sa.Column("phase_4", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("source_row", name="uq_cm_mineral_systems_source_row"), + ) + + op.create_table( + "CM_WorldComparisons", + sa.Column("id", sa.Integer(), nullable=False, autoincrement=True), + sa.Column("source_row", sa.Integer(), nullable=False), + sa.Column("area", sa.String(), nullable=True), + sa.Column("deposit", sa.String(), nullable=True), + sa.Column("reference", sa.String(), nullable=True), + sa.Column("la", sa.String(), nullable=True), + sa.Column("ce", sa.String(), nullable=True), + sa.Column("pr", sa.String(), nullable=True), + sa.Column("nd", sa.String(), nullable=True), + sa.Column("sm", sa.String(), nullable=True), + sa.Column("eu", sa.String(), nullable=True), + sa.Column("gd", sa.String(), nullable=True), + sa.Column("tb", sa.String(), nullable=True), + sa.Column("dy", sa.String(), nullable=True), + sa.Column("ho", sa.String(), nullable=True), + sa.Column("er", sa.String(), nullable=True), + sa.Column("tm", sa.String(), nullable=True), + sa.Column("yb", sa.String(), nullable=True), + sa.Column("lu", sa.String(), nullable=True), + sa.Column("tree", sa.String(), nullable=True), + sa.Column("sc", sa.String(), nullable=True), + sa.Column("y", sa.String(), nullable=True), + sa.Column("metric_tons", sa.String(), nullable=True), + sa.Column("grade_pct", sa.String(), nullable=True), + sa.Column("total_ree", sa.String(), nullable=True), + sa.Column("cutoff_grade_pct", sa.String(), nullable=True), + sa.Column("la2o3", sa.String(), nullable=True), + sa.Column("ce2o3", sa.String(), nullable=True), + sa.Column("pr6o11", sa.String(), nullable=True), + sa.Column("nd2o3", sa.String(), nullable=True), + sa.Column("sm2o3", sa.String(), nullable=True), + sa.Column("eu2o3", sa.String(), nullable=True), + sa.Column("gd2o3", sa.String(), nullable=True), + sa.Column("tb4o7", sa.String(), nullable=True), + sa.Column("dy2o3", sa.String(), nullable=True), + sa.Column("ho2o3", sa.String(), nullable=True), + sa.Column("er2o3", sa.String(), nullable=True), + sa.Column("tm2o3", sa.String(), nullable=True), + sa.Column("yb2o3", sa.String(), nullable=True), + sa.Column("lu2o3", sa.String(), nullable=True), + sa.Column("y2o3", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("source_row", name="uq_cm_world_comparisons_source_row"), + ) + + op.create_table( + "CM_WorldReferences", + sa.Column("id", sa.Integer(), nullable=False, autoincrement=True), + sa.Column("source_row", sa.Integer(), nullable=False), + sa.Column("citation", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("source_row", name="uq_cm_world_references_source_row"), + ) + + op.create_table( + "CM_WorkbookMetadata", + sa.Column("id", sa.Integer(), nullable=False, autoincrement=True), + sa.Column("source_sheet", sa.String(), nullable=False), + sa.Column("source_row", sa.Integer(), nullable=False), + sa.Column("label", sa.String(), nullable=True), + sa.Column("value", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "source_sheet", "source_row", name="uq_cm_workbook_metadata_source_row" + ), + ) + + +def downgrade() -> None: + op.drop_table("CM_WorkbookMetadata") + op.drop_table("CM_WorldReferences") + op.drop_table("CM_WorldComparisons") + op.drop_table("CM_MineralSystems") + op.drop_table("CM_References") + op.drop_index("ix_CM_DetectionLimits_element", table_name="CM_DetectionLimits") + op.drop_table("CM_DetectionLimits") + op.drop_index("ix_CM_ChemicalData_area", table_name="CM_ChemicalData") + op.drop_index("ix_CM_ChemicalData_sample", table_name="CM_ChemicalData") + op.drop_index("ix_CM_ChemicalData_source_sheet", table_name="CM_ChemicalData") + op.drop_table("CM_ChemicalData") diff --git a/cli/cli.py b/cli/cli.py index b4ff204c6..dd4c03dea 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -1359,6 +1359,40 @@ def import_project_area_boundaries_command( ) +@cli.command("load-critical-minerals-workbook") +def load_critical_minerals_workbook_command( + file_path: str = typer.Option( + ..., + "--file", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + help="Path to the McLemoreMasterChem .xlsx critical-minerals workbook.", + ), +): + """ + mirror the McLemore critical-minerals workbook into the CM_legacy staging + tables. Every sheet is copied unchanged -- including the ChemicalData, GIS + and QAQC sheets, which disagree with each other -- and each sheet's rows + replace whatever was loaded for it before. Values are not parsed; see + docs/critical-minerals-legacy-mirror.md. + """ + from db.engine import session_ctx + from services.cm_legacy_mirror import load_cm_workbook + + with session_ctx() as session: + result = load_cm_workbook(file_path, session) + session.commit() + + typer.echo(f"Mirrored {result.total_rows} row(s) from {file_path}:") + width = max(len(sheet) for sheet in result.rows_by_sheet) + for sheet, count in result.rows_by_sheet.items(): + typer.echo(f" {sheet:<{width}} | {count:>6}") + for warning in result.warnings: + typer.echo(f"warning: {warning}", err=True) + + if __name__ == "__main__": cli() diff --git a/db/__init__.py b/db/__init__.py index 4e2e7fb3a..24187c31e 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -58,6 +58,7 @@ from db.thing_aquifer_association import * from db.thing_geologic_formation_association import * from db.aquifer_type import * +from db.cm_legacy import * from db.nma_legacy import * from db.nmw_legacy import * from db.transducer import * diff --git a/db/cm_legacy.py b/db/cm_legacy.py new file mode 100644 index 000000000..5f69dad7d --- /dev/null +++ b/db/cm_legacy.py @@ -0,0 +1,725 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== + +"""1:1 staging mirror of the McLemore critical-minerals chemistry workbook. + +PURPOSE +------- +These models are a FAITHFUL copy of the Earth MRI "Database of chemical +analyses of critical minerals deposits in New Mexico" workbook +(``McLemoreMasterChem``, compiled by V.T. McLemore et al., NMBGMR). They are a +*staging layer*: workbook cells land here unchanged, then a later transform +phase maps them into the Ocotillo data model (Location / Thing / FieldEvent / +FieldActivity / Sample / Observation). + +Mirrors the convention of ``db/nma_legacy.py`` and ``db/nmw_legacy.py``: +``CM_`` table prefix, one table per source sheet, no enforced foreign keys +between mirror tables, no interpretation of values. + +SOURCE SHEETS +------------- + ChemicalData -> CM_ChemicalData (source_sheet='ChemicalData') + GIS -> CM_ChemicalData (source_sheet='GIS') + QAQC -> CM_ChemicalData (source_sheet='QAQC') + DetectionLimits -> CM_DetectionLimits + References -> CM_References + MineralSystems -> CM_MineralSystems + world -> CM_WorldComparisons + world_ref -> CM_WorldReferences + General Information / MetaData / DefinitionOfFields -> CM_WorkbookMetadata + +THREE SHEETS, ONE TABLE +----------------------- +``ChemicalData``, ``GIS`` and ``QAQC`` are mirrored into a single table keyed by +a ``source_sheet`` discriminator because they carry the SAME columns: +``ChemicalData`` and ``GIS`` are byte-identical in header text and order (118 +columns each); ``QAQC`` is those columns minus ``MapSymbol``/``Pd``/``Pt``, with +``latitude``/``longitude`` capitalized. Only the layout differs -- ``GIS`` has a +single header row while ``ChemicalData`` has a title banner (row 1), a header +(row 2) and a units row (row 3). + +RECONCILIATION IS DEFERRED (READ THIS BEFORE QUERYING) +------------------------------------------------------ +``GIS`` is NOT ``ChemicalData`` plus location data, and it is not a clean +subset. It is a stale, hand-maintained fork. As of the 2025-09-17 revision, +every ``GIS`` sample name exists in ``ChemicalData``, but 1704 of the 4848 +shared rows disagree cell-for-cell, in BOTH directions: + + GIS has values ChemicalData lacks: 533 Chem Lab File No., 485 Laboratory, + 85 FeO, 85 Fe2O3 + ChemicalData has values GIS lacks: 633 Total, 184 Area, 11 TREE, + 4 Date analyzed + Outright disagreement: 16 rows on Area ('ZuniMountains'/'Zuni') + Broken formulas: 12 '#VALUE!' Totals in GIS (1 in CD) + +``ChemicalData`` also holds 18 sample rows appended after ``GIS`` was last +synced (BP*, CR1, JP*, SA*, SL*). Neither sheet is authoritative, so BOTH are +mirrored in full and reconciliation is deliberately left to a later phase, to +be ruled on per column by V.T. McLemore. Do not treat any single +``source_sheet`` as complete, and do not de-duplicate across sheets here. + +EVERY COLUMN IS A STRING +------------------------ +Analyte columns hold censored values as text (``<0.1``, ``<10``, ``<0.06``), +Excel error text (``#VALUE!``), and blanks; date columns mix real dates with +free text. Parsing value-plus-qualifier and casting dates belongs to the +transform phase, cross-checked against ``CM_DetectionLimits``. Storing +everything as ``String`` keeps the mirror loadable without dropping cells. + +COLUMN NAMING +------------- +Sheet headers are spreadsheet labels, not SQL identifiers ("Chem Lab File No.", +"Depth/legnth (ft)", "H2O+"), so unlike the NMA/NMW mirrors the source name +cannot be reused verbatim. Names here are derived mechanically: + +* snake_case, lowercased, non-alphanumerics collapsed to ``_``; +* ``+`` -> ``_plus``, ``*`` -> ``_star``, ``%`` -> ``_pct``; +* analyte columns carry the unit the workbook declares for them in its units + row (``sio2_pct``, ``au_ppb``, ``as_ppm``), which also keeps ``As`` and + ``In`` from colliding with the Python/SQL keywords ``as`` and ``in``; +* the source typo in "Depth/legnth (ft)" is preserved as ``depth_legnth_ft`` so + the mapping back to the sheet stays mechanical. + +``SOURCE_HEADER_BY_COLUMN`` records the exact source header for every column, and +``ANALYTE_UNITS`` records the workbook-declared unit for every analyte, so the +units row is not lost. + +ROW IDENTITY +------------ +Sample names are NOT unique in the source (``S1``, ``S10``, ``S100`` and ~340 +others repeat), so mirror rows are keyed on ``(source_sheet, source_row)``, +where ``source_row`` is the 1-based Excel row number. That makes every mirror +row traceable to a cell range in the delivered workbook, and makes a reload +idempotent per sheet. + +KNOWN JUNK IN THE DATA RANGE +---------------------------- +The first data row of ``ChemicalData`` is not a sample: its SAMPLE cell holds +"NOTE: SEE THE ORIGINAL CITATION FOR INFORMATION ON METHODS OF ANALYSES, +QA/QC, DETECTION LIMITS, ETC.". It is mirrored like any other row (fidelity) +and must be excluded by the transform. +""" + +from sqlalchemy import Integer, String, UniqueConstraint +from sqlalchemy.orm import mapped_column + +from db.base import Base + +# Source sheets mirrored into CM_ChemicalData. +CM_SHEET_CHEMICAL_DATA = "ChemicalData" +CM_SHEET_GIS = "GIS" +CM_SHEET_QAQC = "QAQC" +CM_CHEMISTRY_SOURCE_SHEETS = ( + CM_SHEET_CHEMICAL_DATA, + CM_SHEET_GIS, + CM_SHEET_QAQC, +) + +# Columns absent from the QAQC sheet; always NULL for source_sheet='QAQC'. +QAQC_MISSING_COLUMNS = ("map_symbol", "pd_ppm", "pt_ppm") + + +# Unit the workbook's units row (ChemicalData row 3) declares for each analyte +# column. Recorded here because the units row is schema-level, not row-level. +ANALYTE_UNITS: dict[str, str] = { + "sio2_pct": "%", + "tio2_pct": "%", + "al2o3_pct": "%", + "fe2o3t_pct": "%", + "mno_pct": "%", + "mgo_pct": "%", + "cao_pct": "%", + "na2o_pct": "%", + "k2o_pct": "%", + "p2o5_pct": "%", + "loi_pct": "%", + "f_pct": "%", + "s_pct": "%", + "so3_pct": "%", + "so4_pct": "%", + "c_pct": "%", + "co2_pct": "%", + "total_pct": "%", + "feo_pct": "%", + "fe2o3_pct": "%", + "feo_star_pct": "%", + "h2o_plus_pct": "%", + "h2o_minus_pct": "%", + "au_ppb": "ppb", + "ag_ppm": "ppm", + "as_ppm": "ppm", + "b_ppm": "ppm", + "ba_ppm": "ppm", + "be_ppm": "ppm", + "bi_ppm": "ppm", + "br_ppm": "ppm", + "cd_ppm": "ppm", + "cl_ppm": "ppm", + "co_ppm": "ppm", + "cr_ppm": "ppm", + "cs_ppm": "ppm", + "cu_ppm": "ppm", + "ga_ppm": "ppm", + "ge_ppm": "ppm", + "hf_ppm": "ppm", + "hg_ppm": "ppm", + "in_ppm": "ppm", + "li_ppm": "ppm", + "mo_ppm": "ppm", + "nb_ppm": "ppm", + "ni_ppm": "ppm", + "pd_ppm": "ppm", + "pb_ppm": "ppm", + "pt_ppm": "ppm", + "rb_ppm": "ppm", + "re_ppm": "ppm", + "sb_ppm": "ppm", + "sc_ppm": "ppm", + "se_ppm": "ppm", + "sn_ppm": "ppm", + "sr_ppm": "ppm", + "ta_ppm": "ppm", + "te_ppm": "ppm", + "th_ppm": "ppm", + "tl_ppm": "ppm", + "u_ppm": "ppm", + "v_ppm": "ppm", + "w_ppm": "ppm", + "y_ppm": "ppm", + "zn_ppm": "ppm", + "zr_ppm": "ppm", + "la_ppm": "ppm", + "ce_ppm": "ppm", + "pr_ppm": "ppm", + "nd_ppm": "ppm", + "sm_ppm": "ppm", + "eu_ppm": "ppm", + "gd_ppm": "ppm", + "tb_ppm": "ppm", + "dy_ppm": "ppm", + "ho_ppm": "ppm", + "er_ppm": "ppm", + "tm_ppm": "ppm", + "yb_ppm": "ppm", + "lu_ppm": "ppm", + "tree_ppm": "ppm", + "mn_pct": "%", + "fe_pct": "%", + "al_pct": "%", + "ca_pct": "%", + "na_pct": "%", + "k_pct": "%", + "mg_pct": "%", + "p_pct": "%", + "si_pct": "%", + "ti_pct": "%", +} + + +# Exact source header for every mirrored column, per table. The mirror column +# names are derived (see COLUMN NAMING above); this is the round trip back to +# the delivered workbook. +SOURCE_HEADER_BY_COLUMN: dict[str, dict[str, str]] = { + "CM_ChemicalData": { + "sample": "SAMPLE", + "project": "Project", + "area": "Area", + "reference": "Reference", + "date_collected": "Date collected", + "date_analyzed": "Date analyzed", + "chem_lab_file_no": "Chem Lab File No.", + "laboratory": "Laboratory", + "latitude": "latitude", + "longitude": "longitude", + "coordinate_system": "Coordinate system", + "county": "County", + "state": "State", + "lithology": "lithology", + "mineral_system": "Mineral system", + "deposit_types": "Deposit type(s) (from Mineral Systems table)", + "map_symbol": "MapSymbol", + "method_collected": "method collected", + "sample_source": "sample source", + "mineralogy_deposit_type": "MineralogyDepositType", + "depth_legnth_ft": "Depth/legnth (ft)", + "mine_id": "Mine ID", + "location_notes": "LocationNotes", + "comments": "Comments", + "paste_ph": "paste pH", + "paste_conductivity": "paste conductivity", + "tds_mg_l": "TDS (mg/l)", + "sio2_pct": "SiO2", + "tio2_pct": "TiO2", + "al2o3_pct": "Al2O3", + "fe2o3t_pct": "Fe2O3T", + "mno_pct": "MnO", + "mgo_pct": "MgO", + "cao_pct": "CaO", + "na2o_pct": "Na2O", + "k2o_pct": "K2O", + "p2o5_pct": "P2O5", + "loi_pct": "LOI", + "f_pct": "F", + "s_pct": "S", + "so3_pct": "SO3", + "so4_pct": "SO4", + "c_pct": "C", + "co2_pct": "CO2", + "total_pct": "Total", + "feo_pct": "FeO", + "fe2o3_pct": "Fe2O3", + "feo_star_pct": "FeO*", + "h2o_plus_pct": "H2O+", + "h2o_minus_pct": "H2O-", + "au_ppb": "Au", + "ag_ppm": "Ag", + "as_ppm": "As", + "b_ppm": "B", + "ba_ppm": "Ba", + "be_ppm": "Be", + "bi_ppm": "Bi", + "br_ppm": "Br", + "cd_ppm": "Cd", + "cl_ppm": "Cl", + "co_ppm": "Co", + "cr_ppm": "Cr", + "cs_ppm": "Cs", + "cu_ppm": "Cu", + "ga_ppm": "Ga", + "ge_ppm": "Ge", + "hf_ppm": "Hf", + "hg_ppm": "Hg", + "in_ppm": "In", + "li_ppm": "Li", + "mo_ppm": "Mo", + "nb_ppm": "Nb", + "ni_ppm": "Ni", + "pd_ppm": "Pd", + "pb_ppm": "Pb", + "pt_ppm": "Pt", + "rb_ppm": "Rb", + "re_ppm": "Re", + "sb_ppm": "Sb", + "sc_ppm": "Sc", + "se_ppm": "Se", + "sn_ppm": "Sn", + "sr_ppm": "Sr", + "ta_ppm": "Ta", + "te_ppm": "Te", + "th_ppm": "Th", + "tl_ppm": "Tl", + "u_ppm": "U", + "v_ppm": "V", + "w_ppm": "W", + "y_ppm": "Y", + "zn_ppm": "Zn", + "zr_ppm": "Zr", + "la_ppm": "La", + "ce_ppm": "Ce", + "pr_ppm": "Pr", + "nd_ppm": "Nd", + "sm_ppm": "Sm", + "eu_ppm": "Eu", + "gd_ppm": "Gd", + "tb_ppm": "Tb", + "dy_ppm": "Dy", + "ho_ppm": "Ho", + "er_ppm": "Er", + "tm_ppm": "Tm", + "yb_ppm": "Yb", + "lu_ppm": "Lu", + "tree_ppm": "TREE", + "mn_pct": "Mn", + "fe_pct": "Fe", + "al_pct": "Al", + "ca_pct": "Ca", + "na_pct": "Na", + "k_pct": "K", + "mg_pct": "Mg", + "p_pct": "P", + "si_pct": "Si", + "ti_pct": "Ti", + }, + "CM_WorldComparisons": { + "area": "area", + "deposit": "deposit", + "reference": "reference", + "la": "La", + "ce": "Ce", + "pr": "Pr", + "nd": "Nd", + "sm": "Sm", + "eu": "Eu", + "gd": "Gd", + "tb": "Tb", + "dy": "Dy", + "ho": "Ho", + "er": "Er", + "tm": "Tm", + "yb": "Yb", + "lu": "Lu", + "tree": "TREE", + "sc": "Sc", + "y": "Y", + "metric_tons": "metric tons", + "grade_pct": "grade %", + "total_ree": "total REE", + "cutoff_grade_pct": "cutoff grade %", + "la2o3": "La2O3", + "ce2o3": "Ce2O3", + "pr6o11": "Pr6O11", + "nd2o3": "Nd2O3", + "sm2o3": "Sm2O3", + "eu2o3": "Eu2O3", + "gd2o3": "Gd2O3", + "tb4o7": "Tb4O7", + "dy2o3": "Dy2O3", + "ho2o3": "Ho2O3", + "er2o3": "Er2O3", + "tm2o3": "Tm2O3", + "yb2o3": "Yb2O3", + "lu2o3": "Lu2O3", + "y2o3": "Y2O3", + }, + "CM_DetectionLimits": { + "method": "Method (block title, DetectionLimits row 1)", + "element": "Element", + "lower_reporting_limit": "Lower Reporting Limit", + "unit": "Unit", + }, + "CM_MineralSystems": { + "system_name": "System Name", + "synopsis": "Synopsis", + "deposit_types": "Deposit types", + "principal_commodities": "Principal commodities", + "critical_minerals": "Critical minerals", + "references": "References", + "phase_2": "Phase 2", + "phase_3": "Phase 3", + "phase_4": "Phase 4", + }, +} + + +class CM_ChemicalData(Base): + """Mirror of the ChemicalData, GIS and QAQC sheets. + + One row per source spreadsheet row, keyed on (source_sheet, source_row). + See RECONCILIATION IS DEFERRED in the module docstring: the three sheets + disagree and none of them is authoritative. + """ + + __tablename__ = "CM_ChemicalData" + __table_args__ = ( + UniqueConstraint( + "source_sheet", "source_row", name="uq_cm_chemical_data_source_row" + ), + ) + + id = mapped_column(Integer, primary_key=True, autoincrement=True) + # Provenance of the cell range this row came from. + source_sheet = mapped_column(String, nullable=False, index=True) + source_row = mapped_column(Integer, nullable=False) + + # SAMPLE block: sample identity, location and field context. + sample = mapped_column(String, index=True) # SAMPLE + project = mapped_column(String) # Project + area = mapped_column(String, index=True) # Area + reference = mapped_column(String) # Reference + date_collected = mapped_column(String) # Date collected + date_analyzed = mapped_column(String) # Date analyzed + chem_lab_file_no = mapped_column(String) # Chem Lab File No. + laboratory = mapped_column(String) # Laboratory + latitude = mapped_column(String) + longitude = mapped_column(String) + coordinate_system = mapped_column(String) # Coordinate system + county = mapped_column(String) # County + state = mapped_column(String) # State + lithology = mapped_column(String) + mineral_system = mapped_column(String) # Mineral system + deposit_types = mapped_column( + String + ) # Deposit type(s) (from Mineral Systems table) + map_symbol = mapped_column(String) # MapSymbol + method_collected = mapped_column(String) # method collected + sample_source = mapped_column(String) # sample source + mineralogy_deposit_type = mapped_column(String) # MineralogyDepositType + depth_legnth_ft = mapped_column(String) # Depth/legnth (ft) + mine_id = mapped_column(String) # Mine ID + location_notes = mapped_column(String) # LocationNotes + comments = mapped_column(String) # Comments + paste_ph = mapped_column(String) # paste pH + paste_conductivity = mapped_column(String) # paste conductivity + tds_mg_l = mapped_column(String) # TDS (mg/l) + + # Major-element oxides and volatiles, as declared in the units row (%) + sio2_pct = mapped_column(String) # SiO2 + tio2_pct = mapped_column(String) # TiO2 + al2o3_pct = mapped_column(String) # Al2O3 + fe2o3t_pct = mapped_column(String) # Fe2O3T + mno_pct = mapped_column(String) # MnO + mgo_pct = mapped_column(String) # MgO + cao_pct = mapped_column(String) # CaO + na2o_pct = mapped_column(String) # Na2O + k2o_pct = mapped_column(String) # K2O + p2o5_pct = mapped_column(String) # P2O5 + loi_pct = mapped_column(String) # LOI + f_pct = mapped_column(String) # F + s_pct = mapped_column(String) # S + so3_pct = mapped_column(String) # SO3 + so4_pct = mapped_column(String) # SO4 + c_pct = mapped_column(String) # C + co2_pct = mapped_column(String) # CO2 + total_pct = mapped_column(String) # Total + feo_pct = mapped_column(String) # FeO + fe2o3_pct = mapped_column(String) # Fe2O3 + feo_star_pct = mapped_column(String) # FeO* + h2o_plus_pct = mapped_column(String) # H2O+ + h2o_minus_pct = mapped_column(String) # H2O- + + # Precious metals and trace elements (Au in ppb, the rest ppm) + au_ppb = mapped_column(String) # Au + ag_ppm = mapped_column(String) # Ag + as_ppm = mapped_column(String) # As + b_ppm = mapped_column(String) # B + ba_ppm = mapped_column(String) # Ba + be_ppm = mapped_column(String) # Be + bi_ppm = mapped_column(String) # Bi + br_ppm = mapped_column(String) # Br + cd_ppm = mapped_column(String) # Cd + cl_ppm = mapped_column(String) # Cl + co_ppm = mapped_column(String) # Co + cr_ppm = mapped_column(String) # Cr + cs_ppm = mapped_column(String) # Cs + cu_ppm = mapped_column(String) # Cu + ga_ppm = mapped_column(String) # Ga + ge_ppm = mapped_column(String) # Ge + hf_ppm = mapped_column(String) # Hf + hg_ppm = mapped_column(String) # Hg + in_ppm = mapped_column(String) # In + li_ppm = mapped_column(String) # Li + mo_ppm = mapped_column(String) # Mo + nb_ppm = mapped_column(String) # Nb + ni_ppm = mapped_column(String) # Ni + pd_ppm = mapped_column(String) # Pd + pb_ppm = mapped_column(String) # Pb + pt_ppm = mapped_column(String) # Pt + rb_ppm = mapped_column(String) # Rb + re_ppm = mapped_column(String) # Re + sb_ppm = mapped_column(String) # Sb + sc_ppm = mapped_column(String) # Sc + se_ppm = mapped_column(String) # Se + sn_ppm = mapped_column(String) # Sn + sr_ppm = mapped_column(String) # Sr + ta_ppm = mapped_column(String) # Ta + te_ppm = mapped_column(String) # Te + th_ppm = mapped_column(String) # Th + tl_ppm = mapped_column(String) # Tl + u_ppm = mapped_column(String) # U + v_ppm = mapped_column(String) # V + w_ppm = mapped_column(String) # W + y_ppm = mapped_column(String) # Y + zn_ppm = mapped_column(String) # Zn + zr_ppm = mapped_column(String) # Zr + + # Rare earth elements and total REE (ppm) + la_ppm = mapped_column(String) # La + ce_ppm = mapped_column(String) # Ce + pr_ppm = mapped_column(String) # Pr + nd_ppm = mapped_column(String) # Nd + sm_ppm = mapped_column(String) # Sm + eu_ppm = mapped_column(String) # Eu + gd_ppm = mapped_column(String) # Gd + tb_ppm = mapped_column(String) # Tb + dy_ppm = mapped_column(String) # Dy + ho_ppm = mapped_column(String) # Ho + er_ppm = mapped_column(String) # Er + tm_ppm = mapped_column(String) # Tm + yb_ppm = mapped_column(String) # Yb + lu_ppm = mapped_column(String) # Lu + tree_ppm = mapped_column(String) # TREE + + # Whole-rock elemental analyses (%), reported alongside the oxides + mn_pct = mapped_column(String) # Mn + fe_pct = mapped_column(String) # Fe + al_pct = mapped_column(String) # Al + ca_pct = mapped_column(String) # Ca + na_pct = mapped_column(String) # Na + k_pct = mapped_column(String) # K + mg_pct = mapped_column(String) # Mg + p_pct = mapped_column(String) # P + si_pct = mapped_column(String) # Si + ti_pct = mapped_column(String) # Ti + + +class CM_DetectionLimits(Base): + """Mirror of the DetectionLimits sheet (lower reporting limits by element). + + The sheet is a single block whose title row names the analytical method + ("Method C_ICPOES_MS-61"); that title is carried on every row as ``method`` + so the block structure survives without a separate header table. + """ + + __tablename__ = "CM_DetectionLimits" + __table_args__ = ( + UniqueConstraint("source_row", name="uq_cm_detection_limits_source_row"), + ) + + id = mapped_column(Integer, primary_key=True, autoincrement=True) + source_row = mapped_column(Integer, nullable=False) + method = mapped_column(String) + element = mapped_column(String, index=True) + lower_reporting_limit = mapped_column(String) + unit = mapped_column(String) + + +class CM_References(Base): + """Mirror of the References sheet: one full citation per row, no header. + + ``CM_ChemicalData.reference`` holds the short form ("McLemore et al. + (2025b)") that keys into these citations, but the sheet provides no + explicit key column, so the join is left to the transform phase. + """ + + __tablename__ = "CM_References" + __table_args__ = ( + UniqueConstraint("source_row", name="uq_cm_references_source_row"), + ) + + id = mapped_column(Integer, primary_key=True, autoincrement=True) + source_row = mapped_column(Integer, nullable=False) + citation = mapped_column(String) + + +class CM_MineralSystems(Base): + """Mirror of the MineralSystems sheet (Hofstra and Kreiner, 2020). + + The sheet is irregular: columns 1-6 are the systems table proper, while + columns 7-9 hold three independent lists of critical minerals by USGS + phase, and several rows are table notes rather than systems. Cells are + mirrored positionally, notes included; sorting that out is transform work. + """ + + __tablename__ = "CM_MineralSystems" + __table_args__ = ( + UniqueConstraint("source_row", name="uq_cm_mineral_systems_source_row"), + ) + + id = mapped_column(Integer, primary_key=True, autoincrement=True) + source_row = mapped_column(Integer, nullable=False) + system_name = mapped_column(String) + synopsis = mapped_column(String) + deposit_types = mapped_column(String) + principal_commodities = mapped_column(String) + critical_minerals = mapped_column(String) + references = mapped_column(String) + phase_2 = mapped_column(String) + phase_3 = mapped_column(String) + phase_4 = mapped_column(String) + + +class CM_WorldComparisons(Base): + """Mirror of the world sheet: world-class REE deposits used as comparanda. + + Not New Mexico data and not sample data -- published averages for other + deposits plus crustal abundance, carried for context. Citations are in + CM_WorldReferences. The sheet declares no units row. + """ + + __tablename__ = "CM_WorldComparisons" + __table_args__ = ( + UniqueConstraint("source_row", name="uq_cm_world_comparisons_source_row"), + ) + + id = mapped_column(Integer, primary_key=True, autoincrement=True) + source_row = mapped_column(Integer, nullable=False) + + area = mapped_column(String) + deposit = mapped_column(String) + reference = mapped_column(String) + la = mapped_column(String) # La + ce = mapped_column(String) # Ce + pr = mapped_column(String) # Pr + nd = mapped_column(String) # Nd + sm = mapped_column(String) # Sm + eu = mapped_column(String) # Eu + gd = mapped_column(String) # Gd + tb = mapped_column(String) # Tb + dy = mapped_column(String) # Dy + ho = mapped_column(String) # Ho + er = mapped_column(String) # Er + tm = mapped_column(String) # Tm + yb = mapped_column(String) # Yb + lu = mapped_column(String) # Lu + tree = mapped_column(String) # TREE + sc = mapped_column(String) # Sc + y = mapped_column(String) # Y + metric_tons = mapped_column(String) # metric tons + grade_pct = mapped_column(String) # grade % + total_ree = mapped_column(String) # total REE + cutoff_grade_pct = mapped_column(String) # cutoff grade % + la2o3 = mapped_column(String) # La2O3 + ce2o3 = mapped_column(String) # Ce2O3 + pr6o11 = mapped_column(String) # Pr6O11 + nd2o3 = mapped_column(String) # Nd2O3 + sm2o3 = mapped_column(String) # Sm2O3 + eu2o3 = mapped_column(String) # Eu2O3 + gd2o3 = mapped_column(String) # Gd2O3 + tb4o7 = mapped_column(String) # Tb4O7 + dy2o3 = mapped_column(String) # Dy2O3 + ho2o3 = mapped_column(String) # Ho2O3 + er2o3 = mapped_column(String) # Er2O3 + tm2o3 = mapped_column(String) # Tm2O3 + yb2o3 = mapped_column(String) # Yb2O3 + lu2o3 = mapped_column(String) # Lu2O3 + y2o3 = mapped_column(String) # Y2O3 + + +class CM_WorldReferences(Base): + """Mirror of the world_ref sheet: citations for CM_WorldComparisons.""" + + __tablename__ = "CM_WorldReferences" + __table_args__ = ( + UniqueConstraint("source_row", name="uq_cm_world_references_source_row"), + ) + + id = mapped_column(Integer, primary_key=True, autoincrement=True) + source_row = mapped_column(Integer, nullable=False) + citation = mapped_column(String) + + +class CM_WorkbookMetadata(Base): + """Mirror of the workbook's label/value provenance sheets. + + Flattens "General Information", "MetaData" and "DefinitionOfFields" into + (source_sheet, label, value) triples: title, abstract, compilers, original + and revised dates, online resources, and the sheet's own definitions of the + SAMPLE-block fields. Kept so the mirror carries its own provenance rather + than relying on the workbook file staying around. + """ + + __tablename__ = "CM_WorkbookMetadata" + __table_args__ = ( + UniqueConstraint( + "source_sheet", "source_row", name="uq_cm_workbook_metadata_source_row" + ), + ) + + id = mapped_column(Integer, primary_key=True, autoincrement=True) + source_sheet = mapped_column(String, nullable=False) + source_row = mapped_column(Integer, nullable=False) + label = mapped_column(String) + value = mapped_column(String) diff --git a/docs/critical-minerals-legacy-mirror.md b/docs/critical-minerals-legacy-mirror.md new file mode 100644 index 000000000..c178b5729 --- /dev/null +++ b/docs/critical-minerals-legacy-mirror.md @@ -0,0 +1,193 @@ +# Critical minerals legacy mirror (`CM_legacy`) + +Staging mirror of the Earth MRI critical-minerals chemistry workbook +(`McLemoreMasterChem`, compiled by V.T. McLemore et al., NMBGMR) into the +`CM_*` tables. Phase 1 only: the workbook lands in PostgreSQL unchanged. The +transform into the Ocotillo data model (Location / Thing / FieldEvent / +FieldActivity / Sample / Observation) is not designed yet. + +Code: [`db/cm_legacy.py`](../db/cm_legacy.py), +[`services/cm_legacy_mirror.py`](../services/cm_legacy_mirror.py), +migration `d4e5f6a7b8c9`. + +## Loading + +```bash +oco load-critical-minerals-workbook --file /path/to/McLemoreMasterChem_9-18-25.xlsx +``` + +The load is idempotent per sheet: each sheet's rows are deleted and re-inserted, +so a revised workbook can be reloaded without duplicating rows or wiping the +rest of the mirror. The command wraps the whole load in one transaction — a +failure part way through leaves the mirror untouched. + +Row counts from the 2025-09-17 revision: + +| Sheet | Mirror table | Rows | +|---|---|---| +| ChemicalData | `CM_ChemicalData` (`source_sheet='ChemicalData'`) | 4867 | +| GIS | `CM_ChemicalData` (`source_sheet='GIS'`) | 4848 | +| QAQC | `CM_ChemicalData` (`source_sheet='QAQC'`) | 8 | +| DetectionLimits | `CM_DetectionLimits` | 61 | +| References | `CM_References` | 81 | +| MineralSystems | `CM_MineralSystems` | 169 | +| world | `CM_WorldComparisons` | 7 | +| world_ref | `CM_WorldReferences` | 4 | +| General Information / MetaData / DefinitionOfFields | `CM_WorkbookMetadata` | 24 / 9 / 22 | + +## Three sheets, one table + +`ChemicalData`, `GIS` and `QAQC` share a column set, so they mirror into one +table keyed by a `source_sheet` discriminator: + +- `ChemicalData` and `GIS` are byte-identical in header text and order — 118 + columns each. Only the layout differs: `ChemicalData` has a title banner + (row 1), header (row 2) and units row (row 3); `GIS` has a single header row. +- `QAQC` is those columns minus `MapSymbol`, `Pd` and `Pt`, with + `latitude`/`longitude` capitalized. The loader matches headers + case-insensitively; the three absent columns are NULL for those rows + (`QAQC_MISSING_COLUMNS`). + +## Reconciliation is deferred + +**`GIS` is not `ChemicalData` plus location data, and it is not a clean subset.** +It is a stale, hand-maintained fork. It carries the same mixed coordinate +systems (WGS84 / NAD27 / NAD83 / blank) and the same ~876 rows with no +latitude, so it adds no location information at all. + +Every `GIS` sample name exists in `ChemicalData`, but 1704 of the 4848 shared +rows disagree cell-for-cell, in both directions: + +| Direction | Columns | +|---|---| +| `GIS` has values `ChemicalData` lacks | 533 `Chem Lab File No.`, 485 `Laboratory`, 85 `FeO`, 85 `Fe2O3` | +| `ChemicalData` has values `GIS` lacks | 633 `Total`, 184 `Area`, 11 `TREE`, 4 `Date analyzed` | +| Outright disagreement | 16 rows on `Area` (`ZuniMountains` vs `Zuni`) | +| Broken formulas | 12 `#VALUE!` `Total`s in `GIS`, 1 in `ChemicalData` | + +`ChemicalData` also holds 18 sample rows appended after `GIS` was last synced +(`BP*`, `CR1`, `JP*`, `SA*`, `SL*`). + +Neither sheet is authoritative, so both are mirrored in full and the merge is +left to a later phase, **to be ruled on per column by V.T. McLemore**. Until +then: + +- do not treat any single `source_sheet` as complete; +- do not de-duplicate across sheets in the mirror; +- a query that reads only `source_sheet='ChemicalData'` silently drops 533 lab + file numbers, 485 lab names and 170 FeO/Fe2O3 values. + +The loader emits a warning whenever the `ChemicalData` and `GIS` row counts +differ, as a standing reminder that the drift has not been resolved. + +## The reconciliation workbook + +`scripts/cm_reconciliation_report.py` turns everything above into a decision +workbook for whoever owns the source data: + +```bash +python -m scripts.cm_reconciliation_report \ + --workbook "/path/to/McLemoreMasterChem_9-18-25.xlsx" \ + --out CM_reconciliation.xlsx +``` + +Eight sheets. Every yellow column is blank on purpose and carries a dropdown; +nothing in the workbook is decided by the script. + +| Sheet | Rows (2025-09-17 revision) | Purpose | +|---|---|---| +| `README` | — | what it is, who fills it in, what happens next | +| `ColumnDecisions` | 8 | one row per disagreeing column, with a suggested starting point | +| `CellDifferences` | 2123 | every disagreeing cell, both values side by side | +| `RowsOnlyInOneSheet` | 19 | the 18 Pearce (2020) samples appended after the last GIS sync, plus the NOTE row | +| `IntegritySummary` | 13 | integrity findings by issue, worst first | +| `IntegrityDetail` | 4276 | the individual rows behind each finding | +| `DetectionLimitSpread` | 64 | analytes reported against many different `<` limits | +| `DuplicateSampleNames` | 258 | names shared by two or more rows | + +### Integrity findings + +Independent of the sheet-to-sheet drift, from `ChemicalData` alone: + +| Rows | Finding | +|---|---| +| 1594 | date is year-only or free text, not a full date | +| 880 | no coordinates | +| 724 | value impossible for its declared unit — e.g. `F` = 27700 in a column the units row declares as `%`, plus 68 in `P` and 65 in `Mn` | +| 300 | non-numeric analyte text: `bd`, `nd`, `tr`, `nr`, `n/a`, `----`, `>2%` | +| 259 | `Au` censored below 0.01 in a ppb column, i.e. some rows are ppm | +| 224 | `Total` outside 95–105% | +| 159 | `Total` differs from the sum of SiO2–LOI by more than 5 | +| 81 + 8 | latitude / longitude outside the New Mexico bounding box | +| 23 | analyzed before collected | +| 21 | coordinates with no declared datum (NAD27 vs WGS84 is ~100 m here) | +| 2 | positive longitude (missing minus sign) | +| 1 | `Total` is `#VALUE!` | + +`Total` is deliberately exempt from the unit check: it is a sum, so exceeding +100 is not by itself a unit error. It gets the two dedicated checks above +instead. Only `ChemicalData` is integrity-checked — running the same checks over +`GIS` would double every finding without adding information, since the +sheet-level differences are already enumerated. + +## Everything is a string + +Analyte columns hold censored values as text (`<0.1`, `<10`, `<0.06` — 1154 of +them in `Au` alone), Excel error text (`#VALUE!`), and blanks; date columns mix +real dates with free text. Storing every column as `String` keeps the mirror +loadable without dropping cells. Parsing value-plus-qualifier (cross-checked +against `CM_DetectionLimits`), casting dates, and reprojecting coordinates are +all transform work. + +Cell rendering rules (`_cell_to_text`): dates become ISO-8601 rather than Excel +serials, numbers keep Python's round-trippable `repr`, blank and whitespace-only +cells become NULL rather than `''`, and everything else passes through +stripped. + +## Column naming + +Sheet headers are spreadsheet labels, not SQL identifiers (`Chem Lab File No.`, +`Depth/legnth (ft)`, `H2O+`), so unlike the NMA/NMW mirrors the source name +cannot be reused verbatim. Names are derived mechanically: snake_case, +non-alphanumerics collapsed to `_`, `+`→`_plus`, `*`→`_star`, `%`→`_pct`. + +Analyte columns carry the unit the workbook declares for them in its units row +(`sio2_pct`, `au_ppb`, `as_ppm`), which also keeps `As` and `In` from colliding +with the Python and SQL keywords `as` and `in`. The source typo in +`Depth/legnth (ft)` is preserved as `depth_legnth_ft` so the mapping back to the +sheet stays mechanical. + +`SOURCE_HEADER_BY_COLUMN` records the exact source header for every column and +`ANALYTE_UNITS` records the declared unit for every analyte, so the units row is +not lost. + +## Row identity + +Sample names are **not** unique in the source (`S1`, `S10`, `S100` and ~340 +others repeat), so rows are keyed on `(source_sheet, source_row)` where +`source_row` is the 1-based Excel row number. Every mirror row is traceable to a +cell range in the delivered workbook. + +## Known junk in the data range + +The first data row of `ChemicalData` is not a sample: its `SAMPLE` cell holds +"NOTE: SEE THE ORIGINAL CITATION FOR INFORMATION ON METHODS OF ANALYSES, +QA/QC, DETECTION LIMITS, ETC." It is mirrored like any other row and must be +excluded by the transform. + +`CM_MineralSystems` is mirrored positionally, table notes included: the sheet +puts the systems table in columns 1–6 and three unrelated lists of critical +minerals by USGS phase in columns 7–9, and several rows are footnotes rather +than systems. + +Also unresolved in the source data: `F` is declared as `%` in the units row but +several rows report values in the tens of thousands (Gal1 `f_pct` = 27700), +i.e. ppm. The mirror carries the value as given; the transform must decide. + +## Layout is asserted, not guessed + +Each sheet declares which row holds its header, and the load fails loudly if +that row does not contain the expected label, or if a header appears that has no +mirror column. A revised workbook that moves a header row, renames a column or +adds one must be looked at by a human — and given a migration — before it lands +in the mirror. diff --git a/scripts/cm_reconciliation_report.py b/scripts/cm_reconciliation_report.py new file mode 100644 index 000000000..4e6e72a16 --- /dev/null +++ b/scripts/cm_reconciliation_report.py @@ -0,0 +1,1010 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Build the reconciliation workbook for the critical-minerals mirror. + +Reads the delivered McLemore workbook and writes a decision workbook for +V.T. McLemore: where the ChemicalData and GIS sheets disagree, which rows exist +in only one of them, and which values fail a data-integrity check (units that +contradict the declared unit, oxide totals that do not add up, coordinates +outside New Mexico, non-numeric analyte text, inconsistent detection limits, +ambiguous sample names). + +Nothing here writes to the database and nothing is decided here -- every +judgement column is left blank for a human. The CM_legacy mirror stores both +sheets verbatim in the meantime; see docs/critical-minerals-legacy-mirror.md. + + python -m scripts.cm_reconciliation_report --workbook --out +""" + +from __future__ import annotations + +import argparse +import difflib +import re +from collections import Counter, defaultdict +from dataclasses import dataclass +from datetime import date +from pathlib import Path +from typing import Any, Iterable, Sequence + +from db.cm_legacy import ANALYTE_UNITS, SOURCE_HEADER_BY_COLUMN +from services.cm_legacy_mirror import ( + CHEMISTRY_SHEET_LAYOUT, + cell_to_text, + column_by_header, + normalize_header, +) + +# New Mexico bounding box, generous by a few minutes on every side. +NM_LATITUDE = (31.20, 37.05) +NM_LONGITUDE = (-109.10, -102.95) + +# Oxides the workbook's own "Total" column is meant to sum. +MAJOR_OXIDES = ( + "sio2_pct", + "tio2_pct", + "al2o3_pct", + "fe2o3t_pct", + "mno_pct", + "mgo_pct", + "cao_pct", + "na2o_pct", + "k2o_pct", + "p2o5_pct", + "loi_pct", +) +# Oxide totals this far from the sum of the majors are called out. A major- +# element analysis is normally accepted at 100 +/- a couple of percent. +TOTAL_TOLERANCE = 5.0 +MINIMUM_MAJORS_FOR_TOTAL_CHECK = 8 +TOTAL_PLAUSIBLE_RANGE = (95.0, 105.0) +# An analyte with more distinct censoring thresholds than this is being +# reported against detection limits that vary by lab or by decade. +THRESHOLD_SPREAD_FLAG = 3 + +NUMERIC = re.compile(r"^[<>]?\s*-?(\d+\.?\d*|\.\d+)([eE][-+]?\d+)?$") +ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}") + +DECISION_CHOICES = ( + '"ChemicalData,GIS,Most recent value,Case by case,Neither - fix source"' +) +INTEGRITY_CHOICES = ( + '"Fix in source workbook,Handle in transform,Accept as-is,Needs review"' +) + + +@dataclass(frozen=True) +class Value: + """A parsed workbook cell.""" + + text: str | None + number: float | None + kind: str | None # "numeric" | "censored" | "text" | None + + +def parse_value(cell: Any) -> Value: + text = cell_to_text(cell) + if text is None: + return Value(None, None, None) + candidate = text.replace(",", "").strip() + if NUMERIC.match(candidate): + number = float(candidate.lstrip("<> ")) + return Value(text, number, "censored" if candidate[0] in "<>" else "numeric") + return Value(text, None, "text") + + +@dataclass +class Sheet: + """A chemistry sheet flattened to {column: text} records.""" + + name: str + records: list[dict[str, str | None]] + row_numbers: list[int] + + +def read_sheet(workbook: Any, name: str, columns: dict[str, str]) -> Sheet: + header_row_number, first_data_row_number = CHEMISTRY_SHEET_LAYOUT[name] + rows = list(workbook[name].iter_rows(values_only=True)) + header = rows[header_row_number - 1] + index_to_column = { + index: columns[normalize_header(cell)] + for index, cell in enumerate(header) + if cell is not None and normalize_header(cell) in columns + } + records: list[dict[str, str | None]] = [] + row_numbers: list[int] = [] + for offset, row in enumerate(rows[first_data_row_number - 1 :]): + if all(cell_to_text(cell) is None for cell in row): + continue + records.append( + { + column: cell_to_text(row[index]) if index < len(row) else None + for index, column in index_to_column.items() + } + ) + row_numbers.append(first_data_row_number + offset) + return Sheet(name, records, row_numbers) + + +def align( + left: Sheet, right: Sheet +) -> tuple[list[tuple[int, int]], list[int], list[int]]: + """Line the two sheets up on their sample-name sequence. + + Returns aligned (left index, right index) pairs plus the indexes that exist + on only one side. + """ + left_keys = [(record.get("sample") or "").strip() for record in left.records] + right_keys = [(record.get("sample") or "").strip() for record in right.records] + matcher = difflib.SequenceMatcher(a=left_keys, b=right_keys, autojunk=False) + + pairs: list[tuple[int, int]] = [] + left_only: list[int] = [] + right_only: list[int] = [] + for tag, i1, i2, j1, j2 in matcher.get_opcodes(): + if tag == "equal": + pairs.extend(zip(range(i1, i2), range(j1, j2))) + else: + left_only.extend(range(i1, i2)) + right_only.extend(range(j1, j2)) + return pairs, left_only, right_only + + +def classify(left_text: str | None, right_text: str | None) -> str | None: + """Name the kind of disagreement between two cells, or None if they agree.""" + left = (left_text or "").strip() + right = (right_text or "").strip() + if left == right: + return None + if not right: + return "only in ChemicalData" + if not left: + return "only in GIS" + if right.startswith("#"): + return "Excel error in GIS" + if left.startswith("#"): + return "Excel error in ChemicalData" + left_value, right_value = parse_value(left), parse_value(right) + if left_value.number is not None and right_value.number is not None: + if left_value.number == right_value.number: + return "same number, different text" + return "different number" + return "different text" + + +def compare_sheets( + chemical_data: Sheet, gis: Sheet, columns: Sequence[str] +) -> tuple[list[dict[str, Any]], Counter, Counter]: + pairs, chemical_data_only, _ = align(chemical_data, gis) + differences: list[dict[str, Any]] = [] + by_column: Counter = Counter() + by_column_kind: Counter = Counter() + + for left_index, right_index in pairs: + left_record = chemical_data.records[left_index] + right_record = gis.records[right_index] + for column in columns: + kind = classify(left_record.get(column), right_record.get(column)) + if kind is None: + continue + by_column[column] += 1 + by_column_kind[(column, kind)] += 1 + differences.append( + { + "sample": left_record.get("sample"), + "area": left_record.get("area"), + "column": column, + "header": SOURCE_HEADER_BY_COLUMN["CM_ChemicalData"][column], + "chemical_data_value": left_record.get(column), + "gis_value": right_record.get(column), + "kind": kind, + "chemical_data_row": chemical_data.row_numbers[left_index], + "gis_row": gis.row_numbers[right_index], + } + ) + return differences, by_column, by_column_kind + + +def recommend(column: str, kinds: dict[str, int]) -> str: + """Suggest a starting point for each column, to be confirmed or overruled.""" + only_gis = kinds.get("only in GIS", 0) + only_chemical_data = kinds.get("only in ChemicalData", 0) + excel_errors = kinds.get("Excel error in GIS", 0) + kinds.get( + "Excel error in ChemicalData", 0 + ) + conflicts = ( + kinds.get("different number", 0) + + kinds.get("different text", 0) + + kinds.get("same number, different text", 0) + ) + if conflicts: + return "Conflicting values - needs a per-row ruling" + if excel_errors and not (only_gis or only_chemical_data): + return "Take the sheet without the #VALUE! error" + if only_gis and only_chemical_data: + return "Fill blanks from whichever sheet has a value" + if only_gis: + return "Take GIS (it fills blanks ChemicalData never got)" + if only_chemical_data: + return "Take ChemicalData (GIS is missing these)" + return "No action" + + +def integrity_findings( + sheet: Sheet, detection_limits: dict[str, float] +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]: + """Run the data-integrity checks over one sheet. + + Returns (per-cell/row detail, threshold spread rows, duplicate-name rows). + """ + detail: list[dict[str, Any]] = [] + + def flag(record, row_number, issue, column, value, note): + detail.append( + { + "sheet": sheet.name, + "source_row": row_number, + "sample": record.get("sample"), + "area": record.get("area"), + "reference": record.get("reference"), + "issue": issue, + "column": column, + "value": value, + "note": note, + } + ) + + thresholds: dict[str, set[float]] = defaultdict(set) + + for record, row_number in zip(sheet.records, sheet.row_numbers): + # --- analyte values against the unit the workbook declares for them + for column, unit in ANALYTE_UNITS.items(): + value = parse_value(record.get(column)) + if value.kind is None: + continue + if value.kind == "censored": + thresholds[column].add(value.number) + if value.kind == "text": + flag( + record, + row_number, + "Non-numeric analyte value", + column, + value.text, + "Not a number and not a '<' detection limit; the transform " + "cannot store it as a result", + ) + continue + if ( + unit == "%" + and column != "total_pct" + and value.number is not None + and value.number > 100 + ): + flag( + record, + row_number, + "Value impossible for the declared unit", + column, + value.text, + f"Declared as % but reported {value.number:g}; looks like ppm", + ) + if ( + unit == "ppb" + and value.kind == "censored" + and value.number is not None + and value.number < 0.01 + ): + flag( + record, + row_number, + "Detection limit implausible for the declared unit", + column, + value.text, + f"Declared as ppb but censored at {value.number:g}; looks like ppm", + ) + + # --- oxide totals + total = parse_value(record.get("total_pct")) + majors = [parse_value(record.get(column)) for column in MAJOR_OXIDES] + reported = [value.number for value in majors if value.kind == "numeric"] + if total.kind == "numeric": + summed = sum(reported) + enough_majors = len(reported) >= MINIMUM_MAJORS_FOR_TOTAL_CHECK + if enough_majors and abs(summed - total.number) > TOTAL_TOLERANCE: + flag( + record, + row_number, + "Total disagrees with the sum of the major oxides", + "total_pct", + total.text, + f"Majors sum to {summed:.2f} from {len(reported)} oxides, " + f"a difference of {summed - total.number:+.2f}", + ) + elif not ( + TOTAL_PLAUSIBLE_RANGE[0] <= total.number <= TOTAL_PLAUSIBLE_RANGE[1] + ): + flag( + record, + row_number, + "Total outside 95-105%", + "total_pct", + total.text, + ( + "Consistent with the majors, so the analysis itself is " + "incomplete or the sample is not a whole rock" + if enough_majors + else f"Only {len(reported)} major oxides reported, so the " + "Total cannot be checked against them" + ), + ) + elif total.kind == "text": + flag( + record, + row_number, + "Total is not a number", + "total_pct", + total.text, + "Broken spreadsheet formula", + ) + + # --- coordinates + latitude = parse_value(record.get("latitude")) + longitude = parse_value(record.get("longitude")) + datum = (record.get("coordinate_system") or "").strip() + has_latitude = latitude.number is not None + has_longitude = longitude.number is not None + if not has_latitude and not has_longitude: + flag( + record, + row_number, + "No coordinates", + "latitude/longitude", + None, + f"Coordinate system is {datum!r}" if datum else "No datum either", + ) + elif has_latitude != has_longitude: + flag( + record, + row_number, + "Only one of latitude/longitude", + "latitude/longitude", + f"{latitude.text} / {longitude.text}", + "Unusable as a point", + ) + else: + if not datum: + flag( + record, + row_number, + "Coordinates with no declared datum", + "coordinate_system", + f"{latitude.text} / {longitude.text}", + "Cannot be reprojected; NAD27 and WGS84 differ by ~100 m here", + ) + if longitude.number > 0: + flag( + record, + row_number, + "Positive longitude", + "longitude", + longitude.text, + "Missing the minus sign puts the sample in Asia", + ) + if not NM_LATITUDE[0] <= latitude.number <= NM_LATITUDE[1]: + flag( + record, + row_number, + "Latitude outside New Mexico", + "latitude", + latitude.text, + f"Outside {NM_LATITUDE[0]}-{NM_LATITUDE[1]}", + ) + if not NM_LONGITUDE[0] <= longitude.number <= NM_LONGITUDE[1]: + flag( + record, + row_number, + "Longitude outside New Mexico", + "longitude", + longitude.text, + f"Outside {NM_LONGITUDE[0]} to {NM_LONGITUDE[1]}", + ) + + # --- dates + collected = (record.get("date_collected") or "").strip() + analyzed = (record.get("date_analyzed") or "").strip() + for label, value in ( + ("date_collected", collected), + ("date_analyzed", analyzed), + ): + if value and not ISO_DATE.match(value): + flag( + record, + row_number, + "Date is not a full date", + label, + value, + "Year only or free text; stored as text, not a date", + ) + if ( + collected + and analyzed + and ISO_DATE.match(collected) + and ISO_DATE.match(analyzed) + and analyzed[:10] < collected[:10] + ): + flag( + record, + row_number, + "Analyzed before collected", + "date_analyzed", + f"collected {collected[:10]}, analyzed {analyzed[:10]}", + "One of the two dates is wrong", + ) + + threshold_rows = [ + { + "column": column, + "header": SOURCE_HEADER_BY_COLUMN["CM_ChemicalData"][column], + "declared_unit": ANALYTE_UNITS[column], + "distinct_thresholds": len(values), + "lowest": min(values), + "highest": max(values), + "detection_limits_sheet": detection_limits.get( + SOURCE_HEADER_BY_COLUMN["CM_ChemicalData"][column].strip() + ), + "thresholds": ", ".join(f"{value:g}" for value in sorted(values)[:25]), + } + for column, values in sorted(thresholds.items()) + if len(values) > THRESHOLD_SPREAD_FLAG + ] + threshold_rows.sort(key=lambda row: -row["distinct_thresholds"]) + + names = Counter((record.get("sample") or "").strip() for record in sheet.records) + duplicate_rows = [ + { + "sample": name, + "rows": count, + "areas": ", ".join( + sorted( + { + (record.get("area") or "?").strip() + for record in sheet.records + if (record.get("sample") or "").strip() == name + } + ) + ), + "source_rows": ", ".join( + str(number) + for record, number in zip(sheet.records, sheet.row_numbers) + if (record.get("sample") or "").strip() == name + ), + } + for name, count in names.most_common() + if name and count > 1 + ] + + return detail, threshold_rows, duplicate_rows + + +def read_detection_limits(workbook: Any) -> dict[str, float]: + limits: dict[str, float] = {} + for row in list(workbook["DetectionLimits"].iter_rows(values_only=True))[2:]: + element = cell_to_text(row[0]) if row else None + value = parse_value(row[1]) if len(row) > 1 else Value(None, None, None) + if element and value.number is not None: + limits[element.strip()] = value.number + return limits + + +# ---------------------------------------------------------------------- +# workbook writing +# ---------------------------------------------------------------------- + +HEADER_FILL = "FF1F3B4D" +NOTE_FILL = "FFF2F2F2" +DECISION_FILL = "FFFFF3C4" + + +def _style_header( + worksheet: Any, columns: Sequence[str], widths: Sequence[int] +) -> None: + from openpyxl.styles import Alignment, Font, PatternFill + + worksheet.append(list(columns)) + for index, (column, width) in enumerate(zip(columns, widths), start=1): + cell = worksheet.cell(row=1, column=index) + cell.font = Font(bold=True, color="FFFFFFFF") + cell.fill = PatternFill("solid", fgColor=HEADER_FILL) + cell.alignment = Alignment(vertical="center", wrap_text=True) + worksheet.column_dimensions[ + worksheet.cell(row=1, column=index).column_letter + ].width = width + worksheet.freeze_panes = "A2" + + +def _add_table( + workbook: Any, + title: str, + columns: Sequence[str], + widths: Sequence[int], + rows: Iterable[Sequence[Any]], + decision_columns: Sequence[tuple[int, str]] = (), +) -> Any: + from openpyxl.styles import Alignment, PatternFill + from openpyxl.worksheet.datavalidation import DataValidation + + worksheet = workbook.create_sheet(title) + _style_header(worksheet, columns, widths) + count = 0 + for row in rows: + worksheet.append(list(row)) + count += 1 + worksheet.auto_filter.ref = f"A1:{worksheet.cell(row=1, column=len(columns)).column_letter}{max(count + 1, 2)}" + + for column_index, choices in decision_columns: + letter = worksheet.cell(row=1, column=column_index).column_letter + validation = DataValidation(type="list", formula1=choices, allow_blank=True) + worksheet.add_data_validation(validation) + validation.add(f"{letter}2:{letter}{max(count + 1, 2)}") + for row_index in range(2, count + 2): + worksheet.cell(row=row_index, column=column_index).fill = PatternFill( + "solid", fgColor=DECISION_FILL + ) + worksheet.cell(row=1, column=column_index).alignment = Alignment( + vertical="center", wrap_text=True + ) + return worksheet + + +def _add_readme(workbook: Any, facts: Sequence[tuple[str, str]]) -> None: + from openpyxl.styles import Alignment, Font + + worksheet = workbook.create_sheet("README", 0) + worksheet.column_dimensions["A"].width = 34 + worksheet.column_dimensions["B"].width = 104 + for label, value in facts: + worksheet.append([label, value]) + row = worksheet.max_row + worksheet.cell(row=row, column=1).font = Font(bold=not label.startswith(" ")) + worksheet.cell(row=row, column=1).alignment = Alignment(vertical="top") + worksheet.cell(row=row, column=2).alignment = Alignment( + vertical="top", wrap_text=True + ) + + +def build_report(workbook_path: Path, output_path: Path) -> dict[str, int]: + import openpyxl + + source = openpyxl.load_workbook(workbook_path, read_only=True, data_only=True) + columns = column_by_header("CM_ChemicalData") + all_columns = list(SOURCE_HEADER_BY_COLUMN["CM_ChemicalData"]) + + chemical_data = read_sheet(source, "ChemicalData", columns) + gis = read_sheet(source, "GIS", columns) + detection_limits = read_detection_limits(source) + + differences, by_column, by_column_kind = compare_sheets( + chemical_data, gis, all_columns + ) + pairs, chemical_data_only, gis_only = align(chemical_data, gis) + detail, thresholds, duplicates = integrity_findings(chemical_data, detection_limits) + source.close() + + kinds_by_column: dict[str, dict[str, int]] = defaultdict(dict) + for (column, kind), count in by_column_kind.items(): + kinds_by_column[column][kind] = count + + report = openpyxl.Workbook() + report.remove(report.active) + + rows_with_differences = len( + {(row["chemical_data_row"], row["gis_row"]) for row in differences} + ) + issue_counts = Counter(row["issue"] for row in detail) + + _add_table( + report, + "ColumnDecisions", + [ + "Mirror column", + "Workbook header", + "Cells differing", + "Only in GIS", + "Only in ChemicalData", + "Different value", + "Excel error", + "Suggested starting point", + "DECISION", + "Notes", + ], + [24, 34, 12, 12, 14, 14, 12, 46, 22, 46], + ( + [ + column, + SOURCE_HEADER_BY_COLUMN["CM_ChemicalData"][column], + by_column[column], + kinds_by_column[column].get("only in GIS", 0), + kinds_by_column[column].get("only in ChemicalData", 0), + kinds_by_column[column].get("different number", 0) + + kinds_by_column[column].get("different text", 0) + + kinds_by_column[column].get("same number, different text", 0), + kinds_by_column[column].get("Excel error in GIS", 0) + + kinds_by_column[column].get("Excel error in ChemicalData", 0), + recommend(column, kinds_by_column[column]), + None, + None, + ] + for column, _ in by_column.most_common() + ), + decision_columns=[(9, DECISION_CHOICES)], + ) + + _add_table( + report, + "CellDifferences", + [ + "SAMPLE", + "Area", + "Mirror column", + "Workbook header", + "ChemicalData value", + "GIS value", + "Kind of difference", + "ChemicalData row", + "GIS row", + "DECISION", + ], + [20, 18, 22, 32, 24, 24, 26, 16, 12, 22], + ( + [ + row["sample"], + row["area"], + row["column"], + row["header"], + row["chemical_data_value"], + row["gis_value"], + row["kind"], + row["chemical_data_row"], + row["gis_row"], + None, + ] + for row in sorted( + differences, key=lambda row: (row["column"], row["sample"] or "") + ) + ), + decision_columns=[(10, DECISION_CHOICES)], + ) + + _add_table( + report, + "RowsOnlyInOneSheet", + [ + "Sheet", + "SAMPLE", + "Area", + "Reference", + "Source row", + "Has coordinates", + "Note", + "DECISION", + ], + [16, 22, 20, 40, 12, 16, 46, 22], + ( + [ + sheet.name, + sheet.records[index].get("sample"), + sheet.records[index].get("area"), + sheet.records[index].get("reference"), + sheet.row_numbers[index], + ( + "yes" + if parse_value(sheet.records[index].get("latitude")).number + is not None + else "no" + ), + note, + None, + ] + for sheet, indexes, note in ( + ( + chemical_data, + chemical_data_only, + "In ChemicalData only - appended after GIS was last synced", + ), + (gis, gis_only, "In GIS only"), + ) + for index in indexes + ), + decision_columns=[ + (8, '"Add to GIS,Drop - not a sample,Keep as-is,Needs review"') + ], + ) + + _add_table( + report, + "IntegritySummary", + [ + "Issue", + "Rows affected", + "Why it matters", + "RESOLUTION", + "Notes", + ], + [46, 14, 76, 26, 40], + ( + [issue, count, INTEGRITY_NOTES.get(issue, ""), None, None] + for issue, count in issue_counts.most_common() + ), + decision_columns=[(4, INTEGRITY_CHOICES)], + ) + + _add_table( + report, + "IntegrityDetail", + [ + "Sheet", + "Source row", + "SAMPLE", + "Area", + "Reference", + "Issue", + "Column", + "Value", + "Detail", + "RESOLUTION", + ], + [14, 12, 20, 18, 34, 40, 20, 20, 60, 24], + ( + [ + row["sheet"], + row["source_row"], + row["sample"], + row["area"], + row["reference"], + row["issue"], + row["column"], + row["value"], + row["note"], + None, + ] + for row in sorted(detail, key=lambda row: (row["issue"], row["source_row"])) + ), + decision_columns=[(10, INTEGRITY_CHOICES)], + ) + + _add_table( + report, + "DetectionLimitSpread", + [ + "Mirror column", + "Workbook header", + "Declared unit", + "Distinct '<' thresholds", + "Lowest", + "Highest", + "DetectionLimits sheet", + "Thresholds seen (first 25)", + "RESOLUTION", + ], + [22, 24, 14, 20, 12, 12, 20, 62, 24], + ( + [ + row["column"], + row["header"], + row["declared_unit"], + row["distinct_thresholds"], + row["lowest"], + row["highest"], + row["detection_limits_sheet"], + row["thresholds"], + None, + ] + for row in thresholds + ), + decision_columns=[(9, INTEGRITY_CHOICES)], + ) + + _add_table( + report, + "DuplicateSampleNames", + ["SAMPLE", "Rows with this name", "Areas", "Source rows", "RESOLUTION"], + [22, 20, 46, 40, 26], + ( + [row["sample"], row["rows"], row["areas"], row["source_rows"], None] + for row in duplicates + ), + decision_columns=[ + (5, '"Same sample - de-duplicate,Different samples - rename,Needs review"') + ], + ) + + _add_readme( + report, + [ + ("Critical minerals reconciliation", ""), + ( + "What this is", + "Everywhere the delivered workbook contradicts itself, laid out for a " + "decision. Two kinds of problem: the ChemicalData and GIS sheets " + "disagree with each other, and some values fail a data-integrity " + "check regardless of which sheet they came from.", + ), + ( + "Who fills it in", + "V.T. McLemore, or whoever owns the source data. Every yellow column " + "is blank on purpose and has a dropdown; the grey columns are " + "generated and should not be edited.", + ), + ( + "What happens next", + "The CM_legacy staging mirror already holds BOTH sheets verbatim, so " + "nothing has been lost and nothing has been merged. The rulings here " + "become the merge rules for the transform into the Ocotillo data " + "model.", + ), + ("", ""), + ("Source workbook", workbook_path.name), + ("Generated", date.today().isoformat()), + ("ChemicalData rows", str(len(chemical_data.records))), + ("GIS rows", str(len(gis.records))), + ("Rows aligned on SAMPLE", str(len(pairs))), + ("Rows that disagree", str(rows_with_differences)), + ("Cells that disagree", str(len(differences))), + ("Integrity findings", str(len(detail))), + ("", ""), + ("Sheet: ColumnDecisions", "One row per column that differs. Start here."), + ( + "Sheet: CellDifferences", + "Every disagreeing cell, so a column-level ruling can be checked or " + "overridden row by row.", + ), + ( + "Sheet: RowsOnlyInOneSheet", + "Samples present in one sheet and not the other. ChemicalData has " + "rows appended after GIS was last synced; one of them is not a " + "sample at all but the 'NOTE: SEE THE ORIGINAL CITATION...' text.", + ), + ( + "Sheet: IntegritySummary", + "Data-integrity findings grouped by issue, worst first.", + ), + ("Sheet: IntegrityDetail", "The individual rows behind each finding."), + ( + "Sheet: DetectionLimitSpread", + "Analytes reported against many different '<' detection limits, i.e. " + "results pooled across labs and decades without normalizing.", + ), + ( + "Sheet: DuplicateSampleNames", + "SAMPLE is not unique, so it cannot be the key on its own.", + ), + ("", ""), + ( + "Note on units", + "Integrity checks compare each analyte against the unit declared in " + "the workbook's own units row (ChemicalData row 3). A '%' column " + "holding 27700 is the clearest case: that is ppm in a percent " + "column.", + ), + ( + "Note on scope", + "Only ChemicalData is integrity-checked. Checking GIS as well would " + "double every finding without adding information, since the " + "sheet-to-sheet differences are already listed above.", + ), + ], + ) + + output_path.parent.mkdir(parents=True, exist_ok=True) + report.save(output_path) + + return { + "chemical_data_rows": len(chemical_data.records), + "gis_rows": len(gis.records), + "aligned_rows": len(pairs), + "rows_with_differences": rows_with_differences, + "cell_differences": len(differences), + "columns_with_differences": len(by_column), + "rows_only_in_chemical_data": len(chemical_data_only), + "rows_only_in_gis": len(gis_only), + "integrity_findings": len(detail), + "integrity_issue_types": len(issue_counts), + "analytes_with_threshold_spread": len(thresholds), + "duplicate_sample_names": len(duplicates), + } + + +INTEGRITY_NOTES = { + "Value impossible for the declared unit": ( + "The workbook declares this column's unit in its units row, and the value " + "cannot be that unit -- a percentage over 100. Almost certainly ppm " + "reported in a % column. The transform cannot guess which rows to convert." + ), + "Detection limit implausible for the declared unit": ( + "Column is declared ppb but the '<' limit is a ppm-scale number, so some " + "rows in this column are in different units from the rest." + ), + "Non-numeric analyte value": ( + "Text such as 'bd', 'nd', 'tr', 'nr', 'n/a', '----' or '>2%' where a number " + "belongs. Each token needs a ruling: below detection (and at what limit), " + "not determined, trace, or not reported." + ), + "Total disagrees with the sum of the major oxides": ( + "The reported Total is not the sum of SiO2..LOI for that row, so either a " + "major oxide is missing from the row or the Total is stale." + ), + "Total outside 95-105%": ( + "Total is far from 100%, so the analysis is partial rather than a complete " + "whole-rock analysis. Fine for trace-element work, misleading if read as " + "whole rock. Excluded from the unit check above because Total is a sum, " + "not a measurement." + ), + "Total is not a number": ( + "A broken spreadsheet formula ('#VALUE!') rather than a measurement." + ), + "No coordinates": ( + "The sample cannot be mapped or matched to a Location. Some of these rows " + "still declare a coordinate system, which suggests the coordinates were " + "meant to be filled in." + ), + "Only one of latitude/longitude": "Half a coordinate pair is unusable as a point.", + "Coordinates with no declared datum": ( + "NAD27 and WGS84 differ by roughly 100 m in New Mexico, so an undeclared " + "datum is a 100 m position error." + ), + "Positive longitude": "A missing minus sign puts the sample on the other side of the world.", + "Latitude outside New Mexico": ( + "Outside the state bounding box: transposed digits, a swapped lat/long pair, " + "or a genuinely out-of-state sample that should be labelled as such." + ), + "Longitude outside New Mexico": ( + "Outside the state bounding box -- same causes as an out-of-range latitude." + ), + "Date is not a full date": ( + "Year-only or free-text dates cannot become a field-event date without a " + "convention for what day to use." + ), + "Analyzed before collected": "One of the two dates is wrong.", +} + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--workbook", + required=True, + type=Path, + help="Path to the McLemoreMasterChem .xlsx", + ) + parser.add_argument( + "--out", + type=Path, + default=Path("cm_reconciliation.xlsx"), + help="Where to write the reconciliation workbook", + ) + arguments = parser.parse_args(argv) + + summary = build_report(arguments.workbook, arguments.out) + width = max(len(key) for key in summary) + for key, value in summary.items(): + print(f"{key:<{width}} {value:>8}") + print(f"\nwrote {arguments.out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/cm_legacy_mirror.py b/services/cm_legacy_mirror.py new file mode 100644 index 000000000..dcd5da233 --- /dev/null +++ b/services/cm_legacy_mirror.py @@ -0,0 +1,358 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Load the McLemore critical-minerals workbook into the CM_legacy mirror. + +Reads ``McLemoreMasterChem*.xlsx`` and writes each sheet, unchanged, into the +``CM_*`` staging tables (see ``db/cm_legacy.py``). No interpretation happens +here: every cell becomes text, censored values keep their ``<`` prefix, Excel +error text is carried through, and the ChemicalData / GIS / QAQC sheets are all +loaded even though they disagree with each other. + +The load is idempotent per sheet -- existing rows for a sheet are deleted before +its rows are inserted -- so a revised workbook can be reloaded without +duplicating rows or resetting the whole mirror. + +Layout is asserted, not guessed. Each sheet declares which 1-based row holds +its header, and the load fails if that row does not contain the expected label. +A revised workbook that moves a header row, renames a column, or adds one must +be looked at by a human before it lands in the mirror. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import date, datetime +from decimal import Decimal +from pathlib import Path +from typing import Any, Iterator, Sequence + +from sqlalchemy import delete, insert +from sqlalchemy.orm import Session + +from db.cm_legacy import ( + CM_SHEET_CHEMICAL_DATA, + CM_SHEET_GIS, + CM_SHEET_QAQC, + SOURCE_HEADER_BY_COLUMN, + CM_ChemicalData, + CM_DetectionLimits, + CM_MineralSystems, + CM_References, + CM_WorkbookMetadata, + CM_WorldComparisons, + CM_WorldReferences, +) + +# 1-based row holding the column headers, and the first row of data, for each +# sheet mirrored into CM_ChemicalData. ChemicalData carries a title banner in +# row 1 and a units row in row 3; GIS is the same columns with neither. +CHEMISTRY_SHEET_LAYOUT: dict[str, tuple[int, int]] = { + CM_SHEET_CHEMICAL_DATA: (2, 4), + CM_SHEET_GIS: (1, 2), + CM_SHEET_QAQC: (2, 3), +} + +# Sheets flattened into CM_WorkbookMetadata as (label, value) pairs. +METADATA_SHEETS = ("General Information", "MetaData", "DefinitionOfFields") + +MINERAL_SYSTEM_COLUMNS = ( + "system_name", + "synopsis", + "deposit_types", + "principal_commodities", + "critical_minerals", + "references", + "phase_2", + "phase_3", + "phase_4", +) + +INSERT_BATCH_SIZE = 1000 + + +class CMWorkbookError(ValueError): + """The workbook does not have the layout the mirror was built for.""" + + +@dataclass +class CMMirrorLoadResult: + """Rows written per mirror table, plus anything the loader wants flagged.""" + + rows_by_sheet: dict[str, int] = field(default_factory=dict) + warnings: list[str] = field(default_factory=list) + + @property + def total_rows(self) -> int: + return sum(self.rows_by_sheet.values()) + + +def normalize_header(value: Any) -> str: + return " ".join(str(value).split()).lower() + + +def cell_to_text(value: Any) -> str | None: + """Render a cell as text without losing or reinterpreting anything. + + Dates become ISO-8601 (the workbook stores them as Excel serials, which + would otherwise mirror as meaningless integers), numbers keep Python's + round-trippable repr, and everything else -- including ``<0.1`` and + ``#VALUE!`` -- is passed through with surrounding whitespace stripped. + """ + if value is None: + return None + if isinstance(value, datetime): + return ( + value.isoformat(sep=" ") + if any((value.hour, value.minute, value.second, value.microsecond)) + else value.date().isoformat() + ) + if isinstance(value, date): + return value.isoformat() + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + if isinstance(value, (int, float, Decimal)): + return str(value) + text = str(value).strip() + return text or None + + +def _is_blank(row: Sequence[Any]) -> bool: + return all(cell_to_text(v) is None for v in row) + + +def column_by_header(table_name: str) -> dict[str, str]: + """Invert SOURCE_HEADER_BY_COLUMN into {normalized header: column}.""" + return { + normalize_header(header): column + for column, header in SOURCE_HEADER_BY_COLUMN[table_name].items() + } + + +def _map_headers( + sheet_name: str, + header_row: Sequence[Any], + columns_by_header: dict[str, str], +) -> dict[int, str]: + """Map each populated header cell to its mirror column, or fail.""" + mapping: dict[int, str] = {} + unknown: list[str] = [] + for index, cell in enumerate(header_row): + if cell_to_text(cell) is None: + continue + column = columns_by_header.get(normalize_header(cell)) + if column is None: + unknown.append(str(cell).strip()) + continue + mapping[index] = column + if unknown: + raise CMWorkbookError( + f"sheet {sheet_name!r} has {len(unknown)} header(s) the mirror has no " + f"column for: {unknown[:10]}. The workbook changed shape; update " + f"db/cm_legacy.py (and a migration) before loading it." + ) + if not mapping: + raise CMWorkbookError(f"sheet {sheet_name!r} has no recognizable headers") + return mapping + + +def _rows(worksheet: Any) -> list[tuple[Any, ...]]: + return list(worksheet.iter_rows(values_only=True)) + + +def _iter_sheet_records( + worksheet: Any, + sheet_name: str, + header_row_number: int, + first_data_row_number: int, + columns_by_header: dict[str, str], + expected_header: str, +) -> Iterator[dict[str, Any]]: + rows = _rows(worksheet) + if len(rows) < first_data_row_number: + raise CMWorkbookError(f"sheet {sheet_name!r} has no data rows") + + header_row = rows[header_row_number - 1] + if expected_header not in { + normalize_header(c) for c in header_row if c is not None + }: + raise CMWorkbookError( + f"sheet {sheet_name!r} row {header_row_number} is not the header row " + f"(expected to find {expected_header!r})" + ) + + mapping = _map_headers(sheet_name, header_row, columns_by_header) + for offset, row in enumerate(rows[first_data_row_number - 1 :]): + if _is_blank(row): + continue + record: dict[str, Any] = {} + for index, column in mapping.items(): + if index < len(row): + record[column] = cell_to_text(row[index]) + record["source_row"] = first_data_row_number + offset + yield record + + +def _replace( + session: Session, model: Any, records: list[dict[str, Any]], **scope: Any +) -> int: + """Delete the scoped rows, then insert the given ones.""" + statement = delete(model) + for column, value in scope.items(): + statement = statement.where(getattr(model, column) == value) + session.execute(statement) + for start in range(0, len(records), INSERT_BATCH_SIZE): + batch = records[start : start + INSERT_BATCH_SIZE] + if batch: + session.execute(insert(model), batch) + return len(records) + + +def _load_chemistry_sheets(workbook: Any, session: Session) -> dict[str, int]: + columns_by_header = column_by_header("CM_ChemicalData") + counts: dict[str, int] = {} + for sheet_name, (header_row, first_data_row) in CHEMISTRY_SHEET_LAYOUT.items(): + if sheet_name not in workbook.sheetnames: + raise CMWorkbookError(f"workbook is missing the {sheet_name!r} sheet") + records = [ + dict(record, source_sheet=sheet_name) + for record in _iter_sheet_records( + workbook[sheet_name], + sheet_name, + header_row, + first_data_row, + columns_by_header, + expected_header="sample", + ) + ] + counts[sheet_name] = _replace( + session, CM_ChemicalData, records, source_sheet=sheet_name + ) + return counts + + +def _load_detection_limits(workbook: Any, session: Session) -> int: + rows = _rows(workbook["DetectionLimits"]) + method = cell_to_text(rows[0][0]) if rows else None + records = [ + { + "source_row": number, + "method": method, + "element": cell_to_text(row[0]), + "lower_reporting_limit": cell_to_text(row[1]) if len(row) > 1 else None, + "unit": cell_to_text(row[2]) if len(row) > 2 else None, + } + for number, row in enumerate(rows[2:], start=3) + if not _is_blank(row) + ] + return _replace(session, CM_DetectionLimits, records) + + +def _load_citations(workbook: Any, session: Session, sheet: str, model: Any) -> int: + records = [ + {"source_row": number, "citation": cell_to_text(row[0])} + for number, row in enumerate(_rows(workbook[sheet]), start=1) + if not _is_blank(row) + ] + return _replace(session, model, records) + + +def _load_mineral_systems(workbook: Any, session: Session) -> int: + records = [] + for number, row in enumerate(_rows(workbook["MineralSystems"]), start=1): + if _is_blank(row): + continue + record: dict[str, Any] = {"source_row": number} + for index, column in enumerate(MINERAL_SYSTEM_COLUMNS): + record[column] = cell_to_text(row[index]) if index < len(row) else None + records.append(record) + return _replace(session, CM_MineralSystems, records) + + +def _load_world_comparisons(workbook: Any, session: Session) -> int: + records = list( + _iter_sheet_records( + workbook["world"], + "world", + header_row_number=1, + first_data_row_number=2, + columns_by_header=column_by_header("CM_WorldComparisons"), + expected_header="area", + ) + ) + return _replace(session, CM_WorldComparisons, records) + + +def _load_workbook_metadata(workbook: Any, session: Session) -> dict[str, int]: + counts: dict[str, int] = {} + for sheet in METADATA_SHEETS: + records = [ + { + "source_sheet": sheet, + "source_row": number, + "label": cell_to_text(row[0]) if row else None, + "value": cell_to_text(row[1]) if len(row) > 1 else None, + } + for number, row in enumerate(_rows(workbook[sheet]), start=1) + if not _is_blank(row) + ] + counts[sheet] = _replace( + session, CM_WorkbookMetadata, records, source_sheet=sheet + ) + return counts + + +def load_cm_workbook(source_file: Path | str, session: Session) -> CMMirrorLoadResult: + """Mirror every sheet of the critical-minerals workbook into CM_* tables. + + The caller owns the transaction, so a failure part way through leaves the + mirror untouched rather than half-loaded. + """ + import openpyxl + + source_file = Path(source_file) + if not source_file.exists(): + raise CMWorkbookError(f"workbook not found: {source_file}") + + workbook = openpyxl.load_workbook(source_file, read_only=True, data_only=True) + try: + result = CMMirrorLoadResult() + result.rows_by_sheet.update(_load_chemistry_sheets(workbook, session)) + result.rows_by_sheet["DetectionLimits"] = _load_detection_limits( + workbook, session + ) + result.rows_by_sheet["References"] = _load_citations( + workbook, session, "References", CM_References + ) + result.rows_by_sheet["MineralSystems"] = _load_mineral_systems( + workbook, session + ) + result.rows_by_sheet["world"] = _load_world_comparisons(workbook, session) + result.rows_by_sheet["world_ref"] = _load_citations( + workbook, session, "world_ref", CM_WorldReferences + ) + result.rows_by_sheet.update(_load_workbook_metadata(workbook, session)) + finally: + workbook.close() + + chemical_data = result.rows_by_sheet.get(CM_SHEET_CHEMICAL_DATA, 0) + gis = result.rows_by_sheet.get(CM_SHEET_GIS, 0) + if chemical_data != gis: + result.warnings.append( + f"ChemicalData ({chemical_data} rows) and GIS ({gis} rows) disagree on row " + f"count; the sheets are known to have drifted apart and reconciliation is " + f"deferred (see db/cm_legacy.py)" + ) + return result diff --git a/tests/test_cm_legacy.py b/tests/test_cm_legacy.py new file mode 100644 index 000000000..9e0ba559d --- /dev/null +++ b/tests/test_cm_legacy.py @@ -0,0 +1,484 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Tests for the CM_legacy critical-minerals staging mirror.""" + +from datetime import date, datetime +from decimal import Decimal + +import openpyxl +import pytest +from sqlalchemy import delete, select + +from db.cm_legacy import ( + ANALYTE_UNITS, + CM_CHEMISTRY_SOURCE_SHEETS, + QAQC_MISSING_COLUMNS, + SOURCE_HEADER_BY_COLUMN, + CM_ChemicalData, + CM_DetectionLimits, + CM_MineralSystems, + CM_References, + CM_WorkbookMetadata, + CM_WorldComparisons, + CM_WorldReferences, +) +from db.engine import session_ctx +from services.cm_legacy_mirror import ( + CMWorkbookError, + cell_to_text, + load_cm_workbook, +) + +CHEM_HEADERS = [ + "SAMPLE ", + "Project", + "Area", + "Date collected", + "latitude", + "longitude", + "Coordinate system", + "MapSymbol", + "Depth/legnth (ft)", + "paste pH", + "SiO2", + "Total", + "Au", + "As", + "In", + "Pd", + "Pt", +] +CHEM_UNITS = [None] * 10 + ["%", "%", "ppb", "ppm", "ppm", "ppm", "ppm"] +CHEM_ROWS = [ + [ + "Gal1", + "Gallinas", + "Gallinas", + datetime(2019, 8, 28), + "34.19701153", + "-105.74021543", + "WGS84", + "Tv", + 0, + None, + 74.5, + 101.84, + "<10", + 82, + "<0.1", + 4, + "<5", + ], + [ + "Gal2", + "Gallinas", + "ZuniMountains", + datetime(2020, 1, 15), + None, + None, + "NAD27", + None, + 12, + 7.2, + 66.15, + "#VALUE!", + None, + 3, + None, + None, + None, + ], +] + +CM_MODELS = ( + CM_ChemicalData, + CM_DetectionLimits, + CM_References, + CM_MineralSystems, + CM_WorldComparisons, + CM_WorldReferences, + CM_WorkbookMetadata, +) + + +def _write_workbook(path, *, chem_headers=None, extra_header=None, banner=True): + """Build a miniature workbook with the same layout as the real one.""" + headers = list(chem_headers if chem_headers is not None else CHEM_HEADERS) + units = list(CHEM_UNITS) + if extra_header is not None: + headers.append(extra_header) + units.append(None) + + workbook = openpyxl.Workbook() + chemical_data = workbook.active + chemical_data.title = "ChemicalData" + if banner: + chemical_data.append(["Chemical analyses of samples"]) + chemical_data.append(headers) + chemical_data.append(["Units"] + units[1:]) + for row in CHEM_ROWS: + chemical_data.append(row) + + # GIS: same columns, no banner and no units row, and one fewer row -- + # the drift the mirror deliberately preserves. + gis = workbook.create_sheet("GIS") + gis.append(headers) + gis.append(CHEM_ROWS[0]) + + # QAQC: header on row 2, no MapSymbol/Pd/Pt, capitalized Latitude/Longitude. + qaqc = workbook.create_sheet("QAQC") + dropped = {"MapSymbol", "Pd", "Pt"} + keep = [i for i, h in enumerate(headers) if h.strip() not in dropped] + qaqc.append([]) + qaqc.append( + [ + {"latitude": "Latitude", "longitude": "Longitude"}.get( + headers[i], headers[i] + ) + for i in keep + ] + ) + qaqc.append([CHEM_ROWS[0][i] if i < len(CHEM_ROWS[0]) else None for i in keep]) + + detection_limits = workbook.create_sheet("DetectionLimits") + detection_limits.append(["Method C_ICPOES_MS-61"]) + detection_limits.append(["Element", "Lower Reporting Limit", "Unit"]) + detection_limits.append(["Ag", 0.01, "ppm"]) + detection_limits.append(["Al", 100, "ppm"]) + + references = workbook.create_sheet("References") + references.append(["McLemore, V.T., 2025, Critical minerals in New Mexico"]) + references.append(["Hofstra, A.H. and Kreiner, D.C., 2020, Mineral systems"]) + + mineral_systems = workbook.create_sheet("MineralSystems") + mineral_systems.append( + [ + "System Name", + "Synopsis", + "Deposit types", + "Principal commodities", + "Critical minerals", + "References", + "Phase 2", + "Phase 3", + "Phase 4", + ] + ) + mineral_systems.append(["Magmatic REE", "syn", "Peralkaline", "REE", "Nd", "H&K"]) + + world = workbook.create_sheet("world") + world.append(["area", "deposit", "reference", "La", "Ce", "TREE", "grade %"]) + world.append(["crustal abundance", None, None, 31, 63, 200, 0.1]) + + world_ref = workbook.create_sheet("world_ref") + world_ref.append(["Boyer, D.S., 2011, La Paz Rare Earth Project"]) + + general = workbook.create_sheet("General Information") + general.append(["Critical Minerals in New Mexico"]) + general.append(["Title", "Earth MRI database"]) + + metadata = workbook.create_sheet("MetaData") + metadata.append(["Title", "Earth MRI database"]) + + definitions = workbook.create_sheet("DefinitionOfFields") + definitions.append(["Area", "Geographic area, generally mining district"]) + + workbook.save(path) + return path + + +def _clear_mirror(session): + for model in CM_MODELS: + session.execute(delete(model)) + session.commit() + + +@pytest.fixture() +def cm_workbook(tmp_path): + path = _write_workbook(tmp_path / "McLemoreMasterChem_test.xlsx") + yield path + with session_ctx() as session: + _clear_mirror(session) + + +def _load(path): + """Load the workbook the way a caller must: the loader does not commit.""" + with session_ctx() as session: + _clear_mirror(session) + result = load_cm_workbook(path, session) + session.commit() + return result + + +def test_source_headers_cover_every_mirrored_column(): + """Every CM_ChemicalData data column maps back to a workbook header.""" + provenance = {"id", "source_sheet", "source_row"} + columns = {c.name for c in CM_ChemicalData.__table__.columns} - provenance + assert columns == set(SOURCE_HEADER_BY_COLUMN["CM_ChemicalData"]) + assert len(columns) == 118 + + +def test_analyte_units_are_declared_for_every_unit_suffixed_column(): + suffixed = { + c.name + for c in CM_ChemicalData.__table__.columns + if c.name.endswith(("_pct", "_ppm", "_ppb")) + } + # tds_mg_l and depth_legnth_ft carry units in their source header, not in + # the units row, so they are not analytes. + assert suffixed == set(ANALYTE_UNITS) + assert ANALYTE_UNITS["au_ppb"] == "ppb" + assert ANALYTE_UNITS["sio2_pct"] == "%" + assert ANALYTE_UNITS["as_ppm"] == "ppm" + + +@pytest.mark.parametrize( + "value, expected", + [ + (None, None), + ("", None), + (" ", None), + ("<0.1", "<0.1"), + ("#VALUE!", "#VALUE!"), + (" ZuniMountains ", "ZuniMountains"), + (0, "0"), + (74.5, "74.5"), + # A double whose shortest round-trip repr is not its rounded form: + # the mirror keeps every digit rather than truncating. + (101.83999999999999, "101.83999999999999"), + (Decimal("0.010"), "0.010"), + (True, "TRUE"), + (datetime(2019, 8, 28), "2019-08-28"), + (datetime(2019, 8, 28, 13, 45), "2019-08-28 13:45:00"), + (date(2019, 8, 28), "2019-08-28"), + ], +) +def test_cell_rendering_preserves_the_source_value(value, expected): + assert cell_to_text(value) == expected + + +def test_load_mirrors_every_sheet(cm_workbook): + result = _load(cm_workbook) + + assert result.rows_by_sheet == { + "ChemicalData": 2, + "GIS": 1, + "QAQC": 1, + "DetectionLimits": 2, + "References": 2, + "MineralSystems": 2, + "world": 1, + "world_ref": 1, + "General Information": 2, + "MetaData": 1, + "DefinitionOfFields": 1, + } + assert result.total_rows == 16 + + with session_ctx() as session: + assert ( + session.scalar( + select(CM_DetectionLimits).where(CM_DetectionLimits.element == "Ag") + ).method + == "Method C_ICPOES_MS-61" + ) + assert ( + session.scalars(select(CM_References.citation)) + .all()[0] + .startswith("McLemore") + ) + assert ( + session.scalar( + select(CM_WorldComparisons.grade_pct).where( + CM_WorldComparisons.area == "crustal abundance" + ) + ) + == "0.1" + ) + assert session.scalar(select(CM_WorldReferences.citation)).startswith("Boyer") + assert ( + session.scalar( + select(CM_WorkbookMetadata.value).where( + CM_WorkbookMetadata.source_sheet == "DefinitionOfFields", + CM_WorkbookMetadata.label == "Area", + ) + ) + == "Geographic area, generally mining district" + ) + # The MineralSystems header row is mirrored positionally, notes and all. + assert ( + session.scalar( + select(CM_MineralSystems.system_name).where( + CM_MineralSystems.source_row == 1 + ) + ) + == "System Name" + ) + + +def test_all_three_chemistry_sheets_land_in_one_table(cm_workbook): + _load(cm_workbook) + + with session_ctx() as session: + sheets = session.scalars( + select(CM_ChemicalData.source_sheet) + .distinct() + .order_by(CM_ChemicalData.source_sheet) + ).all() + assert set(sheets) == set(CM_CHEMISTRY_SOURCE_SHEETS) + + # Same sample name, mirrored once per sheet -- no cross-sheet dedupe. + gal1 = session.scalars( + select(CM_ChemicalData).where(CM_ChemicalData.sample == "Gal1") + ).all() + assert len(gal1) == 3 + + +def test_source_row_records_the_spreadsheet_row_number(cm_workbook): + _load(cm_workbook) + + with session_ctx() as session: + # ChemicalData data starts on row 4 (banner, header, units). + chemical_data = session.scalars( + select(CM_ChemicalData) + .where(CM_ChemicalData.source_sheet == "ChemicalData") + .order_by(CM_ChemicalData.source_row) + ).all() + assert [row.source_row for row in chemical_data] == [4, 5] + # GIS data starts on row 2 (header only). + assert ( + session.scalar( + select(CM_ChemicalData.source_row).where( + CM_ChemicalData.source_sheet == "GIS" + ) + ) + == 2 + ) + # QAQC data starts on row 3 (blank row, header). + assert ( + session.scalar( + select(CM_ChemicalData.source_row).where( + CM_ChemicalData.source_sheet == "QAQC" + ) + ) + == 3 + ) + + +def test_cells_are_mirrored_without_reinterpretation(cm_workbook): + _load(cm_workbook) + + with session_ctx() as session: + row = session.scalar( + select(CM_ChemicalData).where( + CM_ChemicalData.source_sheet == "ChemicalData", + CM_ChemicalData.source_row == 4, + ) + ) + # Censored values keep their qualifier instead of becoming NULL or 0. + assert row.au_ppb == "<10" + assert row.in_ppm == "<0.1" + assert row.pt_ppm == "<5" + # Numbers keep their value, not a rounded rendering. + assert row.total_pct == "101.84" + assert row.sio2_pct == "74.5" + # Dates become ISO-8601, not Excel serials. + assert row.date_collected == "2019-08-28" + # Blank cells are NULL, not empty strings. + assert row.paste_ph is None + + excel_error = session.scalar( + select(CM_ChemicalData.total_pct).where( + CM_ChemicalData.source_sheet == "ChemicalData", + CM_ChemicalData.source_row == 5, + ) + ) + assert excel_error == "#VALUE!" + + +def test_qaqc_columns_absent_from_the_sheet_are_null(cm_workbook): + _load(cm_workbook) + + with session_ctx() as session: + row = session.scalar( + select(CM_ChemicalData).where(CM_ChemicalData.source_sheet == "QAQC") + ) + for column in QAQC_MISSING_COLUMNS: + assert getattr(row, column) is None + # Capitalized Latitude/Longitude still map to the mirror columns. + assert row.latitude == "34.19701153" + assert row.longitude == "-105.74021543" + + +def test_reload_replaces_rows_per_sheet_without_duplicating(cm_workbook): + _load(cm_workbook) + with session_ctx() as session: + first = load_cm_workbook(cm_workbook, session) + session.commit() + with session_ctx() as session: + second = load_cm_workbook(cm_workbook, session) + session.commit() + + assert first.rows_by_sheet == second.rows_by_sheet + with session_ctx() as session: + assert ( + session.scalar( + select(CM_ChemicalData.source_row).where( + CM_ChemicalData.sample == "Gal1" + ) + ) + is not None + ) + assert ( + len( + session.scalars( + select(CM_ChemicalData).where(CM_ChemicalData.sample == "Gal1") + ).all() + ) + == 3 + ) + + +def test_row_count_drift_between_chemicaldata_and_gis_is_reported(cm_workbook): + result = _load(cm_workbook) + + assert any("reconciliation is deferred" in warning for warning in result.warnings) + + +def test_unknown_header_aborts_the_load(tmp_path): + path = _write_workbook( + tmp_path / "extra_column.xlsx", extra_header="NewAnalyteNobodyMapped" + ) + + with pytest.raises(CMWorkbookError, match="NewAnalyteNobodyMapped"): + _load(path) + + with session_ctx() as session: + assert session.scalars(select(CM_ChemicalData)).all() == [] + + +def test_moved_header_row_aborts_the_load(tmp_path): + # Dropping the banner shifts every ChemicalData row up by one. + path = _write_workbook(tmp_path / "no_banner.xlsx", banner=False) + + with pytest.raises(CMWorkbookError, match="is not the header row"): + _load(path) + + +def test_missing_workbook_raises(): + with pytest.raises(CMWorkbookError, match="workbook not found"): + with session_ctx() as session: + load_cm_workbook("/nonexistent/McLemoreMasterChem.xlsx", session) diff --git a/tests/test_cm_reconciliation_report.py b/tests/test_cm_reconciliation_report.py new file mode 100644 index 000000000..19b9f6bfe --- /dev/null +++ b/tests/test_cm_reconciliation_report.py @@ -0,0 +1,305 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Tests for the critical-minerals reconciliation report.""" + +import openpyxl +import pytest + +from scripts.cm_reconciliation_report import build_report, classify, recommend + +HEADERS = [ + "SAMPLE ", + "Project", + "Area", + "Reference", + "Date collected", + "Date analyzed", + "Laboratory", + "latitude", + "longitude", + "Coordinate system", + "SiO2", + "TiO2", + "Al2O3", + "Fe2O3T", + "MnO", + "MgO", + "CaO", + "Na2O", + "K2O", + "P2O5", + "LOI ", + "F", + "Total", + "Au", +] +UNITS = [None] * 10 + ["%"] * 12 + ["ppb"] + +# A clean row: majors sum to ~100, coordinates in New Mexico, real dates. +CLEAN = [ + "Gal1", + "Gallinas", + "Gallinas", + "McLemore et al. (2025b)", + "2019-08-28", + "2020-04-27", + "ALS", + "34.19701153", + "-105.74021543", + "WGS84", + 60.0, + 0.5, + 16.0, + 5.0, + 0.1, + 2.0, + 5.0, + 3.0, + 5.0, + 0.2, + 3.2, + 0.5, + 100.5, + 12, +] + + +def _row(**overrides): + row = list(CLEAN) + for header, value in overrides.items(): + row[HEADERS.index(header)] = value + return row + + +CHEMICAL_DATA_ROWS = [ + CLEAN, + # F reported in ppm inside a column the workbook declares as %. + _row(**{"SAMPLE ": "Gal2", "F": 27700}), + # Total nowhere near the sum of the majors it is meant to add up. + _row(**{"SAMPLE ": "Gal3", "Total": 60.0}), + # Coordinates in Kansas, and a year-only collection date. + _row(**{"SAMPLE ": "Gal4", "latitude": "39.5", "Date collected": "1993"}), + # Text where an analyte value belongs, and no coordinates at all. + _row(**{"SAMPLE ": "Gal5", "Au": "bd", "latitude": None, "longitude": None}), + # A repeated sample name, so SAMPLE cannot be the key on its own. + _row(**{"SAMPLE ": "Gal1", "Laboratory": "USGS"}), + # Present in ChemicalData only, as the real workbook's tail rows are. + _row(**{"SAMPLE ": "Gal6"}), +] +# GIS drops the last row and disagrees on Laboratory (blank) and Total (#VALUE!). +GIS_ROWS = [ + _row(Laboratory=None), + _row(**{"SAMPLE ": "Gal2", "F": 27700}), + _row(**{"SAMPLE ": "Gal3", "Total": "#VALUE!"}), + _row(**{"SAMPLE ": "Gal4", "latitude": "39.5", "Date collected": "1993"}), + _row(**{"SAMPLE ": "Gal5", "Au": "bd", "latitude": None, "longitude": None}), + _row(**{"SAMPLE ": "Gal1", "Laboratory": "USGS"}), +] + + +@pytest.fixture() +def source_workbook(tmp_path): + workbook = openpyxl.Workbook() + chemical_data = workbook.active + chemical_data.title = "ChemicalData" + chemical_data.append(["Chemical analyses of samples"]) + chemical_data.append(HEADERS) + chemical_data.append(["Units"] + UNITS[1:]) + for row in CHEMICAL_DATA_ROWS: + chemical_data.append(row) + + gis = workbook.create_sheet("GIS") + gis.append(HEADERS) + for row in GIS_ROWS: + gis.append(row) + + detection_limits = workbook.create_sheet("DetectionLimits") + detection_limits.append(["Method C_ICPOES_MS-61"]) + detection_limits.append(["Element", "Lower Reporting Limit", "Unit"]) + detection_limits.append(["Au", 10, "ppb"]) + + path = tmp_path / "McLemoreMasterChem_test.xlsx" + workbook.save(path) + return path + + +@pytest.fixture() +def report(source_workbook, tmp_path): + output = tmp_path / "reconciliation.xlsx" + summary = build_report(source_workbook, output) + return summary, openpyxl.load_workbook(output) + + +def _records(worksheet): + rows = list(worksheet.iter_rows(values_only=True)) + return [dict(zip(rows[0], row)) for row in rows[1:]] + + +@pytest.mark.parametrize( + "chemical_data_value, gis_value, expected", + [ + ("ALS", "ALS", None), + (" ALS ", "ALS", None), + (None, None, None), + ("ALS", None, "only in ChemicalData"), + (None, "ALS", "only in GIS"), + ("100.5", "#VALUE!", "Excel error in GIS"), + ("#VALUE!", "100.5", "Excel error in ChemicalData"), + ("100.5", "60.0", "different number"), + ("100.5", "100.50", "same number, different text"), + ("ZuniMountains", "Zuni", "different text"), + ], +) +def test_classify_names_the_kind_of_disagreement( + chemical_data_value, gis_value, expected +): + assert classify(chemical_data_value, gis_value) == expected + + +@pytest.mark.parametrize( + "kinds, expected_fragment", + [ + ({"only in GIS": 5}, "Take GIS"), + ({"only in ChemicalData": 5}, "Take ChemicalData"), + ({"only in GIS": 5, "only in ChemicalData": 2}, "Fill blanks"), + ({"Excel error in GIS": 3}, "without the #VALUE!"), + ({"different text": 1}, "per-row ruling"), + # A conflict outranks fillable blanks: it cannot be resolved in bulk. + ({"only in GIS": 50, "different number": 1}, "per-row ruling"), + ], +) +def test_recommend_prefers_the_safest_reading(kinds, expected_fragment): + assert expected_fragment in recommend("laboratory", kinds) + + +def test_report_has_a_sheet_for_every_decision(report): + _, workbook = report + + assert workbook.sheetnames == [ + "README", + "ColumnDecisions", + "CellDifferences", + "RowsOnlyInOneSheet", + "IntegritySummary", + "IntegrityDetail", + "DetectionLimitSpread", + "DuplicateSampleNames", + ] + + +def test_summary_counts_the_drift(report): + summary, _ = report + + assert summary["chemical_data_rows"] == 7 + assert summary["gis_rows"] == 6 + assert summary["aligned_rows"] == 6 + assert summary["rows_only_in_chemical_data"] == 1 + assert summary["rows_only_in_gis"] == 0 + assert summary["duplicate_sample_names"] == 1 + + +def test_column_decisions_carry_a_blank_ruling_and_a_dropdown(report): + _, workbook = report + worksheet = workbook["ColumnDecisions"] + records = _records(worksheet) + + laboratory = next(row for row in records if row["Mirror column"] == "laboratory") + assert laboratory["Only in ChemicalData"] == 1 + assert "Take ChemicalData" in laboratory["Suggested starting point"] + assert laboratory["DECISION"] is None + + total = next(row for row in records if row["Mirror column"] == "total_pct") + assert total["Excel error"] == 1 + + validations = worksheet.data_validations.dataValidation + assert len(validations) == 1 + assert "ChemicalData" in validations[0].formula1 + + +def test_cell_differences_show_both_values_side_by_side(report): + _, workbook = report + records = _records(workbook["CellDifferences"]) + + laboratory = next(row for row in records if row["Mirror column"] == "laboratory") + assert laboratory["ChemicalData value"] == "ALS" + assert laboratory["GIS value"] is None + assert laboratory["Kind of difference"] == "only in ChemicalData" + # Row numbers point back into the delivered workbook. + assert laboratory["ChemicalData row"] == 4 + assert laboratory["GIS row"] == 2 + + +def test_rows_only_in_one_sheet_names_the_missing_sample(report): + _, workbook = report + records = _records(workbook["RowsOnlyInOneSheet"]) + + assert [row["SAMPLE"] for row in records] == ["Gal6"] + assert records[0]["Sheet"] == "ChemicalData" + assert records[0]["Has coordinates"] == "yes" + + +def test_integrity_checks_flag_the_unit_contradiction(report): + _, workbook = report + records = _records(workbook["IntegrityDetail"]) + + unit_issues = [ + row + for row in records + if row["Issue"] == "Value impossible for the declared unit" + ] + assert [row["SAMPLE"] for row in unit_issues] == ["Gal2"] + assert unit_issues[0]["Column"] == "f_pct" + assert unit_issues[0]["Value"] == "27700" + assert "looks like ppm" in unit_issues[0]["Detail"] + + issues = {row["Issue"] for row in records} + assert "Total disagrees with the sum of the major oxides" in issues + assert "Latitude outside New Mexico" in issues + assert "Non-numeric analyte value" in issues + assert "No coordinates" in issues + assert "Date is not a full date" in issues + + +def test_a_total_that_is_a_sum_is_not_treated_as_a_unit_error(report): + """Total legitimately exceeds 100; only real measurements get the unit check.""" + _, workbook = report + records = _records(workbook["IntegrityDetail"]) + + assert not [ + row + for row in records + if row["Column"] == "total_pct" + and row["Issue"] == "Value impossible for the declared unit" + ] + + +def test_integrity_summary_explains_each_issue(report): + _, workbook = report + records = _records(workbook["IntegritySummary"]) + + assert records == sorted(records, key=lambda row: -row["Rows affected"]) + for row in records: + assert row["Why it matters"] + assert row["RESOLUTION"] is None + + +def test_duplicate_sample_names_lists_the_source_rows(report): + _, workbook = report + records = _records(workbook["DuplicateSampleNames"]) + + assert records[0]["SAMPLE"] == "Gal1" + assert records[0]["Rows with this name"] == 2 + assert records[0]["Source rows"] == "4, 9" From 7e04e6810c9f9abdf2c5accca3aae2f4e9ccca43 Mon Sep 17 00:00:00 2001 From: Jake Ross Date: Thu, 20 Aug 2026 20:59:51 -0700 Subject: [PATCH 2/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- docs/critical-minerals-legacy-mirror.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/critical-minerals-legacy-mirror.md b/docs/critical-minerals-legacy-mirror.md index c178b5729..e0072b30a 100644 --- a/docs/critical-minerals-legacy-mirror.md +++ b/docs/critical-minerals-legacy-mirror.md @@ -139,7 +139,7 @@ loadable without dropping cells. Parsing value-plus-qualifier (cross-checked against `CM_DetectionLimits`), casting dates, and reprojecting coordinates are all transform work. -Cell rendering rules (`_cell_to_text`): dates become ISO-8601 rather than Excel +Cell rendering rules (`cell_to_text`): dates become ISO-8601 rather than Excel serials, numbers keep Python's round-trippable `repr`, blank and whitespace-only cells become NULL rather than `''`, and everything else passes through stripped. From 89355f8846fac1c6ee976bfa6ba8b2abf0416bac Mon Sep 17 00:00:00 2001 From: Jake Ross Date: Thu, 20 Aug 2026 21:01:09 -0700 Subject: [PATCH 3/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- scripts/cm_reconciliation_report.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/scripts/cm_reconciliation_report.py b/scripts/cm_reconciliation_report.py index 4e6e72a16..03476f9be 100644 --- a/scripts/cm_reconciliation_report.py +++ b/scripts/cm_reconciliation_report.py @@ -116,14 +116,30 @@ class Sheet: def read_sheet(workbook: Any, name: str, columns: dict[str, str]) -> Sheet: + if name not in workbook.sheetnames: + raise ValueError(f"workbook is missing the {name!r} sheet") + header_row_number, first_data_row_number = CHEMISTRY_SHEET_LAYOUT[name] rows = list(workbook[name].iter_rows(values_only=True)) + if len(rows) < header_row_number: + raise ValueError(f"sheet {name!r} has no header row {header_row_number}") + header = rows[header_row_number - 1] - index_to_column = { - index: columns[normalize_header(cell)] - for index, cell in enumerate(header) - if cell is not None and normalize_header(cell) in columns - } + index_to_column: dict[int, str] = {} + unknown: list[str] = [] + for index, cell in enumerate(header): + text = cell_to_text(cell) + if text is None: + continue + column = columns.get(normalize_header(text)) + if column is None: + unknown.append(text) + else: + index_to_column[index] = column + + if unknown: + raise ValueError(f"sheet {name!r} has unmapped header(s): {unknown[:10]}") + records: list[dict[str, str | None]] = [] row_numbers: list[int] = [] for offset, row in enumerate(rows[first_data_row_number - 1 :]): From d064dba859e66ff6d3f2a7613fb04a7548a422f8 Mon Sep 17 00:00:00 2001 From: Jake Ross Date: Thu, 20 Aug 2026 21:01:54 -0700 Subject: [PATCH 4/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- services/cm_legacy_mirror.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/services/cm_legacy_mirror.py b/services/cm_legacy_mirror.py index dcd5da233..c40465944 100644 --- a/services/cm_legacy_mirror.py +++ b/services/cm_legacy_mirror.py @@ -327,12 +327,24 @@ def load_cm_workbook(source_file: Path | str, session: Session) -> CMMirrorLoadR raise CMWorkbookError(f"workbook not found: {source_file}") workbook = openpyxl.load_workbook(source_file, read_only=True, data_only=True) + + required_sheets = { + "DetectionLimits", + "References", + "MineralSystems", + "world", + "world_ref", + *METADATA_SHEETS, + } + missing = sorted(required_sheets - set(workbook.sheetnames)) + if missing: + workbook.close() + raise CMWorkbookError(f"workbook is missing required sheet(s): {missing}") + try: result = CMMirrorLoadResult() result.rows_by_sheet.update(_load_chemistry_sheets(workbook, session)) - result.rows_by_sheet["DetectionLimits"] = _load_detection_limits( - workbook, session - ) + result.rows_by_sheet["DetectionLimits"] = _load_detection_limits(workbook, session) result.rows_by_sheet["References"] = _load_citations( workbook, session, "References", CM_References ) From fda4c7d3f00d8f8f488a5510edfd8a9be54659f3 Mon Sep 17 00:00:00 2001 From: jirhiker <2035568+jirhiker@users.noreply.github.com> Date: Fri, 21 Aug 2026 04:02:23 +0000 Subject: [PATCH 5/5] Formatting changes --- services/cm_legacy_mirror.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/cm_legacy_mirror.py b/services/cm_legacy_mirror.py index c40465944..153648311 100644 --- a/services/cm_legacy_mirror.py +++ b/services/cm_legacy_mirror.py @@ -344,7 +344,9 @@ def load_cm_workbook(source_file: Path | str, session: Session) -> CMMirrorLoadR try: result = CMMirrorLoadResult() result.rows_by_sheet.update(_load_chemistry_sheets(workbook, session)) - result.rows_by_sheet["DetectionLimits"] = _load_detection_limits(workbook, session) + result.rows_by_sheet["DetectionLimits"] = _load_detection_limits( + workbook, session + ) result.rows_by_sheet["References"] = _load_citations( workbook, session, "References", CM_References )