From fb64f68300631b871dc12b569b1f75214802ce00 Mon Sep 17 00:00:00 2001 From: jakeross Date: Fri, 21 Aug 2026 08:40:12 -0700 Subject: [PATCH] fix(seed): load reference data even when migrations seeded a term The development-mode startup seed calls ensure_seed_prereqs, which loaded core/lexicon.json only when lexicon_term was completely empty. Since b2c3d4e5f6a7 (transducer data maturity) the migration inserts a lexicon term of its own, so on a fresh database that emptiness check now sees one row and skips the real lexicon entirely. What followed: init_parameter tripped parameter_default_unit_fkey because 'ft' and 'dimensionless' were never inserted, then seed_all called random.choice on an empty organization category and died with "IndexError: Cannot choose from an empty sequence". FastAPI aborted startup, so the frontend's Cypress job sat on a readiness probe for its full 720s timeout and failed with exit 124. Both initializers leave existing rows alone, so ensure_seed_prereqs now just runs them. init_parameter skips names already stored rather than letting every re-run trip the unique constraint, and a new assert_lexicon_ready names the empty categories instead of failing deep inside the seed with an opaque IndexError. Verified against a fresh database: migrations alone leave exactly one lexicon term, and the seed now loads all 1144 and completes. --- core/initializers.py | 8 ++++ tests/test_seed_prereqs.py | 83 ++++++++++++++++++++++++++++++++++++++ transfers/seed.py | 54 +++++++++++++++++++++---- 3 files changed, 137 insertions(+), 8 deletions(-) create mode 100644 tests/test_seed_prereqs.py diff --git a/core/initializers.py b/core/initializers.py index ee0fecbe2..25420615d 100644 --- a/core/initializers.py +++ b/core/initializers.py @@ -45,7 +45,15 @@ def init_parameter(path: str = None) -> None: default_parameter = json.load(f) with session_ctx() as session: + # A parameter is identified by name and matrix, so skip the ones already + # stored instead of letting every re-run trip the unique constraint. + existing = set( + session.execute(select(Parameter.parameter_name, Parameter.matrix)).all() + ) + for param in default_parameter: + if (param["parameter_name"], param["matrix"]) in existing: + continue try: parameter_obj = Parameter( parameter_name=param["parameter_name"], diff --git a/tests/test_seed_prereqs.py b/tests/test_seed_prereqs.py new file mode 100644 index 000000000..51e8244cc --- /dev/null +++ b/tests/test_seed_prereqs.py @@ -0,0 +1,83 @@ +"""Reference data has to load before the seed touches it. + +Migrations insert a handful of lexicon terms of their own, so "the table has +rows" says nothing about whether core/lexicon.json was ever loaded. +""" + +import contextlib + +import pytest +from sqlalchemy import func, select + +from core.initializers import init_parameter +from db.engine import session_ctx +from db.parameter import Parameter +from transfers import seed as seed_module +from transfers.seed import ( + REQUIRED_LEXICON_CATEGORIES, + assert_lexicon_ready, + ensure_seed_prereqs, + get_terms_by_category, +) + + +def test_ensure_seed_prereqs_loads_reference_data_even_when_tables_have_rows( + monkeypatch, +): + calls = [] + monkeypatch.setattr( + "core.initializers.init_lexicon", lambda: calls.append("lexicon") + ) + monkeypatch.setattr( + "core.initializers.init_parameter", lambda: calls.append("parameter") + ) + + ensure_seed_prereqs() + + assert calls == ["lexicon", "parameter"] + + +def test_assert_lexicon_ready_names_the_empty_categories(monkeypatch): + empty = {"organization", "note_type"} + + @contextlib.contextmanager + def fake_session_ctx(): + yield object() + + monkeypatch.setattr(seed_module, "session_ctx", fake_session_ctx) + monkeypatch.setattr( + seed_module, + "get_terms_by_category", + lambda _session, category: [] if category in empty else ["term"], + ) + + with pytest.raises(RuntimeError) as excinfo: + assert_lexicon_ready() + + message = str(excinfo.value) + assert "organization" in message + assert "note_type" in message + assert "sample_method" not in message + + +def test_required_categories_have_terms_after_reference_data_loads(): + with session_ctx() as session: + empty = [ + category + for category in REQUIRED_LEXICON_CATEGORIES + if not get_terms_by_category(session, category) + ] + + assert empty == [] + + +def test_init_parameter_leaves_existing_parameters_alone(): + with session_ctx() as session: + before = session.scalar(select(func.count()).select_from(Parameter)) + + init_parameter() + + with session_ctx() as session: + after = session.scalar(select(func.count()).select_from(Parameter)) + + assert after == before diff --git a/transfers/seed.py b/transfers/seed.py index bbe2c1885..6dbd7cd47 100644 --- a/transfers/seed.py +++ b/transfers/seed.py @@ -54,18 +54,55 @@ def get_terms_by_category(s, category_name: str) -> list[LexiconTerm]: ) +# Lexicon categories the seed below draws terms from. Every one of them has to +# have at least one term or the seed cannot build a coherent row. +REQUIRED_LEXICON_CATEGORIES = ( + "organization", + "relation", + "analysis_method_type", + "sample_method", + "activity_type", + "sensor_type", + "email_type", + "phone_type", + "address_type", + "well_purpose", + "casing_material", + "monitoring_frequency", + "note_type", + "participant_role", +) + + def ensure_seed_prereqs() -> None: - """Ensure that lexicon and parameter data exist before seeding.""" + """Load the reference lexicon and parameters that the seed data depends on. + + Both initializers leave existing rows alone, so this runs unconditionally. + It used to skip them whenever the tables held any rows at all, which broke + once migrations started inserting lexicon terms of their own: on a fresh + database those few rows made the real lexicon look already-loaded, and the + seed then failed on empty categories. + """ from core.initializers import init_lexicon, init_parameter - with session_ctx() as s: - has_lexicon = s.scalar(select(LexiconTerm.id).limit(1)) is not None - has_parameter = s.scalar(select(Parameter.id).limit(1)) is not None + init_lexicon() + init_parameter() - if not has_lexicon: - init_lexicon() - if not has_parameter: - init_parameter() + +def assert_lexicon_ready() -> None: + """Fail with the empty categories named, rather than deep inside the seed.""" + with session_ctx() as s: + empty = [ + category + for category in REQUIRED_LEXICON_CATEGORIES + if not get_terms_by_category(s, category) + ] + + if empty: + raise RuntimeError( + "Reference lexicon is incomplete; no terms for " + f"{', '.join(empty)}. Seeding cannot continue." + ) def contact_data_exists() -> bool: @@ -79,6 +116,7 @@ def seed_all(n: int = 5, skip_if_exists: bool = False): print("Contact data exists; skipping seeding.") return ensure_seed_prereqs() + assert_lexicon_ready() new_mexico_bounds = [ (36.9, -106.6), # Taos area (35.1, -106.6), # Albuquerque