diff --git a/src/igh_data_transform/transformations/candidates.py b/src/igh_data_transform/transformations/candidates.py index 0b57f58..0536945 100644 --- a/src/igh_data_transform/transformations/candidates.py +++ b/src/igh_data_transform/transformations/candidates.py @@ -26,6 +26,7 @@ "new_includeinpipeline2021", "new_2023includeinevgendatabase", "new_2024includeinpipeline", + "new_includeinpipeline2025", "new_includeinpipeline", ] @@ -172,7 +173,7 @@ def _resolve_rdstage_fk( # Strip product suffix: 'Phase III - Drugs' -> 'Phase III' lookup = lookup.str.rsplit(" - ", n=1).str[0] df = df.copy() - df["_resolved_rdstage_2025"] = df["_vin_currentrndstage_value"].map(lookup) + df["_resolved_rdstage_current"] = df["_vin_currentrndstage_value"].map(lookup) return df @@ -214,12 +215,19 @@ def _expand_temporal_rows(df: pd.DataFrame) -> pd.DataFrame: valid_to = start of candidate's next boundary (None for latest). Must be called before column renaming (uses original bronze names). """ + # R&D stage has no frozen 2025 archive column — unlike 2019/2021/2023/2024, + # IGH never froze one. `_resolved_rdstage_current` is Dataverse's rolling + # "current stage" field, so it sits at the 2025 boundary and the forward-fill + # in `_forward_fill` carries it into 2026. One rolling field therefore yields + # the same stage in both years, which is the honest reading of the source. + # If IGH ever freezes a 2025 stage column, add it here and move the rolling + # column to "2026-01-01". _rdstage_cols = [ ("vin_2019stagepcr", "2019-01-01"), ("new_rdstage2021", "2021-01-01"), ("new_2023currentrdstage", "2023-01-01"), ("new_2024currentrdstage", "2024-01-01"), - ("_resolved_rdstage_2025", "2025-01-01"), + ("_resolved_rdstage_current", "2025-01-01"), ] _pipeline_cols = [ @@ -227,7 +235,11 @@ def _expand_temporal_rows(df: pd.DataFrame) -> pd.DataFrame: ("new_includeinpipeline2021", "2021-01-01"), ("new_2023includeinevgendatabase", "2023-01-01"), ("new_2024includeinpipeline", "2024-01-01"), - ("new_includeinpipeline", "2025-01-01"), + # 2025 was frozen into its own column when IGH closed the collection. + ("new_includeinpipeline2025", "2025-01-01"), + # The unsuffixed column is Dataverse's rolling "current" field and now + # carries 2026. It rolls forward again each year. + ("new_includeinpipeline", "2026-01-01"), ] def _year_map(row: pd.Series, configs: list[tuple[str, str]]) -> dict: @@ -249,7 +261,7 @@ def _forward_fill(year_map: dict, boundaries: list[str]) -> list: return result rows_out: list[dict] = [] - cols_to_drop = _TEMPORAL_SOURCE_COLS + ["_resolved_rdstage_2025"] + cols_to_drop = _TEMPORAL_SOURCE_COLS + ["_resolved_rdstage_current"] # Columns to carry through (everything except temporal source cols) keep_cols = [c for c in df.columns if c not in cols_to_drop] @@ -309,12 +321,12 @@ def transform_candidates( df = _normalize_pipeline_cols(df) # 1c. Capture the strict 2025 pipeline-inclusion value before temporal - # expansion consumes and drops `new_includeinpipeline`. The WHO + # expansion consumes and drops `new_includeinpipeline2025`. The WHO # Priority page needs the *actual* 2025 value (not the forward-filled # flag), so we preserve it candidate-grain under a stable name. Absent # column / NaN stays NaN; the gold CASE maps NaN and "No" to 0. - if "new_includeinpipeline" in df.columns: - df["includeinpipeline_2025_raw"] = df["new_includeinpipeline"] + if "new_includeinpipeline2025" in df.columns: + df["includeinpipeline_2025_raw"] = df["new_includeinpipeline2025"] else: df["includeinpipeline_2025_raw"] = None diff --git a/src/igh_data_transform/transformations/silver_to_gold/config/schema_map.py b/src/igh_data_transform/transformations/silver_to_gold/config/schema_map.py index 1e48ee7..e679931 100644 --- a/src/igh_data_transform/transformations/silver_to_gold/config/schema_map.py +++ b/src/igh_data_transform/transformations/silver_to_gold/config/schema_map.py @@ -54,7 +54,7 @@ "countries_approved_agg": "countries_product_approved", "candidate_type": "CASE WHEN captype_value = 'c1746ad3-93d1-f011-bbd3-00224892cefa' THEN 'Candidate' WHEN captype_value = '545d63d9-93d1-f011-bbd3-00224892cefa' THEN 'Product' ELSE 'Other' END", # Strict 2025 pipeline-inclusion flag for the WHO Priority page. - # 1 only when the raw 2025 `new_includeinpipeline` is exactly Yes; + # 1 only when the raw 2025 `new_includeinpipeline2025` is exactly Yes; # No, Pending, and blank all map to 0. Intentionally distinct from # the forward-filled `fact_pipeline_snapshot.include_in_pipeline`. "new_include_in_pipeline_2025": "CASE WHEN includeinpipeline_2025_raw = 100000000 THEN 1 ELSE 0 END", diff --git a/tests/e2e/test_silver_to_gold_e2e.py b/tests/e2e/test_silver_to_gold_e2e.py index a948bef..91df746 100644 --- a/tests/e2e/test_silver_to_gold_e2e.py +++ b/tests/e2e/test_silver_to_gold_e2e.py @@ -299,6 +299,51 @@ def test_pipeline_snapshot_candidate_key_not_all_null(self, gold_conn): df = _read_table(gold_conn, "fact_pipeline_snapshot") assert df["candidate_key"].notna().any(), "candidate_key is all NULL" + def test_fact_pipeline_snapshot_includes_2026(self, gold_conn): + """The 2026 reporting year reaches Gold. Before the rolling + pipeline-inclusion column was given its own boundary, 2026 values + were folded into 2025.""" + years = pd.read_sql_query( + """ + SELECT DISTINCT dt.year + FROM fact_pipeline_snapshot f + JOIN dim_date dt ON dt.date_key = f.date_key + WHERE dt.year IS NOT NULL + ORDER BY dt.year + """, + gold_conn, + )["year"].tolist() + assert 2026 in years, f"2026 missing from Gold; got {years}" + + def test_2025_and_2026_read_different_source_columns(self, gold_conn): + """2025 must read the frozen `new_includeinpipeline2025` archive and + 2026 the rolling `new_includeinpipeline`. Identical included-counts + would mean the rolling column is still feeding both boundaries — + the exact defect this change fixes. + + Note this deliberately does NOT assert row-count parity between 2025 + and 2024. All 244 candidates created during 2026 carry a rolling R&D + stage, and the stage series is pinned at 2025, so each contributes a + stage-only 2025 row. The raw `includeinpipeline` is NULL on those + rows, but the derivation maps NaN to 0, so those rows carry + `include_in_pipeline` 0 and are invisible to every portal query, + all of which filter on it. + """ + counts = pd.read_sql_query( + """ + SELECT dt.year, COUNT(*) AS n + FROM fact_pipeline_snapshot f + JOIN dim_date dt ON dt.date_key = f.date_key + WHERE dt.year IN (2025, 2026) AND f.include_in_pipeline = 1 + GROUP BY dt.year + """, + gold_conn, + ).set_index("year")["n"] + assert counts.loc[2025] != counts.loc[2026], ( + f"2025 and 2026 both report {counts.loc[2025]} included candidates; " + "the two boundaries appear to be reading the same source column" + ) + # -- fact_clinical_trial_event -- def test_clinical_trial_has_rows(self, gold_conn): diff --git a/tests/unit/test_bronze_to_silver.py b/tests/unit/test_bronze_to_silver.py index d1af3d4..29cdb6e 100644 --- a/tests/unit/test_bronze_to_silver.py +++ b/tests/unit/test_bronze_to_silver.py @@ -458,7 +458,7 @@ def test_lookup_tables_loaded_and_passed_for_candidates(self, tmp_path: Path): assert rows_2025["new_currentrdstage"].iloc[0] == "Phase III" # FK column should be dropped assert "_vin_currentrndstage_value" not in df.columns - assert "_resolved_rdstage_2025" not in df.columns + assert "_resolved_rdstage_current" not in df.columns def test_option_sets_loaded_and_passed_to_transformer(self, tmp_path: Path): """Option sets from Bronze are loaded and passed to transformers.""" diff --git a/tests/unit/test_candidates.py b/tests/unit/test_candidates.py index 96dc2aa..bf83331 100644 --- a/tests/unit/test_candidates.py +++ b/tests/unit/test_candidates.py @@ -29,8 +29,8 @@ def test_resolves_guid_to_stage_name(self): } ) result = _resolve_rdstage_fk(df, rdstageproducts) - assert result["_resolved_rdstage_2025"].iloc[0] == "Phase III" - assert result["_resolved_rdstage_2025"].iloc[1] == "Phase I" + assert result["_resolved_rdstage_current"].iloc[0] == "Phase III" + assert result["_resolved_rdstage_current"].iloc[1] == "Phase I" def test_unknown_guid_resolves_to_nan(self): """Unknown GUID produces NaN in resolved column.""" @@ -47,7 +47,7 @@ def test_unknown_guid_resolves_to_nan(self): } ) result = _resolve_rdstage_fk(df, rdstageproducts) - assert pd.isna(result["_resolved_rdstage_2025"].iloc[0]) + assert pd.isna(result["_resolved_rdstage_current"].iloc[0]) def test_compound_name_preserves_prefix(self): """'Deactivated - Phase IV - Drugs' -> 'Deactivated - Phase IV'.""" @@ -64,7 +64,7 @@ def test_compound_name_preserves_prefix(self): } ) result = _resolve_rdstage_fk(df, rdstageproducts) - assert result["_resolved_rdstage_2025"].iloc[0] == "Deactivated - Phase IV" + assert result["_resolved_rdstage_current"].iloc[0] == "Deactivated - Phase IV" def test_does_not_modify_original(self): """Original DataFrame is not modified.""" @@ -167,7 +167,7 @@ class TestExpandTemporalRows: """Tests for _expand_temporal_rows function. These tests operate on DataFrames after FK resolution, so 2025 data - appears in the _resolved_rdstage_2025 column (not vin_currentrdstage). + appears in the _resolved_rdstage_current column (not vin_currentrdstage). The cross-product expansion also handles includeinpipeline columns. """ @@ -179,8 +179,8 @@ def test_candidate_with_all_temporal_years(self): "vin_name": ["CandA"], "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": ["Phase II"], - "_resolved_rdstage_2025": ["Phase III"], - "new_includeinpipeline": [100000000], + "_resolved_rdstage_current": ["Phase III"], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -200,8 +200,8 @@ def test_candidate_with_only_some_years(self): "vin_name": ["CandA"], "new_2023currentrdstage": [None], "new_2024currentrdstage": ["Phase II"], - "_resolved_rdstage_2025": ["Phase III"], - "new_includeinpipeline": [100000000], + "_resolved_rdstage_current": ["Phase III"], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -216,8 +216,8 @@ def test_candidate_with_only_current_year(self): "vin_name": ["CandA"], "new_2023currentrdstage": [None], "new_2024currentrdstage": [None], - "_resolved_rdstage_2025": ["Phase III"], - "new_includeinpipeline": [100000000], + "_resolved_rdstage_current": ["Phase III"], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -232,8 +232,8 @@ def test_rd_stage_column_sourced_correctly(self): "vin_name": ["CandA"], "new_2023currentrdstage": ["Discovery"], "new_2024currentrdstage": ["Preclinical"], - "_resolved_rdstage_2025": ["Phase I"], - "new_includeinpipeline": [100000000], + "_resolved_rdstage_current": ["Phase I"], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -253,8 +253,8 @@ def test_per_candidate_valid_to_consecutive_years(self): "vin_name": ["CandA"], "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": ["Phase II"], - "_resolved_rdstage_2025": ["Phase III"], - "new_includeinpipeline": [100000000], + "_resolved_rdstage_current": ["Phase III"], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -280,8 +280,8 @@ def test_per_candidate_valid_to_with_gap(self): "new_rdstage2021": ["Discovery"], "new_2023currentrdstage": [None], "new_2024currentrdstage": ["Phase I"], - "_resolved_rdstage_2025": [None], - "new_includeinpipeline": [100000000], + "_resolved_rdstage_current": [None], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -304,8 +304,8 @@ def test_per_candidate_valid_to_single_year(self): "vin_name": ["CandA"], "new_2023currentrdstage": [None], "new_2024currentrdstage": [None], - "_resolved_rdstage_2025": ["Phase III"], - "new_includeinpipeline": [100000000], + "_resolved_rdstage_current": ["Phase III"], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -321,8 +321,8 @@ def test_temporal_source_columns_dropped(self): "vin_name": ["CandA"], "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": ["Phase II"], - "_resolved_rdstage_2025": ["Phase III"], - "new_includeinpipeline": [100000000], + "_resolved_rdstage_current": ["Phase III"], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -331,7 +331,7 @@ def test_temporal_source_columns_dropped(self): "vin_2019stagepcr", "new_2023currentrdstage", "new_2024currentrdstage", - "_resolved_rdstage_2025", + "_resolved_rdstage_current", "new_rdstage2021", "new_includeinpipeline", "new_2024includeinpipeline", @@ -347,8 +347,8 @@ def test_null_year_produces_no_row(self): "vin_name": ["CandA"], "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": [None], - "_resolved_rdstage_2025": ["Phase III"], - "new_includeinpipeline": [100000000], + "_resolved_rdstage_current": ["Phase III"], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -364,8 +364,8 @@ def test_multiple_candidates(self): "vin_name": ["CandA", "CandB"], "new_2023currentrdstage": ["Phase I", "Discovery"], "new_2024currentrdstage": ["Phase II", None], - "_resolved_rdstage_2025": [None, "Preclinical"], - "new_includeinpipeline": [100000000, 100000002], + "_resolved_rdstage_current": [None, "Preclinical"], + "new_includeinpipeline2025": [100000000, 100000002], "vin_product": ["Drugs", "Vaccines"], } ) @@ -385,8 +385,8 @@ def test_multiple_candidates_independent_valid_to(self): "vin_name": ["CandA", "CandB"], "new_2023currentrdstage": ["Phase I", "Discovery"], "new_2024currentrdstage": ["Phase II", None], - "_resolved_rdstage_2025": [None, "Preclinical"], - "new_includeinpipeline": [100000000, 100000002], + "_resolved_rdstage_current": [None, "Preclinical"], + "new_includeinpipeline2025": [100000000, 100000002], "vin_product": ["Drugs", "Vaccines"], } ) @@ -408,10 +408,10 @@ def test_cross_product_boundary_years(self): "new_rdstage2021": [None], "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": [None], - "_resolved_rdstage_2025": ["Phase II"], + "_resolved_rdstage_current": ["Phase II"], "new_includeinpipeline2021": [100000000], "new_2024includeinpipeline": [100000001], - "new_includeinpipeline": [100000002], + "new_includeinpipeline2025": [100000002], "vin_product": ["Drugs"], } ) @@ -435,10 +435,10 @@ def test_forward_fill_across_groups(self): "new_rdstage2021": [None], "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": [None], - "_resolved_rdstage_2025": ["Phase II"], + "_resolved_rdstage_current": ["Phase II"], "new_includeinpipeline2021": [100000000], "new_2024includeinpipeline": [100000001], - "new_includeinpipeline": [100000002], + "new_includeinpipeline2025": [100000002], "vin_product": ["Drugs"], } ) @@ -470,7 +470,7 @@ def test_pipeline_source_columns_dropped(self): "new_2024currentrdstage": ["Phase I"], "new_includeinpipeline2021": [100000000], "new_2024includeinpipeline": [100000001], - "new_includeinpipeline": [100000002], + "new_includeinpipeline2025": [100000002], "vin_product": ["Drugs"], } ) @@ -488,9 +488,9 @@ def test_2019_pipeline_column_contributes_boundary(self): { "vin_candidateid": ["cand-1"], "vin_name": ["CandA"], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "vin_2019pcrpipelineinclusion": [100000000], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -503,9 +503,9 @@ def test_2023_evgen_column_contributes_boundary(self): { "vin_candidateid": ["cand-1"], "vin_name": ["CandA"], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "new_2023includeinevgendatabase": [100000000], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -518,10 +518,10 @@ def test_new_pipeline_columns_dropped_from_output(self): { "vin_candidateid": ["cand-1"], "vin_name": ["CandA"], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "vin_2019pcrpipelineinclusion": [100000000], "new_2023includeinevgendatabase": [100000001], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -539,7 +539,7 @@ def test_full_pipeline_timeline_with_all_five_columns(self): "new_includeinpipeline2021": [100000000], "new_2023includeinevgendatabase": [100000001], "new_2024includeinpipeline": [100000000], - "new_includeinpipeline": [100000001], + "new_includeinpipeline2025": [100000001], "vin_product": ["Drugs"], } ) @@ -566,10 +566,10 @@ def test_null_latest_pipeline_forward_fills_previous_value(self): "vin_candidateid": ["cand-1"], "vin_name": ["CandA"], "new_2024currentrdstage": ["Phase I"], - "_resolved_rdstage_2025": ["Phase II"], + "_resolved_rdstage_current": ["Phase II"], "new_includeinpipeline2021": [100000000], "new_2024includeinpipeline": [None], - "new_includeinpipeline": [None], + "new_includeinpipeline2025": [None], "vin_product": ["Drugs"], } ) @@ -588,8 +588,8 @@ def test_2019_rdstage_column_contributes_boundary(self): "vin_candidateid": ["cand-1"], "vin_name": ["CandA"], "vin_2019stagepcr": ["Phase I"], - "_resolved_rdstage_2025": ["Phase III"], - "new_includeinpipeline": [100000000], + "_resolved_rdstage_current": ["Phase III"], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -603,8 +603,8 @@ def test_2019_rdstage_column_dropped_from_output(self): "vin_candidateid": ["cand-1"], "vin_name": ["CandA"], "vin_2019stagepcr": ["Phase I"], - "_resolved_rdstage_2025": ["Phase III"], - "new_includeinpipeline": [100000000], + "_resolved_rdstage_current": ["Phase III"], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -621,7 +621,7 @@ def test_full_rdstage_timeline_with_all_five_columns(self): "new_rdstage2021": ["Preclinical"], "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": ["Phase II"], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "vin_product": ["Drugs"], } ) @@ -641,6 +641,105 @@ def test_full_rdstage_timeline_with_all_five_columns(self): assert result.loc[3, "new_currentrdstage"] == "Phase II" assert result.loc[4, "new_currentrdstage"] == "Phase III" + def test_rolling_pipeline_column_contributes_2026_boundary(self): + """`new_includeinpipeline` is Dataverse's rolling "current" column. + IGH froze 2025 into `new_includeinpipeline2025` and rolled the + unsuffixed column forward, so it now carries 2026.""" + df = pd.DataFrame( + { + "vin_candidateid": ["cand-1"], + "vin_name": ["CandA"], + "new_includeinpipeline2025": [100000000], + "new_includeinpipeline": [100000001], + "vin_product": ["Drugs"], + } + ) + result = ( + _expand_temporal_rows(df).sort_values("valid_from").reset_index(drop=True) + ) + assert list(result["valid_from"]) == ["2025-01-01", "2026-01-01"] + assert result.loc[0, "includeinpipeline"] == 100000000 + assert result.loc[1, "includeinpipeline"] == 100000001 + + def test_candidate_created_in_2026_has_no_2025_boundary(self): + """When a candidate has no R&D stage value, the stage series + contributes no boundary, so only the rolling pipeline-inclusion + column does — yielding a 2026 boundary alone. The representative + case, a 2026 candidate WITH a stage, is covered by the next test.""" + df = pd.DataFrame( + { + "vin_candidateid": ["cand-new"], + "vin_name": ["CandNew"], + "new_includeinpipeline2025": [None], + "new_includeinpipeline": [100000000], + "vin_product": ["Drugs"], + } + ) + result = _expand_temporal_rows(df) + assert list(result["valid_from"]) == ["2026-01-01"] + assert pd.isna(result.loc[0, "valid_to"]) + + def test_candidate_created_in_2026_gets_stage_only_2025_row(self): + """All 244 candidates Dataverse created during 2026 carry a rolling + R&D stage, and the stage series is pinned at 2025, so they DO get a + 2025 boundary. What makes that harmless is that it carries no + inclusion value — every portal query filters `include_in_pipeline`, + so a NULL-inclusion row is invisible.""" + df = pd.DataFrame( + { + "vin_candidateid": ["cand-new"], + "vin_name": ["CandNew"], + "_resolved_rdstage_current": ["Phase I"], + "new_includeinpipeline2025": [None], + "new_includeinpipeline": [100000000], + "vin_product": ["Drugs"], + } + ) + result = ( + _expand_temporal_rows(df).sort_values("valid_from").reset_index(drop=True) + ) + assert list(result["valid_from"]) == ["2025-01-01", "2026-01-01"] + assert pd.isna(result.loc[0, "includeinpipeline"]) + assert result.loc[0, "new_currentrdstage"] == "Phase I" + assert result.loc[1, "includeinpipeline"] == 100000000 + + def test_rolling_stage_forward_fills_into_2026(self): + """R&D stage has no frozen 2025 archive, so the rolling stage sits at + the 2025 boundary and the cross-group forward-fill carries it into + 2026 unchanged.""" + df = pd.DataFrame( + { + "vin_candidateid": ["cand-1"], + "vin_name": ["CandA"], + "_resolved_rdstage_current": ["Phase II"], + "new_includeinpipeline2025": [100000000], + "new_includeinpipeline": [100000001], + "vin_product": ["Drugs"], + } + ) + result = ( + _expand_temporal_rows(df).sort_values("valid_from").reset_index(drop=True) + ) + assert list(result["valid_from"]) == ["2025-01-01", "2026-01-01"] + assert result.loc[0, "new_currentrdstage"] == "Phase II" + assert result.loc[1, "new_currentrdstage"] == "Phase II" + + def test_frozen_2025_pipeline_column_dropped_from_output(self): + """`new_includeinpipeline2025` is consumed by expansion and must not + leak into Silver alongside the derived `includeinpipeline` column.""" + df = pd.DataFrame( + { + "vin_candidateid": ["cand-1"], + "vin_name": ["CandA"], + "new_includeinpipeline2025": [100000000], + "new_includeinpipeline": [100000001], + "vin_product": ["Drugs"], + } + ) + result = _expand_temporal_rows(df) + assert "new_includeinpipeline2025" not in result.columns + assert "new_includeinpipeline" not in result.columns + class TestTransformCandidates: """Tests for transform_candidates function.""" @@ -667,6 +766,7 @@ def _make_input_df(self, overrides=None): "new_includeinpipeline2021": [100000000.0, 100000002.0, 100000001.0], "new_2023includeinevgendatabase": ["Yes", "No", "Pending"], "new_2024includeinpipeline": [862890000.0, None, None], + "new_includeinpipeline2025": [100000000.0, 100000002.0, 100000001.0], "new_includeinpipeline": [100000000.0, 100000002.0, 100000001.0], "_vin_captype_value": [ "c1746ad3-93d1-f011-bbd3-00224892cefa", @@ -932,7 +1032,7 @@ def test_temporal_source_columns_removed_after_transform(self): "new_2023currentrdstage", "new_2024currentrdstage", "_vin_currentrndstage_value", - "_resolved_rdstage_2025", + "_resolved_rdstage_current", "new_rdstage2021", "vin_2019pcrpipelineinclusion", "new_includeinpipeline2021", @@ -1096,12 +1196,12 @@ def test_includeinpipeline_2025_raw_preserved_at_candidate_grain(self): assert "includeinpipeline_2025_raw" in result.columns - # Candidate A: bronze new_includeinpipeline == 100000000 (Yes) + # Candidate A: bronze new_includeinpipeline2025 == 100000000 (Yes) rows_a = result[result["candidateid"] == "id-1"] assert len(rows_a) > 1 assert (rows_a["includeinpipeline_2025_raw"] == 100000000.0).all() - # Candidate C: bronze new_includeinpipeline == 100000001 (No) — + # Candidate C: bronze new_includeinpipeline2025 == 100000001 (No) — # carried verbatim, NOT coerced. rows_c = result[result["candidateid"] == "id-3"] assert (rows_c["includeinpipeline_2025_raw"] == 100000001.0).all() @@ -1110,7 +1210,7 @@ def test_includeinpipeline_2025_raw_survives_when_all_null(self): """includeinpipeline_2025_raw is never dropped by drop_empty_columns even when every candidate's 2025 value is null (all-None override).""" df = self._make_input_df( - overrides={"new_includeinpipeline": [None, None, None]} + overrides={"new_includeinpipeline2025": [None, None, None]} ) lookup = self._make_lookup_tables() result, _ = transform_candidates(df, lookup_tables=lookup) @@ -1120,6 +1220,25 @@ def test_includeinpipeline_2025_raw_survives_when_all_null(self): # Every value should be null (the source was all-None). assert result["includeinpipeline_2025_raw"].isna().all() + def test_includeinpipeline_2025_raw_reads_frozen_not_rolling(self): + """The strict flag must read the frozen 2025 archive. Reading the + rolling `new_includeinpipeline` would report 2026 inclusion as 2025 + on the WHO Priority page. The two columns are given deliberately + opposite values so a regression cannot pass by coincidence.""" + df = self._make_input_df( + overrides={ + "new_includeinpipeline2025": [100000000.0, None, 100000001.0], + "new_includeinpipeline": [100000001.0, None, 100000000.0], + } + ) + lookup = self._make_lookup_tables() + result, _ = transform_candidates(df, lookup_tables=lookup) + + rows_a = result[result["candidateid"] == "id-1"] + assert (rows_a["includeinpipeline_2025_raw"] == 100000000.0).all() + rows_c = result[result["candidateid"] == "id-3"] + assert (rows_c["includeinpipeline_2025_raw"] == 100000001.0).all() + def test_ctregistrylink_synthesis_applied(self): """CT registry link is cleaned during transform.""" df = self._make_input_df(