From 59c0b65cfdc4b97e56fc614804fbf898cc6e2b1c Mon Sep 17 00:00:00 2001 From: Zuhdil Herry Kurnia Date: Mon, 3 Aug 2026 11:18:09 +0700 Subject: [PATCH 1/5] Rename `_resolved_rdstage_2025` to `_resolved_rdstage_current` The column holds Dataverse's rolling "current R&D stage" value, not a frozen 2025 archive. The old name implied the latter, which is why the 2026 rollover went unnoticed when IGH froze the pipeline-inclusion column but not the stage one. Pure rename with no behavioural change. The comment added to `_rdstage_cols` records why a rolling field is pinned to the 2025 boundary and forward-fills into 2026. --- .../transformations/candidates.py | 13 +++-- tests/unit/test_bronze_to_silver.py | 2 +- tests/unit/test_candidates.py | 54 +++++++++---------- 3 files changed, 38 insertions(+), 31 deletions(-) diff --git a/src/igh_data_transform/transformations/candidates.py b/src/igh_data_transform/transformations/candidates.py index 0b57f58..44c90a0 100644 --- a/src/igh_data_transform/transformations/candidates.py +++ b/src/igh_data_transform/transformations/candidates.py @@ -172,7 +172,7 @@ def _resolve_rdstage_fk( # Strip product suffix: 'Phase III - Drugs' -> 'Phase III' lookup = lookup.str.rsplit(" - ", n=1).str[0] df = df.copy() - df["_resolved_rdstage_2025"] = df["_vin_currentrndstage_value"].map(lookup) + df["_resolved_rdstage_current"] = df["_vin_currentrndstage_value"].map(lookup) return df @@ -214,12 +214,19 @@ def _expand_temporal_rows(df: pd.DataFrame) -> pd.DataFrame: valid_to = start of candidate's next boundary (None for latest). Must be called before column renaming (uses original bronze names). """ + # R&D stage has no frozen 2025 archive column — unlike 2019/2021/2023/2024, + # IGH never froze one. `_resolved_rdstage_current` is Dataverse's rolling + # "current stage" field, so it sits at the 2025 boundary and the forward-fill + # in `_forward_fill` carries it into 2026. One rolling field therefore yields + # the same stage in both years, which is the honest reading of the source. + # If IGH ever freezes a 2025 stage column, add it here and move the rolling + # column to "2026-01-01". _rdstage_cols = [ ("vin_2019stagepcr", "2019-01-01"), ("new_rdstage2021", "2021-01-01"), ("new_2023currentrdstage", "2023-01-01"), ("new_2024currentrdstage", "2024-01-01"), - ("_resolved_rdstage_2025", "2025-01-01"), + ("_resolved_rdstage_current", "2025-01-01"), ] _pipeline_cols = [ @@ -249,7 +256,7 @@ def _forward_fill(year_map: dict, boundaries: list[str]) -> list: return result rows_out: list[dict] = [] - cols_to_drop = _TEMPORAL_SOURCE_COLS + ["_resolved_rdstage_2025"] + cols_to_drop = _TEMPORAL_SOURCE_COLS + ["_resolved_rdstage_current"] # Columns to carry through (everything except temporal source cols) keep_cols = [c for c in df.columns if c not in cols_to_drop] diff --git a/tests/unit/test_bronze_to_silver.py b/tests/unit/test_bronze_to_silver.py index d1af3d4..29cdb6e 100644 --- a/tests/unit/test_bronze_to_silver.py +++ b/tests/unit/test_bronze_to_silver.py @@ -458,7 +458,7 @@ def test_lookup_tables_loaded_and_passed_for_candidates(self, tmp_path: Path): assert rows_2025["new_currentrdstage"].iloc[0] == "Phase III" # FK column should be dropped assert "_vin_currentrndstage_value" not in df.columns - assert "_resolved_rdstage_2025" not in df.columns + assert "_resolved_rdstage_current" not in df.columns def test_option_sets_loaded_and_passed_to_transformer(self, tmp_path: Path): """Option sets from Bronze are loaded and passed to transformers.""" diff --git a/tests/unit/test_candidates.py b/tests/unit/test_candidates.py index 96dc2aa..12c6be9 100644 --- a/tests/unit/test_candidates.py +++ b/tests/unit/test_candidates.py @@ -29,8 +29,8 @@ def test_resolves_guid_to_stage_name(self): } ) result = _resolve_rdstage_fk(df, rdstageproducts) - assert result["_resolved_rdstage_2025"].iloc[0] == "Phase III" - assert result["_resolved_rdstage_2025"].iloc[1] == "Phase I" + assert result["_resolved_rdstage_current"].iloc[0] == "Phase III" + assert result["_resolved_rdstage_current"].iloc[1] == "Phase I" def test_unknown_guid_resolves_to_nan(self): """Unknown GUID produces NaN in resolved column.""" @@ -47,7 +47,7 @@ def test_unknown_guid_resolves_to_nan(self): } ) result = _resolve_rdstage_fk(df, rdstageproducts) - assert pd.isna(result["_resolved_rdstage_2025"].iloc[0]) + assert pd.isna(result["_resolved_rdstage_current"].iloc[0]) def test_compound_name_preserves_prefix(self): """'Deactivated - Phase IV - Drugs' -> 'Deactivated - Phase IV'.""" @@ -64,7 +64,7 @@ def test_compound_name_preserves_prefix(self): } ) result = _resolve_rdstage_fk(df, rdstageproducts) - assert result["_resolved_rdstage_2025"].iloc[0] == "Deactivated - Phase IV" + assert result["_resolved_rdstage_current"].iloc[0] == "Deactivated - Phase IV" def test_does_not_modify_original(self): """Original DataFrame is not modified.""" @@ -167,7 +167,7 @@ class TestExpandTemporalRows: """Tests for _expand_temporal_rows function. These tests operate on DataFrames after FK resolution, so 2025 data - appears in the _resolved_rdstage_2025 column (not vin_currentrdstage). + appears in the _resolved_rdstage_current column (not vin_currentrdstage). The cross-product expansion also handles includeinpipeline columns. """ @@ -179,7 +179,7 @@ def test_candidate_with_all_temporal_years(self): "vin_name": ["CandA"], "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": ["Phase II"], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "new_includeinpipeline": [100000000], "vin_product": ["Drugs"], } @@ -200,7 +200,7 @@ def test_candidate_with_only_some_years(self): "vin_name": ["CandA"], "new_2023currentrdstage": [None], "new_2024currentrdstage": ["Phase II"], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "new_includeinpipeline": [100000000], "vin_product": ["Drugs"], } @@ -216,7 +216,7 @@ def test_candidate_with_only_current_year(self): "vin_name": ["CandA"], "new_2023currentrdstage": [None], "new_2024currentrdstage": [None], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "new_includeinpipeline": [100000000], "vin_product": ["Drugs"], } @@ -232,7 +232,7 @@ def test_rd_stage_column_sourced_correctly(self): "vin_name": ["CandA"], "new_2023currentrdstage": ["Discovery"], "new_2024currentrdstage": ["Preclinical"], - "_resolved_rdstage_2025": ["Phase I"], + "_resolved_rdstage_current": ["Phase I"], "new_includeinpipeline": [100000000], "vin_product": ["Drugs"], } @@ -253,7 +253,7 @@ def test_per_candidate_valid_to_consecutive_years(self): "vin_name": ["CandA"], "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": ["Phase II"], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "new_includeinpipeline": [100000000], "vin_product": ["Drugs"], } @@ -280,7 +280,7 @@ def test_per_candidate_valid_to_with_gap(self): "new_rdstage2021": ["Discovery"], "new_2023currentrdstage": [None], "new_2024currentrdstage": ["Phase I"], - "_resolved_rdstage_2025": [None], + "_resolved_rdstage_current": [None], "new_includeinpipeline": [100000000], "vin_product": ["Drugs"], } @@ -304,7 +304,7 @@ def test_per_candidate_valid_to_single_year(self): "vin_name": ["CandA"], "new_2023currentrdstage": [None], "new_2024currentrdstage": [None], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "new_includeinpipeline": [100000000], "vin_product": ["Drugs"], } @@ -321,7 +321,7 @@ def test_temporal_source_columns_dropped(self): "vin_name": ["CandA"], "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": ["Phase II"], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "new_includeinpipeline": [100000000], "vin_product": ["Drugs"], } @@ -331,7 +331,7 @@ def test_temporal_source_columns_dropped(self): "vin_2019stagepcr", "new_2023currentrdstage", "new_2024currentrdstage", - "_resolved_rdstage_2025", + "_resolved_rdstage_current", "new_rdstage2021", "new_includeinpipeline", "new_2024includeinpipeline", @@ -347,7 +347,7 @@ def test_null_year_produces_no_row(self): "vin_name": ["CandA"], "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": [None], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "new_includeinpipeline": [100000000], "vin_product": ["Drugs"], } @@ -364,7 +364,7 @@ def test_multiple_candidates(self): "vin_name": ["CandA", "CandB"], "new_2023currentrdstage": ["Phase I", "Discovery"], "new_2024currentrdstage": ["Phase II", None], - "_resolved_rdstage_2025": [None, "Preclinical"], + "_resolved_rdstage_current": [None, "Preclinical"], "new_includeinpipeline": [100000000, 100000002], "vin_product": ["Drugs", "Vaccines"], } @@ -385,7 +385,7 @@ def test_multiple_candidates_independent_valid_to(self): "vin_name": ["CandA", "CandB"], "new_2023currentrdstage": ["Phase I", "Discovery"], "new_2024currentrdstage": ["Phase II", None], - "_resolved_rdstage_2025": [None, "Preclinical"], + "_resolved_rdstage_current": [None, "Preclinical"], "new_includeinpipeline": [100000000, 100000002], "vin_product": ["Drugs", "Vaccines"], } @@ -408,7 +408,7 @@ def test_cross_product_boundary_years(self): "new_rdstage2021": [None], "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": [None], - "_resolved_rdstage_2025": ["Phase II"], + "_resolved_rdstage_current": ["Phase II"], "new_includeinpipeline2021": [100000000], "new_2024includeinpipeline": [100000001], "new_includeinpipeline": [100000002], @@ -435,7 +435,7 @@ def test_forward_fill_across_groups(self): "new_rdstage2021": [None], "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": [None], - "_resolved_rdstage_2025": ["Phase II"], + "_resolved_rdstage_current": ["Phase II"], "new_includeinpipeline2021": [100000000], "new_2024includeinpipeline": [100000001], "new_includeinpipeline": [100000002], @@ -488,7 +488,7 @@ def test_2019_pipeline_column_contributes_boundary(self): { "vin_candidateid": ["cand-1"], "vin_name": ["CandA"], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "vin_2019pcrpipelineinclusion": [100000000], "new_includeinpipeline": [100000000], "vin_product": ["Drugs"], @@ -503,7 +503,7 @@ def test_2023_evgen_column_contributes_boundary(self): { "vin_candidateid": ["cand-1"], "vin_name": ["CandA"], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "new_2023includeinevgendatabase": [100000000], "new_includeinpipeline": [100000000], "vin_product": ["Drugs"], @@ -518,7 +518,7 @@ def test_new_pipeline_columns_dropped_from_output(self): { "vin_candidateid": ["cand-1"], "vin_name": ["CandA"], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "vin_2019pcrpipelineinclusion": [100000000], "new_2023includeinevgendatabase": [100000001], "new_includeinpipeline": [100000000], @@ -566,7 +566,7 @@ def test_null_latest_pipeline_forward_fills_previous_value(self): "vin_candidateid": ["cand-1"], "vin_name": ["CandA"], "new_2024currentrdstage": ["Phase I"], - "_resolved_rdstage_2025": ["Phase II"], + "_resolved_rdstage_current": ["Phase II"], "new_includeinpipeline2021": [100000000], "new_2024includeinpipeline": [None], "new_includeinpipeline": [None], @@ -588,7 +588,7 @@ def test_2019_rdstage_column_contributes_boundary(self): "vin_candidateid": ["cand-1"], "vin_name": ["CandA"], "vin_2019stagepcr": ["Phase I"], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "new_includeinpipeline": [100000000], "vin_product": ["Drugs"], } @@ -603,7 +603,7 @@ def test_2019_rdstage_column_dropped_from_output(self): "vin_candidateid": ["cand-1"], "vin_name": ["CandA"], "vin_2019stagepcr": ["Phase I"], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "new_includeinpipeline": [100000000], "vin_product": ["Drugs"], } @@ -621,7 +621,7 @@ def test_full_rdstage_timeline_with_all_five_columns(self): "new_rdstage2021": ["Preclinical"], "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": ["Phase II"], - "_resolved_rdstage_2025": ["Phase III"], + "_resolved_rdstage_current": ["Phase III"], "vin_product": ["Drugs"], } ) @@ -932,7 +932,7 @@ def test_temporal_source_columns_removed_after_transform(self): "new_2023currentrdstage", "new_2024currentrdstage", "_vin_currentrndstage_value", - "_resolved_rdstage_2025", + "_resolved_rdstage_current", "new_rdstage2021", "vin_2019pcrpipelineinclusion", "new_includeinpipeline2021", From ba44de87a01a3a06931a9cb7a691979a24aebe7b Mon Sep 17 00:00:00 2001 From: Zuhdil Herry Kurnia Date: Mon, 3 Aug 2026 11:26:47 +0700 Subject: [PATCH 2/5] Give the rolling pipeline-inclusion column a 2026 boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IGH closed the 2025 data collection and opened 2026, freezing 2025 into `new_includeinpipeline2025` and rolling the unsuffixed `new_includeinpipeline` forward. Both places that read the unsuffixed column were therefore reporting 2026 values as 2025: the temporal boundary table and the strict-2025 flag behind the WHO Priority page. The visible symptom was the 2025 bucket carrying 8826 rows against 8582 for every prior year — the 244 candidates Dataverse created during 2026, backdated into a year they did not exist in. The R&D stage series is deliberately left pinned at 2025. It has no frozen archive column, and only 40 of 9388 candidates have had their stage touched since the rollover, so recovering a 2025 stage snapshot from an older Bronze backup would buy 0.4% accuracy in exchange for a permanent second ETL input. The existing cross-group forward-fill carries the rolling stage into 2026 on its own. The base test fixture keeps both the frozen and rolling columns populated. Replacing one with the other would leave the new 2026 boundary with no test coverage at all. --- .../transformations/candidates.py | 13 +- tests/unit/test_candidates.py | 138 +++++++++++++++--- 2 files changed, 125 insertions(+), 26 deletions(-) diff --git a/src/igh_data_transform/transformations/candidates.py b/src/igh_data_transform/transformations/candidates.py index 44c90a0..0536945 100644 --- a/src/igh_data_transform/transformations/candidates.py +++ b/src/igh_data_transform/transformations/candidates.py @@ -26,6 +26,7 @@ "new_includeinpipeline2021", "new_2023includeinevgendatabase", "new_2024includeinpipeline", + "new_includeinpipeline2025", "new_includeinpipeline", ] @@ -234,7 +235,11 @@ def _expand_temporal_rows(df: pd.DataFrame) -> pd.DataFrame: ("new_includeinpipeline2021", "2021-01-01"), ("new_2023includeinevgendatabase", "2023-01-01"), ("new_2024includeinpipeline", "2024-01-01"), - ("new_includeinpipeline", "2025-01-01"), + # 2025 was frozen into its own column when IGH closed the collection. + ("new_includeinpipeline2025", "2025-01-01"), + # The unsuffixed column is Dataverse's rolling "current" field and now + # carries 2026. It rolls forward again each year. + ("new_includeinpipeline", "2026-01-01"), ] def _year_map(row: pd.Series, configs: list[tuple[str, str]]) -> dict: @@ -316,12 +321,12 @@ def transform_candidates( df = _normalize_pipeline_cols(df) # 1c. Capture the strict 2025 pipeline-inclusion value before temporal - # expansion consumes and drops `new_includeinpipeline`. The WHO + # expansion consumes and drops `new_includeinpipeline2025`. The WHO # Priority page needs the *actual* 2025 value (not the forward-filled # flag), so we preserve it candidate-grain under a stable name. Absent # column / NaN stays NaN; the gold CASE maps NaN and "No" to 0. - if "new_includeinpipeline" in df.columns: - df["includeinpipeline_2025_raw"] = df["new_includeinpipeline"] + if "new_includeinpipeline2025" in df.columns: + df["includeinpipeline_2025_raw"] = df["new_includeinpipeline2025"] else: df["includeinpipeline_2025_raw"] = None diff --git a/tests/unit/test_candidates.py b/tests/unit/test_candidates.py index 12c6be9..4f609b8 100644 --- a/tests/unit/test_candidates.py +++ b/tests/unit/test_candidates.py @@ -180,7 +180,7 @@ def test_candidate_with_all_temporal_years(self): "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": ["Phase II"], "_resolved_rdstage_current": ["Phase III"], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -201,7 +201,7 @@ def test_candidate_with_only_some_years(self): "new_2023currentrdstage": [None], "new_2024currentrdstage": ["Phase II"], "_resolved_rdstage_current": ["Phase III"], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -217,7 +217,7 @@ def test_candidate_with_only_current_year(self): "new_2023currentrdstage": [None], "new_2024currentrdstage": [None], "_resolved_rdstage_current": ["Phase III"], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -233,7 +233,7 @@ def test_rd_stage_column_sourced_correctly(self): "new_2023currentrdstage": ["Discovery"], "new_2024currentrdstage": ["Preclinical"], "_resolved_rdstage_current": ["Phase I"], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -254,7 +254,7 @@ def test_per_candidate_valid_to_consecutive_years(self): "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": ["Phase II"], "_resolved_rdstage_current": ["Phase III"], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -281,7 +281,7 @@ def test_per_candidate_valid_to_with_gap(self): "new_2023currentrdstage": [None], "new_2024currentrdstage": ["Phase I"], "_resolved_rdstage_current": [None], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -305,7 +305,7 @@ def test_per_candidate_valid_to_single_year(self): "new_2023currentrdstage": [None], "new_2024currentrdstage": [None], "_resolved_rdstage_current": ["Phase III"], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -322,7 +322,7 @@ def test_temporal_source_columns_dropped(self): "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": ["Phase II"], "_resolved_rdstage_current": ["Phase III"], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -348,7 +348,7 @@ def test_null_year_produces_no_row(self): "new_2023currentrdstage": ["Phase I"], "new_2024currentrdstage": [None], "_resolved_rdstage_current": ["Phase III"], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -365,7 +365,7 @@ def test_multiple_candidates(self): "new_2023currentrdstage": ["Phase I", "Discovery"], "new_2024currentrdstage": ["Phase II", None], "_resolved_rdstage_current": [None, "Preclinical"], - "new_includeinpipeline": [100000000, 100000002], + "new_includeinpipeline2025": [100000000, 100000002], "vin_product": ["Drugs", "Vaccines"], } ) @@ -386,7 +386,7 @@ def test_multiple_candidates_independent_valid_to(self): "new_2023currentrdstage": ["Phase I", "Discovery"], "new_2024currentrdstage": ["Phase II", None], "_resolved_rdstage_current": [None, "Preclinical"], - "new_includeinpipeline": [100000000, 100000002], + "new_includeinpipeline2025": [100000000, 100000002], "vin_product": ["Drugs", "Vaccines"], } ) @@ -411,7 +411,7 @@ def test_cross_product_boundary_years(self): "_resolved_rdstage_current": ["Phase II"], "new_includeinpipeline2021": [100000000], "new_2024includeinpipeline": [100000001], - "new_includeinpipeline": [100000002], + "new_includeinpipeline2025": [100000002], "vin_product": ["Drugs"], } ) @@ -438,7 +438,7 @@ def test_forward_fill_across_groups(self): "_resolved_rdstage_current": ["Phase II"], "new_includeinpipeline2021": [100000000], "new_2024includeinpipeline": [100000001], - "new_includeinpipeline": [100000002], + "new_includeinpipeline2025": [100000002], "vin_product": ["Drugs"], } ) @@ -470,7 +470,7 @@ def test_pipeline_source_columns_dropped(self): "new_2024currentrdstage": ["Phase I"], "new_includeinpipeline2021": [100000000], "new_2024includeinpipeline": [100000001], - "new_includeinpipeline": [100000002], + "new_includeinpipeline2025": [100000002], "vin_product": ["Drugs"], } ) @@ -490,7 +490,7 @@ def test_2019_pipeline_column_contributes_boundary(self): "vin_name": ["CandA"], "_resolved_rdstage_current": ["Phase III"], "vin_2019pcrpipelineinclusion": [100000000], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -505,7 +505,7 @@ def test_2023_evgen_column_contributes_boundary(self): "vin_name": ["CandA"], "_resolved_rdstage_current": ["Phase III"], "new_2023includeinevgendatabase": [100000000], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -521,7 +521,7 @@ def test_new_pipeline_columns_dropped_from_output(self): "_resolved_rdstage_current": ["Phase III"], "vin_2019pcrpipelineinclusion": [100000000], "new_2023includeinevgendatabase": [100000001], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -539,7 +539,7 @@ def test_full_pipeline_timeline_with_all_five_columns(self): "new_includeinpipeline2021": [100000000], "new_2023includeinevgendatabase": [100000001], "new_2024includeinpipeline": [100000000], - "new_includeinpipeline": [100000001], + "new_includeinpipeline2025": [100000001], "vin_product": ["Drugs"], } ) @@ -569,7 +569,7 @@ def test_null_latest_pipeline_forward_fills_previous_value(self): "_resolved_rdstage_current": ["Phase II"], "new_includeinpipeline2021": [100000000], "new_2024includeinpipeline": [None], - "new_includeinpipeline": [None], + "new_includeinpipeline2025": [None], "vin_product": ["Drugs"], } ) @@ -589,7 +589,7 @@ def test_2019_rdstage_column_contributes_boundary(self): "vin_name": ["CandA"], "vin_2019stagepcr": ["Phase I"], "_resolved_rdstage_current": ["Phase III"], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -604,7 +604,7 @@ def test_2019_rdstage_column_dropped_from_output(self): "vin_name": ["CandA"], "vin_2019stagepcr": ["Phase I"], "_resolved_rdstage_current": ["Phase III"], - "new_includeinpipeline": [100000000], + "new_includeinpipeline2025": [100000000], "vin_product": ["Drugs"], } ) @@ -641,6 +641,80 @@ def test_full_rdstage_timeline_with_all_five_columns(self): assert result.loc[3, "new_currentrdstage"] == "Phase II" assert result.loc[4, "new_currentrdstage"] == "Phase III" + def test_rolling_pipeline_column_contributes_2026_boundary(self): + """`new_includeinpipeline` is Dataverse's rolling "current" column. + IGH froze 2025 into `new_includeinpipeline2025` and rolled the + unsuffixed column forward, so it now carries 2026.""" + df = pd.DataFrame( + { + "vin_candidateid": ["cand-1"], + "vin_name": ["CandA"], + "new_includeinpipeline2025": [100000000], + "new_includeinpipeline": [100000001], + "vin_product": ["Drugs"], + } + ) + result = ( + _expand_temporal_rows(df).sort_values("valid_from").reset_index(drop=True) + ) + assert list(result["valid_from"]) == ["2025-01-01", "2026-01-01"] + assert result.loc[0, "includeinpipeline"] == 100000000 + assert result.loc[1, "includeinpipeline"] == 100000001 + + def test_candidate_created_in_2026_has_no_2025_boundary(self): + """Candidates Dataverse created during 2026 carry only a rolling + value and no frozen 2025 one. They must not be backdated into 2025 — + this is the defect that inflated the 2025 bucket by 244 rows.""" + df = pd.DataFrame( + { + "vin_candidateid": ["cand-new"], + "vin_name": ["CandNew"], + "new_includeinpipeline2025": [None], + "new_includeinpipeline": [100000000], + "vin_product": ["Drugs"], + } + ) + result = _expand_temporal_rows(df) + assert list(result["valid_from"]) == ["2026-01-01"] + assert pd.isna(result.loc[0, "valid_to"]) + + def test_rolling_stage_forward_fills_into_2026(self): + """R&D stage has no frozen 2025 archive, so the rolling stage sits at + the 2025 boundary and the cross-group forward-fill carries it into + 2026 unchanged.""" + df = pd.DataFrame( + { + "vin_candidateid": ["cand-1"], + "vin_name": ["CandA"], + "_resolved_rdstage_current": ["Phase II"], + "new_includeinpipeline2025": [100000000], + "new_includeinpipeline": [100000001], + "vin_product": ["Drugs"], + } + ) + result = ( + _expand_temporal_rows(df).sort_values("valid_from").reset_index(drop=True) + ) + assert list(result["valid_from"]) == ["2025-01-01", "2026-01-01"] + assert result.loc[0, "new_currentrdstage"] == "Phase II" + assert result.loc[1, "new_currentrdstage"] == "Phase II" + + def test_frozen_2025_pipeline_column_dropped_from_output(self): + """`new_includeinpipeline2025` is consumed by expansion and must not + leak into Silver alongside the derived `includeinpipeline` column.""" + df = pd.DataFrame( + { + "vin_candidateid": ["cand-1"], + "vin_name": ["CandA"], + "new_includeinpipeline2025": [100000000], + "new_includeinpipeline": [100000001], + "vin_product": ["Drugs"], + } + ) + result = _expand_temporal_rows(df) + assert "new_includeinpipeline2025" not in result.columns + assert "new_includeinpipeline" not in result.columns + class TestTransformCandidates: """Tests for transform_candidates function.""" @@ -667,6 +741,7 @@ def _make_input_df(self, overrides=None): "new_includeinpipeline2021": [100000000.0, 100000002.0, 100000001.0], "new_2023includeinevgendatabase": ["Yes", "No", "Pending"], "new_2024includeinpipeline": [862890000.0, None, None], + "new_includeinpipeline2025": [100000000.0, 100000002.0, 100000001.0], "new_includeinpipeline": [100000000.0, 100000002.0, 100000001.0], "_vin_captype_value": [ "c1746ad3-93d1-f011-bbd3-00224892cefa", @@ -1110,7 +1185,7 @@ def test_includeinpipeline_2025_raw_survives_when_all_null(self): """includeinpipeline_2025_raw is never dropped by drop_empty_columns even when every candidate's 2025 value is null (all-None override).""" df = self._make_input_df( - overrides={"new_includeinpipeline": [None, None, None]} + overrides={"new_includeinpipeline2025": [None, None, None]} ) lookup = self._make_lookup_tables() result, _ = transform_candidates(df, lookup_tables=lookup) @@ -1120,6 +1195,25 @@ def test_includeinpipeline_2025_raw_survives_when_all_null(self): # Every value should be null (the source was all-None). assert result["includeinpipeline_2025_raw"].isna().all() + def test_includeinpipeline_2025_raw_reads_frozen_not_rolling(self): + """The strict flag must read the frozen 2025 archive. Reading the + rolling `new_includeinpipeline` would report 2026 inclusion as 2025 + on the WHO Priority page. The two columns are given deliberately + opposite values so a regression cannot pass by coincidence.""" + df = self._make_input_df( + overrides={ + "new_includeinpipeline2025": [100000000.0, None, 100000001.0], + "new_includeinpipeline": [100000001.0, None, 100000000.0], + } + ) + lookup = self._make_lookup_tables() + result, _ = transform_candidates(df, lookup_tables=lookup) + + rows_a = result[result["candidateid"] == "id-1"] + assert (rows_a["includeinpipeline_2025_raw"] == 100000000.0).all() + rows_c = result[result["candidateid"] == "id-3"] + assert (rows_c["includeinpipeline_2025_raw"] == 100000001.0).all() + def test_ctregistrylink_synthesis_applied(self): """CT registry link is cleaned during transform.""" df = self._make_input_df( From cb51073c21cac422d88cd82dd96f892869e4d279 Mon Sep 17 00:00:00 2001 From: Zuhdil Herry Kurnia Date: Mon, 3 Aug 2026 12:25:42 +0700 Subject: [PATCH 3/5] Assert the 2026 reporting year reaches the Gold layer Covers both halves of the fix at the star-schema level: that 2026 appears in `fact_pipeline_snapshot`, and that the 2025 and 2026 boundaries read distinct source columns rather than both drawing from the rolling field. Deliberately does not assert row-count parity between 2025 and 2024. All candidates Dataverse creates in 2026 carry a rolling R&D stage, and the stage series has no frozen 2025 archive column of its own, so each such candidate still contributes a stage-only 2025 row. That row carries no pipeline-inclusion value, and every portal query filters on inclusion, so it stays invisible everywhere that matters. Moving the stage series to a 2026 boundary was considered and rejected: it would strip or change 2025 stage data for thousands of existing candidates just to remove a few hundred rows nothing reads. The unit test makes that trade-off explicit at the source: a 2026- created candidate with a stage gets a 2025 boundary with NULL inclusion, alongside its real 2026 boundary with the inclusion value. --- tests/e2e/test_silver_to_gold_e2e.py | 43 ++++++++++++++++++++++++++++ tests/unit/test_candidates.py | 26 +++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/tests/e2e/test_silver_to_gold_e2e.py b/tests/e2e/test_silver_to_gold_e2e.py index a948bef..589cba4 100644 --- a/tests/e2e/test_silver_to_gold_e2e.py +++ b/tests/e2e/test_silver_to_gold_e2e.py @@ -299,6 +299,49 @@ def test_pipeline_snapshot_candidate_key_not_all_null(self, gold_conn): df = _read_table(gold_conn, "fact_pipeline_snapshot") assert df["candidate_key"].notna().any(), "candidate_key is all NULL" + def test_fact_pipeline_snapshot_includes_2026(self, gold_conn): + """The 2026 reporting year reaches Gold. Before the rolling + pipeline-inclusion column was given its own boundary, 2026 values + were folded into 2025.""" + years = pd.read_sql_query( + """ + SELECT DISTINCT dt.year + FROM fact_pipeline_snapshot f + JOIN dim_date dt ON dt.date_key = f.date_key + WHERE dt.year IS NOT NULL + ORDER BY dt.year + """, + gold_conn, + )["year"].tolist() + assert 2026 in years, f"2026 missing from Gold; got {years}" + + def test_2025_and_2026_read_different_source_columns(self, gold_conn): + """2025 must read the frozen `new_includeinpipeline2025` archive and + 2026 the rolling `new_includeinpipeline`. Identical included-counts + would mean the rolling column is still feeding both boundaries — + the exact defect this change fixes. + + Note this deliberately does NOT assert row-count parity between 2025 + and 2024. All 244 candidates created during 2026 carry a rolling R&D + stage, and the stage series is pinned at 2025, so each contributes a + stage-only 2025 row. Those rows carry `include_in_pipeline` NULL and + are invisible to every portal query, all of which filter on it. + """ + counts = pd.read_sql_query( + """ + SELECT dt.year, COUNT(*) AS n + FROM fact_pipeline_snapshot f + JOIN dim_date dt ON dt.date_key = f.date_key + WHERE dt.year IN (2025, 2026) AND f.include_in_pipeline = 1 + GROUP BY dt.year + """, + gold_conn, + ).set_index("year")["n"] + assert counts.loc[2025] != counts.loc[2026], ( + f"2025 and 2026 both report {counts.loc[2025]} included candidates; " + "the two boundaries appear to be reading the same source column" + ) + # -- fact_clinical_trial_event -- def test_clinical_trial_has_rows(self, gold_conn): diff --git a/tests/unit/test_candidates.py b/tests/unit/test_candidates.py index 4f609b8..babbc22 100644 --- a/tests/unit/test_candidates.py +++ b/tests/unit/test_candidates.py @@ -678,6 +678,32 @@ def test_candidate_created_in_2026_has_no_2025_boundary(self): assert list(result["valid_from"]) == ["2026-01-01"] assert pd.isna(result.loc[0, "valid_to"]) + def test_candidate_created_in_2026_gets_stage_only_2025_row(self): + """All 244 candidates Dataverse created during 2026 carry a rolling + R&D stage, and the stage series is pinned at 2025, so they DO get a + 2025 boundary. What makes that harmless is that it carries no + inclusion value — every portal query filters `include_in_pipeline`, + so a NULL-inclusion row is invisible.""" + df = pd.DataFrame( + { + "vin_candidateid": ["cand-new"], + "vin_name": ["CandNew"], + "_resolved_rdstage_current": ["Phase I"], + "new_includeinpipeline2025": [None], + "new_includeinpipeline": [100000000], + "vin_product": ["Drugs"], + } + ) + result = ( + _expand_temporal_rows(df) + .sort_values("valid_from") + .reset_index(drop=True) + ) + assert list(result["valid_from"]) == ["2025-01-01", "2026-01-01"] + assert pd.isna(result.loc[0, "includeinpipeline"]) + assert result.loc[0, "new_currentrdstage"] == "Phase I" + assert result.loc[1, "includeinpipeline"] == 100000000 + def test_rolling_stage_forward_fills_into_2026(self): """R&D stage has no frozen 2025 archive, so the rolling stage sits at the 2025 boundary and the cross-group forward-fill carries it into From 0ab8eb454177fa461127c67e632773ac51917733 Mon Sep 17 00:00:00 2001 From: Zuhdil Herry Kurnia Date: Mon, 3 Aug 2026 12:36:43 +0700 Subject: [PATCH 4/5] Apply ruff format to the 2026 stage-only-row test Collapses a three-line chained call onto one line in test_candidate_created_in_2026_gets_stage_only_2025_row. Pure whitespace change; no assertion or docstring touched. --- tests/unit/test_candidates.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unit/test_candidates.py b/tests/unit/test_candidates.py index babbc22..430f645 100644 --- a/tests/unit/test_candidates.py +++ b/tests/unit/test_candidates.py @@ -695,9 +695,7 @@ def test_candidate_created_in_2026_gets_stage_only_2025_row(self): } ) result = ( - _expand_temporal_rows(df) - .sort_values("valid_from") - .reset_index(drop=True) + _expand_temporal_rows(df).sort_values("valid_from").reset_index(drop=True) ) assert list(result["valid_from"]) == ["2025-01-01", "2026-01-01"] assert pd.isna(result.loc[0, "includeinpipeline"]) From 36990a4dec461ebc0e7efb84c603b169e4a6e6ff Mon Sep 17 00:00:00 2001 From: Zuhdil Herry Kurnia Date: Mon, 3 Aug 2026 12:48:33 +0700 Subject: [PATCH 5/5] Correct comments left describing the pre-2026 column semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The branch fixed the code that mislabelled 2026 data as 2025, but four comments and docstrings still told the old story: - A test docstring claimed 2026-created candidates "must not be backdated into 2025" and blamed a 244-row inflation defect. That prediction was retracted during review: 2025 stayed at 8826 rows, because those candidates carry a rolling R&D stage that is deliberately pinned at 2025, so they correctly get a 2025 row. The docstring now describes what the test actually exercises — the no-stage case, where only the rolling pipeline column contributes a boundary — and points to the adjacent test for the representative with-stage case. - The strict-2025 pipeline-inclusion flag's comment in schema_map.py still named the pre-rollover source column `new_includeinpipeline`; it reads `new_includeinpipeline2025` now that Dataverse rolled its "current" column forward a year. - An e2e docstring said stage-only 2025 rows carry `include_in_pipeline` NULL. The raw `includeinpipeline` column is NULL on those rows, but step 7 of `transform_candidates` maps NaN to 0, so the derived column in Silver and Gold is 0, not NULL. The conclusion (invisible to every `include_in_pipeline = 1` filter) is unchanged; only the mechanism was misdescribed. - Two comments in test_includeinpipeline_2025_raw_preserved_at_candidate_grain said "bronze new_includeinpipeline"; the test reads the frozen new_includeinpipeline2025 column. The fixture happens to give both columns identical values, so this test can't distinguish them — the real frozen-vs-rolling pinning is asserted by test_includeinpipeline_2025_raw_reads_frozen_not_rolling — but the comments should still name the column the test actually reads. No executable code, assertions, or fixture values change in this commit. --- .../silver_to_gold/config/schema_map.py | 2 +- tests/e2e/test_silver_to_gold_e2e.py | 6 ++++-- tests/unit/test_candidates.py | 11 ++++++----- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/igh_data_transform/transformations/silver_to_gold/config/schema_map.py b/src/igh_data_transform/transformations/silver_to_gold/config/schema_map.py index 1e48ee7..e679931 100644 --- a/src/igh_data_transform/transformations/silver_to_gold/config/schema_map.py +++ b/src/igh_data_transform/transformations/silver_to_gold/config/schema_map.py @@ -54,7 +54,7 @@ "countries_approved_agg": "countries_product_approved", "candidate_type": "CASE WHEN captype_value = 'c1746ad3-93d1-f011-bbd3-00224892cefa' THEN 'Candidate' WHEN captype_value = '545d63d9-93d1-f011-bbd3-00224892cefa' THEN 'Product' ELSE 'Other' END", # Strict 2025 pipeline-inclusion flag for the WHO Priority page. - # 1 only when the raw 2025 `new_includeinpipeline` is exactly Yes; + # 1 only when the raw 2025 `new_includeinpipeline2025` is exactly Yes; # No, Pending, and blank all map to 0. Intentionally distinct from # the forward-filled `fact_pipeline_snapshot.include_in_pipeline`. "new_include_in_pipeline_2025": "CASE WHEN includeinpipeline_2025_raw = 100000000 THEN 1 ELSE 0 END", diff --git a/tests/e2e/test_silver_to_gold_e2e.py b/tests/e2e/test_silver_to_gold_e2e.py index 589cba4..91df746 100644 --- a/tests/e2e/test_silver_to_gold_e2e.py +++ b/tests/e2e/test_silver_to_gold_e2e.py @@ -324,8 +324,10 @@ def test_2025_and_2026_read_different_source_columns(self, gold_conn): Note this deliberately does NOT assert row-count parity between 2025 and 2024. All 244 candidates created during 2026 carry a rolling R&D stage, and the stage series is pinned at 2025, so each contributes a - stage-only 2025 row. Those rows carry `include_in_pipeline` NULL and - are invisible to every portal query, all of which filter on it. + stage-only 2025 row. The raw `includeinpipeline` is NULL on those + rows, but the derivation maps NaN to 0, so those rows carry + `include_in_pipeline` 0 and are invisible to every portal query, + all of which filter on it. """ counts = pd.read_sql_query( """ diff --git a/tests/unit/test_candidates.py b/tests/unit/test_candidates.py index 430f645..bf83331 100644 --- a/tests/unit/test_candidates.py +++ b/tests/unit/test_candidates.py @@ -662,9 +662,10 @@ def test_rolling_pipeline_column_contributes_2026_boundary(self): assert result.loc[1, "includeinpipeline"] == 100000001 def test_candidate_created_in_2026_has_no_2025_boundary(self): - """Candidates Dataverse created during 2026 carry only a rolling - value and no frozen 2025 one. They must not be backdated into 2025 — - this is the defect that inflated the 2025 bucket by 244 rows.""" + """When a candidate has no R&D stage value, the stage series + contributes no boundary, so only the rolling pipeline-inclusion + column does — yielding a 2026 boundary alone. The representative + case, a 2026 candidate WITH a stage, is covered by the next test.""" df = pd.DataFrame( { "vin_candidateid": ["cand-new"], @@ -1195,12 +1196,12 @@ def test_includeinpipeline_2025_raw_preserved_at_candidate_grain(self): assert "includeinpipeline_2025_raw" in result.columns - # Candidate A: bronze new_includeinpipeline == 100000000 (Yes) + # Candidate A: bronze new_includeinpipeline2025 == 100000000 (Yes) rows_a = result[result["candidateid"] == "id-1"] assert len(rows_a) > 1 assert (rows_a["includeinpipeline_2025_raw"] == 100000000.0).all() - # Candidate C: bronze new_includeinpipeline == 100000001 (No) — + # Candidate C: bronze new_includeinpipeline2025 == 100000001 (No) — # carried verbatim, NOT coerced. rows_c = result[result["candidateid"] == "id-3"] assert (rows_c["includeinpipeline_2025_raw"] == 100000001.0).all()