From 83f42baae1368ae664d81ab3a171f870c8cf1813 Mon Sep 17 00:00:00 2001 From: Max Nutz Date: Wed, 8 Jul 2026 16:03:05 +0200 Subject: [PATCH 1/4] fix wrong timezone assumptions and adapt tests --- pypsa_validation_processing/class_definitions.py | 3 ++- tests/test_format_timestamps.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pypsa_validation_processing/class_definitions.py b/pypsa_validation_processing/class_definitions.py index a02d865..025992a 100644 --- a/pypsa_validation_processing/class_definitions.py +++ b/pypsa_validation_processing/class_definitions.py @@ -40,7 +40,8 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame: that can be parsed but cannot be localized are replaced with ``pd.NaT`` and reported via ``print`` warnings. """ - fixed_tz = datetime.timezone(datetime.timedelta(hours=1)) + fixed_tz = datetime.timezone(datetime.timedelta(hours=0)) + cols = list(df.columns) idx_name = df.columns.name try: diff --git a/tests/test_format_timestamps.py b/tests/test_format_timestamps.py index 5418584..f8e5564 100644 --- a/tests/test_format_timestamps.py +++ b/tests/test_format_timestamps.py @@ -72,14 +72,14 @@ def test_format_timestamps_hourly_columns(): def test_format_timestamps_preserves_tz_aware_columns(): aware_label = pd.Timestamp( "2050-01-01 00:00:00", - tz=datetime.timezone(datetime.timedelta(hours=1)), + tz=datetime.timezone(datetime.timedelta(hours=0)), ) df = pd.DataFrame([[1.0]], columns=[aware_label]) out = format_timestamps(df) assert out.columns[0] == aware_label - assert out.columns[0].utcoffset() == datetime.timedelta(hours=1) + assert out.columns[0].utcoffset() == datetime.timedelta(hours=0) def test_format_timestamps_keeps_unparsable_columns(): @@ -133,4 +133,4 @@ def test_structure_pyam_from_pandas_formats_time_columns_before_pyam(tmp_path: P if isinstance(c, (pd.Timestamp, datetime.datetime)) ] assert len(time_columns) == 1 - assert time_columns[0].utcoffset() == datetime.timedelta(hours=1) + assert time_columns[0].utcoffset() == datetime.timedelta(hours=0) From 0606f2b0a71eaeba201a4aa75c9fcbc452e8c1c9 Mon Sep 17 00:00:00 2001 From: Max Nutz Date: Thu, 9 Jul 2026 13:57:33 +0200 Subject: [PATCH 2/4] format yearly timesteps as integer, not as pd.timestep --- .../class_definitions.py | 68 +++++++++++-------- tests/test_format_timestamps.py | 3 +- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/pypsa_validation_processing/class_definitions.py b/pypsa_validation_processing/class_definitions.py index 025992a..5ef343f 100644 --- a/pypsa_validation_processing/class_definitions.py +++ b/pypsa_validation_processing/class_definitions.py @@ -20,7 +20,7 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame: - """Normalize timestamp-like columns to tz-aware objects in UTC+01:00. + """Normalize timestamp-like columns to tz-aware objects in UTC+00:00. Parameters ---------- @@ -32,7 +32,7 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame: ------- pd.DataFrame The same DataFrame with columns converted to Python ``datetime`` - objects localized to ``+01:00`` where possible. + objects localized to ``+00:00`` where possible. Notes ----- @@ -52,40 +52,48 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame: converted_list: list[object] = [] nat_list: list[object] = [] - for i, col in enumerate(cols): - is_year_only = isinstance(col, str) and re.match(r"^\d{4}$", col) is not None - parsed_value = parsed[i] + # for yearly aggregated data, use integers as column labels. + if all([(isinstance(elem, str) and re.match(r"^\d{4}$", elem)) for elem in cols]): + converted_list = [int(col) for col in cols] + df.columns = pd.Index(converted_list, name=idx_name) + # for non-aggregated data, use timestamps as column labels. + else: + for i, col in enumerate(cols): + is_year_only = ( + isinstance(col, str) and re.match(r"^\d{4}$", col) is not None + ) + parsed_value = parsed[i] - if pd.isna(parsed_value) and not is_year_only: - continue + if pd.isna(parsed_value) and not is_year_only: + continue - ts = ( - parsed_value - if not pd.isna(parsed_value) - else pd.Timestamp(f"{col}-01-01 00:00:00") - ) + ts = ( + parsed_value + if not pd.isna(parsed_value) + else pd.Timestamp(f"{col}-01-01 00:00:00") + ) - if ts.tz is not None: - cols[i] = ts - converted_list.append(col) - continue + if ts.tz is not None: + cols[i] = ts + converted_list.append(col) + continue - try: - ts_tz = ts.tz_localize(fixed_tz) - except (TypeError, ValueError) as exc: - print( - f"WARNING: format_timestamps: failed to localize column {col!r}: {exc}. " - "Setting label to pd.NaT" - ) - ts_tz = pd.NaT - nat_list.append(col) + try: + ts_tz = ts.tz_localize(fixed_tz) + except (TypeError, ValueError) as exc: + print( + f"WARNING: format_timestamps: failed to localize column {col!r}: {exc}. " + "Setting label to pd.NaT" + ) + ts_tz = pd.NaT + nat_list.append(col) - cols[i] = ts_tz - if not pd.isna(ts_tz): - converted_list.append(col) + cols[i] = ts_tz + if not pd.isna(ts_tz): + converted_list.append(col) - py_datetimes = pd.Index(cols, name=idx_name).to_pydatetime() - df.columns = pd.Index(py_datetimes, dtype="object", name=idx_name) + py_datetimes = pd.Index(cols, name=idx_name).to_pydatetime() + df.columns = pd.Index(py_datetimes, dtype="object", name=idx_name) if nat_list: print("WARNING: format_timestamps: columns set to NaT:", nat_list) return df diff --git a/tests/test_format_timestamps.py b/tests/test_format_timestamps.py index f8e5564..18382f1 100644 --- a/tests/test_format_timestamps.py +++ b/tests/test_format_timestamps.py @@ -130,7 +130,6 @@ def test_structure_pyam_from_pandas_formats_time_columns_before_pyam(tmp_path: P time_columns = [ c for c in passed_data.columns - if isinstance(c, (pd.Timestamp, datetime.datetime)) + if isinstance(c, (pd.Timestamp, datetime.datetime, int)) ] assert len(time_columns) == 1 - assert time_columns[0].utcoffset() == datetime.timedelta(hours=0) From e4c6652c8da37cb34ddcce7327d772a02a8851d9 Mon Sep 17 00:00:00 2001 From: Max Nutz Date: Thu, 9 Jul 2026 13:59:17 +0200 Subject: [PATCH 3/4] adapt default model_name to standard model name added in the explorer-mapping file --- pypsa_validation_processing/configs/config.default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypsa_validation_processing/configs/config.default.yaml b/pypsa_validation_processing/configs/config.default.yaml index f96d4c8..20a1dc5 100644 --- a/pypsa_validation_processing/configs/config.default.yaml +++ b/pypsa_validation_processing/configs/config.default.yaml @@ -10,5 +10,5 @@ map_country_codes_to_names: false # true: map AT -> Austria in output regions; f # Network network_results_path: resources/AT_KN2040/ # path to the folder containing PyPSA network results -model_name: PyPSA-AT v0.0.1-8b8a616 # name of the PyPSA model +model_name: Pypsa-AT v1.0 # name of the PyPSA model scenario_name: KN2040test # name of the PyPSA scenario From adc667fd4648c501539a4735959160107bdab8ab Mon Sep 17 00:00:00 2001 From: Max Nutz Date: Thu, 9 Jul 2026 14:13:15 +0200 Subject: [PATCH 4/4] renew docstring including special case of yearly aggregated data --- pypsa_validation_processing/class_definitions.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pypsa_validation_processing/class_definitions.py b/pypsa_validation_processing/class_definitions.py index 5ef343f..cd8c599 100644 --- a/pypsa_validation_processing/class_definitions.py +++ b/pypsa_validation_processing/class_definitions.py @@ -20,7 +20,8 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame: - """Normalize timestamp-like columns to tz-aware objects in UTC+00:00. + """Normalize timestamp-like columns to tz-aware objects or + formatted integers in UTC+00:00. Parameters ---------- @@ -32,13 +33,18 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame: ------- pd.DataFrame The same DataFrame with columns converted to Python ``datetime`` - objects localized to ``+00:00`` where possible. + objects localized to ``+00:00`` where possible for timeseries. For + yearly aggregated data, columns are converted to integers. Notes ----- Columns that cannot be parsed as timestamps are left unchanged. Values that can be parsed but cannot be localized are replaced with ``pd.NaT`` and reported via ``print`` warnings. + Yearly aggregated data is identified by all column labels being + 4-digit year strings only. In this case, the columns are converted to + integers. For non-aggregated data, columns are converted to Python + ``datetime`` objects localized to UTC+00:00. """ fixed_tz = datetime.timezone(datetime.timedelta(hours=0))