fix wrong timezone assumptions and adapt tests

pypsa_validation_processing/class_definitions.py

@@ -20,7 +20,8 @@
 
 
 def format_timestamps(df: pd.DataFrame) -> pd.DataFrame:
-    """Normalize timestamp-like columns to tz-aware objects in UTC+01:00.
+    """Normalize timestamp-like columns to tz-aware objects or
+    formatted integers in UTC+00:00.
 
     Parameters
     ----------
@@ -32,15 +33,21 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame:
     -------
     pd.DataFrame
         The same DataFrame with columns converted to Python ``datetime``
-        objects localized to ``+01:00`` where possible.
+        objects localized to ``+00:00`` where possible for timeseries. For
+        yearly aggregated data, columns are converted to integers.
 
     Notes
     -----
     Columns that cannot be parsed as timestamps are left unchanged. Values
     that can be parsed but cannot be localized are replaced with ``pd.NaT``
     and reported via ``print`` warnings.
+    Yearly aggregated data is identified by all column labels being
+    4-digit year strings only. In this case, the columns are converted to
+    integers. For non-aggregated data, columns are converted to Python
+    ``datetime`` objects localized to UTC+00:00.
     """
-    fixed_tz = datetime.timezone(datetime.timedelta(hours=1))
+    fixed_tz = datetime.timezone(datetime.timedelta(hours=0))
+
     cols = list(df.columns)
     idx_name = df.columns.name
     try:
@@ -51,40 +58,48 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame:
     converted_list: list[object] = []
     nat_list: list[object] = []
 
-    for i, col in enumerate(cols):
-        is_year_only = isinstance(col, str) and re.match(r"^\d{4}$", col) is not None
-        parsed_value = parsed[i]
+    # for yearly aggregated data, use integers as column labels.
+    if all([(isinstance(elem, str) and re.match(r"^\d{4}$", elem)) for elem in cols]):
+        converted_list = [int(col) for col in cols]
+        df.columns = pd.Index(converted_list, name=idx_name)
+    # for non-aggregated data, use timestamps as column labels.
+    else:
+        for i, col in enumerate(cols):
+            is_year_only = (
+                isinstance(col, str) and re.match(r"^\d{4}$", col) is not None
+            )
+            parsed_value = parsed[i]
 
-        if pd.isna(parsed_value) and not is_year_only:
-            continue
+            if pd.isna(parsed_value) and not is_year_only:
+                continue
 
-        ts = (
-            parsed_value
-            if not pd.isna(parsed_value)
-            else pd.Timestamp(f"{col}-01-01 00:00:00")
-        )
+            ts = (
+                parsed_value
+                if not pd.isna(parsed_value)
+                else pd.Timestamp(f"{col}-01-01 00:00:00")
+            )
 
-        if ts.tz is not None:
-            cols[i] = ts
-            converted_list.append(col)
-            continue
+            if ts.tz is not None:
+                cols[i] = ts
+                converted_list.append(col)
+                continue
 
-        try:
-            ts_tz = ts.tz_localize(fixed_tz)
-        except (TypeError, ValueError) as exc:
-            print(
-                f"WARNING: format_timestamps: failed to localize column {col!r}: {exc}. "
-                "Setting label to pd.NaT"
-            )
-            ts_tz = pd.NaT
-            nat_list.append(col)
+            try:
+                ts_tz = ts.tz_localize(fixed_tz)
+            except (TypeError, ValueError) as exc:
+                print(
+                    f"WARNING: format_timestamps: failed to localize column {col!r}: {exc}. "
+                    "Setting label to pd.NaT"
+                )
+                ts_tz = pd.NaT
+                nat_list.append(col)
 
-        cols[i] = ts_tz
-        if not pd.isna(ts_tz):
-            converted_list.append(col)
+            cols[i] = ts_tz
+            if not pd.isna(ts_tz):
+                converted_list.append(col)
 
-    py_datetimes = pd.Index(cols, name=idx_name).to_pydatetime()
-    df.columns = pd.Index(py_datetimes, dtype="object", name=idx_name)
+        py_datetimes = pd.Index(cols, name=idx_name).to_pydatetime()
+        df.columns = pd.Index(py_datetimes, dtype="object", name=idx_name)
     if nat_list:
         print("WARNING: format_timestamps: columns set to NaT:", nat_list)
     return df

pypsa_validation_processing/configs/config.default.yaml

@@ -10,5 +10,5 @@ map_country_codes_to_names: false # true: map AT -> Austria in output regions; f
 
 # Network
 network_results_path: resources/AT_KN2040/ # path to the folder containing PyPSA network results
-model_name:  PyPSA-AT v0.0.1-8b8a616             # name of the PyPSA model
+model_name:  Pypsa-AT v1.0             # name of the PyPSA model
 scenario_name: KN2040test        # name of the PyPSA scenario

tests/test_format_timestamps.py

@@ -72,14 +72,14 @@ def test_format_timestamps_hourly_columns():
 def test_format_timestamps_preserves_tz_aware_columns():
     aware_label = pd.Timestamp(
         "2050-01-01 00:00:00",
-        tz=datetime.timezone(datetime.timedelta(hours=1)),
+        tz=datetime.timezone(datetime.timedelta(hours=0)),
     )
     df = pd.DataFrame([[1.0]], columns=[aware_label])
 
     out = format_timestamps(df)
 
     assert out.columns[0] == aware_label
-    assert out.columns[0].utcoffset() == datetime.timedelta(hours=1)
+    assert out.columns[0].utcoffset() == datetime.timedelta(hours=0)
 
 
 def test_format_timestamps_keeps_unparsable_columns():
@@ -130,7 +130,6 @@ def test_structure_pyam_from_pandas_formats_time_columns_before_pyam(tmp_path: P
     time_columns = [
         c
         for c in passed_data.columns
-        if isinstance(c, (pd.Timestamp, datetime.datetime))
+        if isinstance(c, (pd.Timestamp, datetime.datetime, int))
     ]
     assert len(time_columns) == 1
-    assert time_columns[0].utcoffset() == datetime.timedelta(hours=1)