diff --git a/pypsa_validation_processing/class_definitions.py b/pypsa_validation_processing/class_definitions.py index a02d865..cd8c599 100644 --- a/pypsa_validation_processing/class_definitions.py +++ b/pypsa_validation_processing/class_definitions.py @@ -20,7 +20,8 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame: - """Normalize timestamp-like columns to tz-aware objects in UTC+01:00. + """Normalize timestamp-like columns to tz-aware objects or + formatted integers in UTC+00:00. Parameters ---------- @@ -32,15 +33,21 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame: ------- pd.DataFrame The same DataFrame with columns converted to Python ``datetime`` - objects localized to ``+01:00`` where possible. + objects localized to ``+00:00`` where possible for timeseries. For + yearly aggregated data, columns are converted to integers. Notes ----- Columns that cannot be parsed as timestamps are left unchanged. Values that can be parsed but cannot be localized are replaced with ``pd.NaT`` and reported via ``print`` warnings. + Yearly aggregated data is identified by all column labels being + 4-digit year strings only. In this case, the columns are converted to + integers. For non-aggregated data, columns are converted to Python + ``datetime`` objects localized to UTC+00:00. """ - fixed_tz = datetime.timezone(datetime.timedelta(hours=1)) + fixed_tz = datetime.timezone(datetime.timedelta(hours=0)) + cols = list(df.columns) idx_name = df.columns.name try: @@ -51,40 +58,48 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame: converted_list: list[object] = [] nat_list: list[object] = [] - for i, col in enumerate(cols): - is_year_only = isinstance(col, str) and re.match(r"^\d{4}$", col) is not None - parsed_value = parsed[i] + # for yearly aggregated data, use integers as column labels. + if all([(isinstance(elem, str) and re.match(r"^\d{4}$", elem)) for elem in cols]): + converted_list = [int(col) for col in cols] + df.columns = pd.Index(converted_list, name=idx_name) + # for non-aggregated data, use timestamps as column labels. + else: + for i, col in enumerate(cols): + is_year_only = ( + isinstance(col, str) and re.match(r"^\d{4}$", col) is not None + ) + parsed_value = parsed[i] - if pd.isna(parsed_value) and not is_year_only: - continue + if pd.isna(parsed_value) and not is_year_only: + continue - ts = ( - parsed_value - if not pd.isna(parsed_value) - else pd.Timestamp(f"{col}-01-01 00:00:00") - ) + ts = ( + parsed_value + if not pd.isna(parsed_value) + else pd.Timestamp(f"{col}-01-01 00:00:00") + ) - if ts.tz is not None: - cols[i] = ts - converted_list.append(col) - continue + if ts.tz is not None: + cols[i] = ts + converted_list.append(col) + continue - try: - ts_tz = ts.tz_localize(fixed_tz) - except (TypeError, ValueError) as exc: - print( - f"WARNING: format_timestamps: failed to localize column {col!r}: {exc}. " - "Setting label to pd.NaT" - ) - ts_tz = pd.NaT - nat_list.append(col) + try: + ts_tz = ts.tz_localize(fixed_tz) + except (TypeError, ValueError) as exc: + print( + f"WARNING: format_timestamps: failed to localize column {col!r}: {exc}. " + "Setting label to pd.NaT" + ) + ts_tz = pd.NaT + nat_list.append(col) - cols[i] = ts_tz - if not pd.isna(ts_tz): - converted_list.append(col) + cols[i] = ts_tz + if not pd.isna(ts_tz): + converted_list.append(col) - py_datetimes = pd.Index(cols, name=idx_name).to_pydatetime() - df.columns = pd.Index(py_datetimes, dtype="object", name=idx_name) + py_datetimes = pd.Index(cols, name=idx_name).to_pydatetime() + df.columns = pd.Index(py_datetimes, dtype="object", name=idx_name) if nat_list: print("WARNING: format_timestamps: columns set to NaT:", nat_list) return df diff --git a/pypsa_validation_processing/configs/config.default.yaml b/pypsa_validation_processing/configs/config.default.yaml index f96d4c8..20a1dc5 100644 --- a/pypsa_validation_processing/configs/config.default.yaml +++ b/pypsa_validation_processing/configs/config.default.yaml @@ -10,5 +10,5 @@ map_country_codes_to_names: false # true: map AT -> Austria in output regions; f # Network network_results_path: resources/AT_KN2040/ # path to the folder containing PyPSA network results -model_name: PyPSA-AT v0.0.1-8b8a616 # name of the PyPSA model +model_name: Pypsa-AT v1.0 # name of the PyPSA model scenario_name: KN2040test # name of the PyPSA scenario diff --git a/tests/test_format_timestamps.py b/tests/test_format_timestamps.py index 5418584..18382f1 100644 --- a/tests/test_format_timestamps.py +++ b/tests/test_format_timestamps.py @@ -72,14 +72,14 @@ def test_format_timestamps_hourly_columns(): def test_format_timestamps_preserves_tz_aware_columns(): aware_label = pd.Timestamp( "2050-01-01 00:00:00", - tz=datetime.timezone(datetime.timedelta(hours=1)), + tz=datetime.timezone(datetime.timedelta(hours=0)), ) df = pd.DataFrame([[1.0]], columns=[aware_label]) out = format_timestamps(df) assert out.columns[0] == aware_label - assert out.columns[0].utcoffset() == datetime.timedelta(hours=1) + assert out.columns[0].utcoffset() == datetime.timedelta(hours=0) def test_format_timestamps_keeps_unparsable_columns(): @@ -130,7 +130,6 @@ def test_structure_pyam_from_pandas_formats_time_columns_before_pyam(tmp_path: P time_columns = [ c for c in passed_data.columns - if isinstance(c, (pd.Timestamp, datetime.datetime)) + if isinstance(c, (pd.Timestamp, datetime.datetime, int)) ] assert len(time_columns) == 1 - assert time_columns[0].utcoffset() == datetime.timedelta(hours=1)