From 1e7052e0785dcb04025804598b560da4c4e5a2ae Mon Sep 17 00:00:00 2001 From: Max Nutz Date: Fri, 31 Jul 2026 11:58:47 +0200 Subject: [PATCH 1/6] implement logging structure with levels in all scripts --- .../class_definitions.py | 64 +++++++++++-------- .../statistics_functions.py | 57 +++++++++++++++-- pypsa_validation_processing/utils.py | 14 ++++ pypsa_validation_processing/workflow.py | 17 ++++- 4 files changed, 118 insertions(+), 34 deletions(-) diff --git a/pypsa_validation_processing/class_definitions.py b/pypsa_validation_processing/class_definitions.py index dfe36c3..06c3632 100644 --- a/pypsa_validation_processing/class_definitions.py +++ b/pypsa_validation_processing/class_definitions.py @@ -3,6 +3,7 @@ import glob import importlib import inspect +import logging import os from pathlib import Path import re @@ -18,6 +19,8 @@ UNITS_MAPPING, ) +logger = logging.getLogger(__name__) + def format_timestamps(df: pd.DataFrame) -> pd.DataFrame: """Normalize timestamp-like columns to tz-aware objects or @@ -40,7 +43,7 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame: ----- Columns that cannot be parsed as timestamps are left unchanged. Values that can be parsed but cannot be localized are replaced with ``pd.NaT`` - and reported via ``print`` warnings. + and reported via logger warnings. Yearly aggregated data is identified by all column labels being 4-digit year strings only. In this case, the columns are converted to integers. For non-aggregated data, columns are converted to Python @@ -87,9 +90,11 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame: try: ts_tz = ts.tz_localize(fixed_tz) except (TypeError, ValueError) as exc: - print( - f"WARNING: format_timestamps: failed to localize column {col!r}: {exc}. " - "Setting label to pd.NaT" + logger.warning( + "format_timestamps: failed to localize column %r: %s. " + "Setting label to pd.NaT", + col, + exc, ) ts_tz = pd.NaT nat_list.append(col) @@ -101,7 +106,7 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame: py_datetimes = pd.Index(cols, name=idx_name).to_pydatetime() df.columns = pd.Index(py_datetimes, dtype="object", name=idx_name) if nat_list: - print("WARNING: format_timestamps: columns set to NaT:", nat_list) + logger.warning("format_timestamps: columns set to NaT: %s", nat_list) return df @@ -332,8 +337,10 @@ def _execute_function_for_variable( ) func = getattr(stats_module, func_name, None) if func is None: - print( - f"WARNING: Variable {variable}: Function '{func_name}' not found in statistics_functions.py" + logger.warning( + "Variable %s: Function '%s' not found in statistics_functions.py", + variable, + func_name, ) return None @@ -663,7 +670,7 @@ def _get_unit_from_common_definitions(self, variable: str) -> str: ----- If multiple units are defined for one variable (in one entry of this variable!), the first one of these variables is taken. If there are more then one definitions for - the searched variable, the first entry is taken and a warning is printed. + the searched variable, the first entry is taken and a warning is logged. """ if self.common_dsd is None: raise RuntimeError("Common definitions are not initialized.") @@ -681,8 +688,10 @@ def _get_unit_from_common_definitions(self, variable: str) -> str: if match.empty: raise KeyError(f"Variable '{variable}' not defined in common definitions.") if len(match) > 1: - print( - f"WARNING: Multiple definitions found for variable '{variable}' in common definitions. Take first one: {match.iloc[0]}" + logger.warning( + "Multiple definitions found for variable '%s' in common definitions. Take first one: %s", + variable, + match.iloc[0], ) target_unit = match.iloc[0][unit_col] @@ -691,19 +700,20 @@ def _get_unit_from_common_definitions(self, variable: str) -> str: f"Unit information not found for variable '{variable}' in common definitions." ) if ("[" in target_unit) and ("]" in target_unit) and ("," in target_unit): - print( - "Several possible units defined for variable '{variable}' in common definitions. Take first one:".format( - variable=variable - ) + logger.info( + "Several possible units defined for variable '%s' in common definitions. Take first one:", + variable, ) try: import ast target_unit = ast.literal_eval(target_unit)[0] - print(target_unit) + logger.info(target_unit) except Exception as exc: - print( - f"WARNING: Failed to parse multiple units for variable '{variable}': {exc}. Using TJ as unit." + logger.warning( + "Failed to parse multiple units for variable '%s': %s. Using TJ as unit.", + variable, + exc, ) target_unit = "TJ" return str(target_unit) @@ -725,7 +735,7 @@ def _get_network_config(self, investment_year): Notes ----- The method searches for a YAML configuration file in the 'configs' subdirectory of the network. If multiple matching - files are found, it uses the first one and prints a warning. If no matching file is found, it prints a warning and returns None. + files are found, it uses the first one and logs a warning. If no matching file is found, it logs a warning and returns None. """ config_pattern = str( self.network_results_path / "configs" / f"config*{investment_year}.yaml" @@ -735,21 +745,23 @@ def _get_network_config(self, investment_year): if matching_files: selected_config_file = matching_files[0] if len(matching_files) > 1: - print( - f"INFO: Multiple config files found for investment year {investment_year}; " - f"using '{selected_config_file}'" + logger.info( + "Multiple config files found for investment year %s; using '%s'", + investment_year, + selected_config_file, ) try: with open(selected_config_file, "r") as f: network_config = yaml.safe_load(f) except Exception as exc: - print( - f"WARNING: Could not load config file '{selected_config_file}': {exc}" + logger.warning( + "Could not load config file '%s': %s", selected_config_file, exc ) else: - print( - f"WARNING: No config file found for investment year {investment_year} " - f"at pattern '{config_pattern}'" + logger.warning( + "No config file found for investment year %s at pattern '%s'", + investment_year, + config_pattern, ) return network_config diff --git a/pypsa_validation_processing/statistics_functions.py b/pypsa_validation_processing/statistics_functions.py index 9ca0cb7..d8bc545 100644 --- a/pypsa_validation_processing/statistics_functions.py +++ b/pypsa_validation_processing/statistics_functions.py @@ -25,6 +25,7 @@ """ from __future__ import annotations +import logging import re from functools import reduce from pathlib import Path @@ -43,6 +44,8 @@ create_location_index_from_copperplate, ) +logger = logging.getLogger(__name__) + def Final_Energy_by_Carrier__Electricity( n: pypsa.Network, @@ -138,6 +141,9 @@ def Final_Energy_by_Carrier__Electricity( series_list = [series for series in series_list if not series.empty] result = pd.concat(series_list) + logger.debug( + f"Final Energy by Carrier|Electricity finished. Resulting shape: {result.shape}. " + ) return result @@ -221,9 +227,12 @@ def Net_Imports__Electricity( ].sum() if isinstance(imports_raw, pd.DataFrame): - return net_imports_grouped.unstack("snapshot") + result = net_imports_grouped.unstack("snapshot") + else: + result = net_imports_grouped - return net_imports_grouped + logger.debug(f"Net Imports|Electricity finished. Resulting shape: {result.shape}. ") + return result def Final_Energy_by_Carrier__Coal( @@ -266,6 +275,9 @@ def Final_Energy_by_Carrier__Coal( groupby_time=aggregate_per_year, **kwargs, ) + logger.debug( + f"Final Energy by Carrier|Coal finished. Resulting shape: {industry.shape}. " + ) return industry @@ -441,6 +453,9 @@ def Final_Energy_by_Carrier__Oil( non_fossil_fraction = non_fossil_fraction.reindex_like(total).fillna(0.0) fossil_oil = total.mul(1 - non_fossil_fraction, axis=0) + logger.debug( + f"Final Energy by Carrier|Oil finished. Resulting shape: {fossil_oil.shape}. " + ) return fossil_oil @@ -539,8 +554,12 @@ def Net_Imports__Oil( ].sum() if isinstance(imports_raw, pd.DataFrame): - return net_imports_grouped.unstack("snapshot") - return net_imports_grouped + result = net_imports_grouped.unstack("snapshot") + else: + result = net_imports_grouped + + logger.debug(f"Net Imports|Oil finished. Resulting shape: {result.shape}. ") + return result def Final_Energy_by_Carrier__Natural_Gas( @@ -656,15 +675,22 @@ def Final_Energy_by_Carrier__Natural_Gas( series_list = [series for series in series_list if not series.empty] if not series_list: - return pd.Series( + result = pd.Series( dtype=float, index=pd.MultiIndex.from_tuples([], names=kwargs["groupby"]), ) + logger.debug( + f"Final Energy by Carrier|Natural Gas finished. Resulting shape: {result.shape}. " + ) + return result total = pd.concat(series_list) total = total.rename(index=UNITS_MAPPING).groupby(kwargs["groupby"]).sum() non_fossil_fraction = non_fossil_fraction.reindex_like(total).fillna(0) result = total.mul(1 - non_fossil_fraction, axis=0) + logger.debug( + f"Final Energy by Carrier|Natural Gas finished. Resulting shape: {result.shape}. " + ) return result @@ -755,6 +781,9 @@ def Final_Energy_by_Carrier__District_Heat( # heat_vent.index = pd.MultiIndex.from_frame(idx_frame_heat_vent) result = (res.groupby(["location", "unit"]).sum()).add(heat_vent, fill_value=0) + logger.debug( + f"Final Energy by Carrier|District Heat finished. Resulting shape: {result.shape}. " + ) return result @@ -826,8 +855,9 @@ def Final_Energy_by_Sector__Transportation( eff = bev_charger_efficiencies.iloc[0] else: eff = bev_charger_efficiencies.mean() - print( - "WARNING: Network includes different efficiencies for BEV chargers. Using mean value for variable Final_Energy_by_Sector__Transportation" + logger.warning( + "Network includes different efficiencies for BEV chargers. " + "Using mean value for variable Final_Energy_by_Sector__Transportation" ) elec = n.statistics.withdrawal( @@ -886,6 +916,9 @@ def Final_Energy_by_Sector__Transportation( series_list = [series for series in series_list if not series.empty] total = pd.concat(series_list) + logger.debug( + f"Final Energy by Sector|Transportation finished. Resulting shape: {total.shape}. " + ) return total @@ -969,6 +1002,9 @@ def Final_Energy_by_Sector__Industry( eff_loss = abs(cc_in) - abs(cc_out) eff_loss = eff_loss.groupby(["location", "unit"]).sum() res = load_statistics.add(eff_loss, fill_value=0) + logger.debug( + f"Final Energy by Sector|Industry finished. Resulting shape: {res.shape}. " + ) return res @@ -1047,6 +1083,9 @@ def Final_Energy_by_Sector__Agriculture( eff_loss = eff_loss.groupby(["location", "unit"]).sum() res = res.add(eff_loss, fill_value=0) + logger.debug( + f"Final Energy by Sector|Agriculture finished. Resulting shape: {res.shape}. " + ) return res @@ -1180,4 +1219,8 @@ def Final_Energy_by_Sector__Residential_and_Commercial( df.groupby(kwargs["groupby"]).sum() for df in series_list if not df.empty ] total = pd.concat(series_list) + logger.debug( + "Final Energy by Sector|Residential and Commercial finished. " + f"Resulting shape: {total.shape}. " + ) return total diff --git a/pypsa_validation_processing/utils.py b/pypsa_validation_processing/utils.py index 2778cd9..a682f33 100644 --- a/pypsa_validation_processing/utils.py +++ b/pypsa_validation_processing/utils.py @@ -1,5 +1,6 @@ """Static information and general utility functions for pypsa_validation_processing.""" +import logging import pandas as pd from pathlib import Path @@ -252,3 +253,16 @@ def create_location_index_from_copperplate( output = raw_input.copy() output.index = new_index return output + + +def setup_logging(level: int | str = logging.WARNING) -> None: + """Configure root logging once for the package entrypoint. + + Parameters + ---------- + level : int or str, optional + Logging level to configure, by default ``logging.WARNING``. + """ + logging.basicConfig( + level=level, format="%(asctime)s %(levelname)s %(name)s: %(message)s" + ) diff --git a/pypsa_validation_processing/workflow.py b/pypsa_validation_processing/workflow.py index 15578cd..cddc146 100644 --- a/pypsa_validation_processing/workflow.py +++ b/pypsa_validation_processing/workflow.py @@ -1,9 +1,13 @@ from __future__ import annotations import argparse +import logging from pathlib import Path from pypsa_validation_processing import Network_Processor +from pypsa_validation_processing.utils import setup_logging + +logger = logging.getLogger(__name__) def get_default_config_path() -> Path: @@ -27,7 +31,7 @@ def resolve_config_path(config_arg: str | None) -> Path: """ if config_arg: return Path(config_arg).expanduser().resolve() - print("WARNING: no config-file provided. Using default config.") + logger.warning("no config-file provided. Using default config.") return get_default_config_path() @@ -47,12 +51,23 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="Path to YAML config file. Defaults to packaged config.", ) + parser.add_argument( + "--log-level", + default="DEBUG", + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + help="Set the logging level. Defaults to WARNING.", + ) return parser def main() -> None: """Parse CLI arguments, run the Network_Processor pipeline, and write output.""" args = build_parser().parse_args() + setup_logging(level=args.log_level) + if args.log_level == "DEBUG": + logger.info( + f"Importing pypsa-Network with logger level {args.log_level} prints several network tables." + ) config_path = resolve_config_path(args.config) processor = Network_Processor(config_path=config_path) From 00dfcfd40b66a26040412ad673d451fe369fd98b Mon Sep 17 00:00:00 2001 From: Max Nutz Date: Fri, 31 Jul 2026 11:59:37 +0200 Subject: [PATCH 2/6] update tests for loggings --- tests/test_format_timestamps.py | 21 ++++++++----- tests/test_network_processor.py | 50 ++++++++++++++++++------------ tests/test_statistics_functions.py | 17 +++++----- tests/test_unit_conversion.py | 12 ++++--- 4 files changed, 61 insertions(+), 39 deletions(-) diff --git a/tests/test_format_timestamps.py b/tests/test_format_timestamps.py index 9e23474..5d3cf4b 100644 --- a/tests/test_format_timestamps.py +++ b/tests/test_format_timestamps.py @@ -3,6 +3,7 @@ from __future__ import annotations import datetime +import logging from pathlib import Path from unittest.mock import MagicMock, patch @@ -89,7 +90,7 @@ def test_format_timestamps_keeps_unparsable_columns(): format_timestamps(df) -def test_format_timestamps_sets_nat_on_localization_failure(capsys): +def test_format_timestamps_sets_nat_on_localization_failure(caplog): class FakeTimestamp: tz = None tzinfo = None @@ -99,15 +100,19 @@ def tz_localize(self, _tz): df = pd.DataFrame([[1.0]], columns=["2050-01-01 00:00:00"]) - with patch( - "pypsa_validation_processing.class_definitions.pd.to_datetime", - return_value=[FakeTimestamp()], - ): - out = format_timestamps(df) + with caplog.at_level(logging.WARNING): + with patch( + "pypsa_validation_processing.class_definitions.pd.to_datetime", + return_value=[FakeTimestamp()], + ): + out = format_timestamps(df) - captured = capsys.readouterr() assert pd.isna(out.columns[0]) - assert "WARNING: format_timestamps: failed to localize column" in captured.out + assert any( + record.levelno == logging.WARNING + and "format_timestamps: failed to localize column" in record.message + for record in caplog.records + ) def test_format_timestamps_falls_back_when_mixed_format_raises_typeerror(): diff --git a/tests/test_network_processor.py b/tests/test_network_processor.py index 7c3baf5..4aac307 100644 --- a/tests/test_network_processor.py +++ b/tests/test_network_processor.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging import re from pathlib import Path from unittest.mock import MagicMock, patch @@ -301,9 +302,9 @@ def test_execute_function_not_found(self, mock_config_file: Path): assert result is None def test_execute_function_warns_when_function_name_not_in_module( - self, mock_config_file: Path, capsys + self, mock_config_file: Path, caplog ): - """Test that a WARNING is printed and None returned for an unresolvable function name.""" + """Test that a WARNING is logged and None returned for an unresolvable function name.""" with patch( "pypsa_validation_processing.class_definitions.pypsa.NetworkCollection" ): @@ -316,14 +317,17 @@ def test_execute_function_warns_when_function_name_not_in_module( } mock_network = MockPyPSANetwork() - result = processor._execute_function_for_variable( - "Bogus Variable", mock_network - ) + with caplog.at_level(logging.WARNING): + result = processor._execute_function_for_variable( + "Bogus Variable", mock_network + ) - captured = capsys.readouterr() assert result is None - assert "WARNING" in captured.out - assert "This_Function_Does_Not_Exist" in captured.out + assert any( + record.levelno == logging.WARNING + and "This_Function_Does_Not_Exist" in record.message + for record in caplog.records + ) def test_execute_function_passes_config_when_accepted(self, mock_config_file: Path): """Test that config is passed to functions that accept it.""" @@ -1578,7 +1582,7 @@ def _setup_processor(self, tmp_path: Path) -> Network_Processor: return Network_Processor(config_path=config_file) def test_warns_and_uses_first_of_multiple_matching_config_files( - self, tmp_path: Path, capsys + self, tmp_path: Path, caplog ): """Test that multiple matching config files trigger an INFO message and use the first.""" processor = self._setup_processor(tmp_path) @@ -1587,35 +1591,41 @@ def test_warns_and_uses_first_of_multiple_matching_config_files( (configs_dir / "config_a_2020.yaml").write_text("foo: bar\n") (configs_dir / "config_b_2020.yaml").write_text("foo: baz\n") - result = processor._get_network_config(2020) + with caplog.at_level(logging.INFO): + result = processor._get_network_config(2020) - captured = capsys.readouterr() - assert "INFO: Multiple config files found" in captured.out + assert any( + "Multiple config files found" in record.message for record in caplog.records + ) assert result is not None - def test_warns_and_returns_none_on_malformed_yaml(self, tmp_path: Path, capsys): + def test_warns_and_returns_none_on_malformed_yaml(self, tmp_path: Path, caplog): """Test that malformed YAML in the matched config file triggers a WARNING.""" processor = self._setup_processor(tmp_path) configs_dir = processor.network_results_path / "configs" configs_dir.mkdir(parents=True, exist_ok=True) (configs_dir / "config_2020.yaml").write_text("foo: [unclosed\n") - result = processor._get_network_config(2020) + with caplog.at_level(logging.WARNING): + result = processor._get_network_config(2020) - captured = capsys.readouterr() - assert "WARNING: Could not load config file" in captured.out + assert any( + "Could not load config file" in record.message for record in caplog.records + ) assert result is None def test_warns_and_returns_none_when_no_config_file_found( - self, tmp_path: Path, capsys + self, tmp_path: Path, caplog ): """Test that no matching config file triggers a WARNING and returns None.""" processor = self._setup_processor(tmp_path) - result = processor._get_network_config(2020) + with caplog.at_level(logging.WARNING): + result = processor._get_network_config(2020) - captured = capsys.readouterr() - assert "WARNING: No config file found" in captured.out + assert any( + "No config file found" in record.message for record in caplog.records + ) assert result is None diff --git a/tests/test_statistics_functions.py b/tests/test_statistics_functions.py index 893e975..27b671a 100644 --- a/tests/test_statistics_functions.py +++ b/tests/test_statistics_functions.py @@ -2,6 +2,8 @@ from __future__ import annotations +import logging + import pandas as pd import pytest @@ -1546,7 +1548,7 @@ def test_raises_without_energy_totals(self, mock_network: MockPyPSANetwork): Final_Energy_by_Sector__Transportation(mock_network) def test_uses_mean_efficiency_and_warns_for_differing_bev_chargers( - self, energy_totals_csv, capsys + self, energy_totals_csv, caplog ): """Multiple distinct BEV charger efficiencies fall back to mean with WARNING.""" network = MockPyPSANetwork( @@ -1559,13 +1561,14 @@ def test_uses_mean_efficiency_and_warns_for_differing_bev_chargers( ) ) - result = Final_Energy_by_Sector__Transportation( - network, energy_totals=energy_totals_csv - ) + with caplog.at_level(logging.WARNING): + result = Final_Energy_by_Sector__Transportation( + network, energy_totals=energy_totals_csv + ) - captured = capsys.readouterr() - assert "WARNING: Network includes different efficiencies for BEV chargers" in ( - captured.out + assert any( + "Network includes different efficiencies for BEV chargers" in record.message + for record in caplog.records ) assert isinstance(result, pd.Series) assert isinstance(result.index, pd.MultiIndex) diff --git a/tests/test_unit_conversion.py b/tests/test_unit_conversion.py index 98430aa..630954f 100644 --- a/tests/test_unit_conversion.py +++ b/tests/test_unit_conversion.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from pathlib import Path from unittest.mock import MagicMock, patch @@ -115,15 +116,18 @@ def test_raises_keyerror_for_unknown_variable(self, processor: Network_Processor processor._get_unit_from_common_definitions("B") def test_warns_and_uses_first_match_for_multiple_definitions( - self, processor: Network_Processor, capsys + self, processor: Network_Processor, caplog ): processor.common_dsd = MagicMock() processor.common_dsd.variable.to_pandas.return_value = pd.DataFrame( {"variable": ["A", "A"], "unit": ["EJ/yr", "TWh/yr"]} ) - assert processor._get_unit_from_common_definitions("A") == "EJ/yr" - captured = capsys.readouterr() - assert "WARNING: Multiple definitions found for variable" in captured.out + with caplog.at_level(logging.WARNING): + assert processor._get_unit_from_common_definitions("A") == "EJ/yr" + assert any( + "Multiple definitions found for variable" in record.message + for record in caplog.records + ) def test_raises_runtime_error_when_common_dsd_not_initialized( self, processor: Network_Processor From 0541a84e18ebba3f5bec9b0f69b92deb57dc35a1 Mon Sep 17 00:00:00 2001 From: Max Nutz Date: Fri, 31 Jul 2026 12:01:27 +0200 Subject: [PATCH 3/6] update documentation with new logging structure --- README.md | 2 +- docs/contributing.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c8f047b..39fa2dc 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,7 @@ At runtime, `Network_Processor` reads this mapping, looks up the function for ea To register a new variable, please first open a new Issue and select Issue Template "New Variable Statistics". In this issue, the following steps are prepared: - Create a new branch linked to the respective issue - Write your pypsa-statistics and add it as a separate function to [statistics_functions.py](https://github.com/maxnutz/pypsa_validation_processing/blob/main/pypsa_validation_processing/statistics_functions.py) (please note the [naming and structural conventions](https://github.com/maxnutz/pypsa_validation_processing/tree/main#variables-statistics---functions)!) -- add a comprehensive docstring to your function +- add a comprehensive docstring to your function and implement loggings - add the mapping variable_name <> function_name to [mapping.default.yaml](https://github.com/maxnutz/pypsa_validation_processing/blob/main/pypsa_validation_processing/configs/mapping.default.yaml) (and your personal mapping-file) - Add a testing routine for your Function to `tests/` - stick to the [testing-README](https://github.com/maxnutz/pypsa_validation_processing/blob/main/tests/README.md) - make sure that the newest version of main is merged into your feature branch diff --git a/docs/contributing.md b/docs/contributing.md index 1efc882..a7649f0 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -56,6 +56,12 @@ Additional optional parameters can be added when a variable needs them, e.g. `co `configs/mapping.default.yaml` maps each IAMC variable name to its corresponding function name in `statistics_functions.py`. At runtime, `Network_Processor` looks up the function for each defined variable via this mapping. Variables without a mapping entry are silently skipped. +## Logging + +- Use the standard library `logging` module. +- Each module gets its own `logger = logging.getLogger(__name__)` near the top of the file. +- `pypsa_validation_processing/workflow.py::main()` configures logging once via `utils.setup_logging()`; statistics functions and helpers must not call it themselves. + ## Testing - Tests live only in `tests/` — see the [testing README](https://github.com/maxnutz/pypsa_validation_processing/blob/main/tests/README.md). From 18999364ec5d739801b7004be434ee73086c2faf Mon Sep 17 00:00:00 2001 From: max_nutz Date: Mon, 3 Aug 2026 08:40:05 +0200 Subject: [PATCH 4/6] Apply sourcery suggestion to README.md Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 39fa2dc..2b42fa7 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,7 @@ At runtime, `Network_Processor` reads this mapping, looks up the function for ea To register a new variable, please first open a new Issue and select Issue Template "New Variable Statistics". In this issue, the following steps are prepared: - Create a new branch linked to the respective issue - Write your pypsa-statistics and add it as a separate function to [statistics_functions.py](https://github.com/maxnutz/pypsa_validation_processing/blob/main/pypsa_validation_processing/statistics_functions.py) (please note the [naming and structural conventions](https://github.com/maxnutz/pypsa_validation_processing/tree/main#variables-statistics---functions)!) -- add a comprehensive docstring to your function and implement loggings +- add a comprehensive docstring to your function and add appropriate logging - add the mapping variable_name <> function_name to [mapping.default.yaml](https://github.com/maxnutz/pypsa_validation_processing/blob/main/pypsa_validation_processing/configs/mapping.default.yaml) (and your personal mapping-file) - Add a testing routine for your Function to `tests/` - stick to the [testing-README](https://github.com/maxnutz/pypsa_validation_processing/blob/main/tests/README.md) - make sure that the newest version of main is merged into your feature branch From 0f72a1a054002137ea264889f6899436d72f09a3 Mon Sep 17 00:00:00 2001 From: Max Nutz Date: Mon, 3 Aug 2026 09:05:52 +0200 Subject: [PATCH 5/6] adapt log-level defaults --- pypsa_validation_processing/workflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pypsa_validation_processing/workflow.py b/pypsa_validation_processing/workflow.py index cddc146..7a981d4 100644 --- a/pypsa_validation_processing/workflow.py +++ b/pypsa_validation_processing/workflow.py @@ -53,7 +53,7 @@ def build_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--log-level", - default="DEBUG", + default="WARNING", choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], help="Set the logging level. Defaults to WARNING.", ) @@ -65,7 +65,7 @@ def main() -> None: args = build_parser().parse_args() setup_logging(level=args.log_level) if args.log_level == "DEBUG": - logger.info( + logger.warning( f"Importing pypsa-Network with logger level {args.log_level} prints several network tables." ) config_path = resolve_config_path(args.config) From c764ad9e7ace2771562d856ec9b640452399ea2c Mon Sep 17 00:00:00 2001 From: Max Nutz Date: Mon, 3 Aug 2026 09:08:22 +0200 Subject: [PATCH 6/6] enhace and adapt tests for log-level CLI and defaults --- tests/test_format_timestamps.py | 5 ++ tests/test_workflow.py | 98 +++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/tests/test_format_timestamps.py b/tests/test_format_timestamps.py index 5d3cf4b..816abe8 100644 --- a/tests/test_format_timestamps.py +++ b/tests/test_format_timestamps.py @@ -113,6 +113,11 @@ def tz_localize(self, _tz): and "format_timestamps: failed to localize column" in record.message for record in caplog.records ) + assert any( + record.levelno == logging.WARNING + and "columns set to NaT" in record.message + for record in caplog.records + ) def test_format_timestamps_falls_back_when_mixed_format_raises_typeerror(): diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 5587e60..a0b0929 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from pathlib import Path from unittest.mock import MagicMock, patch import runpy @@ -116,6 +117,31 @@ def test_parser_accepts_config_path(self): args = parser.parse_args(["--config", "/path/to/config.yaml"]) assert args.config == "/path/to/config.yaml" + def test_parser_has_log_level_argument(self): + """Test that parser has --log-level argument.""" + parser = build_parser() + args = parser.parse_args(["--log-level", "INFO"]) + assert args.log_level == "INFO" + + def test_parser_log_level_defaults_to_warning(self): + """Test that --log-level defaults to WARNING.""" + parser = build_parser() + args = parser.parse_args([]) + assert args.log_level == "WARNING" + + def test_parser_log_level_accepts_all_documented_choices(self): + """Test that --log-level accepts each of its documented choices.""" + parser = build_parser() + for level in ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]: + args = parser.parse_args(["--log-level", level]) + assert args.log_level == level + + def test_parser_log_level_rejects_invalid_choice(self): + """Test that an unsupported --log-level value raises an error.""" + parser = build_parser() + with pytest.raises(SystemExit): + parser.parse_args(["--log-level", "TRACE"]) + # --------------------------------------------------------------------------- # Tests for main workflow @@ -151,6 +177,78 @@ def test_main_execution(self, mock_processor_class): # Expected if config doesn't exist pass + @patch("pypsa_validation_processing.workflow.Network_Processor") + def test_main_passes_log_level_to_setup_logging( + self, mock_processor_class, tmp_path: Path + ): + """Test that main() configures logging with the parsed --log-level.""" + config_file = tmp_path / "config.yaml" + config_file.write_text("dummy: config\n") + mock_processor_class.return_value = MagicMock() + + from pypsa_validation_processing.workflow import main + + with patch.object( + sys, + "argv", + ["workflow.py", "--config", str(config_file), "--log-level", "ERROR"], + ): + with patch( + "pypsa_validation_processing.workflow.setup_logging" + ) as mock_setup_logging: + main() + + mock_setup_logging.assert_called_once_with(level="ERROR") + + @patch("pypsa_validation_processing.workflow.Network_Processor") + def test_main_logs_warning_when_log_level_is_debug( + self, mock_processor_class, tmp_path: Path, caplog + ): + """Test that main() logs a WARNING hint when --log-level is DEBUG.""" + config_file = tmp_path / "config.yaml" + config_file.write_text("dummy: config\n") + mock_processor_class.return_value = MagicMock() + + from pypsa_validation_processing.workflow import main + + with patch.object( + sys, + "argv", + ["workflow.py", "--config", str(config_file), "--log-level", "DEBUG"], + ): + with caplog.at_level(logging.WARNING): + main() + + assert any( + record.levelno == logging.WARNING + and "Importing pypsa-Network" in record.message + for record in caplog.records + ) + + @pytest.mark.parametrize("log_level", ["INFO", "WARNING", "ERROR", "CRITICAL"]) + @patch("pypsa_validation_processing.workflow.Network_Processor") + def test_main_skips_warning_hint_for_non_debug_log_level( + self, mock_processor_class, log_level: str, tmp_path: Path, caplog + ): + """Test that main() does not log the WARNING-specific hint for other levels.""" + config_file = tmp_path / "config.yaml" + config_file.write_text("dummy: config\n") + mock_processor_class.return_value = MagicMock() + + from pypsa_validation_processing.workflow import main + + with patch.object( + sys, + "argv", + ["workflow.py", "--config", str(config_file), "--log-level", log_level], + ): + with caplog.at_level(logging.DEBUG): + main() + + assert not any( + "Importing pypsa-Network" in record.message for record in caplog.records + ) + # --------------------------------------------------------------------------- # Tests for the __main__ execution guard