94 replace print with logging

README.md

@@ -186,7 +186,7 @@ At runtime, `Network_Processor` reads this mapping, looks up the function for ea
 To register a new variable, please first open a new Issue and select Issue Template "New Variable Statistics". In this issue, the following steps are prepared: 
 - Create a new branch linked to the respective issue
 - Write your pypsa-statistics and add it as a separate function to [statistics_functions.py](https://github.com/maxnutz/pypsa_validation_processing/blob/main/pypsa_validation_processing/statistics_functions.py) (please note the [naming and structural conventions](https://github.com/maxnutz/pypsa_validation_processing/tree/main#variables-statistics---functions)!)
-- add a comprehensive docstring to your function 
+- add a comprehensive docstring to your function and add appropriate logging
 - add the mapping variable_name <> function_name to [mapping.default.yaml](https://github.com/maxnutz/pypsa_validation_processing/blob/main/pypsa_validation_processing/configs/mapping.default.yaml) (and your personal mapping-file)
 - Add a testing routine for your Function to `tests/` - stick to the [testing-README](https://github.com/maxnutz/pypsa_validation_processing/blob/main/tests/README.md)
 - make sure that the newest version of main is merged into your feature branch

docs/contributing.md

@@ -56,6 +56,12 @@ Additional optional parameters can be added when a variable needs them, e.g. `co
 
 `configs/mapping.default.yaml` maps each IAMC variable name to its corresponding function name in `statistics_functions.py`. At runtime, `Network_Processor` looks up the function for each defined variable via this mapping. Variables without a mapping entry are silently skipped.
 
+## Logging
+
+- Use the standard library `logging` module.
+- Each module gets its own `logger = logging.getLogger(__name__)` near the top of the file.
+- `pypsa_validation_processing/workflow.py::main()` configures logging once via `utils.setup_logging()`; statistics functions and helpers must not call it themselves.
+
 ## Testing
 
 - Tests live only in `tests/` — see the [testing README](https://github.com/maxnutz/pypsa_validation_processing/blob/main/tests/README.md).

pypsa_validation_processing/class_definitions.py

@@ -3,6 +3,7 @@
 import glob
 import importlib
 import inspect
+import logging
 import os
 from pathlib import Path
 import re
@@ -18,6 +19,8 @@
     UNITS_MAPPING,
 )
 
+logger = logging.getLogger(__name__)
+
 
 def format_timestamps(df: pd.DataFrame) -> pd.DataFrame:
     """Normalize timestamp-like columns to tz-aware objects or
@@ -40,7 +43,7 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame:
     -----
     Columns that cannot be parsed as timestamps are left unchanged. Values
     that can be parsed but cannot be localized are replaced with ``pd.NaT``
-    and reported via ``print`` warnings.
+    and reported via logger warnings.
     Yearly aggregated data is identified by all column labels being
     4-digit year strings only. In this case, the columns are converted to
     integers. For non-aggregated data, columns are converted to Python
@@ -87,9 +90,11 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame:
             try:
                 ts_tz = ts.tz_localize(fixed_tz)
             except (TypeError, ValueError) as exc:
-                print(
-                    f"WARNING: format_timestamps: failed to localize column {col!r}: {exc}. "
-                    "Setting label to pd.NaT"
+                logger.warning(
+                    "format_timestamps: failed to localize column %r: %s. "
+                    "Setting label to pd.NaT",
+                    col,
+                    exc,
                 )
                 ts_tz = pd.NaT
                 nat_list.append(col)
@@ -101,7 +106,7 @@ def format_timestamps(df: pd.DataFrame) -> pd.DataFrame:
         py_datetimes = pd.Index(cols, name=idx_name).to_pydatetime()
         df.columns = pd.Index(py_datetimes, dtype="object", name=idx_name)
     if nat_list:
-        print("WARNING: format_timestamps: columns set to NaT:", nat_list)
+        logger.warning("format_timestamps: columns set to NaT: %s", nat_list)
     return df
 
 
@@ -332,8 +337,10 @@ def _execute_function_for_variable(
         )
         func = getattr(stats_module, func_name, None)
         if func is None:
-            print(
-                f"WARNING: Variable {variable}: Function '{func_name}' not found in statistics_functions.py"
+            logger.warning(
+                "Variable %s: Function '%s' not found in statistics_functions.py",
+                variable,
+                func_name,
             )
             return None
 
@@ -663,7 +670,7 @@ def _get_unit_from_common_definitions(self, variable: str) -> str:
         -----
         If multiple units are defined for one variable (in one entry of this variable!),
         the first one of these variables is taken. If there are more then one definitions for
-        the searched variable, the first entry is taken and a warning is printed.
+        the searched variable, the first entry is taken and a warning is logged.
         """
         if self.common_dsd is None:
             raise RuntimeError("Common definitions are not initialized.")
@@ -681,8 +688,10 @@ def _get_unit_from_common_definitions(self, variable: str) -> str:
         if match.empty:
             raise KeyError(f"Variable '{variable}' not defined in common definitions.")
         if len(match) > 1:
-            print(
-                f"WARNING: Multiple definitions found for variable '{variable}' in common definitions. Take first one: {match.iloc[0]}"
+            logger.warning(
+                "Multiple definitions found for variable '%s' in common definitions. Take first one: %s",
+                variable,
+                match.iloc[0],
             )
 
         target_unit = match.iloc[0][unit_col]
@@ -691,19 +700,20 @@ def _get_unit_from_common_definitions(self, variable: str) -> str:
                 f"Unit information not found for variable '{variable}' in common definitions."
             )
         if ("[" in target_unit) and ("]" in target_unit) and ("," in target_unit):
-            print(
-                "Several possible units defined for variable '{variable}' in common definitions. Take first one:".format(
-                    variable=variable
-                )
+            logger.info(
+                "Several possible units defined for variable '%s' in common definitions. Take first one:",
+                variable,
             )
             try:
                 import ast
 
                 target_unit = ast.literal_eval(target_unit)[0]
-                print(target_unit)
+                logger.info(target_unit)
             except Exception as exc:
-                print(
-                    f"WARNING: Failed to parse multiple units for variable '{variable}': {exc}. Using TJ as unit."
+                logger.warning(
+                    "Failed to parse multiple units for variable '%s': %s. Using TJ as unit.",
+                    variable,
+                    exc,
                 )
                 target_unit = "TJ"
         return str(target_unit)
@@ -725,7 +735,7 @@ def _get_network_config(self, investment_year):
         Notes
         -----
         The method searches for a YAML configuration file in the 'configs' subdirectory of the network. If multiple matching
-        files are found, it uses the first one and prints a warning. If no matching file is found, it prints a warning and returns None.
+        files are found, it uses the first one and logs a warning. If no matching file is found, it logs a warning and returns None.
         """
         config_pattern = str(
             self.network_results_path / "configs" / f"config*{investment_year}.yaml"
@@ -735,21 +745,23 @@ def _get_network_config(self, investment_year):
         if matching_files:
             selected_config_file = matching_files[0]
             if len(matching_files) > 1:
-                print(
-                    f"INFO: Multiple config files found for investment year {investment_year}; "
-                    f"using '{selected_config_file}'"
+                logger.info(
+                    "Multiple config files found for investment year %s; using '%s'",
+                    investment_year,
+                    selected_config_file,
                 )
             try:
                 with open(selected_config_file, "r") as f:
                     network_config = yaml.safe_load(f)
             except Exception as exc:
-                print(
-                    f"WARNING: Could not load config file '{selected_config_file}': {exc}"
+                logger.warning(
+                    "Could not load config file '%s': %s", selected_config_file, exc
                 )
         else:
-            print(
-                f"WARNING: No config file found for investment year {investment_year} "
-                f"at pattern '{config_pattern}'"
+            logger.warning(
+                "No config file found for investment year %s at pattern '%s'",
+                investment_year,
+                config_pattern,
             )
         return network_config
 

pypsa_validation_processing/statistics_functions.py

@@ -25,6 +25,7 @@
 """
 
 from __future__ import annotations
+import logging
 import re
 from functools import reduce
 from pathlib import Path
@@ -43,6 +44,8 @@
     create_location_index_from_copperplate,
 )
 
+logger = logging.getLogger(__name__)
+
 
 def Final_Energy_by_Carrier__Electricity(
     n: pypsa.Network,
@@ -138,6 +141,9 @@ def Final_Energy_by_Carrier__Electricity(
     series_list = [series for series in series_list if not series.empty]
 
     result = pd.concat(series_list)
+    logger.debug(
+        f"Final Energy by Carrier|Electricity finished. Resulting shape: {result.shape}. "
+    )
     return result
 
 
@@ -221,9 +227,12 @@ def Net_Imports__Electricity(
     ].sum()
 
     if isinstance(imports_raw, pd.DataFrame):
-        return net_imports_grouped.unstack("snapshot")
+        result = net_imports_grouped.unstack("snapshot")
+    else:
+        result = net_imports_grouped
 
-    return net_imports_grouped
+    logger.debug(f"Net Imports|Electricity finished. Resulting shape: {result.shape}. ")
+    return result
 
 
 def Final_Energy_by_Carrier__Coal(
@@ -266,6 +275,9 @@ def Final_Energy_by_Carrier__Coal(
         groupby_time=aggregate_per_year,
         **kwargs,
     )
+    logger.debug(
+        f"Final Energy by Carrier|Coal finished. Resulting shape: {industry.shape}. "
+    )
     return industry
 
 
@@ -441,6 +453,9 @@ def Final_Energy_by_Carrier__Oil(
         non_fossil_fraction = non_fossil_fraction.reindex_like(total).fillna(0.0)
 
     fossil_oil = total.mul(1 - non_fossil_fraction, axis=0)
+    logger.debug(
+        f"Final Energy by Carrier|Oil finished. Resulting shape: {fossil_oil.shape}. "
+    )
     return fossil_oil
 
 
@@ -539,8 +554,12 @@ def Net_Imports__Oil(
     ].sum()
 
     if isinstance(imports_raw, pd.DataFrame):
-        return net_imports_grouped.unstack("snapshot")
-    return net_imports_grouped
+        result = net_imports_grouped.unstack("snapshot")
+    else:
+        result = net_imports_grouped
+
+    logger.debug(f"Net Imports|Oil finished. Resulting shape: {result.shape}. ")
+    return result
 
 
 def Final_Energy_by_Carrier__Natural_Gas(
@@ -656,15 +675,22 @@ def Final_Energy_by_Carrier__Natural_Gas(
     series_list = [series for series in series_list if not series.empty]
 
     if not series_list:
-        return pd.Series(
+        result = pd.Series(
             dtype=float,
             index=pd.MultiIndex.from_tuples([], names=kwargs["groupby"]),
         )
+        logger.debug(
+            f"Final Energy by Carrier|Natural Gas finished. Resulting shape: {result.shape}. "
+        )
+        return result
 
     total = pd.concat(series_list)
     total = total.rename(index=UNITS_MAPPING).groupby(kwargs["groupby"]).sum()
     non_fossil_fraction = non_fossil_fraction.reindex_like(total).fillna(0)
     result = total.mul(1 - non_fossil_fraction, axis=0)
+    logger.debug(
+        f"Final Energy by Carrier|Natural Gas finished. Resulting shape: {result.shape}. "
+    )
     return result
 
 
@@ -755,6 +781,9 @@ def Final_Energy_by_Carrier__District_Heat(
     # heat_vent.index = pd.MultiIndex.from_frame(idx_frame_heat_vent)
     result = (res.groupby(["location", "unit"]).sum()).add(heat_vent, fill_value=0)
 
+    logger.debug(
+        f"Final Energy by Carrier|District Heat finished. Resulting shape: {result.shape}. "
+    )
     return result
 
 
@@ -826,8 +855,9 @@ def Final_Energy_by_Sector__Transportation(
         eff = bev_charger_efficiencies.iloc[0]
     else:
         eff = bev_charger_efficiencies.mean()
-        print(
-            "WARNING: Network includes different efficiencies for BEV chargers. Using mean value for variable Final_Energy_by_Sector__Transportation"
+        logger.warning(
+            "Network includes different efficiencies for BEV chargers. "
+            "Using mean value for variable Final_Energy_by_Sector__Transportation"
         )
 
     elec = n.statistics.withdrawal(
@@ -886,6 +916,9 @@ def Final_Energy_by_Sector__Transportation(
     series_list = [series for series in series_list if not series.empty]
 
     total = pd.concat(series_list)
+    logger.debug(
+        f"Final Energy by Sector|Transportation finished. Resulting shape: {total.shape}. "
+    )
     return total
 
 
@@ -969,6 +1002,9 @@ def Final_Energy_by_Sector__Industry(
     eff_loss = abs(cc_in) - abs(cc_out)
     eff_loss = eff_loss.groupby(["location", "unit"]).sum()
     res = load_statistics.add(eff_loss, fill_value=0)
+    logger.debug(
+        f"Final Energy by Sector|Industry finished. Resulting shape: {res.shape}. "
+    )
     return res
 
 
@@ -1047,6 +1083,9 @@ def Final_Energy_by_Sector__Agriculture(
         eff_loss = eff_loss.groupby(["location", "unit"]).sum()
         res = res.add(eff_loss, fill_value=0)
 
+    logger.debug(
+        f"Final Energy by Sector|Agriculture finished. Resulting shape: {res.shape}. "
+    )
     return res
 
 
@@ -1180,4 +1219,8 @@ def Final_Energy_by_Sector__Residential_and_Commercial(
         df.groupby(kwargs["groupby"]).sum() for df in series_list if not df.empty
     ]
     total = pd.concat(series_list)
+    logger.debug(
+        "Final Energy by Sector|Residential and Commercial finished. "
+        f"Resulting shape: {total.shape}. "
+    )
     return total

pypsa_validation_processing/utils.py

@@ -1,5 +1,6 @@
 """Static information and general utility functions for pypsa_validation_processing."""
 
+import logging
 import pandas as pd
 from pathlib import Path
 
@@ -252,3 +253,16 @@ def create_location_index_from_copperplate(
     output = raw_input.copy()
     output.index = new_index
     return output
+
+
+def setup_logging(level: int | str = logging.WARNING) -> None:
+    """Configure root logging once for the package entrypoint.
+
+    Parameters
+    ----------
+    level : int or str, optional
+        Logging level to configure, by default ``logging.WARNING``.
+    """
+    logging.basicConfig(
+        level=level, format="%(asctime)s %(levelname)s %(name)s: %(message)s"
+    )