98 allow for processing of all available variables

README.md

@@ -35,7 +35,7 @@ The file `config.default.yaml` provides a guideline for the two config sections
 ```yaml
 # General section
 country: AT               # ISO 3166-1 alpha-2 country code (e.g. AT) or "all"
-definitions_path: sister_packages/energy-scenarios-at-workflow/definitions      # path to the IAMC variable definitions folder
+definitions_path: sister_packages/energy-scenarios-at-workflow/definitions      # path to the IAMC variable definitions folder; set to False to evaluate all mapping-file variables and skip unit conversion
 convert_units: true       # convert output units to units from definitions_path
 # mapping_path:        # optional: path to mapping YAML; defaults to configs/mapping.default.yaml
 output_path: resources            # path the outputfile should be written to

pypsa_validation_processing/class_definitions.py

@@ -120,6 +120,12 @@ class Network_Processor:
     Outputs are converted to the units of common definitions, set in the
     definitions variable in ``definitions_path`` via
     :meth:`pyam.IamDataFrame.convert_unit` if ``convert_units`` is ``True`` in config.
+
+    If ``definitions_path`` is set to ``False`` (as a YAML bool or the string
+    ``"False"``/``"false"``), definitions are disabled entirely: every
+    variable in the mapping file (``mapping_path``) is evaluated regardless
+    of whether it has a definitions-folder entry, and unit conversion is
+    skipped.
     """
 
     def __init__(
@@ -141,10 +147,16 @@ def __init__(
         settings such as mapping, aggregation, output, and unit conversion
         behavior.
 
+        ``definitions_path`` may also be set to ``False`` (YAML bool, or the
+        string ``"False"``/``"false"``) to disable definitions entirely: all
+        variables from the mapping file are evaluated and unit conversion is
+        skipped.
+
         Raises
         ------
         ValueError
-            If required configuration entries are missing or invalid.
+            If required configuration entries are missing or invalid, or if
+            ``definitions_path`` is unset/empty.
         FileNotFoundError
             If the configured network results or definitions directories do
             not exist.
@@ -165,20 +177,34 @@ def __init__(
             )
 
         definitions_path = self.config.get("definitions_path", None)
-        if definitions_path is None:
+        is_empty = definitions_path is None or (
+            isinstance(definitions_path, str) and definitions_path.strip() == ""
+        )
+        if is_empty:
             raise ValueError(
-                f"'definition_path' not set in config at {self.config_path}"
-            )
-        self.definitions_path: Path = Path(definitions_path)
-        if not self.definitions_path.exists():
-            raise FileNotFoundError(
-                f"Definition folder does not exist: {self.definitions_path}"
+                f"'definitions_path' not set in config at {self.config_path}. "
+                "Set it to a valid definitions folder path, or to False to "
+                "evaluate all variables from the mapping file without unit "
+                "conversion."
             )
 
-        if self.config.get("convert_units", True):
+        self.use_definitions: bool = not self._is_definitions_disabled(definitions_path)
+
+        if self.use_definitions:
+            self.definitions_path: Path | None = Path(definitions_path)
+            if not self.definitions_path.exists():
+                raise FileNotFoundError(
+                    f"Definition folder does not exist: {self.definitions_path}"
+                )
+        else:
+            self.definitions_path = None
+
+        if self.use_definitions and self.config.get("convert_units", True):
             self.common_dsd: nomenclature.DataStructureDefinition | None = (
                 nomenclature.DataStructureDefinition(self.definitions_path)
             )
+        else:
+            self.common_dsd = None
 
         default_mappings_path = (
             Path(__file__).resolve().parent / "configs" / "mapping.default.yaml"
@@ -206,7 +232,9 @@ def __init__(
         )
         self.country_path_token: str = self._sanitize_path_token(self.country)
         self.network_collection = self._read_pypsa_network_collection()
-        self.dsd: nomenclature.DataStructureDefinition = self.read_definitions()
+        self.dsd: nomenclature.DataStructureDefinition | None = (
+            self.read_definitions() if self.use_definitions else None
+        )
         self.functions_dict: dict[str, str | list] = self._read_mappings()
         self.aggregation_level: str = self.config.get("aggregation_level", "country")
         if self.aggregation_level not in ["country", "region"]:
@@ -256,7 +284,7 @@ def __repr__(self) -> str:
             f"  country: {self.country}\n"
             f"  aggregation_level: {self.aggregation_level}\n"
             f"  network_results_path: {self.network_results_path}\n"
-            f"  definitions_path: {self.definitions_path}\n"
+            f"  definitions_path: {self.definitions_path if self.use_definitions else 'False (all variables, no unit conversion)'}\n"
         )
 
     @staticmethod
@@ -268,6 +296,15 @@ def _is_valid_country_identifier(self, country: str) -> bool:
         """Check if country is a valid ISO code or the special value 'all'."""
         return country == "all" or country in EU27_COUNTRY_CODES
 
+    @staticmethod
+    def _is_definitions_disabled(value: object) -> bool:
+        """Check whether a definitions_path config value requests disabling definitions."""
+        if isinstance(value, bool):
+            return value is False
+        if isinstance(value, str):
+            return value.strip().lower() == "false"
+        return False
+
     def _read_config(self) -> dict:
         """Read and return the YAML configuration file."""
         with open(self.config_path, "r") as f:
@@ -768,9 +805,10 @@ def _get_network_config(self, investment_year):
     def calculate_variables_values(self) -> None:
         """Calculate values for all defined variables.
 
-        Iterates over all variables in ``self.dsd``, calls
-        :meth:`_execute_function_for_variable` for each one, and assembles
-        the results.
+        Iterates over all variables in ``self.dsd`` (or, when
+        ``self.use_definitions`` is ``False``, over every variable in
+        ``self.functions_dict``), calls :meth:`_execute_function_for_variable`
+        for each one, and assembles the results.
 
         When ``self.aggregate_per_year`` is ``True`` (default), assembles a
         single :class:`pyam.IamDataFrame` with one column per investment year
@@ -798,8 +836,13 @@ def calculate_variables_values(self) -> None:
             investment_year = n.meta["wildcards"]["planning_horizons"]
             network_config = self._get_network_config(investment_year)
 
+            variables = (
+                self.dsd.variable.to_pandas()["variable"]
+                if self.use_definitions
+                else list(self.functions_dict.keys())
+            )
             results = []
-            for variable in self.dsd.variable.to_pandas()["variable"]:
+            for variable in variables:
                 result = self._execute_function_for_variable(
                     variable, n, config=network_config
                 )

pypsa_validation_processing/workflow.py

@@ -71,7 +71,8 @@ def main() -> None:
     config_path = resolve_config_path(args.config)
 
     processor = Network_Processor(config_path=config_path)
-    processor.read_definitions()
+    if processor.use_definitions:
+        processor.read_definitions()
     processor.calculate_variables_values()
     processor.write_output_to_xlsx()
 

tests/test_network_processor.py

@@ -149,9 +149,73 @@ def test_init_missing_definitions_path_key_raises_value_error(
         config_file = tmp_path / "config.yaml"
         config_file.write_text(config_content)
 
-        with pytest.raises(ValueError, match="definition_path"):
+        with pytest.raises(ValueError, match="definitions_path"):
             Network_Processor(config_path=config_file)
 
+    def test_init_empty_definitions_path_raises_value_error(
+        self, tmp_path: Path, mock_network_results_path: Path
+    ):
+        """Test that an empty-string definitions_path raises ValueError mentioning both options."""
+        config_content = f"""
+country: AT
+model_name: AT_KN2040
+scenario_name: test_scenario
+definitions_path: ""
+network_results_path: {mock_network_results_path}
+"""
+        config_file = tmp_path / "config.yaml"
+        config_file.write_text(config_content)
+
+        with pytest.raises(ValueError, match="definitions_path") as exc_info:
+            Network_Processor(config_path=config_file)
+        assert "False" in str(exc_info.value)
+
+    def test_init_definitions_path_false_bool_disables_definitions(
+        self, tmp_path: Path, mock_network_results_path: Path
+    ):
+        """Test that definitions_path: False (YAML bool) disables definitions."""
+        config_content = f"""
+country: AT
+model_name: AT_KN2040
+scenario_name: test_scenario
+definitions_path: False
+network_results_path: {mock_network_results_path}
+"""
+        config_file = tmp_path / "config.yaml"
+        config_file.write_text(config_content)
+
+        with patch(
+            "pypsa_validation_processing.class_definitions.pypsa.NetworkCollection"
+        ):
+            processor = Network_Processor(config_path=config_file)
+            assert processor.use_definitions is False
+            assert processor.dsd is None
+            assert processor.common_dsd is None
+            assert processor.definitions_path is None
+
+    def test_init_definitions_path_false_string_disables_definitions(
+        self, tmp_path: Path, mock_network_results_path: Path
+    ):
+        """Test that definitions_path: "False" (quoted string) disables definitions."""
+        config_content = f"""
+country: AT
+model_name: AT_KN2040
+scenario_name: test_scenario
+definitions_path: "False"
+network_results_path: {mock_network_results_path}
+"""
+        config_file = tmp_path / "config.yaml"
+        config_file.write_text(config_content)
+
+        with patch(
+            "pypsa_validation_processing.class_definitions.pypsa.NetworkCollection"
+        ):
+            processor = Network_Processor(config_path=config_file)
+            assert processor.use_definitions is False
+            assert processor.dsd is None
+            assert processor.common_dsd is None
+            assert processor.definitions_path is None
+
     def test_init_nonexistent_definitions_path_raises(
         self, tmp_path: Path, mock_network_results_path: Path
     ):
@@ -1734,3 +1798,38 @@ def fake_execute(variable, n, config=None):
         assert "2020" in result.columns
         assert "2030" in result.columns
         assert len(result) == 1
+
+    def test_iterates_mapping_variables_when_definitions_disabled(self, tmp_path: Path):
+        """With use_definitions=False, iterate over functions_dict keys, not self.dsd."""
+        processor = self._setup_processor(tmp_path)
+        processor.aggregate_per_year = True
+        processor.aggregation_level = "region"
+        processor.use_definitions = False
+        processor.dsd = None
+        processor.functions_dict = {
+            "Variable One": "calc_one",
+            "Variable Two": "calc_two",
+        }
+
+        network = MockPyPSANetwork()
+        processor.network_collection = MockNetworkCollection([network])
+
+        def fake_execute(variable, n, config=None):
+            return pd.Series(
+                [1.0],
+                index=pd.MultiIndex.from_tuples(
+                    [("AT1", "MWh_el")], names=["location", "unit"]
+                ),
+            )
+
+        with patch.object(
+            processor, "_execute_function_for_variable", side_effect=fake_execute
+        ) as mock_execute:
+            with patch.object(
+                processor, "structure_pyam_from_pandas", side_effect=lambda df: df
+            ):
+                processor.calculate_variables_values()
+
+        assert mock_execute.call_count == 2
+        called_variables = {call.args[0] for call in mock_execute.call_args_list}
+        assert called_variables == {"Variable One", "Variable Two"}

tests/test_unit_conversion.py

@@ -96,6 +96,44 @@ def test_common_dsd_initialized_correctly(self, tmp_path: Path):
         assert processor.dsd is definitions_dsd
         assert mock_dsd.call_count == 2
 
+    def test_convert_units_false_leaves_common_dsd_none(self, tmp_path: Path):
+        """convert_units: false with a real definitions_path skips common_dsd init."""
+        config_path = _make_config(tmp_path, extra="convert_units: false\n")
+        with patch(
+            "pypsa_validation_processing.class_definitions.pypsa.NetworkCollection"
+        ):
+            with patch(
+                "pypsa_validation_processing.class_definitions.nomenclature.DataStructureDefinition"
+            ):
+                processor = Network_Processor(config_path=config_path)
+        assert processor.common_dsd is None
+
+    def test_definitions_disabled_overrides_convert_units_true(self, tmp_path: Path):
+        """definitions_path: False leaves common_dsd None even if convert_units is true."""
+        nw_path = tmp_path / "networks"
+        nw_path.mkdir(parents=True)
+        (nw_path / "dummy.nc").touch()
+        config_content = f"""
+country: AT
+model_name: test_model
+scenario_name: test_scenario
+definitions_path: False
+convert_units: true
+network_results_path: {tmp_path}
+output_path: {tmp_path / 'output'}
+"""
+        config_path = tmp_path / "config.yaml"
+        config_path.write_text(config_content)
+        with patch(
+            "pypsa_validation_processing.class_definitions.pypsa.NetworkCollection"
+        ):
+            with patch(
+                "pypsa_validation_processing.class_definitions.nomenclature.DataStructureDefinition"
+            ):
+                processor = Network_Processor(config_path=config_path)
+        assert processor.use_definitions is False
+        assert processor.common_dsd is None
+
 
 class TestGetUnitFromCommonDefinitions:
     """Tests for _get_unit_from_common_definitions()."""