diff --git a/README.md b/README.md index 2b42fa7..3ba7a87 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ The file `config.default.yaml` provides a guideline for the two config sections ```yaml # General section country: AT # ISO 3166-1 alpha-2 country code (e.g. AT) or "all" -definitions_path: sister_packages/energy-scenarios-at-workflow/definitions # path to the IAMC variable definitions folder +definitions_path: sister_packages/energy-scenarios-at-workflow/definitions # path to the IAMC variable definitions folder; set to False to evaluate all mapping-file variables and skip unit conversion convert_units: true # convert output units to units from definitions_path # mapping_path: # optional: path to mapping YAML; defaults to configs/mapping.default.yaml output_path: resources # path the outputfile should be written to diff --git a/pypsa_validation_processing/class_definitions.py b/pypsa_validation_processing/class_definitions.py index 06c3632..c6ff586 100644 --- a/pypsa_validation_processing/class_definitions.py +++ b/pypsa_validation_processing/class_definitions.py @@ -120,6 +120,12 @@ class Network_Processor: Outputs are converted to the units of common definitions, set in the definitions variable in ``definitions_path`` via :meth:`pyam.IamDataFrame.convert_unit` if ``convert_units`` is ``True`` in config. + + If ``definitions_path`` is set to ``False`` (as a YAML bool or the string + ``"False"``/``"false"``), definitions are disabled entirely: every + variable in the mapping file (``mapping_path``) is evaluated regardless + of whether it has a definitions-folder entry, and unit conversion is + skipped. """ def __init__( @@ -141,10 +147,16 @@ def __init__( settings such as mapping, aggregation, output, and unit conversion behavior. + ``definitions_path`` may also be set to ``False`` (YAML bool, or the + string ``"False"``/``"false"``) to disable definitions entirely: all + variables from the mapping file are evaluated and unit conversion is + skipped. + Raises ------ ValueError - If required configuration entries are missing or invalid. + If required configuration entries are missing or invalid, or if + ``definitions_path`` is unset/empty. FileNotFoundError If the configured network results or definitions directories do not exist. @@ -165,20 +177,34 @@ def __init__( ) definitions_path = self.config.get("definitions_path", None) - if definitions_path is None: + is_empty = definitions_path is None or ( + isinstance(definitions_path, str) and definitions_path.strip() == "" + ) + if is_empty: raise ValueError( - f"'definition_path' not set in config at {self.config_path}" - ) - self.definitions_path: Path = Path(definitions_path) - if not self.definitions_path.exists(): - raise FileNotFoundError( - f"Definition folder does not exist: {self.definitions_path}" + f"'definitions_path' not set in config at {self.config_path}. " + "Set it to a valid definitions folder path, or to False to " + "evaluate all variables from the mapping file without unit " + "conversion." ) - if self.config.get("convert_units", True): + self.use_definitions: bool = not self._is_definitions_disabled(definitions_path) + + if self.use_definitions: + self.definitions_path: Path | None = Path(definitions_path) + if not self.definitions_path.exists(): + raise FileNotFoundError( + f"Definition folder does not exist: {self.definitions_path}" + ) + else: + self.definitions_path = None + + if self.use_definitions and self.config.get("convert_units", True): self.common_dsd: nomenclature.DataStructureDefinition | None = ( nomenclature.DataStructureDefinition(self.definitions_path) ) + else: + self.common_dsd = None default_mappings_path = ( Path(__file__).resolve().parent / "configs" / "mapping.default.yaml" @@ -206,7 +232,9 @@ def __init__( ) self.country_path_token: str = self._sanitize_path_token(self.country) self.network_collection = self._read_pypsa_network_collection() - self.dsd: nomenclature.DataStructureDefinition = self.read_definitions() + self.dsd: nomenclature.DataStructureDefinition | None = ( + self.read_definitions() if self.use_definitions else None + ) self.functions_dict: dict[str, str | list] = self._read_mappings() self.aggregation_level: str = self.config.get("aggregation_level", "country") if self.aggregation_level not in ["country", "region"]: @@ -256,7 +284,7 @@ def __repr__(self) -> str: f" country: {self.country}\n" f" aggregation_level: {self.aggregation_level}\n" f" network_results_path: {self.network_results_path}\n" - f" definitions_path: {self.definitions_path}\n" + f" definitions_path: {self.definitions_path if self.use_definitions else 'False (all variables, no unit conversion)'}\n" ) @staticmethod @@ -268,6 +296,15 @@ def _is_valid_country_identifier(self, country: str) -> bool: """Check if country is a valid ISO code or the special value 'all'.""" return country == "all" or country in EU27_COUNTRY_CODES + @staticmethod + def _is_definitions_disabled(value: object) -> bool: + """Check whether a definitions_path config value requests disabling definitions.""" + if isinstance(value, bool): + return value is False + if isinstance(value, str): + return value.strip().lower() == "false" + return False + def _read_config(self) -> dict: """Read and return the YAML configuration file.""" with open(self.config_path, "r") as f: @@ -768,9 +805,10 @@ def _get_network_config(self, investment_year): def calculate_variables_values(self) -> None: """Calculate values for all defined variables. - Iterates over all variables in ``self.dsd``, calls - :meth:`_execute_function_for_variable` for each one, and assembles - the results. + Iterates over all variables in ``self.dsd`` (or, when + ``self.use_definitions`` is ``False``, over every variable in + ``self.functions_dict``), calls :meth:`_execute_function_for_variable` + for each one, and assembles the results. When ``self.aggregate_per_year`` is ``True`` (default), assembles a single :class:`pyam.IamDataFrame` with one column per investment year @@ -798,8 +836,13 @@ def calculate_variables_values(self) -> None: investment_year = n.meta["wildcards"]["planning_horizons"] network_config = self._get_network_config(investment_year) + variables = ( + self.dsd.variable.to_pandas()["variable"] + if self.use_definitions + else list(self.functions_dict.keys()) + ) results = [] - for variable in self.dsd.variable.to_pandas()["variable"]: + for variable in variables: result = self._execute_function_for_variable( variable, n, config=network_config ) diff --git a/pypsa_validation_processing/workflow.py b/pypsa_validation_processing/workflow.py index 7a981d4..936cc36 100644 --- a/pypsa_validation_processing/workflow.py +++ b/pypsa_validation_processing/workflow.py @@ -71,7 +71,8 @@ def main() -> None: config_path = resolve_config_path(args.config) processor = Network_Processor(config_path=config_path) - processor.read_definitions() + if processor.use_definitions: + processor.read_definitions() processor.calculate_variables_values() processor.write_output_to_xlsx() diff --git a/tests/test_network_processor.py b/tests/test_network_processor.py index 4aac307..d3dc17c 100644 --- a/tests/test_network_processor.py +++ b/tests/test_network_processor.py @@ -149,9 +149,73 @@ def test_init_missing_definitions_path_key_raises_value_error( config_file = tmp_path / "config.yaml" config_file.write_text(config_content) - with pytest.raises(ValueError, match="definition_path"): + with pytest.raises(ValueError, match="definitions_path"): Network_Processor(config_path=config_file) + def test_init_empty_definitions_path_raises_value_error( + self, tmp_path: Path, mock_network_results_path: Path + ): + """Test that an empty-string definitions_path raises ValueError mentioning both options.""" + config_content = f""" +country: AT +model_name: AT_KN2040 +scenario_name: test_scenario +definitions_path: "" +network_results_path: {mock_network_results_path} +""" + config_file = tmp_path / "config.yaml" + config_file.write_text(config_content) + + with pytest.raises(ValueError, match="definitions_path") as exc_info: + Network_Processor(config_path=config_file) + assert "False" in str(exc_info.value) + + def test_init_definitions_path_false_bool_disables_definitions( + self, tmp_path: Path, mock_network_results_path: Path + ): + """Test that definitions_path: False (YAML bool) disables definitions.""" + config_content = f""" +country: AT +model_name: AT_KN2040 +scenario_name: test_scenario +definitions_path: False +network_results_path: {mock_network_results_path} +""" + config_file = tmp_path / "config.yaml" + config_file.write_text(config_content) + + with patch( + "pypsa_validation_processing.class_definitions.pypsa.NetworkCollection" + ): + processor = Network_Processor(config_path=config_file) + assert processor.use_definitions is False + assert processor.dsd is None + assert processor.common_dsd is None + assert processor.definitions_path is None + + def test_init_definitions_path_false_string_disables_definitions( + self, tmp_path: Path, mock_network_results_path: Path + ): + """Test that definitions_path: "False" (quoted string) disables definitions.""" + config_content = f""" +country: AT +model_name: AT_KN2040 +scenario_name: test_scenario +definitions_path: "False" +network_results_path: {mock_network_results_path} +""" + config_file = tmp_path / "config.yaml" + config_file.write_text(config_content) + + with patch( + "pypsa_validation_processing.class_definitions.pypsa.NetworkCollection" + ): + processor = Network_Processor(config_path=config_file) + assert processor.use_definitions is False + assert processor.dsd is None + assert processor.common_dsd is None + assert processor.definitions_path is None + def test_init_nonexistent_definitions_path_raises( self, tmp_path: Path, mock_network_results_path: Path ): @@ -1734,3 +1798,38 @@ def fake_execute(variable, n, config=None): assert "2020" in result.columns assert "2030" in result.columns assert len(result) == 1 + + def test_iterates_mapping_variables_when_definitions_disabled(self, tmp_path: Path): + """With use_definitions=False, iterate over functions_dict keys, not self.dsd.""" + processor = self._setup_processor(tmp_path) + processor.aggregate_per_year = True + processor.aggregation_level = "region" + processor.use_definitions = False + processor.dsd = None + processor.functions_dict = { + "Variable One": "calc_one", + "Variable Two": "calc_two", + } + + network = MockPyPSANetwork() + processor.network_collection = MockNetworkCollection([network]) + + def fake_execute(variable, n, config=None): + return pd.Series( + [1.0], + index=pd.MultiIndex.from_tuples( + [("AT1", "MWh_el")], names=["location", "unit"] + ), + ) + + with patch.object( + processor, "_execute_function_for_variable", side_effect=fake_execute + ) as mock_execute: + with patch.object( + processor, "structure_pyam_from_pandas", side_effect=lambda df: df + ): + processor.calculate_variables_values() + + assert mock_execute.call_count == 2 + called_variables = {call.args[0] for call in mock_execute.call_args_list} + assert called_variables == {"Variable One", "Variable Two"} diff --git a/tests/test_unit_conversion.py b/tests/test_unit_conversion.py index 630954f..c2d2cc8 100644 --- a/tests/test_unit_conversion.py +++ b/tests/test_unit_conversion.py @@ -96,6 +96,44 @@ def test_common_dsd_initialized_correctly(self, tmp_path: Path): assert processor.dsd is definitions_dsd assert mock_dsd.call_count == 2 + def test_convert_units_false_leaves_common_dsd_none(self, tmp_path: Path): + """convert_units: false with a real definitions_path skips common_dsd init.""" + config_path = _make_config(tmp_path, extra="convert_units: false\n") + with patch( + "pypsa_validation_processing.class_definitions.pypsa.NetworkCollection" + ): + with patch( + "pypsa_validation_processing.class_definitions.nomenclature.DataStructureDefinition" + ): + processor = Network_Processor(config_path=config_path) + assert processor.common_dsd is None + + def test_definitions_disabled_overrides_convert_units_true(self, tmp_path: Path): + """definitions_path: False leaves common_dsd None even if convert_units is true.""" + nw_path = tmp_path / "networks" + nw_path.mkdir(parents=True) + (nw_path / "dummy.nc").touch() + config_content = f""" +country: AT +model_name: test_model +scenario_name: test_scenario +definitions_path: False +convert_units: true +network_results_path: {tmp_path} +output_path: {tmp_path / 'output'} +""" + config_path = tmp_path / "config.yaml" + config_path.write_text(config_content) + with patch( + "pypsa_validation_processing.class_definitions.pypsa.NetworkCollection" + ): + with patch( + "pypsa_validation_processing.class_definitions.nomenclature.DataStructureDefinition" + ): + processor = Network_Processor(config_path=config_path) + assert processor.use_definitions is False + assert processor.common_dsd is None + class TestGetUnitFromCommonDefinitions: """Tests for _get_unit_from_common_definitions()."""