diff --git a/CHANGES.txt b/CHANGES.txt index 4ffa49c..2b22bfd 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,7 +1,9 @@ Next release ============ -- ... +Features +-------- +- New base.context.dump task (thanks @vlap) ScriptEngine 1.2.0 diff --git a/docs/sphinx/base-tasks.rst b/docs/sphinx/base-tasks.rst index f2ca669..c40896d 100644 --- a/docs/sphinx/base-tasks.rst +++ b/docs/sphinx/base-tasks.rst @@ -8,10 +8,10 @@ with just the base task package. The base task package contains the following tasks, described in more detail below:: - base.echo, base.chdir, base.command, base.context, base.context.from, - base.copy, base.exit, base.find, base.getenv, base.include, base.link, - base.make_dir, base.move, base.remove, base.setenv, base.task_timer, - base.template, base.time, base.unsetenv + base.echo, base.chdir, base.command, base.context, base.context.load, + base.context.dump, base.copy, base.exit, base.find, base.getenv, + base.include, base.link, base.make_dir, base.move, base.remove, + base.setenv, base.task_timer, base.template, base.time, base.unsetenv ``base.echo`` @@ -162,6 +162,47 @@ possibly nested, dictionary (i.e. single values or lists are not allowed). This task has been renamed to ``base.context.load`` (the old name was ``base.context.from``). +``base.context.dump`` +^^^^^^^^^^^^^^^^^^^^^ +Dumps the current ScriptEngine context data, or a subset of context keys, to a YAML file:: + + base.context.dump: + file: # required + keys: # optional + root: # optional + +This task exports the evaluated runtime state of the context. Note that it dumps the context data +dictionary, not the ScriptEngine script definitions or execution structure. Dynamic constructs +(like ``base.include`` or task logic) and internal engine parameters (the ``se`` namespace) are not +part of the output. + +By default (if ``keys`` is not specified), ``base.context.dump`` dumps the full context data to the +given ``file``:: + + - base.context.dump: + file: all_context.yml + +To dump only specific context keys, use the ``keys`` argument:: + + - base.context.dump: + file: experiment-config.yml + keys: + - experiment + - model_config + +The ``root`` argument wraps the dumped dictionary under an optional top-level key:: + + - base.context.dump: + file: experiment-config.yml + root: "ic_meta" + keys: + - experiment + - model_config + +.. versionadded:: 1.3 + Add ``base.context.dump`` task. + + Control flow ------------ @@ -331,7 +372,7 @@ will set the environment variables ``$LD_LIBRARY_PATH`` to Allow dotted keys for nested context parameters. ``base.unsetenv`` -^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^ Unsets one or more environment variables:: - base.unsetenv: diff --git a/pyproject.toml b/pyproject.toml index e14124d..f3e62f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ se = "scriptengine.cli.se:main" "base.command" = "scriptengine.tasks.base.command:Command" "base.context" = "scriptengine.tasks.base.context:Context" "base.context.load" = "scriptengine.tasks.base.context:ContextLoad" +"base.context.dump" = "scriptengine.tasks.base.context:ContextDump" "base.copy" = "scriptengine.tasks.base.file.copy:Copy" "base.echo" = "scriptengine.tasks.base.echo:Echo" "base.exit" = "scriptengine.tasks.base.exit:Exit" diff --git a/src/scriptengine/tasks/base/context.py b/src/scriptengine/tasks/base/context.py index 597aa26..53b8447 100644 --- a/src/scriptengine/tasks/base/context.py +++ b/src/scriptengine/tasks/base/context.py @@ -70,7 +70,7 @@ def run(self, context): elif file_arg: self.log_info(f"Load context update from file: {file_arg}") try: - with open(file_arg) as f: + with open(str(file_arg)) as f: dict_from_file = yaml.load(f, Loader=yaml.SafeLoader) except (FileNotFoundError, PermissionError, IsADirectoryError) as e: self.log_error(e) @@ -93,3 +93,52 @@ def run(self, context): raise ScriptEngineTaskError return SEContext(context_update_d) + + +class ContextDump(Task): + """ + This task dumps the context, or a subset of context keys, to a YAML file. + + Examples: + - base.context.dump: + file: saveic/experiment-config.yml + root: "ic_meta" + keys: + - experiment + - model_config + + - base.context.dump: + file: all_context.yml + """ + + _required_arguments = ("file",) + + @timed_runner + def run(self, context): + file_arg = self.getarg("file", context) + keys_arg = self.getarg("keys", context, default=None) + root_arg = self.getarg("root", context, default=None) + + self.log_info(f"Dump context to file: {file_arg}") + + if keys_arg is not None: + keys_list = [keys_arg] if isinstance(keys_arg, str) else keys_arg + if not isinstance(keys_list, list): + self.log_error( + f"The 'keys' argument must be a string or list (was a '{type(keys_arg).__name__}')" + ) + raise ScriptEngineTaskRunError + data = {k: context[k] for k in keys_list if k in context} + else: + # Dump full context excluding internal 'se' namespace + data = {k: v for k, v in context.items() if k != "se"} + + if root_arg is not None: + data = {str(root_arg): data} + + try: + with open(str(file_arg), "w") as f: + yaml.dump(data, f, sort_keys=False) + except (FileNotFoundError, PermissionError, IsADirectoryError, OSError) as e: + self.log_error(e) + raise ScriptEngineTaskRunError diff --git a/src/scriptengine/yaml/parser.py b/src/scriptengine/yaml/parser.py index 73436ed..03a82ac 100644 --- a/src/scriptengine/yaml/parser.py +++ b/src/scriptengine/yaml/parser.py @@ -49,6 +49,19 @@ def rrule_constructor(loader, node): yaml.add_constructor("!rrule", rrule_constructor) +def string_class_representer(tag): + return lambda dumper, node: dumper.represent_scalar(tag, str(node)) + + +yaml.add_representer(NoParseString, string_class_representer("!noparse")) +yaml.add_representer(NoParseYamlString, string_class_representer("!noparse_yaml")) +yaml.add_representer(NoParseJinjaString, string_class_representer("!noparse_jinja")) +yaml.add_representer( + dateutil.rrule.rrule, + lambda dumper, node: dumper.represent_scalar("!rrule", str(node), style="|"), +) + + def parse(data): """Recursively parses data and returns a ScriptEngine Task or Job, or a list of those. The data is supposed to come from YAML-parsing a ScriptEngine script.""" diff --git a/tests/tasks/base/test_context.py b/tests/tasks/base/test_context.py index 8bae6d8..0d6c00f 100644 --- a/tests/tasks/base/test_context.py +++ b/tests/tasks/base/test_context.py @@ -212,3 +212,126 @@ def test_context_load_file_not_dict(tmp_path): ) with pytest.raises(ScriptEngineTaskRunError): t.run(SEContext()) + + +def test_context_dump_file(tmp_path): + f = tmp_path / "f.yml" + t1 = from_yaml( + """ + base.context: + foo: 1 + bar: 2 + """ + ) + t2 = from_yaml( + f""" + base.context.dump: + file: {f} + """ + ) + ctx = SEContext() + ctx += t1.run(ctx) + t2.run(ctx) + assert f.exists() + assert yaml.safe_load(f.read_text()) == {"foo": 1, "bar": 2} + + +def test_context_dump_keys(tmp_path): + f = tmp_path / "f.yml" + t1 = from_yaml( + """ + base.context: + foo: 1 + bar: 2 + baz: 3 + """ + ) + t2 = from_yaml( + f""" + base.context.dump: + file: {f} + keys: + - foo + - bar + """ + ) + ctx = SEContext() + ctx += t1.run(ctx) + t2.run(ctx) + assert f.exists() + assert yaml.safe_load(f.read_text()) == {"foo": 1, "bar": 2} + + +def test_context_dump_root(tmp_path): + f = tmp_path / "f.yml" + t1 = from_yaml( + """ + base.context: + foo: 1 + bar: 2 + """ + ) + t2 = from_yaml( + f""" + base.context.dump: + file: {f} + root: ic_meta + keys: + - foo + """ + ) + ctx = SEContext() + ctx += t1.run(ctx) + t2.run(ctx) + assert f.exists() + assert yaml.safe_load(f.read_text()) == {"ic_meta": {"foo": 1}} + + +def test_context_dump_no_args(): + t = from_yaml( + """ + base.context.dump: + """ + ) + with pytest.raises(ScriptEngineTaskError): + t.run(SEContext()) + + +def test_context_dump_keys_string(tmp_path): + f = tmp_path / "f.yml" + t1 = from_yaml( + """ + base.context: + foo: 1 + bar: 2 + """ + ) + t2 = from_yaml( + f""" + base.context.dump: + file: {f} + keys: foo + """ + ) + ctx = SEContext() + ctx += t1.run(ctx) + t2.run(ctx) + assert f.exists() + assert yaml.safe_load(f.read_text()) == {"foo": 1} + + +def test_context_dump_keys_invalid_type(tmp_path): + f = tmp_path / "f.yml" + t = from_yaml( + f""" + base.context.dump: + file: {f} + keys: + a: 1 + """ + ) + with pytest.raises(ScriptEngineTaskRunError): + t.run(SEContext()) + + +