From da33f7a7504403931415eada89810fe1c1926d98 Mon Sep 17 00:00:00 2001 From: Vladimir Lapin Date: Fri, 14 Aug 2026 02:32:31 +0200 Subject: [PATCH 1/5] feat(tasks): add base.context.dump task --- docs/sphinx/base-tasks.rst | 57 ++++++++++- pyproject.toml | 1 + src/scriptengine/tasks/base/context.py | 48 +++++++++ src/scriptengine/yaml/parser.py | 17 ++++ tests/tasks/base/test_context.py | 136 +++++++++++++++++++++++++ 5 files changed, 254 insertions(+), 5 deletions(-) diff --git a/docs/sphinx/base-tasks.rst b/docs/sphinx/base-tasks.rst index f2ca669..e52d86a 100644 --- a/docs/sphinx/base-tasks.rst +++ b/docs/sphinx/base-tasks.rst @@ -8,10 +8,10 @@ with just the base task package. The base task package contains the following tasks, described in more detail below:: - base.echo, base.chdir, base.command, base.context, base.context.from, - base.copy, base.exit, base.find, base.getenv, base.include, base.link, - base.make_dir, base.move, base.remove, base.setenv, base.task_timer, - base.template, base.time, base.unsetenv + base.echo, base.chdir, base.command, base.context, base.context.load, + base.context.dump, base.copy, base.exit, base.find, base.getenv, + base.include, base.link, base.make_dir, base.move, base.remove, + base.setenv, base.task_timer, base.template, base.time, base.unsetenv ``base.echo`` @@ -162,6 +162,53 @@ possibly nested, dictionary (i.e. single values or lists are not allowed). This task has been renamed to ``base.context.load`` (the old name was ``base.context.from``). +``base.context.dump`` +^^^^^^^^^^^^^^^^^^^^^ +Dumps the ScriptEngine context, or a subset of context keys, to a YAML file:: + + base.context.dump: + file: # required + keys: # optional + root: # optional + +This task is the direct counterpart to ``base.context.load``. It writes context data as pure YAML, +making the resulting file directly readable by ``base.context.load`` in other scripts. + +By default (if ``keys`` is not specified), ``base.context.dump`` dumps the full ScriptEngine +context (excluding the internal ``se`` namespace) to the given ``file``:: + + - base.context.dump: + file: all_context.yml + +To dump only specific context keys, use the ``keys`` argument:: + + - base.context.dump: + file: experiment-config.yml + keys: + - experiment + - model_config + +The ``root`` argument wraps the dumped dictionary under a top-level root key. This is particularly +useful when downstream scripts want to load the metadata into an isolated namespace (e.g. ``ic_meta``) +using standard ``base.context.load`` without modifying active root context:: + + - base.context.dump: + file: experiment-config.yml + root: "ic_meta" + keys: + - experiment + - model_config + +The resulting file ``experiment-config.yml`` can then be loaded back into the context in any +downstream script using ``base.context.load``:: + + - base.context.load: + file: experiment-config.yml + +.. versionadded:: 1.3 + Add ``base.context.dump`` task. + + Control flow ------------ @@ -331,7 +378,7 @@ will set the environment variables ``$LD_LIBRARY_PATH`` to Allow dotted keys for nested context parameters. ``base.unsetenv`` -^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^ Unsets one or more environment variables:: - base.unsetenv: diff --git a/pyproject.toml b/pyproject.toml index e14124d..f3e62f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ se = "scriptengine.cli.se:main" "base.command" = "scriptengine.tasks.base.command:Command" "base.context" = "scriptengine.tasks.base.context:Context" "base.context.load" = "scriptengine.tasks.base.context:ContextLoad" +"base.context.dump" = "scriptengine.tasks.base.context:ContextDump" "base.copy" = "scriptengine.tasks.base.file.copy:Copy" "base.echo" = "scriptengine.tasks.base.echo:Echo" "base.exit" = "scriptengine.tasks.base.exit:Exit" diff --git a/src/scriptengine/tasks/base/context.py b/src/scriptengine/tasks/base/context.py index 597aa26..34b1437 100644 --- a/src/scriptengine/tasks/base/context.py +++ b/src/scriptengine/tasks/base/context.py @@ -93,3 +93,51 @@ def run(self, context): raise ScriptEngineTaskError return SEContext(context_update_d) + + +class ContextDump(Task): + """ + This task dumps the context, or a subset of context keys, to a YAML file. + + Examples: + - base.context.dump: + file: saveic/experiment-config.yml + root: "ic_meta" + keys: + - experiment + - model_config + + - base.context.dump: + file: all_context.yml + """ + + _required_arguments = ("file",) + + @timed_runner + def run(self, context): + file_arg = self.getarg("file", context) + keys_arg = self.getarg("keys", context, default=None) + root_arg = self.getarg("root", context, default=None) + + self.log_info(f"Dump context to file: {file_arg}") + + if keys_arg is not None: + if not isinstance(keys_arg, list): + self.log_error( + f"The 'keys' argument must be a list (was a '{type(keys_arg).__name__}')" + ) + raise ScriptEngineTaskRunError + data = {k: context[k] for k in keys_arg if k in context} + else: + # Dump full context excluding internal 'se' namespace + data = {k: v for k, v in context.items() if k != "se"} + + if root_arg is not None: + data = {str(root_arg): data} + + try: + with open(file_arg, "w") as f: + yaml.dump(data, f, sort_keys=False) + except (FileNotFoundError, PermissionError, IsADirectoryError, OSError) as e: + self.log_error(e) + raise ScriptEngineTaskRunError diff --git a/src/scriptengine/yaml/parser.py b/src/scriptengine/yaml/parser.py index 73436ed..85b88b1 100644 --- a/src/scriptengine/yaml/parser.py +++ b/src/scriptengine/yaml/parser.py @@ -37,6 +37,9 @@ def constructor(loader, node): yaml.add_constructor("!noparse", string_class_constructor(NoParseString)) yaml.add_constructor("!noparse_yaml", string_class_constructor(NoParseYamlString)) yaml.add_constructor("!noparse_jinja", string_class_constructor(NoParseJinjaString)) +yaml.add_constructor("!noparse", string_class_constructor(NoParseString), Loader=yaml.SafeLoader) +yaml.add_constructor("!noparse_yaml", string_class_constructor(NoParseYamlString), Loader=yaml.SafeLoader) +yaml.add_constructor("!noparse_jinja", string_class_constructor(NoParseJinjaString), Loader=yaml.SafeLoader) def rrule_constructor(loader, node): @@ -47,6 +50,20 @@ def rrule_constructor(loader, node): yaml.add_constructor("!rrule", rrule_constructor) +yaml.add_constructor("!rrule", rrule_constructor, Loader=yaml.SafeLoader) + + +def string_class_representer(tag): + return lambda dumper, node: dumper.represent_scalar(tag, str(node)) + + +yaml.add_representer(NoParseString, string_class_representer("!noparse")) +yaml.add_representer(NoParseYamlString, string_class_representer("!noparse_yaml")) +yaml.add_representer(NoParseJinjaString, string_class_representer("!noparse_jinja")) +yaml.add_representer( + dateutil.rrule.rrule, + lambda dumper, node: dumper.represent_scalar("!rrule", str(node), style="|"), +) def parse(data): diff --git a/tests/tasks/base/test_context.py b/tests/tasks/base/test_context.py index 8bae6d8..b28e291 100644 --- a/tests/tasks/base/test_context.py +++ b/tests/tasks/base/test_context.py @@ -212,3 +212,139 @@ def test_context_load_file_not_dict(tmp_path): ) with pytest.raises(ScriptEngineTaskRunError): t.run(SEContext()) + + +def test_context_dump_file(tmp_path): + f = tmp_path / "f.yml" + t1 = from_yaml( + """ + base.context: + foo: 1 + bar: 2 + """ + ) + t2 = from_yaml( + f""" + base.context.dump: + file: {f} + """ + ) + ctx = SEContext() + ctx += t1.run(ctx) + t2.run(ctx) + assert f.exists() + assert yaml.load(f.read_text(), Loader=yaml.SafeLoader) == {"foo": 1, "bar": 2} + + +def test_context_dump_keys(tmp_path): + f = tmp_path / "f.yml" + t1 = from_yaml( + """ + base.context: + foo: 1 + bar: 2 + baz: 3 + """ + ) + t2 = from_yaml( + f""" + base.context.dump: + file: {f} + keys: + - foo + - bar + """ + ) + ctx = SEContext() + ctx += t1.run(ctx) + t2.run(ctx) + assert f.exists() + assert yaml.load(f.read_text(), Loader=yaml.SafeLoader) == {"foo": 1, "bar": 2} + + +def test_context_dump_root_and_load(tmp_path): + f = tmp_path / "f.yml" + t1 = from_yaml( + """ + base.context: + foo: 1 + bar: 2 + """ + ) + t2 = from_yaml( + f""" + base.context.dump: + file: {f} + root: ic_meta + keys: + - foo + """ + ) + t3 = from_yaml( + f""" + base.context.load: + file: {f} + """ + ) + ctx = SEContext() + ctx += t1.run(ctx) + t2.run(ctx) + new_ctx = SEContext({"foo": 99}) + new_ctx += t3.run(new_ctx) + assert new_ctx["foo"] == 99 + assert new_ctx["ic_meta"] == {"foo": 1} + + +def test_context_dump_load_roundtrip(tmp_path): + f = tmp_path / "f.yml" + t1 = from_yaml( + """ + base.context: + foo: 1 + bar: 2 + """ + ) + t2 = from_yaml( + f""" + base.context.dump: + file: {f} + """ + ) + t3 = from_yaml( + f""" + base.context.load: + file: {f} + """ + ) + ctx = SEContext() + ctx += t1.run(ctx) + t2.run(ctx) + new_ctx = SEContext() + new_ctx += t3.run(new_ctx) + assert new_ctx["foo"] == 1 + assert new_ctx["bar"] == 2 + + +def test_context_dump_no_args(): + t = from_yaml( + """ + base.context.dump: + """ + ) + with pytest.raises(ScriptEngineTaskError): + t.run(SEContext()) + + +def test_context_dump_keys_not_a_list(tmp_path): + f = tmp_path / "f.yml" + t = from_yaml( + f""" + base.context.dump: + file: {f} + keys: not_a_list + """ + ) + with pytest.raises(ScriptEngineTaskRunError): + t.run(SEContext()) + + From f688c6ff531e652719712290529aaab91bfc4b0f Mon Sep 17 00:00:00 2001 From: Uwe Fladrich Date: Tue, 18 Aug 2026 10:41:42 +0200 Subject: [PATCH 2/5] Avoid warnings int open() calls --- src/scriptengine/tasks/base/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scriptengine/tasks/base/context.py b/src/scriptengine/tasks/base/context.py index 34b1437..9399532 100644 --- a/src/scriptengine/tasks/base/context.py +++ b/src/scriptengine/tasks/base/context.py @@ -70,7 +70,7 @@ def run(self, context): elif file_arg: self.log_info(f"Load context update from file: {file_arg}") try: - with open(file_arg) as f: + with open(str(file_arg)) as f: dict_from_file = yaml.load(f, Loader=yaml.SafeLoader) except (FileNotFoundError, PermissionError, IsADirectoryError) as e: self.log_error(e) @@ -136,7 +136,7 @@ def run(self, context): data = {str(root_arg): data} try: - with open(file_arg, "w") as f: + with open(str(file_arg), "w") as f: yaml.dump(data, f, sort_keys=False) except (FileNotFoundError, PermissionError, IsADirectoryError, OSError) as e: self.log_error(e) From 3df3a7c305c59c834bb98c79d6d6e6df6e8a7a01 Mon Sep 17 00:00:00 2001 From: Vladimir Lapin Date: Thu, 20 Aug 2026 16:13:54 +0200 Subject: [PATCH 3/5] refactor: remove SafeLoader constructors and clarify base.context.dump docs --- docs/sphinx/base-tasks.rst | 16 +++++++++++----- src/scriptengine/yaml/parser.py | 4 ---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/sphinx/base-tasks.rst b/docs/sphinx/base-tasks.rst index e52d86a..ac770fd 100644 --- a/docs/sphinx/base-tasks.rst +++ b/docs/sphinx/base-tasks.rst @@ -171,8 +171,14 @@ Dumps the ScriptEngine context, or a subset of context keys, to a YAML file:: keys: # optional root: # optional -This task is the direct counterpart to ``base.context.load``. It writes context data as pure YAML, -making the resulting file directly readable by ``base.context.load`` in other scripts. +This task serializes the current in-memory context data state (the evaluated variables and configuration) +to a YAML file. It is designed to export data state rather than reproducing ScriptEngine scripts or task +definitions: + +* **State vs. scripts**: It captures the evaluated data values currently held in the context. Dynamic constructs such as ``base.include``, loops, conditionals, or task structures are not part of the context data and are not exported. +* **Evaluated values**: Jinja expressions already resolved in the context are exported as their final evaluated values. Values marked with ``!noparse`` retain their YAML tags. +* **Internal namespace**: The internal ``se`` namespace (containing engine-level execution paths, loop states, and runtime metadata) is automatically excluded from the dump. +* **Custom tags**: ScriptEngine tags (such as ``!rrule`` and ``!noparse``) are preserved using custom YAML representers. By default (if ``keys`` is not specified), ``base.context.dump`` dumps the full ScriptEngine context (excluding the internal ``se`` namespace) to the given ``file``:: @@ -190,7 +196,7 @@ To dump only specific context keys, use the ``keys`` argument:: The ``root`` argument wraps the dumped dictionary under a top-level root key. This is particularly useful when downstream scripts want to load the metadata into an isolated namespace (e.g. ``ic_meta``) -using standard ``base.context.load`` without modifying active root context:: +using ``base.context.load`` without modifying the active root context:: - base.context.dump: file: experiment-config.yml @@ -199,8 +205,8 @@ using standard ``base.context.load`` without modifying active root context:: - experiment - model_config -The resulting file ``experiment-config.yml`` can then be loaded back into the context in any -downstream script using ``base.context.load``:: +The resulting YAML file can be inspected, used by external model tools and downstream workflows, +or loaded back into the context in another script using ``base.context.load``:: - base.context.load: file: experiment-config.yml diff --git a/src/scriptengine/yaml/parser.py b/src/scriptengine/yaml/parser.py index 85b88b1..03a82ac 100644 --- a/src/scriptengine/yaml/parser.py +++ b/src/scriptengine/yaml/parser.py @@ -37,9 +37,6 @@ def constructor(loader, node): yaml.add_constructor("!noparse", string_class_constructor(NoParseString)) yaml.add_constructor("!noparse_yaml", string_class_constructor(NoParseYamlString)) yaml.add_constructor("!noparse_jinja", string_class_constructor(NoParseJinjaString)) -yaml.add_constructor("!noparse", string_class_constructor(NoParseString), Loader=yaml.SafeLoader) -yaml.add_constructor("!noparse_yaml", string_class_constructor(NoParseYamlString), Loader=yaml.SafeLoader) -yaml.add_constructor("!noparse_jinja", string_class_constructor(NoParseJinjaString), Loader=yaml.SafeLoader) def rrule_constructor(loader, node): @@ -50,7 +47,6 @@ def rrule_constructor(loader, node): yaml.add_constructor("!rrule", rrule_constructor) -yaml.add_constructor("!rrule", rrule_constructor, Loader=yaml.SafeLoader) def string_class_representer(tag): From 3e8e48b22d4f6aea4512dcba79e3d3a4058188f6 Mon Sep 17 00:00:00 2001 From: Vladimir Lapin Date: Thu, 20 Aug 2026 16:18:51 +0200 Subject: [PATCH 4/5] docs: simplify base.context.dump description and tests --- docs/sphinx/base-tasks.rst | 28 ++++++------------- tests/tasks/base/test_context.py | 48 ++++---------------------------- 2 files changed, 13 insertions(+), 63 deletions(-) diff --git a/docs/sphinx/base-tasks.rst b/docs/sphinx/base-tasks.rst index ac770fd..89a9288 100644 --- a/docs/sphinx/base-tasks.rst +++ b/docs/sphinx/base-tasks.rst @@ -164,24 +164,20 @@ possibly nested, dictionary (i.e. single values or lists are not allowed). ``base.context.dump`` ^^^^^^^^^^^^^^^^^^^^^ -Dumps the ScriptEngine context, or a subset of context keys, to a YAML file:: +Dumps the current ScriptEngine context data, or a subset of context keys, to a YAML file:: base.context.dump: file: # required keys: # optional root: # optional -This task serializes the current in-memory context data state (the evaluated variables and configuration) -to a YAML file. It is designed to export data state rather than reproducing ScriptEngine scripts or task -definitions: +This task exports the evaluated runtime state of the context. Note that it dumps the context data +dictionary, not the ScriptEngine script definitions or execution structure. Dynamic constructs +(like ``base.include`` or task logic) and internal engine parameters (the ``se`` namespace) are not +part of the output. -* **State vs. scripts**: It captures the evaluated data values currently held in the context. Dynamic constructs such as ``base.include``, loops, conditionals, or task structures are not part of the context data and are not exported. -* **Evaluated values**: Jinja expressions already resolved in the context are exported as their final evaluated values. Values marked with ``!noparse`` retain their YAML tags. -* **Internal namespace**: The internal ``se`` namespace (containing engine-level execution paths, loop states, and runtime metadata) is automatically excluded from the dump. -* **Custom tags**: ScriptEngine tags (such as ``!rrule`` and ``!noparse``) are preserved using custom YAML representers. - -By default (if ``keys`` is not specified), ``base.context.dump`` dumps the full ScriptEngine -context (excluding the internal ``se`` namespace) to the given ``file``:: +By default (if ``keys`` is not specified), ``base.context.dump`` dumps the full context data to the +given ``file``:: - base.context.dump: file: all_context.yml @@ -194,9 +190,7 @@ To dump only specific context keys, use the ``keys`` argument:: - experiment - model_config -The ``root`` argument wraps the dumped dictionary under a top-level root key. This is particularly -useful when downstream scripts want to load the metadata into an isolated namespace (e.g. ``ic_meta``) -using ``base.context.load`` without modifying the active root context:: +The ``root`` argument wraps the dumped dictionary under an optional top-level key:: - base.context.dump: file: experiment-config.yml @@ -205,12 +199,6 @@ using ``base.context.load`` without modifying the active root context:: - experiment - model_config -The resulting YAML file can be inspected, used by external model tools and downstream workflows, -or loaded back into the context in another script using ``base.context.load``:: - - - base.context.load: - file: experiment-config.yml - .. versionadded:: 1.3 Add ``base.context.dump`` task. diff --git a/tests/tasks/base/test_context.py b/tests/tasks/base/test_context.py index b28e291..172f287 100644 --- a/tests/tasks/base/test_context.py +++ b/tests/tasks/base/test_context.py @@ -233,7 +233,7 @@ def test_context_dump_file(tmp_path): ctx += t1.run(ctx) t2.run(ctx) assert f.exists() - assert yaml.load(f.read_text(), Loader=yaml.SafeLoader) == {"foo": 1, "bar": 2} + assert yaml.safe_load(f.read_text()) == {"foo": 1, "bar": 2} def test_context_dump_keys(tmp_path): @@ -259,10 +259,10 @@ def test_context_dump_keys(tmp_path): ctx += t1.run(ctx) t2.run(ctx) assert f.exists() - assert yaml.load(f.read_text(), Loader=yaml.SafeLoader) == {"foo": 1, "bar": 2} + assert yaml.safe_load(f.read_text()) == {"foo": 1, "bar": 2} -def test_context_dump_root_and_load(tmp_path): +def test_context_dump_root(tmp_path): f = tmp_path / "f.yml" t1 = from_yaml( """ @@ -280,49 +280,11 @@ def test_context_dump_root_and_load(tmp_path): - foo """ ) - t3 = from_yaml( - f""" - base.context.load: - file: {f} - """ - ) - ctx = SEContext() - ctx += t1.run(ctx) - t2.run(ctx) - new_ctx = SEContext({"foo": 99}) - new_ctx += t3.run(new_ctx) - assert new_ctx["foo"] == 99 - assert new_ctx["ic_meta"] == {"foo": 1} - - -def test_context_dump_load_roundtrip(tmp_path): - f = tmp_path / "f.yml" - t1 = from_yaml( - """ - base.context: - foo: 1 - bar: 2 - """ - ) - t2 = from_yaml( - f""" - base.context.dump: - file: {f} - """ - ) - t3 = from_yaml( - f""" - base.context.load: - file: {f} - """ - ) ctx = SEContext() ctx += t1.run(ctx) t2.run(ctx) - new_ctx = SEContext() - new_ctx += t3.run(new_ctx) - assert new_ctx["foo"] == 1 - assert new_ctx["bar"] == 2 + assert f.exists() + assert yaml.safe_load(f.read_text()) == {"ic_meta": {"foo": 1}} def test_context_dump_no_args(): From 963f4b6d26f00b3b2cf0447a658d9a14e15c2080 Mon Sep 17 00:00:00 2001 From: Vladimir Lapin Date: Thu, 20 Aug 2026 16:26:56 +0200 Subject: [PATCH 5/5] feat(tasks): allow string keys in base.context.dump and update CHANGES --- CHANGES.txt | 4 +++- docs/sphinx/base-tasks.rst | 2 +- src/scriptengine/tasks/base/context.py | 7 ++++--- tests/tasks/base/test_context.py | 29 ++++++++++++++++++++++++-- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 4ffa49c..2b22bfd 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,7 +1,9 @@ Next release ============ -- ... +Features +-------- +- New base.context.dump task (thanks @vlap) ScriptEngine 1.2.0 diff --git a/docs/sphinx/base-tasks.rst b/docs/sphinx/base-tasks.rst index 89a9288..c40896d 100644 --- a/docs/sphinx/base-tasks.rst +++ b/docs/sphinx/base-tasks.rst @@ -168,7 +168,7 @@ Dumps the current ScriptEngine context data, or a subset of context keys, to a Y base.context.dump: file: # required - keys: # optional + keys: # optional root: # optional This task exports the evaluated runtime state of the context. Note that it dumps the context data diff --git a/src/scriptengine/tasks/base/context.py b/src/scriptengine/tasks/base/context.py index 9399532..53b8447 100644 --- a/src/scriptengine/tasks/base/context.py +++ b/src/scriptengine/tasks/base/context.py @@ -122,12 +122,13 @@ def run(self, context): self.log_info(f"Dump context to file: {file_arg}") if keys_arg is not None: - if not isinstance(keys_arg, list): + keys_list = [keys_arg] if isinstance(keys_arg, str) else keys_arg + if not isinstance(keys_list, list): self.log_error( - f"The 'keys' argument must be a list (was a '{type(keys_arg).__name__}')" + f"The 'keys' argument must be a string or list (was a '{type(keys_arg).__name__}')" ) raise ScriptEngineTaskRunError - data = {k: context[k] for k in keys_arg if k in context} + data = {k: context[k] for k in keys_list if k in context} else: # Dump full context excluding internal 'se' namespace data = {k: v for k, v in context.items() if k != "se"} diff --git a/tests/tasks/base/test_context.py b/tests/tasks/base/test_context.py index 172f287..0d6c00f 100644 --- a/tests/tasks/base/test_context.py +++ b/tests/tasks/base/test_context.py @@ -297,16 +297,41 @@ def test_context_dump_no_args(): t.run(SEContext()) -def test_context_dump_keys_not_a_list(tmp_path): +def test_context_dump_keys_string(tmp_path): + f = tmp_path / "f.yml" + t1 = from_yaml( + """ + base.context: + foo: 1 + bar: 2 + """ + ) + t2 = from_yaml( + f""" + base.context.dump: + file: {f} + keys: foo + """ + ) + ctx = SEContext() + ctx += t1.run(ctx) + t2.run(ctx) + assert f.exists() + assert yaml.safe_load(f.read_text()) == {"foo": 1} + + +def test_context_dump_keys_invalid_type(tmp_path): f = tmp_path / "f.yml" t = from_yaml( f""" base.context.dump: file: {f} - keys: not_a_list + keys: + a: 1 """ ) with pytest.raises(ScriptEngineTaskRunError): t.run(SEContext()) +