From 5815ac41556de79d45333b087a422cd0ce20e02c Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 16 Apr 2026 13:50:24 -0300 Subject: [PATCH 1/7] #371: Added command to generate build-steps dependency dot file --- exasol/slc/api/__init__.py | 2 + .../slc/api/generate_build_steps_dot_graph.py | 24 ++++++ .../generate_build_steps_dot_graph.py | 78 +++++++++++++++++++ exasol/slc/tool/commands/__init__.py | 2 + .../generate_build_steps_dot_graph.py | 34 ++++++++ ...test_api_generate_build_steps_dot_graph.py | 41 ++++++++++ 6 files changed, 181 insertions(+) create mode 100644 exasol/slc/api/generate_build_steps_dot_graph.py create mode 100644 exasol/slc/internal/generate_build_steps_dot_graph.py create mode 100644 exasol/slc/tool/commands/generate_build_steps_dot_graph.py create mode 100644 test/test_api_generate_build_steps_dot_graph.py diff --git a/exasol/slc/api/__init__.py b/exasol/slc/api/__init__.py index e76ac097..5ce12940 100644 --- a/exasol/slc/api/__init__.py +++ b/exasol/slc/api/__init__.py @@ -3,6 +3,7 @@ from .clean import clean_all_images, clean_flavor_images from .deploy import deploy from .export import export +from .generate_build_steps_dot_graph import generate_build_steps_dot_graph from .generate_language_activation import generate_language_activation from .generate_package_diffs import generate_package_diffs from .push import push @@ -19,6 +20,7 @@ "clean_flavor_images", "deploy", "export", + "generate_build_steps_dot_graph", "generate_language_activation", "push", "push_test_container", diff --git a/exasol/slc/api/generate_build_steps_dot_graph.py b/exasol/slc/api/generate_build_steps_dot_graph.py new file mode 100644 index 00000000..d37a2d88 --- /dev/null +++ b/exasol/slc/api/generate_build_steps_dot_graph.py @@ -0,0 +1,24 @@ +from exasol_integration_test_docker_environment.lib.utils.api_function_decorators import ( + cli_function, +) + +from exasol.slc.internal.generate_build_steps_dot_graph import generate_dot + + +@cli_function +def generate_build_steps_dot_graph( + flavor_path: str, + output_path: str | None = None, +) -> str: + """ + Generate a .dot dependency graph from a flavor's build_steps.py. + + :param flavor_path: Path to the flavor directory. + :param output_path: Optional path where to write the .dot file. + :return: The .dot file content as a string. + :raises FileNotFoundError: if build_steps.py not found in the flavor. + """ + return generate_dot( + flavor_path=flavor_path, + output_path=output_path, + ) diff --git a/exasol/slc/internal/generate_build_steps_dot_graph.py b/exasol/slc/internal/generate_build_steps_dot_graph.py new file mode 100644 index 00000000..0aae822c --- /dev/null +++ b/exasol/slc/internal/generate_build_steps_dot_graph.py @@ -0,0 +1,78 @@ +import importlib +import inspect +import sys +from pathlib import Path + +from exasol.slc.internal.tasks.build.docker_flavor_image_task import ( + DockerFlavorAnalyzeImageTask, +) + + +def _load_build_steps_module(build_steps_path: Path): + spec = importlib.util.spec_from_file_location("build_steps", build_steps_path) + module = importlib.util.module_from_spec(spec) + sys.modules["build_steps"] = module + spec.loader.exec_module(module) + return module + + +def _collect_build_step_classes(module) -> list[type]: + return [ + obj + for obj in vars(module).values() + if inspect.isclass(obj) + and issubclass(obj, DockerFlavorAnalyzeImageTask) + and obj is not DockerFlavorAnalyzeImageTask + ] + + +def _build_dependency_edges( + build_step_classes: list[type], +) -> tuple[list[str], list[tuple[str, str]]]: + nodes: list[str] = [] + edges: list[tuple[str, str]] = [] + for cls in build_step_classes: + step_name = cls.get_build_step(cls) + nodes.append(step_name) + for cls in build_step_classes: + step_name = cls.get_build_step(cls) + requires = cls.requires_tasks(cls) + if requires: + for dep_name in requires: + edges.append((dep_name, step_name)) + return nodes, edges + + +def generate_dot(flavor_path: str, output_path: str | None = None) -> str: + """ + Generate a .dot dependency graph from a flavor's build_steps.py. + + :param flavor_path: Path to the flavor directory. + :param output_path: Optional path where to write the .dot file. + :return: The .dot file content as a string. + """ + flavor_dir = Path(flavor_path) + build_steps_path = flavor_dir / "flavor_base" / "build_steps.py" + if not build_steps_path.exists(): + raise FileNotFoundError( + f"build_steps.py not found at {build_steps_path}" + ) + + module = _load_build_steps_module(build_steps_path) + build_step_classes = _collect_build_step_classes(module) + nodes, edges = _build_dependency_edges(build_step_classes) + + lines = ["strict digraph {"] + for node in nodes: + lines.append(f'"{node}" [label="{node}"];') + for source, target in edges: + lines.append(f'"{source}" -> "{target}";') + lines.append("}") + dot_content = "\n".join(lines) + "\n" + + if output_path is not None: + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + output_file.write_text(dot_content, encoding="utf-8") + + return dot_content diff --git a/exasol/slc/tool/commands/__init__.py b/exasol/slc/tool/commands/__init__.py index e76ac097..5ce12940 100644 --- a/exasol/slc/tool/commands/__init__.py +++ b/exasol/slc/tool/commands/__init__.py @@ -3,6 +3,7 @@ from .clean import clean_all_images, clean_flavor_images from .deploy import deploy from .export import export +from .generate_build_steps_dot_graph import generate_build_steps_dot_graph from .generate_language_activation import generate_language_activation from .generate_package_diffs import generate_package_diffs from .push import push @@ -19,6 +20,7 @@ "clean_flavor_images", "deploy", "export", + "generate_build_steps_dot_graph", "generate_language_activation", "push", "push_test_container", diff --git a/exasol/slc/tool/commands/generate_build_steps_dot_graph.py b/exasol/slc/tool/commands/generate_build_steps_dot_graph.py new file mode 100644 index 00000000..642fe79e --- /dev/null +++ b/exasol/slc/tool/commands/generate_build_steps_dot_graph.py @@ -0,0 +1,34 @@ +import click +from exasol_integration_test_docker_environment.lib.utils.cli_function_decorators import ( + add_options, +) + +from exasol.slc import api +from exasol.slc.tool.cli import cli +from exasol.slc.tool.options.flavor_options import single_flavor_options + + +@cli.command( + short_help="Generates a .dot dependency graph from a flavor's build_steps.py." +) +@add_options(single_flavor_options) +@click.option( + "--output-path", + required=False, + default=None, + help="Path where to write the .dot file. If not specified, prints to stdout.", + type=click.Path(exists=False), +) +def generate_build_steps_dot_graph( + flavor_path: str, + output_path: str | None, +): + """ + Generate a .dot file visualizing the build step dependencies of a flavor. + """ + result = api.generate_build_steps_dot_graph( + flavor_path=flavor_path, + output_path=output_path, + ) + if output_path is None: + print(result) diff --git a/test/test_api_generate_build_steps_dot_graph.py b/test/test_api_generate_build_steps_dot_graph.py new file mode 100644 index 00000000..72eca649 --- /dev/null +++ b/test/test_api_generate_build_steps_dot_graph.py @@ -0,0 +1,41 @@ +import tempfile +import unittest +from pathlib import Path + +import utils as exaslct_utils # type: ignore # pylint: disable=import-error + +from exasol.slc.api import generate_build_steps_dot_graph + + +class GenerateBuildStepsDotGraphTest(unittest.TestCase): + + def test_generate_dot_graph_matches_expected(self): + flavor_path = exaslct_utils.get_test_flavor() + expected_dot_file = ( + flavor_path / "flavor_base" / "test-flavor.dot" + ) + expected_content = expected_dot_file.read_text(encoding="utf-8") + result = generate_build_steps_dot_graph( + flavor_path=str(flavor_path), + ) + self.assertEqual(expected_content, result) + + def test_generate_dot_graph_writes_to_file(self): + flavor_path = exaslct_utils.get_test_flavor() + expected_dot_file = ( + flavor_path / "flavor_base" / "test-flavor.dot" + ) + expected_content = expected_dot_file.read_text(encoding="utf-8") + with tempfile.TemporaryDirectory() as tmp_dir: + output_path = str(Path(tmp_dir) / "output.dot") + result = generate_build_steps_dot_graph( + flavor_path=str(flavor_path), + output_path=output_path, + ) + written_content = Path(output_path).read_text(encoding="utf-8") + self.assertEqual(expected_content, written_content) + self.assertEqual(expected_content, result) + + +if __name__ == "__main__": + unittest.main() From c36419412b5eb763b058da0033d7595d30b1e47b Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 16 Apr 2026 13:51:43 -0300 Subject: [PATCH 2/7] Updated changelog --- doc/changes/unreleased.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 7e595947..fe16e596 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -2,6 +2,10 @@ ## Summary +## Features + + - #371: Added command to generate build-steps dependency dot file + ## Bugs - #367: Fixed gen_package_diff From 94bb64b1726362e1a6689992e0312387de5b30a3 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 16 Apr 2026 14:06:06 -0300 Subject: [PATCH 3/7] Changed default output path --- .../generate_build_steps_dot_graph.py | 10 +++-- .../generate_build_steps_dot_graph.py | 6 +-- ...test_api_generate_build_steps_dot_graph.py | 42 +++++++++++-------- 3 files changed, 32 insertions(+), 26 deletions(-) diff --git a/exasol/slc/internal/generate_build_steps_dot_graph.py b/exasol/slc/internal/generate_build_steps_dot_graph.py index 0aae822c..e87212a2 100644 --- a/exasol/slc/internal/generate_build_steps_dot_graph.py +++ b/exasol/slc/internal/generate_build_steps_dot_graph.py @@ -49,6 +49,7 @@ def generate_dot(flavor_path: str, output_path: str | None = None) -> str: :param flavor_path: Path to the flavor directory. :param output_path: Optional path where to write the .dot file. + Defaults to /build_steps.dot. :return: The .dot file content as a string. """ flavor_dir = Path(flavor_path) @@ -70,9 +71,10 @@ def generate_dot(flavor_path: str, output_path: str | None = None) -> str: lines.append("}") dot_content = "\n".join(lines) + "\n" - if output_path is not None: - output_file = Path(output_path) - output_file.parent.mkdir(parents=True, exist_ok=True) - output_file.write_text(dot_content, encoding="utf-8") + if output_path is None: + output_path = str(flavor_dir / "build_steps.dot") + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + output_file.write_text(dot_content, encoding="utf-8") return dot_content diff --git a/exasol/slc/tool/commands/generate_build_steps_dot_graph.py b/exasol/slc/tool/commands/generate_build_steps_dot_graph.py index 642fe79e..e1a96017 100644 --- a/exasol/slc/tool/commands/generate_build_steps_dot_graph.py +++ b/exasol/slc/tool/commands/generate_build_steps_dot_graph.py @@ -16,7 +16,7 @@ "--output-path", required=False, default=None, - help="Path where to write the .dot file. If not specified, prints to stdout.", + help="Path where to write the .dot file. Defaults to /build_steps.dot.", type=click.Path(exists=False), ) def generate_build_steps_dot_graph( @@ -26,9 +26,7 @@ def generate_build_steps_dot_graph( """ Generate a .dot file visualizing the build step dependencies of a flavor. """ - result = api.generate_build_steps_dot_graph( + api.generate_build_steps_dot_graph( flavor_path=flavor_path, output_path=output_path, ) - if output_path is None: - print(result) diff --git a/test/test_api_generate_build_steps_dot_graph.py b/test/test_api_generate_build_steps_dot_graph.py index 72eca649..c62cfe71 100644 --- a/test/test_api_generate_build_steps_dot_graph.py +++ b/test/test_api_generate_build_steps_dot_graph.py @@ -1,3 +1,4 @@ +import shutil import tempfile import unittest from pathlib import Path @@ -6,30 +7,35 @@ from exasol.slc.api import generate_build_steps_dot_graph +EXPECTED_DOT_FILE = ( + exaslct_utils.DEFAULT_FLAVOR_FLAVORS_ROOT_DIRECTORY + / "test-flavor" + / "build_steps.dot" +) + class GenerateBuildStepsDotGraphTest(unittest.TestCase): def test_generate_dot_graph_matches_expected(self): - flavor_path = exaslct_utils.get_test_flavor() - expected_dot_file = ( - flavor_path / "flavor_base" / "test-flavor.dot" - ) - expected_content = expected_dot_file.read_text(encoding="utf-8") - result = generate_build_steps_dot_graph( - flavor_path=str(flavor_path), - ) - self.assertEqual(expected_content, result) - - def test_generate_dot_graph_writes_to_file(self): - flavor_path = exaslct_utils.get_test_flavor() - expected_dot_file = ( - flavor_path / "flavor_base" / "test-flavor.dot" - ) - expected_content = expected_dot_file.read_text(encoding="utf-8") + expected_content = EXPECTED_DOT_FILE.read_text(encoding="utf-8") + with tempfile.TemporaryDirectory() as tmp_dir: + flavor_copy = Path(tmp_dir) / "test-flavor" + shutil.copytree(exaslct_utils.get_test_flavor(), flavor_copy) + result = generate_build_steps_dot_graph( + flavor_path=str(flavor_copy), + ) + default_output = flavor_copy / "build_steps.dot" + self.assertTrue(default_output.exists()) + written_content = default_output.read_text(encoding="utf-8") + self.assertEqual(expected_content, written_content) + self.assertEqual(expected_content, result) + + def test_generate_dot_graph_writes_to_custom_path(self): + expected_content = EXPECTED_DOT_FILE.read_text(encoding="utf-8") with tempfile.TemporaryDirectory() as tmp_dir: - output_path = str(Path(tmp_dir) / "output.dot") + output_path = str(Path(tmp_dir) / "custom.dot") result = generate_build_steps_dot_graph( - flavor_path=str(flavor_path), + flavor_path=str(exaslct_utils.get_test_flavor()), output_path=output_path, ) written_content = Path(output_path).read_text(encoding="utf-8") From 04e0848d4ea601a655e6b06b161efdca0207e0f7 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 16 Apr 2026 14:11:35 -0300 Subject: [PATCH 4/7] Fixed linter and formatting --- .../generate_build_steps_dot_graph.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/exasol/slc/internal/generate_build_steps_dot_graph.py b/exasol/slc/internal/generate_build_steps_dot_graph.py index e87212a2..67e7721c 100644 --- a/exasol/slc/internal/generate_build_steps_dot_graph.py +++ b/exasol/slc/internal/generate_build_steps_dot_graph.py @@ -1,22 +1,26 @@ -import importlib +import importlib.util import inspect import sys from pathlib import Path +from types import ModuleType from exasol.slc.internal.tasks.build.docker_flavor_image_task import ( DockerFlavorAnalyzeImageTask, ) -def _load_build_steps_module(build_steps_path: Path): +def _load_build_steps_module(build_steps_path: Path) -> ModuleType: spec = importlib.util.spec_from_file_location("build_steps", build_steps_path) + assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) sys.modules["build_steps"] = module spec.loader.exec_module(module) return module -def _collect_build_step_classes(module) -> list[type]: +def _collect_build_step_classes( + module: ModuleType, +) -> list[type[DockerFlavorAnalyzeImageTask]]: return [ obj for obj in vars(module).values() @@ -27,16 +31,16 @@ def _collect_build_step_classes(module) -> list[type]: def _build_dependency_edges( - build_step_classes: list[type], + build_step_classes: list[type[DockerFlavorAnalyzeImageTask]], ) -> tuple[list[str], list[tuple[str, str]]]: nodes: list[str] = [] edges: list[tuple[str, str]] = [] for cls in build_step_classes: - step_name = cls.get_build_step(cls) + step_name = cls.get_build_step(cls) # type: ignore[arg-type] nodes.append(step_name) for cls in build_step_classes: - step_name = cls.get_build_step(cls) - requires = cls.requires_tasks(cls) + step_name = cls.get_build_step(cls) # type: ignore[arg-type] + requires = cls.requires_tasks(cls) # type: ignore[arg-type] if requires: for dep_name in requires: edges.append((dep_name, step_name)) @@ -55,9 +59,7 @@ def generate_dot(flavor_path: str, output_path: str | None = None) -> str: flavor_dir = Path(flavor_path) build_steps_path = flavor_dir / "flavor_base" / "build_steps.py" if not build_steps_path.exists(): - raise FileNotFoundError( - f"build_steps.py not found at {build_steps_path}" - ) + raise FileNotFoundError(f"build_steps.py not found at {build_steps_path}") module = _load_build_steps_module(build_steps_path) build_step_classes = _collect_build_step_classes(module) From 5b7d2a3f6acb6bbda7ad649fe851163ada5d5227 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 16 Apr 2026 14:21:23 -0300 Subject: [PATCH 5/7] Added shape=box and added test dot file --- .../slc/internal/generate_build_steps_dot_graph.py | 2 +- .../flavors/test-flavor/build_steps.dot | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 test/resources/default_flavor/flavors/test-flavor/build_steps.dot diff --git a/exasol/slc/internal/generate_build_steps_dot_graph.py b/exasol/slc/internal/generate_build_steps_dot_graph.py index 67e7721c..b1ac5546 100644 --- a/exasol/slc/internal/generate_build_steps_dot_graph.py +++ b/exasol/slc/internal/generate_build_steps_dot_graph.py @@ -67,7 +67,7 @@ def generate_dot(flavor_path: str, output_path: str | None = None) -> str: lines = ["strict digraph {"] for node in nodes: - lines.append(f'"{node}" [label="{node}"];') + lines.append(f'"{node}" [label="{node}", shape=box];') for source, target in edges: lines.append(f'"{source}" -> "{target}";') lines.append("}") diff --git a/test/resources/default_flavor/flavors/test-flavor/build_steps.dot b/test/resources/default_flavor/flavors/test-flavor/build_steps.dot new file mode 100644 index 00000000..05c8cf56 --- /dev/null +++ b/test/resources/default_flavor/flavors/test-flavor/build_steps.dot @@ -0,0 +1,13 @@ +strict digraph { +"build_run" [label="build_run", shape=box]; +"base_test_build_run" [label="base_test_build_run", shape=box]; +"flavor_customization" [label="flavor_customization", shape=box]; +"flavor_test_build_run" [label="flavor_test_build_run", shape=box]; +"release" [label="release", shape=box]; +"security_scan" [label="security_scan", shape=box]; +"build_run" -> "flavor_customization"; +"flavor_customization" -> "flavor_test_build_run"; +"base_test_build_run" -> "flavor_test_build_run"; +"flavor_customization" -> "release"; +"release" -> "security_scan"; +} From c8b78186ebaf7c3224786c08c60a5d7ed888ac8e Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 17 Apr 2026 13:42:54 -0300 Subject: [PATCH 6/7] Replaced assert with exception --- exasol/slc/internal/generate_build_steps_dot_graph.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/exasol/slc/internal/generate_build_steps_dot_graph.py b/exasol/slc/internal/generate_build_steps_dot_graph.py index b1ac5546..c9d29768 100644 --- a/exasol/slc/internal/generate_build_steps_dot_graph.py +++ b/exasol/slc/internal/generate_build_steps_dot_graph.py @@ -11,7 +11,8 @@ def _load_build_steps_module(build_steps_path: Path) -> ModuleType: spec = importlib.util.spec_from_file_location("build_steps", build_steps_path) - assert spec is not None and spec.loader is not None + if spec is None or spec.loader is None: + raise RuntimeError("Unable to load build_steps module") module = importlib.util.module_from_spec(spec) sys.modules["build_steps"] = module spec.loader.exec_module(module) From 442f15fcea8f6158133d5d0008dc824c7c4fca32 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 24 Apr 2026 13:31:55 -0300 Subject: [PATCH 7/7] Test