diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 7e595947..fe16e596 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -2,6 +2,10 @@ ## Summary +## Features + + - #371: Added command to generate build-steps dependency dot file + ## Bugs - #367: Fixed gen_package_diff diff --git a/exasol/slc/api/__init__.py b/exasol/slc/api/__init__.py index e76ac097..5ce12940 100644 --- a/exasol/slc/api/__init__.py +++ b/exasol/slc/api/__init__.py @@ -3,6 +3,7 @@ from .clean import clean_all_images, clean_flavor_images from .deploy import deploy from .export import export +from .generate_build_steps_dot_graph import generate_build_steps_dot_graph from .generate_language_activation import generate_language_activation from .generate_package_diffs import generate_package_diffs from .push import push @@ -19,6 +20,7 @@ "clean_flavor_images", "deploy", "export", + "generate_build_steps_dot_graph", "generate_language_activation", "push", "push_test_container", diff --git a/exasol/slc/api/generate_build_steps_dot_graph.py b/exasol/slc/api/generate_build_steps_dot_graph.py new file mode 100644 index 00000000..d37a2d88 --- /dev/null +++ b/exasol/slc/api/generate_build_steps_dot_graph.py @@ -0,0 +1,24 @@ +from exasol_integration_test_docker_environment.lib.utils.api_function_decorators import ( + cli_function, +) + +from exasol.slc.internal.generate_build_steps_dot_graph import generate_dot + + +@cli_function +def generate_build_steps_dot_graph( + flavor_path: str, + output_path: str | None = None, +) -> str: + """ + Generate a .dot dependency graph from a flavor's build_steps.py. + + :param flavor_path: Path to the flavor directory. + :param output_path: Optional path where to write the .dot file. + :return: The .dot file content as a string. + :raises FileNotFoundError: if build_steps.py not found in the flavor. + """ + return generate_dot( + flavor_path=flavor_path, + output_path=output_path, + ) diff --git a/exasol/slc/internal/generate_build_steps_dot_graph.py b/exasol/slc/internal/generate_build_steps_dot_graph.py new file mode 100644 index 00000000..c9d29768 --- /dev/null +++ b/exasol/slc/internal/generate_build_steps_dot_graph.py @@ -0,0 +1,83 @@ +import importlib.util +import inspect +import sys +from pathlib import Path +from types import ModuleType + +from exasol.slc.internal.tasks.build.docker_flavor_image_task import ( + DockerFlavorAnalyzeImageTask, +) + + +def _load_build_steps_module(build_steps_path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location("build_steps", build_steps_path) + if spec is None or spec.loader is None: + raise RuntimeError("Unable to load build_steps module") + module = importlib.util.module_from_spec(spec) + sys.modules["build_steps"] = module + spec.loader.exec_module(module) + return module + + +def _collect_build_step_classes( + module: ModuleType, +) -> list[type[DockerFlavorAnalyzeImageTask]]: + return [ + obj + for obj in vars(module).values() + if inspect.isclass(obj) + and issubclass(obj, DockerFlavorAnalyzeImageTask) + and obj is not DockerFlavorAnalyzeImageTask + ] + + +def _build_dependency_edges( + build_step_classes: list[type[DockerFlavorAnalyzeImageTask]], +) -> tuple[list[str], list[tuple[str, str]]]: + nodes: list[str] = [] + edges: list[tuple[str, str]] = [] + for cls in build_step_classes: + step_name = cls.get_build_step(cls) # type: ignore[arg-type] + nodes.append(step_name) + for cls in build_step_classes: + step_name = cls.get_build_step(cls) # type: ignore[arg-type] + requires = cls.requires_tasks(cls) # type: ignore[arg-type] + if requires: + for dep_name in requires: + edges.append((dep_name, step_name)) + return nodes, edges + + +def generate_dot(flavor_path: str, output_path: str | None = None) -> str: + """ + Generate a .dot dependency graph from a flavor's build_steps.py. + + :param flavor_path: Path to the flavor directory. + :param output_path: Optional path where to write the .dot file. + Defaults to /build_steps.dot. + :return: The .dot file content as a string. + """ + flavor_dir = Path(flavor_path) + build_steps_path = flavor_dir / "flavor_base" / "build_steps.py" + if not build_steps_path.exists(): + raise FileNotFoundError(f"build_steps.py not found at {build_steps_path}") + + module = _load_build_steps_module(build_steps_path) + build_step_classes = _collect_build_step_classes(module) + nodes, edges = _build_dependency_edges(build_step_classes) + + lines = ["strict digraph {"] + for node in nodes: + lines.append(f'"{node}" [label="{node}", shape=box];') + for source, target in edges: + lines.append(f'"{source}" -> "{target}";') + lines.append("}") + dot_content = "\n".join(lines) + "\n" + + if output_path is None: + output_path = str(flavor_dir / "build_steps.dot") + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + output_file.write_text(dot_content, encoding="utf-8") + + return dot_content diff --git a/exasol/slc/tool/commands/__init__.py b/exasol/slc/tool/commands/__init__.py index e76ac097..5ce12940 100644 --- a/exasol/slc/tool/commands/__init__.py +++ b/exasol/slc/tool/commands/__init__.py @@ -3,6 +3,7 @@ from .clean import clean_all_images, clean_flavor_images from .deploy import deploy from .export import export +from .generate_build_steps_dot_graph import generate_build_steps_dot_graph from .generate_language_activation import generate_language_activation from .generate_package_diffs import generate_package_diffs from .push import push @@ -19,6 +20,7 @@ "clean_flavor_images", "deploy", "export", + "generate_build_steps_dot_graph", "generate_language_activation", "push", "push_test_container", diff --git a/exasol/slc/tool/commands/generate_build_steps_dot_graph.py b/exasol/slc/tool/commands/generate_build_steps_dot_graph.py new file mode 100644 index 00000000..e1a96017 --- /dev/null +++ b/exasol/slc/tool/commands/generate_build_steps_dot_graph.py @@ -0,0 +1,32 @@ +import click +from exasol_integration_test_docker_environment.lib.utils.cli_function_decorators import ( + add_options, +) + +from exasol.slc import api +from exasol.slc.tool.cli import cli +from exasol.slc.tool.options.flavor_options import single_flavor_options + + +@cli.command( + short_help="Generates a .dot dependency graph from a flavor's build_steps.py." +) +@add_options(single_flavor_options) +@click.option( + "--output-path", + required=False, + default=None, + help="Path where to write the .dot file. Defaults to /build_steps.dot.", + type=click.Path(exists=False), +) +def generate_build_steps_dot_graph( + flavor_path: str, + output_path: str | None, +): + """ + Generate a .dot file visualizing the build step dependencies of a flavor. + """ + api.generate_build_steps_dot_graph( + flavor_path=flavor_path, + output_path=output_path, + ) diff --git a/test/resources/default_flavor/flavors/test-flavor/build_steps.dot b/test/resources/default_flavor/flavors/test-flavor/build_steps.dot new file mode 100644 index 00000000..05c8cf56 --- /dev/null +++ b/test/resources/default_flavor/flavors/test-flavor/build_steps.dot @@ -0,0 +1,13 @@ +strict digraph { +"build_run" [label="build_run", shape=box]; +"base_test_build_run" [label="base_test_build_run", shape=box]; +"flavor_customization" [label="flavor_customization", shape=box]; +"flavor_test_build_run" [label="flavor_test_build_run", shape=box]; +"release" [label="release", shape=box]; +"security_scan" [label="security_scan", shape=box]; +"build_run" -> "flavor_customization"; +"flavor_customization" -> "flavor_test_build_run"; +"base_test_build_run" -> "flavor_test_build_run"; +"flavor_customization" -> "release"; +"release" -> "security_scan"; +} diff --git a/test/test_api_generate_build_steps_dot_graph.py b/test/test_api_generate_build_steps_dot_graph.py new file mode 100644 index 00000000..c62cfe71 --- /dev/null +++ b/test/test_api_generate_build_steps_dot_graph.py @@ -0,0 +1,47 @@ +import shutil +import tempfile +import unittest +from pathlib import Path + +import utils as exaslct_utils # type: ignore # pylint: disable=import-error + +from exasol.slc.api import generate_build_steps_dot_graph + +EXPECTED_DOT_FILE = ( + exaslct_utils.DEFAULT_FLAVOR_FLAVORS_ROOT_DIRECTORY + / "test-flavor" + / "build_steps.dot" +) + + +class GenerateBuildStepsDotGraphTest(unittest.TestCase): + + def test_generate_dot_graph_matches_expected(self): + expected_content = EXPECTED_DOT_FILE.read_text(encoding="utf-8") + with tempfile.TemporaryDirectory() as tmp_dir: + flavor_copy = Path(tmp_dir) / "test-flavor" + shutil.copytree(exaslct_utils.get_test_flavor(), flavor_copy) + result = generate_build_steps_dot_graph( + flavor_path=str(flavor_copy), + ) + default_output = flavor_copy / "build_steps.dot" + self.assertTrue(default_output.exists()) + written_content = default_output.read_text(encoding="utf-8") + self.assertEqual(expected_content, written_content) + self.assertEqual(expected_content, result) + + def test_generate_dot_graph_writes_to_custom_path(self): + expected_content = EXPECTED_DOT_FILE.read_text(encoding="utf-8") + with tempfile.TemporaryDirectory() as tmp_dir: + output_path = str(Path(tmp_dir) / "custom.dot") + result = generate_build_steps_dot_graph( + flavor_path=str(exaslct_utils.get_test_flavor()), + output_path=output_path, + ) + written_content = Path(output_path).read_text(encoding="utf-8") + self.assertEqual(expected_content, written_content) + self.assertEqual(expected_content, result) + + +if __name__ == "__main__": + unittest.main()