diff --git a/bzl/bundle_rules.bzl b/bzl/bundle_rules.bzl index ace5d796d..a9ce54b9a 100644 --- a/bzl/bundle_rules.bzl +++ b/bzl/bundle_rules.bzl @@ -72,6 +72,7 @@ CodeTargetSourcesInfo = provider( doc = "Source files collected from an implementation target and its dependencies.", fields = { "sources": "Depset of direct and transitive source files.", + "kind": "Bazel rule kind of the target selected by code_targets.", }, ) @@ -101,6 +102,7 @@ def _collect_code_target_sources_impl(target, ctx): direct = _source_files_from_attributes(ctx), transitive = dependency_sources, ), + kind = ctx.rule.kind, )] _collect_code_target_sources = aspect( @@ -167,6 +169,7 @@ def _rebase_bundle_entry(entry, mount_at, attach_to, data): external = entry.external, repository = entry.repository, data = data, + code_targets = entry.code_targets, ) def _entries_visible_through(ctx, child): @@ -213,6 +216,13 @@ def _docs_bundle_impl(ctx): own_source_files = [] own_external_runfiles = [] own_data = depset(direct = ctx.files.data) + own_code_targets = [ + struct( + label = _format_bazel_label(target.label), + type = target[CodeTargetSourcesInfo].kind, + ) + for target in ctx.attr.code_targets + ] if ctx.files.srcs: runtime_path = _bundle_runtime_path(ctx) @@ -229,6 +239,7 @@ def _docs_bundle_impl(ctx): external = external, repository = ctx.label.workspace_name, data = own_data, + code_targets = own_code_targets, )) own_source_files.extend(ctx.files.srcs) # Local sources are read directly from the workspace by ``bazel run``. @@ -246,6 +257,7 @@ def _docs_bundle_impl(ctx): external = False, repository = ctx.label.workspace_name, data = own_data, + code_targets = own_code_targets, )) child_source_files = [] @@ -305,11 +317,12 @@ _docs_bundle = rule( "bundle_mount_ats": attr.string_list(), "bundle_attach_tos": attr.string_list(), "data": attr.label_list(allow_files = True), + "code_targets": attr.label_list(aspects = [_collect_code_target_sources]), }, doc = "Internal rule that carries bundle files and their documentation-tree locations.", ) -def create_bundle(name, bundles, srcs = [], sourcelinks = [], strip_prefix = "", entry_doc = "index", data = [], visibility = None, **kwargs): +def create_bundle(name, bundles, srcs = [], sourcelinks = [], strip_prefix = "", entry_doc = "index", data = [], code_targets = [], visibility = None, **kwargs): """Create a reusable documentation bundle from files and child declarations.""" parsed_bundles = [_parse_bundle_declaration(declaration) for declaration in bundles] _docs_bundle( @@ -322,6 +335,7 @@ def create_bundle(name, bundles, srcs = [], sourcelinks = [], strip_prefix = "", bundle_mount_ats = [bundle.mount_at for bundle in parsed_bundles], bundle_attach_tos = [bundle.attach_to for bundle in parsed_bundles], data = data, + code_targets = code_targets, visibility = visibility, **kwargs ) @@ -400,17 +414,28 @@ def _code_targets_sourcelinks_impl(ctx): target[CodeTargetSourcesInfo].sources for target in ctx.attr.code_targets ]) - if not source_files.to_list(): - fail("code_targets must declare source files through filegroups, srcs, hdrs, or textual_hdrs") output = ctx.actions.declare_file(ctx.label.name + ".json") + target_map = ctx.actions.declare_file(ctx.label.name + "_targets.json") + target_map_entries = [] + for target in ctx.attr.code_targets: + target_info = target[CodeTargetSourcesInfo] + for source in target_info.sources.to_list(): + target_map_entries.append({ + "file": source.path, + "bazel_target": _format_bazel_label(target.label), + "bazel_type": target_info.kind, + }) + ctx.actions.write(target_map, json.encode(target_map_entries)) + arguments = ctx.actions.args() arguments.add("--output", output.path) + arguments.add("--target-map", target_map.path) arguments.add_all(source_files) ctx.actions.run( executable = ctx.executable._generate_sourcelinks, arguments = [arguments], - inputs = source_files, + inputs = depset(direct = [target_map], transitive = [source_files]), outputs = [output], mnemonic = "GenerateCodeTargetSourcelinks", ) @@ -437,3 +462,8 @@ def generate_code_target_sourcelinks(name, code_targets, visibility = None): visibility = visibility, ) return ":" + name + +def _format_bazel_label(label): + """Return an apparent-style label without Bzlmod's internal ``@@`` prefix.""" + result = "//" + label.package + ":" + label.name + return "@" + label.workspace_name + result if label.workspace_name else result diff --git a/bzl/mount_rules.bzl b/bzl/mount_rules.bzl index dd42571eb..3485fcdcf 100644 --- a/bzl/mount_rules.bzl +++ b/bzl/mount_rules.bzl @@ -53,3 +53,29 @@ def create_mounts_manifest(name, bundle): bundle = bundle, ) return ":" + name + +def _bundle_target_manifest_impl(ctx): + """Write the code targets associated with each documentation-tree subtree.""" + mappings = [] + for entry in ctx.attr.bundle[DocsBundleInfo].entries: + if entry.code_targets: + mappings.append({ + "mount_at": entry.mount_at, + "targets": [ + {"bazel_target": target.label, "bazel_type": target.type} + for target in entry.code_targets + ], + }) + out = ctx.actions.declare_file(ctx.label.name + ".json") + ctx.actions.write(out, json.encode({"mappings": mappings})) + return [DefaultInfo(files = depset([out]))] + +_bundle_target_manifest = rule( + implementation = _bundle_target_manifest_impl, + attrs = {"bundle": attr.label(providers = [DocsBundleInfo])}, + doc = "Writes Bazel target metadata associated with documentation bundles.", +) + +def create_bundle_target_manifest(name, bundle): + _bundle_target_manifest(name = name, bundle = bundle) + return ":" + name diff --git a/docs.bzl b/docs.bzl index cdc3a735c..12dc75857 100644 --- a/docs.bzl +++ b/docs.bzl @@ -59,6 +59,7 @@ load( load( "@score_docs_as_code//:bzl/mount_rules.bzl", "create_mounts_manifest", + "create_bundle_target_manifest", ) def _generated_conf_impl(ctx): @@ -142,6 +143,7 @@ def docs_bundle(name, source_dir = None, data = [], entry_doc = "index", bundles entry_doc = entry_doc, bundles = bundles, data = data, + code_targets = code_targets, visibility = visibility, **kwargs ) @@ -294,6 +296,10 @@ def docs( bundle = ":docs_bundle", known_good = known_good, ) + create_bundle_target_manifest( + name = "bundle_target_manifest", + bundle = ":docs_bundle", + ) external_docs_runfiles( name = "_external_docs_runfiles", @@ -306,7 +312,7 @@ def docs( # bundles do need runfiles, so keep only those sources. docs_data = ( data + external_needs + metamodel_label + - [":sourcelinks_json", ":_external_docs_runfiles"] + + [":sourcelinks_json", ":bundle_target_manifest", ":_external_docs_runfiles"] + mounts_manifest_label ) if config_is_generated: @@ -324,6 +330,7 @@ def docs( # resolved by score_mounts through ``RUNFILES_DIR``. "MOUNTS_MANIFEST": "$(rlocationpath :_mounts_manifest)" if bundles else "", "SCORE_SOURCELINKS": "$(location :sourcelinks_json)", + "SCORE_BAZEL_TARGETS": "$(rlocationpath :bundle_target_manifest)", } if config_is_generated: # The generated file is named conf.py. Run targets pass its containing @@ -408,6 +415,7 @@ def docs( "auto", "--define=external_needs_source=" + str(data + external_needs), "--define=score_sourcelinks_json=$(location :sourcelinks_json)", + "--define=score_bazel_targets=$(location :bundle_target_manifest)", "--define=score_source_code_linker_plain_links=1", ] + ( # ``sphinx_docs`` is a sandboxed build action, so it needs the @@ -416,7 +424,7 @@ def docs( ) + (["--define=score_metamodel_yaml=$(location " + str(metamodel) + ")"] if metamodel else []), formats = ["needs"], sphinx = ":sphinx_build", - tools = data + external_needs + metamodel_label + [":sourcelinks_json", ":docs_bundle"] + mounts_manifest_label, + tools = data + external_needs + metamodel_label + [":sourcelinks_json", ":bundle_target_manifest", ":docs_bundle"] + mounts_manifest_label, visibility = ["//visibility:public"], # Persistent workers cause stale symlinks after dependency version # changes, corrupting the Bazel cache. diff --git a/docs/reference/bazel_macros.rst b/docs/reference/bazel_macros.rst index f188c93f5..4f12d6e2b 100644 --- a/docs/reference/bazel_macros.rst +++ b/docs/reference/bazel_macros.rst @@ -184,10 +184,15 @@ Signature: ``docs_bundle(name, source_dir = None, entry_doc = "index", bundles = same bundle be mounted at different locations by different consumers without changing its canonical entry page. -- ``code_targets`` (list of Bazel labels, optional) +- ``code_targets`` (list of Bazel labels, optional). For every source-code link + found in these targets, the corresponding need receives ``bazel_target`` (the + target label) and ``bazel_type`` (for example ``cc_library``). Implementation targets or filegroups to scan for requirement tags. Implementation target ``srcs``, ``hdrs``, and ``textual_hdrs`` are collected recursively from their ``deps``; filegroups expand to their files. The bundle + may also reference an empty target (for example a component template before + implementation starts); its documentation needs still receive the target + metadata. owns one cached scan result; Bazel only regenerates it when its collected source inputs change. diff --git a/scripts_bazel/generate_sourcelinks_cli.py b/scripts_bazel/generate_sourcelinks_cli.py index e5b134a45..3064e8c8d 100644 --- a/scripts_bazel/generate_sourcelinks_cli.py +++ b/scripts_bazel/generate_sourcelinks_cli.py @@ -18,6 +18,7 @@ """ import argparse +import json import logging import sys from pathlib import Path @@ -51,6 +52,19 @@ def clean_external_prefix(path: Path) -> Path: return Path("".join(filepath_split[1:])) +def _load_target_map(path: Path | None) -> dict[str, list[tuple[str, str]]]: + """Return the direct ``code_targets`` owning each scanned source file.""" + if path is None: + return {} + entries = json.loads(path.read_text(encoding="utf-8")) + target_map: dict[str, list[tuple[str, str]]] = {} + for entry in entries: + target_map.setdefault(entry["file"], []).append( + (entry["bazel_target"], entry["bazel_type"]) + ) + return target_map + + def main(): parser = argparse.ArgumentParser( description="Generate source code links JSON from source files" @@ -61,6 +75,11 @@ def main(): type=Path, help="Output JSON file path", ) + _ = parser.add_argument( + "--target-map", + type=Path, + help="JSON mapping source paths to their Bazel target labels and rule kinds", + ) _ = parser.add_argument( "files", nargs="*", @@ -71,6 +90,7 @@ def main(): args = parser.parse_args() all_need_references = [] + target_map = _load_target_map(args.target_map) metadata = DefaultMetaData() metadata_set = False @@ -84,6 +104,10 @@ def main(): references = _extract_references_from_file( abs_file_path.parent, Path(abs_file_path.name), clean_path ) + target_data = target_map.get(str(file_path), []) + for reference in references: + reference.bazel_target = ", ".join(target for target, _ in target_data) + reference.bazel_type = ", ".join(kind for _, kind in target_data) all_need_references.extend(references) store_source_code_links_with_metadata_json( file=args.output, metadata=metadata, needlist=all_need_references diff --git a/scripts_bazel/tests/generate_sourcelinks_cli_test.py b/scripts_bazel/tests/generate_sourcelinks_cli_test.py index 4aa50ed53..d8259fc3f 100644 --- a/scripts_bazel/tests/generate_sourcelinks_cli_test.py +++ b/scripts_bazel/tests/generate_sourcelinks_cli_test.py @@ -135,6 +135,44 @@ def test_generate_sourcelinks_cli_parses_cpp_traceability_tag( assert data[1]["need"] == "tool_req__docs_arch_types" +def test_generate_sourcelinks_cli_adds_bazel_target_metadata( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + test_file = tmp_path / "test_source.cc" + test_file.write_text("// req-Id: tool_req__docs_arch_types\n") + target_map = tmp_path / "targets.json" + target_map.write_text( + json.dumps( + [ + { + "file": str(test_file), + "bazel_target": "//components/filesystem:filesystem", + "bazel_type": "cc_library", + } + ] + ) + ) + output_file = tmp_path / "output.json" + + monkeypatch.setattr( + sys, + "argv", + [ + str(_MY_PATH.parent / "generate_sourcelinks_cli.py"), + "--output", + str(output_file), + "--target-map", + str(target_map), + str(test_file), + ], + ) + + assert scripts_bazel.generate_sourcelinks_cli.main() == 0 + data = json.loads(output_file.read_text()) + assert data[1]["bazel_target"] == "//components/filesystem:filesystem" + assert data[1]["bazel_type"] == "cc_library" + + def test_generate_sourcelinks_cli_parse_external_module( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ): diff --git a/src/extensions/score_metamodel/metamodel.yaml b/src/extensions/score_metamodel/metamodel.yaml index 606632156..cff972182 100644 --- a/src/extensions/score_metamodel/metamodel.yaml +++ b/src/extensions/score_metamodel/metamodel.yaml @@ -17,6 +17,8 @@ needs_types_base_options: # req-Id: tool_req__docs_dd_link_source_code_link source_code_link: ^https://github.com/.* testlink: ^https://github.com/.* + bazel_target: ^.*$ + bazel_type: ^.*$ # Version will be mandatory global option in future releases # For now giving grace periods to consumers mandatory_options: diff --git a/src/extensions/score_source_code_linker/__init__.py b/src/extensions/score_source_code_linker/__init__.py index ef1acfd90..34da9c426 100644 --- a/src/extensions/score_source_code_linker/__init__.py +++ b/src/extensions/score_source_code_linker/__init__.py @@ -20,6 +20,7 @@ # req-Id: tool_req__docs_dd_link_source_code_link # This whole directory implements the above mentioned tool requirements +import json import os from copy import deepcopy from pathlib import Path @@ -63,6 +64,7 @@ from src.helper_lib import ( find_git_root, find_ws_root, + get_runfiles_dir, ) LOGGER = get_logger(__name__) @@ -70,6 +72,41 @@ # LOGGER.setLevel("DEBUG") +def _bundle_targets_for_doc(docname: str) -> list[tuple[str, str]]: + """Return code targets associated with the most-specific bundle for a page.""" + raw_manifest_path = os.environ.get("SCORE_BAZEL_TARGETS") + if not raw_manifest_path: + return [] + manifest_path = Path(raw_manifest_path) + if not manifest_path.is_file(): + manifest_path = get_runfiles_dir() / raw_manifest_path + try: + mappings = json.loads(manifest_path.read_text(encoding="utf-8"))["mappings"] + except (OSError, KeyError, TypeError, json.JSONDecodeError) as error: + LOGGER.warning( + "Could not read Bazel target metadata from %s: %s", + raw_manifest_path, + error, + type="score_source_code_linker", + ) + return [] + + matching_mappings = [ + mapping + for mapping in mappings + if not mapping["mount_at"] + or docname == mapping["mount_at"] + or docname.startswith(mapping["mount_at"] + "/") + ] + if not matching_mappings: + return [] + best_match = max(matching_mappings, key=lambda mapping: len(mapping["mount_at"])) + return [ + (target["bazel_target"], target["bazel_type"]) + for target in best_match["targets"] + ] + + # re-qid: gd_req__req_attr_impl # ╭──────────────────────────────────────╮ # │ JSON FILE RELATED FUNCS │ @@ -151,6 +188,10 @@ def setup_source_code_linker(app: Sphinx, ws_root: Path | None): "options": ["source_code_link", "testlink"], }, ) + for option in ("bazel_target", "bazel_type"): + app.config.needs_fields.setdefault( + option, {"schema": {"type": "string"}, "default": ""} + ) score_sourcelinks_json = os.environ.get("SCORE_SOURCELINKS") if not score_sourcelinks_json: @@ -160,6 +201,14 @@ def setup_source_code_linker(app: Sphinx, ws_root: Path | None): if score_sourcelinks_json: # Reuse existing code paths that expect this env var. os.environ["SCORE_SOURCELINKS"] = score_sourcelinks_json + + score_bazel_targets = os.environ.get("SCORE_BAZEL_TARGETS") + if not score_bazel_targets: + score_bazel_targets = str( + getattr(app.config, "score_bazel_targets", "") + ).strip() + if score_bazel_targets: + os.environ["SCORE_BAZEL_TARGETS"] = score_bazel_targets if score_sourcelinks_json: # No need to generate the JSON file if this env var is set # because it points to an existing file with the needed data. @@ -343,6 +392,13 @@ def setup(app: Sphinx) -> dict[str, str | bool]: types=str, description="Path to pre-generated source code links JSON from Bazel via SCORE_SOURCELINKS env var", ) + app.add_config_value( + "score_bazel_targets", + default="", + rebuild="env", + types=str, + description="Path to Bazel target metadata for documentation bundles.", + ) app.add_config_value( "score_source_code_linker_plain_links", default=False, @@ -498,6 +554,18 @@ def _apply_links_to_need( _render_test_link(plain_links, metadata, test_link) for test_link in links.TestLinks ) + source_targets = [ + (link.bazel_target, link.bazel_type) + for link in links.CodeLinks + if link.bazel_target + ] + bundle_targets = _bundle_targets_for_doc(str(need["docname"])) + bazel_targets = sorted({target for target, _ in source_targets + bundle_targets}) + bazel_types = sorted({kind for _, kind in source_targets + bundle_targets}) + if bazel_targets: + need_as_dict["bazel_target"] = ", ".join(bazel_targets) + if bazel_types: + need_as_dict["bazel_type"] = ", ".join(bazel_types) # NOTE: Removing & adding the need is important to make sure # the needs gets 're-evaluated'. @@ -505,6 +573,20 @@ def _apply_links_to_need( needs_data.add_need(need) +def _apply_bundle_targets_to_need(needs_data: SphinxNeedsData, need: NeedItem) -> None: + """Add a bundle's code targets even when no source tag references the need.""" + bundle_targets = _bundle_targets_for_doc(str(need["docname"])) + if not bundle_targets: + return + need_as_dict = cast(dict[str, object], need) + need_as_dict["bazel_target"] = ", ".join( + sorted({target for target, _ in bundle_targets}) + ) + need_as_dict["bazel_type"] = ", ".join(sorted({kind for _, kind in bundle_targets})) + needs_data.remove_need(need["id"]) + needs_data.add_need(need) + + # re-qid: gd_req__req__attr_impl def inject_links_into_needs(app: Sphinx, env: BuildEnvironment) -> None: """ @@ -524,6 +606,9 @@ def inject_links_into_needs(app: Sphinx, env: BuildEnvironment) -> None: _log_existing_links(needs) + for need in needs_copy.values(): + _apply_bundle_targets_to_need(needs_data, need) + scl_by_module = load_repo_source_links_json( get_cache_filename(app.outdir, "score_repo_grouped_scl_cache.json") ) diff --git a/src/extensions/score_source_code_linker/needlinks.py b/src/extensions/score_source_code_linker/needlinks.py index 2998240fc..d0b784c80 100644 --- a/src/extensions/score_source_code_linker/needlinks.py +++ b/src/extensions/score_source_code_linker/needlinks.py @@ -47,6 +47,8 @@ class NeedLink: repo_name: str = "local_repo" hash: str = "" url: str = "" + bazel_target: str = "" + bazel_type: str = "" # Adding hashing & equality as this is needed to make comparisions. # Since the Dataclass is not 'frozen = true' it isn't automatically hashable @@ -62,6 +64,8 @@ def __hash__(self): self.repo_name, self.hash, self.url, + self.bazel_target, + self.bazel_type, ) ) @@ -77,6 +81,8 @@ def __eq__(self, other: Any): and self.repo_name == other.repo_name and self.hash == other.hash and self.url == other.url + and self.bazel_target == other.bazel_target + and self.bazel_type == other.bazel_type ) # Normal 'dictionary conversion'. Converts all fields @@ -129,6 +135,8 @@ def needlink_decoder(d: dict[str, Any]) -> NeedLink | dict[str, Any]: repo_name=d.get("repo_name", ""), hash=d.get("hash", ""), url=d.get("url", ""), + bazel_target=d.get("bazel_target", ""), + bazel_type=d.get("bazel_type", ""), ) # It's something else, pass it on to other decoders return d diff --git a/src/tests/docs_bzl/scenarios/nested_bundles/child/example.cc b/src/tests/docs_bzl/scenarios/nested_bundles/child/example.cc index a8c3055ec..9726b3dae 100644 --- a/src/tests/docs_bzl/scenarios/nested_bundles/child/example.cc +++ b/src/tests/docs_bzl/scenarios/nested_bundles/child/example.cc @@ -1,4 +1,4 @@ -// req-traceability: REQ_CHILD +// req-traceability: doc_concept__child int example() { return 0; diff --git a/src/tests/docs_bzl/scenarios/nested_bundles/child/example.py b/src/tests/docs_bzl/scenarios/nested_bundles/child/example.py index ca83634c4..5adec3fc6 100644 --- a/src/tests/docs_bzl/scenarios/nested_bundles/child/example.py +++ b/src/tests/docs_bzl/scenarios/nested_bundles/child/example.py @@ -11,4 +11,4 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -# req-traceability: REQ_CHILD +# req-traceability: doc_concept__child diff --git a/src/tests/docs_bzl/scenarios/nested_bundles/child/landing.rst b/src/tests/docs_bzl/scenarios/nested_bundles/child/landing.rst index 48974f8fb..1f9a97c19 100644 --- a/src/tests/docs_bzl/scenarios/nested_bundles/child/landing.rst +++ b/src/tests/docs_bzl/scenarios/nested_bundles/child/landing.rst @@ -15,6 +15,16 @@ Child landing page ================== +.. doc_concept:: Child implementation trace + :id: doc_concept__child + :status: valid + :version: 1 + +.. doc_concept:: Child bundle trace without source tag + :id: doc_concept__unlinked + :status: valid + :version: 1 + .. toctree:: index diff --git a/src/tests/docs_bzl/test_nested_bundles.py b/src/tests/docs_bzl/test_nested_bundles.py index 0bf1861dc..d7e54856c 100644 --- a/src/tests/docs_bzl/test_nested_bundles.py +++ b/src/tests/docs_bzl/test_nested_bundles.py @@ -62,6 +62,42 @@ def test_nested_bundles_render_and_preserve_metadata(): "src/tests/docs_bzl/scenarios/nested_bundles/child/example.cc", "src/tests/docs_bzl/scenarios/nested_bundles/child/filegroup_source.py", } + by_file = {link["file"]: link for link in sourcelinks} + assert ( + by_file["src/tests/docs_bzl/scenarios/nested_bundles/child/example.py"][ + "bazel_target" + ] + == "//src/tests/docs_bzl/scenarios/nested_bundles:example_binary" + ) + assert ( + by_file["src/tests/docs_bzl/scenarios/nested_bundles/child/example.py"][ + "bazel_type" + ] + == "py_binary" + ) + assert ( + by_file["src/tests/docs_bzl/scenarios/nested_bundles/child/example.cc"][ + "bazel_target" + ] + == "//src/tests/docs_bzl/scenarios/nested_bundles:example_executable" + ) + assert ( + by_file["src/tests/docs_bzl/scenarios/nested_bundles/child/example.cc"][ + "bazel_type" + ] + == "cc_binary" + ) + needs = json.loads((result.build_dir / "needs.json").read_text(encoding="utf-8")) + need = needs["needs"]["doc_concept__child"] + assert need["bazel_target"] == ( + "//src/tests/docs_bzl/scenarios/nested_bundles:example_binary, " + "//src/tests/docs_bzl/scenarios/nested_bundles:example_executable, " + "//src/tests/docs_bzl/scenarios/nested_bundles:nested_filegroup_sources" + ) + assert need["bazel_type"] == "cc_binary, filegroup, py_binary" + unlinked_need = needs["needs"]["doc_concept__unlinked"] + assert unlinked_need["bazel_target"] == need["bazel_target"] + assert unlinked_need["bazel_type"] == need["bazel_type"] assert (result.build_dir / "concepts" / "example_bundle" / "index.html").is_file() assert ( result.build_dir / "concepts" / "example_bundle" / "child" / "landing.html"