DR-008 Option 4: two-stage test-execution workflow (PR 2 of 2)

.bazelrc

@@ -72,6 +72,11 @@ build:eb-aarch64 --config=_common
 build:eb-aarch64 --extra_toolchains=@score_ebclfsa_toolchain//:aarch64-linux-sdk_0.1.0-ebclfsa
 build:eb-aarch64 --extra_toolchains=@ferrocene_aarch64_ebclfsa//:rust_ferrocene_toolchain
 build:eb-aarch64 --platforms=@score_bazel_platforms//:aarch64-linux-sdk_0.1.0-ebclfsa
+# GCC-13 emits aarch64 outline-atomic calls (__aarch64_ldadd4_acq_rel, __aarch64_cas*, ...)
+# whose helpers live in libgcc.a. Ferrocene rustc drives the final link with -nodefaultlibs,
+# so libgcc.a is never pulled in -> undefined references when Rust binaries link C++ libs.
+# Force inline atomics instead.
+build:eb-aarch64 --copt=-mno-outline-atomics
 
 build:autosd-x86_64 --config=_common
 build:autosd-x86_64 --force_pic
@@ -111,6 +116,13 @@ test:unit-tests --test_tag_filters=-manual,-miri
 # Coverage configuration for C++
 coverage --features=coverage
 coverage --combined_report=lcov
+# Exclude tests the modules tag out of coverage (e.g. *_tsan_test): gcov-instrumenting a
+# ThreadSanitizer binary reports false data races on the non-atomic __gcov* counters.
+# Mirrors the modules' own .bazelrc so central-mode coverage matches in-module coverage.
+coverage --test_tag_filters=-no-coverage
+# Make gcov counter updates atomic so any remaining multithreaded coverage test is race-free.
+coverage --copt=-fprofile-update=atomic
+coverage --linkopt=-fprofile-update=atomic
 
 # user specific overrides (like proxy settings)
 try-import %workspace%/user.bazelrc

bazel_common/score_modules_target_sw.MODULE.bazel

@@ -15,13 +15,14 @@
 # Do not edit manually - use scripts/known_good/update_module_from_known_good.py
 
 bazel_dep(name = "score_baselibs")
-single_version_override(
+git_override(
     module_name = "score_baselibs",
+    commit = "ce204159f37ee7815907369a8f678583bf102306",
     patch_strip = 1,
     patches = [
         "//patches/baselibs:001-restore-trace-library-mock-visibility.patch",
     ],
-    version = "0.2.9",
+    remote = "https://github.com/eclipse-score/baselibs.git",
 )
 
 bazel_dep(name = "score_communication")

bazel_common/score_rust_toolchains.MODULE.bazel

@@ -12,7 +12,11 @@
 # *******************************************************************************
 bazel_dep(name = "rules_rust", version = "0.68.1-score")
 
-bazel_dep(name = "score_toolchains_rust", version = "0.8.0", dev_dependency = True)
+# 0.9.1 declares the ferrocene *_miri toolchain targets that score_lifecycle_health and
+# score_persistency register in their own .bazelrc. Stage 2 layers ref_int's config over the
+# module's rather than replacing it, so those registrations survive and --extra_toolchains
+# accumulates — ref_int cannot un-register them, and the version must match here instead.
+bazel_dep(name = "score_toolchains_rust", version = "0.9.1", dev_dependency = True)
 
 ferrocene = use_extension(
     "@score_toolchains_rust//extensions:ferrocene_toolchain_ext.bzl",

ci/stage2/README.md

@@ -0,0 +1,112 @@
+# DR-008 Stage 2 — mission, and the decisions ref_int made deliberately
+
+Read this before changing anything in `ci/stage2/`. It exists because this area has already
+been implemented the wrong way once, and the misalignment came from paraphrasing the goal
+instead of writing it down.
+
+## The mission (acceptance criteria for Stage 2)
+
+1. CI runs Stage 1 → Stage 2 matrix over the `target_sw` modules → aggregate report.
+2. Each Stage-2 job reports a **non-zero test count**. A job that configures but runs no
+   tests is a failure, not a pass.
+3. Each module's regenerated `MODULE.bazel.lock` agrees with `resolved_versions.json` for
+   every dependency present in both.
+4. Every failure is classified: **0 tests executed ⇒ ref_int harness defect**;
+   **>0 tests with failures ⇒ integration finding owned by the module team**.
+5. No module is built with semantics different from its own CI **unless ref_int decided so
+   deliberately and recorded it in the table below**.
+
+Explicitly *not* the mission: "ref_int owns every flag a module builds with." That was an
+over-reading of the review guidance and it cost a Critical regression (see below).
+
+## What the review guidance actually said
+
+The maintainer asked which `.bazelrc` Stage 2 uses, was told "the downstream one", and did
+not object. What they objected to was the *next* answer:
+
+> "config names are coming from ref_int's `known_good.json`, but the actual flag definitions
+> are downstream" — **"that needs to be changed / so we have a control over the flags that
+> are passed to modules / please check with config and toolchain from ref_int / if there is
+> still an issue check toolchain difference"**
+
+So the defect was a **dangling reference**: ref_int named a config (`--config=per-x86_64-linux`)
+whose meaning lived in the module. The fix is for ref_int to **define** the configs it names.
+It is *not* to discard the module's `.bazelrc`.
+
+## Mechanism: layer, never `--noworkspace_rc`
+
+Stage 2 passes `--bazelrc=<ref_int>/ci/stage2/module.bazelrc` and **keeps** the module's own
+`.bazelrc`. Bazel reads rc files in order, and that order is what gives ref_int control:
+
+| Flag kind | Two rc files disagree | Outcome |
+|---|---|---|
+| single-valued (`--platforms`, `--test_tag_filters`, `--instrumentation_filter`) | module first, ref_int last | **ref_int wins** |
+| accumulating (`--extra_toolchains`, `--copt`, `--per_file_copt`, `--aspects`) | both apply | both kept |
+
+`--noworkspace_rc` was tried and **reverted**. It discarded module settings unrelated to the
+configs ref_int names — `score_communication`'s generic-trace-library **stub** selection, its
+sandbox settings and its own libclang/cc toolchains; `score_kyron`'s Rust coverage
+instrumentation; `score_baselibs`/`score_orchestrator`'s clippy aspects. Net effect was worse
+than the bug it fixed: all 8 modules configured, 3 silently mis-configured. Loud failures
+traded for quiet wrongness.
+
+Consequence of layering to keep in mind: ref_int **cannot un-register** a toolchain the module
+registers, because `--extra_toolchains` accumulates. Version skew therefore has to be fixed at
+the source (see `score_toolchains_rust` below), not papered over in the rc.
+
+## Why per-module opt-in configs still exist (and when they go away)
+
+`known_good.json`'s `bazel_config` carries **additive opt-ins only**. `stage2-linux-x86_64` is
+emitted unconditionally by `quality_runners.py`, so a module with an empty or mistyped
+`bazel_config` can never run with no platform/toolchain again.
+
+| Config | Who selects it | Why it is not in the base | Removable when |
+|---|---|---|---|
+| `stage2-gcc` | all except `score_communication` | communication's `gcc.toolchain()` call passes `use_base_constraints_only = True`, so its generated target is `:x86_64-linux`, not `:x86_64-linux-gcc_12.2.0`. It registers its own cc toolchain unconditionally, so it needs nothing from ref_int here. | ref_int injects the toolchain *declarations* into the module (design option β) |
+| `stage2-rust` | all except `score_time` | `score_time` declares no `score_toolchains_rust`, so `@score_toolchains_rust//...` is not a resolvable apparent repo name in its graph | the Phase-1 transitive-pinning fix lands (PR #278): injecting a `bazel_dep` stub for every module in the resolved set makes the repo present everywhere, and this folds into the base |
+
+Both are named after a **capability**, not a module. That distinction matters: the file may
+branch, but the branch condition must be derivable from the module's own sources — never
+duplicated by hand for a module name.
+
+Deleted and not to be reintroduced: `stage2-libclang-communication`. It was a `//`-relative
+label in a ref_int-owned file that only resolved because the module happened to be the Bazel
+root. Under layering `score_communication` registers its own libclang toolchain
+unconditionally (`.bazelrc`: `common --extra_toolchains=//bazel/toolchains:score_communication_libclang_toolchain`),
+so ref_int does not need to name it at all.
+
+## Decisions ref_int made deliberately (criterion 5)
+
+Anything here is a place where Stage 2 differs from the module's own CI **on purpose**.
+Anything *not* here that differs is a bug.
+
+| Decision | Effect | Rationale |
+|---|---|---|
+| `--test_tag_filters=-manual,-miri,-no-coverage` | miri and `no-coverage`-tagged tests do not run in Stage 2 | ref_int registers no miri toolchain; `*_tsan_test` under `bazel coverage` reports false races on non-atomic `__gcov*` counters |
+| `--copt/--linkopt=-fprofile-update=atomic` | gcov counters updated atomically | makes any remaining multithreaded coverage test race-free |
+| `--repo_env=ANDROID_HOME=` | `android_sdk_repository` gets an empty stub | GitHub runners leave `ANDROID_HOME` set after the SDK is removed; only `score_baselibs` guards against this itself |
+| Rust coverage extraction is skipped in module context | no `*_rust` coverage rows for Rust-only modules | the `rust_coverage_*` targets are generated in ref_int's `rust_coverage/BUILD` and do not exist inside a module checkout. **Tracked gap, reported explicitly in the aggregate report — not silently absent.** |
+| `score_communication` keeps its own coverage report generator | its `coverage.bazelrc` (imported unconditionally by its `.bazelrc`) emits a pre-built llvm-cov HTML zip, so genhtml is skipped and it contributes no lcov numbers | its llvm-cov pipeline is the path its own CI exercises; forcing uniform gcov is a larger, separately-testable change |
+
+## Known limitations
+
+**Transitive dependencies are not pinned.** `overwrite()` injects an override only for deps the
+module declares itself, so everything reached transitively resolves via the module's own MVS.
+Measured on `score_time`: of ref_int's 155 resolved modules, 127 agree, **3 were built at a
+different version** (`boringssl` 0.20241024.0 vs resolved 0.20251124.0, `zlib` 1.3.1.bcr.5 vs
+1.3.1.bcr.8, `score_rules_imagefs` 0.0.3 vs 0.0.1) and 7 are absent from its graph entirely. So
+the DR-008 claim currently holds for the directly-declared slice, not the whole resolved set.
+
+`verify_stage2_resolution.py` splits these deliberately: a dep ref_int **did** inject an
+override for that still mismatches is an injection failure and **fails** the job; a transitive
+one **warns**. Tracked against Phase 1 (PR #278) — the fix is to inject a `bazel_dep` stub plus
+override for every module in the resolved set. Once that lands, run the verifier with
+`--strict` and fold `stage2-rust` into the base config.
+
+
+
+Stage 2 runs `bazel coverage` with `--build_tests_only`, so **non-test targets are never
+analyzed**. A resolved set in which a module's own binary cannot build will not be caught. Live
+example: `score_time`'s `TimeDaemon`/`TimeSlave` depend on
+`@score_lifecycle_health//src/lifecycle_client_lib`, which does not exist at the lifecycle
+commit ref_int resolves — Stage 2 is green regardless. This is a scope limit, not a pass.

ci/stage2/module.bazelrc

@@ -0,0 +1,91 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+#
+# DR-008 Stage 2 configuration, owned by reference_integration. Layered ON TOP of the
+# module's own .bazelrc via --bazelrc, so ref_int defines the configs it names in
+# known_good.json instead of dangling a name whose meaning lives downstream.
+#
+# Read last, so single-valued flags here win; --extra_toolchains and other accumulating
+# flags add to what the module already registers. Do NOT add --noworkspace_rc — see
+# ci/stage2/README.md for what that broke and why the module's rc must stay.
+
+# rules_android is pulled in transitively (grpc-java -> rules_jvm_external) and evaluates
+# android_sdk_repository, which fails when ANDROID_HOME points at an incomplete SDK, as on
+# CI runners after the disk-cleanup step. Only score_baselibs guards against this itself.
+common --repo_env=ANDROID_HOME=
+
+# ─── stage2-linux-x86_64: emitted unconditionally by quality_runners.py ──────
+build:stage2-linux-x86_64 --host_platform=@score_bazel_platforms//:x86_64-linux-gcc_12.2.0-posix
+build:stage2-linux-x86_64 --platforms=@score_bazel_platforms//:x86_64-linux-gcc_12.2.0-posix
+
+# Test selection and coverage policy. `coverage` inherits `test` inherits `build`.
+# -miri: ref_int registers no miri toolchain. -no-coverage: gcov-instrumenting a TSAN
+# binary reports false races on the non-atomic __gcov* counters.
+test:stage2-linux-x86_64 --build_tests_only
+test:stage2-linux-x86_64 --test_tag_filters=-manual,-miri,-no-coverage
+test:stage2-linux-x86_64 --test_output=errors
+test:stage2-linux-x86_64 --test_summary=testcase
+test:stage2-linux-x86_64 --test_verbose_timeout_warnings
+test:stage2-linux-x86_64 --test_timeout=1200
+test:stage2-linux-x86_64 --nocache_test_results
+
+coverage:stage2-linux-x86_64 --features=coverage
+coverage:stage2-linux-x86_64 --combined_report=lcov
+# Make gcov counter updates atomic so a multithreaded coverage test is race-free.
+coverage:stage2-linux-x86_64 --copt=-fprofile-update=atomic
+coverage:stage2-linux-x86_64 --linkopt=-fprofile-update=atomic
+
+# ─── stage2-gcc: score's gcc x86_64 toolchain ────────────────────────────────
+# Opt-in because the target name is generated by each module's own gcc.toolchain() call:
+# score_communication passes use_base_constraints_only = True, which yields :x86_64-linux
+# instead, and it registers its own cc toolchain unconditionally — so it omits this.
+build:stage2-gcc --extra_toolchains=@score_gcc_x86_64_toolchain//:x86_64-linux-gcc_12.2.0
+
+# ─── stage2-rust: ferrocene Rust toolchain ───────────────────────────────────
+# Opt-in because score_time declares no score_toolchains_rust, so the apparent repo name
+# does not resolve in its graph. Folds into the base once Phase 1 injects a bazel_dep stub
+# for every module in the resolved set (PR #278).
+build:stage2-rust --extra_toolchains=@score_toolchains_rust//toolchains/ferrocene:ferrocene_x86_64_unknown_linux_gnu
+
+# ─── ferrocene-coverage: Rust coverage instrumentation ───────────────────────
+# Applied automatically by quality_runners.py (never opt-in via known_good.json) whenever a
+# module has Rust to extract coverage for — see stage2_config_flags. Without rustc emitting
+# .profraw during the SAME run genhtml/ferrocene_report later reads, extraction finds
+# nothing to report.
+#
+# score_kyron, score_persistency and score_lifecycle_health already define this name
+# identically in their own .bazelrc (verified against their known_good pins); layering
+# repeats the same values there, which is a no-op for a list-accumulating rustc flag set
+# to the same value twice. score_logging's own .bazelrc has no Rust instrumentation config
+# at all, so this is what actually supplies it there. Mirrors ref_int's root .bazelrc.
+build:ferrocene-coverage --@rules_rust//rust/settings:extra_rustc_flag=-Cinstrument-coverage
+build:ferrocene-coverage --@rules_rust//rust/settings:extra_rustc_flag=-Clink-dead-code
+build:ferrocene-coverage --@rules_rust//rust/settings:extra_rustc_flag=-Ccodegen-units=1
+build:ferrocene-coverage --@rules_rust//rust/settings:extra_rustc_flag=-Cdebuginfo=2
+build:ferrocene-coverage --@rules_rust//rust/settings:extra_exec_rustc_flag=-Cinstrument-coverage
+build:ferrocene-coverage --@rules_rust//rust/settings:extra_exec_rustc_flag=-Clink-dead-code
+build:ferrocene-coverage --@rules_rust//rust/settings:extra_exec_rustc_flag=-Ccodegen-units=1
+build:ferrocene-coverage --@rules_rust//rust/settings:extra_exec_rustc_flag=-Cdebuginfo=2
+test:ferrocene-coverage --run_under=@score_tooling//coverage:llvm_profile_wrapper
+
+# score_persistency's own rust_coverage_config (known_good.json) is ferrocene-coverage-per,
+# not plain ferrocene-coverage — its own .bazelrc does not define this name at all (checked
+# against its known_good pin), so this is the sole source. Only the two score_baselibs
+# settings are ported: they're real deps persistency itself declares, so they resolve
+# inside its own checkout same as ref_int's. ref_int's root .bazelrc also carries a third,
+# @score_logging-relative one, but that's ref_int-graph-specific in a way these two aren't —
+# and persistency's own known_good.json `extra_test_config` already passes it directly on
+# every Stage-2 invocation, so nothing is lost by leaving it out here.
+build:ferrocene-coverage-per --config=ferrocene-coverage
+build:ferrocene-coverage-per --@score_baselibs//src/log:safety_level=qm
+build:ferrocene-coverage-per --@score_baselibs//score/json:base_library=nlohmann

docs/verification_report/coverage_summary.md

@@ -1,3 +1,5 @@
 # Coverage Analysis Summary
 
-## Template for a table with Coverage execution summary
+| module | lines | functions | branches |
+| --- | --- | --- | --- |
+| score_communication_cpp |  |  |  |

docs/verification_report/unit_test_summary.md

@@ -1,3 +1,5 @@
 # Unit Test Execution Summary
 
-## Template for a table with Unit Test execution summary
+| module | passed | failed | skipped | total |
+| --- | --- | --- | --- | --- |
+| score_communication | 3301 | 0 | 0 | 3301 |