diff --git a/bleep-bsp-tests/src/scala/bleep/analysis/BuildStateReducerTest.scala b/bleep-bsp-tests/src/scala/bleep/analysis/BuildStateReducerTest.scala index b31c8200c..2c56d5afd 100644 --- a/bleep-bsp-tests/src/scala/bleep/analysis/BuildStateReducerTest.scala +++ b/bleep-bsp-tests/src/scala/bleep/analysis/BuildStateReducerTest.scala @@ -20,6 +20,44 @@ class BuildStateReducerTest extends AnyFunSuite with Matchers { private def reduce(events: BuildEvent*): BuildState = events.foldLeft(BuildState.empty)(BuildStateReducer.reduce) + // ========================================================================== + // Task-time / parallelism accounting + // ========================================================================== + + test("totalTaskTimeMs counts a suite's whole fork occupancy, not the sum of its test durations") { + // A slow-booting integration suite spends most of its wall time on setup — starting an app and + // its containers — before any test method runs. The suite occupied its fork for 27s + // (SuiteStarted..SuiteFinished); the two test methods report 5ms each. Task time must be the 27s, + // or parallelism reads ~1x under fan-out. + val state = reduce( + BuildEvent.SuiteStarted(cpn("proj"), sn("com.example.SlowBootIT"), ts), + BuildEvent.TestFinished(cpn("proj"), sn("com.example.SlowBootIT"), tn("a"), TestStatus.Passed, 5, None, None, ts + 26000, None), + BuildEvent.TestFinished(cpn("proj"), sn("com.example.SlowBootIT"), tn("b"), TestStatus.Passed, 5, None, None, ts + 26500, None), + BuildEvent.SuiteFinished(cpn("proj"), sn("com.example.SlowBootIT"), SuiteOutcome.Executed(2, 0, 0, 0), 27000, ts + 27000) + ) + state.totalTaskTimeMs shouldBe 27000L + } + + test("totalTaskTimeMs sums two overlapping suites so parallelism exceeds wall time") { + // Two suites, each occupying a fork for 10s, started together: 20s of task time in 10s of wall. + val state = reduce( + BuildEvent.SuiteStarted(cpn("a"), sn("A"), ts), + BuildEvent.SuiteStarted(cpn("b"), sn("B"), ts), + BuildEvent.SuiteFinished(cpn("a"), sn("A"), SuiteOutcome.Executed(1, 0, 0, 0), 10000, ts + 10000), + BuildEvent.SuiteFinished(cpn("b"), sn("B"), SuiteOutcome.Executed(1, 0, 0, 0), 10000, ts + 10000) + ) + state.totalTaskTimeMs shouldBe 20000L + } + + test("a suite whose end is reported by both SuiteFinished and SuiteError is counted once") { + val state = reduce( + BuildEvent.SuiteStarted(cpn("proj"), sn("S"), ts), + BuildEvent.SuiteFinished(cpn("proj"), sn("S"), SuiteOutcome.Executed(0, 1, 0, 0), 8000, ts + 8000), + BuildEvent.SuiteError(cpn("proj"), sn("S"), "exited 1", bleep.bsp.protocol.ProcessExit.ExitCode(1), 8000, ts + 8001) + ) + state.totalTaskTimeMs shouldBe 8000L + } + // ========================================================================== // SuiteFinished synthetic failure tests // ========================================================================== diff --git a/bleep-bsp-tests/src/scala/bleep/analysis/BuildSummaryVerdictTest.scala b/bleep-bsp-tests/src/scala/bleep/analysis/BuildSummaryVerdictTest.scala index df9508ed3..c4886e1c4 100644 --- a/bleep-bsp-tests/src/scala/bleep/analysis/BuildSummaryVerdictTest.scala +++ b/bleep-bsp-tests/src/scala/bleep/analysis/BuildSummaryVerdictTest.scala @@ -23,6 +23,40 @@ class BuildSummaryVerdictTest extends AnyFunSuite with Matchers { state.toSummary(durationMs = 0L, wasCancelled = false).toEither } + /** Fold in the authoritative per-run counts the server sends as its response payload, exactly as `bleep test` and the MCP tools do via + * [[bleep.history.TranscriptFormat]] — the path where `suitesTotal`/`suitesCompleted` are set from discovery rather than accumulated from streamed events. + */ + private def verdictWithResult(events: List[E], trr: BleepBspProtocol.TestRunResult): Either[bleep.BleepException, Unit] = { + val trrEvent = BuildEvent.TestRunCompleted( + totalPassed = trr.totalPassed, + totalFailed = trr.totalFailed, + totalSkipped = trr.totalSkipped, + totalIgnored = trr.totalIgnored, + suitesTotal = trr.suitesTotal, + suitesCompleted = trr.suitesCompleted, + suitesFailed = trr.suitesFailed, + suitesCancelled = trr.suitesCancelled, + durationMs = trr.durationMs, + timestamp = 0L + ) + val state = (events.flatMap(BuildEvent.fromProtocol) :+ trrEvent).foldLeft(BuildState.empty)(BuildStateReducer.reduce) + state.toSummary(durationMs = 0L, wasCancelled = false).toEither + } + + private def runResult(passed: Int, suitesTotal: Int, suitesCompleted: Int): BleepBspProtocol.TestRunResult = + BleepBspProtocol.TestRunResult( + totalPassed = passed, + totalFailed = 0, + totalSkipped = 0, + totalIgnored = 0, + suitesTotal = suitesTotal, + suitesCompleted = suitesCompleted, + suitesFailed = 0, + suitesCancelled = 0, + durationMs = 0L, + historyId = None + ) + private def leftMessage(events: List[E]): String = verdict(events).left.map(_.getMessage) match { case Left(msg) => msg @@ -209,6 +243,38 @@ class BuildSummaryVerdictTest extends AnyFunSuite with Matchers { ) shouldBe Right(()) } + test("a run that completed fewer suites than it discovered is not a pass, even with zero failures") { + // The `bleep test dfmt/test` report: 64 passed, 0 failed, yet only 10 of 29 discovered suites ran. `failed: 0` made it look green; the 19 that never + // reported a result are the whole story. The verdict must fail so CI's "test every project" gate cannot mistake a fraction for full coverage. + val msg = verdictWithResult(Nil, runResult(passed = 64, suitesTotal = 29, suitesCompleted = 10)).left.map(_.getMessage) match { + case Left(m) => m + case Right(()) => fail("a run that left 19 of 29 suites unaccounted must be judged a failure") + } + msg should include("did not finish") + msg should include("10 of 29") + msg should include("19") + } + + test("the same shortfall from streamed SuiteStarted/SuiteFinished events (batch announces every suite up front) also fails") { + // Per-project batch mode announces a SuiteStarted for every suite when the batch begins, then a SuiteFinished as each completes. If the fork stops after a + // subset, `suitesTotal` (announced) exceeds `suitesCompleted` (finished) with no failure event anywhere — the verdict has to catch it from the counts alone. + verdict( + List( + E.SuiteStarted(proj("app"), SuiteName("A"), timestamp = 1L), + E.SuiteStarted(proj("app"), SuiteName("B"), timestamp = 1L), + E.SuiteStarted(proj("app"), SuiteName("C"), timestamp = 1L) + ) ++ passedTest("app").map { + case ts: E.TestFinished => ts.copy(suite = SuiteName("A")) + case sf: E.SuiteFinished => sf.copy(suite = SuiteName("A")) + case other => other + } + ).isLeft shouldBe true + } + + test("a run where every discovered suite completed is Right") { + verdictWithResult(passedTest("app"), runResult(passed = 42, suitesTotal = 29, suitesCompleted = 29)) shouldBe Right(()) + } + test("a clean run is Right") { verdict(compileFinished("app", CompileStatus.Success, skippedBecause = None) +: passedTest("app")) shouldBe Right(()) } diff --git a/bleep-bsp-tests/src/scala/bleep/analysis/LinkDagIntegrationTest.scala b/bleep-bsp-tests/src/scala/bleep/analysis/LinkDagIntegrationTest.scala index c5ae08a37..9936c2497 100644 --- a/bleep-bsp-tests/src/scala/bleep/analysis/LinkDagIntegrationTest.scala +++ b/bleep-bsp-tests/src/scala/bleep/analysis/LinkDagIntegrationTest.scala @@ -269,6 +269,7 @@ class LinkDagIntegrationTest extends AnyFunSuite with Matchers { }, discover = (_, _, _) => sys.error("DiscoverTask should not appear in a link DAG"), test = (_, _, _) => sys.error("TestSuiteTask should not appear in a link DAG"), + testBatch = (_, _) => sys.error("TestBatchTask should not appear in a link DAG"), sourcegen = (_, _) => sys.error("SourcegenTask should not appear here"), annotationProcessor = (_, _) => sys.error("ResolveAnnotationProcessorsTask should not appear here"), symbolProcessor = (_, _) => sys.error("ResolveSymbolProcessorsTask should not appear here") @@ -316,6 +317,7 @@ class LinkDagIntegrationTest extends AnyFunSuite with Matchers { ), discover = (_, _, _) => sys.error("DiscoverTask should not appear in a link DAG"), test = (_, _, _) => sys.error("TestSuiteTask should not appear in a link DAG"), + testBatch = (_, _) => sys.error("TestBatchTask should not appear in a link DAG"), sourcegen = (_, _) => sys.error("SourcegenTask should not appear here"), annotationProcessor = (_, _) => sys.error("ResolveAnnotationProcessorsTask should not appear here"), symbolProcessor = (_, _) => sys.error("ResolveSymbolProcessorsTask should not appear here") @@ -361,8 +363,9 @@ class LinkDagIntegrationTest extends AnyFunSuite with Matchers { mayAdmitCompile = _ => IO.pure(true), compile = (_, _) => IO.pure(TaskResult.Success), link = (_, _) => IO.pure((TaskResult.Failure("Link error", List.empty), LinkResult.Failure("Link error", List.empty))), - discover = (_, _, _) => IO.pure((TaskResult.Success, TaskDag.DiscoveryResult(Nil, 0, isTestProject = false))), + discover = (_, _, _) => IO.pure((TaskResult.Success, TaskDag.DiscoveryResult(Nil, 0, isTestProject = false, suiteParallelism = None, batches = Nil))), test = (_, _, _) => sys.error("TestSuiteTask should not appear in this DAG"), + testBatch = (_, _) => sys.error("TestBatchTask should not appear in this DAG"), sourcegen = (_, _) => sys.error("SourcegenTask should not appear here"), annotationProcessor = (_, _) => sys.error("ResolveAnnotationProcessorsTask should not appear here"), symbolProcessor = (_, _) => sys.error("ResolveSymbolProcessorsTask should not appear here") diff --git a/bleep-bsp-tests/src/scala/bleep/analysis/MaxConcurrentSuitesDagTest.scala b/bleep-bsp-tests/src/scala/bleep/analysis/MaxConcurrentSuitesDagTest.scala new file mode 100644 index 000000000..ff4f0ccb8 --- /dev/null +++ b/bleep-bsp-tests/src/scala/bleep/analysis/MaxConcurrentSuitesDagTest.scala @@ -0,0 +1,134 @@ +package bleep.analysis + +import bleep.bsp.{Outcome, TaskDag} +import bleep.bsp.TaskDag.{TaskId, _} +import bleep.model.{CrossProjectName, ProjectName, SuiteName} +import bleep.testing.FrameworkSelection +import cats.effect.{IO, Ref} +import cats.effect.std.Queue +import cats.effect.unsafe.implicits.global +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +/** `maxConcurrentSuites` turns a project's discovered suites into round-robin chains of ordering-only `runAfter` edges. The properties that matter: + * + * - at bound 1 the suites run strictly sequentially in alphabetical order — surefire's usual class order, which schema-bootstrapping test setups rely on + * - a failing suite does NOT skip the rest of its chain: `runAfter` is ordering, not failure propagation + * - without a bound no chains exist and every suite is schedulable at once + */ +class MaxConcurrentSuitesDagTest extends AnyFunSuite with Matchers { + + private def testMachine(cpu: Int): bleep.MachineResources = + bleep.MachineResources.create(totalCpu = cpu, totalMemoryMb = 64 * 1024, logger = ryddig.TypedLogger.DevNull, longWaitWarnMs = 60000L) + + private def projectName(name: String): CrossProjectName = + CrossProjectName(ProjectName(name), None) + + private val selection = FrameworkSelection.JUnitPlatform("junit") + + /** Runs a one-project test DAG whose discovery yields `suiteNames` (deliberately unsorted) under the given parallelism bound. Returns the interleaved + * start/finish event log and the final dag. + */ + private def run( + project: CrossProjectName, + suiteNames: List[String], + parallelism: Option[Int], + failingSuites: Set[String] + ): (List[String], Dag) = { + val dag = TaskDag.buildTestDag( + Set(project), + BuildContext( + allProjectDeps = Map.empty, + platforms = Map(project -> LinkPlatform.Jvm), + sourcegen = SourcegenPlan.empty, + apPlan = AnnotationProcessorPlan.empty, + kspPlan = SymbolProcessorPlan.empty, + testProjects = Set(project) + ) + ) + + (for { + log <- Ref.of[IO, List[String]](Nil) + executor = TaskDag.executor( + Handlers( + mayAdmitCompile = _ => IO.pure(true), + compile = (_, _) => IO.pure(TaskResult.Success), + link = (_, _) => sys.error("no link on JVM"), + discover = (_, _, _) => + IO.pure( + ( + TaskResult.Success, + TaskDag.DiscoveryResult( + suiteNames.map(_ -> selection), + suiteNames.size, + isTestProject = true, + suiteParallelism = parallelism, + batches = Nil + ) + ) + ), + test = (task, _, _) => + for { + _ <- log.update(_ :+ s"start:${task.suiteName.value}") + // yield so that, were another suite schedulable, it could interleave between our start and finish + _ <- IO.cede + _ <- log.update(_ :+ s"finish:${task.suiteName.value}") + } yield if (failingSuites(task.suiteName.value)) TaskResult.Failure(s"${task.suiteName.value} failed", Nil) else TaskResult.Success, + testBatch = (_, _) => sys.error("no test batch here"), + sourcegen = (_, _) => sys.error("no sourcegen here"), + annotationProcessor = (_, _) => sys.error("no annotation processors here"), + symbolProcessor = (_, _) => sys.error("no symbol processors here") + ) + ) + eventQueue <- Queue.unbounded[IO, Option[DagEvent]] + killSignal <- Outcome.neverKillSignal + finalDag <- executor.execute(dag, testMachine(4), TaskDag.ForkHeaps.default, eventQueue, killSignal) + events <- log.get + } yield (events, finalDag)).unsafeRunSync() + } + + test("suiteParallelism=1: suites run sequentially in alphabetical order") { + val project = projectName("app-test") + val (events, dag) = run(project, List("b.Suite", "c.Suite", "a.Suite"), parallelism = Some(1), failingSuites = Set.empty) + + events shouldBe List("start:a.Suite", "finish:a.Suite", "start:b.Suite", "finish:b.Suite", "start:c.Suite", "finish:c.Suite") + dag.completed should contain allOf ( + TaskId.Test(project, SuiteName("a.Suite")), + TaskId.Test(project, SuiteName("b.Suite")), + TaskId.Test( + project, + SuiteName("c.Suite") + ) + ) + } + + test("suiteParallelism=1: a failing suite does not skip the rest of the chain") { + val project = projectName("app-test") + val (events, dag) = run(project, List("a.Suite", "b.Suite", "c.Suite"), parallelism = Some(1), failingSuites = Set("a.Suite")) + + events shouldBe List("start:a.Suite", "finish:a.Suite", "start:b.Suite", "finish:b.Suite", "start:c.Suite", "finish:c.Suite") + dag.failed should contain(TaskId.Test(project, SuiteName("a.Suite"))) + dag.skipped shouldBe empty + dag.completed should contain allOf (TaskId.Test(project, SuiteName("b.Suite")), TaskId.Test(project, SuiteName("c.Suite"))) + } + + test("suiteParallelism=2: each suite waits only for the one two places ahead of it") { + val project = projectName("app-test") + val (events, dag) = run(project, List("a.Suite", "b.Suite", "c.Suite"), parallelism = Some(2), failingSuites = Set.empty) + + // c chains after a (index 2 waits for index 2-2=0); b is unchained + events.indexOf("start:c.Suite") should be > events.indexOf("finish:a.Suite") + dag.completed.count { case TaskId.Test(_, _) => true; case _ => false } shouldBe 3 + } + + test("no suiteParallelism: no ordering edges, all suites complete") { + val project = projectName("app-test") + val (events, dag) = run(project, List("b.Suite", "a.Suite"), parallelism = None, failingSuites = Set.empty) + + events should have size 4 + dag.completed should contain allOf (TaskId.Test(project, SuiteName("a.Suite")), TaskId.Test(project, SuiteName("b.Suite"))) + dag.tasks.values.collect { case t: TestSuiteTask => t }.foreach { t => + t.runAfter shouldBe empty + } + } +} diff --git a/bleep-bsp-tests/src/scala/bleep/analysis/SourcegenDagIntegrationTest.scala b/bleep-bsp-tests/src/scala/bleep/analysis/SourcegenDagIntegrationTest.scala index 35b8ad202..b5d7d93b8 100644 --- a/bleep-bsp-tests/src/scala/bleep/analysis/SourcegenDagIntegrationTest.scala +++ b/bleep-bsp-tests/src/scala/bleep/analysis/SourcegenDagIntegrationTest.scala @@ -311,6 +311,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { link = (_, _) => sys.error("LinkTask should not appear here"), discover = (_, _, _) => sys.error("DiscoverTask should not appear here"), test = (_, _, _) => sys.error("TestSuiteTask should not appear here"), + testBatch = (_, _) => sys.error("TestBatchTask should not appear here"), sourcegen = (t, _) => IO(order.add(s"sourcegen:${t.script.main}"): Unit).as(TaskResult.Success), annotationProcessor = (_, _) => sys.error("ResolveAnnotationProcessorsTask should not appear here"), symbolProcessor = (_, _) => sys.error("ResolveSymbolProcessorsTask should not appear here") @@ -362,6 +363,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { link = (_, _) => sys.error("LinkTask should not appear here"), discover = (_, _, _) => sys.error("DiscoverTask should not appear here"), test = (_, _, _) => sys.error("TestSuiteTask should not appear here"), + testBatch = (_, _) => sys.error("TestBatchTask should not appear here"), sourcegen = (_, _) => IO(sourcegenCalled.set(true)).as(TaskResult.Success), annotationProcessor = (_, _) => sys.error("ResolveAnnotationProcessorsTask should not appear here"), symbolProcessor = (_, _) => sys.error("ResolveSymbolProcessorsTask should not appear here") @@ -413,6 +415,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { link = (_, _) => sys.error("LinkTask should not appear here"), discover = (_, _, _) => sys.error("DiscoverTask should not appear here"), test = (_, _, _) => sys.error("TestSuiteTask should not appear here"), + testBatch = (_, _) => sys.error("TestBatchTask should not appear here"), sourcegen = (_, _) => IO.pure(TaskResult.Failure("script threw", Nil)), annotationProcessor = (_, _) => sys.error("ResolveAnnotationProcessorsTask should not appear here"), symbolProcessor = (_, _) => sys.error("ResolveSymbolProcessorsTask should not appear here") @@ -463,6 +466,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { link = (_, _) => sys.error("LinkTask should not appear here"), discover = (_, _, _) => sys.error("DiscoverTask should not appear here"), test = (_, _, _) => sys.error("TestSuiteTask should not appear here"), + testBatch = (_, _) => sys.error("TestBatchTask should not appear here"), sourcegen = (_, _) => IO.pure(TaskResult.Success), // up-to-date fast path annotationProcessor = (_, _) => sys.error("ResolveAnnotationProcessorsTask should not appear here"), symbolProcessor = (_, _) => sys.error("ResolveSymbolProcessorsTask should not appear here") @@ -511,6 +515,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { link = (_, _) => sys.error("LinkTask should not appear here"), discover = (_, _, _) => sys.error("DiscoverTask should not appear here"), test = (_, _, _) => sys.error("TestSuiteTask should not appear here"), + testBatch = (_, _) => sys.error("TestBatchTask should not appear here"), sourcegen = (_, _) => IO.pure(TaskResult.Success), annotationProcessor = (_, _) => sys.error("ResolveAnnotationProcessorsTask should not appear here"), symbolProcessor = (_, _) => sys.error("ResolveSymbolProcessorsTask should not appear here") @@ -565,6 +570,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { link = (_, _) => sys.error("LinkTask should not appear here"), discover = (_, _, _) => sys.error("DiscoverTask should not appear here"), test = (_, _, _) => sys.error("TestSuiteTask should not appear here"), + testBatch = (_, _) => sys.error("TestBatchTask should not appear here"), sourcegen = (_, _) => IO.pure(TaskResult.Failure("boom", Nil)), annotationProcessor = (_, _) => sys.error("ResolveAnnotationProcessorsTask should not appear here"), symbolProcessor = (_, _) => sys.error("ResolveSymbolProcessorsTask should not appear here") @@ -617,6 +623,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { link = (_, _) => sys.error("LinkTask should not appear here"), discover = (_, _, _) => sys.error("DiscoverTask should not appear here"), test = (_, _, _) => sys.error("TestSuiteTask should not appear here"), + testBatch = (_, _) => sys.error("TestBatchTask should not appear here"), sourcegen = (_, taskKill) => taskKill.get.map(reason => TaskResult.Killed(reason)), annotationProcessor = (_, _) => sys.error("ResolveAnnotationProcessorsTask should not appear here"), symbolProcessor = (_, _) => sys.error("ResolveSymbolProcessorsTask should not appear here") @@ -666,6 +673,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { link = (_, _) => sys.error("LinkTask should not appear here"), discover = (_, _, _) => sys.error("DiscoverTask should not appear here"), test = (_, _, _) => sys.error("TestSuiteTask should not appear here"), + testBatch = (_, _) => sys.error("TestBatchTask should not appear here"), sourcegen = (_, _) => record("sourcegen:start") >> IO.sleep(scala.concurrent.duration.DurationInt(100).millis) >> @@ -714,6 +722,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { link = (_, _) => sys.error("LinkTask should not appear here"), discover = (_, _, _) => sys.error("DiscoverTask should not appear here"), test = (_, _, _) => sys.error("TestSuiteTask should not appear here"), + testBatch = (_, _) => sys.error("TestBatchTask should not appear here"), sourcegen = (_, _) => sys.error("SourcegenTask should not appear here"), annotationProcessor = (_, _) => sys.error("ResolveAnnotationProcessorsTask should not appear here"), symbolProcessor = (_, _) => sys.error("ResolveSymbolProcessorsTask should not appear here") @@ -768,6 +777,7 @@ class SourcegenDagIntegrationTest extends AnyFunSuite with Matchers { link = (_, _) => sys.error("LinkTask should not appear here"), discover = (_, _, _) => sys.error("DiscoverTask should not appear here"), test = (_, _, _) => sys.error("TestSuiteTask should not appear here"), + testBatch = (_, _) => sys.error("TestBatchTask should not appear here"), sourcegen = (_, _) => IO.raiseError(new RuntimeException("generator blew up")), annotationProcessor = (_, _) => sys.error("ResolveAnnotationProcessorsTask should not appear here"), symbolProcessor = (_, _) => sys.error("ResolveSymbolProcessorsTask should not appear here") diff --git a/bleep-bsp-tests/src/scala/bleep/analysis/SymbolProcessorDagIntegrationTest.scala b/bleep-bsp-tests/src/scala/bleep/analysis/SymbolProcessorDagIntegrationTest.scala index 2769c5399..ece81c242 100644 --- a/bleep-bsp-tests/src/scala/bleep/analysis/SymbolProcessorDagIntegrationTest.scala +++ b/bleep-bsp-tests/src/scala/bleep/analysis/SymbolProcessorDagIntegrationTest.scala @@ -112,6 +112,7 @@ class SymbolProcessorDagIntegrationTest extends AnyFunSuite with Matchers { link = (_, _) => sys.error("LinkTask should not appear here"), discover = (_, _, _) => sys.error("DiscoverTask should not appear here"), test = (_, _, _) => sys.error("TestSuiteTask should not appear here"), + testBatch = (_, _) => sys.error("TestBatchTask should not appear here"), sourcegen = (_, _) => sys.error("SourcegenTask should not appear here"), annotationProcessor = (_, _) => sys.error("ResolveAnnotationProcessorsTask should not appear here"), symbolProcessor = (kspt, _) => IO { timeline.add(s"ksp:${kspt.project.value}"); (TaskResult.Success, 2) } @@ -145,6 +146,7 @@ class SymbolProcessorDagIntegrationTest extends AnyFunSuite with Matchers { link = (_, _) => sys.error("LinkTask should not appear here"), discover = (_, _, _) => sys.error("DiscoverTask should not appear here"), test = (_, _, _) => sys.error("TestSuiteTask should not appear here"), + testBatch = (_, _) => sys.error("TestBatchTask should not appear here"), sourcegen = (_, _) => sys.error("SourcegenTask should not appear here"), annotationProcessor = (_, _) => sys.error("ResolveAnnotationProcessorsTask should not appear here"), symbolProcessor = (_, _) => IO((TaskResult.Failure("simulated KSP misconfig", Nil), 0)) diff --git a/bleep-bsp-tests/src/scala/bleep/bsp/DeclaredTestFrameworkTest.scala b/bleep-bsp-tests/src/scala/bleep/bsp/DeclaredTestFrameworkTest.scala index 3d0ea2e27..f690d9762 100644 --- a/bleep-bsp-tests/src/scala/bleep/bsp/DeclaredTestFrameworkTest.scala +++ b/bleep-bsp-tests/src/scala/bleep/bsp/DeclaredTestFrameworkTest.scala @@ -2,6 +2,7 @@ package bleep.bsp import bleep.model import bleep.testing.FrameworkSelection +import ryddig.TypedLogger import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import sbt.testing._ @@ -60,7 +61,8 @@ class DeclaredTestFrameworkTest extends AnyFunSuite with Matchers { project, classesDirContaining(classOf[MarkerSuite]), currentClasspath, - declaredFrameworks = List(classOf[MarkerFramework].getName) + declaredFrameworks = List(classOf[MarkerFramework].getName), + logger = TypedLogger.DevNull ) suites.map(_.className) shouldBe List(classOf[MarkerSuite].getName) @@ -74,7 +76,8 @@ class DeclaredTestFrameworkTest extends AnyFunSuite with Matchers { project, classesDirContaining(classOf[MarkerSuite]), currentClasspath, - declaredFrameworks = List("com.novocode.junit.JUnitFramework") + declaredFrameworks = List("com.novocode.junit.JUnitFramework"), + logger = TypedLogger.DevNull ) } thrown.getMessage should include("mytest") @@ -88,7 +91,8 @@ class DeclaredTestFrameworkTest extends AnyFunSuite with Matchers { project, classesDirContaining(classOf[MarkerSuite]), currentClasspath, - declaredFrameworks = Nil + declaredFrameworks = Nil, + logger = TypedLogger.DevNull ) withClue("nothing built in should match a class whose only marker is a base class bleep does not know: ")(suites shouldBe empty) } diff --git a/bleep-bsp-tests/src/scala/bleep/testing/ExitAttributionTest.scala b/bleep-bsp-tests/src/scala/bleep/testing/ExitAttributionTest.scala index fa173dc53..30816e9ea 100644 --- a/bleep-bsp-tests/src/scala/bleep/testing/ExitAttributionTest.scala +++ b/bleep-bsp-tests/src/scala/bleep/testing/ExitAttributionTest.scala @@ -40,6 +40,40 @@ class ExitAttributionTest extends AnyFunSuite with Matchers { } } +/** Capturing what a fork wrote before it died — the diagnostic that turns "N suites never reported a result" into a reason. + * + * The failure that motivated this: a JVM handed an option it rejects prints "Unrecognized VM option ..." to stderr and exits non-zero, but bleep read only the + * bytes `available()` at the instant the process was seen dead — which is usually zero, because the flush lands a beat later — and dropped the message. + * Draining an exited fork to EOF is what actually gets it. Tested against real processes (a bare `Process`, no pool, no BSP server), because the bug lived + * entirely in how the streams of a just-exited process are read. + */ +class DescribeChildOutputTest extends AnyFunSuite with Matchers { + + private def run(cmd: List[String]): Process = { + val p = new ProcessBuilder(cmd*).start() + p.waitFor() + p + } + + test("an exited fork's stderr is drained to EOF, not just what was already available at exit") { + // The exact failure this exists for, without depending on any particular JVM: a process that writes to stderr and exits. `available()` reports 0 the instant + // the process is seen dead, so the old read missed this; draining to EOF gets it. (A real bad-JVM-option failure — "Unrecognized VM option ..." — is this same + // shape; that end-to-end path, on the project's resolved JVM, is covered by an integration test rather than here.) + val marker = "STARTUP-FAILURE-MARKER" + val cmd = + if (scala.util.Properties.isWin) List("cmd", "/c", s"echo $marker 1>&2 & exit 3") + else List("/bin/sh", "-c", s"echo $marker 1>&2; exit 3") + val described = JvmPool.describeChildOutput(run(cmd), exited = true) + described should include("stderr") + described should include(marker) + } + + test("a fork that wrote nothing says so, rather than returning an empty diagnostic") { + val cmd = if (scala.util.Properties.isWin) List("cmd", "/c", "exit 0") else List("/bin/sh", "-c", "exit 0") + JvmPool.describeChildOutput(run(cmd), exited = true) should include("no output") + } +} + /** The start-stagger scales with the run, rather than being a constant tuned on one machine. */ class MaxConcurrentStartsTest extends AnyFunSuite with Matchers { diff --git a/bleep-bsp/src/scala/bleep/bsp/BspMetrics.scala b/bleep-bsp/src/scala/bleep/bsp/BspMetrics.scala index 8c08a82e2..b308bc605 100644 --- a/bleep-bsp/src/scala/bleep/bsp/BspMetrics.scala +++ b/bleep-bsp/src/scala/bleep/bsp/BspMetrics.scala @@ -396,8 +396,17 @@ object BspMetrics { def onForkStart(pid: Long, label: String, xmxMb: Option[Long]): Unit = recordForkStart(pid, label, xmxMb) def onForkEnd(pid: Long, lifetimeMs: Long, exit: String, killedByUs: Option[String]): Unit = recordForkEnd(pid, lifetimeMs, exit, killedByUs) def onForkReused(pid: Long, label: String): Unit = recordForkReused(pid, label) + def onForkKill(pid: Long, reason: String, wasAlive: Boolean, graceMillis: Long): Unit = recordForkKill(pid, reason, wasAlive, graceMillis) } + /** Every bleep-initiated kill, so a fork's death can be attributed after the fact: a `fork_end` for a pid with no preceding `fork_kill` was not bleep's + * doing. `was_alive=false` marks a redundant escalation over an already-dead fork. + */ + def recordForkKill(pid: Long, reason: String, wasAlive: Boolean, graceMillis: Long): Unit = + writeEvent( + s"""{"type":"fork_kill","ts":${now()},"pid":$pid,"reason":"${esc(reason)}","was_alive":$wasAlive,"grace_ms":$graceMillis}""" + ) + def recordForkStart(pid: Long, label: String, xmxMb: Option[Long]): Unit = writeEvent( s"""{"type":"fork_start","ts":${now()},"pid":$pid,"label":"${esc(label)}","xmx_mb":${xmxMb.getOrElse(-1L)}}""" diff --git a/bleep-bsp/src/scala/bleep/bsp/ClasspathTestDiscovery.scala b/bleep-bsp/src/scala/bleep/bsp/ClasspathTestDiscovery.scala index 95b970003..a1f1f51b9 100644 --- a/bleep-bsp/src/scala/bleep/bsp/ClasspathTestDiscovery.scala +++ b/bleep-bsp/src/scala/bleep/bsp/ClasspathTestDiscovery.scala @@ -2,6 +2,7 @@ package bleep.bsp import bleep.model.CrossProjectName import bleep.testing.FrameworkSelection +import ryddig.Logger import sbt.testing._ import java.io.File @@ -10,6 +11,7 @@ import java.net.URLClassLoader import java.nio.file.{Files, Path} import scala.jdk.CollectionConverters._ import scala.util.Try +import scala.util.control.NonFatal /** Discovered test suite ready for execution. * @@ -191,7 +193,8 @@ object ClasspathTestDiscovery { project: CrossProjectName, classesDir: Path, classpath: List[Path], - declaredFrameworks: List[String] + declaredFrameworks: List[String], + logger: Logger ): List[DiscoveredTestSuite] = { if (!Files.isDirectory(classesDir)) { return Nil @@ -206,14 +209,14 @@ object ClasspathTestDiscovery { val classNames = classFiles.map(f => classFileToClassName(classesDir, f)) // Strategy 1: sbt-testing Framework fingerprints - val frameworkDiscovered = discoverViaFrameworks(project, classNames, classLoader, declaredFrameworks) + val frameworkDiscovered = discoverViaFrameworks(project, classNames, classLoader, declaredFrameworks, logger) // Get classes not yet discovered val discoveredClassNames = frameworkDiscovered.map(_.className).toSet val remainingClasses = classNames.filterNot(discoveredClassNames.contains) // Strategy 2: Direct annotation scanning (JUnit 4/5, TestNG, kotlin.test) - val annotationDiscovered = discoverViaAnnotations(project, remainingClasses, classLoader) + val annotationDiscovered = discoverViaAnnotations(project, remainingClasses, classLoader, logger) // Get classes still not discovered val annotationDiscoveredNames = annotationDiscovered.map(_.className).toSet @@ -247,7 +250,8 @@ object ClasspathTestDiscovery { project: CrossProjectName, classNames: List[String], classLoader: URLClassLoader, - declaredFrameworks: List[String] + declaredFrameworks: List[String], + logger: Logger ): List[DiscoveredTestSuite] = { val frameworks = loadFrameworks(project, classLoader, declaredFrameworks) @@ -261,7 +265,7 @@ object ClasspathTestDiscovery { } classNames.flatMap { className => - matchFingerprint(className, classLoader, fingerprintsByFramework).map { case (fw, _) => + matchFingerprint(className, classLoader, fingerprintsByFramework, logger).map { case (fw, _) => DiscoveredTestSuite(project, className, selectionForFramework(fw)) } } @@ -317,58 +321,82 @@ object ClasspathTestDiscovery { private def instantiateFramework(classLoader: URLClassLoader, fqn: String): Framework = classLoader.loadClass(fqn).getDeclaredConstructor().newInstance().asInstanceOf[Framework] + /** Reflecting over a project's compiled classes can land on one whose supertypes or annotations name a type no longer on the classpath — almost always an + * orphaned `.class` an incremental compile left behind after a package rename, whose source is gone but whose output lingers. `Class.getDeclaredMethods` / + * `getAnnotations` / `Class.isAssignableFrom` then throw a `LinkageError` (`NoClassDefFoundError`). That is an `Error`, not an `Exception`, so + * `scala.util.Try` and `scala.util.control.NonFatal` do NOT hold it: it used to escape discovery, tear down the whole client handler, and surface to the + * user as "BSP server connection lost (server may have crashed)" — pointing at memory when the real cause was a stale build. One un-reflectable class must + * cost only that class: skip it, name it and the likely remedy, and let discovery continue. + * + * `LinkageError` is caught deliberately and separately from `NonFatal`; it is not a `VirtualMachineError`, so this does not swallow `OutOfMemoryError` or + * `StackOverflowError`, which still propagate. + */ + private def skipUnreflectable[A](className: String, logger: Logger)(f: => Option[A]): Option[A] = + try f + catch { + case e: LinkageError => + logger.warn( + s"test discovery skipping $className: ${e.getClass.getSimpleName}: ${e.getMessage}. " + + "This is usually an orphaned .class an incremental compile left after a rename — `bleep clean` on that project clears it." + ) + None + case NonFatal(_) => None + } + /** Try to match a class against fingerprints */ private def matchFingerprint( className: String, classLoader: ClassLoader, - fingerprints: List[(Framework, Fingerprint)] - ): Option[(Framework, Fingerprint)] = { - val clazz = Try(classLoader.loadClass(className)).toOption - - clazz.flatMap { cls => - // Skip abstract classes and interfaces - if (Modifier.isAbstract(cls.getModifiers) || cls.isInterface) None - else - fingerprints.find { case (_, fp) => - fp match { - case sfp: SubclassFingerprint => - Try { - val superclass = classLoader.loadClass(sfp.superclassName()) - // A module fingerprint describes the object's own class, so every question — does it extend the right thing, does it have the constructor the - // fingerprint asks for — has to be asked of that class rather than of the name the scan happened to land on. - val subject: Option[Class[?]] = - if (!sfp.isModule) Some(cls) - // The scan yields `Foo` when the compiler emitted a mirror class beside `Foo$`, and `Foo$` when it did not. Appending `$` unconditionally - // asked for `Foo$$` in the second case, which exists for nothing, so any framework whose only fingerprint is a module one went undiscovered: - // minitest declares exactly one, and its suites were never found. - else if (className.endsWith("$")) Some(cls) - else Try(classLoader.loadClass(className + "$")).toOption - - subject.exists { subjectClass => - // Checked against `subjectClass`, not `cls`. A Scala 3 mirror class declares no constructor at all, so asking it this question rejected every - // module suite whose fingerprint required one — which is all of them. - val hasConstructor = !sfp.requireNoArgConstructor || hasNoArgConstructor(subjectClass) - hasConstructor && superclass.isAssignableFrom(subjectClass) - } - }.getOrElse(false) - - case afp: AnnotatedFingerprint => - val annotationClass = Try(classLoader.loadClass(afp.annotationName())).toOption - annotationClass.exists { annClass => - if (afp.isModule) { - val moduleClass = if (className.endsWith("$")) Some(cls) else Try(classLoader.loadClass(className + "$")).toOption - moduleClass.exists(_.getAnnotations.exists(a => annClass.isAssignableFrom(a.annotationType()))) - } else { - cls.getAnnotations.exists(a => annClass.isAssignableFrom(a.annotationType())) + fingerprints: List[(Framework, Fingerprint)], + logger: Logger + ): Option[(Framework, Fingerprint)] = + skipUnreflectable(className, logger) { + val clazz = Try(classLoader.loadClass(className)).toOption + + clazz.flatMap { cls => + // Skip abstract classes and interfaces + if (Modifier.isAbstract(cls.getModifiers) || cls.isInterface) None + else + fingerprints.find { case (_, fp) => + fp match { + case sfp: SubclassFingerprint => + Try { + val superclass = classLoader.loadClass(sfp.superclassName()) + // A module fingerprint describes the object's own class, so every question — does it extend the right thing, does it have the constructor the + // fingerprint asks for — has to be asked of that class rather than of the name the scan happened to land on. + val subject: Option[Class[?]] = + if (!sfp.isModule) Some(cls) + // The scan yields `Foo` when the compiler emitted a mirror class beside `Foo$`, and `Foo$` when it did not. Appending `$` unconditionally + // asked for `Foo$$` in the second case, which exists for nothing, so any framework whose only fingerprint is a module one went undiscovered: + // minitest declares exactly one, and its suites were never found. + else if (className.endsWith("$")) Some(cls) + else Try(classLoader.loadClass(className + "$")).toOption + + subject.exists { subjectClass => + // Checked against `subjectClass`, not `cls`. A Scala 3 mirror class declares no constructor at all, so asking it this question rejected every + // module suite whose fingerprint required one — which is all of them. + val hasConstructor = !sfp.requireNoArgConstructor || hasNoArgConstructor(subjectClass) + hasConstructor && superclass.isAssignableFrom(subjectClass) + } + }.getOrElse(false) + + case afp: AnnotatedFingerprint => + val annotationClass = Try(classLoader.loadClass(afp.annotationName())).toOption + annotationClass.exists { annClass => + if (afp.isModule) { + val moduleClass = if (className.endsWith("$")) Some(cls) else Try(classLoader.loadClass(className + "$")).toOption + moduleClass.exists(_.getAnnotations.exists(a => annClass.isAssignableFrom(a.annotationType()))) + } else { + cls.getAnnotations.exists(a => annClass.isAssignableFrom(a.annotationType())) + } } - } - case _ => - false + case _ => + false + } } - } + } } - } // ============================================================================ // Strategy 2: Direct annotation scanning @@ -378,13 +406,14 @@ object ClasspathTestDiscovery { project: CrossProjectName, classNames: List[String], // URLClassLoader, not ClassLoader: the runtime probes need to ask what is on *these* URLs, not what the parent can also reach. - classLoader: URLClassLoader + classLoader: URLClassLoader, + logger: Logger ): List[DiscoveredTestSuite] = { val junitAvailable = junitRuntimeOnClasspath(classLoader) val testngBridge = testngBridgeClasses.find(c => onProjectClasspath(classLoader, c)) classNames.flatMap { className => - detectFrameworkByAnnotation(className, classLoader).flatMap { displayName => + detectFrameworkByAnnotation(className, classLoader, logger).flatMap { displayName => selectionForAnnotation(displayName, junitAvailable, testngBridge) .map(selection => DiscoveredTestSuite(project, className, selection)) } @@ -452,29 +481,32 @@ object ClasspathTestDiscovery { /** Detect test framework by scanning for test annotations */ private def detectFrameworkByAnnotation( className: String, - classLoader: ClassLoader + classLoader: ClassLoader, + logger: Logger ): Option[String] = - Try(classLoader.loadClass(className)).toOption.flatMap { cls => - // Skip abstract classes and interfaces - if (Modifier.isAbstract(cls.getModifiers) || cls.isInterface) None - else { - // Check for class-level @Test annotation (TestNG style) - val classLevelAnnotation = findTestAnnotation(cls.getAnnotations, classLoader) - - // Check for method-level test annotations - val methodLevelAnnotation = cls.getDeclaredMethods.flatMap { method => - findTestAnnotation(method.getAnnotations, classLoader) - }.headOption - - // Determine framework from annotation - (classLevelAnnotation orElse methodLevelAnnotation).map { - case ann if ann.contains("jupiter") => "JUnit Jupiter" - case ann if ann.contains("junit") => "JUnit" - case ann if ann.contains("testng") => "TestNG" - case ann if ann.contains("kotlin") => "kotlin.test" - case ann if ann.contains("jqwik") => "jqwik" - case ann if ann.contains("suite") => "JUnit Platform Suite" - case _ => "JUnit" // Default + skipUnreflectable(className, logger) { + Try(classLoader.loadClass(className)).toOption.flatMap { cls => + // Skip abstract classes and interfaces + if (Modifier.isAbstract(cls.getModifiers) || cls.isInterface) None + else { + // Check for class-level @Test annotation (TestNG style) + val classLevelAnnotation = findTestAnnotation(cls.getAnnotations, classLoader) + + // Check for method-level test annotations + val methodLevelAnnotation = cls.getDeclaredMethods.flatMap { method => + findTestAnnotation(method.getAnnotations, classLoader) + }.headOption + + // Determine framework from annotation + (classLevelAnnotation orElse methodLevelAnnotation).map { + case ann if ann.contains("jupiter") => "JUnit Jupiter" + case ann if ann.contains("junit") => "JUnit" + case ann if ann.contains("testng") => "TestNG" + case ann if ann.contains("kotlin") => "kotlin.test" + case ann if ann.contains("jqwik") => "jqwik" + case ann if ann.contains("suite") => "JUnit Platform Suite" + case _ => "JUnit" // Default + } } } } diff --git a/bleep-bsp/src/scala/bleep/bsp/MultiWorkspaceBspServer.scala b/bleep-bsp/src/scala/bleep/bsp/MultiWorkspaceBspServer.scala index 6728b4576..2ce1514f2 100644 --- a/bleep-bsp/src/scala/bleep/bsp/MultiWorkspaceBspServer.scala +++ b/bleep-bsp/src/scala/bleep/bsp/MultiWorkspaceBspServer.scala @@ -1959,6 +1959,9 @@ class MultiWorkspaceBspServer( val testHandler: (TaskDag.TestSuiteTask, Option[TaskDag.LinkResult], Deferred[IO, KillReason]) => IO[TaskDag.TaskResult] = (_, _, _) => sys.error("TestSuiteTask should not appear in compile/link DAG") + val testBatchHandler: (TaskDag.TestBatchTask, Deferred[IO, KillReason]) => IO[TaskDag.TaskResult] = + (_, _) => sys.error("TestBatchTask should not appear in compile/link DAG") + val apHandler = makeAnnotationProcessorHandler(started, params.originId, apResults) val kspHandler = makeSymbolProcessorHandler(started, params.originId) @@ -1969,6 +1972,7 @@ class MultiWorkspaceBspServer( link = linkHandler, discover = discoverHandler, test = testHandler, + testBatch = testBatchHandler, sourcegen = sourcegenHandler, annotationProcessor = apHandler, symbolProcessor = kspHandler, @@ -2471,6 +2475,10 @@ class MultiWorkspaceBspServer( def ioProgram(traceRecorder: TraceRecorder) = for { eventQueue <- Queue.bounded[IO, Option[TaskDag.DagEvent]](100000) totalSuitesRef <- Ref.of[IO, Int](0) + // Suite completion counted from the per-suite SuiteFinished stream, not from finished DAG tasks — a batched project is one task standing in for many + // suites, so a task count would report zero completed for it. Every suite emits a SuiteFinished whether it ran alone or in a batch. + suitesCompletedRef <- Ref.of[IO, Int](0) + suitesFailedRef <- Ref.of[IO, Int](0) totalPassedRef <- Ref.of[IO, Int](0) totalFailedRef <- Ref.of[IO, Int](0) totalSkippedRef <- Ref.of[IO, Int](0) @@ -2506,7 +2514,17 @@ class MultiWorkspaceBspServer( started.build.explodedProjects(discoverTask.project).testTags.value.view.mapValues(_.values.toSet).toMap // Discovery runs on every target the client named, libraries included. Only a project that declared itself a test project is claiming there // are suites here, so only that project's empty scan is a contradiction worth failing the run over. - val isTestProject = started.build.explodedProjects(discoverTask.project).isTestProject.getOrElse(false) + val discoverProject = started.build.explodedProjects(discoverTask.project) + val isTestProject = discoverProject.isTestProject.getOrElse(false) + // How many of a project's suites run at once. Mode-aware default: per-project (the default mode) shares ONE fork and runs suites SEQUENTIALLY — + // maven's forkCount=1 reuseForks=true, the safe, memory-frugal default (one live suite's heap at a time; cores are saturated by running many + // projects' forks at once, not many suites within one). An unset value is 1. per-suite forks per suite, so an unset value stays unbounded — the + // machine-wide governor bounds how many forks run at once. Either way maxConcurrentSuites raises the ceiling. + val suiteParallelism: Option[Int] = + discoverProject.testFork.getOrElse(model.TestForkMode.PerProject) match { + case model.TestForkMode.PerProject => Some(discoverProject.maxConcurrentSuites.getOrElse(1)) + case model.TestForkMode.PerSuite => discoverProject.maxConcurrentSuites + } val tagFiltered = if (!tagsActive) regexFiltered else { @@ -2515,6 +2533,46 @@ class MultiWorkspaceBspServer( val keptSet = keptFqdns.toSet regexFiltered.filter { case (fqdn, _) => keptSet(fqdn) } } + // In per-project mode (the default) a project's suites of ONE framework run through a single execution: for JUnit Platform one + // `launcher.execute()`; for sbt test-interface one `Framework`/`Runner` with all their tasks and one `done()` — maven's `forkCount=1 + // reuseForks=true`, which is what stateful frameworks (munit, ZIO Test) need and what the sbt interface's one-runner-per-framework contract + // specifies. Group the JVM suites by framework into batches, and choose each batch's degree of concurrency: + // - JUnit Platform: the user's `maxConcurrentSuites` (default 1). The JUnit engine owns a lock-aware scheduler (`@ResourceLock`/`@Execution`), + // so running several of its classes at once is a knob its ecosystem is built for. + // - sbt test-interface: ALWAYS 1 (sequential). These frameworks share one `Runner` and have no conflict graph, so concurrent suites in a + // shared JVM are unsafe (shared statics, unattributable async output). Concurrency for sbt suites is `testFork: per-suite` — a fork each, + // OS-isolated. `maxConcurrentSuites` has no effect on sbt suites here; we warn if a project set it with no JUnit suites to apply it to. + // PlatformRunner (JS/Native) suites are never batched here — they run through their platform's own runner. + val batchGroups: List[(List[(String, bleep.testing.FrameworkSelection)], Int)] = + discoverProject.testFork.getOrElse(model.TestForkMode.PerProject) match { + case model.TestForkMode.PerProject => + val userParallelism = discoverProject.maxConcurrentSuites + val junit = tagFiltered.filter(_._2.isInstanceOf[bleep.testing.FrameworkSelection.JUnitPlatform]).toList + val junitGroup = + if (junit.isEmpty) Nil + else List((junit, math.min(junit.size, userParallelism.getOrElse(1)))) + val sbtGroups = + tagFiltered.toList + .collect { + // A framework that reports per-suite output from Runner.done() cannot share a Runner across suites (one done() for the batch would + // drop it); it is left out of the batch and runs in its own fork, below. + case s @ (_, sel: bleep.testing.FrameworkSelection.SbtTestInterface) if !bleep.testing.FrameworkSelection.needsIsolatedFork(sel) => + (sel.frameworkClass, s) + } + .groupBy(_._1) + .toList + .sortBy(_._1) + // sbt-interface groups always run sequentially in the shared fork (degree 1); concurrency for them is testFork: per-suite. + .map { case (_, pairs) => (pairs.map(_._2), 1) } + if (junit.isEmpty && userParallelism.exists(_ > 1)) + sendLogMessage( + s"${discoverTask.project.value}: maxConcurrentSuites=${userParallelism.get} has no effect — all its frameworks are sbt-interface, " + + "which run sequentially in per-project mode. Use `testFork: per-suite` to run these suites concurrently (a fork per suite).", + MessageType.Warning + ) + junitGroup ++ sbtGroups + case model.TestForkMode.PerSuite => Nil + } // Only treat an empty result as an error when the user asked to *include* something (--only or --only-tag). // A pure --exclude / --exclude-tag emptying the set is the user explicitly skipping, not a misconfiguration. @@ -2561,9 +2619,9 @@ class MultiWorkspaceBspServer( else "filter" val msg = s"$triggered matched no test suites in $projectName ($whichFilters): $pipeline. " + hints.mkString(" ") - (TaskDag.TaskResult.Failure(msg, Nil), TaskDag.DiscoveryResult(Nil, suites.size, isTestProject)) + (TaskDag.TaskResult.Failure(msg, Nil), TaskDag.DiscoveryResult(Nil, suites.size, isTestProject, suiteParallelism, batches = Nil)) } else { - (result, TaskDag.DiscoveryResult(tagFiltered, suites.size, isTestProject)) + (result, TaskDag.DiscoveryResult(tagFiltered.toList, suites.size, isTestProject, suiteParallelism, batches = batchGroups)) } } @@ -2601,17 +2659,49 @@ class MultiWorkspaceBspServer( // JVM (default) - use JvmPool val projectDir = started.build.explodedProjects.get(testTask.project).flatMap(_.folder).map(rp => started.buildPaths.buildDir.resolve(rp.toString)) - // Project-level JVM options from platform config (e.g. -Djava.util.logging.manager for Quarkus) - val projectJvmOptions = started.resolvedProject(testTask.project).platform match { + // Project-level JVM options from platform config (e.g. a custom -Djava.util.logging.manager). + // This includes the `-Duser.dir` the build states (or the sbt-compatible build-dir default + // from `Defaults`) — working-directory semantics are the BUILD's decision, expressed in + // bleep.yaml, never adjusted here. + val declaredJvmOptions = started.resolvedProject(testTask.project).platform match { case Some(p: ResolvedProject.Platform.Jvm) => p.options case _ => Nil } + // A sourcegen may declare JVM options its output requires by writing them to the + // project's `forkJvmOptions` file (one per line). This is how a code-generating test + // project's model-writer hands the fork a generated path or a custom LogManager + // without every project restating them in bleep.yaml. Generic: bleep + // knows nothing of what the options mean. Read here so a changed file re-forks + // through the normal option-keyed pool. + val sourcegenJvmOptions = { + val f = started.projectPaths(testTask.project).forkJvmOptions + if (java.nio.file.Files.exists(f)) + java.nio.file.Files + .readAllLines(f) + .asScala + .map(_.trim) + .filter(l => l.nonEmpty && !l.startsWith("#")) + .toList + else Nil + } + val projectJvmOptions = declaredJvmOptions ++ sourcegenJvmOptions + // per-project (the default, maven's one-JVM-per-module) runs every suite of this project in one shared fork; per-suite forks per suite. + // The sharing key is the project, so all its suites land on the same fork. A framework that must not share a fork (it reports per-suite + // output from a once-per-run done()) always gets its own, whatever the project's mode. How many run at once is bounded by the DAG's + // suite-parallelism chains, not here. + val sharing: bleep.testing.SessionSharing = + if (bleep.testing.FrameworkSelection.needsIsolatedFork(testTask.selection)) bleep.testing.SessionSharing.Exclusive + else + project.testFork.getOrElse(model.TestForkMode.PerProject) match { + case model.TestForkMode.PerProject => bleep.testing.SessionSharing.Shared(testTask.project.value) + case model.TestForkMode.PerSuite => bleep.testing.SessionSharing.Exclusive + } TestRunner.runSuite( project = testTask.project, suiteName = testTask.suiteName.value, selection = testTask.selection, classpath = classpath, - pool = jvmPool, + executor = jvmPool, eventQueue = eventQueue, options = TestRunner.Options( // Only what someone asked for, in precedence order: the project's own options, then this run's `--jvm-opt`. The configured heap is NOT @@ -2622,7 +2712,8 @@ class MultiWorkspaceBspServer( testArgs = testOptions.testArgs, idleTimeout = idleTimeout, environment = testEnv, - workingDirectory = projectDir + workingDirectory = projectDir, + sharing = sharing ), resolveSourcePath = className => bleep.analysis.ZincSourceLookup.relativeSourceForProject( @@ -2635,6 +2726,51 @@ class MultiWorkspaceBspServer( } } + // A whole project's JUnit suites as one batched execution (per-project mode). JVM-only, so no platform branching; the fork does one execute so an + // application-scoped fixture is built once and reused across the classes. + val testBatchHandler: (TaskDag.TestBatchTask, Deferred[IO, KillReason]) => IO[TaskDag.TaskResult] = + (batchTask, taskKillSignal) => + IO.blocking(getTestClasspath(started, batchTask.project)).flatMap { classpath => + val testEnv = computeTestEnvironment(started, batchTask.project, testOptions.env) + val projectDir = + started.build.explodedProjects.get(batchTask.project).flatMap(_.folder).map(rp => started.buildPaths.buildDir.resolve(rp.toString)) + val declaredJvmOptions = started.resolvedProject(batchTask.project).platform match { + case Some(p: ResolvedProject.Platform.Jvm) => p.options + case _ => Nil + } + val sourcegenJvmOptions = { + val f = started.projectPaths(batchTask.project).forkJvmOptions + if (java.nio.file.Files.exists(f)) + java.nio.file.Files.readAllLines(f).asScala.map(_.trim).filter(l => l.nonEmpty && !l.startsWith("#")).toList + else Nil + } + TestRunner.runBatch( + project = batchTask.project, + suites = batchTask.suites, + parallelism = batchTask.parallelism, + classpath = classpath, + executor = jvmPool, + eventQueue = eventQueue, + options = TestRunner.Options( + jvmOptions = declaredJvmOptions ++ sourcegenJvmOptions ++ testOptions.jvmOptions, + defaultHeapMb = MachineResources.forkHeapMb(serverConfig.testRunnerHeap), + testArgs = testOptions.testArgs, + idleTimeout = idleTimeout, + environment = testEnv, + workingDirectory = projectDir, + sharing = bleep.testing.SessionSharing.Exclusive + ), + resolveSourcePath = className => + bleep.analysis.ZincSourceLookup.relativeSourceForProject( + bleep.analysis.AnalysisCache.Ref(analysisCache, started.buildPaths.workspaceKey), + started.buildPaths.variantBuildDir(batchTask.project).resolve(".zinc").resolve("analysis.zip"), + className + ), + killSignal = taskKillSignal, + logger = logger + ) + } + // Link handler for non-JVM platforms (Scala.js, Scala Native, Kotlin/JS, Kotlin/Native) val linkHandler: (TaskDag.LinkTask, Deferred[IO, KillReason]) => IO[(TaskDag.TaskResult, TaskDag.LinkResult)] = (linkTask, killSignal) => @@ -2660,6 +2796,7 @@ class MultiWorkspaceBspServer( link = linkHandler, discover = discoverHandler, test = testHandler, + testBatch = testBatchHandler, sourcegen = sourcegenHandler, annotationProcessor = apHandler, symbolProcessor = kspHandler, @@ -2679,6 +2816,8 @@ class MultiWorkspaceBspServer( eventQueue, params.originId, totalSuitesRef, + suitesCompletedRef, + suitesFailedRef, totalPassedRef, totalFailedRef, totalSkippedRef, @@ -2754,7 +2893,9 @@ class MultiWorkspaceBspServer( skipped <- totalSkippedRef.get ignored <- totalIgnoredRef.get suites <- totalSuitesRef.get - } yield (testResult, passed, failed, skipped, ignored, suites) + suitesDone <- suitesCompletedRef.get + suitesFail <- suitesFailedRef.get + } yield (testResult, passed, failed, skipped, ignored, suites, suitesDone, suitesFail) for { // Create trace recorder (noop if not enabled) @@ -2773,7 +2914,7 @@ class MultiWorkspaceBspServer( _ <- IO(clearStaleDiagnostics(diagnosticTracker)) testResult <- IO(ioResult match { - case Right((result, totalPassed, totalFailed, totalSkipped, totalIgnored, totalSuites)) => + case Right((result, totalPassed, totalFailed, totalSkipped, totalIgnored, totalSuites, suitesCompletedTally, suitesFailedTally)) => // Send TestRunFinished event val durationMs = System.currentTimeMillis() - startTime val timestamp = System.currentTimeMillis() @@ -2793,10 +2934,14 @@ class MultiWorkspaceBspServer( else if (result.failed.nonEmpty || result.errored.nonEmpty || result.timedOut.nonEmpty || result.killed.nonEmpty) StatusCode.Error else StatusCode.Ok - // Compute suite-level counts from DAG result - val suiteTaskIds = result.tasks.collect { case (id, _: TaskDag.TestSuiteTask) => id }.toSet - val suitesCompleted = suiteTaskIds.count(id => result.completed.contains(id) || result.failed.contains(id) || result.timedOut.contains(id)) - val suitesFailed = suiteTaskIds.count(id => result.failed.contains(id) || result.errored.contains(id)) + // Suite completion/failure come from the per-suite SuiteFinished stream (batch-agnostic: a batched project is one task but many finished suites). + // Cancellation stays task-derived — a cancelled suite emits no SuiteFinished, so it is a task that reached neither completion nor failure. + val suitesCompleted = suitesCompletedTally + val suitesFailed = suitesFailedTally + val suiteTaskIds = result.tasks.collect { + case (id, _: TaskDag.TestSuiteTask) => id + case (id, _: TaskDag.TestBatchTask) => id + }.toSet val suitesCancelled = suiteTaskIds.count(id => result.killed.contains(id) || result.skipped.contains(id)) bspInfo(s"Test completed: $totalPassed passed, $totalFailed failed, $totalSkipped skipped (${durationMs}ms)") @@ -3310,14 +3455,23 @@ class MultiWorkspaceBspServer( val resolved = started.resolvedProject(project) val classpath = resolved.classpath.map(p => Path.of(p.toString)).toList - val suites = ClasspathTestDiscovery.discover(project, classesDir, classpath, resolved.testFrameworks) + val suites = ClasspathTestDiscovery.discover(project, classesDir, classpath, resolved.testFrameworks, logger) + + // Scala.js and Scala Native enumerate through the same classpath scan — their JVM-facing `.class` frontend is present — but they must never run in a + // JVM fork: the test framework on their classpath is the JS/Native artifact (`munit_sjs1`, `munit_native0.5`), which throws "trying to run Scala.js + // binaries on the JVM" the instant it is instantiated. Their suites run through the platform's own linked runner (node / native binary), which + // `testHandler` selects by project platform. So their selection is PlatformRunner, matching Kotlin JS/Native above — and, crucially, keeping them out + // of the per-project JVM batch, which regroups SbtTestInterface selections into one fork and has no platform branch of its own. + val isScalaJsOrNative = platformOpt.contains(model.PlatformId.Js) || platformOpt.contains(model.PlatformId.Native) + def selectionOf(s: DiscoveredTestSuite): bleep.testing.FrameworkSelection = + if (isScalaJsOrNative) bleep.testing.FrameworkSelection.PlatformRunner(s.selection.displayName) else s.selection if (suites.isEmpty) { debugLog(s"No test suites discovered in ${project.value}") (TaskDag.TaskResult.Success, Nil) } else { debugLog(s"Discovered ${suites.size} test suites in ${project.value}: ${suites.map(_.className).mkString(", ")}") - (TaskDag.TaskResult.Success, suites.map(s => (s.className, s.selection))) + (TaskDag.TaskResult.Success, suites.map(s => (s.className, selectionOf(s)))) } } } @@ -3962,6 +4116,7 @@ class MultiWorkspaceBspServer( case lt: TaskDag.LinkTask => (TraceCategory.Link, lt.project.value) case dt: TaskDag.DiscoverTask => (TraceCategory.Discover, dt.project.value) case tt: TaskDag.TestSuiteTask => (TraceCategory.Test, s"${tt.project.value}:${tt.suiteName.value}") + case bt: TaskDag.TestBatchTask => (TraceCategory.Test, s"${bt.project.value} (batch of ${bt.suites.size})") case sgt: TaskDag.SourcegenTask => (TraceCategory.Sourcegen, s"${sgt.script.project.value}/${sgt.script.main}") case apt: TaskDag.ResolveAnnotationProcessorsTask => (TraceCategory.ResolveAnnotationProcessors, apt.project.value) case kspt: TaskDag.RunSymbolProcessorsTask => (TraceCategory.RunSymbolProcessors, kspt.project.value) @@ -4167,6 +4322,8 @@ class MultiWorkspaceBspServer( queue: Queue[IO, Option[TaskDag.DagEvent]], originId: Option[String], totalSuitesRef: Ref[IO, Int], + suitesCompletedRef: Ref[IO, Int], + suitesFailedRef: Ref[IO, Int], totalPassedRef: Ref[IO, Int], totalFailedRef: Ref[IO, Int], totalSkippedRef: Ref[IO, Int], @@ -4189,6 +4346,8 @@ class MultiWorkspaceBspServer( Some(BleepBspProtocol.Event.DiscoveryStarted(dt.project, timestamp)) case tt: TaskDag.TestSuiteTask => Some(BleepBspProtocol.Event.SuiteStarted(tt.project, tt.suiteName, timestamp)) + case _: TaskDag.TestBatchTask => + None // A batch has no single suite to start; each suite's own SuiteFinished conveys its result as the batch runs. case _: TaskDag.SourcegenTask => None // Sourcegen is reported via DagEvent.SourcegenStarted/Finished, not TaskStarted/Finished case _: TaskDag.ResolveAnnotationProcessorsTask => @@ -4196,8 +4355,23 @@ class MultiWorkspaceBspServer( case _: TaskDag.RunSymbolProcessorsTask => None // KSP execution is reported via DagEvent.RunSymbolProcessors{Started,Finished} } - traceRecorder.recordStart(cat, name) >> - IO(protocolEvent.foreach(e => sendTestEvent(originId, task.id.value, e, recorder))) + val emitStart = task match { + case bt: TaskDag.TestBatchTask => + // One batch task stands in for many suites, so it announces the start of each — pairing every suite's own SuiteFinished the way a suite-by-suite + // run does, so the client tracks them as started-then-completed rather than never-started. + bt.suites.traverse_ { case (suite, _) => + IO( + sendTestEvent( + originId, + s"suite:${bt.project.value}:${suite.value}", + BleepBspProtocol.Event.SuiteStarted(bt.project, suite, timestamp), + recorder + ) + ) + } + case _ => IO(protocolEvent.foreach(e => sendTestEvent(originId, task.id.value, e, recorder))) + } + traceRecorder.recordStart(cat, name) >> emitStart case TaskDag.DagEvent.TaskFinished(task, result, durationMs, timestamp) => val (cat, name) = taskCatName(task) @@ -4254,6 +4428,12 @@ class MultiWorkspaceBspServer( Some(BleepBspProtocol.Event.SuiteTimedOut(tt.project, tt.suiteName, durationMs, threadDump, timestamp)) } + case _: TaskDag.TestBatchTask => + // Every suite's own SuiteFinished (or SuiteError/etc.) was already emitted as the batch ran, so the batch task's own result adds nothing — + // emitting a batch-level event here would double-count. An infra failure that killed the fork mid-batch leaves its unfinished suites uncounted, + // which is the honest picture: they did not run to completion. + None + case _: TaskDag.SourcegenTask => None // Sourcegen is reported via DagEvent.SourcegenFinished, not TaskFinished case _: TaskDag.ResolveAnnotationProcessorsTask => @@ -4318,6 +4498,17 @@ class MultiWorkspaceBspServer( totalFailedRef.update(_ + failedContribution) >> totalSkippedRef.update(_ + outcome.skippedCount) >> totalIgnoredRef.update(_ + outcome.ignoredCount) >> + // Every finished suite counts once toward completion, and once toward failed if its outcome is a failure — the batch-agnostic authoritative tally. + suitesCompletedRef.update(_ + 1) >> + suitesFailedRef.update(_ + (if (outcome.isFailure) 1 else 0)) >> + IO(sendTestEvent(originId, s"suite:$project:$suite", protocolEvent, recorder)) + + case TaskDag.DagEvent.SuiteTimedOut(project, suite, timeoutMs, threadDump, timestamp) => + // Same authoritative tallies as a finished suite: one completed, one failed — a timeout is a failure. The client-side reducer additionally counts + // it toward `testsTimedOut`, which is what makes the run's verdict say "timed out" rather than "did not finish". + val protocolEvent = BleepBspProtocol.Event.SuiteTimedOut(project, suite, timeoutMs, threadDump, timestamp) + suitesCompletedRef.update(_ + 1) >> + suitesFailedRef.update(_ + 1) >> IO(sendTestEvent(originId, s"suite:$project:$suite", protocolEvent, recorder)) case linkEvent: TaskDag.DagEvent.LinkStarted => processLinkEvent(linkEvent, originId, traceRecorder, recorder) @@ -4878,7 +5069,7 @@ class MultiWorkspaceBspServer( val resolved = started.resolvedProject(crossName) val classpath = resolved.classpath.map(p => Path.of(p.toString)).toList - val suites = ClasspathTestDiscovery.discover(crossName, classesDir, classpath, resolved.testFrameworks) + val suites = ClasspathTestDiscovery.discover(crossName, classesDir, classpath, resolved.testFrameworks, logger) debugLog(s"handleScalaTestClasses: project=${crossName.value}, classesDir=$classesDir, found ${suites.size} test classes") diff --git a/bleep-bsp/src/scala/bleep/bsp/TaskDag.scala b/bleep-bsp/src/scala/bleep/bsp/TaskDag.scala index 87dcf6e95..3e7445163 100644 --- a/bleep-bsp/src/scala/bleep/bsp/TaskDag.scala +++ b/bleep-bsp/src/scala/bleep/bsp/TaskDag.scala @@ -49,6 +49,11 @@ object TaskDag { val value: String = s"test:${project.value}:${suiteName.value}" } + /** A whole project's JUnit suites run as one batched execution — one task, not one per suite. */ + case class TestBatch(project: CrossProjectName) extends TaskId { + val value: String = s"test-batch:${project.value}" + } + /** Identity for a sourcegen script in the DAG. * * Two `ScriptDef.Main` values collapse to the same task iff they share the same script project + main class. A single `SourcegenTask` runs the script once @@ -83,6 +88,9 @@ object TaskDag { def id: TaskId def project: CrossProjectName def dependencies: Set[TaskId] + + /** Ordering-only predecessors: scheduling waits for these to finish (in any state) without propagating their failure. See TestSuiteTask. */ + def runAfter: Set[TaskId] = Set.empty } /** A task's claim on the machine. `cpu` is in cores; `memoryMb` is off-heap memory for a forked process. */ @@ -113,6 +121,8 @@ object TaskDag { // (measured RSS), and only the pool knows which. It holds that reservation itself, for a // lifetime this task does not share — one JVM serves many suites. case _: TestSuiteTask => Cost(MachineResources.ResourceKind.TestFork, cpu = 1, memoryMb = 0L) + // One fork running `parallelism` classes at once, so it claims that many cores (never more than it has suites to run). Fork memory is the pool's, as above. + case bt: TestBatchTask => Cost(MachineResources.ResourceKind.TestFork, cpu = math.max(1, math.min(bt.parallelism, bt.suites.size)), memoryMb = 0L) } /** Compile a project. @@ -280,14 +290,28 @@ object TaskDag { suites: List[(String, bleep.testing.FrameworkSelection)], discoveredBeforeFilters: Int, /** Whether the project declares `isTestProject: true` — not whether it was named as a target, which every discovered project was. */ - isTestProject: Boolean + isTestProject: Boolean, + /** The project's `maxConcurrentSuites`: how many of its suites may run in parallel forks. None = unbounded (the default). 1 = all suites run sequentially + * through one warm fork, maven-style. + */ + suiteParallelism: Option[Int], + /** Per-project batches, one per framework: the suites to run through a single execution (JUnit Platform: one `launcher.execute()`; sbt test-interface: + * one `Framework`/`Runner`, one `done()` — maven's one-execute-per-module), paired with the degree of parallelism bleep chose. Decided by the discover + * handler from `testFork == per-project`, grouping JVM suites by framework. Empty = run suite-by-suite (per-suite mode, and any PlatformRunner suites). + */ + batches: List[(List[(String, bleep.testing.FrameworkSelection)], Int)] ) /** Execute a test suite */ case class TestSuiteTask( project: CrossProjectName, suiteName: SuiteName, - selection: bleep.testing.FrameworkSelection + selection: bleep.testing.FrameworkSelection, + /** Ordering-only predecessors: this suite waits for them to reach a terminal state but does NOT inherit their failure — a red suite must not skip the + * rest of its project's chain, just as maven's surefire keeps going after a failing class. Used by `maxConcurrentSuites` to serialize a project's suites + * through one warm fork. + */ + override val runAfter: Set[TaskId] ) extends Task { val id: TaskId = TaskId.Test(project, suiteName) val dependencies: Set[TaskId] = Set(TaskId.Discover(project)) @@ -296,6 +320,20 @@ object TaskDag { // reservation itself, from spawn until the process is destroyed — a lifetime this task does not } + /** Run ALL of a project's JUnit suites as one batched execution in a single fork — maven surefire's one-execute-per-module, which is what keeps an + * execution-scoped fixture (an application the framework boots for the run) built once and reused across the classes rather than rebuilt per class. junit's + * engine runs `parallelism` classes at once inside the fork, a number bleep chose. One task, not one per suite: the resource cost is one fork doing that + * much work. + */ + case class TestBatchTask( + project: CrossProjectName, + suites: List[(SuiteName, bleep.testing.FrameworkSelection)], + parallelism: Int + ) extends Task { + val id: TaskId = TaskId.TestBatch(project) + val dependencies: Set[TaskId] = Set(TaskId.Discover(project)) + } + /** Result of task execution. * * Semantics: @@ -426,6 +464,18 @@ object TaskDag { timestamp: Long ) extends DagEvent + // A suite the idle-timeout watchdog stopped before it reported a result. A per-project batch kills the whole fork on timeout, so its own TimedOut result + // names no suite; each suite that had not reported is emitted here so the run counts it as a timeout (and the verdict says "timed out") instead of the + // anonymous "N suites never reported a result". A TestSuiteTask reports its own timeout from the task result (see abnormalTaskEvent), so this is the + // batch-only equivalent. + case class SuiteTimedOut( + project: CrossProjectName, + suite: SuiteName, + timeoutMs: Long, + threadDump: Option[String], + timestamp: Long + ) extends DagEvent + // Sourcegen events (mirror Link events: Started around handler, Finished with result) case class SourcegenStarted( scriptProject: CrossProjectName, @@ -511,10 +561,12 @@ object TaskDag { val task = tasks(taskId) // A task is complete if it's in any terminal state (including timedOut) val depsComplete = task.dependencies.forall(d => finished.contains(d)) + // runAfter is ordering only: wait for a terminal state, ignore what that state was + val orderingComplete = task.runAfter.forall(d => finished.contains(d)) val depsFailed = task.dependencies.exists(propagatesFailure) if (depsFailed) None // Will be skipped - else if (depsComplete) Some(task) + else if (depsComplete && orderingComplete) Some(task) else None } } @@ -916,6 +968,8 @@ object TaskDag { * path from convention: the linker already knows where it wrote, and a second derivation is a second thing to keep in step with it. */ test: (TestSuiteTask, Option[LinkResult], Deferred[IO, KillReason]) => IO[TaskResult], + /** Run a whole project's JUnit suites as one batched execution. JVM-only (JUnit Platform has no non-JVM linked form), so no LinkResult. */ + testBatch: (TestBatchTask, Deferred[IO, KillReason]) => IO[TaskResult], sourcegen: (SourcegenTask, Deferred[IO, KillReason]) => IO[TaskResult], annotationProcessor: (ResolveAnnotationProcessorsTask, Deferred[IO, KillReason]) => IO[(TaskResult, Int)], symbolProcessor: (RunSymbolProcessorsTask, Deferred[IO, KillReason]) => IO[(TaskResult, Int)], @@ -1120,11 +1174,28 @@ object TaskDag { (result, discovery) <- handlers.discover(dt, linkOutput, taskKill) _ <- result match { case TaskResult.Success => - // Add test tasks for discovered suites - val testTasks = discovery.suites.map { case (suiteName, selection) => - TestSuiteTask(dt.project, SuiteName(suiteName), selection) - } - dagRef.update(dag => testTasks.foldLeft(dag)(_.addTask(_))) >> + // One batched execution per framework group (per-project mode) — an execution-scoped fixture / a framework's Runner is built once for + // all its classes. Whatever a batch does not cover (per-suite mode, or PlatformRunner suites) runs suite-by-suite. + val batchTasks: List[Task] = + discovery.batches.map { case (groupSuites, degree) => + val ordered = groupSuites.sortBy(_._1).map { case (n, sel) => (SuiteName(n), sel) } + TestBatchTask(dt.project, ordered, degree) + } + val batchedNames: Set[String] = discovery.batches.flatMap(_._1.map(_._1)).toSet + // Suite-by-suite for the rest. With a suite-parallelism bound, a project's suites form that many round-robin chains, alphabetically + // ordered — surefire's usual class order, which schema-bootstrapping setups rely on. At bound 1 all suites run through one warm fork + // sequentially. + val perSuite = discovery.suites.filterNot { case (n, _) => batchedNames(n) }.sortBy(_._1) + val bound = discovery.suiteParallelism.getOrElse(Int.MaxValue) + val suiteTasks: List[Task] = + perSuite.zipWithIndex.map { case ((suiteName, selection), idx) => + val after: Set[TaskId] = + if (idx < bound) Set.empty + else Set(TaskId.Test(dt.project, SuiteName(perSuite(idx - bound)._1))) + TestSuiteTask(dt.project, SuiteName(suiteName), selection, runAfter = after) + } + val newTasks: List[Task] = batchTasks ++ suiteTasks + dagRef.update(dag => newTasks.foldLeft(dag)(_.addTask(_))) >> emit( DagEvent.SuitesDiscovered( dt.project, @@ -1154,6 +1225,10 @@ object TaskDag { withRecovery(s"Test ${tt.suiteName.value}", taskKill)(handlers.test(tt, linkResult, taskKill)) } + case bt: TestBatchTask => + // Same cancellation story as TestSuiteTask: the handler races execution vs the kill signal internally. + withRecovery(s"Test batch ${bt.project.value}", taskKill)(handlers.testBatch(bt, taskKill)) + // These three emit their own Started/Finished pair, and the Finished carries the // error message. Recovery therefore wraps ONLY the handler call, with the emit // driven by the recovered result — if withRecovery wrapped the whole diff --git a/bleep-bsp/src/scala/bleep/bsp/TestRunner.scala b/bleep-bsp/src/scala/bleep/bsp/TestRunner.scala index 74501a848..92aef0174 100644 --- a/bleep-bsp/src/scala/bleep/bsp/TestRunner.scala +++ b/bleep-bsp/src/scala/bleep/bsp/TestRunner.scala @@ -4,18 +4,20 @@ import bleep.MachineResources import bleep.bsp.protocol.KillReason import bleep.bsp.protocol.{BleepBspProtocol, OutputChannel, ProcessExit, SuiteOutcome, TestStatus} import bleep.model.{CrossProjectName, SuiteName, TestName} -import bleep.testing.{FrameworkSelection, JvmPool, TestJvm, TestProtocol} +import bleep.testing.{FrameworkSelection, SessionSharing, TestExecutor, TestProtocol, TestSession, TestSessionRequest} import cats.effect._ import cats.effect.std.Queue import cats.syntax.all._ import java.nio.file.Path +import ryddig.Logger import scala.concurrent.duration._ -/** Test runner that executes test suites in forked JVMs. +/** Test runner that executes test suites and streams their events back through the DAG event queue. * - * Uses JvmPool for efficient JVM reuse and streams test events back through the DAG event queue. Uses Deferred-based kill signals for explicit cancellation - * handling. + * Where a suite runs is [[TestExecutor]]'s business, not this one's: a pooled forked JVM talking over a socket, or a classloader in the server itself. This + * drives whichever it is handed through the same protocol, with the same idle timeout and the same Deferred-based kill signal — so cancellation, timeouts and + * reporting behave identically whether or not there is a process on the other end. */ object TestRunner { @@ -29,7 +31,9 @@ object TestRunner { testArgs: List[String], idleTimeout: FiniteDuration, environment: Map[String, String], - workingDirectory: Option[Path] + workingDirectory: Option[Path], + /** Shared (the default — this suite's project runs all its suites in one fork) or Exclusive (a fork per suite). Set from the project's `testFork`. */ + sharing: SessionSharing ) object Options { @@ -39,7 +43,8 @@ object TestRunner { testArgs = Nil, idleTimeout = 2.minutes, environment = Map.empty, - workingDirectory = None + workingDirectory = None, + sharing = SessionSharing.Exclusive ) } @@ -53,8 +58,8 @@ object TestRunner { * how to run the suite: which runner, and for the sbt path which `Framework` class * @param classpath * full classpath for the test JVM - * @param pool - * the JVM pool to acquire from + * @param executor + * where to run the suite: a pool of forked JVMs, or this process * @param eventQueue * queue to emit DAG events to * @param options @@ -72,7 +77,7 @@ object TestRunner { suiteName: String, selection: FrameworkSelection, classpath: List[Path], - pool: JvmPool, + executor: TestExecutor, eventQueue: Queue[IO, Option[TaskDag.DagEvent]], options: Options, resolveSourcePath: String => Option[String], @@ -80,7 +85,18 @@ object TestRunner { ): IO[TaskDag.TaskResult] = { val runnerClass = "bleep.testing.runner.ForkedTestRunner" - pool.acquire(suiteName, classpath, options.jvmOptions, options.defaultHeapMb, runnerClass, options.environment, options.workingDirectory).use { jvm => + val request = TestSessionRequest( + label = suiteName, + classpath = classpath, + jvmOptions = options.jvmOptions, + defaultHeapMb = options.defaultHeapMb, + runnerClass = runnerClass, + environment = options.environment, + workingDirectory = options.workingDirectory, + sharing = options.sharing + ) + + executor.acquire(request).use { jvm => // Recorded here rather than in the pool because this is the only place that knows both which JVM was handed out and what is about to run on it. The pid // joins these to the fork_start/fork_end pair, which is what lets a test run be reconstructed: which suites shared a JVM, and which JVM was killed // under which suite. @@ -105,6 +121,275 @@ object TestRunner { } } + /** Run a whole project's JUnit suites as ONE batched execution. + * + * The maven one-execute-per-module shape: all the classes go through a single execution in one fork, so an execution-scoped fixture (an application the + * framework boots for the run) is built once and reused across them instead of rebuilt per class. junit's engine runs `parallelism` classes at once, a + * number bleep chose. The per-suite events emitted are exactly what a suite-by-suite run emits (each response carries its own suite), so nothing downstream + * can tell the difference — only the fork does one execute instead of N. + */ + def runBatch( + project: CrossProjectName, + suites: List[(SuiteName, FrameworkSelection)], + parallelism: Int, + classpath: List[Path], + executor: TestExecutor, + eventQueue: Queue[IO, Option[TaskDag.DagEvent]], + options: Options, + resolveSourcePath: String => Option[String], + killSignal: Deferred[IO, KillReason], + logger: Logger + ): IO[TaskDag.TaskResult] = { + val runnerClass = "bleep.testing.runner.ForkedTestRunner" + val classNames = suites.map(_._1.value) + val selection = suites.head._2 // all JUnit-Platform (the batch is only formed for JUnit) + val request = TestSessionRequest( + label = s"${project.value} (batch of ${suites.size})", + classpath = classpath, + jvmOptions = options.jvmOptions, + defaultHeapMb = options.defaultHeapMb, + runnerClass = runnerClass, + environment = options.environment, + workingDirectory = options.workingDirectory, + // One execute, one fork: exclusive. (A shared session multiplexes independent suites — the opposite model.) + sharing = SessionSharing.Exclusive + ) + executor + .acquire(request) + .use { jvm => + val startedAt = System.currentTimeMillis() + IO(BspMetrics.recordSuiteScheduled(jvm.pid, project.value, s"", selection.displayName)).attempt >> + executeBatch( + project, + classNames, + parallelism, + selection, + jvm, + eventQueue, + options.idleTimeout, + options.testArgs, + resolveSourcePath, + killSignal, + logger + ) + .flatTap { result => + IO( + BspMetrics.recordSuiteFinished( + jvm.pid, + project.value, + s"", + System.currentTimeMillis() - startedAt, + result.getClass.getSimpleName.stripSuffix("$") + ) + ).attempt + } + } + .handleErrorWith { e => + // A fork that dies before the protocol handshake (a bad -Xmx, an `Unrecognized VM option`, a shadowing test-runner) fails here, in acquisition, before + // executeBatch runs — so no suite ever reported, and awaitProtocolConnection has drained the fork's own stderr onto this exception. Left to propagate, + // it becomes only the DAG's errored-task record, which the batch mapping drops (a batch's suites normally speak for themselves). Attribute the death to + // each suite in the batch so the cause — the fork's stderr, "without ever connecting" — reaches the client and history, then re-raise so the task is + // still Errored. + val msg = Option(e.getMessage).getOrElse(e.getClass.getName) + val ts = System.currentTimeMillis() + suites.traverse_ { case (suite, _) => + eventQueue.offer(Some(TaskDag.DagEvent.SuiteFinished(project, suite, SuiteOutcome.Errored(msg, None), 0L, ts))) + } >> IO.raiseError(e) + } + } + + /** Why a batch fork stopped without reporting the rest of its suites. The fork's own stderr (fd 2) is where a JVM records an OutOfMemoryError, a native + * crash, or a `System.exit` — none of which travels over the per-suite protocol — so drain it here; say whether the fork is still alive (wedged) or gone; + * and add a thread dump when it is wedged. Best-effort: every probe is `.attempt`ed, so producing the diagnostic can never itself fail the run. + */ + private def forkDeathDiagnostic(jvm: TestSession): IO[String] = + for { + alive <- jvm.isAlive.attempt.map(_.getOrElse(true)) + stderr <- jvm.drainStderr.attempt.map(_.getOrElse(Nil)) + dump <- if (alive) jvm.dumpThreads.attempt.map(_.getOrElse(Nil)) else IO.pure(Nil) + } yield { + val liveNote = + if (alive) " The fork is still alive — it stopped producing results without exiting, so it is wedged rather than dead." + else " The fork had exited." + val errNote = + if (stderr.nonEmpty) s"\n fork stderr (tail):\n${stderr.takeRight(50).map(" " + _).mkString("\n")}" + else " It wrote nothing to its own stderr." + val dumpNote = + if (dump.nonEmpty) s"\n fork thread dump (head):\n${dump.take(80).map(" " + _).mkString("\n")}" else "" + liveNote + errNote + dumpNote + } + + private def executeBatch( + project: CrossProjectName, + classNames: List[String], + parallelism: Int, + selection: FrameworkSelection, + jvm: TestSession, + eventQueue: Queue[IO, Option[TaskDag.DagEvent]], + idleTimeout: FiniteDuration, + args: List[String], + resolveSourcePath: String => Option[String], + killSignal: Deferred[IO, KillReason], + logger: Logger + ): IO[TaskDag.TaskResult] = { + def now: IO[Long] = IO.realTime.map(_.toMillis) + def emit(event: TaskDag.DagEvent): IO[Unit] = eventQueue.offer(Some(event)) + val startTime = System.currentTimeMillis() + + for { + lastActivityAt <- Ref.of[IO, Long](startTime) + outcomes <- Ref.of[IO, Map[String, SuiteOutcome]](Map.empty) + failuresPerSuite <- Ref.of[IO, Map[String, List[String]]](Map.empty) + forkError <- Ref.of[IO, Option[String]](None) + + processResponses = + jvm + .runSuites(classNames, parallelism, selection, args) + .evalMap { + case TestProtocol.TestResponse.TestStarted(suite, test) => + now.flatMap(ts => lastActivityAt.set(ts) >> emit(TaskDag.DagEvent.TestStarted(project, SuiteName(suite), TestName(test), ts))) + + case TestProtocol.TestResponse.TestFinished(suite, test, statusStr, durationMs, message, throwable, location) => + val status = TestStatus.fromString(statusStr) + val track = if (status.isFailure) failuresPerSuite.update(m => m.updated(suite, test :: m.getOrElse(suite, Nil))) else IO.unit + track >> now.flatMap { ts => + lastActivityAt.set(ts) >> + emit( + TaskDag.DagEvent.TestFinished( + project, + SuiteName(suite), + TestName(test), + status, + durationMs, + message, + throwable, + ts, + location.map(loc => loc.copy(path = resolveSourcePath(loc.declaringClass))) + ) + ) + } + + case TestProtocol.TestResponse.SuiteDone(suite, outcome, durationMs) => + outcomes.update(_ + (suite -> outcome)) >> + now.flatMap(ts => lastActivityAt.set(ts) >> emit(TaskDag.DagEvent.SuiteFinished(project, SuiteName(suite), outcome, durationMs, ts))) + + case TestProtocol.TestResponse.Log(level, message, suite) => + if (level == "debug") IO.delay(MultiWorkspaceBspServer.debugLogStatic(s"[${suite.getOrElse(project.value)}] $message")) + else + suite match { + case Some(s) => + now.flatMap(ts => + emit(TaskDag.DagEvent.Output(project, SuiteName(s), message, OutputChannel.fromIsError(level == "error" || level == "stderr"), ts)) + ) + case None => + IO.unit // batch output not attributable to a single suite (a framework thread) — dropped, as its structured events already carried the result + } + + case TestProtocol.TestResponse.Error(message, details) => + // A fork-level error (the JVM died) has no suite — it fails the whole batch. Carry the + // runner's detail (the "likely System.exit()" hint and the fork's stderr tail) so the + // batch's failure says why, not just that it died. + forkError.set(Some(withDetail(message, details))) + + case TestProtocol.TestResponse.BatchComplete => IO.unit + case TestProtocol.TestResponse.Ready => IO.unit + case TestProtocol.TestResponse.ThreadDump(_) => IO.unit + } + .compile + .drain + + idleTimeoutIO = { + val checkInterval = 1.second + def loop: IO[Unit] = for { + nowMs <- IO.realTime.map(_.toMillis) + lastActivity <- lastActivityAt.get + elapsed = nowMs - lastActivity + _ <- if (elapsed >= idleTimeout.toMillis) IO.unit else IO.sleep(checkInterval) >> loop + } yield () + loop + } + + result <- IO.racePair(processResponses, IO.race(idleTimeoutIO, killSignal.get)).flatMap { + case Left((_, raceFiber)) => + // The batch completed. Aggregate: a fork-level death is an Error; a class that never reported is an Error; otherwise the first failing suite decides, + // else Success. Per-suite results already went out as SuiteFinished events, so this is only the batch task's own status. + raceFiber.cancel >> (for { + fe <- forkError.get + outs <- outcomes.get + fps <- failuresPerSuite.get + result <- fe match { + case Some(msg) => + // The fork reported a death (or bleep synthesised one when its stream ended). Append the fork diagnostic — its own stderr, where a shutdown + // hook prints the thread dump that names a System.exit caller — so a clean-looking "exited 0" carries the reason with it. + forkDeathDiagnostic(jvm).flatMap { diag => + val full = s"$msg$diag" + IO(logger.warn(s"batch fork error for ${project.value}:\n$full")) >> + IO.pure(TaskDag.TaskResult.Error(error = full, processExit = ProcessExit.Unknown)) + } + case None => + val missing = classNames.filterNot(outs.contains) + if (missing.nonEmpty) + // The fork stopped after reporting some suites but not these. It almost always died mid-run, and a JVM that dies of an OutOfMemoryError, a + // native crash, or `System.exit` says so on its OWN stderr (fd 2) — which the protocol, carrying only per-suite events, never delivered. So + // this used to be a dead end: "produced no result", cause unknown. Drain that stderr (and note whether the fork is even still alive) so the + // reason travels with the failure. + forkDeathDiagnostic(jvm).flatMap { diag => + val msg = s"${missing.size} of ${classNames.size} batched suites never reported a result " + + s"(${missing.take(8).mkString(", ")}${if (missing.size > 8) ", …" else ""}).$diag" + // The DAG keeps only the errored task's id and drops this message, so emit it as output too — attributed to the first suite that never + // reported (the one the fork was on when it went) — so the reason reaches history and the client, not just this returned value. + IO(logger.warn(s"batch fork diagnostic for ${project.value}:\n$msg")) >> + now.flatMap { ts => + msg.linesIterator.toList + .traverse_(line => + emit(TaskDag.DagEvent.Output(project, SuiteName(missing.head), line, OutputChannel.fromIsError(isError = true), ts)) + ) + } >> IO.pure(TaskDag.TaskResult.Error(error = msg, processExit = ProcessExit.Unknown)) + } + else { + val perSuite = classNames.map(c => taskResultFor(c, outs(c), fps.getOrElse(c, Nil))) + IO.pure( + perSuite + .collectFirst { case f: TaskDag.TaskResult.Failure => f } + .orElse(perSuite.collectFirst { case e: TaskDag.TaskResult.Error => e }) + .getOrElse(TaskDag.TaskResult.Success) + ) + } + } + } yield result) + + case Right((suiteFiber, raceOutcome)) => + // Idle timeout or kill: the whole fork goes (there is one execute; there is no per-suite thread to interrupt without ending the run). + val cleanup: IO[Unit] = IO.uncancelable(_ => jvm.kill.attempt >> suiteFiber.cancel.attempt.void) + raceOutcome match { + case Outcome.Succeeded(fa) => + fa.flatMap { + case Left(_) => + IO.race(jvm.dumpThreads.attempt, IO.sleep(5.seconds)) + .map { + case Left(Right(lines)) if lines.nonEmpty => Some(lines.mkString("\n")) + case _ => None + } + .flatMap { dump => + // The idle watchdog kills the whole fork, so every suite that had not reported is a casualty of the timeout — name each one, so the run + // counts it as timed out (and the verdict says so) instead of the anonymous "N suites never reported a result". The batch task's own + // TimedOut result names no suite; this is where the suite identities live. + (outcomes.get, now).flatMapN { (outs, ts) => + classNames.filterNot(outs.contains).traverse_ { c => + emit(TaskDag.DagEvent.SuiteTimedOut(project, SuiteName(c), idleTimeout.toMillis, dump, ts)) + } + } >> cleanup >> IO.pure(TaskDag.TaskResult.TimedOut(dump)) + } + case Right(reason) => cleanup >> IO.pure(TaskDag.TaskResult.Killed(reason)) + } + case Outcome.Errored(e) => + cleanup >> IO.pure(TaskDag.TaskResult.Error(error = s"Error during batch: ${e.getMessage}", processExit = ProcessExit.Unknown)) + case Outcome.Canceled() => cleanup >> IO.pure(TaskDag.TaskResult.Killed(KillReason.UserRequest)) + } + } + } yield result + } + /** Execute a test suite with idle timeout and kill signal handling. * * The idle timeout resets each time a test completes. If no test completes within the timeout period, the suite is considered hung and killed. @@ -113,7 +398,7 @@ object TestRunner { project: CrossProjectName, suiteName: String, selection: FrameworkSelection, - jvm: TestJvm, + jvm: TestSession, eventQueue: Queue[IO, Option[TaskDag.DagEvent]], testArgs: List[String], idleTimeout: FiniteDuration, @@ -189,16 +474,22 @@ object TestRunner { now.flatMap(ts => emit(TaskDag.DagEvent.Output(project, SuiteName(effectiveSuite), message, OutputChannel.fromIsError(isError), ts))) } - case TestProtocol.TestResponse.Error(message, _) => + case TestProtocol.TestResponse.Error(message, details) => // Infrastructure error (JVM died mid-stream, or malformed response) — no authoritative // SuiteDone. Record it as the terminal signal so we emit SuiteError, not a green suite. - terminal.set(Some(Left(message))) + // Keep the runner's diagnostic detail: JvmPool assembles the "likely System.exit()" hint + // and the fork's stderr tail here, and dropping them leaves the user with a bare "died + // unexpectedly" that cannot be acted on. + terminal.set(Some(Left(withDetail(message, details)))) case TestProtocol.TestResponse.Ready => IO.unit case TestProtocol.TestResponse.ThreadDump(_) => IO.unit + + case TestProtocol.TestResponse.BatchComplete => + IO.unit // a single-suite run never batches; only runSuites produces this } .compile .drain @@ -283,9 +574,12 @@ object TestRunner { } .handleError(e => System.err.println(s"[TestRunner] stderr drain failed: ${e.getClass.getName}: ${e.getMessage}")) - // Helper for cleanup - uncancelable and recovers from errors + // Helper for cleanup - uncancelable and recovers from errors. + // killSuite, not kill: on an exclusive fork the two are the same (the fork is this suite), but on a per-project shared fork this stops ONLY this + // suite (a CancelSuite interrupting its thread) and leaves its siblings running. Killing the whole fork here would take a project's other in-flight + // suites down with a single one's timeout or cancellation. def cleanup: IO[Unit] = IO.uncancelable { _ => - drainStderrToEvents.attempt >> jvm.kill.attempt >> suiteFiber.cancel.attempt.void + drainStderrToEvents.attempt >> jvm.killSuite(suiteName).attempt >> suiteFiber.cancel.attempt.void } // On idle timeout the test runner JVM is alive but stuck. Run jstack against it so @@ -355,6 +649,13 @@ object TestRunner { * [[SuiteOutcome.isFailure]], which decides the same question for the summary; when these two disagreed, the run showed "1 failed" with no failure to point * at. */ + /** Append the runner's diagnostic detail to a fork-death message, on its own lines. The detail is what makes such a death actionable — the exit-status + * reading ("likely called System.exit()") and the fork's stderr tail, both assembled in [[bleep.testing.JvmPool]] — and the display renders a multi-line + * message verbatim, so the user finally sees why the JVM went instead of only that it did. + */ + private def withDetail(message: String, details: Option[String]): String = + details.filter(_.trim.nonEmpty).fold(message)(d => s"$message\n$d") + private def taskResultFor(suiteName: String, outcome: SuiteOutcome, failures: List[String]): TaskDag.TaskResult = outcome match { case SuiteOutcome.Executed(_, failed, _, _) if failed > 0 => diff --git a/bleep-cli/src/scala/bleep/commands/BuildCreateNew.scala b/bleep-cli/src/scala/bleep/commands/BuildCreateNew.scala index 9a1e56c59..e48a147de 100644 --- a/bleep-cli/src/scala/bleep/commands/BuildCreateNew.scala +++ b/bleep-cli/src/scala/bleep/commands/BuildCreateNew.scala @@ -125,6 +125,8 @@ object BuildCreateNew { isTestProject = None, testFrameworks = model.JsonSet.empty[model.TestFrameworkName], testTags = model.JsonMap.empty, + maxConcurrentSuites = None, + testFork = None, sourcegen = model.JsonSet.empty[model.ScriptDef], libraryVersionSchemes = model.JsonSet.empty[model.LibraryVersionScheme], ignoreEvictionErrors = None, @@ -278,6 +280,8 @@ object BuildCreateNew { isTestProject = None, testFrameworks = model.JsonSet.empty[model.TestFrameworkName], testTags = model.JsonMap.empty, + maxConcurrentSuites = None, + testFork = None, sourcegen = model.JsonSet.empty[model.ScriptDef], libraryVersionSchemes = model.JsonSet.empty[model.LibraryVersionScheme], ignoreEvictionErrors = None, diff --git a/bleep-cli/src/scala/bleep/mavenimport/buildFromMavenPom.scala b/bleep-cli/src/scala/bleep/mavenimport/buildFromMavenPom.scala index 60774332d..fce5b7bda 100644 --- a/bleep-cli/src/scala/bleep/mavenimport/buildFromMavenPom.scala +++ b/bleep-cli/src/scala/bleep/mavenimport/buildFromMavenPom.scala @@ -239,6 +239,8 @@ object buildFromMavenPom { isTestProject = None, testFrameworks = model.JsonSet.empty[model.TestFrameworkName], testTags = model.JsonMap.empty, + maxConcurrentSuites = None, + testFork = None, sourcegen = model.JsonSet.empty[model.ScriptDef], libraryVersionSchemes = model.JsonSet.empty[model.LibraryVersionScheme], ignoreEvictionErrors = None, @@ -279,6 +281,8 @@ object buildFromMavenPom { isTestProject = Some(true), testFrameworks = testFrameworks, testTags = model.JsonMap.empty, + maxConcurrentSuites = None, + testFork = None, sourcegen = model.JsonSet.empty[model.ScriptDef], libraryVersionSchemes = model.JsonSet.empty[model.LibraryVersionScheme], ignoreEvictionErrors = None, diff --git a/bleep-cli/src/scala/bleep/mavenimport/generateBuildFromMaven.scala b/bleep-cli/src/scala/bleep/mavenimport/generateBuildFromMaven.scala index 5e430f720..cbcfd84ac 100644 --- a/bleep-cli/src/scala/bleep/mavenimport/generateBuildFromMaven.scala +++ b/bleep-cli/src/scala/bleep/mavenimport/generateBuildFromMaven.scala @@ -86,6 +86,8 @@ object generateBuildFromMaven { isTestProject = None, testFrameworks = model.JsonSet.empty[model.TestFrameworkName], testTags = model.JsonMap.empty, + maxConcurrentSuites = None, + testFork = None, sourcegen = model.JsonSet.empty[model.ScriptDef], libraryVersionSchemes = model.JsonSet.empty[model.LibraryVersionScheme], ignoreEvictionErrors = None, diff --git a/bleep-cli/src/scala/bleep/sbtimport/buildFromBloopFiles.scala b/bleep-cli/src/scala/bleep/sbtimport/buildFromBloopFiles.scala index ed4cbd386..b0bfcf05d 100644 --- a/bleep-cli/src/scala/bleep/sbtimport/buildFromBloopFiles.scala +++ b/bleep-cli/src/scala/bleep/sbtimport/buildFromBloopFiles.scala @@ -195,6 +195,8 @@ object buildFromBloopFiles { isTestProject = if (projectType.testLike) Some(true) else None, testFrameworks = testFrameworks, testTags = model.JsonMap.empty, + maxConcurrentSuites = None, + testFork = None, sourcegen = model.JsonSet.empty[model.ScriptDef], libraryVersionSchemes = model.JsonSet.fromIterable(libraryVersionSchemes), ignoreEvictionErrors = convertEvictionErrorLevel(inputProject.sbtExportFile.evictionErrorLevel), diff --git a/bleep-cli/src/scala/bleep/sbtimport/generateBuild.scala b/bleep-cli/src/scala/bleep/sbtimport/generateBuild.scala index a007e20a7..84d981318 100644 --- a/bleep-cli/src/scala/bleep/sbtimport/generateBuild.scala +++ b/bleep-cli/src/scala/bleep/sbtimport/generateBuild.scala @@ -86,6 +86,8 @@ object generateBuild { isTestProject = None, testFrameworks = model.JsonSet.empty[model.TestFrameworkName], testTags = model.JsonMap.empty, + maxConcurrentSuites = None, + testFork = None, sourcegen = model.JsonSet.empty[model.ScriptDef], libraryVersionSchemes = model.JsonSet.empty[model.LibraryVersionScheme], ignoreEvictionErrors = None, diff --git a/bleep-core/src/scala/bleep/history/TranscriptFormat.scala b/bleep-core/src/scala/bleep/history/TranscriptFormat.scala index 7b174841b..1b9e88552 100644 --- a/bleep-core/src/scala/bleep/history/TranscriptFormat.scala +++ b/bleep-core/src/scala/bleep/history/TranscriptFormat.scala @@ -287,6 +287,16 @@ object TranscriptFormat { fields += "ignored" -> Json.fromInt(summary.testsIgnored) durationMs.foreach(d => fields += "durationMs" -> Json.fromLong(d)) + // Suite-level coverage, so a caller reading only this JSON can see whether every discovered suite actually ran. `success` alone cannot convey it (and used to + // hide it entirely over MCP): a run that executed 10 of 29 suites is not the same as one that ran all 29, even when both report `failed: 0`. `toEither` + // already turns a shortfall into `success: false`, but these fields let an agent act on the exact gap rather than parse the summary string. + if (summary.suitesTotal > 0) { + fields += "suitesTotal" -> Json.fromInt(summary.suitesTotal) + fields += "suitesCompleted" -> Json.fromInt(summary.suitesCompleted) + val suitesDidNotFinish = summary.suitesTotal - summary.suitesCompleted - summary.suitesCancelled + if (suitesDidNotFinish > 0) fields += "suitesDidNotFinish" -> Json.fromInt(suitesDidNotFinish) + } + val summaryParts = List.newBuilder[String] problem match { case None => diff --git a/bleep-core/src/scala/bleep/testing/BuildDisplay.scala b/bleep-core/src/scala/bleep/testing/BuildDisplay.scala index 996baba2f..b13dd4e2f 100644 --- a/bleep-core/src/scala/bleep/testing/BuildDisplay.scala +++ b/bleep-core/src/scala/bleep/testing/BuildDisplay.scala @@ -139,6 +139,7 @@ case class BuildSummary( else { val testProblems = testsFailed + testsTimedOut + testsCancelled val testsObserved = testsPassed + testsFailed + testsSkipped + testsIgnored + testsTimedOut + testsCancelled + val suitesUnaccounted = suitesTotal - suitesCompleted - suitesCancelled if (testProblems > 0 || suitesCancelled > 0) { val parts = List.newBuilder[String] parts += s"$testsPassed passed" @@ -147,6 +148,16 @@ case class BuildSummary( if (testsCancelled > 0) parts += s"$testsCancelled cancelled" if (suitesCancelled > 0) parts += s"$suitesCancelled suites cancelled" Left(new bleep.BleepException.Text(s"Tests failed: ${parts.result().mkString(", ")}")) + } else if (suitesUnaccounted > 0) { + // Suites were discovered and their run announced, but fewer reported a result than were announced — the run stopped short of executing all of them. + // Whatever the mechanism (a fork that died mid-run, a scheduler that returned before every suite ran), a partial run is NOT a pass: the same silent-green + // hazard as a zero-suite build, one level up. A green `bleep test` that ran 10 of 29 suites lets CI's "test every project" gate mistake a fraction for full + // coverage. This mirrors what [[BuildDisplay]] already shows as "N did not finish"; the verdict must agree with the summary the user is reading. + Left( + new bleep.BleepException.Text( + s"Tests did not finish: $suitesCompleted of $suitesTotal suites completed, $suitesUnaccounted never reported a result. A partial run is not a pass." + ) + ) } else if (testProjectsWithoutSuites.nonEmpty) { // A project reaches discovery only when it is `isTestProject: true` and its classes compiled, and the count checked here is the one taken *before* // `--only` / `--exclude` / tag filters. So this is not the user narrowing a run to nothing: it is compiled test classes that no framework recognised — diff --git a/bleep-core/src/scala/bleep/testing/BuildState.scala b/bleep-core/src/scala/bleep/testing/BuildState.scala index 1d660c655..490d9adc7 100644 --- a/bleep-core/src/scala/bleep/testing/BuildState.scala +++ b/bleep-core/src/scala/bleep/testing/BuildState.scala @@ -172,6 +172,14 @@ object BuildState { */ object BuildStateReducer { + /** How long a suite occupied its fork, for the task-time / parallelism metric: from when it started to `now`. Guarded on `runningSuites` so that a suite + * whose end is reported by two events (SuiteFinished then SuiteError) is counted once — whichever removes it from `runningSuites` first. A suite with no + * recorded start (its SuiteStarted was lost) contributes 0 rather than a bogus large delta. + */ + private def suiteOccupancyMs(state: BuildState, key: SuiteKey, now: Long): Long = + if (state.runningSuites.contains(key)) state.suiteStartTimes.get(key).map(now - _).getOrElse(0L) + else 0L + def reduce(state: BuildState, event: BuildEvent): BuildState = event match { case BuildEvent.SourcegenStarted(scriptMain, _, _) => @@ -282,11 +290,15 @@ object BuildStateReducer { testsSkipped = state.testsSkipped + (if (status == TestStatus.Skipped || status == TestStatus.AssumptionFailed) 1 else 0), testsIgnored = state.testsIgnored + (if (status == TestStatus.Ignored || status == TestStatus.Pending) 1 else 0), failures = updatedFailures, - skipped = updatedSkipped, - totalTaskTimeMs = state.totalTaskTimeMs + durationMs + skipped = updatedSkipped + // NOT totalTaskTimeMs: an individual test's duration is time already inside its suite's + // fork occupancy, which the suite-terminal handlers count. Adding it here double-counted + // the test methods and, worse, ignored the suite-level boot — a slow-booting suite spends ~25s + // starting an app and its containers before any test method runs, so summing test durations + // saw ~2s for a suite that held a fork for 27s, and parallelism read ~1x under real fan-out. ) - case BuildEvent.SuiteFinished(project, suite, outcome, _, _) => + case BuildEvent.SuiteFinished(project, suite, outcome, _, timestamp) => val key = SuiteKey(project, suite) // Check if SuiteError already counted this suite (SuiteError can arrive before SuiteFinished) val alreadyCounted = state.failures.exists(f => f.project == project && f.suite == suite && f.category == FailureCategory.ProcessError) @@ -343,7 +355,8 @@ object BuildStateReducer { runningSuites = state.runningSuites - key, suiteStartTimes = state.suiteStartTimes - key, pendingOutput = state.pendingOutput - key, - failures = syntheticFailures ++ failuresWithSuiteOutput + failures = syntheticFailures ++ failuresWithSuiteOutput, + totalTaskTimeMs = state.totalTaskTimeMs + suiteOccupancyMs(state, key, timestamp) ) case BuildEvent.Output(project, suite, line, _, _) => @@ -377,8 +390,9 @@ object BuildStateReducer { case _: BuildEvent.LockContention | _: BuildEvent.LockAcquired => state // Lock contention events don't affect build state — handled by display - case BuildEvent.SuiteTimedOut(project, suite, timeoutMs, threadDumpInfo, _) => + case BuildEvent.SuiteTimedOut(project, suite, timeoutMs, threadDumpInfo, timestamp) => val key = SuiteKey(project, suite) + val occupancy = suiteOccupancyMs(state, key, timestamp) // jstack dump arrives via `threadDumpInfo.singleThreadStack` (see ReactiveBsp's SuiteTimedOut translation); // expose it as `failure.throwable` so BuildDisplay's summary Timeouts section renders it under "Stack trace:". val timeoutFailure = TestFailure( @@ -399,11 +413,13 @@ object BuildStateReducer { runningSuites = state.runningSuites - key, suiteStartTimes = state.suiteStartTimes - key, pendingOutput = state.pendingOutput - key, - failures = timeoutFailure :: state.failures + failures = timeoutFailure :: state.failures, + totalTaskTimeMs = state.totalTaskTimeMs + occupancy ) - case BuildEvent.SuiteError(project, suite, error, processExit, _, _) => + case BuildEvent.SuiteError(project, suite, error, processExit, _, timestamp) => val key = SuiteKey(project, suite) + val occupancy = suiteOccupancyMs(state, key, timestamp) val desc = processExit match { case ProcessExit.Signal(sig) => s"Process crashed (signal $sig)" case ProcessExit.ExitCode(code) => s"Process exited with code $code" @@ -430,7 +446,8 @@ object BuildStateReducer { runningSuites = state.runningSuites - key, suiteStartTimes = state.suiteStartTimes - key, pendingOutput = state.pendingOutput - key, - failures = state.failures.map(f => if (f eq existing) merged else f) + failures = state.failures.map(f => if (f eq existing) merged else f), + totalTaskTimeMs = state.totalTaskTimeMs + occupancy ) case None => val errorFailure = TestFailure( @@ -451,7 +468,8 @@ object BuildStateReducer { runningSuites = state.runningSuites - key, suiteStartTimes = state.suiteStartTimes - key, pendingOutput = state.pendingOutput - key, - failures = errorFailure :: state.failures + failures = errorFailure :: state.failures, + totalTaskTimeMs = state.totalTaskTimeMs + occupancy ) } @@ -473,14 +491,15 @@ object BuildStateReducer { failures = errorFailure :: state.failures ) - case BuildEvent.SuiteCancelled(project, suite, reason, _) => + case BuildEvent.SuiteCancelled(project, suite, reason, timestamp) => val key = SuiteKey(project, suite) state.copy( suitesCompleted = state.suitesCompleted + 1, suitesCancelled = state.suitesCancelled + 1, runningSuites = state.runningSuites - key, suiteStartTimes = state.suiteStartTimes - key, - cancelledSuites = CancelledSuite(project, suite, reason) :: state.cancelledSuites + cancelledSuites = CancelledSuite(project, suite, reason) :: state.cancelledSuites, + totalTaskTimeMs = state.totalTaskTimeMs + suiteOccupancyMs(state, key, timestamp) ) case BuildEvent.LinkStarted(project, _, _) => diff --git a/bleep-core/src/scala/bleep/testing/FancyBuildDisplay.scala b/bleep-core/src/scala/bleep/testing/FancyBuildDisplay.scala index 10683b210..0f497ede4 100644 --- a/bleep-core/src/scala/bleep/testing/FancyBuildDisplay.scala +++ b/bleep-core/src/scala/bleep/testing/FancyBuildDisplay.scala @@ -23,6 +23,19 @@ import scala.jdk.CollectionConverters._ /** Terminal UI for build execution - functional and readable */ object FancyBuildDisplay { + /** How many test suites are running right now, for the "Tests (N running)" heading. + * + * Two sources, because the two test paths report at different granularities. A JVM project tracks every suite individually, so each of its running suites is + * already in `runningSuites` and the project itself must not be counted again. A JS or Native project reports only that it is testing — no suite events at + * all — so it contributes nothing to `runningSuites` and is counted once, as the one suite-ish thing known to be in flight. + * + * Counting every `Testing` item, as this did, added the project on top of its own suites: a run over nine JVM projects with nineteen suites in flight + * announced twenty-eight, and the number grew with the number of projects rather than with the work. It read as the runner over-subscribing the machine — + * which the governor will not do, since it admits one CPU permit per suite and holds it for the suite's whole life. + */ + private[testing] def runningTestCount(runningSuites: Int, displayItems: List[ProjectDisplayItem]): Int = + runningSuites + displayItems.count(_.isInstanceOf[ProjectDisplayItem.Testing.Bsp]) + private val spinnerFrames = Array("⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏") // Icons must be width-1 per Wcwidth to avoid rendering artifacts. @@ -722,8 +735,10 @@ object FancyBuildDisplay { // Line 3: compile summary + parallelism + exit hint val wallTimeMs = state.elapsedMs + // A running suite's elapsed time IS the fork it occupies; the individual tests inside it are + // not separate fork-holders, so they must not be added on top (that double-counted, and the + // reducer likewise counts finished work per-suite, not per-test — see suiteOccupancyMs). val runningTaskTimeMs = state.compilingProjects.values.map(_.elapsedMs).sum + - state.runningTests.values.map(_.elapsedMs).sum + state.runningSuites.values.map(_.elapsedMs).sum val totalTaskTimeMs = state.core.totalTaskTimeMs + runningTaskTimeMs val parallelism = @@ -1183,7 +1198,7 @@ object FancyBuildDisplay { } } - val runningCount = state.runningSuites.size + displayItems.count(_.isInstanceOf[ProjectDisplayItem.Testing]) + val runningCount = FancyBuildDisplay.runningTestCount(state.runningSuites.size, displayItems) val title = if (runningCount > 0) s"Tests ($runningCount running)" else "Tests" PaneData(items.toArray, title, Palette.info, Palette.border) } diff --git a/bleep-core/src/scala/bleep/testing/ForkedTestRunnerProtocol.scala b/bleep-core/src/scala/bleep/testing/ForkedTestRunnerProtocol.scala index c1c777f26..415756a85 100644 --- a/bleep-core/src/scala/bleep/testing/ForkedTestRunnerProtocol.scala +++ b/bleep-core/src/scala/bleep/testing/ForkedTestRunnerProtocol.scala @@ -13,4 +13,13 @@ object ForkedTestRunnerProtocol { * Must match `bleep.testing.runner.ForkedTestRunner.PROTOCOL_PORT_PROPERTY`. */ val PortProperty: String = "bleep.test.protocolPort" + + /** System property carrying the file the fork writes its exit diagnostic to (thread dump, whether the command loop had exited, which suites were still + * running). A FILE, not fd 2: when a fork exits from under a run — a test's `System.exit`, or the JVM tearing down — whatever it writes to stderr races the + * pipe closing and is routinely lost, which is exactly how "N suites never reported a result" ended up with no cause. A file survives that teardown, and the + * parent reads it after the fork dies. Its ABSENCE is itself a signal: a `Runtime.halt` (or a hard OS kill) runs no shutdown hooks, so no file is written. + * + * Must match `bleep.testing.runner.ForkedTestRunner.EXIT_LOG_PROPERTY`. + */ + val ExitLogProperty: String = "bleep.test.exitLog" } diff --git a/bleep-core/src/scala/bleep/testing/FrameworkSelection.scala b/bleep-core/src/scala/bleep/testing/FrameworkSelection.scala index 07929ddfd..3ef5f3664 100644 --- a/bleep-core/src/scala/bleep/testing/FrameworkSelection.scala +++ b/bleep-core/src/scala/bleep/testing/FrameworkSelection.scala @@ -34,4 +34,21 @@ object FrameworkSelection { * producing a command the other side would have to interpret. */ case class PlatformRunner(displayName: String) extends FrameworkSelection + + /** sbt frameworks that report a suite's failure detail from `Runner.done()` — called once per run — or on their own threads after the suite's tasks return, + * so running a project's suites through one shared Runner (one `done()`, per-project mode) drops that per-suite output. Keyed by framework class, the real + * identifier. Such a suite gets its own fork instead, where `done()` fires for it alone. Add to this only for a framework proven to lose output when it + * shares a fork — most do not. + */ + private val needsIsolatedForkClasses: Set[String] = Set( + "weaver.framework.CatsEffect", + "hedgehog.sbt.Framework" + ) + + /** Whether this suite must run in a fork of its own rather than share one across the project's suites. See [[needsIsolatedForkClasses]]. */ + def needsIsolatedFork(selection: FrameworkSelection): Boolean = + selection match { + case SbtTestInterface(_, frameworkClass) => needsIsolatedForkClasses(frameworkClass) + case _ => false + } } diff --git a/bleep-core/src/scala/bleep/testing/InProcessTestExecutor.scala b/bleep-core/src/scala/bleep/testing/InProcessTestExecutor.scala new file mode 100644 index 000000000..894318bd1 --- /dev/null +++ b/bleep-core/src/scala/bleep/testing/InProcessTestExecutor.scala @@ -0,0 +1,249 @@ +package bleep.testing + +import cats.effect._ +import fs2.Stream + +import java.net.{URL, URLClassLoader} +import java.nio.file.Path +import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} +import java.util.concurrent.{ConcurrentHashMap, Executors, LinkedBlockingQueue, ThreadFactory} +import scala.concurrent.ExecutionContext +import scala.jdk.CollectionConverters._ + +/** Runs test suites inside the bleep-bsp server, with no fork at all. + * + * A suite costs a classloader and a thread here, against a JVM start and a socket handshake in [[JvmPool]] — the difference a run notices is seconds per + * distinct classpath, and it is why this exists. What it costs instead is the process boundary, and the losses are specific rather than theoretical: + * + * - **`System.exit` takes the server with it.** The forked runner installs a `SecurityManager` to turn a test's exit into a suite error; JEP 486 made that + * permanently unavailable from JDK 24, which is what bleep runs on. There is no supported way to stop it in process. A test that exits kills the daemon + * and every other workspace's build with it. + * - **A wedged suite cannot be killed.** [[kill]] interrupts the suite's thread, and a thread that ignores interruption keeps running. The forked path + * destroys a process and is done; here the idle timeout fires, the suite is reported, and the thread stays. + * - **Global state is shared.** System properties, the default locale and time zone, `System.out`, singletons, whatever a framework caches statically: one + * copy, shared by every suite running concurrently and by the server itself. + * - **Output is not attributed per suite.** `System.out` is one stream for the whole JVM, so with suites running concurrently there is no way to say which + * one printed a line without hijacking it globally and dispatching by thread. Not done here: what a test prints reaches the daemon's own stdout, and + * [[InProcessSession.drainStderr]] has nothing to hand back. + * + * None of that is dressed up as a degraded mode: a request naming `jvmOptions`, an environment or a working directory is refused, because those are properties + * of a process and there is no process to set them on. Silently running the tests without settings the user asked for is the failure mode this refuses to + * have. + * + * Every suite still runs its own [[SuiteRunner]] instance loaded from the project's own classpath, so framework loading, fingerprint ordering and event + * translation are the same code the fork runs, reached the same way, producing the same [[TestProtocol]] lines. + * + * @param maxConcurrentSuites + * threads kept for running suites. Not a limit on how many the DAG will admit — that is the machine governor's job, and it admits test tasks by CPU. This + * only sizes the pool they land on. + */ +class InProcessTestExecutor(maxConcurrentSuites: Int) extends TestExecutor { + + /** The only runner entry point this executor knows how to be. A request naming another one wants something this cannot provide. */ + private val KnownRunnerClass = "bleep.testing.runner.ForkedTestRunner" + + /** One loader per distinct classpath, shared by every suite that runs against it — the analogue of [[JvmPool]] keying its forks by classpath, and the reason + * a second suite on the same project starts instantly. + */ + private val loaders = new ConcurrentHashMap[String, URLClassLoader]() + + private val threadCounter = new AtomicInteger(0) + + private val suitePool = Executors.newFixedThreadPool( + math.max(1, maxConcurrentSuites), + new ThreadFactory { + def newThread(r: Runnable): Thread = { + // Named so a thread dump of a stuck in-process run says which threads are tests. Without a process to point at, the thread name is the only handle a + // person has on a suite that will not finish. + val t = new Thread(r, s"bleep-test-inprocess-${threadCounter.incrementAndGet()}") + t.setDaemon(true) + t + } + } + ) + + private val suiteEc: ExecutionContext = ExecutionContext.fromExecutorService(suitePool) + + override def acquire(request: TestSessionRequest): Resource[IO, TestSession] = + Resource.eval { + IO { + if (request.jvmOptions.nonEmpty) + sys.error( + s"${request.label}: in-process test execution cannot honour jvmOptions ${request.jvmOptions.mkString(", ")} — they configure a JVM, and this runs in the one already started. Fork this project's tests, or drop the options." + ) + if (request.environment.nonEmpty) + sys.error( + s"${request.label}: in-process test execution cannot set environment variables (${request.environment.keys.toList.sorted.mkString(", ")}) — a process inherits its environment at start and cannot change its own. Fork this project's tests." + ) + request.workingDirectory.foreach { wd => + sys.error( + s"${request.label}: in-process test execution cannot set the working directory to $wd — a JVM has exactly one, shared with the server and every other suite. Fork this project's tests." + ) + } + if (request.runnerClass != KnownRunnerClass) + sys.error(s"${request.label}: unknown test runner class ${request.runnerClass}; in-process execution only knows $KnownRunnerClass") + request.sharing match { + case SessionSharing.Exclusive => () + case SessionSharing.Shared(key) => + sys.error( + s"${request.label}: in-process test execution cannot honour a per-project shared fork (project '$key') — sharing one fork across a project's suites is a property of a forked JVM, and this runs in the server. Fork this project's tests." + ) + } + + new InProcessSession(loaderFor(request.classpath), suiteEc) + } + } + + /** The classpath, loaded flat, under the platform loader. + * + * Flat and platform-parented on purpose. Nothing of bleep's is visible to the tests: not its Scala library (the project's may be 2.12, 2.13 or 3), not its + * cats-effect, not its copy of the test frameworks. The only types that cross between the server and a suite are `java.*` ones — a `Consumer` going + * in and encoded protocol lines coming back — so there is no shared bleep class to conflict, and no equivalent of sbt's layering question to get wrong. + * + * sbt reaches the same place from the other direction: its `Flat` strategy is documented as the one to use when layering causes trouble, and layering exists + * there to reuse loaded classes across runs. Here the loader is already reused across suites by being cached per classpath, so layering would buy the one + * thing it is for and cost the isolation. + */ + private def loaderFor(classpath: List[Path]): URLClassLoader = + loaders.computeIfAbsent( + classpath.map(_.toString).mkString(java.io.File.pathSeparator), + _ => { + val urls: Array[URL] = classpath.map(_.toUri.toURL).toArray + new URLClassLoader(urls, ClassLoader.getPlatformClassLoader) + } + ) + + override def shutdown: IO[Unit] = + IO { + loaders.values().asScala.foreach { loader => + // A loader that will not close keeps its jars mapped; say so rather than leaving the run to wonder why the files are locked on Windows. + loader.close() + } + loaders.clear() + suitePool.shutdownNow() + () + } + + override def size: IO[Int] = IO(loaders.size()) +} + +object InProcessTestExecutor { + + /** Marks the end of a suite's response stream. Identity-compared, so it can never collide with a response. */ + private[testing] object EndOfSuite +} + +/** One classpath's worth of in-process test execution. + * + * Not exclusive, unlike a forked [[TestJvm]]: several sessions share a loader and run concurrently, which is the point — the DAG admits N test tasks and all N + * of them run here at once. + */ +private class InProcessSession(loader: URLClassLoader, suiteEc: ExecutionContext) extends TestSession { + + /** The thread currently running a suite, so [[kill]] has something to interrupt and [[dumpThreads]] something to point at. */ + private val runningThread = new AtomicReference[Thread](null) + + /** The server's own pid. The tests really do run in this process, so this is the truth rather than a stand-in. */ + override val pid: Long = ProcessHandle.current().pid() + + override def runSuite( + className: String, + selection: FrameworkSelection, + args: List[String] + ): Stream[IO, TestProtocol.TestResponse] = { + // Encoded, not passed as objects. The command crosses into a classloader that shares none of bleep's types with this one, and the same encoding is what + // goes down the socket to a fork — so the two paths are handed byte-identical instructions and decoded by the same decoder on the way back. + val commandLine = TestProtocol.encodeCommand(TestProtocol.TestCommand.RunSuite(className, selection, args)) + + Stream.eval(IO(new LinkedBlockingQueue[AnyRef]())).flatMap { queue => + val sink: java.util.function.Consumer[String] = (line: String) => + queue.put(TestProtocol.decodeResponse(line) match { + case Right(response) => response + case Left(err) => TestProtocol.TestResponse.Error(s"Protocol error: ${err.getMessage}", Some(s"Line: $line")) + }) + + val runSuiteOnThread = IO + .interruptible { + runningThread.set(Thread.currentThread()) + try { + val runnerClass = loader.loadClass("bleep.testing.runner.SuiteRunner") + val ctor = runnerClass.getConstructor(classOf[java.util.function.Consumer[?]], classOf[ClassLoader], classOf[java.util.List[?]]) + val runner = ctor.newInstance(sink, loader, java.util.Collections.emptyList[java.io.Flushable]()) + runnerClass.getMethod("runSerialized", classOf[String]).invoke(runner, commandLine) + () + } catch { + case t: Throwable => + // The suite never got to report itself, so nothing else will. Emitted as the same terminal Error a dead fork produces, which the caller already + // knows how to turn into a failed suite rather than a green one. + val cause = t match { + case e: java.lang.reflect.InvocationTargetException if e.getCause != null => e.getCause + case other => other + } + queue.put( + TestProtocol.TestResponse.Error( + s"in-process runner failed for $className: ${cause.getClass.getName}: ${cause.getMessage}", + Some(stackTraceOf(cause)) + ) + ) + } finally { + runningThread.set(null) + queue.put(InProcessTestExecutor.EndOfSuite) + } + } + .evalOn(suiteEc) + + Stream + .eval(runSuiteOnThread.start) + .flatMap { fiber => + Stream + .repeatEval(IO.interruptible(queue.take())) + .takeWhile(_ ne InProcessTestExecutor.EndOfSuite) + .map(_.asInstanceOf[TestProtocol.TestResponse]) + .onFinalize(fiber.cancel) + } + } + } + + private def stackTraceOf(t: Throwable): String = { + val sw = new java.io.StringWriter() + t.printStackTrace(new java.io.PrintWriter(sw)) + sw.toString + } + + /** Read straight off this JVM's own threads. No `jstack` and no attach: the threads in question are ours. */ + override def getThreadDump: IO[Option[TestProtocol.TestResponse.ThreadDump]] = + dumpThreads.map(lines => if (lines.isEmpty) None else Some(TestProtocol.TestResponse.ThreadDump(threadDumpEntries))) + + override def dumpThreads: IO[List[String]] = + IO(threadDumpEntries.flatMap(entry => s"\"${entry.name}\" ${entry.state}" :: entry.stackTrace.map("\tat " + _))) + + private def threadDumpEntries: List[TestProtocol.TestResponse.ThreadInfo] = + Thread.getAllStackTraces.asScala.toList.map { case (thread, frames) => + TestProtocol.TestResponse.ThreadInfo(thread.getName, thread.getState.toString, frames.toList.map(_.toString)) + } + + /** Nothing to drain: in process there is no second stream to read, because the suite writes to the server's own stdout. See the class comment on + * [[InProcessTestExecutor]] — this is a stated loss, not an empty result standing in for one. + */ + override def drainStderr: IO[List[String]] = IO.pure(Nil) + + /** Always: this session is the server, and if the server were gone nobody would be asking. */ + override def isAlive: IO[Boolean] = IO.pure(true) + + /** Interrupt the suite's thread. Best-effort by construction — a thread that does not check for interruption keeps running, and there is no process to + * destroy instead. The caller's idle timeout still reports the suite; what it cannot do is reclaim the thread. + */ + override def kill: IO[Unit] = + IO { + val thread = runningThread.get() + if (thread != null) thread.interrupt() + } + + override def killSuite(className: String): IO[Unit] = + // Each in-process session runs a single suite on its own thread, so stopping that suite is the same interrupt as `kill`. + kill + + override def runSuites(classNames: List[String], parallelism: Int, selection: FrameworkSelection, args: List[String]): Stream[IO, TestProtocol.TestResponse] = + // In-process runs one suite per session on its own thread; a batched one-execution run belongs to a forked JVM, where reuse across classes is worth having. + Stream.raiseError[IO](new IllegalStateException("runSuites (one-execution batch) is a forked-JVM path; in-process runs suites individually")) +} diff --git a/bleep-core/src/scala/bleep/testing/JvmPool.scala b/bleep-core/src/scala/bleep/testing/JvmPool.scala index 244599cab..3f56c9412 100644 --- a/bleep-core/src/scala/bleep/testing/JvmPool.scala +++ b/bleep-core/src/scala/bleep/testing/JvmPool.scala @@ -9,6 +9,7 @@ import fs2.Stream import java.io._ import java.net.{InetAddress, ServerSocket, Socket, SocketTimeoutException} import java.nio.charset.StandardCharsets +import java.nio.file.Files import java.nio.file.Path import java.security.MessageDigest import java.util.concurrent.TimeUnit @@ -28,7 +29,7 @@ import scala.util.control.NonFatal * - Explicit shutdown (no shutdown hooks) * - Health checks before reuse */ -trait JvmPool { +trait JvmPool extends TestExecutor { /** Acquire a JVM suitable for the given classpath and options. * @@ -47,6 +48,33 @@ trait JvmPool { workingDirectory: Option[Path] ): Resource[IO, TestJvm] + /** The [[TestExecutor]] shape of the above: same call, with the arguments gathered into the request every executor is handed. + * + * The one branch a pool makes on the way in: an [[SessionSharing.Exclusive]] request gets a fork of its own (the classic path above), a + * [[SessionSharing.Shared]] one joins the single fork its project shares (`acquireShared`). Everything past this point — `TestRunner`, the event stream, the + * idle timeout — is handed a [[TestSession]] and cannot tell which it got. + */ + final override def acquire(request: TestSessionRequest): Resource[IO, TestSession] = + request.sharing match { + case SessionSharing.Exclusive => + acquire( + request.label, + request.classpath, + request.jvmOptions, + request.defaultHeapMb, + request.runnerClass, + request.environment, + request.workingDirectory + ) + case SessionSharing.Shared(key) => + acquireShared(request, key) + } + + /** Join (or, as the first of a project's suites, create) the one fork the project named by `key` shares. The fork is torn down when the last suite holding it + * releases. The returned session's `runSuite` may be called concurrently by different suites; how many do so at once is bounded by admission, not here. + */ + def acquireShared(request: TestSessionRequest, key: String): Resource[IO, TestSession] + /** Shutdown all JVMs in the pool. * * This MUST be called when done with the pool. Use guarantee to ensure it runs. @@ -58,7 +86,7 @@ trait JvmPool { } /** A handle to a forked JVM running the test runner */ -trait TestJvm { +trait TestJvm extends TestSession { /** Process ID of this JVM */ def pid: Long @@ -227,7 +255,15 @@ object JvmPool { else process.exitValue() match { case 0 => - ExitDescription("EOF on stdout, exited 0", Some("The JVM exited cleanly without sending a suite result — it likely called System.exit().")) + ExitDescription( + "EOF on stdout, exited 0", + Some( + "The JVM exited cleanly (0) without sending a suite result. Something ended the process out from under the run: a System.exit(0), a " + + "Runtime.halt(0), or its last non-daemon thread finishing. The fork's exit log distinguishes them — if one was written, a shutdown hook " + + "ran (System.exit or a normal exit) and it names the caller; its ABSENCE means no hook ran at all, i.e. Runtime.halt(0) or a hard kill. A " + + "daemon-thread watchdog that calls halt() to bound a subprocess is a classic source when it is armed inside a shared, long-lived fork." + ) + ) case 137 => ExitDescription( "killed by SIGKILL (exit 137)", @@ -255,6 +291,57 @@ object JvmPool { } } + /** Whatever a fork wrote before it stopped, for a spawn-failure diagnostic — the message the user reads when a test JVM never connects back. + * + * `exited` decides HOW to read, and it is the whole point of this existing. An exited fork has flushed and closed its streams: drain to EOF, because that is + * the only way to get the tail — the JVM's "Unrecognized VM option ...", "Could not create the Java Virtual Machine", an `hs_err` pointer — lands on stderr + * a beat AFTER the process is seen dead, and `available()` reports 0 at that instant. Reading only what was `available()` is exactly how that message got + * lost, turning a fully-explained failure into a bare "N suites never reported a result". A still-running fork has open streams, so take only what is + * already buffered, without blocking the very code whose job is to report a hang. Bounded to `maxBytes` per stream. + * + * Takes a bare [[Process]] so it is unit-testable without a pool or a BSP server: spawn `java -version`, which exits non-zero with the reason + * on stderr, and assert it comes back here. + */ + private[testing] def describeChildOutput(process: Process, exited: Boolean, maxBytes: Int = MaxChildOutputBytes): String = { + def read(stream: InputStream): String = if (exited) drainToEof(stream, maxBytes) else drainAvailable(stream, maxBytes) + val quoted = + List("stderr" -> read(process.getErrorStream), "stdout" -> read(process.getInputStream)) + .collect { case (name, text) if text.trim.nonEmpty => s"\n $name: ${text.trim}" } + if (quoted.isEmpty) " The fork wrote no output." else quoted.mkString + } + + /** Bytes already sitting in the pipe, never blocking — safe on a process that may still be running. Stops as soon as nothing more is buffered, so a live fork + * that has more to say later is not waited on. + */ + private[testing] def drainAvailable(stream: InputStream, maxBytes: Int): String = { + val collected = new ByteArrayOutputStream + val buf = new Array[Byte](8192) + var more = true + while (more && collected.size < maxBytes) { + val ready = stream.available() + if (ready <= 0) more = false + else { + val n = stream.read(buf, 0, math.min(buf.length, math.min(ready, maxBytes - collected.size))) + if (n <= 0) more = false else collected.write(buf, 0, n) + } + } + new String(collected.toByteArray, StandardCharsets.UTF_8) + } + + /** Read to EOF. Safe ONLY on a process that has already exited — its writer is closed, so `read` returns -1 rather than blocking. This is what actually + * captures a startup failure's full stderr, which `drainAvailable` races and misses. Bounded to `maxBytes`. + */ + private[testing] def drainToEof(stream: InputStream, maxBytes: Int): String = { + val collected = new ByteArrayOutputStream + val buf = new Array[Byte](8192) + var more = true + while (more && collected.size < maxBytes) { + val n = stream.read(buf, 0, math.min(buf.length, maxBytes - collected.size)) + if (n < 0) more = false else collected.write(buf, 0, n) + } + new String(collected.toByteArray, StandardCharsets.UTF_8) + } + /** Key for pooling JVMs */ private case class JvmKey(classpathHash: String, optionsHash: String, envHash: String, cwdHash: String) { @@ -305,6 +392,14 @@ object JvmPool { * is actually destroyed. Must be run exactly where the process is killed — see `JvmPoolImpl.destroy`. */ val releaseMemory: IO[Unit], + /** File the fork writes its exit diagnostic to (see [[ForkedTestRunnerProtocol.ExitLogProperty]]). Read by [[readExitLog]] after the fork dies, when it + * is the only surviving account of an exit the parent otherwise sees as a bare "exited 0". + */ + val exitLogPath: Path, + /** Where [[kill]] announces itself. Not the pool's `listener` field reached directly, because this class is not nested in `JvmPoolImpl`; the pool passes + * it at construction so the single kill chokepoint can report every termination on the same channel as the other fork events. + */ + val listener: JvmPoolListener, /** When this fork was created. Taken at construction, not from `process.info().startInstant()` when it dies: by then the process has been killed and the * OS no longer reports a start instant for it, which is why every fork_end carried a lifetime of -1. * @@ -414,8 +509,28 @@ object JvmPool { @volatile private var _killedByUs: Option[String] = None def killedByUs: Option[String] = _killedByUs - def kill(reason: String): Unit = { - if (_killedByUs.isEmpty) _killedByUs = Some(reason) + /** Terminate the fork, escalating instead of going straight to SIGKILL. + * + * `graceMillis` is how long the child gets to die on its own terms: half after the socket close (which it reads as end-of-commands and exits on), half + * after SIGTERM. Both routes run the JVM's shutdown hooks — and for a fork that started an application those hooks are what stop it and the testcontainers + * it started. With ryuk disabled (required for testcontainers reuse, and common) those hooks are the ONLY container cleanup there is; SIGKILLing first + * thing is how a machine ends up with dozens of orphaned databases. + * + * Pass 0 when the fork has forfeited its grace — it never completed the startup handshake, or a collective shutdown deadline already gave it time. + */ + def kill(reason: String, graceMillis: Long): Unit = { + // Every bleep-initiated socket close funnels through here (stdin/protocolSocket close below), + // so this one announcement accounts for every fork bleep tears down. If a fork's socket goes + // to EOF and no onForkKill named its pid first, bleep did not close it — the fork exited on + // its own (a test's System.exit, a natural end, or an OS kill). That distinction is exactly + // what was ambiguous when "N suites never reported a result" had no cause; recording every + // kill on the fork-event channel (joined to fork_end by pid) settles it after the fact. + val wasAlive = process.isAlive + listener.onForkKill(process.pid(), reason, wasAlive, graceMillis) + // Only claim the kill if there is something left to kill: a fork that already exited on its + // own (e.g. gracefully during shutdown's deadline) must not be attributed to bleep — this + // flag is the only thing separating our kills from natural exits and OS kills. + if (wasAlive && _killedByUs.isEmpty) _killedByUs = Some(reason) alive = false try stdin.close() @@ -424,9 +539,17 @@ object JvmPool { try protocolSocket.close() catch { case NonFatal(_) => } - // Kill the entire process tree, not just the direct child. - // If the test runner spawned sub-processes (e.g., for some test frameworks), - // those would otherwise be orphaned and consume system resources. + def waitForExit(millis: Long): Boolean = + millis > 0 && (try process.waitFor(millis, java.util.concurrent.TimeUnit.MILLISECONDS) + catch { case NonFatal(_) => false }) + if (!waitForExit(graceMillis / 2)) { + process.destroy(): Unit // SIGTERM: shutdown hooks still run if the JVM is responsive + if (!waitForExit(graceMillis / 2)) { + process.destroyForcibly(): Unit + } + } + // Sweep whatever the child left behind, however it died. A gracefully-exited JVM reaps its + // own children; this catches the rest so orphaned sub-processes don't consume the machine. try process .descendants() @@ -435,7 +558,6 @@ object JvmPool { catch { case _: Exception => () } ) catch { case NonFatal(_) => } - process.destroyForcibly() try process.waitFor(5, java.util.concurrent.TimeUnit.SECONDS): Unit catch { case NonFatal(_) => } @@ -451,6 +573,19 @@ object JvmPool { } sb.toString() } + + /** The fork's exit diagnostic, if it wrote one, then deleted. Empty when the file is absent — which is itself informative: a `Runtime.halt` or a hard OS + * kill runs no shutdown hooks, so the fork never got to write it, distinguishing those from a `System.exit` (hooks run, file present). + */ + def readExitLog(): String = + try + if (java.nio.file.Files.exists(exitLogPath)) { + val content = new String(java.nio.file.Files.readAllBytes(exitLogPath), StandardCharsets.UTF_8) + try java.nio.file.Files.deleteIfExists(exitLogPath): Unit + catch { case NonFatal(_) => } + content + } else "" + catch { case NonFatal(_) => "" } } /** Max consecutive spawn failures per key before refusing to spawn. Prevents infinite retry when test runner jar is incompatible. */ @@ -535,7 +670,8 @@ object JvmPool { * actually surrendered yet. */ private def destroy(jvm: ManagedJvm, destroyReason: String): IO[Unit] = - observeCost(jvm).attempt >> IO(jvm.kill(destroyReason)).attempt >> announceEnd(jvm).attempt >> allJvms.update(_ - jvm) >> jvm.releaseMemory + observeCost(jvm).attempt >> IO.blocking(jvm.kill(destroyReason, graceMillis = 10000)).attempt >> announceEnd(jvm).attempt >> allJvms.update(_ - jvm) >> + jvm.releaseMemory /** Announced after `kill`, so the exit description is final and `killedByUs` is set — that flag is the only thing separating a fork bleep terminated from * one the OS killed, since both report exit 137. @@ -630,33 +766,6 @@ object JvmPool { } } yield jvm - /** Whatever the fork wrote before it stopped, read without ever blocking. - * - * Only bytes already sitting in the pipe are taken, and only up to [[MaxChildOutputBytes]]. Nothing is draining these streams at this point — the reader - * threads belong to `ManagedJvm`, which does not exist yet on this path — so a blocking read here would hang the very code whose job is to report a hang. - */ - private def describeChildOutput(process: Process): String = { - val quoted = - List("stderr" -> drainAvailable(process.getErrorStream), "stdout" -> drainAvailable(process.getInputStream)) - .collect { case (name, text) if text.trim.nonEmpty => s"\n $name: ${text.trim}" } - if (quoted.isEmpty) " The fork wrote no output." else quoted.mkString - } - - private def drainAvailable(stream: InputStream): String = { - val collected = new ByteArrayOutputStream - val buf = new Array[Byte](8192) - var more = true - while (more && collected.size < MaxChildOutputBytes) { - val ready = stream.available() - if (ready <= 0) more = false - else { - val n = stream.read(buf, 0, math.min(buf.length, math.min(ready, MaxChildOutputBytes - collected.size))) - if (n <= 0) more = false else collected.write(buf, 0, n) - } - } - new String(collected.toByteArray, StandardCharsets.UTF_8) - } - /** Wait for a freshly spawned fork to connect back, giving up the moment that becomes impossible rather than always serving the full sentence. * * Polled instead of one long `accept`, because the answer is usually available long before the deadline: a fork that died during JVM startup is never @@ -672,10 +781,15 @@ object JvmPool { val deadlineNanos = System.nanoTime() + ProtocolConnectTimeout.toNanos listener.setSoTimeout(ProtocolPollInterval.toMillis.toInt) - def giveUp(reason: String): Nothing = { + def giveUp(reason: String, exited: Boolean): Nothing = { // Read what the fork wrote before killing it. `destroyForcibly` closes these pipes as the process is reaped, and a read landing on the far side of // that comes back "Stream closed", replacing the diagnosis this exists to produce. - val childOutput = describeChildOutput(process) + // + // `exited` decides HOW we read. A fork that already exited (a bad JVM option, a startup crash) has written its whole story to stderr — "Unrecognized VM + // option", "Could not create the Java Virtual Machine" — and closed it; we must drain to EOF to get it, because `available()` races the flush and + // usually reports 0 the instant the process is detected dead, which is exactly how that message got lost. A fork still running has an open stderr, so we + // can only take what is already buffered without blocking the very code meant to report a hang. + val childOutput = describeChildOutput(process, exited) if (process.isAlive) { process.destroyForcibly(): Unit process.waitFor(5, TimeUnit.SECONDS): Unit @@ -688,11 +802,12 @@ object JvmPool { try connected = listener.accept() catch { case _: SocketTimeoutException => - if (!process.isAlive) giveUp(s"the fork exited with code ${process.exitValue()} without ever connecting") + if (!process.isAlive) giveUp(s"the fork exited with code ${process.exitValue()} without ever connecting", exited = true) else if (System.nanoTime() >= deadlineNanos) giveUp( s"the fork was still running $ProtocolConnectTimeout later and had not connected, so it is not speaking this server's protocol — check " + - "whether another bleep-test-runner is shadowing the one bleep puts on the test classpath" + "whether another bleep-test-runner is shadowing the one bleep puts on the test classpath", + exited = false ) } connected @@ -758,7 +873,13 @@ object JvmPool { val protocolListener = new ServerSocket(0, 1, InetAddress.getLoopbackAddress) val protocolPort = protocolListener.getLocalPort - val cmdWithProtocol = cmd.head :: s"-D${ForkedTestRunnerProtocol.PortProperty}=$protocolPort" :: cmd.tail + val exitLogPath = Files.createTempFile("bleep-test-fork-exit-", ".log") + Files.delete(exitLogPath) // the fork (re)creates it only if it actually reaches its shutdown; its absence is a signal (see ExitLogProperty) + val cmdWithProtocol = + cmd.head :: + s"-D${ForkedTestRunnerProtocol.PortProperty}=$protocolPort" :: + s"-D${ForkedTestRunnerProtocol.ExitLogProperty}=$exitLogPath" :: + cmd.tail val pb = new ProcessBuilder(cmdWithProtocol*) pb.directory(cwdOverride.getOrElse(workingDirectory).toFile) pb.redirectErrorStream(false) @@ -792,7 +913,7 @@ object JvmPool { val stderr = new BufferedReader(new InputStreamReader(process.getErrorStream)) val processStdout = new BufferedReader(new InputStreamReader(process.getInputStream)) - new ManagedJvm(process, stdin, stdout, stderr, processStdout, protocolSocket, key, jvmCommand, releaseMemory) + new ManagedJvm(process, stdin, stdout, stderr, processStdout, protocolSocket, key, jvmCommand, releaseMemory, exitLogPath, listener) } .flatTap(jvm => allJvms.update(_ + jvm)) .flatTap(jvm => @@ -845,7 +966,9 @@ object JvmPool { // A slow start is not a dead JVM. Under load — 18 cores saturated, dozens of JVMs paging in a // large classpath — reaching Ready can legitimately take a while, and killing at 30s turned // "this machine is busy" into a SIGKILL we then blamed on the OS. - .onError { case _ => IO(jvm.kill("bleep: no Ready handshake within the startup timeout")) } + // No grace: a fork that never completed the handshake has not run any suite, so it has no + // application or containers to wind down. + .onError { case _ => IO.blocking(jvm.kill("bleep: no Ready handshake within the startup timeout", graceMillis = 0)) } override def shutdown: IO[Unit] = // CRITICAL: Use uncancelable to ensure cleanup completes even during cancellation @@ -861,10 +984,25 @@ object JvmPool { } catch { case NonFatal(_) => } } } - // Give them a moment to shutdown gracefully - _ <- IO.sleep(500.millis) + // Wait for graceful exits under one shared deadline. The Shutdown command makes a healthy + // runner exit on its own, running its shutdown hooks — for a fork running an application that is where + // it stops and its testcontainers get removed. The + // old fixed 500ms then SIGKILL truncated exactly those hooks, so every run leaked its + // containers when ryuk was disabled. Well-behaved forks exit as fast as ever; the + // deadline only costs time on forks that are actually winding something down. _ <- IO.blocking { - jvms.foreach(_.kill("bleep: pool shutdown")) + val deadlineNanos = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(10) + jvms.foreach { jvm => + val remaining = deadlineNanos - System.nanoTime() + if (remaining > 0) { + try jvm.process.waitFor(remaining, java.util.concurrent.TimeUnit.NANOSECONDS): Unit + catch { case NonFatal(_) => } + } + } + } + // Stragglers forfeited their grace; kill() with none escalates straight to SIGKILL. + _ <- IO.blocking { + jvms.foreach(_.kill("bleep: pool shutdown", graceMillis = 0)) } // Shutdown kills directly rather than going through `destroy`, so without this the JVMs that survived to the end of a run — usually most of them — // would have a fork_start and never a fork_end, and their lifetimes would be unknowable. @@ -939,6 +1077,27 @@ object JvmPool { } } + override def runSuites( + classNames: List[String], + parallelism: Int, + selection: FrameworkSelection, + args: List[String] + ): Stream[IO, TestProtocol.TestResponse] = { + val command = TestProtocol.TestCommand.RunSuites(classNames, parallelism, selection, args) + val body = + Stream.eval(IO(jvm.markSuiteStarted()) >> sendCommand(command)) >> + readResponses.takeThrough { + // One execute for the whole set; every class's own SuiteDone already went by, so the batch ends at BatchComplete (or a fork-level Error). + case TestProtocol.TestResponse.BatchComplete => false + case _: TestProtocol.TestResponse.Error => false + case _ => true + } + body.onFinalizeCase { + case Resource.ExitCase.Succeeded => IO(jvm.markSuiteFinished()) + case _ => IO.unit + } + } + private def sendCommand(cmd: TestProtocol.TestCommand): IO[Unit] = IO.blocking { jvm.stdin.println(TestProtocol.encodeCommand(cmd)) @@ -957,13 +1116,24 @@ object JvmPool { // `SuiteError`, not the silent `SuiteFinished(0,0,0,0,...)` path. Previously this returned `None` + `unNoneTerminate` — silent zero-count finish. jvm.markDead() val pid = jvm.process.pid() + // Give a just-closed process a beat to finish dying so its exit code is final and its shutdown-hook exit log is fully written before we read them. + try jvm.process.waitFor(2, java.util.concurrent.TimeUnit.SECONDS): Unit + catch { case NonFatal(_) => } val stderrTail = jvm.readStderr() + val exitLog = jvm.readExitLog() // Reap it and say HOW it died. "EOF on stdout" alone is undiagnosable — it looks the // same whether the JVM exited, crashed, or was killed by the OS. The exit status // distinguishes them, and an externally-signalled death (128+signal, so 137 = SIGKILL) // is the fingerprint of the kernel reclaiming memory, which no in-process log can show. val exitDescription = JvmPool.describeExit(jvm.process, jvm.killedByUs) - val details = List(exitDescription.detail, Option.when(stderrTail.nonEmpty)(s"stderr tail:\n$stderrTail")).flatten match { + // The exit log is the fork's own account, written to a FILE that survives the pipe teardown that loses stderr. Its ABSENCE is a signal too: on a + // clean exit-0 death with no log, no shutdown hook ran — a `Runtime.halt` or a hard kill, not a `System.exit`. + val exitLogPart = + if (exitLog.nonEmpty) Some(s"fork exit log:\n$exitLog") + else if (exitDescription.summary.contains("exited 0")) + Some("fork wrote no exit log — no shutdown hook ran (Runtime.halt, or a hard external kill), not a System.exit.") + else None + val details = List(exitDescription.detail, exitLogPart, Option.when(stderrTail.nonEmpty)(s"stderr tail:\n$stderrTail")).flatten match { case Nil => None case lines => Some(lines.mkString("\n")) } @@ -1013,7 +1183,246 @@ object JvmPool { IO(jvm.isAlive) override def kill: IO[Unit] = - IO(jvm.kill("bleep: explicit kill (suite timeout or cancellation)")) + // On cancellation the fork is healthy: the socket close makes it exit on its own, running + // the shutdown hooks that stop an app's containers. On a suite timeout the JVM may + // be wedged, in which case the grace period merely delays the SIGKILL it was always + // getting — after the suite already burned its idle timeout, that delay is noise. + IO.blocking(jvm.kill("bleep: explicit kill (suite timeout or cancellation)", graceMillis = 10000)) + + override def killSuite(className: String): IO[Unit] = + // Exclusive: the fork runs only this suite, so stopping the suite is stopping the fork. + kill + } + + // ============================ Shared per-project sessions ============================ + + /** One project's shared fork, and how many of its suites are currently holding it. The session is created behind a Deferred so that when several suites of + * a project ask at once, exactly one of them spawns the fork and the rest wait on it rather than each spawning their own. + */ + private case class SharedSlot(session: Deferred[IO, Either[Throwable, SharedProjectSession]], refCount: Int) + + // Allocated here rather than threaded through the constructor so `SharedProjectSession` can stay an inner class with direct access to `destroy`, + // `ManagedJvm` and the rest of the pool. Ref.unsafe is the same shortcut the pool's per-key queues take with `unsafeRunSync`. + private val sharedSlots: Ref[IO, Map[String, SharedSlot]] = Ref.unsafe(Map.empty) + + override def acquireShared(request: TestSessionRequest, key: String): Resource[IO, TestSession] = + // One permit per suite, exactly as the exclusive path takes — this run's parallelism is bounded the same whether or not suites share a fork. The shared + // session underneath is refcounted separately, so the fork outlives any one suite and is torn down only when the last suite releases. + Resource.make(semaphore.acquire)(_ => semaphore.release).flatMap { _ => + Resource.make(acquireSharedSession(request, key))(_ => releaseSharedSession(key)).map(s => s: TestSession) + } + + private def acquireSharedSession(request: TestSessionRequest, key: String): IO[SharedProjectSession] = + Deferred[IO, Either[Throwable, SharedProjectSession]].flatMap { fresh => + sharedSlots + .modify { slots => + slots.get(key) match { + case Some(slot) => (slots.updated(key, slot.copy(refCount = slot.refCount + 1)), (slot.session, false)) + case None => (slots.updated(key, SharedSlot(fresh, refCount = 1)), (fresh, true)) + } + } + .flatMap { case (deferred, isCreator) => + if (isCreator) + // Drop the slot on failure so a later suite can try again; the waiters parked on this Deferred get the same failure and fail their own acquire, + // so none of them will release (Resource.make only releases what it acquired). + buildSharedSession(request, key).attempt.flatMap { outcome => + val cleanup = outcome match { + case Left(_) => sharedSlots.update(_ - key) + case Right(_) => IO.unit + } + cleanup >> deferred.complete(outcome) >> IO.fromEither(outcome) + } + else + deferred.get.flatMap(IO.fromEither) + } + } + + private def releaseSharedSession(key: String): IO[Unit] = + sharedSlots + .modify { slots => + slots.get(key) match { + case Some(slot) if slot.refCount > 1 => (slots.updated(key, slot.copy(refCount = slot.refCount - 1)), None) + case Some(slot) => (slots - key, Some(slot.session)) // last suite out + case None => (slots, None) + } + } + .flatMap { + case Some(deferred) => deferred.get.flatMap { case Right(s) => s.teardown; case Left(_) => IO.unit } + case None => IO.unit + } + + private def buildSharedSession(request: TestSessionRequest, key: String): IO[SharedProjectSession] = { + val _ = key + val boundedOptions = MachineResources.withHeapBound(request.jvmOptions, request.defaultHeapMb) + val jvmKey = JvmKey(request.classpath, boundedOptions, request.environment, request.workingDirectory) + getOrCreate(request.label, jvmKey, request.classpath, boundedOptions, request.runnerClass, request.environment, request.workingDirectory) + .flatMap(SharedProjectSession.start) + } + + /** A [[TestSession]] over one fork that runs several suites at once. + * + * A single reader fiber pulls the fork's response lines off the socket and routes each to the queue of the suite it names — every response carries its + * suite, so no protocol change is needed to tell concurrent suites apart. `runSuite` registers a queue, sends the RunSuite, and streams from that queue + * until the suite's terminal; different suites call it concurrently. A fork-level Error (its death) is broadcast to every in-flight suite. Cancelling one + * suite's stream sends CancelSuite for it, interrupting just that suite's thread in the fork and leaving its siblings running. + */ + private class SharedProjectSession private ( + jvm: ManagedJvm, + reader: FiberIO[Unit], + queues: TrieMap[String, Queue[IO, TestProtocol.TestResponse]], + threadDumps: Queue[IO, TestProtocol.TestResponse.ThreadDump] + ) extends TestSession { + + override def pid: Long = jvm.process.pid() + + override def runSuites( + classNames: List[String], + parallelism: Int, + selection: FrameworkSelection, + args: List[String] + ): Stream[IO, TestProtocol.TestResponse] = + // A shared session multiplexes many independent suites; a batched one-execution run is the other model (an exclusive fork), and mixing them on the same + // fork would have two owners of the response stream. The batch path never acquires a shared session, so reaching here is a routing bug. + Stream.raiseError[IO](new IllegalStateException("runSuites (one-execution batch) must run on an exclusive fork, not a shared per-project session")) + + override def runSuite(className: String, selection: FrameworkSelection, args: List[String]): Stream[IO, TestProtocol.TestResponse] = + Stream + .eval { + for { + q <- Queue.unbounded[IO, TestProtocol.TestResponse] + // Register the queue BEFORE the command, so a response cannot arrive before there is somewhere to route it. + _ <- IO(queues.put(className, q)) + _ <- sendCommand(TestProtocol.TestCommand.RunSuite(className, selection, args)) + } yield q + } + .flatMap { q => + Stream + .fromQueueUnterminated(q) + .takeThrough { + case _: TestProtocol.TestResponse.SuiteDone => false + case _: TestProtocol.TestResponse.Error => false + case _ => true + } + .onFinalizeCase { + // Clean exit: the terminal was consumed, just unregister. Cancelled/errored mid-suite: tell the fork to interrupt THIS suite (its siblings + // keep running), then unregister. Unlike an exclusive fork, we never kill the process here — it belongs to the whole project. + case Resource.ExitCase.Succeeded => IO(queues.remove(className)).void + case _ => sendCommand(TestProtocol.TestCommand.CancelSuite(className)).attempt >> IO(queues.remove(className)).void + } + } + + private def sendCommand(cmd: TestProtocol.TestCommand): IO[Unit] = + // Concurrent suites share this one writer; synchronize so two commands cannot interleave mid-line on the socket. + IO.blocking { + jvm.stdin.synchronized { + jvm.stdin.println(TestProtocol.encodeCommand(cmd)) + jvm.stdin.flush() + } + } + + override def getThreadDump: IO[Option[TestProtocol.TestResponse.ThreadDump]] = + (sendCommand(TestProtocol.TestCommand.GetThreadDump) >> threadDumps.take.map(Some(_))).timeout(5.seconds).handleError(_ => None) + + override def dumpThreads: IO[List[String]] = + IO.blocking(jvm.dumpThreads()) + + override def drainStderr: IO[List[String]] = + IO.blocking { + val output = jvm.readStderr() + if (output.isEmpty) Nil else output.split('\n').toList + } + + override def isAlive: IO[Boolean] = + IO(jvm.isAlive) + + override def kill: IO[Unit] = + // Kills the whole fork — every suite on it. Used only when the session as a whole is being killed (a shared fork does not idle-timeout on one suite; + // that cancels the suite via CancelSuite instead, below). + IO.blocking(jvm.kill("bleep: explicit kill of shared project fork", graceMillis = 10000)) + + override def killSuite(className: String): IO[Unit] = + // The point of a shared fork: stop one suite without touching its siblings. CancelSuite interrupts just that suite's thread in the fork; the process + // and the other suites on it keep running. If the interrupt does not take (a genuinely wedged thread) the suite stays stuck, but the fork is still + // reclaimed when the project's last suite releases the session. + sendCommand(TestProtocol.TestCommand.CancelSuite(className)).attempt.void + + /** Destroy the fork and reap the reader. Called once, when the last suite releases the session. + * + * Destroy FIRST, cancel second, and the order is load-bearing: the reader is blocked in a socket `readLine`, and a blocking socket read does not respond + * to `Thread.interrupt` — which is all `IO.interruptible`'s cancellation has. So cancelling first would hang forever on a still-open socket. Destroying + * closes the socket, the `readLine` returns end-of-stream, the reader loop ends on its own, and the `cancel` that follows just reaps an already-finished + * fiber. + */ + def teardown: IO[Unit] = + destroy(jvm, "bleep: last suite of the project's shared fork released it") >> reader.cancel + } + + private object SharedProjectSession { + def start(jvm: ManagedJvm): IO[SharedProjectSession] = + for { + queues <- IO(new TrieMap[String, Queue[IO, TestProtocol.TestResponse]]()) + threadDumps <- Queue.unbounded[IO, TestProtocol.TestResponse.ThreadDump] + fiber <- readerLoop(jvm, queues, threadDumps).start + } yield new SharedProjectSession(jvm, fiber, queues, threadDumps) + + /** Read the fork's response lines forever, routing each to the suite it names. Ends when the socket does (fork death), which it reports to every + * in-flight suite so none hangs waiting for a terminal that will never come. + */ + private def readerLoop( + jvm: ManagedJvm, + queues: TrieMap[String, Queue[IO, TestProtocol.TestResponse]], + threadDumps: Queue[IO, TestProtocol.TestResponse.ThreadDump] + ): IO[Unit] = { + def broadcast(err: TestProtocol.TestResponse.Error): IO[Unit] = + IO(queues.values.toList).flatMap(_.traverse_(_.offer(err))) + + def route(resp: TestProtocol.TestResponse): IO[Unit] = + resp match { + case TestProtocol.TestResponse.Ready => IO.unit // consumed at spawn; never seen mid-session + case td: TestProtocol.TestResponse.ThreadDump => threadDumps.offer(td) + case e: TestProtocol.TestResponse.Error => broadcast(e) // fork-level, names no suite: every suite on this JVM is affected + case other => + suiteOf(other) match { + case Some(suite) => queues.get(suite).fold(IO.unit)(_.offer(other)) // no subscriber => drop (a late or unattributed line) + case None => IO.unit // a null-suite Log: unattributable console noise, dropped + } + } + + // A read returning null (clean EOF) and a read THROWING (the socket closed under it — exactly what `teardown` does to unblock this reader) mean the same + // thing: no more lines. Fold them together with `.attempt` so a socket-closed exception is an ordinary end of stream, not an escaped failure that leaves + // in-flight suites hanging. + def endOfStream: IO[Unit] = + IO(jvm.markDead()) >> IO + .blocking { + val pid = jvm.process.pid() + val exit = JvmPool.describeExit(jvm.process, jvm.killedByUs) + TestProtocol.TestResponse.Error(s"Shared test JVM (pid=$pid) died unexpectedly (${exit.summary})", exit.detail) + } + .flatMap(broadcast) + + def loop: IO[Unit] = + IO.interruptible(jvm.stdout.readLine()).attempt.flatMap { + case Left(_) | Right(null) => endOfStream + case Right(line) => + TestProtocol.decodeResponse(line) match { + case Right(resp) => route(resp) >> loop + case Left(err) => + // A garbled line means the one shared stream is corrupt; there is no per-suite recovery, so fail every in-flight suite and stop. + IO(jvm.markProtocolDirty()) >> broadcast(TestProtocol.TestResponse.Error(s"Protocol error: ${err.getMessage}", Some(s"Line: $line"))) + } + } + + loop + } + + private def suiteOf(resp: TestProtocol.TestResponse): Option[String] = + resp match { + case ts: TestProtocol.TestResponse.TestStarted => Some(ts.suite) + case tf: TestProtocol.TestResponse.TestFinished => Some(tf.suite) + case sd: TestProtocol.TestResponse.SuiteDone => Some(sd.suite) + case l: TestProtocol.TestResponse.Log => l.suite + case _ => None + } } } } diff --git a/bleep-core/src/scala/bleep/testing/JvmPoolListener.scala b/bleep-core/src/scala/bleep/testing/JvmPoolListener.scala index 5ca5331f1..07bc2c462 100644 --- a/bleep-core/src/scala/bleep/testing/JvmPoolListener.scala +++ b/bleep-core/src/scala/bleep/testing/JvmPoolListener.scala @@ -27,6 +27,13 @@ trait JvmPoolListener { * saving, which is otherwise invisible. */ def onForkReused(pid: Long, label: String): Unit + + /** The pool moved to terminate a fork — the single chokepoint every bleep-initiated kill passes through (eviction, suite-idle timeout, cancellation, pool + * shutdown). Recorded so a fork's death can be told apart from a self-inflicted one: if a fork exits and no `onForkKill` names its pid first, bleep did not + * end it — a test's `System.exit`/`Runtime.halt`, a natural end, or an OS kill did. `wasAlive` is whether the process was still running when the kill was + * issued (a redundant escalation over an already-dead fork reports `false`). Joins the other fork events on pid. + */ + def onForkKill(pid: Long, reason: String, wasAlive: Boolean, graceMillis: Long): Unit } object JvmPoolListener { @@ -34,5 +41,6 @@ object JvmPoolListener { def onForkStart(pid: Long, label: String, xmxMb: Option[Long]): Unit = () def onForkEnd(pid: Long, lifetimeMs: Long, exit: String, killedByUs: Option[String]): Unit = () def onForkReused(pid: Long, label: String): Unit = () + def onForkKill(pid: Long, reason: String, wasAlive: Boolean, graceMillis: Long): Unit = () } } diff --git a/bleep-core/src/scala/bleep/testing/TestExecutor.scala b/bleep-core/src/scala/bleep/testing/TestExecutor.scala new file mode 100644 index 000000000..25e0d43c2 --- /dev/null +++ b/bleep-core/src/scala/bleep/testing/TestExecutor.scala @@ -0,0 +1,123 @@ +package bleep.testing + +import cats.effect._ +import fs2.Stream + +import java.nio.file.Path + +/** Whether the session this request asks for is the caller's alone, or shared with the rest of its project's suites. + * + * This is what makes [[bleep.model.TestForkMode]] real at the pool: `per-suite` asks for [[Exclusive]] and gets a fork to itself (one suite in flight, the + * pool reuses forks serially); `per-project` asks for [[Shared]] and every one of the project's suites lands on the SAME fork, which runs them concurrently. + * The key is what "same" means — all of a project's suites pass the same key. How many run at once is not this type's business: admission decides it (the DAG + * serialises a project's suites into `maxConcurrentSuites` chains), so a suite that is not allowed to start yet never reaches the pool holding a machine + * permit. + */ +sealed trait SessionSharing +object SessionSharing { + case object Exclusive extends SessionSharing + case class Shared(key: String) extends SessionSharing +} + +/** Everything the caller knows about the suite it is about to run, before anything decides *where* it runs. + * + * A request, not a JVM description, because half of these fields only mean something to a fork. `jvmOptions`, `environment` and `workingDirectory` are + * properties of a process, and an executor that is not starting one cannot honour them — see [[InProcessTestExecutor]], which refuses the request rather than + * running the tests under settings the user asked for and did not get. + */ +case class TestSessionRequest( + label: String, + classpath: List[Path], + jvmOptions: List[String], + /** Heap for a fork whose `jvmOptions` state no `-Xmx`. Meaningless without a fork: in process the tests share the server's heap. */ + defaultHeapMb: Long, + runnerClass: String, + environment: Map[String, String], + workingDirectory: Option[Path], + /** Exclusive (a fork to this suite alone) or Shared (this suite's project runs all its suites in one fork). See [[SessionSharing]]. */ + sharing: SessionSharing +) + +/** Hands out somewhere to run a test suite. + * + * Two implementations, and the difference between them is the process boundary: + * + * - [[JvmPool]] forks JVMs and pools them by classpath, talking to each over a socket. Isolation is the operating system's: a suite that calls + * `System.exit`, wedges a thread or corrupts a static kills a process bleep can replace, and `jvmOptions`, environment and working directory are all + * honestly settable because there is a process to set them on. + * - [[InProcessTestExecutor]] runs suites in the server itself, in a classloader per classpath. No fork to pay for, so a suite starts in milliseconds rather + * than seconds — and no process boundary either, which is the whole trade. + * + * Both hand back a [[TestSession]] that speaks the same protocol, so `TestRunner` cannot tell them apart. That is deliberate and load-bearing: the forked path + * encodes [[TestProtocol]] lines onto a socket, and the in-process path hands the identical lines to a queue. One wire format, one decoder, one set of answers + * to "did this suite run" — a second implementation of framework loading and fingerprint matching would be a second set, and they would drift. + */ +trait TestExecutor { + + /** Somewhere to run suites with this classpath. Released back when the caller is done with it. */ + def acquire(request: TestSessionRequest): Resource[IO, TestSession] + + /** Tear down everything this executor is holding — processes, threads, classloaders. + * + * MUST be called. Use `guarantee`. + */ + def shutdown: IO[Unit] + + /** How many sessions are currently held open. Forked JVMs, or live classloaders. */ + def size: IO[Int] +} + +/** Somewhere a suite can be run, obtained from a [[TestExecutor]]. */ +trait TestSession { + + /** The OS process the tests actually execute in: a fork's pid, or the server's own when they run in it. + * + * Not an identifier for the session — two in-process sessions share this number, because they really are the same process. It exists so a metric or a thread + * dump names something a person can find with `ps`. + */ + def pid: Long + + /** Run a test suite and stream back responses. `selection` says *how* to run it, decided where the classpath is known; see [[FrameworkSelection]]. */ + def runSuite( + className: String, + selection: FrameworkSelection, + args: List[String] + ): Stream[IO, TestProtocol.TestResponse] + + /** Run a whole set of classes of ONE framework through a single execution — for JUnit Platform, one `launcher.execute()`; for sbt test-interface, one + * `Framework`/`Runner` with all their tasks and one `done()` (maven's `forkCount=1 reuseForks=true`). Streams every class's responses, each tagged with its + * suite, until the batch terminator. This is what keeps an execution-scoped fixture — an application booted for the run — built once and reused across the + * classes rather than rebuilt per class, and (for sbt) what stateful frameworks require. `parallelism` bounds how many classes run at once (1 = sequential). + * Only the forked JVM session honours it; other sessions run suite-by-suite. + */ + def runSuites( + classNames: List[String], + parallelism: Int, + selection: FrameworkSelection, + args: List[String] + ): Stream[IO, TestProtocol.TestResponse] + + /** A thread dump, as the protocol carries it. */ + def getThreadDump: IO[Option[TestProtocol.TestResponse.ThreadDump]] + + /** A thread dump as plain lines, for the summary a timed-out suite prints. Best-effort — never throws. */ + def dumpThreads: IO[List[String]] + + /** Any stderr the session has buffered that did not come through the protocol. Empty where there is no separate stderr to drain. */ + def drainStderr: IO[List[String]] + + /** Is this session still usable? */ + def isAlive: IO[Boolean] + + /** Stop whatever is running here, now. A process to destroy, or a thread to interrupt. */ + def kill: IO[Unit] + + /** Stop ONE suite, by whatever means fits this session, without harming any other suite that shares it. + * + * The difference between this and [[kill]] is the whole reason a shared session exists. On an exclusive session there is only the one suite, so stopping it + * is stopping the session — this is `kill`. On a per-project shared session the fork is running several suites at once, and stopping one must leave the rest + * alone: it interrupts just that suite's thread in the fork (a `CancelSuite`), never the process. Called when a single suite times out, is cancelled, or is + * killed — where [[kill]] would take its siblings down with it. + */ + def killSuite(className: String): IO[Unit] +} diff --git a/bleep-core/src/scala/bleep/testing/TestProtocol.scala b/bleep-core/src/scala/bleep/testing/TestProtocol.scala index 5f51be335..fd308ed39 100644 --- a/bleep-core/src/scala/bleep/testing/TestProtocol.scala +++ b/bleep-core/src/scala/bleep/testing/TestProtocol.scala @@ -28,6 +28,29 @@ object TestProtocol { args: List[String] ) extends TestCommand + /** Run a whole project's JUnit-Platform suites in ONE launcher execution, at a bleep-chosen degree of parallelism. + * + * This is the maven-surefire shape for JUnit: all of a module's classes go through one `launcher.execute()`, so anything scoped to that one execution — an + * application booted for the run, a shared `LauncherSessionListener` registration — is set up once and reused across every class, instead of rebuilt per + * class. `parallelism` is bleep's decision, not junit's: 1 serialises (what a singleton-per-JVM application needs), N runs N classes at once. junit's + * engine is the executor of that number, nothing more. Only the JUnit-Platform runner honours this; sbt test-interface frameworks are driven + * suite-by-suite on bleep's own threads, where there is no cross-suite execution scope to preserve. + */ + case class RunSuites( + classNames: List[String], + parallelism: Int, + selection: FrameworkSelection, + args: List[String] + ) extends TestCommand + + /** Cancel one in-flight suite by class name without touching the fork or its siblings. + * + * Only meaningful when the fork is running several suites at once (a per-project shared session): it interrupts just that suite's thread, so the other + * suites sharing the JVM keep running. An exclusive fork cancels by having its socket closed instead — there is only the one suite, and the whole process + * goes. See [[bleep.model.TestForkMode]]. + */ + case class CancelSuite(className: String) extends TestCommand + /** Gracefully shut down the forked JVM */ case object Shutdown extends TestCommand @@ -77,15 +100,55 @@ object TestProtocol { } yield RunSuite(className, selection, args) } + // Only JUnit-Platform suites are ever batched into one execution (that is the runner with a cross-class scope worth preserving), so the wire form carries + // the one runner + display name shared by all the classes, plus the class list and bleep's parallelism. + implicit val runSuitesEncoder: Encoder[RunSuites] = Encoder.instance { rs => + // A batch's classes all share one framework, so one runner + (for sbt) one frameworkClass covers them. JUnit Platform needs no frameworkClass; sbt does. + val (runner, frameworkClass) = rs.selection match { + case FrameworkSelection.JUnitPlatform(_) => (RunnerWire.JUnitPlatform, None) + case FrameworkSelection.SbtTestInterface(_, cls) => (RunnerWire.SbtTestInterface, Some(cls)) + case other => sys.error(s"RunSuites is for JUnit-Platform or sbt-test-interface; got $other for ${rs.classNames.mkString(", ")}") + } + Json.obj( + "classNames" -> rs.classNames.asJson, + "parallelism" -> rs.parallelism.asJson, + "framework" -> rs.selection.displayName.asJson, + "runner" -> runner.asJson, + "frameworkClass" -> frameworkClass.asJson, + "args" -> rs.args.asJson + ) + } + + implicit val runSuitesDecoder: Decoder[RunSuites] = Decoder.instance { cursor => + for { + classNames <- cursor.downField("classNames").as[List[String]] + parallelism <- cursor.downField("parallelism").as[Int] + displayName <- cursor.downField("framework").as[String] + runner <- cursor.downField("runner").as[String] + frameworkClass <- cursor.downField("frameworkClass").as[Option[String]] + args <- cursor.downField("args").as[List[String]] + selection <- (runner, frameworkClass) match { + case (RunnerWire.JUnitPlatform, _) => Right(FrameworkSelection.JUnitPlatform(displayName)) + case (RunnerWire.SbtTestInterface, Some(cls)) => Right(FrameworkSelection.SbtTestInterface(displayName, cls)) + case (RunnerWire.SbtTestInterface, None) => Left(DecodingFailure(s"${RunnerWire.SbtTestInterface} RunSuites requires frameworkClass", cursor.history)) + case (other, _) => Left(DecodingFailure(s"Unknown runner for RunSuites: $other", cursor.history)) + } + } yield RunSuites(classNames, parallelism, selection, args) + } + implicit val encoder: Encoder[TestCommand] = Encoder.instance { - case rs: RunSuite => Json.obj("type" -> "RunSuite".asJson, "data" -> rs.asJson) - case Shutdown => Json.obj("type" -> "Shutdown".asJson) - case GetThreadDump => Json.obj("type" -> "GetThreadDump".asJson) + case rs: RunSuite => Json.obj("type" -> "RunSuite".asJson, "data" -> rs.asJson) + case rs: RunSuites => Json.obj("type" -> "RunSuites".asJson, "data" -> rs.asJson) + case cs: CancelSuite => Json.obj("type" -> "CancelSuite".asJson, "className" -> cs.className.asJson) + case Shutdown => Json.obj("type" -> "Shutdown".asJson) + case GetThreadDump => Json.obj("type" -> "GetThreadDump".asJson) } implicit val decoder: Decoder[TestCommand] = Decoder.instance { cursor => cursor.downField("type").as[String].flatMap { case "RunSuite" => cursor.downField("data").as[RunSuite] + case "RunSuites" => cursor.downField("data").as[RunSuites] + case "CancelSuite" => cursor.downField("className").as[String].map(CancelSuite(_)) case "Shutdown" => Right(Shutdown) case "GetThreadDump" => Right(GetThreadDump) case other => Left(DecodingFailure(s"Unknown command type: $other", cursor.history)) @@ -144,6 +207,11 @@ object TestProtocol { throwable: Option[String] ) extends TestResponse + /** The single batched execution (a `RunSuites`) has returned. Every class's own terminal (`SuiteDone`/error) has already been sent and demultiplexed to its + * suite; this just tells the parent the one execute is over, so it can stop reading without counting terminals against a class list. + */ + case object BatchComplete extends TestResponse + /** Thread dump from the forked JVM */ case class ThreadDump( threads: List[ThreadInfo] @@ -226,18 +294,20 @@ object TestProtocol { case l: Log => Json.obj("type" -> "Log".asJson, "data" -> l.asJson) case e: Error => Json.obj("type" -> "Error".asJson, "data" -> e.asJson) case td: ThreadDump => Json.obj("type" -> "ThreadDump".asJson, "data" -> td.asJson) + case BatchComplete => Json.obj("type" -> "BatchComplete".asJson) } implicit val decoder: Decoder[TestResponse] = Decoder.instance { cursor => cursor.downField("type").as[String].flatMap { - case "Ready" => Right(Ready) - case "TestStarted" => cursor.downField("data").as[TestStarted] - case "TestFinished" => cursor.downField("data").as[TestFinished] - case "SuiteDone" => cursor.downField("data").as[SuiteDone] - case "Log" => cursor.downField("data").as[Log] - case "Error" => cursor.downField("data").as[Error] - case "ThreadDump" => cursor.downField("data").as[ThreadDump] - case other => Left(DecodingFailure(s"Unknown response type: $other", cursor.history)) + case "Ready" => Right(Ready) + case "TestStarted" => cursor.downField("data").as[TestStarted] + case "TestFinished" => cursor.downField("data").as[TestFinished] + case "SuiteDone" => cursor.downField("data").as[SuiteDone] + case "Log" => cursor.downField("data").as[Log] + case "Error" => cursor.downField("data").as[Error] + case "ThreadDump" => cursor.downField("data").as[ThreadDump] + case "BatchComplete" => Right(BatchComplete) + case other => Left(DecodingFailure(s"Unknown response type: $other", cursor.history)) } } } diff --git a/bleep-model/src/scala/bleep/ProjectPaths.scala b/bleep-model/src/scala/bleep/ProjectPaths.scala index 09a4d2dad..a9a6fd443 100644 --- a/bleep-model/src/scala/bleep/ProjectPaths.scala +++ b/bleep-model/src/scala/bleep/ProjectPaths.scala @@ -9,6 +9,14 @@ case class ProjectPaths(dir: Path, targetDir: Path, sourcesDirs: ProjectPaths.Di val incrementalAnalysis: Path = targetDir / s"inc_compile.zip" + + /** Extra JVM options a test fork of this project needs, one per line (blank lines and `#` comments ignored). A sourcegen may write this file to declare + * options its generated output requires at runtime — the mechanism a code-generating test project uses to hand its fork a generated path or a custom + * LogManager, instead of every project restating them in `bleep.yaml`. Absent means no extra options. Read when the fork is assembled, so a changed file + * re-forks through the normal option-keyed pool. + */ + val forkJvmOptions: Path = + targetDir / "bleep-fork-jvm-options" } object ProjectPaths { diff --git a/bleep-model/src/scala/bleep/model/Project.scala b/bleep-model/src/scala/bleep/model/Project.scala index 20865b0ac..53ff66946 100644 --- a/bleep-model/src/scala/bleep/model/Project.scala +++ b/bleep-model/src/scala/bleep/model/Project.scala @@ -35,6 +35,17 @@ case class Project( * runner discovers. CLI surface: `bleep test --only-tag slow --exclude-tag flaky`. */ testTags: JsonMap[String, JsonSet[String]], + /** Ceiling on how many of this project's test suites run at once. Default `1` — suites run one at a time. Its reach depends on `testFork`: in per-project + * mode (the default) it only speeds up JUnit Platform suites (the JUnit engine runs that many of the project's JUnit classes at once inside the one shared + * fork); sbt-interface frameworks always run sequentially there, so a value > 1 on an sbt-only project has no effect (bleep warns). In per-suite mode it + * bounds how many *forks* run at once (unset = unbounded). Either way the machine-wide governor caps the total across all projects. + */ + maxConcurrentSuites: Option[Int], + /** Where this project's test suites run relative to the JVM hosting them: one fork for the whole project (`per-project`, the default — maven's + * `forkCount=1 reuseForks=true`) or a fork per suite (`per-suite`, OS-level isolation). Unset = per-project. See [[TestForkMode]]. `maxConcurrentSuites` + * bounds concurrency in both modes — inside the one fork for per-project (JUnit only), across forks for per-suite. + */ + testFork: Option[TestForkMode], sourcegen: JsonSet[ScriptDef], libraryVersionSchemes: JsonSet[LibraryVersionScheme], ignoreEvictionErrors: Option[IgnoreEvictionErrors], @@ -60,6 +71,8 @@ case class Project( isTestProject = if (isTestProject == other.isTestProject) isTestProject else None, testFrameworks = testFrameworks.intersect(other.testFrameworks), testTags = testTags.intersect(other.testTags), + maxConcurrentSuites = if (maxConcurrentSuites == other.maxConcurrentSuites) maxConcurrentSuites else None, + testFork = if (testFork == other.testFork) testFork else None, sourcegen = sourcegen.intersect(other.sourcegen), libraryVersionSchemes = libraryVersionSchemes.intersect(other.libraryVersionSchemes), ignoreEvictionErrors = if (ignoreEvictionErrors == other.ignoreEvictionErrors) ignoreEvictionErrors else None, @@ -89,6 +102,8 @@ case class Project( isTestProject = if (isTestProject == other.isTestProject) None else isTestProject, testFrameworks = testFrameworks.removeAll(other.testFrameworks), testTags = testTags.removeAll(other.testTags), + maxConcurrentSuites = if (maxConcurrentSuites == other.maxConcurrentSuites) None else maxConcurrentSuites, + testFork = if (testFork == other.testFork) None else testFork, sourcegen = sourcegen.removeAll(other.sourcegen), libraryVersionSchemes = libraryVersionSchemes.removeAll(other.libraryVersionSchemes), ignoreEvictionErrors = if (ignoreEvictionErrors == other.ignoreEvictionErrors) None else ignoreEvictionErrors, @@ -116,6 +131,8 @@ case class Project( isTestProject = isTestProject.orElse(other.isTestProject), testFrameworks = testFrameworks.union(other.testFrameworks), testTags = testTags.union(other.testTags), + maxConcurrentSuites = maxConcurrentSuites.orElse(other.maxConcurrentSuites), + testFork = testFork.orElse(other.testFork), sourcegen = sourcegen.union(other.sourcegen), libraryVersionSchemes = libraryVersionSchemes.union(other.libraryVersionSchemes), ignoreEvictionErrors = ignoreEvictionErrors.orElse(other.ignoreEvictionErrors), @@ -142,6 +159,8 @@ case class Project( isTestProject, testFrameworks, testTags, + maxConcurrentSuites, + testFork, sourceGeneratorsScripts, libraryVersionSchemes, ignoreEvictionErrors, @@ -165,6 +184,8 @@ case class Project( isTestProject.isEmpty && testFrameworks.isEmpty && testTags.isEmpty && + maxConcurrentSuites.isEmpty && + testFork.isEmpty && sourceGeneratorsScripts.isEmpty && libraryVersionSchemes.isEmpty && ignoreEvictionErrors.isEmpty && @@ -192,6 +213,8 @@ object Project { isTestProject = None, testFrameworks = JsonSet.empty, testTags = JsonMap.empty, + maxConcurrentSuites = None, + testFork = None, sourcegen = JsonSet.empty, libraryVersionSchemes = JsonSet.empty, ignoreEvictionErrors = None, diff --git a/bleep-model/src/scala/bleep/model/TestForkMode.scala b/bleep-model/src/scala/bleep/model/TestForkMode.scala new file mode 100644 index 000000000..b182350ad --- /dev/null +++ b/bleep-model/src/scala/bleep/model/TestForkMode.scala @@ -0,0 +1,37 @@ +package bleep +package model + +import io.circe.{Decoder, DecodingFailure, Encoder} + +/** Where a project's test suites run relative to the JVM that hosts them — the fork granularity. + * + * - [[PerProject]] (the default) runs every one of the project's suites in a single forked JVM — maven surefire's `forkCount=1 reuseForks=true`. JVM-wide + * state carries across suites: a booted application and its dev-service containers, a shared Testcontainers instance, schema an earlier suite created. + * Suites run one at a time by default. `maxConcurrentSuites` only speeds up JUnit Platform suites: raising it lets the JUnit engine run that many of the + * project's JUnit classes at once inside the one fork. sbt-interface frameworks (ScalaTest, MUnit, utest, Specs2, ScalaCheck, weaver, …) always run + * sequentially in this mode — they share one `Runner` and have no lock-aware scheduler, so concurrency in a shared JVM is unsafe; use [[PerSuite]] for + * concurrent sbt suites. + * - [[PerSuite]] forks a JVM per suite, pooled by classpath. Suites are isolated by the operating system: one that calls `System.exit`, wedges a thread or + * corrupts a static kills a process bleep can replace. `maxConcurrentSuites` then bounds how many such forks run at once (unset = unbounded; the + * machine-wide governor still caps the total). This is how you run sbt suites concurrently — every framework, isolated. + */ +sealed abstract class TestForkMode(val value: String) + +object TestForkMode { + case object PerSuite extends TestForkMode("per-suite") + case object PerProject extends TestForkMode("per-project") + + val All: List[TestForkMode] = List(PerSuite, PerProject) + val byName: Map[String, TestForkMode] = All.map(x => x.value -> x).toMap + + def fromString(str: String): Either[String, TestForkMode] = + byName.get(str).toRight(s"'$str' not among ${byName.keys.mkString(", ")}") + + implicit val decoder: Decoder[TestForkMode] = + Decoder.instance { c => + c.as[String].flatMap(str => fromString(str).left.map(err => DecodingFailure(err, c.history))) + } + + implicit val encoder: Encoder[TestForkMode] = + Encoder.encodeString.contramap(_.value) +} diff --git a/bleep-model/src/scala/bleep/rewrites/Defaults.scala b/bleep-model/src/scala/bleep/rewrites/Defaults.scala index 0585fadfb..a99aaca8b 100644 --- a/bleep-model/src/scala/bleep/rewrites/Defaults.scala +++ b/bleep-model/src/scala/bleep/rewrites/Defaults.scala @@ -62,7 +62,21 @@ object Defaults { def project(proj: model.Project): model.Project = proj.copy( scala = proj.scala.map(x => x.copy(setup = Some(x.setup.fold(DefaultCompileSetup)(_.union(DefaultCompileSetup))))), - platform = proj.platform.map(x => if (x.name.contains(model.PlatformId.Jvm)) x.union(Defaults.Jvm) else x), + platform = proj.platform.map { x => + if (x.name.contains(model.PlatformId.Jvm)) { + // The default `-Duser.dir=${BUILD_DIR}` preserves sbt's working-directory semantics + // (tests run from the build root there). It is a DEFAULT: a build that states its own + // `-Duser.dir` — the maven importer emits `${PROJECT_DIR}`, matching surefire's + // `${basedir}` — must not end up with two competing flags, since Options.union keeps + // both and the JVM then obeys whichever happens to render last. + val declaresUserDir = x.jvmOptions.values.exists { + case model.Options.Opt.Flag(name) => name.startsWith("-Duser.dir=") + case _ => false + } + val defaults = if (declaresUserDir) Defaults.Jvm.copy(jvmOptions = model.Options.empty) else Defaults.Jvm + x.union(defaults) + } else x + }, `source-layout` = proj.`source-layout`.orElse { Some(defaultSourceLayout(proj)) } diff --git a/bleep-site/sidebars.js b/bleep-site/sidebars.js index 24918e87b..cd8d5fc24 100644 --- a/bleep-site/sidebars.js +++ b/bleep-site/sidebars.js @@ -245,6 +245,16 @@ const sidebars = { label: "Memory & parallelism", id: "usage/resource-management", }, + { + type: "doc", + label: "Running tests", + id: "usage/testing", + }, + { + type: "doc", + label: "Test tags", + id: "usage/test-tags", + }, { type: "doc", label: "Proxies, TLS & air-gapped networks", diff --git a/bleep-test-runner/src/main/java/bleep/testing/runner/ForkedTestRunner.java b/bleep-test-runner/src/main/java/bleep/testing/runner/ForkedTestRunner.java index 18d6edf3d..dcc4b118c 100644 --- a/bleep-test-runner/src/main/java/bleep/testing/runner/ForkedTestRunner.java +++ b/bleep-test-runner/src/main/java/bleep/testing/runner/ForkedTestRunner.java @@ -6,11 +6,11 @@ import java.nio.charset.StandardCharsets; import java.security.Permission; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import sbt.testing.*; /** @@ -33,17 +33,104 @@ public class ForkedTestRunner { // Flag to indicate we're shutting down private static final AtomicBoolean shuttingDown = new AtomicBoolean(false); - // Currently running test task thread for cancellation - private static final AtomicReference currentTask = new AtomicReference<>(null); + // Suites currently running, keyed by class name, so a CancelSuite can interrupt exactly one + // without touching the fork or its siblings. A project's class names are unique, so the name is a + // sufficient key. When the fork runs one suite at a time (an exclusive per-suite session) this + // map + // simply never holds more than one entry. + private static final Map runningSuites = new ConcurrentHashMap<>(); + + // The suite the CURRENT thread is running, for tagging that thread's captured output. + // Thread-local, + // not global: with several suites in flight at once, each runs on its own thread, and a line + // written on a suite's thread belongs to that suite. Output from framework/async threads that + // never + // set this (a Vert.x event loop, a Netty worker) has no owning suite and is tagged null. + private static final ThreadLocal currentSuite = new ThreadLocal<>(); + + // The suite a batch is currently running, across all threads. A batch runs one suite at a time by + // default, so this is unambiguous, and it is the fallback that lets output from a framework's own + // threads (which never set the thread-local `currentSuite`) still be attributed to the right + // suite. + private static volatile String activeBatchSuite = null; + + // Set true once the command loop exits on its own terms (Shutdown, EOF, or a broken protocol). A + // JVM shutdown that happens while this is still false was NOT + // asked for by bleep — a test called System.exit()/Runtime.halt(), which on JDK 24+ (no + // SecurityManager) bleep cannot block. The exit diagnostic hook below + // fires only in that case, and dumps every thread so the one blocked in Shutdown.exit names the + // caller. Without it a System.exit is a silent exit-0 the parent + // can only guess at ("likely System.exit()"). + private static volatile boolean loopExitedNormally = false; - // Currently running suite name (for output tagging) - private static volatile String currentSuite = null; + /** + * Set (or clear, with null) the suite whose output a batch is currently producing. Passed to + * {@link SuiteRunner#runSuites} and used by the JUnit batch loop so output — on the running + * thread or on a framework's own — is attributed to the suite in flight, the way {@link + * #startSuiteThread} does for a single suite on its own thread. + */ + static void setCurrentSuite(String suite) { + if (suite == null) { + currentSuite.remove(); + activeBatchSuite = null; + } else { + currentSuite.set(suite); + activeBatchSuite = suite; + } + } public static void main(String[] args) { // Save original streams for protocol communication PrintStream originalOut = System.out; PrintStream originalErr = System.err; + // Name who kills the fork out from under a run. A test that calls System.exit()/Runtime.halt() + // ends the whole JVM — every suite that had not reported is + // simply lost, and the parent sees only a clean exit it can do no better than call "likely + // System.exit()". This hook fires ONLY on such an unrequested + // shutdown (loopExitedNormally is still false) and writes a full thread dump to the ORIGINAL + // stderr (fd 2, which the parent drains — not the protocol, which + // is already tearing down): the thread blocked in `Shutdown`/`Runtime.exit` is the caller, + // stack and all. Written straight to fd 2, no allocation-heavy + // machinery, because a shutdown is not a good time to need the heap. + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> { + if (loopExitedNormally) return; + originalErr.println( + "bleep test runner: the fork is shutting down before the run finished — a" + + " test called System.exit()/Runtime.halt(), which bleep cannot block on" + + " JDK 24+. Every unreported suite is lost. Thread dump follows; the" + + " thread in java.lang.Shutdown / Runtime.exit is the caller:"); + for (Map.Entry e : + Thread.getAllStackTraces().entrySet()) { + originalErr.println( + " \"" + e.getKey().getName() + "\" " + e.getKey().getState()); + for (StackTraceElement frame : e.getValue()) + originalErr.println(" at " + frame); + } + originalErr.flush(); + }, + "bleep-exit-diagnostic")); + + // Always-on companion to the fd-2 hook above, writing the same account to a FILE the parent + // reads after the fork dies. fd 2 is lost when the exit races the pipe teardown (the case that + // left "N suites never reported a result" with no cause); a file is not. It runs on every + // orderly shutdown — including one bleep asked for — so it is unconditional; the parent decides + // what to make of it. It does NOT run on Runtime.halt or a hard kill, so an absent file is + // itself the answer: no shutdown hook ran. + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> + writeExitLog( + loopExitedNormally + ? "shutdown after the command loop ended on its own terms" + : "shutdown while the command loop was still running (System.exit by a" + + " test, or the JVM tearing down)"), + "bleep-exit-logfile")); + // The protocol runs over a loopback socket the parent is already listening on, NOT over this // process's stdin/stdout. // @@ -101,6 +188,21 @@ public static void main(String[] args) { // Install security manager to catch System.exit (if supported) installSecurityManager(); + // One LauncherSession for this whole fork: a LauncherSessionListener fires once here, not + // once + // per suite — which is what a test harness written for maven's one-fork-per-module assumes, + // and what keeps concurrent suites from racing on a "register this global once" listener. + // + // Guarded because it is a JUnit-Platform concept and JUnitPlatformRunner links against + // org.junit.platform.launcher.*. A fork for an sbt test-interface project (ScalaTest, MUnit, + // utest, ...) has no JUnit Platform on its runtime classpath (junit-platform-launcher is a + // `provided`, compile-only dependency of bleep-test-runner), so merely referencing that class + // would NoClassDefFoundError and kill the fork before it reaches Ready. No JUnit suites will + // run in such a fork, so there is nothing to share; skip it. + if (junitPlatformOnClasspath()) { + JUnitPlatformRunner.enableSharedSession(); + } + // Signal ready send(TestProtocol.encodeReady()); @@ -108,58 +210,147 @@ public static void main(String[] args) { new BufferedReader( new InputStreamReader(protocolSocket.getInputStream(), StandardCharsets.UTF_8)); - // Main command loop + // Main command loop. + // + // A RunSuite starts a thread and the loop goes straight back to reading, so several suites + // can + // be in flight at once (a per-project shared session dispatches them concurrently). The loop + // is + // never blocked on a running suite, which is what lets a CancelSuite for one suite — or a + // Shutdown — be acted on while others keep running. An exclusive per-suite session sends one + // RunSuite at a time and never overlaps them; the same code serves it with a map of size one. boolean running = true; + // A failure to READ the protocol socket means the parent (bleep) is gone, and every + // subsequent + // read will fail the same way — so exit, do not retry. This loop used to wrap the read in the + // same catch as dispatch and go straight back to reading: an IOException from a broken/reset + // socket (the parent gave up on the fork, or the client disconnected) turned an abandoned + // fork + // into an immortal 100%-CPU zombie that re-encoded the same error forever. Observed in the + // wild: a fork stuck ~2h with `main` pegged in TestProtocol.encodeError. Those zombies pile + // up + // and starve the machine, which then kills freshly-spawned forks at startup — a + // self-inflicted + // cascade. Dispatch errors (a single bad command) are still reported and tolerated, bounded + // by + // a consecutive-error cap so no other persistently-failing state can spin either. + int consecutiveErrors = 0; + // Why the loop ended, recorded at each exit so the fork's own stderr (fd 2, drained by the + // parent) says which side hung up. "The fork exited while a suite was still running" is + // otherwise undiagnosable from the parent, which only sees the socket go quiet: EOF and a + // reset both look like "died unexpectedly". This names the cause on the fork side. + String loopExitCause = "loop condition became false without an explicit cause"; while (running && !shuttingDown.get()) { + String line; try { - String line = in.readLine(); - if (line == null) { - // EOF - parent process closed stdin, shut down + line = in.readLine(); + } catch (IOException e) { + // Protocol socket broken — the parent is unreachable. Stop; the finally block cleans up. + loopExitCause = "IOException reading the command socket (parent unreachable): " + e; + break; + } + if (line == null) { + // EOF - parent closed the protocol socket, shut down + loopExitCause = "EOF on the command socket (parent closed its end)"; + break; + } + try { + TestProtocol.ParsedCommand cmd = TestProtocol.parseCommand(line); + if (cmd instanceof TestProtocol.ParsedCommand.Shutdown) { + loopExitCause = "received an explicit Shutdown command"; running = false; - } else { - TestProtocol.ParsedCommand cmd = TestProtocol.parseCommand(line); - if (cmd instanceof TestProtocol.ParsedCommand.Shutdown) { - running = false; - } else if (cmd instanceof TestProtocol.ParsedCommand.RunSuite) { - TestProtocol.ParsedCommand.RunSuite runSuite = - (TestProtocol.ParsedCommand.RunSuite) cmd; - // Run in current thread so we can interrupt it - currentTask.set(Thread.currentThread()); - try { - runSuite( - runSuite.className, - runSuite.framework, - runSuite.runner, - runSuite.frameworkClass, - runSuite.args, - capturedOut, - capturedErr); - } finally { - currentTask.set(null); - } - } else if (cmd instanceof TestProtocol.ParsedCommand.GetThreadDump) { - send(generateThreadDump()); - } else if (cmd instanceof TestProtocol.ParsedCommand.Invalid) { - TestProtocol.ParsedCommand.Invalid invalid = (TestProtocol.ParsedCommand.Invalid) cmd; - send(TestProtocol.encodeError("Failed to decode command: " + invalid.message, null)); - } + } else if (cmd instanceof TestProtocol.ParsedCommand.RunSuite) { + startSuiteThread((TestProtocol.ParsedCommand.RunSuite) cmd, capturedOut, capturedErr); + } else if (cmd instanceof TestProtocol.ParsedCommand.RunSuites) { + startSuitesThread((TestProtocol.ParsedCommand.RunSuites) cmd, capturedOut, capturedErr); + } else if (cmd instanceof TestProtocol.ParsedCommand.CancelSuite) { + String toCancel = ((TestProtocol.ParsedCommand.CancelSuite) cmd).className; + Thread t = runningSuites.get(toCancel); + // Interrupt only that suite's thread. A suite already finished (t == null) is a no-op — + // the cancel raced its completion, which is harmless. + if (t != null) t.interrupt(); + } else if (cmd instanceof TestProtocol.ParsedCommand.GetThreadDump) { + send(generateThreadDump()); + } else if (cmd instanceof TestProtocol.ParsedCommand.Invalid) { + TestProtocol.ParsedCommand.Invalid invalid = (TestProtocol.ParsedCommand.Invalid) cmd; + send(TestProtocol.encodeError("Failed to decode command: " + invalid.message, null)); } + consecutiveErrors = 0; } catch (Exception e) { - if (e instanceof InterruptedException) { - // We were interrupted (cancellation) - continue loop to get next command - Thread.interrupted(); // Clear interrupt flag - continue; - } send( TestProtocol.encodeError( - "Error in command loop: " + e.getMessage(), stackTraceToString(e))); + "Error in command loop: " + e.getMessage(), SuiteRunner.stackTraceToString(e))); + if (++consecutiveErrors >= 50) { + loopExitCause = "hit the consecutive-error cap (50) in the command loop"; + break; + } + } + } + + // The loop ended on its own terms (Shutdown, EOF, a broken protocol, or too many errors), so + // the shutdown that follows is expected — the exit diagnostic + // hook stays quiet. Anything that shut the JVM down BEFORE reaching here was a test's own + // System.exit/halt, which is exactly what the hook reports. + loopExitedNormally = true; + + // Leaving the loop (Shutdown or EOF): interrupt whatever is still running so a wedged or + // cancelled suite lets go, and give the threads a moment to emit their terminal responses + // before the JVM's shutdown hooks (which stop a Quarkus app and its containers) run. + // + // Record the teardown to fd 2 (drained by the parent). If the loop ended while suites were + // still running, those suites are being ABANDONED — their results never reach the parent, + // which reports them as "never reported a result". Naming them, and the cause, here is what + // turns that dead end into a reason. + java.util.Collection stillRunning = new java.util.ArrayList<>(runningSuites.values()); + originalErr.println( + "bleep test runner: command loop ended — " + + loopExitCause + + ". " + + stillRunning.size() + + " suite thread(s) still running at teardown" + + (stillRunning.isEmpty() + ? "." + : " (they will be interrupted and, if they do not stop, abandoned): " + + runningSuites.keySet())); + originalErr.flush(); + shuttingDown.set(true); + for (Thread t : runningSuites.values()) t.interrupt(); + long deadline = System.currentTimeMillis() + 5000; + for (Thread t : runningSuites.values()) { + long remaining = deadline - System.currentTimeMillis(); + if (remaining > 0) { + try { + t.join(remaining); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } } } + java.util.List abandoned = new java.util.ArrayList<>(); + for (Map.Entry e : runningSuites.entrySet()) + if (e.getValue().isAlive()) abandoned.add(e.getKey()); + if (!abandoned.isEmpty()) { + originalErr.println( + "bleep test runner: " + + abandoned.size() + + " suite thread(s) did not stop within 5s of interrupt and are being abandoned" + + " (the JVM will exit with them still running): " + + abandoned); + originalErr.flush(); + } } catch (Exception e) { send( TestProtocol.encodeError( - "Fatal error in test runner: " + e.getMessage(), stackTraceToString(e))); + "Fatal error in test runner: " + e.getMessage(), SuiteRunner.stackTraceToString(e))); } finally { + // Close the shared LauncherSession, running its listeners' launcherSessionClosed — where a + // booted application and its containers are asked to stop. Guarded for the same reason as the + // open above: an sbt-interface fork has no JUnit Platform on its classpath, so touching + // JUnitPlatformRunner here would NoClassDefFoundError in the finally and mask the real + // result. + if (junitPlatformOnClasspath()) { + JUnitPlatformRunner.closeSharedSession(); + } // Restore original streams System.setOut(originalOut); System.setErr(originalErr); @@ -178,11 +369,74 @@ public static void main(String[] args) { */ static final String PROTOCOL_PORT_PROPERTY = "bleep.test.protocolPort"; - private static void send(String message) { + /** + * System property carrying the file this fork writes its exit diagnostic to. A file, not fd 2: + * when the JVM tears down, whatever a shutdown hook writes to stderr races the pipe closing and + * is routinely lost — which is how a fork that exits from under a run leaves the parent with a + * bare "exited 0" and no cause. A file survives that; the parent reads it after the fork dies. + * Must match {@code bleep.testing.ForkedTestRunnerProtocol.ExitLogProperty}. + */ + static final String EXIT_LOG_PROPERTY = "bleep.test.exitLog"; + + /** + * Write the fork's exit diagnostic — whether the command loop had exited on its own terms, which + * suites were still running, and a full thread dump — to {@link #EXIT_LOG_PROPERTY}'s file. + * Called from a shutdown hook, so it runs on any orderly exit ({@code System.exit}, main + * returning) but NOT on {@code Runtime.halt} or a hard OS kill, which run no hooks: the file's + * ABSENCE afterwards tells the parent it was one of those. Best-effort and swallows its own + * errors — a shutdown is no time to throw, and a missing diagnostic must never mask the exit it + * was trying to explain. + */ + private static void writeExitLog(String cause) { + String path = System.getProperty(EXIT_LOG_PROPERTY); + if (path == null) return; + try (PrintWriter w = + new PrintWriter( + new OutputStreamWriter(new FileOutputStream(path), StandardCharsets.UTF_8))) { + w.println("bleep test fork exit diagnostic"); + w.println(" cause: " + cause); + w.println(" loopExitedNormally: " + loopExitedNormally); + w.println(" suites still registered as running: " + runningSuites.keySet()); + w.println(" thread dump:"); + for (Map.Entry e : Thread.getAllStackTraces().entrySet()) { + w.println(" \"" + e.getKey().getName() + "\" " + e.getKey().getState()); + for (StackTraceElement frame : e.getValue()) w.println(" at " + frame); + } + w.flush(); + } catch (Throwable ignored) { + // best-effort; never let the diagnostic's failure mask the exit + } + } + + // Synchronized: several suite threads share this one socket, and a response must reach the parent + // as one whole line. Without the lock two println/flush pairs could interleave mid-line and the + // parent would fail to decode the spliced JSON. + private static synchronized void send(String message) { protocolOut.println(message); protocolOut.flush(); } + /** + * Is JUnit Platform's launcher on this fork's classpath? Only then may we touch {@link + * JUnitPlatformRunner}, which links against {@code org.junit.platform.launcher.*}. sbt + * test-interface forks (ScalaTest, MUnit, utest, ...) have no JUnit Platform — {@code + * junit-platform-launcher} is a {@code provided}, compile-only dependency of bleep-test-runner — + * so referencing that class in such a fork NoClassDefFoundErrors. Probed with the class the + * runner's shared-session lifecycle needs; loaded lazily (initialize=false) so the check itself + * never triggers the failure it is guarding against. + */ + private static boolean junitPlatformOnClasspath() { + try { + Class.forName( + "org.junit.platform.launcher.TestExecutionListener", + false, + ForkedTestRunner.class.getClassLoader()); + return true; + } catch (ClassNotFoundException e) { + return false; + } + } + /** * Security manager that catches System.exit calls. Note: SecurityManager is deprecated in Java * 17+ and may not be available. @@ -222,453 +476,99 @@ public void checkExit(int status) { } } - private static void runSuite( - String className, - String frameworkName, - TestProtocol.RunnerKind runnerKind, - String frameworkClass, - List args, + /** + * Start a suite on its own thread and return at once, so the command loop can keep serving. + * + *

Each suite gets a fresh {@link SuiteRunner} (which holds no static state, precisely so N of + * them can share this JVM). The thread registers itself under the suite's class name for the + * lifetime of the run so a CancelSuite can find and interrupt it, and tags its own captured + * output via the thread-local {@link #currentSuite}. What stays a fork's concern is what only a + * fork has: the socket the lines go out on and the streams the tests write to. + */ + private static void startSuiteThread( + TestProtocol.ParsedCommand.RunSuite runSuite, OutputStream capturedOut, OutputStream capturedErr) { - - // Set current suite for output tagging - currentSuite = className; - - send( - TestProtocol.encodeLog( - // bleep talking to itself about which suite it was handed. Not the user's test output. - "debug", - "runSuite called: className=" + className + ", frameworkName=" + frameworkName)); - - // The server decided this, with the project's classpath in front of it. Nothing here re-derives - // it from frameworkName, which is a display label. - if (runnerKind == TestProtocol.RunnerKind.JUNIT_PLATFORM) { - JUnitPlatformRunner junitRunner = new JUnitPlatformRunner(protocolOut); - junitRunner.runSuite(className, capturedOut, capturedErr); - return; - } - - long startTime = System.currentTimeMillis(); - - // Counters declared outside try so they're accessible in catch for SuiteDone reporting - final int[] passed = {0}; - final int[] failed = {0}; - final int[] skipped = {0}; - final int[] ignored = {0}; - - try { - // Flush any pending output before starting - capturedOut.flush(); - capturedErr.flush(); - - // Load the framework - send(TestProtocol.encodeLog("debug", "Loading framework: " + frameworkClass)); - Framework framework = loadFramework(frameworkClass); - send(TestProtocol.encodeLog("debug", "Framework loaded: " + framework.getClass().getName())); - - // Get the runner - Runner runner = - framework.runner( - args.toArray(new String[0]), new String[0], ForkedTestRunner.class.getClassLoader()); - - // Try each fingerprint from the framework until we find one that produces tasks. - // Different fingerprints match different test patterns (e.g. @Test annotation vs - // TestCase subclass), so we need to find the right one for this class. - Fingerprint[] fingerprints = framework.fingerprints(); - - if (fingerprints.length == 0) { - send(TestProtocol.encodeError("Framework has no fingerprints: " + frameworkName, null)); - return; - } - - Task[] tasks = null; - - // Try fingerprints that agree with what the class actually is before the rest. - // - // "First fingerprint that yields a task" is not enough on its own. A framework that declares - // both a class and a module fingerprint — specs2 does — is - // free to hand back a task for either without checking, and only fails later when it tries to - // load the form that does not exist: a `class Fixture extends - // Specification` matched against the module fingerprint produced a task whose whole error - // message was "example.Specs2Fixture$". Whether a suite is a - // Scala object is not a guess; the compiler emits `Fixture$` for one and not for the other. - Fingerprint[] ordered = orderFingerprintsFor(className, fingerprints); - - for (Fingerprint fingerprint : ordered) { - TaskDef taskDef = - new TaskDef(className, fingerprint, true, new Selector[] {new SuiteSelector()}); - Task[] candidate = runner.tasks(new TaskDef[] {taskDef}); - if (candidate.length > 0) { - tasks = candidate; - send( - TestProtocol.encodeLog( - "debug", "Matched fingerprint: " + describeFingerprint(fingerprint))); - break; - } - } - - if (tasks == null || tasks.length == 0) { - // No fingerprint produced a task: the loaded framework does not recognize this class as - // a suite. Not an empty suite (the framework never claimed it) — a framework mismatch. - send( - TestProtocol.encodeSuiteNoFrameworkMatched( - className, - System.currentTimeMillis() - startTime, - "No test framework recognized " + className + " as a suite")); - return; - } - - // Custom event handler to capture test events - EventHandler eventHandler = - new EventHandler() { - @Override - public void handle(Event event) { - String status; - switch (event.status()) { - case Success: - status = "passed"; - passed[0]++; - break; - case Failure: - status = "failed"; - failed[0]++; - break; - case Error: - status = "error"; - failed[0]++; - break; - case Skipped: - status = "skipped"; - skipped[0]++; - break; - case Ignored: - status = "ignored"; - ignored[0]++; - break; - case Canceled: - status = "assumption-failed"; - skipped[0]++; - break; - case Pending: - status = "pending"; - ignored[0]++; - break; - default: - status = "unknown"; - break; - } - - String throwableStr = null; - String message = null; - StackTraceElement location = null; - if (event.throwable() != null && event.throwable().isDefined()) { - Throwable t = event.throwable().get(); - message = t.getMessage(); - throwableStr = stackTraceToString(t); - location = failureLocation(t, className); - } - - // Extract test name from selector if available - String testName = extractTestName(event); - - // Flush output before reporting test finished + String className = runSuite.className; + Thread t = + new Thread( + () -> { + currentSuite.set(className); try { - capturedOut.flush(); - capturedErr.flush(); - } catch (IOException e) { - // Ignore + new SuiteRunner( + ForkedTestRunner::send, + ForkedTestRunner.class.getClassLoader(), + Arrays.asList(capturedOut, capturedErr)) + .runSuite( + className, + runSuite.framework, + runSuite.runner.name(), + runSuite.frameworkClass, + runSuite.args); + } finally { + currentSuite.remove(); + runningSuites.remove(className, Thread.currentThread()); } - - send( - TestProtocol.encodeTestFinished( - className, - testName, - status, - event.duration(), - message, - throwableStr, - location == null ? null : location.getClassName(), - location == null ? null : location.getFileName(), - location == null ? 0 : location.getLineNumber())); - } - }; - - // Execute tasks - Logger logger = createLogger(className); - executeTasks(tasks, eventHandler, new Logger[] {logger}); - - // Done - runner.done(); - - // Final flush - capturedOut.flush(); - capturedErr.flush(); - - long durationMs = System.currentTimeMillis() - startTime; - int total = passed[0] + failed[0] + skipped[0] + ignored[0]; - if (total == 0) { - // The framework claimed the class (a task ran) but no test fired an event: an empty suite. - send(TestProtocol.encodeSuiteEmpty(className, durationMs)); - } else { - send( - TestProtocol.encodeSuiteExecuted( - className, passed[0], failed[0], skipped[0], ignored[0], durationMs)); - } - - } catch (InterruptedException e) { - // Cancelled - report and re-throw to exit the run - send(TestProtocol.encodeLog("warn", "Suite " + className + " was cancelled")); - throw new RuntimeException(e); - } catch (SecurityException e) { - if (e.getMessage() != null && e.getMessage().contains("System.exit")) { - send( - TestProtocol.encodeSuiteErrored( - className, - System.currentTimeMillis() - startTime, - "Test attempted a blocked System.exit", - null)); - } else { - throw e; - } - } catch (Throwable e) { - // Must catch Throwable (not just Exception): a framework may let an Error (AssertionError, - // or a LinkageError propagated from executeTasks) escape. Report it as an errored suite — - // NOT SuiteExecuted with faked counts — so the outcome carries the real reason. - send(TestProtocol.encodeLog("error", stackTraceToString(e))); - Throwable reported = - (e instanceof SuiteExecutionException && e.getCause() != null) ? e.getCause() : e; - send( - TestProtocol.encodeSuiteErrored( - className, - System.currentTimeMillis() - startTime, - "Error running suite " - + className - + ": " - + reported.getClass().getName() - + ": " - + reported.getMessage(), - stackTraceToString(reported))); - } - } - - private static void executeTasks(Task[] tasks, EventHandler eventHandler, Logger[] loggers) - throws InterruptedException { - for (Task task : tasks) { - // Check for interruption before each task - if (Thread.interrupted()) { - throw new InterruptedException(); - } - - try { - Task[] nestedTasks = task.execute(eventHandler, loggers); - // Recursively execute nested tasks - executeTasks(nestedTasks, eventHandler, loggers); - } catch (InterruptedException e) { - throw e; - } catch (Throwable e) { - // Do NOT swallow and continue. A Throwable escaping task.execute — LinkageError, - // NoClassDefFoundError, ExceptionInInitializerError, typically from a stale sibling - // compile — means this suite's classpath cannot be trusted, not that one test failed. - // Swallowing it here let the suite fall through to SuiteDone(...,0,0,0) and be reported - // PASSED: a green build over a suite that never ran. Propagate so the caller's handler - // records a real failure with a non-zero count. - throw new SuiteExecutionException(e); - } - } - } - - /** - * Wraps a non-interruption Throwable that escaped {@code task.execute} so it propagates out of - * {@link #executeTasks} (whose only checked throw is InterruptedException) to runSuite's outer - * handler, which reports it as a suite failure. - */ - private static final class SuiteExecutionException extends RuntimeException { - SuiteExecutionException(Throwable cause) { - super(cause); - } + }, + "suite-" + className); + // Register before start so a CancelSuite arriving immediately still finds the thread. + runningSuites.put(className, t); + t.start(); } /** - * Instantiate an sbt.testing.Framework by class name. + * Run a project's suites of one framework as a single batch on its own thread — JUnit Platform + * classes through {@link JUnitPlatformRunner#runSuites} (each class its own execution on the + * shared session, up to the degree bleep chose at once), sbt-interface suites sequentially + * through one {@link SuiteRunner#runSuites} ({@code Runner}/{@code done()} once). * - *

One line, because the server sends the class rather than a label to guess from. This used to - * special-case JUnit, Kotest and TestNG, probe lists of candidate classes, and fall back to - * treating the display name as a class name — which is how "Spock" and "kotlin.test" arrived at - * Class.forName verbatim. - */ - private static Framework loadFramework(String frameworkClass) throws Exception { - Class clazz = Class.forName(frameworkClass); - return (Framework) clazz.getDeclaredConstructor().newInstance(); - } - - /** Check if this framework should use JUnit Platform Launcher directly. */ - private static Logger createLogger(final String suiteName) { - return new Logger() { - @Override - public boolean ansiCodesSupported() { - // Frameworks ask this before colourising. Answering an unconditional `true` meant `bleep - // test --no-color` still got ANSI escapes from ScalaTest, - // hedgehog and friends: the flag lives in the client JVM and this runs in a forked one, so - // the only thing that crosses is the environment. The - // client sets NO_COLOR when the user asks for no colour, and the no-color.org convention - // means a user who sets it themselves is honoured too. - String noColor = System.getenv("NO_COLOR"); - return noColor == null || noColor.isEmpty(); - } - - @Override - public void error(String msg) { - send(TestProtocol.encodeLog("error", msg)); - } - - @Override - public void warn(String msg) { - send(TestProtocol.encodeLog("warn", msg)); - } - - @Override - public void info(String msg) { - send(TestProtocol.encodeLog("info", msg)); - } - - @Override - public void debug(String msg) { - send(TestProtocol.encodeLog("debug", msg)); - } - - @Override - public void trace(Throwable t) { - send(TestProtocol.encodeLog("error", stackTraceToString(t))); - } - }; - } - - /** - * Puts fingerprints whose `isModule` matches the class on disk first, keeping the framework's own - * order within each group. Nothing is discarded: a framework that disagrees with this reading - * still gets every fingerprint tried, just second. + *

The batch is a single unit of work: it registers under one key, its per-suite results are + * tagged by class, and a {@link TestProtocol#encodeBatchComplete} is sent when it finishes, + * telling the parent to stop reading. Cancellation of a batch is fork-level (there is no + * per-suite thread here to interrupt). */ - private static Fingerprint[] orderFingerprintsFor(String className, Fingerprint[] fingerprints) { - Class asModule = loadClass(className + "$"); - Class asPlain = loadClass(className); - boolean isModule = asModule != null; - - // Ranked, highest first, keeping the framework's own order within a rank: - // 2 — the class really does extend what the fingerprint names - // 1 — only the class/object shape agrees - // 0 — neither - // - // Shape alone is not enough to tell a framework's fingerprints apart when several describe - // objects. Weaver declares one for suites and another for global - // resources; picking by shape chose the resource one and the run died with - // "example.WeaverFixture$ is not an instance of weaver.IOGlobalResource". What the - // class extends is the question the fingerprint is actually asking, so ask that first. - List> byRank = new ArrayList<>(); - for (int i = 0; i < 3; i++) byRank.add(new ArrayList<>()); - for (Fingerprint fp : fingerprints) { - Boolean declaredModule = fingerprintIsModule(fp); - boolean shapeAgrees = declaredModule != null && declaredModule == isModule; - // Each fingerprint is checked against the class it is talking about: a module fingerprint - // means `Foo$`, a class fingerprint means `Foo`. Checking both against the object's class - // scored a class fingerprint naming `org.scalacheck.Properties` just as highly as the module - // one — `Foo$` extends Properties either way — and picking it made ScalaCheck unrunnable. - // A Scala 3 mirror class extends nothing, so the wrong shape now scores itself out. - Class meant = (declaredModule != null && declaredModule) ? asModule : asPlain; - boolean extendsIt = - meant != null - && fingerprintSuperclass(fp).map(sup -> sup.isAssignableFrom(meant)).orElse(false); - int rank = extendsIt ? 2 : (shapeAgrees ? 1 : 0); - byRank.get(2 - rank).add(fp); - } - - List ordered = new ArrayList<>(); - for (List rank : byRank) ordered.addAll(rank); - return ordered.toArray(new Fingerprint[0]); - } - - /** The class a SubclassFingerprint names, when it names one and it can be loaded. */ - private static Optional> fingerprintSuperclass(Fingerprint fp) { - if (!(fp instanceof SubclassFingerprint)) return Optional.empty(); - return Optional.ofNullable(loadClass(((SubclassFingerprint) fp).superclassName())); - } - - private static Class loadClass(String name) { - try { - return Class.forName(name, false, ForkedTestRunner.class.getClassLoader()); - } catch (ClassNotFoundException | LinkageError e) { - return null; - } - } - - /** Null when the fingerprint kind says nothing about module-ness. */ - private static Boolean fingerprintIsModule(Fingerprint fp) { - if (fp instanceof SubclassFingerprint) return ((SubclassFingerprint) fp).isModule(); - if (fp instanceof AnnotatedFingerprint) return ((AnnotatedFingerprint) fp).isModule(); - return null; - } - - private static String describeFingerprint(Fingerprint fp) { - if (fp instanceof SubclassFingerprint) { - SubclassFingerprint sfp = (SubclassFingerprint) fp; - return "SubclassFingerprint(" + sfp.superclassName() + ", isModule=" + sfp.isModule() + ")"; - } else if (fp instanceof AnnotatedFingerprint) { - AnnotatedFingerprint afp = (AnnotatedFingerprint) fp; - return "AnnotatedFingerprint(" + afp.annotationName() + ", isModule=" + afp.isModule() + ")"; - } - return fp.toString(); - } - - private static String stackTraceToString(Throwable t) { - StringWriter sw = new StringWriter(); - t.printStackTrace(new PrintWriter(sw)); - return sw.toString(); - } - - /** - * The first stack frame belonging to the suite class itself, which is where the failing assertion - * lives for every framework we support — the frames above it are inside the assertion library. - * - *

Deliberately not "the first frame with a line number": that points at someone else's source, - * and an annotation on the wrong file is worse than no annotation. Returns null when the - * throwable has no frame in the suite, which is normal for a failure thrown from a helper or a - * fixture. - * - *

Inner and anonymous classes ({@code MyTest$$anon$1}) still belong to the suite, so match on - * the {@code $} boundary rather than equality alone. Causes are walked because assertion - * libraries routinely wrap. - */ - private static StackTraceElement failureLocation(Throwable t, String suiteClass) { - for (Throwable current = t; current != null; current = current.getCause()) { - for (StackTraceElement frame : current.getStackTrace()) { - String cn = frame.getClassName(); - boolean inSuite = cn.equals(suiteClass) || cn.startsWith(suiteClass + "$"); - if (inSuite && frame.getFileName() != null && frame.getLineNumber() > 0) { - return frame; - } - } - if (current.getCause() == current) break; // self-referential cause, seen in the wild - } - return null; + private static void startSuitesThread( + TestProtocol.ParsedCommand.RunSuites runSuites, + OutputStream capturedOut, + OutputStream capturedErr) { + Thread t = + new Thread( + () -> { + try { + if (runSuites.runner == TestProtocol.RunnerKind.JUNIT_PLATFORM) { + new JUnitPlatformRunner( + ForkedTestRunner::send, Arrays.asList(capturedOut, capturedErr)) + .runSuites(runSuites.classNames, runSuites.parallelism); + } else { + // sbt test-interface: one Framework/Runner for all the project's suites of this + // framework, done() once — maven's forkCount=1 reuseForks=true, which stateful + // frameworks need. Suites run sequentially (these frameworks have no lock-aware + // scheduler); setCurrentSuite tags captured output with the running suite. + new SuiteRunner( + ForkedTestRunner::send, + ForkedTestRunner.class.getClassLoader(), + Arrays.asList(capturedOut, capturedErr)) + .runSuites( + runSuites.classNames, + runSuites.framework, + runSuites.frameworkClass, + runSuites.args, + ForkedTestRunner::setCurrentSuite); + } + } finally { + runningSuites.remove(BATCH_KEY, Thread.currentThread()); + } + }, + "suites-batch"); + runningSuites.put(BATCH_KEY, t); + t.start(); } /** - * Extract the test name from an event. Tries to get the test method name from the selector, falls - * back to fullyQualifiedName. + * The single key a batched (RunSuites) execution registers under; a project runs at most one + * batch at a time. */ - private static String extractTestName(Event event) { - Selector selector = event.selector(); - - if (selector instanceof TestSelector) { - // TestSelector contains the test method name - return ((TestSelector) selector).testName(); - } else if (selector instanceof NestedTestSelector) { - // NestedTestSelector for nested tests - return ((NestedTestSelector) selector).testName(); - } else { - // Fall back to fully qualified name for suite-level events - return event.fullyQualifiedName(); - } - } + private static final String BATCH_KEY = "batch"; /** Generate a thread dump of all threads in the JVM. Returns encoded JSON response. */ private static String generateThreadDump() { @@ -695,11 +595,19 @@ private static String generateThreadDump() { return TestProtocol.encodeThreadDump(entries); } - /** Output stream that captures writes and sends them via protocol. */ + /** + * Output stream that captures writes and sends them via protocol, one buffer per writing thread. + * + *

System.out is one stream shared by every thread in the JVM, so with several suites running + * at once their bytes would interleave in a single buffer and a line could come out half from one + * suite and half from another. A per-thread buffer keeps each writer's partial line to itself, + * and the completed line is tagged with whatever suite that thread is running ({@link + * #currentSuite}) — or none, for a framework thread that belongs to no single suite. When only + * one suite runs at a time this is exactly the old behaviour with one live buffer. + */ private static class CapturingOutputStream extends OutputStream { private final String name; - private final StringBuilder buffer = new StringBuilder(); - private final Object lock = new Object(); + private final ThreadLocal buffer = ThreadLocal.withInitial(StringBuilder::new); CapturingOutputStream(String name) { this.name = name; @@ -707,39 +615,43 @@ private static class CapturingOutputStream extends OutputStream { @Override public void write(int b) { - synchronized (lock) { - if (b == '\n') { - flush(); - } else { - buffer.append((char) b); - } + if (b == '\n') { + flush(); + } else { + buffer.get().append((char) b); } } @Override public void write(byte[] b, int off, int len) { - synchronized (lock) { - String s = new String(b, off, len); - for (int i = 0; i < s.length(); i++) { - char c = s.charAt(i); - if (c == '\n') { - flush(); - } else { - buffer.append(c); - } + StringBuilder buf = buffer.get(); + String s = new String(b, off, len); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == '\n') { + flush(); + } else { + buf.append(c); } } } @Override public void flush() { - synchronized (lock) { - if (buffer.length() > 0) { - String level = "stderr".equals(name) ? "error" : "info"; - // Include current suite in log message if available - send(TestProtocol.encodeLog(currentSuite, level, buffer.toString())); - buffer.setLength(0); - } + StringBuilder buf = buffer.get(); + if (buf.length() > 0) { + String level = "stderr".equals(name) ? "error" : "info"; + // Prefer the writing thread's own suite; fall back to the batch's active suite for output + // on + // a framework/async thread (a ZIO fiber, a specs2 worker) that never set the thread-local. + // Correct because a batch runs one suite at a time by default; without it such output has + // no + // owning suite and, on a batch's shared fork, is dropped instead of landing in + // . + String suite = currentSuite.get(); + if (suite == null) suite = activeBatchSuite; + send(TestProtocol.encodeLog(suite, level, buf.toString())); + buf.setLength(0); } } } diff --git a/bleep-test-runner/src/main/java/bleep/testing/runner/JUnitPlatformRunner.java b/bleep-test-runner/src/main/java/bleep/testing/runner/JUnitPlatformRunner.java index cf5399e43..bb778041d 100644 --- a/bleep-test-runner/src/main/java/bleep/testing/runner/JUnitPlatformRunner.java +++ b/bleep-test-runner/src/main/java/bleep/testing/runner/JUnitPlatformRunner.java @@ -2,12 +2,18 @@ import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; -import java.io.OutputStream; -import java.io.PrintWriter; +import java.io.Flushable; +import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.engine.reporting.ReportEntry; +import org.junit.platform.engine.support.descriptor.ClassSource; +import org.junit.platform.engine.support.descriptor.MethodSource; import org.junit.platform.launcher.Launcher; import org.junit.platform.launcher.LauncherDiscoveryRequest; import org.junit.platform.launcher.TestExecutionListener; @@ -20,22 +26,28 @@ * Runs JUnit 5 tests via JUnit Platform Launcher directly, bypassing sbt test-interface. * *

This enables proper JUnit Platform lifecycle including LauncherSessionListener SPI, which is - * required for frameworks like Quarkus that set up custom classloaders (FacadeClassLoader) during - * session initialization. + * required for frameworks that set up custom classloaders during session initialization. * - *

Using openSession() instead of create() triggers: - Quarkus's CustomLauncherInterceptor → - * FacadeClassLoader as TCCL - Spring Boot's test context management - Any other - * LauncherSessionListener implementations + *

Using openSession() instead of create() triggers any registered LauncherSessionListener + * implementations — e.g. a framework that installs a custom classloader as the thread-context + * classloader, or a test-context manager. */ class JUnitPlatformRunner { /** Fully-qualified name of the session interface, absent before JUnit Platform 1.8. */ private static final String LAUNCHER_SESSION = "org.junit.platform.launcher.LauncherSession"; - private final PrintWriter protocolOut; + private final Consumer sink; - JUnitPlatformRunner(PrintWriter protocolOut) { - this.protocolOut = protocolOut; + /** + * Streams to flush before a result is reported. Empty in process; the fork's captured pair + * otherwise. + */ + private final List toFlush; + + JUnitPlatformRunner(Consumer sink, List toFlush) { + this.sink = sink; + this.toFlush = toFlush; } /** A {@link Launcher} plus whatever has to be closed afterwards. */ @@ -58,6 +70,51 @@ public void close() throws Exception { } } + // ---- Shared-session mode (set by a fork, off in-process) ---- + // + // In a forked JVM every suite shares ONE LauncherSession, so a LauncherSessionListener fires once + // for the whole fork rather than once per suite (and, with concurrent suites, N racing opens). + // The + // session is opened lazily by the first JUnit suite and closed at fork shutdown. In-process + // leaves + // this off: there the daemon JVM runs many projects' suites under different classloaders, and one + // shared session across them would be wrong — each opens its own, as before. + private static volatile boolean SHARE_SESSION = false; + private static volatile LauncherHandle SHARED_HANDLE = null; + private static final Object SHARED_LOCK = new Object(); + + /** Turn on one-session-per-JVM. Called once by a fork at startup, before any suite runs. */ + static void enableSharedSession() { + SHARE_SESSION = true; + } + + /** + * Close the shared session, if one was opened. Called once at fork shutdown; runs + * launcherSessionClosed SPI. + */ + static void closeSharedSession() { + synchronized (SHARED_LOCK) { + if (SHARED_HANDLE != null) { + try { + SHARED_HANDLE.close(); + } catch (Exception ignored) { + // Shutdown path: nothing useful to do if the session will not close. + } + SHARED_HANDLE = null; + } + } + } + + /** The one shared launcher, opened on first use under lock. */ + private LauncherHandle sharedHandle() { + LauncherHandle h = SHARED_HANDLE; + if (h != null) return h; + synchronized (SHARED_LOCK) { + if (SHARED_HANDLE == null) SHARED_HANDLE = openLauncher(); + return SHARED_HANDLE; + } + } + /** * Obtain a launcher, using the session lifecycle when the platform on the classpath has one. * @@ -84,8 +141,7 @@ private LauncherHandle openLauncher() { TestProtocol.encodeLog( "debug", "JUnit Platform predates LauncherSession (1.8); using LauncherFactory.create()." - + " LauncherSessionListener extensions (Quarkus, Spring Boot) do not exist on" - + " this version.")); + + " LauncherSessionListener extensions do not exist on this version.")); return new LauncherHandle(LauncherFactory.create(), null); } @@ -109,10 +165,8 @@ private LauncherHandle openLauncher() { * Run a single test class using JUnit Platform Launcher with full session lifecycle. * * @param className fully qualified test class name - * @param capturedOut captured stdout stream - * @param capturedErr captured stderr stream */ - void runSuite(String className, OutputStream capturedOut, OutputStream capturedErr) { + void runSuite(String className) { long startTime = System.currentTimeMillis(); String currentSuite = className; @@ -129,16 +183,21 @@ void runSuite(String className, OutputStream capturedOut, OutputStream capturedE int[] failed = {0}; int[] skipped = {0}; int[] ignored = {0}; + Map testStartNanos = new ConcurrentHashMap<>(); try { // Flush any pending output - capturedOut.flush(); - capturedErr.flush(); - - // Open a LauncherSession where the platform has one — this triggers LauncherSessionListener - // SPI. - // Quarkus's CustomLauncherInterceptor creates FacadeClassLoader here. - try (LauncherHandle handle = openLauncher()) { + flushAll(); + + // A LauncherSession is where LauncherSessionListener SPI fires (a framework's registrar, or + // an interceptor that builds a custom classloader). In fork mode one + // session is opened for the whole JVM and shared by every suite, so those listeners fire + // exactly ONCE — maven surefire's one-session-per-fork semantics. That is the difference + // between a listener that pre-registers a global once and N concurrent suites each racing to + // register it. Per-suite otherwise (in-process, or a platform predating sessions). + boolean shareSession = SHARE_SESSION; + LauncherHandle handle = shareSession ? sharedHandle() : openLauncher(); + try { Launcher launcher = handle.launcher; LauncherDiscoveryRequest request = @@ -179,6 +238,7 @@ public void testPlanExecutionStarted(TestPlan testPlan) { @Override public void executionStarted(TestIdentifier testIdentifier) { if (testIdentifier.isTest() && !isChildlessVintageClass(testIdentifier)) { + testStartNanos.put(testIdentifier.getUniqueId(), System.nanoTime()); String testName = testIdentifier.getDisplayName(); send(TestProtocol.encodeTestStarted(currentSuite, testName)); } @@ -199,7 +259,7 @@ public void executionFinished( } String testName = testIdentifier.getDisplayName(); - long durationMs = 0; // JUnit Platform doesn't provide per-test duration in listener + long durationMs = elapsedMs(testStartNanos.remove(testIdentifier.getUniqueId())); String status; String message = null; @@ -245,8 +305,7 @@ public void executionFinished( // Flush output before reporting try { - capturedOut.flush(); - capturedErr.flush(); + flushAll(); } catch (Exception e) { // Ignore } @@ -287,8 +346,7 @@ private void reportContainerFailure( failed[0]++; try { - capturedOut.flush(); - capturedErr.flush(); + flushAll(); } catch (Exception e) { // Ignore } @@ -352,11 +410,14 @@ public void reportingEntryPublished( } launcher.execute(request, listener); + } finally { + // A shared session is closed once, at fork shutdown; a per-suite one is this suite's to + // close. + if (!shareSession) handle.close(); } // Flush and report done - capturedOut.flush(); - capturedErr.flush(); + flushAll(); long durationMs = System.currentTimeMillis() - startTime; int total = passed[0] + failed[0] + skipped[0] + ignored[0]; @@ -386,9 +447,118 @@ public void reportingEntryPublished( } } + /** + * Run a project's classes through the shared LauncherSession — one {@code launcher.execute} per + * class — at a bleep-chosen degree of parallelism. + * + *

The shared session is what makes an execution-scoped fixture — an application booted for the + * run — build once and be reused by every class, as under maven surefire's + * one-execute-per-module. Per-class results are still reported: the listener attributes each test + * and container to the requested class it belongs to (a {@code @Nested Foo$Bar} test back to + * {@code Foo}) and sends that class's own SuiteDone when its container finishes, so the parent + * demultiplexes per suite. + * + *

Parallelism is bleep's own: a fixed thread pool runs {@code parallelism} classes at once, + * each as its own {@code launcher.execute}. bleep sets NO JUnit configuration parameters, so a + * {@code junit-platform.properties} on the classpath still governs parallelism WITHIN a class + * (jupiter's own parallel execution) — bleep neither enables nor overrides it. And because each + * class runs in a separate execution, {@code @ResourceLock} across classes is not coordinated by + * the engine; keep {@code parallelism} at 1 (the default) when a project's classes share mutable + * state, which is also what a singleton-per-JVM application requires. + */ + void runSuites(List classNames, int parallelism) { + // One launcher.execute PER class, on the shared LauncherSession — not one execute selecting all + // classes. The one-execute form lost tests for engines whose test tree is not keyed by the + // selected class: a cucumber scenario's class source is the glue/feature, a spek test's is the + // spec node, so neither maps back to the requested Fixture and it reported zero. Per-class + // execute attributes every test in that execute to the one class it selected — exactly what the + // single-suite path already does correctly. The session is opened once (SHARE_SESSION) and + // reused across the classes, so a session-scoped fixture — a booted application, a + // LauncherSessionListener — is still built once, which was the point of batching. `parallelism` + // bounds how many classes execute at once (1 = sequential, the safe default). + try { + if (parallelism <= 1) { + for (String c : classNames) { + ForkedTestRunner.setCurrentSuite(c); + try { + runSuite(c); + } finally { + ForkedTestRunner.setCurrentSuite(null); + } + } + } else { + java.util.concurrent.ExecutorService pool = + java.util.concurrent.Executors.newFixedThreadPool( + Math.min(parallelism, Math.max(1, classNames.size()))); + try { + java.util.List> futures = new java.util.ArrayList<>(); + for (String c : classNames) { + futures.add( + pool.submit( + () -> { + ForkedTestRunner.setCurrentSuite(c); + try { + runSuite(c); + } finally { + ForkedTestRunner.setCurrentSuite(null); + } + return null; + })); + } + for (java.util.concurrent.Future f : futures) { + try { + f.get(); + } catch (java.util.concurrent.ExecutionException e) { + send( + TestProtocol.encodeLog( + "error", stackTraceToString(e.getCause() == null ? e : e.getCause()))); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + } finally { + pool.shutdownNow(); + } + } + } finally { + send(TestProtocol.encodeBatchComplete()); + } + } + + /** + * The class a test or container belongs to, from its source. Null for engine roots and anything + * without a class source. + */ + private static String classOf(TestIdentifier id) { + return id.getSource() + .map( + s -> { + if (s instanceof MethodSource) return ((MethodSource) s).getClassName(); + if (s instanceof ClassSource) return ((ClassSource) s).getClassName(); + return null; + }) + .orElse(null); + } + + /** + * Wall-clock duration of one test, measured by the runner because JUnit Platform's {@link + * TestExecutionListener} does not carry one. The start is stamped in {@code executionStarted} + * (keyed by unique id, safe under the concurrent classes of a batch) and read back here with the + * entry removed. {@link System#nanoTime()} so a wall-clock adjustment mid-suite cannot make it + * negative; a missing start (a test that finished without a matching start, e.g. skipped) reads 0 + * rather than a bogus age-of-the-map. + */ + private static long elapsedMs(Long startNanos) { + return startNanos == null ? 0L : Math.max(0L, (System.nanoTime() - startNanos) / 1_000_000L); + } + private void send(String message) { - protocolOut.println(message); - protocolOut.flush(); + sink.accept(message); + } + + private void flushAll() throws IOException { + for (Flushable f : toFlush) f.flush(); } /** diff --git a/bleep-test-runner/src/main/java/bleep/testing/runner/SuiteRunner.java b/bleep-test-runner/src/main/java/bleep/testing/runner/SuiteRunner.java new file mode 100644 index 000000000..f12811112 --- /dev/null +++ b/bleep-test-runner/src/main/java/bleep/testing/runner/SuiteRunner.java @@ -0,0 +1,644 @@ +package bleep.testing.runner; + +import java.io.Flushable; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; +import sbt.testing.*; + +/** + * Runs one test suite and reports it as encoded {@link TestProtocol} lines. + * + *

Everything about how a suite is run lives here: loading the framework, ordering its + * fingerprints against what the class on disk actually is, executing the tasks it hands back, and + * turning sbt-testing events into protocol messages. None of that knows where the lines go or which + * classes it is looking at — a {@link Consumer} takes the output and a {@link ClassLoader} supplies + * the classes. + * + *

Those two parameters are the whole reason this is not part of {@link ForkedTestRunner}. In a + * fork, the sink writes to the parent's socket and the loader is the fork's own system loader. In + * process, the sink pushes onto a queue and the loader is one built over the project's test + * classpath. Same code, same wire format, same semantics — which is the point: a second + * implementation of fingerprint ordering would be a second set of answers to "did this suite run", + * and the two would drift. + * + *

No static mutable state, deliberately. In a fork there is one suite at a time; in process + * there are as many as the DAG admits, concurrently, in this JVM. Anything static here would be + * shared between them. + */ +public final class SuiteRunner { + + private final Consumer sink; + private final ClassLoader loader; + + /** + * Streams to flush before a test result is reported, so captured output arrives ahead of the + * event it belongs to. Empty in process, where there is nothing between the test and the console. + */ + private final List toFlush; + + public SuiteRunner(Consumer sink, ClassLoader loader, List toFlush) { + this.sink = sink; + this.loader = loader; + this.toFlush = toFlush; + } + + /** + * Run the suite named by an encoded {@link TestProtocol} RunSuite command. + * + *

The entry point for a caller on the other side of a classloader boundary: every parameter + * and the result are platform types, so nothing needs this class's own types to call it. An + * in-process caller reaches this reflectively and decodes the lines the sink receives with the + * same decoder the forked path uses. + */ + public void runSerialized(String runSuiteCommandLine) { + TestProtocol.ParsedCommand cmd = TestProtocol.parseCommand(runSuiteCommandLine); + if (!(cmd instanceof TestProtocol.ParsedCommand.RunSuite)) { + throw new IllegalArgumentException( + "expected an encoded RunSuite command, got: " + runSuiteCommandLine); + } + TestProtocol.ParsedCommand.RunSuite runSuite = (TestProtocol.ParsedCommand.RunSuite) cmd; + runSuite( + runSuite.className, + runSuite.framework, + runSuite.runner.name(), + runSuite.frameworkClass, + runSuite.args); + } + + private void flushAll() throws IOException { + for (Flushable f : toFlush) f.flush(); + } + + public void runSuite( + String className, + String frameworkName, + String runnerKind, + String frameworkClass, + List args) { + + sink.accept( + TestProtocol.encodeLog( + // bleep talking to itself about which suite it was handed. Not the user's test output. + "debug", + "runSuite called: className=" + className + ", frameworkName=" + frameworkName)); + + // The server decided this, with the project's classpath in front of it. Nothing here re-derives + // it from frameworkName, which is a display label. + if (TestProtocol.RunnerKind.JUNIT_PLATFORM.name().equals(runnerKind)) { + JUnitPlatformRunner junitRunner = new JUnitPlatformRunner(sink, toFlush); + junitRunner.runSuite(className); + return; + } + + Framework framework; + Runner runner; + try { + flushAll(); + sink.accept(TestProtocol.encodeLog("debug", "Loading framework: " + frameworkClass)); + framework = loadFramework(frameworkClass); + sink.accept( + TestProtocol.encodeLog("debug", "Framework loaded: " + framework.getClass().getName())); + runner = framework.runner(args.toArray(new String[0]), new String[0], loader); + } catch (Throwable e) { + sink.accept(TestProtocol.encodeLog("error", stackTraceToString(e))); + sink.accept( + TestProtocol.encodeSuiteErrored( + className, + 0, + "Error loading framework " + + frameworkClass + + ": " + + e.getClass().getName() + + ": " + + e.getMessage(), + stackTraceToString(e))); + return; + } + + // One Runner, one done() — even for a single suite. The lifecycle lives in the caller so the + // batch path ([[runSuites]]) can share one Runner across a project's suites, which is the sbt + // interface's contract (one runner per framework per run) and what stateful frameworks need. + try { + runOneSuiteOn(framework, runner, frameworkName, className); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } finally { + try { + runner.done(); + } catch (Throwable ignored) { + // done() is best-effort cleanup; a failure here must not mask the suite's own result. + } + } + } + + /** + * Run every suite in {@code classNames} through ONE {@link Runner} — maven surefire's {@code + * forkCount=1 reuseForks=true}, and what sbt and mill do: {@link Framework#runner} once, all + * suites through it, {@link Runner#done()} once. A fresh Runner (and done()) per suite — the + * shape a shared fork used before — breaks frameworks that keep per-JVM state (munit, ZIO Test), + * which set up on runner creation and tear down on done(); N of each in one JVM corrupted their + * results. + * + *

Suites run sequentially, one at a time — surefire's {@code reuseForks=true}. sbt-interface + * frameworks share one {@code Runner} and have no lock-aware scheduler, so bleep never runs their + * suites concurrently in a shared fork; concurrency for them is a fork per suite (per-suite + * mode). Each suite's events are attributed by the class name bound to it here. {@code + * setCurrentSuite} lets the caller tag captured output with the suite running on the current + * thread (null clears it). + */ + public void runSuites( + List classNames, + String frameworkName, + String frameworkClass, + List args, + Consumer setCurrentSuite) { + Framework framework; + Runner runner; + try { + flushAll(); + framework = loadFramework(frameworkClass); + runner = framework.runner(args.toArray(new String[0]), new String[0], loader); + } catch (Throwable e) { + sink.accept(TestProtocol.encodeLog("error", stackTraceToString(e))); + for (String className : classNames) { + sink.accept( + TestProtocol.encodeSuiteErrored( + className, + 0, + "Error loading framework " + + frameworkClass + + ": " + + e.getClass().getName() + + ": " + + e.getMessage(), + stackTraceToString(e))); + } + return; + } + + try { + for (String className : classNames) { + if (Thread.interrupted()) break; + setCurrentSuite.accept(className); + try { + runOneSuiteOn(framework, runner, frameworkName, className); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } finally { + setCurrentSuite.accept(null); + } + } + } finally { + try { + runner.done(); + } catch (Throwable ignored) { + // best-effort cleanup + } + try { + flushAll(); + } catch (IOException ignored) { + // best-effort + } + // The batch terminator: the parent's runSuites stream ends on this (each suite's own + // SuiteDone + // has already gone by). Without it the parent waits forever for a run that has finished. + sink.accept(TestProtocol.encodeBatchComplete()); + } + } + + /** + * Run one suite on an already-created {@link Runner}. Does NOT create the runner or call {@link + * Runner#done()} — the caller owns that lifecycle so a batch can share one Runner across a + * project's suites. Reports the suite's terminal outcome; a per-suite failure is reported and + * swallowed so a batch's other suites still run, while an interruption propagates to stop the + * run. + */ + private void runOneSuiteOn( + Framework framework, Runner runner, String frameworkName, String className) + throws InterruptedException { + long startTime = System.currentTimeMillis(); + + // Counters declared outside try so they're accessible in catch for SuiteDone reporting + final int[] passed = {0}; + final int[] failed = {0}; + final int[] skipped = {0}; + final int[] ignored = {0}; + + try { + // Try each fingerprint from the framework until we find one that produces tasks. + // Different fingerprints match different test patterns (e.g. @Test annotation vs + // TestCase subclass), so we need to find the right one for this class. + Fingerprint[] fingerprints = framework.fingerprints(); + + if (fingerprints.length == 0) { + sink.accept( + TestProtocol.encodeError("Framework has no fingerprints: " + frameworkName, null)); + return; + } + + Task[] tasks = null; + + // Try fingerprints that agree with what the class actually is before the rest. + // + // "First fingerprint that yields a task" is not enough on its own. A framework that declares + // both a class and a module fingerprint — specs2 does — is + // free to hand back a task for either without checking, and only fails later when it tries to + // load the form that does not exist: a `class Fixture extends + // Specification` matched against the module fingerprint produced a task whose whole error + // message was "example.Specs2Fixture$". Whether a suite is a + // Scala object is not a guess; the compiler emits `Fixture$` for one and not for the other. + Fingerprint[] ordered = orderFingerprintsFor(className, fingerprints); + + for (Fingerprint fingerprint : ordered) { + TaskDef taskDef = + new TaskDef(className, fingerprint, true, new Selector[] {new SuiteSelector()}); + Task[] candidate = runner.tasks(new TaskDef[] {taskDef}); + if (candidate.length > 0) { + tasks = candidate; + sink.accept( + TestProtocol.encodeLog( + "debug", "Matched fingerprint: " + describeFingerprint(fingerprint))); + break; + } + } + + if (tasks == null || tasks.length == 0) { + // No fingerprint produced a task: the loaded framework does not recognize this class as + // a suite. Not an empty suite (the framework never claimed it) — a framework mismatch. + sink.accept( + TestProtocol.encodeSuiteNoFrameworkMatched( + className, + System.currentTimeMillis() - startTime, + "No test framework recognized " + className + " as a suite")); + return; + } + + // Custom event handler to capture test events + EventHandler eventHandler = + new EventHandler() { + @Override + public void handle(Event event) { + String status; + switch (event.status()) { + case Success: + status = "passed"; + passed[0]++; + break; + case Failure: + status = "failed"; + failed[0]++; + break; + case Error: + status = "error"; + failed[0]++; + break; + case Skipped: + status = "skipped"; + skipped[0]++; + break; + case Ignored: + status = "ignored"; + ignored[0]++; + break; + case Canceled: + status = "assumption-failed"; + skipped[0]++; + break; + case Pending: + status = "pending"; + ignored[0]++; + break; + default: + status = "unknown"; + break; + } + + String throwableStr = null; + String message = null; + StackTraceElement location = null; + if (event.throwable() != null && event.throwable().isDefined()) { + Throwable t = event.throwable().get(); + message = t.getMessage(); + throwableStr = stackTraceToString(t); + location = failureLocation(t, className); + } + + // Extract test name from selector if available + String testName = extractTestName(event); + + // Flush output before reporting test finished + try { + flushAll(); + } catch (IOException e) { + // Ignore + } + + sink.accept( + TestProtocol.encodeTestFinished( + className, + testName, + status, + event.duration(), + message, + throwableStr, + location == null ? null : location.getClassName(), + location == null ? null : location.getFileName(), + location == null ? 0 : location.getLineNumber())); + } + }; + + // Execute tasks + Logger logger = createLogger(className); + executeTasks(tasks, eventHandler, new Logger[] {logger}); + + // NB: no runner.done() here — the caller owns the Runner's lifecycle and calls done() once, + // after all of a batch's suites, per the sbt interface's one-runner-per-framework contract. + + // Final flush + flushAll(); + + long durationMs = System.currentTimeMillis() - startTime; + int total = passed[0] + failed[0] + skipped[0] + ignored[0]; + if (total == 0) { + // The framework claimed the class (a task ran) but no test fired an event: an empty suite. + sink.accept(TestProtocol.encodeSuiteEmpty(className, durationMs)); + } else { + sink.accept( + TestProtocol.encodeSuiteExecuted( + className, passed[0], failed[0], skipped[0], ignored[0], durationMs)); + } + + } catch (InterruptedException e) { + // Cancelled — report and propagate so a batch stops rather than starting its next suite. + sink.accept(TestProtocol.encodeLog("warn", "Suite " + className + " was cancelled")); + throw e; + } catch (SecurityException e) { + if (e.getMessage() != null && e.getMessage().contains("System.exit")) { + sink.accept( + TestProtocol.encodeSuiteErrored( + className, + System.currentTimeMillis() - startTime, + "Test attempted a blocked System.exit", + null)); + } else { + // Report and swallow (do not abort a batch's siblings); a non-exit SecurityException here + // is unexpected, so it is surfaced as this suite's error rather than rethrown. + sink.accept(TestProtocol.encodeLog("error", stackTraceToString(e))); + sink.accept( + TestProtocol.encodeSuiteErrored( + className, + System.currentTimeMillis() - startTime, + "Error running suite " + + className + + ": " + + e.getClass().getName() + + ": " + + e.getMessage(), + stackTraceToString(e))); + } + } catch (Throwable e) { + // Must catch Throwable (not just Exception): a framework may let an Error (AssertionError, + // or a LinkageError propagated from executeTasks) escape. Report it as an errored suite — + // NOT SuiteExecuted with faked counts — so the outcome carries the real reason. + sink.accept(TestProtocol.encodeLog("error", stackTraceToString(e))); + Throwable reported = + (e instanceof SuiteExecutionException && e.getCause() != null) ? e.getCause() : e; + sink.accept( + TestProtocol.encodeSuiteErrored( + className, + System.currentTimeMillis() - startTime, + "Error running suite " + + className + + ": " + + reported.getClass().getName() + + ": " + + reported.getMessage(), + stackTraceToString(reported))); + } + } + + private void executeTasks(Task[] tasks, EventHandler eventHandler, Logger[] loggers) + throws InterruptedException { + for (Task task : tasks) { + // Check for interruption before each task + if (Thread.interrupted()) { + throw new InterruptedException(); + } + + try { + Task[] nestedTasks = task.execute(eventHandler, loggers); + // Recursively execute nested tasks + executeTasks(nestedTasks, eventHandler, loggers); + } catch (InterruptedException e) { + throw e; + } catch (Throwable e) { + // Do NOT swallow and continue. A Throwable escaping task.execute — LinkageError, + // NoClassDefFoundError, ExceptionInInitializerError, typically from a stale sibling + // compile — means this suite's classpath cannot be trusted, not that one test failed. + // Swallowing it here let the suite fall through to SuiteDone(...,0,0,0) and be reported + // PASSED: a green build over a suite that never ran. Propagate so the caller's handler + // records a real failure with a non-zero count. + throw new SuiteExecutionException(e); + } + } + } + + /** + * Wraps a non-interruption Throwable that escaped {@code task.execute} so it propagates out of + * {@link #executeTasks} (whose only checked throw is InterruptedException) to runSuite's outer + * handler, which reports it as a suite failure. + */ + private static final class SuiteExecutionException extends RuntimeException { + SuiteExecutionException(Throwable cause) { + super(cause); + } + } + + /** + * Instantiate an sbt.testing.Framework by class name. + * + *

One line, because the server sends the class rather than a label to guess from. This used to + * special-case JUnit, Kotest and TestNG, probe lists of candidate classes, and fall back to + * treating the display name as a class name — which is how "Spock" and "kotlin.test" arrived at + * Class.forName verbatim. + */ + private Framework loadFramework(String frameworkClass) throws Exception { + Class clazz = loader.loadClass(frameworkClass); + return (Framework) clazz.getDeclaredConstructor().newInstance(); + } + + /** Check if this framework should use JUnit Platform Launcher directly. */ + private Logger createLogger(final String suiteName) { + return new Logger() { + @Override + public boolean ansiCodesSupported() { + // Frameworks ask this before colourising. Answering an unconditional `true` meant `bleep + // test --no-color` still got ANSI escapes from ScalaTest, + // hedgehog and friends: the flag lives in the client JVM and this runs in a forked one, so + // the only thing that crosses is the environment. The + // client sets NO_COLOR when the user asks for no colour, and the no-color.org convention + // means a user who sets it themselves is honoured too. + String noColor = System.getenv("NO_COLOR"); + return noColor == null || noColor.isEmpty(); + } + + @Override + public void error(String msg) { + sink.accept(TestProtocol.encodeLog("error", msg)); + } + + @Override + public void warn(String msg) { + sink.accept(TestProtocol.encodeLog("warn", msg)); + } + + @Override + public void info(String msg) { + sink.accept(TestProtocol.encodeLog("info", msg)); + } + + @Override + public void debug(String msg) { + sink.accept(TestProtocol.encodeLog("debug", msg)); + } + + @Override + public void trace(Throwable t) { + sink.accept(TestProtocol.encodeLog("error", stackTraceToString(t))); + } + }; + } + + /** + * Puts fingerprints whose `isModule` matches the class on disk first, keeping the framework's own + * order within each group. Nothing is discarded: a framework that disagrees with this reading + * still gets every fingerprint tried, just second. + */ + private Fingerprint[] orderFingerprintsFor(String className, Fingerprint[] fingerprints) { + Class asModule = loadClass(className + "$"); + Class asPlain = loadClass(className); + boolean isModule = asModule != null; + + // Ranked, highest first, keeping the framework's own order within a rank: + // 2 — the class really does extend what the fingerprint names + // 1 — only the class/object shape agrees + // 0 — neither + // + // Shape alone is not enough to tell a framework's fingerprints apart when several describe + // objects. Weaver declares one for suites and another for global + // resources; picking by shape chose the resource one and the run died with + // "example.WeaverFixture$ is not an instance of weaver.IOGlobalResource". What the + // class extends is the question the fingerprint is actually asking, so ask that first. + List> byRank = new ArrayList<>(); + for (int i = 0; i < 3; i++) byRank.add(new ArrayList<>()); + for (Fingerprint fp : fingerprints) { + Boolean declaredModule = fingerprintIsModule(fp); + boolean shapeAgrees = declaredModule != null && declaredModule == isModule; + // Each fingerprint is checked against the class it is talking about: a module fingerprint + // means `Foo$`, a class fingerprint means `Foo`. Checking both against the object's class + // scored a class fingerprint naming `org.scalacheck.Properties` just as highly as the module + // one — `Foo$` extends Properties either way — and picking it made ScalaCheck unrunnable. + // A Scala 3 mirror class extends nothing, so the wrong shape now scores itself out. + Class meant = (declaredModule != null && declaredModule) ? asModule : asPlain; + boolean extendsIt = + meant != null + && fingerprintSuperclass(fp).map(sup -> sup.isAssignableFrom(meant)).orElse(false); + int rank = extendsIt ? 2 : (shapeAgrees ? 1 : 0); + byRank.get(2 - rank).add(fp); + } + + List ordered = new ArrayList<>(); + for (List rank : byRank) ordered.addAll(rank); + return ordered.toArray(new Fingerprint[0]); + } + + /** The class a SubclassFingerprint names, when it names one and it can be loaded. */ + private Optional> fingerprintSuperclass(Fingerprint fp) { + if (!(fp instanceof SubclassFingerprint)) return Optional.empty(); + return Optional.ofNullable(loadClass(((SubclassFingerprint) fp).superclassName())); + } + + private Class loadClass(String name) { + try { + return Class.forName(name, false, loader); + } catch (ClassNotFoundException | LinkageError e) { + return null; + } + } + + /** Null when the fingerprint kind says nothing about module-ness. */ + private Boolean fingerprintIsModule(Fingerprint fp) { + if (fp instanceof SubclassFingerprint) return ((SubclassFingerprint) fp).isModule(); + if (fp instanceof AnnotatedFingerprint) return ((AnnotatedFingerprint) fp).isModule(); + return null; + } + + private String describeFingerprint(Fingerprint fp) { + if (fp instanceof SubclassFingerprint) { + SubclassFingerprint sfp = (SubclassFingerprint) fp; + return "SubclassFingerprint(" + sfp.superclassName() + ", isModule=" + sfp.isModule() + ")"; + } else if (fp instanceof AnnotatedFingerprint) { + AnnotatedFingerprint afp = (AnnotatedFingerprint) fp; + return "AnnotatedFingerprint(" + afp.annotationName() + ", isModule=" + afp.isModule() + ")"; + } + return fp.toString(); + } + + public static String stackTraceToString(Throwable t) { + StringWriter sw = new StringWriter(); + t.printStackTrace(new PrintWriter(sw)); + return sw.toString(); + } + + /** + * The first stack frame belonging to the suite class itself, which is where the failing assertion + * lives for every framework we support — the frames above it are inside the assertion library. + * + *

Deliberately not "the first frame with a line number": that points at someone else's source, + * and an annotation on the wrong file is worse than no annotation. Returns null when the + * throwable has no frame in the suite, which is normal for a failure thrown from a helper or a + * fixture. + * + *

Inner and anonymous classes ({@code MyTest$$anon$1}) still belong to the suite, so match on + * the {@code $} boundary rather than equality alone. Causes are walked because assertion + * libraries routinely wrap. + */ + private StackTraceElement failureLocation(Throwable t, String suiteClass) { + for (Throwable current = t; current != null; current = current.getCause()) { + for (StackTraceElement frame : current.getStackTrace()) { + String cn = frame.getClassName(); + boolean inSuite = cn.equals(suiteClass) || cn.startsWith(suiteClass + "$"); + if (inSuite && frame.getFileName() != null && frame.getLineNumber() > 0) { + return frame; + } + } + if (current.getCause() == current) break; // self-referential cause, seen in the wild + } + return null; + } + + /** + * Extract the test name from an event. Tries to get the test method name from the selector, falls + * back to fullyQualifiedName. + */ + private String extractTestName(Event event) { + Selector selector = event.selector(); + + if (selector instanceof TestSelector) { + // TestSelector contains the test method name + return ((TestSelector) selector).testName(); + } else if (selector instanceof NestedTestSelector) { + // NestedTestSelector for nested tests + return ((NestedTestSelector) selector).testName(); + } else { + // Fall back to fully qualified name for suite-level events + return event.fullyQualifiedName(); + } + } +} diff --git a/bleep-test-runner/src/main/java/bleep/testing/runner/TestProtocol.java b/bleep-test-runner/src/main/java/bleep/testing/runner/TestProtocol.java index b21d7992c..2b7d6190f 100644 --- a/bleep-test-runner/src/main/java/bleep/testing/runner/TestProtocol.java +++ b/bleep-test-runner/src/main/java/bleep/testing/runner/TestProtocol.java @@ -16,6 +16,8 @@ private TestProtocol() {} // === Commands (parent -> forked JVM) === public static final String CMD_RUN_SUITE = "RunSuite"; + public static final String CMD_RUN_SUITES = "RunSuites"; + public static final String CMD_CANCEL_SUITE = "CancelSuite"; public static final String CMD_SHUTDOWN = "Shutdown"; public static final String CMD_GET_THREAD_DUMP = "GetThreadDump"; @@ -273,6 +275,49 @@ public static ParsedCommand parseCommand(String line) { return new ParsedCommand.Shutdown(); } else if (CMD_GET_THREAD_DUMP.equals(type)) { return new ParsedCommand.GetThreadDump(); + } else if (CMD_CANCEL_SUITE.equals(type)) { + String className = extractStringField(line, "className"); + if (className == null) { + return new ParsedCommand.Invalid("Missing className for CancelSuite"); + } + return new ParsedCommand.CancelSuite(className); + } else if (CMD_RUN_SUITES.equals(type)) { + int dataStart = line.indexOf("\"data\""); + if (dataStart < 0) { + return new ParsedCommand.Invalid("Missing data field for RunSuites"); + } + String dataSection = line.substring(dataStart); + List classNames = extractStringArray(dataSection, "classNames"); + String framework = extractStringField(dataSection, "framework"); + String runner = extractStringField(dataSection, "runner"); + Integer parallelism = extractIntField(dataSection, "parallelism"); + if (classNames == null + || classNames.isEmpty() + || framework == null + || runner == null + || parallelism == null) { + return new ParsedCommand.Invalid( + "Missing classNames, framework, runner or parallelism for RunSuites"); + } + RunnerKind kind = RunnerKind.fromWire(runner); + if (kind == null) { + return new ParsedCommand.Invalid("Unknown runner: " + runner); + } + // frameworkClass + args are the sbt-interface batch's payload (which Framework to build and + // its args); JUnit Platform ignores them (it configures via parameters and needs no class). + String frameworkClass = extractStringField(dataSection, "frameworkClass"); + List args = extractStringArray(dataSection, "args"); + if (kind == RunnerKind.SBT_TEST_INTERFACE && frameworkClass == null) { + return new ParsedCommand.Invalid( + "RunSuites for sbt-test-interface requires frameworkClass"); + } + return new ParsedCommand.RunSuites( + classNames, + parallelism, + framework, + kind, + frameworkClass, + args == null ? java.util.Collections.emptyList() : args); } else if (CMD_RUN_SUITE.equals(type)) { // Extract data object fields int dataStart = line.indexOf("\"data\""); @@ -359,6 +404,25 @@ private static String jsonString(String s) { return sb.toString(); } + /** Extract an integer field value from JSON (simple parsing). Null if absent or unparseable. */ + private static Integer extractIntField(String json, String fieldName) { + String pattern = "\"" + fieldName + "\""; + int idx = json.indexOf(pattern); + if (idx < 0) return null; + int colon = json.indexOf(':', idx + pattern.length()); + if (colon < 0) return null; + int i = colon + 1; + while (i < json.length() && Character.isWhitespace(json.charAt(i))) i++; + int start = i; + while (i < json.length() && (json.charAt(i) == '-' || Character.isDigit(json.charAt(i)))) i++; + if (i == start) return null; + try { + return Integer.parseInt(json.substring(start, i)); + } catch (NumberFormatException e) { + return null; + } + } + /** Extract a string field value from JSON (simple parsing). */ private static String extractStringField(String json, String fieldName) { String pattern = "\"" + fieldName + "\""; @@ -515,6 +579,11 @@ public static String encodeShutdown() { return "{\"type\":\"Shutdown\"}"; } + /** The single batched execution has returned; every class's own terminal was already sent. */ + public static String encodeBatchComplete() { + return "{\"type\":\"BatchComplete\"}"; + } + // === Parsed command types === public abstract static class ParsedCommand { @@ -548,6 +617,49 @@ public RunSuite( } } + /** + * Run a whole set of JUnit-Platform classes in one launcher execution, at the given + * parallelism. + */ + public static final class RunSuites extends ParsedCommand { + public final List classNames; + public final int parallelism; + public final String framework; + public final RunnerKind runner; + + /** sbt-interface only: the Framework class to instantiate. Null for JUnit Platform. */ + public final String frameworkClass; + + /** + * Framework args (sbt-interface); empty for JUnit Platform, which configures via parameters. + */ + public final List args; + + public RunSuites( + List classNames, + int parallelism, + String framework, + RunnerKind runner, + String frameworkClass, + List args) { + this.classNames = classNames; + this.parallelism = parallelism; + this.framework = framework; + this.runner = runner; + this.frameworkClass = frameworkClass; + this.args = args; + } + } + + /** Interrupt one in-flight suite by name; the fork and its other suites keep running. */ + public static final class CancelSuite extends ParsedCommand { + public final String className; + + public CancelSuite(String className) { + this.className = className; + } + } + public static final class Shutdown extends ParsedCommand {} public static final class GetThreadDump extends ParsedCommand {} diff --git a/bleep-tests/src/scala/bleep/BatchModeIT.scala b/bleep-tests/src/scala/bleep/BatchModeIT.scala new file mode 100644 index 000000000..945ef07a2 --- /dev/null +++ b/bleep-tests/src/scala/bleep/BatchModeIT.scala @@ -0,0 +1,176 @@ +package bleep + +import bleep.commands.{DisplayMode, ReactiveBsp} +import ryddig.{Stored, TypedLogger} + +import java.nio.file.{Files, Path} + +/** Per-project batch mode (`testFork: per-project`) runs every one of a project's JUnit-Platform suites through a SINGLE `launcher.execute()` in one fork, so + * an execution-scoped fixture — an application booted for the run — is built once and reused, the way maven surefire's one-execute-per-module works. + * + * The whole promise is that nothing downstream can tell the difference: each suite must still report its own result, its own captured output, and real + * per-test durations, exactly as a suite-by-suite run does — only the fork does one execute instead of N. These tests pin that promise against the JUnit XML + * the run writes, which is the authoritative per-suite record (see [[JUnitReports]]): a batch that silently merged suites, dropped a sibling's output into the + * wrong ``, or reported `time="0"` for every case would pass a mere "the run was green" check and fail here. + */ +class BatchModeIT extends IntegrationTestHarness { + + private val project = model.CrossProjectName(model.ProjectName("batched-test"), None) + + /** Three JUnit-Platform suites in one project, `testFork: per-project` so they batch into one fork. `maxConcurrentSuites: 2` runs them concurrently, which is + * the path where per-thread output capture and per-suite attribution have to hold under interleaving. + */ + private def yamlFor(jupiter: String): String = + s"""projects: + | batched-test: + | platform: + | name: jvm + | isTestProject: true + | testFork: per-project + | maxConcurrentSuites: 2 + | dependencies: + | - org.junit.jupiter:junit-jupiter:$jupiter + |""".stripMargin + + private val jupiter = "5.10.1" + + /** A suite that prints a suite-specific marker to stdout, records the pid of the JVM it ran in, and passes two tests. Parameterised so three near-identical + * suites differ only in name and marker — enough to tell their results, output and JVM apart. + */ + private def suiteSource(name: String, extraTest: String): String = + s"""package com.example; + | + |import static org.junit.jupiter.api.Assertions.assertEquals; + |import static org.junit.jupiter.api.Assertions.assertTrue; + | + |import java.nio.file.Files; + |import java.nio.file.Path; + |import org.junit.jupiter.api.Test; + | + |class ${name}Test { + | @Test + | void recordsAndPrints() throws Exception { + | System.out.println("MARKER-$name"); + | Path dir = Path.of(System.getProperty("batch.piddir")); + | Files.writeString(dir.resolve("$name"), String.valueOf(ProcessHandle.current().pid())); + | assertEquals(2, 1 + 1); + | } + | + | @Test + | void alsoPasses() { + | assertTrue(true); + | } + |$extraTest + |} + |""".stripMargin + + /** A test that sleeps a measurable amount, so its `` proves the runner is timing tests itself (JUnit Platform's listener carries no duration — + * without our own stopwatch every case would be `time="0"`). + */ + private val slowTest = + """ + | @Test + | void slow() throws Exception { + | Thread.sleep(60); + | assertTrue(true); + | } + |""".stripMargin + + private def runBatch(ws: Workspace, pidDir: Path, reportDir: Path): (Started, TypedLogger[Array[Stored]]) = { + val (started, _, storingLogger) = ws.start() + Files.createDirectories(pidDir) + // A run with a failing suite returns Left; the per-suite record is the XML, asserted by the caller. + val _ = ReactiveBsp + .test( + watch = false, + projects = Array(project), + displayMode = DisplayMode.NoTui, + jvmOptions = List(s"-Dbatch.piddir=$pidDir"), + testArgs = Nil, + only = Nil, + exclude = Nil, + includeTags = Nil, + excludeTags = Nil, + flamegraph = false, + cancel = false, + junitReportDir = Some(reportDir), + diffBase = None, + diffOutput = OutputMode.Text, + clientEnv = Map.empty + ) + .run(started) + (started, storingLogger) + } + + integrationTest("per-project batch: each suite reports separately, in one JVM, with attributed output and real durations") { ws => + ws.yaml(yamlFor(jupiter)) + ws.file("batched-test/src/java/com/example/AlphaTest.java", suiteSource("Alpha", slowTest)) + ws.file("batched-test/src/java/com/example/BetaTest.java", suiteSource("Beta", "")) + ws.file("batched-test/src/java/com/example/GammaTest.java", suiteSource("Gamma", "")) + + val pidDir = ws.root.resolve("pids") + val reportDir = ws.root.resolve("junit-reports") + val (_, storingLogger) = runBatch(ws, pidDir, reportDir) + + val suites = JUnitReports.read(reportDir) + def suite(name: String): JUnitReports.Suite = + suites.find(_.name == s"com.example.${name}Test").getOrElse(fail(s"no suite com.example.${name}Test; got ${suites.map(_.name)}")) + + // 1. Per-suite results survive the batch: three distinct suites, each green, none merged into another. + assert(suites.map(_.name).toSet == Set("com.example.AlphaTest", "com.example.BetaTest", "com.example.GammaTest"), suites.map(_.describe)) + assert(suite("Alpha").passed == 3, suite("Alpha").describe) // recordsAndPrints + alsoPasses + slow + assert(suite("Beta").passed == 2, suite("Beta").describe) + assert(suite("Gamma").passed == 2, suite("Gamma").describe) + assertSuitePassed(storingLogger, "com.example.BetaTest", tests = 2) + + // 2. One fork for the whole project: every suite recorded the same pid. + val pids = List("Alpha", "Beta", "Gamma").map(n => Files.readString(pidDir.resolve(n)).trim) + assert(pids.distinct.size == 1, s"expected all suites in one JVM, got pids $pids") + + // 3. Output is attributed to its own suite — Alpha's marker in Alpha's , and nowhere else. + assert(suite("Alpha").systemOut.contains("MARKER-Alpha"), s"Alpha system-out: ${suite("Alpha").systemOut}") + assert(!suite("Beta").systemOut.contains("MARKER-Alpha"), s"Beta system-out leaked Alpha's marker: ${suite("Beta").systemOut}") + assert(suite("Beta").systemOut.contains("MARKER-Beta"), s"Beta system-out: ${suite("Beta").systemOut}") + + // 4. Real per-test durations: the 60ms sleeper is timed, not reported as zero. + val slow = suite("Alpha").cases.find(_.name.startsWith("slow")).getOrElse(fail(s"no slow case in ${suite("Alpha").cases.map(_.name)}")) + assert(slow.timeSeconds >= 0.03, s"slow test timed as ${slow.timeSeconds}s — durations are not being measured") + } + + integrationTest("per-project batch: a failing suite does not sink its siblings") { ws => + ws.yaml(yamlFor(jupiter)) + ws.file( + "batched-test/src/java/com/example/AlphaTest.java", + s"""package com.example; + | + |import static org.junit.jupiter.api.Assertions.assertEquals; + | + |import org.junit.jupiter.api.Test; + | + |class AlphaTest { + | @Test + | void boom() { + | assertEquals("expected", "actual"); + | } + |} + |""".stripMargin + ) + ws.file("batched-test/src/java/com/example/BetaTest.java", suiteSource("Beta", "")) + ws.file("batched-test/src/java/com/example/GammaTest.java", suiteSource("Gamma", "")) + + val pidDir = ws.root.resolve("pids") + val reportDir = ws.root.resolve("junit-reports") + runBatch(ws, pidDir, reportDir) + + val suites = JUnitReports.read(reportDir) + def suite(name: String): JUnitReports.Suite = + suites.find(_.name == s"com.example.${name}Test").getOrElse(fail(s"no suite com.example.${name}Test; got ${suites.map(_.name)}")) + + // Alpha's one test failed and is reported as a failure... + assert(suite("Alpha").failures + suite("Alpha").errors >= 1, suite("Alpha").describe) + assert(suite("Alpha").cases.exists(c => c.status == "failure" || c.status == "error"), suite("Alpha").describe) + // ...while its batch-mates ran to completion and came back green — the failure did not take the shared execute down with it. + assert(suite("Beta").passed == 2, suite("Beta").describe) + assert(suite("Gamma").passed == 2, suite("Gamma").describe) + } +} diff --git a/bleep-tests/src/scala/bleep/InProcessTestExecutorTest.scala b/bleep-tests/src/scala/bleep/InProcessTestExecutorTest.scala new file mode 100644 index 000000000..fc474bf52 --- /dev/null +++ b/bleep-tests/src/scala/bleep/InProcessTestExecutorTest.scala @@ -0,0 +1,103 @@ +package bleep + +import bleep.testing.{FrameworkSelection, InProcessTestExecutor, TestProtocol, TestSessionRequest} +import bleep.bsp.protocol.SuiteOutcome +import cats.effect.unsafe.implicits.global +import org.scalatest.funsuite.AnyFunSuite + +import java.io.File +import java.nio.file.{Path, Paths} + +/** A real ScalaTest suite, run by [[InProcessTestExecutor]], in the JVM running this test. + * + * The classpath comes from `java.class.path`, which inside a bleep test fork is the test classpath bleep assembled — so it carries ScalaTest, the sbt test + * interface, and `bleep-test-runner` itself. That is exactly the shape of classpath the executor is built for, available here without resolving anything. + * + * Worth doing at this level rather than asserting on the wiring: the mechanism this covers is a reflective call across a classloader boundary into a + * `SuiteRunner` loaded from the project's own copy, with only `java.*` types crossing. Every way that can be wrong — the constructor signature not matching, + * the platform parent hiding something the framework needs, protocol lines not decoding — is invisible to a type checker and shows up only when a framework is + * actually loaded and asked to run something. + */ +class InProcessTestExecutorTest extends AnyFunSuite { + + private val fixtureFqn = "bleep.InProcessFixtureSuite" + + private def currentClasspath: List[Path] = + System.getProperty("java.class.path").split(File.pathSeparator).toList.filter(_.nonEmpty).map(Paths.get(_)) + + private def runFixture(suite: String): List[TestProtocol.TestResponse] = { + val executor = new InProcessTestExecutor(maxConcurrentSuites = 2) + val request = TestSessionRequest( + label = suite, + classpath = currentClasspath, + jvmOptions = Nil, + defaultHeapMb = 0L, + runnerClass = "bleep.testing.runner.ForkedTestRunner", + environment = Map.empty, + workingDirectory = None, + sharing = bleep.testing.SessionSharing.Exclusive + ) + try + executor + .acquire(request) + .use(session => session.runSuite(suite, FrameworkSelection.SbtTestInterface("scalatest", "org.scalatest.tools.Framework"), Nil).compile.toList) + .unsafeRunSync() + finally executor.shutdown.unsafeRunSync() + } + + test("a ScalaTest suite runs in this process and reports every test it executed") { + val responses = runFixture(fixtureFqn) + + val finished = responses.collect { case f: TestProtocol.TestResponse.TestFinished => f } + assert( + finished.map(_.test).sorted == List("addition still works", "strings are still strings"), + s"expected both fixture tests to report; got:\n${responses.mkString("\n")}" + ) + assert(finished.forall(_.status == "passed"), s"fixture tests should pass: ${finished.map(f => s"${f.test}=${f.status}")}") + + val done = responses.collect { case d: TestProtocol.TestResponse.SuiteDone => d } + assert(done.size == 1, s"expected exactly one terminal SuiteDone, got ${done.size}") + assert(done.head.outcome == SuiteOutcome.Executed(passed = 2, failed = 0, skipped = 0, ignored = 0), s"outcome was ${done.head.outcome}") + } + + test("a class that cannot be run is reported as errored, never as an empty green suite") { + // The distinction that matters most in this whole path: a class that produced no test results must not read as a suite that simply had none. ScalaTest + // does claim `java.lang.String` at fingerprint time and then refuses it at execute time, so the failure arrives as a throwable out of `task.execute` — + // which is the case `SuiteRunner.executeTasks` deliberately propagates rather than swallowing, because swallowing it reported a suite that never ran as + // passing. Asserted here on the in-process path for the same reason it matters on the forked one. + val responses = runFixture("java.lang.String") + val done = responses.collect { case d: TestProtocol.TestResponse.SuiteDone => d } + assert(done.size == 1, s"expected exactly one terminal SuiteDone, got:\n${responses.mkString("\n")}") + done.head.outcome match { + case SuiteOutcome.Errored(message, _) => + assert(message.contains("java.lang.String"), s"the error should name the class it could not run: $message") + case other => + fail(s"a class that ran nothing must not report as $other") + } + } + + test("the executor refuses a request it cannot honour rather than running the tests without it") { + val executor = new InProcessTestExecutor(maxConcurrentSuites = 1) + val withOptions = TestSessionRequest( + label = "some-suite", + classpath = currentClasspath, + jvmOptions = List("-Xmx4g"), + defaultHeapMb = 0L, + runnerClass = "bleep.testing.runner.ForkedTestRunner", + environment = Map.empty, + workingDirectory = None, + sharing = bleep.testing.SessionSharing.Exclusive + ) + val thrown = intercept[RuntimeException](executor.acquire(withOptions).use(_ => cats.effect.IO.unit).unsafeRunSync()) + assert(thrown.getMessage.contains("-Xmx4g"), s"the refusal should name what it could not honour: ${thrown.getMessage}") + executor.shutdown.unsafeRunSync() + } +} + +/** The suite [[InProcessTestExecutorTest]] runs through the in-process executor. Only passing tests: it is discovered and run by the outer build as well, being + * an ordinary suite on this project's test classpath, and a fixture that failed on purpose would fail that run too. + */ +class InProcessFixtureSuite extends AnyFunSuite { + test("addition still works")(assert(1 + 1 == 2)) + test("strings are still strings")(assert("bleep".nonEmpty)) +} diff --git a/bleep-tests/src/scala/bleep/JUnitReports.scala b/bleep-tests/src/scala/bleep/JUnitReports.scala index 7a9234bf0..6beb3a6dc 100644 --- a/bleep-tests/src/scala/bleep/JUnitReports.scala +++ b/bleep-tests/src/scala/bleep/JUnitReports.scala @@ -20,7 +20,7 @@ object JUnitReports { * Both matter and frameworks disagree about which to use: munit puts a full diff in the attribute, while ScalaTest and hedgehog leave it empty and write the * failure into the body. Reading only one of the two makes half the frameworks look like they report failures with nothing to say. */ - case class Case(name: String, className: String, status: String, message: Option[String], detail: String) + case class Case(name: String, className: String, status: String, message: Option[String], detail: String, timeSeconds: Double) /** `systemOut` and `systemErr` are the `` / `` sections: everything the test program printed. * @@ -72,7 +72,8 @@ object JUnitReports { className = tc.getAttribute("classname"), status = child.map(_.getTagName).getOrElse("passed"), message = child.map(_.getAttribute("message")).filter(_.nonEmpty), - detail = child.map(c => (c.getAttribute("message") + "\n" + c.getTextContent).trim).getOrElse("") + detail = child.map(c => (c.getAttribute("message") + "\n" + c.getTextContent).trim).getOrElse(""), + timeSeconds = doubleAttr(tc, "time") ) } ) @@ -84,6 +85,11 @@ object JUnitReports { if (raw.isEmpty) 0 else raw.toInt } + private def doubleAttr(e: Element, name: String): Double = { + val raw = e.getAttribute(name) + if (raw.isEmpty) 0.0 else raw.toDouble + } + private def elements(parent: Element, tag: String): List[Element] = childElements(parent).filter(_.getTagName == tag) diff --git a/bleep-tests/src/scala/bleep/RunningTestCountTest.scala b/bleep-tests/src/scala/bleep/RunningTestCountTest.scala new file mode 100644 index 000000000..67731e38f --- /dev/null +++ b/bleep-tests/src/scala/bleep/RunningTestCountTest.scala @@ -0,0 +1,49 @@ +package bleep.testing + +import bleep.model +import org.scalatest.funsuite.AnyFunSuite + +/** What the "Tests (N running)" heading is allowed to count. + * + * The number is read as a statement about how hard the machine is being driven, so it being wrong is not cosmetic: a run over nine JVM projects with nineteen + * suites in flight announced twenty-eight, on a machine whose governor admits eighteen — which reads as the scheduler over-subscribing, and sends you looking + * at admission code that turns out to be correct. + */ +class RunningTestCountTest extends AnyFunSuite { + + private def project(name: String) = model.CrossProjectName(model.ProjectName(name), None) + + private def jvmProject(name: String) = + ProjectDisplayItem.Testing.Reactive( + project = project(name), + suitesCompleted = 0, + suitesTotal = 10, + failures = 0, + runningTests = Nil + ) + + private def platformProject(name: String) = + ProjectDisplayItem.Testing.Bsp(project = project(name), platform = model.PlatformId.Js, elapsedMs = 1000L) + + test("a JVM project's suites are counted once, not once per suite plus once per project") { + // The shape from the report: nine projects testing, nineteen suites actually running. + val items = List.tabulate(9)(i => jvmProject(s"proj-$i")) + assert(FancyBuildDisplay.runningTestCount(runningSuites = 19, displayItems = items) == 19) + } + + test("a JS or Native project counts once, because it reports no suites of its own") { + // These emit no per-suite events, so `runningSuites` knows nothing about them. Counting the project is the only way the heading reflects that they are + // running at all. + val items = List(platformProject("js-app"), platformProject("native-app")) + assert(FancyBuildDisplay.runningTestCount(runningSuites = 0, displayItems = items) == 2) + } + + test("a mixed run adds the platform projects to the suite count and nothing else") { + val items = List(jvmProject("jvm-app"), platformProject("js-app")) + assert(FancyBuildDisplay.runningTestCount(runningSuites = 5, displayItems = items) == 6) + } + + test("nothing running is nothing running") { + assert(FancyBuildDisplay.runningTestCount(runningSuites = 0, displayItems = Nil) == 0) + } +} diff --git a/bleep-tests/src/scala/bleep/history/TranscriptFormatTest.scala b/bleep-tests/src/scala/bleep/history/TranscriptFormatTest.scala index 39026aab8..01046c6f0 100644 --- a/bleep-tests/src/scala/bleep/history/TranscriptFormatTest.scala +++ b/bleep-tests/src/scala/bleep/history/TranscriptFormatTest.scala @@ -119,6 +119,43 @@ class TranscriptFormatTest extends AnyFunSuite with Matchers { private def formatTest(events: List[E]): io.circe.Json = TranscriptFormat.formatTestResult(events, testRunResult = None, includeThrowables = false, query = None, limit = None, offset = None) + private def formatTestWith(events: List[E], trr: BleepBspProtocol.TestRunResult): io.circe.Json = + TranscriptFormat.formatTestResult(events, testRunResult = Some(trr), includeThrowables = false, query = None, limit = None, offset = None) + + private def runResult(passed: Int, suitesTotal: Int, suitesCompleted: Int): BleepBspProtocol.TestRunResult = + BleepBspProtocol.TestRunResult( + totalPassed = passed, + totalFailed = 0, + totalSkipped = 0, + totalIgnored = 0, + suitesTotal = suitesTotal, + suitesCompleted = suitesCompleted, + suitesFailed = 0, + suitesCancelled = 0, + durationMs = 0L, + historyId = None + ) + + test("a partial run — fewer suites completed than discovered — is success:false over MCP and carries the suite counts") { + // The `bleep_test dfmt/test` report: the MCP surface returned {"success":true,"passed":64,...} with NO suite line at all, so an agent driving bleep over MCP + // could not know 19 of 29 suites never ran. The `success` boolean is the only verdict it has, and it lied. This pins both halves of the fix: the verdict flips + // to false, and the discovered-vs-completed counts are present in the JSON so the gap is actionable, not merely implied by a summary string. + val json = formatTestWith(Nil, runResult(passed = 64, suitesTotal = 29, suitesCompleted = 10)) + json.hcursor.get[Boolean]("success") shouldBe Right(false) + json.hcursor.get[String]("summary").toOption.get should include("did not finish") + json.hcursor.get[Int]("suitesTotal") shouldBe Right(29) + json.hcursor.get[Int]("suitesCompleted") shouldBe Right(10) + json.hcursor.get[Int]("suitesDidNotFinish") shouldBe Right(19) + } + + test("a run where every discovered suite completed is success:true and omits the did-not-finish count") { + val json = formatTestWith(List(passedTest("app")), runResult(passed = 29, suitesTotal = 29, suitesCompleted = 29)) + json.hcursor.get[Boolean]("success") shouldBe Right(true) + json.hcursor.get[Int]("suitesTotal") shouldBe Right(29) + json.hcursor.get[Int]("suitesCompleted") shouldBe Right(29) + json.hcursor.get[Int]("suitesDidNotFinish").toOption shouldBe None + } + private def passedTest(p: String): E = E.TestFinished( proj(p), diff --git a/bleep.yaml b/bleep.yaml index 900be216b..348905d47 100644 --- a/bleep.yaml +++ b/bleep.yaml @@ -369,12 +369,21 @@ projects: # wall-time. The full `build` job in GHA still runs them; native-image jobs exclude them with `--exclude-tag slow`. slow: - "**IT" - # The framework version sweep: every framework x every version it declares x every Scala version x every platform, each one a full inner build resolved - # from Maven Central. It exists to catch what the pinned matrix cannot — an older framework release whose suites bleep discovers or dispatches wrongly — - # and the axes only move when someone edits TestFrameworkFixture. So no CI job runs it; run it deliberately with `--only-tag matrix` when changing - # discovery, runner selection or the fork protocol. The `build` job excludes it explicitly. + # The framework matrices, run deliberately with `--only-tag matrix`, never by a CI job (the `build` job excludes it explicitly). Each case is a full inner + # build that links and runs a real project on a real toolchain (node for JS, LLVM for Native) resolving frameworks from Maven Central — the slowest and + # most environment-sensitive suites we have, and their axes only move when someone edits TestFrameworkFixture, so they belong to a deliberate run rather + # than every PR: + # - `**TestFrameworkIT` the pinned matrix: each framework at its pinned version x every Scala version x every platform (JVM/JS/Native/Kotlin). + # - `**TestFrameworkVersionIT` the version sweep: additionally every version a framework declares, to catch an older release bleep dispatches wrongly. + # - the JS/Native link-behaviour suites below are not framework sweeps, but they are kept manual for the same reason: each links and runs a real project + # on a real toolchain (node for JS, LLVM for Native), resolving frameworks from Maven Central. They pass — run them with `--only-tag matrix` — but need + # a toolchain the plain `build` job should not have to carry, so they run deliberately rather than on every PR. matrix: + - "**TestFrameworkIT" - "**TestFrameworkVersionIT" + - "**ScalaJsTestModuleKindIT" + - "**ScalaNativeSharedBinaryIT" + - "**TestLinkOutputDirIT" scripts: dependencies: - build.bleep::bleep-plugin-native-image:${BLEEP_VERSION} diff --git a/bleepscript/src/main/java/bleepscript/ProjectPaths.java b/bleepscript/src/main/java/bleepscript/ProjectPaths.java index 39bbd6e62..4e596feef 100644 --- a/bleepscript/src/main/java/bleepscript/ProjectPaths.java +++ b/bleepscript/src/main/java/bleepscript/ProjectPaths.java @@ -22,4 +22,13 @@ public record ProjectPaths( sourceDirs = List.copyOf(sourceDirs); resourceDirs = List.copyOf(resourceDirs); } + + /** + * Extra JVM options a test fork of this project needs, one per line. A sourcegen may write this + * file to declare options its generated output requires at runtime; bleep appends them when it + * assembles the fork. Kept identical to {@code bleep.ProjectPaths.forkJvmOptions}. + */ + public Path forkJvmOptions() { + return targetDir.resolve("bleep-fork-jvm-options"); + } } diff --git a/docs/usage/testing.mdx b/docs/usage/testing.mdx new file mode 100644 index 000000000..78a4c26d7 --- /dev/null +++ b/docs/usage/testing.mdx @@ -0,0 +1,180 @@ +--- +title: Running tests +--- + +# Running tests + +A test project in bleep is an ordinary project with `isTestProject: true`. There is no +separate test scope and no second source root to configure — tests are a project that +depends on the code under test: + +```yaml +projects: + myapp: {} + myapp-test: + isTestProject: true + dependsOn: myapp + dependencies: + - org.scalatest::scalatest:3.2.19 +``` + +```bash +bleep test # every test project +bleep test myapp-test # one project (deps compile first) +bleep test -w # watch mode: re-run on change +``` + +`bleep test` with no project argument runs every test project; naming non-test projects +just compiles them. Test frameworks are **auto-detected** from the classpath — ScalaTest, +JUnit 4/5 (JUnit Platform), MUnit, utest, Specs2, ScalaCheck, and weaver all work without +configuration. + +## How a project's suites run + +Two knobs, each answering one question, plus one rule: + +| Setting | Question it answers | Default | +| --- | --- | --- | +| `testFork` | *Where* do a project's suites run? | `per-project` (one shared JVM) | +| `maxConcurrentSuites` | *How many* run at once? | `1` — sequential | + +**The rule:** in a shared JVM, only JUnit Platform parallelizes; for sbt-interface +frameworks, concurrency means separate forks (`per-suite`). Everything below follows from it. + +### `testFork: per-project` (the default) + +Every one of a project's suites runs in **one forked JVM** — the same model as maven +surefire's `forkCount=1 reuseForks=true`. JVM-wide state is built once and reused across +suites: a booted application, dev-service containers, a shared Testcontainers instance, +a schema an earlier suite created. Suites run **one at a time** by default (`maxConcurrentSuites: 1`), +which is memory-frugal — one live suite's heap at a time — and correct for every framework. +Your cores stay busy because bleep runs **many projects' forks at once** (bounded by the +machine-wide governor), not many suites within one fork. + +```yaml +myapp-test: + isTestProject: true + # testFork: per-project # implied + # maxConcurrentSuites: 1 # implied (sequential) +``` + +In per-project mode, `maxConcurrentSuites` **only affects JUnit Platform** suites — it runs that +many of the project's JUnit classes at once in the one fork. **sbt-interface frameworks always run +sequentially here** (they share one `Runner`); set it on an sbt-only project and bleep warns it has +no effect. Either way it bounds **this project's** concurrency — the machine-wide governor still caps +the total across all projects. + +One caveat: bleep runs each class as its own JUnit execution today, so `@ResourceLock` is not +coordinated *across* classes — keep it at `1` when a project's classes share mutable state. (The full +sbt-vs-JUnit story is in [Test frameworks](/docs/appendix/test-frameworks).) + +```yaml +web-test: + isTestProject: true + maxConcurrentSuites: 4 # up to 4 of this project's JUnit classes at once +``` + +> Under the hood, per-project runs each suite as its own execution on a context built once +> for the fork: for JUnit Platform, one launcher execution per class on a single +> `LauncherSession` — so a session-scoped fixture (a booted application, a +> `LauncherSessionListener`) is set up once; for sbt frameworks, all the suites through one +> `Runner` with a single `done()`. This is automatic; you never configure it. See +> [Framework support](#framework-support-sbt-interface-vs-junit-platform) below. + +### `testFork: per-suite` + +Each suite runs in **its own forked JVM**, pooled by classpath. Suites are isolated by the +operating system: one that calls `System.exit`, wedges a thread, or corrupts a static kills +a process bleep can replace, without touching the others. This is also **how you run +sbt-interface suites concurrently** — a fork each, safely isolated. `maxConcurrentSuites` +then bounds how many of **this project's** forks run at once (unset = unbounded); the +machine-wide governor still caps the total across all projects. + +```yaml +flaky-tests: + isTestProject: true + testFork: per-suite # OS-level isolation, one suite can't poison another +``` + +```yaml +scalatest-test: + isTestProject: true + testFork: per-suite # concurrency for sbt suites: a fork each + maxConcurrentSuites: 4 # up to 4 forks at once +``` + +### Choosing between them + +| Reach for | When | +| --- | --- | +| `per-project` (default) | Normal unit tests; anything that boots an app or shares expensive fixtures (Testcontainers, an embedded DB). Fewer JVM starts, warm reuse, one live suite's heap at a time. | +| `per-project` + `maxConcurrentSuites` > 1 | JUnit Platform suites you want to run concurrently in that one JVM (that many classes at once). No effect on sbt frameworks. | +| `per-suite` | Suites that must not share a JVM (`System.exit`, thread leaks, static corruption), **or** sbt-interface suites you want to run concurrently. | + +### Framework support: sbt test interface vs JUnit Platform + +Bleep discovers and runs suites through two runner surfaces, and both execution modes +apply to both: + +* **sbt test interface** — ScalaTest, MUnit, utest, Specs2, ScalaCheck, weaver, ZIO Test, + and the rest of the Scala ecosystem. +* **JUnit Platform** — JUnit 5 and the engines built on it (Cucumber, Spek, jqwik, …). + +What per-project does for each, and why it's correct: + +| | per-suite | per-project | +| --- | --- | --- | +| **sbt interface** | one fork per suite | all the project's suites of one framework through **one `Runner`, one `done()`** in the shared fork — the interface's one-runner-per-framework contract, so stateful frameworks (munit, ZIO) behave. Always **sequential** — `maxConcurrentSuites` does not apply | +| **JUnit Platform** | one fork per suite | each class its own execution on **one shared `LauncherSession`** in the shared fork — session-scoped fixtures (a booted application, a `LauncherSessionListener`) build once, and per-class attribution is exact. `maxConcurrentSuites` runs that many classes at once | + +In per-project mode, `maxConcurrentSuites` raises concurrency **only for JUnit Platform**, +because JUnit's tests are the ones written for parallel execution; sbt frameworks share one +`Runner` and run one suite at a time. +To run sbt suites concurrently, use `testFork: per-suite` (a fork each). A framework that +reports a suite's failure detail only from its once-per-run `done()` can't share a `Runner` +at all; bleep gives such frameworks their own fork per suite automatically (weaver, hedgehog +today). And the machine-wide governor bounds the total across all projects, so `bleep test` +never oversubscribes your cores. + +## JVM options and heap + +Options for the forked test JVM come from three places, applied in order: + +1. the project's `platform.jvmOptions` (also picked up from maven's surefire `argLine` on import); +2. any fork options a sourcegen step declared for the project; +3. `--jvm-opt` on the command line. + +```bash +bleep test myapp-test --jvm-opt -Dmy.flag=true --jvm-opt -Duser.timezone=UTC +``` + +The per-fork heap defaults to a fixed size; change it once for the whole build with +`bleep config test-runner-heap ` (and `test-runner-heap-clear` to reset). A project +`-Xmx` in `platform.jvmOptions` still wins for that project's fork. + +## Selecting what runs + +```bash +bleep test --only FooSpec # suites whose name matches (substring, or FQDN to disambiguate) +bleep test --exclude SlowSpec # drop matching suites (takes precedence over --only) +bleep test --test-arg -oD # pass arguments straight to the framework +``` + +For **tag-based** selection across frameworks (`--only-tag`, `--exclude-tag`, and the +project pre-filter that skips compiling untagged projects), see [Test tags](/docs/usage/test-tags). + +## Reports and history + +* `--junit-report

` writes one JUnit XML per suite, for CI test-reporting UIs. +* Every run is recorded as a transcript; `bleep test --diff` shows only what changed + versus a previous run. See [Run history & diffs](/docs/usage/run-history). + +## Reference + +| Field | Type | Meaning | +| --- | --- | --- | +| `isTestProject` | `true` | Marks the project as tests; turns on suite discovery. | +| `testFork` | `per-project` \| `per-suite` | Fork granularity. Default `per-project`. | +| `maxConcurrentSuites` | integer | Ceiling on **this project's** suites at once (default `1`); the machine-wide governor caps the total across all projects. Per-project: JUnit classes inside the one fork (no effect on sbt frameworks — they stay sequential). Per-suite: number of forks (default unbounded). | +| `testFrameworks` | list | Escape hatch to name an **sbt test-interface** `Framework` class bleep doesn't auto-detect; normally unset. JUnit Platform engines are found via the platform SPI and cannot be named here. | +| `testTags` | map | Tag → suite class-name patterns for `--only-tag`/`--exclude-tag`. See [Test tags](/docs/usage/test-tags). | diff --git a/schema.json b/schema.json index ac1a70ac6..079f375c9 100644 --- a/schema.json +++ b/schema.json @@ -637,10 +637,21 @@ "platform": { "$ref": "#/$defs/Platform" }, + "maxConcurrentSuites": { + "description": "Ceiling on how many of this project's test suites run at once. Default 1 — suites run one at a time. Its reach depends on testFork: in per-project mode (the default) it only speeds up JUnit Platform suites — the JUnit engine runs that many of the project's JUnit classes at once inside the one shared fork, scheduled with its @ResourceLock/@Execution graph. sbt-interface frameworks (ScalaTest, MUnit, utest, Specs2, ScalaCheck, weaver, …) always run sequentially in per-project mode, so a value > 1 on an sbt-only project has no effect (bleep warns); use testFork: per-suite to run sbt suites concurrently, a fork each. In per-suite mode it bounds how many forks run at once and is unbounded by default. The machine-wide governor caps the total across all projects either way.", + "type": "integer", + "minimum": 1 + }, + "testFork": { + "description": "Where this project's test suites run relative to the JVM hosting them — the fork granularity. 'per-project' (the default) runs every suite in one forked JVM (maven's forkCount=1 reuseForks=true), so a booted application or shared fixture is reused across suites and suites run sequentially; raise maxConcurrentSuites to run JUnit Platform classes concurrently inside it (sbt frameworks stay sequential). 'per-suite' forks a JVM per suite for OS-level isolation, and is how you run sbt suites concurrently; maxConcurrentSuites then bounds how many forks run at once. Unset = per-project.", + "type": "string", + "enum": ["per-project", "per-suite"] + }, "isTestProject": { "type": "boolean" }, "testFrameworks": { + "description": "Escape hatch to name an sbt test-interface Framework class (implements sbt.testing.Framework) that bleep does not auto-detect; normally unset. The class is instantiated and its fingerprints drive discovery, so it must be a Framework, not a suite. JUnit Platform engines are NOT named here — they are found through the platform's own SPI whenever a JUnit runtime is on the classpath.", "oneOf": [ { "$ref": "#/$defs/TestFrameworkName" diff --git a/snapshot-tests/bloop/resolve-cache.json.gz b/snapshot-tests/bloop/resolve-cache.json.gz index 7861c53a4..ddbb6defa 100644 Binary files a/snapshot-tests/bloop/resolve-cache.json.gz and b/snapshot-tests/bloop/resolve-cache.json.gz differ diff --git a/snapshot-tests/converter/resolve-cache.json.gz b/snapshot-tests/converter/resolve-cache.json.gz index 36177603e..8f2aa3ab9 100644 Binary files a/snapshot-tests/converter/resolve-cache.json.gz and b/snapshot-tests/converter/resolve-cache.json.gz differ diff --git a/snapshot-tests/create-new-build/resolve-cache.json.gz b/snapshot-tests/create-new-build/resolve-cache.json.gz index f445db74f..db984d2c5 100644 Binary files a/snapshot-tests/create-new-build/resolve-cache.json.gz and b/snapshot-tests/create-new-build/resolve-cache.json.gz differ diff --git a/snapshot-tests/doobie/resolve-cache.json.gz b/snapshot-tests/doobie/resolve-cache.json.gz index a884b0606..e48523ae7 100644 Binary files a/snapshot-tests/doobie/resolve-cache.json.gz and b/snapshot-tests/doobie/resolve-cache.json.gz differ diff --git a/snapshot-tests/http4s/resolve-cache.json.gz b/snapshot-tests/http4s/resolve-cache.json.gz index f1875f1e5..4272e9ae2 100644 Binary files a/snapshot-tests/http4s/resolve-cache.json.gz and b/snapshot-tests/http4s/resolve-cache.json.gz differ diff --git a/snapshot-tests/sbt/resolve-cache.json.gz b/snapshot-tests/sbt/resolve-cache.json.gz index 6a44acca7..19a5914ac 100644 Binary files a/snapshot-tests/sbt/resolve-cache.json.gz and b/snapshot-tests/sbt/resolve-cache.json.gz differ diff --git a/snapshot-tests/scalameta/resolve-cache.json.gz b/snapshot-tests/scalameta/resolve-cache.json.gz index ef1ba8f14..2fb7b5b17 100644 Binary files a/snapshot-tests/scalameta/resolve-cache.json.gz and b/snapshot-tests/scalameta/resolve-cache.json.gz differ diff --git a/snapshot-tests/tapir/resolve-cache.json.gz b/snapshot-tests/tapir/resolve-cache.json.gz index c3731c195..94bf53162 100644 Binary files a/snapshot-tests/tapir/resolve-cache.json.gz and b/snapshot-tests/tapir/resolve-cache.json.gz differ