Export JVM metrics and capture a heap dump on OOM

CLAUDE.md

@@ -162,6 +162,11 @@ The `readingbat-kotest` module provides `TestSupport` with helpers:
   will not launch on `ubuntu-latest` without libgtk-4, gstreamer and friends, which the test workflow installs via
   `playwright install-deps webkit`. Chromium happens to run without that step, so a new WebKit spec is free locally on
   macOS and fails in CI until those deps are present.
+- **Never measure memory inside a test task.** Kover attaches the IntelliJ coverage agent to every `Test`
+  task, and its `ClassFinder` holds a strong reference to every classloader it sees. Script evaluation creates
+  a classloader per eval, so heap climbs about 1 MB per Kotlin eval under `./gradlew test` — and not at all in
+  production or `./gradlew run`. That phantom leak survived a full investigation before a GC-root trace showed
+  the agent holding it (#128). Disable Kover instrumentation for the task, or measure outside Gradle.
 
 ### Key Dependencies
 

Makefile

@@ -46,7 +46,7 @@ uberjar: ## Build the executable uberjar
 	./gradlew uberjar
 
 uber: uberjar ## Build and run the uberjar
-	java -jar build/libs/server.jar
+	java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=build -jar build/libs/server.jar
 
 run: ## Run the application
 	./gradlew run

README.md

@@ -215,6 +215,11 @@ Required environment variables for production:
 - `RESEND_API_KEY` (for email notifications)
 - `STATIC_URL_PREFIX` — *optional*; defaults to `/static`, serving images and icons from the jar.
   Set it to a CDN origin to serve them from there instead. The app serves them either way.
+- `JAVA_TOOL_OPTIONS` — *optional but recommended*:
+  `-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=<writable dir>`. The flags cost nothing until an
+  OOM and are the only way to learn what was holding memory when one happens. Point the path at a
+  volume that outlives the process — a container restart otherwise takes the dump with it. The
+  server logs at startup whether dumps are enabled.
 
 ## 🧪 Testing
 

gradle/libs.versions.toml

@@ -59,6 +59,7 @@ common-service-utils = { module = "com.pambrose.common-utils:service-utils", ver
 # Prometheus
 prometheus-proxy = { module = "com.pambrose:prometheus-proxy", version.ref = "prometheus-proxy" }
 simple-client = { module = "io.prometheus:simpleclient", version.ref = "prometheus" }
+simple-client-hotspot = { module = "io.prometheus:simpleclient_hotspot", version.ref = "prometheus" }
 
 # Script Engines
 java-scripting = { module = "ch.obermuhlner:java-scriptengine", version.ref = "java-scriptengine" }

readingbat-core/build.gradle.kts

@@ -12,6 +12,9 @@ description = "Ktor web server, DSL engine, and database layer for ReadingBat pr
 
 application {
   mainClass = "TestMain"
+  // Capture the evidence if a local run dies of an OOM: without a dump there is no record of what
+  // was holding memory.
+  applicationDefaultJvmArgs = listOf("-XX:+HeapDumpOnOutOfMemoryError", "-XX:HeapDumpPath=build")
 }
 
 dependencies {
@@ -26,6 +29,7 @@ dependencies {
   implementation(libs.bundles.exposed)
 
   implementation(libs.simple.client)
+  implementation(libs.simple.client.hotspot)
 
   runtimeOnly(libs.python.scripting)
   runtimeOnly(libs.kotlin.scripting)
@@ -114,3 +118,4 @@ if (isMac) {
 tasks.register("stage") {
   dependsOn(tasks.named("build"))
 }
+

readingbat-core/src/main/kotlin/com/readingbat/common/Metrics.kt

@@ -28,6 +28,7 @@ import com.readingbat.dsl.agentLaunchId
 import com.readingbat.server.GeoInfo.Companion.geoInfoMap
 import com.readingbat.server.Intercepts.requestTimingMap
 import com.readingbat.server.ws.ChallengeWs.answerWsConnections
+import io.prometheus.client.hotspot.DefaultExports
 import kotlin.time.Duration.Companion.hours
 import kotlin.time.Duration.Companion.minutes
 
@@ -190,6 +191,11 @@ class Metrics {
 
   /** Initializes sampler gauges that periodically report cache sizes and active session counts. */
   fun init(contentSource: () -> com.readingbat.dsl.ReadingBatContent) {
+    // JVM heap, GC, thread and class-loading collectors. Until these existed the only memory signal
+    // this server produced was the OOM itself. Heap that fails to fall back after a GC is the symptom
+    // to watch. DefaultExports guards against double registration internally.
+    DefaultExports.initialize()
+
     gauge {
       name("server_start_time_seconds")
       labelNames(AGENT_ID)

readingbat-core/src/main/kotlin/com/readingbat/server/ReadingBatServer.kt

@@ -60,6 +60,7 @@ import com.readingbat.server.routes.sysAdminRoutes
 import com.readingbat.server.routes.userRoutes
 import com.readingbat.server.ws.LoggingWs
 import com.readingbat.server.ws.WsCommon.wsRoutes
+import com.sun.management.HotSpotDiagnosticMXBean
 import com.zaxxer.hikari.HikariConfig
 import com.zaxxer.hikari.HikariDataSource
 import io.github.oshai.kotlinlogging.KotlinLogging
@@ -73,6 +74,7 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.launch
 import org.jetbrains.exposed.v1.jdbc.Database
+import java.lang.management.ManagementFactory
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
 import javax.script.ScriptEngineManager
@@ -274,6 +276,36 @@ internal fun runInitialContentLoad(load: () -> Unit): Boolean =
     }
     .isSuccess
 
+/**
+ * Reports whether the JVM will write a heap dump on `OutOfMemoryError`.
+ *
+ * A dump taken at the moment of an OOM is the only record of what was holding memory, and the flags
+ * cost nothing until then. Enable with
+ * `JAVA_TOOL_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/some/writable/dir"`, and
+ * make sure that directory survives the process, or a container restart takes the evidence with it.
+ *
+ * Logged rather than enforced: the flags are set by whoever launches the JVM, not by this code, so
+ * the most this can do is say what it sees.
+ */
+private fun logHeapDumpConfig() {
+  runCatching {
+    val bean = ManagementFactory.getPlatformMXBean(HotSpotDiagnosticMXBean::class.java)
+    val enabled = bean.getVMOption("HeapDumpOnOutOfMemoryError").value.toBoolean()
+    val path = bean.getVMOption("HeapDumpPath").value.ifBlank { "the working directory" }
+
+    if (enabled)
+      logger.info { "Heap dump on OutOfMemoryError is enabled, writing to: $path" }
+    else
+      logger.warn {
+        "Heap dump on OutOfMemoryError is disabled -- an OOM will leave no evidence behind. " +
+          "Enable with JAVA_TOOL_OPTIONS=\"-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=<dir>\""
+      }
+  }.onFailure {
+    // Not a HotSpot JVM, or the option is unavailable -- nothing to report.
+    logger.debug { "Unable to read heap dump configuration: ${it.message}" }
+  }
+}
+
 /**
  * Ktor application module entry point. Initializes HOCON properties, the database,
  * Prometheus agent, metrics, DSL content, plugin installations, request intercepts,
@@ -286,6 +318,8 @@ fun Application.module() {
   logger.info { "Loaded script engines: ${ScriptEngineManager().engineFactories.map { it.engineName }}" }
   check(ScriptEngineManager().engineFactories.count() == 3) { "Missing script engines" }
 
+  logHeapDumpConfig()
+
   adminUsersRef.store((ADMIN_USERS.configValueOrNull(this)?.getList() ?: emptyList()).toHashSet())
 
   if (isDbmsEnabled()) {

readingbat-core/src/test/kotlin/com/readingbat/common/JvmMetricsTest.kt

@@ -0,0 +1,56 @@
+/*
+ * Copyright © 2026 Paul Ambrose (pambrose@mac.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.readingbat.common
+
+import com.readingbat.TestData
+import com.readingbat.kotest.TestSupport.initTestProperties
+import com.readingbat.server.ReadingBatServer
+import io.kotest.core.spec.style.StringSpec
+import io.kotest.matchers.collections.shouldContainAll
+import io.prometheus.client.CollectorRegistry
+
+/**
+ * `Metrics.init` registers the JVM collectors, so heap, GC and thread state are scrapeable.
+ *
+ * Worth pinning: the server previously exported its own cache-size gauges but nothing about the
+ * JVM, which meant memory problems were invisible until the process died. That is the difference
+ * between watching heap fail to fall back after a GC and finding out from an OOM.
+ */
+class JvmMetricsTest : StringSpec() {
+  init {
+    "Metrics.init registers the JVM collectors" {
+      initTestProperties()
+      ReadingBatServer.metrics.init { TestData.readTestContent() }
+
+      val exported =
+        CollectorRegistry.defaultRegistry
+          .metricFamilySamples()
+          .asSequence()
+          .map { it.name }
+          .toSet()
+
+      exported shouldContainAll
+        [
+          "jvm_memory_bytes_used", // the heap gauge to alert on
+          "jvm_gc_collection_seconds",
+          "jvm_threads_current",
+          "jvm_classes_loaded",
+        ]
+    }
+  }
+}

website/readingbat-core/docs/configuration/index.md

@@ -120,6 +120,7 @@ precedence when defined:
 | `RESEND_SENDER_EMAIL` | Resend sender email address | No |
 | `IPGEOLOCATION_KEY` | IP geolocation API key | Yes |
 | `STATIC_URL_PREFIX` | URL prefix for static assets (default `/static`) | No |
+| `JAVA_TOOL_OPTIONS` | JVM flags; set `-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=<dir>` | No |
 | `AGENT_ENABLED` | Enable Prometheus proxy agent | No |
 | `REDIRECT_HOSTNAME` | Hostname for redirects | No |
 | `OAUTH_CALLBACK_URL_PREFIX` | OAuth callback URL prefix | No |