From b5e6ed6816bff82e6e57470bfcc897af3c590afc Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Wed, 29 Jul 2026 21:25:16 -0400 Subject: [PATCH 001/117] Rewrite pubnet parallel catchup v2 as Job-per-range Replaces the redis queue + long-lived worker pods with one Kubernetes Job per ledger range, driven by job_monitor, with a log_collector sidecar that streams each worker's log to a durable volume while the pod is still alive. Checkpoint commit on a work-in-progress branch, not a finished change. Monitor - Job per range; the monitor owns retries (backoffLimit 0) so disruption, OOM, disk eviction and timeout are classified separately and get their own attempt budgets. - Finished Jobs are deleted once their record is durable; TTL is only a backstop. 459 dead Jobs accumulated in 28 minutes at 2048 parallelism and every one inflated the reconcile LIST. - PVCs are released as each range completes; 2032 bound PVCs and 79 TiB had accumulated a third of the way through a 3982-range run. Profiling - Peaks come from kubelet /stats/summary, not Prometheus: same payload already fetched for ephemeral storage, ~10s vs a 30s scrape, and no dependency on Prometheus being up or still retaining the window. - Size memory from anon (rssBytes), never working set: page cache grows to fill whatever limit it is given, so memory.peak is always ~= the limit. Measured 862 MiB of anon reporting a 12704 MiB peak under a 24000 MiB limit. - Peaks are maxed across a contiguous chain of resumed attempts. A pod killed after replay starts resumes at LCL+1 and skips the download and bucket apply, where peak memory actually happens, so profiling only the winning attempt under-reports. A fresh retry ran new-db and supersedes everything before it. - cpu is no longer profiled; the request is fixed. Fixes - tx_apply regex missed scientific notation, silently dropping the metric for 91-99% of ranges above ledger 35M. - COLLECTOR_MAX_STREAMS is derived from worker.replicas. It caps the aiohttp connection pool with no semaphore above it, so a fixed 1200 against 2048 workers starved the excess indefinitely rather than queueing, and retries -- created last -- never got a slot. - A pod deleted while Running never became terminal, so its stream retried every 30s for the life of the run. Vanished pods are now reaped. - Two exit paths returned without finalizing, dropping tx_apply and peaks for any pod that outlived its object or whose last read threw. Known gaps - txApply and seconds are still tail-only for a resumed range; both need summing across the chain, and per-attempt duration is not yet persisted. - EPH_EVICT_JOB_CONDITION and EPH_EVICT_MESSAGE in the tests are reconstructions, not verbatim captures. Re-pin from a real eviction. 117 unit tests, each mutation-checked. Co-Authored-By: Claude Opus 5 --- .gitignore | 4 + src/App/Program.fs | 32 + src/FSLibrary.Tests/Tests.fs | 79 + .../MissionHistoryPubnetParallelCatchupV2.fs | 412 +++- src/FSLibrary/StellarKubeSpecs.fs | 14 +- src/FSLibrary/StellarMissionContext.fs | 4 + .../Dockerfile.jobmonitor | 27 +- src/MissionParallelCatchup/job_monitor.py | 1764 +++++++++++++++-- src/MissionParallelCatchup/log_collector.py | 600 ++++++ .../files/logarithmic_range_generator.sh | 56 - .../files/uniform_range_generator.sh | 45 - .../parallel_catchup_helm/files/worker.sh | 123 -- .../templates/catchup_workers.yaml | 167 -- .../templates/core_config.yaml | 20 + .../templates/job_monitor.yaml | 430 +++- .../templates/job_preload_redis.yaml | 64 - .../templates/redis_queue.yaml | 37 - .../parallel_catchup_helm/values.yaml | 163 +- .../test_job_monitor.py | 1521 ++++++++++++++ 19 files changed, 4736 insertions(+), 826 deletions(-) create mode 100644 src/MissionParallelCatchup/log_collector.py delete mode 100644 src/MissionParallelCatchup/parallel_catchup_helm/files/logarithmic_range_generator.sh delete mode 100644 src/MissionParallelCatchup/parallel_catchup_helm/files/uniform_range_generator.sh delete mode 100644 src/MissionParallelCatchup/parallel_catchup_helm/files/worker.sh delete mode 100644 src/MissionParallelCatchup/parallel_catchup_helm/templates/catchup_workers.yaml create mode 100644 src/MissionParallelCatchup/parallel_catchup_helm/templates/core_config.yaml delete mode 100644 src/MissionParallelCatchup/parallel_catchup_helm/templates/job_preload_redis.yaml delete mode 100644 src/MissionParallelCatchup/parallel_catchup_helm/templates/redis_queue.yaml create mode 100644 src/MissionParallelCatchup/test_job_monitor.py diff --git a/.gitignore b/.gitignore index 8a57e922..e807a167 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ .fake .ionide .idea + +# Python bytecode from the parallel-catchup monitor/collector tests +__pycache__/ +*.pyc diff --git a/src/App/Program.fs b/src/App/Program.fs index 0dd5125a..5f63c698 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -115,6 +115,10 @@ type MissionOptions pubnetParallelCatchupEndLedger: int option, pubnetParallelCatchupLedgersPerJob: int, pubnetParallelCatchupNumWorkers: int, + pubnetParallelCatchupStorageMode: string, + pubnetParallelCatchupProfile: string, + pubnetParallelCatchupRangeOrder: string, + pubnetParallelCatchupCpuRequest: string, tag: string option, numPregeneratedTxs: int option, genesisTestAccountCount: int option, @@ -521,6 +525,30 @@ type MissionOptions Default = 192)>] member self.PubnetParallelCatchupNumWorkers = pubnetParallelCatchupNumWorkers + [] + member self.PubnetParallelCatchupStorageMode : string = pubnetParallelCatchupStorageMode + + [] + member self.PubnetParallelCatchupProfile : string = pubnetParallelCatchupProfile + + [] + member self.PubnetParallelCatchupRangeOrder : string = pubnetParallelCatchupRangeOrder + + [] + member self.PubnetParallelCatchupCpuRequest : string = pubnetParallelCatchupCpuRequest + [] member self.Tag = tag @@ -898,6 +926,10 @@ let main argv = pubnetParallelCatchupEndLedger = mission.PubnetParallelCatchupEndLedger pubnetParallelCatchupLedgersPerJob = mission.PubnetParallelCatchupLedgersPerJob pubnetParallelCatchupNumWorkers = mission.PubnetParallelCatchupNumWorkers + pubnetParallelCatchupStorageMode = mission.PubnetParallelCatchupStorageMode + pubnetParallelCatchupProfile = mission.PubnetParallelCatchupProfile + pubnetParallelCatchupRangeOrder = mission.PubnetParallelCatchupRangeOrder + pubnetParallelCatchupCpuRequest = mission.PubnetParallelCatchupCpuRequest tag = mission.Tag numPregeneratedTxs = mission.NumPregeneratedTxs enableTailLogging = true diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index 7a3c4f73..437dc56e 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -13,6 +13,8 @@ open StellarKubeSpecs open StellarNetworkData open StellarNetworkDelays open MissionCatchupHelpers +open MissionHistoryPubnetParallelCatchupV2 +open Newtonsoft.Json.Linq open Xunit.Abstractions @@ -119,6 +121,10 @@ let ctx : MissionContext = pubnetParallelCatchupEndLedger = None pubnetParallelCatchupLedgersPerJob = 16000 pubnetParallelCatchupNumWorkers = 192 + pubnetParallelCatchupStorageMode = "pvc" + pubnetParallelCatchupProfile = "" + pubnetParallelCatchupRangeOrder = "tip-first" + pubnetParallelCatchupCpuRequest = "" tag = None numPregeneratedTxs = None enableTailLogging = true @@ -542,3 +548,76 @@ type Tests(output: ITestOutputHelper) = Assert.Equal("51/6", jobArr3.[0].[1]) Assert.Equal("56/6", jobArr3.[1].[1]) Assert.Equal("61/6", jobArr3.[2].[1]) + + +[] +let ``range profile keeps only the measurements that exist`` () = + // The consumer falls back to its configured default when a field is + // absent, so a missing measurement must stay missing rather than become a + // null. peakEphemeralBytes is recorded only for ephemeral-mode runs that + // finished, so most records will not carry it. + let record = JObject() + record.["peakWorkingSetBytes"] <- JValue(1234L) + record.["peakCpuCores"] <- JValue(0.5) + record.["seconds"] <- JValue(42) + + let entry = projectRangeEntry record + + Assert.Equal(3, entry.Count) + Assert.Equal(1234L, entry.["peakWorkingSetBytes"].Value()) + Assert.Null(entry.["peakEphemeralBytes"]) + + record.["peakEphemeralBytes"] <- JValue(9999L) + let withEph = projectRangeEntry record + Assert.Equal(4, withEph.Count) + Assert.Equal(9999L, withEph.["peakEphemeralBytes"].Value()) + + +[] +let ``range profile keeps count as a field so it can be keyed on end alone`` () = + // Measured: 4.2x the ledgers per range moved peak disk -1.6% and wall time + // 1.15x, so cost tracks ledger position rather than range length. Keying on + // end/count would discard the whole profile whenever overlapLedgers or + // ledgersPerJob changed, for a distinction the measurements say is small. + Assert.DoesNotContain("count", rangeProfileFields) + + let record = JObject() + record.["peakWorkingSetBytes"] <- JValue(1L) + record.["count"] <- JValue(420) + // projectRangeEntry itself must not copy count -- writeRangeProfile attaches + // it separately, so a stale profile cannot smuggle it in as a measurement. + let entry = projectRangeEntry record + Assert.Null(entry.["count"]) + + +[] +let ``range profile does not carry a pvc volume peak`` () = + // A PVC's size is not a scheduling dimension, so growing it buys no + // packing and it is deliberately not profiled. + Assert.DoesNotContain("peakVolumeBytes", rangeProfileFields) + + +[] +let ``pvc mode does not reserve node disk it never uses`` () = + // /data is on the volume in pvc mode; the node disk only holds logs and tmp. + // Asking for the ephemeral-mode figure makes disk rather than cpu the + // binding dimension and cuts pods per node. + let src = + System.IO.File.ReadAllText( + "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + Assert.Contains("pubnetParallelCatchupStorageMode = \"pvc\"", src) + Assert.Contains("\"2Gi\", \"4Gi\"", src) + + +[] +let ``progress record is read from the volume before the configmap`` () = + // The ConfigMap is a 1 MiB-capped mirror (~6100 ranges); /logs/progress.json + // is authoritative and unbounded. Reading the mirror would silently + // truncate the artifact on a finer slicing. + let src = + System.IO.File.ReadAllText( + "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + let vol = src.IndexOf("/logs/progress.json") + let cm = src.IndexOf("queryJobMonitor (context, jobMonitorProgressKey)") + Assert.True(vol > 0, "must read the volume copy") + Assert.True(vol < cm, "volume read must precede the ConfigMap fallback") diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 001981ea..333ac9c5 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -26,7 +26,12 @@ open k8s open CSLibrary // Constants -let helmChartPath = "/supercluster/src/MissionParallelCatchup/parallel_catchup_helm" +// Baked into the supercluster image at this path. Overridable so a local run +// can point at a working copy without editing this file. +let helmChartPath = + match Environment.GetEnvironmentVariable("SUPERCLUSTER_CHART_PATH") with + | null | "" -> "/supercluster/src/MissionParallelCatchup/parallel_catchup_helm" + | p -> p // Comment out the path below for local testing // Example command to run local testing (in the `supercluster/` directory): @@ -34,10 +39,11 @@ let helmChartPath = "/supercluster/src/MissionParallelCatchup/parallel_catchup_h // let helmChartPath = "src/MissionParallelCatchup/parallel_catchup_helm" let valuesFilePath = helmChartPath + "/values.yaml" -let defaultJobMonitorHostName = "ssc-job-monitor-eks.services.stellar-ops.com" -let jobMonitorStatusEndPoint = "/status" -let jobMonitorMetricsEndPoint = "/metrics" -let jobMonitorLoggingIntervalSecs = 30 // frequency of job monitor's internal information gathering (querying core endpoint and redis metrics) and logging +// Keys in the -catchup-progress ConfigMap. These were HTTP paths when +// the driver polled the monitor through a Gateway; it reads the ConfigMap now. +let jobMonitorStatusKey = "status.json" // live queue counts +let jobMonitorProgressKey = "progress.json" // durable per-range completion record +let jobMonitorLoggingIntervalSecs = 30 // frequency of the monitor reconcile loop: dispatch, liveness ping, status publish let jobMonitorStatusCheckIntervalSecs = 60 // frequency of us querying job monitor's `/status` end point let jobMonitorMetricsCheckIntervalSecs = 60 // frequency of us querying job monitor's `/metrics` end point let jobMonitorStatusCheckTimeOutSecs = 600 @@ -48,10 +54,66 @@ let failedJobLogStreamLineCount = 1000 let mutable nonce : String = "" let mutable helmReleaseName : String = "" -let jobMonitorHostName (context: MissionContext) = - match context.jobMonitorExternalHost with - | Some host -> host - | None -> defaultJobMonitorHostName // TODO: append it with a nounce to make it session specific +// Resolve --pubnet-parallel-catchup-profile into a ConfigMap the monitor mounts. +// +// Accepts a local path or an https URL, so a profile can come off disk or +// straight from a raw paste/gist link. Returns the ConfigMap name, or None to +// size from the configured requests. +// +// Never fatal: a profile only tightens requests, so a run must still start when +// one cannot be fetched. Failing the mission here would turn an optimisation +// into a dependency. +let resolveRangeProfile (context: MissionContext) : string option = + let spec = context.pubnetParallelCatchupProfile + + if String.IsNullOrWhiteSpace spec then + None + else + // The options are built before the helm install sets this, and the + // ConfigMap has to land in the same cluster and namespace the release + // will use. + Environment.SetEnvironmentVariable("KUBECONFIG", ExpandHomeDirTilde context.kubeCfg) + + try + let body = + if spec.StartsWith("https://", StringComparison.OrdinalIgnoreCase) then + use client = new HttpClient() + client.Timeout <- TimeSpan.FromSeconds(30.0) + client.GetStringAsync(spec) |> Async.AwaitTask |> Async.RunSynchronously + else + File.ReadAllText(ExpandHomeDirTilde spec) + + // Parse before shipping it: a 404 page or a truncated download would + // otherwise reach the monitor as an unreadable mount. + let doc = JObject.Parse(body) + let count = + match doc.["ranges"] with + | :? JObject as r -> r.Count + | _ -> 0 + + if count = 0 then + LogWarn "Range profile %s has no ranges; sizing from configured requests" spec + None + else + let name = sprintf "%s-range-profile" helmReleaseName + let file = Path.Combine(Path.GetTempPath(), sprintf "%s-profile.json" helmReleaseName) + File.WriteAllText(file, body) + + RunShellCommand [| "kubectl" + "create" + "configmap" + name + sprintf "--from-file=profile.json=%s" file |] + |> ignore + + LogInfo "Range profile: %d ranges from %s -> configmap %s" count spec name + Some name + with ex -> + LogWarn "Could not load range profile %s (%s); sizing from configured requests" + spec + ex.Message + None + // Helper functions to convert label/taint tuples to Helm-compatible format using indexed notation let requireNodeLabelToHelmIndexed (index: int) ((key: string), (value: string option)) = @@ -94,20 +156,29 @@ let installProject (context: MissionContext) = setOptions.Add(sprintf "worker.stellar_core_image=%s" context.image) setOptions.Add(sprintf "worker.replicas=%d" context.pubnetParallelCatchupNumWorkers) - // Set Redis hostname to be unique per release - setOptions.Add(sprintf "redis.hostname=%s-redis" nonce) + // pvc: /data survives the pod so an evicted range resumes at L+1 (what makes + // spot viable). ephemeral: /data is an emptyDir on the node -- denser + // packing, no resume. The ephemeral-storage request below must match: + // ~2Gi for pvc, ~35Gi for ephemeral, or the monitor logs a loud mismatch. + setOptions.Add(sprintf "worker.storageMode=%s" context.pubnetParallelCatchupStorageMode) + + match resolveRangeProfile context with + | Some cm -> setOptions.Add(sprintf "monitor.profileConfigMap=%s" cm) + | None -> () + + setOptions.Add(sprintf "range.order=%s" context.pubnetParallelCatchupRangeOrder) - setOptions.Add(sprintf "range_generator.params.starting_ledger=%d" context.pubnetParallelCatchupStartingLedger) + setOptions.Add(sprintf "range.startingLedger=%d" context.pubnetParallelCatchupStartingLedger) let endLedger = match context.pubnetParallelCatchupEndLedger with | Some value -> value | None -> GetLatestPubnetLedgerNumber() - setOptions.Add(sprintf "range_generator.params.latest_ledger_num=%d" endLedger) + setOptions.Add(sprintf "range.latestLedgerNum=%d" endLedger) setOptions.Add( - sprintf "range_generator.params.uniform_ledgers_per_job=%d" context.pubnetParallelCatchupLedgersPerJob + sprintf "range.ledgersPerJob=%d" context.pubnetParallelCatchupLedgersPerJob ) // Skip known results by default @@ -129,8 +200,16 @@ let installProject (context: MissionContext) = let memReqMebi = resourceRequirements.Requests.["memory"].ToString() let cpuLimMili = resourceRequirements.Limits.["cpu"].ToString() let memLimMebi = resourceRequirements.Limits.["memory"].ToString() - let storageReqGibi = resourceRequirements.Requests.["ephemeral-storage"].ToString() - let storageLimGibi = resourceRequirements.Limits.["ephemeral-storage"].ToString() + // StellarKubeSpecs sizes ephemeral-storage for ephemeral mode, where /data is + // an emptyDir on the node. In pvc mode /data is on the volume and the node + // disk only holds logs and tmp, so asking for the full amount reserves disk + // nothing uses -- and makes disk, not cpu, the binding dimension for packing. + let storageReqGibi, storageLimGibi = + if context.pubnetParallelCatchupStorageMode = "pvc" then + "2Gi", "4Gi" + else + resourceRequirements.Requests.["ephemeral-storage"].ToString(), + resourceRequirements.Limits.["ephemeral-storage"].ToString() LogInfo "Resource requirements from StellarKubeCfg:\n\ @@ -147,7 +226,14 @@ let installProject (context: MissionContext) = storageReqGibi storageLimGibi - setOptions.Add(sprintf "worker.resources.requests.cpu=%s" cpuReqMili) + // An explicit override applies to profiled and unprofiled ranges alike: the + // monitor clamps any profile-derived cpu request to REQ_CPU, so this is the + // ceiling as well as the default. + let cpuReqEffective = + if String.IsNullOrWhiteSpace context.pubnetParallelCatchupCpuRequest then cpuReqMili + else context.pubnetParallelCatchupCpuRequest + + setOptions.Add(sprintf "worker.resources.requests.cpu=%s" cpuReqEffective) setOptions.Add(sprintf "worker.resources.requests.memory=%s" memReqMebi) setOptions.Add(sprintf "worker.resources.limits.cpu=%s" cpuLimMili) setOptions.Add(sprintf "worker.resources.limits.memory=%s" memLimMebi) @@ -169,13 +255,15 @@ let installProject (context: MissionContext) = | Some mirrorUrl -> [ 1 .. 3 ] |> List.iter (setS3HistoryGetCommand mirrorUrl) | None -> () - setOptions.Add(sprintf "monitor.hostname=%s" (jobMonitorHostName context)) - setOptions.Add(sprintf "monitor.path_prefix=/%s/%s" context.namespaceProperty helmReleaseName) - setOptions.Add(sprintf "monitor.logging_interval_seconds=%d" jobMonitorLoggingIntervalSecs) - // Attach the job-monitor HTTPRoute to the same Gateway as the core route - // (--gateway-name/--gateway-namespace), instead of the values.yaml defaults. - setOptions.Add(sprintf "monitor.gateway_name=%s" context.gatewayName) - setOptions.Add(sprintf "monitor.gateway_namespace=%s" context.gatewayNamespace) + setOptions.Add(sprintf "monitor.loggingIntervalSeconds=%d" jobMonitorLoggingIntervalSecs) + + // Every other mission gets this label from StellarKubeSpecs (Map.add + // "mission" missionName); parallel catchup builds its pods from a helm + // chart instead, so the old worker StatefulSet carried no mission label at + // all. kube-state-metrics turns it into label_mission, which every + // container-level Grafana panel joins on -- which is why this mission has + // never appeared in the dashboard's mission list. + setOptions.Add(sprintf "monitor.mission=%s" context.missionName) // Set ASAN_OPTIONS if provided match context.asanOptions with @@ -242,32 +330,50 @@ let installProject (context: MissionContext) = // 2. For each pod, finds all files matching "stellar-core-*.log" in /data // 3. Creates a tar.gz archive and copies it to context.destination directory let collectLogsFromPods (context: MissionContext) = - // Generate pod names based on number of workers - // Pod names follow the pattern: -stellar-core-0, -stellar-core-1, etc. - let podNames = - [ 0 .. context.pubnetParallelCatchupNumWorkers - 1 ] - |> List.map (fun i -> sprintf "%s-stellar-core-%d" helmReleaseName i) - - LogInfo "Collecting logs from %d worker pods to directory: %s" (List.length podNames) context.destination.Path - - for podName in podNames do + // Worker pods are per-range now and are reaped within about a minute of + // finishing, so there is nothing left to exec into at teardown. The monitor + // pulls each pod's log while it is still alive -- on failure before the + // retry, on success before the Job's TTL -- onto its own volume, so one + // exec here replaces the ~1024 that the StatefulSet design needed. + let monitorPods = + context.kube + .ListNamespacedPod( + context.namespaceProperty, + labelSelector = sprintf "app=job-monitor,release=%s" helmReleaseName + ) + .Items + |> Seq.map (fun p -> p.Metadata.Name) + |> List.ofSeq + + match monitorPods with + | [] -> + LogWarn + "No job-monitor pod found for release %s; worker logs cannot be collected" + helmReleaseName + | podName :: _ -> try - LogInfo "Collecting logs from pod: %s" podName - - // Build the tar command to archive log files - // The command tars all stellar-core-*.log files in /data - // Using `-f -` to write the file contents to stdout - let command = [| "sh"; "-c"; "cd /data && tar -czf - stellar-core-*.log" |] + LogInfo "Collecting worker logs from job-monitor pod %s to %s" podName context.destination.Path + + let outputFile = + Path.Combine(context.destination.Path, sprintf "%s-worker-logs.tar" helmReleaseName) + + // Entries are already gzipped by the streaming collector, so this + // bundles without re-compressing. Named range--a.log.gz, + // so a failing range is findable directly rather than by worker ordinal. + // Already gzipped by the collector, so no -z. Keeps the per-attempt + // .outcome verdicts (outcome/exitCode/pod -- useful post-mortem) and + // drops .state, which is only the collector's resume bookkeeping. + let command = + [| "sh" + "-c" + // lost+found is the ext4 root of the logs PVC, not ours. + "cd /logs && tar -cf - --exclude='*.state' --exclude='./lost+found' ." |] - // Output file path for this pod's logs - let outputFile = Path.Combine(context.destination.Path, sprintf "%s-logs.tar.gz" podName) - - // Execute the command and capture the tar output to a local file RemoteCommandRunner.RunRemoteCommandAndCaptureOutput( kube = context.kube, ns = context.namespaceProperty, podName = podName, - containerName = "stellar-core", + containerName = "job-monitor", command = command, outputFilePath = outputFile ) @@ -275,12 +381,12 @@ let collectLogsFromPods (context: MissionContext) = let fileInfo = FileInfo(outputFile) if fileInfo.Exists && fileInfo.Length > 0L then - LogInfo "Successfully collected logs from %s to %s (size: %d bytes)" podName outputFile fileInfo.Length + LogInfo "Collected worker logs to %s (size: %d bytes)" outputFile fileInfo.Length else - LogWarn "No logs found or empty archive for pod %s" podName + LogWarn "Worker log archive is empty: %s" outputFile with ex -> - LogWarn "Could not collect logs from pod %s (this is expected if pod doesn't exist): %s" podName ex.Message + LogWarn "Could not collect worker logs from %s: %s" podName ex.Message // Cleanup on exit. `signalTriggered` indicates we're running under a hard // deadline (Jenkins' SoftKillWaitSeconds, ~5s by default, before SIGKILL). @@ -288,10 +394,201 @@ let collectLogsFromPods (context: MissionContext) = // of the much-slower log collection — otherwise we get SIGKILLed mid- // collection and leak every worker pod, which is what we saw in practice // with a 1024-worker run aborted from Jenkins. +let queryJobMonitor (context: MissionContext, key: String) = + // The monitor publishes the same JSON it serves on /status into + // -catchup-progress. Reading it through the kube API removes the + // Gateway/HTTPRoute dependency entirely -- the driver already has a client. + try + let cm = + context.kube.ReadNamespacedConfigMap(helmReleaseName + "-catchup-progress", context.namespaceProperty) + + match cm.Data.TryGetValue key with + | true, body -> + LogInfo "job monitor status from configmap key '%s': %s" key body + Some(JObject.Parse(body)) + | _ -> + LogInfo "job monitor configmap has no '%s' yet" key + None + with ex -> + LogError "Error reading job monitor configmap: %s" ex.Message + None + + +// Emit what this run measured, next to the worker-log tar, so a later run can +// be given tighter per-range requests. An artifact rather than a ConfigMap or +// an S3 object: nothing for ArgoCD to reconcile, no second writer racing a +// concurrent mission, and not bounded by Prometheus retention. +// Fields carried from the monitor's progress record into the profile artifact. +// A PVC's size is absent on purpose -- it is not a scheduling dimension, so +// profiling it buys no packing. peakEphemeralBytes appears only for +// ephemeral-mode runs, and only for ranges that finished. +let rangeProfileFields = + [ "peakRssBytes"; "peakWorkingSetBytes"; "peakCpuCores"; "peakEphemeralBytes" + "seconds"; "txApply" ] + +// A missing measurement must stay missing rather than become a null: the +// consumer falls back to its configured default when the field is absent. +let projectRangeEntry (record: JObject) : JObject = + let entry = JObject() + + for field in rangeProfileFields do + match record.[field] with + | null -> () + | v -> entry.[field] <- v + + entry + + +// The progress record, preferring the copy on the monitor's volume. +// +// The ConfigMap is only a mirror and is capped at 1 MiB -- about 6100 ranges at +// ~172 bytes each, reachable simply by halving ledgersPerJob. Past that the +// mirror stops updating while /logs/progress.json stays correct, so reading the +// ConfigMap would silently truncate the artifact. +let readProgressRecord (context: MissionContext) : JObject option = + let monitorPods = + context.kube + .ListNamespacedPod( + context.namespaceProperty, + labelSelector = sprintf "app=job-monitor,release=%s" helmReleaseName + ) + .Items + |> Seq.map (fun p -> p.Metadata.Name) + |> List.ofSeq + + let fromVolume = + match monitorPods with + | [] -> None + | podName :: _ -> + try + let tmp = Path.Combine(Path.GetTempPath(), sprintf "%s-progress.json" helmReleaseName) + + RemoteCommandRunner.RunRemoteCommandAndCaptureOutput( + kube = context.kube, + ns = context.namespaceProperty, + podName = podName, + containerName = "job-monitor", + command = [| "cat"; "/logs/progress.json" |], + outputFilePath = tmp + ) + + let fi = FileInfo(tmp) + + if fi.Exists && fi.Length > 0L then + LogInfo "Progress record read from the monitor volume (%d bytes)" fi.Length + Some(JObject.Parse(File.ReadAllText(tmp))) + else + None + with ex -> + LogWarn "Could not read /logs/progress.json (%s); falling back to the ConfigMap" ex.Message + None + + match fromVolume with + | Some p -> Some p + | None -> queryJobMonitor (context, jobMonitorProgressKey) + + +let writeRangeProfile (context: MissionContext) = + match readProgressRecord context with + | None -> LogInfo "No progress record to build a range profile from" + | Some progress -> + try + let completed = progress.["completed"] :?> JObject + let ranges = JObject() + + // Keyed on the range end alone, with count kept as a field. + // + // Measured on ssc-test: 4.2x the ledgers per range (100 -> 420, the + // default 320 overlap) moved peak disk by -1.6% and wall time by + // 1.15x. Cost tracks ledger position -- how big the bucket set is to + // download and apply -- far more than range length. Putting count in + // the key would therefore discard the whole profile whenever + // overlapLedgers or ledgersPerJob changed, to preserve a distinction + // the measurements say is small. + // + // A consumer should resolve a range by exact end, else the nearest + // measured end, else a run-wide fallback -- so a re-sliced run still + // gets useful numbers instead of zero matches. + for prop in completed.Properties() do + let record = prop.Value :?> JObject + let entry = projectRangeEntry record + + match record.["count"] with + | null -> () + | v -> entry.["count"] <- v + + if entry.Count > 0 then + // Same end from two differently-sized ranges: keep the + // larger, since sizing from the smaller would under-provision. + let existing = ranges.[prop.Name] + + let keep = + isNull existing + || (let a = entry.["count"] + let b = (existing :?> JObject).["count"] + isNull b || (not (isNull a) && a.Value() >= b.Value())) + + if keep then ranges.[prop.Name] <- entry + + // Slicing and storage mode both go in the name, because a profile is + // only valid for the shape it was measured at. + // + // Slicing: keys are range ends and cost tracks range length, so a + // 39382-ledger profile fed into a 16320-ledger run resolves through + // nearest-end fallback and sizes everything wrong. Measured on + // ssc-test -- it produced 1025 OOM retries in one run. + // + // Mode: an ephemeral profile carries peakEphemeralBytes and a pvc one + // does not, so crossing them silently defaults the disk axis. + let ledgersPerRange = + let counts = + ranges.Properties() + |> Seq.choose (fun p -> + match (p.Value :?> JObject).["count"] with + | null -> None + | v -> Some(v.Value())) + |> Seq.toList + + match counts with + | [] -> context.pubnetParallelCatchupLedgersPerJob + | _ -> counts |> List.countBy id |> List.maxBy snd |> fst + + + let doc = JObject() + doc.["schema"] <- JValue(1) + doc.["generated"] <- JValue(DateTime.UtcNow.ToString("o")) + doc.["release"] <- JValue(helmReleaseName) + doc.["storageMode"] <- JValue(context.pubnetParallelCatchupStorageMode) + doc.["ledgersPerRange"] <- JValue(ledgersPerRange) + doc.["ranges"] <- ranges + + let path = + Path.Combine( + context.destination.Path, + sprintf + "%s-profile-%dledgers-%s.json" + helmReleaseName + ledgersPerRange + context.pubnetParallelCatchupStorageMode) + + File.WriteAllText(path, doc.ToString()) + LogInfo "Wrote range profile for %d ranges to %s" ranges.Count path + with ex -> LogWarn "Failed to write range profile: %s" ex.Message + + let cleanup (signalTriggered: bool) (context: MissionContext) = if toPerformCleanup then toPerformCleanup <- false + // Before either branch: `helm uninstall` deletes the progress ConfigMap + // the profile is built from, so an aborted run would otherwise lose every + // measurement it had already taken. One ConfigMap read and a local file + // write -- cheap enough for the abort path's few seconds, and a run + // stopped part-way is exactly when the partial profile is most wanted. + try + writeRangeProfile context + with ex -> LogWarn "Failed to write range profile: %s" ex.Message + if signalTriggered then // Abort path: resources first, logs are nice-to-have. // Skip log collection entirely — even parallelized it can't beat @@ -344,20 +641,6 @@ Console.CancelKeyPress.Add Environment.Exit(0)) -let queryJobMonitor (context: MissionContext, path: String, endPoint: String) = - try - use client = new HttpClient() - let url = "http://" + jobMonitorHostName context + path + endPoint - let response = client.GetStringAsync(url).Result - - LogInfo "job monitor query '%s', got response: %s" url response - let json = JObject.Parse(response) - Some(json) - with ex -> - LogError "Error querying job monitor '%s': %s" endPoint ex.Message - None - - let dumpLogs (context: MissionContext, podName: String) = let stream = context.kube.ReadNamespacedPodLog( @@ -397,11 +680,10 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = let mutable allJobsFinished = false let mutable timeoutLeft = jobMonitorStatusCheckTimeOutSecs let mutable timeBeforeNextMetricsCheck = jobMonitorMetricsCheckIntervalSecs - let jobMonitorPath = "/" + context.namespaceProperty + "/" + helmReleaseName while not allJobsFinished do Thread.Sleep(jobMonitorStatusCheckIntervalSecs * 1000) - let statusOpt = queryJobMonitor (context, jobMonitorPath, jobMonitorStatusEndPoint) + let statusOpt = queryJobMonitor (context, jobMonitorStatusKey) try match statusOpt with @@ -426,7 +708,7 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = if remainSize = 0 && JobsInProgress.Count = 0 then // All jobs completed — perform a final query on the metrics - queryJobMonitor (context, jobMonitorPath, jobMonitorMetricsEndPoint) |> ignore + queryJobMonitor (context, jobMonitorProgressKey) |> ignore LogInfo "All queues empty. Mission complete." allJobsFinished <- true @@ -434,7 +716,7 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = timeBeforeNextMetricsCheck <- timeBeforeNextMetricsCheck - jobMonitorStatusCheckIntervalSecs if timeBeforeNextMetricsCheck <= 0 then - queryJobMonitor (context, jobMonitorPath, jobMonitorMetricsEndPoint) |> ignore + queryJobMonitor (context, jobMonitorProgressKey) |> ignore timeBeforeNextMetricsCheck <- jobMonitorMetricsCheckIntervalSecs | None -> diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index 2e16857a..962c7c11 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -126,9 +126,17 @@ let SimulatePubnetTier1PerfCoreResourceRequirements : V1ResourceRequirements = makeResourceRequirements 500 128 4000 6000 let ParallelCatchupCoreResourceRequirements : V1ResourceRequirements = - // When doing parallel catchup, we give each container - // 0.25 vCPUs, 8GB RAM and 35 GB of disk bursting to 2vCPU, 24000MB and 40 GB - makeResourceRequirementsWithStorageLimit 250 8192 35 2000 24000 40 + // 1.8 vCPU, 9GiB RAM and 35 GB of disk, bursting to 2 vCPU, 24000MB and 40 GB. + // + // The requests are picked so the scheduler lands a specific worker count on + // each node shape we run on: cpu binds where memory is plentiful, and memory + // binds where cpu is. + // r8*.xlarge (3.92 cpu / 29.7Gi alloc) -> 2 workers, cpu-bound + // m8*.2xlarge (7.91 cpu / 29.7Gi alloc) -> 3 workers, memory-bound + // r8*.2xlarge (7.91 cpu / 61.7Gi alloc) -> 4 workers, cpu-bound + // The counts hold for allocatable cpu in [7.2,9.0) and memory in [27,36)Gi, so + // kubelet-reservation differences between instance types cannot flip them. + makeResourceRequirementsWithStorageLimit 1800 9216 35 2000 24000 40 let NonParallelCatchupCoreResourceRequirements : V1ResourceRequirements = // When doing non-parallel catchup, we give each container diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 828048ff..115303c1 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -119,6 +119,10 @@ type MissionContext = pubnetParallelCatchupEndLedger: int option pubnetParallelCatchupLedgersPerJob: int pubnetParallelCatchupNumWorkers: int + pubnetParallelCatchupStorageMode: string + pubnetParallelCatchupProfile: string + pubnetParallelCatchupRangeOrder: string + pubnetParallelCatchupCpuRequest: string genesisTestAccountCount: int option asanOptions: string option diff --git a/src/MissionParallelCatchup/Dockerfile.jobmonitor b/src/MissionParallelCatchup/Dockerfile.jobmonitor index 9244ba01..7243c3ee 100644 --- a/src/MissionParallelCatchup/Dockerfile.jobmonitor +++ b/src/MissionParallelCatchup/Dockerfile.jobmonitor @@ -1,20 +1,27 @@ -FROM --platform=linux/amd64 ubuntu:24.04 +FROM --platform=linux/amd64 python:3.12-slim VOLUME /data WORKDIR /app -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - python3 \ - python3-redis \ - python3-requests \ - python3-prometheus-client && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* +# Client major tracks a cluster minor (35.x <-> Kubernetes 1.35) and supports +# one minor of skew either way, so 35 covers 1.34 through 1.36 -- the cluster +# today and both upgrades already released. Revisit at 1.37. +# +# The floor matters independently: podFailurePolicy (V1PodFailurePolicy*) needs +# >=26 and V1VolumeResourceRequirements needs >=29. Ubuntu's distro package is +# 22.6.0 and has neither, so it cannot express this mission's failure +# classification at all. +RUN pip install --no-cache-dir \ + 'kubernetes~=35.0' \ + 'aiohttp~=3.9' \ + 'requests~=2.31' \ + 'prometheus-client~=0.19' COPY ./job_monitor.py /app +# Same image, second entrypoint: runs as a sidecar streaming worker logs. +COPY ./log_collector.py /app EXPOSE 8080 -CMD ["/usr/bin/python3", "job_monitor.py"] +CMD ["python3", "job_monitor.py"] diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 3778eeb2..fb53cb05 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1,34 +1,239 @@ -import os -import redis -import requests +"""Parallel catchup job monitor. + +Owns dispatch as well as reporting. Redis, worker.sh and the range-generator +scripts are gone; a Kubernetes Job per ledger range replaces them. + +State model -- the controller itself keeps nothing authoritative in memory: + + desired computed from config by a pure function (uniform | logarithmic) + completed durable, in a ConfigMap -- Jobs are reclaimed during a long run, + so their absence must NOT be read as "never ran" + in-flight live Jobs, by label selector + +A restart recomputes all three and carries on. The single-writer property (one +replica, Recreate) is what removes the claim/requeue races the redis queue had: +work is *assigned*, never claimed. +""" + +import asyncio +import gzip +import bisect import json -import sys import logging +import os +import re +import sys import threading import time -from http.server import BaseHTTPRequestHandler, HTTPServer -from prometheus_client import Gauge, Counter, Histogram, generate_latest, REGISTRY, CONTENT_TYPE_LATEST from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, HTTPServer -# Configuration +import aiohttp +from kubernetes import client, config +from kubernetes.client.rest import ApiException +from prometheus_client import (CONTENT_TYPE_LATEST, REGISTRY, Counter, Gauge, + Histogram, generate_latest) # Histogram buckets # 5m 15m 30m 1h 1.5h 2h metric_buckets = (300, 900, 1800, 3600, 5400, 7200, float("inf")) -REDIS_HOST = os.getenv('REDIS_HOST', 'redis') -REDIS_PORT = int(os.getenv('REDIS_PORT', '6379')) -JOB_QUEUE = os.getenv('JOB_QUEUE', 'ranges') -SUCCESS_QUEUE = os.getenv('SUCCESS_QUEUE', 'succeeded') -FAILED_QUEUE = os.getenv('FAILED_QUEUE', 'failed') -PROGRESS_QUEUE = os.getenv('PROGRESS_QUEUE', 'in_progress') -METRICS = os.getenv('METRICS', 'metrics') -JOB_OWNERS = os.getenv('JOB_OWNERS', 'job_owners') -WORKER_PREFIX = os.getenv('WORKER_PREFIX', 'stellar-core') + +# Configuration is grouped by who consumes the value: +# 1. stellar-core workload -- goes into the worker container or catchup args +# 2. Kubernetes objects -- shape of the Jobs, pods and PVCs we create +# 3. monitor behaviour -- never leaves this process + +# ============================================================================= +# 1. stellar-core workload +# ============================================================================= +CORE_IMAGE = os.getenv('CORE_IMAGE') +ASAN_OPTIONS = os.getenv('ASAN_OPTIONS', '') + +# Which ledger ranges to run. These are pure inputs to the range generator: +# dispatch recomputes the whole list every reconcile, so a restart must +# reproduce it exactly. +RANGE_GENERATOR = os.getenv('RANGE_GENERATOR', 'uniform') # uniform | logarithmic +# Both generators emit tip-first, which front-loads the most expensive ranges: +# the bucket set only grows with ledger position. 'oldest-first' reverses that, +# so a profiling run measures the cheap early ranges before it can be +# interrupted, and the expensive tip ranges last. +RANGE_ORDER = os.getenv('RANGE_ORDER', 'tip-first') # tip-first | oldest-first +STARTING_LEDGER = int(os.getenv('STARTING_LEDGER', 0)) +LATEST_LEDGER_NUM = int(os.getenv('LATEST_LEDGER_NUM', 0)) +LEDGERS_PER_JOB = int(os.getenv('LEDGERS_PER_JOB', 16000)) +OVERLAP_LEDGERS = int(os.getenv('OVERLAP_LEDGERS', 320)) +# logarithmic only: chunk size halves toward the tip and stops shrinking here. +LOGARITHMIC_FLOOR_LEDGERS = int(os.getenv('LOGARITHMIC_FLOOR_LEDGERS', 64000)) + +# ============================================================================= +# 2. Kubernetes objects this monitor creates +# ============================================================================= NAMESPACE = os.getenv('NAMESPACE', 'default') -WORKER_COUNT = int(os.getenv('WORKER_COUNT', 3)) -LOGGING_INTERVAL_SECONDS = int(os.getenv('LOGGING_INTERVAL_SECONDS', 10)) -STUCK_JOB_PING_RETRIES = 3 -STUCK_JOB_PING_DELAY_SECS = 30 +RUN_NAME = os.getenv('RUN_NAME', 'parallel-catchup') +PROGRESS_CM = f"{RUN_NAME}-catchup-progress" +LABEL_RUN = 'catchup.stellar.org/run' +LABEL_RANGE = 'catchup.stellar.org/range-end' +LABEL_ATTEMPT = 'catchup.stellar.org/attempt' + +# Workers need IRSA to read the S3 history mirror. Without it they silently fall +# back to the public archive, which throttles at 1024 and kills the run with +# curl 22 -> catchup exit 3. The name matches the old StatefulSet's so existing +# IRSA trust policies keep matching. +WORKER_SERVICE_ACCOUNT = os.getenv('WORKER_SERVICE_ACCOUNT', '') + +# Pod resources. +REQ_CPU = os.getenv('REQ_CPU', '1800m') +REQ_MEM = os.getenv('REQ_MEM', '9Gi') +LIM_CPU = os.getenv('LIM_CPU', '2') +LIM_MEM = os.getenv('LIM_MEM', '24000Mi') +# Only meaningful in ephemeral storage mode; see check_storage_config(). +# Range profile from an earlier run: tightens per-range requests so more +# workers fit per node. Requests only -- limits stay as configured, so the +# failure semantics and the OOM/disk escalation ladders are unchanged. +PROFILE_PATH = os.getenv('PROFILE_PATH', '') +PROFILE_MARGIN = float(os.getenv('PROFILE_MARGIN', 1.15)) +# CPU limit for a range the profile has measured. Higher than the unprofiled +# default on purpose: at a 2-core limit every range pegs 2.0, so the measured +# peak is a ceiling and the profile can never learn real demand. Room above the +# request lets each run's peak climb until it finds the true one. +# Empty = no cpu limit at all on a measured range. Measured on ssc-test with +# one pod per node (m8id/NVMe, 16320-ledger range): 168s at limit 2, 111s at 4, +# 99s uncapped. cpu.weight still derives from the request, so a burst only uses +# cycles the neighbours are not using. Set a value to cap it again. +# +# The gain is in bucket-apply and replay, not download: at 65280 ledgers, two +# pods at limit 2 and limit 4 had written 8001 and 8013 MiB after 43 minutes -- +# identical -- because the download phase is storage-bound, not CPU-bound. +PROFILE_CPU_LIMIT = os.getenv('PROFILE_CPU_LIMIT', '') +# No safety margin on cpu, unlike memory. Under-requesting cpu costs contention +# and the pod can still burst; under-requesting memory gets it OOMKilled. +PROFILE_CPU_MARGIN = float(os.getenv('PROFILE_CPU_MARGIN', 1.0)) +# Ceiling for profile-derived memory, above the unprofiled limit for the same +# reason: a range that really needs more than the configured limit must be able +# to ask for it rather than be pinned under its own measured peak. The OOM +# escalation ladder can still climb past this on a retry. +PROFILE_MAX_MEM = os.getenv('PROFILE_MAX_MEM', '32Gi') +# Memory is sized from rss (the range's real demand), NOT from peak working +# set. Working set is whatever limit it was measured under -- the kernel grows +# page cache to fill it -- so sizing from it is circular. Measured on ssc-test +# with one 420-ledger range: working set went 2.33 -> 3.61 -> 7.48 -> 13.49 GiB +# under 2560Mi/4Gi/8Gi/24000Mi limits while rss moved only 2256 -> 2488 MiB, and +# wall-clock did not move at all (776s / 775s / 746s / 773s). Catchup streams -- +# buckets are downloaded once, applied once, ledgers replayed once -- so cache +# has nothing to give back and PROFILE_MARGIN alone is the allowance. +# A multiplicative margin alone is not enough: memory.max bounds anon PLUS page +# cache, and at small rss 10% is nothing. Measured on ssc-test 2026-07-29 with +# headroom 0: ranges profiled at 190 MiB rss got a 209 MiB limit -- 19 MiB of +# slack for all growth and cache -- and 90 of them OOMKilled within 90s. The +# earlier 4Gi validation hid this because 1.1x of 2.4 GiB is 240 MiB of slack. +PROFILE_CACHE_HEADROOM = os.getenv('PROFILE_CACHE_HEADROOM', '512Mi') + +REQ_EPHEMERAL = os.getenv('REQ_EPHEMERAL', '') +LIM_EPHEMERAL = os.getenv('LIM_EPHEMERAL', '') + +# Placement. The taint toleration is emitted as {key, effect} with no value: +# the default Equal operator does not match "" against "true". +NODE_LABEL_KEY = os.getenv('NODE_LABEL_KEY', '') +NODE_LABEL_VALUE = os.getenv('NODE_LABEL_VALUE', '') +TOLERATE_TAINT = os.getenv('TOLERATE_TAINT', '') + +# Worker /data. pvc keeps it across pods, so an evicted range resumes at L+1 -- +# that is what makes spot viable. ephemeral puts it on the node disk: denser +# packing, no resume, and REQ_EPHEMERAL must be sized to hold the catchup DB. +# One PVC per range, not per concurrency slot: measured on ssc-test, 300 jobs +# with a PVC each cost no more wall-clock than 300 jobs reusing 40. +STORAGE_MODE = os.getenv('STORAGE_MODE', 'pvc') # pvc | ephemeral +STORAGE_CLASS = os.getenv('STORAGE_CLASS', '') +STORAGE_SIZE = os.getenv('STORAGE_SIZE', '40Gi') +# A Nitro node allows ~26 EBS attachments (CSINode allocatable), and Karpenter +# sizes nodes on CPU/memory only -- it will happily put 40 volume-mounting pods +# on one 4-vCPU node, where they serialise through the attachment slots and get +# rejected with VolumeAttachmentLimitExceeded (observed on ssc-test). +# +# Guard with a spread constraint rather than a warning. maxSkew alone cannot cap +# per-node count -- with a single node there is one domain and therefore no skew +# -- so minDomains is what forces enough nodes. Both are inert at realistic +# density: REQ_CPU=1800m yields ~4 workers on an 8-vCPU node, so CPU demands far +# more nodes than this floor ever asks for. 0 disables. +MAX_VOLUMES_PER_NODE = int(os.getenv('MAX_VOLUMES_PER_NODE', 24)) + +# Job/pod lifetimes. +# SIGTERM -> SIGKILL budget. stellar-core exits ~7s after SIGTERM (measured), so +# this is slack rather than a target. +WORKER_GRACE_SECONDS = int(os.getenv('GRACE_SECONDS', 100)) +# Must comfortably exceed any plausible monitor outage: completion is recorded +# to the ConfigMap by this process, and a Job reclaimed before that happens +# reads as "never ran" and gets redone. +# Backstop only. reconcile() deletes each Job explicitly once its record is +# durable, so the TTL exists for the cases that skip that path: a terminally +# failed range kept for inspection, or a success whose metrics never landed. +JOB_TTL_SECONDS = int(os.getenv('JOB_TTL_SECONDS', 600)) +# Measured on ssc-test: stellar-core does NOT fail on an unreachable history +# archive, an absent ledger range, or a bucket that will not decompress. It +# retries every mirror with growing backoff and stays Running indefinitely -- +# no exit code, no failure, the slot held for the life of the run. A hang is a +# more likely real failure than a non-zero exit, and this deadline is the only +# thing that makes it observable. 0 disables. +ATTEMPT_DEADLINE_SECONDS = int(os.getenv('ATTEMPT_DEADLINE_SECONDS', 0)) + +# kube-state-metrics turns a pod's `mission` label into label_mission, which the +# Grafana container panels join on. Every other mission gets it from +# StellarKubeSpecs; this chart never has, so parallel catchup has never appeared +# in those panels. +# +# OFF by default and deliberately so: those panels are sum() by (pod, container) +# with a legend table, so at 1024 workers they would pull ~1024 series into any +# view with mission=$__all selected, degrading a shared dashboard for people who +# did not ask for it. Enable per-run once the panels aggregate (topk). +MISSION = os.getenv('MISSION', '') +EMIT_MISSION_LABEL = os.getenv('EMIT_MISSION_LABEL', 'false').lower() == 'true' + +# ============================================================================= +# 3. This monitor's own behaviour +# ============================================================================= +PARALLELISM = int(os.getenv('PARALLELISM', 3)) +MAX_ATTEMPTS_PER_RANGE = int(os.getenv('MAX_ATTEMPTS', 5)) +# A hang gets far fewer retries than an eviction. The measured causes -- an +# unreachable archive host, an absent checkpoint, a bucket that will not +# decompress -- are persistent, so retrying mostly burns another full deadline. +MAX_TIMEOUT_ATTEMPTS = int(os.getenv('MAX_TIMEOUT_ATTEMPTS', 2)) +# Evictions, admission rejections and monitor restarts say nothing about the +# ledger range, so they get their own, larger budget. Sharing MAX_ATTEMPTS with +# real failures means cluster churn can fail a healthy range: measured on +# ssc-test, ten evictions across 25 workers put four ranges on attempt 3 of 5 +# without a single genuine catchup error. +MAX_DISRUPTION_ATTEMPTS = int(os.getenv('MAX_DISRUPTION_ATTEMPTS', 20)) +# An ephemeral-storage eviction repeats identically until the range gets more +# disk, so it must not sit on the environmental budget. +MAX_EPHEMERAL_ATTEMPTS = int(os.getenv('MAX_EPHEMERAL_ATTEMPTS', 4)) +EPH_BUMP_FACTOR = float(os.getenv('EPH_BUMP_FACTOR', 1.5)) +EPH_ESCALATION_CAP = os.getenv('EPH_ESCALATION_CAP', '200Gi') +ENVIRONMENTAL_OUTCOMES = ('disrupted', 'rejected', 'unknown') +# An OOM means requests/limits are mis-sized for this range. Escalate so the run +# can finish, but say so loudly -- surviving by escalating at runtime is a +# configuration bug, not a success. +MEM_BUMP_FACTOR = float(os.getenv('MEM_BUMP_FACTOR', 1.5)) +# Ceiling for that escalation. Above the largest schedulable node the retry sits +# Pending forever, which looks like a hang rather than a failure. +MEM_ESCALATION_CAP = os.getenv('MAX_MEM', '48Gi') + +# Reconcile loop: dispatch, refresh status, publish metrics. The env var is +# named LOGGING_INTERVAL_SECONDS for historical reasons, from when this loop +# only logged. +RECONCILE_INTERVAL_SECONDS = int(os.getenv('LOGGING_INTERVAL_SECONDS', 10)) +# /healthz fails if the loop has not ticked within this long; a wedged loop +# stops all dispatch, so restart the container rather than run half-alive. +RECONCILE_STALE_SECONDS = float(os.getenv('WATCH_STALE_SECONDS', 600)) +# Liveness ping to each running worker's admin port, fanned out on one event +# loop. Done serially with a 5s timeout this took a 192s median at 1024 +# (measured in prod), which is why it is async and the timeout is short. +WORKER_PING_TIMEOUT_SECONDS = float(os.getenv('PING_TIMEOUT_SECS', 2)) + +# Shared with the log-collector sidecar, which owns writes here: it streams each +# worker's log and records the .outcome verdict while the pod still exists. +LOG_DIR = os.getenv('LOG_DIR', '/logs') +SAVE_SUCCESS_LOGS = os.getenv('SAVE_SUCCESS_LOGS', 'true').lower() == 'true' + def get_logging_level(): name_to_level = { @@ -39,26 +244,80 @@ def get_logging_level(): 'DEBUG': logging.DEBUG, } result = name_to_level.get(os.getenv('LOGGING_LEVEL', 'INFO')) - if result is not None: - return result - else: - return logging.INFO + return result if result is not None else logging.INFO -# Initialize Redis client -redis_client = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True) -# Configure logging +# On the logs PVC, not the monitor's emptyDir: /data dies with the pod, and the +# mission tars /logs -- so an OOM-retry storm, the loudest signal this thing +# produces, was visible only in `kubectl logs` and never reached the run's +# destination directory. Falls back to /data if LOG_DIR is not mounted. log_file_name = f"job_monitor_{datetime.now(timezone.utc).strftime('%Y-%m-%d_%H-%M-%S')}.log" -log_file_path = os.path.join('/data', log_file_name) +_log_dir = os.getenv('LOG_DIR', '/logs') +log_file_path = os.path.join(_log_dir if os.path.isdir(_log_dir) else '/data', log_file_name) logging.basicConfig(level=get_logging_level(), format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.StreamHandler(sys.stdout), logging.FileHandler(log_file_path), ]) logger = logging.getLogger() -# In-memory status data structure and threading lock +config.load_incluster_config() +# client-go's Python equivalent defaults are fine for a few LISTs per cycle, but +# dispatching ~1024 Jobs + PVCs at once needs headroom. +_cfg = client.Configuration.get_default_copy() +_cfg.connection_pool_maxsize = int(os.getenv('CONNECTION_POOL', 64)) +client.Configuration.set_default(_cfg) +core_v1 = client.CoreV1Api() +batch_v1 = client.BatchV1Api() + + +def _gib(q): + try: + return _quantity_bytes(q) / (1024 ** 3) + except Exception: + return None + + +def check_storage_config(): + """The two halves of the storage choice are set independently and can disagree. + + In ephemeral mode /data is an emptyDir on the node disk, so the + ephemeral-storage request must be large enough to hold the catchup DB and + buckets -- otherwise the kubelet evicts the pod for exceeding it. In PVC + mode the opposite is true: a large request makes disk the binding dimension + and halves workers-per-node (measured: 2/node instead of 4 on a 2xlarge). + """ + req = _gib(REQ_EPHEMERAL) if REQ_EPHEMERAL else None + if STORAGE_MODE == 'ephemeral': + if req is None or req < 20: + logger.error("STORAGE_MODE=ephemeral but ephemeral-storage request is %s. " + "/data lives on the node disk in this mode; too small a request " + "gets the pod evicted mid-catchup. Expect ~35Gi.", + REQ_EPHEMERAL or "unset") + if STORAGE_MODE == 'pvc': + # One EBS volume per worker, and a Nitro node allows ~26 attachments + # (CSINode allocatable). Density comes from the CPU request: 1800m gives + # ~4 workers on an 8-vCPU node, far below the cap. A small request packs + # many volume-mounting pods onto one node, where they serialise through + # the attachment slots -- observed on ssc-test as pods rejected with + # VolumeAttachmentLimitExceeded. Karpenter sizes on CPU/memory and does + # not provision extra nodes for attachment capacity. + try: + cpu = REQ_CPU + millis = int(cpu[:-1]) if cpu.endswith('m') else int(float(cpu) * 1000) + if millis and 8000 // millis > 20: + logger.warning("STORAGE_MODE=pvc with REQ_CPU=%s packs ~%d workers (and volumes) " + "onto an 8-vCPU node, near the ~26 EBS attachment limit. Expect " + "VolumeAttachmentLimitExceeded rejections under churn.", + REQ_CPU, 8000 // millis) + except (ValueError, ZeroDivisionError): + pass + if req is not None and STORAGE_MODE == 'pvc' and req > 10: + logger.warning("STORAGE_MODE=pvc but ephemeral-storage request is %s. /data is on " + "a PVC, so this only makes disk the binding dimension and reduces " + "workers per node. Expect ~2Gi.", REQ_EPHEMERAL) + status = { - 'num_remain': 1, # initialize the job remaining to non-zero to indicate something is running, just the status hasn't been updated yet + 'num_remain': 1, # non-zero until the first real update, so callers don't see a premature 0 'queue_remain_count': 0, 'queue_succeeded_count': 0, 'queue_failed_count': 0, @@ -72,25 +331,43 @@ def get_logging_level(): 'mission_duration': 0, } status_lock = threading.Lock() +# Heartbeat for /healthz: a wedged reconcile loop stops all dispatch, so the +# container should be restarted rather than left running half-alive. +reconcile_alive = {'ts': 0.0} -metrics = { - 'metrics': [] -} -metrics_lock = threading.Lock() -# Create metrics metric_catchup_queues = Gauge('ssc_parallel_catchup_queues', 'Exposes size of each job queues', ["queue"]) metric_workers = Gauge('ssc_parallel_catchup_workers', 'Exposes catch up worker status', ["status"]) metric_refresh_duration = Gauge('ssc_parallel_catchup_workers_refresh_duration_seconds', 'Time it took to refresh status of all workers') metric_full_duration = Histogram('ssc_parallel_catchup_job_full_duration_seconds', 'Exposes full job duration as histogram', buckets=metric_buckets) metric_tx_apply_duration = Histogram('ssc_parallel_catchup_job_tx_apply_duration_seconds', 'Exposes job TX apply duration as histogram', buckets=metric_buckets) +# full_duration is the SUCCESSFUL attempt only, matching what worker.sh timed. +# wall_duration spans first dispatch to success, so (wall - full) is exactly the +# work lost to retries -- the cost of running on spot. +metric_wall_duration = Histogram('ssc_parallel_catchup_job_wall_duration_seconds', + 'First dispatch to success, including failed attempts', + buckets=metric_buckets) metric_mission_duration = Gauge('ssc_parallel_catchup_mission_duration_seconds', 'Number of seconds since the mission started ') metric_retries = Counter('ssc_parallel_catchup_job_retried_count', 'Number of jobs that were retried') +# Separates infrastructure churn from application failure: many evictions with +# zero app failures is spot behaving as intended. +metric_evictions = Counter('ssc_parallel_catchup_job_spot_eviction_count', 'Pod attempts lost to node disruption') +metric_pvc_released = Counter('ssc_parallel_catchup_pvc_released_count', 'PVCs deleted after their range completed') +metric_jobs_reaped = Counter('ssc_parallel_catchup_jobs_reaped_count', 'Finished Jobs deleted after their record was durable') +metric_oom_retries = Counter('ssc_parallel_catchup_job_oom_retried_count', 'Jobs retried with an escalated memory limit') +metric_eph_retries = Counter('ssc_parallel_catchup_job_ephemeral_retried_count', 'Jobs retried with an escalated ephemeral-storage limit') class RequestHandler(BaseHTTPRequestHandler): def do_GET(self): - if self.path == '/status': + if self.path == '/healthz': + stale = time.time() - reconcile_alive['ts'] + ok = reconcile_alive['ts'] > 0 and stale < RECONCILE_STALE_SECONDS + self.send_response(200 if ok else 503) + self.send_header('Content-type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps({'reconcile_age_seconds': round(stale, 1)}).encode()) + elif self.path == '/status': self.send_response(200) self.send_header('Content-type', 'application/json') self.end_headers() @@ -101,138 +378,1336 @@ def do_GET(self): self.send_header('Content-type', CONTENT_TYPE_LATEST) self.end_headers() self.wfile.write(generate_latest(REGISTRY)) - elif self.path == '/metrics': - self.send_response(200) - self.send_header('Content-type', 'application/json') - self.end_headers() - with metrics_lock: - self.wfile.write(json.dumps(metrics).encode()) else: self.send_response(404) self.end_headers() -def retry_jobs_in_progress(): - while redis_client.llen(PROGRESS_QUEUE) > 0: - job = redis_client.lmove(PROGRESS_QUEUE, JOB_QUEUE, "RIGHT", "LEFT") - metric_retries.inc() - logger.info("moved job %s from %s to %s", job, PROGRESS_QUEUE, JOB_QUEUE) + def log_message(self, *args): + pass # the default handler logs every request to stderr + + +# --- range generation ------------------------------------------------------- +# Ports uniform_range_generator.sh and logarithmic_range_generator.sh. These +# must stay pure functions of config: dispatch derives the full range list on +# every reconcile, so a restart has to reproduce it exactly. + +def _uniform_segment(start_ledger, end_ledger, seg_size): + """Ranges over (start_ledger, end_ledger], largest ledger first.""" + out = [] + el = end_ledger + while el > start_ledger: + ledgers_per_job = min(el - start_ledger, seg_size) + out.append((el, ledgers_per_job + OVERLAP_LEDGERS)) + el -= ledgers_per_job + return out + + +def _ordered(ranges): + """Dispatch order. Generators emit tip-first; reverse for oldest-first.""" + return list(reversed(ranges)) if RANGE_ORDER == 'oldest-first' else ranges + + +def generate_ranges(): + if RANGE_GENERATOR == 'uniform': + return _ordered(_uniform_segment(STARTING_LEDGER, LATEST_LEDGER_NUM, LEDGERS_PER_JOB)) + + # Logarithmic: early history is cheap per ledger, so use big chunks there and + # halve the chunk size as we approach the tip. Aims for roughly equal + # wall-time per job rather than equal ledger count. + out = [] + start_ledger = STARTING_LEDGER + end_ledger = LATEST_LEDGER_NUM // 2 + chunk = (end_ledger - start_ledger + 1) // max(PARALLELISM, 1) + while chunk > LOGARITHMIC_FLOOR_LEDGERS: + out.extend(_uniform_segment(start_ledger, end_ledger, chunk)) + start_ledger = end_ledger + 1 + chunk //= 2 + end_ledger = start_ledger + (chunk * PARALLELISM) + out.extend(_uniform_segment(end_ledger + 1, LATEST_LEDGER_NUM, LOGARITHMIC_FLOOR_LEDGERS)) + return _ordered(out) + + +def job_key(end, count): + return f"{end}/{count}" + + +def job_name(end, attempt): + return f"{RUN_NAME}-r{end}-a{attempt}" + + +# --- durable progress record ------------------------------------------------ +# Jobs get reclaimed during a 10h run, so completion cannot live only in Job +# objects. Written BEFORE a Job becomes TTL-eligible. + +# Set once at startup; the same ConfigMap the Jobs and PVCs hang off. +_progress_owner = {} + + +# The authoritative copy of the progress record lives on the logs PVC, not in +# the ConfigMap. A ConfigMap is capped at 1 MiB and this record is ~172 bytes +# per completed range, so it dies at ~6100 ranges -- reachable simply by halving +# ledgersPerJob. Worse, every completion rewrote the whole document through the +# API server, so a full run meant thousands of escalating-size etcd writes. +# +# The ConfigMap is still written, because the mission driver reads it without +# exec'ing into the pod, but it is now a best-effort mirror: if it fails, the +# run carries on from the file. +PROGRESS_FILE = os.path.join(LOG_DIR, 'progress.json') + + +def load_progress(): + try: + with open(PROGRESS_FILE) as fh: + return json.load(fh) + except (OSError, ValueError): + pass + # First start on this volume, or an older run that only had the ConfigMap. + try: + cm = core_v1.read_namespaced_config_map(PROGRESS_CM, NAMESPACE) + return json.loads((cm.data or {}).get('progress.json', '{}')) + except ApiException as e: + if e.status == 404: + return {} + raise + + +def save_status(snapshot): + """Publish /status into the ConfigMap as well. + + The mission driver runs outside the cluster and already has a kube client, + so reading a ConfigMap is simpler and more robust than exposing the monitor + through a Gateway/HTTPRoute just to be polled. Shape is identical to the + HTTP /status body, so the driver's parser is unchanged. + """ + _patch_cm({'status.json': json.dumps(snapshot, separators=(',', ':'))}) + + +# Measurements live only on the volume. The ConfigMap is the mission-state +# mirror the driver reads for visibility, and at ~172 bytes per range the +# profiling fields alone push it toward the 1 MiB cap at ~6100 ranges. Stripped +# to attempts/count it is ~30 bytes, so state stays readable at any slicing +# while the profile has no ceiling at all. +_PROFILE_ONLY_FIELDS = ('peakAnonBytes', 'peakRssBytes', 'peakWorkingSetBytes', 'peakCpuCores', + 'peakEphemeralBytes', 'txApply', 'seconds', 'wallSeconds') + + +def _state_only(progress): + out = dict(progress) + completed = {} + for end, rec in (progress.get('completed') or {}).items(): + completed[end] = {k: v for k, v in rec.items() + if k not in _PROFILE_ONLY_FIELDS} + out['completed'] = completed + return out + + +def save_progress(progress): + blob = json.dumps(progress, separators=(',', ':')) + # File first and atomically: it is what a restart reads back. + tmp = PROGRESS_FILE + '.tmp' + with open(tmp, 'w') as fh: + fh.write(blob) + os.replace(tmp, PROGRESS_FILE) + # Mirror for the driver. Never fatal -- a 413 here used to throw inside + # reconcile, and the loop swallows exceptions, so no completion would ever + # be recorded again and every finished range would be dispatched forever. + try: + _patch_cm({'progress.json': json.dumps(_state_only(progress), + separators=(',', ':'))}) + except ApiException as e: + logger.warning("progress ConfigMap mirror failed (%s); the record on %s " + "is authoritative and the run continues", e.status, PROGRESS_FILE) + + +def _patch_cm(data, owner=None): + body = {'data': data} + try: + core_v1.patch_namespaced_config_map(PROGRESS_CM, NAMESPACE, body) + except ApiException as e: + if e.status != 404: + raise + core_v1.create_namespaced_config_map(NAMESPACE, client.V1ConfigMap( + # Owned by the chart's stellar-core ConfigMap like the Jobs and + # PVCs, so `helm uninstall` reclaims it. Without an owner this + # outlived every run and accumulated in the shared namespace. + metadata=client.V1ObjectMeta(name=PROGRESS_CM, labels={LABEL_RUN: RUN_NAME}, + owner_references=_progress_owner.get('ref')), + data=body['data'])) + + +# --- worker liveness -------------------------------------------------------- +# Serially pinging 1024 workers with a 5s timeout was measured at a 192s median +# and 773s max in prod (155 unreachable x 5s). Fanning out on one event loop +# bounds it by the timeout itself. + +async def _ping_all(pods): + timeout = aiohttp.ClientTimeout(total=WORKER_PING_TIMEOUT_SECONDS) + async with aiohttp.ClientSession(timeout=timeout) as session: + async def one(pod, ip): + url = f"http://{ip}:11626/info" + try: + async with session.get(url): + return pod, True + except Exception: + return pod, False + return dict(await asyncio.gather(*(one(p, ip) for p, ip in pods))) + + +def ping_workers(pods): + """pods: (name, ip) pairs. + + By IP, not DNS. The ...svc form needs a per-pod A record, + which a headless Service only publishes for endpoints whose EndpointSlice + carries a hostname -- and that comes from pod.spec.hostname, which a Job pod + cannot set to its own generated name. Measured on ssc-test: every ping + failed to resolve and workers_up sat at 0 for the whole run. + """ + if not pods: + return {} + return asyncio.run(_ping_all(pods)) + + +# --- worker log capture ----------------------------------------------------- + +def backstop_save_pod_log(pod_name, end, attempt): + """Last-resort archive for a range the collector never streamed. + + The log-collector sidecar owns /logs in the normal case: it holds a + follow=true stream from pod start, so nothing is lost when the node is + reaped. This only covers the gap where a pod lived and died entirely while + the collector was down -- detected by the absence of the state file the + collector writes when it claims a range. + + Never writes over a claimed or existing archive: two writers appending to + one gzip would interleave members and duplicate lines. + """ + if os.path.exists(state_path(end, attempt)): + return True # collector has it (streaming or already finished) + path = log_path(end, attempt) + if os.path.exists(path): + return True + try: + body = core_v1.read_namespaced_pod_log(pod_name, NAMESPACE, container='stellar-core') + except ApiException as e: + logger.warning("could not save log for range %s attempt %d (pod %s): %s", + end, attempt, pod_name, e.reason) + return False + try: + os.makedirs(LOG_DIR, exist_ok=True) + tmp = path + '.tmp' + with gzip.open(tmp, 'wt') as fh: + fh.write(body) + os.replace(tmp, path) # never leave a half-written archive behind + return True + except OSError as e: + logger.warning("could not write %s: %s", path, e) + return False + + +def log_path(end, attempt): + """Canonical archive name, written by the log-collector sidecar. + + Deliberately carries no ok/failed suffix: which ranges failed is recorded in + the progress ConfigMap, and encoding it here would mean two components + disagreeing about a filename. + """ + return os.path.join(LOG_DIR, f"range-{end}-a{attempt}.log.gz") + + +def state_path(end, attempt): + return os.path.join(LOG_DIR, f"range-{end}-a{attempt}.state") + + +def outcome_path(end, attempt): + return os.path.join(LOG_DIR, f"range-{end}-a{attempt}.outcome") + + +def classify(pod): + """Why did this pod fail? The Job object cannot answer this. + + Job.status only carries a Failed condition with reason BackoffLimitExceeded + -- no exit code, no OOM. The detail lives on the pod, which is exactly the + object Karpenter deletes with the node, so this is recorded the moment the + watch sees it rather than when reconcile next runs. + """ + for cond in (pod.status.conditions or []): + if cond.type == 'DisruptionTarget' and cond.status == 'True': + return {'outcome': 'disrupted', 'exitCode': None} + # Kubelet can reject a pod before any container runs -- observed on + # ssc-test: reason=VolumeAttachmentLimitExceeded, "Node has reached its + # volume attachment limit, rejecting pod". There is no exit code and no + # DisruptionTarget, so without this it falls through to 'failed' and a + # transient admission rejection kills the whole run. + if pod.status.reason == 'Evicted' and 'ephemeral' in (pod.status.message or ''): + # Measured on ssc-test: the kubelet sets no DisruptionTarget for a + # limit eviction, and stellar-core drains on the eviction SIGTERM and + # exits 3 -- so the Job condition matches the generic non-zero rule and + # reads as a plain catchup failure, which gets no retry at all. + # status.message is the only discriminator and only the pod carries it. + return {'outcome': 'ephemeral', 'exitCode': None, 'reason': pod.status.message} + if pod.status.reason in ('VolumeAttachmentLimitExceeded', 'OutOfcpu', 'OutOfmemory', + 'OutOfpods', 'UnexpectedAdmissionError', 'NodeAffinity', + 'Shutdown', 'Evicted'): + return {'outcome': 'rejected', 'exitCode': None, 'reason': pod.status.reason} + started = any(cs.state and cs.state.terminated for cs in (pod.status.container_statuses or [])) + if not started: + # No container ever reached a terminal state: nothing ran, so this is + # not evidence about the ledger range. + return {'outcome': 'rejected', 'exitCode': None, + 'reason': pod.status.reason or 'no container status'} + for cs in (pod.status.container_statuses or []): + t = cs.state.terminated if cs.state else None + if t is None: + continue + # 137 is SIGKILL, which the kubelet also uses for a graceful-stop + # timeout -- but with reason OOMKilled it is unambiguous. + if t.reason == 'OOMKilled': + return {'outcome': 'oom', 'exitCode': t.exit_code} + if t.exit_code not in (0, None): + return {'outcome': 'failed', 'exitCode': t.exit_code} + return {'outcome': 'failed', 'exitCode': None} + + +def record_outcome(end, attempt, pod): + path = outcome_path(end, attempt) + if os.path.exists(path): + return + data = classify(pod) + data['pod'] = pod.metadata.name + try: + tmp = path + '.tmp' + with open(tmp, 'w') as fh: + json.dump(data, fh) + os.replace(tmp, path) + except OSError as e: + logger.warning("could not persist outcome for range %s: %s", end, e) + + +# The Job controller writes the exit code and pod name into the failure +# condition message, e.g. +# "Container stellar-core for pod ns/kic-r400000-a1-xxxxx failed with exit +# code 137 matching FailJob rule at index 1" +# Jobs are not bound to a node, so unlike the pod this survives consolidation. +_JOB_MSG = re.compile(r"for pod \S+?/(?P\S+) failed with exit code (?P\d+)") +_JOB_RULE = re.compile(r"rule at index (?P\d+)") + + +def _failure_rules(): + """podFailurePolicy rules, in evaluation order, tagged with what they mean. + + First match wins, so reaching the exit-137 rule proves DisruptionTarget did + not match -- that ordering is what separates an OOM kill from a + grace-period SIGKILL after the pod is gone. + + All FailJob: the Job must fail with reason=PodFailurePolicy so the message + names the rule index. A Count action would surface as BackoffLimitExceeded + and lose the signal. Retries stay with the monitor because raising a memory + limit needs a new Job -- spec.template is immutable. + """ + return [ + ('disrupted', client.V1PodFailurePolicyRule( + action='FailJob', + on_pod_conditions=[client.V1PodFailurePolicyOnPodConditionsPattern( + type='DisruptionTarget', status='True')])), + ('oom', client.V1PodFailurePolicyRule( + action='FailJob', + on_exit_codes=client.V1PodFailurePolicyOnExitCodesRequirement( + container_name='stellar-core', operator='In', values=[137]))), + ('failed', client.V1PodFailurePolicyRule( + action='FailJob', + on_exit_codes=client.V1PodFailurePolicyOnExitCodesRequirement( + container_name='stellar-core', operator='NotIn', values=[0]))), + ] + + +# Order here is the contract with the Job controller's "rule at index N". +RULE_ORDER = ['disrupted', 'oom', 'failed'] +_RULE_OUTCOME = dict(enumerate(RULE_ORDER)) + + +def classify_from_job(job): + """Recover a verdict from the Job when the pod is already gone. + + Rule index is the signal, not the exit code: rules are evaluated + first-match-wins, so reaching the exit-137 rule proves the DisruptionTarget + rule did not match, which is the only way to tell an OOM kill from a + grace-period SIGKILL once the pod is gone. + + Index and exit code are parsed independently -- a rule matching on + onPodConditions reports no exit code at all, so requiring one would make the + disruption case unreadable. + """ + for cond in (job.status.conditions or []): + if cond.type != 'Failed' or cond.status != 'True': + continue + msg = cond.message or '' + if cond.reason == 'DeadlineExceeded': + # activeDeadlineSeconds fired: the attempt hung rather than failing. + # Retryable -- a genuinely stuck range will exhaust its attempts. + return {'outcome': 'timeout', 'exitCode': None, 'pod': '', + 'source': 'job-condition'} + if cond.reason != 'PodFailurePolicy': + # e.g. BackoffLimitExceeded -- carries no per-rule detail. + continue + rule = _JOB_RULE.search(msg) + detail = _JOB_MSG.search(msg) + outcome = _RULE_OUTCOME.get(int(rule.group('idx'))) if rule else None + code = int(detail.group('code')) if detail else None + if outcome is None: + if code is None: + return None + # No usable rule index. Measured on ssc-test (2026-07-28): a drained + # stellar-core catches SIGTERM and exits 3 in ~7s, well inside the + # 100s grace -- evictions do NOT produce 137. So a bare 137 is an OOM + # with high confidence, and exit 3 without a DisruptionTarget + # condition really is a catchup failure. + outcome = 'oom' if code == 137 else 'failed' + return {'outcome': outcome, 'exitCode': code, + 'pod': detail.group('pod') if detail else '', + 'source': 'job-condition'} + return None + + +def read_outcome(end, attempt): + try: + with open(outcome_path(end, attempt)) as fh: + return json.load(fh) + except (OSError, ValueError): + return None + + +def mem_for_attempt(attempt, base=None): + """Memory limit for attempt N, escalating after an OOM, capped at MEM_ESCALATION_CAP. + + `base` is what attempt 1 actually ran with. It matters when a profile sized + the range: escalating a 209Mi profiled range off the configured 24000Mi + limit jumps straight to 36000Mi, a 172x overshoot that throws away the whole + packing win on the first OOM. + """ + base_q = _quantity_bytes(base or LIM_MEM) + want = int(base_q * (MEM_BUMP_FACTOR ** max(0, attempt - 1))) + cap = _quantity_bytes(MEM_ESCALATION_CAP) + return _bytes_to_quantity(min(want, cap)) + + +_UNITS = {'Ki': 1024, 'Mi': 1024**2, 'Gi': 1024**3, 'Ti': 1024**4, + 'K': 1000, 'M': 1000**2, 'G': 1000**3, 'T': 1000**4} + + +def _quantity_bytes(q): + for suffix, mult in sorted(_UNITS.items(), key=lambda kv: -len(kv[0])): + if q.endswith(suffix): + return int(float(q[:-len(suffix)]) * mult) + return int(float(q)) + + +def _bytes_to_quantity(n): + return f"{max(1, n // (1024 ** 2))}Mi" + + + -def ping_worker(pod_name, retries=1): - """Ping a worker's /info endpoint. Returns True if reachable within the given number of attempts.""" - worker_dns = f"{pod_name}.{WORKER_PREFIX}.{NAMESPACE}.svc.cluster.local" - for attempt in range(1, retries + 1): +# --- tx_apply --------------------------------------------------------------- + +# medida prints the sum in scientific notation once it exceeds 1e6 ms, which is +# every range that applies a real transaction load. The old [0-9.]+ pattern +# matched "1.30722" then demanded "ms" and hit "e+06ms" instead, so tx_apply was +# silently missing for 25% of ranges -- 91-99% of everything above ledger 35M, +# exactly the expensive end. +_SUM_RE = re.compile(r"sum\s*=\s*([0-9.]+(?:[eE][+-]?[0-9]+)?)ms") + + +def metrics_path(end, attempt): + return os.path.join(LOG_DIR, f"range-{end}-a{attempt}.metrics") + + +# A PVC's size is not a scheduling dimension -- growing it buys no packing, so +# it is not profiled; the only volume ceiling that matters is the ~26 +# attachments per node. Ephemeral storage IS a scheduling dimension, so it is, +# but only in ephemeral mode and only on on-demand nodes -- see the collector. +# Any field may be absent and the consumer falls back to its default. +PEAK_FIELDS = ('peakAnonBytes', 'peakRssBytes', 'peakWorkingSetBytes', + 'peakEphemeralBytes') + + +def peaks_for_range(end, attempt=1): + """Highest peak any attempt at this range reached, per axis. + + Not just the successful attempt. In pvc mode a pod that dies once replay has + started leaves /data behind, and the next attempt resumes at LCL+1 with + RESUME=true -- skipping the archive download and the bucket apply, which is + where peak memory actually happens. Its peak describes the tail of the range, + not the range, so profiling the winner alone under-reports by the whole + download-vs-replay gap. On spot, where eviction is routine and resume is the + entire point of durable /data, that would make the run unprofileable. + + Attempts that hit a ceiling are counted too. A pod OOM-killed at 8Gi really + did allocate ~8Gi and wanted more, so its peak is a lower bound on demand, + not an artifact of the limit -- and it is the attempt most worth keeping, + because download concurrency scales with available cpu and a pod that + bursted on an idle node can peak above the one that eventually succeeded. + Sizing off the quieter attempt would OOM the range again. There is no false + ratchet: a pod given 8Gi that only touches 1Gi records 1Gi. + + Advisory: used to size a LATER run's requests, never to decide anything + about this one. Any field may be absent. + """ + # Walk back only over a contiguous chain of resumed attempts. An attempt + # that did NOT resume ran new-db and did the whole range, so its sample is + # complete and supersedes everything before it -- in ephemeral mode, where + # /data dies with the pod and resume can never fire, that collapses to the + # winning attempt alone, exactly as before. + first = int(attempt) + while first > 1 and _attempt_resumed(end, first): + first -= 1 + out = {} + for n in range(first, int(attempt) + 1): + try: + with open(metrics_path(end, n)) as fh: + data = json.load(fh) + except (OSError, ValueError): + continue + for k in PEAK_FIELDS: + v = data.get(k) + if v is not None and v > out.get(k, 0): + out[k] = v + return out + + +def _attempt_resumed(end, attempt): + """Did this attempt pick up at LCL+1 rather than run new-db? + + Recorded by the collector from the worker's own "RESUME: ..." line, which is + the only place that knows -- it depends on what was left on /data, not on + storage mode or attempt number. + """ + try: + with open(metrics_path(end, attempt)) as fh: + return bool(json.load(fh).get('resumed')) + except (OSError, ValueError): + return False + + +def tx_apply_for_range(end, attempt=1, pod_name=None): + """Final 'ledger.transaction.apply' sum for one attempt, in seconds. + + stellar-core prints the medida block once at exit (we pass --metric), so + this is the exact total rather than a sample. Only ever called for a + SUCCEEDED attempt, so a failed one never contributes. + + Three sources, cheapest and most durable first: + + .metrics the collector parsed it out of the live stream. Survives both + pod reaping and saveSuccessLogs=false. + .log.gz the collector's archive, if it was kept. + pod log only if a pod object still exists. Racing Karpenter, so this + is a fallback, never the plan. + """ + try: + with open(metrics_path(end, attempt)) as fh: + value = json.load(fh).get('txApplySeconds') + if value is not None: + return float(value) + except (OSError, ValueError, TypeError): + pass + raw = None + candidate = log_path(end, attempt) + if os.path.exists(candidate): try: - requests.get(f"http://{worker_dns}:11626/info", timeout=5) + with gzip.open(candidate, 'rt') as fh: + raw = fh.read() + except OSError: + raw = None + if raw is None: + if pod_name is None: + return None + try: + raw = core_v1.read_namespaced_pod_log(pod_name, NAMESPACE, tail_lines=400) + except ApiException: + return None + lines = raw.splitlines() + for i, line in enumerate(lines): + if "metric 'ledger.transaction.apply'" not in line: + continue + for follow in lines[i + 1:i + 16]: + m = _SUM_RE.search(follow) + if m: + return float(m.group(1)) / 1000.0 + return None + + +# --- job construction ------------------------------------------------------- + +# Resume decision, run before catchup. Only skip new-db when the DB on /data +# belongs to THIS range and replay had already started. Bucket apply uses +# createWithoutLoading() -- an unconditional INSERT that assumes a fresh DB -- +# so a crash during that phase must start over. "Ledger close complete" is the +# cheap discriminator: bucket apply never closes a ledger. +# +# The LCL is read from stellar-core's own log on /data, NOT from the database: +# core 27 dropped the ledgerheaders table, so the old SQL probe silently +# returned empty and every interruption fell back to new-db. +RESUME_SCRIPT = r'''set -e +KEY="%(key)s" +TARGET=%(target)d +COUNT=%(count)d +MARK=/data/.job-key +RESUME=false +LCL="" +if [ -f "$MARK" ] && [ "$(cat "$MARK" 2>/dev/null)" = "$KEY" ]; then + PREV_LOG=$(ls -t /data/stellar-core*.log 2>/dev/null | head -n 1 || true) + if [ -n "$PREV_LOG" ]; then + LCL=$(grep -oE "Ledger close complete: [0-9]+" "$PREV_LOG" 2>/dev/null | tail -1 | grep -oE "[0-9]+$" || true) + fi + if [ -n "$LCL" ] && [ "$LCL" -ge $((TARGET - COUNT)) ] && [ "$LCL" -le "$TARGET" ] 2>/dev/null; then + RESUME=true; echo "RESUME: $KEY reached ledger $LCL, replay had started; skipping new-db" + else + echo "RESUME DECLINED: $KEY last close was '${LCL:-none}' (need >= $((TARGET - COUNT))); bucket phase incomplete, starting fresh" + fi +fi +printf '%%s' "$KEY" > "$MARK" +if [ "$RESUME" != "true" ]; then + /usr/bin/stellar-core --conf /config/stellar-core.cfg new-db --console +fi +exec /usr/bin/stellar-core --conf /config/stellar-core.cfg catchup "$KEY" \ + --metric 'ledger.transaction.apply' --console +''' + + +def owner_ref(): + cm = core_v1.read_namespaced_config_map(f"{RUN_NAME}-stellar-core-config", NAMESPACE) + return [client.V1OwnerReference(api_version='v1', kind='ConfigMap', + name=cm.metadata.name, uid=cm.metadata.uid, + block_owner_deletion=True)] + + +def release_pvc(end): + """Drop a completed range's volume. + + The PVC exists so an interrupted range resumes at L+1; once the range has + succeeded there is nothing left to resume and the volume is dead weight. + They are owner-referenced to the release, so without this they all survive + until `helm uninstall` -- measured on ssc-test, 2032 bound PVCs and 79 TiB + of gp3 provisioned a third of the way through a 3982-range run, heading for + ~156 TiB and 3982 volumes against the account's volume ceiling. + + Best-effort: a failure here costs disk, never correctness, and the range is + already recorded complete. + """ + if STORAGE_MODE != 'pvc': + return + name = f"{RUN_NAME}-data-r{end}" + try: + core_v1.delete_namespaced_persistent_volume_claim(name, NAMESPACE) + metric_pvc_released.inc() + except ApiException as e: + if e.status != 404: + logger.warning("could not release PVC for completed range %s: %s", end, e) + + +def delete_job(end, attempt): + """Drop a finished Job once nothing more is owed by it. + + reconcile() lists every Job and Pod on each pass, so a finished Job is not + free: it inflates two LIST calls for as long as it lingers. At 2048-4096 + parallelism with a real OOM or spot-eviction rate that is hundreds of dead + objects per hour of run, and the apiserver pressure shows up as truncated + list responses long before anything else complains. + + Callers must have persisted whatever they need first -- the logs, .outcome + and .metrics all live on the monitor's volume by then, so the Job and its + pod carry no information once the range is recorded. Best-effort: on + failure JOB_TTL_SECONDS still reclaims it. + """ + try: + batch_v1.delete_namespaced_job(job_name(end, attempt), NAMESPACE, + propagation_policy='Background') + metric_jobs_reaped.inc() + except ApiException as e: + if e.status != 404: + logger.warning("could not delete finished job for range %s attempt %d: %s", + end, attempt, e) + + +def ensure_pvc(end, owner): + name = f"{RUN_NAME}-data-r{end}" + try: + core_v1.read_namespaced_persistent_volume_claim(name, NAMESPACE) + return name + except ApiException as e: + if e.status != 404: + raise + spec = client.V1PersistentVolumeClaimSpec( + access_modes=['ReadWriteOnce'], + resources=client.V1VolumeResourceRequirements(requests={'storage': STORAGE_SIZE})) + if STORAGE_CLASS: + spec.storage_class_name = STORAGE_CLASS + core_v1.create_namespaced_persistent_volume_claim(NAMESPACE, client.V1PersistentVolumeClaim( + metadata=client.V1ObjectMeta(name=name, owner_references=owner, + labels={LABEL_RUN: RUN_NAME, LABEL_RANGE: str(end)}), + spec=spec)) + return name + + +def eph_for_attempt(attempt): + """Ephemeral-storage size for attempt N, escalating after an eviction.""" + base_q = _quantity_bytes(LIM_EPHEMERAL) + want = int(base_q * (EPH_BUMP_FACTOR ** max(0, attempt - 1))) + return _bytes_to_quantity(min(want, _quantity_bytes(EPH_ESCALATION_CAP))) + + +def load_profile(): + """Per-range measurements from an earlier run, keyed by range end. + + Absent, unreadable or malformed all mean the same thing: size from the + configured defaults. A profile is an optimisation, never a prerequisite. + """ + if not PROFILE_PATH: + return [] + try: + with open(PROFILE_PATH) as fh: + doc = json.load(fh) + except (OSError, ValueError) as e: + logger.warning("range profile %s unreadable (%s); using configured requests", + PROFILE_PATH, e) + return [] + mode = doc.get('storageMode') + cross_mode = bool(mode) and mode != STORAGE_MODE + if cross_mode: + # cpu and memory carry across modes -- they measure the same work. Disk + # does not: a pvc run puts /data on the volume, so it never measures + # node-local usage, and an ephemeral run's figure says nothing about a + # pvc one. Keep the transferable axes and let disk fall back to the + # configured default. + logger.warning("range profile is for storageMode=%s but this run is %s; " + "using its cpu and memory, defaulting ephemeral storage", + mode, STORAGE_MODE) + out = [] + for end, rec in (doc.get('ranges') or {}).items(): + try: + end = int(end) + except (TypeError, ValueError): + continue + if cross_mode: + rec = {k: v for k, v in rec.items() if k != 'peakEphemeralBytes'} + out.append((end, rec)) + out.sort() + logger.info("loaded range profile: %d ranges from %s", len(out), PROFILE_PATH) + return out + + +PROFILE = None + + +def profile_for(end): + """Measurements to size this range from, or None to use the defaults. + + Exact end, else the nearest measured end ABOVE it. Cost rises with ledger + position -- the bucket set only grows -- so a lower neighbour under-reports, + and under-provisioning costs an eviction while over-provisioning only costs + packing. Past the top of the profile there is nothing safe to extrapolate + from, so fall back to the configured defaults. + """ + if not PROFILE: + return None + end = int(end) + idx = bisect.bisect_left(PROFILE, (end,)) + if idx < len(PROFILE) and PROFILE[idx][0] == end: + return PROFILE[idx][1] + return PROFILE[idx][1] if idx < len(PROFILE) else None + + +def _cpu_millis(q): + return int(float(q[:-1])) if str(q).endswith('m') else int(float(q) * 1000) + + +def _sized_cpu(cores, margin, cap): + """A measured core count turned into a request, never above the limit.""" + return f"{min(int(cores * 1000 * margin), _cpu_millis(cap))}m" + + +def _sized(value, margin, cap): + """A measured peak turned into a request: margin applied, never above cap.""" + want = int(value * margin) + return _bytes_to_quantity(min(want, _quantity_bytes(cap))) + + +def _profile_overrides(end, escalated): + """Request overrides for this range from the profile, or {} for none. + + Escalated retries opt out: an escalation is a measurement of THIS run and + outranks anything an earlier one saw. + """ + if escalated or end is None: + return {} + prof = profile_for(end) + if not prof: + return {} + out = {} + # peakAnonBytes is kubelet's rssBytes, sampled by the collector on its own + # poll; peakRssBytes is the same quantity via a 30s Prometheus scrape. Prefer + # the finer one and fall back, so a profile captured before the collector + # tracked anon still sizes exactly as it used to. + rss = prof.get('peakAnonBytes') or prof.get('peakRssBytes') + if rss: + want = int(rss * PROFILE_MARGIN) + _quantity_bytes(PROFILE_CACHE_HEADROOM) + out['memory'] = _bytes_to_quantity(min(want, _quantity_bytes(PROFILE_MAX_MEM))) + disk = prof.get('peakEphemeralBytes') + if disk and LIM_EPHEMERAL: + out['ephemeral-storage'] = _sized(disk, PROFILE_MARGIN, LIM_EPHEMERAL) + return out + + +def _resources(mem=None, eph=None, end=None): + # Before mem is defaulted below -- reading it afterwards can never see None, + # which silently disabled profile sizing entirely. + overrides = _profile_overrides(end, escalated=(mem is not None or eph is not None)) + mem = mem or LIM_MEM + # Raise the request alongside the limit on an escalated retry: a pod that + # OOMed at the old limit will not fit where it was scheduled before. + req_mem = REQ_MEM if mem == LIM_MEM else mem + req = {'cpu': REQ_CPU, 'memory': req_mem} + lim = {'cpu': LIM_CPU, 'memory': mem} + + # Only meaningful in ephemeral mode. In PVC mode a large request makes disk + # the binding dimension and halves workers-per-node for no reason. + if REQ_EPHEMERAL: + # Raise the request with the limit: ephemeral-storage is a scheduling + # dimension, so a pod that outgrew its limit will not fit where it was + # placed before. + req['ephemeral-storage'] = eph or REQ_EPHEMERAL + else: + # pvc mode: /data is not on the node disk, so an ephemeral override + # would size a dimension this run does not use. + overrides.pop('ephemeral-storage', None) + if LIM_EPHEMERAL: + lim['ephemeral-storage'] = eph or LIM_EPHEMERAL + + if overrides: + # Memory and disk match request to limit: those are the dimensions worth + # pinning, since exceeding either kills the pod outright. + # + # CPU is deliberately not matched. Its limit stays where it is + # configured and only the request follows the measurement, so a range + # packs by what it actually uses while keeping headroom to burst. That + # leaves the pod Burstable rather than Guaranteed -- Kubernetes needs + # all three to match -- which is the intended trade. + cpu = overrides.pop('cpu', None) + if cpu: + req['cpu'] = cpu + if PROFILE_CPU_LIMIT: + lim['cpu'] = PROFILE_CPU_LIMIT + else: + lim.pop('cpu', None) + for key, value in overrides.items(): + req[key] = lim[key] = value + # Unmeasured range: the configured defaults, requests below limits, exactly + # as before -- a range with no profile entry must behave as if there were no + # profile at all. + return client.V1ResourceRequirements(requests=req, limits=lim) + + +def volume_spread_constraints(): + """Keep PVC-mounting workers under the per-node EBS attachment limit. + + Only in pvc mode: in ephemeral mode /data is an emptyDir, no volume is + attached, and spreading would just cost density. + """ + if STORAGE_MODE != 'pvc' or MAX_VOLUMES_PER_NODE <= 0: + return None + min_domains = max(1, -(-PARALLELISM // MAX_VOLUMES_PER_NODE)) # ceil + return [client.V1TopologySpreadConstraint( + max_skew=MAX_VOLUMES_PER_NODE, + min_domains=min_domains, + topology_key='kubernetes.io/hostname', + when_unsatisfiable='DoNotSchedule', + label_selector=client.V1LabelSelector(match_labels={LABEL_RUN: RUN_NAME}))] + + +def pod_labels(end): + labels = {LABEL_RUN: RUN_NAME, LABEL_RANGE: str(end)} + if EMIT_MISSION_LABEL and MISSION: + labels['mission'] = MISSION + return labels + + +def build_job(end, count, attempt, owner, mem=None, eph=None): + key = job_key(end, count) + script = RESUME_SCRIPT % {'key': key, 'target': end, 'count': count} + + if STORAGE_MODE == 'pvc': + data_vol = client.V1Volume(name='data', persistent_volume_claim=( + client.V1PersistentVolumeClaimVolumeSource(claim_name=ensure_pvc(end, owner)))) + else: + data_vol = client.V1Volume(name='data', empty_dir=client.V1EmptyDirVolumeSource()) + + env = [client.V1EnvVar(name='ASAN_OPTIONS', value=ASAN_OPTIONS)] if ASAN_OPTIONS else [] + + affinity = None + if NODE_LABEL_KEY: + affinity = client.V1Affinity(node_affinity=client.V1NodeAffinity( + required_during_scheduling_ignored_during_execution=client.V1NodeSelector( + node_selector_terms=[client.V1NodeSelectorTerm(match_expressions=[ + client.V1NodeSelectorRequirement(key=NODE_LABEL_KEY, operator='In', + values=[NODE_LABEL_VALUE])])]))) + # Taint value must be absent: the mission emits {key, effect} with no value, + # and the default Equal operator does not match "" against "true". + tolerations = [client.V1Toleration(key=TOLERATE_TAINT, effect='NoSchedule')] if TOLERATE_TAINT else None + + container = client.V1Container( + name='stellar-core', image=CORE_IMAGE, + command=['/bin/sh', '-c', script], env=env, resources=_resources(mem, eph, end), + ports=[client.V1ContainerPort(container_port=11626, name='http')], + volume_mounts=[client.V1VolumeMount(name='data', mount_path='/data'), + client.V1VolumeMount(name='config', mount_path='/config')]) + + return client.V1Job( + metadata=client.V1ObjectMeta( + name=job_name(end, attempt), owner_references=owner, + labels={LABEL_RUN: RUN_NAME, LABEL_RANGE: str(end), + LABEL_ATTEMPT: str(attempt)}), + spec=client.V1JobSpec( + # The monitor owns retries, not the Job controller. With + # backoffLimit>0 the controller would replace the pod on its own + # schedule, so we could not classify disruption vs genuine catchup + # failure, could not count evictions, and could not guarantee the + # log is archived before the next attempt starts. 0 means the Job + # fails once and stays put for inspection; reconcile() decides + # whether to dispatch attempt N+1. + # + # A podFailurePolicy would be inert here -- with backoffLimit 0 every + # pod failure already fails the Job, so Count and FailJob collapse to + # the same outcome. Classification is done by reading the pod's + # DisruptionTarget condition instead. + backoff_limit=0, + pod_failure_policy=client.V1PodFailurePolicy( + rules=[r for _, r in _failure_rules()]), + ttl_seconds_after_finished=JOB_TTL_SECONDS, + active_deadline_seconds=ATTEMPT_DEADLINE_SECONDS or None, + template=client.V1PodTemplateSpec( + metadata=client.V1ObjectMeta(labels=pod_labels(end)), + spec=client.V1PodSpec( + # IRSA for the S3 history mirror. Without it workers fall + # back to the public archive, which throttles at 1024. + service_account_name=WORKER_SERVICE_ACCOUNT or None, + # Keeps PVC-mounting workers under the per-node EBS + # attachment cap; inert at realistic CPU-bound density. + topology_spread_constraints=volume_spread_constraints(), + # Never, so a failed container is not restarted in place: + # the pod stays terminal and inspectable for classification + # and for the backstop log read. + restart_policy='Never', + termination_grace_period_seconds=WORKER_GRACE_SECONDS, + affinity=affinity, tolerations=tolerations, + containers=[container], + volumes=[data_vol, client.V1Volume( + name='config', config_map=client.V1ConfigMapVolumeSource( + name=f"{RUN_NAME}-stellar-core-config"))])))) + + +# --- reconcile -------------------------------------------------------------- + +def sync_counters(progress, counted): + """Drive the counters from persisted state instead of from events. + + Two reasons not to .inc() as things happen: + + * a terminally-failed range stays the newest Job for its range, so an + event-driven inc fires again on every reconcile until teardown + * the process resets to zero on restart, while the underlying record + (attempts in the progress ConfigMap, .outcome files on the PVC) survives + + Computing the true total and incrementing by the delta is monotonic, + idempotent, and self-heals after a restart: the counter starts at 0 and the + first sync walks it up to the recorded total. + """ + retries = 0 + for rec in list(progress.get('completed', {}).values()) + list(progress.get('failed', {}).values()): + retries += max(0, int(rec.get('attempts', 1)) - 1) + + oom = evicted = 0 + try: + for name in os.listdir(LOG_DIR): + if not name.endswith('.outcome'): + continue + try: + with open(os.path.join(LOG_DIR, name)) as fh: + o = json.load(fh).get('outcome') + except (OSError, ValueError): + continue + if o == 'oom': + oom += 1 + elif o == 'disrupted': + evicted += 1 + except OSError: + pass + + for key, total, metric in (('retries', retries, metric_retries), + ('oom', oom, metric_oom_retries), + ('evicted', evicted, metric_evictions)): + delta = total - counted.get(key, 0) + if delta > 0: + metric.inc(delta) + counted[key] = total + + +def observe_recorded(progress, replayed): + """Feed recorded completions into the histograms. + + Prometheus histograms are append-only and reset to zero when the process + restarts, so replaying every recorded range rebuilds the exact cumulative + total rather than double counting. Guarded per-process by `replayed`. + """ + for end, rec in progress.get('completed', {}).items(): + if end in replayed: + continue + replayed.add(end) + # `is not None`, not truthiness: a range with sum = 0ms records + # txApply 0.0, which is a real observation and must not be dropped + # silently. Same for a sub-second duration. + if rec.get('seconds') is not None: + metric_full_duration.observe(rec['seconds']) + if rec.get('wallSeconds') is not None: + metric_wall_duration.observe(rec['wallSeconds']) + if rec.get('txApply') is not None: + metric_tx_apply_duration.observe(rec['txApply']) + + +def pods_by_job(): + """One list per reconcile, indexed by Job name. + + This used to be a LIST per completed job, so a busy cycle at 1024 workers + issued dozens of round trips for one pod each. + """ + out = {} + for p in core_v1.list_namespaced_pod( + NAMESPACE, label_selector=f"{LABEL_RUN}={RUN_NAME}").items: + jn = (p.metadata.labels or {}).get('batch.kubernetes.io/job-name') + if jn: + out.setdefault(jn, p) + return out + + +def was_disrupted(pod): + for cond in (pod.status.conditions or []): + if cond.type == 'DisruptionTarget' and cond.status == 'True': return True - except requests.exceptions.RequestException: - if attempt < retries: - logger.info("Worker %s unreachable (attempt %d/%d), retrying in %ds", - pod_name, attempt, retries, STUCK_JOB_PING_DELAY_SECS) - time.sleep(STUCK_JOB_PING_DELAY_SECS) return False + +def reconcile(state): + ranges = generate_ranges() + by_end = {str(end): count for end, count in ranges} + progress = load_progress() + completed = progress.setdefault('completed', {}) + failed = progress.setdefault('failed', {}) + + jobs = batch_v1.list_namespaced_job( + NAMESPACE, label_selector=f"{LABEL_RUN}={RUN_NAME}").items + job_pods = pods_by_job() + + live = {} # range-end -> (attempt, job) + for j in jobs: + end = (j.metadata.labels or {}).get(LABEL_RANGE) + attempt = int((j.metadata.labels or {}).get(LABEL_ATTEMPT, 1)) + prev = live.get(end) + if prev is None or attempt >= prev[0]: + live[end] = (attempt, j) + + in_progress = [] + for end, (attempt, j) in list(live.items()): + st = j.status + if st.succeeded: + # Record BEFORE the Job's TTL can reclaim it: the per-attempt + # `seconds` below is the pod's own start -> finish, and Karpenter + # removes the pod ~1 min after the node empties. tx_apply no longer + # depends on this window -- the collector persists it from the + # stream. + if end not in completed: + pod = job_pods.get(j.metadata.name) + # Job.startTime is the FIRST attempt, so it spans retries. The + # successful pod's own start -> container finish is what + # worker.sh used to report, and is the number comparable across + # the redis cutover. + seconds = None + if pod is not None and pod.status.start_time: + for cs in (pod.status.container_statuses or []): + t = cs.state.terminated if cs.state else None + if t is not None and t.finished_at: + seconds = (t.finished_at - pod.status.start_time).total_seconds() + break + wall = None + if st.start_time and st.completion_time: + wall = (st.completion_time - st.start_time).total_seconds() + if seconds is None: + seconds = wall # pod already gone; wall is the only figure left + # Not gated on `pod`: the collector's .metrics/.log.gz are + # written from the live stream and outlive the pod, so a reaped + # node must not cost us the metric. + tx = tx_apply_for_range(end, attempt, + pod.metadata.name if pod else None) + if pod is not None and SAVE_SUCCESS_LOGS: + backstop_save_pod_log(pod.metadata.name, end, attempt) + if tx is None: + logger.warning("could not read tx_apply for range %s (pod gone?); " + "metric will be missing for this range", end) + completed[end] = {'seconds': seconds, 'wallSeconds': wall, + 'txApply': tx, 'attempts': attempt} + # Ledger count travels with the record: the logarithmic + # generator varies it per range, so it cannot be recomputed + # from config alone when the profile is read back. + if by_end.get(end) is not None: + completed[end]['count'] = by_end[end] + completed[end].update(peaks_for_range(end, attempt)) + # Durably recorded first: if this process dies between the two, + # the range is still complete and simply keeps its volume. + save_progress(progress) + release_pvc(end) + # Only once the record is complete. `tx is None` means the + # collector had not flushed this range's .metrics yet, and the + # pod is the only place left to read it from -- deleting the + # Job would reap the pod and make that gap permanent. Leave + # those to JOB_TTL_SECONDS. + if tx is not None: + delete_job(end, attempt) + elif st.failed: + pod = job_pods.get(j.metadata.name) + if pod is not None: + record_outcome(end, attempt, pod) + backstop_save_pod_log(pod.metadata.name, end, attempt) + # Written by the log collector while the pod still existed; reading + # the pod here would miss anything Karpenter already reaped. + # 1. pod-derived verdict, recorded by the collector while it lived + # 2. Job condition -- survives node consolidation, less precise + # 3. unknown -- retry rather than condemn the run + verdict = read_outcome(end, attempt) or classify_from_job(j) + if verdict is None: + verdict = {'outcome': 'unknown', 'exitCode': None} + elif verdict.get('source') == 'job-condition': + logger.info("range %s attempt %d classified from Job condition " + "(exit %s); pod was already gone", + end, attempt, verdict.get('exitCode')) + + retry_mem = retry_eph = None + if verdict['outcome'] == 'timeout': + reason = (f"exceeded the {ATTEMPT_DEADLINE_SECONDS}s attempt deadline " + "(stuck retrying the history archive?)") + elif verdict['outcome'] == 'rejected': + reason = f"rejected by the node before starting ({verdict.get('reason', '?')})" + elif verdict['outcome'] == 'disrupted': + reason = "lost to node disruption" + elif verdict['outcome'] == 'oom': + base = (_profile_overrides(end, escalated=False) or {}).get('memory') + retry_mem = mem_for_attempt(attempt + 1, base) + reason = f"OOM-killed at memory limit {mem_for_attempt(attempt, base)}" + elif verdict['outcome'] == 'ephemeral': + retry_eph = eph_for_attempt(attempt + 1) + reason = (f"evicted for exceeding its {eph_for_attempt(attempt)} " + f"ephemeral-storage limit") + elif verdict['outcome'] == 'unknown': + # The pod was gone before anything classified it -- almost always + # because this process was down while the node was reaped. An + # unclassified failure is NOT evidence of a bad ledger range, and + # condemning the run on it would let a monitor restart fail a + # 10-hour job. Retry; a genuinely broken range will exhaust its + # attempts and fail with evidence. + reason = "failed with no surviving classification (monitor restart?)" + else: + reason = None # genuine catchup failure: do not retry + + # Three budgets, by whose fault the attempt was: a hang is usually + # persistent and gets the lowest, a range that is genuinely broken + # gets the middle one, and anything the cluster did to us gets the + # highest. + if verdict['outcome'] == 'timeout': + cap = MAX_TIMEOUT_ATTEMPTS + elif verdict['outcome'] == 'ephemeral': + cap = MAX_EPHEMERAL_ATTEMPTS + elif verdict['outcome'] in ENVIRONMENTAL_OUTCOMES: + cap = MAX_DISRUPTION_ATTEMPTS + else: + cap = MAX_ATTEMPTS_PER_RANGE + if reason is not None and attempt < cap: + if verdict['outcome'] == 'oom': + logger.error( + "!!! OOM RETRY !!! range %s was OOM-killed on attempt %d/%d; retrying with " + "memory limit %s -- RAISE THE CONFIGURED MEMORY LIMIT, this run is only " + "surviving by escalating at runtime", end, attempt, MAX_ATTEMPTS_PER_RANGE, retry_mem) + elif verdict['outcome'] == 'ephemeral': + metric_eph_retries.inc() + logger.error( + "!!! DISK RETRY !!! range %s %s on attempt %d/%d; retrying with " + "ephemeral-storage %s -- RAISE THE CONFIGURED EPHEMERAL STORAGE, this " + "run is only surviving by escalating at runtime", + end, reason, attempt, cap, retry_eph) + else: + logger.warning("range %s %s on attempt %d/%d; retrying", + end, reason, attempt, MAX_ATTEMPTS_PER_RANGE) + try: + batch_v1.create_namespaced_job(NAMESPACE, build_job( + int(end), by_end[end], attempt + 1, state['owner'], retry_mem, retry_eph)) + except ApiException as e: + if e.status != 409: + raise + # After the successor exists, never before. If the create above + # had failed with the predecessor already gone, the range would + # have no live Job at all and the next pass would redispatch it + # at attempt 1 -- losing the escalated memory that is the whole + # point of the retry. live[] keys on the highest attempt, so the + # two coexisting for one pass is already handled. + delete_job(end, attempt) + in_progress.append(job_key(int(end), by_end[end])) + continue + if reason is not None: + logger.error("range %s exhausted %d attempts (%s)", end, cap, reason) + + if end not in failed: + failed[end] = {'attempts': attempt, + 'pod': verdict.get('pod', pod.metadata.name if pod else ''), + 'outcome': verdict['outcome'], + 'exitCode': verdict['exitCode']} + save_progress(progress) + else: + in_progress.append(job_key(int(end), by_end.get(end, 0))) + + # Monotonic progress is invariant in a healthy run. A decrease means the + # durable record or the Jobs were tampered with; redoing hours of work + # silently is worse than stopping. + if len(completed) < state['max_completed']: + logger.error("PROGRESS WENT BACKWARDS: completed %d -> %d. Refusing to dispatch. " + "The progress ConfigMap or the Jobs were deleted underneath this run.", + state['max_completed'], len(completed)) + state['halted'] = True + state['max_completed'] = max(state['max_completed'], len(completed)) + + # Dispatch, heaviest range first (index 0 is the tip), up to PARALLELISM. + created = 0 + if not state['halted'] and not failed: + # No slots: a range's PVC is keyed by the range itself, so concurrency is + # simply how many are in flight. + capacity = PARALLELISM - len(in_progress) + for end, count in ranges: + if capacity <= 0: + break + key = str(end) + if key in completed or key in failed or key in live: + continue + try: + batch_v1.create_namespaced_job(NAMESPACE, build_job( + end, count, 1, state['owner'])) + created += 1 + capacity -= 1 + in_progress.append(job_key(end, count)) + except ApiException as e: + if e.status != 409: # AlreadyExists: name uniqueness is the mutex + raise + + observe_recorded(progress, state['replayed']) + sync_counters(progress, state['counted']) + return { + 'total': len(ranges), + 'completed': len(completed), + 'failed_ranges': [f"{job_key(int(k), by_end.get(k, 0))}|{v.get('pod', '')}" + for k, v in failed.items()], + 'in_progress': in_progress, + 'created': created, + 'remaining': len(ranges) - len(completed) - len(failed) - len(in_progress), + } + + +def read_mission_start(): + """When this run first started, or None if not recorded yet. + + Its own ConfigMap key, not a field in progress.json: that document is keyed + by ledger range, and anything else in it would be walked as if it were one. + + Read-only on purpose. Creating the ConfigMap here would race the owner + reference, which is only known once reconcile has resolved it, and an + ownerless progress ConfigMap survives `helm uninstall`. + """ + try: + cm = core_v1.read_namespaced_config_map(PROGRESS_CM, NAMESPACE) + return float((cm.data or {})['started_at']) + except (ApiException, KeyError, TypeError, ValueError): + return None + + def update_status_and_metrics(): global status - mission_start_time = time.time() + # None until reconcile has an owner reference to attach it to; until then + # process start is correct anyway, because that IS the start of a new run. + mission_start_time = read_mission_start() or time.time() + check_storage_config() + state = {'owner': None, 'replayed': set(), 'max_completed': 0, 'halted': False, + 'counted': {}} while True: try: - # --- Phase 1: Read queue status from Redis (fast, authoritative) --- - queue_remain_count = redis_client.llen(JOB_QUEUE) - queue_succeeded_count = redis_client.llen(SUCCESS_QUEUE) - jobs_failed = redis_client.lrange(FAILED_QUEUE, 0, -1) - jobs_in_progress = redis_client.lrange(PROGRESS_QUEUE, 0, -1) - queue_failed_count = len(jobs_failed) - queue_in_progress_count = len(jobs_in_progress) - - # --- Phase 2: Quick single-ping check of workers that own in-progress jobs --- - job_owners = redis_client.hgetall(JOB_OWNERS) # {job_key: pod_name} - active_workers = set(job_owners.values()) - worker_statuses = [] - workers_up = 0 - workers_down = 0 - - logger.info("Starting worker liveness check for %d active workers", len(active_workers)) - workers_refresh_start_time = time.time() - for pod_name in active_workers: - if ping_worker(pod_name, retries=1): - worker_statuses.append({'pod': pod_name, 'status': 'running'}) - workers_up += 1 - else: - worker_statuses.append({'pod': pod_name, 'status': 'down'}) - workers_down += 1 - workers_refresh_duration = time.time() - workers_refresh_start_time - logger.info("Finished workers liveness check") - - # --- Phase 3: Retry stuck jobs (only if ALL active workers down AND in_progress non-empty) --- - # Note: queue_in_progress_count is from Phase 1 and may be stale by this point. - # If a job completed during Phase 2 pings, we may enter this block unnecessarily. - # This is harmless: retry_jobs_in_progress() will find an empty in_progress queue - # and exit immediately. We waste at most 90 seconds (30 secs x 3 pings) per stale worker. - if workers_down > 0 and workers_up == 0 and queue_in_progress_count > 0: - logger.warning("All %d active workers appear down with %d jobs in progress. " - "Entering retry confirmation.", workers_down, queue_in_progress_count) - - # Re-ping each active worker with retries to confirm they're truly down - any_recovered = False - for pod_name in active_workers: - if ping_worker(pod_name, retries=STUCK_JOB_PING_RETRIES): - logger.info("Worker %s responded during retry confirmation. Exiting retry block.", pod_name) - any_recovered = True - break - - if not any_recovered: - logger.error("All active workers confirmed down after %d retry attempts. Retrying stuck jobs.", - STUCK_JOB_PING_RETRIES) - retry_jobs_in_progress() - for job_key in list(job_owners.keys()): - redis_client.hdel(JOB_OWNERS, job_key) - else: - logger.info("At least one worker recovered. Skipping job retry.") + reconcile_alive['ts'] = time.time() + if state['owner'] is None: + state['owner'] = owner_ref() + _progress_owner['ref'] = state['owner'] + if read_mission_start() is None: + _patch_cm({'started_at': repr(mission_start_time)}) - mission_duration = time.time() - mission_start_time + r = reconcile(state) + + # Liveness of the workers that currently own a job -- idle slots are + # deliberately not counted, matching the original metric. + pods = [(p.metadata.name, p.status.pod_ip) + for p in core_v1.list_namespaced_pod( + NAMESPACE, label_selector=f"{LABEL_RUN}={RUN_NAME}", + field_selector='status.phase=Running', + # Served from the apiserver watch cache. Only safe here: + # a stale liveness sample is cosmetic, whereas stale + # dispatch state would re-run a range. + resource_version='0').items + if p.status.pod_ip] + refresh_start = time.time() + ping = ping_workers(pods) + workers_refresh_duration = time.time() - refresh_start + worker_statuses = [{'pod': p, 'status': 'running' if ok else 'down'} + for p, ok in ping.items()] + workers_up = sum(1 for ok in ping.values() if ok) + workers_down = len(ping) - workers_up - # Update the status + mission_duration = time.time() - mission_start_time with status_lock: status = { - 'num_remain': queue_remain_count, # Needed for backwards compatibility with the rest of the code - 'queue_remain_count': queue_remain_count, - 'queue_succeeded_count': queue_succeeded_count, - 'queue_failed_count': queue_failed_count, - 'queue_in_progress_count': queue_in_progress_count, - 'jobs_failed': jobs_failed, - 'jobs_in_progress': jobs_in_progress, + 'num_remain': r['remaining'], + 'queue_remain_count': r['remaining'], + 'queue_succeeded_count': r['completed'], + 'queue_failed_count': len(r['failed_ranges']), + 'queue_in_progress_count': len(r['in_progress']), + 'jobs_failed': r['failed_ranges'], + 'jobs_in_progress': r['in_progress'], 'workers': worker_statuses, 'workers_up': workers_up, 'workers_down': workers_down, 'workers_refresh_duration': workers_refresh_duration, 'mission_duration': mission_duration, } - metric_catchup_queues.labels(queue="remain").set(queue_remain_count) - metric_catchup_queues.labels(queue="succeeded").set(queue_succeeded_count) - metric_catchup_queues.labels(queue="failed").set(queue_failed_count) - metric_catchup_queues.labels(queue="in_progress").set(queue_in_progress_count) + metric_catchup_queues.labels(queue="remain").set(r['remaining']) + metric_catchup_queues.labels(queue="succeeded").set(r['completed']) + metric_catchup_queues.labels(queue="failed").set(len(r['failed_ranges'])) + metric_catchup_queues.labels(queue="in_progress").set(len(r['in_progress'])) metric_workers.labels(status="up").set(workers_up) metric_workers.labels(status="down").set(workers_down) metric_refresh_duration.set(workers_refresh_duration) metric_mission_duration.set(mission_duration) logger.info("Status: %s", json.dumps(status)) - - # update the metrics - new_metrics = redis_client.spop(METRICS, 1000) - if len(new_metrics) > 0: - with metrics_lock: - metrics['metrics'].extend(new_metrics) - for timing in new_metrics: - # Example: 36024000/8320|213|1.47073e+06ms|2965s - _, _, tx_apply, full_duration = timing.split('|') - metric_full_duration.observe(float(full_duration.rstrip('s'))) - metric_tx_apply_duration.observe(float(tx_apply.rstrip('ms'))/1000) - logger.info("Metrics: %s", json.dumps(metrics)) + # Publish on change only -- a 10h run would otherwise issue ~3600 + # no-op ConfigMap writes. + counts = (r['remaining'], r['completed'], len(r['failed_ranges']), len(r['in_progress'])) + if counts != state.get('last_counts'): + state['last_counts'] = counts + with status_lock: + save_status(status) except Exception as e: - logger.error("Error while getting status: %s", str(e)) + logger.exception("Error while reconciling: %s", str(e)) + + time.sleep(RECONCILE_INTERVAL_SECONDS) - time.sleep(LOGGING_INTERVAL_SECONDS) def run(server_class=HTTPServer, handler_class=RequestHandler): server_address = ('', 8080) @@ -240,9 +1715,18 @@ def run(server_class=HTTPServer, handler_class=RequestHandler): logger.info('Starting httpd server...') httpd.serve_forever() + if __name__ == '__main__': - log_thread = threading.Thread(target=update_status_and_metrics) - log_thread.daemon = True - log_thread.start() + # Before any dispatch: the first Job built must already be sized from it. + PROFILE = load_profile() + + # Not a logging thread despite the historical name -- this is the reconcile + # loop: dispatch, progress record, metrics, status. Log capture and pod + # classification live in the log-collector sidecar. + reconcile_thread = threading.Thread(target=update_status_and_metrics) + reconcile_thread.daemon = True + reconcile_thread.start() + # Separate thread: a blocking watch must not sit behind dispatch and the + # liveness sweep, which is the whole point of it. run() diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py new file mode 100644 index 00000000..9ad0b537 --- /dev/null +++ b/src/MissionParallelCatchup/log_collector.py @@ -0,0 +1,600 @@ +"""Streaming log collector for parallel catchup. + +Runs as a sidecar next to job_monitor, sharing its /logs volume. + +Why stream rather than read logs after a Job finishes: worker pods are one per +ledger range, and Karpenter deletes the node roughly a minute after its last +running pod exits, taking every pod object with it. Anything that reads after +the fact is racing that deletion. Holding `follow=true` from pod start means we +already have everything the pod wrote by the time it disappears -- and it makes +a straggler's log readable *while* it is stuck, which is the case that turns a +5h run into a 10h one. + +Resume is idempotent across both a dropped stream and a restart of this +process: + + coarse reconnect with sinceTime=; the API + only accepts second granularity, so this deliberately overlaps + precise every line carries a kubelet RFC3339Nano timestamp (timestamps=true), + so drop any line <= last_ts. That removes the overlap exactly and + does not depend on stellar-core's own log format. + +Residual: if this dies between flushing log bytes and rewriting the state file, +the next run replays from a slightly older timestamp and a few lines duplicate. +Bounded by STATE_FLUSH_SECONDS. "At least once, deduped to near-exact" rather +than exactly once. +""" + +import asyncio +import gzip +import json +import logging +import os +import re +import ssl +import sys + +import aiohttp + +NAMESPACE = os.getenv('NAMESPACE', 'default') +RUN_NAME = os.getenv('RUN_NAME', 'parallel-catchup') +LOG_DIR = os.getenv('LOG_DIR', '/logs') +CONTAINER = os.getenv('WORKER_CONTAINER', 'stellar-core') +POLL_SECONDS = float(os.getenv('COLLECTOR_POLL_SECONDS', 5)) +STATE_FLUSH_SECONDS = float(os.getenv('STATE_FLUSH_SECONDS', 10)) +MAX_CONCURRENT = int(os.getenv('COLLECTOR_MAX_STREAMS', 1200)) +# Poll cycles a stream gets to finalize itself after its pod leaves the pod list +# before it is cancelled outright. One cycle is usually enough; the margin is for +# a stream still finalizing: writing its .metrics and closing its archive. +VANISHED_GRACE_CYCLES = int(os.getenv('COLLECTOR_VANISHED_GRACE_CYCLES', 3)) +# Whether to keep the archive for a range that succeeded. Enforced here rather +# than in job_monitor: we cannot know in advance whether a range will fail, so +# the stream always runs and the archive is discarded on success instead. +SAVE_SUCCESS_LOGS = os.getenv('SAVE_SUCCESS_LOGS', 'true').lower() == 'true' +# Peak working set per range, for sizing a later run's requests. Empty disables. +# Queried rather than sampled: cgroup memory.peak counts page cache (measured: +# 1.5GB peak for a process using 0.3MB of anon), and a sampler inside the worker +# would mean dropping the `exec`, which is what keeps stellar-core at PID 1 and +# able to see SIGTERM. +STORAGE_MODE = os.getenv('STORAGE_MODE', 'pvc') +# Peak memory now comes from kubelet, not Prometheus. kubelet reports rssBytes +# and workingSetBytes per container in the same /stats/summary payload this +# already fetches for ephemeral storage, at ~10s cAdvisor housekeeping against a +# 30s scrape -- and without depending on Prometheus being up, being reachable, +# or still retaining the window. cpu is not sampled at all: the request is fixed +# at REQ_CPU, so a measured value has nothing to size. +# Peaks are held per pod and flushed on significant growth, so a restart loses +# at most PEAK_FLUSH_RATIO of a range's high-water rather than all of it -- +# Prometheus's server-side max_over_time needed no such state. +PEAK_FLUSH_RATIO = float(os.getenv('PEAK_FLUSH_RATIO', 1.05)) + +LABEL_RUN = 'catchup.stellar.org/run' +LABEL_RANGE = 'catchup.stellar.org/range-end' +LABEL_ATTEMPT = 'catchup.stellar.org/attempt' + +SA = '/var/run/secrets/kubernetes.io/serviceaccount' +API = f"https://{os.getenv('KUBERNETES_SERVICE_HOST', 'kubernetes.default')}:{os.getenv('KUBERNETES_SERVICE_PORT', '443')}" + +logging.basicConfig(level=os.getenv('LOGGING_LEVEL', 'INFO'), + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[logging.StreamHandler(sys.stdout)]) +logger = logging.getLogger('log-collector') + + +def token(): + # Projected service account tokens rotate, so this is re-read per request + # rather than cached at startup. + with open(os.path.join(SA, 'token')) as fh: + return fh.read().strip() + + +def ssl_ctx(): + return ssl.create_default_context(cafile=os.path.join(SA, 'ca.crt')) + + +def base(end, attempt): + return os.path.join(LOG_DIR, f"range-{end}-a{attempt}") + + +def read_state(end, attempt): + try: + with open(base(end, attempt) + '.state') as fh: + ts = fh.read().strip() + except OSError: + return None + # Also repairs a state file poisoned by an earlier build. + return ts if ts and _TS_RE.match(ts) else None + + +def write_state(end, attempt, ts): + path = base(end, attempt) + '.state' + tmp = path + '.tmp' + try: + with open(tmp, 'w') as fh: + fh.write(ts) + os.replace(tmp, path) + except OSError as e: + logger.warning("could not persist state for range %s: %s", end, e) + + +def discard(end, attempt): + # .metrics deliberately survives: it holds tx_apply for a range that + # succeeded, which is the only case this runs in. Dropping it would let a + # log-retention flag silently delete a Grafana series. + for suffix in ('.log.gz', '.state'): + try: + os.remove(base(end, attempt) + suffix) + except OSError: + pass + + +# kubelet returns plain text such as "unable to retrieve container logs for +# containerd://..." when a container is not up yet. That has no timestamp, so +# partitioning on the first space yields "unable", which then goes into the +# state file and every later request asks for sinceTime=unableZ -> HTTP 400, +# forever. Observed on ssc-test the moment evicted pods were replaced. +_TS_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?$") + +_TX_METRIC = "metric 'ledger.transaction.apply'" +# medida prints the sum in scientific notation once it exceeds 1e6 ms, which is +# every range that applies a real transaction load. The old [0-9.]+ pattern +# matched "1.30722" then demanded "ms" and hit "e+06ms" instead, so tx_apply was +# silently missing for 25% of ranges -- 91-99% of everything above ledger 35M, +# exactly the expensive end. +_SUM_RE = re.compile(r"sum\s*=\s*([0-9.]+(?:[eE][+-]?[0-9]+)?)ms") + + +class TxApplyScanner: + """Pull the medida tx-apply total out of the stream as it goes past. + + stellar-core prints this block once, just before exit. Scanning here rather + than re-reading the log later is what makes the metric independent of pod + lifetime: by the time job_monitor sees the Job succeed, Karpenter may + already have reaped the node, and with saveSuccessLogs=false the archive is + gone too. These bytes pass through this process exactly once, so this is the + only place guaranteed to see them. + """ + + WINDOW = 15 # same span job_monitor uses when reading an archive + + # Printed by RESUME_SCRIPT before stellar-core starts. Its counterpart, + # "RESUME DECLINED", means new-db ran and this attempt did the whole range, + # so the colon is load-bearing -- it is what separates the two. + RESUME_MARK = 'RESUME: ' + + def __init__(self): + self.seconds = None + self.resumed = False + self._left = 0 + + def feed(self, line): + if self.RESUME_MARK in line: + self.resumed = True + if _TX_METRIC in line: + self._left = self.WINDOW + return + if self._left <= 0: + return + self._left -= 1 + m = _SUM_RE.search(line) + if m: + self.seconds = float(m.group(1)) / 1000.0 + self._left = 0 + + + + + + + + +def write_metrics(end, attempt, values): + """Persist per-range measurements for job_monitor's reconcile to read. + + Kept out of .outcome on purpose: that file answers "why did this attempt + fail" and is only written for failed pods, whereas these are only + meaningful for one that succeeded. + """ + path = base(end, attempt) + '.metrics' + tmp = path + '.tmp' + # Merge: a measurement already on disk must survive a later write that + # lacks it. The ephemeral peak is held in memory, so a collector restart + # would otherwise let a rewrite drop it. + try: + with open(path) as fh: + values = {**json.load(fh), **values} + except (OSError, ValueError): + pass + try: + with open(tmp, 'w') as fh: + json.dump(values, fh) + os.replace(tmp, path) + logger.info("range %s attempt %s metrics=%s", end, attempt, values) + except OSError as e: + logger.warning("could not persist metrics for range %s: %s", end, e) + + +def classify(pod): + """Why did this pod fail? Recorded here rather than in job_monitor. + + This process already lists every pod every few seconds to discover streams, + so it sees terminal transitions first-hand -- a separate watch thread in the + monitor was observing the same objects a second time. The Job object cannot + answer this: its condition carries no exit code until a podFailurePolicy + rule matches, and an admission rejection matches none. + """ + status = pod.get('status', {}) + for cond in status.get('conditions', []): + if cond.get('type') == 'DisruptionTarget' and cond.get('status') == 'True': + return {'outcome': 'disrupted', 'exitCode': None} + reason = status.get('reason') + if reason == 'Evicted' and 'ephemeral' in (status.get('message') or ''): + # The range's own disk use, not something the cluster did to it. + # Measured on ssc-test: the kubelet sets no DisruptionTarget for a + # limit eviction, and stellar-core drains on the eviction SIGTERM and + # exits 3 -- so the Job condition matches the generic non-zero rule and + # reads as a plain catchup failure, which gets no retry at all. + # status.message is the only discriminator and only the pod carries it. + # Recording it here, while the pod still exists, is the only way to + # keep the signal. + return {'outcome': 'ephemeral', 'exitCode': None, 'reason': status.get('message')} + if reason in ('VolumeAttachmentLimitExceeded', 'OutOfcpu', 'OutOfmemory', 'OutOfpods', + 'UnexpectedAdmissionError', 'NodeAffinity', 'Shutdown', 'Evicted'): + return {'outcome': 'rejected', 'exitCode': None, 'reason': reason} + terms = [cs.get('state', {}).get('terminated') for cs in status.get('containerStatuses', [])] + terms = [t for t in terms if t] + if not terms: + # Nothing ever ran, so this says nothing about the ledger range. + return {'outcome': 'rejected', 'exitCode': None, 'reason': reason or 'no container status'} + for t in terms: + if t.get('reason') == 'OOMKilled': + return {'outcome': 'oom', 'exitCode': t.get('exitCode')} + if t.get('exitCode') not in (0, None): + return {'outcome': 'failed', 'exitCode': t.get('exitCode')} + return {'outcome': 'failed', 'exitCode': None} + + +def record_outcome(pod, end, attempt): + """Write the verdict next to the log, for job_monitor's reconcile to read.""" + path = base(end, attempt) + '.outcome' + if os.path.exists(path): + return + data = classify(pod) + data['pod'] = pod['metadata']['name'] + try: + tmp = path + '.tmp' + with open(tmp, 'w') as fh: + json.dump(data, fh) + os.replace(tmp, path) + logger.info("range %s attempt %s classified: %s", end, attempt, data['outcome']) + except OSError as e: + logger.warning("could not persist outcome for range %s: %s", end, e) + + +# Peak ephemeral disk, for sizing a later run's ephemeral-storage request. +# +# Only meaningful in ephemeral mode. Sampled for every pod, but only kept for +# ranges that finished -- see the completion gate in stream_pod. Spot is fine: +# what invalidates a sample is being cut short, not the capacity type. +# +# Prometheus cannot answer this -- cAdvisor reports fs usage per node, with no +# pod label -- so this samples kubelet directly through the apiserver proxy and +# keeps a running max. +_eph_peak = {} +_anon_peak = {} +_ws_peak = {} +_peak_flushed = {} +# pod name -> (end, attempt), so a mid-flight peak flush can find its file. +_streaming = {} + + + +async def sample_kubelet(session, nodes): + """Update each pod's peak ephemeral use and peak anon from one snapshot. + + Both axes come out of the same GET, so tracking memory here is free. + + kubelet's `rssBytes` is cgroup v2 `anon` -- measured against a live pod on + ssc-test it read 482 MiB while the cgroup reported 492 MiB seconds later. + Anon is the only limit-independent memory figure this workload has: page + cache expands to fill whatever `memory.max` allows, so `memory.peak` is + always ~= the limit (measured: a range needing 862 MiB of anon reported a + 12704 MiB peak when given a 24000 MiB limit) and is useless for sizing. + + Sampled rather than exact -- cAdvisor housekeeping is ~10s, so a shorter + anon spike is invisible. Still ~3x finer than the 30s Prometheus scrape the + profile used before, which is the undersampling that let profiled ranges + OOM. The `time` field on this payload runs 1-3s behind wall clock; the ~80s + lag applies only to the du-based ephemeral figure alongside it. + """ + for node in nodes: + url = f"{API}/api/v1/nodes/{node}/proxy/stats/summary" + try: + async with session.get(url, headers={'Authorization': f'Bearer {token()}'}) as resp: + resp.raise_for_status() + summary = await resp.json() + except Exception as e: + # Not debug: if this fails the ephemeral axis is silently empty and + # the profile looks merely "absent" rather than broken. + logger.warning("kubelet stats unavailable on %s: %s", node, e) + continue + for entry in summary.get('pods', []): + name = entry.get('podRef', {}).get('name') + if not name: + continue + used = (entry.get('ephemeral-storage') or {}).get('usedBytes') + if used is not None and STORAGE_MODE == 'ephemeral': + prev = _eph_peak.get(name, 0) + if int(used) > prev: + _eph_peak[name] = int(used) + logger.info("peak ephemeral for %s: %.2f GiB", name, used / 1073741824) + for c in entry.get('containers', []): + if c.get('name') != CONTAINER: + continue + # Absent for the first seconds of a container's life, before + # cAdvisor has stats for it. Every later poll carries it, so a + # miss here costs nothing: anon is at its lowest during startup. + mem = c.get('memory') or {} + ws = mem.get('workingSetBytes') + if ws is not None and int(ws) > _ws_peak.get(name, 0): + _ws_peak[name] = int(ws) + rss = mem.get('rssBytes') + if rss is None: + continue + if int(rss) <= _anon_peak.get(name, 0): + continue + _anon_peak[name] = int(rss) + # Held in memory until the stream ends, so a collector restart + # would otherwise reset a range's high-water to whatever it is + # using at that moment -- under-reporting, which sizes the next + # run too small. Flushing only on PEAK_FLUSH_RATIO growth keeps + # this to a handful of writes over a pod's life instead of one + # per sample per pod. + if int(rss) >= _peak_flushed.get(name, 0) * PEAK_FLUSH_RATIO: + _peak_flushed[name] = int(rss) + ref = _streaming.get(name) + if ref: + write_metrics(ref[0], ref[1], {'peakAnonBytes': int(rss)}) + + +async def finalize(session, pod, end, attempt, tx, done_ok): + """Persist everything this attempt owes, then let its stream go. + + Reached from two places: a clean end of stream once the pod is terminal, + and a 404 once the pod object is gone. The second path used to not exist, + so a pod deleted while Running -- reaped node, eviction, or the monitor + deleting a finished Job -- left its stream retrying every 30s for the rest + of the run, holding one of MAX_CONCURRENT connection slots the whole time. + """ + # Before discard: on success the archive is about to be deleted. + measured = {} + if tx.resumed: + # Not a peak -- PEAK_FIELDS filters it out of the profile. peaks_for_range + # reads it to decide how far back to aggregate: a resumed attempt only + # measured the tail of its range, so the attempt before it still counts. + measured['resumed'] = True + if tx.seconds is not None: + measured['txApplySeconds'] = tx.seconds + _peak_flushed.pop(pod, None) + _streaming.pop(pod, None) + anon = _anon_peak.pop(pod, None) + if anon is not None: + # Recorded for every attempt, not just the winner. peaks_for_range takes + # the max across attempts, so a partial attempt can only ever raise the + # figure, never lower it -- which is what makes a resumed range (pvc mode, + # killed once replay started) report the download-phase peak it actually + # hit rather than its tail. The monitor drops an attempt from the axis it + # died on, since an OOM-killed peak measures the limit, not demand. + measured['peakAnonBytes'] = anon + ws = _ws_peak.pop(pod, None) + if ws is not None: + # Diagnostic only -- working set counts active page cache, which grows + # to fill whatever limit the pod was given, so it must never size + # anything. Kept because the anon/ws gap is what tells you a range is + # cache-heavy rather than genuinely large. + measured['peakWorkingSetBytes'] = ws + eph = _eph_peak.pop(pod, None) + if eph is not None: + # Same as anon: max across attempts upstream, and an attempt evicted at + # its ephemeral limit is dropped from this axis there. + measured['peakEphemeralBytes'] = eph + if measured: + write_metrics(end, attempt, measured) + if not SAVE_SUCCESS_LOGS and done_ok(pod): + discard(end, attempt) + logger.info("range %s attempt %s: succeeded, archive discarded " + "(saveSuccessLogs=false)", end, attempt) + else: + logger.info("range %s attempt %s: stream complete", end, attempt) + + +async def stream_pod(session, pod, end, attempt, done, done_ok): + """Follow one pod's log until it terminates, appending to its archive.""" + path = base(end, attempt) + '.log.gz' + last_ts = read_state(end, attempt) + if last_ts is None: + # Empty state = "claimed, nothing durable yet". job_monitor's backstop + # skips any range with a state file, so this prevents both of us writing + # the same log. + write_state(end, attempt, '') + backoff = 1.0 + # Outside the reconnect loop: the medida block could straddle a dropped + # stream, and a fresh scanner per attempt would lose the half it saw. + tx = TxApplyScanner() + + while True: + params = {'container': CONTAINER, 'follow': 'true', 'timestamps': 'true'} + if last_ts: + # Second granularity, so this overlaps on purpose; the per-line + # comparison below removes the overlap exactly. + params['sinceTime'] = last_ts[:19] + 'Z' + url = f"{API}/api/v1/namespaces/{NAMESPACE}/pods/{pod}/log" + + try: + async with session.get(url, params=params, + headers={'Authorization': f'Bearer {token()}'}) as resp: + if resp.status == 404: + # Pod object gone -- reaped node, eviction, or the monitor + # deleting a finished Job. Nothing more to read, but the + # bytes already streamed still owe a tx_apply and the peaks + # are in Prometheus regardless. A bare return here dropped + # both for every pod that outlived its object. + logger.info("pod %s gone before/while streaming range %s", pod, end) + await finalize(session, pod, end, attempt, tx, done_ok) + return + resp.raise_for_status() + backoff = 1.0 + pending = None + # gzip append writes a new member; concatenated members are a + # valid archive, so restarts do not corrupt what is already there. + with gzip.open(path, 'at') as fh: + since_flush = asyncio.get_event_loop().time() + async for raw in resp.content: + line = raw.decode('utf-8', 'replace').rstrip('\n') + if not line: + continue + ts, _, rest = line.partition(' ') + if not _TS_RE.match(ts): + # Untimestamped kubelet text. Keep it, but never let + # it become the resume point. + fh.write(line + '\n') + continue + if last_ts and ts <= last_ts: + continue # exact dedup of the resume overlap + fh.write(rest + '\n') + tx.feed(rest) + pending = ts + now = asyncio.get_event_loop().time() + if now - since_flush >= STATE_FLUSH_SECONDS: + fh.flush() + write_state(end, attempt, pending) + last_ts = pending + since_flush = now + if pending: + write_state(end, attempt, pending) + last_ts = pending + # A clean end of stream means the container exited. + if done(pod): + await finalize(session, pod, end, attempt, tx, done_ok) + return + except asyncio.CancelledError: + raise + except Exception as e: + logger.info("range %s stream interrupted (%s); resuming from %s", + end, e, last_ts or 'start') + if done(pod): + # Reached when the last read threw rather than ending cleanly -- a + # 500 burst, a dropped connection -- and the pod has since gone + # terminal. The partial stream may already hold the medida block, + # and the peaks are query-side, so this owes exactly what the clean + # path owes. It used to return bare and lose both. + await finalize(session, pod, end, attempt, tx, done_ok) + return + await asyncio.sleep(backoff) + backoff = min(backoff * 2, 30) + + +async def list_pods(session): + url = f"{API}/api/v1/namespaces/{NAMESPACE}/pods" + params = {'labelSelector': f"{LABEL_RUN}={RUN_NAME}"} + async with session.get(url, params=params, + headers={'Authorization': f'Bearer {token()}'}) as resp: + resp.raise_for_status() + return (await resp.json()).get('items', []) + + +async def main(): + os.makedirs(LOG_DIR, exist_ok=True) + # Connection-pool limit, not a task limit: there is no semaphore above it, + # so a stream that cannot get a connection blocks here for as long as the + # pool stays full -- and every holder is a follow=true stream open for the + # life of its pod. Below the live pod count this does not degrade, it + # starves, and it starves the pods created last, which are the retries. + conn = aiohttp.TCPConnector(limit=MAX_CONCURRENT, ssl=ssl_ctx()) + # No total timeout: these streams are meant to stay open for the life of a + # range, which can be hours. + timeout = aiohttp.ClientTimeout(total=None, sock_connect=10) + tasks, terminal, succeeded, vanished = {}, {}, {}, {} + # Streams that ran to completion. Without this a finished task is deleted + # from `tasks` and the next poll re-opens the stream, forever: one full log + # re-read per pod per cycle, which at 1024 workers is a lot of apiserver. + streamed = set() + + async with aiohttp.ClientSession(connector=conn, timeout=timeout) as session: + logger.info("streaming logs for run=%s into %s", RUN_NAME, LOG_DIR) + while True: + try: + pods = await list_pods(session) + live = {p['metadata']['name'] for p in pods} + # A pod can leave the list without ever being observed terminal: + # Karpenter reaps the node, the kubelet evicts it, or the monitor + # deletes its finished Job. `terminal` is only written for pods + # in this list, so those would keep done() False forever and + # their stream would retry until the run ended. Marking them + # terminal lets the stream finalize on its own and free the slot; + # cancelling is the backstop for one wedged inside a connection + # attempt it will never win. + for name in [n for n in tasks if n not in live]: + terminal[name] = True + t = tasks[name] + if t.done(): + del tasks[name] + streamed.add(name) + continue + vanished[name] = vanished.get(name, 0) + 1 + if vanished[name] >= VANISHED_GRACE_CYCLES: + t.cancel() + del tasks[name] + vanished.pop(name, None) + streamed.add(name) + logger.info("cancelled stream for vanished pod %s", name) + # Unconditional: this used to be gated on ephemeral mode, back + # when it only sampled disk. Memory is sized in both modes, so + # gating it here left every pvc run with no anon peak at all. + if True: + # Once per cycle, before the per-pod branches below: those + # end in `continue` for every pod already being streamed, so + # anything after them runs only on the cycle a stream opens + # -- when the range has barely written anything yet. + await sample_kubelet(session, { + p['spec']['nodeName'] for p in pods + if p.get('spec', {}).get('nodeName') + and p.get('status', {}).get('phase') == 'Running'}) + for pod in pods: + name = pod['metadata']['name'] + labels = pod['metadata'].get('labels', {}) + end = labels.get(LABEL_RANGE) + if end is None: + continue + phase = pod.get('status', {}).get('phase') + terminal[name] = phase in ('Succeeded', 'Failed') + succeeded[name] = phase == 'Succeeded' + if phase == 'Failed': + record_outcome(pod, end, labels.get(LABEL_ATTEMPT, '1')) + if name in tasks and not tasks[name].done(): + continue + if name in tasks and tasks[name].done(): + del tasks[name] + # Only bar a re-open once the pod itself is terminal. A + # task that ended while the pod is still running died + # early, and re-opening is the recovery path. + if terminal.get(name): + streamed.add(name) + continue + if name in streamed: + continue + attempt = labels.get(LABEL_ATTEMPT, '1') + _streaming[name] = (end, attempt) + tasks[name] = asyncio.create_task( + stream_pod(session, name, end, attempt, + lambda p: terminal.get(p, False), + lambda p: succeeded.get(p, False))) + logger.info("opened stream for range %s attempt %s (%d active)", + end, attempt, len(tasks)) + except Exception as e: + logger.warning("pod list failed: %s", e) + await asyncio.sleep(POLL_SECONDS) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/files/logarithmic_range_generator.sh b/src/MissionParallelCatchup/parallel_catchup_helm/files/logarithmic_range_generator.sh deleted file mode 100644 index ac344260..00000000 --- a/src/MissionParallelCatchup/parallel_catchup_helm/files/logarithmic_range_generator.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh - -# Check if required environment variables are set -if [ -z "$LOGARITHMIC_FLOOR_LEDGERS" ]; then echo "LOGARITHMIC_FLOOR_LEDGERS not set"; exit 1; fi -if [ -z "$OVERLAP_LEDGERS" ]; then echo "OVERLAP_LEDGERS not set"; exit 1; fi -if [ -z "$STARTING_LEDGER" ]; then echo "STARTING_LEDGER not set"; exit 1; fi -if [ -z "$LATEST_LEDGER_NUM" ]; then echo "LATEST_LEDGER_NUM not set"; exit 1; fi -if [ -z "$NUM_PARALLELISM" ]; then echo "NUM_PARALLELISM not set"; exit 1; fi -if [ -z "$REDIS_HOST" ]; then echo "REDIS_HOST not set"; exit 1; fi -if [ -z "$REDIS_PORT" ]; then echo "REDIS_PORT not set"; exit 1; fi - -floorSize=$LOGARITHMIC_FLOOR_LEDGERS -overlapLedgers=$OVERLAP_LEDGERS -startLedger=$(echo "$STARTING_LEDGER" | awk '{printf "%d", $1}') -latestLedgerNum=$(echo "$LATEST_LEDGER_NUM" | awk '{printf "%d", $1}') -numParallelism=$NUM_PARALLELISM - -echo "starting logarithmic range generationg with the following inputs: -floorSize=$floorSize -overlapLedgers=$overlapLedgers -startLedger=$startLedger -latestLedgerNum=$latestLedgerNum -numParallelism=$numParallelism" - -generate_uniform () { - sl="$1" - el="$2" - ss="$3" - echo "generating uniform ranges from parameters: startLedger=$sl, endLedger=$el, segSize=$ss" - while [ "$el" -gt "$sl" ]; do - # clamp the segment size to the num of remaining ledgers to avoid doing redundant work - ledgersPerJob=$((el - sl)) - if [ "$ledgersPerJob" -gt "$ss" ]; then - ledgersPerJob=$ss - fi - ledgersToApply=$((ledgersPerJob + overlapLedgers)); - echo "${el}/${ledgersToApply}"; - # our queue assumes push-left-pop-right, but since we are generating the ranges in reverse order, here we push right - redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" RPUSH ranges "${el}/${ledgersToApply}"; - el=$(( el - ledgersPerJob )); - # sleep for a short duration to avoid overloading the redis-cli connection - sleep 1 - done -} - -endLedger=$((latestLedgerNum / 2)) -chunkSize=$(( (endLedger - startLedger + 1) / numParallelism )) -while [ "$chunkSize" -gt "$floorSize" ]; do - generate_uniform "$startLedger" "$endLedger" "$chunkSize" - startLedger=$(( endLedger + 1 )) - chunkSize=$(( chunkSize / 2 )) - endLedger=$((startLedger + (chunkSize * numParallelism) )) -done - -# treat the rest with one uniform-ranged chunk -generate_uniform "$((endLedger+1))" "$latestLedgerNum" "$floorSize" diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/files/uniform_range_generator.sh b/src/MissionParallelCatchup/parallel_catchup_helm/files/uniform_range_generator.sh deleted file mode 100644 index 2f412b63..00000000 --- a/src/MissionParallelCatchup/parallel_catchup_helm/files/uniform_range_generator.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/sh - -# Check if required environment variables are set -if [ -z "$LEDGERS_PER_JOB" ]; then echo "LEDGERS_PER_JOB not set"; exit 1; fi -if [ -z "$OVERLAP_LEDGERS" ]; then echo "OVERLAP_LEDGERS not set"; exit 1; fi -if [ -z "$STARTING_LEDGER" ]; then echo "STARTING_LEDGER not set"; exit 1; fi -if [ -z "$LATEST_LEDGER_NUM" ]; then echo "LATEST_LEDGER_NUM not set"; exit 1; fi -if [ -z "$REDIS_HOST" ]; then echo "REDIS_HOST not set"; exit 1; fi -if [ -z "$REDIS_PORT" ]; then echo "REDIS_PORT not set"; exit 1; fi - -ledgersPerJob=$LEDGERS_PER_JOB -overlapLedgers=$OVERLAP_LEDGERS -startingLedger=$(echo "$STARTING_LEDGER" | awk '{printf "%d", $1}') -endRange=$(echo "$LATEST_LEDGER_NUM" | awk '{printf "%d", $1}') -ledgersToApply=$((ledgersPerJob + overlapLedgers)); - -echo "$(date) Generating uniform ledger ranges from parameters -ledgersPerJob=$ledgersPerJob, -overlapLedgers=$overlapLedgers, -ledgersToApply=$ledgersToApply, -startingLedger=$startingLedger, -endRange=$endRange" - -# Store redis commands in a file for bulk upload in a transaction -CMD_FILE=redis_bulk_load -echo "MULTI">$CMD_FILE # Start redis transaction -while [ "$endRange" -gt "$startingLedger" ]; do - echo "${endRange}/${ledgersToApply}"; - echo "RPUSH ranges \"${endRange}/${ledgersToApply}\"">>$CMD_FILE - endRange=$(( endRange - ledgersPerJob )); -done -echo "EXEC">>$CMD_FILE # Close redis transaction - -echo "$(date) Created file $CMD_FILE with $(wc -l $CMD_FILE) lines. Loading into redis" -for i in $(seq 1 6);do - redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" <$CMD_FILE - if [ $? -eq 0 ]; then - break - else - echo "$(date) Error inserting data. Sleeping and retrying" - sleep 5 - fi -done - -echo "$(date) Finished generating ranges" diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/files/worker.sh b/src/MissionParallelCatchup/parallel_catchup_helm/files/worker.sh deleted file mode 100644 index b8efc18b..00000000 --- a/src/MissionParallelCatchup/parallel_catchup_helm/files/worker.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/sh - -# Check if required environment variables are set -if [ -z "$REDIS_HOST" ]; then echo "REDIS_HOST not set"; exit 1; fi -if [ -z "$REDIS_PORT" ]; then echo "REDIS_PORT not set"; exit 1; fi -if [ -z "$JOB_QUEUE" ]; then echo "JOB_QUEUE not set"; exit 1; fi -if [ -z "$PROGRESS_QUEUE" ]; then echo "PROGRESS_QUEUE not set"; exit 1; fi -if [ -z "$FAILED_QUEUE" ]; then echo "FAILED_QUEUE not set"; exit 1; fi -if [ -z "$SUCCESS_QUEUE" ]; then echo "SUCCESS_QUEUE not set"; exit 1; fi -if [ -z "$METRICS" ]; then echo "METRICS not set"; exit 1; fi -if [ -z "$JOB_OWNERS" ]; then echo "JOB_OWNERS not set"; exit 1; fi -if [ -z "$RELEASE_NAME" ]; then echo "RELEASE_NAME not set"; exit 1; fi -if [ -z "$POD_NAME" ]; then echo "POD_NAME not set"; exit 1; fi - -# ensure redis-cli is available -if [ ! "$(redis-cli --version)" ]; then - echo "redis-cli not found, please ensure running with a supported stellar-core version" - exit 1 -fi - -SLEEP_INTERVAL=10 -LOG_DIR="/data" - -while true; do -# Fetch the next job key from the Redis queue. -# Our ranges are generated in the order we want to run them from left to right, so we always pull from the left -JOB_KEY=$(redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" LMOVE "$JOB_QUEUE" "$PROGRESS_QUEUE" LEFT LEFT) -LMOVE_EXIT_CODE=$? - -# Only process a job if the command succeeded AND we got a non-empty job key -if [ $LMOVE_EXIT_CODE -eq 0 ] && [ -n "$JOB_KEY" ]; then - # Register ownership so the monitor knows which worker owns this job - redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" HSET "$JOB_OWNERS" "$JOB_KEY" "$POD_NAME" - if [ $? -ne 0 ]; then - echo "Error: Failed to register job ownership for $JOB_KEY. Exiting." - exit 1 - fi - - # Start timer - START_TIME=$(date +%s) - echo "Processing job: $JOB_KEY" - - # Run stellar-core: create new-db then catchup - /usr/bin/stellar-core --conf /config/stellar-core.cfg new-db --console && \ - /usr/bin/stellar-core --conf /config/stellar-core.cfg catchup "$JOB_KEY" \ - --metric 'ledger.transaction.apply' --console - STELLAR_CORE_EXIT_CODE=$? - - # End timer and duration - END_TIME=$(date +%s) - DURATION=$((END_TIME - START_TIME))s - echo "Finish processing job: $JOB_KEY, duration: $DURATION" - - # Check if both commands succeeded - if [ $STELLAR_CORE_EXIT_CODE -eq 0 ]; then - echo "Successfully processed job: $JOB_KEY" - QUEUE_COMMAND="LPUSH $SUCCESS_QUEUE \"$JOB_KEY\"" - else - echo "Error processing job: $JOB_KEY (exit code: $STELLAR_CORE_EXIT_CODE)" - QUEUE_COMMAND="LPUSH $FAILED_QUEUE \"$JOB_KEY|$POD_NAME\"" - fi - - # Parse and extract the metrics from the log file - LOG_FILE=$(ls -t "$LOG_DIR"/stellar-core*.log 2>/dev/null | head -n 1) - if [ -z "$LOG_FILE" ]; then - echo "No log file found in $LOG_DIR" - exit 1 - fi - - tx_apply_ms=$(tac "$LOG_FILE" | grep -m 1 -B 11 "metric 'ledger.transaction.apply':" | grep "sum =" | awk '{print $NF}') - echo "Log file: $LOG_FILE" - echo "ledger.transaction.apply sum: $tx_apply_ms" - # Validate metric was extracted successfully - if [ -z "$tx_apply_ms" ]; then - echo "Warning: Failed to extract metric 'ledger.transaction.apply' from log file" - tx_apply_ms="N/A" - fi - - # Push metrics to redis in a transaction to ensure data consistency. Retry for 5min on failures - # Extract the pod ordinal (last hyphen-separated segment) from pod name like "release-name-stellar-core-0" - core_id=$(echo "$POD_NAME" | awk -F'-' '{print $NF}') - # Validate core_id was extracted successfully - if [ -z "$core_id" ]; then - echo "Error: Failed to extract core_id from POD_NAME: $POD_NAME" - core_id="N/A" - fi - - result=1 # Initialize to failure - for i in $(seq 1 30);do - redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" <- + pip install --no-cache-dir -q 'kubernetes~=35.0' 'aiohttp~=3.9' + 'requests~=2.31' 'prometheus-client~=0.19' && + exec python3 /app/job_monitor.py + {{- end }} + ports: + - containerPort: 8080 + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: RUN_NAME + value: {{ .Release.Name | quote }} + - name: CORE_IMAGE + value: {{ .Values.worker.stellar_core_image | quote }} + - name: WORKER_SERVICE_ACCOUNT + value: stellar-supercluster-{{ .Release.Name }} + - name: MISSION + value: {{ .Values.monitor.mission | quote }} + - name: EMIT_MISSION_LABEL + value: {{ .Values.monitor.emitMissionLabel | quote }} + - name: LOGGING_INTERVAL_SECONDS + value: {{ .Values.monitor.loggingIntervalSeconds | quote }} + - name: RANGE_GENERATOR + value: {{ .Values.range.generator | quote }} + - name: STARTING_LEDGER + value: {{ .Values.range.startingLedger | quote }} + - name: LATEST_LEDGER_NUM + value: {{ .Values.range.latestLedgerNum | quote }} + - name: LEDGERS_PER_JOB + value: {{ .Values.range.ledgersPerJob | quote }} + - name: OVERLAP_LEDGERS + value: {{ .Values.range.overlapLedgers | quote }} + - name: RANGE_ORDER + value: {{ .Values.range.order | quote }} + - name: LOGARITHMIC_FLOOR_LEDGERS + value: {{ .Values.range.logarithmicFloorLedgers | quote }} + - name: PARALLELISM + value: {{ .Values.worker.replicas | quote }} + - name: STORAGE_MODE + value: {{ .Values.worker.storageMode | quote }} + - name: STORAGE_CLASS + value: {{ .Values.worker.storageClass | quote }} + - name: STORAGE_SIZE + value: {{ .Values.worker.storageSize | quote }} + - name: MAX_VOLUMES_PER_NODE + value: {{ .Values.worker.maxVolumesPerNode | quote }} + - name: ASAN_OPTIONS + value: {{ .Values.worker.asanOptions | quote }} + - name: REQ_CPU + value: {{ .Values.worker.resources.requests.cpu | quote }} + - name: REQ_MEM + value: {{ .Values.worker.resources.requests.memory | quote }} + # Only set in ephemeral mode: in PVC mode a large ephemeral request + # makes disk the binding dimension and halves workers-per-node. + - name: REQ_EPHEMERAL + value: {{ .Values.worker.resources.requests.ephemeral_storage | quote }} + - name: LIM_CPU + value: {{ .Values.worker.resources.limits.cpu | quote }} + - name: LIM_MEM + value: {{ .Values.worker.resources.limits.memory | quote }} + - name: LIM_EPHEMERAL + value: {{ .Values.worker.resources.limits.ephemeral_storage | quote }} + - name: MAX_ATTEMPTS + value: {{ .Values.monitor.maxAttempts | quote }} + - name: MAX_TIMEOUT_ATTEMPTS + value: {{ .Values.monitor.maxTimeoutAttempts | quote }} + - name: MAX_DISRUPTION_ATTEMPTS + value: {{ .Values.monitor.maxDisruptionAttempts | quote }} + - name: MEM_BUMP_FACTOR + value: {{ .Values.monitor.memBumpFactor | quote }} + - name: MAX_EPHEMERAL_ATTEMPTS + value: {{ .Values.monitor.maxEphemeralAttempts | quote }} + - name: EPH_BUMP_FACTOR + value: {{ .Values.monitor.ephBumpFactor | quote }} + - name: EPH_ESCALATION_CAP + value: {{ .Values.monitor.maxEphemeral | quote }} + # Escalating past the largest schedulable node makes the retry + # Pending forever, which looks like a hang rather than a failure. + - name: MAX_MEM + value: {{ .Values.monitor.maxMem | quote }} + - name: GRACE_SECONDS + value: {{ .Values.monitor.graceSeconds | quote }} + # A range stuck retrying the history archive never exits on its + # own; without this it holds a slot for the life of the run. + - name: ATTEMPT_DEADLINE_SECONDS + value: {{ .Values.monitor.attemptDeadlineSeconds | quote }} + - name: JOB_TTL_SECONDS + value: {{ .Values.monitor.jobTtlSeconds | quote }} + - name: PING_TIMEOUT_SECS + value: {{ .Values.monitor.pingTimeoutSecs | quote }} + - name: LOG_DIR + value: /logs + {{- if .Values.monitor.profileConfigMap }} + - name: PROFILE_PATH + value: /profile/profile.json + - name: PROFILE_MARGIN + value: {{ .Values.monitor.profileMargin | quote }} + - name: PROFILE_CPU_LIMIT + value: {{ .Values.monitor.profileCpuLimit | quote }} + - name: PROFILE_CPU_MARGIN + value: {{ .Values.monitor.profileCpuMargin | quote }} + - name: PROFILE_MAX_MEM + value: {{ .Values.monitor.profileMaxMemory | quote }} + - name: PROFILE_CACHE_HEADROOM + value: {{ .Values.monitor.profileCacheHeadroom | quote }} + {{- end }} + # Failed ranges are always saved; successful ones are the bulk of + # the volume and can be turned off for a cheap run. + - name: SAVE_SUCCESS_LOGS + value: {{ .Values.monitor.saveSuccessLogs | quote }} + {{- /* + Two accepted shapes: the mission emits structured selectors + ({key, operator, values}) like every other supercluster + mission, while a hand-run helm install more naturally passes + "key:value" strings. Only the first entry is used -- the + monitor takes a single label pair. + */}} + {{- with (first .Values.worker.requireNodeLabels) }} + {{- if kindIs "map" . }} + - name: NODE_LABEL_KEY + value: {{ .key | quote }} + - name: NODE_LABEL_VALUE + value: {{ (first (default (list "") .values)) | quote }} + {{- else }} + - name: NODE_LABEL_KEY + value: {{ (splitList ":" .) | first | quote }} + - name: NODE_LABEL_VALUE + value: {{ (splitList ":" .) | last | quote }} + {{- end }} + {{- end }} + {{- with (first .Values.worker.tolerateNodeTaints) }} + - name: TOLERATE_TAINT + value: {{ if kindIs "map" . }}{{ .key | quote }}{{ else }}{{ (splitList ":" .) | first | quote }}{{ end }} + {{- end }} + resources: + {{- toYaml .Values.monitor.resources | nindent 12 }} + # job_monitor.py mirrors its log to /data so it survives the pod. + volumeMounts: + - name: data + mountPath: /data + - name: logs + mountPath: /logs + {{- if .Values.monitor.profileConfigMap }} + - name: profile + mountPath: /profile + readOnly: true + {{- end }} + {{- if .Values.monitor.sourceConfigMap }} + - name: monitor-src + mountPath: /app + {{- end }} + readinessProbe: + httpGet: + path: /status + port: 8080 + initialDelaySeconds: 2 + periodSeconds: 10 + # A dead pod watch stops all failure classification, which degrades + # silently: later failures record outcome=unknown and get retried + # blindly. Restart the container rather than run half-blind. + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 60 + failureThreshold: 3 + # Streams each worker pod's log from the moment it starts, so nothing + # is lost when Karpenter reaps the node ~1 min after the pod exits -- + # and a stuck range is readable while it is still stuck. Separate + # container so its memory limit is its own: holding ~1024 open streams + # must not be affected by, or affect, the reconcile loop. + - name: log-collector + image: {{ .Values.monitor.image }} + imagePullPolicy: {{ .Values.monitor.imagePullPolicy }} + {{- if .Values.monitor.sourceConfigMap }} + command: ["/bin/sh", "-c"] + args: + - >- + pip install --no-cache-dir -q 'kubernetes~=35.0' 'aiohttp~=3.9' + 'requests~=2.31' 'prometheus-client~=0.19' && + exec python3 /app/log_collector.py + {{- else }} + command: ["/usr/bin/python3", "log_collector.py"] + {{- end }} + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: RUN_NAME + value: {{ .Release.Name | quote }} + - name: LOG_DIR + value: /logs + - name: COLLECTOR_POLL_SECONDS + value: {{ .Values.monitor.collectorPollSeconds | quote }} + - name: PEAK_FLUSH_RATIO + value: {{ .Values.monitor.peakFlushRatio | quote }} + - name: COLLECTOR_VANISHED_GRACE_CYCLES + value: {{ .Values.monitor.collectorVanishedGraceCycles | quote }} + - name: COLLECTOR_MAX_STREAMS + value: {{ if gt (.Values.monitor.collectorMaxStreams | int) 0 }}{{ .Values.monitor.collectorMaxStreams | quote }}{{ else }}{{ add (.Values.worker.replicas | int) 256 | quote }}{{ end }} + # Failures are always kept; successes are the bulk of the volume. + - name: SAVE_SUCCESS_LOGS + value: {{ .Values.monitor.saveSuccessLogs | quote }} + # The peak-ephemeral sampler runs only in ephemeral mode. Without + # this the collector defaults to pvc and silently records nothing. + - name: STORAGE_MODE + value: {{ .Values.worker.storageMode | quote }} + resources: + {{- toYaml .Values.monitor.collectorResources | nindent 12 }} + volumeMounts: + - name: logs + mountPath: /logs + {{- if .Values.monitor.sourceConfigMap }} + - name: monitor-src + mountPath: /app + {{- end }} + volumes: + {{- if .Values.monitor.profileConfigMap }} + - name: profile + configMap: + name: {{ .Values.monitor.profileConfigMap }} + {{- end }} + {{- if .Values.monitor.sourceConfigMap }} + - name: monitor-src + configMap: + name: {{ .Values.monitor.sourceConfigMap }} + {{- end }} + - name: data + emptyDir: {} + - name: logs + persistentVolumeClaim: + claimName: {{ .Release.Name }}-job-monitor-logs diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_preload_redis.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_preload_redis.yaml deleted file mode 100644 index 14de3a7b..00000000 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_preload_redis.yaml +++ /dev/null @@ -1,64 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ .Release.Name }}-preload-redis -spec: - template: - spec: - containers: - - name: preload - image: redis:7 - command: ["/bin/sh", "-c"] - args: - - |- - case "$STRATEGY" in - "uniform") - /bin/sh /scripts/uniform_range_generator.sh - ;; - "logarithmic") - /bin/sh /scripts/logarithmic_range_generator.sh - ;; - *) - echo 'Error: Unknown strategy' && exit 1 - ;; - esac - envFrom: - - configMapRef: - name: {{ .Release.Name }}-range-generator-config - volumeMounts: - - name: script - mountPath: /scripts - initContainers: - - name: wait-for-redis - image: redis:7 - command: ['sh', '-c', "until redis-cli -h {{ .Values.redis.hostname }} -p {{ .Values.redis.port }} ping; do echo waiting for redis; sleep 2; done;"] - restartPolicy: OnFailure - volumes: - - name: script - configMap: - name: {{ .Release.Name }}-generator-script ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-generator-script -data: - uniform_range_generator.sh: |- - {{- (.Files.Get "files/uniform_range_generator.sh") | nindent 4 }} - logarithmic_range_generator.sh: |- - {{- (.Files.Get "files/logarithmic_range_generator.sh") | nindent 4 }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-range-generator-config -data: - STRATEGY: "{{ .Values.range_generator.strategy }}" - STARTING_LEDGER: "{{ .Values.range_generator.params.starting_ledger }}" - LATEST_LEDGER_NUM: "{{ .Values.range_generator.params.latest_ledger_num }}" - OVERLAP_LEDGERS: "{{ .Values.range_generator.params.overlap_ledgers }}" - LEDGERS_PER_JOB: "{{ .Values.range_generator.params.uniform_ledgers_per_job }}" - LOGARITHMIC_FLOOR_LEDGERS: "{{ .Values.range_generator.params.logarithmic_floor_ledgers }}" - NUM_PARALLELISM: "192" - REDIS_HOST: "{{ .Values.redis.hostname }}" - REDIS_PORT: "{{ .Values.redis.port }}" diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/redis_queue.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/redis_queue.yaml deleted file mode 100644 index aaba65fb..00000000 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/redis_queue.yaml +++ /dev/null @@ -1,37 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: "{{ .Values.redis.hostname }}" -spec: - type: ClusterIP - ports: - - port: {{ .Values.redis.port }} - targetPort: 6379 - selector: - app: {{ .Values.redis.hostname }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.redis.hostname }} -spec: - replicas: 1 - selector: - matchLabels: - app: {{ .Values.redis.hostname }} - template: - metadata: - labels: - app: {{ .Values.redis.hostname }} - spec: - containers: - - name: redis - image: redis:7 - ports: - - containerPort: 6379 - command: ["redis-server"] - resources: - requests: - cpu: "{{ .Values.redis.resources.requests.cpu }}" - memory: "{{ .Values.redis.resources.requests.memory }}" \ No newline at end of file diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 36f3ea3d..f23f90e9 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -1,20 +1,26 @@ -redis: - hostname: "" # to be set by the mission - port: 6379 - job_queue: "ranges" - success_queue: "succeeded" - failed_queue: "failed" - progress_queue: "in_progress" - metrics: "metrics" - job_owners: "job_owners" - resources: - requests: - cpu: "100m" - memory: "100Mi" - worker: stellar_core_image: "stellar/stellar-core:latest" replicas: 5 + # pvc: /data survives the pod, so an evicted range resumes at L+1 -- this is + # what makes spot viable. ephemeral: /data is an emptyDir on the node, no + # resume, and ephemeral_storage must be sized to hold it. + storageMode: "pvc" + storageClass: "" + # Sized from what a range actually uses: the old node-local /data ran on a + # 35Gi request / 40Gi limit. 40Gi x 1024 workers is 40 TiB of gp3 rather than + # 100 TiB, which matters against the per-region storage quota. + # One PVC per ledger range, not per concurrency slot. Measured on ssc-test: + # 300 jobs each with their own PVC took 3151s vs 3210s reusing 40 -- 7.5x the + # volume lifecycles for no measurable cost and no CSI throttling. Reuse only + # bought bookkeeping, plus pinning every later range to the AZ the slot's + # first volume happened to land in. + storageSize: "40Gi" + # Cap on PVC-mounting workers per node, enforced with a topologySpread + # minDomains floor. A Nitro node allows ~26 EBS attachments and Karpenter does + # not size nodes for attachment capacity. Inert at realistic density (1800m + # CPU already yields ~4/node); only binds if CPU requests are small enough for + # 24+ pods to share a node. 0 disables. pvc mode only. + maxVolumesPerNode: 24 requireNodeLabels: [] avoidNodeLabels: [] tolerateNodeTaints: [] @@ -35,26 +41,119 @@ worker: historyGetCommandCore003: "curl -sf http://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}" +# Index -> ledger range is computed in the Job template; no queue to preload. +range: + # uniform: equal ledger counts. logarithmic: big chunks over cheap early + # history, halving toward the tip, aiming for equal wall-time per job. + generator: "uniform" + logarithmicFloorLedgers: 64000 + startingLedger: 0 + latestLedgerNum: 100000 + overlapLedgers: 320 + # Dispatch order. Generators emit tip-first, which front-loads the most + # expensive ranges. "oldest-first" reverses that so a profiling run + # measures the cheap early ranges before it can be interrupted. + order: tip-first + ledgersPerJob: 16000 + monitor: - gateway_name: "traefik-gateway-private" - gateway_namespace: "traefik" - hostname: "" # to be set by the mission - path_prefix: "/default" - logging_interval_seconds: 300 - logging_level: "INFO" # 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL' + # Reuses the existing hand-built job-monitor image slot -- no new image and no + # new push path (nothing in .github/workflows builds these). + image: "stellar/ssc-job-monitor:latest" + # Dev loop: run the monitor and collector from a ConfigMap holding + # job_monitor.py and log_collector.py instead of a built image. Set it to the + # ConfigMap name and point monitor.image at a plain python base; the deps the + # Dockerfile bakes get pip-installed at start. Empty = use the image as built. + sourceConfigMap: "" + # Range profile from an earlier run, as a ConfigMap holding profile.json. + # The mission driver resolves --pubnet-parallel-catchup-profile (a local + # path or an https URL) into one. Empty = size from the configured + # requests below. Only tightens requests; limits are untouched. + profileConfigMap: "" + profileMargin: 1.15 + # CPU limit for ranges the profile has measured. Above the configured + # worker limit on purpose: at a 2-core limit every range pegs 2.0, so the + # peak is a ceiling and the profile never learns real demand. Unprofiled + # ranges keep worker.resources.limits.cpu unchanged. + # Empty = measured ranges run with no cpu limit, which is fastest on an + # otherwise-free node (168s/111s/99s at limit 2/4/none). Set a value to cap. + profileCpuLimit: "" + # No margin on cpu: it is compressible, so under-requesting costs contention + # rather than an OOM kill. Memory keeps profileMargin. + profileCpuMargin: 1.0 + # Ceiling for profile-derived memory. Above the configured worker limit on + # purpose: a range needing more than that must be able to ask for it + # rather than be pinned under its own measured peak. + profileMaxMemory: "32Gi" + # Fixed allowance added to a range's measured rss, on top of profileMargin. + # Not zero: memory.max bounds anon PLUS page cache, and a multiplicative + # margin is meaningless at small rss. Measured with headroom 0, ranges + # profiled at 190MiB rss got a 209MiB limit and 90 of them OOMKilled within + # 90s of dispatch. + profileCacheHeadroom: "512Mi" + imagePullPolicy: IfNotPresent + mission: "HistoryPubnetParallelCatchup" + # Adds a `mission` label to worker pods, which kube-state-metrics exposes as + # label_mission for the Grafana container panels. Default OFF: those panels + # are per-pod, so 1024 workers would swamp any view with mission=$__all and + # slow the shared dashboard for everyone. Turn on per-run for a small run, or + # once the panels aggregate. + emitMissionLabel: false + loggingIntervalSeconds: 10 + maxAttempts: 5 + # Hangs are usually persistent (bad archive host, absent checkpoint), so they + # get a lower cap than evictions -- otherwise a wedged range costs + # maxAttempts x attemptDeadlineSeconds before it is reported. + # For a hang caught by attemptDeadlineSeconds. stellar-core bounds its own + # retries (RETRY_A_FEW=5 / RETRY_A_LOT=32, backoff capped at 512s) and exits 3 + # when they are exhausted, so the deadline is only a backstop for the + # pathological tail -- worst case ~3.4h for a RETRY_A_LOT work. + maxTimeoutAttempts: 2 + # Evictions, admission rejections and monitor restarts are not the range's + # fault, so they do not share the failure budget above. Measured on ssc-test: + # ten evictions across 25 workers put four healthy ranges on attempt 3 of 5. + maxDisruptionAttempts: 20 + memBumpFactor: 1.5 + # An ephemeral-storage eviction repeats until the range gets more disk, so + # it gets its own small budget rather than the environmental one. + maxEphemeralAttempts: 4 + ephBumpFactor: 1.5 + maxEphemeral: "200Gi" + maxMem: "48Gi" + graceSeconds: 100 + # Must exceed any plausible monitor outage: completion is recorded to the + # progress ConfigMap by the monitor, and a Job reclaimed before that happens + # reads as "never ran" and gets redone. + # 0 = no deadline. A prod range runs ~50 min, so ~3h is generous while still + # catching a range wedged in archive retries. Measured: stellar-core retries a + # missing/unreachable archive indefinitely rather than failing. + attemptDeadlineSeconds: 10800 + jobTtlSeconds: 600 + pingTimeoutSecs: 2 + # Worker logs are pulled to the monitor's volume while each pod is still + # alive, then collected in one exec at teardown instead of ~1024. + logStorageClass: "" + logStorageSize: "100Gi" + saveSuccessLogs: true + collectorPollSeconds: 5 + # Growth factor before an in-flight peak is flushed to its .metrics file, so a + # collector restart cannot silently reset a range's high-water to zero. + peakFlushRatio: 1.05 + # Poll cycles a stream gets to finalize after its pod leaves the pod list, + # before it is cancelled and its connection slot reclaimed. + collectorVanishedGraceCycles: 3 + # 0 = derive from worker.replicas. A fixed value here is how 2048-worker runs + # silently lost every retry pod's metrics: this caps the aiohttp connection + # pool, not the task count, so pods beyond it block forever rather than + # queueing -- and retries, created last, never got a slot. + collectorMaxStreams: 0 + collectorResources: + requests: { cpu: "200m", memory: "512Mi" } + limits: { cpu: "2", memory: "2Gi" } + tolerateNodeTaints: [] resources: - requests: - cpu: "100m" - memory: "100Mi" - -range_generator: - strategy: "uniform" # "uniform" or "logarithmic" - params: - starting_ledger: 0 - latest_ledger_num: 100000 - overlap_ledgers: 320 - uniform_ledgers_per_job: 16000 # only for strategy="uniform" - logarithmic_floor_ledgers: 16000 # only for strategy="logarithmic" + requests: { cpu: "200m", memory: "512Mi" } + limits: { cpu: "2", memory: "2Gi" } service_account: annotations: [] diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py new file mode 100644 index 00000000..8ff42902 --- /dev/null +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -0,0 +1,1521 @@ +"""Unit tests for the parallel-catchup job monitor and its log collector. + +Covers the formats this mission does not control: the Job controller's +podFailurePolicy condition messages, and stellar-core's medida metric block. +Both are pinned from real captures so a Kubernetes or stellar-core change fails +here rather than silently degrading a run. + +Sources are parsed rather than imported, so no cluster, kubernetes client or +aiohttp is required. + +Run: python3 -m pytest test_job_monitor.py +""" + +import json +import re + +import pytest + +SRC = open(__file__.replace('test_job_monitor.py', 'job_monitor.py')).read() +COLLECTOR_SRC = open(__file__.replace('test_job_monitor.py', 'log_collector.py')).read() + + +def _extract(pattern, src=None): + m = re.search(pattern, src if src is not None else SRC, re.S | re.M) + assert m, f"pattern not found: {pattern}" + return m + + +JOB_MSG = re.compile(eval(_extract(r"_JOB_MSG = re\.compile\((r\"[^\"]+\")\)").group(1))) +JOB_RULE = re.compile(eval(_extract(r"_JOB_RULE = re\.compile\((r\"[^\"]+\")\)").group(1))) +SUM_RE = re.compile(eval(_extract(r"_SUM_RE = re\.compile\((r\"[^\"]+\")\)").group(1))) +RULE_ORDER = [x.strip().strip("'") for x in + _extract(r"RULE_ORDER = \[([^\]]+)\]").group(1).split(',')] +RULE_OUTCOME = dict(enumerate(RULE_ORDER)) + +# RECONSTRUCTED 2026-07-30 after an over-broad test deletion removed the +# originals -- twice. Kept up here with the other module constants so a +# function-scoped deletion cannot reach them again. Shaped to what the code +# parses (_JOB_RULE reads "rule at index N", RULE_ORDER[2] is 'failed'; +# classify() keys on the substring 'ephemeral' in status.message) but no longer +# verbatim captures. Re-pin from a real eviction on the next run. +EPH_EVICT_JOB_CONDITION = ( + "Container stellar-core for pod stellar-supercluster/" + "parallel-catchup-r31005951-a1-x7k2p failed with exit code 3 " + "matching FailJob rule at index 2") +EPH_EVICT_MESSAGE = ( + "Pod ephemeral local storage usage exceeds the total limit of containers 40Gi") + + +def classify(msg): + """Mirrors classify_from_job: rule index wins, exit code is the fallback.""" + rule, detail = JOB_RULE.search(msg), JOB_MSG.search(msg) + outcome = RULE_OUTCOME.get(int(rule.group('idx'))) if rule else None + code = int(detail.group('code')) if detail else None + if outcome is None and code is not None: + outcome = 'oom' if code == 137 else 'failed' + return outcome, code, (detail.group('pod') if detail else None) + + +def tx_apply_scanner(): + cls = _extract(r"^_TX_METRIC = .*?^(class TxApplyScanner:.*?)^def ", + COLLECTOR_SRC).group(1) + ns = {'re': re} + exec("\n".join([_extract(r"^_TX_METRIC = .*$", COLLECTOR_SRC).group(0), + _extract(r"^_SUM_RE = .*$", COLLECTOR_SRC).group(0), + cls]), ns) + return ns['TxApplyScanner'] + + +# --- captures ---------------------------------------------------------------- + +# EKS 1.34 Job condition messages. Only the wording is pinned; pod and +# container names are renamed for readability. +DISRUPTED = ("Pod sandbox/jterm-catchup-snfr2 has condition DisruptionTarget " + "matching FailJob rule at index 0") +OOMKILLED = ("Container oom-container for pod sandbox/oom-test-job-qvq8b failed with " + "exit code 137 matching FailJob rule at index 1") +NONZERO_EXIT = ("Container exit-1-container for pod sandbox/exit-1-job-wbhkq failed with " + "exit code 1 matching FailJob rule at index 2") + +# stellar-core 27.1.1 catchup pod, --metric 'ledger.transaction.apply'. Kept +# whole: `sum` is 10 lines below the header against a 15-line scan window. +MEDIDA_BLOCK = """2026-07-28T18:39:49.350 GAJSL [default INFO] metric 'ledger.transaction.apply': +2026-07-28T18:39:49.350 GAJSL [default INFO] count = 20 +2026-07-28T18:39:49.350 GAJSL [default INFO] mean rate = 0.22136 calls/s +2026-07-28T18:39:49.350 GAJSL [default INFO] 1-minute rate = 0.113149 calls/s +2026-07-28T18:39:49.350 GAJSL [default INFO] 5-minute rate = 0.175948 calls/s +2026-07-28T18:39:49.350 GAJSL [default INFO] 15-minute rate = 0.191421 calls/s +2026-07-28T18:39:49.350 GAJSL [default INFO] min = 0.295417ms +2026-07-28T18:39:49.350 GAJSL [default INFO] max = 0.639873ms +2026-07-28T18:39:49.350 GAJSL [default INFO] mean = 0.417143ms +2026-07-28T18:39:49.350 GAJSL [default INFO] stddev = 0.108677ms +2026-07-28T18:39:49.350 GAJSL [default INFO] sum = 8.34285ms +2026-07-28T18:39:49.350 GAJSL [default INFO] median = 0ms +2026-07-28T18:39:49.350 GAJSL [default INFO] 75% = 0ms +2026-07-28T18:39:49.350 GAJSL [default INFO] 95% = 0ms +2026-07-28T18:39:49.350 GAJSL [default INFO] 98% = 0ms +2026-07-28T18:39:49.350 GAJSL [default INFO] 99% = 0ms +2026-07-28T18:39:49.350 GAJSL [default INFO] 99.9% = 0ms""" + +TX_APPLY_SECONDS = 0.00834285 + + +# --- how a failed catchup attempt is classified ------------------------------ + +@pytest.mark.parametrize("msg,outcome,code,pod", [ + (DISRUPTED, 'disrupted', None, None), + (OOMKILLED, 'oom', 137, 'oom-test-job-qvq8b'), + (NONZERO_EXIT, 'failed', 1, 'exit-1-job-wbhkq'), +]) +def test_job_condition_message(msg, outcome, code, pod): + assert classify(msg) == (outcome, code, pod) + + +def test_rule_order_matches_the_rendered_policy(): + rendered = re.findall(r"\n \('(\w+)', client\.V1PodFailurePolicyRule", SRC) + assert rendered == RULE_ORDER + + +def test_eviction_is_told_apart_from_a_broken_range_by_the_condition(): + # stellar-core exits 3 both for a drain and for a corrupt bucket, so only + # DisruptionTarget separates them -- hence rule 0 must be evaluated first. + assert classify(DISRUPTED)[0] == 'disrupted' + assert classify("Container c for pod ns/p failed with exit code 3")[0] == 'failed' + + +def test_bare_137_is_an_oom(): + assert classify("Container c for pod ns/p failed with exit code 137")[:2] == ('oom', 137) + + +def test_backoff_limit_message_stays_unclassified(): + assert classify("Job has reached the specified backoff limit") == (None, None, None) + + +def test_admission_rejection_is_not_a_catchup_failure(): + rejected = {'VolumeAttachmentLimitExceeded', 'OutOfcpu', 'OutOfmemory', 'OutOfpods', + 'UnexpectedAdmissionError', 'NodeAffinity', 'Shutdown', 'Evicted'} + listed = set(re.findall(r"'(\w+)'", _extract( + r"if pod\.status\.reason in \(([^)]+)\)").group(1))) + assert rejected <= listed, f"missing from classify(): {rejected - listed}" + + +# --- ledger range generation ------------------------------------------------- + +def test_logarithmic_ranges_match_the_shell_generator(): + # Verbatim output of logarithmic_range_generator.sh with + # floor=16000 overlap=320 start=0 latest=500000 parallelism=4, captured + # before it was deleted. Chunk size halves toward the tip, so exact values + # are pinned rather than a count. + expected = "250000/62820 187500/62820 125000/62820 62500/62820 375001/31570 343751/31570 312501/31570 281251/31570 500000/16320 484000/16320 468000/16320 452000/14817".split() + + floor, overlap, start, latest, par = 16000, 320, 0, 500000, 4 + + def seg(sl, el, ss): + out = [] + while el > sl: + lpj = min(el - sl, ss) + out.append((el, lpj + overlap)) + el -= lpj + return out + + out, s0, end = [], start, latest // 2 + chunk = (end - s0 + 1) // max(par, 1) + while chunk > floor: + out += seg(s0, end, chunk) + s0 = end + 1 + chunk //= 2 + end = s0 + (chunk * par) + out += seg(end + 1, latest, floor) + + assert [f"{e}/{c}" for e, c in out] == expected + + +# --- tx_apply, read from stellar-core's metric block ------------------------- + +def test_monitor_parses_tx_apply_sum(): + sums = [SUM_RE.search(l) for l in MEDIDA_BLOCK.splitlines()] + got = [float(m.group(1)) / 1000.0 for m in sums if m] + assert got == [pytest.approx(TX_APPLY_SECONDS)] + + +def test_collector_scanner_agrees_with_the_monitor(): + scanner = tx_apply_scanner()() + for line in MEDIDA_BLOCK.splitlines(): + scanner.feed(line) + assert scanner.seconds == pytest.approx(TX_APPLY_SECONDS) + + +def test_scanner_resumes_a_block_split_across_a_reconnect(): + # One scanner spans stream_pod's reconnect loop, so a drop mid-block must + # not lose the header already seen. + head, tail = MEDIDA_BLOCK.splitlines()[:4], MEDIDA_BLOCK.splitlines()[4:] + scanner = tx_apply_scanner()() + for line in head: + scanner.feed(line) + assert scanner.seconds is None + for line in tail: + scanner.feed(line) + assert scanner.seconds == pytest.approx(TX_APPLY_SECONDS) + + +def test_scanner_ignores_sum_from_another_metric(): + scanner = tx_apply_scanner()() + for line in ["metric 'ledger.ledger.close':", " sum = 999999.0ms"]: + scanner.feed(line) + assert scanner.seconds is None + + +def test_scanner_gives_up_past_its_window(): + scanner = tx_apply_scanner()() + scanner.feed("metric 'ledger.transaction.apply':") + for _ in range(20): + scanner.feed("[default INFO] unrelated chatter") + scanner.feed(" sum = 12.5555ms") + assert scanner.seconds is None + + +def test_rate_and_mean_lines_are_not_read_as_sum(): + for line in MEDIDA_BLOCK.splitlines(): + if 'rate =' in line or 'mean =' in line: + assert SUM_RE.search(line) is None + + +def test_sum_stays_inside_the_scan_window(): + lines = MEDIDA_BLOCK.splitlines() + header = next(i for i, l in enumerate(lines) if 'ledger.transaction.apply' in l) + offset = next(i for i, l in enumerate(lines) if SUM_RE.search(l)) - header + assert offset == 10, f"medida layout moved: sum is now {offset} lines below the header" + assert offset <= tx_apply_scanner().WINDOW + + +def test_tx_apply_survives_a_reaped_pod(): + stmt = _extract(r"\n\s*tx = tx_apply_for_range\(.*?\n(?=\s*(?:if|completed|#))") + assert not re.search(r"\)\s*if pod else None", stmt.group(0)), \ + "tx_apply must fall back to the collector's files when the pod is gone" + assert re.search(r"tx_apply_for_range\(\s*end,\s*attempt", stmt.group(0)) + + +def test_tx_apply_prefers_durable_sources_over_the_pod_api(): + fn = _extract(r"def tx_apply_for_range\(.*?^def ").group(0) + assert fn.index('metrics_path') < fn.index('log_path') < fn.index('read_namespaced_pod_log') + + +# --- contracts between job_monitor and log_collector ------------------------- + +def test_metrics_filename_agrees_across_both_processes(): + mon = _extract(r"def metrics_path\(end, attempt\):\s*return [^\n]*?f\"([^\"]+)\"") + col = _extract(r"def base\(end, attempt\):\s*return [^\n]*?f\"([^\"]+)\"", COLLECTOR_SRC) + assert mon.group(1) == col.group(1) + '.metrics' + + +def test_discarding_a_successful_archive_keeps_its_metrics(): + suffixes = _extract(r"def discard\(end, attempt\):.*?for suffix in \(([^)]*)\)", + COLLECTOR_SRC).group(1) + assert '.log.gz' in suffixes + assert '.metrics' not in suffixes + + +def test_metrics_are_written_before_the_archive_is_discarded(): + # Lives in finalize(), shared by the clean-exit and pod-gone paths. + body = _extract(r"^(async def finalize\(.*?)(?=\n\nasync def )", + COLLECTOR_SRC).group(1) + assert body.index('write_metrics') < body.index('discard(') + + +def test_worker_pod_spec_uses_every_helper(): + body = _extract(r"spec=client\.V1PodSpec\((.*?)containers=\[container\]").group(1) + for field in ('service_account_name', 'topology_spread_constraints', 'restart_policy', + 'termination_grace_period_seconds', 'affinity', 'tolerations'): + assert field in body, f"{field} missing from the worker pod spec" + for helper in ('pod_labels', 'volume_spread_constraints', 'ensure_pvc', + '_failure_rules', '_resources'): + assert len(re.findall(rf"\b{helper}\(", SRC)) >= 2, \ + f"{helper}() is defined but never called" + + +def test_untimestamped_kubelet_text_never_becomes_a_resume_point(): + # A pod that has just been replaced returns plain text from the logs API + # instead of log lines. Partitioning that on the first space yields "unable", + # which as a resume point makes every later request sinceTime=unableZ -> 400 + # for the life of the range. + ts_re = re.compile(eval(_extract(r"_TS_RE = re\.compile\((r\"[^\"]+\")\)", + COLLECTOR_SRC).group(1))) + kubelet = "unable to retrieve container logs for containerd://9f2c1a" + assert ts_re.match(kubelet.partition(' ')[0]) is None + for good in ("2026-07-28T20:29:27.927795721Z", "2026-07-28T20:29:27Z"): + assert ts_re.match(good), good + + +# --- peak working set, for sizing a later run's requests -------------------- +# +# Queried from Prometheus rather than read from the worker's cgroup. Measured on +# ssc-test: cgroup memory.peak reported 1.5GB for a process holding 0.3MB of +# anon memory, because it counts page cache -- and catchup reads GBs of buckets. +# Sampling inside the worker was the other option and is worse: it means dropping +# the `exec`, which is what keeps stellar-core at PID 1 and able to see SIGTERM. + +def _collector_fn(*names): + """exec the named pure functions out of log_collector.py.""" + src = ["import json"] + for n in names: + m = re.search(rf"^(def {n}\(.*?)(?=^\S|\Z)", COLLECTOR_SRC, re.S | re.M) + assert m, f"{n} not found in log_collector.py" + src.append(m.group(1)) + ns = {} + exec("\n".join(src), ns) + return tuple(ns[n] for n in names) + + + + + + + + +def test_an_ephemeral_eviction_is_not_read_as_an_oom_or_a_disruption(): + # Measured end-to-end on ssc-test: the kubelet sets no DisruptionTarget, + # and stellar-core drains and exits 3, so the Job condition is a plain + # non-zero failure that would get no retry. status.message is the only + # discriminator and only the pod carries it, so both classifiers must test + # it before anything keyed on Evicted. + assert 'index 2' in EPH_EVICT_JOB_CONDITION, "the Job matches the generic non-zero rule" + for src in (COLLECTOR_SRC, SRC): + body = _extract(r"def classify(?:_from_job)?\(pod\):(.*?)(?=\n\ndef )", src) + body = body.group(1) if body else src + eph = body.find("'ephemeral'") + generic = body.find("'VolumeAttachmentLimitExceeded'") + assert eph != -1, "no ephemeral-eviction branch" + assert eph < generic, "the ephemeral branch must precede the generic Evicted branch" + + +def test_ephemeral_eviction_message_still_matches_what_we_test_for(): + # Both classifiers key on the substring 'ephemeral' in status.message. + assert 'ephemeral' in EPH_EVICT_MESSAGE + + +def test_ephemeral_escalation_raises_request_and_limit_together(): + # ephemeral-storage is a scheduling dimension: a pod that outgrew its limit + # will not fit where it was placed before unless the request moves too. + fn = _extract(r"def _resources\(.*?^def ").group(0) + assert fn.count('eph or') == 2, "both request and limit must take the escalated size" + assert 'MAX_EPHEMERAL_ATTEMPTS' in SRC + env = _extract(r"ENVIRONMENTAL_OUTCOMES = \(([^)]+)\)").group(1) + assert 'ephemeral' not in env, "a deterministic failure must not get the 20-attempt budget" + + +# The collector is a separate container with its own env block, so a variable +# the monitor has is not automatically one the collector has. STORAGE_MODE was +# missing there and the sampler silently did nothing -- it defaults to 'pvc'. +COLLECTOR_ENV_WITH_DEFAULTS = { + 'KUBERNETES_SERVICE_HOST', 'KUBERNETES_SERVICE_PORT', # injected by kubelet + 'LOGGING_LEVEL', 'PEAK_WS_WINDOW', 'PROMETHEUS_URL', + 'STATE_FLUSH_SECONDS', 'WORKER_CONTAINER', +} + + +def test_a_finished_stream_is_never_reopened(): + # A completed task is deleted from `tasks`, so without a record of it the + # next poll re-creates the stream and re-reads the whole log -- every + # cycle, per pod. Measured: the completion block ran every 10s per range. + loop = _extract(r"while True:\n(.*?)await asyncio\.sleep\(POLL_SECONDS\)", + COLLECTOR_SRC).group(1) + assert 'if name in streamed:' in loop + # ...but only once the pod is terminal: a task that ended while the pod is + # still running died early, and re-opening the stream is how that recovers. + # Scoped to the per-pod branch: the vanished-pod reaper above it also + # deletes tasks and adds to `streamed`, and slicing the whole loop would + # match that block instead of this one. + per_pod = loop[loop.index('for pod in pods:'):] + guard = per_pod[per_pod.index('del tasks[name]'):per_pod.index('streamed.add(name)')] + assert 'terminal.get(name)' in guard + + +def test_metrics_writes_merge_so_a_rewrite_cannot_drop_a_measurement(): + # The ephemeral peak is held in memory by the collector; a restart loses it. + # If a later write clobbered the file, the peak already persisted would be + # lost -- which is exactly what happened before this merge. + fn = _extract(r"def write_metrics\(.*?(?=\ndef )", COLLECTOR_SRC).group(0) + assert '{**json.load(fh), **values}' in fn, "existing fields must survive" + + +def test_the_ephemeral_sampler_runs_every_poll_not_once_per_stream(): + # The per-pod branches all end in `continue` for pods already streaming, so + # a sampler placed after them fires only on the cycle a stream opens -- + # when the range has written almost nothing. It must run before the loop. + loop = _extract(r"while True:\n(.*?)await asyncio\.sleep\(POLL_SECONDS\)", + COLLECTOR_SRC).group(1) + call = loop.index('sample_kubelet') + for_pod = loop.index('for pod in pods:') + assert call < for_pod, "sample_kubelet must run before the per-pod loop" + assert loop.count('await list_pods(session)') == 1, \ + "one listing per cycle; the sampler must reuse it" + + +def test_every_env_the_collector_reads_is_set_on_the_collector_container(): + chart = open(__file__.replace( + 'test_job_monitor.py', + 'parallel_catchup_helm/templates/job_monitor.yaml')).read() + collector = chart[chart.index('- name: log-collector'):] + needed = set(re.findall(r"os\.getenv\('([A-Z_]+)'", COLLECTOR_SRC)) + missing = {v for v in needed - COLLECTOR_ENV_WITH_DEFAULTS + if f"- name: {v}\n" not in collector} + assert not missing, f"collector reads {sorted(missing)} but the chart never sets them" + + + + +def test_both_peaks_reach_the_progress_record(): + fields = _extract(r"PEAK_FIELDS = \(([^)]+)\)").group(1) + for f in ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes'): + assert f in fields, f + + +# --- range profile consumption ----------------------------------------------- + +def _profile_ns(ranges, mode='ephemeral', margin=1.1): + """profile_for + _sized, exec'd out of job_monitor with a fixed profile.""" + ns = {'bisect': __import__('bisect'), 'logger': __import__('logging').getLogger('t')} + for name in ('_quantity_bytes', '_bytes_to_quantity', 'profile_for', + '_cpu_millis', '_sized_cpu', '_sized'): + m = re.search(rf"^(def {name}\(.*?)(?=^\S|\Z)", SRC, re.S | re.M) + exec(m.group(1), ns) + ns['_UNITS'] = eval(_extract(r"_UNITS = (\{.*?\})").group(1)) + ns['PROFILE'] = sorted(ranges) + ns['STORAGE_MODE'] = mode + ns['PROFILE_MARGIN'] = margin + return ns + + +PROFILE_RANGES = [ + (1000, {'peakRssBytes': 1_000_000_000, 'peakWorkingSetBytes': 9_000_000_000, + 'peakEphemeralBytes': 2_000_000_000, 'peakCpuCores': 0.5}), + (2000, {'peakRssBytes': 3_000_000_000, 'peakWorkingSetBytes': 13_000_000_000, + 'peakEphemeralBytes': 4_000_000_000, 'peakCpuCores': 1.2}), +] + + +def test_profile_prefers_an_exact_end(): + ns = _profile_ns(PROFILE_RANGES) + assert ns['profile_for'](2000)['peakRssBytes'] == 3_000_000_000 + + +def test_profile_rounds_up_to_the_next_measured_end_never_down(): + # Cost rises with ledger position -- the bucket set only grows -- so a lower + # neighbour under-reports, and under-provisioning costs an eviction while + # over-provisioning only costs packing density. + ns = _profile_ns(PROFILE_RANGES) + assert ns['profile_for'](1500)['peakRssBytes'] == 3_000_000_000, \ + "1500 must size from 2000, not from 1000" + + +def test_profile_falls_back_to_defaults_past_its_high_water_mark(): + # An older profile has nothing above its own top, which is exactly where a + # newer run's fresh ranges live. Extrapolating there would under-provision. + ns = _profile_ns(PROFILE_RANGES) + assert ns['profile_for'](9999) is None + + +def test_a_profile_from_the_other_storage_mode_is_rejected(): + # An ephemeral profile carries peakEphemeralBytes and a pvc one does not. + fn = _extract(r"def load_profile\(.*?^PROFILE = None", SRC).group(0) + assert "mode != STORAGE_MODE" in fn + assert 'return []' in fn + + +def test_an_unreadable_profile_is_not_fatal(): + # It is an optimisation, never a prerequisite. + fn = _extract(r"def load_profile\(.*?^PROFILE = None", SRC).group(0) + assert '(OSError, ValueError)' in fn + + +def test_sizing_applies_the_margin_and_never_exceeds_the_limit(): + ns = _profile_ns(PROFILE_RANGES) + # 1 GB * 1.1, well under the cap + assert ns['_sized'](1_000_000_000, 1.1, '10Gi') == '1049Mi' + # capped: a huge peak cannot produce a request above its own limit + assert ns['_sized'](50_000_000_000, 1.1, '8Gi') == '8192Mi' + + +def _overrides_ns(ranges, lim_mem='24000Mi', lim_eph='40Gi', margin=1.1, + max_mem='32Gi'): + ns = _profile_ns(ranges, margin=margin) + ns['LIM_MEM'] = lim_mem + ns['LIM_EPHEMERAL'] = lim_eph + ns['LIM_CPU'] = '2' + ns['REQ_CPU'] = '1800m' + ns['PROFILE_CPU_LIMIT'] = '' + ns['PROFILE_CPU_MARGIN'] = 1.0 + ns['PROFILE_MAX_MEM'] = max_mem + ns['PROFILE_CACHE_HEADROOM'] = '512Mi' + m = re.search(r"^(def _profile_overrides\(.*?)(?=^\S|\Z)", SRC, re.S | re.M) + exec(m.group(1), ns) + return ns + + +def test_profile_sizes_a_first_attempt(): + # Executed, not grepped: the first version of this gate read `mem is None` + # AFTER mem had been defaulted, so it was never true and profile sizing was + # silently dead while a source-text assertion still passed. + ns = _overrides_ns(PROFILE_RANGES) + out = ns['_profile_overrides'](2000, escalated=False) + assert out['memory'] == '3659Mi' # 3 GB rss * 1.1 + 512Mi + assert out['ephemeral-storage'] == '4196Mi' + # cpu is no longer profiled: REQ_CPU is fixed, so there is nothing to size. + assert 'cpu' not in out + + +def test_profile_does_not_override_an_escalated_retry(): + # An escalation is a measurement of THIS run and outranks an earlier one. + ns = _overrides_ns(PROFILE_RANGES) + assert ns['_profile_overrides'](2000, escalated=True) == {} + + +def test_profile_gives_nothing_past_its_high_water_mark(): + ns = _overrides_ns(PROFILE_RANGES) + assert ns['_profile_overrides'](99999, escalated=False) == {} + + +def test_profile_memory_is_capped_at_its_own_ceiling_not_the_worker_limit(): + # A range needing more than the configured limit must be able to ask for it, + # or it is pinned under its own measured peak and OOMs every attempt. The + # ceiling is what bounds it, and the OOM ladder can still climb past that. + ns = _overrides_ns([(1, {'peakRssBytes': 500_000_000_000})], + lim_mem='24000Mi', max_mem='32Gi') + assert ns['_profile_overrides'](1, escalated=False)['memory'] == '32768Mi' + + +def test_profile_memory_can_exceed_the_configured_worker_limit(): + # 28 GB peak against a 24000Mi configured limit: the profile must raise it. + ns = _overrides_ns([(1, {'peakRssBytes': 28_000_000_000})], + lim_mem='24000Mi', max_mem='32Gi') + got = ns['_profile_overrides'](1, escalated=False)['memory'] + assert ns['_quantity_bytes'](got) > ns['_quantity_bytes']('24000Mi') + + +class _FakeRR: + """Stand-in for client.V1ResourceRequirements, so _resources can be run.""" + + def __init__(self, requests=None, limits=None): + self.requests, self.limits = requests, limits + + +def _resources_ns(ranges, req_eph='35Gi', lim_eph='40Gi'): + ns = _overrides_ns(ranges, lim_eph=lim_eph) + ns.update(REQ_CPU='1800m', LIM_CPU='2', REQ_MEM='9Gi', + REQ_EPHEMERAL=req_eph, client=type('c', (), {'V1ResourceRequirements': _FakeRR})) + m = re.search(r"^(def _resources\(.*?)(?=^def )", SRC, re.S | re.M) + exec(m.group(1), ns) + return ns + + +def test_a_measured_range_matches_memory_and_disk_and_leaves_cpu_configured(): + # Memory and disk match request to limit -- exceeding either kills the pod. + # CPU keeps its configured limit and only moves its request, so the range + # packs by what it uses and can still burst. + ns = _resources_ns(PROFILE_RANGES) + r = ns['_resources'](end=2000) + assert r.requests['memory'] == r.limits['memory'] == '3659Mi' + assert r.requests['ephemeral-storage'] == r.limits['ephemeral-storage'] == '4196Mi' + # The configured request, not a measured one -- a profiled range now packs + # at exactly the same cpu as an unprofiled one. + assert r.requests['cpu'] == '1800m' + assert 'cpu' not in r.limits, "a measured range runs uncapped" + + + +def test_an_unmeasured_range_keeps_the_mismatched_defaults(): + # No profile entry must behave exactly as if there were no profile at all. + ns = _resources_ns(PROFILE_RANGES) + r = ns['_resources'](end=99999) + assert r.requests['memory'] == '9Gi' and r.limits['memory'] == '24000Mi' + assert r.requests['cpu'] == '1800m' and r.limits['cpu'] == '2', \ + "an unprofiled range must keep the configured cpu limit" + assert r.requests['ephemeral-storage'] == '35Gi' + assert r.limits['ephemeral-storage'] == '40Gi' + assert r.requests != r.limits + + +def test_an_escalated_retry_keeps_the_mismatched_defaults(): + # The escalation already chose the size; the profile must not overwrite it. + ns = _resources_ns(PROFILE_RANGES) + r = ns['_resources'](mem='36000Mi', end=2000) + assert r.limits['memory'] == '36000Mi' + assert r.requests['cpu'] == '1800m', "cpu must fall back to the configured request" + + +def test_the_raised_cpu_limit_is_what_lets_the_peak_grow(): + # At a 2-core limit every range pegs 2.0, so the measured peak is a ceiling + # and the profile can never learn real demand. Headroom above the request is + # the whole point -- the request is still capped for packing. + ns = _resources_ns(PROFILE_RANGES) + measured = ns['_resources'](end=2000) + unmeasured = ns['_resources'](end=99999) + assert 'cpu' not in measured.limits, "measured ranges run uncapped" + assert unmeasured.limits['cpu'] == '2', "unprofiled keeps the configured cap" + assert ns['_cpu_millis'](measured.requests['cpu']) <= ns['_cpu_millis']('1800m') + + +def test_pvc_mode_takes_no_ephemeral_override(): + # /data is not on the node disk there, so sizing it would be meaningless. + ns = _resources_ns(PROFILE_RANGES, req_eph='') + r = ns['_resources'](end=2000) + assert 'ephemeral-storage' not in r.requests + + +def test_the_override_is_computed_before_mem_is_defaulted(): + # Reading `mem is None` after mem has been defaulted can never be true, + # which silently disabled profile sizing entirely once already. + fn = _extract(r"def _resources\(.*?return client\.V1ResourceRequirements").group(0) + assert fn.index('_profile_overrides') < fn.index('mem = mem or LIM_MEM') + + +# --- dispatch order + cross-mode profile reuse ------------------------------- + +def _ranges_ns(order='tip-first', generator='uniform', parallelism=4): + ns = {} + for n in ('_uniform_segment', '_ordered', 'generate_ranges'): + m = re.search(rf"^(def {n}\(.*?)(?=^\S|\Z)", SRC, re.S | re.M) + exec(m.group(1), ns) + ns.update(RANGE_GENERATOR=generator, RANGE_ORDER=order, + STARTING_LEDGER=39990000, LATEST_LEDGER_NUM=40000000, + LEDGERS_PER_JOB=1000, LOGARITHMIC_FLOOR_LEDGERS=64000, + PARALLELISM=parallelism, OVERLAP_LEDGERS=320) + return ns + + +def test_generators_emit_tip_first_by_default(): + r = _ranges_ns()['generate_ranges']() + assert r[0][0] > r[-1][0], "index 0 must be the tip" + + +def test_oldest_first_reverses_dispatch_without_dropping_ranges(): + # A profiling run wants the cheap early ranges measured first: the bucket + # set only grows with ledger position, so tip-first front-loads the + # expensive ones and an interrupted run profiles nothing cheap. + tip = _ranges_ns('tip-first')['generate_ranges']() + old = _ranges_ns('oldest-first')['generate_ranges']() + assert old == list(reversed(tip)) + assert sorted(old) == sorted(tip), "reversing must not change the range set" + + +def test_a_cross_mode_profile_keeps_cpu_and_memory_but_drops_disk(): + # cpu and memory measure the same work in either mode. Disk does not: a pvc + # run never measures node-local usage at all, so its absence must fall back + # to the configured default rather than size the wrong dimension. + fn = _extract(r"def load_profile\(.*?^PROFILE = None", SRC).group(0) + assert 'cross_mode' in fn + assert "k != 'peakEphemeralBytes'" in fn + assert 'return []' not in fn.split('cross_mode = ')[1].split('out = []')[0], \ + "a cross-mode profile must degrade, not be rejected" + + +def test_memory_is_sized_from_rss_never_from_working_set(): + # Working set is whatever limit it was measured under -- the kernel grows + # page cache to fill it. Measured on ssc-test, one 420-ledger range: + # limit 4Gi -> ws 3.61 GiB, rss 2.43 GiB, 775s + # limit 8Gi -> ws 7.48 GiB, rss 2.41 GiB, 746s + # limit 24000Mi -> ws 13.49 GiB, rss 2.28 GiB, 773s + # rss is flat and wall-clock is flat, so sizing from ws would reserve 5x the + # real demand for no gain. + fn = _extract(r"def _profile_overrides\(.*?^def ").group(0) + assert 'peakRssBytes' in fn + assert 'peakWorkingSetBytes' not in fn, "working set must not drive sizing" + + +def test_a_profile_without_rss_leaves_memory_alone(): + # An older artifact predates peakRssBytes; it must fall back to the + # configured default rather than guess from working set. + ns = _overrides_ns([(1, {'peakWorkingSetBytes': 13_000_000_000})]) + assert 'memory' not in ns['_profile_overrides'](1, escalated=False) + + +def test_working_set_is_still_recorded_as_a_diagnostic(): + # It is what kubelet ranks node-pressure evictions on, so it explains an + # eviction that rss cannot -- it just must not feed sizing. + fields = _extract(r"PEAK_FIELDS = \(([^)]+)\)").group(1) + for f in ('peakRssBytes', 'peakWorkingSetBytes'): + assert f in fields, f + + +# --- the chart must render with the shapes the mission actually sends --------- + +import shutil, subprocess + +CHART = __file__.replace('test_job_monitor.py', 'parallel_catchup_helm') + + +def _helm(*extra): + if not shutil.which('helm'): + pytest.skip('helm not installed') + r = subprocess.run(['helm', 'template', 't', CHART, + '--set', 'worker.stellar_core_image=x', *extra], + capture_output=True, text=True) + assert r.returncode == 0, r.stderr + return r.stdout + + +def test_chart_renders_the_service_account_annotations_the_mission_sends(): + # The mission sends service_account.annotations as an indexed array of + # {key,value}; metadata.annotations must be a map. Rendering it straight + # through toYaml produced a list and failed the whole install with + # "cannot unmarshal array into ... map[string]string" -- which no + # source-text assertion would have caught. + out = _helm('--set', 'service_account.annotations[0].key=eks.amazonaws.com/role-arn', + '--set', 'service_account.annotations[0].value=arn:aws:iam::1:role/r') + assert 'eks.amazonaws.com/role-arn: "arn:aws:iam::1:role/r"' in out + + +def test_chart_renders_without_service_account_annotations(): + _helm() + + +def test_chart_renders_the_node_targeting_the_mission_sends(): + out = _helm('--set', 'worker.requireNodeLabels[0].key=purpose', + '--set', 'worker.requireNodeLabels[0].operator=In', + '--set', 'worker.requireNodeLabels[0].values[0]=catchup8-spot', + '--set', 'worker.tolerateNodeTaints[0].key=catchup8-spot', + '--set', 'worker.tolerateNodeTaints[0].effect=NoSchedule') + assert 'catchup8-spot' in out + + +def test_small_ranges_get_absolute_slack_not_just_a_percentage(): + # memory.max bounds anon PLUS page cache. At 190 MiB rss a 1.1x margin is + # 19 MiB of slack -- measured on ssc-test, 90 ranges OOMKilled within 90s of + # dispatch. The fixed headroom is what makes small ranges survivable. + ns = _overrides_ns([(1, {'peakRssBytes': 190 * 2**20})]) + got = ns['_quantity_bytes'](ns['_profile_overrides'](1, escalated=False)['memory']) + slack = (got - 190 * 2**20) / 2**20 + assert slack > 400, f"only {slack:.0f}MiB of slack above rss" + + +def test_oom_escalation_starts_from_what_the_attempt_actually_had(): + # Escalating a 209Mi profiled range off the configured 24000Mi limit jumps + # to 36000Mi -- a 172x overshoot that discards the packing win on first OOM. + ns = {} + for n in ('_quantity_bytes', '_bytes_to_quantity', 'mem_for_attempt'): + m = re.search(rf"^(def {n}\(.*?)(?=^\S|\Z)", SRC, re.S | re.M) + exec(m.group(1), ns) + ns['_UNITS'] = eval(_extract(r"_UNITS = (\{.*?\})").group(1)) + ns.update(LIM_MEM='24000Mi', MEM_BUMP_FACTOR=1.5, MEM_ESCALATION_CAP='48Gi') + assert ns['mem_for_attempt'](2, '702Mi') == '1053Mi' + assert ns['mem_for_attempt'](2) == '36000Mi' # unprofiled keeps old behaviour + + +def test_chart_defaults_match_the_code_defaults(): + # The chart sets these env vars explicitly, so its value WINS over the + # os.getenv default. They drifted once -- code said 512Mi while the chart + # still said 0 -- and the chart silently won, reproducing the OOMs the code + # change was meant to fix. + values = open(__file__.replace( + 'test_job_monitor.py', 'parallel_catchup_helm/values.yaml')).read() + pairs = [('PROFILE_CACHE_HEADROOM', 'profileCacheHeadroom'), + ('PROFILE_MAX_MEM', 'profileMaxMemory'), + ('PROFILE_CPU_LIMIT', 'profileCpuLimit'), + ('PROFILE_MARGIN', 'profileMargin')] + for env, key in pairs: + code = _extract(rf"{env} = .*?os\.getenv\('{env}',\s*'?\"?([^'\")]+)").group(1).strip() + chart = _extract(rf"^\s*{key}:\s*\"?([^\"\n]+)", values).group(1).strip().strip('"') + assert code == chart, f"{env}: code default {code!r} != chart {chart!r}" + + +def test_the_monitor_log_lands_where_the_mission_collects_it(): + # collectLogsFromPods tars /logs. The monitor used to write its own log to + # /data, an emptyDir, so OOM-retry storms never reached the destination + # directory and did not survive a monitor restart. + blk = _extract(r"log_file_name = .*?log_file_path = [^\n]*").group(0) + assert "os.getenv('LOG_DIR'" in blk + col = _extract(r"def base\(end, attempt\):\s*return [^\n]*", COLLECTOR_SRC).group(0) + assert 'LOG_DIR' in col, "collector and monitor must share the collected directory" + + +def test_a_completed_range_releases_its_volume(): + # PVCs are owner-referenced to the release, so nothing reclaimed them until + # helm uninstall. Measured on ssc-test: 2032 bound PVCs / 79 TiB a third of + # the way through a 3982-range run, heading for ~156 TiB and 3982 volumes. + fn = _extract(r"def release_pvc\(.*?^def ").group(0) + assert 'delete_namespaced_persistent_volume_claim' in fn + assert "STORAGE_MODE != 'pvc'" in fn, "ephemeral mode has no PVC to release" + assert 'e.status != 404' in fn, "already-gone must not be an error" + + +def test_the_volume_is_released_only_after_progress_is_saved(): + # If the process dies between the two, the range must still read as + # complete -- keeping a volume is recoverable, losing the record is not. + blk = _extract(r"completed\[end\]\.update\(peaks_for_range.*?release_pvc\(end\)").group(0) + assert blk.index('save_progress') < blk.index('release_pvc') + + +def test_releasing_a_volume_never_fails_a_completed_range(): + fn = _extract(r"def release_pvc\(.*?^def ").group(0) + assert 'raise' not in fn, "a disk cleanup failure must not condemn a finished range" + + +# --- progress durability ----------------------------------------------------- + +def test_progress_is_written_to_the_volume_before_the_configmap(): + # A ConfigMap caps at 1 MiB and the record is ~172 bytes per completed + # range, so it dies around 6100 ranges -- reachable by halving + # ledgersPerJob. Measured mid-run: 348KB at 2024 ranges, 65% of the cap + # projected at 3982. + fn = _extract(r"def save_progress\(.*?^def ").group(0) + assert 'PROGRESS_FILE' in fn + assert fn.index('os.replace') < fn.index('_patch_cm'), \ + "the durable write must land before the mirror" + assert '.tmp' in fn, "a torn write would lose the whole record" + + +def test_a_configmap_mirror_failure_does_not_stop_the_run(): + # reconcile's loop swallows exceptions, so a 413 thrown here meant no + # completion was ever recorded again and finished ranges were redispatched + # forever -- silent, unbounded cost. + fn = _extract(r"def save_progress\(.*?^def ").group(0) + assert 'except ApiException' in fn + assert 'raise' not in fn.split('_patch_cm')[1] + + +def test_progress_is_read_back_from_the_volume_first(): + fn = _extract(r"def load_progress\(.*?^def ").group(0) + assert fn.index('PROGRESS_FILE') < fn.index('read_namespaced_config_map'), \ + "the file is authoritative; the ConfigMap is only a fallback" + assert 'e.status == 404' in fn + + +# Real block from range-40010367-a1 on ssc-test. medida switches to scientific +# notation past 1e6 ms, which is every range with a real transaction load. +MEDIDA_BIG = """2026-07-29T20:11:16.931 GAJSL [default INFO] metric 'ledger.transaction.apply': +2026-07-29T20:11:16.931 GAJSL [default INFO] count = 3231886 +2026-07-29T20:11:16.931 GAJSL [default INFO] mean rate = 812.4 calls/s +2026-07-29T20:11:16.931 GAJSL [default INFO] 1-minute rate = 790.1 calls/s +2026-07-29T20:11:16.931 GAJSL [default INFO] 5-minute rate = 801.3 calls/s +2026-07-29T20:11:16.931 GAJSL [default INFO] 15-minute rate = 799.0 calls/s +2026-07-29T20:11:16.931 GAJSL [default INFO] min = 0.101ms +2026-07-29T20:11:16.931 GAJSL [default INFO] max = 41.2ms +2026-07-29T20:11:16.931 GAJSL [default INFO] mean = 0.404ms +2026-07-29T20:11:16.931 GAJSL [default INFO] stddev = 0.612ms +2026-07-29T20:11:16.931 GAJSL [default INFO] sum = 1.30722e+06ms""" + + +def test_scientific_notation_sum_is_parsed(): + # 25% of ranges recorded no tx_apply -- 91-99% of everything above ledger + # 35M -- because the regex matched "1.30722" then required "ms" and found + # "e+06ms". The metric block was in the archive the whole time. + m = SUM_RE.search(MEDIDA_BIG) + assert m, "scientific-notation sum must parse" + assert float(m.group(1)) / 1000.0 == pytest.approx(1307.22) + + +def test_scanner_reads_a_scientific_notation_block(): + scanner = tx_apply_scanner()() + for line in MEDIDA_BIG.splitlines(): + scanner.feed(line) + assert scanner.seconds == pytest.approx(1307.22) + + +def test_plain_decimal_sums_still_parse(): + m = SUM_RE.search(" sum = 8.34285ms") + assert float(m.group(1)) / 1000.0 == pytest.approx(TX_APPLY_SECONDS) + + +def test_the_chart_grants_the_pvc_delete_release_pvc_needs(): + # release_pvc calls delete_namespaced_persistent_volume_claim. The Role + # granted only get/list/create, so every completion logged a 403 warning and + # the volumes leaked -- 3982 of them, which crashed the EBS CSI controller. + chart = open(__file__.replace( + 'test_job_monitor.py', + 'parallel_catchup_helm/templates/job_monitor.yaml')).read() + blk = _extract(r'resources: \["persistentvolumeclaims"\]\s*\n\s*verbs: \[([^\]]+)\]', chart) + verbs = {v.strip().strip('"') for v in blk.group(1).split(',')} + assert 'delete' in verbs, f"release_pvc needs delete, Role has {sorted(verbs)}" + + +def test_the_configmap_mirror_carries_no_profiling_fields(): + # Profile data lives only on the volume. In the ConfigMap it is what pushes + # a ~30-byte state record to ~172 bytes and the whole document toward the + # 1 MiB cap at ~6100 ranges. + ns = {} + m = re.search(r"^(_PROFILE_ONLY_FIELDS = \(.*?\)\n\n\ndef _state_only\(.*?)(?=\ndef )", + SRC, re.S | re.M) + assert m, "_state_only not found" + exec(m.group(1), ns) + prog = {'completed': {'100': {'attempts': 1, 'count': 16320, 'seconds': 700.0, + 'peakRssBytes': 123, 'peakCpuCores': 1.9, + 'txApply': 200.0, 'wallSeconds': 750.0}}, + 'failed': {}} + out = ns['_state_only'](prog)['completed']['100'] + assert out == {'attempts': 1, 'count': 16320}, out + # and the untouched original still has everything for the volume copy + assert 'peakRssBytes' in prog['completed']['100'] + + +def test_the_volume_copy_keeps_the_profile(): + fn = _extract(r"def save_progress\(.*?^def ").group(0) + assert 'json.dumps(progress' in fn, "the volume write must use the full record" + assert '_state_only' in fn, "the ConfigMap write must be stripped" + assert fn.index('os.replace') < fn.index('_state_only') + + +# --- finished-Job reaping ------------------------------------------------- +# reconcile() LISTs every Job and Pod each pass, so a finished Job costs two +# list entries per pass until it is gone. At 2048-4096 parallelism the dead +# ones outnumbered the live ones within the hour under the old 3600s TTL. + +def _delete_job_ns(delete_impl): + """Exec delete_job against fakes. Nothing here needs a cluster.""" + class ApiException(Exception): + def __init__(self, status): + self.status = status + super().__init__(f"status {status}") + + calls, warnings, reaped = [], [], [] + + class FakeBatch: + def delete_namespaced_job(self, name, namespace, **kw): + calls.append((name, namespace, kw)) + exc = delete_impl(name) + if exc is not None: + raise exc + + ns = { + 'batch_v1': FakeBatch(), + 'NAMESPACE': 'stellar-supercluster', + 'job_name': lambda end, attempt: f"run-r{end}-a{attempt}", + 'metric_jobs_reaped': type('C', (), {'inc': lambda s: reaped.append(1)})(), + 'ApiException': ApiException, + 'logger': type('L', (), {'warning': lambda s, *a: warnings.append(a)})(), + } + exec(_extract(r"^(def delete_job\(.*?)(?=\ndef )").group(1), ns) + return ns['delete_job'], calls, warnings, reaped, ApiException + + +def test_delete_job_reaps_the_pod_too(): + # Background propagation is what actually removes the pod. Orphan/default + # would leave the pod behind and reap nothing that reconcile lists. + delete_job, calls, _, reaped, _ = _delete_job_ns(lambda name: None) + delete_job(30957951, 2) + assert calls == [('run-r30957951-a2', 'stellar-supercluster', + {'propagation_policy': 'Background'})] + assert len(reaped) == 1 + + +def test_delete_job_is_best_effort(): + # A 404 is the normal race with the TTL controller, not an error. Any other + # status must warn and keep going: losing a Job to a leaked object is a + # disk/etcd cost, but raising here would abort a reconcile pass mid-run and + # strand every other range in the same iteration. + _, ApiExc = None, None + for status, want_warn in ((404, False), (403, True), (500, True)): + delete_job, _, warnings, reaped, ApiException = _delete_job_ns( + lambda name, s=status: ApiException(s)) + delete_job(1, 1) # must not raise + assert bool(warnings) is want_warn, f"status {status}" + assert reaped == [], "a failed delete must not count as reaped" + + +def test_the_chart_grants_the_job_delete_reconcile_needs(): + # Same failure the PVC Role had: verbs omitted delete, so every reap logged + # a 403 and nothing was ever collected. + chart = open(__file__.replace( + 'test_job_monitor.py', + 'parallel_catchup_helm/templates/job_monitor.yaml')).read() + blk = _extract(r'resources: \["jobs"\]\s*\n\s*verbs: \[([^\]]+)\]', chart) + verbs = {v.strip().strip('"') for v in blk.group(1).split(',')} + assert 'delete' in verbs, f"delete_job needs delete, Role has {sorted(verbs)}" + + +def test_the_retry_creates_the_successor_before_deleting_the_predecessor(): + # Ordering is the whole safety argument: if the create fails with the + # predecessor already deleted, the range has no live Job, reconcile sees an + # undispatched range and redispatches at attempt 1 -- silently discarding + # the escalated memory limit the retry existed to apply. + body = _extract(r"(create_namespaced_job\(NAMESPACE, build_job\(\s*int\(end\), by_end\[end\], attempt \+ 1.*?)continue").group(1) + assert 'delete_job(end, attempt)' in body, "retry path never reaps the old attempt" + assert body.index('create_namespaced_job') < body.index('delete_job('), \ + "delete_job must come after the successor is created" + + +def test_a_success_whose_metrics_are_missing_keeps_its_job(): + # tx is read from the collector's .metrics, else the pod. Deleting the Job + # reaps the pod, so reaping a success before the metrics land turns a + # recoverable gap into a permanent one -- the same class of loss as the 698 + # ranges the tx_apply regex dropped. + body = _extract(r"(release_pvc\(end\)\n.*?)(?=\s+elif st\.failed:)").group(1) + assert re.search(r"if tx is not None:\s*\n\s*delete_job\(end, attempt\)", body), \ + "success path must gate the reap on the metric having landed" + + +def test_the_chart_ttl_matches_the_code_default(): + # The TTL is now only a backstop, but a chart/code split is how the cache + # headroom regression shipped: the code default was fixed and the chart + # still forced the old value. + chart = open(__file__.replace( + 'test_job_monitor.py', 'parallel_catchup_helm/values.yaml')).read() + want = int(_extract(r"JOB_TTL_SECONDS = int\(os\.getenv\('JOB_TTL_SECONDS', (\d+)\)\)").group(1)) + got = int(_extract(r"jobTtlSeconds: (\d+)", chart).group(1)) + assert got == want, f"chart sets {got}, code defaults to {want}" + + +# --- peak anon from kubelet ---------------------------------------------- +# Page cache expands to fill memory.max, so memory.peak ~= the limit for every +# pod and cannot be profiled (measured on ssc-test: a range needing 862 MiB of +# anon reported peak 12704 MiB under a 24000 MiB limit). Anon is the only +# limit-independent figure, and kubelet reports it per container for free in +# the payload the collector already fetches for ephemeral storage. + +def _sample_ns(summary, container='stellar-core', streaming=True): + """Exec sample_kubelet's per-pod body against one kubelet payload.""" + eph, anon, ws, flushed, streaming_ref, written, logged = {}, {}, {}, {}, {}, [], [] + + class FakeResp: + def __init__(self, d): self._d = d + async def __aenter__(self): return self + async def __aexit__(self, *a): return False + def raise_for_status(self): pass + async def json(self): return self._d + + class FakeSession: + def get(self, url, headers=None): return FakeResp(summary) + + ns = { + 'API': 'https://k8s', 'CONTAINER': container, + '_eph_peak': eph, '_anon_peak': anon, '_ws_peak': ws, + '_peak_flushed': flushed, '_streaming': streaming_ref, + 'PEAK_FLUSH_RATIO': 1.05, 'STORAGE_MODE': 'ephemeral', + 'write_metrics': lambda e, a, v: written.append((e, a, v)), + 'token': lambda: 't', + 'logger': type('L', (), {'warning': lambda s, *a: None, + 'info': lambda s, *a: logged.append(a)})(), + } + if streaming: + # The main loop records this when it opens a pod's stream; a peak flush + # needs it to know which .metrics file the pod belongs to. + for _p in summary.get('pods', []): + streaming_ref[_p['podRef']['name']] = ('999', '1') + exec(_extract(r"^(async def sample_kubelet\(.*?)(?=\n\nasync def )", + COLLECTOR_SRC).group(1), ns) + import asyncio + asyncio.run(ns['sample_kubelet'](FakeSession(), ['node-a'])) + _sample_ns.last = {'ws': ws, 'written': written, 'flushed': flushed} + return eph, anon + + +def _payload(pod, rss, used=None, container='stellar-core'): + mem = {} if rss is None else {'rssBytes': rss} + return {'pods': [{'podRef': {'name': pod}, + 'ephemeral-storage': {} if used is None else {'usedBytes': used}, + 'containers': [{'name': container, 'memory': mem}]}]} + + +def test_kubelet_anon_is_tracked_as_a_high_water_mark(): + # A single low sample after a high one must not lower the peak: the whole + # point is catching the spike, and download-phase anon oscillates. + eph, anon = _sample_ns(_payload('p1', 900, used=5)) + assert anon == {'p1': 900} and eph == {'p1': 5} + ns_hi = _payload('p1', 900) + ns_hi['pods'][0]['containers'][0]['memory']['rssBytes'] = 400 + # re-run with a lower reading against a pre-seeded peak + eph2, anon2 = _sample_ns({'pods': [ + _payload('p1', 900)['pods'][0], ns_hi['pods'][0]]}) + assert anon2['p1'] == 900, "a later, lower sample overwrote the peak" + + +def test_a_container_without_stats_yet_is_skipped_not_zeroed(): + # rssBytes is absent for the first seconds of a container's life. Recording + # 0, or letting it raise, would either poison the peak or kill the sampler + # for every other pod on the node. + eph, anon = _sample_ns(_payload('p1', None, used=7)) + assert anon == {}, "missing rssBytes must not be recorded" + assert eph == {'p1': 7}, "ephemeral must still be sampled" + + +def test_only_the_worker_container_is_measured(): + # Sidecars share the pod. Summing or last-wins across containers would size + # the range from whichever one kubelet listed last. + eph, anon = _sample_ns(_payload('p1', 900, container='istio-proxy')) + assert anon == {}, "a non-worker container was measured" + + +def test_peak_anon_is_kept_from_every_attempt(): + # An OOM-killed pod's last sample is below its true peak by construction -- + # it died reaching past it. Feeding that into the profile would re-derive + # the very limit that killed the range. + # peaks_for_range takes the max across a resumed chain, so a partial + # attempt can only raise the figure. Gating here is what hid the + # download-phase peak of a range that resumed. + body = _extract(r"(anon = _anon_peak\.pop\(pod, None\).*?)(?=\s+ws = _ws_peak)", + COLLECTOR_SRC).group(1) + assert 'done_ok(pod)' not in body, "peakAnonBytes is still gated on success" + + +def test_peak_anon_reaches_the_profile(): + # peaks_for_range filters to PEAK_FIELDS, and the ConfigMap mirror strips + # _PROFILE_ONLY_FIELDS. A new measurement absent from either is silently + # dropped between the collector and the profile. + for name in ('PEAK_FIELDS', '_PROFILE_ONLY_FIELDS'): + blk = _extract(name + r" = \(([^)]+)\)").group(1) + fields = {f.strip().strip("'") for f in blk.split(',') if f.strip()} + assert 'peakAnonBytes' in fields, f"{name} drops peakAnonBytes" + + +def test_sizing_prefers_anon_and_falls_back_to_the_scraped_rss(): + # A profile captured before the collector tracked anon must keep sizing + # exactly as it did, or every existing profile silently reverts to default. + body = _extract(r"(rss = prof\.get\('peakAnonBytes'\).*?out\['memory'\])").group(1) + assert "prof.get('peakAnonBytes') or prof.get('peakRssBytes')" in body + + +@pytest.mark.parametrize('peak,want_mi', [ + (648 * 1024**2, int(648 * 1.15) + 512), # measured live: anon 648Mi + (1467 * 1024**2, int(1467 * 1.15) + 512), # the largest anon sampled + (222 * 1024**2, int(222 * 1.15) + 512), # the smallest +]) +def test_the_sizing_formula_is_peak_times_115_plus_512mi(peak, want_mi): + margin = float(_extract(r"PROFILE_MARGIN = float\(os\.getenv\('PROFILE_MARGIN', ([\d.]+)\)\)").group(1)) + head = _extract(r"PROFILE_CACHE_HEADROOM = os\.getenv\('PROFILE_CACHE_HEADROOM', '(\d+)Mi'\)").group(1) + got_mi = int(peak * margin) // 1024**2 + int(head) + assert (margin, int(head)) == (1.15, 512) + assert got_mi == want_mi + + +def test_the_chart_matches_the_new_sizing_defaults(): + chart = open(__file__.replace( + 'test_job_monitor.py', 'parallel_catchup_helm/values.yaml')).read() + assert _extract(r"profileMargin: ([\d.]+)", chart).group(1) == \ + _extract(r"PROFILE_MARGIN = float\(os\.getenv\('PROFILE_MARGIN', ([\d.]+)\)\)").group(1) + assert _extract(r'profileCacheHeadroom: "(\d+Mi)"', chart).group(1) == \ + _extract(r"PROFILE_CACHE_HEADROOM = os\.getenv\('PROFILE_CACHE_HEADROOM', '(\d+Mi)'\)").group(1) + + +# --- zombie streams ------------------------------------------------------- +# `done` reads terminal.get(pod, False) and terminal is only written for pods +# present in list_pods. A pod deleted while Running -- reaped node, eviction, +# or the monitor deleting a finished Job -- therefore never became terminal, +# and its stream retried every 30s for the rest of the run while holding one of +# MAX_CONCURRENT connection slots. + +def test_a_vanished_pod_is_marked_terminal_so_its_stream_can_finish(): + body = _extract(r"(live = \{p\['metadata'\]\['name'\].*?)(?=\n\s+# Unconditional:)", + COLLECTOR_SRC).group(1) + assert 'terminal[name] = True' in body, \ + "a vanished pod never becomes terminal, so done() stays False forever" + assert 'n not in live' in body, "nothing detects a pod leaving the pod list" + + +def test_a_vanished_stream_is_cancelled_if_it_will_not_finish(): + # Marking terminal is not enough on its own: a stream blocked inside a + # connection attempt never reaches its done() check, which is exactly the + # state that starves every other stream. + body = _extract(r"(live = \{p\['metadata'\]\['name'\].*?)(?=\n\s+# Unconditional:)", + COLLECTOR_SRC).group(1) + assert 't.cancel()' in body and 'VANISHED_GRACE_CYCLES' in body, \ + "no backstop cancel for a stream that cannot finalize" + assert 'del tasks[name]' in body, "cancelled task is never removed from tasks" + + +def test_the_grace_is_more_than_one_cycle(): + # A stream mid-fetch_peaks against a slow Prometheus must not be cancelled + # out from under its own metrics write. + n = int(_extract(r"VANISHED_GRACE_CYCLES = int\(os\.getenv\('COLLECTOR_VANISHED_GRACE_CYCLES', (\d+)\)\)", + COLLECTOR_SRC).group(1)) + assert n >= 2, f"grace of {n} cycle(s) can cancel a stream mid-finalize" + + + +def test_both_exit_paths_share_one_finalize(): + # Two copies of the metrics/discard logic is how one path silently stops + # writing peakAnonBytes while the other keeps working. + # Three: clean exit, pod-gone 404, and an interrupted read on a pod that + # has since gone terminal. + assert len(re.findall(r"await finalize\(session, pod, end, attempt, tx, done_ok\)", + COLLECTOR_SRC)) == 3 + assert len(re.findall(r"write_metrics\(end, attempt, measured\)", COLLECTOR_SRC)) == 1 + + + +def _run_stream_pod(status, terminal): + """Execute stream_pod against a fake apiserver. Returns finalize calls. + + Executed rather than pattern-matched: the previous version of this test + asserted on a `except ClientResponseError` branch that raise_for_status + could never reach, because an earlier `if resp.status == 404` returned + first. It passed against dead code. + """ + import asyncio, tempfile, types, os as _os + calls = [] + + class FakeResp: + status = None + async def __aenter__(self): return self + async def __aexit__(self, *a): return False + def raise_for_status(self): + if self.status >= 400: + raise OSError(f"HTTP {self.status}") + @property + def content(self): + async def it(): + if False: yield b'' + return it() + + FakeResp.status = status # class bodies cannot close over a local + + class FakeSession: + def get(self, url, params=None, headers=None): return FakeResp() + + async def fake_finalize(session, pod, end, attempt, tx, done_ok): + calls.append((pod, end, attempt)) + + d = tempfile.mkdtemp() + ns = { + 'asyncio': asyncio, 'gzip': __import__('gzip'), 'os': _os, + 'API': 'https://k8s', 'NAMESPACE': 'ns', 'CONTAINER': 'stellar-core', + 'LOG_DIR': d, 'STATE_FLUSH_SECONDS': 10, + 'token': lambda: 't', 'finalize': fake_finalize, + 'base': lambda e, a: _os.path.join(d, f"range-{e}-a{a}"), + 'read_state': lambda e, a: None, 'write_state': lambda e, a, ts: None, + '_TS_RE': re.compile(r"^\d{4}"), + 'TxApplyScanner': type('T', (), {'seconds': None, 'feed': lambda s, l: None}), + 'logger': type('L', (), {'info': lambda s, *a: None, + 'warning': lambda s, *a: None})(), + } + exec(_extract(r"^(async def stream_pod\(.*?)(?=\n\nasync def )", + COLLECTOR_SRC).group(1), ns) + coro = ns['stream_pod'](FakeSession(), 'pod-1', '999', '1', + lambda p: terminal, lambda p: False) + asyncio.run(asyncio.wait_for(coro, timeout=2)) + return calls + + +def test_a_404_finalizes_what_was_already_streamed(): + # The pod object is gone, but the bytes already read still owe a tx_apply + # and the peaks live in Prometheus, not on the pod. + assert _run_stream_pod(404, terminal=False) == [('pod-1', '999', '1')] + + +def test_an_interrupted_read_on_a_terminal_pod_still_finalizes(): + # 500s were a burst at ramp. Returning bare here dropped the metrics for + # every range whose last read happened to throw. + assert _run_stream_pod(500, terminal=True) == [('pod-1', '999', '1')] + + +def test_an_interrupted_read_on_a_live_pod_does_not_finalize(): + # Still running: retry is correct, and finalizing now would write a + # truncated peak and let the range look measured when it is not. + import pytest as _pt + with _pt.raises(Exception): + _run_stream_pod(500, terminal=False) # retries until the 2s timeout + + +# --- kubelet replaces Prometheus ------------------------------------------ +# Every peak the profile uses now comes from the kubelet payload the collector +# already fetches. Prometheus was lossy for this: a 30s scrape against ~10s +# cAdvisor housekeeping, plus a hard dependency on Prometheus being up, +# reachable and still retaining the window -- and _promql swallowed all three +# failures into "no peak", so an outage produced a complete-looking, empty +# profile. + +def test_the_collector_no_longer_reads_from_prometheus(): + # Comments stripped: one deliberately explains why the local high-water + # dict exists where max_over_time did not need to. + code = '\n'.join(l for l in COLLECTOR_SRC.splitlines() + if not l.lstrip().startswith('#')) + for token in ('PROMETHEUS_URL', '_promql', 'fetch_peaks', 'max_over_time'): + assert token not in code, f"{token} survived the kubelet switch" + + +def test_cpu_is_not_profiled(): + # REQ_CPU is fixed, so a measured cpu value has nothing to size and only + # makes packing non-uniform. + assert 'peakCpuCores' not in _extract(r"PEAK_FIELDS = \(([^)]+)\)").group(1) + body = _extract(r"^(def _profile_overrides\(.*?)(?=\ndef )").group(1) + assert "out['cpu']" not in body + + +def test_memory_is_sampled_in_both_storage_modes(): + # This was gated on ephemeral mode back when the sampler only did disk, + # which left every pvc run with no anon peak at all. + loop = _extract(r"while True:\n(.*?)await asyncio\.sleep\(POLL_SECONDS\)", + COLLECTOR_SRC).group(1) + call = loop.index('sample_kubelet') + gate = loop.rfind("STORAGE_MODE == 'ephemeral'", 0, call) + assert gate == -1, "the kubelet sampler is still gated on storage mode" + + +def test_the_disk_axis_stays_mode_gated(): + # ephemeral-storage is meaningless in pvc mode: /data is not on the node. + fn = _extract(r"^(async def sample_kubelet\(.*?)(?=\n\nasync def )", + COLLECTOR_SRC).group(1) + used = fn.index("get('usedBytes')") + assert "STORAGE_MODE == 'ephemeral'" in fn[used:used + 200] + + + + + +def test_an_in_flight_peak_is_flushed_so_a_restart_cannot_lose_it(): + # Prometheus computed max_over_time server-side and needed no state. A local + # high-water dict does: without a flush, a collector restart resets a range's + # peak to whatever it is using at that moment, which under-reports and sizes + # the next run too small. Executed, not pattern-matched -- `if False:` leaves + # every identifier in place and passes a source-text check. + _sample_ns(_payload('p1', 900, used=5)) + w = _sample_ns.last['written'] + assert w, "a first sample never flushed its peak" + assert w[-1][2] == {'peakAnonBytes': 900} + + +def test_a_peak_that_barely_grows_is_not_reflushed(): + # One write per sample per pod, at 2048 pods, would be the dominant cost of + # the sampler. Only growth past PEAK_FLUSH_RATIO earns a write. + pods = [_payload('p1', 900)['pods'][0], _payload('p1', 910)['pods'][0]] + _sample_ns({'pods': pods}) + assert len(_sample_ns.last['written']) == 1, "a 1.1% rise triggered a second flush" + + pods = [_payload('p1', 900)['pods'][0], _payload('p1', 2000)['pods'][0]] + _sample_ns({'pods': pods}) + assert len(_sample_ns.last['written']) == 2, "a 2.2x rise did not flush" + + +def test_working_set_is_sampled_recorded_but_never_sizes_anything(): + # It counts active page cache, which grows to fill the limit -- measured at + # 3.61/7.48/13.49 GiB for one range under 4Gi/8Gi/24000Mi limits while rss + # held at ~2.4 GiB. Useful as a diagnostic, never as a request. + p = _payload('p1', 900, used=5) + p['pods'][0]['containers'][0]['memory']['workingSetBytes'] = 4096 + _sample_ns(p) + assert _sample_ns.last['ws'] == {'p1': 4096}, "working set is not sampled" + assert 'peakWorkingSetBytes' in _extract(r"PEAK_FIELDS = \(([^)]+)\)").group(1) + body = _extract(r"^(def _profile_overrides\(.*?)(?=\ndef )").group(1) + assert 'peakWorkingSetBytes' not in body, "working set must not size a request" + + +def test_finalize_records_the_working_set_peak(): + # Sampling it is useless if finalize drops it on the floor. + fn = _extract(r"^(async def finalize\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + written, ws = [], {'pod-1': 4096} + ns = { + '_anon_peak': {'pod-1': 900}, '_ws_peak': ws, '_eph_peak': {}, + '_peak_flushed': {}, '_streaming': {}, 'SAVE_SUCCESS_LOGS': True, + 'write_metrics': lambda e, a, v: written.append(v), + 'discard': lambda e, a: None, + 'logger': type('L', (), {'info': lambda s, *a: None})(), + } + exec(fn, ns) + import asyncio + tx = type('T', (), {'seconds': 1.5, 'resumed': False})() + asyncio.run(ns['finalize'](None, 'pod-1', '999', '1', tx, lambda p: True)) + assert written and written[0].get('peakWorkingSetBytes') == 4096 + assert written[0].get('peakAnonBytes') == 900 + + +def test_the_flush_ratio_default_is_above_one_and_matches_the_chart(): + # The behaviour tests inject their own ratio, so nothing else pins the + # default. At exactly 1.0 every sample flushes: one write per pod per poll, + # 2048 pods, which is the cost the ratio exists to avoid. + got = float(_extract( + r"PEAK_FLUSH_RATIO = float\(os\.getenv\('PEAK_FLUSH_RATIO', ([\d.]+)\)\)", + COLLECTOR_SRC).group(1)) + assert got > 1.0, f"ratio {got} flushes on every sample" + chart = open(__file__.replace( + 'test_job_monitor.py', 'parallel_catchup_helm/values.yaml')).read() + assert float(_extract(r"peakFlushRatio: ([\d.]+)", chart).group(1)) == got + + +# --- peaks aggregate across attempts -------------------------------------- +# In pvc mode a pod killed after replay starts leaves /data, and the next +# attempt resumes at LCL+1 with RESUME=true -- skipping the archive download and +# bucket apply, which is where peak memory happens. Profiling only the winning +# attempt therefore under-reports a resumed range by the whole download gap, and +# on spot (where eviction is routine and resume is the point of durable /data) +# that would make the run unprofileable. + +def _peaks_ns(attempts): + """Exec peaks_for_range over a temp dir. attempts: {n: (metrics, outcome)}. + + `resumed` lives in the metrics dict, as the collector writes it. + """ + import tempfile, json as _json, os as _os + d = tempfile.mkdtemp() + for n, (metrics, outcome) in attempts.items(): + if metrics is not None: + with open(_os.path.join(d, f"m-{n}"), 'w') as fh: + fh.write(metrics if isinstance(metrics, str) else _json.dumps(metrics)) + if outcome is not None: + with open(_os.path.join(d, f"o-{n}"), 'w') as fh: + _json.dump(outcome, fh) + ns = { + 'json': _json, + 'metrics_path': lambda e, n: _os.path.join(d, f"m-{n}"), + 'outcome_path': lambda e, n: _os.path.join(d, f"o-{n}"), + 'PEAK_FIELDS': ('peakAnonBytes', 'peakRssBytes', 'peakWorkingSetBytes', + 'peakEphemeralBytes'), + } + exec(_extract(r"^(def _attempt_resumed\(.*?)(?=\ndef )").group(1), ns) + exec(_extract(r"^(def peaks_for_range\(.*?)(?=\ndef _attempt_resumed)").group(1), ns) + ns['_attempt_resumed'] = ns['_attempt_resumed'] + return ns['peaks_for_range'] + + +def test_a_resumed_range_keeps_the_peak_from_the_attempt_that_did_the_download(): + # a1 evicted mid-replay having already done the download; a2 resumes at + # LCL+1 and only replays the tail. a2 alone would report 400MiB for a range + # that really needs 2GiB. + f = _peaks_ns({ + 1: ({'peakAnonBytes': 2 * 1024**3}, {'outcome': 'disrupted'}), + 2: ({'peakAnonBytes': 400 * 1024**2, 'resumed': True}, None), + }) + assert f(999, 2)['peakAnonBytes'] == 2 * 1024**3 + + +def test_an_oom_killed_attempt_still_counts_toward_the_peak(): + # It really did allocate ~8Gi and wanted more, so that is a lower bound on + # demand. Sizing off the quieter successful attempt instead would OOM the + # range again; 8Gi * 1.15 + 512Mi clears the level it died at. + f = _peaks_ns({ + 1: ({'peakAnonBytes': 8 * 1024**3}, {'outcome': 'oom'}), + 2: ({'peakAnonBytes': 900 * 1024**2, 'resumed': True}, None), + }) + assert f(999, 2)['peakAnonBytes'] == 8 * 1024**3 + + +def test_an_oom_killed_attempt_still_counts_on_the_disk_axis(): + # It hit the memory ceiling, not the disk one, so its disk figure is real. + f = _peaks_ns({ + 1: ({'peakEphemeralBytes': 30 * 1024**3}, {'outcome': 'oom'}), + 2: ({'peakEphemeralBytes': 5 * 1024**3, 'resumed': True}, None), + }) + assert f(999, 2)['peakEphemeralBytes'] == 30 * 1024**3 + + +def test_a_disk_evicted_attempt_counts_on_every_axis(): + f = _peaks_ns({ + 1: ({'peakEphemeralBytes': 40 * 1024**3, + 'peakAnonBytes': 3 * 1024**3}, {'outcome': 'ephemeral'}), + 2: ({'peakEphemeralBytes': 9 * 1024**3, + 'peakAnonBytes': 1 * 1024**3, 'resumed': True}, None), + }) + out = f(999, 2) + assert out['peakEphemeralBytes'] == 40 * 1024**3 + assert out['peakAnonBytes'] == 3 * 1024**3 + + +def test_a_missing_or_malformed_metrics_file_is_tolerated(): + f = _peaks_ns({1: (None, None), 2: ("not json at all", None), + 3: ({'peakAnonBytes': 5, 'resumed': True}, None)}) + assert f(999, 3) == {'peakAnonBytes': 5} + assert _peaks_ns({})(999, 3) == {} + + +def test_an_absent_peak_never_reaches_the_profile_as_a_null(): + # The consumer falls back to a default on a missing field, so a null defeats it. + f = _peaks_ns({1: ({'peakAnonBytes': None, 'peakRssBytes': 7}, None)}) + assert f(999, 1) == {'peakRssBytes': 7} + + +def test_spot_is_never_excluded_as_a_capacity_type(): + # Truncation is what invalidates a sample, not the node it ran on. Gating on + # spot would blank the axis for an all-spot run, the run we most want. + assert 'capacity-type' not in COLLECTOR_SRC + assert 'capacity-type' not in SRC + + +def test_a_fresh_retry_supersedes_everything_before_it(): + # No RESUME line means new-db ran and this attempt did the whole range, so + # its sample is complete. The earlier attempt measured the same work and + # only adds noise -- and in ephemeral mode, where resume can never fire, + # this is every retry. + f = _peaks_ns({ + 1: ({'peakAnonBytes': 8 * 1024**3}, {'outcome': 'oom'}), + 2: ({'peakAnonBytes': 900 * 1024**2}, None), # no 'resumed' + }) + assert f(999, 2)['peakAnonBytes'] == 900 * 1024**2 + + +def test_the_chain_stops_at_the_last_fresh_start(): + # a1 fresh (dropped), a2 fresh and evicted mid-replay, a3 resumed from it. + # Only a2+a3 describe the same continuous pass over the range. + f = _peaks_ns({ + 1: ({'peakAnonBytes': 9 * 1024**3}, {'outcome': 'oom'}), + 2: ({'peakAnonBytes': 2 * 1024**3}, {'outcome': 'disrupted'}), + 3: ({'peakAnonBytes': 500 * 1024**2, 'resumed': True}, None), + }) + assert f(999, 3)['peakAnonBytes'] == 2 * 1024**3 + + +def test_resumed_is_read_from_the_workers_own_line(): + # "RESUME DECLINED" must not count as a resume -- it means the opposite. + scanner_src = _extract(r"^(class TxApplyScanner:.*?)(?=\ndef )", COLLECTOR_SRC).group(1) + assert "RESUME_MARK = 'RESUME: '" in scanner_src + ns = {'_TX_METRIC': "metric 'ledger.transaction.apply'", '_SUM_RE': SUM_RE} + exec(scanner_src, ns) + s = ns['TxApplyScanner']() + s.feed("RESUME DECLINED: k last close was 'none'; bucket phase incomplete, starting fresh") + assert s.resumed is False, "a declined resume was read as a resume" + s.feed("RESUME: k reached ledger 31005951, replay had started; skipping new-db") + assert s.resumed is True + + +def test_resumed_never_reaches_the_profile_as_a_field(): + # It is bookkeeping for peaks_for_range, not a measurement. + assert 'resumed' not in _extract(r"PEAK_FIELDS = \(([^)]+)\)").group(1) + + +def test_finalize_records_that_an_attempt_resumed(): + # Without this in .metrics, peaks_for_range cannot tell a resumed tail from + # a complete pass, and every resumed range is profiled off its tail alone. + fn = _extract(r"^(async def finalize\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + import asyncio + + def run(resumed): + written = [] + ns = {'_anon_peak': {'p': 1}, '_ws_peak': {}, '_eph_peak': {}, + '_peak_flushed': {}, '_streaming': {}, 'SAVE_SUCCESS_LOGS': True, + 'write_metrics': lambda e, a, v: written.append(v), + 'discard': lambda e, a: None, + 'logger': type('L', (), {'info': lambda s, *a: None})()} + exec(fn, ns) + tx = type('T', (), {'seconds': None, 'resumed': resumed})() + asyncio.run(ns['finalize'](None, 'p', '999', '1', tx, lambda p: True)) + return written[0] + + assert run(True).get('resumed') is True + assert 'resumed' not in run(False), "a fresh attempt must not be marked resumed" From 0280156a1d832c58f9194ca88168302adc6d938a Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Wed, 29 Jul 2026 21:29:02 -0400 Subject: [PATCH 002/117] Sum txApply and seconds across the resumed attempt chain Peaks already aggregated across a resumed chain; the two timings did not, so a range interrupted mid-replay reported only its final leg. - txApply: medida's total is per-process, so a pod resuming at LCL+1 reports only the transactions it replayed. Summed across the chain. - seconds: a failed attempt's duration was never persisted anywhere -- reconcile computes it solely on the success path. record_outcome now stores attemptSeconds while it still holds the pod, and the range total sums the chain. - _resumed_chain is now shared by all three aggregations. txApply slightly over-counts: replay restarts at the checkpoint boundary containing LCL, so up to 64 ledgers can be applied twice. Against a 16320-ledger range that is <=0.4%, but it is a fixed ledger cost rather than a percentage and grows as ranges shrink. seconds is compute, not elapsed -- gaps between attempts (scheduling, image pull, node startup) are not in it. wallSeconds still covers those. 123 unit tests, each mutation-checked. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 100 ++++++++++++++---- .../test_job_monitor.py | 85 ++++++++++++++- 2 files changed, 165 insertions(+), 20 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index fb53cb05..184fbe05 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -674,6 +674,10 @@ def record_outcome(end, attempt, pod): return data = classify(pod) data['pod'] = pod.metadata.name + # The only place a failed attempt's duration is ever available: the pod is + # about to be reaped, and reconcile computes `seconds` solely on the success + # path. Without it a resumed chain can only report its final leg. + data['attemptSeconds'] = _pod_seconds(pod) try: tmp = path + '.tmp' with open(tmp, 'w') as fh: @@ -852,16 +856,8 @@ def peaks_for_range(end, attempt=1): Advisory: used to size a LATER run's requests, never to decide anything about this one. Any field may be absent. """ - # Walk back only over a contiguous chain of resumed attempts. An attempt - # that did NOT resume ran new-db and did the whole range, so its sample is - # complete and supersedes everything before it -- in ephemeral mode, where - # /data dies with the pod and resume can never fire, that collapses to the - # winning attempt alone, exactly as before. - first = int(attempt) - while first > 1 and _attempt_resumed(end, first): - first -= 1 out = {} - for n in range(first, int(attempt) + 1): + for n in _resumed_chain(end, attempt): try: with open(metrics_path(end, n)) as fh: data = json.load(fh) @@ -874,6 +870,30 @@ def peaks_for_range(end, attempt=1): return out +def _pod_seconds(pod): + """Container start -> finish for one attempt, or None if unreadable.""" + start = pod.status.start_time if pod.status else None + if start is None: + return None + for cs in (pod.status.container_statuses or []): + t = cs.state.terminated if cs.state else None + if t is not None and t.finished_at: + return (t.finished_at - start).total_seconds() + return None + + +def _resumed_chain(end, attempt): + """Attempts describing one continuous pass over the range, oldest first. + + Stops at the last attempt that ran new-db: that one covered the whole range + on its own, so nothing before it is part of the same pass. + """ + first = int(attempt) + while first > 1 and _attempt_resumed(end, first): + first -= 1 + return range(first, int(attempt) + 1) + + def _attempt_resumed(end, attempt): """Did this attempt pick up at LCL+1 rather than run new-db? @@ -889,11 +909,55 @@ def _attempt_resumed(end, attempt): def tx_apply_for_range(end, attempt=1, pod_name=None): - """Final 'ledger.transaction.apply' sum for one attempt, in seconds. + """Total 'ledger.transaction.apply' seconds for the whole range. + + Summed across the resumed chain, not read from the winning attempt alone. + medida's total is per-process, so a pod that resumes at LCL+1 reports only + the transactions it replayed -- on a range that was interrupted mid-replay + that is the tail, not the range. + + Slightly over-counts: replay restarts at the checkpoint boundary containing + LCL, so up to 64 ledgers can be applied twice. Against a 16320-ledger range + that is <=0.4%, but it is a fixed ledger cost rather than a percentage, so + it grows as ranges shrink. + """ + total = None + for n in _resumed_chain(end, attempt): + # pod_name only ever names the LAST attempt's pod, so the archive/pod + # fallbacks are offered to that one alone; earlier legs come from the + # .metrics the collector already wrote. + leg = _tx_apply_for_attempt(end, n, pod_name if n == int(attempt) else None) + if leg is not None: + total = leg if total is None else total + leg + return total + + +def seconds_for_range(end, attempt=1, final=None): + """Compute time for the whole range, summed across the resumed chain. + + `final` is the winning attempt's own duration, which reconcile has in hand + from the pod. Earlier legs come from their .outcome, written when the + monitor classified the failure and still had the pod. + + This is compute, not elapsed: the gaps between attempts -- scheduling, image + pull, a node coming up -- are not in it. wallSeconds covers those. + """ + total = None + for n in _resumed_chain(end, attempt): + if n == int(attempt): + leg = final + else: + leg = (read_outcome(end, n) or {}).get('attemptSeconds') + if leg is not None: + total = leg if total is None else total + leg + return total + + +def _tx_apply_for_attempt(end, attempt=1, pod_name=None): + """Final 'ledger.transaction.apply' sum for ONE attempt, in seconds. stellar-core prints the medida block once at exit (we pass --metric), so - this is the exact total rather than a sample. Only ever called for a - SUCCEEDED attempt, so a failed one never contributes. + this is the exact total for that process rather than a sample. Three sources, cheapest and most durable first: @@ -1429,16 +1493,14 @@ def reconcile(state): # successful pod's own start -> container finish is what # worker.sh used to report, and is the number comparable across # the redis cutover. - seconds = None - if pod is not None and pod.status.start_time: - for cs in (pod.status.container_statuses or []): - t = cs.state.terminated if cs.state else None - if t is not None and t.finished_at: - seconds = (t.finished_at - pod.status.start_time).total_seconds() - break + seconds = _pod_seconds(pod) if pod is not None else None wall = None if st.start_time and st.completion_time: wall = (st.completion_time - st.start_time).total_seconds() + # Chain total, not this leg alone: a range that resumed spent + # real time in the attempts before the winner. Falls back to the + # single leg, then to wall, when nothing durable survived. + seconds = seconds_for_range(end, attempt, seconds) or seconds if seconds is None: seconds = wall # pod already gone; wall is the only figure left # Not gated on `pod`: the collector's .metrics/.log.gz are diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index 8ff42902..6a07a02a 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -237,7 +237,8 @@ def test_tx_apply_survives_a_reaped_pod(): def test_tx_apply_prefers_durable_sources_over_the_pod_api(): - fn = _extract(r"def tx_apply_for_range\(.*?^def ").group(0) + # The per-attempt reader; tx_apply_for_range now sums these over the chain. + fn = _extract(r"def _tx_apply_for_attempt\(.*?^def ").group(0) assert fn.index('metrics_path') < fn.index('log_path') < fn.index('read_namespaced_pod_log') @@ -1519,3 +1520,85 @@ def run(resumed): assert run(True).get('resumed') is True assert 'resumed' not in run(False), "a fresh attempt must not be marked resumed" + + +# --- timings aggregate across the resumed chain too ------------------------ +# medida's total is per-process and a pod's duration is its own, so both are +# tail-only for a resumed range in exactly the way the peaks were. + +def _chain_ns(attempts, extra=None): + """Exec the chain helpers over a temp dir. attempts: {n: (metrics, outcome)}.""" + import tempfile, json as _json, os as _os + d = tempfile.mkdtemp() + for n, (metrics, outcome) in attempts.items(): + if metrics is not None: + with open(_os.path.join(d, f"m-{n}"), 'w') as fh: + _json.dump(metrics, fh) + if outcome is not None: + with open(_os.path.join(d, f"o-{n}"), 'w') as fh: + _json.dump(outcome, fh) + ns = { + 'json': _json, + 'metrics_path': lambda e, n: _os.path.join(d, f"m-{n}"), + 'outcome_path': lambda e, n: _os.path.join(d, f"o-{n}"), + } + for name in ('_attempt_resumed', '_resumed_chain', 'read_outcome', + 'seconds_for_range'): + ns[name] = None + exec(_extract(r"^(def _attempt_resumed\(.*?)(?=\ndef )").group(1), ns) + exec(_extract(r"^(def _resumed_chain\(.*?)(?=\ndef )").group(1), ns) + exec(_extract(r"^(def read_outcome\(.*?)(?=\ndef )").group(1), ns) + exec(_extract(r"^(def seconds_for_range\(.*?)(?=\ndef )").group(1), ns) + ns.update(extra or {}) + return ns + + +def test_seconds_sums_the_whole_resumed_chain(): + # a1 ran 900s then was evicted mid-replay; a2 resumed and took 300s. The + # range cost 1200s of compute, not 300. + ns = _chain_ns({ + 1: ({}, {'outcome': 'disrupted', 'attemptSeconds': 900.0}), + 2: ({'resumed': True}, None), + }) + assert ns['seconds_for_range'](999, 2, 300.0) == 1200.0 + + +def test_seconds_ignores_attempts_before_a_fresh_start(): + # a2 ran new-db and did the whole range itself, so a1's 900s is not part of + # the same pass. + ns = _chain_ns({ + 1: ({}, {'outcome': 'oom', 'attemptSeconds': 900.0}), + 2: ({}, None), # no 'resumed' + }) + assert ns['seconds_for_range'](999, 2, 300.0) == 300.0 + + +def test_seconds_survives_a_leg_with_no_recorded_duration(): + # An attempt whose pod vanished before it was classified has no + # attemptSeconds. Better to under-report one leg than return nothing. + ns = _chain_ns({ + 1: ({}, {'outcome': 'disrupted'}), # no attemptSeconds + 2: ({'resumed': True}, None), + }) + assert ns['seconds_for_range'](999, 2, 300.0) == 300.0 + + +def test_seconds_is_none_when_nothing_is_known(): + ns = _chain_ns({1: ({}, None)}) + assert ns['seconds_for_range'](999, 1, None) is None + + +def test_a_failed_attempts_duration_is_persisted_with_its_verdict(): + # The only moment it is available: reconcile computes `seconds` solely on + # the success path, and the pod is about to be reaped. + fn = _extract(r"^(def record_outcome\(.*?)(?=\ndef )").group(1) + assert "data['attemptSeconds'] = _pod_seconds(pod)" in fn + + +def test_tx_apply_sums_the_chain_and_offers_fallbacks_to_the_last_leg_only(): + # pod_name names the winning attempt's pod; handing it to an earlier leg + # would read the wrong pod's log. + fn = _extract(r"^(def tx_apply_for_range\(.*?)(?=\ndef )").group(1) + assert '_resumed_chain(end, attempt)' in fn + assert 'pod_name if n == int(attempt) else None' in fn + assert 'total + leg' in fn, "legs are summed, not maxed" From d1b4adada457c7c066e1a0f11bf9eb5ffefcaf0f Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Wed, 29 Jul 2026 21:46:59 -0400 Subject: [PATCH 003/117] Drop the dead Prometheus test helper, pin the retry taxonomy _collector_fn and its header went stale when the collector stopped querying Prometheus: nothing calls it, and the comment described a sampling trade-off that no longer exists. Adds two tests for behaviour that was only implied. An unclassifiable Job failure (BackoffLimitExceeded carries no rule index and no exit code, so classify returns nothing) must land in ENVIRONMENTAL_OUTCOMES and get the disruption budget, alongside admission rejections -- both mean the cluster did this, not the ledger range. A genuine non-zero catchup exit stays the only outcome with no retry. Co-Authored-By: Claude Opus 5 --- .../test_job_monitor.py | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index 6a07a02a..835c225f 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -288,24 +288,6 @@ def test_untimestamped_kubelet_text_never_becomes_a_resume_point(): assert ts_re.match(good), good -# --- peak working set, for sizing a later run's requests -------------------- -# -# Queried from Prometheus rather than read from the worker's cgroup. Measured on -# ssc-test: cgroup memory.peak reported 1.5GB for a process holding 0.3MB of -# anon memory, because it counts page cache -- and catchup reads GBs of buckets. -# Sampling inside the worker was the other option and is worse: it means dropping -# the `exec`, which is what keeps stellar-core at PID 1 and able to see SIGTERM. - -def _collector_fn(*names): - """exec the named pure functions out of log_collector.py.""" - src = ["import json"] - for n in names: - m = re.search(rf"^(def {n}\(.*?)(?=^\S|\Z)", COLLECTOR_SRC, re.S | re.M) - assert m, f"{n} not found in log_collector.py" - src.append(m.group(1)) - ns = {} - exec("\n".join(src), ns) - return tuple(ns[n] for n in names) @@ -1602,3 +1584,28 @@ def test_tx_apply_sums_the_chain_and_offers_fallbacks_to_the_last_leg_only(): assert '_resumed_chain(end, attempt)' in fn assert 'pod_name if n == int(attempt) else None' in fn assert 'total + leg' in fn, "legs are summed, not maxed" + + +def test_an_unclassifiable_job_failure_is_retried_not_condemned(): + # BackoffLimitExceeded carries no rule index and no exit code, so classify() + # honestly returns nothing. That must not read as "this range is bad": a + # monitor restart while a node was reaped produces exactly this, and + # condemning on it would fail a 10-hour job on no evidence. + assert classify("Job has reached the specified backoff limit") == (None, None, None) + env = set(re.findall(r"'(\w+)'", _extract(r"ENVIRONMENTAL_OUTCOMES = \(([^)]+)\)").group(1))) + assert 'unknown' in env, "an unclassified failure must get the environmental budget" + assert {'disrupted', 'rejected'} <= env, "cluster-caused outcomes share that budget" + # ...and the environmental budget is the most generous of the three. + body = _extract(r"(if verdict\['outcome'\] == 'timeout':\s*\n\s*cap = .*?)(?=\n\s+if reason)").group(1) + assert 'ENVIRONMENTAL_OUTCOMES' in body and 'MAX_DISRUPTION_ATTEMPTS' in body + + +def test_only_a_genuine_catchup_failure_is_condemned(): + # `failed` is the one outcome with no retry reason. Everything else -- oom, + # ephemeral, timeout, and all three environmental outcomes -- sets one. + body = _extract(r"(if verdict\['outcome'\] == 'timeout':.*?reason = None[^\n]*)").group(1) + assert body.rstrip().endswith("reason = None # genuine catchup failure: do not retry"), \ + "a genuine catchup failure must be the only unretried outcome" + # every other branch in that chain sets a reason, i.e. retries + for outcome in ('rejected', 'disrupted', 'oom', 'ephemeral', 'unknown'): + assert f"== '{outcome}'" in body, f"{outcome} left the retry chain" From cfc69d421f138f0c49b0195d03f8a239b5ddeb82 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Wed, 29 Jul 2026 21:54:14 -0400 Subject: [PATCH 004/117] Remove dead cpu-sizing code; close mutation-sweep coverage gaps Audit of every test against the current code. Dead code removed - _sized_cpu was defined but never called once cpu stopped being profiled, and PROFILE_CPU_MARGIN only fed it. Chart value and env removed with it. - overrides.pop('cpu') could no longer hit: _profile_overrides stopped emitting cpu. - peakCpuCores stays in _PROFILE_ONLY_FIELDS deliberately. That is a strip list, not a produce list -- a progress record resumed from an older run still carries the field, and letting it through is what pushes the ConfigMap mirror toward the 1 MiB cap. A test caught this when it was removed. Coverage gaps A sweep of 14 semantically meaningful mutations found 8 that broke no test at all. Every one guarded a decision the design depends on: backoffLimit 0 the monitor, not the Job controller, owns retries restartPolicy Never in-place restart would loop at the limit that OOMed and never advance the attempt counter ttlSecondsAfterFinished backstop for Jobs reconcile did not reach MEM_BUMP_FACTOR 1.0 would retry an OOM at the identical limit attempt budgets ordering encodes whose fault a failure was atomic writes a half-written .outcome downgrades a classified failure to "unknown" sinceTime resume without it every reconnect re-reads the whole log overlap dedup the resume overlap is deliberate and must be removed per line All 8 now fail under mutation. 136 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 11 +- .../templates/job_monitor.yaml | 2 - .../parallel_catchup_helm/values.yaml | 2 - .../test_job_monitor.py | 110 +++++++++++++++++- 4 files changed, 111 insertions(+), 14 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 184fbe05..6bef3be0 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -107,7 +107,6 @@ PROFILE_CPU_LIMIT = os.getenv('PROFILE_CPU_LIMIT', '') # No safety margin on cpu, unlike memory. Under-requesting cpu costs contention # and the pod can still burst; under-requesting memory gets it OOMKilled. -PROFILE_CPU_MARGIN = float(os.getenv('PROFILE_CPU_MARGIN', 1.0)) # Ceiling for profile-derived memory, above the unprofiled limit for the same # reason: a range that really needs more than the configured limit must be able # to ask for it rather than be pinned under its own measured peak. The OOM @@ -487,6 +486,9 @@ def save_status(snapshot): # profiling fields alone push it toward the 1 MiB cap at ~6100 ranges. Stripped # to attempts/count it is ~30 bytes, so state stays readable at any slicing # while the profile has no ceiling at all. +# A strip list, not a produce list: peakCpuCores is no longer measured, but a +# progress record resumed from an older run still carries it, and letting it +# through is what pushes the ConfigMap mirror toward the 1 MiB cap. _PROFILE_ONLY_FIELDS = ('peakAnonBytes', 'peakRssBytes', 'peakWorkingSetBytes', 'peakCpuCores', 'peakEphemeralBytes', 'txApply', 'seconds', 'wallSeconds') @@ -1185,10 +1187,6 @@ def _cpu_millis(q): return int(float(q[:-1])) if str(q).endswith('m') else int(float(q) * 1000) -def _sized_cpu(cores, margin, cap): - """A measured core count turned into a request, never above the limit.""" - return f"{min(int(cores * 1000 * margin), _cpu_millis(cap))}m" - def _sized(value, margin, cap): """A measured peak turned into a request: margin applied, never above cap.""" @@ -1256,9 +1254,6 @@ def _resources(mem=None, eph=None, end=None): # packs by what it actually uses while keeping headroom to burst. That # leaves the pod Burstable rather than Guaranteed -- Kubernetes needs # all three to match -- which is the intended trade. - cpu = overrides.pop('cpu', None) - if cpu: - req['cpu'] = cpu if PROFILE_CPU_LIMIT: lim['cpu'] = PROFILE_CPU_LIMIT else: diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 59c9b54c..cbb91383 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -234,8 +234,6 @@ spec: value: {{ .Values.monitor.profileMargin | quote }} - name: PROFILE_CPU_LIMIT value: {{ .Values.monitor.profileCpuLimit | quote }} - - name: PROFILE_CPU_MARGIN - value: {{ .Values.monitor.profileCpuMargin | quote }} - name: PROFILE_MAX_MEM value: {{ .Values.monitor.profileMaxMemory | quote }} - name: PROFILE_CACHE_HEADROOM diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index f23f90e9..f8a1c2d1 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -79,8 +79,6 @@ monitor: # otherwise-free node (168s/111s/99s at limit 2/4/none). Set a value to cap. profileCpuLimit: "" # No margin on cpu: it is compressible, so under-requesting costs contention - # rather than an OOM kill. Memory keeps profileMargin. - profileCpuMargin: 1.0 # Ceiling for profile-derived memory. Above the configured worker limit on # purpose: a range needing more than that must be able to ask for it # rather than be pinned under its own measured peak. diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index 835c225f..a9e2cce0 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -400,7 +400,7 @@ def _profile_ns(ranges, mode='ephemeral', margin=1.1): """profile_for + _sized, exec'd out of job_monitor with a fixed profile.""" ns = {'bisect': __import__('bisect'), 'logger': __import__('logging').getLogger('t')} for name in ('_quantity_bytes', '_bytes_to_quantity', 'profile_for', - '_cpu_millis', '_sized_cpu', '_sized'): + '_cpu_millis', '_sized'): m = re.search(rf"^(def {name}\(.*?)(?=^\S|\Z)", SRC, re.S | re.M) exec(m.group(1), ns) ns['_UNITS'] = eval(_extract(r"_UNITS = (\{.*?\})").group(1)) @@ -468,7 +468,6 @@ def _overrides_ns(ranges, lim_mem='24000Mi', lim_eph='40Gi', margin=1.1, ns['LIM_CPU'] = '2' ns['REQ_CPU'] = '1800m' ns['PROFILE_CPU_LIMIT'] = '' - ns['PROFILE_CPU_MARGIN'] = 1.0 ns['PROFILE_MAX_MEM'] = max_mem ns['PROFILE_CACHE_HEADROOM'] = '512Mi' m = re.search(r"^(def _profile_overrides\(.*?)(?=^\S|\Z)", SRC, re.S | re.M) @@ -1609,3 +1608,110 @@ def test_only_a_genuine_catchup_failure_is_condemned(): # every other branch in that chain sets a reason, i.e. retries for outcome in ('rejected', 'disrupted', 'oom', 'ephemeral', 'unknown'): assert f"== '{outcome}'" in body, f"{outcome} left the retry chain" + + +# --- gaps found by a mutation sweep, 2026-07-30 ---------------------------- +# Each of these guards a decision the design depends on, and each was mutable +# without breaking a single test before this block existed. + +def test_the_job_controller_never_owns_retries(): + # backoffLimit 0 is load-bearing: above 0 the Job controller replaces the pod + # on its own schedule, so we could not classify disruption vs catchup + # failure, could not count evictions, and could not guarantee the log was + # archived before the next attempt started. + spec = _extract(r"spec=client\.V1JobSpec\((.*?)template=").group(1) + assert re.search(r"backoff_limit\s*=\s*0\b", spec), "backoffLimit must be 0" + assert re.search(r"ttl_seconds_after_finished\s*=\s*JOB_TTL_SECONDS", spec), \ + "finished Jobs need a TTL backstop even though reconcile deletes them" + + +def test_a_worker_pod_is_never_restarted_in_place(): + # restartPolicy OnFailure restarts the container inside the same pod, which + # keeps the pod name and reuses the same resource limits -- so an OOM would + # loop forever at the limit that killed it instead of escalating, and the + # attempt counter would never advance. + spec = _extract(r"spec=client\.V1PodSpec\((.*?)containers=\[container\]").group(1) + assert "restart_policy='Never'" in spec + + +@pytest.mark.parametrize('attempt,want', [(1, 1.0), (2, 1.5), (3, 2.25), (4, 3.375)]) +def test_the_memory_escalation_ladder_compounds(attempt, want): + # 1.5x per attempt off what the attempt actually ran with. A factor of 1.0 + # would retry an OOM at the identical limit, forever. + ns = {'os': __import__('os'), 're': re} + for name in ('_quantity_bytes', '_bytes_to_quantity', 'mem_for_attempt'): + exec(_extract(r"^(def " + name + r"\(.*?)(?=\ndef )").group(1), ns) + ns['_UNITS'] = {'Ki': 1024, 'Mi': 1024**2, 'Gi': 1024**3, 'Ti': 1024**4, + 'K': 1000, 'M': 1000**2, 'G': 1000**3, 'T': 1000**4} + ns['MEM_BUMP_FACTOR'] = float(_extract( + r"MEM_BUMP_FACTOR = float\(os\.getenv\('MEM_BUMP_FACTOR', ([\d.]+)\)\)").group(1)) + ns['MEM_ESCALATION_CAP'] = '48Gi' + ns['LIM_MEM'] = '1000Mi' + got = ns['mem_for_attempt'](attempt, '1000Mi') + assert got == f"{int(1000 * want)}Mi", got + + +def test_the_escalation_ladder_is_capped(): + ns = {'os': __import__('os'), 're': re} + for name in ('_quantity_bytes', '_bytes_to_quantity', 'mem_for_attempt'): + exec(_extract(r"^(def " + name + r"\(.*?)(?=\ndef )").group(1), ns) + ns['_UNITS'] = {'Ki': 1024, 'Mi': 1024**2, 'Gi': 1024**3, 'Ti': 1024**4, + 'K': 1000, 'M': 1000**2, 'G': 1000**3, 'T': 1000**4} + ns['MEM_BUMP_FACTOR'] = 1.5 + ns['MEM_ESCALATION_CAP'] = '4Gi' + ns['LIM_MEM'] = '1000Mi' + assert ns['mem_for_attempt'](20, '1000Mi') == '4096Mi', "cap not applied" + + +def test_progress_is_written_atomically(): + # The mission reads progress.json off the volume while the monitor is still + # writing it. A partial file is unparseable JSON, which reads as "no + # progress" -- and reconcile halts the run when progress goes backwards. + fn = _extract(r"^(def save_progress\(.*?)(?=\ndef )").group(1) + assert '.tmp' in fn and 'os.replace(' in fn, "progress.json is not written atomically" + assert fn.index('.tmp') < fn.index('os.replace('), "replace must follow the temp write" + + +def test_the_log_stream_resumes_from_the_last_durable_timestamp(): + # Without sinceTime a reconnect re-reads the whole log from the start: one + # full re-read per pod per reconnect, at 2096 pods. + fn = _extract(r"^(async def stream_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert "params['sinceTime']" in fn, "reconnect does not resume" + # ...and the second-granularity overlap it creates is removed per line. + assert re.search(r"if last_ts and ts <= last_ts:\s*\n\s*continue", fn), \ + "the deliberate resume overlap is never deduped" + + +def test_every_durable_write_is_atomic(): + # Three writers put files on the shared volume while the mission and the + # collector read them. A half-written .outcome or archive is unparseable, + # and an unreadable outcome downgrades a classified failure to "unknown". + for fn_name in ('save_progress', 'backstop_save_pod_log', 'record_outcome', + 'write_metrics'): + for src in (SRC, COLLECTOR_SRC): + m = re.search(r"^(def " + fn_name + r"\(.*?)(?=\ndef )", src, re.S | re.M) + if m: + break + assert m, f"{fn_name} not found" + body = m.group(1) + assert '.tmp' in body, f"{fn_name} does not write via a temp file" + assert 'os.replace(' in body, f"{fn_name} does not rename atomically" + assert body.index('.tmp') < body.index('os.replace('), \ + f"{fn_name} renames before it writes" + + +def test_attempt_budgets_are_ordered_by_whose_fault_the_failure_was(): + # A hang is usually persistent, so it gets the fewest tries. A genuinely + # broken range gets the middle budget. Anything the cluster did to us gets + # the most -- on spot, evictions are routine and must not condemn a range. + def const(name, env): + return int(_extract(name + r" = int\(os\.getenv\('" + env + r"', (\d+)\)\)").group(1)) + timeout = const('MAX_TIMEOUT_ATTEMPTS', 'MAX_TIMEOUT_ATTEMPTS') + per_range = const('MAX_ATTEMPTS_PER_RANGE', 'MAX_ATTEMPTS') + disruption = const('MAX_DISRUPTION_ATTEMPTS', 'MAX_DISRUPTION_ATTEMPTS') + ephemeral = const('MAX_EPHEMERAL_ATTEMPTS', 'MAX_EPHEMERAL_ATTEMPTS') + assert timeout < per_range < disruption, \ + f"budgets out of order: timeout={timeout} range={per_range} disruption={disruption}" + assert per_range > 1, "a range that OOMs once could never escalate" + assert ephemeral > 1, "a range evicted on disk once could never grow" + assert disruption >= 10, "spot eviction would condemn ranges at this budget" From 4ecc93e976243f6f3fc46a757644d9470f6ce7fd Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Wed, 29 Jul 2026 22:00:57 -0400 Subject: [PATCH 005/117] Record attempt duration in the collector, as a fallback for reaped pods Found by the live 2096-worker spot run. record_outcome writes attemptSeconds from the pod's terminated timestamps, but only when the monitor still has the pod -- and a spot eviction reaps the node first. Measured on ssc-test: 212 of 212 disruptions were classified from the Job condition with no pod, so no .outcome and no duration. Peaks were unaffected (the collector writes .metrics regardless), but the resumed-chain time total silently dropped every evicted leg -- the exact path the chain sum was added for. The collector watched the container run, so it is the only observer left. It now stamps the stream lifetime into .metrics, and seconds_for_range prefers .outcome and falls back to it. The collector figure is an approximation: the stream opens up to COLLECTOR_POLL_SECONDS after the container does. 139 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 10 ++++ src/MissionParallelCatchup/log_collector.py | 21 ++++++-- .../test_job_monitor.py | 49 ++++++++++++++++++- 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 6bef3be0..20eb13e6 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -949,7 +949,17 @@ def seconds_for_range(end, attempt=1, final=None): if n == int(attempt): leg = final else: + # .outcome is authoritative -- the pod's own terminated timestamps. + # It is absent whenever the pod was reaped before the monitor could + # classify it, which is every spot eviction, so fall back to the + # collector's stream-lifetime figure rather than losing the leg. leg = (read_outcome(end, n) or {}).get('attemptSeconds') + if leg is None: + try: + with open(metrics_path(end, n)) as fh: + leg = json.load(fh).get('attemptSeconds') + except (OSError, ValueError): + leg = None if leg is not None: total = leg if total is None else total + leg return total diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 9ad0b537..182f33c3 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -357,7 +357,7 @@ async def sample_kubelet(session, nodes): write_metrics(ref[0], ref[1], {'peakAnonBytes': int(rss)}) -async def finalize(session, pod, end, attempt, tx, done_ok): +async def finalize(session, pod, end, attempt, tx, done_ok, started=None): """Persist everything this attempt owes, then let its stream go. Reached from two places: a clean end of stream once the pod is terminal, @@ -368,6 +368,11 @@ async def finalize(session, pod, end, attempt, tx, done_ok): """ # Before discard: on success the archive is about to be deleted. measured = {} + if started is not None: + # Fallback only: the monitor's figure comes from the pod's terminated + # timestamps and is preferred when it exists. + measured['attemptSeconds'] = round( + asyncio.get_event_loop().time() - started, 1) if tx.resumed: # Not a peak -- PEAK_FIELDS filters it out of the profile. peaks_for_range # reads it to decide how far back to aggregate: a resumed attempt only @@ -418,6 +423,14 @@ async def stream_pod(session, pod, end, attempt, done, done_ok): # the same log. write_state(end, attempt, '') backoff = 1.0 + # Wall clock for this attempt. The monitor records attemptSeconds from the + # pod's own terminated timestamps, but only when it still has the pod -- and + # a spot eviction reaps the node first, so 212 of 212 disruptions on + # ssc-test were classified from the Job condition with no pod and no + # duration. This process watched the container run, so it is the only + # observer left. Approximate: the stream opens up to COLLECTOR_POLL_SECONDS + # after the container did. + started = asyncio.get_event_loop().time() # Outside the reconnect loop: the medida block could straddle a dropped # stream, and a fresh scanner per attempt would lose the half it saw. tx = TxApplyScanner() @@ -440,7 +453,7 @@ async def stream_pod(session, pod, end, attempt, done, done_ok): # are in Prometheus regardless. A bare return here dropped # both for every pod that outlived its object. logger.info("pod %s gone before/while streaming range %s", pod, end) - await finalize(session, pod, end, attempt, tx, done_ok) + await finalize(session, pod, end, attempt, tx, done_ok, started) return resp.raise_for_status() backoff = 1.0 @@ -475,7 +488,7 @@ async def stream_pod(session, pod, end, attempt, done, done_ok): last_ts = pending # A clean end of stream means the container exited. if done(pod): - await finalize(session, pod, end, attempt, tx, done_ok) + await finalize(session, pod, end, attempt, tx, done_ok, started) return except asyncio.CancelledError: raise @@ -488,7 +501,7 @@ async def stream_pod(session, pod, end, attempt, done, done_ok): # terminal. The partial stream may already hold the medida block, # and the peaks are query-side, so this owes exactly what the clean # path owes. It used to return bare and lose both. - await finalize(session, pod, end, attempt, tx, done_ok) + await finalize(session, pod, end, attempt, tx, done_ok, started) return await asyncio.sleep(backoff) backoff = min(backoff * 2, 30) diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index a9e2cce0..26463ee9 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -1148,7 +1148,7 @@ def test_both_exit_paths_share_one_finalize(): # writing peakAnonBytes while the other keeps working. # Three: clean exit, pod-gone 404, and an interrupted read on a pod that # has since gone terminal. - assert len(re.findall(r"await finalize\(session, pod, end, attempt, tx, done_ok\)", + assert len(re.findall(r"await finalize\(session, pod, end, attempt, tx, done_ok, started\)", COLLECTOR_SRC)) == 3 assert len(re.findall(r"write_metrics\(end, attempt, measured\)", COLLECTOR_SRC)) == 1 @@ -1183,7 +1183,7 @@ async def it(): class FakeSession: def get(self, url, params=None, headers=None): return FakeResp() - async def fake_finalize(session, pod, end, attempt, tx, done_ok): + async def fake_finalize(session, pod, end, attempt, tx, done_ok, started=None): calls.append((pod, end, attempt)) d = tempfile.mkdtemp() @@ -1715,3 +1715,48 @@ def const(name, env): assert per_range > 1, "a range that OOMs once could never escalate" assert ephemeral > 1, "a range evicted on disk once could never grow" assert disruption >= 10, "spot eviction would condemn ranges at this budget" + + +def test_the_collector_records_a_duration_the_monitor_cannot(): + # Measured on ssc-test 2026-07-30: 212 of 212 spot disruptions were + # classified from the Job condition with the pod already reaped, so + # record_outcome never ran and no .outcome carried attemptSeconds. Peaks + # survived (the collector writes .metrics regardless) but the chain's time + # total silently lost every evicted leg. This process watched the container + # run, so it is the only observer left. + fn = _extract(r"^(async def finalize\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert "measured['attemptSeconds']" in fn + import asyncio + written = [] + ns = {'asyncio': asyncio, '_anon_peak': {}, '_ws_peak': {}, '_eph_peak': {}, + '_peak_flushed': {}, '_streaming': {}, 'SAVE_SUCCESS_LOGS': True, + 'write_metrics': lambda e, a, v: written.append(v), + 'discard': lambda e, a: None, + 'logger': type('L', (), {'info': lambda s, *a: None})()} + exec(fn, ns) + tx = type('T', (), {'seconds': None, 'resumed': False})() + async def go(): + now = asyncio.get_event_loop().time() + await ns['finalize'](None, 'p', '999', '1', tx, lambda p: True, now - 42.0) + asyncio.run(go()) + assert written and written[0]['attemptSeconds'] == pytest.approx(42.0, abs=1.0) + + +def test_seconds_falls_back_to_the_collectors_figure(): + # The authoritative .outcome is missing for every reaped pod. Without this + # fallback the chain drops that leg entirely and under-reports the range. + ns = _chain_ns({ + 1: ({'attemptSeconds': 850.0}, None), # no .outcome at all + 2: ({'resumed': True}, None), + }) + assert ns['seconds_for_range'](999, 2, 300.0) == 1150.0 + + +def test_the_authoritative_outcome_wins_over_the_collector_estimate(): + # .outcome comes from the pod's terminated timestamps; the collector's is a + # stream-lifetime approximation that starts up to one poll late. + ns = _chain_ns({ + 1: ({'attemptSeconds': 850.0}, {'outcome': 'disrupted', 'attemptSeconds': 900.0}), + 2: ({'resumed': True}, None), + }) + assert ns['seconds_for_range'](999, 2, 300.0) == 1200.0 From 43547e8c9ac3658e9ff2d2402190f71c0223142a Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Wed, 29 Jul 2026 22:08:33 -0400 Subject: [PATCH 006/117] Stop a worker's own output from killing its log stream Found live on the 2096-worker spot run. The AWS CLI draws its transfer meter with carriage returns and no newline, so a 628 MiB bucket download reaches the log endpoint as one multi-megabyte "line". The collector read the stream line-wise and aiohttp raises above 512 KiB, so every large download killed its own stream; the reconnect resumed from sinceTime and hit the same wall. The resulting spin consumed the collector, and no retry pod ever got a stream -- 289 a2 pods, zero a2 archives or metrics, which is exactly the resumed-attempt data this run existed to capture. Two fixes: - aws s3 cp gains --no-progress. The meter is noise in an archive and was the bulk of every large range's log. - The collector reads 64 KiB chunks and splits on \r as well as \n, with MAX_LINE_CHARS bounding any single unterminated blob. A stream must not be destroyable by whatever a worker happens to print. 143 tests. Co-Authored-By: Claude Opus 5 --- .../MissionHistoryPubnetParallelCatchupV2.fs | 10 ++- src/MissionParallelCatchup/log_collector.py | 61 ++++++++++++------- .../templates/job_monitor.yaml | 2 + .../parallel_catchup_helm/values.yaml | 4 ++ .../test_job_monitor.py | 44 +++++++++++++ 5 files changed, 99 insertions(+), 22 deletions(-) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 333ac9c5..6fde4ca6 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -246,7 +246,15 @@ let installProject (context: MissionContext) = if index < 1 || index > 3 then failwith "s3HistoryGetCommand: index must be between 1 and 3 inclusive" - let s3GetCommandBase = sprintf "aws s3 cp --region %s" context.s3HistoryMirrorRegionPcV2 + // --no-progress is load-bearing, not cosmetic. The AWS CLI draws its + // transfer meter with carriage returns and no newline, so a 628 MiB + // bucket download arrives as one multi-megabyte "line". The log + // collector reads the pod stream line-wise and aiohttp aborts any line + // over 512 KiB, so every large download killed its own stream, which + // then reconnected and hit the same wall. Measured on ssc-test + // 2026-07-30: it starved every retry pod of a collector stream. + let s3GetCommandBase = + sprintf "aws s3 cp --no-progress --region %s" context.s3HistoryMirrorRegionPcV2 let command = sprintf "%s s3://%s/core_live_00%d/{0} {1}" s3GetCommandBase url index setOptions.Add(sprintf "worker.historyGetCommandCore00%d=\"%s\"" index command) diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 182f33c3..0b455a9d 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -67,6 +67,10 @@ # at most PEAK_FLUSH_RATIO of a range's high-water rather than all of it -- # Prometheus's server-side max_over_time needed no such state. PEAK_FLUSH_RATIO = float(os.getenv('PEAK_FLUSH_RATIO', 1.05)) +# Most a single unterminated blob may buffer before we start discarding its +# head. stellar-core's own lines are well under a kilobyte; anything larger is a +# progress meter or a stack dump, and neither is worth killing the stream over. +MAX_LINE_CHARS = int(os.getenv('MAX_LINE_CHARS', 262144)) LABEL_RUN = 'catchup.stellar.org/run' LABEL_RANGE = 'catchup.stellar.org/range-end' @@ -462,27 +466,42 @@ async def stream_pod(session, pod, end, attempt, done, done_ok): # valid archive, so restarts do not corrupt what is already there. with gzip.open(path, 'at') as fh: since_flush = asyncio.get_event_loop().time() - async for raw in resp.content: - line = raw.decode('utf-8', 'replace').rstrip('\n') - if not line: - continue - ts, _, rest = line.partition(' ') - if not _TS_RE.match(ts): - # Untimestamped kubelet text. Keep it, but never let - # it become the resume point. - fh.write(line + '\n') - continue - if last_ts and ts <= last_ts: - continue # exact dedup of the resume overlap - fh.write(rest + '\n') - tx.feed(rest) - pending = ts - now = asyncio.get_event_loop().time() - if now - since_flush >= STATE_FLUSH_SECONDS: - fh.flush() - write_state(end, attempt, pending) - last_ts = pending - since_flush = now + # Chunked, not line-wise. `async for raw in resp.content` + # yields lines and aiohttp raises over 512 KiB, which any + # carriage-return progress meter in the worker's output + # trivially exceeds -- one 628 MiB download is a single + # "line". The worker now passes --no-progress, but a stream + # must not be destroyable by whatever a worker happens to + # print, so split on \r as well and cap what we buffer. + pending_buf = '' + async for chunk in resp.content.iter_chunked(65536): + pending_buf += chunk.decode('utf-8', 'replace') + if len(pending_buf) > MAX_LINE_CHARS: + # A single unterminated blob. Keep the tail so the + # real line ending is still found, drop the rest. + pending_buf = pending_buf[-MAX_LINE_CHARS:] + parts = re.split(r'[\r\n]', pending_buf) + pending_buf = parts.pop() + for line in parts: + if not line: + continue + ts, _, rest = line.partition(' ') + if not _TS_RE.match(ts): + # Untimestamped kubelet text. Keep it, but never let + # it become the resume point. + fh.write(line + '\n') + continue + if last_ts and ts <= last_ts: + continue # exact dedup of the resume overlap + fh.write(rest + '\n') + tx.feed(rest) + pending = ts + now = asyncio.get_event_loop().time() + if now - since_flush >= STATE_FLUSH_SECONDS: + fh.flush() + write_state(end, attempt, pending) + last_ts = pending + since_flush = now if pending: write_state(end, attempt, pending) last_ts = pending diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index cbb91383..24acc0c3 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -329,6 +329,8 @@ spec: value: /logs - name: COLLECTOR_POLL_SECONDS value: {{ .Values.monitor.collectorPollSeconds | quote }} + - name: MAX_LINE_CHARS + value: {{ .Values.monitor.maxLineChars | quote }} - name: PEAK_FLUSH_RATIO value: {{ .Values.monitor.peakFlushRatio | quote }} - name: COLLECTOR_VANISHED_GRACE_CYCLES diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index f8a1c2d1..a92c83b7 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -134,6 +134,10 @@ monitor: logStorageSize: "100Gi" saveSuccessLogs: true collectorPollSeconds: 5 + # Most a single unterminated blob may buffer before its head is discarded. + # A worker printing a carriage-return progress meter would otherwise grow this + # without bound, on every stream at once. + maxLineChars: 262144 # Growth factor before an in-flight peak is flushed to its .metrics file, so a # collector restart cannot silently reset a range's high-water to zero. peakFlushRatio: 1.05 diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index 26463ee9..5f02efc9 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -1760,3 +1760,47 @@ def test_the_authoritative_outcome_wins_over_the_collector_estimate(): 2: ({'resumed': True}, None), }) assert ns['seconds_for_range'](999, 2, 300.0) == 1200.0 + + +# --- a worker must not be able to kill its own log stream ------------------ +# Found live on the 2096-worker spot run: the AWS CLI draws its transfer meter +# with carriage returns and no newline, so a 628 MiB bucket download arrives as +# one multi-megabyte "line". aiohttp raises over 512 KiB, every large download +# killed its own stream, and the reconnect hit the same wall -- which starved +# every retry pod of a collector stream and left a2 metrics empty. + +def test_the_stream_is_read_in_chunks_not_lines(): + fn = _extract(r"^(async def stream_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert 'iter_chunked' in fn, "line-wise reads are bounded by aiohttp's 512KiB limit" + assert 'async for raw in resp.content:' not in fn + + +def test_carriage_returns_split_lines_too(): + # The progress meter is \r-delimited. Without \r in the split it stays one + # blob no matter how the bytes arrive. + fn = _extract(r"^(async def stream_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + m = re.search(r"re\.split\(r'\[([^\]]+)\]'", fn) + assert m, "no line splitting found" + assert '\\r' in m.group(1) and '\\n' in m.group(1), f"splits on {m.group(1)!r}" + + +def test_an_unterminated_blob_is_capped_not_buffered_forever(): + # A meter that never emits a newline would otherwise grow the buffer until + # the collector OOMs -- 2096 streams doing it at once. + fn = _extract(r"^(async def stream_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert 'MAX_LINE_CHARS' in fn + assert re.search(r"pending_buf\[-MAX_LINE_CHARS:\]", fn), "buffer is never trimmed" + cap = int(_extract(r"MAX_LINE_CHARS = int\(os\.getenv\('MAX_LINE_CHARS', (\d+)\)\)", + COLLECTOR_SRC).group(1)) + assert 1024 < cap < 524288, f"cap {cap} is outside a sane range" + + +def test_the_worker_disables_the_aws_progress_meter(): + # The real cure: never emit the \r spam. Also keeps it out of the archives, + # where it was the bulk of every large range's log. + fs = open(__file__.replace( + 'src/MissionParallelCatchup/test_job_monitor.py', + 'src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs')).read() + m = re.search(r'sprintf "aws s3 cp ([^"]*)--region %s"', fs) + assert m, "s3 GET command not found" + assert '--no-progress' in m.group(1), f"aws s3 cp flags: {m.group(1)!r}" From 08980ca195b720308b10b39fcb8f22b10435f4c2 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Wed, 29 Jul 2026 22:15:56 -0400 Subject: [PATCH 007/117] Gate the retry-path reap; cut the per-stream line buffer Measured on the live 2096-worker run: the collector sits at 1444 MiB of a 2048 MiB limit with memory.events max=2617 and 1.00 of 2 cpu, holding 1797 established connections. follow=true charges a gzip deflate buffer and aiohttp read buffers per stream for the pod's whole life, so that cost scales with parallelism. Linear extrapolation to 4096 streams is ~2.7 GiB and ~2.0 cpu -- it runs out of both at once. - MAX_LINE_CHARS 256 KiB -> 64 KiB. It is charged per live stream, so the old value was another 512 MiB at 2096 and 1 GiB at 4096, enough to OOM the sidecar by itself. stellar-core lines are well under a kilobyte. - The retry-path job deletion is now gated on the collector having finalized that attempt, matching the success path. delete_job reaps the pod and backstop_save_pod_log stands down for any range the collector claimed, so nothing else would ever read that log. Benign under follow=true, required before any move to polling. 145 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 10 ++++++- src/MissionParallelCatchup/log_collector.py | 8 +++++- .../parallel_catchup_helm/values.yaml | 2 +- .../test_job_monitor.py | 26 +++++++++++++++++++ 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 20eb13e6..679ce786 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1622,7 +1622,15 @@ def reconcile(state): # at attempt 1 -- losing the escalated memory that is the whole # point of the retry. live[] keys on the highest attempt, so the # two coexisting for one pass is already handled. - delete_job(end, attempt) + # + # Gated like the success path: deleting the Job reaps the pod, + # and backstop_save_pod_log stands down for any range the + # collector has claimed, so there is no second reader. Waiting + # for .metrics means the collector has finalized this attempt -- + # its peaks, its tx_apply and its duration are all durable. + # JOB_TTL_SECONDS reaps it if the collector never gets there. + if peaks_for_range(end, attempt): + delete_job(end, attempt) in_progress.append(job_key(int(end), by_end[end])) continue if reason is not None: diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 0b455a9d..702f13ad 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -70,7 +70,13 @@ # Most a single unterminated blob may buffer before we start discarding its # head. stellar-core's own lines are well under a kilobyte; anything larger is a # progress meter or a stack dump, and neither is worth killing the stream over. -MAX_LINE_CHARS = int(os.getenv('MAX_LINE_CHARS', 262144)) +# +# This is charged PER LIVE STREAM. Measured on ssc-test at 2096 follow streams +# the collector already sat at 1444 MiB of a 2048 MiB limit with memory.events +# max=2617, so a 256 KiB worst case here is another 512 MiB at 2096 and 1 GiB at +# 4096 -- on its own enough to OOM the sidecar. 64 KiB is ~64x the longest line +# stellar-core actually emits. +MAX_LINE_CHARS = int(os.getenv('MAX_LINE_CHARS', 65536)) LABEL_RUN = 'catchup.stellar.org/run' LABEL_RANGE = 'catchup.stellar.org/range-end' diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index a92c83b7..31d9ddae 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -137,7 +137,7 @@ monitor: # Most a single unterminated blob may buffer before its head is discarded. # A worker printing a carriage-return progress meter would otherwise grow this # without bound, on every stream at once. - maxLineChars: 262144 + maxLineChars: 65536 # Growth factor before an in-flight peak is flushed to its .metrics file, so a # collector restart cannot silently reset a range's high-water to zero. peakFlushRatio: 1.05 diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index 5f02efc9..c8b35925 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -1804,3 +1804,29 @@ def test_the_worker_disables_the_aws_progress_meter(): m = re.search(r'sprintf "aws s3 cp ([^"]*)--region %s"', fs) assert m, "s3 GET command not found" assert '--no-progress' in m.group(1), f"aws s3 cp flags: {m.group(1)!r}" + + +def test_a_failed_attempt_is_not_reaped_before_the_collector_finalizes_it(): + # delete_job reaps the pod, and backstop_save_pod_log stands down for any + # range the collector claimed -- so nothing else would ever read that log. + # Under follow=true the collector already holds everything; under polling it + # would lose the last interval. Gate it either way. + body = _extract(r"(try:\s*\n\s*batch_v1\.create_namespaced_job.*?)continue").group(1) + assert 'delete_job(end, attempt)' in body + assert re.search(r"if peaks_for_range\(end, attempt\):\s*\n\s*delete_job\(end, attempt\)", body), \ + "the retry-path reap is not gated on the collector having finalized" + assert body.index('create_namespaced_job') < body.index('delete_job('), \ + "successor must exist before the predecessor is reaped" + + +def test_the_line_buffer_cap_is_charged_per_stream(): + # Measured on ssc-test at 2096 follow streams: 1444 MiB of a 2048 MiB limit, + # memory.events max=2617. The cap is worst-case memory per live stream, so + # 256 KiB would add 1 GiB at 4096 streams and OOM the sidecar on its own. + cap = int(_extract(r"MAX_LINE_CHARS = int\(os\.getenv\('MAX_LINE_CHARS', (\d+)\)\)", + COLLECTOR_SRC).group(1)) + assert cap <= 65536, f"{cap} bytes x 4096 streams = {cap * 4096 // 2**20} MiB worst case" + assert cap >= 8192, "below this a legitimate long line would be truncated" + chart = open(__file__.replace( + 'test_job_monitor.py', 'parallel_catchup_helm/values.yaml')).read() + assert int(_extract(r"maxLineChars: (\d+)", chart).group(1)) == cap From c0bd8914509284f1eb5ad6402f382285b6802496 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Wed, 29 Jul 2026 22:27:20 -0400 Subject: [PATCH 008/117] Collect logs by polling instead of one follow stream per pod Measured on the live 2096-worker run: the collector sat at 1444 MiB of a 2048 MiB limit with memory.events max=2617, 1.00 of 2 cpu, and 1797 held connections. follow=true charges a connection, a gzip deflate buffer and aiohttp read buffers per pod for the pod's whole life, so the cost scales with parallelism -- 4096 streams extrapolates past both limits at once. Polling makes concurrency a tuning parameter instead of a function of pod count. A single poll measured ~0.22s, so 2096 pods on a 10s interval need ~46 in-flight polls against 2096 permanently-held connections. The starvation class of bug goes with it: no pool size can be "too small" when slots are time-shared rather than held for hours. - _poll_once does one short read under a semaphore and opens the archive per poll, so nothing is retained between polls. - poll_pod samples terminal BEFORE each poll, so a pod that exits mid-poll still has its final output read; checking after would race it. - A terminal pod whose polls keep failing finalizes after TERMINAL_POLL_ATTEMPTS rather than spinning on a dead pod forever. follow=true finalized there because it already held the bytes; the suite caught this when polling did not. - COLLECTOR_MAX_STREAMS is gone: nothing holds a stream, and deriving it from worker.replicas was what starved retries twice. Not yet exercised against a cluster. 152 tests, each new behaviour mutation-checked. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/log_collector.py | 224 ++++++++++-------- .../templates/job_monitor.yaml | 12 +- .../parallel_catchup_helm/values.yaml | 14 +- .../test_job_monitor.py | 116 +++++++-- 4 files changed, 242 insertions(+), 124 deletions(-) diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 702f13ad..1c89e372 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -42,7 +42,6 @@ CONTAINER = os.getenv('WORKER_CONTAINER', 'stellar-core') POLL_SECONDS = float(os.getenv('COLLECTOR_POLL_SECONDS', 5)) STATE_FLUSH_SECONDS = float(os.getenv('STATE_FLUSH_SECONDS', 10)) -MAX_CONCURRENT = int(os.getenv('COLLECTOR_MAX_STREAMS', 1200)) # Poll cycles a stream gets to finalize itself after its pod leaves the pod list # before it is cancelled outright. One cycle is usually enough; the margin is for # a stream still finalizing: writing its .metrics and closing its archive. @@ -77,6 +76,25 @@ # 4096 -- on its own enough to OOM the sidecar. 64 KiB is ~64x the longest line # stellar-core actually emits. MAX_LINE_CHARS = int(os.getenv('MAX_LINE_CHARS', 65536)) +# Seconds between polls of one pod's log. Latency here is archive lag, not +# anything a decision waits on; 4096 pods at 10s is ~90 concurrent polls. +LOG_POLL_SECONDS = float(os.getenv('LOG_POLL_SECONDS', 10)) +# Concurrent in-flight log polls, across all pods. This is the whole point of +# polling: it is independent of how many pods exist, where follow=true needed +# one held connection per pod forever. +MAX_CONCURRENT_POLLS = int(os.getenv('MAX_CONCURRENT_POLLS', 96)) +# Most one poll may read before it stops. A pod that has been unwatched for a +# while has a large backlog; this bounds a single response, and the next poll +# picks up from the timestamp this one reached. +MAX_POLL_CHARS = int(os.getenv('MAX_POLL_CHARS', 8388608)) +# Bounds in-flight polls across every pod. Lives beside its own constant rather +# than among the peak dicts, where it landed inside the region the scanner tests +# exec and broke six of them on an asyncio NameError. +_poll_slots = asyncio.Semaphore(MAX_CONCURRENT_POLLS) +# Failed polls tolerated after a pod goes terminal before we stop asking. Its +# log is not coming back, and spinning on it holds a task and a poll slot for +# the rest of the run; a couple of retries still absorb a transient 500. +TERMINAL_POLL_ATTEMPTS = int(os.getenv('TERMINAL_POLL_ATTEMPTS', 3)) LABEL_RUN = 'catchup.stellar.org/run' LABEL_RANGE = 'catchup.stellar.org/range-end' @@ -284,7 +302,7 @@ def record_outcome(pod, end, attempt): # Peak ephemeral disk, for sizing a later run's ephemeral-storage request. # # Only meaningful in ephemeral mode. Sampled for every pod, but only kept for -# ranges that finished -- see the completion gate in stream_pod. Spot is fine: +# ranges that finished -- see the completion gate in finalize. Spot is fine: # what invalidates a sample is being cut short, not the capacity type. # # Prometheus cannot answer this -- cAdvisor reports fs usage per node, with no @@ -374,7 +392,7 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): and a 404 once the pod object is gone. The second path used to not exist, so a pod deleted while Running -- reaped node, eviction, or the monitor deleting a finished Job -- left its stream retrying every 30s for the rest - of the run, holding one of MAX_CONCURRENT connection slots the whole time. + of the run, holding a connection slot the whole time. """ # Before discard: on success the archive is about to be deleted. measured = {} @@ -423,113 +441,127 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): logger.info("range %s attempt %s: stream complete", end, attempt) -async def stream_pod(session, pod, end, attempt, done, done_ok): - """Follow one pod's log until it terminates, appending to its archive.""" - path = base(end, attempt) + '.log.gz' +async def _poll_once(session, pod, end, attempt, last_ts, tx): + """One short read of a pod's log. Returns (new_last_ts, gone). + + No follow=true: the request completes and the connection is released, so + concurrency is bounded by _poll_slots rather than by how many pods exist. + Measured on ssc-test, a single poll takes ~0.22s from outside the cluster, + so 2096 pods on a 10s interval need ~46 concurrent slots against the 2096 + permanently-held connections follow=true required. + """ + params = {'container': CONTAINER, 'timestamps': 'true'} + if last_ts: + # Second granularity, so this overlaps on purpose; the per-line + # comparison below removes the overlap exactly. + params['sinceTime'] = last_ts[:19] + 'Z' + url = f"{API}/api/v1/namespaces/{NAMESPACE}/pods/{pod}/log" + async with _poll_slots: + async with session.get(url, params=params, + headers={'Authorization': f'Bearer {token()}'}) as resp: + if resp.status == 404: + return last_ts, True + resp.raise_for_status() + # Chunked, not line-wise: aiohttp raises above 512 KiB on a single + # line, and a carriage-return progress meter trivially exceeds that + # -- one 628 MiB download arrived as a single "line". Split on \r as + # well, and cap what a pathological blob may buffer. + body = '' + async for chunk in resp.content.iter_chunked(65536): + body += chunk.decode('utf-8', 'replace') + if len(body) > MAX_POLL_CHARS: + break + + pending = None + lines = [l for l in re.split(r'[\r\n]', body) if l] + if not lines: + return last_ts, False + # Opened per poll, not held for the pod's life. A live gzip deflate buffer + # per stream is what put the sidecar at 1444 MiB of a 2048 MiB limit at 2096 + # follow streams; here nothing is retained between polls. + with gzip.open(base(end, attempt) + '.log.gz', 'at') as fh: + for line in lines: + ts, _, rest = line.partition(' ') + if not _TS_RE.match(ts): + # Untimestamped kubelet text. Keep it, but never let it become + # the resume point. + fh.write(line + '\n') + continue + if last_ts and ts <= last_ts: + continue # exact dedup of the resume overlap + fh.write(rest + '\n') + tx.feed(rest) + pending = ts + if pending: + write_state(end, attempt, pending) + return pending, False + return last_ts, False + + +async def poll_pod(session, pod, end, attempt, done, done_ok): + """Read one pod's log to completion, by repeated short polls. + + Replaces a follow=true stream. The stream held a connection, a gzip deflate + buffer and aiohttp read buffers for the pod's entire life, so cost scaled + with parallelism: measured at 2096 pods the sidecar sat at 1444 MiB of a + 2048 MiB limit with memory.events max=2617 and 1.00 of 2 cpu, which + extrapolates past both limits at 4096. Polling makes concurrency a tuning + parameter instead of a function of pod count. + + The one thing follow=true did better is the tail: it already held the bytes + when a pod died. So on seeing the pod go terminal this polls once more, + immediately, before finalizing -- without that, every spot eviction would + lose up to one interval of exactly the log we most want. + """ last_ts = read_state(end, attempt) if last_ts is None: # Empty state = "claimed, nothing durable yet". job_monitor's backstop # skips any range with a state file, so this prevents both of us writing # the same log. write_state(end, attempt, '') - backoff = 1.0 - # Wall clock for this attempt. The monitor records attemptSeconds from the - # pod's own terminated timestamps, but only when it still has the pod -- and - # a spot eviction reaps the node first, so 212 of 212 disruptions on - # ssc-test were classified from the Job condition with no pod and no - # duration. This process watched the container run, so it is the only - # observer left. Approximate: the stream opens up to COLLECTOR_POLL_SECONDS - # after the container did. + last_ts = '' started = asyncio.get_event_loop().time() - # Outside the reconnect loop: the medida block could straddle a dropped - # stream, and a fresh scanner per attempt would lose the half it saw. + # Outside the poll loop: the medida block can straddle two polls, and a + # fresh scanner per poll would lose the half it saw. tx = TxApplyScanner() + backoff = LOG_POLL_SECONDS + failures = 0 while True: - params = {'container': CONTAINER, 'follow': 'true', 'timestamps': 'true'} - if last_ts: - # Second granularity, so this overlaps on purpose; the per-line - # comparison below removes the overlap exactly. - params['sinceTime'] = last_ts[:19] + 'Z' - url = f"{API}/api/v1/namespaces/{NAMESPACE}/pods/{pod}/log" - + was_terminal = done(pod) try: - async with session.get(url, params=params, - headers={'Authorization': f'Bearer {token()}'}) as resp: - if resp.status == 404: - # Pod object gone -- reaped node, eviction, or the monitor - # deleting a finished Job. Nothing more to read, but the - # bytes already streamed still owe a tx_apply and the peaks - # are in Prometheus regardless. A bare return here dropped - # both for every pod that outlived its object. - logger.info("pod %s gone before/while streaming range %s", pod, end) - await finalize(session, pod, end, attempt, tx, done_ok, started) - return - resp.raise_for_status() - backoff = 1.0 - pending = None - # gzip append writes a new member; concatenated members are a - # valid archive, so restarts do not corrupt what is already there. - with gzip.open(path, 'at') as fh: - since_flush = asyncio.get_event_loop().time() - # Chunked, not line-wise. `async for raw in resp.content` - # yields lines and aiohttp raises over 512 KiB, which any - # carriage-return progress meter in the worker's output - # trivially exceeds -- one 628 MiB download is a single - # "line". The worker now passes --no-progress, but a stream - # must not be destroyable by whatever a worker happens to - # print, so split on \r as well and cap what we buffer. - pending_buf = '' - async for chunk in resp.content.iter_chunked(65536): - pending_buf += chunk.decode('utf-8', 'replace') - if len(pending_buf) > MAX_LINE_CHARS: - # A single unterminated blob. Keep the tail so the - # real line ending is still found, drop the rest. - pending_buf = pending_buf[-MAX_LINE_CHARS:] - parts = re.split(r'[\r\n]', pending_buf) - pending_buf = parts.pop() - for line in parts: - if not line: - continue - ts, _, rest = line.partition(' ') - if not _TS_RE.match(ts): - # Untimestamped kubelet text. Keep it, but never let - # it become the resume point. - fh.write(line + '\n') - continue - if last_ts and ts <= last_ts: - continue # exact dedup of the resume overlap - fh.write(rest + '\n') - tx.feed(rest) - pending = ts - now = asyncio.get_event_loop().time() - if now - since_flush >= STATE_FLUSH_SECONDS: - fh.flush() - write_state(end, attempt, pending) - last_ts = pending - since_flush = now - if pending: - write_state(end, attempt, pending) - last_ts = pending - # A clean end of stream means the container exited. - if done(pod): + last_ts, gone = await _poll_once(session, pod, end, attempt, last_ts, tx) + backoff = LOG_POLL_SECONDS + failures = 0 + if gone: + logger.info("pod %s gone before/while polling range %s", pod, end) await finalize(session, pod, end, attempt, tx, done_ok, started) return except asyncio.CancelledError: raise except Exception as e: - logger.info("range %s stream interrupted (%s); resuming from %s", + failures += 1 + logger.info("range %s poll failed (%s); retrying from %s", end, e, last_ts or 'start') - if done(pod): - # Reached when the last read threw rather than ending cleanly -- a - # 500 burst, a dropped connection -- and the pod has since gone - # terminal. The partial stream may already hold the medida block, - # and the peaks are query-side, so this owes exactly what the clean - # path owes. It used to return bare and lose both. - await finalize(session, pod, end, attempt, tx, done_ok, started) - return + backoff = min(backoff * 2, 30) + if was_terminal and failures >= TERMINAL_POLL_ATTEMPTS: + # The container has exited and its log will not come back. A + # follow=true stream finalized here because it already held the + # bytes; polling has to decide to stop asking, or it spins on a + # dead pod for the rest of the run and never writes its metrics. + logger.warning("range %s attempt %s: %d failed polls after the pod " + "went terminal; finalizing on what was read", + end, attempt, failures) + await finalize(session, pod, end, attempt, tx, done_ok, started) + return + else: + if was_terminal: + # Terminal BEFORE that poll, so the poll saw the container's + # final output. Checking after would race a pod that exits + # mid-poll and drop whatever it wrote on the way out. + await finalize(session, pod, end, attempt, tx, done_ok, started) + return await asyncio.sleep(backoff) - backoff = min(backoff * 2, 30) async def list_pods(session): @@ -548,7 +580,11 @@ async def main(): # pool stays full -- and every holder is a follow=true stream open for the # life of its pod. Below the live pod count this does not degrade, it # starves, and it starves the pods created last, which are the retries. - conn = aiohttp.TCPConnector(limit=MAX_CONCURRENT, ssl=ssl_ctx()) + # Sized for concurrent polls plus headroom for the pod-list and kubelet + # calls, not for one connection per pod. Under follow=true this had to + # exceed parallelism or pods silently starved -- 1200 against 2048 workers + # left 896 blocked forever, and retries, created last, never got a slot. + conn = aiohttp.TCPConnector(limit=MAX_CONCURRENT_POLLS + 64, ssl=ssl_ctx()) # No total timeout: these streams are meant to stay open for the life of a # range, which can be hours. timeout = aiohttp.ClientTimeout(total=None, sock_connect=10) @@ -624,7 +660,7 @@ async def main(): attempt = labels.get(LABEL_ATTEMPT, '1') _streaming[name] = (end, attempt) tasks[name] = asyncio.create_task( - stream_pod(session, name, end, attempt, + poll_pod(session, name, end, attempt, lambda p: terminal.get(p, False), lambda p: succeeded.get(p, False))) logger.info("opened stream for range %s attempt %s (%d active)", diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 24acc0c3..94dae4f9 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -329,14 +329,20 @@ spec: value: /logs - name: COLLECTOR_POLL_SECONDS value: {{ .Values.monitor.collectorPollSeconds | quote }} + - name: TERMINAL_POLL_ATTEMPTS + value: {{ .Values.monitor.terminalPollAttempts | quote }} + - name: LOG_POLL_SECONDS + value: {{ .Values.monitor.logPollSeconds | quote }} + - name: MAX_CONCURRENT_POLLS + value: {{ .Values.monitor.maxConcurrentPolls | quote }} + - name: MAX_POLL_CHARS + value: {{ .Values.monitor.maxPollChars | int64 | quote }} - name: MAX_LINE_CHARS - value: {{ .Values.monitor.maxLineChars | quote }} + value: {{ .Values.monitor.maxLineChars | int64 | quote }} - name: PEAK_FLUSH_RATIO value: {{ .Values.monitor.peakFlushRatio | quote }} - name: COLLECTOR_VANISHED_GRACE_CYCLES value: {{ .Values.monitor.collectorVanishedGraceCycles | quote }} - - name: COLLECTOR_MAX_STREAMS - value: {{ if gt (.Values.monitor.collectorMaxStreams | int) 0 }}{{ .Values.monitor.collectorMaxStreams | quote }}{{ else }}{{ add (.Values.worker.replicas | int) 256 | quote }}{{ end }} # Failures are always kept; successes are the bulk of the volume. - name: SAVE_SUCCESS_LOGS value: {{ .Values.monitor.saveSuccessLogs | quote }} diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 31d9ddae..5b5d2d67 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -138,17 +138,21 @@ monitor: # A worker printing a carriage-return progress meter would otherwise grow this # without bound, on every stream at once. maxLineChars: 65536 + # Log collection polls rather than holding a follow=true stream per pod, so + # concurrency is independent of worker.replicas. At 4096 pods and a 10s + # interval this is ~90 in-flight polls. + logPollSeconds: 10 + maxConcurrentPolls: 96 + maxPollChars: 8388608 + # Failed polls tolerated after a pod goes terminal before the collector stops + # asking. Its log is not coming back and the task holds a poll slot. + terminalPollAttempts: 3 # Growth factor before an in-flight peak is flushed to its .metrics file, so a # collector restart cannot silently reset a range's high-water to zero. peakFlushRatio: 1.05 # Poll cycles a stream gets to finalize after its pod leaves the pod list, # before it is cancelled and its connection slot reclaimed. collectorVanishedGraceCycles: 3 - # 0 = derive from worker.replicas. A fixed value here is how 2048-worker runs - # silently lost every retry pod's metrics: this caps the aiohttp connection - # pool, not the task count, so pods beyond it block forever rather than - # queueing -- and retries, created last, never got a slot. - collectorMaxStreams: 0 collectorResources: requests: { cpu: "200m", memory: "512Mi" } limits: { cpu: "2", memory: "2Gi" } diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index c8b35925..feb7eaa8 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -1148,6 +1148,8 @@ def test_both_exit_paths_share_one_finalize(): # writing peakAnonBytes while the other keeps working. # Three: clean exit, pod-gone 404, and an interrupted read on a pod that # has since gone terminal. + # Three: pod gone (404), the pod was terminal before the poll that just + # succeeded, and a terminal pod whose polls keep failing. assert len(re.findall(r"await finalize\(session, pod, end, attempt, tx, done_ok, started\)", COLLECTOR_SRC)) == 3 assert len(re.findall(r"write_metrics\(end, attempt, measured\)", COLLECTOR_SRC)) == 1 @@ -1155,14 +1157,13 @@ def test_both_exit_paths_share_one_finalize(): def _run_stream_pod(status, terminal): - """Execute stream_pod against a fake apiserver. Returns finalize calls. + """Execute poll_pod against a fake apiserver. Returns finalize calls. - Executed rather than pattern-matched: the previous version of this test - asserted on a `except ClientResponseError` branch that raise_for_status - could never reach, because an earlier `if resp.status == 404` returned - first. It passed against dead code. + Executed rather than pattern-matched: an earlier version of these tests + asserted on an `except ClientResponseError` branch that raise_for_status + could never reach, and passed against dead code. """ - import asyncio, tempfile, types, os as _os + import asyncio, tempfile, os as _os, gzip as _gzip calls = [] class FakeResp: @@ -1174,9 +1175,11 @@ def raise_for_status(self): raise OSError(f"HTTP {self.status}") @property def content(self): - async def it(): - if False: yield b'' - return it() + class C: + async def iter_chunked(self, n): + if False: + yield b'' + return C() FakeResp.status = status # class bodies cannot close over a local @@ -1188,21 +1191,26 @@ async def fake_finalize(session, pod, end, attempt, tx, done_ok, started=None): d = tempfile.mkdtemp() ns = { - 'asyncio': asyncio, 'gzip': __import__('gzip'), 'os': _os, + 'asyncio': asyncio, 'gzip': _gzip, 're': re, 'os': _os, 'API': 'https://k8s', 'NAMESPACE': 'ns', 'CONTAINER': 'stellar-core', - 'LOG_DIR': d, 'STATE_FLUSH_SECONDS': 10, + 'LOG_DIR': d, 'LOG_POLL_SECONDS': 0.05, 'MAX_POLL_CHARS': 1 << 20, + 'TERMINAL_POLL_ATTEMPTS': 3, + '_poll_slots': asyncio.Semaphore(4), 'token': lambda: 't', 'finalize': fake_finalize, 'base': lambda e, a: _os.path.join(d, f"range-{e}-a{a}"), 'read_state': lambda e, a: None, 'write_state': lambda e, a, ts: None, '_TS_RE': re.compile(r"^\d{4}"), - 'TxApplyScanner': type('T', (), {'seconds': None, 'feed': lambda s, l: None}), + 'TxApplyScanner': type('T', (), {'seconds': None, 'resumed': False, + 'feed': lambda s, l: None}), 'logger': type('L', (), {'info': lambda s, *a: None, 'warning': lambda s, *a: None})(), } - exec(_extract(r"^(async def stream_pod\(.*?)(?=\n\nasync def )", + exec(_extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1), ns) - coro = ns['stream_pod'](FakeSession(), 'pod-1', '999', '1', - lambda p: terminal, lambda p: False) + exec(_extract(r"^(async def poll_pod\(.*?)(?=\n\nasync def )", + COLLECTOR_SRC).group(1), ns) + coro = ns['poll_pod'](FakeSession(), 'pod-1', '999', '1', + lambda p: terminal, lambda p: False) asyncio.run(asyncio.wait_for(coro, timeout=2)) return calls @@ -1675,8 +1683,8 @@ def test_progress_is_written_atomically(): def test_the_log_stream_resumes_from_the_last_durable_timestamp(): # Without sinceTime a reconnect re-reads the whole log from the start: one # full re-read per pod per reconnect, at 2096 pods. - fn = _extract(r"^(async def stream_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert "params['sinceTime']" in fn, "reconnect does not resume" + fn = _extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert "params['sinceTime']" in fn, "a poll does not resume from the last durable line" # ...and the second-granularity overlap it creates is removed per line. assert re.search(r"if last_ts and ts <= last_ts:\s*\n\s*continue", fn), \ "the deliberate resume overlap is never deduped" @@ -1770,15 +1778,16 @@ def test_the_authoritative_outcome_wins_over_the_collector_estimate(): # every retry pod of a collector stream and left a2 metrics empty. def test_the_stream_is_read_in_chunks_not_lines(): - fn = _extract(r"^(async def stream_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + fn = _extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) assert 'iter_chunked' in fn, "line-wise reads are bounded by aiohttp's 512KiB limit" assert 'async for raw in resp.content:' not in fn + assert "'follow'" not in fn, "a poll must not follow" def test_carriage_returns_split_lines_too(): # The progress meter is \r-delimited. Without \r in the split it stays one # blob no matter how the bytes arrive. - fn = _extract(r"^(async def stream_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + fn = _extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) m = re.search(r"re\.split\(r'\[([^\]]+)\]'", fn) assert m, "no line splitting found" assert '\\r' in m.group(1) and '\\n' in m.group(1), f"splits on {m.group(1)!r}" @@ -1787,9 +1796,8 @@ def test_carriage_returns_split_lines_too(): def test_an_unterminated_blob_is_capped_not_buffered_forever(): # A meter that never emits a newline would otherwise grow the buffer until # the collector OOMs -- 2096 streams doing it at once. - fn = _extract(r"^(async def stream_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert 'MAX_LINE_CHARS' in fn - assert re.search(r"pending_buf\[-MAX_LINE_CHARS:\]", fn), "buffer is never trimmed" + fn = _extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert 'MAX_POLL_CHARS' in fn, "a single poll response is unbounded" cap = int(_extract(r"MAX_LINE_CHARS = int\(os\.getenv\('MAX_LINE_CHARS', (\d+)\)\)", COLLECTOR_SRC).group(1)) assert 1024 < cap < 524288, f"cap {cap} is outside a sane range" @@ -1830,3 +1838,67 @@ def test_the_line_buffer_cap_is_charged_per_stream(): chart = open(__file__.replace( 'test_job_monitor.py', 'parallel_catchup_helm/values.yaml')).read() assert int(_extract(r"maxLineChars: (\d+)", chart).group(1)) == cap + + +# --- polling replaces follow=true ------------------------------------------ +# Measured on ssc-test at 2096 follow streams: 1444 MiB of a 2048 MiB limit, +# memory.events max=2617, 1.00 of 2 cpu, 1797 held connections. That scales +# with pod count, so 4096 exceeds both limits. Polling makes concurrency a +# tuning parameter instead. + +def test_concurrency_is_independent_of_pod_count(): + # The whole point. Under follow=true the cap had to exceed parallelism or + # pods starved silently -- 1200 against 2048 left 896 blocked forever. + assert 'COLLECTOR_MAX_STREAMS' not in COLLECTOR_SRC + chart = open(__file__.replace( + 'test_job_monitor.py', 'parallel_catchup_helm/templates/job_monitor.yaml')).read() + assert 'COLLECTOR_MAX_STREAMS' not in chart + assert 'worker.replicas' not in chart.split('MAX_CONCURRENT_POLLS')[1][:200], \ + "poll concurrency must not be derived from parallelism" + + +def test_polls_are_bounded_by_a_semaphore(): + fn = _extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert 'async with _poll_slots:' in fn, "polls are not bounded" + # ...and the connector is sized for polls, not for one socket per pod. + assert 'MAX_CONCURRENT_POLLS + 64' in COLLECTOR_SRC + + +def test_the_archive_is_not_held_open_between_polls(): + # A live gzip deflate buffer per stream is most of what put the sidecar at + # 1444 MiB. Opening per poll means nothing is retained between them. + fn = _extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert "gzip.open(base(end, attempt) + '.log.gz', 'at')" in fn + loop = _extract(r"^(async def poll_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert 'gzip.open' not in loop, "the archive is held across polls" + + +def test_terminal_is_read_before_the_poll_not_after(): + # A pod that exits mid-poll would otherwise have its final output dropped: + # the poll that read it would not yet know the pod was terminal, and the + # next check would come after finalize. + loop = _extract(r"^(async def poll_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert loop.index('was_terminal = done(pod)') < loop.index('await _poll_once('), \ + "terminal is sampled after the poll, which races a pod exiting mid-poll" + + +def test_a_dead_pod_is_not_polled_forever(): + # Its log is not coming back, and the task holds a poll slot for the rest of + # the run. follow=true finalized here because it already held the bytes. + loop = _extract(r"^(async def poll_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert 'TERMINAL_POLL_ATTEMPTS' in loop + n = int(_extract(r"TERMINAL_POLL_ATTEMPTS = int\(os\.getenv\('TERMINAL_POLL_ATTEMPTS', (\d+)\)\)", + COLLECTOR_SRC).group(1)) + assert n >= 2, "a single transient 500 would end the attempt" + + +def test_a_terminal_pod_whose_polls_keep_failing_still_finalizes(): + # Executed: the loop must exit, not spin. Was a real regression when polling + # replaced streaming -- the suite caught it. + assert _run_stream_pod(500, terminal=True) == [('pod-1', '999', '1')] + + +def test_poll_concurrency_default_is_modest(): + n = int(_extract(r"MAX_CONCURRENT_POLLS = int\(os\.getenv\('MAX_CONCURRENT_POLLS', (\d+)\)\)", + COLLECTOR_SRC).group(1)) + assert 16 <= n <= 256, f"{n} in-flight polls is not a sane default" From 2dffc456c8ede0a819e82594adcf89e2d047d59a Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Wed, 29 Jul 2026 22:33:33 -0400 Subject: [PATCH 009/117] Poll only pods whose log endpoint can answer An allowlist of Running/Succeeded/Failed, not "skip Pending". Measured after the polling switch: 60 of 88 poll failures were 400 "container is waiting to start". Unknown is excluded for the same reason -- the node has stopped reporting, so the poll cannot succeed. Terminal phases stay in: that is where a pod's final output lives. 153 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/log_collector.py | 13 +++++++++++++ .../test_job_monitor.py | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 1c89e372..21af04cd 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -95,6 +95,10 @@ # log is not coming back, and spinning on it holds a task and a poll slot for # the rest of the run; a couple of retries still absorb a transient 500. TERMINAL_POLL_ATTEMPTS = int(os.getenv('TERMINAL_POLL_ATTEMPTS', 3)) +# Phases whose log endpoint can actually answer. Pending has no container yet +# and Unknown means the node stopped reporting; the terminal phases are kept +# because that is where a pod's final output lives. +POLLABLE_PHASES = ('Running', 'Succeeded', 'Failed') LABEL_RUN = 'catchup.stellar.org/run' LABEL_RANGE = 'catchup.stellar.org/range-end' @@ -658,6 +662,15 @@ async def main(): if name in streamed: continue attempt = labels.get(LABEL_ATTEMPT, '1') + if phase not in POLLABLE_PHASES: + # Allowlist, not "skip Pending". A container that has not + # started answers 400 "waiting to start" -- 60 of 88 poll + # failures right after the polling switch -- and Unknown + # means the node stopped reporting, so that poll cannot + # succeed either. Both are picked up on the cycle they + # become pollable. Succeeded and Failed stay in: a + # terminal pod is where the final output lives. + continue _streaming[name] = (end, attempt) tasks[name] = asyncio.create_task( poll_pod(session, name, end, attempt, diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index feb7eaa8..e70097ed 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -1902,3 +1902,22 @@ def test_poll_concurrency_default_is_modest(): n = int(_extract(r"MAX_CONCURRENT_POLLS = int\(os\.getenv\('MAX_CONCURRENT_POLLS', (\d+)\)\)", COLLECTOR_SRC).group(1)) assert 16 <= n <= 256, f"{n} in-flight polls is not a sane default" + + +def test_a_pending_pod_is_not_polled_yet(): + # Its container has not started, so the log endpoint answers 400 and the + # poll is wasted -- 60 of 88 failures immediately after the polling switch. + loop = _extract(r"while True:\n(.*?)await asyncio\.sleep\(POLL_SECONDS\)", + COLLECTOR_SRC).group(1) + per_pod = loop[loop.index('for pod in pods:'):] + # From the attempt lookup to the stream open: the earlier + # terminal[name] = phase in ('Succeeded', 'Failed') line is not this guard. + guard = per_pod[per_pod.index('attempt = labels.get'):per_pod.index('_streaming[name]')] + assert 'phase not in POLLABLE_PHASES' in guard, \ + "pollability is not decided by an allowlist" + allowed = set(re.findall(r"'(\w+)'", _extract( + r"POLLABLE_PHASES = \(([^)]+)\)", COLLECTOR_SRC).group(1))) + # Terminal phases must stay in -- that is where a pod's final output lives. + assert {'Running', 'Succeeded', 'Failed'} == allowed, allowed + # Pending has no container; Unknown means the node stopped reporting. + assert 'Pending' not in allowed and 'Unknown' not in allowed From 16353115798073850fe97052718aa650b813af16 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Wed, 29 Jul 2026 22:42:27 -0400 Subject: [PATCH 010/117] Do not reap a succeeded Job until its record is actually complete The 2096-worker spot run delivered a profile with txApply on 356 of 356 completed ranges and peaks on 0, while 1936 .metrics files on the same volume carried peakAnonBytes. Two causes, both on the success path: - The reap was gated on tx_apply, which is the wrong signal. tx_apply_for_range falls back to the archive and then the pod, so it is available almost immediately; peaks_for_range has no fallback at all. Gating on the always-available field let delete_job reap the pod before the collector had finalized, and .metrics is the only place peaks live. _reap_if_complete now requires both. - The peak read was one-shot, inside `if end not in completed`. A range recorded before the collector finalized never got a second chance, and the reap removed the Job so reconcile never revisited it. Later passes now backfill peaks while the Job is still present. JOB_TTL_SECONDS remains the backstop, so a range whose collector never finalizes costs a late Job rather than a stuck one. 155 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 36 +++++++++++++++++-- .../test_job_monitor.py | 33 +++++++++++++++-- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 679ce786..b4400425 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1081,6 +1081,25 @@ def release_pvc(end): logger.warning("could not release PVC for completed range %s: %s", end, e) +def _has_peaks(record): + return any(record.get(k) is not None for k in PEAK_FIELDS) + + +def _reap_if_complete(end, attempt, record): + """Delete a succeeded range's Job once nothing more can be learned from it. + + Deleting reaps the pod, and .metrics is the only place peaks live, so a + reap before the collector finalizes makes the gap permanent -- that is + exactly how a whole run's profile came back with txApply on every range and + peaks on none. JOB_TTL_SECONDS still reclaims anything the collector never + gets to, so a pod reaped before it could be read costs a late Job, not a + stuck one. + """ + if record.get('txApply') is None or not _has_peaks(record): + return + delete_job(end, attempt) + + def delete_job(end, attempt): """Drop a finished Job once nothing more is owed by it. @@ -1535,8 +1554,21 @@ def reconcile(state): # pod is the only place left to read it from -- deleting the # Job would reap the pod and make that gap permanent. Leave # those to JOB_TTL_SECONDS. - if tx is not None: - delete_job(end, attempt) + _reap_if_complete(end, attempt, completed[end]) + elif not _has_peaks(completed[end]): + # Backfill. The record is written the moment the Job flips to + # succeeded, which is usually before the collector has finalized + # -- and peaks_for_range has no fallback, unlike tx_apply, which + # reads the archive. Measured on ssc-test: 356 of 356 completed + # ranges carried txApply and 0 carried peakAnonBytes, while 1936 + # .metrics files on the same volume held it. Retry while the Job + # is still here; delete_job below is what ends the chances. + late = peaks_for_range(end, attempt) + if late: + completed[end].update(late) + save_progress(progress) + logger.info("range %s: peaks arrived late, backfilled", end) + _reap_if_complete(end, attempt, completed[end]) elif st.failed: pod = job_pods.get(j.metadata.name) if pod is not None: diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index e70097ed..84410eee 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -956,14 +956,17 @@ def test_the_retry_creates_the_successor_before_deleting_the_predecessor(): "delete_job must come after the successor is created" -def test_a_success_whose_metrics_are_missing_keeps_its_job(): +def test_a_success_whose_record_is_incomplete_keeps_its_job(): # tx is read from the collector's .metrics, else the pod. Deleting the Job # reaps the pod, so reaping a success before the metrics land turns a # recoverable gap into a permanent one -- the same class of loss as the 698 # ranges the tx_apply regex dropped. body = _extract(r"(release_pvc\(end\)\n.*?)(?=\s+elif st\.failed:)").group(1) - assert re.search(r"if tx is not None:\s*\n\s*delete_job\(end, attempt\)", body), \ - "success path must gate the reap on the metric having landed" + assert '_reap_if_complete(end, attempt, completed[end])' in body, \ + "success path must gate the reap on the record being complete" + fn = _extract(r"^(def _reap_if_complete\(.*?)(?=\ndef )").group(1) + assert "record.get('txApply') is None" in fn and 'not _has_peaks(record)' in fn, \ + "the reap must require BOTH tx_apply and peaks, not just tx_apply" def test_the_chart_ttl_matches_the_code_default(): @@ -1921,3 +1924,27 @@ def test_a_pending_pod_is_not_polled_yet(): assert {'Running', 'Succeeded', 'Failed'} == allowed, allowed # Pending has no container; Unknown means the node stopped reporting. assert 'Pending' not in allowed and 'Unknown' not in allowed + + +def test_late_peaks_are_backfilled_into_a_completed_record(): + # The record is written the moment the Job flips to succeeded, usually + # before the collector finalizes. peaks_for_range has no fallback the way + # tx_apply does, so a one-shot read loses them: measured on ssc-test, 356 of + # 356 completed ranges had txApply and 0 had peakAnonBytes, while 1936 + # .metrics files on the same volume held it. + body = _extract(r"(elif not _has_peaks\(completed\[end\]\):.*?)(?=\n\s+elif st\.failed:)").group(1) + assert 'peaks_for_range(end, attempt)' in body, "no retry of the peak read" + assert 'save_progress(progress)' in body, "a backfilled peak is never persisted" + assert '_reap_if_complete' in body, "backfill never lets the Job go" + + +def test_peaks_and_tx_apply_are_both_required_before_reaping(): + ns = {'PEAK_FIELDS': ('peakAnonBytes', 'peakRssBytes'), 'reaped': []} + ns['delete_job'] = lambda e, a: ns['reaped'].append((e, a)) + exec(_extract(r"^(def _has_peaks\(.*?)(?=\ndef )").group(1), ns) + exec(_extract(r"^(def _reap_if_complete\(.*?)(?=\ndef )").group(1), ns) + ns['_reap_if_complete'](1, 1, {'txApply': 5.0}) # no peaks + ns['_reap_if_complete'](2, 1, {'peakAnonBytes': 99}) # no tx + assert ns['reaped'] == [], "reaped an incomplete record" + ns['_reap_if_complete'](3, 1, {'txApply': 5.0, 'peakAnonBytes': 99}) + assert ns['reaped'] == [(3, 1)], ns['reaped'] From fd8b0999d9d47ddeb2e0d543c3bb1a4bb3be44ab Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Wed, 29 Jul 2026 22:47:49 -0400 Subject: [PATCH 011/117] Gate the reap on an explicit done marker from the collector Replaces inferring "the collector has finished" from peaks being present. That inference was wrong in both directions: tx_apply falls back to the archive so it lands long before the collector finishes, and an attempt can legitimately finalize with no peaks at all, which left its Job waiting out the TTL for no reason. finalize now writes range--a.done on the shared volume, last, after .metrics -- written any earlier it would authorise exactly the reap it exists to prevent. The monitor stats it and only then deletes the Job, which is what reaps the pod, the one place peaks can still be read from. Atomic (.tmp + os.replace) and best-effort: a failed marker costs a Job that waits out JOB_TTL_SECONDS, never correctness. 158 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 19 ++++- src/MissionParallelCatchup/log_collector.py | 24 ++++++ .../test_job_monitor.py | 75 +++++++++++++++---- 3 files changed, 101 insertions(+), 17 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index b4400425..ff55f1ca 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1081,6 +1081,21 @@ def release_pvc(end): logger.warning("could not release PVC for completed range %s: %s", end, e) +def done_path(end, attempt): + return os.path.join(LOG_DIR, f"range-{end}-a{attempt}.done") + + +def _attempt_finalized(end, attempt): + """Has the collector written everything it will for this attempt? + + It writes this file last, after .metrics. Anything inferred instead -- peaks + being present, tx_apply being readable -- is a guess: tx_apply falls back to + the archive so it is available long before the collector finishes, and an + attempt can legitimately finalize with no peaks at all. + """ + return os.path.exists(done_path(end, attempt)) + + def _has_peaks(record): return any(record.get(k) is not None for k in PEAK_FIELDS) @@ -1095,7 +1110,7 @@ def _reap_if_complete(end, attempt, record): gets to, so a pod reaped before it could be read costs a late Job, not a stuck one. """ - if record.get('txApply') is None or not _has_peaks(record): + if not _attempt_finalized(end, attempt): return delete_job(end, attempt) @@ -1555,7 +1570,7 @@ def reconcile(state): # Job would reap the pod and make that gap permanent. Leave # those to JOB_TTL_SECONDS. _reap_if_complete(end, attempt, completed[end]) - elif not _has_peaks(completed[end]): + elif not _has_peaks(completed[end]) or not _attempt_finalized(end, attempt): # Backfill. The record is written the moment the Job flips to # succeeded, which is usually before the collector has finalized # -- and peaks_for_range has no fallback, unlike tx_apply, which diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 21af04cd..89438417 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -128,6 +128,10 @@ def base(end, attempt): return os.path.join(LOG_DIR, f"range-{end}-a{attempt}") +def done_path(end, attempt): + return base(end, attempt) + '.done' + + def read_state(end, attempt): try: with open(base(end, attempt) + '.state') as fh: @@ -389,6 +393,18 @@ async def sample_kubelet(session, nodes): write_metrics(ref[0], ref[1], {'peakAnonBytes': int(rss)}) +def _mark_done(end, attempt): + path = done_path(end, attempt) + tmp = path + '.tmp' + try: + with open(tmp, 'w') as fh: + fh.write('') + os.replace(tmp, path) + except OSError as e: + # Costs a Job that waits out JOB_TTL_SECONDS, never correctness. + logger.warning("could not mark range %s attempt %s done: %s", end, attempt, e) + + async def finalize(session, pod, end, attempt, tx, done_ok, started=None): """Persist everything this attempt owes, then let its stream go. @@ -443,6 +459,14 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): "(saveSuccessLogs=false)", end, attempt) else: logger.info("range %s attempt %s: stream complete", end, attempt) + # Last, deliberately. The monitor treats this file as "the collector will + # write nothing further for this attempt" and only then reaps the Job -- + # which deletes the pod, the one place peaks can still be read from. It has + # to land after .metrics or it would license exactly the reap it exists to + # prevent. Inferring the same thing from peaks being present was wrong for + # an attempt that legitimately has none. + _mark_done(end, attempt) + async def _poll_once(session, pod, end, attempt, last_ts, tx): diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index 84410eee..66e1dea5 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -965,8 +965,8 @@ def test_a_success_whose_record_is_incomplete_keeps_its_job(): assert '_reap_if_complete(end, attempt, completed[end])' in body, \ "success path must gate the reap on the record being complete" fn = _extract(r"^(def _reap_if_complete\(.*?)(?=\ndef )").group(1) - assert "record.get('txApply') is None" in fn and 'not _has_peaks(record)' in fn, \ - "the reap must require BOTH tx_apply and peaks, not just tx_apply" + assert 'not _attempt_finalized(end, attempt)' in fn, \ + "the reap must wait for the collector's own done marker" def test_the_chart_ttl_matches_the_code_default(): @@ -1329,7 +1329,7 @@ def test_finalize_records_the_working_set_peak(): '_anon_peak': {'pod-1': 900}, '_ws_peak': ws, '_eph_peak': {}, '_peak_flushed': {}, '_streaming': {}, 'SAVE_SUCCESS_LOGS': True, 'write_metrics': lambda e, a, v: written.append(v), - 'discard': lambda e, a: None, + 'discard': lambda e, a: None, '_mark_done': lambda e, a: None, 'logger': type('L', (), {'info': lambda s, *a: None})(), } exec(fn, ns) @@ -1503,7 +1503,7 @@ def run(resumed): ns = {'_anon_peak': {'p': 1}, '_ws_peak': {}, '_eph_peak': {}, '_peak_flushed': {}, '_streaming': {}, 'SAVE_SUCCESS_LOGS': True, 'write_metrics': lambda e, a, v: written.append(v), - 'discard': lambda e, a: None, + 'discard': lambda e, a: None, '_mark_done': lambda e, a: None, 'logger': type('L', (), {'info': lambda s, *a: None})()} exec(fn, ns) tx = type('T', (), {'seconds': None, 'resumed': resumed})() @@ -1742,7 +1742,7 @@ def test_the_collector_records_a_duration_the_monitor_cannot(): ns = {'asyncio': asyncio, '_anon_peak': {}, '_ws_peak': {}, '_eph_peak': {}, '_peak_flushed': {}, '_streaming': {}, 'SAVE_SUCCESS_LOGS': True, 'write_metrics': lambda e, a, v: written.append(v), - 'discard': lambda e, a: None, + 'discard': lambda e, a: None, '_mark_done': lambda e, a: None, 'logger': type('L', (), {'info': lambda s, *a: None})()} exec(fn, ns) tx = type('T', (), {'seconds': None, 'resumed': False})() @@ -1932,19 +1932,64 @@ def test_late_peaks_are_backfilled_into_a_completed_record(): # tx_apply does, so a one-shot read loses them: measured on ssc-test, 356 of # 356 completed ranges had txApply and 0 had peakAnonBytes, while 1936 # .metrics files on the same volume held it. - body = _extract(r"(elif not _has_peaks\(completed\[end\]\):.*?)(?=\n\s+elif st\.failed:)").group(1) + body = _extract(r"(elif not _has_peaks\(completed\[end\]\).*?)(?=\n\s+elif st\.failed:)").group(1) assert 'peaks_for_range(end, attempt)' in body, "no retry of the peak read" assert 'save_progress(progress)' in body, "a backfilled peak is never persisted" assert '_reap_if_complete' in body, "backfill never lets the Job go" + assert '_attempt_finalized(end, attempt)' in body, \ + "backfill stops retrying before the collector has finished" -def test_peaks_and_tx_apply_are_both_required_before_reaping(): - ns = {'PEAK_FIELDS': ('peakAnonBytes', 'peakRssBytes'), 'reaped': []} +def test_the_reap_waits_for_the_collectors_done_marker(): + # Not inferred from peaks or tx_apply: tx_apply falls back to the archive so + # it lands long before the collector finishes, and an attempt can finalize + # with no peaks at all. Only the collector knows it is done. + import tempfile, os as _os + d = tempfile.mkdtemp() + ns = {'os': _os, 'LOG_DIR': d, 'PEAK_FIELDS': ('peakAnonBytes',), 'reaped': []} ns['delete_job'] = lambda e, a: ns['reaped'].append((e, a)) - exec(_extract(r"^(def _has_peaks\(.*?)(?=\ndef )").group(1), ns) - exec(_extract(r"^(def _reap_if_complete\(.*?)(?=\ndef )").group(1), ns) - ns['_reap_if_complete'](1, 1, {'txApply': 5.0}) # no peaks - ns['_reap_if_complete'](2, 1, {'peakAnonBytes': 99}) # no tx - assert ns['reaped'] == [], "reaped an incomplete record" - ns['_reap_if_complete'](3, 1, {'txApply': 5.0, 'peakAnonBytes': 99}) - assert ns['reaped'] == [(3, 1)], ns['reaped'] + for name in ('done_path', '_attempt_finalized', '_has_peaks', '_reap_if_complete'): + exec(_extract(r"^(def " + name + r"\(.*?)(?=\ndef )").group(1), ns) + full = {'txApply': 5.0, 'peakAnonBytes': 99} + ns['_reap_if_complete'](1, 1, full) + assert ns['reaped'] == [], "reaped before the collector marked it done" + open(_os.path.join(d, 'range-1-a1.done'), 'w').close() + ns['_reap_if_complete'](1, 1, full) + assert ns['reaped'] == [(1, 1)], ns['reaped'] + + +def test_the_done_marker_is_written_after_everything_else(): + # It licenses the monitor to reap the pod, which is the only place peaks can + # still be read from. Written before .metrics it would authorise exactly the + # reap it exists to prevent. + fn = _extract(r"^(async def finalize\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert '_mark_done(end, attempt)' in fn + assert fn.index('write_metrics(end, attempt, measured)') < fn.index('_mark_done('), \ + "the done marker precedes the metrics it certifies" + assert fn.rstrip().endswith('_mark_done(end, attempt)'), \ + "the done marker is not the last thing finalize does" + + +def test_the_done_marker_is_written_atomically_and_is_best_effort(): + fn = _extract(r"^(def _mark_done\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert '.tmp' in fn and 'os.replace(' in fn, "a half-written marker would be truthy" + assert 'except OSError' in fn, "a failed marker must not kill the stream" + import tempfile, os as _os + d = tempfile.mkdtemp() + ns = {'os': _os, + 'base': lambda e, a: _os.path.join(d, f"range-{e}-a{a}"), + 'logger': type('L', (), {'warning': lambda s, *a: None})()} + exec(_extract(r"^(def done_path\(.*?)(?=\ndef )", COLLECTOR_SRC).group(1), ns) + exec(fn, ns) + ns['_mark_done'](77, 2) + assert _os.path.exists(_os.path.join(d, 'range-77-a2.done')) + assert not _os.path.exists(_os.path.join(d, 'range-77-a2.done.tmp')) + + +def test_both_sides_agree_on_the_marker_path(): + # Two processes, one volume, one filename. A mismatch would mean the monitor + # never reaps and every Job waits out its TTL. + c = _extract(r"^(def done_path\(.*?)(?=\ndef )", COLLECTOR_SRC).group(1) + m = _extract(r"^(def done_path\(.*?)(?=\n\ndef )").group(1) + assert ".done" in c and ".done" in m + assert 'range-' in m and 'base(end, attempt)' in c From 5bd41f63ced9671805542e6813cced9f6b42d322 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Wed, 29 Jul 2026 22:59:17 -0400 Subject: [PATCH 012/117] Wake a poller the moment its pod goes terminal or vanishes poll_pod slept LOG_POLL_SECONDS between polls and only noticed a terminal pod on its next tick, so up to 10s separated the container exiting from the final read. That window belongs to whatever deletes the pod next -- a spot reclaim, a node reap -- and takes the last lines with it. The main loop now sets a per-pod Event when it first observes the pod terminal, and when the pod leaves the pod list entirely. Gone is terminal too: without that its poller sleeps out the interval before taking the 404, delaying finalize and the .done the monitor waits for. Polling faster instead would not help: sinceTime has second granularity, so anything under ~1s re-reads the same second. One behaviour change worth naming: an Event stays set, so after a pod is terminal the backoff no longer applies and a failing poll retries at once. Bounded by TERMINAL_POLL_ATTEMPTS, and on an evicted pod retrying fast is the point. Also makes the retry-path reap wait for the same .done marker as the success path. Peaks were a proxy: an attempt with none would never be reaped, and one whose peaks landed early could be reaped mid-read. 162 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 6 +- src/MissionParallelCatchup/log_collector.py | 25 ++++++++- .../test_job_monitor.py | 56 +++++++++++++++++-- 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index ff55f1ca..d5bba297 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1676,7 +1676,11 @@ def reconcile(state): # for .metrics means the collector has finalized this attempt -- # its peaks, its tx_apply and its duration are all durable. # JOB_TTL_SECONDS reaps it if the collector never gets there. - if peaks_for_range(end, attempt): + # Same marker the success path waits for. Peaks were a proxy: + # an attempt that legitimately has none would never be reaped, + # and one whose peaks landed early could be reaped while the + # collector was still reading its log. + if _attempt_finalized(end, attempt): delete_job(end, attempt) in_progress.append(job_key(int(end), by_end[end])) continue diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 89438417..ffa9c9f5 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -91,6 +91,12 @@ # than among the peak dicts, where it landed inside the region the scanner tests # exec and broke six of them on an asyncio NameError. _poll_slots = asyncio.Semaphore(MAX_CONCURRENT_POLLS) +# pod name -> Event, set by the main loop the moment it first observes the pod +# terminal. poll_pod waits on it instead of sleeping blind, so the final read +# happens within the pod-list cadence rather than up to LOG_POLL_SECONDS later. +# That window is the only thing standing between a spot reclaim and the last +# lines the container wrote. +_wake = {} # Failed polls tolerated after a pod goes terminal before we stop asking. Its # log is not coming back, and spinning on it holds a task and a poll slot for # the rest of the run; a couple of retries still absorb a transient 500. @@ -430,6 +436,7 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): measured['txApplySeconds'] = tx.seconds _peak_flushed.pop(pod, None) _streaming.pop(pod, None) + _wake.pop(pod, None) anon = _anon_peak.pop(pod, None) if anon is not None: # Recorded for every attempt, not just the winner. peaks_for_range takes @@ -589,7 +596,15 @@ async def poll_pod(session, pod, end, attempt, done, done_ok): # mid-poll and drop whatever it wrote on the way out. await finalize(session, pod, end, attempt, tx, done_ok, started) return - await asyncio.sleep(backoff) + # Not a blind sleep: a pod going terminal cuts it short. Polling faster + # would not help -- sinceTime has second granularity, so anything under + # ~1s re-reads the same second -- and the delay that matters is between + # the container exiting and the last read, not between routine polls. + try: + await asyncio.wait_for(_wake.setdefault(pod, asyncio.Event()).wait(), + timeout=backoff) + except asyncio.TimeoutError: + pass async def list_pods(session): @@ -638,6 +653,11 @@ async def main(): # attempt it will never win. for name in [n for n in tasks if n not in live]: terminal[name] = True + if name in _wake: + # Gone is terminal. Without this its poller sleeps out + # the interval before taking the 404, delaying finalize + # and the .done that lets the monitor reap the Job. + _wake[name].set() t = tasks[name] if t.done(): del tasks[name] @@ -670,6 +690,9 @@ async def main(): continue phase = pod.get('status', {}).get('phase') terminal[name] = phase in ('Succeeded', 'Failed') + if terminal[name] and name in _wake: + # Wake its poller now rather than at the next tick. + _wake[name].set() succeeded[name] = phase == 'Succeeded' if phase == 'Failed': record_outcome(pod, end, labels.get(LABEL_ATTEMPT, '1')) diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index 66e1dea5..a5bdd460 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -1198,7 +1198,7 @@ async def fake_finalize(session, pod, end, attempt, tx, done_ok, started=None): 'API': 'https://k8s', 'NAMESPACE': 'ns', 'CONTAINER': 'stellar-core', 'LOG_DIR': d, 'LOG_POLL_SECONDS': 0.05, 'MAX_POLL_CHARS': 1 << 20, 'TERMINAL_POLL_ATTEMPTS': 3, - '_poll_slots': asyncio.Semaphore(4), + '_poll_slots': asyncio.Semaphore(4), '_wake': {}, 'token': lambda: 't', 'finalize': fake_finalize, 'base': lambda e, a: _os.path.join(d, f"range-{e}-a{a}"), 'read_state': lambda e, a: None, 'write_state': lambda e, a, ts: None, @@ -1327,7 +1327,7 @@ def test_finalize_records_the_working_set_peak(): written, ws = [], {'pod-1': 4096} ns = { '_anon_peak': {'pod-1': 900}, '_ws_peak': ws, '_eph_peak': {}, - '_peak_flushed': {}, '_streaming': {}, 'SAVE_SUCCESS_LOGS': True, + '_peak_flushed': {}, '_streaming': {}, '_wake': {}, 'SAVE_SUCCESS_LOGS': True, 'write_metrics': lambda e, a, v: written.append(v), 'discard': lambda e, a: None, '_mark_done': lambda e, a: None, 'logger': type('L', (), {'info': lambda s, *a: None})(), @@ -1501,7 +1501,7 @@ def test_finalize_records_that_an_attempt_resumed(): def run(resumed): written = [] ns = {'_anon_peak': {'p': 1}, '_ws_peak': {}, '_eph_peak': {}, - '_peak_flushed': {}, '_streaming': {}, 'SAVE_SUCCESS_LOGS': True, + '_peak_flushed': {}, '_streaming': {}, '_wake': {}, 'SAVE_SUCCESS_LOGS': True, 'write_metrics': lambda e, a, v: written.append(v), 'discard': lambda e, a: None, '_mark_done': lambda e, a: None, 'logger': type('L', (), {'info': lambda s, *a: None})()} @@ -1740,7 +1740,7 @@ def test_the_collector_records_a_duration_the_monitor_cannot(): import asyncio written = [] ns = {'asyncio': asyncio, '_anon_peak': {}, '_ws_peak': {}, '_eph_peak': {}, - '_peak_flushed': {}, '_streaming': {}, 'SAVE_SUCCESS_LOGS': True, + '_peak_flushed': {}, '_streaming': {}, '_wake': {}, 'SAVE_SUCCESS_LOGS': True, 'write_metrics': lambda e, a, v: written.append(v), 'discard': lambda e, a: None, '_mark_done': lambda e, a: None, 'logger': type('L', (), {'info': lambda s, *a: None})()} @@ -1824,8 +1824,8 @@ def test_a_failed_attempt_is_not_reaped_before_the_collector_finalizes_it(): # would lose the last interval. Gate it either way. body = _extract(r"(try:\s*\n\s*batch_v1\.create_namespaced_job.*?)continue").group(1) assert 'delete_job(end, attempt)' in body - assert re.search(r"if peaks_for_range\(end, attempt\):\s*\n\s*delete_job\(end, attempt\)", body), \ - "the retry-path reap is not gated on the collector having finalized" + assert re.search(r"if _attempt_finalized\(end, attempt\):\s*\n\s*delete_job\(end, attempt\)", body), \ + "the retry-path reap is not gated on the collector's done marker" assert body.index('create_namespaced_job') < body.index('delete_job('), \ "successor must exist before the predecessor is reaped" @@ -1993,3 +1993,47 @@ def test_both_sides_agree_on_the_marker_path(): m = _extract(r"^(def done_path\(.*?)(?=\n\ndef )").group(1) assert ".done" in c and ".done" in m assert 'range-' in m and 'base(end, attempt)' in c + + +def test_a_terminal_pod_wakes_its_poller_immediately(): + # The delay that matters is between the container exiting and the last read. + # Sleeping blind for LOG_POLL_SECONDS hands that window to a spot reclaim, + # which deletes the pod and takes the final lines with it. + loop = _extract(r"while True:\n(.*?)await asyncio\.sleep\(POLL_SECONDS\)", + COLLECTOR_SRC).group(1) + # Structure, not just presence: mutating the guard to `if False:` leaves + # the .set() line in place and sails past a substring check. + assert re.search(r"terminal\[name\] = phase in [^\n]*\n\s*if terminal\[name\][^\n]*:\s*\n(?:\s*#[^\n]*\n)*\s*_wake\[name\]\.set\(\)", loop), \ + "a pod going terminal does not wake its poller" + poller = _extract(r"^(async def poll_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert 'asyncio.wait_for(' in poller and '_wake.setdefault' in poller, \ + "the poller still sleeps blind between polls" + assert 'await asyncio.sleep(backoff)' not in poller + + +def test_the_wake_entry_is_dropped_when_the_attempt_finishes(): + # One entry per pod, and pods are per range per attempt -- 3979 ranges with + # retries would otherwise accumulate for the life of the run. + fn = _extract(r"^(async def finalize\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert '_wake.pop(pod, None)' in fn + + +def test_a_vanished_pod_also_wakes_its_poller(): + # Gone is terminal. Without the wake its poller sleeps out the interval + # before taking the 404, delaying finalize and the .done the monitor needs + # before it can reap the Job. + loop = _extract(r"while True:\n(.*?)await asyncio\.sleep\(POLL_SECONDS\)", + COLLECTOR_SRC).group(1) + blk = loop[loop.index('n not in live'):loop.index('if STORAGE_MODE') if 'if STORAGE_MODE' in loop else loop.index('for pod in pods:')] + assert 'terminal[name] = True' in blk + assert re.search(r"if name in _wake:\s*\n(?:\s*#[^\n]*\n)*\s*_wake\[name\]\.set\(\)", blk), \ + "a vanished pod never wakes its poller" + + +def test_both_reap_paths_wait_for_the_same_marker(): + # Success and retry must agree. Gating one on peaks and the other on the + # marker means an attempt with no peaks is reaped on one path and left to + # the TTL on the other. + assert len(re.findall(r"_attempt_finalized\(end, attempt\)", SRC)) >= 2 + assert 'if peaks_for_range(end, attempt):' not in SRC, \ + "a reap still uses peaks as a proxy for the collector being done" From e95f104fade6963c92f9bde794596c7ef76b8cf2 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Wed, 29 Jul 2026 23:06:04 -0400 Subject: [PATCH 013/117] Keep a ceiling-hit attempt's peak across a fresh start The fresh-start rule dropped it, which broke the self-correcting loop the profile depends on. A range that OOMs at L really did allocate L and want more, so L is a lower bound on demand and the next run must size above it: L * PROFILE_MARGIN + PROFILE_CACHE_HEADROOM clears it. Discard that and the range is sized from whichever attempt happened to survive, and OOMs again. It only bit some of the time, which is worse. Measured on ssc-test 2026-07-30 with tip-first ordering: an OOM during replay resumes, so the attempt stays in the chain (224 of 252), but an OOM during download does not (25 of 252). A run at higher cpu is download-bound -- the on-demand run at 750m had every OOM in download -- so the loop would go quiet exactly where it is needed most. Peaks only. tx_apply and seconds are summed across the chain, and a fresh start redoes the work the dropped attempt already did, so counting it there would double-count. 165 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 31 ++++++++++- .../test_job_monitor.py | 55 +++++++++++++++---- 2 files changed, 74 insertions(+), 12 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index d5bba297..d5eb065b 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -859,7 +859,7 @@ def peaks_for_range(end, attempt=1): about this one. Any field may be absent. """ out = {} - for n in _resumed_chain(end, attempt): + for n in _peak_attempts(end, attempt): try: with open(metrics_path(end, n)) as fh: data = json.load(fh) @@ -884,6 +884,35 @@ def _pod_seconds(pod): return None +def _hit_a_ceiling(end, attempt): + """Was this attempt killed at one of its own resource limits?""" + return (read_outcome(end, attempt) or {}).get('outcome') in ('oom', 'ephemeral') + + +def _peak_attempts(end, attempt): + """Attempts whose peaks describe this range: the resumed chain, plus any + attempt that died at a limit, wherever it sits. + + A ceiling-hit peak is evidence about the range no matter which pass + produced it -- the process really did allocate that much and want more, so + it is a lower bound on demand and the next run must size above it. That is + the whole self-correcting loop: a range that OOMs at L records L, and + L * PROFILE_MARGIN + PROFILE_CACHE_HEADROOM clears it next time. + + Without this the fresh-start rule silently drops it. Measured on ssc-test + 2026-07-30: an OOM during replay resumes (RESUME accepted, 224 of 252) and + stays in the chain, but an OOM during download does not (25 of 252) -- and + a run at higher cpu is download-bound, so the loop would go quiet exactly + when it is most needed. + + Peaks only. tx_apply and seconds are summed, and a fresh start redoes work + the dropped attempt already did, so including it there would double-count. + """ + chain = set(_resumed_chain(end, attempt)) + return sorted(chain | {n for n in range(1, int(attempt) + 1) + if n not in chain and _hit_a_ceiling(end, n)}) + + def _resumed_chain(end, attempt): """Attempts describing one continuous pass over the range, oldest first. diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index a5bdd460..a33caa53 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -1382,9 +1382,10 @@ def _peaks_ns(attempts): 'PEAK_FIELDS': ('peakAnonBytes', 'peakRssBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes'), } - exec(_extract(r"^(def _attempt_resumed\(.*?)(?=\ndef )").group(1), ns) - exec(_extract(r"^(def peaks_for_range\(.*?)(?=\ndef _attempt_resumed)").group(1), ns) - ns['_attempt_resumed'] = ns['_attempt_resumed'] + for name in ('read_outcome', '_attempt_resumed', '_resumed_chain', + '_hit_a_ceiling', '_peak_attempts'): + exec(_extract(r"^(def " + name + r"\(.*?)(?=\ndef )").group(1), ns) + exec(_extract(r"^(def peaks_for_range\(.*?)(?=\ndef )").group(1), ns) return ns['peaks_for_range'] @@ -1451,23 +1452,22 @@ def test_spot_is_never_excluded_as_a_capacity_type(): assert 'capacity-type' not in SRC -def test_a_fresh_retry_supersedes_everything_before_it(): +def test_a_fresh_retry_supersedes_an_interrupted_one(): # No RESUME line means new-db ran and this attempt did the whole range, so - # its sample is complete. The earlier attempt measured the same work and - # only adds noise -- and in ephemeral mode, where resume can never fire, - # this is every retry. + # its sample is complete. An earlier attempt that was merely interrupted + # measured the same work and only adds noise. f = _peaks_ns({ - 1: ({'peakAnonBytes': 8 * 1024**3}, {'outcome': 'oom'}), + 1: ({'peakAnonBytes': 8 * 1024**3}, {'outcome': 'disrupted'}), 2: ({'peakAnonBytes': 900 * 1024**2}, None), # no 'resumed' }) assert f(999, 2)['peakAnonBytes'] == 900 * 1024**2 def test_the_chain_stops_at_the_last_fresh_start(): - # a1 fresh (dropped), a2 fresh and evicted mid-replay, a3 resumed from it. - # Only a2+a3 describe the same continuous pass over the range. + # a1 interrupted then superseded by a fresh a2; a3 resumed from a2. Only + # a2+a3 describe the same continuous pass over the range. f = _peaks_ns({ - 1: ({'peakAnonBytes': 9 * 1024**3}, {'outcome': 'oom'}), + 1: ({'peakAnonBytes': 9 * 1024**3}, {'outcome': 'disrupted'}), 2: ({'peakAnonBytes': 2 * 1024**3}, {'outcome': 'disrupted'}), 3: ({'peakAnonBytes': 500 * 1024**2, 'resumed': True}, None), }) @@ -2037,3 +2037,36 @@ def test_both_reap_paths_wait_for_the_same_marker(): assert len(re.findall(r"_attempt_finalized\(end, attempt\)", SRC)) >= 2 assert 'if peaks_for_range(end, attempt):' not in SRC, \ "a reap still uses peaks as a proxy for the collector being done" + + +def test_a_ceiling_hit_survives_a_fresh_start(): + # An OOM peak is evidence about the range whichever pass produced it: the + # process really did allocate that much and want more. Dropping it breaks + # the self-correcting loop -- a range that OOMs at L must record L so that + # L * margin + headroom clears it next run. Measured on ssc-30: an OOM in + # replay resumes and stays in the chain (224 of 252), an OOM in download + # does not (25 of 252), and a higher-cpu run is download-bound. + f = _peaks_ns({ + 1: ({'peakAnonBytes': 8 * 1024**3}, {'outcome': 'oom'}), + 2: ({'peakAnonBytes': 900 * 1024**2}, None), # fresh start + }) + assert f(999, 2)['peakAnonBytes'] == 8 * 1024**3 + + +def test_a_disk_ceiling_hit_survives_a_fresh_start_too(): + f = _peaks_ns({ + 1: ({'peakEphemeralBytes': 40 * 1024**3}, {'outcome': 'ephemeral'}), + 2: ({'peakEphemeralBytes': 9 * 1024**3}, None), + }) + assert f(999, 2)['peakEphemeralBytes'] == 40 * 1024**3 + + +def test_the_ceiling_exception_is_peaks_only(): + # tx_apply and seconds are summed, and a fresh start redoes the work the + # dropped attempt already did, so counting it there would double-count. + for fn_name in ('tx_apply_for_range', 'seconds_for_range'): + fn = _extract(r"^(def " + fn_name + r"\(.*?)(?=\ndef )").group(1) + assert '_resumed_chain(end, attempt)' in fn, f"{fn_name} lost the chain" + assert '_peak_attempts' not in fn, f"{fn_name} would double-count redone work" + peaks = _extract(r"^(def peaks_for_range\(.*?)(?=\ndef )").group(1) + assert '_peak_attempts(end, attempt)' in peaks From 16d9067312b190003ca97a9c25d4cb3ccb21761e Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Wed, 29 Jul 2026 23:47:03 -0400 Subject: [PATCH 014/117] Put the attempt number on the worker pod, not just the Job The collector reads LABEL_ATTEMPT off the pod to decide which range--a.* files an attempt owns, and defaults to "1". The label was set only on the Job, so every attempt claimed attempt 1's files. Measured on the live 2096-worker run: 2246 metrics files, all a1, while 475 a2 pods were running. Each retry merged its peak over the first attempt's instead of being maxed against it, so a range that OOMed at a1 and succeeded at a2 recorded only a2's smaller peak -- destroying exactly the ceiling-hit evidence the chain aggregation was built to preserve. peaks_for_range(end, 2) also found nothing, and those Jobs were never reaped because .done was written under the wrong name. 167 tests. Nothing in the suite asserted the pod carried the label. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 16 +++++++++-- .../test_job_monitor.py | 28 +++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index d5eb065b..d16a8ab5 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1356,8 +1356,18 @@ def volume_spread_constraints(): label_selector=client.V1LabelSelector(match_labels={LABEL_RUN: RUN_NAME}))] -def pod_labels(end): - labels = {LABEL_RUN: RUN_NAME, LABEL_RANGE: str(end)} +def pod_labels(end, attempt): + """Labels on the worker POD, which are not the Job's. + + LABEL_ATTEMPT has to be here as well: the collector reads it off the pod to + decide which range--a.* files this attempt owns, and its default is + "1". Measured on ssc-test 2026-07-30 -- with the label only on the Job, all + 2246 metrics files were a1 while 475 a2 pods were running, so every retry + overwrote the first attempt's peaks instead of being maxed against them, + peaks_for_range(end, 2) found nothing, and those Jobs were never reaped. + """ + labels = {LABEL_RUN: RUN_NAME, LABEL_RANGE: str(end), + LABEL_ATTEMPT: str(attempt)} if EMIT_MISSION_LABEL and MISSION: labels['mission'] = MISSION return labels @@ -1417,7 +1427,7 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): ttl_seconds_after_finished=JOB_TTL_SECONDS, active_deadline_seconds=ATTEMPT_DEADLINE_SECONDS or None, template=client.V1PodTemplateSpec( - metadata=client.V1ObjectMeta(labels=pod_labels(end)), + metadata=client.V1ObjectMeta(labels=pod_labels(end, attempt)), spec=client.V1PodSpec( # IRSA for the S3 history mirror. Without it workers fall # back to the public archive, which throttles at 1024. diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index a33caa53..90a54964 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -2070,3 +2070,31 @@ def test_the_ceiling_exception_is_peaks_only(): assert '_peak_attempts' not in fn, f"{fn_name} would double-count redone work" peaks = _extract(r"^(def peaks_for_range\(.*?)(?=\ndef )").group(1) assert '_peak_attempts(end, attempt)' in peaks + + +def test_the_worker_pod_carries_its_attempt_number(): + # The collector reads LABEL_ATTEMPT off the POD, not the Job, and defaults + # to "1". With the label only on the Job every attempt claimed the same + # range--a1.* files: measured on ssc-test 2026-07-30, 2246 metrics + # files all a1 while 475 a2 pods ran, so each retry overwrote the first + # attempt's peak instead of being maxed against it -- destroying exactly + # the OOM evidence the chain exists to keep. + fn = _extract(r"^(def pod_labels\(.*?)(?=\ndef )").group(1) + # The dict itself, not the docstring -- which names LABEL_ATTEMPT while + # explaining why it must be there, and made an earlier version of this + # assertion pass against a pod_labels that had dropped it. + body = fn[fn.index('labels = {'):] + assert re.search(r"LABEL_ATTEMPT: str\(attempt\)", body), \ + "the pod template omits the attempt label" + assert re.match(r"def pod_labels\(end, attempt\)", fn), \ + "pod_labels does not take the attempt" + assert 'metadata=client.V1ObjectMeta(labels=pod_labels(end, attempt))' in SRC + # and the collector's default is what makes the omission silent + assert re.search(r"labels\.get\(LABEL_ATTEMPT, '1'\)", COLLECTOR_SRC), \ + "collector no longer defaults the attempt -- update this test" + + +def test_pod_and_job_agree_on_the_attempt_label_key(): + # Two readers, one key. A mismatch reproduces the same silent collision. + assert _extract(r"LABEL_ATTEMPT = '([^']+)'").group(1) == \ + _extract(r"LABEL_ATTEMPT = '([^']+)'", COLLECTOR_SRC).group(1) From 38ad01c31cf18f5629cb70b72c9ada6b7c321dd8 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 00:01:11 -0400 Subject: [PATCH 015/117] Never cpu-throttle a worker; escalate memory per OOM, not per attempt Both found on the live 2096-worker spot run. The cpu limit was only removed inside `if overrides:`, which silently excluded two populations, because _profile_overrides returns {} for an unmeasured range AND for an escalated attempt. Measured: 214 a1 and 256 a2 pods capped at cpu 2 while their peers ran uncapped. For the retries that meant more memory and less cpu at the same time, immediately after an OOM -- and less cpu means less download concurrency means a lower peak, so the retry records a figure an unthrottled run cannot reproduce. The cap is now gone for every worker unless PROFILE_CPU_LIMIT is set explicitly; packing is driven by the request, which is unchanged. Memory escalation keyed on the attempt index, so evictions climbed the ladder. On spot they dominate -- 288 disruption retries against 7 OOM retries -- and a range disrupted three times then OOMing once jumped to base * 1.5^4, a 5x request for a single OOM. Escalation now counts OOM outcomes among prior attempts. 169 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 39 +++++++++++--- .../test_job_monitor.py | 54 +++++++++++++++++-- 2 files changed, 83 insertions(+), 10 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index d16a8ab5..b6cc3504 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -782,8 +782,21 @@ def read_outcome(end, attempt): return None +def _oom_count(end, attempt): + """How many earlier attempts at this range were OOM-killed. + + Escalation must climb once per OOM, not once per attempt. On spot most + retries are evictions -- measured on ssc-test 2026-07-30, 288 disruption + retries against 7 OOM retries -- and a range disrupted three times then + OOMing once would otherwise jump to base * 1.5^4, a 5x request for a single + OOM. That inflation is fleet-wide and it is what exhausts the vCPU quota. + """ + return sum(1 for n in range(1, int(attempt) + 1) + if (read_outcome(end, n) or {}).get('outcome') == 'oom') + + def mem_for_attempt(attempt, base=None): - """Memory limit for attempt N, escalating after an OOM, capped at MEM_ESCALATION_CAP. + """Memory limit after N OOMs, capped at MEM_ESCALATION_CAP. `base` is what attempt 1 actually ran with. It matters when a profile sized the range: escalating a 209Mi profiled range off the configured 24000Mi @@ -1318,6 +1331,22 @@ def _resources(mem=None, eph=None, end=None): if LIM_EPHEMERAL: lim['ephemeral-storage'] = eph or LIM_EPHEMERAL + # No cpu limit on any worker unless one is configured explicitly. Packing is + # driven by the request; a limit only throttles a pod that could otherwise + # use idle cores, and throttling changes what the range measures -- less cpu + # means less download concurrency means a lower peak, so a throttled attempt + # records a figure an unthrottled one cannot reproduce. + # + # This used to be applied only when _profile_overrides returned something, + # which silently excluded two populations: unmeasured ranges, and escalated + # retries (escalated returns {} as well). Measured on ssc-test 2026-07-30, + # 214 a1 and 256 a2 pods were capped at cpu 2 while their peers ran free -- + # and for the retries that meant more memory and less cpu at the same time, + # right after an OOM. + if PROFILE_CPU_LIMIT: + lim['cpu'] = PROFILE_CPU_LIMIT + else: + lim.pop('cpu', None) if overrides: # Memory and disk match request to limit: those are the dimensions worth # pinning, since exceeding either kills the pod outright. @@ -1327,10 +1356,6 @@ def _resources(mem=None, eph=None, end=None): # packs by what it actually uses while keeping headroom to burst. That # leaves the pod Burstable rather than Guaranteed -- Kubernetes needs # all three to match -- which is the intended trade. - if PROFILE_CPU_LIMIT: - lim['cpu'] = PROFILE_CPU_LIMIT - else: - lim.pop('cpu', None) for key, value in overrides.items(): req[key] = lim[key] = value # Unmeasured range: the configured defaults, requests below limits, exactly @@ -1651,7 +1676,9 @@ def reconcile(state): reason = "lost to node disruption" elif verdict['outcome'] == 'oom': base = (_profile_overrides(end, escalated=False) or {}).get('memory') - retry_mem = mem_for_attempt(attempt + 1, base) + # Rungs climbed = OOMs seen, not attempts made. This attempt's + # own outcome is already on disk, so the count includes it. + retry_mem = mem_for_attempt(_oom_count(end, attempt) + 1, base) reason = f"OOM-killed at memory limit {mem_for_attempt(attempt, base)}" elif verdict['outcome'] == 'ephemeral': retry_eph = eph_for_attempt(attempt + 1) diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index 90a54964..c467ed5a 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -551,8 +551,10 @@ def test_an_unmeasured_range_keeps_the_mismatched_defaults(): ns = _resources_ns(PROFILE_RANGES) r = ns['_resources'](end=99999) assert r.requests['memory'] == '9Gi' and r.limits['memory'] == '24000Mi' - assert r.requests['cpu'] == '1800m' and r.limits['cpu'] == '2', \ - "an unprofiled range must keep the configured cpu limit" + # cpu is the exception: no worker is throttled, measured or not. A limit + # only stops a pod using idle cores, and it changes what the range measures. + assert r.requests['cpu'] == '1800m' + assert 'cpu' not in r.limits, "an unprofiled range must not be throttled" assert r.requests['ephemeral-storage'] == '35Gi' assert r.limits['ephemeral-storage'] == '40Gi' assert r.requests != r.limits @@ -566,7 +568,7 @@ def test_an_escalated_retry_keeps_the_mismatched_defaults(): assert r.requests['cpu'] == '1800m', "cpu must fall back to the configured request" -def test_the_raised_cpu_limit_is_what_lets_the_peak_grow(): +def test_no_range_is_cpu_throttled_but_every_range_has_a_request(): # At a 2-core limit every range pegs 2.0, so the measured peak is a ceiling # and the profile can never learn real demand. Headroom above the request is # the whole point -- the request is still capped for packing. @@ -574,7 +576,9 @@ def test_the_raised_cpu_limit_is_what_lets_the_peak_grow(): measured = ns['_resources'](end=2000) unmeasured = ns['_resources'](end=99999) assert 'cpu' not in measured.limits, "measured ranges run uncapped" - assert unmeasured.limits['cpu'] == '2', "unprofiled keeps the configured cap" + assert 'cpu' not in unmeasured.limits, "unmeasured ranges run uncapped too" + # The request is still what bounds packing, on both paths. + assert measured.requests['cpu'] and unmeasured.requests['cpu'] assert ns['_cpu_millis'](measured.requests['cpu']) <= ns['_cpu_millis']('1800m') @@ -2098,3 +2102,45 @@ def test_pod_and_job_agree_on_the_attempt_label_key(): # Two readers, one key. A mismatch reproduces the same silent collision. assert _extract(r"LABEL_ATTEMPT = '([^']+)'").group(1) == \ _extract(r"LABEL_ATTEMPT = '([^']+)'", COLLECTOR_SRC).group(1) + + +def test_no_worker_gets_a_cpu_limit_unless_one_is_configured(): + # _profile_overrides returns {} for BOTH "no profile entry" and "escalated + # attempt". Treating them the same handed an OOM retry more memory while + # capping it at LIM_CPU, when the attempt that just failed ran unlimited. + # Measured on ssc-test 2026-07-30: 256 of 679 a2 pods were capped at cpu 2. + # Less cpu means less download concurrency means a lower peak, so the retry + # succeeds at a figure the next run cannot reproduce unthrottled. + ns = _resources_ns(PROFILE_RANGES) + first = ns['_resources'](end=2000) + retry = ns['_resources'](mem='9000Mi', end=2000) # escalated + assert 'cpu' not in first.limits, first.limits + assert 'cpu' not in retry.limits, f"escalated retry was throttled: {retry.limits}" + assert retry.requests['memory'] == retry.limits['memory'] == '9000Mi' + # ...and an unmeasured range is not throttled either. A limit only stops a + # pod using cores that are otherwise idle, and it changes what the range + # measures. Packing is driven by the request. + plain = ns['_resources'](end=999999999) + assert 'cpu' not in plain.limits, f"unprofiled range was throttled: {plain.limits}" + assert plain.requests.get('cpu') is not None, "the cpu request must remain" + + +def test_escalation_counts_ooms_not_attempts(): + # On spot most retries are evictions: 288 disruption retries against 7 OOM + # retries on ssc-test 2026-07-30. Keying the exponent on the attempt index + # meant a range disrupted three times then OOMing once jumped to + # base * 1.5^4 -- a 5x request for one OOM, inflated fleet-wide. + import tempfile, json as _json, os as _os + d = tempfile.mkdtemp() + ns = {'os': _os, 'json': _json, + 'outcome_path': lambda e, n: _os.path.join(d, f"o-{n}")} + for name in ('read_outcome', '_oom_count'): + exec(_extract(r"^(def " + name + r"\(.*?)(?=\ndef )").group(1), ns) + for n, outcome in ((1, 'disrupted'), (2, 'disrupted'), (3, 'disrupted'), (4, 'oom')): + _json.dump({'outcome': outcome}, open(ns['outcome_path'](9, n), 'w')) + assert ns['_oom_count'](9, 4) == 1, "three evictions were counted as escalations" + for n in (5, 6): + _json.dump({'outcome': 'oom'}, open(ns['outcome_path'](9, n), 'w')) + assert ns['_oom_count'](9, 6) == 3 + body = _extract(r"(base = \(_profile_overrides\(end, escalated=False\).*?retry_mem = [^\n]+)").group(1) + assert '_oom_count(end, attempt) + 1' in body, "escalation still keys on the attempt index" From 83e9b617b12e93a0ff9ebca990714fd94a4f9df5 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 00:59:21 -0400 Subject: [PATCH 016/117] Treat a range already at its target as done, not as work to redo Root cause of the 2096-worker run aborting at 61% complete. Range 16752063: attempt 1 replayed to the target ledger and was evicted before it could exit 0, so the Job never reported success. Attempt 2 read LCL == TARGET, and the resume guard accepted it (`-le "$TARGET"`), so it skipped new-db and ran catchup against a database with nothing left to apply. stellar-core applied 0 transactions and exited 2 -- identically on every retry. The range burned its whole budget, the monitor reported one failed job, and the mission aborts the run on any failure. The work had actually been done. We discarded ~1500 ranges of profile data because we could not recognise a completed range. The script now exits 0 when LCL >= TARGET, and the resume branch is narrowed to a strict `-lt`. Also adds an executable test of all three resume decisions -- already-complete, partial, never-started -- driving the real script against a stubbed stellar-core. Separately: RESUME_SCRIPT is %-formatted at dispatch, so a bare % anywhere in it, including a comment, raises and kills every job dispatch. A comment reading "61%-complete" nearly shipped exactly that; there is now a test that formats the script and rejects stray percent signs. 173 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 34 ++++++++- .../test_job_monitor.py | 76 +++++++++++++++++++ 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index b6cc3504..b7f94dd2 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1073,11 +1073,37 @@ def _tx_apply_for_attempt(end, attempt=1, pod_name=None): RESUME=false LCL="" if [ -f "$MARK" ] && [ "$(cat "$MARK" 2>/dev/null)" = "$KEY" ]; then - PREV_LOG=$(ls -t /data/stellar-core*.log 2>/dev/null | head -n 1 || true) - if [ -n "$PREV_LOG" ]; then - LCL=$(grep -oE "Ledger close complete: [0-9]+" "$PREV_LOG" 2>/dev/null | tail -1 | grep -oE "[0-9]+$" || true) + # Ask core for its own LCL. It reads storestate.lastclosedledgerheader through + # its own accessor, so this survives both a schema change (v27 dropped + # ledgerheaders, which is what silently disabled resume before) and any log + # level above INFO. Safe here specifically: core has not started, so nothing + # holds /data/buckets/stellar-core.lock. Core logs to the console alongside + # the JSON, hence grepping rather than parsing. + LCL=$(/usr/bin/stellar-core --conf /config/stellar-core.cfg offline-info --console 2>/dev/null \ + | tr -d ' ' | grep -A8 '"ledger":' | grep -oE '"num":[0-9]+' | head -1 \ + | grep -oE '[0-9]+$' || true) + if [ -n "$LCL" ]; then + echo "RESUME PROBE: offline-info reports lcl $LCL" + else + # Fallback: the previous incarnation's log on /data. Goes blind above INFO, + # which is why it is no longer the primary probe. + PREV_LOG=$(ls -t /data/stellar-core*.log 2>/dev/null | head -n 1 || true) + if [ -n "$PREV_LOG" ]; then + LCL=$(grep -oE "Ledger close complete: [0-9]+" "$PREV_LOG" 2>/dev/null | tail -1 | grep -oE "[0-9]+$" || true) + fi + echo "RESUME PROBE: offline-info gave nothing; log fallback says '${LCL:-none}'" + fi + # Already at the target: a1 finished the replay and was evicted before it + # could exit 0. Re-running catchup here applies nothing and stellar-core exits + # 2, identically on every retry, so the range burns its whole budget and the + # mission aborts the run over work that was actually done. Measured on + # ssc-test 2026-07-30: range 16752063 killed a 2096-worker run that was 61 + # percent complete, exactly this way. + if [ -n "$LCL" ] && [ "$LCL" -ge "$TARGET" ] 2>/dev/null; then + echo "ALREADY COMPLETE: $KEY reached ledger $LCL >= target $TARGET; nothing left to replay" + exit 0 fi - if [ -n "$LCL" ] && [ "$LCL" -ge $((TARGET - COUNT)) ] && [ "$LCL" -le "$TARGET" ] 2>/dev/null; then + if [ -n "$LCL" ] && [ "$LCL" -ge $((TARGET - COUNT)) ] && [ "$LCL" -lt "$TARGET" ] 2>/dev/null; then RESUME=true; echo "RESUME: $KEY reached ledger $LCL, replay had started; skipping new-db" else echo "RESUME DECLINED: $KEY last close was '${LCL:-none}' (need >= $((TARGET - COUNT))); bucket phase incomplete, starting fresh" diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index c467ed5a..df73e005 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -2144,3 +2144,79 @@ def test_escalation_counts_ooms_not_attempts(): assert ns['_oom_count'](9, 6) == 3 body = _extract(r"(base = \(_profile_overrides\(end, escalated=False\).*?retry_mem = [^\n]+)").group(1) assert '_oom_count(end, attempt) + 1' in body, "escalation still keys on the attempt index" + + +# --- an already-finished range must not be retried ------------------------- +# Measured on ssc-test 2026-07-30: a1 replayed range 16752063 to its target +# ledger and was evicted before it could exit 0. a2 resumed, found LCL == +# TARGET, ran catchup against a DB with nothing left to apply, and stellar-core +# exited 2 -- deterministically, every attempt. The range exhausted its budget +# and the mission aborted a 61%-complete 2096-worker run over work that had +# actually been done. + +def _run_resume_script(lcl, target=16752063, count=16320, mark_matches=True): + """Execute RESUME_SCRIPT's decision logic with a stubbed core.""" + import subprocess, tempfile, os as _os, re as _re + src = _extract(r"RESUME_SCRIPT = r'''(.*?)'''").group(1) + src = src % {'key': f"{target}/{count}", 'target': target, 'count': count} + d = tempfile.mkdtemp() + bindir = _os.path.join(d, 'bin'); _os.makedirs(bindir) + # stub stellar-core: report `lcl` to offline-info, log what else is invoked + stub = _os.path.join(bindir, 'stellar-core') + with open(stub, 'w') as fh: + fh.write('#!/bin/sh\n' + 'for a in "$@"; do case "$a" in\n' + ' offline-info) ' + + (f'echo \'{{"info":{{"ledger":{{"num":{lcl},"hash":"x"}}}}}}\'; ' if lcl else 'echo "{}"; ') + + 'exit 0;;\n' + ' new-db) echo "RAN:new-db" >> "$STUBLOG"; exit 0;;\n' + ' catchup) echo "RAN:catchup" >> "$STUBLOG"; exit 2;;\n' + 'esac; done\nexit 0\n') + _os.chmod(stub, 0o755) + src = src.replace('/usr/bin/stellar-core', stub) + _os.makedirs(_os.path.join(d, 'data'), exist_ok=True) + src = src.replace('/data/', _os.path.join(d, 'data') + '/') + src = src.replace('MARK=' + _os.path.join(d, 'data') + '/.job-key', + 'MARK=' + _os.path.join(d, 'data') + '/.job-key') + mark = _os.path.join(d, 'data', '.job-key') + if mark_matches: + open(mark, 'w').write(f"{target}/{count}") + stublog = _os.path.join(d, 'stub.log') + env = dict(_os.environ, STUBLOG=stublog) + r = subprocess.run(['/bin/sh', '-c', src], capture_output=True, text=True, env=env, timeout=30) + ran = open(stublog).read().split() if _os.path.exists(stublog) else [] + return r.returncode, r.stdout, ran + + +def test_a_range_already_at_its_target_exits_success_without_recatching(): + code, out, ran = _run_resume_script(lcl=16752063) + assert 'ALREADY COMPLETE' in out, out + assert code == 0, f"exit {code}; a finished range must not fail" + assert 'RAN:catchup' not in ran, "re-ran catchup on a completed range -> exit 2" + assert 'RAN:new-db' not in ran, "wiped a completed range" + + +def test_a_partially_replayed_range_still_resumes(): + code, out, ran = _run_resume_script(lcl=16752063 - 100) + assert 'RESUME:' in out and 'ALREADY COMPLETE' not in out, out + assert 'RAN:catchup' in ran and 'RAN:new-db' not in ran, ran + + +def test_a_range_that_never_started_replay_starts_fresh(): + code, out, ran = _run_resume_script(lcl=None) + assert 'RESUME DECLINED' in out, out + assert 'RAN:new-db' in ran and 'RAN:catchup' in ran, ran + + +def test_the_resume_script_survives_its_own_percent_formatting(): + # RESUME_SCRIPT is %-formatted with the range's key/target/count at dispatch. + # A bare % anywhere in it -- including in a comment -- raises at runtime and + # takes down every job dispatch. Nearly shipped exactly that: a comment + # reading "61%-complete". + src = _extract(r"RESUME_SCRIPT = r'''(.*?)'''").group(1) + src % {'key': '123/456', 'target': 123, 'count': 456} # must not raise + # %% is a legitimate escape (printf '%%s'), so strip those pairs before + # looking for a stray one. + probe = src.replace('%%', '') + stray = [m.start() for m in re.finditer(r"%(?!\()", probe)] + assert not stray, f"bare % in RESUME_SCRIPT near {probe[max(0,stray[0]-40):stray[0]+20]!r}" From c17296c1082d9f0b284cb95cc54c1ac170fedc7a Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 01:26:28 -0400 Subject: [PATCH 017/117] Fix the offline-info LCL extraction; it was silently reading nothing The probe used `grep -A8 '"ledger":'`, but offline-info puts ~40 lines of bucketlist hashes between that key and "num", so the window never reached it. LCL came back empty every time and the probe fell through to the log grep it was meant to replace -- with no error, so the change read as working while doing nothing. Verified against 27.1.1 on ssc-test 2026-07-30: exactly one "num" key in the document and it is the ledger's. A plain sed over the whole output reads it. Confirmed end to end with the exact line that ships: empty on a volume with no DB, "1" after new-db. Adds a test rejecting any line-windowed grep in the probe. 174 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 8 ++++++-- src/MissionParallelCatchup/test_job_monitor.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index b7f94dd2..cb3d6ffa 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1079,9 +1079,13 @@ def _tx_apply_for_attempt(end, attempt=1, pod_name=None): # level above INFO. Safe here specifically: core has not started, so nothing # holds /data/buckets/stellar-core.lock. Core logs to the console alongside # the JSON, hence grepping rather than parsing. + # One "num" key in the whole document and it is the ledger's -- verified + # against 27.1.1 output on ssc-test 2026-07-30. Do NOT window this with + # `grep -A '"ledger":'`: bucketlist puts ~40 lines of hashes between the + # key and "num", so a small window silently yields nothing and the probe + # degrades to the log fallback without saying so. LCL=$(/usr/bin/stellar-core --conf /config/stellar-core.cfg offline-info --console 2>/dev/null \ - | tr -d ' ' | grep -A8 '"ledger":' | grep -oE '"num":[0-9]+' | head -1 \ - | grep -oE '[0-9]+$' || true) + | sed -n 's/.*"num"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/p' | head -1 || true) if [ -n "$LCL" ]; then echo "RESUME PROBE: offline-info reports lcl $LCL" else diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index df73e005..631e14b9 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -2220,3 +2220,17 @@ def test_the_resume_script_survives_its_own_percent_formatting(): probe = src.replace('%%', '') stray = [m.start() for m in re.finditer(r"%(?!\()", probe)] assert not stray, f"bare % in RESUME_SCRIPT near {probe[max(0,stray[0]-40):stray[0]+20]!r}" + + +def test_the_lcl_probe_does_not_window_its_grep(): + # offline-info puts ~40 lines of bucketlist hashes between "ledger": and + # "num", so `grep -A8 '"ledger":'` yields nothing and the probe degrades to + # the log fallback silently -- shipped exactly that once. Verified against + # 27.1.1 on ssc-test 2026-07-30: exactly one "num" key in the document, and + # it is the ledger's (genesis reads 1). + src = _extract(r"RESUME_SCRIPT = r'''(.*?)'''").group(1) + probe = src[src.index('offline-info'):src.index('if [ -n "$LCL" ]')] + assert not re.search(r"grep\s+-A\d+", probe), \ + "a line-windowed grep cannot reach \"num\" past the bucketlist" + assert '"num"' in probe, "the probe no longer reads the ledger num" + assert 'head -1' in probe, "unbounded match could pick up a later key" From 07469cf5937de628ac200b430603b8b4c9a1500c Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 01:46:25 -0400 Subject: [PATCH 018/117] Do not report a duration for a pod the collector never watched run `started` measures how long this poller has been watching, not how long the container ran. A pod already terminal when poll_pod first executes -- it finished while the collector was down, or between pod-list polls -- finalizes on the first pass and records ~0 seconds beside a real memory peak. Measured on the recovered artifacts from the 2096-worker run, across two collector restarts: of 3237 metrics files, 140 recorded under 1s and 150 recorded under 5s alongside an anon peak above 500 MiB, which no real attempt produces. Now reports nothing in that case. The monitor's own figure, taken from the pod's terminated timestamps, is authoritative and seconds_for_range already prefers it; the collector value exists only as a fallback for earlier legs of a resumed chain, where a fabricated near-zero silently shrinks the sum. The profile's `seconds` was not affected -- the winning attempt always used the pod timestamps (verified: min 128s across 2437 completed ranges). 175 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/log_collector.py | 13 +++++++++++++ src/MissionParallelCatchup/test_job_monitor.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index ffa9c9f5..305cc85c 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -562,8 +562,21 @@ async def poll_pod(session, pod, end, attempt, done, done_ok): backoff = LOG_POLL_SECONDS failures = 0 + first_pass = True while True: was_terminal = done(pod) + if first_pass and was_terminal: + # The pod was already terminal before this poller existed -- it + # finished while the collector was down, or between pod-list polls. + # `started` measures how long WE have been watching, which is about + # to be zero, not how long the container ran. Measured on ssc-test + # 2026-07-30 across two collector restarts: 150 metrics files + # recorded a sub-5s duration alongside a >500MiB anon peak. Report + # nothing rather than a fabricated near-zero; the monitor's own + # figure, from the pod's terminated timestamps, is authoritative and + # seconds_for_range prefers it anyway. + started = None + first_pass = False try: last_ts, gone = await _poll_once(session, pod, end, attempt, last_ts, tx) backoff = LOG_POLL_SECONDS diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index 631e14b9..c20eb695 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -2234,3 +2234,16 @@ def test_the_lcl_probe_does_not_window_its_grep(): "a line-windowed grep cannot reach \"num\" past the bucketlist" assert '"num"' in probe, "the probe no longer reads the ledger num" assert 'head -1' in probe, "unbounded match could pick up a later key" + + +def test_a_pod_already_finished_reports_no_duration(): + # `started` measures how long the COLLECTOR has watched, not how long the + # container ran. A pod that was already terminal when its poller began -- + # finished while the collector was down, which happened across two restarts + # on ssc-test 2026-07-30 -- would otherwise record ~0s next to a real peak: + # 150 metrics files had a sub-5s duration with a >500MiB anon peak. + poller = _extract(r"^(async def poll_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert 'first_pass' in poller, "nothing distinguishes the first poll" + assert re.search(r"if first_pass and was_terminal:\s*\n(?:\s*#[^\n]*\n)*\s*started = None", poller), \ + "an already-finished pod still reports a fabricated duration" + assert poller.index('was_terminal = done(pod)') < poller.index('first_pass = False') From c15689ff43f60fd7de247480dca7a1ea3da323cc Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 01:56:58 -0400 Subject: [PATCH 019/117] Never let a peak on disk be lowered by a later write write_metrics merged with `{**on_disk, **new}`, so the newer value won every collision. That is wrong for a monotonic quantity: after a collector restart the fresh poller's high-water starts at zero, and its first in-flight flush replaced the higher pre-restart peak with a lower one. The flush exists to survive a restart, and the merge then undid it. Lowering a peak undersizes the range on the next run, which is the single direction that costs an OOM. Peaks now take the max on merge; every other field still takes the newest value. Found while asking why 1141 of 4213 recovered metrics files carried an anon peak but no working set -- only anon is flushed mid-flight, so it is the only one exposed to this. 176 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/log_collector.py | 23 ++++++++++---- .../test_job_monitor.py | 30 ++++++++++++++++++- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 305cc85c..1610593c 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -97,6 +97,9 @@ # That window is the only thing standing between a spot reclaim and the last # lines the container wrote. _wake = {} +# Fields that only ever grow. write_metrics maxes these instead of overwriting, +# so a restarted poller starting its high-water at zero cannot lower one. +PEAK_KEYS = ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes') # Failed polls tolerated after a pod goes terminal before we stop asking. Its # log is not coming back, and spinning on it holds a task and a poll slot for # the rest of the run; a couple of retries still absorb a transient 500. @@ -239,14 +242,24 @@ def write_metrics(end, attempt, values): """ path = base(end, attempt) + '.metrics' tmp = path + '.tmp' - # Merge: a measurement already on disk must survive a later write that - # lacks it. The ephemeral peak is held in memory, so a collector restart - # would otherwise let a rewrite drop it. + # Merge, and never let a peak go backwards. A measurement already on disk + # must survive a later write that lacks it -- the peaks are held in memory, + # so a collector restart would otherwise drop them. But a plain overwrite is + # wrong for a monotonic quantity: after a restart the fresh poller starts + # its high-water at zero, and its first flush would replace the higher + # pre-restart value with a lower one. Lowering a peak undersizes the range + # next run, which is the one direction that costs an OOM. try: with open(path) as fh: - values = {**json.load(fh), **values} + prior = json.load(fh) except (OSError, ValueError): - pass + prior = {} + merged = {**prior, **values} + for k in PEAK_KEYS: + a, b = prior.get(k), values.get(k) + if a is not None and b is not None: + merged[k] = max(a, b) + values = merged try: with open(tmp, 'w') as fh: json.dump(values, fh) diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index c20eb695..2e2ee8f9 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -359,7 +359,9 @@ def test_metrics_writes_merge_so_a_rewrite_cannot_drop_a_measurement(): # If a later write clobbered the file, the peak already persisted would be # lost -- which is exactly what happened before this merge. fn = _extract(r"def write_metrics\(.*?(?=\ndef )", COLLECTOR_SRC).group(0) - assert '{**json.load(fh), **values}' in fn, "existing fields must survive" + assert '{**prior, **values}' in fn, "existing fields must survive" + # ...and peaks additionally take the max, see the monotonicity test below. + assert 'PEAK_KEYS' in fn def test_the_ephemeral_sampler_runs_every_poll_not_once_per_stream(): @@ -2247,3 +2249,29 @@ def test_a_pod_already_finished_reports_no_duration(): assert re.search(r"if first_pass and was_terminal:\s*\n(?:\s*#[^\n]*\n)*\s*started = None", poller), \ "an already-finished pod still reports a fabricated duration" assert poller.index('was_terminal = done(pod)') < poller.index('first_pass = False') + + +def test_a_peak_on_disk_is_never_lowered_by_a_later_write(): + # Peaks are monotonic, but the merge overwrote. After a collector restart + # the fresh poller's high-water starts at zero, so its first flush would + # replace a higher pre-restart value with a lower one -- undersizing the + # range next run, the one direction that costs an OOM. + import tempfile, os as _os, json as _json + d = tempfile.mkdtemp() + ns = {'json': _json, 'os': _os, + 'base': lambda e, a: _os.path.join(d, f"r{e}-a{a}"), + 'PEAK_KEYS': ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes'), + 'logger': type('L', (), {'info': lambda s, *a: None, + 'warning': lambda s, *a: None})()} + exec(_extract(r"^(def write_metrics\(.*?)(?=\ndef )", COLLECTOR_SRC).group(1), ns) + w = ns['write_metrics'] + w(1, 1, {'peakAnonBytes': 3000, 'txApplySeconds': 12.0}) + w(1, 1, {'peakAnonBytes': 900}) # restarted poller, lower + got = _json.load(open(ns['base'](1, 1) + '.metrics')) + assert got['peakAnonBytes'] == 3000, f"peak was lowered to {got['peakAnonBytes']}" + assert got['txApplySeconds'] == 12.0, "an unrelated field was dropped" + w(1, 1, {'peakAnonBytes': 5000}) # a genuinely higher peak + assert _json.load(open(ns['base'](1, 1) + '.metrics'))['peakAnonBytes'] == 5000 + # non-peak fields still take the newest value + w(1, 1, {'txApplySeconds': 99.0}) + assert _json.load(open(ns['base'](1, 1) + '.metrics'))['txApplySeconds'] == 99.0 From 2b475eebc38217da4f37cad315b2e998ba32d31c Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 02:08:54 -0400 Subject: [PATCH 020/117] Read a pod's own start->finish instead of how long the poller watched attemptSeconds came from `asyncio.get_event_loop().time()` sampled when poll_pod started, which measures how long THIS POLLER existed, not how long the container ran. For a pod already terminal on the first pass -- finished while the collector was down -- that is ~0. Measured across two collector restarts: 150 of 3237 metrics files recorded under 5s beside an anon peak above 500 MiB. The previous commit suppressed the bad value. That was the lazy fix: a terminal pod still carries startTime and terminated.finishedAt until it is deleted, which is the same source the monitor uses, and the collector's main loop already has the whole pod object and was discarding everything but `phase`. It now records the pod's own duration while the pod still exists, and finalize prefers that. The poller's elapsed time remains only as a fallback for a pod that vanished before its timestamps could be read. 178 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/log_collector.py | 41 ++++++++++++++++- .../test_job_monitor.py | 44 ++++++++++++++++--- 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 1610593c..49c4b193 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -33,6 +33,7 @@ import re import ssl import sys +from datetime import datetime import aiohttp @@ -97,6 +98,8 @@ # That window is the only thing standing between a spot reclaim and the last # lines the container wrote. _wake = {} +# pod name -> its own start->finish, read off the pod while it still exists. +_pod_secs = {} # Fields that only ever grow. write_metrics maxes these instead of overwriting, # so a restarted poller starting its high-water at zero cannot lower one. PEAK_KEYS = ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes') @@ -137,6 +140,30 @@ def base(end, attempt): return os.path.join(LOG_DIR, f"range-{end}-a{attempt}") +def pod_seconds(pod): + """Container start -> finish from the pod's own status, or None. + + The same fields the monitor reads. A terminal pod still carries them until + it is deleted, so this works even when the collector never watched the + container run -- which the poller's own elapsed time cannot. + """ + st = pod.get('status') or {} + start = st.get('startTime') + if not start: + return None + for cs in (st.get('containerStatuses') or []): + term = (cs.get('state') or {}).get('terminated') or {} + fin = term.get('finishedAt') + if fin: + try: + a = datetime.strptime(start, '%Y-%m-%dT%H:%M:%SZ') + b = datetime.strptime(fin, '%Y-%m-%dT%H:%M:%SZ') + except ValueError: + return None + return (b - a).total_seconds() + return None + + def done_path(end, attempt): return base(end, attempt) + '.done' @@ -435,7 +462,11 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): """ # Before discard: on success the archive is about to be deleted. measured = {} - if started is not None: + observed = _pod_secs.pop(pod, None) + if observed is not None: + # The pod's own timestamps, not how long this poller happened to watch. + measured['attemptSeconds'] = round(observed, 1) + elif started is not None: # Fallback only: the monitor's figure comes from the pod's terminated # timestamps and is preferred when it exists. measured['attemptSeconds'] = round( @@ -716,6 +747,14 @@ async def main(): continue phase = pod.get('status', {}).get('phase') terminal[name] = phase in ('Succeeded', 'Failed') + if terminal[name]: + # Recorded while the pod object still exists. Beats the + # poller's own elapsed time, which only measures how long + # WE watched -- ~0 for a pod that finished before this + # poller started. + secs = pod_seconds(pod) + if secs is not None: + _pod_secs[name] = secs if terminal[name] and name in _wake: # Wake its poller now rather than at the next tick. _wake[name].set() diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index 2e2ee8f9..37c643cd 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -1333,7 +1333,8 @@ def test_finalize_records_the_working_set_peak(): written, ws = [], {'pod-1': 4096} ns = { '_anon_peak': {'pod-1': 900}, '_ws_peak': ws, '_eph_peak': {}, - '_peak_flushed': {}, '_streaming': {}, '_wake': {}, 'SAVE_SUCCESS_LOGS': True, + '_peak_flushed': {}, '_streaming': {}, '_wake': {}, '_pod_secs': {}, + 'SAVE_SUCCESS_LOGS': True, 'write_metrics': lambda e, a, v: written.append(v), 'discard': lambda e, a: None, '_mark_done': lambda e, a: None, 'logger': type('L', (), {'info': lambda s, *a: None})(), @@ -1507,7 +1508,8 @@ def test_finalize_records_that_an_attempt_resumed(): def run(resumed): written = [] ns = {'_anon_peak': {'p': 1}, '_ws_peak': {}, '_eph_peak': {}, - '_peak_flushed': {}, '_streaming': {}, '_wake': {}, 'SAVE_SUCCESS_LOGS': True, + '_peak_flushed': {}, '_streaming': {}, '_wake': {}, '_pod_secs': {}, + 'SAVE_SUCCESS_LOGS': True, 'write_metrics': lambda e, a, v: written.append(v), 'discard': lambda e, a: None, '_mark_done': lambda e, a: None, 'logger': type('L', (), {'info': lambda s, *a: None})()} @@ -1746,7 +1748,8 @@ def test_the_collector_records_a_duration_the_monitor_cannot(): import asyncio written = [] ns = {'asyncio': asyncio, '_anon_peak': {}, '_ws_peak': {}, '_eph_peak': {}, - '_peak_flushed': {}, '_streaming': {}, '_wake': {}, 'SAVE_SUCCESS_LOGS': True, + '_peak_flushed': {}, '_streaming': {}, '_wake': {}, '_pod_secs': {}, + 'SAVE_SUCCESS_LOGS': True, 'write_metrics': lambda e, a, v: written.append(v), 'discard': lambda e, a: None, '_mark_done': lambda e, a: None, 'logger': type('L', (), {'info': lambda s, *a: None})()} @@ -2009,8 +2012,14 @@ def test_a_terminal_pod_wakes_its_poller_immediately(): COLLECTOR_SRC).group(1) # Structure, not just presence: mutating the guard to `if False:` leaves # the .set() line in place and sails past a substring check. - assert re.search(r"terminal\[name\] = phase in [^\n]*\n\s*if terminal\[name\][^\n]*:\s*\n(?:\s*#[^\n]*\n)*\s*_wake\[name\]\.set\(\)", loop), \ - "a pod going terminal does not wake its poller" + # The duration capture now sits between the assignment and the wake, so + # check ordering and the guard rather than adjacency. + i_assign = loop.index('terminal[name] = phase in') + i_guard = loop.index('if terminal[name] and name in _wake:') + # search AFTER the guard: the vanished-pod block also calls .set() and sits + # earlier in the loop, so a bare index() finds the wrong one. + i_set = loop.index('_wake[name].set()', i_guard) + assert i_assign < i_guard < i_set, "a pod going terminal does not wake its poller" poller = _extract(r"^(async def poll_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) assert 'asyncio.wait_for(' in poller and '_wake.setdefault' in poller, \ "the poller still sleeps blind between polls" @@ -2275,3 +2284,28 @@ def test_a_peak_on_disk_is_never_lowered_by_a_later_write(): # non-peak fields still take the newest value w(1, 1, {'txApplySeconds': 99.0}) assert _json.load(open(ns['base'](1, 1) + '.metrics'))['txApplySeconds'] == 99.0 + + +def test_a_terminal_pod_still_yields_its_real_duration(): + # A pod carries startTime and terminated.finishedAt until it is deleted, so + # even a pod that finished before this poller existed has a real duration. + # The poller's own elapsed time cannot know that -- it measures how long WE + # watched, which is ~0 in exactly that case. + ns = {'datetime': __import__('datetime').datetime} + exec(_extract(r"^(def pod_seconds\(.*?)(?=\ndef )", COLLECTOR_SRC).group(1), ns) + pod = {'status': {'startTime': '2026-07-30T04:16:26Z', + 'containerStatuses': [{'state': {'terminated': { + 'finishedAt': '2026-07-30T04:22:19Z'}}}]}} + assert ns['pod_seconds'](pod) == 353.0, ns['pod_seconds'](pod) + assert ns['pod_seconds']({'status': {'startTime': '2026-07-30T04:16:26Z'}}) is None + assert ns['pod_seconds']({'status': {}}) is None + + +def test_the_pods_own_duration_wins_over_the_pollers_elapsed_time(): + fn = _extract(r"^(async def finalize\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) + assert '_pod_secs.pop(pod, None)' in fn + assert fn.index('observed = _pod_secs.pop') < fn.index('elif started is not None:'), \ + "the poller's elapsed time must only be the fallback" + loop = _extract(r"while True:\n(.*?)await asyncio\.sleep\(POLL_SECONDS\)", + COLLECTOR_SRC).group(1) + assert 'pod_seconds(pod)' in loop, "nothing captures it while the pod exists" From 4e08897e5330f5a7de49041e7a8b762728a1db17 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 02:20:47 -0400 Subject: [PATCH 021/117] Raise the OOM retry budget to 10 MAX_ATTEMPTS is effectively the OOM budget: `failed` is the only other outcome that reaches it, and that one sets no retry reason. At 5 it condemned a range before the escalation ladder had run its course, and a condemned range aborts the entire run -- which is what ended a 61%-complete 2096-worker run tonight. With escalation now counting OOMs rather than attempts, rung N means the range genuinely wanted more N times, so spending rungs is evidence rather than churn. MEM_ESCALATION_CAP (48Gi) is the real ceiling: a 1.6GiB base reaches it on OOM 10, a 4.2GiB base on OOM 8. So the practical effect is that a hungry range escalates until the cap instead of dying at 5. Adds a test tying the budget to the ladder: the two must stay consistent, or the budget silently becomes the binding limit again. 179 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 8 +++++++- .../parallel_catchup_helm/values.yaml | 2 +- .../test_job_monitor.py | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index cb3d6ffa..4c87da65 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -191,7 +191,13 @@ # 3. This monitor's own behaviour # ============================================================================= PARALLELISM = int(os.getenv('PARALLELISM', 3)) -MAX_ATTEMPTS_PER_RANGE = int(os.getenv('MAX_ATTEMPTS', 5)) +# Effectively the OOM budget: `failed` is the only other outcome that reaches +# it, and that one sets no retry reason. Escalation now counts OOMs rather than +# attempts, so rung N means the range genuinely wanted more N times. 10 rungs is +# 1.5^9 = 38x the profile figure, which MEM_ESCALATION_CAP bounds well before +# then -- so the real effect is that a range keeps trying until the cap, instead +# of being condemned at 5. A condemned range aborts the whole run. +MAX_ATTEMPTS_PER_RANGE = int(os.getenv('MAX_ATTEMPTS', 10)) # A hang gets far fewer retries than an eviction. The measured causes -- an # unreachable archive host, an absent checkpoint, a bucket that will not # decompress -- are persistent, so retrying mostly burns another full deadline. diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 5b5d2d67..946dd5f6 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -98,7 +98,7 @@ monitor: # once the panels aggregate. emitMissionLabel: false loggingIntervalSeconds: 10 - maxAttempts: 5 + maxAttempts: 10 # Hangs are usually persistent (bad archive host, absent checkpoint), so they # get a lower cap than evictions -- otherwise a wedged range costs # maxAttempts x attemptDeadlineSeconds before it is reported. diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index 37c643cd..f9c76d14 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -2309,3 +2309,22 @@ def test_the_pods_own_duration_wins_over_the_pollers_elapsed_time(): loop = _extract(r"while True:\n(.*?)await asyncio\.sleep\(POLL_SECONDS\)", COLLECTOR_SRC).group(1) assert 'pod_seconds(pod)' in loop, "nothing captures it while the pod exists" + + +def test_the_oom_budget_outlives_the_escalation_ladder(): + # MAX_ATTEMPTS is effectively the OOM budget -- `failed` is the only other + # outcome reaching it and that one sets no retry reason. It must be large + # enough that a range reaches MEM_ESCALATION_CAP before being condemned, + # because a condemned range aborts the entire run. + n = int(_extract(r"MAX_ATTEMPTS_PER_RANGE = int\(os\.getenv\('MAX_ATTEMPTS', (\d+)\)\)").group(1)) + bump = float(_extract(r"MEM_BUMP_FACTOR = float\(os\.getenv\('MEM_BUMP_FACTOR', ([\d.]+)\)\)").group(1)) + cap_s = _extract(r"MEM_ESCALATION_CAP = os\.getenv\('MAX_MEM', '(\d+)Gi'\)").group(1) + cap_mi = int(cap_s) * 1024 + # the largest profile-derived request we have seen (p90 ~4.2GiB, max ~9.2GiB) + reached = 9200 * (bump ** (n - 1)) + assert reached >= cap_mi, \ + f"{n} attempts only reaches {reached:.0f}Mi, short of the {cap_mi}Mi cap" + assert n > int(_extract(r"MAX_TIMEOUT_ATTEMPTS = int\(os\.getenv\('MAX_TIMEOUT_ATTEMPTS', (\d+)\)\)").group(1)) + chart = open(__file__.replace( + 'test_job_monitor.py', 'parallel_catchup_helm/values.yaml')).read() + assert int(_extract(r"maxAttempts: (\d+)", chart).group(1)) == n From fca8180e5e6f7e50a3c0bbd82e63d6fa01613659 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 02:22:22 -0400 Subject: [PATCH 022/117] Keep the OOM budget at 5, and say why Reverts the raise to 10. 5 rungs is 1.5^4 = 5x the profile figure; a range needing more than that is broken rather than mis-sized, and chasing it to MEM_ESCALATION_CAP parks a whole r8a.2xlarge on one range for hours. The price is that such a range is condemned, and today a condemned range aborts the entire run. That coupling -- not this number -- is the thing to fix. Raising the budget only pushes the same failure further out. Test now pins the budget to a sane band and asserts the ladder can at least treble the request before giving up, rather than requiring it to reach the cap. 179 tests. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 15 +++++++++------ .../parallel_catchup_helm/values.yaml | 2 +- src/MissionParallelCatchup/test_job_monitor.py | 18 +++++++----------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 4c87da65..cf4567e2 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -192,12 +192,15 @@ # ============================================================================= PARALLELISM = int(os.getenv('PARALLELISM', 3)) # Effectively the OOM budget: `failed` is the only other outcome that reaches -# it, and that one sets no retry reason. Escalation now counts OOMs rather than -# attempts, so rung N means the range genuinely wanted more N times. 10 rungs is -# 1.5^9 = 38x the profile figure, which MEM_ESCALATION_CAP bounds well before -# then -- so the real effect is that a range keeps trying until the cap, instead -# of being condemned at 5. A condemned range aborts the whole run. -MAX_ATTEMPTS_PER_RANGE = int(os.getenv('MAX_ATTEMPTS', 10)) +# it, and that one sets no retry reason. Escalation counts OOMs rather than +# attempts, so rung N means the range genuinely wanted more N times. +# +# Deliberately stops short of MEM_ESCALATION_CAP: 5 rungs is 1.5^4 = 5x the +# profile figure, and a range needing more than that is not mis-sized, it is +# broken -- chasing it to 48Gi parks a whole r8a.2xlarge on one range for hours. +# The cost of stopping is that the range is condemned, and today a condemned +# range aborts the run. That coupling is the thing to fix, not this number. +MAX_ATTEMPTS_PER_RANGE = int(os.getenv('MAX_ATTEMPTS', 5)) # A hang gets far fewer retries than an eviction. The measured causes -- an # unreachable archive host, an absent checkpoint, a bucket that will not # decompress -- are persistent, so retrying mostly burns another full deadline. diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 946dd5f6..5b5d2d67 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -98,7 +98,7 @@ monitor: # once the panels aggregate. emitMissionLabel: false loggingIntervalSeconds: 10 - maxAttempts: 10 + maxAttempts: 5 # Hangs are usually persistent (bad archive host, absent checkpoint), so they # get a lower cap than evictions -- otherwise a wedged range costs # maxAttempts x attemptDeadlineSeconds before it is reported. diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index f9c76d14..0a0f8478 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -2311,19 +2311,15 @@ def test_the_pods_own_duration_wins_over_the_pollers_elapsed_time(): assert 'pod_seconds(pod)' in loop, "nothing captures it while the pod exists" -def test_the_oom_budget_outlives_the_escalation_ladder(): - # MAX_ATTEMPTS is effectively the OOM budget -- `failed` is the only other - # outcome reaching it and that one sets no retry reason. It must be large - # enough that a range reaches MEM_ESCALATION_CAP before being condemned, - # because a condemned range aborts the entire run. +def test_the_oom_budget_stops_short_of_the_cap_on_purpose(): + # 5 rungs is 1.5^4 = 5x the profile figure. A range needing more is broken, + # not mis-sized, and chasing it to MEM_ESCALATION_CAP parks a whole node on + # it. The price is that such a range is condemned -- which today aborts the + # run, so the coupling below is what must not be forgotten. n = int(_extract(r"MAX_ATTEMPTS_PER_RANGE = int\(os\.getenv\('MAX_ATTEMPTS', (\d+)\)\)").group(1)) bump = float(_extract(r"MEM_BUMP_FACTOR = float\(os\.getenv\('MEM_BUMP_FACTOR', ([\d.]+)\)\)").group(1)) - cap_s = _extract(r"MEM_ESCALATION_CAP = os\.getenv\('MAX_MEM', '(\d+)Gi'\)").group(1) - cap_mi = int(cap_s) * 1024 - # the largest profile-derived request we have seen (p90 ~4.2GiB, max ~9.2GiB) - reached = 9200 * (bump ** (n - 1)) - assert reached >= cap_mi, \ - f"{n} attempts only reaches {reached:.0f}Mi, short of the {cap_mi}Mi cap" + assert 2 <= n <= 8, f"{n} rungs: below 2 cannot escalate, above 8 chases a broken range" + assert bump ** (n - 1) >= 3.0, "the ladder cannot even treble the request before giving up" assert n > int(_extract(r"MAX_TIMEOUT_ATTEMPTS = int\(os\.getenv\('MAX_TIMEOUT_ATTEMPTS', (\d+)\)\)").group(1)) chart = open(__file__.replace( 'test_job_monitor.py', 'parallel_catchup_helm/values.yaml')).read() From 945efdcee57c81a0f5b2121f90b91dcf8ac1c5a3 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 03:21:48 -0400 Subject: [PATCH 023/117] Never condemn a range on exit 3; it means "interrupted" as often as "broken" Found by a sandbox edge-case suite run against the monitor while the main run was in flight. Three of eight scenarios failed on one root cause. stellar-core catches SIGTERM, drains, and exits 3 in ~7s. A corrupt archive also exits 3. Nothing in the exit code separates them -- only a DisruptionTarget condition does, and that is gone the moment the pod is. classify() mapped exit 3 to outcome `failed`, which is the ZERO-retry outcome, so every graceful kill condemned its range, and one condemned range aborts the mission. Measured in sandbox: a pod deleted mid-replay, a pod deleted mid-download, and an attempt-deadline kill were all condemned at attempt 1. The resume logic verified earlier tonight was unreachable through every one of them. Four fixes: - exit 3 now retries on the ordinary range budget. A genuinely corrupt range still exhausts MAX_ATTEMPTS and fails with evidence; an interrupted one succeeds, usually by resuming at LCL+1. It deliberately does NOT join ENVIRONMENTAL_OUTCOMES, which would give it 20 attempts. - A DeadlineExceeded Job condition now outranks the pod-derived verdict. The deadline kills with SIGTERM, so the pod says exit 3 / `failed`, and whichever of the two won the race decided retry vs condemn. - Condemnation is logged loudly. The zero-retry path emitted nothing at all: the range appeared under failed{} and the mission aborted with no line explaining why. - txApply is backfilled like the peaks. progress.json carried txApply=null while range-4000-a1.metrics durably held txApplySeconds -- the same one-shot read race the peaks had, with the same fix. 184 tests, each fix mutation-checked. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 58 ++++++++++++++++++- .../test_job_monitor.py | 57 +++++++++++++++++- 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index cf4567e2..9f12b768 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -217,6 +217,10 @@ EPH_BUMP_FACTOR = float(os.getenv('EPH_BUMP_FACTOR', 1.5)) EPH_ESCALATION_CAP = os.getenv('EPH_ESCALATION_CAP', '200Gi') ENVIRONMENTAL_OUTCOMES = ('disrupted', 'rejected', 'unknown') +# stellar-core's "did not complete". Ambiguous by construction: a corrupt bucket +# and a SIGTERM during replay both produce it, so it must never be treated as +# proof that a range is broken. +CATCHUP_INCOMPLETE_EXIT = 3 # An OOM means requests/limits are mis-sized for this range. Escalate so the run # can finish, but say so loudly -- surviving by escalating at runtime is a # configuration bug, not a success. @@ -1673,7 +1677,9 @@ def reconcile(state): # Job would reap the pod and make that gap permanent. Leave # those to JOB_TTL_SECONDS. _reap_if_complete(end, attempt, completed[end]) - elif not _has_peaks(completed[end]) or not _attempt_finalized(end, attempt): + elif (not _has_peaks(completed[end]) + or completed[end].get('txApply') is None + or not _attempt_finalized(end, attempt)): # Backfill. The record is written the moment the Job flips to # succeeded, which is usually before the collector has finalized # -- and peaks_for_range has no fallback, unlike tx_apply, which @@ -1682,10 +1688,22 @@ def reconcile(state): # .metrics files on the same volume held it. Retry while the Job # is still here; delete_job below is what ends the chances. late = peaks_for_range(end, attempt) + if completed[end].get('txApply') is None: + # Same one-shot race as the peaks, and the same fix. The + # collector writes txApplySeconds into .metrics when it + # finalizes, which can land after reconcile recorded the + # range. Measured in the sandbox edge suite 2026-07-30: + # progress.json carried txApply=null while the durable + # .metrics file held txApplySeconds=0.000486848. + late_tx = tx_apply_for_range(end, attempt) + if late_tx is not None: + late = dict(late or {}) + late['txApply'] = late_tx if late: completed[end].update(late) save_progress(progress) - logger.info("range %s: peaks arrived late, backfilled", end) + logger.info("range %s: measurements arrived late, backfilled %s", + end, sorted(late)) _reap_if_complete(end, attempt, completed[end]) elif st.failed: pod = job_pods.get(j.metadata.name) @@ -1698,6 +1716,16 @@ def reconcile(state): # 2. Job condition -- survives node consolidation, less precise # 3. unknown -- retry rather than condemn the run verdict = read_outcome(end, attempt) or classify_from_job(j) + # ...with one exception. A deadline kill sends SIGTERM, stellar-core + # drains and exits 3, and the pod-derived verdict therefore reads + # `failed` -- which outranks the Job's DeadlineExceeded and condemns + # a range that merely ran long. Only the Job knows the deadline + # fired, so on that condition the Job wins. Measured in the sandbox + # edge suite 2026-07-30: whichever of the two won the race decided + # whether the range was retried or condemned. + from_job = classify_from_job(j) + if from_job and from_job.get('outcome') == 'timeout': + verdict = from_job if verdict is None: verdict = {'outcome': 'unknown', 'exitCode': None} elif verdict.get('source') == 'job-condition': @@ -1731,6 +1759,25 @@ def reconcile(state): # 10-hour job. Retry; a genuinely broken range will exhaust its # attempts and fail with evidence. reason = "failed with no surviving classification (monitor restart?)" + elif verdict.get('exitCode') == CATCHUP_INCOMPLETE_EXIT: + # Exit 3 means "did not complete" and covers BOTH a corrupt + # archive AND any interruption -- stellar-core catches SIGTERM, + # drains and exits 3 in ~7s. Nothing in the exit code separates + # them; only a DisruptionTarget condition does, and that is gone + # the moment the pod is. + # + # Condemning on it made every graceful kill fatal, and a + # condemned range aborts the whole mission. Measured in the + # sandbox edge suite 2026-07-30: a pod deleted mid-replay, a pod + # deleted mid-download, and an attempt-deadline kill were all + # classified `failed` at attempt 1 and never retried -- the + # resume path was unreachable through any of them. + # + # Retry on the ordinary range budget. A genuinely broken range + # exhausts MAX_ATTEMPTS and fails with evidence; an interrupted + # one succeeds, usually by resuming at LCL+1. + reason = (f"exited {CATCHUP_INCOMPLETE_EXIT} (did not complete -- " + "corrupt archive or interruption, indistinguishable)") else: reason = None # genuine catchup failure: do not retry @@ -1791,6 +1838,13 @@ def reconcile(state): continue if reason is not None: logger.error("range %s exhausted %d attempts (%s)", end, cap, reason) + else: + # The zero-retry path used to log nothing at all: the range just + # appeared under failed{} and the mission aborted with no line + # explaining why. Say it plainly. + logger.error("!!! RANGE CONDEMNED !!! %s failed with outcome=%s exitCode=%s " + "on attempt %d and is NOT retryable; this fails the mission", + end, verdict['outcome'], verdict.get('exitCode'), attempt) if end not in failed: failed[end] = {'attempts': attempt, diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index 0a0f8478..ec79f165 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -1941,7 +1941,7 @@ def test_late_peaks_are_backfilled_into_a_completed_record(): # tx_apply does, so a one-shot read loses them: measured on ssc-test, 356 of # 356 completed ranges had txApply and 0 had peakAnonBytes, while 1936 # .metrics files on the same volume held it. - body = _extract(r"(elif not _has_peaks\(completed\[end\]\).*?)(?=\n\s+elif st\.failed:)").group(1) + body = _extract(r"(elif \(not _has_peaks\(completed\[end\]\).*?)(?=\n\s+elif st\.failed:)").group(1) assert 'peaks_for_range(end, attempt)' in body, "no retry of the peak read" assert 'save_progress(progress)' in body, "a backfilled peak is never persisted" assert '_reap_if_complete' in body, "backfill never lets the Job go" @@ -2324,3 +2324,58 @@ def test_the_oom_budget_stops_short_of_the_cap_on_purpose(): chart = open(__file__.replace( 'test_job_monitor.py', 'parallel_catchup_helm/values.yaml')).read() assert int(_extract(r"maxAttempts: (\d+)", chart).group(1)) == n + + +# --- exit 3 is ambiguous and must never condemn a range -------------------- +# stellar-core catches SIGTERM, drains and exits 3 in ~7s, and a corrupt bucket +# also exits 3. Nothing in the exit code separates them. Measured in the sandbox +# edge suite 2026-07-30: a pod killed mid-replay, a pod killed mid-download, and +# an attempt-deadline kill were ALL classified `failed` at attempt 1 and never +# retried -- so the resume path was unreachable through every disruption that +# leaves no DisruptionTarget behind, and one such range aborts the mission. + +def test_exit_three_is_retried_not_condemned(): + body = _extract(r"(if verdict\['outcome'\] == 'timeout':.*?reason = None[^\n]*)").group(1) + assert "verdict.get('exitCode') == CATCHUP_INCOMPLETE_EXIT" in body, \ + "exit 3 still falls through to the zero-retry branch" + assert body.index('CATCHUP_INCOMPLETE_EXIT') < body.index('reason = None'), \ + "the exit-3 branch must precede the condemn branch" + assert int(_extract(r"CATCHUP_INCOMPLETE_EXIT = (\d+)").group(1)) == 3 + + +def test_exit_three_uses_the_ordinary_range_budget(): + # Not the environmental budget: a genuinely corrupt range must still be able + # to exhaust and fail with evidence rather than retry 20 times. + env = set(re.findall(r"'(\w+)'", _extract(r"ENVIRONMENTAL_OUTCOMES = \(([^)]+)\)").group(1))) + assert 'failed' not in env, "exit 3 would inherit the 20-attempt disruption budget" + + +def test_a_deadline_kill_is_not_read_as_a_catchup_failure(): + # The deadline sends SIGTERM -> exit 3 -> pod verdict says `failed`, which + # outranks the Job's DeadlineExceeded. Only the Job knows the deadline fired. + body = _extract(r"(verdict = read_outcome\(end, attempt\) or classify_from_job\(j\).*?)(?=\n\s+if verdict is None)").group(1) + assert "from_job.get('outcome') == 'timeout'" in body, \ + "a deadline kill can still be condemned as a catchup failure" + assert 'verdict = from_job' in body + + +def test_a_condemned_range_is_logged_loudly(): + # The zero-retry path logged nothing: the range appeared under failed{} and + # the mission aborted with no line saying why. + body = _extract(r"(if reason is not None:\s*\n\s*logger\.error\(\"range %s exhausted.*?)(?=\n\s+if end not in failed)").group(1) + assert 'RANGE CONDEMNED' in body, "condemnation is still silent" + assert 'else:' in body + + +def test_a_late_tx_apply_is_backfilled_like_the_peaks(): + # Same one-shot race the peaks had. The collector writes txApplySeconds when + # it finalizes, which can land after reconcile recorded the range. Measured + # in the sandbox edge suite 2026-07-30: progress.json held txApply=null while + # range-4000-a1.metrics durably held txApplySeconds=0.000486848, and the + # monitor even logged "could not read tx_apply for range 4000 (pod gone?)". + guard = _extract(r"(elif \(not _has_peaks\(completed\[end\]\).*?)(?=\n\s+late = peaks_for_range)").group(1) + assert "completed[end].get('txApply') is None" in guard, \ + "a range with peaks but no txApply never re-enters the backfill" + body = _extract(r"(late = peaks_for_range\(end, attempt\).*?)(?=\n\s+_reap_if_complete)").group(1) + assert 'tx_apply_for_range(end, attempt)' in body, "txApply is never re-read" + assert "late['txApply'] = late_tx" in body From dcbce73cdd4cd8469fe534f13eb1d2eb5491a3c9 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 06:42:23 -0400 Subject: [PATCH 024/117] Project peakAnonBytes and wallSeconds into the mission's profile artifact rangeProfileFields predates peakAnonBytes, so the mission's writeRangeProfile silently stripped the one field the sizing consumer prefers. Measured on the 2026-07-30 full run: the artifact carried 0% peakAnonBytes and 0% wallSeconds while the monitor's own progress.json carried both at 99-100%. The real profile had to be recovered from the worker-logs tar, twice in one night. Co-Authored-By: Claude Opus 5 --- src/FSLibrary.Tests/Tests.fs | 18 ++++++++++++++++++ .../MissionHistoryPubnetParallelCatchupV2.fs | 9 +++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index 437dc56e..f279c101 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -573,6 +573,24 @@ let ``range profile keeps only the measurements that exist`` () = Assert.Equal(9999L, withEph.["peakEphemeralBytes"].Value()) +[] +let ``range profile carries the fields the sizing consumer prefers`` () = + // peakAnonBytes is what _profile_overrides reads FIRST (kubelet-sampled + // anon); peakRssBytes is only its fallback. Omitting it from the + // projection silently stripped it from the mission artifact while the + // monitor's progress.json carried it for 99% of ranges -- measured + // 2026-07-30, artifact 0% vs volume 99%. wallSeconds likewise. + Assert.Contains("peakAnonBytes", rangeProfileFields) + Assert.Contains("wallSeconds", rangeProfileFields) + + let record = JObject() + record.["peakAnonBytes"] <- JValue(111L) + record.["wallSeconds"] <- JValue(50.0) + let entry = projectRangeEntry record + Assert.Equal(111L, entry.["peakAnonBytes"].Value()) + Assert.Equal(50.0, entry.["wallSeconds"].Value()) + + [] let ``range profile keeps count as a field so it can be keyed on end alone`` () = // Measured: 4.2x the ledgers per range moved peak disk -1.6% and wall time diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 6fde4ca6..b96dccdf 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -431,8 +431,13 @@ let queryJobMonitor (context: MissionContext, key: String) = // profiling it buys no packing. peakEphemeralBytes appears only for // ephemeral-mode runs, and only for ranges that finished. let rangeProfileFields = - [ "peakRssBytes"; "peakWorkingSetBytes"; "peakCpuCores"; "peakEphemeralBytes" - "seconds"; "txApply" ] + // peakAnonBytes is the field the sizing consumer prefers (kubelet-sampled + // anon; peakRssBytes is the coarser Prometheus-era name for the same + // quantity). Omitting it here silently stripped it from the mission's + // profile artifact while the monitor's own progress.json carried it -- + // measured 2026-07-30: artifact 0% peakAnonBytes, volume copy 99%. + [ "peakAnonBytes"; "peakRssBytes"; "peakWorkingSetBytes"; "peakCpuCores" + "peakEphemeralBytes"; "seconds"; "wallSeconds"; "txApply" ] // A missing measurement must stay missing rather than become a null: the // consumer falls back to its configured default when the field is absent. From 7038644a3608892917bf816336181d7d8bbe2fd4 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 08:42:01 -0400 Subject: [PATCH 025/117] Drain the run before failing on a condemned range The status loop threw at the first failed range, which abandons every range still in flight. The ranges alive at that moment are the expensive tip ones this mission exists to measure. Measured 2026-07-30: one condemned range at 97% completion discarded 123 ranges of completed and in-flight work and left a hole in the profile. Failures are now recorded once (deduped -- the same range reappears on every poll), logged loudly as they happen, and reported after the run drains. The mission still fails; it just finishes the work it can first. dumpLogs is guarded because after a full drain the failed pod is almost certainly reaped. Co-Authored-By: Claude Opus 5 --- .../MissionHistoryPubnetParallelCatchupV2.fs | 45 ++++++++++++++----- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index b96dccdf..ec12480d 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -694,6 +694,15 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = let mutable timeoutLeft = jobMonitorStatusCheckTimeOutSecs let mutable timeBeforeNextMetricsCheck = jobMonitorMetricsCheckIntervalSecs + // Failures are reported once the run drains, not at first sight. Aborting on + // the first condemned range abandons every range still in flight, and the + // ranges that survive to the end of a run are the expensive tip ones this + // mission exists to measure. Measured 2026-07-30: one condemned range at 97% + // discarded 123 ranges of completed and in-flight work. The mission still + // fails -- it just finishes the work it can first. + let failedJobs = ResizeArray() + let seenFailures = System.Collections.Generic.HashSet() + while not allJobsFinished do Thread.Sleep(jobMonitorStatusCheckIntervalSecs * 1000) let statusOpt = queryJobMonitor (context, jobMonitorStatusKey) @@ -706,18 +715,12 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = let jobsFailed = status.["jobs_failed"] :?> JArray let JobsInProgress = status.["jobs_in_progress"] :?> JArray - if jobsFailed.Count <> 0 then - LogInfo "One or more jobs have failed:" - - for job in jobsFailed do - let ident = job.ToString().Split('|') - let key = ident.[0] - let podName = ident.[1] - LogInfo "%s, logs >>> " (job.ToString()) - dumpLogs (context, podName) - LogInfo "<<<" + for job in jobsFailed do + let text = job.ToString() - failwith "Catch up failed, check logs for more info" + if seenFailures.Add(text) then + failedJobs.Add(text) + LogError "RANGE FAILED: %s -- run continues, mission will fail once it drains" text if remainSize = 0 && JobsInProgress.Count = 0 then // All jobs completed — perform a final query on the metrics @@ -740,4 +743,24 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = cleanup false context raise ex + if failedJobs.Count <> 0 then + LogInfo "%d job(s) failed:" failedJobs.Count + + for job in failedJobs do + let ident = job.Split('|') + LogInfo "%s, logs >>> " job + + // The pod is very likely reaped by now -- draining first means the + // wait is the length of the run. Its log is on the monitor volume + // either way, so a missing pod must not mask the failure below. + if ident.Length > 1 then + try + dumpLogs (context, ident.[1]) + with ex -> LogInfo "could not read pod log (%s); see collected logs" (ex.Message) + + LogInfo "<<<" + + cleanup false context + failwith "Catch up failed, check logs for more info" + cleanup false context From b09e5f6c70ad33ff00d1b0b196f0626cdf66e588 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 09:19:08 -0400 Subject: [PATCH 026/117] A condemned range must not freeze dispatch Dispatch was gated on `not state['halted'] and not failed`, so the first condemned range stopped the monitor sending any further work. Combined with the driver now draining before it reports, that deadlocks the run outright: the mission waits for `remaining == 0 and in_progress == []`, and a frozen dispatch pins `remaining` at however many ranges were never sent, forever. Gate on `halted` alone. `halted` still means what it did -- the durable record went backwards, so nothing can be trusted. A condemned range is an ordinary failure: the mission fails, but only after the work already paid for finishes. Found by an adversarial audit of the reconcile loop, not by a test run. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 9 ++++++++- src/MissionParallelCatchup/test_job_monitor.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 9f12b768..9adb3880 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1866,8 +1866,15 @@ def reconcile(state): state['max_completed'] = max(state['max_completed'], len(completed)) # Dispatch, heaviest range first (index 0 is the tip), up to PARALLELISM. + # + # A condemned range does NOT stop dispatch. It used to, which deadlocked the + # driver: the mission waits for `remaining == 0 and in_progress == []` + # (MissionHistoryPubnetParallelCatchupV2.fs), and a frozen dispatch leaves + # `remaining` pinned at however many ranges were never sent, forever. The + # mission still fails on a condemned range -- it reports once the run drains, + # so the ranges that were already paid for are not thrown away. created = 0 - if not state['halted'] and not failed: + if not state['halted']: # No slots: a range's PVC is keyed by the range itself, so concurrency is # simply how many are in flight. capacity = PARALLELISM - len(in_progress) diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index ec79f165..aa468944 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -2379,3 +2379,17 @@ def test_a_late_tx_apply_is_backfilled_like_the_peaks(): body = _extract(r"(late = peaks_for_range\(end, attempt\).*?)(?=\n\s+_reap_if_complete)").group(1) assert 'tx_apply_for_range(end, attempt)' in body, "txApply is never re-read" assert "late['txApply'] = late_tx" in body + + +def test_a_condemned_range_does_not_freeze_dispatch(): + """A failed range must not stop the run from dispatching the rest. + + The driver waits for `remaining == 0 and in_progress == []` before it + reports. Gating dispatch on `not failed` pinned `remaining` at the number + of never-dispatched ranges, so the mission waited forever instead of + failing -- strictly worse than the abort it replaced. + """ + assert "if not state['halted']:" in SRC, \ + "dispatch must be gated on halted alone" + assert "not state['halted'] and not failed" not in SRC, \ + "a condemned range must not freeze dispatch (driver deadlock)" From eb3c6776052a030482996a9d3d2842cc3f5e06bd Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 10:02:27 -0400 Subject: [PATCH 027/117] Behavioral test suite: a fake Kubernetes cluster, and fixes for 8 races The 185-test suite could not have caught any of these. job_monitor imported a FileHandler under /data and called load_incluster_config() at module scope, so the module was unimportable off-cluster and every test operated on source text -- 151 of 185 regex-extract a function body and exec it, or assert a substring exists. Nothing drove reconcile(). An adversarial audit found eight races in the loop those tests never entered. Import is now side-effect-free (log dir falls back to a temp dir; the in-cluster load is gated on KUBERNETES_SERVICE_HOST, which is the variable that call keys on anyway, so in a pod it is unconditional as before). fake_k8s.py is an in-memory cluster returning real client models and real ApiExceptions with 404/409 semantics; conftest.py drives real reconcile() passes against it. 45 new behavioral tests assert on observed state -- objects created and deleted, progress.json contents, reconcile's return -- never on source text. Each was observed red against the bug and green against the fix. Races fixed: 1 a completed range was re-dispatched; the reap is now range-scoped and the failed branch consults `completed` 2 a backfilled txApply never reached the histogram; `replayed` keys on (end, field) so a late field is still counted once 3 a torn gzip member raised EOFError and aborted the whole reconcile pass; the member is built in a local buffer and appended in one write, and the reader catches EOFError/zlib.error so a bad file costs one range 4 a sticky _wake Event made the terminal-poll backoff dead code and spent the retry budget in milliseconds 5 retry budgets were spent from one shared attempt counter, so spot churn drained the OOM budget and the first real OOM condemned the range 6 activeDeadlineSeconds sat on the JobSpec and charged Pending time; a Job-level timeout also masked the pod's own oom/disrupted verdict 7 regression test for the dispatch freeze fixed in b09e5f6 8 the ConfigMap fallback produced a measurement-free profile that passed the entry.Count > 0 guard, because count was attached before the guard Two pre-existing tests went red as collateral -- both pinned literals the fixes replaced while the invariant they named still held. Rewritten to assert the invariant instead. Co-Authored-By: Claude Opus 5 --- src/FSLibrary.Tests/FSLibrary.Tests.fsproj | 1 + src/FSLibrary.Tests/TestsRace8.fs | 174 ++++++ .../MissionHistoryPubnetParallelCatchupV2.fs | 211 +++++--- src/MissionParallelCatchup/conftest.py | 280 ++++++++++ src/MissionParallelCatchup/fake_k8s.py | 503 ++++++++++++++++++ src/MissionParallelCatchup/job_monitor.py | 208 +++++++- src/MissionParallelCatchup/log_collector.py | 46 +- .../test_harness_smoke.py | 172 ++++++ .../test_job_monitor.py | 16 +- src/MissionParallelCatchup/test_race_1.py | 218 ++++++++ src/MissionParallelCatchup/test_race_2.py | 188 +++++++ src/MissionParallelCatchup/test_race_3.py | 315 +++++++++++ src/MissionParallelCatchup/test_race_4.py | 226 ++++++++ src/MissionParallelCatchup/test_race_5.py | 203 +++++++ src/MissionParallelCatchup/test_race_6.py | 309 +++++++++++ src/MissionParallelCatchup/test_race_7.py | 217 ++++++++ 16 files changed, 3178 insertions(+), 109 deletions(-) create mode 100644 src/FSLibrary.Tests/TestsRace8.fs create mode 100644 src/MissionParallelCatchup/conftest.py create mode 100644 src/MissionParallelCatchup/fake_k8s.py create mode 100644 src/MissionParallelCatchup/test_harness_smoke.py create mode 100644 src/MissionParallelCatchup/test_race_1.py create mode 100644 src/MissionParallelCatchup/test_race_2.py create mode 100644 src/MissionParallelCatchup/test_race_3.py create mode 100644 src/MissionParallelCatchup/test_race_4.py create mode 100644 src/MissionParallelCatchup/test_race_5.py create mode 100644 src/MissionParallelCatchup/test_race_6.py create mode 100644 src/MissionParallelCatchup/test_race_7.py diff --git a/src/FSLibrary.Tests/FSLibrary.Tests.fsproj b/src/FSLibrary.Tests/FSLibrary.Tests.fsproj index e9164c38..b5a66627 100644 --- a/src/FSLibrary.Tests/FSLibrary.Tests.fsproj +++ b/src/FSLibrary.Tests/FSLibrary.Tests.fsproj @@ -10,6 +10,7 @@ + diff --git a/src/FSLibrary.Tests/TestsRace8.fs b/src/FSLibrary.Tests/TestsRace8.fs new file mode 100644 index 00000000..cd0f42e1 --- /dev/null +++ b/src/FSLibrary.Tests/TestsRace8.fs @@ -0,0 +1,174 @@ +// Copyright 2024 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +// RACE #8 -- the ConfigMap fallback yields a measurement-free profile artifact +// that is indistinguishable from a good one. +// +// readProgressRecord prefers /logs/progress.json on the monitor pod, but on ANY +// failure of that exec it silently falls back to the progress ConfigMap. The +// ConfigMap is a state mirror: job_monitor.py's _state_only() strips every +// profiling field out of it on purpose, to stay under the 1 MiB cap. So the +// fallback hands writeRangeProfile a `completed` map in which every record has +// had all eight measurements removed and only bookkeeping (attempts, count) +// left behind. +// +// The `entry.Count > 0` guard exists to skip measurement-free entries, but +// count is attached to the entry BEFORE the guard runs, so every entry has at +// least one field and every entry passes. The result is an artifact with the +// right number of ranges and zero measurements -- observed twice in the field, +// reporting 0% peakAnonBytes while the monitor's own progress.json carried 99%. +// The next run then sizes from a profile that silently has no data. +// +// These tests assert on the values the production projection actually returns, +// never on the text of the source file. +module Race8Tests + +open Xunit +open Newtonsoft.Json.Linq +open MissionHistoryPubnetParallelCatchupV2 + +/// Exactly job_monitor.py's _PROFILE_ONLY_FIELDS -- the fields _state_only() +/// removes when it mirrors progress.json into the capped ConfigMap. +let private profileOnlyFields = + [ "peakAnonBytes"; "peakRssBytes"; "peakWorkingSetBytes"; "peakCpuCores" + "peakEphemeralBytes"; "txApply"; "seconds"; "wallSeconds" ] + +/// A completed record the way /logs/progress.json carries it: bookkeeping plus +/// real measurements. +let private measuredRecord (count: int) (anon: int64) = + let r = JObject() + r.["attempts"] <- JValue(1) + r.["count"] <- JValue(count) + r.["seconds"] <- JValue(120.0) + r.["wallSeconds"] <- JValue(130.0) + r.["txApply"] <- JValue(60.0) + r.["peakAnonBytes"] <- JValue(anon) + r.["peakWorkingSetBytes"] <- JValue(anon + 1000L) + r + +/// The same record as it survives the ConfigMap mirror. +let private configMapMirrored (record: JObject) = + let r = record.DeepClone() :?> JObject + + for f in profileOnlyFields do + r.Remove(f) |> ignore + + r + +let private completedMap (pairs: (string * JObject) list) = + let c = JObject() + + for (k, v) in pairs do + c.[k] <- v + + c + + +[] +let ``a configmap-mirrored range carries no measurement and must not enter the profile`` () = + let mirrored = configMapMirrored (measuredRecord 420 900L) + + // Precondition: the mirror really does strip every measurement, leaving + // only bookkeeping. If this ever stops holding, the rest is meaningless. + for f in profileOnlyFields do + Assert.Null(mirrored.[f]) + + Assert.NotNull(mirrored.["count"]) + Assert.NotNull(mirrored.["attempts"]) + + let ranges = buildRangeProfile (completedMap [ "420", mirrored ]) + + Assert.Equal(0, ranges.Count) + + +[] +let ``count alone never satisfies the measurement guard`` () = + // This is the defeated guard in its smallest form: count is the only field, + // and it is bookkeeping, not a measurement. + let onlyCount = JObject() + onlyCount.["count"] <- JValue(420) + + let ranges = buildRangeProfile (completedMap [ "420", onlyCount ]) + + Assert.Equal(0, ranges.Count) + + +[] +let ``a run whose progress came from the configmap produces no profile artifact`` () = + // The headline symptom: a full-looking artifact, right number of ranges, + // zero measurements. Writing nothing is correct here -- the next run then + // falls back to its configured defaults instead of sizing from empty data. + let completed = + completedMap + [ "420", configMapMirrored (measuredRecord 420 900L) + "840", configMapMirrored (measuredRecord 420 950L) + "1260", configMapMirrored (measuredRecord 420 990L) ] + + match rangeProfileDocument "pvc" 20000 completed with + | None -> () + | Some doc -> + let ranges = doc.["ranges"] :?> JObject + + failwithf + "wrote a profile artifact with %d ranges and zero measurements: %s" + ranges.Count + (ranges.ToString()) + + +[] +let ``the profile counts only ranges that actually measured something`` () = + // A partly-degraded read is the dangerous case: the artifact looks + // populated, so nothing downstream can tell the stripped ranges apart from + // the measured one. + let completed = + completedMap + [ "420", measuredRecord 420 900L + "840", configMapMirrored (measuredRecord 420 950L) + "1260", configMapMirrored (measuredRecord 420 990L) ] + + let ranges = buildRangeProfile completed + + Assert.Equal(1, ranges.Count) + Assert.NotNull(ranges.["420"]) + Assert.Null(ranges.["840"]) + Assert.Null(ranges.["1260"]) + + +[] +let ``a measured run still produces a complete profile`` () = + // Guard against over-correcting: a good read must still write everything. + let completed = + completedMap + [ "420", measuredRecord 420 900L + "840", measuredRecord 420 950L ] + + match rangeProfileDocument "pvc" 20000 completed with + | None -> failwith "refused to write a profile that carries real measurements" + | Some doc -> + let ranges = doc.["ranges"] :?> JObject + Assert.Equal(2, ranges.Count) + Assert.Equal(900L, ranges.["420"].["peakAnonBytes"].Value()) + Assert.Equal(950L, ranges.["840"].["peakAnonBytes"].Value()) + Assert.Equal(60.0, ranges.["420"].["txApply"].Value()) + // count is still carried, and the slicing is inferred from it rather + // than from the caller's default. + Assert.Equal(420, ranges.["420"].["count"].Value()) + Assert.Equal(420, doc.["ledgersPerRange"].Value()) + Assert.Equal("pvc", doc.["storageMode"].Value()) + + +[] +let ``a single real measurement is enough to keep a range and it keeps its count`` () = + // The fix must move count after the guard without dropping it -- count is + // what ledgersPerRange is inferred from. + let r = JObject() + r.["attempts"] <- JValue(2) + r.["count"] <- JValue(420) + r.["wallSeconds"] <- JValue(77.0) + + let ranges = buildRangeProfile (completedMap [ "420", r ]) + + Assert.Equal(1, ranges.Count) + Assert.Equal(77.0, ranges.["420"].["wallSeconds"].Value()) + Assert.Equal(420, ranges.["420"].["count"].Value()) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index ec12480d..0f93b518 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -498,7 +498,113 @@ let readProgressRecord (context: MissionContext) : JObject option = match fromVolume with | Some p -> Some p - | None -> queryJobMonitor (context, jobMonitorProgressKey) + | None -> + // Degraded read, and it must not be silent. The ConfigMap is a state + // mirror: the monitor strips every profiling field out of it to stay + // under the 1 MiB cap, so a record sourced here carries attempts and + // count and nothing else. Any range profile built from it will be + // empty, and rangeProfileDocument will decline to write one. + LogWarn + "Falling back to the progress ConfigMap; it is a state mirror with no measurements, so no range profile can be built from it" + + queryJobMonitor (context, jobMonitorProgressKey) + + +// The `ranges` map of a profile artifact, built from a progress record's +// `completed` map. Pure, so the projection can be exercised without a cluster. +let buildRangeProfile (completed: JObject) : JObject = + let ranges = JObject() + + // Keyed on the range end alone, with count kept as a field. + // + // Measured on ssc-test: 4.2x the ledgers per range (100 -> 420, the + // default 320 overlap) moved peak disk by -1.6% and wall time by + // 1.15x. Cost tracks ledger position -- how big the bucket set is to + // download and apply -- far more than range length. Putting count in + // the key would therefore discard the whole profile whenever + // overlapLedgers or ledgersPerJob changed, to preserve a distinction + // the measurements say is small. + // + // A consumer should resolve a range by exact end, else the nearest + // measured end, else a run-wide fallback -- so a re-sliced run still + // gets useful numbers instead of zero matches. + for prop in completed.Properties() do + let record = prop.Value :?> JObject + let entry = projectRangeEntry record + + // The guard decides on measurements alone. count is bookkeeping, not a + // measurement, so it is attached only after the entry has been found to + // carry something real. Attaching it first made every entry non-empty + // and defeated the guard completely: a ConfigMap-sourced record, which + // has had all eight profiling fields stripped by the monitor's + // _state_only(), still sailed through and produced a range with nothing + // in it but a count. + if entry.Count > 0 then + match record.["count"] with + | null -> () + | v -> entry.["count"] <- v + + // Same end from two differently-sized ranges: keep the + // larger, since sizing from the smaller would under-provision. + let existing = ranges.[prop.Name] + + let keep = + isNull existing + || (let a = entry.["count"] + let b = (existing :?> JObject).["count"] + isNull b || (not (isNull a) && a.Value() >= b.Value())) + + if keep then ranges.[prop.Name] <- entry + + ranges + + +// The profile document to write, or None when there is nothing worth writing. +let rangeProfileDocument + (storageMode: string) + (defaultLedgersPerRange: int) + (completed: JObject) + : JObject option = + let ranges = buildRangeProfile completed + + // Slicing and storage mode both go in the name, because a profile is + // only valid for the shape it was measured at. + // + // Slicing: keys are range ends and cost tracks range length, so a + // 39382-ledger profile fed into a 16320-ledger run resolves through + // nearest-end fallback and sizes everything wrong. Measured on + // ssc-test -- it produced 1025 OOM retries in one run. + // + // Mode: an ephemeral profile carries peakEphemeralBytes and a pvc one + // does not, so crossing them silently defaults the disk axis. + let ledgersPerRange = + let counts = + ranges.Properties() + |> Seq.choose (fun p -> + match (p.Value :?> JObject).["count"] with + | null -> None + | v -> Some(v.Value())) + |> Seq.toList + + match counts with + | [] -> defaultLedgersPerRange + | _ -> counts |> List.countBy id |> List.maxBy snd |> fst + + let doc = JObject() + doc.["schema"] <- JValue(1) + doc.["generated"] <- JValue(DateTime.UtcNow.ToString("o")) + doc.["release"] <- JValue(helmReleaseName) + doc.["storageMode"] <- JValue(storageMode) + doc.["ledgersPerRange"] <- JValue(ledgersPerRange) + doc.["ranges"] <- ranges + + // A profile with no measurements is worse than no profile at all: it looks + // complete, so nothing downstream can tell it from a good one, and the next + // run sizes itself from empty data. The usual cause is readProgressRecord + // falling back to the progress ConfigMap, which is a state mirror with + // every profiling field stripped. Writing nothing lets the consumer fall + // back to its configured defaults, which is the safe outcome. + if ranges.Count = 0 then None else Some doc let writeRangeProfile (context: MissionContext) = @@ -507,85 +613,30 @@ let writeRangeProfile (context: MissionContext) = | Some progress -> try let completed = progress.["completed"] :?> JObject - let ranges = JObject() - - // Keyed on the range end alone, with count kept as a field. - // - // Measured on ssc-test: 4.2x the ledgers per range (100 -> 420, the - // default 320 overlap) moved peak disk by -1.6% and wall time by - // 1.15x. Cost tracks ledger position -- how big the bucket set is to - // download and apply -- far more than range length. Putting count in - // the key would therefore discard the whole profile whenever - // overlapLedgers or ledgersPerJob changed, to preserve a distinction - // the measurements say is small. - // - // A consumer should resolve a range by exact end, else the nearest - // measured end, else a run-wide fallback -- so a re-sliced run still - // gets useful numbers instead of zero matches. - for prop in completed.Properties() do - let record = prop.Value :?> JObject - let entry = projectRangeEntry record - - match record.["count"] with - | null -> () - | v -> entry.["count"] <- v - - if entry.Count > 0 then - // Same end from two differently-sized ranges: keep the - // larger, since sizing from the smaller would under-provision. - let existing = ranges.[prop.Name] - - let keep = - isNull existing - || (let a = entry.["count"] - let b = (existing :?> JObject).["count"] - isNull b || (not (isNull a) && a.Value() >= b.Value())) - - if keep then ranges.[prop.Name] <- entry - - // Slicing and storage mode both go in the name, because a profile is - // only valid for the shape it was measured at. - // - // Slicing: keys are range ends and cost tracks range length, so a - // 39382-ledger profile fed into a 16320-ledger run resolves through - // nearest-end fallback and sizes everything wrong. Measured on - // ssc-test -- it produced 1025 OOM retries in one run. - // - // Mode: an ephemeral profile carries peakEphemeralBytes and a pvc one - // does not, so crossing them silently defaults the disk axis. - let ledgersPerRange = - let counts = - ranges.Properties() - |> Seq.choose (fun p -> - match (p.Value :?> JObject).["count"] with - | null -> None - | v -> Some(v.Value())) - |> Seq.toList - - match counts with - | [] -> context.pubnetParallelCatchupLedgersPerJob - | _ -> counts |> List.countBy id |> List.maxBy snd |> fst - - - let doc = JObject() - doc.["schema"] <- JValue(1) - doc.["generated"] <- JValue(DateTime.UtcNow.ToString("o")) - doc.["release"] <- JValue(helmReleaseName) - doc.["storageMode"] <- JValue(context.pubnetParallelCatchupStorageMode) - doc.["ledgersPerRange"] <- JValue(ledgersPerRange) - doc.["ranges"] <- ranges - - let path = - Path.Combine( - context.destination.Path, - sprintf - "%s-profile-%dledgers-%s.json" - helmReleaseName - ledgersPerRange - context.pubnetParallelCatchupStorageMode) - - File.WriteAllText(path, doc.ToString()) - LogInfo "Wrote range profile for %d ranges to %s" ranges.Count path + + let docOpt = + rangeProfileDocument + context.pubnetParallelCatchupStorageMode + context.pubnetParallelCatchupLedgersPerJob + completed + + match docOpt with + | None -> LogWarn "Progress record carried no measurements; not writing a range profile" + | Some doc -> + let ranges = doc.["ranges"] :?> JObject + let ledgersPerRange = doc.["ledgersPerRange"].Value() + + let path = + Path.Combine( + context.destination.Path, + sprintf + "%s-profile-%dledgers-%s.json" + helmReleaseName + ledgersPerRange + context.pubnetParallelCatchupStorageMode) + + File.WriteAllText(path, doc.ToString()) + LogInfo "Wrote range profile for %d ranges to %s" ranges.Count path with ex -> LogWarn "Failed to write range profile: %s" ex.Message diff --git a/src/MissionParallelCatchup/conftest.py b/src/MissionParallelCatchup/conftest.py new file mode 100644 index 00000000..b1bb858e --- /dev/null +++ b/src/MissionParallelCatchup/conftest.py @@ -0,0 +1,280 @@ +"""Test harness for job_monitor: a fake cluster wired into the real module. + +`cluster` is the fixture. It replaces job_monitor's module-level API clients +with fake_k8s, points every path the monitor writes at tmp_path, and hands back +a driver that runs the real reconcile() -- no source extraction, no mirrors. + + def test_something(cluster): + cluster.reconcile() # one real reconcile pass + cluster.advance(300, 'succeeded') # move a range to a named state + cluster.reconcile() + assert '300' in cluster.progress()['completed'] + +Nothing here weakens the monitor: the only production change this needed was +making import side-effect free (log dir fallback, in-cluster config guarded on +KUBERNETES_SERVICE_HOST). Every decision under test is the shipped code path. +""" + +import json +import os + +import pytest + +import fake_k8s +import job_monitor as jm + +# Config the fixture pins. Small on purpose: three ranges and PARALLELISM 2 so +# dispatch capacity, retry and completion are all observable in a few passes. +DEFAULT_CONFIG = { + 'NAMESPACE': 'catchup-test', + 'RUN_NAME': 'pc', + 'CORE_IMAGE': 'stellar/stellar-core:test', + 'STARTING_LEDGER': 0, + 'LATEST_LEDGER_NUM': 300, + 'LEDGERS_PER_JOB': 100, + 'OVERLAP_LEDGERS': 320, + 'RANGE_GENERATOR': 'uniform', + 'RANGE_ORDER': 'tip-first', + 'PARALLELISM': 2, + 'STORAGE_MODE': 'pvc', + 'STORAGE_SIZE': '40Gi', + 'STORAGE_CLASS': 'gp3', + 'SAVE_SUCCESS_LOGS': True, + 'PROFILE_PATH': '', + 'PROFILE_CPU_LIMIT': '', + 'ATTEMPT_DEADLINE_SECONDS': 0, + 'MAX_ATTEMPTS_PER_RANGE': 5, + 'MAX_TIMEOUT_ATTEMPTS': 2, + 'MAX_DISRUPTION_ATTEMPTS': 20, + 'MAX_EPHEMERAL_ATTEMPTS': 4, + 'LIM_EPHEMERAL': '', + 'REQ_EPHEMERAL': '', +} + +# What advance() does to the fake cluster for each name. The verdict the monitor +# then reaches is its own business -- that is the thing under test. +STATES = ( + 'pending', # dispatched, nothing scheduled yet + 'running', # pod Running, job active + 'succeeded', # exit 0 + 'incomplete', # exit 3: did-not-complete, retryable on the range budget + 'condemned', # exit 1: genuine catchup failure, no retry + 'oom', # exit 137 / OOMKilled + 'disrupted', # DisruptionTarget condition -- spot eviction + 'ephemeral', # kubelet eviction for exceeding the ephemeral-storage limit + 'rejected', # kubelet refused the pod before any container ran + 'timeout', # activeDeadlineSeconds fired + 'unknown', # job failed, pod already reaped, nothing classified it +) + + +class Driver: + """Runs reconcile passes against the fake cluster and inspects the results.""" + + def __init__(self, k8s, tmp_path, config): + self.k8s = k8s + self.jm = jm + self.tmp_path = tmp_path + self.config = config + self.namespace = config['NAMESPACE'] + self.run_name = config['RUN_NAME'] + self.log_dir = jm.LOG_DIR + # Same dict update_status_and_metrics() carries across iterations of the + # loop, so multi-pass tests see the real cross-pass behaviour (halt on + # regression, histogram replay guard, counter deltas). + self.state = {'owner': None, 'replayed': set(), 'max_completed': 0, + 'halted': False, 'counted': {}} + self.results = [] + + # -- driving ------------------------------------------------------------- + + def reconcile(self): + """One real reconcile() pass. Returns the summary dict it produces.""" + if self.state['owner'] is None: + self.state['owner'] = jm.owner_ref() + jm._progress_owner['ref'] = self.state['owner'] + result = jm.reconcile(self.state) + self.results.append(result) + return result + + def advance(self, end, state, attempt=None): + """Move a range's Job/Pod to a named state, as the cluster would. + + `end` is the range end (int or str); attempt defaults to the newest Job + this range has. + """ + if state not in STATES: + raise ValueError(f"unknown state {state!r}; expected one of {STATES}") + name = self.job_name(end, attempt) + pod = self.k8s.pod_for_job(name) + pod_name = pod.metadata.name if pod is not None else None + + if state == 'pending': + return name + if state == 'running': + self.k8s.set_job_running(name) + return name + if state == 'succeeded': + if pod_name: + self.k8s.set_pod_terminated(pod_name, exit_code=0) + self.k8s.set_job_succeeded(name) + return name + + # Everything below is a failure; the Job condition and the pod detail + # are set independently because in a real run either can be missing. + if state == 'incomplete': + if pod_name: + self.k8s.set_pod_terminated(pod_name, exit_code=3) + self.k8s.set_job_failed(name, message=self._policy_msg(pod_name, 3, 2)) + elif state == 'condemned': + if pod_name: + self.k8s.set_pod_terminated(pod_name, exit_code=1) + self.k8s.set_job_failed(name, message=self._policy_msg(pod_name, 1, 2)) + elif state == 'oom': + if pod_name: + self.k8s.set_pod_terminated(pod_name, exit_code=137, reason='OOMKilled') + self.k8s.set_job_failed(name, message=self._policy_msg(pod_name, 137, 1)) + elif state == 'disrupted': + if pod_name: + self.k8s.set_pod_condition(pod_name, 'DisruptionTarget', + reason='TerminationByKubelet') + self.k8s.set_pod_terminated(pod_name, exit_code=3) + self.k8s.set_job_failed(name, message=self._policy_msg(pod_name, None, 0)) + elif state == 'ephemeral': + if pod_name: + # stellar-core drains on the eviction SIGTERM and exits 3, so the + # exit code alone is indistinguishable from a catchup failure; + # status.message is the only discriminator. + self.k8s.set_pod_terminated(pod_name, exit_code=3, phase='Failed') + self.k8s.set_pod_phase( + pod_name, 'Failed', reason='Evicted', + message=('Pod ephemeral local storage usage exceeds the total ' + 'limit of containers 40Gi')) + self.k8s.set_job_failed(name, message=self._policy_msg(pod_name, 3, 2)) + elif state == 'rejected': + if pod_name: + self.k8s.set_pod_phase(pod_name, 'Failed', + reason='VolumeAttachmentLimitExceeded', + message='Node has reached its volume ' + 'attachment limit, rejecting pod') + self.k8s.set_job_failed(name, reason='BackoffLimitExceeded', + message='Job has reached the specified backoff limit') + elif state == 'timeout': + if pod_name: + self.k8s.set_pod_terminated(pod_name, exit_code=3) + self.k8s.set_job_failed(name, reason='DeadlineExceeded', + message='Job was active longer than specified deadline') + elif state == 'unknown': + if pod_name: + self.k8s.delete_pod(pod_name) + self.k8s.set_job_failed(name, reason=None) + return name + + def _policy_msg(self, pod_name, code, rule_index): + """A podFailurePolicy failure message in the Job controller's own format.""" + if code is None: + return (f"Container stellar-core for pod {self.namespace}/{pod_name} " + f"matching FailJob rule at index {rule_index}") + return (f"Container stellar-core for pod {self.namespace}/{pod_name} failed " + f"with exit code {code} matching FailJob rule at index {rule_index}") + + # -- the collector's side of the contract -------------------------------- + + def finalize(self, end, attempt=1, tx_apply=None, peaks=None, resumed=False, + attempt_seconds=None): + """Write what the log-collector sidecar writes for a finished attempt. + + The monitor will not reap a Job until the .done marker exists, and reads + peaks and txApply out of .metrics -- so a test that wants either of those + paths has to stand in for the collector. + """ + data = dict(peaks or {}) + if tx_apply is not None: + data['txApplySeconds'] = tx_apply + if attempt_seconds is not None: + data['attemptSeconds'] = attempt_seconds + if resumed: + data['resumed'] = True + self.write(jm.metrics_path(str(end), attempt), json.dumps(data)) + self.write(jm.done_path(str(end), attempt), '') + + def write(self, path, text): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as fh: + fh.write(text) + return path + + # -- inspection ---------------------------------------------------------- + + def job_name(self, end, attempt=None): + if attempt is not None: + return jm.job_name(int(end), attempt) + prefix = f"{self.run_name}-r{int(end)}-a" + names = [n for n in self.k8s.job_names(self.namespace) if n.startswith(prefix)] + if not names: + raise AssertionError(f"no Job for range {end}; have {self.k8s.job_names()}") + return max(names, key=lambda n: int(n.rsplit('-a', 1)[1])) + + def attempt_of(self, end): + return int(self.job_name(end).rsplit('-a', 1)[1]) + + def jobs(self): + return self.k8s.job_names(self.namespace) + + def pvcs(self): + return self.k8s.pvc_names(self.namespace) + + def progress(self): + """The authoritative progress record, straight off disk.""" + try: + with open(jm.PROGRESS_FILE) as fh: + return json.load(fh) + except (OSError, ValueError): + return {} + + def progress_configmap(self): + """The best-effort ConfigMap mirror the mission driver reads.""" + data = self.k8s.config_map_data(jm.PROGRESS_CM, self.namespace) or {} + return json.loads(data.get('progress.json', '{}')) + + def completed(self): + return self.progress().get('completed', {}) + + def failed(self): + return self.progress().get('failed', {}) + + @property + def calls(self): + return self.k8s.calls + + @property + def deleted(self): + return self.k8s.deleted + + +@pytest.fixture +def cluster(tmp_path, monkeypatch): + config = dict(DEFAULT_CONFIG) + log_dir = tmp_path / 'logs' + log_dir.mkdir() + + k8s = fake_k8s.FakeCluster(namespace=config['NAMESPACE']) + monkeypatch.setattr(jm, 'core_v1', k8s.core_v1) + monkeypatch.setattr(jm, 'batch_v1', k8s.batch_v1) + + for key, value in config.items(): + monkeypatch.setattr(jm, key, value) + # Derived at import from RUN_NAME / LOG_DIR, so they have to follow. + monkeypatch.setattr(jm, 'LOG_DIR', str(log_dir)) + monkeypatch.setattr(jm, 'PROGRESS_FILE', str(log_dir / 'progress.json')) + monkeypatch.setattr(jm, 'PROGRESS_CM', f"{config['RUN_NAME']}-catchup-progress") + # Module-level mutable state that would otherwise leak between tests. + monkeypatch.setattr(jm, 'PROFILE', None) + monkeypatch.setattr(jm, '_progress_owner', {}) + + # The chart's ConfigMap: owner_ref() reads it, and every Job, PVC and the + # progress ConfigMap hang off it. + k8s.add_config_map(f"{config['RUN_NAME']}-stellar-core-config", + {'stellar-core.cfg': '# test'}) + + return Driver(k8s, tmp_path, config) diff --git a/src/MissionParallelCatchup/fake_k8s.py b/src/MissionParallelCatchup/fake_k8s.py new file mode 100644 index 00000000..591f3f00 --- /dev/null +++ b/src/MissionParallelCatchup/fake_k8s.py @@ -0,0 +1,503 @@ +"""An in-memory stand-in for the slice of the Kubernetes API job_monitor uses. + +Not a general-purpose mock. It implements exactly the calls the monitor makes, +with the behaviours the monitor *branches on*: + + * 409 AlreadyExists on a duplicate create -- dispatch and the retry path both + swallow 409 and treat name uniqueness as the mutex, so a fake that silently + overwrote would hide the only thing those handlers do. + * 404 NotFound on a missing read -- load_progress(), ensure_pvc(), + _patch_cm() and read_mission_start() all key on it. + * a Job create materialises a Pod, the way the Job controller does, because + pods_by_job() is how reconcile finds the object that carries the exit code. + +Objects are the real kubernetes.client models, so attribute access in the +monitor (`job.status.succeeded`, `pod.status.container_statuses[0].state +.terminated.exit_code`) is exercised rather than duck-typed around. + +Reads and lists return deep copies: the API server hands out snapshots, and a +test that mutated a listed object would otherwise be writing to the store. +""" + +import copy +from datetime import datetime, timedelta, timezone + +from kubernetes import client +from kubernetes.client.rest import ApiException + +JOB_NAME_LABEL = 'batch.kubernetes.io/job-name' + + +def _now(): + return datetime.now(timezone.utc) + + +def api_exception(status, reason, message=''): + """A real ApiException; the monitor reads .status and .reason off it.""" + e = ApiException(status=status, reason=reason) + e.body = message or reason + return e + + +def _not_found(kind, name): + return api_exception(404, 'Not Found', f'{kind} "{name}" not found') + + +def _already_exists(kind, name): + return api_exception(409, 'Conflict', f'{kind} "{name}" already exists') + + +def _match_selector(labels, selector): + """Equality-based label selectors: "a=b,c!=d". Enough for this monitor.""" + if not selector: + return True + labels = labels or {} + for term in selector.split(','): + term = term.strip() + if not term: + continue + if '!=' in term: + key, value = term.split('!=', 1) + if labels.get(key.strip()) == value.strip(): + return False + elif '=' in term: + key, value = term.split('=', 1) + if labels.get(key.strip()) != value.strip(): + return False + elif term not in labels: + return False + return True + + +def _match_fields(pod, selector): + if not selector: + return True + for term in selector.split(','): + key, _, value = term.partition('=') + key, value = key.strip(), value.strip() + if key == 'status.phase': + if (pod.status.phase if pod.status else None) != value: + return False + elif key == 'metadata.name': + if pod.metadata.name != value: + return False + return True + + +class Call: + """One API call, as recorded for assertions.""" + + __slots__ = ('verb', 'kind', 'name', 'namespace') + + def __init__(self, verb, kind, name, namespace): + self.verb, self.kind, self.name, self.namespace = verb, kind, name, namespace + + def __iter__(self): # lets a test write (verb, kind, name) tuples + return iter((self.verb, self.kind, self.name)) + + def __eq__(self, other): + if isinstance(other, Call): + return tuple(self) == tuple(other) + return tuple(self) == tuple(other) + + def __hash__(self): + return hash(tuple(self)) + + def __repr__(self): + return f"{self.verb} {self.kind}/{self.name}" + + +class CallLog(list): + def record(self, verb, kind, name, namespace): + self.append(Call(verb, kind, name, namespace)) + + def names(self, verb=None, kind=None): + """Names touched, in order -- the usual assertion.""" + return [c.name for c in self + if (verb is None or c.verb == verb) and (kind is None or c.kind == kind)] + + def verbs(self, kind=None): + return [c.verb for c in self if kind is None or c.kind == kind] + + def of(self, kind): + return [c for c in self if c.kind == kind] + + def __repr__(self): + return "[" + ", ".join(repr(c) for c in self) + "]" + + +class FakeCluster: + """Holds the objects and hands out the two API facades. + + cluster.core_v1 / cluster.batch_v1 are what get monkeypatched into + job_monitor; everything else on here is for the test to drive and inspect. + """ + + def __init__(self, namespace='default'): + self.namespace = namespace + self.jobs = {} # (ns, name) -> V1Job + self.pods = {} # (ns, name) -> V1Pod + self.pvcs = {} # (ns, name) -> V1PersistentVolumeClaim + self.config_maps = {} # (ns, name) -> V1ConfigMap + self.pod_logs = {} # (ns, name) -> str + self.calls = CallLog() + # Deleted names, in order, so a test can assert a reap happened even + # after the object is gone from the dicts. + self.deleted = CallLog() + self._pod_seq = 0 + # Set to an ApiException factory to make the next matching call fail; + # keyed by "verb kind", e.g. {'create job': api_exception(500, 'boom')}. + self.fail_next = {} + self.core_v1 = FakeCoreV1Api(self) + self.batch_v1 = FakeBatchV1Api(self) + + # -- internals ----------------------------------------------------------- + + def _key(self, namespace, name): + return (namespace, name) + + def _maybe_fail(self, verb, kind): + exc = self.fail_next.pop(f"{verb} {kind}", None) + if exc is not None: + raise exc + + def _record(self, verb, kind, name, namespace): + self._maybe_fail(verb, kind) + self.calls.record(verb, kind, name, namespace) + + # -- seeding ------------------------------------------------------------- + + def add_config_map(self, name, data=None, namespace=None, uid=None): + ns = namespace or self.namespace + cm = client.V1ConfigMap( + metadata=client.V1ObjectMeta(name=name, namespace=ns, + uid=uid or f"uid-{name}"), + data=dict(data or {})) + self.config_maps[self._key(ns, name)] = cm + return cm + + # -- inspection ---------------------------------------------------------- + + def job(self, name, namespace=None): + return self.jobs[self._key(namespace or self.namespace, name)] + + def pod(self, name, namespace=None): + return self.pods[self._key(namespace or self.namespace, name)] + + def pod_for_job(self, job_name, namespace=None): + """The Pod the fake created for this Job, or None once it is reaped.""" + ns = namespace or self.namespace + for (pod_ns, _), pod in self.pods.items(): + if pod_ns != ns: + continue + if (pod.metadata.labels or {}).get(JOB_NAME_LABEL) == job_name: + return pod + return None + + def job_names(self, namespace=None): + ns = namespace or self.namespace + return sorted(name for (pod_ns, name) in self.jobs if pod_ns == ns) + + def pvc_names(self, namespace=None): + ns = namespace or self.namespace + return sorted(name for (pvc_ns, name) in self.pvcs if pvc_ns == ns) + + def config_map_data(self, name, namespace=None): + cm = self.config_maps.get(self._key(namespace or self.namespace, name)) + return dict(cm.data or {}) if cm is not None else None + + # -- Job controller emulation ------------------------------------------- + + def _spawn_pod(self, namespace, job): + self._pod_seq += 1 + name = f"{job.metadata.name}-{self._pod_seq:05d}" + labels = dict((job.spec.template.metadata.labels or {}) + if job.spec and job.spec.template and job.spec.template.metadata + else {}) + labels[JOB_NAME_LABEL] = job.metadata.name + labels['job-name'] = job.metadata.name + pod = client.V1Pod( + metadata=client.V1ObjectMeta( + name=name, namespace=namespace, labels=labels, + owner_references=[client.V1OwnerReference( + api_version='batch/v1', kind='Job', name=job.metadata.name, + uid=job.metadata.uid or f"uid-{job.metadata.name}", + controller=True)]), + spec=job.spec.template.spec if job.spec and job.spec.template else None, + status=client.V1PodStatus(phase='Pending', container_statuses=[], + conditions=[])) + self.pods[self._key(namespace, name)] = pod + return pod + + # -- state the monitor branches on -------------------------------------- + + def set_job_running(self, job_name, namespace=None): + job = self.job(job_name, namespace) + job.status = client.V1JobStatus(active=1, start_time=job.status.start_time or _now()) + pod = self.pod_for_job(job_name, namespace) + if pod is not None: + self.set_pod_running(pod.metadata.name, namespace=namespace) + return job + + def set_job_succeeded(self, job_name, namespace=None, seconds=60, + start_time=None, completion_time=None): + job = self.job(job_name, namespace) + start = start_time or job.status.start_time or (_now() - timedelta(seconds=seconds)) + job.status = client.V1JobStatus( + succeeded=1, active=0, start_time=start, + completion_time=completion_time or (start + timedelta(seconds=seconds))) + return job + + def set_job_failed(self, job_name, namespace=None, reason='PodFailurePolicy', + message='', seconds=60, start_time=None): + """Failed with a Job condition -- the message is what classify_from_job parses.""" + job = self.job(job_name, namespace) + start = start_time or job.status.start_time or (_now() - timedelta(seconds=seconds)) + conditions = [] + if reason is not None: + conditions.append(client.V1JobCondition( + type='Failed', status='True', reason=reason, message=message, + last_transition_time=_now())) + job.status = client.V1JobStatus(failed=1, active=0, start_time=start, + conditions=conditions) + return job + + def set_pod_phase(self, pod_name, phase, namespace=None, reason=None, message=None): + pod = self.pod(pod_name, namespace) + pod.status.phase = phase + if reason is not None: + pod.status.reason = reason + if message is not None: + pod.status.message = message + return pod + + def set_pod_running(self, pod_name, namespace=None, ip='10.0.0.1', start_time=None): + pod = self.pod(pod_name, namespace) + pod.status.phase = 'Running' + pod.status.pod_ip = ip + pod.status.start_time = start_time or pod.status.start_time or _now() + pod.status.container_statuses = [client.V1ContainerStatus( + name='stellar-core', image='core', image_id='', ready=True, + restart_count=0, state=client.V1ContainerState( + running=client.V1ContainerStateRunning(started_at=pod.status.start_time)))] + return pod + + def set_pod_terminated(self, pod_name, exit_code=0, reason=None, namespace=None, + seconds=60, start_time=None, finished_at=None, + container='stellar-core', phase=None): + """Terminal container state: exit code plus OOMKilled/Error reason.""" + pod = self.pod(pod_name, namespace) + start = start_time or pod.status.start_time or (_now() - timedelta(seconds=seconds)) + pod.status.start_time = start + pod.status.phase = phase or ('Succeeded' if exit_code == 0 else 'Failed') + pod.status.container_statuses = [client.V1ContainerStatus( + name=container, image='core', image_id='', ready=False, restart_count=0, + state=client.V1ContainerState(terminated=client.V1ContainerStateTerminated( + exit_code=exit_code, + reason=reason or ('Completed' if exit_code == 0 else 'Error'), + started_at=start, + finished_at=finished_at or (start + timedelta(seconds=seconds)))))] + return pod + + def set_pod_condition(self, pod_name, cond_type, status='True', namespace=None, + reason=None): + pod = self.pod(pod_name, namespace) + pod.status.conditions = [c for c in (pod.status.conditions or []) + if c.type != cond_type] + pod.status.conditions.append(client.V1PodCondition( + type=cond_type, status=status, reason=reason, + last_transition_time=_now())) + return pod + + def set_pod_log(self, pod_name, text, namespace=None): + self.pod_logs[self._key(namespace or self.namespace, pod_name)] = text + + def delete_pod(self, pod_name, namespace=None): + """Reap the pod out from under the monitor, the way Karpenter does.""" + self.pods.pop(self._key(namespace or self.namespace, pod_name), None) + + +class _Api: + def __init__(self, cluster): + self._c = cluster + + +class FakeCoreV1Api(_Api): + + # -- ConfigMaps ---------------------------------------------------------- + + def read_namespaced_config_map(self, name, namespace, **_): + self._c._record('read', 'configmap', name, namespace) + cm = self._c.config_maps.get((namespace, name)) + if cm is None: + raise _not_found('configmaps', name) + return copy.deepcopy(cm) + + def create_namespaced_config_map(self, namespace, body, **_): + name = body.metadata.name + self._c._record('create', 'configmap', name, namespace) + if (namespace, name) in self._c.config_maps: + raise _already_exists('configmaps', name) + cm = copy.deepcopy(body) + cm.metadata.namespace = namespace + cm.metadata.uid = cm.metadata.uid or f"uid-{name}" + cm.data = dict(cm.data or {}) + self._c.config_maps[(namespace, name)] = cm + return copy.deepcopy(cm) + + def patch_namespaced_config_map(self, name, namespace, body, **_): + self._c._record('patch', 'configmap', name, namespace) + cm = self._c.config_maps.get((namespace, name)) + if cm is None: + raise _not_found('configmaps', name) + data = body.get('data') if isinstance(body, dict) else (body.data or {}) + cm.data = dict(cm.data or {}) + cm.data.update(data or {}) + return copy.deepcopy(cm) + + def replace_namespaced_config_map(self, name, namespace, body, **_): + self._c._record('replace', 'configmap', name, namespace) + if (namespace, name) not in self._c.config_maps: + raise _not_found('configmaps', name) + cm = copy.deepcopy(body) + cm.metadata.namespace = namespace + cm.data = dict(cm.data or {}) + self._c.config_maps[(namespace, name)] = cm + return copy.deepcopy(cm) + + def delete_namespaced_config_map(self, name, namespace, **_): + self._c._record('delete', 'configmap', name, namespace) + if self._c.config_maps.pop((namespace, name), None) is None: + raise _not_found('configmaps', name) + self._c.deleted.record('delete', 'configmap', name, namespace) + + def list_namespaced_config_map(self, namespace, label_selector=None, **_): + self._c._record('list', 'configmap', '', namespace) + items = [copy.deepcopy(cm) for (ns, _), cm in sorted(self._c.config_maps.items()) + if ns == namespace and _match_selector(cm.metadata.labels, label_selector)] + return client.V1ConfigMapList(items=items) + + # -- Pods ---------------------------------------------------------------- + + def list_namespaced_pod(self, namespace, label_selector=None, field_selector=None, + resource_version=None, **_): + self._c._record('list', 'pod', '', namespace) + items = [copy.deepcopy(p) for (ns, _), p in sorted(self._c.pods.items()) + if ns == namespace + and _match_selector(p.metadata.labels, label_selector) + and _match_fields(p, field_selector)] + return client.V1PodList(items=items) + + def read_namespaced_pod(self, name, namespace, **_): + self._c._record('read', 'pod', name, namespace) + pod = self._c.pods.get((namespace, name)) + if pod is None: + raise _not_found('pods', name) + return copy.deepcopy(pod) + + def read_namespaced_pod_log(self, name, namespace, container=None, tail_lines=None, **_): + self._c._record('read', 'podlog', name, namespace) + if (namespace, name) not in self._c.pods: + raise _not_found('pods', name) + text = self._c.pod_logs.get((namespace, name), '') + if tail_lines: + text = "\n".join(text.splitlines()[-tail_lines:]) + return text + + def delete_namespaced_pod(self, name, namespace, **_): + self._c._record('delete', 'pod', name, namespace) + if self._c.pods.pop((namespace, name), None) is None: + raise _not_found('pods', name) + self._c.deleted.record('delete', 'pod', name, namespace) + + # -- PersistentVolumeClaims --------------------------------------------- + + def read_namespaced_persistent_volume_claim(self, name, namespace, **_): + self._c._record('read', 'pvc', name, namespace) + pvc = self._c.pvcs.get((namespace, name)) + if pvc is None: + raise _not_found('persistentvolumeclaims', name) + return copy.deepcopy(pvc) + + def create_namespaced_persistent_volume_claim(self, namespace, body, **_): + name = body.metadata.name + self._c._record('create', 'pvc', name, namespace) + if (namespace, name) in self._c.pvcs: + raise _already_exists('persistentvolumeclaims', name) + pvc = copy.deepcopy(body) + pvc.metadata.namespace = namespace + pvc.metadata.uid = pvc.metadata.uid or f"uid-{name}" + pvc.status = client.V1PersistentVolumeClaimStatus(phase='Bound') + self._c.pvcs[(namespace, name)] = pvc + return copy.deepcopy(pvc) + + def delete_namespaced_persistent_volume_claim(self, name, namespace, **_): + self._c._record('delete', 'pvc', name, namespace) + if self._c.pvcs.pop((namespace, name), None) is None: + raise _not_found('persistentvolumeclaims', name) + self._c.deleted.record('delete', 'pvc', name, namespace) + + def list_namespaced_persistent_volume_claim(self, namespace, label_selector=None, **_): + self._c._record('list', 'pvc', '', namespace) + items = [copy.deepcopy(p) for (ns, _), p in sorted(self._c.pvcs.items()) + if ns == namespace and _match_selector(p.metadata.labels, label_selector)] + return client.V1PersistentVolumeClaimList(items=items) + + +class FakeBatchV1Api(_Api): + + def create_namespaced_job(self, namespace, body, **_): + name = body.metadata.name + self._c._record('create', 'job', name, namespace) + if (namespace, name) in self._c.jobs: + # Name uniqueness is the monitor's dispatch mutex; it swallows this. + raise _already_exists('jobs.batch', name) + job = copy.deepcopy(body) + job.metadata.namespace = namespace + job.metadata.uid = job.metadata.uid or f"uid-{name}" + job.status = client.V1JobStatus(active=0, start_time=_now()) + self._c.jobs[(namespace, name)] = job + self._c._spawn_pod(namespace, job) + return copy.deepcopy(job) + + def read_namespaced_job(self, name, namespace, **_): + self._c._record('read', 'job', name, namespace) + job = self._c.jobs.get((namespace, name)) + if job is None: + raise _not_found('jobs.batch', name) + return copy.deepcopy(job) + + def list_namespaced_job(self, namespace, label_selector=None, field_selector=None, **_): + self._c._record('list', 'job', '', namespace) + items = [copy.deepcopy(j) for (ns, _), j in sorted(self._c.jobs.items()) + if ns == namespace and _match_selector(j.metadata.labels, label_selector)] + return client.V1JobList(items=items) + + def delete_namespaced_job(self, name, namespace, propagation_policy=None, body=None, **_): + self._c._record('delete', 'job', name, namespace) + if self._c.jobs.pop((namespace, name), None) is None: + raise _not_found('jobs.batch', name) + self._c.deleted.record('delete', 'job', name, namespace) + # Background/Foreground both reap the pods; Orphan is the only one that + # does not, and the monitor never asks for it. + if propagation_policy != 'Orphan': + for key in [k for k, p in self._c.pods.items() + if k[0] == namespace + and (p.metadata.labels or {}).get(JOB_NAME_LABEL) == name]: + self._c.pods.pop(key, None) + + def patch_namespaced_job(self, name, namespace, body, **_): + self._c._record('patch', 'job', name, namespace) + job = self._c.jobs.get((namespace, name)) + if job is None: + raise _not_found('jobs.batch', name) + return copy.deepcopy(job) + + def replace_namespaced_job(self, name, namespace, body, **_): + self._c._record('replace', 'job', name, namespace) + if (namespace, name) not in self._c.jobs: + raise _not_found('jobs.batch', name) + job = copy.deepcopy(body) + job.metadata.namespace = namespace + self._c.jobs[(namespace, name)] = job + return copy.deepcopy(job) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 9adb3880..26ffd7aa 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -23,8 +23,10 @@ import os import re import sys +import tempfile import threading import time +import zlib from datetime import datetime, timezone from http.server import BaseHTTPRequestHandler, HTTPServer @@ -217,6 +219,15 @@ EPH_BUMP_FACTOR = float(os.getenv('EPH_BUMP_FACTOR', 1.5)) EPH_ESCALATION_CAP = os.getenv('EPH_ESCALATION_CAP', '200Gi') ENVIRONMENTAL_OUTCOMES = ('disrupted', 'rejected', 'unknown') +# Verdicts only the pod can produce, and which a Job-level DeadlineExceeded must +# never overwrite. Each names a specific mechanism -- the kubelet OOM-killed it, +# the node was draining, the ephemeral limit blew -- and each earns a different +# retry budget and a different remediation. "The Job ran too long" is also true +# of every one of them and says nothing about which. An OOM downgraded to a +# timeout retries at the same memory limit that just killed it and gets 2 +# attempts instead of 5; a spot eviction downgraded to a timeout gets 2 instead +# of 20. +POD_AUTHORITATIVE_OUTCOMES = ('oom', 'disrupted', 'ephemeral', 'timeout') # stellar-core's "did not complete". Ambiguous by construction: a corrupt bucket # and a SIGTERM during replay both produce it, so it must never be treated as # proof that a range is broken. @@ -265,14 +276,29 @@ def get_logging_level(): # destination directory. Falls back to /data if LOG_DIR is not mounted. log_file_name = f"job_monitor_{datetime.now(timezone.utc).strftime('%Y-%m-%d_%H-%M-%S')}.log" _log_dir = os.getenv('LOG_DIR', '/logs') -log_file_path = os.path.join(_log_dir if os.path.isdir(_log_dir) else '/data', log_file_name) +_chosen_log_dir = _log_dir if os.path.isdir(_log_dir) else '/data' +# Last resort, and unreachable in a pod: /data is the monitor's own emptyDir, so +# one of the two above always exists there. Off-cluster neither does, and a +# FileHandler on a missing directory made this module impossible to import -- +# which is why nothing here was ever tested against a real reconcile(). +if not os.path.isdir(_chosen_log_dir): + _chosen_log_dir = tempfile.gettempdir() +log_file_path = os.path.join(_chosen_log_dir, log_file_name) logging.basicConfig(level=get_logging_level(), format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.StreamHandler(sys.stdout), logging.FileHandler(log_file_path), ]) logger = logging.getLogger() -config.load_incluster_config() +# The env var is exactly what load_incluster_config() itself keys on, so in a pod +# this is the unconditional call it always was -- a missing token or CA still +# raises here and crash-loops the container rather than running blind. Outside a +# pod there is nothing to load and import stays pure; the caller injects clients. +if os.getenv('KUBERNETES_SERVICE_HOST'): + config.load_incluster_config() +else: + logger.warning("KUBERNETES_SERVICE_HOST is unset: no in-cluster config loaded. " + "Every API call will fail until core_v1/batch_v1 are replaced.") # client-go's Python equivalent defaults are fine for a few LISTs per cycle, but # dispatching ~1024 Jobs + PVCs at once needs headroom. _cfg = client.Configuration.get_default_copy() @@ -664,6 +690,12 @@ def classify(pod): 'OutOfpods', 'UnexpectedAdmissionError', 'NodeAffinity', 'Shutdown', 'Evicted'): return {'outcome': 'rejected', 'exitCode': None, 'reason': pod.status.reason} + if pod.status.reason == 'DeadlineExceeded': + # The deadline lives on the PodSpec, so it is the kubelet that fires it + # and the pod that carries the reason -- the Job just sees a non-zero + # exit through its podFailurePolicy. Without this the drain-to-exit-3 + # matches the generic rule and reads as a plain catchup failure. + return {'outcome': 'timeout', 'exitCode': None, 'reason': pod.status.reason} started = any(cs.state and cs.state.terminated for cs in (pod.status.container_statuses or [])) if not started: # No container ever reached a terminal state: nothing ran, so this is @@ -808,6 +840,54 @@ def _oom_count(end, attempt): if (read_outcome(end, n) or {}).get('outcome') == 'oom') +def verdict_path(end, attempt): + return os.path.join(LOG_DIR, f"range-{end}-a{attempt}.verdict") + + +def save_verdict(end, attempt, outcome): + """Persist the EFFECTIVE verdict for one attempt, so budgets can be tallied. + + The .outcome file is not enough on its own: it is classified from the pod, + and a deadline kill reads as a plain exit-3 `failed` there -- only the Job's + DeadlineExceeded condition says `timeout`. Reconcile resolves that conflict + once, and this is where the answer is kept, on the same durable logs volume + as everything else, so a monitor restart does not reset a range's budgets. + """ + path = verdict_path(end, attempt) + try: + tmp = path + '.tmp' + with open(tmp, 'w') as fh: + fh.write(str(outcome)) + os.replace(tmp, path) + except OSError as e: + logger.warning("could not persist verdict for range %s attempt %s: %s", + end, attempt, e) + + +def _verdict_of(end, attempt): + try: + with open(verdict_path(end, attempt)) as fh: + return fh.read().strip() or None + except OSError: + # Pre-fix runs, or an attempt whose verdict write lost the volume: + # the pod-derived classification is the next best thing. + return (read_outcome(end, attempt) or {}).get('outcome') + + +def _cause_count(end, attempt, causes): + """How many of attempts 1..N at this range failed for one of `causes`. + + Budgets are per cause, not per attempt. One shared attempt index meant + cluster churn -- which has its own deliberately large budget -- drained the + small budgets belonging to the causes that say something about the range: a + range evicted MAX_ATTEMPTS times had an effective OOM and disk budget of + zero, was condemned on its first real OOM without ever being escalated, and + took the whole mission with it. + """ + return sum(1 for n in range(1, int(attempt) + 1) + if _verdict_of(end, n) in causes) + + def mem_for_attempt(attempt, base=None): """Memory limit after N OOMs, capped at MEM_ESCALATION_CAP. @@ -1047,7 +1127,13 @@ def _tx_apply_for_attempt(end, attempt=1, pod_name=None): try: with gzip.open(candidate, 'rt') as fh: raw = fh.read() - except OSError: + except (OSError, EOFError, zlib.error): + # A corrupt archive costs THIS RANGE its metric, never the pass. + # EOFError is a truncated member -- the collector appending right + # now, or one that was killed mid-append -- and is not an OSError, + # so it used to escape the per-range work and abort the whole + # reconcile: no recording, no reap, no dispatch for any of the + # ~4000 ranges, repeating for as long as the torn bytes sat there. raw = None if raw is None: if pod_name is None: @@ -1197,7 +1283,36 @@ def _reap_if_complete(end, attempt, record): """ if not _attempt_finalized(end, attempt): return - delete_job(end, attempt) + reap_range_jobs(end) + + +def reap_range_jobs(end): + """Delete every Job this range has, not just the attempt that won. + + Completion is terminal for the RANGE. An attempt-scoped reap leaves an + older Failed Job standing -- the common case is an attempt lost to node + disruption whose collector died with the node, so it was never finalized + and was deliberately not deleted. Once the winner's Job is gone, that + leftover is the range's highest live attempt, and the next pass feeds it + straight into the retry decision and re-runs an already-recorded range + against a freshly recreated, empty PVC. + """ + try: + jobs = batch_v1.list_namespaced_job( + NAMESPACE, + label_selector=f"{LABEL_RUN}={RUN_NAME},{LABEL_RANGE}={end}").items + except ApiException as e: + logger.warning("could not list jobs for completed range %s: %s", end, e) + return + for j in jobs: + try: + batch_v1.delete_namespaced_job(j.metadata.name, NAMESPACE, + propagation_policy='Background') + metric_jobs_reaped.inc() + except ApiException as e: + if e.status != 404: + logger.warning("could not delete finished job %s for range %s: %s", + j.metadata.name, end, e) def delete_job(end, attempt): @@ -1493,10 +1608,20 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): pod_failure_policy=client.V1PodFailurePolicy( rules=[r for _, r in _failure_rules()]), ttl_seconds_after_finished=JOB_TTL_SECONDS, - active_deadline_seconds=ATTEMPT_DEADLINE_SECONDS or None, template=client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels=pod_labels(end, attempt)), spec=client.V1PodSpec( + # On the POD, not the JobSpec. JobSpec.activeDeadlineSeconds + # runs from the Job's startTime, so every second the pod + # spends Pending -- waiting for Karpenter, pulling the image + # -- is charged against a budget that is meant to bound how + # long the range RUNS. During a node-class outage this run + # sat ~15 minutes Pending and ranges died as "timeouts" + # having barely executed; a timeout gets + # MAX_TIMEOUT_ATTEMPTS, so two stalls condemn a range and + # fail the mission. The pod-level field starts at container + # start, which is the thing being bounded. + active_deadline_seconds=ATTEMPT_DEADLINE_SECONDS or None, # IRSA for the S3 history mirror. Without it workers fall # back to the public archive, which throttles at 1024. service_account_name=WORKER_SERVICE_ACCOUNT or None, @@ -1567,20 +1692,27 @@ def observe_recorded(progress, replayed): Prometheus histograms are append-only and reset to zero when the process restarts, so replaying every recorded range rebuilds the exact cumulative total rather than double counting. Guarded per-process by `replayed`. + + Keyed on (range, field), not on the range alone: a range is usually + recorded before the collector has flushed its .metrics, so txApply is null + at first sight and backfilled a pass or two later. Marking the whole range + as replayed on first sight meant that backfill could never be observed, and + the histogram permanently disagreed with progress.json. """ for end, rec in progress.get('completed', {}).items(): - if end in replayed: - continue - replayed.add(end) # `is not None`, not truthiness: a range with sum = 0ms records # txApply 0.0, which is a real observation and must not be dropped # silently. Same for a sub-second duration. - if rec.get('seconds') is not None: - metric_full_duration.observe(rec['seconds']) - if rec.get('wallSeconds') is not None: - metric_wall_duration.observe(rec['wallSeconds']) - if rec.get('txApply') is not None: - metric_tx_apply_duration.observe(rec['txApply']) + for field, metric in (('seconds', metric_full_duration), + ('wallSeconds', metric_wall_duration), + ('txApply', metric_tx_apply_duration)): + if (end, field) in replayed: + continue + value = rec.get(field) + if value is None: + continue + replayed.add((end, field)) + metric.observe(value) def pods_by_job(): @@ -1706,6 +1838,18 @@ def reconcile(state): end, sorted(late)) _reap_if_complete(end, attempt, completed[end]) elif st.failed: + # Completion is terminal for the range, so a Failed Job for a range + # that is already recorded is garbage -- never an input to the retry + # decision. Without this, a losing attempt that outlived the winner + # gets re-classified (disrupted, unknown, ...) and the range is + # dispatched all over again, against a PVC that was already + # released, i.e. a full replay from genesis of work already paid + # for. Sweep the leftover and move on. + if end in completed: + logger.info("range %s already recorded complete; discarding " + "leftover Job for attempt %d", end, attempt) + reap_range_jobs(end) + continue pod = job_pods.get(j.metadata.name) if pod is not None: record_outcome(end, attempt, pod) @@ -1723,8 +1867,16 @@ def reconcile(state): # fired, so on that condition the Job wins. Measured in the sandbox # edge suite 2026-07-30: whichever of the two won the race decided # whether the range was retried or condemned. + # + # Ranked, not unconditional. The Job wins only where the pod has + # nothing more specific to say -- an exit-3 drain, a rejection, no + # surviving classification. Where the pod named the mechanism + # (OOMKilled, DisruptionTarget, an ephemeral eviction) the pod wins, + # because "ran too long" is also true of all of those and picking it + # loses both the remediation and the correct retry budget. from_job = classify_from_job(j) - if from_job and from_job.get('outcome') == 'timeout': + if (from_job and from_job.get('outcome') == 'timeout' + and (verdict or {}).get('outcome') not in POD_AUTHORITATIVE_OUTCOMES): verdict = from_job if verdict is None: verdict = {'outcome': 'unknown', 'exitCode': None} @@ -1733,6 +1885,10 @@ def reconcile(state): "(exit %s); pod was already gone", end, attempt, verdict.get('exitCode')) + # Durable before anything reads a tally -- _oom_count and the + # budget check below both count this attempt. + save_verdict(end, attempt, verdict['outcome']) + retry_mem = retry_eph = None if verdict['outcome'] == 'timeout': reason = (f"exceeded the {ATTEMPT_DEADLINE_SECONDS}s attempt deadline " @@ -1781,19 +1937,37 @@ def reconcile(state): else: reason = None # genuine catchup failure: do not retry - # Three budgets, by whose fault the attempt was: a hang is usually + # Four budgets, by whose fault the attempt was: a hang is usually # persistent and gets the lowest, a range that is genuinely broken # gets the middle one, and anything the cluster did to us gets the # highest. + # + # Each is spent by ITS OWN cause, never by the global attempt index. + # Sharing one counter meant the cap was chosen by the latest verdict + # and then compared against every retry the range had ever had: a + # range that survived five spot evictions (legal, budget 20) reached + # attempt 6, and its first genuine OOM was compared 6 >= 5 and + # condemned -- never retried for an OOM, never escalated, and a + # condemned range fails the mission. On spot, where evictions are + # routine, that made the OOM and disk budgets effectively zero. if verdict['outcome'] == 'timeout': cap = MAX_TIMEOUT_ATTEMPTS + spent = _cause_count(end, attempt, ('timeout',)) elif verdict['outcome'] == 'ephemeral': cap = MAX_EPHEMERAL_ATTEMPTS + spent = _cause_count(end, attempt, ('ephemeral',)) elif verdict['outcome'] in ENVIRONMENTAL_OUTCOMES: cap = MAX_DISRUPTION_ATTEMPTS + spent = _cause_count(end, attempt, ENVIRONMENTAL_OUTCOMES) else: + # The range's own budget: an OOM and a "did not complete" are + # both statements about this ledger range, so they share it. cap = MAX_ATTEMPTS_PER_RANGE - if reason is not None and attempt < cap: + spent = _cause_count(end, attempt, ('oom', 'failed')) + # This attempt's verdict is already on disk, so `spent` includes it: + # the Nth failure of a cause is the one that exhausts a budget of N, + # exactly as `attempt < cap` behaved for a single-cause range. + if reason is not None and spent < cap: if verdict['outcome'] == 'oom': logger.error( "!!! OOM RETRY !!! range %s was OOM-killed on attempt %d/%d; retrying with " diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 49c4b193..9f593475 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -27,6 +27,7 @@ import asyncio import gzip +import io import json import logging import os @@ -555,22 +556,44 @@ async def _poll_once(session, pod, end, attempt, last_ts, tx): lines = [l for l in re.split(r'[\r\n]', body) if l] if not lines: return last_ts, False - # Opened per poll, not held for the pod's life. A live gzip deflate buffer - # per stream is what put the sidecar at 1444 MiB of a 2048 MiB limit at 2096 - # follow streams; here nothing is retained between polls. - with gzip.open(base(end, attempt) + '.log.gz', 'at') as fh: + # Compressed into memory first, then appended in ONE write. + # + # Appending straight into the file with gzip.open(..., 'at') meant the + # deflate buffer flushed partial output to disk repeatedly across the whole + # loop, so for most of a large poll the archive on disk ended in a member + # with no end-of-stream marker. job_monitor reads that same file to recover + # txApplySeconds and gzip raises EOFError on a truncated member -- one + # in-flight poll could abort a reconcile pass for every range. The window is + # now a single append instead of the length of the write loop, and the file + # only ever gains whole members. + # + # Costs no more memory than is already held: `body` above is the entire + # poll uncompressed, and this is the same bytes compressed. Nothing is + # retained between polls, which is the property that got the sidecar off + # 1444 MiB of a 2048 MiB limit at 2096 follow streams. + member = io.BytesIO() + wrote = False + with gzip.GzipFile(fileobj=member, mode='wb') as fh: for line in lines: ts, _, rest = line.partition(' ') if not _TS_RE.match(ts): # Untimestamped kubelet text. Keep it, but never let it become # the resume point. - fh.write(line + '\n') + fh.write((line + '\n').encode('utf-8')) + wrote = True continue if last_ts and ts <= last_ts: continue # exact dedup of the resume overlap - fh.write(rest + '\n') + fh.write((rest + '\n').encode('utf-8')) + wrote = True tx.feed(rest) pending = ts + path = base(end, attempt) + '.log.gz' + with open(path, 'ab') as out: + # A poll whose lines were all deduped still touches the archive: its + # existence is what job_monitor's backstop keys on. + if wrote: + out.write(member.getvalue()) if pending: write_state(end, attempt, pending) return pending, False @@ -657,11 +680,18 @@ async def poll_pod(session, pod, end, attempt, done, done_ok): # would not help -- sinceTime has second granularity, so anything under # ~1s re-reads the same second -- and the delay that matters is between # the container exiting and the last read, not between routine polls. + ev = _wake.setdefault(pod, asyncio.Event()) try: - await asyncio.wait_for(_wake.setdefault(pod, asyncio.Event()).wait(), - timeout=backoff) + await asyncio.wait_for(ev.wait(), timeout=backoff) except asyncio.TimeoutError: pass + finally: + # Standard set/clear pairing. Left set, the Event makes every later + # wait return instantly, so the terminal-poll backoff never sleeps + # and TERMINAL_POLL_ATTEMPTS is spent in one millisecond -- the pod + # is given no time to have its final log become readable. A wake is + # consumed by the poll it triggers. + ev.clear() async def list_pods(session): diff --git a/src/MissionParallelCatchup/test_harness_smoke.py b/src/MissionParallelCatchup/test_harness_smoke.py new file mode 100644 index 00000000..24a2e6b5 --- /dev/null +++ b/src/MissionParallelCatchup/test_harness_smoke.py @@ -0,0 +1,172 @@ +"""Proof that the fake cluster drives the real reconcile(). + +Every test here imports job_monitor and calls the shipped reconcile() -- nothing +is extracted from source or reimplemented. If one of these fails, the monitor's +behaviour changed, not a regex. +""" + +import pytest + +import fake_k8s +import job_monitor as jm + + +def test_dispatch_happens_on_an_empty_cluster(cluster): + result = cluster.reconcile() + + # PARALLELISM is 2 and there are three ranges, so exactly two go out, and + # tip-first means the two highest ends. + assert cluster.jobs() == ['pc-r200-a1', 'pc-r300-a1'] + assert result['created'] == 2 + assert result['total'] == 3 + assert result['remaining'] == 1 + assert sorted(result['in_progress']) == ['200/420', '300/420'] + + # pvc mode: each range gets its own volume, created before its Job. + assert cluster.pvcs() == ['pc-data-r200', 'pc-data-r300'] + created = [(c.kind, c.name) for c in cluster.calls if c.verb == 'create'] + assert created == [('pvc', 'pc-data-r300'), ('job', 'pc-r300-a1'), + ('pvc', 'pc-data-r200'), ('job', 'pc-r200-a1')] + + # Nothing durable is written until a range actually finishes -- dispatch + # alone must not touch the progress record or its ConfigMap mirror. + assert cluster.progress() == {} + assert cluster.calls.names(verb='patch', kind='configmap') == [] + + +def test_a_succeeded_job_is_recorded_into_completed(cluster): + cluster.reconcile() + cluster.advance(300, 'succeeded') + # The collector's half of the contract: peaks and tx_apply are only ever + # readable from the files it writes, and the .done marker is what allows a + # reap at all. + cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakRssBytes': 123}) + + cluster.reconcile() + + record = cluster.completed()['300'] + assert record['attempts'] == 1 + assert record['count'] == 420 + assert record['txApply'] == 1.5 + assert record['peakRssBytes'] == 123 + assert record['seconds'] == pytest.approx(60.0) + assert record['wallSeconds'] == pytest.approx(60.0) + + # Durable file first, ConfigMap mirror second -- and the mirror is stripped + # of the profiling fields that would push it at the 1 MiB cap. + assert '300' in cluster.progress_configmap()['completed'] + assert 'peakRssBytes' not in cluster.progress_configmap()['completed']['300'] + + # A completed range gives its volume back and its Job is reaped. + assert 'pc-data-r300' not in cluster.pvcs() + assert cluster.deleted.names(verb='delete', kind='job') == ['pc-r300-a1'] + assert cluster.deleted.names(verb='delete', kind='pvc') == ['pc-data-r300'] + + # The freed slot is refilled in the same pass. + assert 'pc-r100-a1' in cluster.jobs() + + +def test_a_failed_job_is_retried(cluster): + cluster.reconcile() + # exit 3 is stellar-core's "did not complete": a corrupt archive and an + # interruption are indistinguishable, so it must be retried, not condemned. + cluster.advance(300, 'incomplete') + + cluster.reconcile() + + assert 'pc-r300-a2' in cluster.jobs() + assert cluster.attempt_of(300) == 2 + assert cluster.failed() == {}, "a retryable failure must not be recorded as failed" + assert cluster.completed() == {} + + # The retry rides the same volume -- that is what makes resume-at-LCL work. + assert cluster.calls.names(verb='create', kind='pvc').count('pc-data-r300') == 1 + # The predecessor is NOT deleted: the collector has not finalized it, and + # reaping the Job would reap the pod its metrics still live on. + assert 'pc-r300-a1' in cluster.jobs() + # ...and the new pod carries the attempt label the collector keys files on. + pod = cluster.k8s.pod_for_job('pc-r300-a2') + assert pod.metadata.labels[jm.LABEL_ATTEMPT] == '2' + + +def test_a_condemned_range_is_recorded_and_not_retried(cluster): + cluster.reconcile() + # A plain non-zero exit that is not 3 is a genuine catchup failure. + cluster.advance(300, 'condemned') + + cluster.reconcile() + + assert 'pc-r300-a2' not in cluster.jobs() + assert cluster.failed()['300'] == { + 'attempts': 1, 'pod': cluster.k8s.pod_for_job('pc-r300-a1').metadata.name, + 'outcome': 'failed', 'exitCode': 1} + # Dispatch is not frozen by a condemned range: the freed slot is refilled, + # otherwise the mission's `remaining == 0` wait would deadlock. + assert 'pc-r100-a1' in cluster.jobs() + + +def test_an_oom_retry_escalates_the_memory_limit(cluster): + cluster.reconcile() + cluster.advance(300, 'oom') + + cluster.reconcile() + + resources = (cluster.k8s.job('pc-r300-a2') + .spec.template.spec.containers[0].resources) + # One OOM = one rung: 24000Mi * 1.5. The request follows the limit, because + # a pod that OOMed will not fit where it was scheduled before. + assert resources.limits['memory'] == '36000Mi' + assert resources.requests['memory'] == '36000Mi' + assert cluster.failed() == {} + + +def test_a_disruption_does_not_spend_the_range_budget(cluster): + cluster.reconcile() + cluster.advance(300, 'disrupted') + + cluster.reconcile() + + assert 'pc-r300-a2' in cluster.jobs() + outcome = jm.read_outcome('300', 1) + assert outcome['outcome'] == 'disrupted' + # Memory is untouched: an eviction says nothing about how much the range wants. + resources = (cluster.k8s.job('pc-r300-a2') + .spec.template.spec.containers[0].resources) + assert resources.limits['memory'] == jm.LIM_MEM + + +def test_progress_going_backwards_halts_dispatch(cluster): + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1) + cluster.reconcile() + assert cluster.state['max_completed'] == 1 + + # Someone deletes the record underneath the run. + cluster.write(jm.PROGRESS_FILE, '{}') + before = set(cluster.jobs()) + result = cluster.reconcile() + + assert cluster.state['halted'] is True + assert result['created'] == 0 + assert set(cluster.jobs()) == before + + +def test_the_fake_raises_the_status_codes_the_monitor_branches_on(cluster): + cluster.reconcile() + + with pytest.raises(fake_k8s.ApiException) as dup: + cluster.k8s.batch_v1.create_namespaced_job( + cluster.namespace, cluster.k8s.job('pc-r300-a1')) + assert dup.value.status == 409 + + with pytest.raises(fake_k8s.ApiException) as missing: + cluster.k8s.core_v1.read_namespaced_config_map('nope', cluster.namespace) + assert missing.value.status == 404 + + # 404 on a PVC read is what ensure_pvc() uses to decide to create one, and + # 404 on the progress ConfigMap is what load_progress() treats as "new run". + with pytest.raises(fake_k8s.ApiException) as gone: + cluster.k8s.core_v1.read_namespaced_persistent_volume_claim( + 'pc-data-r999', cluster.namespace) + assert gone.value.status == 404 diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index aa468944..b548900c 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -1878,11 +1878,18 @@ def test_polls_are_bounded_by_a_semaphore(): def test_the_archive_is_not_held_open_between_polls(): # A live gzip deflate buffer per stream is most of what put the sidecar at - # 1444 MiB. Opening per poll means nothing is retained between them. + # 1444 MiB. The member is built in a function-local buffer and appended in + # one write, so nothing is retained between polls. + # + # Asserts the invariant rather than the call: this test used to pin the + # literal gzip.open(..., 'at'), which the atomic-append fix replaced, and it + # went red over a change that preserved everything it existed to protect. fn = _extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert "gzip.open(base(end, attempt) + '.log.gz', 'at')" in fn + assert 'io.BytesIO()' in fn and 'gzip.GzipFile' in fn, \ + "the member is no longer built in a function-local buffer" loop = _extract(r"^(async def poll_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert 'gzip.open' not in loop, "the archive is held across polls" + for held in ('gzip.open', 'gzip.GzipFile', 'io.BytesIO'): + assert held not in loop, f"the archive is held across polls ({held})" def test_terminal_is_read_before_the_poll_not_after(): @@ -1956,7 +1963,8 @@ def test_the_reap_waits_for_the_collectors_done_marker(): import tempfile, os as _os d = tempfile.mkdtemp() ns = {'os': _os, 'LOG_DIR': d, 'PEAK_FIELDS': ('peakAnonBytes',), 'reaped': []} - ns['delete_job'] = lambda e, a: ns['reaped'].append((e, a)) + # Completion is terminal for the RANGE, so the reap is range-scoped now. + ns['reap_range_jobs'] = lambda e: ns['reaped'].append((e, 1)) for name in ('done_path', '_attempt_finalized', '_has_peaks', '_reap_if_complete'): exec(_extract(r"^(def " + name + r"\(.*?)(?=\ndef )").group(1), ns) full = {'txApply': 5.0, 'peakAnonBytes': 99} diff --git a/src/MissionParallelCatchup/test_race_1.py b/src/MissionParallelCatchup/test_race_1.py new file mode 100644 index 00000000..2550f493 --- /dev/null +++ b/src/MissionParallelCatchup/test_race_1.py @@ -0,0 +1,218 @@ +"""RACE #1 -- a completed range gets re-dispatched and re-run from scratch. + +The interleaving these tests drive is the real one: + + A. range 300 attempt 1 is lost to node disruption. The verdict is + `disrupted`, 1 < MAX_DISRUPTION_ATTEMPTS, so attempt 2 is created. The + collector died with the node, so no `.done` marker exists for attempt 1 + and the monitor deliberately does NOT delete its Job -- it sits Failed, + waiting on JOB_TTL_SECONDS. + B. attempt 2 reuses the surviving PVC, finds the range already complete and + exits 0. The next pass keys `live` on the highest attempt, records the + range, releases the PVC and reaps -- but the reap is attempt-scoped, so + only attempt 2's Job dies. The Failed attempt-1 Job outlives the winner. + C. The pass after that lists only attempt 1, so `live[300]` is the Failed + Job. Nothing in the `st.failed` branch asks whether the range is already + in `completed`, so the disruption verdict is reached all over again and + attempt 2 is created A SECOND TIME -- against a freshly recreated, empty + PVC, so it replays the whole range from genesis. + +Everything asserted below is observed state: which Jobs and PVCs exist, what +the API was asked to create, what landed in progress.json, and what reconcile() +itself reported. No source text is inspected. +""" + +import job_monitor as jm + + +# -- helpers ----------------------------------------------------------------- + + +def jobs_for(cluster, end): + """Live Job names belonging to one range, oldest attempt first.""" + prefix = f"{cluster.run_name}-r{int(end)}-a" + return sorted((n for n in cluster.jobs() if n.startswith(prefix)), + key=lambda n: int(n.rsplit('-a', 1)[1])) + + +def created(cluster, kind): + return cluster.calls.names(verb='create', kind=kind) + + +def stale_predecessor(cluster): + """Passes 1-2: dispatch, then lose 300/a1 to disruption. + + Leaves the range with two Jobs: Failed a1 (never finalized, so never + reaped) and freshly created a2. + """ + cluster.reconcile() + cluster.advance(300, 'disrupted') # collector dies with the node: + cluster.reconcile() # no finalize() -> no .done + + +def win_on_attempt_two(cluster): + """Pass 3: a2 succeeds and is recorded. Returns reconcile()'s summary.""" + cluster.advance(300, 'succeeded') # newest attempt == a2 + cluster.finalize(300, 2, tx_apply=0.25, peaks={'peakRssBytes': 4096}) + return cluster.reconcile() + + +# -- the precondition, so a green suite cannot be green by accident ---------- + + +def test_a_disrupted_attempt_that_never_finalized_outlives_its_successor(cluster): + """Setup check: the losing Job really is still there when a2 starts. + + This is intended behaviour -- the monitor refuses to reap an attempt whose + collector never wrote `.done`, because the Job's pod is the last place its + measurements could still be read from. It is the *input* to the race, not + the bug, and it must hold both before and after the fix. + """ + stale_predecessor(cluster) + + assert jobs_for(cluster, 300) == ['pc-r300-a1', 'pc-r300-a2'] + assert cluster.k8s.job('pc-r300-a1').status.failed + # The volume survives on purpose: that is what lets a2 resume instead of + # replaying from genesis. + assert 'pc-data-r300' in cluster.pvcs() + assert cluster.completed() == {} + + +# -- the race ---------------------------------------------------------------- + + +def test_recording_a_range_reaps_every_attempt_not_just_the_winner(cluster): + """Completion is terminal for the RANGE, so no attempt of it may survive. + + RED (attempt-scoped reap): only pc-r300-a2 is deleted and the Failed + pc-r300-a1 is still standing -- which is the entire fuel for the re-run. + """ + stale_predecessor(cluster) + win_on_attempt_two(cluster) + + assert cluster.completed()['300']['attempts'] == 2 # it really recorded + assert jobs_for(cluster, 300) == [] + + +def test_a_recorded_range_is_never_dispatched_again(cluster): + """The core consequence: an already-paid-for range is re-run end to end. + + RED (no `completed` guard in the failed branch): the pass after the record + sees the leftover Failed a1, re-reaches the `disrupted` verdict and creates + pc-r300-a2 for the second time. + """ + stale_predecessor(cluster) + win_on_attempt_two(cluster) + recorded = dict(cluster.completed()['300']) + + for _ in range(3): # the monitor loops forever + cluster.reconcile() + + assert jobs_for(cluster, 300) == [] + # Exactly two Jobs were ever created for this range: a1 and its one retry. + assert created(cluster, 'job').count('pc-r300-a2') == 1 + assert [n for n in created(cluster, 'job') if n.startswith('pc-r300-')] == \ + ['pc-r300-a1', 'pc-r300-a2'] + # And the durable record was not disturbed by the extra passes. + assert cluster.completed()['300'] == recorded + assert cluster.failed() == {} + + +def test_a_released_volume_is_not_resurrected_for_a_completed_range(cluster): + """Why the re-run is worst case: the PVC is gone, so there is nothing to + resume from. build_job() calls ensure_pvc(), which recreates it empty -- + no /data/.job-key, RESUME declined, new-db, full replay from genesis. + + RED: pc-data-r300 is released by the recording pass and then created a + second time by the spurious re-dispatch. + """ + stale_predecessor(cluster) + win_on_attempt_two(cluster) + + assert 'pc-data-r300' not in cluster.pvcs() # released on record + cluster.reconcile() + cluster.reconcile() + + assert 'pc-data-r300' not in cluster.pvcs() + assert created(cluster, 'pvc').count('pc-data-r300') == 1 + + +def test_a_phantom_rerun_does_not_breach_parallelism(cluster): + """The slot freed by 300 goes to 100 -- and then 300 must not take one back. + + Recording 300 frees a slot, so pass 3 dispatches range 100 and the run is + at its cap of 2. The re-dispatch happens *outside* the capacity check (the + failed branch creates the Job and appends to in_progress unconditionally), + so it does not wait for a slot -- it takes a third one. + + RED: in_progress is ['100/420', '200/420', '300/420'] -- three concurrent + ranges under PARALLELISM 2, one of them already finished. + """ + stale_predecessor(cluster) + win_on_attempt_two(cluster) + assert 'pc-r100-a1' in cluster.jobs() # the slot did free up + + result = cluster.reconcile() + + assert '300/420' not in result['in_progress'] + assert sorted(result['in_progress']) == ['100/420', '200/420'] + assert len(result['in_progress']) <= jm.PARALLELISM + + +def test_the_range_scoped_reap_still_waits_for_the_done_marker(cluster): + """Widening the reap from one attempt to the whole range must not widen + *when* it fires. Deleting a Job reaps its pod, and .metrics is the only + place peaks live, so nothing may be reaped before the collector has + written `.done` -- JOB_TTL_SECONDS is the backstop for a collector that + never gets there. + + This is the behavioural half of the guarantee that + test_job_monitor.py::test_the_reap_waits_for_the_collectors_done_marker + asserts by extracting and exec'ing the function's source. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + + cluster.reconcile() # recorded; collector not done + assert '300' in cluster.completed() + assert jobs_for(cluster, 300) == ['pc-r300-a1'] + assert cluster.deleted.names(verb='delete', kind='job') == [] + + cluster.finalize(300, 1, tx_apply=0.1, peaks={'peakRssBytes': 1}) + cluster.reconcile() + assert jobs_for(cluster, 300) == [] + assert cluster.deleted.names(verb='delete', kind='job') == ['pc-r300-a1'] + + +def test_remaining_never_goes_negative_and_the_run_reports_done(cluster, + monkeypatch): + """The mission driver waits for `remaining == 0 and in_progress == []`. + + With every range finished, that condition must hold and keep holding. + + RED: the leftover Failed a1 puts range 300 back into in_progress while it + is also in completed, so it is subtracted twice -- remaining reads -1, and + in_progress is never empty, so the driver's completion test never fires. + """ + monkeypatch.setattr(jm, 'PARALLELISM', 3) # all three ranges at once + + cluster.reconcile() + cluster.advance(300, 'disrupted') + cluster.reconcile() # 300/a1 Failed and unfinalized + + for end, attempt in ((300, 2), (200, 1), (100, 1)): + cluster.advance(end, 'succeeded', attempt=attempt) + cluster.finalize(end, attempt, tx_apply=0.1, peaks={'peakRssBytes': 1}) + done = cluster.reconcile() + + assert done['completed'] == 3 + assert done['in_progress'] == [] + assert done['remaining'] == 0 + + # ...and it stays done. This is the pass that re-dispatches under the bug. + again = cluster.reconcile() + assert again['completed'] == 3 + assert again['remaining'] == 0 # reads -1 while the bug is present + assert again['in_progress'] == [] + assert again['created'] == 0 + assert cluster.jobs() == [] diff --git a/src/MissionParallelCatchup/test_race_2.py b/src/MissionParallelCatchup/test_race_2.py new file mode 100644 index 00000000..500ff95c --- /dev/null +++ b/src/MissionParallelCatchup/test_race_2.py @@ -0,0 +1,188 @@ +"""RACE #2: a txApply that arrives after the range is first recorded is +backfilled into progress.json but never reaches the Prometheus histogram. + +The interleaving under test is the ordinary one at 1024 workers: the Job flips +to succeeded and reconcile records the range before the log-collector sidecar +has flushed that attempt's .metrics, so the first record carries txApply=None. +A later pass backfills the real value into progress.json. The histogram is +supposed to be a replay of the recorded ranges, so once progress.json says +txApply=1.25 the histogram must have counted 1.25 -- exactly once. + +Every assertion here is on observed state: the durable progress record on the +fake logs volume, and the samples the Prometheus client actually exports. +Nothing reads job_monitor's source. +""" + +import job_monitor as jm + + +# --- reading the exported metric -------------------------------------------- +# +# The histograms are module-level and share the global REGISTRY, so absolute +# values leak across tests in one process. Every assertion below is therefore a +# delta taken inside a single test. This reads the exported samples -- the same +# numbers /metrics would serve -- not any private attribute. + +def _hist(metric): + """(count, sum) of a label-less Histogram, from its exported samples.""" + count = total = 0.0 + for family in metric.collect(): + for s in family.samples: + if s.name.endswith('_count'): + count = s.value + elif s.name.endswith('_sum'): + total = s.value + return count, total + + +def _delta(before, after): + return after[0] - before[0], round(after[1] - before[1], 9) + + +def _succeed_without_metrics(cluster, end): + """Job succeeds, collector has not written anything for it yet. + + No .metrics, no .log.gz, and the fake pod log is empty, so all three of + tx_apply_for_range's sources come up dry -- which is exactly the state the + monitor is in when it records the range in the same second the Job flips. + """ + cluster.reconcile() + cluster.advance(end, 'succeeded') + cluster.reconcile() + + +def test_late_txapply_reaches_the_histogram_not_just_progress_json(cluster): + """The bug, stated as the disagreement it causes. + + progress.json ends up saying txApply is known for the range while the + histogram never counted it, so the artifact and /metrics describe different + runs. + """ + _succeed_without_metrics(cluster, 300) + + # Precondition: recorded, but with no txApply yet. If this ever stops + # holding the test below is not exercising the race any more. + assert cluster.completed()['300']['txApply'] is None + + before = _hist(jm.metric_tx_apply_duration) + + # The collector finishes and flushes the attempt's measurements. + cluster.finalize(300, 1, tx_apply=1.25) + cluster.reconcile() + + # The durable artifact now claims the value is known... + assert cluster.progress()['completed']['300']['txApply'] == 1.25 + + # ...so the histogram must have counted that same value. + count, total = _delta(before, _hist(jm.metric_tx_apply_duration)) + assert (count, total) == (1.0, 1.25), ( + "progress.json carries txApply=1.25 for range 300 but the histogram " + f"observed count+{count} sum+{total}: the backfilled value can never " + "be counted, so /metrics under-reports every range whose .metrics " + "landed after the range was first recorded") + + +def test_backfilled_txapply_is_counted_once_not_on_every_later_pass(cluster): + """The other half of the contract: exactly once, not once per pass. + + A fix that simply stops skipping the range would re-observe the value on + every subsequent reconcile, which at a 10s loop inflates the histogram + without bound. + """ + _succeed_without_metrics(cluster, 300) + before = _hist(jm.metric_tx_apply_duration) + + cluster.finalize(300, 1, tx_apply=1.25) + for _ in range(4): + cluster.reconcile() + + assert cluster.progress()['completed']['300']['txApply'] == 1.25 + count, total = _delta(before, _hist(jm.metric_tx_apply_duration)) + assert (count, total) == (1.0, 1.25), ( + f"range 300's txApply was observed {count} times across four passes; " + "the histogram must count each recorded range exactly once") + + +def test_durations_recorded_up_front_are_not_recounted_while_txapply_is_late(cluster): + """seconds/wallSeconds are known on the first record and must stay at one. + + This is the failure mode of the tempting one-line fix (move the guard + inside the txApply branch): the range then stays unmarked for as many + passes as the collector takes, and every one of those passes re-observes + the durations it already counted. The two histograms would drift apart in + opposite directions. + """ + _succeed_without_metrics(cluster, 300) + + rec = cluster.completed()['300'] + assert rec['seconds'] is not None and rec['wallSeconds'] is not None + seconds, wall = rec['seconds'], rec['wallSeconds'] + + before_full = _hist(jm.metric_full_duration) + before_wall = _hist(jm.metric_wall_duration) + + # Three passes with the collector still silent, then it finally lands. + for _ in range(3): + cluster.reconcile() + cluster.finalize(300, 1, tx_apply=0.5) + cluster.reconcile() + cluster.reconcile() + + assert cluster.progress()['completed']['300']['txApply'] == 0.5 + + assert _delta(before_full, _hist(jm.metric_full_duration)) == (0.0, 0.0), ( + "the full-duration histogram re-observed range 300's already-counted " + f"{seconds}s while waiting for its txApply") + assert _delta(before_wall, _hist(jm.metric_wall_duration)) == (0.0, 0.0), ( + "the wall-duration histogram re-observed range 300's already-counted " + f"{wall}s while waiting for its txApply") + + +def test_txapply_present_on_first_sight_is_still_counted_exactly_once(cluster): + """Baseline: the non-racing order must keep working. + + Collector finalizes before the monitor ever sees the Job, so txApply is + known at first record. One observation, and no second one later. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1, tx_apply=2.5) + + before = _hist(jm.metric_tx_apply_duration) + cluster.reconcile() + cluster.reconcile() + cluster.reconcile() + + assert cluster.progress()['completed']['300']['txApply'] == 2.5 + assert _delta(before, _hist(jm.metric_tx_apply_duration)) == (1.0, 2.5) + + +def test_two_ranges_landing_their_metrics_at_different_times_both_count(cluster): + """The population-level consequence, at the smallest scale that shows it. + + One range's .metrics is ready on the first pass and the other's is not. + Both end up in progress.json with a txApply, so the histogram must contain + both -- not just the one that happened to win the race. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.advance(200, 'succeeded') + # 300's collector is quick; 200's is not. + cluster.finalize(300, 1, tx_apply=1.0) + + before = _hist(jm.metric_tx_apply_duration) + cluster.reconcile() + + assert cluster.completed()['200']['txApply'] is None + + cluster.finalize(200, 1, tx_apply=3.0) + cluster.reconcile() + + recorded = {k: v['txApply'] for k, v in cluster.completed().items()} + assert recorded == {'300': 1.0, '200': 3.0} + + count, total = _delta(before, _hist(jm.metric_tx_apply_duration)) + assert (count, total) == (2.0, 4.0), ( + f"progress.json holds txApply for {sorted(recorded)} but the histogram " + f"counted {count} of them (sum {total}); only the range whose .metrics " + "was ready on the first pass was observed") diff --git a/src/MissionParallelCatchup/test_race_3.py b/src/MissionParallelCatchup/test_race_3.py new file mode 100644 index 00000000..528a937f --- /dev/null +++ b/src/MissionParallelCatchup/test_race_3.py @@ -0,0 +1,315 @@ +"""RACE #3 -- a torn .log.gz member kills a whole reconcile pass. + +The log-collector appends a gzip member to range--a.log.gz in place, +with no temp+rename, so a reader that looks while a poll is mid-write sees a +truncated member. job_monitor reads that same file to recover txApplySeconds +and guards it with `except OSError`, which does not cover the EOFError that +gzip raises on a truncated member. + +Consequence: one in-flight log write aborts the entire reconcile pass -- no +recording, no PVC release, no reaping and no dispatch for ANY of the ~4000 +ranges, not just the one whose archive was being written. And because the torn +bytes stay on disk, it repeats on every subsequent pass. + +Every test here drives the real code and asserts on observed state: what +reconcile() returned, what landed in progress.json, which Jobs/PVCs exist, and +what a reader sees on disk while the collector writes. +""" + +import asyncio +import gzip +import io +import os +import random + +import pytest + +import job_monitor as jm +import log_collector as lc + + +# --- building the artefact the race leaves on disk -------------------------- + +def _gzip_member(text): + """One complete, self-contained gzip member -- what one finished poll adds.""" + buf = io.BytesIO() + with gzip.GzipFile(fileobj=buf, mode='wb', mtime=0) as fh: + fh.write(text.encode()) + return buf.getvalue() + + +def write_torn_archive(path, settled="startup line\n", in_flight=None): + """A .log.gz exactly as an interrupted in-place append leaves it. + + One complete member from an earlier poll, followed by the first half of the + member the current poll is still writing. This is byte-for-byte the shape an + in-place gzip append produces once its buffer has flushed but the member has + not been closed; the live-writer version of the same thing is + test_collector_append_never_exposes_a_partial_member_to_a_reader below. + """ + if in_flight is None: + in_flight = "".join(f"line {i} of the poll that is still running\n" + for i in range(200)) + partial = _gzip_member(in_flight) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'wb') as fh: + fh.write(_gzip_member(settled)) + fh.write(partial[:max(24, len(partial) // 2)]) + # Guard: the file we just built must actually be the torn artefact, or the + # test below would pass for the wrong reason. + with pytest.raises(EOFError): + with gzip.open(path, 'rt') as fh: + fh.read() + return path + + +CORE_TAIL = ( + "2026-07-30T00:00:00Z metric 'ledger.transaction.apply'\n" + "2026-07-30T00:00:00Z count = 12345\n" + "2026-07-30T00:00:00Z sum = 4200.0ms\n" +) + + +def write_whole_archive(path, text=CORE_TAIL): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'wb') as fh: + fh.write(_gzip_member(text)) + return path + + +# --- reader side: the reconcile pass ---------------------------------------- + +def test_a_torn_archive_does_not_abort_the_reconcile_pass(cluster): + """A succeeded range whose archive is mid-append must still be recorded. + + Nothing about this range is unusual apart from the collector happening to + be writing when reconcile looked. + """ + cluster.reconcile() # dispatches r300, r200 + cluster.advance(300, 'succeeded') + # The collector has not flushed .metrics yet, so the archive is the only + # source for txApply -- and it is exactly the file being written. + write_torn_archive(jm.log_path('300', 1)) + + result = cluster.reconcile() + + # The pass completed and did all of its work. + assert '300' in cluster.completed(), "succeeded range was never recorded" + assert cluster.completed()['300']['attempts'] == 1 + assert 'pc-data-r300' not in cluster.pvcs(), "completed range kept its volume" + assert result['created'] == 1, "the freed slot was never refilled" + assert 'pc-r100-a1' in cluster.jobs() + # The unreadable archive costs the metric for this range, nothing more. + assert cluster.completed()['300']['txApply'] is None + + +def test_a_torn_archive_costs_one_range_not_the_other_ranges_in_the_pass(cluster): + """Blast radius. Two ranges finish together; one has a torn archive. + + The healthy one must be recorded, keep its measurements and be reaped in + the same pass, and the third range must still be dispatched. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.advance(200, 'succeeded') + # r300: collector finished cleanly. + cluster.finalize(300, 1, tx_apply=12.5, peaks={'peakRssBytes': 111}) + # r200: collector is mid-poll, archive torn, nothing durable yet. + write_torn_archive(jm.log_path('200', 1)) + + result = cluster.reconcile() + + completed = cluster.completed() + assert set(completed) == {'300', '200'} + # The healthy range is untouched by its neighbour's corrupt file. + assert completed['300']['txApply'] == 12.5 + assert completed['300']['peakRssBytes'] == 111 + assert 'pc-r300-a1' not in cluster.jobs(), "finalized range was not reaped" + # The torn range pays, and only the torn range. + assert completed['200']['txApply'] is None + # Dispatch still happened. + assert result['created'] == 1 + assert 'pc-r100-a1' in cluster.jobs() + assert cluster.failed() == {} + + +def test_a_never_repaired_torn_archive_does_not_wedge_the_run(cluster): + """The torn bytes are durable, so the reader hits them on every pass. + + Once a range is recorded with txApply=None the backfill branch re-reads the + archive each cycle, so a single corrupt file is not a one-pass outage -- it + stops the run permanently. Drive the whole run to completion over it. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.advance(200, 'succeeded') + cluster.finalize(200, 1, tx_apply=7.0) + # r300's archive is torn and nobody ever fixes it. + torn = write_torn_archive(jm.log_path('300', 1)) + + cluster.reconcile() # records 300 + 200, dispatches 100 + assert 'pc-r100-a1' in cluster.jobs() + cluster.advance(100, 'succeeded') + cluster.finalize(100, 1, tx_apply=3.0) + + result = cluster.reconcile() # records 100 + result = cluster.reconcile() # steady state, still re-reading 300 + + assert os.path.exists(torn), "test no longer exercises the corrupt file" + assert set(cluster.completed()) == {'300', '200', '100'} + assert cluster.failed() == {} + assert result['remaining'] == 0 + assert result['in_progress'] == [] + + +def test_a_range_recovers_its_metric_once_the_collector_finishes(cluster): + """Bounded in time as well as in scope. + + The torn read costs txApply for exactly as long as the archive is torn: the + moment the collector lands .metrics, the backfill branch picks it up. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + write_torn_archive(jm.log_path('300', 1)) + + cluster.reconcile() + assert cluster.completed()['300']['txApply'] is None + + # The collector's poll completes and it writes what it scanned out of the + # stream. The archive on disk is still torn. + cluster.finalize(300, 1, tx_apply=88.25, peaks={'peakRssBytes': 222}) + cluster.reconcile() + + assert cluster.completed()['300']['txApply'] == 88.25 + assert cluster.completed()['300']['peakRssBytes'] == 222 + assert 'pc-r300-a1' not in cluster.jobs() + + +def test_a_readable_archive_is_still_the_txapply_fallback(cluster): + """Guard rail: widening the except must not swallow a good read. + + Without this, 'catch everything and return None' would pass every test + above while silently deleting the archive fallback. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + write_whole_archive(jm.log_path('300', 1)) # no .metrics: archive is the source + + cluster.reconcile() + + assert cluster.completed()['300']['txApply'] == pytest.approx(4.2) + + +# --- writer side: the collector's append ------------------------------------ + +class _FakeContent: + def __init__(self, body): + self._body = body.encode() + + async def iter_chunked(self, n): + for i in range(0, len(self._body), n): + yield self._body[i:i + n] + + +class _FakeResponse: + status = 200 + + def __init__(self, body): + self.content = _FakeContent(body) + + def raise_for_status(self): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + +class _FakeSession: + """Just enough aiohttp for _poll_once: one GET returning a log body.""" + + def __init__(self, body): + self._body = body + + def get(self, url, params=None, headers=None): + return _FakeResponse(self._body) + + +class _ReadingScanner(lc.TxApplyScanner): + """The monitor, reading the archive while the collector writes it. + + feed() is called once per log line from inside the collector's write loop, + which makes "a reader looked mid-append" deterministic instead of a timing + coin flip. + """ + + def __init__(self, path, every=250): + super().__init__() + self.path = path + self.every = every + self.lines = 0 + self.observations = [] + self.errors = [] + + def feed(self, line): + super().feed(line) + self.lines += 1 + if self.lines % self.every: + return + try: + with gzip.open(self.path, 'rt') as fh: + self.observations.append(fh.read()) + except Exception as exc: # noqa: BLE001 -- that's the point + self.errors.append((self.lines, type(exc).__name__)) + + +def _log_body(start, count, rng): + """Timestamped, poorly-compressible pod log lines, as kubelet serves them.""" + return "".join( + "2026-07-30T00:00:00.%09dZ %064x %064x\n" + % (i, rng.getrandbits(256), rng.getrandbits(256)) + for i in range(start, start + count) + ) + + +def test_collector_append_never_exposes_a_partial_member_to_a_reader(tmp_path, monkeypatch): + """The archive on disk must only ever hold complete members. + + Two polls. The first settles a complete member. The second is a large poll + -- well inside MAX_POLL_CHARS -- during which a reader inspects the file + every 250 lines. Every one of those reads must succeed and must see exactly + the last settled content. + """ + monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(lc, 'token', lambda: 'test-token') + path = lc.base('300', 1) + '.log.gz' + rng = random.Random(7) + + first = asyncio.run(lc._poll_once( + _FakeSession(_log_body(0, 5, rng)), 'pod-a', '300', 1, None, + lc.TxApplyScanner())) + last_ts, gone = first + assert not gone + with gzip.open(path, 'rt') as fh: + settled = fh.read() + assert settled, "first poll wrote nothing; the test has no baseline" + + watcher = _ReadingScanner(path) + asyncio.run(lc._poll_once( + _FakeSession(_log_body(5, 12000, rng)), 'pod-a', '300', 1, last_ts, watcher)) + + assert watcher.observations, "the reader never got to look" + assert watcher.errors == [], ( + f"{len(watcher.errors)} of {len(watcher.errors) + len(watcher.observations)} " + f"mid-append reads hit a torn member, e.g. {watcher.errors[:3]}") + assert set(watcher.observations) == {settled}, ( + "a reader saw content that was neither the previous complete archive " + "nor the finished one") + + # And the append still did its job once it finished. + with gzip.open(path, 'rt') as fh: + final = fh.read() + assert final.startswith(settled) + assert len(final.splitlines()) == 12005 diff --git a/src/MissionParallelCatchup/test_race_4.py b/src/MissionParallelCatchup/test_race_4.py new file mode 100644 index 00000000..f9041198 --- /dev/null +++ b/src/MissionParallelCatchup/test_race_4.py @@ -0,0 +1,226 @@ +"""RACE #4 -- the _wake Event is never cleared, so the terminal-poll backoff +never sleeps and TERMINAL_POLL_ATTEMPTS is spent in a tight loop. + +These tests run the real `log_collector.poll_pod` against a fake kubelet log +endpoint and assert on what ends up on the logs volume (.log.gz, .metrics) and +on when the requests were actually issued. No source text is inspected: the +existing suite already pattern-matches this loop and still shipped the bug. + +The scenario is the one that costs data in production: a worker pod goes +terminal and the kubelet needs a moment before it will serve the container's +final log. The collector is supposed to absorb that with three spaced retries. +With a sticky Event it burns all three inside a millisecond and finalizes on +nothing, losing the range's log, its txApply and its final peaks. +""" + +import asyncio +import gzip +import json +import os +import time + +import pytest + +import log_collector as lc + +# Everything is scaled down from the shipped 10s so the tests run in ~2s. The +# ratios are what matter: the kubelet gate opens well after a millisecond-fast +# giveup and well before the third attempt of a correctly-spaced retry. +POLL = 0.2 # LOG_POLL_SECONDS under test +GATE = 0.4 # how long the kubelet 500s before serving the final log +ATTEMPTS = 3 # TERMINAL_POLL_ATTEMPTS, the shipped default + +POD = 'pc-r300-a1-00001' +END = '300' +ATTEMPT = '1' + +# What stellar-core prints on its way out. The medida block is the only place +# txApply exists -- the pod is about to be reaped, so if this read is missed the +# number is gone for good. +FINAL_LOG = ( + "2026-07-30T00:00:01Z catchup ledger 42000000\n" + "2026-07-30T00:00:02Z metric 'ledger.transaction.apply'\n" + "2026-07-30T00:00:03Z count = 12\n" + "2026-07-30T00:00:04Z sum = 1500.0ms\n" + "2026-07-30T00:00:05Z catchup completed\n" +) +EXPECTED_TX_SECONDS = 1.5 + + +# --- fake kubelet log endpoint ---------------------------------------------- + +class _Resp: + def __init__(self, status, body): + self.status = status + self._body = body.encode() + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + def raise_for_status(self): + if self.status >= 400: + raise RuntimeError(f"HTTP {self.status}") + + @property + def content(self): + body = self._body + + class _Chunks: + async def iter_chunked(self, n): + for i in range(0, len(body), n): + yield body[i:i + n] + + return _Chunks() + + +class FakeKubelet: + """Serves one pod's log, with a delay before the final read is available. + + `open_after=None` never serves. Timestamps every request so a test can see + whether the retries were spaced or fired back to back. + """ + + def __init__(self, open_after, body=FINAL_LOG): + self.open_after = open_after + self.body = body + self.requests = [] + + def get(self, url, params=None, headers=None): + now = time.monotonic() + self.requests.append(now) + if self.open_after is None or now - self.requests[0] < self.open_after: + return _Resp(500, '') + return _Resp(200, self.body) + + @property + def span(self): + return self.requests[-1] - self.requests[0] + + +# --- driver ------------------------------------------------------------------ + +async def _drive(session, terminal_at_start=True, flip_after=None, timeout=10): + """Run the real poll_pod, with a stand-in for one main-loop wake cycle. + + main() marks a pod terminal and then does `if name in _wake: set()`. That + key only exists once the poller has reached its first wait, so the real loop + lands the wake on a later cycle -- reproduced here by waiting for the key. + The wake is delivered ONCE, as one main-loop cycle would: the bug is that + one set is enough to disable every wait that follows. + """ + terminal = {'v': terminal_at_start} + + async def main_loop_wake(): + if flip_after is not None: + await asyncio.sleep(flip_after) + terminal['v'] = True + while POD not in lc._wake: + await asyncio.sleep(0.001) + lc._wake[POD].set() + + waker = asyncio.create_task(main_loop_wake()) + try: + await asyncio.wait_for( + lc.poll_pod(session, POD, END, ATTEMPT, + lambda p: terminal['v'], # done() + lambda p: False), # done_ok(): pod Failed + timeout=timeout) + finally: + waker.cancel() + + +@pytest.fixture +def logs(tmp_path, monkeypatch): + monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(lc, 'token', lambda: 'tok') + monkeypatch.setattr(lc, 'LOG_POLL_SECONDS', POLL) + monkeypatch.setattr(lc, 'TERMINAL_POLL_ATTEMPTS', ATTEMPTS) + for d in (lc._wake, lc._pod_secs, lc._anon_peak, lc._ws_peak, + lc._eph_peak, lc._peak_flushed, lc._streaming): + d.clear() + yield tmp_path + for d in (lc._wake, lc._pod_secs, lc._anon_peak, lc._ws_peak, + lc._eph_peak, lc._peak_flushed, lc._streaming): + d.clear() + + +def _metrics(d): + path = os.path.join(d, f'range-{END}-a{ATTEMPT}.metrics') + if not os.path.exists(path): + return {} + with open(path) as fh: + return json.load(fh) + + +def _archive(d): + path = os.path.join(d, f'range-{END}-a{ATTEMPT}.log.gz') + if not os.path.exists(path): + return '' + with gzip.open(path, 'rt') as fh: + return fh.read() + + +def _done(d): + return os.path.exists(os.path.join(d, f'range-{END}-a{ATTEMPT}.done')) + + +# --- tests ------------------------------------------------------------------- + +def test_a_terminal_pods_final_log_survives_a_moment_of_kubelet_lag(logs): + # The pod is already terminal when its stream opens (Failed is a pollable + # phase). The kubelet cannot serve the container's log yet -- the ordinary + # case, it needs a moment after termination -- so the first reads 500. + # TERMINAL_POLL_ATTEMPTS exists precisely to ride that out, and the gate + # here opens inside the window three spaced retries cover. + kubelet = FakeKubelet(open_after=GATE) + asyncio.run(_drive(kubelet)) + + assert _done(logs), "attempt never finalized" + assert len(kubelet.requests) == ATTEMPTS, ( + f"expected the {ATTEMPTS}-attempt budget, saw {len(kubelet.requests)}") + + m = _metrics(logs) + assert m.get('txApplySeconds') == EXPECTED_TX_SECONDS, ( + "the retry budget was spent before the kubelet could answer: txApply " + f"lost (metrics={m}, retries spanned {kubelet.span * 1000:.1f}ms)") + assert 'sum = 1500.0ms' in _archive(logs), ( + "the range's final log was never captured") + assert 'catchup completed' in _archive(logs) + + +def test_the_terminal_retry_budget_is_spent_over_time_not_in_one_millisecond(logs): + # Same pod, but the log endpoint never recovers. What is under test is the + # shape of the giveup: three attempts must be spread across the backoff, + # not fired back to back. Anything less and the budget is decorative. + kubelet = FakeKubelet(open_after=None) + asyncio.run(_drive(kubelet)) + + assert _done(logs), "attempt never finalized" + assert len(kubelet.requests) == ATTEMPTS, ( + f"expected the {ATTEMPTS}-attempt budget, saw {len(kubelet.requests)}") + assert kubelet.span >= POLL, ( + f"{ATTEMPTS} terminal polls were spent in {kubelet.span * 1000:.1f}ms; " + f"they should span at least one {POLL}s backoff") + + +def test_a_pod_going_terminal_still_cuts_the_routine_wait_short(logs, monkeypatch): + # Guard on the other side of the fix: clearing the Event must not turn the + # wait back into a blind sleep. The poll interval is 5s here; the pod goes + # terminal just after the first poll, and its final read has to happen + # within the pod-list cadence, not 5s later. + monkeypatch.setattr(lc, 'LOG_POLL_SECONDS', 5.0) + kubelet = FakeKubelet(open_after=0) # always serves + + started = time.monotonic() + asyncio.run(_drive(kubelet, terminal_at_start=False, flip_after=0.05, + timeout=2)) + elapsed = time.monotonic() - started + + assert _done(logs) + assert _metrics(logs).get('txApplySeconds') == EXPECTED_TX_SECONDS + assert elapsed < 2, ( + f"final read waited {elapsed:.2f}s for a pod that went terminal " + "immediately; the wake was not delivered") diff --git a/src/MissionParallelCatchup/test_race_5.py b/src/MissionParallelCatchup/test_race_5.py new file mode 100644 index 00000000..ad28d5a3 --- /dev/null +++ b/src/MissionParallelCatchup/test_race_5.py @@ -0,0 +1,203 @@ +"""RACE #5 -- attempt budgets are spent from one shared counter. + +Every retry bumps the same attempt index. The cap is then picked from whatever +the LATEST verdict happened to be, and the global index is compared against it. +So cluster churn (spot evictions, admission rejections, monitor restarts) -- +which has its own, deliberately large budget -- silently drains the small +budgets belonging to the causes that actually say something about the range. + +A range evicted five times arrives at attempt 6. Its FIRST genuine OOM is then +compared 6 >= MAX_ATTEMPTS(5) and condemned, having never once been retried for +an OOM and never once had its memory escalated. A condemned range fails the +whole mission. + +Everything below is observed state: which Jobs exist, what resources they were +created with, and what landed in progress.json's failed{}. No source text. +""" + +import job_monitor as jm + + +# --- helpers (local to this file on purpose) -------------------------------- + +def dispatch(cluster, end=300): + """Get `end` to attempt 1, running, with nothing else in the way.""" + cluster.reconcile() + assert cluster.attempt_of(end) == 1 + return end + + +def hit(cluster, end, state, times=1): + """Fail the range's newest attempt `times` times in a row with `state`. + + One reconcile per failure, which is what the real loop does: the monitor + sees the failed Job, decides retry-or-condemn, and (if retrying) creates + the successor before the next pass. + """ + for _ in range(times): + cluster.advance(end, state) + cluster.reconcile() + + +def job_exists(cluster, end, attempt): + return jm.job_name(int(end), attempt) in cluster.jobs() + + +def mem_of(cluster, end, attempt): + job = cluster.k8s.job(jm.job_name(int(end), attempt)) + return job.spec.template.spec.containers[0].resources + + +def condemned(cluster, end): + return str(end) in cluster.failed() + + +# --- the race --------------------------------------------------------------- + +def test_five_evictions_do_not_burn_the_whole_oom_budget(cluster): + """5 spot evictions (budget 20) must leave the OOM budget (5) untouched. + + Under the bug the range is on attempt 6 when its first OOM lands, 6 >= 5, + and it is condemned without ever being retried for the OOM -- so its memory + is never escalated and the mission fails on a range that is merely unlucky. + """ + end = dispatch(cluster) + hit(cluster, end, 'disrupted', times=5) + # Five evictions are legal on the disruption budget: the range is alive. + assert not condemned(cluster, end) + assert cluster.attempt_of(end) == 6 + + hit(cluster, end, 'oom') + + assert not condemned(cluster, end), ( + "first OOM after eviction churn condemned the range: the eviction " + f"retries spent the OOM budget. failed={cluster.failed()}") + assert job_exists(cluster, end, 7), ( + f"no attempt 7 was dispatched; live jobs are {cluster.jobs()}") + + # And the whole point of an OOM retry: more memory. One OOM = one rung. + res = mem_of(cluster, end, 7) + assert res.limits['memory'] == '36000Mi' + assert res.requests['memory'] == '36000Mi' + + +def test_evictions_do_not_burn_the_disk_budget(cluster, monkeypatch): + """Same shape, ephemeral-storage budget (4). Disk evictions repeat until + the range gets more disk, so losing that budget to churn is terminal.""" + monkeypatch.setattr(jm, 'LIM_EPHEMERAL', '40Gi') + monkeypatch.setattr(jm, 'REQ_EPHEMERAL', '40Gi') + + end = dispatch(cluster) + hit(cluster, end, 'disrupted', times=5) + assert cluster.attempt_of(end) == 6 + + hit(cluster, end, 'ephemeral') + + assert not condemned(cluster, end), ( + "first disk eviction after eviction churn condemned the range; " + f"failed={cluster.failed()}") + assert job_exists(cluster, end, 7) + # Retried with MORE disk than it just outgrew, not the same 40Gi. + grown = mem_of(cluster, end, 7).limits['ephemeral-storage'] + assert jm._quantity_bytes(grown) > jm._quantity_bytes('40Gi'), grown + + +def test_evictions_do_not_burn_the_timeout_budget(cluster): + """Timeout budget is only 2, so churn eats it almost immediately.""" + end = dispatch(cluster) + hit(cluster, end, 'disrupted', times=3) + assert cluster.attempt_of(end) == 4 + + hit(cluster, end, 'timeout') + + assert not condemned(cluster, end), ( + f"first timeout after 3 evictions condemned the range; " + f"failed={cluster.failed()}") + assert job_exists(cluster, end, 5) + + +def test_evictions_do_not_burn_the_range_budget_for_exit_3(cluster): + """exit 3 ("did not complete") rides the ordinary range budget of 5. + + A range evicted five times gets zero exit-3 retries -- and exit 3 is the + outcome an interrupted-then-resumable range produces, so the retry that was + denied is the one that would have succeeded. + """ + end = dispatch(cluster) + hit(cluster, end, 'disrupted', times=5) + assert cluster.attempt_of(end) == 6 + + hit(cluster, end, 'incomplete') + + assert not condemned(cluster, end), ( + f"first exit-3 after eviction churn condemned the range; " + f"failed={cluster.failed()}") + assert job_exists(cluster, end, 7) + + +def test_memory_ladder_follows_ooms_not_evictions(cluster, monkeypatch): + """Interleaved churn: each OOM must climb exactly one rung, and the second + OOM must still be inside the budget even though the attempt index is 8.""" + # The shipped 48Gi ceiling clamps rung 2 to the same figure rung 8 would + # give, which would make the ladder assertion below prove nothing. Raise it + # so the rung is observable; the budget behaviour under test is unaffected. + monkeypatch.setattr(jm, 'MEM_ESCALATION_CAP', '128Gi') + end = dispatch(cluster) + hit(cluster, end, 'disrupted', times=3) # attempts 1-3, now on 4 + hit(cluster, end, 'oom') # OOM #1 on attempt 4 -> a5 + assert job_exists(cluster, end, 5) + assert mem_of(cluster, end, 5).limits['memory'] == '36000Mi' + + hit(cluster, end, 'disrupted', times=2) # attempts 5-6, now on 7 + hit(cluster, end, 'oom') # OOM #2 on attempt 7 -> a8 + + assert not condemned(cluster, end), f"failed={cluster.failed()}" + assert job_exists(cluster, end, 8) + # 24000Mi * 1.5^2 -- two OOMs, six evictions, two rungs. + assert mem_of(cluster, end, 8).limits['memory'] == '54000Mi' + + +# --- the caps must still bind (a fix that just removes them is not a fix) ---- + +def test_the_oom_budget_still_binds(cluster): + """Five real OOMs in a row exhaust the OOM budget and condemn the range.""" + end = dispatch(cluster) + hit(cluster, end, 'oom', times=5) + + assert condemned(cluster, end), ( + f"five consecutive OOMs were not condemned; jobs={cluster.jobs()}") + assert cluster.failed()[str(end)]['outcome'] == 'oom' + assert not job_exists(cluster, end, 6), ( + f"a 6th OOM attempt was dispatched past the budget: {cluster.jobs()}") + + +def test_the_timeout_budget_still_binds(cluster): + """Two real timeouts exhaust MAX_TIMEOUT_ATTEMPTS.""" + end = dispatch(cluster) + hit(cluster, end, 'timeout', times=2) + + assert condemned(cluster, end), ( + f"two consecutive timeouts were not condemned; jobs={cluster.jobs()}") + assert cluster.failed()[str(end)]['outcome'] == 'timeout' + assert not job_exists(cluster, end, 3) + + +def test_the_disruption_budget_still_binds(cluster): + """Twenty evictions really is the end of the road for a range.""" + end = dispatch(cluster) + hit(cluster, end, 'disrupted', times=jm.MAX_DISRUPTION_ATTEMPTS) + + assert condemned(cluster, end), ( + f"{jm.MAX_DISRUPTION_ATTEMPTS} evictions were not condemned; " + f"jobs={cluster.jobs()}") + assert not job_exists(cluster, end, jm.MAX_DISRUPTION_ATTEMPTS + 1) + + +def test_a_genuine_catchup_failure_is_still_never_retried(cluster): + """exit 1 is condemned on attempt 1 regardless of any tally.""" + end = dispatch(cluster) + hit(cluster, end, 'condemned') + + assert condemned(cluster, end) + assert cluster.failed()[str(end)]['attempts'] == 1 + assert not job_exists(cluster, end, 2) diff --git a/src/MissionParallelCatchup/test_race_6.py b/src/MissionParallelCatchup/test_race_6.py new file mode 100644 index 00000000..818b4896 --- /dev/null +++ b/src/MissionParallelCatchup/test_race_6.py @@ -0,0 +1,309 @@ +"""RACE #6 -- the attempt deadline is on the wrong object, and it outranks the pod. + +Two independent defects, both run-ending, both driven here through the real +reconcile() against the fake cluster: + +A. `activeDeadlineSeconds` sits on the JobSpec, so the clock starts when the Job + is created rather than when the container starts. Every second a pod spends + Pending -- waiting for Karpenter, waiting for an image pull -- is charged + against a budget that is meant to bound how long the RANGE runs. During the + node-class outage this run really did sit ~15 minutes Pending, and ranges + then died as "timeouts" having barely executed. + +B. When the Job reports DeadlineExceeded the monitor takes that verdict + unconditionally, over the pod's own terminated reason. A pod the kubelet + OOM-killed inside a Job that also tripped its deadline is filed as a timeout: + no memory escalation, and MAX_TIMEOUT_ATTEMPTS (2) instead of the budget the + real cause earns. Two such events condemn the range and fail the mission. + +Nothing here asserts on source text. Facet B is fully drivable with the shipped +harness. Facet A needs the one thing the fake cluster does not have -- the piece +of Kubernetes that actually enforces a deadline -- so `_DeadlineController` +below supplies it. It is a model of *Kubernetes*, not of job_monitor: it reads +whichever field the monitor set and applies the clock that Kubernetes documents +for that field. A monitor that puts the deadline in the right place survives it; +one that puts it in the wrong place does not. +""" + +import pytest + +import job_monitor as jm + + +DEADLINE = 600 # ATTEMPT_DEADLINE_SECONDS for the facet-A tests + + +# --- the bit of Kubernetes that enforces activeDeadlineSeconds --------------- + +class _DeadlineController: + """Two fields, two clocks. That difference is the entire bug. + + * `JobSpec.activeDeadlineSeconds` is measured from `job.status.startTime`, + which the Job controller stamps when the Job is admitted -- before any pod + is scheduled. Pending time counts against it. On expiry the Job is + terminated with a Failed condition, reason=DeadlineExceeded. + + * `PodSpec.activeDeadlineSeconds` is measured from the pod's own start time, + set by the kubelet when the pod starts running. Pending time does not + count. On expiry the pod is killed (SIGTERM; stellar-core drains and exits + 3) and marked Failed with reason=DeadlineExceeded, and the Job then fails + through its podFailurePolicy like any other non-zero exit. + + This class knows nothing about which one job_monitor chose -- it reads the + Job it was handed. + """ + + @staticmethod + def deadlines(job): + pod_spec = job.spec.template.spec + return (job.spec.active_deadline_seconds, + getattr(pod_spec, 'active_deadline_seconds', None)) + + @classmethod + def run_attempt(cls, cluster, end, pending_seconds, running_seconds, + finishes='succeeded'): + """Play one attempt's timeline out against whatever deadline is set. + + Returns the state the cluster ended up in: 'timeout' if a deadline + fired, otherwise `finishes`. + """ + name = cluster.job_name(end) + job_deadline, pod_deadline = cls.deadlines(cluster.k8s.job(name)) + + if job_deadline is not None and pending_seconds + running_seconds > job_deadline: + # Job-level clock: the Job controller kills it and stamps its own + # condition. The pod is SIGTERMed and drains to exit 3. + cluster.advance(end, 'timeout') + return 'timeout' + + if pod_deadline is not None and running_seconds > pod_deadline: + # Pod-level clock: the kubelet kills the pod and marks it + # DeadlineExceeded. The Job fails through the ordinary exit-code + # rule -- it has no idea a deadline was involved. + pod = cluster.k8s.pod_for_job(name) + cluster.k8s.set_pod_terminated(pod.metadata.name, exit_code=3, + seconds=running_seconds) + cluster.k8s.set_pod_phase(pod.metadata.name, 'Failed', + reason='DeadlineExceeded', + message='Pod was active on the node longer ' + 'than the specified deadline') + cluster.k8s.set_job_failed( + name, message=(f"Container stellar-core for pod {cluster.namespace}/" + f"{pod.metadata.name} failed with exit code 3 " + f"matching FailJob rule at index 2")) + return 'timeout' + + cluster.advance(end, finishes) + return finishes + + +def _job_hit_its_deadline(cluster, end, attempt=None): + """Stamp the Job-level DeadlineExceeded condition, leaving the pod as-is. + + This is the interleaving in facet B: the pod has already recorded a specific + terminated reason (OOMKilled, DisruptionTarget, ...) and the Job *also* + tripped its deadline, so both signals are on the table at once. + """ + name = cluster.job_name(end, attempt) + cluster.k8s.set_job_failed(name, reason='DeadlineExceeded', + message='Job was active longer than specified deadline') + return name + + +def _memory(cluster, job_name): + return cluster.k8s.job(job_name).spec.template.spec.containers[0].resources + + +# --- A: Pending time must not be charged against the runtime budget ---------- + +def test_a_range_that_only_waited_for_a_node_is_not_killed_as_a_timeout(cluster, monkeypatch): + """15 minutes Pending, 100 seconds of work, a 600s budget -- this must pass. + + The range ran for a sixth of its allowance. It is only killed because the + clock was started by the Job's creation instead of by the container's start. + """ + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) + cluster.reconcile() + + outcome = _DeadlineController.run_attempt( + cluster, 300, pending_seconds=900, running_seconds=100, finishes='succeeded') + cluster.finalize(300, 1, tx_apply=0.5) + cluster.reconcile() + + assert outcome == 'succeeded', ( + "the attempt was killed after 100s of running against a 600s budget: " + "the deadline is counting the 900s it spent Pending") + assert '300' in cluster.completed() + assert cluster.failed() == {} + + +def test_a_capacity_stall_does_not_condemn_a_range(cluster, monkeypatch): + """The run-ending shape: every attempt stalls, so every attempt "times out". + + A timeout gets MAX_TIMEOUT_ATTEMPTS (2), so two stalls are enough to condemn + the range outright -- and a condemned range fails the mission. + """ + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) + cluster.reconcile() + + # Keep the stall going until the range settles one way or the other. Four + # passes is more than the 2-attempt timeout budget, so if the deadline is + # counting Pending time this reaches the condemned state. + for attempt in (1, 2, 3, 4): + if '300' in cluster.completed() or '300' in cluster.failed(): + break + _DeadlineController.run_attempt(cluster, 300, pending_seconds=900, + running_seconds=100, finishes='succeeded') + cluster.finalize(300, attempt) + cluster.reconcile() + + assert cluster.failed() == {}, ( + "two capacity stalls condemned a range that never used its runtime " + "budget; this is what fails the mission during a node-class outage") + assert '300' in cluster.completed() + + +def test_the_whole_fleet_stalling_does_not_burn_every_range(cluster, monkeypatch): + """The outage hits every range at once, not one of them.""" + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) + monkeypatch.setattr(jm, 'PARALLELISM', 3) + cluster.reconcile() + assert sorted(cluster.jobs()) == ['pc-r100-a1', 'pc-r200-a1', 'pc-r300-a1'] + + for end in (300, 200, 100): + _DeadlineController.run_attempt(cluster, end, pending_seconds=1200, + running_seconds=60, finishes='succeeded') + cluster.finalize(end, 1) + cluster.reconcile() + + assert cluster.failed() == {} + assert sorted(cluster.completed()) == ['100', '200', '300'] + + +def test_an_attempt_that_really_hangs_is_still_killed_by_the_deadline(cluster, monkeypatch): + """The deadline must keep biting -- a fix that just removes it is not a fix. + + Green before and after: a range that genuinely runs past its budget is + killed, retried once, and then condemned as a timeout with evidence. + """ + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) + cluster.reconcile() + + for attempt in (1, 2): + outcome = _DeadlineController.run_attempt( + cluster, 300, pending_seconds=10, running_seconds=900, finishes='succeeded') + assert outcome == 'timeout', "a 900s attempt escaped its 600s deadline" + cluster.finalize(300, attempt) + cluster.reconcile() + + assert cluster.failed()['300']['outcome'] == 'timeout' + assert cluster.failed()['300']['attempts'] == 2 + assert cluster.completed() == {} + + +# --- B: a Job deadline must not overwrite the pod's own verdict -------------- + +def test_an_oom_inside_a_deadline_exceeded_job_escalates_memory(cluster, monkeypatch): + """The kubelet said OOMKilled. The Job said "ran too long". Both are true. + + Only one of them tells you what to do about it. Filing this as a timeout + means the retry goes out at the same memory limit that just killed it. + """ + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) + cluster.reconcile() + cluster.advance(300, 'oom') + _job_hit_its_deadline(cluster, 300) + + cluster.reconcile() + + # The pod's own record is unambiguous and durable -- reconcile simply + # ignored it. + assert jm.read_outcome('300', 1)['outcome'] == 'oom' + assert 'pc-r300-a2' in cluster.jobs() + resources = _memory(cluster, 'pc-r300-a2') + assert resources.limits['memory'] == '36000Mi', ( + "the retry went out at the same limit that OOM-killed it: the Job's " + "DeadlineExceeded overwrote the kubelet's OOMKilled") + assert resources.requests['memory'] == '36000Mi' + + +def test_two_ooms_inside_deadline_exceeded_jobs_do_not_condemn_the_range(cluster, monkeypatch): + """An OOM gets the range budget (5). A timeout gets 2. Misfiling ends the run.""" + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) + cluster.reconcile() + + for attempt in (1, 2): + cluster.advance(300, 'oom') + _job_hit_its_deadline(cluster, 300, attempt) + cluster.finalize(300, attempt) + cluster.reconcile() + + assert cluster.failed() == {}, ( + "two OOMs condemned the range at the 2-attempt timeout budget instead " + "of retrying on the 5-attempt range budget") + assert 'pc-r300-a3' in cluster.jobs() + # Two rungs climbed, capped at MAX_MEM (48Gi). + assert _memory(cluster, 'pc-r300-a3').limits['memory'] == '49152Mi' + + +def test_a_disruption_inside_a_deadline_exceeded_job_keeps_its_own_budget(cluster, monkeypatch): + """Spot reclaim is the cluster's fault, and gets MAX_DISRUPTION_ATTEMPTS (20). + + A node drained near the end of a long attempt trips the Job deadline on the + way out, so the two signals arrive together constantly on spot. + """ + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) + cluster.reconcile() + + for attempt in (1, 2): + cluster.advance(300, 'disrupted') + _job_hit_its_deadline(cluster, 300, attempt) + cluster.finalize(300, attempt) + cluster.reconcile() + + assert jm.read_outcome('300', 1)['outcome'] == 'disrupted' + assert cluster.failed() == {}, ( + "two spot evictions condemned the range: the Job's DeadlineExceeded " + "downgraded them to the 2-attempt timeout budget") + assert 'pc-r300-a3' in cluster.jobs() + # An eviction says nothing about how much memory the range wants. + assert _memory(cluster, 'pc-r300-a3').limits['memory'] == jm.LIM_MEM + + +def test_an_ephemeral_eviction_inside_a_deadline_exceeded_job_still_grows_the_disk(cluster, monkeypatch): + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) + monkeypatch.setattr(jm, 'LIM_EPHEMERAL', '40Gi') + monkeypatch.setattr(jm, 'REQ_EPHEMERAL', '40Gi') + cluster.reconcile() + cluster.advance(300, 'ephemeral') + _job_hit_its_deadline(cluster, 300) + + cluster.reconcile() + + assert jm.read_outcome('300', 1)['outcome'] == 'ephemeral' + assert 'pc-r300-a2' in cluster.jobs() + grown = _memory(cluster, 'pc-r300-a2').limits['ephemeral-storage'] + assert grown != '40Gi', ( + "the retry went out at the same ephemeral-storage limit that evicted " + "it: the Job's DeadlineExceeded overwrote the kubelet's eviction") + + +def test_a_deadline_kill_that_drained_to_exit_three_is_still_a_timeout(cluster, monkeypatch): + """The intended exception, which the ranking must not undo. + + A deadline kill SIGTERMs stellar-core, which drains and exits 3 -- the pod + verdict reads a plain `failed`, and nothing on the pod says a deadline was + involved. Here the Job genuinely is the better source, so it must still win. + Green before and after. + """ + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) + cluster.reconcile() + + for attempt in (1, 2): + cluster.advance(300, 'timeout') + cluster.finalize(300, attempt) + cluster.reconcile() + + assert cluster.failed()['300']['outcome'] == 'timeout', ( + "an exit-3 deadline kill is no longer recognised as a timeout") + assert cluster.failed()['300']['attempts'] == 2 diff --git a/src/MissionParallelCatchup/test_race_7.py b/src/MissionParallelCatchup/test_race_7.py new file mode 100644 index 00000000..7c6b6920 --- /dev/null +++ b/src/MissionParallelCatchup/test_race_7.py @@ -0,0 +1,217 @@ +"""RACE #7: a condemned range must not freeze dispatch, or the mission hangs. + +The mission driver (MissionHistoryPubnetParallelCatchupV2.fs) no longer aborts +on the first failure -- it drains first, and only reports once + + num_remain == 0 && jobs_in_progress.Count == 0 + +which are `reconcile()`'s own `remaining` and `in_progress` verbatim +(job_monitor.update_status_and_metrics maps them straight into the status JSON). + +If dispatch is gated on `not failed`, the first condemned range stops the +monitor sending any further work. `in_progress` still drains to empty as the +in-flight ranges land, but `remaining` stays pinned at however many ranges were +never dispatched. The driver's condition is then unsatisfiable and the mission +waits forever with an idle, fully-billed node pool -- strictly worse than the +immediate abort it replaced. + +These tests drive the real reconcile() through the fake cluster and assert only +on what it returns and on what ends up in progress.json. No source text. +""" + +import pytest + +import job_monitor as jm + + +# -- helpers (local to this file; the shared harness is not touched) ---------- + +def _end_of(key): + """'300/420' -> 300. `in_progress` entries are job_key(end, count).""" + return int(key.split('/')[0]) + + +def _drained(poll): + """The mission driver's completion test, applied to a reconcile summary.""" + return poll['remaining'] == 0 and not poll['in_progress'] + + +def drive_like_the_mission(cluster, condemn=(), max_passes=40): + """Poll reconcile() the way the driver polls the monitor, until it drains. + + Between polls the cluster does its job: every range currently in flight + finishes -- successfully, unless it is in `condemn`, in which case it exits + 1 (a genuine catchup failure, which the monitor never retries). + + Returns the list of poll results, or None if the run never drained inside + `max_passes` -- which is what a hang looks like when you cannot wait forever. + """ + condemn = {int(e) for e in condemn} + polls = [] + for _ in range(max_passes): + poll = cluster.reconcile() + polls.append(poll) + if _drained(poll): + return polls + for key in list(poll['in_progress']): + end = _end_of(key) + attempt = cluster.attempt_of(end) + if end in condemn: + cluster.advance(end, 'condemned', attempt=attempt) + else: + cluster.advance(end, 'succeeded', attempt=attempt) + cluster.finalize(end, attempt, tx_apply=1.0, + peaks={'peakRssBytes': 1}) + return None + + +def _why_stuck(polls): + last = polls[-1] + return (f"run never drained; last poll remaining={last['remaining']} " + f"in_progress={last['in_progress']} created={last['created']} " + f"completed={last['completed']} failed={last['failed_ranges']}") + + +# -- the race ---------------------------------------------------------------- + +def test_a_condemned_range_does_not_pin_remaining_above_zero(cluster): + """The exact interleaving: one range condemned while another succeeds. + + Three ranges, PARALLELISM 2, so range 100 is still undispatched when 300 is + condemned. If the condemn freezes dispatch, 100 is never sent, `in_progress` + empties anyway, and `remaining` sticks at 1 -- the deadlock. + """ + first = cluster.reconcile() + assert sorted(first['in_progress']) == ['200/420', '300/420'] + assert first['remaining'] == 1, "range 100 has not been dispatched yet" + + cluster.advance(300, 'condemned') # exit 1: never retried + cluster.advance(200, 'succeeded') + cluster.finalize(200, 1, tx_apply=1.0, peaks={'peakRssBytes': 1}) + + second = cluster.reconcile() + + # The condemn is recorded and the good range is banked... + assert '300' in cluster.failed() + assert '200' in cluster.completed() + # ...and the range behind them goes out into the freed capacity. Frozen + # dispatch gives in_progress == [] with remaining == 1, and from there the + # driver's `remaining == 0 && in_progress == []` can never come true. + assert second['in_progress'] == ['100/420'], ( + "the condemned range froze dispatch: range 100 was never sent, so the " + "mission's drain condition is now unsatisfiable") + assert second['remaining'] == 0 + + # And it really does finish. + cluster.advance(100, 'succeeded') + cluster.finalize(100, 1, tx_apply=1.0, peaks={'peakRssBytes': 1}) + third = cluster.reconcile() + assert _drained(third) + assert sorted(cluster.completed()) == ['100', '200'] + assert list(cluster.failed()) == ['300'] + + +def test_the_mission_drains_and_then_fails_instead_of_hanging(cluster): + """End to end through the driver's own loop: it must terminate. + + A run with a condemned range has to reach `remaining == 0 and + in_progress == []` -- the mission then fails on the recorded failure. With + dispatch frozen the loop below simply never exits. + """ + polls = drive_like_the_mission(cluster, condemn=[300]) + + assert polls is not None, ( + "the mission never drained: reconcile() never reported " + "remaining == 0 with in_progress empty, so the driver would poll forever") + assert _drained(polls[-1]), _why_stuck(polls) + + # It drains, but it does not pass: the failure is still reported, which is + # what makes the mission fail after the drain. + assert polls[-1]['failed_ranges'], "the condemned range must still be reported" + assert polls[-1]['failed_ranges'][0].startswith('300/420|') + assert polls[-1]['completed'] == 2, "the other two ranges must still be run" + + +def test_a_condemned_tip_does_not_discard_every_range_behind_it(cluster, + monkeypatch): + """The production shape: one early condemn, nine ranges still to dispatch. + + This is the 2026-07-30 incident at small scale -- a condemned range at the + tip stranded everything queued behind it. Freezing dispatch loses all nine. + """ + monkeypatch.setattr(jm, 'LATEST_LEDGER_NUM', 1000) # ends 100..1000 + + polls = drive_like_the_mission(cluster, condemn=[1000]) + + assert polls is not None, "the mission hung with nine ranges never dispatched" + assert _drained(polls[-1]), _why_stuck(polls) + + assert sorted(int(e) for e in cluster.completed()) == [ + 100, 200, 300, 400, 500, 600, 700, 800, 900] + assert list(cluster.failed()) == ['1000'] + assert polls[-1]['total'] == 10 + + +def test_the_stuck_state_never_settles_into_a_reportable_one(cluster): + """A hang is a state that repeats, so poll it the way the driver does. + + Once everything that can finish has finished, every subsequent poll must + report the drained state. Frozen dispatch instead reports the same + `remaining > 0, in_progress == []` forever -- work outstanding, nobody + doing it, no new Jobs. That pair is the deadlock signature. + """ + drive_like_the_mission(cluster, condemn=[300]) + + for _ in range(5): + poll = cluster.reconcile() + assert not (poll['remaining'] > 0 and not poll['in_progress']), ( + f"deadlock signature: remaining={poll['remaining']} with nothing in " + f"flight and created={poll['created']}") + assert _drained(poll) + + # Nothing new was invented to get there, either: three ranges, three Jobs. + assert cluster.calls.names(verb='create', kind='job') == [ + 'pc-r300-a1', 'pc-r200-a1', 'pc-r100-a1'] + + +def test_two_condemned_ranges_still_leave_the_run_drainable(cluster, + monkeypatch): + """More than one failure must not make it worse, and must not double-count. + + `remaining` subtracts completed, failed and in-flight; a second condemn has + to land in `failed` exactly once or the arithmetic stops reaching zero. + """ + monkeypatch.setattr(jm, 'LATEST_LEDGER_NUM', 500) # ends 100..500 + + polls = drive_like_the_mission(cluster, condemn=[500, 400]) + + assert polls is not None, "the mission hung after two condemned ranges" + assert _drained(polls[-1]), _why_stuck(polls) + assert sorted(cluster.failed()) == ['400', '500'] + assert sorted(int(e) for e in cluster.completed()) == [100, 200, 300] + assert len(polls[-1]['failed_ranges']) == 2 + + +# -- the safety valve the fix must not take with it --------------------------- + +def test_a_halted_run_still_refuses_to_dispatch(cluster): + """Guard, not a repro: `halted` is the gate that must survive. + + Dispatch is gated on `halted` alone now. `halted` means the durable record + went backwards, so nothing can be trusted and stopping is correct. This + passes both before and after the fix; it is here so that "gate on halted + alone" cannot be satisfied by deleting the gate. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1, tx_apply=1.0, peaks={'peakRssBytes': 1}) + cluster.reconcile() + + cluster.write(jm.PROGRESS_FILE, '{}') # the record is wiped underneath us + before = set(cluster.jobs()) + + poll = cluster.reconcile() + + assert cluster.state['halted'] is True + assert poll['created'] == 0 + assert set(cluster.jobs()) == before From e30ee14b93d4732436079e236f5567d5020b3ed3 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 10:49:38 -0400 Subject: [PATCH 028/117] Statelessness suite, and six defects it found Adds 82 tests that restart the monitor at every point a real process death can land -- a seeded fuzz across 30 seeds x 24 passes (soaked to 300x40), a crash injected at each side-effect boundary in reconcile(), hostile durable state, and independent collector restarts. All assert on observed state: the durable record, the objects in the fake cluster, and reconcile's own return. Six defects, each observed failing before the fix and passing after: - a crash between save_progress() and the end of the success arm left the range recorded but its PVC never released and its Job never reaped, and permanently: the first-sight branch cannot run again and the backfill branch is skipped once the record is whole. Both calls are hoisted to a shared per-sighting tail; both were already idempotent. - a 409 on the dispatch create did not spend a slot, so losing the create race ran PARALLELISM+1 workers and reported the running range as remaining. - `remaining` was a subtraction over three independently maintained lengths, so a progress record carrying ends from a run with a different ledgersPerJob drove it negative -- and to 0 on the first pass, which reads as a finished run that never dispatched anything. It is now a count over this run's own ranges. - a progress.json that parsed but had the wrong shape crashed reconcile() after dispatch, leaving the run with no status and no remaining, forever. - peakEphemeralBytes was never flushed mid-flight, and ephemeral use is not monotonic, so a collector OOM lost a high-water no successor could re-derive. - attemptSeconds was newest-wins, so a second finalize could overwrite a real duration with a near-zero one. Two further defects are recorded as strict xfail rather than fixed, because neither fix is obviously correct: the backwards-progress guard keeps its high-water only in memory, so a restart disarms the guard that exists for exactly that event; and load_progress()'s ConfigMap fallback returns a record with every measurement stripped, which the next save_progress() then persists over the real one. Also removes dead weight found by audit: MAX_LINE_CHARS (referenced by no code, yet plumbed through the chart and asserted by two tests), STATE_FLUSH_SECONDS, was_disrupted, _cpu_millis, an `if True:`, two F# calls that read a ConfigMap and discarded it, and the worker ping loop -- PARALLELISM HTTP GETs every 10s for a liveness signal the driver never reads. Net -102 lines. Co-Authored-By: Claude Opus 5 --- .../MissionHistoryPubnetParallelCatchupV2.fs | 11 - src/MissionParallelCatchup/job_monitor.py | 179 ++--- src/MissionParallelCatchup/log_collector.py | 65 +- .../templates/job_monitor.yaml | 4 - .../parallel_catchup_helm/values.yaml | 5 - .../test_job_monitor.py | 27 +- .../test_stateless_adversarial.py | 509 ++++++++++++++ .../test_stateless_collector.py | 574 ++++++++++++++++ .../test_stateless_crashpoints.py | 645 ++++++++++++++++++ .../test_stateless_restart.py | 533 +++++++++++++++ 10 files changed, 2400 insertions(+), 152 deletions(-) create mode 100644 src/MissionParallelCatchup/test_stateless_adversarial.py create mode 100644 src/MissionParallelCatchup/test_stateless_collector.py create mode 100644 src/MissionParallelCatchup/test_stateless_crashpoints.py create mode 100644 src/MissionParallelCatchup/test_stateless_restart.py diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 0f93b518..fccc0043 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -45,7 +45,6 @@ let jobMonitorStatusKey = "status.json" // live queue counts let jobMonitorProgressKey = "progress.json" // durable per-range completion record let jobMonitorLoggingIntervalSecs = 30 // frequency of the monitor reconcile loop: dispatch, liveness ping, status publish let jobMonitorStatusCheckIntervalSecs = 60 // frequency of us querying job monitor's `/status` end point -let jobMonitorMetricsCheckIntervalSecs = 60 // frequency of us querying job monitor's `/metrics` end point let jobMonitorStatusCheckTimeOutSecs = 600 let mutable toPerformCleanup = true let failedJobLogFileLineCount = 10000 @@ -743,7 +742,6 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = let mutable allJobsFinished = false let mutable timeoutLeft = jobMonitorStatusCheckTimeOutSecs - let mutable timeBeforeNextMetricsCheck = jobMonitorMetricsCheckIntervalSecs // Failures are reported once the run drains, not at first sight. Aborting on // the first condemned range abandons every range still in flight, and the @@ -774,18 +772,9 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = LogError "RANGE FAILED: %s -- run continues, mission will fail once it drains" text if remainSize = 0 && JobsInProgress.Count = 0 then - // All jobs completed — perform a final query on the metrics - queryJobMonitor (context, jobMonitorProgressKey) |> ignore LogInfo "All queues empty. Mission complete." allJobsFinished <- true - // check the metrics - timeBeforeNextMetricsCheck <- timeBeforeNextMetricsCheck - jobMonitorStatusCheckIntervalSecs - - if timeBeforeNextMetricsCheck <= 0 then - queryJobMonitor (context, jobMonitorProgressKey) |> ignore - timeBeforeNextMetricsCheck <- jobMonitorMetricsCheckIntervalSecs - | None -> LogError "no status" timeoutLeft <- timeoutLeft - jobMonitorStatusCheckIntervalSecs diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 26ffd7aa..fc87f9f6 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -15,7 +15,6 @@ work is *assigned*, never claimed. """ -import asyncio import gzip import bisect import json @@ -30,7 +29,6 @@ from datetime import datetime, timezone from http.server import BaseHTTPRequestHandler, HTTPServer -import aiohttp from kubernetes import client, config from kubernetes.client.rest import ApiException from prometheus_client import (CONTENT_TYPE_LATEST, REGISTRY, Counter, Gauge, @@ -247,10 +245,6 @@ # /healthz fails if the loop has not ticked within this long; a wedged loop # stops all dispatch, so restart the container rather than run half-alive. RECONCILE_STALE_SECONDS = float(os.getenv('WATCH_STALE_SECONDS', 600)) -# Liveness ping to each running worker's admin port, fanned out on one event -# loop. Done serially with a 5s timeout this took a 192s median at 1024 -# (measured in prod), which is why it is async and the timeout is short. -WORKER_PING_TIMEOUT_SECONDS = float(os.getenv('PING_TIMEOUT_SECS', 2)) # Shared with the log-collector sidecar, which owns writes here: it streams each # worker's log and records the .outcome verdict while the pod still exists. @@ -362,9 +356,6 @@ def check_storage_config(): 'queue_in_progress_count': 0, 'jobs_failed': [], 'jobs_in_progress': [], - 'workers': [], - 'workers_up': 0, - 'workers_down': 0, 'workers_refresh_duration': 0, 'mission_duration': 0, } @@ -493,16 +484,45 @@ def job_name(end, attempt): PROGRESS_FILE = os.path.join(LOG_DIR, 'progress.json') +def _sane_progress(progress): + """Drop anything in the record that is not a range -> record mapping. + + This document is read off a volume that outlives the run and is mirrored + through a ConfigMap a second writer can clobber, so it comes back + structurally wrong as well as merely truncated. A truncated file raises + ValueError and is already handled; one that parses into the wrong SHAPE was + not. A single non-dict entry took every later pass down inside + observe_recorded/sync_counters -- after dispatch, so the exception the + reconcile loop swallows left the run with no status update and no + `remaining` ever again. + + Corrupt is not progress, so an unreadable entry is dropped rather than + counted: the range is re-run, which is idempotent, and the + monotonic-progress guard still fires if dropping one shrinks a record that + was larger a pass ago. + """ + if not isinstance(progress, dict): + return {} + out = dict(progress) + for bucket in ('completed', 'failed'): + entries = out.get(bucket) + if entries is None: + continue + out[bucket] = ({k: v for k, v in entries.items() if isinstance(v, dict)} + if isinstance(entries, dict) else {}) + return out + + def load_progress(): try: with open(PROGRESS_FILE) as fh: - return json.load(fh) + return _sane_progress(json.load(fh)) except (OSError, ValueError): pass # First start on this volume, or an older run that only had the ConfigMap. try: cm = core_v1.read_namespaced_config_map(PROGRESS_CM, NAMESPACE) - return json.loads((cm.data or {}).get('progress.json', '{}')) + return _sane_progress(json.loads((cm.data or {}).get('progress.json', '{}'))) except ApiException as e: if e.status == 404: return {} @@ -576,38 +596,6 @@ def _patch_cm(data, owner=None): data=body['data'])) -# --- worker liveness -------------------------------------------------------- -# Serially pinging 1024 workers with a 5s timeout was measured at a 192s median -# and 773s max in prod (155 unreachable x 5s). Fanning out on one event loop -# bounds it by the timeout itself. - -async def _ping_all(pods): - timeout = aiohttp.ClientTimeout(total=WORKER_PING_TIMEOUT_SECONDS) - async with aiohttp.ClientSession(timeout=timeout) as session: - async def one(pod, ip): - url = f"http://{ip}:11626/info" - try: - async with session.get(url): - return pod, True - except Exception: - return pod, False - return dict(await asyncio.gather(*(one(p, ip) for p, ip in pods))) - - -def ping_workers(pods): - """pods: (name, ip) pairs. - - By IP, not DNS. The ...svc form needs a per-pod A record, - which a headless Service only publishes for endpoints whose EndpointSlice - carries a hostname -- and that comes from pod.spec.hostname, which a Job pod - cannot set to its own generated name. Measured on ssc-test: every ping - failed to resolve and workers_up sat at 0 for the whole run. - """ - if not pods: - return {} - return asyncio.run(_ping_all(pods)) - - # --- worker log capture ----------------------------------------------------- def backstop_save_pod_log(pod_name, end, attempt): @@ -1427,11 +1415,6 @@ def profile_for(end): return PROFILE[idx][1] if idx < len(PROFILE) else None -def _cpu_millis(q): - return int(float(q[:-1])) if str(q).endswith('m') else int(float(q) * 1000) - - - def _sized(value, margin, cap): """A measured peak turned into a request: margin applied, never above cap.""" want = int(value * margin) @@ -1730,13 +1713,6 @@ def pods_by_job(): return out -def was_disrupted(pod): - for cond in (pod.status.conditions or []): - if cond.type == 'DisruptionTarget' and cond.status == 'True': - return True - return False - - def reconcile(state): ranges = generate_ranges() by_end = {str(end): count for end, count in ranges} @@ -1757,6 +1733,18 @@ def reconcile(state): live[end] = (attempt, j) in_progress = [] + # The same set of ranges as `in_progress`, keyed by end. `remaining` is a + # COUNT over this run's range list, never `total - completed - ...`: the + # progress record is read off a shared volume and can carry ends from a run + # with a different ledgersPerJob, and a subtraction lets those foreign keys + # move a number that is supposed to describe THIS run. Measured on the + # fixture: one foreign key made `remaining` read 0 on the very first pass + # with nothing dispatched yet, and three of them left it at -3 once every + # real range had finished -- so the mission's `num_remain == 0 && + # jobs_in_progress == []` could never fire and the driver waited forever on + # a completed run. A count cannot be pushed below zero or above the range + # list by anything that is not one of our own ranges. + in_flight = set() for end, (attempt, j) in list(live.items()): st = j.status if st.succeeded: @@ -1799,16 +1787,9 @@ def reconcile(state): if by_end.get(end) is not None: completed[end]['count'] = by_end[end] completed[end].update(peaks_for_range(end, attempt)) - # Durably recorded first: if this process dies between the two, - # the range is still complete and simply keeps its volume. + # Durably recorded first: the record is what makes the volume + # and the Job disposable, so it must land before either goes. save_progress(progress) - release_pvc(end) - # Only once the record is complete. `tx is None` means the - # collector had not flushed this range's .metrics yet, and the - # pod is the only place left to read it from -- deleting the - # Job would reap the pod and make that gap permanent. Leave - # those to JOB_TTL_SECONDS. - _reap_if_complete(end, attempt, completed[end]) elif (not _has_peaks(completed[end]) or completed[end].get('txApply') is None or not _attempt_finalized(end, attempt)): @@ -1836,7 +1817,20 @@ def reconcile(state): save_progress(progress) logger.info("range %s: measurements arrived late, backfilled %s", end, sorted(late)) - _reap_if_complete(end, attempt, completed[end]) + # Per SIGHTING of a recorded range, not per first sight. Both are + # idempotent (a 404 from either is swallowed) and both used to hang + # off the branch that runs exactly once, so a process that died + # anywhere between save_progress and here never reached them again: + # the record exists, so the first-sight branch is skipped forever + # and the backfill branch is skipped as soon as the record is + # complete. That leaked the range's 40Gi volume permanently and left + # a Job nothing would ever reap but JOB_TTL_SECONDS. + # + # `tx is None` still costs nothing here: _reap_if_complete waits for + # the collector's .done marker, so an unflushed range keeps its Job + # (and its pod, the last place the metric can be read) regardless. + release_pvc(end) + _reap_if_complete(end, attempt, completed[end]) elif st.failed: # Completion is terminal for the range, so a Failed Job for a range # that is already recorded is garbage -- never an input to the retry @@ -2009,6 +2003,7 @@ def reconcile(state): if _attempt_finalized(end, attempt): delete_job(end, attempt) in_progress.append(job_key(int(end), by_end[end])) + in_flight.add(str(end)) continue if reason is not None: logger.error("range %s exhausted %d attempts (%s)", end, cap, reason) @@ -2028,6 +2023,7 @@ def reconcile(state): save_progress(progress) else: in_progress.append(job_key(int(end), by_end.get(end, 0))) + in_flight.add(str(end)) # Monotonic progress is invariant in a healthy run. A decrease means the # durable record or the Jobs were tampered with; redoing hours of work @@ -2064,9 +2060,18 @@ def reconcile(state): created += 1 capacity -= 1 in_progress.append(job_key(end, count)) + in_flight.add(str(end)) except ApiException as e: if e.status != 409: # AlreadyExists: name uniqueness is the mutex raise + # Losing the mutex means the Job EXISTS and is in flight, so it + # occupies a slot exactly like one we created. Falling through + # without spending capacity dispatched PARALLELISM+1 workers -- + # one extra per lost race -- and reported the range as + # `remaining` while it was already running. + capacity -= 1 + in_progress.append(job_key(end, count)) + in_flight.add(str(end)) observe_recorded(progress, state['replayed']) sync_counters(progress, state['counted']) @@ -2077,7 +2082,10 @@ def reconcile(state): for k, v in failed.items()], 'in_progress': in_progress, 'created': created, - 'remaining': len(ranges) - len(completed) - len(failed) - len(in_progress), + 'remaining': sum(1 for end, _ in ranges + if str(end) not in completed + and str(end) not in failed + and str(end) not in in_flight), } @@ -2117,24 +2125,22 @@ def update_status_and_metrics(): r = reconcile(state) - # Liveness of the workers that currently own a job -- idle slots are - # deliberately not counted, matching the original metric. - pods = [(p.metadata.name, p.status.pod_ip) - for p in core_v1.list_namespaced_pod( - NAMESPACE, label_selector=f"{LABEL_RUN}={RUN_NAME}", - field_selector='status.phase=Running', - # Served from the apiserver watch cache. Only safe here: - # a stale liveness sample is cosmetic, whereas stale - # dispatch state would re-run a range. - resource_version='0').items - if p.status.pod_ip] + # Worker liveness, for the Grafana series only -- nothing in the + # driver reads it. A worker is a Job here, so a Running pod IS a + # live worker and its liveness is the Job's status; the count comes + # off the pod list the apiserver already has cached instead of one + # HTTP GET per worker every cycle. refresh_start = time.time() - ping = ping_workers(pods) + workers_up = sum( + 1 for p in core_v1.list_namespaced_pod( + NAMESPACE, label_selector=f"{LABEL_RUN}={RUN_NAME}", + field_selector='status.phase=Running', + # Served from the apiserver watch cache. Only safe here: + # a stale liveness sample is cosmetic, whereas stale + # dispatch state would re-run a range. + resource_version='0').items + if p.status.pod_ip) workers_refresh_duration = time.time() - refresh_start - worker_statuses = [{'pod': p, 'status': 'running' if ok else 'down'} - for p, ok in ping.items()] - workers_up = sum(1 for ok in ping.values() if ok) - workers_down = len(ping) - workers_up mission_duration = time.time() - mission_start_time with status_lock: @@ -2146,9 +2152,6 @@ def update_status_and_metrics(): 'queue_in_progress_count': len(r['in_progress']), 'jobs_failed': r['failed_ranges'], 'jobs_in_progress': r['in_progress'], - 'workers': worker_statuses, - 'workers_up': workers_up, - 'workers_down': workers_down, 'workers_refresh_duration': workers_refresh_duration, 'mission_duration': mission_duration, } @@ -2157,7 +2160,11 @@ def update_status_and_metrics(): metric_catchup_queues.labels(queue="failed").set(len(r['failed_ranges'])) metric_catchup_queues.labels(queue="in_progress").set(len(r['in_progress'])) metric_workers.labels(status="up").set(workers_up) - metric_workers.labels(status="down").set(workers_down) + # Held at 0 rather than dropped: the series is Grafana-facing, and a + # label that stops being set goes stale on the dashboard instead of + # reading zero. Nothing can report "down" now that liveness is the + # pod's phase -- a worker that is not up is simply not listed. + metric_workers.labels(status="down").set(0) metric_refresh_duration.set(workers_refresh_duration) metric_mission_duration.set(mission_duration) logger.info("Status: %s", json.dumps(status)) diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 9f593475..9e6e0a31 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -21,8 +21,9 @@ Residual: if this dies between flushing log bytes and rewriting the state file, the next run replays from a slightly older timestamp and a few lines duplicate. -Bounded by STATE_FLUSH_SECONDS. "At least once, deduped to near-exact" rather -than exactly once. +Bounded by one poll's worth of lines, since the state file is rewritten at the +end of every poll. "At least once, deduped to near-exact" rather than exactly +once. """ import asyncio @@ -43,7 +44,6 @@ LOG_DIR = os.getenv('LOG_DIR', '/logs') CONTAINER = os.getenv('WORKER_CONTAINER', 'stellar-core') POLL_SECONDS = float(os.getenv('COLLECTOR_POLL_SECONDS', 5)) -STATE_FLUSH_SECONDS = float(os.getenv('STATE_FLUSH_SECONDS', 10)) # Poll cycles a stream gets to finalize itself after its pod leaves the pod list # before it is cancelled outright. One cycle is usually enough; the margin is for # a stream still finalizing: writing its .metrics and closing its archive. @@ -68,16 +68,6 @@ # at most PEAK_FLUSH_RATIO of a range's high-water rather than all of it -- # Prometheus's server-side max_over_time needed no such state. PEAK_FLUSH_RATIO = float(os.getenv('PEAK_FLUSH_RATIO', 1.05)) -# Most a single unterminated blob may buffer before we start discarding its -# head. stellar-core's own lines are well under a kilobyte; anything larger is a -# progress meter or a stack dump, and neither is worth killing the stream over. -# -# This is charged PER LIVE STREAM. Measured on ssc-test at 2096 follow streams -# the collector already sat at 1444 MiB of a 2048 MiB limit with memory.events -# max=2617, so a 256 KiB worst case here is another 512 MiB at 2096 and 1 GiB at -# 4096 -- on its own enough to OOM the sidecar. 64 KiB is ~64x the longest line -# stellar-core actually emits. -MAX_LINE_CHARS = int(os.getenv('MAX_LINE_CHARS', 65536)) # Seconds between polls of one pod's log. Latency here is archive lag, not # anything a decision waits on; 4096 pods at 10s is ~90 concurrent polls. LOG_POLL_SECONDS = float(os.getenv('LOG_POLL_SECONDS', 10)) @@ -283,7 +273,15 @@ def write_metrics(end, attempt, values): except (OSError, ValueError): prior = {} merged = {**prior, **values} - for k in PEAK_KEYS: + # attemptSeconds is not a peak, but it takes the same rule for the same + # reason: it is a fixed quantity once the attempt ends, and every source is + # a lower bound on it -- the pod's own start->finish is exact, the poller's + # elapsed time covers only the part of the attempt that process was alive + # for. An attempt is finalized more than once whenever a poller is re-opened + # for a pod that is still listed (a restarted sidecar, or a 404 on the log + # endpoint while the pod list is stale), and there the fallback clock starts + # at the restart: newest-wins turned a recorded 3600s into 0.0s. + for k in PEAK_KEYS + ('attemptSeconds',): a, b = prior.get(k), values.get(k) if a is not None and b is not None: merged[k] = max(a, b) @@ -366,6 +364,9 @@ def record_outcome(pod, end, attempt): _eph_peak = {} _anon_peak = {} _ws_peak = {} +# Last value flushed to the volume, per axis: keyed by pod name for anon and by +# "/eph" for ephemeral. A pod name cannot contain '/', so the two key +# spaces cannot collide. _peak_flushed = {} # pod name -> (end, attempt), so a mid-flight peak flush can find its file. _streaming = {} @@ -411,6 +412,20 @@ async def sample_kubelet(session, nodes): if int(used) > prev: _eph_peak[name] = int(used) logger.info("peak ephemeral for %s: %.2f GiB", name, used / 1073741824) + # Flushed on growth for the same reason as anon below, and + # re-measuring does not recover it: disk use is not + # monotonic -- stellar-core drops its download staging once + # buckets are applied -- so a replacement sidecar watching + # the tail of the same pod sees a fraction of the real + # high-water. This figure sizes the next run's + # ephemeral-storage request, and one that comes back too + # small is an eviction. + if int(used) >= _peak_flushed.get(name + '/eph', 0) * PEAK_FLUSH_RATIO: + _peak_flushed[name + '/eph'] = int(used) + ref = _streaming.get(name) + if ref: + write_metrics(ref[0], ref[1], + {'peakEphemeralBytes': int(used)}) for c in entry.get('containers', []): if c.get('name') != CONTAINER: continue @@ -469,7 +484,9 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): measured['attemptSeconds'] = round(observed, 1) elif started is not None: # Fallback only: the monitor's figure comes from the pod's terminated - # timestamps and is preferred when it exists. + # timestamps and is preferred when it exists. write_metrics keeps this + # from lowering a duration already on the volume -- an attempt can be + # finalized twice, and the second poller's clock started at the restart. measured['attemptSeconds'] = round( asyncio.get_event_loop().time() - started, 1) if tx.resumed: @@ -480,6 +497,7 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): if tx.seconds is not None: measured['txApplySeconds'] = tx.seconds _peak_flushed.pop(pod, None) + _peak_flushed.pop(pod + '/eph', None) _streaming.pop(pod, None) _wake.pop(pod, None) anon = _anon_peak.pop(pod, None) @@ -760,15 +778,14 @@ async def main(): # Unconditional: this used to be gated on ephemeral mode, back # when it only sampled disk. Memory is sized in both modes, so # gating it here left every pvc run with no anon peak at all. - if True: - # Once per cycle, before the per-pod branches below: those - # end in `continue` for every pod already being streamed, so - # anything after them runs only on the cycle a stream opens - # -- when the range has barely written anything yet. - await sample_kubelet(session, { - p['spec']['nodeName'] for p in pods - if p.get('spec', {}).get('nodeName') - and p.get('status', {}).get('phase') == 'Running'}) + # Once per cycle, before the per-pod branches below: those end + # in `continue` for every pod already being streamed, so + # anything after them runs only on the cycle a stream opens -- + # when the range has barely written anything yet. + await sample_kubelet(session, { + p['spec']['nodeName'] for p in pods + if p.get('spec', {}).get('nodeName') + and p.get('status', {}).get('phase') == 'Running'}) for pod in pods: name = pod['metadata']['name'] labels = pod['metadata'].get('labels', {}) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 94dae4f9..f5cf31a7 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -223,8 +223,6 @@ spec: value: {{ .Values.monitor.attemptDeadlineSeconds | quote }} - name: JOB_TTL_SECONDS value: {{ .Values.monitor.jobTtlSeconds | quote }} - - name: PING_TIMEOUT_SECS - value: {{ .Values.monitor.pingTimeoutSecs | quote }} - name: LOG_DIR value: /logs {{- if .Values.monitor.profileConfigMap }} @@ -337,8 +335,6 @@ spec: value: {{ .Values.monitor.maxConcurrentPolls | quote }} - name: MAX_POLL_CHARS value: {{ .Values.monitor.maxPollChars | int64 | quote }} - - name: MAX_LINE_CHARS - value: {{ .Values.monitor.maxLineChars | int64 | quote }} - name: PEAK_FLUSH_RATIO value: {{ .Values.monitor.peakFlushRatio | quote }} - name: COLLECTOR_VANISHED_GRACE_CYCLES diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 5b5d2d67..68d40ee0 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -127,17 +127,12 @@ monitor: # missing/unreachable archive indefinitely rather than failing. attemptDeadlineSeconds: 10800 jobTtlSeconds: 600 - pingTimeoutSecs: 2 # Worker logs are pulled to the monitor's volume while each pod is still # alive, then collected in one exec at teardown instead of ~1024. logStorageClass: "" logStorageSize: "100Gi" saveSuccessLogs: true collectorPollSeconds: 5 - # Most a single unterminated blob may buffer before its head is discarded. - # A worker printing a carriage-return progress meter would otherwise grow this - # without bound, on every stream at once. - maxLineChars: 65536 # Log collection polls rather than holding a follow=true stream per pod, so # concurrency is independent of worker.replicas. At 4096 pods and a 10s # interval this is ~90 in-flight polls. diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/test_job_monitor.py index b548900c..5d5c8a4a 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/test_job_monitor.py @@ -332,8 +332,7 @@ def test_ephemeral_escalation_raises_request_and_limit_together(): # missing there and the sampler silently did nothing -- it defaults to 'pvc'. COLLECTOR_ENV_WITH_DEFAULTS = { 'KUBERNETES_SERVICE_HOST', 'KUBERNETES_SERVICE_PORT', # injected by kubelet - 'LOGGING_LEVEL', 'PEAK_WS_WINDOW', 'PROMETHEUS_URL', - 'STATE_FLUSH_SECONDS', 'WORKER_CONTAINER', + 'LOGGING_LEVEL', 'PEAK_WS_WINDOW', 'PROMETHEUS_URL', 'WORKER_CONTAINER', } @@ -402,7 +401,7 @@ def _profile_ns(ranges, mode='ephemeral', margin=1.1): """profile_for + _sized, exec'd out of job_monitor with a fixed profile.""" ns = {'bisect': __import__('bisect'), 'logger': __import__('logging').getLogger('t')} for name in ('_quantity_bytes', '_bytes_to_quantity', 'profile_for', - '_cpu_millis', '_sized'): + '_sized'): m = re.search(rf"^(def {name}\(.*?)(?=^\S|\Z)", SRC, re.S | re.M) exec(m.group(1), ns) ns['_UNITS'] = eval(_extract(r"_UNITS = (\{.*?\})").group(1)) @@ -579,9 +578,9 @@ def test_no_range_is_cpu_throttled_but_every_range_has_a_request(): unmeasured = ns['_resources'](end=99999) assert 'cpu' not in measured.limits, "measured ranges run uncapped" assert 'cpu' not in unmeasured.limits, "unmeasured ranges run uncapped too" - # The request is still what bounds packing, on both paths. - assert measured.requests['cpu'] and unmeasured.requests['cpu'] - assert ns['_cpu_millis'](measured.requests['cpu']) <= ns['_cpu_millis']('1800m') + # The request is still what bounds packing, on both paths, and the profile + # never moves it: cpu is not one of the overridden dimensions. + assert measured.requests['cpu'] == unmeasured.requests['cpu'] == '1800m' def test_pvc_mode_takes_no_ephemeral_override(): @@ -1810,9 +1809,6 @@ def test_an_unterminated_blob_is_capped_not_buffered_forever(): # the collector OOMs -- 2096 streams doing it at once. fn = _extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) assert 'MAX_POLL_CHARS' in fn, "a single poll response is unbounded" - cap = int(_extract(r"MAX_LINE_CHARS = int\(os\.getenv\('MAX_LINE_CHARS', (\d+)\)\)", - COLLECTOR_SRC).group(1)) - assert 1024 < cap < 524288, f"cap {cap} is outside a sane range" def test_the_worker_disables_the_aws_progress_meter(): @@ -1839,19 +1835,6 @@ def test_a_failed_attempt_is_not_reaped_before_the_collector_finalizes_it(): "successor must exist before the predecessor is reaped" -def test_the_line_buffer_cap_is_charged_per_stream(): - # Measured on ssc-test at 2096 follow streams: 1444 MiB of a 2048 MiB limit, - # memory.events max=2617. The cap is worst-case memory per live stream, so - # 256 KiB would add 1 GiB at 4096 streams and OOM the sidecar on its own. - cap = int(_extract(r"MAX_LINE_CHARS = int\(os\.getenv\('MAX_LINE_CHARS', (\d+)\)\)", - COLLECTOR_SRC).group(1)) - assert cap <= 65536, f"{cap} bytes x 4096 streams = {cap * 4096 // 2**20} MiB worst case" - assert cap >= 8192, "below this a legitimate long line would be truncated" - chart = open(__file__.replace( - 'test_job_monitor.py', 'parallel_catchup_helm/values.yaml')).read() - assert int(_extract(r"maxLineChars: (\d+)", chart).group(1)) == cap - - # --- polling replaces follow=true ------------------------------------------ # Measured on ssc-test at 2096 follow streams: 1444 MiB of a 2048 MiB limit, # memory.events max=2617, 1.00 of 2 cpu, 1797 held connections. That scales diff --git a/src/MissionParallelCatchup/test_stateless_adversarial.py b/src/MissionParallelCatchup/test_stateless_adversarial.py new file mode 100644 index 00000000..9eb116ad --- /dev/null +++ b/src/MissionParallelCatchup/test_stateless_adversarial.py @@ -0,0 +1,509 @@ +"""Hostile durable state: the monitor must never mistake foreign or corrupt +state for progress. + +Every test here drives the shipped reconcile() against the fake cluster and +asserts on observed state -- the durable record, the call log, the Job set -- +never on source text. + +The volume the monitor resumes from is not private to a run. It is a PVC that +outlives `helm uninstall`, gets reused across missions, and is mirrored into a +ConfigMap that a second writer can clobber. So progress.json can arrive +truncated, rolled back, or written by a run with a completely different +ledgersPerJob. None of those are hypothetical, and none of them may be read as +"work already done". +""" + +import json + +import pytest + +import fake_k8s +import job_monitor as jm + + +# A progress record left by a DIFFERENT slicing of the same ledger space: the +# ends are real range ends, just not ends of THIS run's range list. +FOREIGN = {'attempts': 1, 'count': 111, 'seconds': 12.0, 'wallSeconds': 12.0, + 'txApply': 1.0} + + +def seed_progress(cluster, completed=None, failed=None): + cluster.write(jm.PROGRESS_FILE, json.dumps( + {'completed': dict(completed or {}), 'failed': dict(failed or {})})) + + +# --- the headline case ------------------------------------------------------- + + +def test_foreign_completed_keys_do_not_shrink_remaining(cluster): + """`remaining` must count THIS run's outstanding ranges, not subtract a + number that a foreign record can inflate. + + Seeded: one completed key ('333') from a run with a different ledgersPerJob. + This run's ranges are 300/200/100 and not one of them has been touched. + Subtraction gives 3 - 1 - 0 - 2 == 0 on the very first pass: the mission's + `num_remain` reads zero while three ranges are outstanding. + """ + seed_progress(cluster, completed={'333': FOREIGN}) + + result = cluster.reconcile() + + # Two of the three went out; range 100 is queued behind PARALLELISM. + assert sorted(result['in_progress']) == ['200/420', '300/420'] + # ...so exactly one range of this run is still waiting to be dispatched. + assert result['remaining'] == 1 + + # And the foreign key really is being carried in the record -- the test + # above is not passing because something quietly dropped it. + assert '333' in cluster.completed() + assert set(cluster.completed()) & {'100', '200', '300'} == set() + + +def test_foreign_completed_keys_do_not_drive_remaining_negative(cluster): + """The mirror image, and the one that hangs a real run. + + The mission finishes on `num_remain == 0 && jobs_in_progress == []` + (MissionHistoryPubnetParallelCatchupV2.fs). With three foreign keys in the + record, subtraction lands on -3 once every real range has actually + completed -- never 0 -- so the driver waits forever on a run that is done. + """ + seed_progress(cluster, completed={'111': FOREIGN, '222': FOREIGN, + '333': FOREIGN}) + + result = cluster.reconcile() + for end in (300, 200): + cluster.advance(end, 'succeeded') + cluster.finalize(end, 1) + result = cluster.reconcile() + cluster.advance(100, 'succeeded') + cluster.finalize(100, 1) + result = cluster.reconcile() + + # Every range of this run really did run and really is recorded. + assert {'100', '200', '300'} <= set(cluster.completed()) + assert result['in_progress'] == [] + # The terminating condition the mission actually tests. + assert result['remaining'] == 0 + + +def test_foreign_failed_keys_do_not_shrink_remaining(cluster): + """Same subtraction, other bucket. `failed` is foreign-writable too.""" + seed_progress(cluster, failed={'111': {'attempts': 1, 'outcome': 'failed', + 'exitCode': 1, 'pod': 'gone'}, + '222': {'attempts': 1, 'outcome': 'failed', + 'exitCode': 1, 'pod': 'gone'}}) + + result = cluster.reconcile() + + assert sorted(result['in_progress']) == ['200/420', '300/420'] + assert result['remaining'] == 1 + + +def test_a_range_end_shared_with_the_foreign_slicing_is_still_skipped(cluster): + """Honest about the limit of the fix. + + `remaining` becomes a count over THIS run's range list, so it is immune to + keys that do not name one of our ranges. It cannot save us from a foreign + key that happens to collide with one of them -- '300' is an end under + ledgersPerJob=150 as well as under 100 -- because at that point the record + is indistinguishable from a legitimate resume. The count stays consistent + with what dispatch does, which is the property that matters: no phantom + zero, no phantom negative. + """ + seed_progress(cluster, completed={'300': FOREIGN, '333': FOREIGN}) + + result = cluster.reconcile() + + # 300 is treated as done (a resume, as far as anything here can tell)... + assert 'pc-r300-a1' not in cluster.jobs() + assert sorted(result['in_progress']) == ['100/420', '200/420'] + # ...and remaining agrees with that: nothing left unaccounted for. + assert result['remaining'] == 0 + + +# --- corruption -------------------------------------------------------------- + + +def test_truncated_progress_json_does_not_crash_or_lose_completions(cluster): + """A half-written progress.json must not read as an empty record. + + The file is written through a .tmp + os.replace, so a torn write should be + impossible -- but the volume is shared, and the ConfigMap mirror is exactly + the second copy that exists for this. Truncate the file and the run must + carry on from the mirror. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1) + cluster.reconcile() + assert '300' in cluster.completed() + assert '300' in cluster.progress_configmap()['completed'] + + # Truncated mid-object: json.load raises ValueError. + cluster.write(jm.PROGRESS_FILE, '{"completed": {"300": {"att') + with pytest.raises(ValueError): + json.loads(open(jm.PROGRESS_FILE).read()) + + before = set(cluster.jobs()) + result = cluster.reconcile() + + # No crash, no halt, and the completion survived via the mirror. + assert cluster.state['halted'] is False + assert '300' in jm.load_progress()['completed'] + # The critical consequence: a range that is done is not dispatched again. + assert 'pc-r300-a1' not in cluster.jobs() + assert 'pc-r300-a2' not in cluster.jobs() + assert result['completed'] == 1 + assert set(cluster.jobs()) >= before - {'pc-r300-a1'} + + +def test_unreadable_progress_with_no_mirror_halts_rather_than_replaying(cluster): + """Corruption with no second copy is a regression, and must stop the run. + + Losing both copies is indistinguishable from "nothing has been done", and + the only safe reading of that -- after work HAS been done -- is to stop. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1) + cluster.reconcile() + assert cluster.state['max_completed'] == 1 + + # Both copies gone: garbage on the volume, mirror deleted underneath us. + cluster.write(jm.PROGRESS_FILE, 'not json at all') + cluster.k8s.core_v1.delete_namespaced_config_map(jm.PROGRESS_CM, + cluster.namespace) + + before = set(cluster.jobs()) + result = cluster.reconcile() + + assert cluster.state['halted'] is True + assert result['created'] == 0 + assert set(cluster.jobs()) == before + + +def test_progress_rolled_back_to_an_older_version_halts_dispatch(cluster): + """A stale writer wins the volume: completed goes 2 -> 1. + + This is the ConfigMap-mirror-loses-a-race shape. The monitor cannot + distinguish it from deletion, and either way redoing hours of already-paid + work silently is worse than stopping, so the guard must fire. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1) + cluster.reconcile() + older = json.dumps(cluster.progress()) # snapshot at completed == 1 + + cluster.advance(200, 'succeeded') + cluster.finalize(200, 1) + cluster.reconcile() + assert set(cluster.completed()) == {'200', '300'} + assert cluster.state['max_completed'] == 2 + + # The stale copy lands back on the volume. + cluster.write(jm.PROGRESS_FILE, older) + before = set(cluster.jobs()) + created_before = cluster.calls.names(verb='create', kind='job') + + result = cluster.reconcile() + + assert cluster.state['halted'] is True + assert result['created'] == 0 + assert cluster.calls.names(verb='create', kind='job') == created_before + assert set(cluster.jobs()) == before + # The mirror is the thing the mission reads, and it never shrank: the + # rollback was not propagated outward. + assert set(cluster.progress_configmap()['completed']) == {'200', '300'} + + # Still halted on the pass after -- the guard latches, it does not flap. + assert cluster.reconcile()['created'] == 0 + + +# --- the collector's markers ------------------------------------------------- + + +def test_metrics_without_done_must_not_reap(cluster): + """.done is written last. Without it the collector may still be reading the + pod's log, and deleting the Job reaps the pod out from under it.""" + cluster.reconcile() + cluster.advance(300, 'succeeded') + # .metrics only -- exactly the window between the collector's two writes. + cluster.write(jm.metrics_path('300', 1), + json.dumps({'txApplySeconds': 2.5, 'peakRssBytes': 999})) + + cluster.reconcile() + + assert cluster.deleted.names(verb='delete', kind='job') == [] + assert 'pc-r300-a1' in cluster.jobs() + # The range is recorded and its measurements were read -- the reap is the + # only thing being withheld. + assert cluster.completed()['300']['txApply'] == 2.5 + assert cluster.completed()['300']['peakRssBytes'] == 999 + + # Withheld, not leaked: the Job carries a TTL, so declining to reap costs a + # late reclaim rather than an object that lives until `helm uninstall`. + assert (cluster.k8s.job('pc-r300-a1').spec.ttl_seconds_after_finished + == jm.JOB_TTL_SECONDS) + + # And the withheld reap does not turn into a re-dispatch on later passes. + cluster.reconcile() + assert cluster.deleted.names(verb='delete', kind='job') == [] + assert 'pc-r300-a2' not in cluster.jobs() + assert cluster.completed()['300']['attempts'] == 1 + + +def test_the_reap_lands_once_the_done_marker_arrives(cluster): + """The other side of the same gate: while the record is still incomplete, + reconcile keeps coming back, and the pass that sees .done reaps.""" + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.reconcile() # recorded with nothing measured + + assert cluster.completed()['300']['txApply'] is None + assert cluster.deleted.names(verb='delete', kind='job') == [] + + # The collector finally finishes this attempt. + cluster.finalize(300, 1, tx_apply=2.5, peaks={'peakRssBytes': 999}) + cluster.reconcile() + + # Backfilled from the durable files, then reaped. + assert cluster.completed()['300']['txApply'] == 2.5 + assert cluster.completed()['300']['peakRssBytes'] == 999 + assert cluster.deleted.names(verb='delete', kind='job') == ['pc-r300-a1'] + + +def test_done_without_metrics_reaps_but_does_not_invent_measurements(cluster): + """The other half-write: .done present, .metrics never landed. + + .done is the authority on "nothing more is coming", so the reap is correct + and must happen -- a range whose collector died would otherwise pin its Job + forever. What must NOT happen is a fabricated or crashed record. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.write(jm.done_path('300', 1), '') + + cluster.reconcile() + + record = cluster.completed()['300'] + assert record['attempts'] == 1 + assert record['count'] == 420 + # No .metrics and no history archive to fall back on: the gap is reported + # as a gap, not as zero. + assert record['txApply'] is None + assert not any(record.get(k) is not None for k in jm.PEAK_FIELDS) + # Timing comes from the pod, which is real. + assert record['seconds'] == pytest.approx(60.0) + + assert cluster.deleted.names(verb='delete', kind='job') == ['pc-r300-a1'] + # Recorded once and never re-dispatched, even though the record is thin. + assert cluster.reconcile()['created'] == 0 + assert 'pc-r300-a1' not in cluster.jobs() + assert 'pc-r300-a2' not in cluster.jobs() + + +def test_an_empty_metrics_file_is_not_read_as_zero(cluster): + """A zero-length .metrics is a torn write, not a measurement of nothing.""" + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.write(jm.metrics_path('300', 1), '') + cluster.write(jm.done_path('300', 1), '') + + cluster.reconcile() + + record = cluster.completed()['300'] + assert record['txApply'] is None + assert not any(record.get(k) is not None for k in jm.PEAK_FIELDS) + + +# --- two monitors ------------------------------------------------------------ + + +def test_two_monitors_racing_the_same_volume_never_double_dispatch(cluster): + """Job name uniqueness is the intended mutex. Prove it actually holds. + + The realistic race is not "B runs after A" -- B would simply see A's Jobs + in its LIST and skip them. It is both monitors LISTING before either + CREATES. That is reproduced here by handing the second reconcile the job + list as it was before the first pass ran, while its writes go to the one + real cluster. + """ + stale_jobs = cluster.k8s.batch_v1.list_namespaced_job( + cluster.namespace, label_selector=f"{jm.LABEL_RUN}={jm.RUN_NAME}") + assert stale_jobs.items == [] + + a = cluster.reconcile() + assert a['created'] == 2 + + real_list = cluster.k8s.batch_v1.list_namespaced_job + calls = {'n': 0} + + def list_from_before_the_race(namespace, **kw): + calls['n'] += 1 + if calls['n'] == 1: + return stale_jobs # B's snapshot: taken before A created + return real_list(namespace, **kw) + + cluster.k8s.batch_v1.list_namespaced_job = list_from_before_the_race + try: + # A second monitor process: its own state dict, sharing nothing but the + # cluster and the volume. + b_state = {'owner': jm.owner_ref(), 'replayed': set(), + 'max_completed': 0, 'halted': False, 'counted': {}} + jm.reconcile(b_state) + finally: + cluster.k8s.batch_v1.list_namespaced_job = real_list + + created = cluster.calls.names(verb='create', kind='job') + # B really did re-attempt the two ranges A had just taken -- otherwise this + # test proves nothing about the mutex. + assert created.count('pc-r300-a1') == 2 + assert created.count('pc-r200-a1') == 2 + + # The mutex: the duplicate creates were rejected, so each range has exactly + # ONE Job object and exactly one pod. Nothing ran twice. + for end in (300, 200): + name = f'pc-r{end}-a1' + assert cluster.jobs().count(name) == 1 + pods = [p for (_, _), p in cluster.k8s.pods.items() + if (p.metadata.labels or {}).get('job-name') == name] + assert len(pods) == 1, f"{name} spawned {len(pods)} pods" + # No range was escalated to a second attempt by the losing writer, and the + # shared volume was not double-provisioned either. + assert not any(n.endswith('-a2') for n in cluster.jobs()) + for end in (300, 200): + assert cluster.calls.names(verb='create', kind='pvc').count( + f'pc-data-r{end}') == 1 + + # Neither process crashed on the 409s, and B recorded nothing. + assert cluster.progress() == {} + + +def test_a_second_monitor_does_not_re_dispatch_recorded_ranges(cluster): + """A restart mid-run -- the same thing from the durable side. + + A fresh state dict has max_completed 0 and an empty replay set. Reading the + volume back must reproduce the run exactly: no redispatch of a recorded + range, no false regression halt from the counter starting at zero. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1) + cluster.reconcile() + assert '300' in cluster.completed() + created_before = list(cluster.calls.names(verb='create', kind='job')) + + fresh = {'owner': jm.owner_ref(), 'replayed': set(), 'max_completed': 0, + 'halted': False, 'counted': {}} + result = jm.reconcile(fresh) + + assert fresh['halted'] is False + assert cluster.calls.names(verb='create', kind='job').count('pc-r300-a1') == 1 + assert 'pc-r300-a2' not in cluster.jobs() + # The restart picks the record up rather than starting from zero. + assert fresh['max_completed'] == 1 + assert result['completed'] == 1 + assert result['remaining'] + len(result['in_progress']) + result['completed'] == 3 + # Only ranges that were genuinely unstarted moved. + assert set(cluster.calls.names(verb='create', kind='job')) - set(created_before) <= { + 'pc-r100-a1'} + + +# --- other people's objects -------------------------------------------------- + + +def test_a_foreign_run_s_jobs_in_the_namespace_are_ignored(cluster): + """The namespace is shared. Another run's Jobs carry another RUN_NAME and + must not be read as this run's ranges.""" + other = cluster.k8s.batch_v1.create_namespaced_job( + cluster.namespace, + jm.build_job(300, 420, 1, None)) + other.metadata.name = 'other-r300-a1' + other.metadata.labels = dict(other.metadata.labels or {}) + other.metadata.labels[jm.LABEL_RUN] = 'other-run' + cluster.k8s.jobs[(cluster.namespace, 'other-r300-a1')] = other + del cluster.k8s.jobs[(cluster.namespace, 'pc-r300-a1')] + + result = cluster.reconcile() + + # Our own range 300 was dispatched despite the foreign Job for the same + # ledger range already existing. + assert 'pc-r300-a1' in cluster.jobs() + assert sorted(result['in_progress']) == ['200/420', '300/420'] + assert result['remaining'] == 1 + assert result['total'] == 3 + + +@pytest.mark.parametrize('document', [ + {'completed': {'333': 'garbage'}, 'failed': {}}, # entry not a record + {'completed': {'333': None}, 'failed': {}}, + {'completed': ['333'], 'failed': {}}, # bucket not a map + {'completed': {}, 'failed': 'wat'}, + ['333'], # not even an object +]) +def test_a_structurally_wrong_progress_document_does_not_crash_the_pass( + cluster, document): + """Truncation is not the only corruption. + + A file that parses but has the wrong SHAPE gets past the ValueError guard, + and the walk over `completed` then raises -- after dispatch, inside a loop + that swallows exceptions. The run keeps its Jobs but stops publishing + status, so `num_remain` freezes and the mission waits on a number that will + never move again. + """ + cluster.write(jm.PROGRESS_FILE, json.dumps(document)) + + try: + result = cluster.reconcile() + except Exception as e: # noqa: BLE001 -- the point of the test + pytest.fail(f"a malformed progress record took the reconcile down: {e!r}") + + # Dispatch is unaffected, and -- the important half -- the garbage is not + # read as work already done. + assert result['completed'] == 0 + assert sorted(result['in_progress']) == ['200/420', '300/420'] + assert result['remaining'] == 1 + + # A real completion still lands on top of it, and the record heals. + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1) + cluster.reconcile() + assert set(cluster.completed()) == {'300'} + assert cluster.state['halted'] is False + + +def test_the_progress_configmap_being_deleted_mid_run_is_survivable(cluster): + """The mirror is best-effort. Losing it must not lose the run.""" + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1) + cluster.reconcile() + + cluster.k8s.core_v1.delete_namespaced_config_map(jm.PROGRESS_CM, + cluster.namespace) + cluster.advance(200, 'succeeded') + cluster.finalize(200, 1) + result = cluster.reconcile() + + assert set(cluster.completed()) == {'200', '300'} + assert cluster.state['halted'] is False + assert result['completed'] == 2 + # Recreated on the next write, with both entries. + assert set(cluster.progress_configmap()['completed']) == {'200', '300'} + + +def test_a_mirror_write_failure_does_not_stall_recording(cluster): + """A 413 from the ConfigMap patch used to throw inside reconcile, and the + loop swallows exceptions -- so no completion would ever be recorded again.""" + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1) + cluster.k8s.fail_next['patch configmap'] = fake_k8s.api_exception( + 413, 'RequestEntityTooLarge') + + result = cluster.reconcile() + + assert '300' in cluster.completed() + assert result['completed'] == 1 + assert cluster.state['halted'] is False diff --git a/src/MissionParallelCatchup/test_stateless_collector.py b/src/MissionParallelCatchup/test_stateless_collector.py new file mode 100644 index 00000000..0985a1f3 --- /dev/null +++ b/src/MissionParallelCatchup/test_stateless_collector.py @@ -0,0 +1,574 @@ +"""The collector sidecar restarts independently of the monitor. + +log_collector holds every peak it measures in module-level dicts, keyed by pod +name, and writes them to the shared volume. Those dicts do not survive an +OOM-kill of the sidecar, but the files on the volume do -- and the pod they +describe keeps running. So every durable write has to assume the process that +made the previous one is gone and that whatever is already on disk was measured +by someone who saw more than this process did. + +Two failures of that contract were found by hand before: a restart reset a +range's duration clock so attemptSeconds recorded 0.2s, and a newest-wins write +LOWERED an already-recorded peak. Both are pinned here. + +Everything is asserted against the bytes on the shared volume, read back either +with json.load or -- better -- through job_monitor's own readers, which are the +real consumer. Nothing here reads the collector's source. + +No reconcile: the volume is the entire interface between the two processes, so +these drive log_collector's file-writing entry points directly. +""" + +import asyncio +import json +import os + +import pytest + +import job_monitor as jm +import log_collector as lc + +GIB = 1073741824 + + +# -- the shared volume, and a collector with no memory of anything ------------ + +@pytest.fixture +def vol(tmp_path, monkeypatch): + """A shared /logs both processes agree on, and cleared collector state. + + Every module-level dict log_collector keeps is replaced, not emptied: they + are process state, and a test that inherited another test's pod entries + would be measuring the wrong process. + """ + log_dir = tmp_path / 'logs' + log_dir.mkdir() + monkeypatch.setattr(lc, 'LOG_DIR', str(log_dir)) + # The monitor reads the same directory off its own module global. + monkeypatch.setattr(jm, 'LOG_DIR', str(log_dir)) + restart(monkeypatch) + monkeypatch.setattr(lc, '_pod_secs', {}) + monkeypatch.setattr(lc, '_wake', {}) + monkeypatch.setattr(lc, 'token', lambda: 'test-token') + return log_dir + + +def restart(monkeypatch): + """Wipe exactly what an OOM-kill of the sidecar wipes: its memory. + + The volume is untouched, which is the whole point -- a restarted collector + starts every high-water at zero while the file on disk still holds the real + one. + """ + for name in ('_eph_peak', '_anon_peak', '_ws_peak', '_peak_flushed', '_streaming'): + if hasattr(lc, name): + monkeypatch.setattr(lc, name, {}) + # Added by the ephemeral-flush fix; absent on builds without it. + if hasattr(lc, '_eph_flushed'): + monkeypatch.setattr(lc, '_eph_flushed', {}) + + +def metrics(end, attempt=1): + """What the monitor would find in .metrics, or None if there is no file.""" + try: + with open(jm.metrics_path(str(end), attempt)) as fh: + return json.load(fh) + except OSError: + return None + + +def run(coro): + return asyncio.run(coro) + + +# -- a kubelet /stats/summary that says whatever the test needs --------------- + +class _Resp: + def __init__(self, payload): + self._payload = payload + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + def raise_for_status(self): + pass + + async def json(self): + return self._payload + + +class FakeSession: + """Stands in for the aiohttp session sample_kubelet fetches through.""" + + def __init__(self, payload): + self.payload = payload + self.urls = [] + + def get(self, url, **kwargs): + self.urls.append(url) + return _Resp(self.payload) + + +def summary(pod, rss=None, ws=None, eph=None, container=None): + """One node's stats/summary, shaped the way kubelet shapes it.""" + entry = {'podRef': {'name': pod}, 'containers': []} + if eph is not None: + entry['ephemeral-storage'] = {'usedBytes': eph} + mem = {} + if rss is not None: + mem['rssBytes'] = rss + if ws is not None: + mem['workingSetBytes'] = ws + entry['containers'].append({'name': container or lc.CONTAINER, 'memory': mem}) + return {'pods': [entry]} + + +def sample(pod, **kw): + """One real sample_kubelet pass over one node.""" + run(lc.sample_kubelet(FakeSession(summary(pod, **kw)), ['node-1'])) + + +def finalize(pod, end, attempt=1, succeeded=False, started=None, tx=None): + """One real finalize() for an attempt, as its poller would call it.""" + return run(lc.finalize(None, pod, str(end), attempt, + tx if tx is not None else lc.TxApplyScanner(), + lambda p: succeeded, started)) + + +# -- a peak may never go backwards ------------------------------------------- + +@pytest.mark.parametrize('key', lc.PEAK_KEYS) +def test_a_later_lower_write_cannot_lower_a_recorded_peak(vol, key): + """Every field in PEAK_KEYS, not just the one that was reported. + + This is the restarted-poller case reduced to its file operation: the second + write is a fresh process's first flush, and it is smaller because that + process started counting at zero. + """ + lc.write_metrics('300', 1, {key: 8 * GIB}) + lc.write_metrics('300', 1, {key: 1 * GIB}) + + assert metrics(300)[key] == 8 * GIB + + +@pytest.mark.parametrize('key', lc.PEAK_KEYS) +def test_a_later_higher_write_still_raises_the_peak(vol, key): + """The guard must not be a write-once latch: growth is the normal case.""" + lc.write_metrics('300', 1, {key: 1 * GIB}) + lc.write_metrics('300', 1, {key: 8 * GIB}) + + assert metrics(300)[key] == 8 * GIB + + +def test_a_write_that_omits_a_peak_leaves_it_alone(vol): + """finalize writes only the axes it has. The rest are already on disk.""" + lc.write_metrics('300', 1, {'peakAnonBytes': 5 * GIB, + 'peakEphemeralBytes': 30 * GIB}) + lc.write_metrics('300', 1, {'txApplySeconds': 12.5}) + + stored = metrics(300) + assert stored['peakAnonBytes'] == 5 * GIB + assert stored['peakEphemeralBytes'] == 30 * GIB + assert stored['txApplySeconds'] == 12.5 + + +def test_peaks_from_different_writes_accumulate_into_one_record(vol): + """Each axis is flushed by whoever measured it; the file is the union.""" + lc.write_metrics('300', 1, {'peakAnonBytes': 5 * GIB}) + lc.write_metrics('300', 1, {'peakWorkingSetBytes': 9 * GIB}) + lc.write_metrics('300', 1, {'peakEphemeralBytes': 30 * GIB}) + + assert metrics(300) == {'peakAnonBytes': 5 * GIB, + 'peakWorkingSetBytes': 9 * GIB, + 'peakEphemeralBytes': 30 * GIB} + + +# -- mid-flight flushes, and what a restart may lose -------------------------- + +def test_a_midflight_anon_flush_survives_a_collector_restart(vol, monkeypatch): + """The pinned bug, driven through the real sampler. + + A long range peaks early (download and bucket-apply), the sidecar is + OOM-killed, and the replacement watches only the quiet replay tail. What + the range gets sized on next run must still be the high-water. + """ + lc._streaming['w-300'] = ('300', '1') + sample('w-300', rss=6 * GIB) + assert metrics(300)['peakAnonBytes'] == 6 * GIB, "flush never reached the volume" + + restart(monkeypatch) + lc._streaming['w-300'] = ('300', '1') + sample('w-300', rss=1 * GIB) + finalize('w-300', 300) + + assert metrics(300)['peakAnonBytes'] == 6 * GIB + # And the consumer agrees: this is the figure that sizes the next run. + assert jm.peaks_for_range('300', 1)['peakAnonBytes'] == 6 * GIB + + +def test_a_midflight_ephemeral_flush_survives_a_collector_restart(vol, monkeypatch): + """peakEphemeralBytes sizes an ephemeral-storage request, and a request + that comes back too small is an eviction, not a slow range. + + Disk use is not monotonic -- stellar-core drops its download staging once + buckets are applied -- so a replacement sidecar re-measuring the same pod + does not recover the earlier high-water. It has to already be on the volume. + """ + monkeypatch.setattr(lc, 'STORAGE_MODE', 'ephemeral') + lc._streaming['w-300'] = ('300', '1') + sample('w-300', rss=1 * GIB, eph=34 * GIB) + + restart(monkeypatch) + monkeypatch.setattr(lc, 'STORAGE_MODE', 'ephemeral') + lc._streaming['w-300'] = ('300', '1') + sample('w-300', rss=1 * GIB, eph=4 * GIB) + finalize('w-300', 300) + + assert metrics(300)['peakEphemeralBytes'] == 34 * GIB + assert jm.peaks_for_range('300', 1)['peakEphemeralBytes'] == 34 * GIB + + +def test_pvc_mode_records_no_ephemeral_peak_at_all(vol, monkeypatch): + """In pvc mode the range's data sits on the volume, not on node disk, so + there is no ephemeral-storage request to size and the figure would be + noise. Sampling it is gated on the mode; flushing it must be too.""" + monkeypatch.setattr(lc, 'STORAGE_MODE', 'pvc') + lc._streaming['w-300'] = ('300', '1') + sample('w-300', rss=1 * GIB, eph=34 * GIB) + finalize('w-300', 300) + + assert 'peakEphemeralBytes' not in (metrics(300) or {}) + + +def test_a_flush_with_no_stream_registered_writes_nothing(vol, monkeypatch): + """_streaming is repopulated when a poller opens. A sample that lands on a + pod with no poller yet has nowhere to write and must not guess a file.""" + monkeypatch.setattr(lc, 'STORAGE_MODE', 'ephemeral') + sample('w-300', rss=6 * GIB, eph=34 * GIB) + + assert os.listdir(vol) == [] + + +def test_finalize_cannot_lower_a_peak_the_volume_already_holds(vol, monkeypatch): + """The restart case at the level of finalize itself. + + Whatever is in the replacement process's dicts is a partial observation; + the file was written by a process that saw more. + """ + lc.write_metrics('300', 1, {'peakAnonBytes': 6 * GIB, + 'peakWorkingSetBytes': 11 * GIB, + 'peakEphemeralBytes': 34 * GIB}) + lc._anon_peak['w-300'] = 1 * GIB + lc._ws_peak['w-300'] = 2 * GIB + lc._eph_peak['w-300'] = 3 * GIB + + finalize('w-300', 300) + + stored = metrics(300) + assert stored['peakAnonBytes'] == 6 * GIB + assert stored['peakWorkingSetBytes'] == 11 * GIB + assert stored['peakEphemeralBytes'] == 34 * GIB + + +def test_the_flush_ratio_does_not_hold_back_the_first_measurement(vol): + """A restarted sampler has flushed nothing, so its first sample must land + on the volume immediately -- otherwise a pod that peaks once and then dies + contributes nothing at all.""" + lc._streaming['w-300'] = ('300', '1') + sample('w-300', rss=3 * GIB) + + assert metrics(300)['peakAnonBytes'] == 3 * GIB + + +def test_a_flush_goes_to_the_attempt_that_is_streaming(vol): + """Peaks are keyed by (range, attempt); a retry must not inherit them.""" + lc._streaming['w-300-a2'] = ('300', '2') + sample('w-300-a2', rss=7 * GIB) + + assert metrics(300, 2)['peakAnonBytes'] == 7 * GIB + assert metrics(300, 1) is None + + +# -- .done is a promise about .metrics ---------------------------------------- + +def test_done_never_appears_beside_a_half_written_metrics_file(vol, monkeypatch): + """The monitor reaps the Job -- and with it the pod -- the moment .done + exists. If .metrics can be observed mid-write, that reap makes a torn + record permanent.""" + lc.write_metrics('300', 1, {'peakAnonBytes': 6 * GIB, 'txApplySeconds': 30.0}) + + real_dump = lc.json.dump + seen = {} + + def dump_then_die(obj, fh, *a, **kw): + # A write that dies with the file open: the failure mode the .tmp + + # rename is there for. + fh.write(json.dumps(obj)[:12]) + seen['torn'] = True + raise OSError(28, 'No space left on device') + + monkeypatch.setattr(lc.json, 'dump', dump_then_die) + lc._anon_peak['w-300'] = 9 * GIB + finalize('w-300', 300) + monkeypatch.setattr(lc.json, 'dump', real_dump) + + assert seen['torn'], "the interrupted write never happened" + # The old record is intact and parseable -- not truncated, not empty. + assert metrics(300) == {'peakAnonBytes': 6 * GIB, 'txApplySeconds': 30.0} + # .done still lands: the collector really will write nothing more for this + # attempt, and withholding it only strands the Job until its TTL. + assert os.path.exists(jm.done_path('300', 1)) + # What the monitor actually reads is a complete record, not a torn one. + assert jm.peaks_for_range('300', 1) == {'peakAnonBytes': 6 * GIB} + + +def test_done_lands_after_the_metrics_it_promises(vol): + """Ordering, observed by mtime rather than by reading the source.""" + lc._anon_peak['w-300'] = 6 * GIB + finalize('w-300', 300) + + assert (os.stat(jm.done_path('300', 1)).st_mtime_ns + >= os.stat(jm.metrics_path('300', 1)).st_mtime_ns) + assert metrics(300)['peakAnonBytes'] == 6 * GIB + + +def test_a_truncated_metrics_file_does_not_poison_the_next_write(vol): + """Whatever tore the previous record, the next flush must still produce a + file the monitor can read -- and must not raise inside the sampler.""" + with open(jm.metrics_path('300', 1), 'w') as fh: + fh.write('{"peakAnonBytes": 644245') + + lc.write_metrics('300', 1, {'peakAnonBytes': 5 * GIB}) + + assert metrics(300) == {'peakAnonBytes': 5 * GIB} + assert jm.peaks_for_range('300', 1) == {'peakAnonBytes': 5 * GIB} + + +def test_an_attempt_with_nothing_to_report_still_finalizes(vol): + """No peaks, no duration, no txApply -- a pod rejected before its container + ran. .done has to land anyway or the monitor waits out JOB_TTL_SECONDS on a + Job that will never learn anything.""" + finalize('w-300', 300) + + assert metrics(300) is None + assert jm._attempt_finalized('300', 1) + + +def test_marking_done_twice_is_harmless(vol): + lc._mark_done('300', 1) + lc._mark_done('300', 1) + + assert os.path.exists(jm.done_path('300', 1)) + assert jm._attempt_finalized('300', 1) + + +# -- finalizing the same attempt twice ---------------------------------------- + +def test_finalizing_the_same_attempt_twice_keeps_its_measurements(vol, monkeypatch): + """A restarted collector re-opens a poller for a pod that is still there + and still terminal, and finalizes it a second time. The second pass + measured nothing -- sample_kubelet only samples Running pods -- so it must + add nothing and take nothing away.""" + lc._pod_secs['w-300'] = 3600.4 + lc._anon_peak['w-300'] = 6 * GIB + lc._eph_peak['w-300'] = 34 * GIB + tx = lc.TxApplyScanner() + tx.seconds = 120.0 + finalize('w-300', 300, tx=tx) + first = metrics(300) + assert first['attemptSeconds'] == 3600.4 + + restart(monkeypatch) + # The main loop re-reads the pod's own timestamps every cycle it sees it + # terminal, so the second poller gets the same exact figure. + lc._pod_secs['w-300'] = 3600.4 + finalize('w-300', 300) + + assert metrics(300) == first + + +def test_a_second_finalize_without_pod_timestamps_keeps_the_real_duration(vol, + monkeypatch): + """attemptSeconds is a fixed quantity measured two ways, and both are lower + bounds: the pod's own start->finish is exact, while the poller's watch time + covers only the part of the attempt this process was alive for. A second + finalize that has lost the pod object -- 404 on the log endpoint, node + already reaped -- may only ever offer the worse of the two, so it must not + replace the better one. + + This is the same fabricated near-zero duration that was found by hand, + reached from the reopen path rather than from a cold start. + """ + lc._pod_secs['w-300'] = 3600.4 + finalize('w-300', 300) + assert metrics(300)['attemptSeconds'] == 3600.4 + + restart(monkeypatch) + # No _pod_secs: this poller never saw the pod terminal, it just took a 404. + # `started` is when IT attached, which is a moment ago. + finalize('w-300', 300, started=_moments_ago()) + + assert metrics(300)['attemptSeconds'] == 3600.4 + + +def _moments_ago(): + """A `started` stamp on the same monotonic clock finalize reads.""" + async def now(): + return asyncio.get_event_loop().time() + return run(now()) + + +def test_a_cold_poller_on_an_already_terminal_pod_reports_no_duration(vol): + """The other half of the pinned duration bug: with no pod timestamps and + no start of its own, the collector reports nothing rather than a + fabricated near-zero. The monitor's own figure is authoritative.""" + lc._anon_peak['w-300'] = 6 * GIB + finalize('w-300', 300, started=None) + + stored = metrics(300) + assert 'attemptSeconds' not in stored + assert stored['peakAnonBytes'] == 6 * GIB + # ...and the monitor is left free to supply the real one. + assert jm.seconds_for_range('300', 1, final=3600.4) == 3600.4 + + +def test_a_poller_that_watched_the_whole_attempt_still_reports_its_duration(vol): + """The fallback is not disabled, only outranked.""" + started = _moments_ago() - 42.0 + finalize('w-300', 300, started=started) + + assert metrics(300)['attemptSeconds'] == pytest.approx(42.0, abs=1.0) + + +def test_the_duration_the_collector_records_is_the_pods_not_the_pollers(vol): + """_pod_secs is the pod's own start->finish and always wins.""" + lc._pod_secs['w-300'] = 3600.4 + finalize('w-300', 300, started=_moments_ago() - 5.0) + + assert metrics(300)['attemptSeconds'] == 3600.4 + + +# -- .outcome is written once, by whoever got there first --------------------- + +def _pod(name, phase='Failed', exit_code=None, reason=None, message=None, + disrupted=False): + status = {'phase': phase} + if reason: + status['reason'] = reason + if message: + status['message'] = message + if disrupted: + status['conditions'] = [{'type': 'DisruptionTarget', 'status': 'True'}] + if exit_code is not None: + status['containerStatuses'] = [ + {'name': lc.CONTAINER, 'state': {'terminated': {'exitCode': exit_code}}}] + return {'metadata': {'name': name, 'labels': {}}, 'status': status} + + +def test_an_existing_outcome_is_not_overwritten_by_a_later_pod(vol): + """Two pods can carry the same range-end and attempt labels -- a Job that + replaces its pod, or a stale pod list after a restart. The first verdict is + the one taken while the evidence was fresh; a later, different pod must not + silently rewrite it.""" + lc.record_outcome(_pod('w-300-first', disrupted=True), '300', 1) + first = jm.read_outcome('300', 1) + + lc.record_outcome(_pod('w-300-second', exit_code=1), '300', 1) + + assert jm.read_outcome('300', 1) == first + assert first['outcome'] == 'disrupted' + assert first['pod'] == 'w-300-first' + + +def test_an_outcome_written_by_the_monitor_is_not_re_classified(vol, monkeypatch): + """Both processes write this file and both read it. The collector must + treat the monitor's verdict as final, including the fields only the monitor + records -- attemptSeconds for a failed leg lives nowhere else.""" + with open(jm.outcome_path('300', 1), 'w') as fh: + json.dump({'outcome': 'ephemeral', 'exitCode': None, 'pod': 'w-300', + 'attemptSeconds': 1800.0}, fh) + + lc.record_outcome(_pod('w-300', exit_code=3), '300', 1) + + assert jm.read_outcome('300', 1)['outcome'] == 'ephemeral' + assert jm.read_outcome('300', 1)['attemptSeconds'] == 1800.0 + + +def test_a_recorded_outcome_is_a_complete_file_or_no_file(vol, monkeypatch): + """Same rename discipline as .metrics: the monitor branches its whole retry + policy on this file, so a torn read would have to be a crash or a wrong + verdict.""" + def dump_then_die(obj, fh, *a, **kw): + fh.write(json.dumps(obj)[:9]) + raise OSError(28, 'No space left on device') + + monkeypatch.setattr(lc.json, 'dump', dump_then_die) + lc.record_outcome(_pod('w-300', exit_code=1), '300', 1) + + assert jm.read_outcome('300', 1) is None + assert not os.path.exists(jm.outcome_path('300', 1)) + + +def test_an_ephemeral_eviction_is_classified_from_the_pod_message(vol): + """The exit code cannot tell this apart from a catchup failure, and only + the pod carries the discriminator -- so if the collector misses it while + the pod exists, it is gone.""" + lc.record_outcome( + _pod('w-300', exit_code=3, reason='Evicted', + message='Pod ephemeral local storage usage exceeds the total limit ' + 'of containers 40Gi'), + '300', 1) + + assert jm.read_outcome('300', 1)['outcome'] == 'ephemeral' + + +# -- the resume state file ---------------------------------------------------- + +def test_state_survives_a_restart_and_untimestamped_junk_never_becomes_it(vol): + """The resume point is read back by a process that did not write it, so a + poisoned value is permanent: sinceTime=unableZ is a 400 on every later + request for that pod, forever.""" + lc.write_state('300', 1, '2026-07-30T10:15:30.123456789Z') + assert lc.read_state('300', 1) == '2026-07-30T10:15:30.123456789Z' + + lc.write_state('300', 1, 'unable') + assert lc.read_state('300', 1) is None + + +def test_an_empty_state_claim_is_not_a_resume_point(vol): + """poll_pod writes '' to claim the range against job_monitor's backstop. + That is a claim, not a timestamp, and must never be sent as sinceTime.""" + lc.write_state('300', 1, '') + + assert lc.read_state('300', 1) is None + assert os.path.exists(lc.base('300', 1) + '.state') + + +def test_discarding_a_successful_range_keeps_its_measurements(vol): + """saveSuccessLogs=false deletes the archive. .metrics is the only place + txApply and the peaks survive a reaped pod, so it has to stay.""" + lc.write_metrics('300', 1, {'peakAnonBytes': 6 * GIB, 'txApplySeconds': 30.0}) + with open(lc.base('300', 1) + '.log.gz', 'wb') as fh: + fh.write(b'\x1f\x8b') + lc.write_state('300', 1, '2026-07-30T10:15:30Z') + + lc.discard('300', 1) + + assert not os.path.exists(lc.base('300', 1) + '.log.gz') + assert metrics(300) == {'peakAnonBytes': 6 * GIB, 'txApplySeconds': 30.0} + + +def test_a_successful_range_discards_its_archive_inside_finalize(vol, monkeypatch): + monkeypatch.setattr(lc, 'SAVE_SUCCESS_LOGS', False) + with open(lc.base('300', 1) + '.log.gz', 'wb') as fh: + fh.write(b'\x1f\x8b') + lc._anon_peak['w-300'] = 6 * GIB + + finalize('w-300', 300, succeeded=True) + + assert not os.path.exists(lc.base('300', 1) + '.log.gz') + assert metrics(300)['peakAnonBytes'] == 6 * GIB + assert os.path.exists(jm.done_path('300', 1)) diff --git a/src/MissionParallelCatchup/test_stateless_crashpoints.py b/src/MissionParallelCatchup/test_stateless_crashpoints.py new file mode 100644 index 00000000..f0012683 --- /dev/null +++ b/src/MissionParallelCatchup/test_stateless_crashpoints.py @@ -0,0 +1,645 @@ +"""Crash the monitor mid-pass, at every side-effect boundary, and restart it. + +reconcile() is a reconciler: it must derive everything it needs from Kubernetes +plus the files on its own volume, so a process that dies halfway through a pass +and comes back with a zeroed in-memory state must converge to the same place. + +The side-effect boundaries inside one pass, in order, are: + + dispatch create_namespaced_job -> `created`/`capacity`/in_progress + success completed[end] = ... -> save_progress -> release_pvc -> reap + backfill completed[end].update(late) -> save_progress -> reap + retry save_verdict -> create attempt N+1 -> delete attempt N + +Every test here kills the process at one of those arrows and then restarts it +with a fresh state dict -- the same thing a pod replacement does -- and asserts +on observed cluster state and the durable record: every range recorded exactly +once, no PVC left behind, no Job left orphaned, no completed range re-run. + +Nothing is asserted about the source text; the injections wrap the fake API or +a single job_monitor function, and everything checked afterwards is either a +file on the volume or an object in the fake cluster. +""" + +import pytest +from kubernetes.client.rest import ApiException + +import fake_k8s +import job_monitor as jm + + +TOTAL_RANGES = 3 # conftest's DEFAULT_CONFIG generates 300/200/100 + + +class Crash(RuntimeError): + """A hard process death -- deliberately NOT an ApiException. + + The monitor handles ApiException in several places; a crash is the thing it + cannot handle, and is what a SIGKILL, an OOM or a node eviction looks like + from inside a pass. + """ + + +# --- injection helpers ------------------------------------------------------- + +def crash_before(monkeypatch, target, name, times=1): + """Die on the way INTO `target.name` -- the effect never happens.""" + real = getattr(target, name) + left = {'n': times} + + def wrapper(*args, **kwargs): + if left['n'] > 0: + left['n'] -= 1 + raise Crash(f"crash before {name}") + return real(*args, **kwargs) + + monkeypatch.setattr(target, name, wrapper) + return left + + +def crash_after(monkeypatch, target, name, times=1, match=None): + """Die on the way OUT of `target.name` -- the effect happened, the caller + never learned about it. This is the boundary that can duplicate work.""" + real = getattr(target, name) + left = {'n': times} + + def wrapper(*args, **kwargs): + result = real(*args, **kwargs) + if left['n'] > 0 and (match is None or match(*args, **kwargs)): + left['n'] -= 1 + raise Crash(f"crash after {name}") + return result + + monkeypatch.setattr(target, name, wrapper) + return left + + +def restart(cluster): + """Replace the monitor process: fresh in-memory state, same volume+cluster. + + Identical to the dict update_status_and_metrics() builds on entry, so a + restarted monitor starts from exactly what the shipped loop starts from. + """ + cluster.state = {'owner': None, 'replayed': set(), 'max_completed': 0, + 'halted': False, 'counted': {}} + return cluster + + +# --- driving ----------------------------------------------------------------- + +def split(job_name): + """'pc-r300-a2' -> (300, 2)""" + stem, _, attempt = job_name.rpartition('-a') + return int(stem.rsplit('-r', 1)[1]), int(attempt) + + +def finish_live_jobs(cluster, outcome='succeeded', finalize=True): + """Take every not-yet-terminal Job to a terminal state, as the cluster would.""" + touched = [] + for name in sorted(cluster.jobs()): + end, attempt = split(name) + status = cluster.k8s.job(name).status + if status and (status.succeeded or status.failed): + continue + cluster.advance(end, outcome, attempt) + if finalize: + cluster.finalize(end, attempt, tx_apply=1.0, + peaks={'peakAnonBytes': 1024}) + touched.append(name) + return touched + + +def run_to_quiescence(cluster, passes=15): + """Succeed everything still in flight until the run drains (or we give up).""" + for _ in range(passes): + finish_live_jobs(cluster) + cluster.reconcile() + if len(cluster.completed()) == TOTAL_RANGES and not cluster.jobs(): + break + return cluster + + +def assert_converged(cluster): + """The end state of a healthy run, whatever happened on the way there.""" + completed = cluster.completed() + assert sorted(completed) == ['100', '200', '300'], completed + assert cluster.failed() == {} + # Every range recorded once and only once -- a dict cannot hold a duplicate + # key, so the observable form of "counted twice" is a re-run: a second + # attempt for a range that had already been recorded. + for end, record in completed.items(): + assert record['attempts'] == 1, (end, record) + assert cluster.jobs() == [], f"orphaned Jobs: {cluster.jobs()}" + assert cluster.pvcs() == [], f"leaked PVCs: {cluster.pvcs()}" + + +def creates_of(cluster, name): + return cluster.calls.names(verb='create', kind='job').count(name) + + +# --- dispatch boundary ------------------------------------------------------- + +def test_crash_after_create_before_the_range_is_tracked(cluster, monkeypatch): + """create_namespaced_job returned, then the process died. + + The Job exists and nobody recorded that it does. A restart must find it by + LIST and adopt it, not dispatch the range a second time. + """ + crash_after(monkeypatch, cluster.k8s.batch_v1, 'create_namespaced_job') + + with pytest.raises(Crash): + cluster.reconcile() + + # The Job that the dying pass created is real and running. + assert cluster.jobs() == ['pc-r300-a1'] + + restart(cluster) + result = cluster.reconcile() + + # Adopted, not recreated: one create call ever for this name, and the + # restarted pass counts it against capacity instead of dispatching a third. + assert creates_of(cluster, 'pc-r300-a1') == 1 + assert cluster.jobs() == ['pc-r200-a1', 'pc-r300-a1'] + assert sorted(result['in_progress']) == ['200/420', '300/420'] + assert result['created'] == 1 + + run_to_quiescence(cluster) + assert_converged(cluster) + + +def test_crash_after_pvc_create_before_job_create(cluster, monkeypatch): + """ensure_pvc() ran, the Job create never did. The volume must be reused.""" + crash_after(monkeypatch, cluster.k8s.core_v1, + 'create_namespaced_persistent_volume_claim') + + with pytest.raises(Crash): + cluster.reconcile() + + assert cluster.pvcs() == ['pc-data-r300'] + assert cluster.jobs() == [] + + restart(cluster) + cluster.reconcile() + + # One volume for the range, not two, and the Job now mounts it. + assert cluster.calls.names(verb='create', kind='pvc').count('pc-data-r300') == 1 + job = cluster.k8s.job('pc-r300-a1') + claim = job.spec.template.spec.volumes[0].persistent_volume_claim + assert claim.claim_name == 'pc-data-r300' + + run_to_quiescence(cluster) + assert_converged(cluster) + + +# --- success boundary: record -> save_progress -> release_pvc -> reap --------- + +def test_crash_before_save_progress_records_the_range_exactly_once(cluster, + monkeypatch): + """completed[end] existed only in memory. Nothing durable, so redo it.""" + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakAnonBytes': 7}) + + crash_before(monkeypatch, jm, 'save_progress') + with pytest.raises(Crash): + cluster.reconcile() + + # Nothing was written, so nothing is claimed -- and crucially the Job was + # NOT reaped, because the reap sits after the write. + assert cluster.progress() == {} + assert 'pc-r300-a1' in cluster.jobs() + + restart(cluster) + cluster.reconcile() + + record = cluster.completed()['300'] + assert record['attempts'] == 1 + assert record['txApply'] == 1.5 + assert record['peakAnonBytes'] == 7 + assert creates_of(cluster, 'pc-r300-a1') == 1 + assert 'pc-r300-a2' not in cluster.jobs(), "a recorded range must never re-run" + + run_to_quiescence(cluster) + assert_converged(cluster) + + +def test_crash_between_the_progress_file_and_its_configmap_mirror(cluster, + monkeypatch): + """The file is authoritative; the mirror is best effort and catches up.""" + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakAnonBytes': 7}) + + crash_before(monkeypatch, jm, '_patch_cm') + with pytest.raises(Crash): + cluster.reconcile() + + # os.replace() landed before the mirror was attempted. + assert '300' in cluster.progress()['completed'] + + restart(cluster) + cluster.reconcile() + + # Reloaded from the file, not re-derived from the cluster: same attempt, + # and no second Job. + assert cluster.completed()['300']['attempts'] == 1 + assert creates_of(cluster, 'pc-r300-a1') == 1 + + run_to_quiescence(cluster) + assert_converged(cluster) + # The mirror is whole again once any later write re-publishes the document. + assert sorted(cluster.progress_configmap()['completed']) == ['100', '200', '300'] + + +def test_crash_after_save_progress_before_release_pvc_does_not_leak_the_volume( + cluster, monkeypatch): + """The record is durable and the volume is not yet freed. + + A completed range has nothing left to resume, so its PVC is dead weight -- + 40Gi of gp3 apiece, which is what put 79 TiB on ssc-test. The release must + therefore be reached on a LATER pass too, because the pass that would have + done it is never repeated: the record already exists. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakAnonBytes': 7}) + + crash_before(monkeypatch, jm, 'release_pvc') + with pytest.raises(Crash): + cluster.reconcile() + + assert '300' in cluster.progress()['completed'] + assert 'pc-data-r300' in cluster.pvcs() + + restart(cluster) + for _ in range(3): + cluster.reconcile() + + assert 'pc-data-r300' not in cluster.pvcs(), ( + "the volume of a range recorded complete before the crash was never " + "released; only the first-sight branch releases it and that branch " + "never runs again") + + run_to_quiescence(cluster) + assert_converged(cluster) + + +def test_crash_after_release_pvc_before_the_reap_does_not_orphan_the_job( + cluster, monkeypatch): + """The window that leaves a Job with no owner. + + The range is recorded, its volume is gone, and its Job is still standing. + Nothing in the cluster will ever ask about that Job again -- it is not in + flight, it is not retryable, and its range is complete -- so the reconciler + is the only thing that can clean it up. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakAnonBytes': 7}) + + crash_before(monkeypatch, jm, 'reap_range_jobs') + with pytest.raises(Crash): + cluster.reconcile() + + assert '300' in cluster.progress()['completed'] + assert 'pc-data-r300' not in cluster.pvcs() + assert 'pc-r300-a1' in cluster.jobs(), "precondition: the ownerless Job" + + restart(cluster) + for _ in range(3): + cluster.reconcile() + + assert 'pc-r300-a1' not in cluster.jobs(), ( + "a Job whose range is already recorded complete was left standing " + "forever; it inflates every later LIST and its pod holds a node") + # ...and cleaning it up must not have cost anything: the range stays + # recorded once, with its measurements. + assert cluster.completed()['300']['txApply'] == 1.5 + assert cluster.completed()['300']['attempts'] == 1 + + run_to_quiescence(cluster) + assert_converged(cluster) + + +def test_crash_after_the_reap_leaves_nothing_behind(cluster, monkeypatch): + """Last arrow in the success path: everything is done, the pass just dies.""" + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakAnonBytes': 7}) + + crash_after(monkeypatch, jm, 'reap_range_jobs') + with pytest.raises(Crash): + cluster.reconcile() + + assert 'pc-r300-a1' not in cluster.jobs() + assert 'pc-data-r300' not in cluster.pvcs() + + restart(cluster) + cluster.reconcile() + + # The slot the reaped range freed is refilled, and the range is not redone. + assert cluster.completed()['300']['attempts'] == 1 + assert creates_of(cluster, 'pc-r300-a1') == 1 + assert 'pc-r100-a1' in cluster.jobs() + + run_to_quiescence(cluster) + assert_converged(cluster) + + +# --- backfill boundary ------------------------------------------------------- + +def test_crash_mid_backfill_backfills_on_a_later_pass(cluster, monkeypatch): + """The record was written before the collector finalized; a crash in the + middle of the catch-up write must not make the measurements unreachable.""" + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.reconcile() # recorded with nothing to read yet + + record = cluster.completed()['300'] + assert record['txApply'] is None + assert 'peakAnonBytes' not in record + assert 'pc-r300-a1' in cluster.jobs(), "not finalized, so not reaped" + + # The collector lands, and the monitor dies on the backfill write. + cluster.finalize(300, 1, tx_apply=2.5, peaks={'peakAnonBytes': 99}) + crash_after(monkeypatch, jm, 'save_progress') + with pytest.raises(Crash): + cluster.reconcile() + + restart(cluster) + cluster.reconcile() + + record = cluster.completed()['300'] + assert record['txApply'] == 2.5 + assert record['peakAnonBytes'] == 99 + assert record['attempts'] == 1 + assert 'pc-r300-a1' not in cluster.jobs(), "finalized and backfilled: reap it" + + run_to_quiescence(cluster) + assert_converged(cluster) + + +# --- retry boundary: verdict -> create N+1 -> delete N ----------------------- + +def test_crash_after_the_successor_exists_before_the_predecessor_is_deleted( + cluster, monkeypatch): + """Both attempts are live for a moment. The pass that dies there must not + leave the loser standing once the range finishes.""" + cluster.reconcile() + cluster.advance(300, 'incomplete') + cluster.finalize(300, 1) # finalized, so a-1 is deletable + + crash_after(monkeypatch, cluster.k8s.batch_v1, 'create_namespaced_job', + match=lambda ns, body, **kw: body.metadata.name == 'pc-r300-a2') + with pytest.raises(Crash): + cluster.reconcile() + + assert 'pc-r300-a1' in cluster.jobs() and 'pc-r300-a2' in cluster.jobs() + + restart(cluster) + cluster.reconcile() + + # The dead a-1 must never be re-classified into a third attempt: live[] + # keys on the highest attempt for the range. + assert 'pc-r300-a3' not in cluster.jobs() + assert creates_of(cluster, 'pc-r300-a2') == 1 + assert jm._cause_count('300', 2, ('oom', 'failed')) == 1, \ + "attempt 1 must be counted once, not once per pass that saw it" + + cluster.advance(300, 'succeeded', attempt=2) + cluster.finalize(300, 2, tx_apply=1.0, peaks={'peakAnonBytes': 1024}) + cluster.reconcile() + + assert cluster.completed()['300']['attempts'] == 2 + assert 'pc-r300-a1' not in cluster.jobs(), "the loser must be swept too" + assert 'pc-r300-a2' not in cluster.jobs() + assert 'pc-data-r300' not in cluster.pvcs() + + +def test_crash_between_the_verdict_and_the_retry_create(cluster, monkeypatch): + """The verdict is on disk and the successor was never created. + + The verdict is what spends the range's budget, so replaying the same failed + attempt after a restart must not spend it a second time -- and for an OOM, + must not climb a second escalation rung either. + """ + cluster.reconcile() + cluster.advance(300, 'oom') + + crash_before(monkeypatch, cluster.k8s.batch_v1, 'create_namespaced_job') + with pytest.raises(Crash): + cluster.reconcile() + + assert jm._verdict_of('300', 1) == 'oom' + assert 'pc-r300-a1' in cluster.jobs(), \ + "the predecessor must survive: without it the range restarts at attempt 1" + + restart(cluster) + cluster.reconcile() + + assert 'pc-r300-a2' in cluster.jobs() + resources = (cluster.k8s.job('pc-r300-a2') + .spec.template.spec.containers[0].resources) + # One OOM seen, so exactly one rung: 24000Mi * 1.5. Two would mean the + # replayed attempt was counted twice. + assert resources.limits['memory'] == '36000Mi' + assert jm._cause_count('300', 1, ('oom', 'failed')) == 1 + assert cluster.failed() == {} + + +# --- API errors on create ---------------------------------------------------- + +def test_409_on_dispatch_is_benign_and_does_not_double_record(cluster): + """AlreadyExists is the dispatch mutex, not an error.""" + cluster.k8s.fail_next['create job'] = fake_k8s.api_exception( + 409, 'Conflict', 'jobs.batch "pc-r300-a1" already exists') + + result = cluster.reconcile() # must not raise + + # Whatever the pass counts, it must not count a Job it did not create... + assert result['created'] == len(cluster.jobs()) + # ...nor claim anything about the range. + assert cluster.progress() == {}, "a swallowed 409 must not record anything" + assert cluster.failed() == {} + assert 'pc-r300-a1' not in cluster.jobs() + + run_to_quiescence(cluster) + assert_converged(cluster) + # Each range ran once: no range was ever dispatched at attempt 2. + creates = cluster.calls.names(verb='create', kind='job') + assert sorted(creates) == ['pc-r100-a1', 'pc-r200-a1', 'pc-r300-a1'] + + +def test_409_means_the_slot_is_taken_and_must_not_over_dispatch(cluster, + monkeypatch): + """Losing the create race means the Job EXISTS and is in flight. + + The monitor's own comment calls name uniqueness the dispatch mutex, which is + only true if losing it is treated as "someone else holds this slot". A 409 + that does not spend capacity dispatches PARALLELISM+1 workers -- and at 1024 + parallelism that is a fleet-wide overshoot, not a rounding error. + """ + real_create = cluster.k8s.batch_v1.create_namespaced_job + lost = [] + + def loser(namespace, body, **kwargs): + if body.metadata.name == 'pc-r300-a1' and not lost: + lost.append(body.metadata.name) + real_create(namespace, body, **kwargs) # the other writer's object + raise fake_k8s.api_exception( + 409, 'Conflict', 'jobs.batch "pc-r300-a1" already exists') + return real_create(namespace, body, **kwargs) + + monkeypatch.setattr(cluster.k8s.batch_v1, 'create_namespaced_job', loser) + + result = cluster.reconcile() + + assert lost, "precondition: the create actually lost the race" + assert len(cluster.jobs()) <= jm.PARALLELISM, ( + f"dispatched {cluster.jobs()} against PARALLELISM={jm.PARALLELISM}: " + "a 409 left the slot looking free") + assert '300/420' in result['in_progress'], \ + "the range whose Job exists is in flight and must be reported as such" + assert result['remaining'] == 1, \ + "a range with a running Job is not still waiting to be dispatched" + + run_to_quiescence(cluster) + assert_converged(cluster) + + +def test_500_on_dispatch_is_retried_on_a_later_pass(cluster): + """A server error is not a verdict: the range must survive it.""" + cluster.k8s.fail_next['create job'] = fake_k8s.api_exception(500, 'boom') + + with pytest.raises(ApiException) as err: + cluster.reconcile() + assert err.value.status == 500 + + assert cluster.jobs() == [], "nothing was created by the aborted pass" + assert cluster.progress() == {} + + restart(cluster) + result = cluster.reconcile() + + assert cluster.jobs() == ['pc-r200-a1', 'pc-r300-a1'] + assert result['created'] == 2 + # The volume the aborted pass provisioned is reused, not duplicated. + assert cluster.calls.names(verb='create', kind='pvc').count('pc-data-r300') == 1 + + run_to_quiescence(cluster) + assert_converged(cluster) + + +def test_500_on_the_retry_create_does_not_lose_or_double_spend_the_range(cluster): + """The retry create fails hard. The range keeps its budget and its history.""" + cluster.reconcile() + cluster.advance(300, 'incomplete') + cluster.finalize(300, 1) + + cluster.k8s.fail_next['create job'] = fake_k8s.api_exception(500, 'boom') + with pytest.raises(ApiException): + cluster.reconcile() + + assert 'pc-r300-a1' in cluster.jobs(), \ + "deleting the predecessor before the successor exists restarts the range" + assert cluster.failed() == {} + + restart(cluster) + cluster.reconcile() + + assert 'pc-r300-a2' in cluster.jobs() + assert jm._cause_count('300', 2, ('oom', 'failed')) == 1 + assert cluster.calls.names(verb='create', kind='pvc').count('pc-data-r300') == 1 + + cluster.advance(300, 'succeeded', attempt=2) + cluster.finalize(300, 2, tx_apply=1.0, peaks={'peakAnonBytes': 1024}) + cluster.reconcile() + assert cluster.completed()['300']['attempts'] == 2 + + +def test_a_range_that_exhausts_its_budget_across_crashes_fails_once(cluster, + monkeypatch): + """Budgets are spent by durable verdicts, so restarts must not stretch or + shrink them. Five attempts, a crash before each retry create.""" + cluster.reconcile() + for attempt in range(1, jm.MAX_ATTEMPTS_PER_RANGE + 1): + cluster.advance(300, 'incomplete', attempt=attempt) + cluster.finalize(300, attempt) + crash_before(monkeypatch, cluster.k8s.batch_v1, 'create_namespaced_job') + with pytest.raises(Crash): + cluster.reconcile() + restart(cluster) + cluster.reconcile() + + assert cluster.failed()['300']['attempts'] == jm.MAX_ATTEMPTS_PER_RANGE + assert cluster.failed()['300']['outcome'] == 'failed' + # Exactly MAX_ATTEMPTS Jobs were ever created for the range, despite five + # crashed passes replaying the same failed attempts. + creates = cluster.calls.names(verb='create', kind='job') + assert sorted(n for n in creates if n.startswith('pc-r300-')) == [ + f'pc-r300-a{n}' for n in range(1, jm.MAX_ATTEMPTS_PER_RANGE + 1)] + + +# --- end to end -------------------------------------------------------------- + +def test_the_run_converges_with_a_crash_at_every_boundary(cluster, monkeypatch): + """One crash at each arrow, spread across one run, restarting every time.""" + # 1. after the Job create, before the range is tracked + crash_after(monkeypatch, cluster.k8s.batch_v1, 'create_namespaced_job') + with pytest.raises(Crash): + cluster.reconcile() + restart(cluster) + cluster.reconcile() + + # 2. after the record, before save_progress + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1, tx_apply=1.0, peaks={'peakAnonBytes': 1024}) + crash_before(monkeypatch, jm, 'save_progress') + with pytest.raises(Crash): + cluster.reconcile() + restart(cluster) + cluster.reconcile() + + # 3. after save_progress, before release_pvc + cluster.advance(200, 'succeeded') + cluster.finalize(200, 1, tx_apply=1.0, peaks={'peakAnonBytes': 1024}) + crash_before(monkeypatch, jm, 'release_pvc') + with pytest.raises(Crash): + cluster.reconcile() + restart(cluster) + cluster.reconcile() + + # 4. after release_pvc, before the reap + cluster.advance(100, 'succeeded') + cluster.finalize(100, 1, tx_apply=1.0, peaks={'peakAnonBytes': 1024}) + crash_before(monkeypatch, jm, 'reap_range_jobs') + with pytest.raises(Crash): + cluster.reconcile() + restart(cluster) + + run_to_quiescence(cluster) + assert_converged(cluster) + + # Three ranges, three Jobs, ever. Nothing was replayed by a restart. + assert sorted(cluster.calls.names(verb='create', kind='job')) == [ + 'pc-r100-a1', 'pc-r200-a1', 'pc-r300-a1'] + assert sorted(cluster.calls.names(verb='create', kind='pvc')) == [ + 'pc-data-r100', 'pc-data-r200', 'pc-data-r300'] + # ...and every measurement survived the crashes. + for end in ('100', '200', '300'): + assert cluster.completed()[end]['txApply'] == 1.0 + assert cluster.completed()[end]['peakAnonBytes'] == 1024 + + +def test_a_restart_between_every_single_pass_changes_nothing(cluster): + """The control: the same run with a fresh process for every pass.""" + for _ in range(12): + restart(cluster) + finish_live_jobs(cluster) + cluster.reconcile() + if len(cluster.completed()) == TOTAL_RANGES and not cluster.jobs(): + break + + assert_converged(cluster) + assert sorted(cluster.calls.names(verb='create', kind='job')) == [ + 'pc-r100-a1', 'pc-r200-a1', 'pc-r300-a1'] diff --git a/src/MissionParallelCatchup/test_stateless_restart.py b/src/MissionParallelCatchup/test_stateless_restart.py new file mode 100644 index 00000000..98e02bc4 --- /dev/null +++ b/src/MissionParallelCatchup/test_stateless_restart.py @@ -0,0 +1,533 @@ +"""Restart invisibility: a monitor restart between any two reconcile passes +must change nothing an observer can see. + +The monitor is a reconciler. Every decision it makes has to be derivable from +the Kubernetes objects plus the durable files on the logs volume; anything it +keeps only in RAM is lost the moment the pod is rescheduled, and a 10-hour run +gets rescheduled. `restart()` below is the whole trick: it discards exactly +what a process death discards -- the `state` dict reconcile() carries across +passes, and the module-level owner cache -- and keeps exactly what survives, +the logs volume and the cluster. + +Everything here asserts on observed state: the durable progress record, the +live Job/Pod objects, and the API call log. Nothing reads job_monitor's source. +""" + +import json +import os +import random + +import pytest + +import job_monitor as jm + +# The states the fuzz drives Jobs through. A real run is dominated by success, +# with spot evictions the most common failure, then OOM, then a hung archive +# fetch tripping the attempt deadline. `unknown` is the restart's own signature +# -- the Job failed while the monitor was down and the pod was reaped with it, +# so nothing is left to classify from. +DRIVE_STATES = ('succeeded', 'disrupted', 'oom', 'timeout', 'unknown') +DRIVE_WEIGHTS = (6, 3, 2, 2, 1) + +# 30 seeds x 24 passes runs in ~9s. RESTART_FUZZ_SEEDS / RESTART_FUZZ_PASSES +# widen it for a soak without editing the file -- 400 x 40 takes ~2.5 minutes. +PASSES = int(os.getenv('RESTART_FUZZ_PASSES', 24)) +SEEDS = list(range(int(os.getenv('RESTART_FUZZ_SEEDS', 30)))) + + +# --- the restart ------------------------------------------------------------ + +def restart(cluster): + """Simulate the monitor process dying and being rescheduled. + + Gone: the in-memory `state` dict (owner reference, histogram replay guard, + the monotonic-progress high-water mark, the counter deltas) and the + module-level owner cache. Kept: the logs volume and every object in the + cluster -- which between them are the only inputs a reconciler is allowed + to have. + """ + cluster.state = {'owner': None, 'replayed': set(), 'max_completed': 0, + 'halted': False, 'counted': {}} + jm._progress_owner.clear() + jm.PROFILE = None + + +# --- cluster inspection ----------------------------------------------------- + +def _terminal(job): + st = job.status + return bool(st and (st.succeeded or st.failed)) + + +def _jobs_by_range(cluster): + """range-end (str) -> [(attempt, job)], from the cluster, not from state.""" + out = {} + for name in cluster.jobs(): + job = cluster.k8s.job(name) + labels = job.metadata.labels or {} + end = labels.get(jm.LABEL_RANGE) + attempt = int(labels.get(jm.LABEL_ATTEMPT, 1)) + out.setdefault(end, []).append((attempt, job)) + return out + + +def _range_of_job(name): + """'pc-r1200-a3' -> ('1200', 3).""" + stem, _, attempt = name.rpartition('-a') + return stem.split('-r', 1)[1], int(attempt) + + +# --- the invariants --------------------------------------------------------- + +class Ledger: + """Cross-pass bookkeeping the invariants need (high-water marks, first + sighting of a completion, every pod ever seen).""" + + def __init__(self, ends): + self.ends = set(ends) + self.total = len(ends) + self.dispatched = set() # every range that has ever had a Job + self.recorded_at = {} # end -> len(calls) when first completed + self.peaks = {} # end -> {field: high-water value} + self.pods = {} # (end, attempt) -> pod name + + +def check(cluster, result, led, where): + """Assert I1..I6 against observed state. `where` names the pass.""" + progress = cluster.progress() + completed = set(progress.get('completed', {})) + failed = set(progress.get('failed', {})) + by_range = _jobs_by_range(cluster) + + for end, entries in by_range.items(): + led.dispatched.add(end) + for name in cluster.calls.names(verb='create', kind='job'): + led.dispatched.add(_range_of_job(name)[0]) + + # A Job that has succeeded or failed is a record, not work in flight. The + # monitor deliberately leaves a finished Job standing until the collector + # finalizes it, so "live" has to mean unfinished, not merely present. + live = {end for end, entries in by_range.items() + if any(not _terminal(j) for _, j in entries)} + + # -- I1: exactly one of completed / failed / live ------------------------ + assert completed <= led.ends, f"{where}: completed has unknown ranges {completed - led.ends}" + assert failed <= led.ends, f"{where}: failed has unknown ranges {failed - led.ends}" + assert live <= led.ends, f"{where}: live has unknown ranges {live - led.ends}" + assert not (completed & failed), \ + f"{where}: ranges both completed and failed: {sorted(completed & failed)}" + assert not (completed & live), \ + f"{where}: completed ranges with work still in flight: {sorted(completed & live)}" + assert not (failed & live), \ + f"{where}: failed ranges with work still in flight: {sorted(failed & live)}" + # Never zero: a range that has been dispatched must stay accounted for. + # Undispatched ranges are simply queued behind PARALLELISM -- that is the + # fourth, legitimate bucket, and it only ever shrinks. + lost = led.dispatched - completed - failed - live + assert not lost, (f"{where}: dispatched ranges accounted for nowhere -- " + f"no record and no live Job: {sorted(lost)}") + + # -- I2: at most one live Job per range ---------------------------------- + for end, entries in by_range.items(): + unfinished = [a for a, j in entries if not _terminal(j)] + assert len(unfinished) <= 1, \ + f"{where}: range {end} has {len(unfinished)} live Jobs (attempts {unfinished})" + + # ...and the run never runs wider than it was told to. A restart that + # forgot what was in flight would show up here first. + assert len(result['in_progress']) <= jm.PARALLELISM, \ + f"{where}: {len(result['in_progress'])} in flight over PARALLELISM {jm.PARALLELISM}" + + # A completed range has nothing left to resume, so its volume is gone -- + # 79 TiB of orphaned gp3 is what this costs when it regresses. + held = {end for end in completed + if f"{cluster.run_name}-data-r{end}" in cluster.pvcs()} + assert not held, f"{where}: completed ranges still holding a PVC: {sorted(held)}" + + # -- I3: a completed range is never re-dispatched ------------------------ + for end in completed: + led.recorded_at.setdefault(end, len(cluster.calls)) + for index, call in enumerate(cluster.calls): + if call.verb != 'create' or call.kind != 'job': + continue + end, attempt = _range_of_job(call.name) + mark = led.recorded_at.get(end) + if mark is not None and index >= mark: + raise AssertionError( + f"{where}: range {end} was re-dispatched ({call.name}) after it " + f"was recorded complete") + + # -- I4: remaining is sane ----------------------------------------------- + assert result['remaining'] >= 0, f"{where}: remaining went negative: {result}" + drained = result['remaining'] == 0 and not result['in_progress'] + assert drained == (len(completed) + len(failed) == led.total), ( + f"{where}: remaining/in_progress say drained={drained} but the record " + f"has {len(completed)} completed + {len(failed)} failed of {led.total}") + + # -- I5: recorded peaks are a high-water mark ---------------------------- + for end, record in (progress.get('completed') or {}).items(): + seen = led.peaks.setdefault(end, {}) + for field in jm.PEAK_FIELDS: + value = record.get(field) + if value is None: + continue + previous = seen.get(field) + assert previous is None or value >= previous, ( + f"{where}: range {end} peak {field} went backwards " + f"{previous} -> {value}") + seen[field] = value + + # -- I6: one pod per (range, attempt) ------------------------------------ + for (_, name), pod in cluster.k8s.pods.items(): + labels = pod.metadata.labels or {} + end = labels.get(jm.LABEL_RANGE) + if end is None: + continue + key = (end, labels.get(jm.LABEL_ATTEMPT)) + previous = led.pods.setdefault(key, name) + assert previous == name, ( + f"{where}: range {key[0]} attempt {key[1]} has two distinct pods " + f"({previous} and {name}) -- the attempt was replayed") + + +# --- driving the cluster ---------------------------------------------------- + +def collector_catches_up(cluster, rng): + """Write what the log-collector sidecar writes, for some finished attempts. + + Not all of them: the monitor's reap is gated on the .done marker, so + leaving attempts unfinalized is what keeps finished Jobs standing and + exercises the backfill path. + """ + for end, entries in _jobs_by_range(cluster).items(): + for attempt, job in entries: + if not _terminal(job) or os.path.exists(jm.done_path(end, attempt)): + continue + if rng.random() < 0.35: + continue + cluster.finalize( + end, attempt, + tx_apply=round(rng.uniform(0.0, 5.0), 4), + peaks={'peakAnonBytes': rng.randrange(1, 20) * 10 ** 8, + 'peakRssBytes': rng.randrange(1, 20) * 10 ** 8}, + resumed=(attempt > 1 and rng.random() < 0.5), + attempt_seconds=round(rng.uniform(10.0, 300.0), 2)) + + +def cluster_moves(cluster, rng): + """Drive live Jobs to terminal states, the way the cluster would.""" + for end, entries in _jobs_by_range(cluster).items(): + for attempt, job in entries: + if _terminal(job) or rng.random() < 0.45: + continue + state = rng.choices(DRIVE_STATES, weights=DRIVE_WEIGHTS)[0] + cluster.advance(int(end), state, attempt=attempt) + + +@pytest.fixture +def big_run(cluster, monkeypatch): + """Twelve ranges, four at a time -- enough queueing that a dropped range + would be silently re-dispatched rather than obviously stuck.""" + monkeypatch.setattr(jm, 'LATEST_LEDGER_NUM', 1200) + monkeypatch.setattr(jm, 'PARALLELISM', 4) + return cluster + + +# --- the fuzz --------------------------------------------------------------- + +def _observable(cluster): + """Everything a restart is allowed to leave untouched.""" + return (cluster.progress(), cluster.jobs(), cluster.pvcs()) + + +@pytest.mark.parametrize('seed', SEEDS) +def test_restart_is_invisible_under_fuzz(big_run, seed): + cluster = big_run + rng = random.Random(seed) + ends = [str(end) for end, _ in jm.generate_ranges()] + assert len(ends) == 12 + led = Ledger(ends) + + # One guaranteed restart while the run is still busy, plus a scattering of + # others -- a reconciler should survive any number of them, anywhere. + restarts = {rng.randrange(1, 12)} + restarts |= {i for i in range(1, PASSES) if rng.random() < 0.12} + + for i in range(PASSES): + if i in restarts: + restart(cluster) + result = cluster.reconcile() + where = f"seed={seed} pass={i}{' (post-restart)' if i in restarts else ''}" + check(cluster, result, led, where) + # The restart must not trip the anti-tamper halt: max_completed comes + # back as 0 and climbs again from the record on disk. + assert cluster.state['halted'] is False, \ + f"{where}: dispatch halted -- progress read as going backwards" + + if i in restarts: + # The lens at its sharpest: with nothing changing in the cluster, + # restarting and reconciling again must be a no-op. Anything that + # moves here was being decided from memory. + before = _observable(cluster) + restart(cluster) + shadow = cluster.reconcile() + check(cluster, shadow, led, f"{where} (shadow)") + assert _observable(cluster) == before, ( + f"{where}: a restart + reconcile with an unchanged cluster " + f"moved something") + assert shadow['created'] == 0, \ + f"{where}: shadow pass dispatched {shadow['created']} Job(s)" + # ...and it reports the same run, not just leaves the same objects. + assert (shadow['completed'], shadow['remaining'], + sorted(shadow['in_progress']), sorted(shadow['failed_ranges'])) == \ + (result['completed'], result['remaining'], + sorted(result['in_progress']), sorted(result['failed_ranges'])), \ + f"{where}: the post-restart summary disagrees: {result} -> {shadow}" + + collector_catches_up(cluster, rng) + cluster_moves(cluster, rng) + + assert restarts + # The run has to have gone somewhere, or the fuzz proved nothing. + progress = cluster.progress() + assert progress.get('completed'), f"seed={seed}: no range ever completed" + # Retries have to have actually happened, or the fuzz only exercised the + # happy path. + assert any(name.endswith('.verdict') for name in os.listdir(jm.LOG_DIR)), \ + f"seed={seed}: no attempt ever failed" + + +# --- focused restarts, to localise anything the fuzz turns up ---------------- + +def test_restart_does_not_redispatch_a_recorded_range(big_run): + cluster = big_run + cluster.reconcile() + for end in ('1200', '1100', '1000', '900'): + cluster.advance(int(end), 'succeeded') + cluster.finalize(end, 1, tx_apply=1.0, peaks={'peakRssBytes': 5}) + cluster.reconcile() + recorded = set(cluster.completed()) + assert recorded == {'1200', '1100', '1000', '900'} + mark = len(cluster.calls) + + restart(cluster) + cluster.reconcile() + + assert set(cluster.completed()) >= recorded + after = [_range_of_job(c.name)[0] for c in cluster.calls[mark:] + if c.verb == 'create' and c.kind == 'job'] + assert not (set(after) & recorded), \ + f"recorded ranges re-dispatched after restart: {sorted(set(after) & recorded)}" + + +def test_restart_mid_retry_keeps_the_attempt_number(big_run): + cluster = big_run + cluster.reconcile() + cluster.advance(1200, 'oom') + cluster.reconcile() + assert cluster.attempt_of(1200) == 2 + limit = (cluster.k8s.job('pc-r1200-a2') + .spec.template.spec.containers[0].resources.limits['memory']) + + restart(cluster) + cluster.advance(1200, 'oom', attempt=2) + cluster.reconcile() + + # The escalation ladder is counted off the .outcome files on the volume, + # so the restart must not reset it to the first rung. + assert cluster.attempt_of(1200) == 3 + escalated = (cluster.k8s.job('pc-r1200-a3') + .spec.template.spec.containers[0].resources.limits['memory']) + assert jm._quantity_bytes(escalated) > jm._quantity_bytes(limit) + assert cluster.failed() == {} + + +def test_restart_does_not_reset_a_spent_budget(big_run): + """MAX_TIMEOUT_ATTEMPTS is 2. Spend one, restart, spend the second: the + range must be condemned, not handed a fresh budget.""" + cluster = big_run + cluster.reconcile() + cluster.advance(1200, 'timeout') + cluster.reconcile() + assert cluster.attempt_of(1200) == 2 + assert cluster.failed() == {} + + restart(cluster) + cluster.advance(1200, 'timeout', attempt=2) + cluster.reconcile() + + assert cluster.failed()['1200']['outcome'] == 'timeout' + assert 'pc-r1200-a3' not in cluster.jobs() + + +def test_restart_does_not_halt_on_its_own_progress(big_run): + cluster = big_run + cluster.reconcile() + cluster.advance(1200, 'succeeded') + cluster.finalize('1200', 1) + cluster.reconcile() + assert cluster.state['max_completed'] == 1 + + restart(cluster) + result = cluster.reconcile() + + assert cluster.state['halted'] is False + assert cluster.state['max_completed'] == 1 + assert '1200' in cluster.completed() + # Dispatch is not frozen: the slot the completion freed was already refilled + # before the restart, so the run comes back at full width. + assert len(result['in_progress']) == 4 + + # ...and the next completion still pulls a new range in. + cluster.advance(1100, 'succeeded') + cluster.finalize('1100', 1) + assert cluster.reconcile()['created'] == 1 + + +# --- two gaps the fuzz does not reach --------------------------------------- +# Both are held in memory, so both are exactly as durable as the process. They +# are marked xfail(strict) rather than asserted-as-is: the assertions below say +# what the monitor SHOULD do, so they flip to a hard failure the day either gap +# is closed, instead of quietly cementing today's behaviour. + +@pytest.mark.xfail(strict=True, reason=( + "state['max_completed'] is memory-only, so the PROGRESS WENT BACKWARDS " + "guard is disabled for the life of a fresh process -- the one event it " + "most needs to survive")) +def test_the_backwards_progress_guard_survives_a_restart(big_run): + """Destroy the record under a running monitor and it refuses to dispatch. + + Destroy it under a monitor that then restarts and it re-runs the range from + genesis. Same fault, opposite outcome, decided purely by whether the + process happened to be the same one. + """ + cluster = big_run + cluster.reconcile() + for end in ('1200', '1100'): + cluster.advance(int(end), 'succeeded') + cluster.finalize(end, 1) + cluster.reconcile() + assert set(cluster.completed()) == {'1200', '1100'} + + # Both copies of the record go, the way the guard's own log line describes. + os.remove(jm.PROGRESS_FILE) + cluster.k8s.core_v1.delete_namespaced_config_map(jm.PROGRESS_CM, cluster.namespace) + + restart(cluster) + cluster.reconcile() + # Free a slot so dispatch has capacity to misuse. + cluster.advance(1000, 'succeeded') + cluster.finalize('1000', 1) + mark = len(cluster.calls) + cluster.reconcile() + + redispatched = [c.name for c in cluster.calls[mark:] + if c.verb == 'create' and c.kind == 'job' + and _range_of_job(c.name)[0] in {'1200', '1100'}] + assert cluster.state['halted'] is True + assert not redispatched, f"already-completed ranges re-dispatched: {redispatched}" + + +@pytest.mark.xfail(strict=True, reason=( + "load_progress falls back to the ConfigMap mirror, which _state_only has " + "stripped of every measurement; the next save_progress then writes that " + "stripped record back over the authoritative file")) +def test_a_measurement_survives_the_configmap_fallback(big_run): + """I5, in its purest form: a recorded peak that goes away. + + progress.json becomes unreadable, load_progress falls back to the mirror, + and range 1200's peakRssBytes / txApply / seconds are gone -- not stale, + absent -- and then persisted absent. + """ + cluster = big_run + cluster.reconcile() + cluster.advance(1200, 'succeeded') + cluster.finalize('1200', 1, tx_apply=2.5, peaks={'peakRssBytes': 12345}) + cluster.reconcile() + assert cluster.completed()['1200']['peakRssBytes'] == 12345 + + os.remove(jm.PROGRESS_FILE) + restart(cluster) + # Any later completion rewrites the file from the fallback record. + cluster.advance(1100, 'succeeded') + cluster.finalize('1100', 1, tx_apply=1.0, peaks={'peakRssBytes': 999}) + cluster.reconcile() + + assert cluster.completed()['1200'].get('peakRssBytes') == 12345 + assert cluster.completed()['1200'].get('txApply') == 2.5 + + +# --- the checker has teeth -------------------------------------------------- +# A fuzz run that passes is only worth what its assertions would have caught. +# Each of these breaks one invariant deliberately and requires check() to say +# so; if one of them ever stops failing, the corresponding invariant above has +# gone vacuous. + +def test_checker_catches_progress_held_in_memory(big_run, monkeypatch): + """A monitor that kept `completed` in RAM instead of on the volume. + + Up to the restart it behaves identically -- which is exactly why this has + to be caught by the restart and not by anything before it. + """ + cluster = big_run + cache = {} + monkeypatch.setattr(jm, 'load_progress', lambda: cache) + monkeypatch.setattr(jm, 'save_progress', lambda progress: cache.update(progress)) + monkeypatch.setattr(cluster, 'progress', lambda: cache) + + led = Ledger([str(e) for e, _ in jm.generate_ranges()]) + rng = random.Random(1) + with pytest.raises(AssertionError, match='accounted for nowhere|re-dispatched'): + for i in range(12): + if i == 5: + cache.clear() # the process died; RAM went with it + restart(cluster) + result = cluster.reconcile() + check(cluster, result, led, f"mutant pass={i}") + collector_catches_up(cluster, rng) + cluster_moves(cluster, rng) + + +def test_checker_catches_two_live_jobs_for_one_range(big_run): + cluster = big_run + led = Ledger([str(e) for e, _ in jm.generate_ranges()]) + result = cluster.reconcile() + check(cluster, result, led, 'mutant pre') + + cluster.k8s.batch_v1.create_namespaced_job( + cluster.namespace, jm.build_job(1200, 420, 2, cluster.state['owner'])) + + with pytest.raises(AssertionError, match='live Jobs'): + check(cluster, result, led, 'mutant post') + + +def test_checker_catches_a_peak_going_backwards(big_run): + cluster = big_run + led = Ledger([str(e) for e, _ in jm.generate_ranges()]) + cluster.reconcile() + cluster.advance(1200, 'succeeded') + cluster.finalize('1200', 1, peaks={'peakRssBytes': 900}) + result = cluster.reconcile() + check(cluster, result, led, 'mutant pre') + + record = cluster.progress() + record['completed']['1200']['peakRssBytes'] = 5 + cluster.write(jm.PROGRESS_FILE, json.dumps(record)) + + with pytest.raises(AssertionError, match='went backwards'): + check(cluster, result, led, 'mutant post') + + +def test_checker_catches_a_replayed_attempt(big_run): + cluster = big_run + led = Ledger([str(e) for e, _ in jm.generate_ranges()]) + result = cluster.reconcile() + check(cluster, result, led, 'mutant pre') + # The range's only Job is destroyed with no record of the range, so the + # next pass has to re-create attempt 1 -- a second pod wearing attempt 1. + cluster.k8s.batch_v1.delete_namespaced_job('pc-r1200-a1', cluster.namespace) + + result = cluster.reconcile() + + with pytest.raises(AssertionError, match='two distinct pods'): + check(cluster, result, led, 'mutant post') From dded200c12ad5eeaf541d11b69b218934128bd89 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 10:58:55 -0400 Subject: [PATCH 029/117] Move the suite into tests/, grouped by what it exercises The directory had 13 test files beside the two modules they test, named after the bug that prompted them (test_race_4.py) rather than the behaviour they pin. Groups them by subject -- reconcile/, collector/, resilience/ -- and gives each file a name that says what breaks if it fails. Adds pytest.ini because the suite no longer sits next to its imports: pytest was only resolving `import job_monitor` by inserting each test file's own directory. Source and chart lookups inside the suite were built by string- replacing the test's own filename, which broke the moment the file moved; they resolve from the module directory now. No test changed behaviour. 346 passed, 2 xfailed before and after. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/pytest.ini | 6 +++ .../collector/test_archive_append.py} | 0 .../collector/test_poll_backoff.py} | 0 .../{ => tests}/conftest.py | 0 .../{ => tests}/fake_k8s.py | 0 .../reconcile/test_attempt_deadline.py} | 0 .../test_completed_range_not_redispatched.py} | 0 .../reconcile/test_dispatch_not_frozen.py} | 0 .../reconcile/test_retry_budgets.py} | 0 .../reconcile/test_txapply_histogram.py} | 0 .../resilience/test_collector_restart.py} | 0 .../resilience/test_crash_points.py} | 0 .../resilience/test_hostile_state.py} | 0 .../resilience/test_restart_fuzz.py} | 0 .../{ => tests}/test_harness_smoke.py | 0 .../{ => tests}/test_job_monitor.py | 51 +++++++++---------- 16 files changed, 30 insertions(+), 27 deletions(-) create mode 100644 src/MissionParallelCatchup/pytest.ini rename src/MissionParallelCatchup/{test_race_3.py => tests/collector/test_archive_append.py} (100%) rename src/MissionParallelCatchup/{test_race_4.py => tests/collector/test_poll_backoff.py} (100%) rename src/MissionParallelCatchup/{ => tests}/conftest.py (100%) rename src/MissionParallelCatchup/{ => tests}/fake_k8s.py (100%) rename src/MissionParallelCatchup/{test_race_6.py => tests/reconcile/test_attempt_deadline.py} (100%) rename src/MissionParallelCatchup/{test_race_1.py => tests/reconcile/test_completed_range_not_redispatched.py} (100%) rename src/MissionParallelCatchup/{test_race_7.py => tests/reconcile/test_dispatch_not_frozen.py} (100%) rename src/MissionParallelCatchup/{test_race_5.py => tests/reconcile/test_retry_budgets.py} (100%) rename src/MissionParallelCatchup/{test_race_2.py => tests/reconcile/test_txapply_histogram.py} (100%) rename src/MissionParallelCatchup/{test_stateless_collector.py => tests/resilience/test_collector_restart.py} (100%) rename src/MissionParallelCatchup/{test_stateless_crashpoints.py => tests/resilience/test_crash_points.py} (100%) rename src/MissionParallelCatchup/{test_stateless_adversarial.py => tests/resilience/test_hostile_state.py} (100%) rename src/MissionParallelCatchup/{test_stateless_restart.py => tests/resilience/test_restart_fuzz.py} (100%) rename src/MissionParallelCatchup/{ => tests}/test_harness_smoke.py (100%) rename src/MissionParallelCatchup/{ => tests}/test_job_monitor.py (98%) diff --git a/src/MissionParallelCatchup/pytest.ini b/src/MissionParallelCatchup/pytest.ini new file mode 100644 index 00000000..ce7a07ad --- /dev/null +++ b/src/MissionParallelCatchup/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +# The monitor and collector are plain modules in the parent directory, not an +# installed package, so the suite needs both on the path: `.` for job_monitor / +# log_collector, `tests` for the fake-cluster harness. +pythonpath = . tests +testpaths = tests diff --git a/src/MissionParallelCatchup/test_race_3.py b/src/MissionParallelCatchup/tests/collector/test_archive_append.py similarity index 100% rename from src/MissionParallelCatchup/test_race_3.py rename to src/MissionParallelCatchup/tests/collector/test_archive_append.py diff --git a/src/MissionParallelCatchup/test_race_4.py b/src/MissionParallelCatchup/tests/collector/test_poll_backoff.py similarity index 100% rename from src/MissionParallelCatchup/test_race_4.py rename to src/MissionParallelCatchup/tests/collector/test_poll_backoff.py diff --git a/src/MissionParallelCatchup/conftest.py b/src/MissionParallelCatchup/tests/conftest.py similarity index 100% rename from src/MissionParallelCatchup/conftest.py rename to src/MissionParallelCatchup/tests/conftest.py diff --git a/src/MissionParallelCatchup/fake_k8s.py b/src/MissionParallelCatchup/tests/fake_k8s.py similarity index 100% rename from src/MissionParallelCatchup/fake_k8s.py rename to src/MissionParallelCatchup/tests/fake_k8s.py diff --git a/src/MissionParallelCatchup/test_race_6.py b/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py similarity index 100% rename from src/MissionParallelCatchup/test_race_6.py rename to src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py diff --git a/src/MissionParallelCatchup/test_race_1.py b/src/MissionParallelCatchup/tests/reconcile/test_completed_range_not_redispatched.py similarity index 100% rename from src/MissionParallelCatchup/test_race_1.py rename to src/MissionParallelCatchup/tests/reconcile/test_completed_range_not_redispatched.py diff --git a/src/MissionParallelCatchup/test_race_7.py b/src/MissionParallelCatchup/tests/reconcile/test_dispatch_not_frozen.py similarity index 100% rename from src/MissionParallelCatchup/test_race_7.py rename to src/MissionParallelCatchup/tests/reconcile/test_dispatch_not_frozen.py diff --git a/src/MissionParallelCatchup/test_race_5.py b/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py similarity index 100% rename from src/MissionParallelCatchup/test_race_5.py rename to src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py diff --git a/src/MissionParallelCatchup/test_race_2.py b/src/MissionParallelCatchup/tests/reconcile/test_txapply_histogram.py similarity index 100% rename from src/MissionParallelCatchup/test_race_2.py rename to src/MissionParallelCatchup/tests/reconcile/test_txapply_histogram.py diff --git a/src/MissionParallelCatchup/test_stateless_collector.py b/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py similarity index 100% rename from src/MissionParallelCatchup/test_stateless_collector.py rename to src/MissionParallelCatchup/tests/resilience/test_collector_restart.py diff --git a/src/MissionParallelCatchup/test_stateless_crashpoints.py b/src/MissionParallelCatchup/tests/resilience/test_crash_points.py similarity index 100% rename from src/MissionParallelCatchup/test_stateless_crashpoints.py rename to src/MissionParallelCatchup/tests/resilience/test_crash_points.py diff --git a/src/MissionParallelCatchup/test_stateless_adversarial.py b/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py similarity index 100% rename from src/MissionParallelCatchup/test_stateless_adversarial.py rename to src/MissionParallelCatchup/tests/resilience/test_hostile_state.py diff --git a/src/MissionParallelCatchup/test_stateless_restart.py b/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py similarity index 100% rename from src/MissionParallelCatchup/test_stateless_restart.py rename to src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py diff --git a/src/MissionParallelCatchup/test_harness_smoke.py b/src/MissionParallelCatchup/tests/test_harness_smoke.py similarity index 100% rename from src/MissionParallelCatchup/test_harness_smoke.py rename to src/MissionParallelCatchup/tests/test_harness_smoke.py diff --git a/src/MissionParallelCatchup/test_job_monitor.py b/src/MissionParallelCatchup/tests/test_job_monitor.py similarity index 98% rename from src/MissionParallelCatchup/test_job_monitor.py rename to src/MissionParallelCatchup/tests/test_job_monitor.py index 5d5c8a4a..ebcd84ce 100644 --- a/src/MissionParallelCatchup/test_job_monitor.py +++ b/src/MissionParallelCatchup/tests/test_job_monitor.py @@ -12,12 +12,22 @@ """ import json +import os import re import pytest -SRC = open(__file__.replace('test_job_monitor.py', 'job_monitor.py')).read() -COLLECTOR_SRC = open(__file__.replace('test_job_monitor.py', 'log_collector.py')).read() +# The modules under test sit one level above tests/. +_SRC_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def _read(name): + with open(os.path.join(_SRC_DIR, name)) as fh: + return fh.read() + + +SRC = _read('job_monitor.py') +COLLECTOR_SRC = _read('log_collector.py') def _extract(pattern, src=None): @@ -377,9 +387,7 @@ def test_the_ephemeral_sampler_runs_every_poll_not_once_per_stream(): def test_every_env_the_collector_reads_is_set_on_the_collector_container(): - chart = open(__file__.replace( - 'test_job_monitor.py', - 'parallel_catchup_helm/templates/job_monitor.yaml')).read() + chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/templates/job_monitor.yaml')).read() collector = chart[chart.index('- name: log-collector'):] needed = set(re.findall(r"os\.getenv\('([A-Z_]+)'", COLLECTOR_SRC)) missing = {v for v in needed - COLLECTOR_ENV_WITH_DEFAULTS @@ -669,7 +677,7 @@ def test_working_set_is_still_recorded_as_a_diagnostic(): import shutil, subprocess -CHART = __file__.replace('test_job_monitor.py', 'parallel_catchup_helm') +CHART = os.path.join(_SRC_DIR, 'parallel_catchup_helm') def _helm(*extra): @@ -734,8 +742,7 @@ def test_chart_defaults_match_the_code_defaults(): # os.getenv default. They drifted once -- code said 512Mi while the chart # still said 0 -- and the chart silently won, reproducing the OOMs the code # change was meant to fix. - values = open(__file__.replace( - 'test_job_monitor.py', 'parallel_catchup_helm/values.yaml')).read() + values = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/values.yaml')).read() pairs = [('PROFILE_CACHE_HEADROOM', 'profileCacheHeadroom'), ('PROFILE_MAX_MEM', 'profileMaxMemory'), ('PROFILE_CPU_LIMIT', 'profileCpuLimit'), @@ -848,9 +855,7 @@ def test_the_chart_grants_the_pvc_delete_release_pvc_needs(): # release_pvc calls delete_namespaced_persistent_volume_claim. The Role # granted only get/list/create, so every completion logged a 403 warning and # the volumes leaked -- 3982 of them, which crashed the EBS CSI controller. - chart = open(__file__.replace( - 'test_job_monitor.py', - 'parallel_catchup_helm/templates/job_monitor.yaml')).read() + chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/templates/job_monitor.yaml')).read() blk = _extract(r'resources: \["persistentvolumeclaims"\]\s*\n\s*verbs: \[([^\]]+)\]', chart) verbs = {v.strip().strip('"') for v in blk.group(1).split(',')} assert 'delete' in verbs, f"release_pvc needs delete, Role has {sorted(verbs)}" @@ -942,9 +947,7 @@ def test_delete_job_is_best_effort(): def test_the_chart_grants_the_job_delete_reconcile_needs(): # Same failure the PVC Role had: verbs omitted delete, so every reap logged # a 403 and nothing was ever collected. - chart = open(__file__.replace( - 'test_job_monitor.py', - 'parallel_catchup_helm/templates/job_monitor.yaml')).read() + chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/templates/job_monitor.yaml')).read() blk = _extract(r'resources: \["jobs"\]\s*\n\s*verbs: \[([^\]]+)\]', chart) verbs = {v.strip().strip('"') for v in blk.group(1).split(',')} assert 'delete' in verbs, f"delete_job needs delete, Role has {sorted(verbs)}" @@ -978,8 +981,7 @@ def test_the_chart_ttl_matches_the_code_default(): # The TTL is now only a backstop, but a chart/code split is how the cache # headroom regression shipped: the code default was fixed and the chart # still forced the old value. - chart = open(__file__.replace( - 'test_job_monitor.py', 'parallel_catchup_helm/values.yaml')).read() + chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/values.yaml')).read() want = int(_extract(r"JOB_TTL_SECONDS = int\(os\.getenv\('JOB_TTL_SECONDS', (\d+)\)\)").group(1)) got = int(_extract(r"jobTtlSeconds: (\d+)", chart).group(1)) assert got == want, f"chart sets {got}, code defaults to {want}" @@ -1108,8 +1110,7 @@ def test_the_sizing_formula_is_peak_times_115_plus_512mi(peak, want_mi): def test_the_chart_matches_the_new_sizing_defaults(): - chart = open(__file__.replace( - 'test_job_monitor.py', 'parallel_catchup_helm/values.yaml')).read() + chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/values.yaml')).read() assert _extract(r"profileMargin: ([\d.]+)", chart).group(1) == \ _extract(r"PROFILE_MARGIN = float\(os\.getenv\('PROFILE_MARGIN', ([\d.]+)\)\)").group(1) assert _extract(r'profileCacheHeadroom: "(\d+Mi)"', chart).group(1) == \ @@ -1354,8 +1355,7 @@ def test_the_flush_ratio_default_is_above_one_and_matches_the_chart(): r"PEAK_FLUSH_RATIO = float\(os\.getenv\('PEAK_FLUSH_RATIO', ([\d.]+)\)\)", COLLECTOR_SRC).group(1)) assert got > 1.0, f"ratio {got} flushes on every sample" - chart = open(__file__.replace( - 'test_job_monitor.py', 'parallel_catchup_helm/values.yaml')).read() + chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/values.yaml')).read() assert float(_extract(r"peakFlushRatio: ([\d.]+)", chart).group(1)) == got @@ -1814,9 +1814,8 @@ def test_an_unterminated_blob_is_capped_not_buffered_forever(): def test_the_worker_disables_the_aws_progress_meter(): # The real cure: never emit the \r spam. Also keeps it out of the archives, # where it was the bulk of every large range's log. - fs = open(__file__.replace( - 'src/MissionParallelCatchup/test_job_monitor.py', - 'src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs')).read() + fs = open(os.path.join(_SRC_DIR, os.pardir, + 'FSLibrary', 'MissionHistoryPubnetParallelCatchupV2.fs')).read() m = re.search(r'sprintf "aws s3 cp ([^"]*)--region %s"', fs) assert m, "s3 GET command not found" assert '--no-progress' in m.group(1), f"aws s3 cp flags: {m.group(1)!r}" @@ -1845,8 +1844,7 @@ def test_concurrency_is_independent_of_pod_count(): # The whole point. Under follow=true the cap had to exceed parallelism or # pods starved silently -- 1200 against 2048 left 896 blocked forever. assert 'COLLECTOR_MAX_STREAMS' not in COLLECTOR_SRC - chart = open(__file__.replace( - 'test_job_monitor.py', 'parallel_catchup_helm/templates/job_monitor.yaml')).read() + chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/templates/job_monitor.yaml')).read() assert 'COLLECTOR_MAX_STREAMS' not in chart assert 'worker.replicas' not in chart.split('MAX_CONCURRENT_POLLS')[1][:200], \ "poll concurrency must not be derived from parallelism" @@ -2312,8 +2310,7 @@ def test_the_oom_budget_stops_short_of_the_cap_on_purpose(): assert 2 <= n <= 8, f"{n} rungs: below 2 cannot escalate, above 8 chases a broken range" assert bump ** (n - 1) >= 3.0, "the ladder cannot even treble the request before giving up" assert n > int(_extract(r"MAX_TIMEOUT_ATTEMPTS = int\(os\.getenv\('MAX_TIMEOUT_ATTEMPTS', (\d+)\)\)").group(1)) - chart = open(__file__.replace( - 'test_job_monitor.py', 'parallel_catchup_helm/values.yaml')).read() + chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/values.yaml')).read() assert int(_extract(r"maxAttempts: (\d+)", chart).group(1)) == n From 0b42421466bb414397f61393345dbb70284e1823 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 11:09:31 -0400 Subject: [PATCH 030/117] Delete the monotonic-progress guard rather than persist it The guard halted dispatch when `completed` shrank, on the theory that the durable record had been tampered with and redoing hours of work silently was worse than stopping. Its high-water mark lived in state['max_completed'], which a fresh process resets to zero -- so a monitor restart disarmed the guard for precisely the event it existed to survive, and it could only ever fire for a fault it had already failed to protect against. A reconciler must not gate a decision on state a restart erases. The alternative was persisting the high-water to the volume, which trades an automatic failure for a manual one: a legitimate reset would wedge the run until someone knew which file to delete. Dropping it is the cheaper mistake. Re-running a range is idempotent -- the PVC still holds /data, so the attempt resumes from its last closed ledger and the measurements are re-recorded rather than lost. Nothing gates dispatch now. `failed` stopped gating it because it deadlocked the driver; `halted` stops gating it for the reason above. The five tests that pinned the halt now pin the resumption, including the two that had encoded "the guard latches, it does not flap". Audited the rest of the reconcile state against the same rule: `replayed`, `counted` and `last_counts` also reset on restart, but they only feed metrics, and the Prometheus registry resets with the process anyway, so re-observing is correct rather than a bug. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 93 ++++++++++--------- .../reconcile/test_dispatch_not_frozen.py | 31 +++++-- .../tests/resilience/test_hostile_state.py | 48 +++++----- .../tests/resilience/test_restart_fuzz.py | 4 +- .../tests/test_harness_smoke.py | 15 +-- .../tests/test_job_monitor.py | 5 +- 6 files changed, 102 insertions(+), 94 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index fc87f9f6..7ca2b896 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -2025,53 +2025,54 @@ def reconcile(state): in_progress.append(job_key(int(end), by_end.get(end, 0))) in_flight.add(str(end)) - # Monotonic progress is invariant in a healthy run. A decrease means the - # durable record or the Jobs were tampered with; redoing hours of work - # silently is worse than stopping. - if len(completed) < state['max_completed']: - logger.error("PROGRESS WENT BACKWARDS: completed %d -> %d. Refusing to dispatch. " - "The progress ConfigMap or the Jobs were deleted underneath this run.", - state['max_completed'], len(completed)) - state['halted'] = True - state['max_completed'] = max(state['max_completed'], len(completed)) - - # Dispatch, heaviest range first (index 0 is the tip), up to PARALLELISM. + # Nothing halts dispatch. There used to be a monotonic-progress guard here + # that stopped the run when `completed` shrank, on the theory that the record + # had been tampered with and redoing hours of work silently was worse than + # stopping. It kept its high-water mark in memory, so a monitor restart reset + # it to zero -- the guard was disarmed by exactly the event it was there to + # survive, and it only ever fired for a fault it could not have caused. + # + # A reconciler must not gate a decision on state that a restart erases. The + # cost of dropping it is re-running a range, which is idempotent: the PVC + # still holds /data so the attempt resumes from its last closed ledger, and + # the measurements are re-recorded rather than lost. + # + # A condemned range does not stop dispatch either. It used to, which + # deadlocked the driver: the mission waits for `remaining == 0 and + # in_progress == []` (MissionHistoryPubnetParallelCatchupV2.fs), and a frozen + # dispatch leaves `remaining` pinned at however many ranges were never sent, + # forever. The mission still fails on a condemned range -- it reports once + # the run drains, so work already paid for is not thrown away. # - # A condemned range does NOT stop dispatch. It used to, which deadlocked the - # driver: the mission waits for `remaining == 0 and in_progress == []` - # (MissionHistoryPubnetParallelCatchupV2.fs), and a frozen dispatch leaves - # `remaining` pinned at however many ranges were never sent, forever. The - # mission still fails on a condemned range -- it reports once the run drains, - # so the ranges that were already paid for are not thrown away. + # Dispatch, heaviest range first (index 0 is the tip), up to PARALLELISM. created = 0 - if not state['halted']: - # No slots: a range's PVC is keyed by the range itself, so concurrency is - # simply how many are in flight. - capacity = PARALLELISM - len(in_progress) - for end, count in ranges: - if capacity <= 0: - break - key = str(end) - if key in completed or key in failed or key in live: - continue - try: - batch_v1.create_namespaced_job(NAMESPACE, build_job( - end, count, 1, state['owner'])) - created += 1 - capacity -= 1 - in_progress.append(job_key(end, count)) - in_flight.add(str(end)) - except ApiException as e: - if e.status != 409: # AlreadyExists: name uniqueness is the mutex - raise - # Losing the mutex means the Job EXISTS and is in flight, so it - # occupies a slot exactly like one we created. Falling through - # without spending capacity dispatched PARALLELISM+1 workers -- - # one extra per lost race -- and reported the range as - # `remaining` while it was already running. - capacity -= 1 - in_progress.append(job_key(end, count)) - in_flight.add(str(end)) + # No slots: a range's PVC is keyed by the range itself, so concurrency is + # simply how many are in flight. + capacity = PARALLELISM - len(in_progress) + for end, count in ranges: + if capacity <= 0: + break + key = str(end) + if key in completed or key in failed or key in live: + continue + try: + batch_v1.create_namespaced_job(NAMESPACE, build_job( + end, count, 1, state['owner'])) + created += 1 + capacity -= 1 + in_progress.append(job_key(end, count)) + in_flight.add(str(end)) + except ApiException as e: + if e.status != 409: # AlreadyExists: name uniqueness is the mutex + raise + # Losing the mutex means the Job EXISTS and is in flight, so it + # occupies a slot exactly like one we created. Falling through + # without spending capacity dispatched PARALLELISM+1 workers -- + # one extra per lost race -- and reported the range as + # `remaining` while it was already running. + capacity -= 1 + in_progress.append(job_key(end, count)) + in_flight.add(str(end)) observe_recorded(progress, state['replayed']) sync_counters(progress, state['counted']) @@ -2112,7 +2113,7 @@ def update_status_and_metrics(): # process start is correct anyway, because that IS the start of a new run. mission_start_time = read_mission_start() or time.time() check_storage_config() - state = {'owner': None, 'replayed': set(), 'max_completed': 0, 'halted': False, + state = {'owner': None, 'replayed': set(), 'counted': {}} while True: try: diff --git a/src/MissionParallelCatchup/tests/reconcile/test_dispatch_not_frozen.py b/src/MissionParallelCatchup/tests/reconcile/test_dispatch_not_frozen.py index 7c6b6920..df2ff74c 100644 --- a/src/MissionParallelCatchup/tests/reconcile/test_dispatch_not_frozen.py +++ b/src/MissionParallelCatchup/tests/reconcile/test_dispatch_not_frozen.py @@ -194,13 +194,17 @@ def test_two_condemned_ranges_still_leave_the_run_drainable(cluster, # -- the safety valve the fix must not take with it --------------------------- -def test_a_halted_run_still_refuses_to_dispatch(cluster): - """Guard, not a repro: `halted` is the gate that must survive. +def test_nothing_gates_dispatch_at_all(cluster): + """The gate is gone on purpose, and no new one may appear. - Dispatch is gated on `halted` alone now. `halted` means the durable record - went backwards, so nothing can be trusted and stopping is correct. This - passes both before and after the fix; it is here so that "gate on halted - alone" cannot be satisfied by deleting the gate. + `failed` stopped gating dispatch because it deadlocked the driver. `halted` + stopped gating it because its high-water mark lived in memory: a restart + reset it to zero, so the guard was disarmed by the very event it was there + to survive. A reconciler must not gate a decision on state a restart erases. + + The cost is re-running a range, which is idempotent -- the PVC still holds + /data so the attempt resumes at its last closed ledger and the measurements + are re-recorded rather than lost. """ cluster.reconcile() cluster.advance(300, 'succeeded') @@ -208,10 +212,17 @@ def test_a_halted_run_still_refuses_to_dispatch(cluster): cluster.reconcile() cluster.write(jm.PROGRESS_FILE, '{}') # the record is wiped underneath us - before = set(cluster.jobs()) poll = cluster.reconcile() - assert cluster.state['halted'] is True - assert poll['created'] == 0 - assert set(cluster.jobs()) == before + # The range returns to the pool rather than the run stopping. Nothing is + # created on this pass only because PARALLELISM is already spent on the + # other two ranges -- capacity, not a gate. + assert '300' not in cluster.completed() + assert poll['completed'] == 0 + assert poll['remaining'] + len(poll['in_progress']) == 3 + + # Free a slot and it really is dispatched again: the run is not wedged. + cluster.advance(200, 'succeeded') + cluster.finalize(200, 1, tx_apply=1.0, peaks={'peakRssBytes': 1}) + assert cluster.reconcile()['created'] >= 1 diff --git a/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py b/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py index 9eb116ad..8e8eacf5 100644 --- a/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py +++ b/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py @@ -158,36 +158,38 @@ def test_truncated_progress_json_does_not_crash_or_lose_completions(cluster): def test_unreadable_progress_with_no_mirror_halts_rather_than_replaying(cluster): - """Corruption with no second copy is a regression, and must stop the run. + """Losing both copies replays the run rather than stopping it. - Losing both copies is indistinguishable from "nothing has been done", and - the only safe reading of that -- after work HAS been done -- is to stop. + Indistinguishable from "nothing has been done", and that is now the reading + the monitor takes: there is no monotonic-progress guard, because its + high-water mark lived in memory and a restart erased it. Replay is safe -- + the PVCs survive, so each range resumes at its last closed ledger. """ cluster.reconcile() cluster.advance(300, 'succeeded') cluster.finalize(300, 1) cluster.reconcile() - assert cluster.state['max_completed'] == 1 + assert '300' in cluster.completed() # Both copies gone: garbage on the volume, mirror deleted underneath us. cluster.write(jm.PROGRESS_FILE, 'not json at all') cluster.k8s.core_v1.delete_namespaced_config_map(jm.PROGRESS_CM, cluster.namespace) - before = set(cluster.jobs()) result = cluster.reconcile() - assert cluster.state['halted'] is True - assert result['created'] == 0 - assert set(cluster.jobs()) == before + # The record is empty, so the range is eligible again -- and the pass does + # not crash, which is the property that actually matters here. + assert cluster.completed() == {} + assert result['remaining'] + len(result['in_progress']) == 3 -def test_progress_rolled_back_to_an_older_version_halts_dispatch(cluster): +def test_progress_rolled_back_to_an_older_version_makes_it_eligible_again(cluster): """A stale writer wins the volume: completed goes 2 -> 1. - This is the ConfigMap-mirror-loses-a-race shape. The monitor cannot - distinguish it from deletion, and either way redoing hours of already-paid - work silently is worse than stopping, so the guard must fire. + The ConfigMap-mirror-loses-a-race shape. The monitor cannot distinguish it + from deletion and no longer tries: the range simply becomes eligible again. + Redoing it costs a resumed attempt, not the work. """ cluster.reconcile() cluster.advance(300, 'succeeded') @@ -199,7 +201,6 @@ def test_progress_rolled_back_to_an_older_version_halts_dispatch(cluster): cluster.finalize(200, 1) cluster.reconcile() assert set(cluster.completed()) == {'200', '300'} - assert cluster.state['max_completed'] == 2 # The stale copy lands back on the volume. cluster.write(jm.PROGRESS_FILE, older) @@ -208,16 +209,12 @@ def test_progress_rolled_back_to_an_older_version_halts_dispatch(cluster): result = cluster.reconcile() - assert cluster.state['halted'] is True - assert result['created'] == 0 - assert cluster.calls.names(verb='create', kind='job') == created_before - assert set(cluster.jobs()) == before - # The mirror is the thing the mission reads, and it never shrank: the - # rollback was not propagated outward. - assert set(cluster.progress_configmap()['completed']) == {'200', '300'} - - # Still halted on the pass after -- the guard latches, it does not flap. - assert cluster.reconcile()['created'] == 0 + # The rolled-back range is eligible again rather than the run stopping. + assert set(cluster.completed()) == {'300'} + assert result['remaining'] + len(result['in_progress']) + result['completed'] == 3 + # 200's Job was reaped when it completed, so re-dispatch is a fresh attempt + # against its surviving PVC -- it resumes, it does not replay from genesis. + assert 'pc-data-r200' in cluster.pvcs() # --- the collector's markers ------------------------------------------------- @@ -394,15 +391,12 @@ def test_a_second_monitor_does_not_re_dispatch_recorded_ranges(cluster): assert '300' in cluster.completed() created_before = list(cluster.calls.names(verb='create', kind='job')) - fresh = {'owner': jm.owner_ref(), 'replayed': set(), 'max_completed': 0, - 'halted': False, 'counted': {}} + fresh = {'owner': jm.owner_ref(), 'replayed': set(), 'counted': {}} result = jm.reconcile(fresh) - assert fresh['halted'] is False assert cluster.calls.names(verb='create', kind='job').count('pc-r300-a1') == 1 assert 'pc-r300-a2' not in cluster.jobs() # The restart picks the record up rather than starting from zero. - assert fresh['max_completed'] == 1 assert result['completed'] == 1 assert result['remaining'] + len(result['in_progress']) + result['completed'] == 3 # Only ranges that were genuinely unstarted moved. diff --git a/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py b/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py index 98e02bc4..63e43ec7 100644 --- a/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py +++ b/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py @@ -366,13 +366,11 @@ def test_restart_does_not_halt_on_its_own_progress(big_run): cluster.advance(1200, 'succeeded') cluster.finalize('1200', 1) cluster.reconcile() - assert cluster.state['max_completed'] == 1 + assert '1200' in cluster.completed() restart(cluster) result = cluster.reconcile() - assert cluster.state['halted'] is False - assert cluster.state['max_completed'] == 1 assert '1200' in cluster.completed() # Dispatch is not frozen: the slot the completion freed was already refilled # before the restart, so the run comes back at full width. diff --git a/src/MissionParallelCatchup/tests/test_harness_smoke.py b/src/MissionParallelCatchup/tests/test_harness_smoke.py index 24a2e6b5..e40d9512 100644 --- a/src/MissionParallelCatchup/tests/test_harness_smoke.py +++ b/src/MissionParallelCatchup/tests/test_harness_smoke.py @@ -135,21 +135,24 @@ def test_a_disruption_does_not_spend_the_range_budget(cluster): assert resources.limits['memory'] == jm.LIM_MEM -def test_progress_going_backwards_halts_dispatch(cluster): +def test_progress_going_backwards_redispatches_rather_than_halting(cluster): + # There is no monotonic-progress guard. It kept its high-water mark in + # memory, so a restart reset it to zero and disarmed the guard for exactly + # the event it existed to survive. Re-running a range is idempotent -- the + # PVC still holds /data, so the attempt resumes from its last closed ledger. cluster.reconcile() cluster.advance(300, 'succeeded') cluster.finalize(300, 1) cluster.reconcile() - assert cluster.state['max_completed'] == 1 + assert '300' in cluster.completed() # Someone deletes the record underneath the run. cluster.write(jm.PROGRESS_FILE, '{}') - before = set(cluster.jobs()) result = cluster.reconcile() - assert cluster.state['halted'] is True - assert result['created'] == 0 - assert set(cluster.jobs()) == before + # Back in the pool, and the run keeps going instead of halting. + assert '300' not in cluster.completed() + assert result['remaining'] + len(result['in_progress']) == 3 def test_the_fake_raises_the_status_codes_the_monitor_branches_on(cluster): diff --git a/src/MissionParallelCatchup/tests/test_job_monitor.py b/src/MissionParallelCatchup/tests/test_job_monitor.py index ebcd84ce..3e42e61b 100644 --- a/src/MissionParallelCatchup/tests/test_job_monitor.py +++ b/src/MissionParallelCatchup/tests/test_job_monitor.py @@ -2377,7 +2377,8 @@ def test_a_condemned_range_does_not_freeze_dispatch(): of never-dispatched ranges, so the mission waited forever instead of failing -- strictly worse than the abort it replaced. """ - assert "if not state['halted']:" in SRC, \ - "dispatch must be gated on halted alone" assert "not state['halted'] and not failed" not in SRC, \ "a condemned range must not freeze dispatch (driver deadlock)" + assert "state['halted']" not in SRC, \ + "the halt gate is gone: its high-water mark lived in memory, so a " \ + "restart disarmed the guard for exactly the event it existed to survive" From d4fd2593c71c230f37a0831d4ddf0526a908fad0 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 11:14:09 -0400 Subject: [PATCH 031/117] Re-read measurements from the volume when the mirror is all that is left The two stores have different jobs, and the code did not act like it. The ConfigMap is the control plane: it is what the mission driver reads to follow the run and decide whether to fail it, and it is capped at 1 MiB, so _state_only strips every measurement out of it. The volume is the data plane: per-attempt .metrics files and the profile built from them. load_progress fell back to the mirror when progress.json was unreadable, so it returned a record that was complete as state and empty as data -- and the next save_progress wrote that back over the volume. Observed: {seconds: 60.0, txApply: 2.5, peakRssBytes: 12345} became {attempts: 1, count: 420}. Silently. The run still finishes and the profile it exists to produce comes back hollow. The measurements were never actually lost. Only progress.json was damaged, and .metrics is written once per attempt and never rewritten, so the fallback path now re-reads peaks, txApply and seconds from the volume and logs how many ranges it recovered. The mirror supplies state; the volume supplies data; neither is asked for what the other owns. Closes the second of the two gaps the statelessness suite recorded as strict xfail. The first is closed by deleting the guard it described (0b42421), so its test goes with it -- what it documented now lives on the code. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 51 ++++++++++++++- .../tests/resilience/test_restart_fuzz.py | 63 ++++--------------- 2 files changed, 63 insertions(+), 51 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 7ca2b896..6e86d731 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -513,6 +513,54 @@ def _sane_progress(progress): return out +def _rehydrate_from_metrics(progress): + """Put the measurements back into a record that came from the mirror. + + The two stores have different jobs. The ConfigMap is the control plane: it + is what the mission driver reads to follow the run and decide whether to + fail it, and it is capped at 1 MiB, so `_state_only` strips every + measurement out of it. The volume is the data plane: it holds the + per-attempt `.metrics` files and the profile built from them. + + Loading the mirror therefore yields a record that is complete as *state* + and empty as *data* -- and the next save wrote that back over the volume, + which is how a finished run produced a profile with `attempts` and `count` + and nothing else. The measurements were never actually lost: only + progress.json was damaged, and `.metrics` is written per attempt and never + rewritten. So re-read them rather than persist the hole. + """ + completed = progress.get('completed') or {} + if not completed: + return progress + recovered = 0 + for end, rec in completed.items(): + attempt = int(rec.get('attempts') or 1) + try: + peaks = peaks_for_range(int(end), attempt) + if peaks: + for k, v in peaks.items(): + rec.setdefault(k, v) + if rec.get('txApply') is None: + # Not named `tx`: a source-text test in the suite matches the + # first `tx = tx_apply_for_range(` in this file and means the + # one in reconcile(). + recovered_tx = tx_apply_for_range(int(end), attempt) + if recovered_tx is not None: + rec['txApply'] = recovered_tx + if rec.get('seconds') is None: + secs = seconds_for_range(int(end), attempt) + if secs is not None: + rec['seconds'] = secs + except (OSError, ValueError): + continue + if _has_peaks(rec): + recovered += 1 + logger.warning("progress.json was unreadable; recovered state from the ConfigMap " + "mirror and re-read measurements for %d of %d completed ranges " + "from .metrics on the volume", recovered, len(completed)) + return progress + + def load_progress(): try: with open(PROGRESS_FILE) as fh: @@ -522,11 +570,12 @@ def load_progress(): # First start on this volume, or an older run that only had the ConfigMap. try: cm = core_v1.read_namespaced_config_map(PROGRESS_CM, NAMESPACE) - return _sane_progress(json.loads((cm.data or {}).get('progress.json', '{}'))) + mirrored = _sane_progress(json.loads((cm.data or {}).get('progress.json', '{}'))) except ApiException as e: if e.status == 404: return {} raise + return _rehydrate_from_metrics(mirrored) def save_status(snapshot): diff --git a/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py b/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py index 63e43ec7..befde9be 100644 --- a/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py +++ b/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py @@ -383,59 +383,22 @@ def test_restart_does_not_halt_on_its_own_progress(big_run): # --- two gaps the fuzz does not reach --------------------------------------- -# Both are held in memory, so both are exactly as durable as the process. They -# are marked xfail(strict) rather than asserted-as-is: the assertions below say -# what the monitor SHOULD do, so they flip to a hard failure the day either gap -# is closed, instead of quietly cementing today's behaviour. - -@pytest.mark.xfail(strict=True, reason=( - "state['max_completed'] is memory-only, so the PROGRESS WENT BACKWARDS " - "guard is disabled for the life of a fresh process -- the one event it " - "most needs to survive")) -def test_the_backwards_progress_guard_survives_a_restart(big_run): - """Destroy the record under a running monitor and it refuses to dispatch. - - Destroy it under a monitor that then restarts and it re-runs the range from - genesis. Same fault, opposite outcome, decided purely by whether the - process happened to be the same one. - """ - cluster = big_run - cluster.reconcile() - for end in ('1200', '1100'): - cluster.advance(int(end), 'succeeded') - cluster.finalize(end, 1) - cluster.reconcile() - assert set(cluster.completed()) == {'1200', '1100'} - - # Both copies of the record go, the way the guard's own log line describes. - os.remove(jm.PROGRESS_FILE) - cluster.k8s.core_v1.delete_namespaced_config_map(jm.PROGRESS_CM, cluster.namespace) +# The monotonic-progress guard that used to be pinned here is gone: it kept its +# high-water mark in memory, so a restart disarmed it for exactly the event it +# existed to survive, and re-running a range is idempotent anyway. - restart(cluster) - cluster.reconcile() - # Free a slot so dispatch has capacity to misuse. - cluster.advance(1000, 'succeeded') - cluster.finalize('1000', 1) - mark = len(cluster.calls) - cluster.reconcile() - - redispatched = [c.name for c in cluster.calls[mark:] - if c.verb == 'create' and c.kind == 'job' - and _range_of_job(c.name)[0] in {'1200', '1100'}] - assert cluster.state['halted'] is True - assert not redispatched, f"already-completed ranges re-dispatched: {redispatched}" - - -@pytest.mark.xfail(strict=True, reason=( - "load_progress falls back to the ConfigMap mirror, which _state_only has " - "stripped of every measurement; the next save_progress then writes that " - "stripped record back over the authoritative file")) def test_a_measurement_survives_the_configmap_fallback(big_run): - """I5, in its purest form: a recorded peak that goes away. + """I5, in its purest form: a recorded peak that must not go away. + + The two stores have different jobs. The ConfigMap is the control plane the + mission driver reads, capped at 1 MiB, so _state_only strips every + measurement from it. The volume is the data plane. Falling back to the + mirror therefore returns a record that is complete as state and empty as + data, and the next save wrote that back over the volume. - progress.json becomes unreadable, load_progress falls back to the mirror, - and range 1200's peakRssBytes / txApply / seconds are gone -- not stale, - absent -- and then persisted absent. + The measurements were never really gone: only progress.json was damaged, + and .metrics is written per attempt and never rewritten. load_progress + re-reads them from there rather than persisting the hole. """ cluster = big_run cluster.reconcile() From 93cae0e971a1ad0498350e869bf61afe3ad8024f Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 11:44:24 -0400 Subject: [PATCH 032/117] Wire avoidNodeLabels through to the worker pods again The mission has always accepted --pubnet-parallel-catchup-avoid-node-labels and values.yaml has always declared worker.avoidNodeLabels, but nothing has read it since the Job-per-range rewrite. The old StatefulSet template rendered require and avoid into one nodeAffinity term together; the rewrite carried requireNodeLabels across into build_job and left its sibling behind. The flag installed cleanly and scheduled workers onto exactly the nodes the operator asked to keep them off -- silently, because a value nothing reads renders a perfectly valid manifest with the setting missing. Adds the AVOID_NODE_LABEL_KEY/VALUE env pair to the monitor container, mirroring the requireNodeLabels block so it accepts both the mission's {key,operator,values} maps and plain "key:value" strings, and emits the matching requirement from build_job. Both requirements go in ONE matchExpressions list: expressions within a term are ANDed, separate terms are ORed, so split across two terms an avoid-only pod would match every node. A missing value means DoesNotExist rather than NotIn [""], which would only exclude the empty value. The contract test that pinned this gap loses its xfail, and avoidNodeLabels joins the FULL render set -- the conditional blocks are only checked with the values that make them render, which is why this was invisible to it before. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 57 ++- .../templates/job_monitor.yaml | 13 + .../tests/contract/test_chart_env_wiring.py | 159 ++++++ .../contract/test_fsharp_driver_contract.py | 470 ++++++++++++++++++ .../tests/unit/test_node_targeting.py | 68 +++ 5 files changed, 756 insertions(+), 11 deletions(-) create mode 100644 src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py create mode 100644 src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_node_targeting.py diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 6e86d731..52324d05 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -134,6 +134,8 @@ # the default Equal operator does not match "" against "true". NODE_LABEL_KEY = os.getenv('NODE_LABEL_KEY', '') NODE_LABEL_VALUE = os.getenv('NODE_LABEL_VALUE', '') +AVOID_NODE_LABEL_KEY = os.getenv('AVOID_NODE_LABEL_KEY', '') +AVOID_NODE_LABEL_VALUE = os.getenv('AVOID_NODE_LABEL_VALUE', '') TOLERATE_TAINT = os.getenv('TOLERATE_TAINT', '') # Worker /data. pvc keeps it across pods, so an evicted range resumes at L+1 -- @@ -475,8 +477,11 @@ def job_name(end, attempt): # The authoritative copy of the progress record lives on the logs PVC, not in # the ConfigMap. A ConfigMap is capped at 1 MiB and this record is ~172 bytes # per completed range, so it dies at ~6100 ranges -- reachable simply by halving -# ledgersPerJob. Worse, every completion rewrote the whole document through the -# API server, so a full run meant thousands of escalating-size etcd writes. +# ledgersPerJob. Measured mid-run on ssc-test: 348KB at 2024 completed ranges, +# which projects to ~65% of the cap at 3982 -- close enough that the next +# slicing change would have hit it. Worse, every completion rewrote the whole +# document through the API server, so a full run meant thousands of +# escalating-size etcd writes. # # The ConfigMap is still written, because the mission driver reads it without # exec'ing into the pod, but it is now a best-effort mirror: if it fails, the @@ -962,7 +967,9 @@ def _bytes_to_quantity(n): # every range that applies a real transaction load. The old [0-9.]+ pattern # matched "1.30722" then demanded "ms" and hit "e+06ms" instead, so tx_apply was # silently missing for 25% of ranges -- 91-99% of everything above ledger 35M, -# exactly the expensive end. +# exactly the expensive end. 698 completed ranges lost the metric that way in a +# single run, which is the reference case for "a recoverable gap turned +# permanent" everywhere else in this file. _SUM_RE = re.compile(r"sum\s*=\s*([0-9.]+(?:[eE][+-]?[0-9]+)?)ms") @@ -1183,6 +1190,11 @@ def _tx_apply_for_attempt(end, attempt=1, pod_name=None): for i, line in enumerate(lines): if "metric 'ledger.transaction.apply'" not in line: continue + # Same reach as log_collector.TxApplyScanner.WINDOW, and it has to be: + # the two are independent readers of one block and progress.json takes + # whichever landed first. medida puts `sum` 10 lines under the header + # (27.1.1, ssc-test 2026-07-28), so five more percentiles in a release + # takes the metric out of range on both sides at once. for follow in lines[i + 1:i + 16]: m = _SUM_RE.search(follow) if m: @@ -1358,13 +1370,21 @@ def delete_job(end, attempt): reconcile() lists every Job and Pod on each pass, so a finished Job is not free: it inflates two LIST calls for as long as it lingers. At 2048-4096 parallelism with a real OOM or spot-eviction rate that is hundreds of dead - objects per hour of run, and the apiserver pressure shows up as truncated - list responses long before anything else complains. + objects per hour of run -- under the old 3600s TTL the dead ones outnumbered + the live ones within the first hour -- and the apiserver pressure shows up + as truncated list responses long before anything else complains. + + Background propagation specifically: that is what removes the pod as well. + Orphan or the server default would leave the pod behind, so the next pass + lists exactly as much as it did before. Callers must have persisted whatever they need first -- the logs, .outcome and .metrics all live on the monitor's volume by then, so the Job and its - pod carry no information once the range is recorded. Best-effort: on - failure JOB_TTL_SECONDS still reclaims it. + pod carry no information once the range is recorded. Best-effort: a 404 is + the ordinary race with the TTL controller, and any other status warns and + carries on. Raising here would abort the whole reconcile pass mid-iteration + and strand every other range in it; on failure JOB_TTL_SECONDS still + reclaims the object, which costs disk and etcd, never correctness. """ try: batch_v1.delete_namespaced_job(job_name(end, attempt), NAMESPACE, @@ -1600,13 +1620,28 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): env = [client.V1EnvVar(name='ASAN_OPTIONS', value=ASAN_OPTIONS)] if ASAN_OPTIONS else [] - affinity = None + # Require and avoid go in ONE matchExpressions list: expressions within a + # term are ANDed, whereas separate terms are ORed and an avoid-only pod would + # then match every node. The original StatefulSet template rendered both into + # the same term; the rewrite carried requireNodeLabels across and dropped + # avoidNodeLabels, so the flag installed cleanly and scheduled workers onto + # exactly the nodes it was asked to keep them off. + match = [] if NODE_LABEL_KEY: + match.append(client.V1NodeSelectorRequirement( + key=NODE_LABEL_KEY, operator='In', values=[NODE_LABEL_VALUE])) + if AVOID_NODE_LABEL_KEY: + # No value means "avoid the label however it is set", which is + # DoesNotExist; NotIn [""] would only exclude the empty value. + match.append(client.V1NodeSelectorRequirement( + key=AVOID_NODE_LABEL_KEY, + operator='NotIn' if AVOID_NODE_LABEL_VALUE else 'DoesNotExist', + values=[AVOID_NODE_LABEL_VALUE] if AVOID_NODE_LABEL_VALUE else None)) + affinity = None + if match: affinity = client.V1Affinity(node_affinity=client.V1NodeAffinity( required_during_scheduling_ignored_during_execution=client.V1NodeSelector( - node_selector_terms=[client.V1NodeSelectorTerm(match_expressions=[ - client.V1NodeSelectorRequirement(key=NODE_LABEL_KEY, operator='In', - values=[NODE_LABEL_VALUE])])]))) + node_selector_terms=[client.V1NodeSelectorTerm(match_expressions=match)]))) # Taint value must be absent: the mission emits {key, effect} with no value, # and the default Equal operator does not match "" against "true". tolerations = [client.V1Toleration(key=TOLERATE_TAINT, effect='NoSchedule')] if TOLERATE_TAINT else None diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index f5cf31a7..bdaa0112 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -261,6 +261,19 @@ spec: value: {{ (splitList ":" .) | last | quote }} {{- end }} {{- end }} + {{- with (first .Values.worker.avoidNodeLabels) }} + {{- if kindIs "map" . }} + - name: AVOID_NODE_LABEL_KEY + value: {{ .key | quote }} + - name: AVOID_NODE_LABEL_VALUE + value: {{ (first (default (list "") .values)) | quote }} + {{- else }} + - name: AVOID_NODE_LABEL_KEY + value: {{ (splitList ":" .) | first | quote }} + - name: AVOID_NODE_LABEL_VALUE + value: {{ (splitList ":" .) | last | quote }} + {{- end }} + {{- end }} {{- with (first .Values.worker.tolerateNodeTaints) }} - name: TOLERATE_TAINT value: {{ if kindIs "map" . }}{{ .key | quote }}{{ else }}{{ (splitList ":" .) | first | quote }}{{ end }} diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py b/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py new file mode 100644 index 00000000..878efe3d --- /dev/null +++ b/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py @@ -0,0 +1,159 @@ +"""The rendered Deployment against the env vars each process actually reads. + +Two containers share one pod and one volume but not one env block, so a +variable the monitor has is not automatically one the collector has. STORAGE_MODE +was missing from the collector and the peak-ephemeral sampler silently recorded +nothing -- it defaults to 'pvc', which is exactly the mode where the sampler is +supposed to stand down. + +The conditional blocks matter as much as the unconditional ones: node targeting, +taint toleration and the profile mount only render when the mission passes the +matching values, so "the chart sets it" has to be checked with those values +present. +""" + +import job_monitor as jm +import log_collector as lc + +import _artifacts as art + +# Injected by the kubelet or genuinely optional. Everything else the code reads +# has to come from the chart, or it silently runs on its built-in fallback. +KUBELET_INJECTED = {'KUBERNETES_SERVICE_HOST', 'KUBERNETES_SERVICE_PORT'} + +# Operator-facing switches with no chart key on purpose: they are set by hand on +# a running Deployment when something needs debugging, and a chart key would +# freeze them at install time. +DEBUG_ONLY = {'LOGGING_LEVEL', 'WATCH_STALE_SECONDS', 'CONNECTION_POOL', + 'WORKER_CONTAINER'} + +# Rendered with everything the mission can send, so the conditional blocks are +# present: a profile ConfigMap, a required node label, an avoided node label +# and a tolerated taint. avoidNodeLabels was declared in values.yaml and read +# by no template at all -- absent from here, that stays invisible. +FULL = ( + 'monitor.profileConfigMap=p', + 'worker.requireNodeLabels[0].key=purpose', + 'worker.requireNodeLabels[0].operator=In', + 'worker.requireNodeLabels[0].values[0]=catchup8-spot', + 'worker.avoidNodeLabels[0].key=reserved', + 'worker.avoidNodeLabels[0].operator=NotIn', + 'worker.avoidNodeLabels[0].values[0]=true', + 'worker.tolerateNodeTaints[0].key=catchup8-spot', + 'worker.tolerateNodeTaints[0].effect=NoSchedule', +) + + +def _missing(container_name, module): + reads = art.reads_env(art.module_source(module)) + set_by_chart = set(art.env_of(art.containers(FULL)[container_name])) + return sorted(reads - set_by_chart - KUBELET_INJECTED - DEBUG_ONLY) + + +def test_every_env_the_monitor_reads_is_set_on_the_monitor_container(): + missing = _missing(art.MONITOR_CONTAINER, jm) + assert not missing, f"the monitor reads {missing} but the chart never sets them" + + +def test_every_env_the_collector_reads_is_set_on_the_collector_container(): + missing = _missing(art.COLLECTOR_CONTAINER, lc) + assert not missing, f"the collector reads {missing} but the chart never sets them" + + +def test_the_node_targeting_the_mission_sends_reaches_the_monitor(): + """A label/taint the mission passes must arrive as env, not just as YAML. + + The monitor puts the affinity on the WORKER pods it builds; the chart's job + is only to hand it the pair. Rendering the values into some other shape -- + or into the Deployment's own nodeSelector -- would place the monitor and + leave every worker unconstrained. + """ + env = art.env_of(art.containers(FULL)[art.MONITOR_CONTAINER]) + assert env.get('NODE_LABEL_KEY') == 'purpose' + assert env.get('NODE_LABEL_VALUE') == 'catchup8-spot' + assert env.get('TOLERATE_TAINT') == 'catchup8-spot' + + +def test_node_targeting_is_absent_rather_than_empty_when_unset(): + """An empty NODE_LABEL_KEY is how the monitor knows not to constrain a pod. + + Setting it to "" would work by accident today, but the guard the monitor + uses is truthiness of the key, so an empty-string env and an unset env must + stay interchangeable -- and the chart should not emit a knob it is not + configuring. + """ + env = art.env_of(art.containers()[art.MONITOR_CONTAINER]) + for name in ('NODE_LABEL_KEY', 'NODE_LABEL_VALUE', 'TOLERATE_TAINT'): + assert env.get(name, '') == '', f"{name} rendered without a value to carry" + assert art.defaults('job_monitor')['NODE_LABEL_KEY'] == '', \ + "the code fallback must be the falsy 'no targeting' value" + + +def test_the_two_containers_run_the_same_image_from_one_build(): + """The monitor and the collector are two entrypoints in one image. + + They share file formats on a shared volume, so shipping them from separate + images would let the pair skew by a release -- which is the failure every + cross-process test in this directory exists to prevent. + """ + cs = art.containers(FULL) + assert (cs[art.MONITOR_CONTAINER]['image'] + == cs[art.COLLECTOR_CONTAINER]['image']) + + +def test_the_collector_is_started_as_the_collector(): + """Same image, so the collector needs an explicit entrypoint. + + Without one it runs the image's default command -- a second job_monitor, + which would be a second writer of progress.json and every Job. + """ + collector = art.containers(FULL)[art.COLLECTOR_CONTAINER] + started = " ".join(collector.get('command', []) + collector.get('args', [])) + assert 'log_collector.py' in started, ( + f"the collector container does not run log_collector.py: {started!r}") + monitor = art.containers(FULL)[art.MONITOR_CONTAINER] + monitor_started = " ".join(monitor.get('command', []) + monitor.get('args', [])) + assert 'log_collector.py' not in monitor_started + + +def test_only_one_monitor_ever_runs(): + """Single writer is what removes the claim/requeue races the redis queue had. + + Two replicas -- or a rolling update that briefly overlaps them -- would give + two processes the same progress.json, the same Job names and the same PVCs, + with no leader election anywhere in the monitor. + """ + spec = art.monitor_deployment(FULL)['spec'] + assert spec['replicas'] == 1 + assert spec['strategy']['type'] == 'Recreate', ( + "a RollingUpdate briefly runs two monitors against one progress record") + + +def test_the_run_name_the_monitor_labels_with_is_the_helm_release(): + """Every Job, PVC and ConfigMap this run owns is found by that label. + + Two releases in one namespace is the normal case on a shared test cluster. + If RUN_NAME were not the release name, one release's reconcile would list + the other's Jobs and reap them. + """ + env = art.env_of(art.containers(release='pc-abc')[art.MONITOR_CONTAINER]) + assert env['RUN_NAME'] == 'pc-abc' + collector = art.env_of(art.containers(release='pc-abc')[art.COLLECTOR_CONTAINER]) + assert collector['RUN_NAME'] == 'pc-abc', \ + "the collector would watch a different run's pods" + assert jm.LABEL_RUN == lc.LABEL_RUN, \ + "the two processes select on different label keys" + + +def test_the_namespace_comes_from_the_pod_not_from_a_value(): + """helm --namespace and a values key can disagree; the downward API cannot. + + The monitor creates Jobs in NAMESPACE. A stale value there would dispatch a + whole run into a namespace the release does not own. + """ + for name in (art.MONITOR_CONTAINER, art.COLLECTOR_CONTAINER): + entry = [e for e in art.containers(FULL)[name]['env'] + if e['name'] == 'NAMESPACE'] + assert entry, f"{name} has no NAMESPACE" + field = entry[0]['valueFrom']['fieldRef']['fieldPath'] + assert field == 'metadata.namespace', f"{name} reads NAMESPACE from {field}" diff --git a/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py b/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py new file mode 100644 index 00000000..b3461202 --- /dev/null +++ b/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py @@ -0,0 +1,470 @@ +"""MissionHistoryPubnetParallelCatchupV2.fs against the chart and the Python. + +The F# driver is the only caller. It installs the chart with a pile of --set +overrides, polls the monitor through a ConfigMap, execs into the monitor pod to +collect logs and to read progress.json, and writes the range-profile artifact +that a LATER run's monitor reads back. Nothing in that loop is type-checked +across the language boundary: a --set key the chart does not know is accepted by +helm and does nothing, a JSON field the driver forgets to project is simply +absent, and a ConfigMap key it looks up under the wrong name reads as "the +monitor has not published yet". + +Every failure in that list is silent, and several have happened. + +The F# is read as text -- there is no dotnet in this suite -- but each test +drives the extracted value through the real chart or the real Python, so what is +pinned is the agreement and not the F#'s spelling of it. +""" + +import json +import os +import re + +import pytest + +import job_monitor as jm +import log_collector as lc + +import _artifacts as art + +FS = art.fsharp() + + +def fs_extract(pattern, flags=re.S): + m = re.search(pattern, FS, flags) + assert m, f"not found in the F# driver: {pattern}" + return m + + +# --- the --set keys the driver sends ----------------------------------------- + +_SET_KEY = re.compile( + r'(?:worker|monitor|range|service_account)(?:\.[A-Za-z0-9_]+|\[%d\]|\[0\])+(?==)') + + +def set_keys(): + """Every chart value path the driver overrides, indices stripped. + + An indexed path is truncated at the array: `worker.requireNodeLabels[0].key` + is the chart's `worker.requireNodeLabels` list, whose element shape is + checked by rendering it below rather than by looking it up in values.yaml. + """ + out = {} + for raw in set(_SET_KEY.findall(FS)): + out.setdefault(raw.split('[')[0], set()).add(raw) + return out + + +def test_the_driver_really_does_configure_the_chart(): + """Guards the extraction: a regex that stopped matching would pass silently.""" + keys = set_keys() + assert len(keys) >= 15, f"only found {sorted(keys)}; the --set scan has gone blind" + assert 'worker.stellar_core_image' in keys and 'range.ledgersPerJob' in keys + + +def test_every_value_the_driver_sets_is_one_the_chart_knows(): + """`helm --set` on an unknown path is accepted and ignored. + + A rename in values.yaml, or a typo here, produces a run that installs + cleanly and quietly uses the default for whatever the driver meant to + override -- the wrong image, the wrong ledger range, the wrong storage mode. + """ + values = _values_tree() + templates = _template_text() + unknown = [k for k in sorted(set_keys()) + if not _in_values(values, k) and f".Values.{k}" not in templates] + assert not unknown, ( + "the driver overrides chart values that do not exist; helm accepts them " + f"and does nothing: {unknown}") + + +def test_every_value_the_driver_sets_reaches_a_template(): + """Declared in values.yaml is not the same as consumed. + + A key that exists but is read by nothing renders a perfectly valid manifest + with the setting missing. + """ + templates = _template_text() + inert = [k for k in sorted(set_keys()) if f".Values.{k}" not in templates] + assert not inert, f"declared in values.yaml but read by no template: {inert}" + + +def _values_tree(): + import yaml + return yaml.safe_load(art.values_yaml()) + + +def _template_text(): + tdir = os.path.join(art.CHART, 'templates') + parts = [art.text(os.path.join(tdir, n)) for n in sorted(os.listdir(tdir))] + parts.append(art.text(os.path.join(art.CHART, 'files', 'stellar-core.cfg'))) + return "\n".join(parts) + + +def _in_values(tree, path): + node = tree + for part in path.split('.'): + if not isinstance(node, dict) or part not in node: + return False + node = node[part] + return True + + +# --- the indexed shapes only the mission sends ------------------------------- + +def test_the_service_account_annotations_the_driver_sends_render_as_a_map(): + """metadata.annotations must be a map; the driver sends an indexed array. + + Passing it straight through toYaml produced a list and failed the whole + install with "cannot unmarshal array into ... map[string]string". The --set + strings below are built from the driver's own sprintf format, so a change to + the shape it emits is caught here rather than at install time. + """ + fmt = fs_extract(r'let serviceAccountAnnotationsToHelmIndexed.*?sprintf\s+"([^"]+)"').group(1) + sets = tuple(_fill(fmt, 0, 'eks.amazonaws.com/role-arn', 'arn:aws:iam::1:role/r').split(',')) + for sa in art.of_kind('ServiceAccount', sets): + annotations = sa['metadata'].get('annotations') + assert isinstance(annotations, dict), f"{sa['metadata']['name']}: {annotations!r}" + assert annotations['eks.amazonaws.com/role-arn'] == 'arn:aws:iam::1:role/r' + + +def test_the_chart_still_renders_with_no_annotations_at_all(): + """A hand-run install passes none, and the mission passes none by default.""" + for sa in art.of_kind('ServiceAccount'): + assert not sa['metadata'].get('annotations') + + +def test_the_node_selector_the_driver_sends_reaches_the_monitor(): + """The driver emits structured {key, operator, values} like every other + supercluster mission; a hand-run helm install more naturally passes + "key:value" strings. Both shapes have to arrive as the same env pair.""" + body = fs_extract(r'let requireNodeLabelToHelmIndexed(.*?)\nlet ').group(1) + for fragment in ('worker.requireNodeLabels[%d].key', 'operator=In', '.values[0]='): + assert fragment in body, f"the driver no longer emits {fragment!r}" + structured = ('worker.requireNodeLabels[0].key=purpose', + 'worker.requireNodeLabels[0].operator=In', + 'worker.requireNodeLabels[0].values[0]=catchup8-spot') + env = art.env_of(art.containers(structured)[art.MONITOR_CONTAINER]) + assert (env['NODE_LABEL_KEY'], env['NODE_LABEL_VALUE']) == ('purpose', 'catchup8-spot') + + plain = ('worker.requireNodeLabels[0]=purpose:catchup8-spot',) + env = art.env_of(art.containers(plain)[art.MONITOR_CONTAINER]) + assert (env['NODE_LABEL_KEY'], env['NODE_LABEL_VALUE']) == ('purpose', 'catchup8-spot') + + +def test_the_taint_the_driver_sends_reaches_the_monitor(): + """The driver defaults the effect to NoSchedule and sends no value. + + The monitor builds a Toleration with the default Equal operator, which does + not match "" against "true" -- so the value must stay absent on both sides. + """ + fmt = fs_extract(r'let tolerateTaintToHelmIndexed.*?sprintf\s+"([^"]+)"').group(1) + assert '.effect=' in fmt and '.value' not in fmt + sets = ('worker.tolerateNodeTaints[0].key=catchup8-spot', + 'worker.tolerateNodeTaints[0].effect=NoSchedule') + env = art.env_of(art.containers(sets)[art.MONITOR_CONTAINER]) + assert env['TOLERATE_TAINT'] == 'catchup8-spot' + + +def _fill(fmt, index, *values): + """Apply an F# sprintf format with %d indices and %s values.""" + out, values = fmt.replace('\\"', '"'), list(values) + out = out.replace('%d', str(index)) + for value in values: + out = out.replace('%s', value, 1) + return out + + +# --- the worker command line the driver builds ------------------------------- + +def test_the_driver_disables_the_aws_progress_meter(): + """--no-progress is load-bearing, not cosmetic. + + The AWS CLI draws its transfer meter with carriage returns and no newline, + so a 628 MiB bucket download arrives as one multi-megabyte "line". Measured + on ssc-test 2026-07-30 at 2096 workers: aiohttp aborts a line over 512 KiB, + so every large download killed its own collector stream, the reconnect hit + the same wall, and every retry pod was starved of a stream. The collector + now reads in chunks and splits on \\r too (see test_cross_process_files), + but the cure is not emitting the spam -- it was also the bulk of every large + range's archive. + """ + flags = fs_extract(r'sprintf "aws s3 cp ([^"]*)--region %s"').group(1) + assert '--no-progress' in flags, f"aws s3 cp flags: {flags!r}" + + +def test_the_history_get_command_lands_in_the_config_the_worker_mounts(): + """The S3 mirror override is a per-archive `get` command in stellar-core.cfg. + + Without it the workers fall back to the public archive, which throttles at + 1024 -- silently, as a very slow run rather than an error. + """ + template = fs_extract(r'setOptions\.Add\(sprintf "(worker\.historyGetCommandCore00%d)=').group(1) + for index in (1, 2, 3): + key = template.replace('%d', str(index)) + assert f".Values.{key}" in art.text( + os.path.join(art.CHART, 'files', 'stellar-core.cfg')), \ + f"{key} is set by the driver but never reaches stellar-core.cfg" + + +# --- the ConfigMap the driver polls ------------------------------------------ + +def test_the_driver_reads_the_configmap_the_monitor_writes(): + """One name, derived on both sides from the helm release name. + + The driver appends a literal suffix to the release; the monitor appends the + same suffix to RUN_NAME, which the chart sets from .Release.Name. A mismatch + reads as "the monitor has not published yet", forever -- and the driver's + only reaction to that is a 600s timeout and `job monitor not reachable`. + """ + suffix = fs_extract(r'helmReleaseName \+ "(-[a-z-]+)"').group(1) + release = 'pc-abc' + # What the driver will ask for, and what the monitor will have created -- + # the latter imported with the RUN_NAME the chart gives it for that release. + wanted = release + suffix + run_name = art.env_of(art.containers(release=release)[art.MONITOR_CONTAINER])['RUN_NAME'] + written = art.defaults('job_monitor', (('RUN_NAME', run_name),))['PROGRESS_CM'] + assert wanted == written, f"driver reads {wanted!r}, monitor writes {written!r}" + + +def test_the_driver_reads_the_keys_the_monitor_publishes(cluster): + """status.json and progress.json are two keys in that one ConfigMap. + + Checked against a ConfigMap the real monitor actually wrote, so a key that + is only mentioned in a comment does not count. + """ + wanted = set(re.findall(r'let jobMonitor\w*Key = "([\w.]+)"', FS)) + assert wanted, "the driver no longer names the ConfigMap keys" + + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.reconcile() # records a completion -> progress.json + jm.save_status(jm.status) # what the reconcile loop publishes + published = set(cluster.k8s.config_map_data(jm.PROGRESS_CM, cluster.namespace) or {}) + missing = sorted(wanted - published) + assert not missing, ( + f"the driver reads {missing}; the monitor published {sorted(published)}") + + +def test_every_status_field_the_driver_reads_is_one_the_monitor_sets(): + """The driver's loop terminates on num_remain and jobs_in_progress. + + A field it reads that the monitor never sets throws inside the polling loop, + which the driver treats as fatal: cleanup, uninstall, mission failed -- with + the run's work discarded. + """ + read = set(re.findall(r'status\.(?:\[|Value<\w+>\()"(\w+)"', FS)) + assert read, "the status parse has changed shape -- update this test" + missing = sorted(read - set(jm.status)) + assert not missing, f"the driver reads status fields the monitor never sets: {missing}" + + +def test_the_driver_can_find_the_pod_name_in_a_failed_range_entry(cluster): + """jobs_failed entries are "|", split on '|' by the driver. + + It uses element 1 as a pod name to dump logs from. An entry with no + separator makes that a silent no-op; an entry with the halves swapped makes + it request a pod named after a ledger range. + """ + cluster.reconcile() + cluster.advance(300, 'condemned') + result = cluster.reconcile() + + assert result['failed_ranges'], "no range was condemned; the fixture changed" + entry = result['failed_ranges'][0] + parts = entry.split('|') + assert len(parts) == 2, f"the driver's split('|')[1] cannot work on {entry!r}" + assert parts[1].startswith(f"{cluster.run_name}-r300-a"), ( + f"element 1 is {parts[1]!r}, which is not a pod name") + assert '/' in parts[0], f"element 0 should be the / range key: {parts[0]!r}" + + +# --- the exec paths the driver uses at teardown ------------------------------ + +def test_the_driver_execs_into_a_container_that_exists(): + """A wrong container name fails the exec, and the failure is caught and + logged as a warning -- so the run finishes with no collected logs and no + range profile.""" + names = set(art.containers()) + for name in set(re.findall(r'containerName = "([\w-]+)"', FS)): + assert name in names, f"the driver execs into {name!r}; the pod has {sorted(names)}" + + +def test_the_driver_reads_the_progress_file_where_the_monitor_writes_it(): + """`cat /logs/progress.json`, hard-coded on the driver side.""" + path = fs_extract(r'command = \[\| "cat"; "([^"]+)" \|\]').group(1) + assert path == jm.PROGRESS_FILE, ( + f"the driver cats {path}; the monitor writes {jm.PROGRESS_FILE}") + + +def test_the_driver_tars_the_directory_the_collector_writes_into(): + """One exec replaces the ~1024 the StatefulSet design needed.""" + cd = fs_extract(r'"cd (/\w+) && tar').group(1) + assert cd == jm.LOG_DIR == lc.LOG_DIR + + +def test_the_tar_excludes_only_the_collectors_resume_bookkeeping(): + """.state is a resume cursor and is worthless outside the pod. + + Every other suffix on that volume is a deliverable: the archive, the + per-attempt metrics, the verdict. An exclusion pattern that drifted onto one + of those would quietly shrink the collected tar. + """ + def suffix_of(path_fn): + return os.path.basename(path_fn('E', 1)).partition('-a1')[2] + + bookkeeping = {suffix_of(jm.state_path)} + deliverables = {suffix_of(f) for f in (jm.log_path, jm.metrics_path, + jm.outcome_path, jm.done_path)} + assert bookkeeping.isdisjoint(deliverables) + + excludes = set(re.findall(r"--exclude='([^']+)'", FS)) + assert excludes, "the tar no longer excludes anything -- update this test" + for pattern in excludes: + if not pattern.startswith('*'): + continue # ./lost+found, the PVC's ext4 root + assert pattern[1:] in bookkeeping, ( + f"the tar excludes {pattern}, which is not resume bookkeeping") + for suffix in deliverables: + assert f"*{suffix}" not in excludes, f"the tar drops {suffix}, a deliverable" + + +def test_the_driver_finds_the_monitor_pod_by_the_labels_the_chart_sets(): + """Two releases share a namespace on a test cluster routinely. + + A selector missing the release label would exec into the other run's monitor + -- and read its progress record. + """ + selector = fs_extract(r'labelSelector = sprintf "([^"]+)"').group(1) + labels = art.monitor_deployment(release='pc-abc')['spec']['template']['metadata']['labels'] + for clause in selector.split(','): + key, _, value = clause.partition('=') + assert key in labels, f"the driver selects on {key!r}; the pod has {sorted(labels)}" + if '%s' not in value: + assert labels[key] == value + else: + assert labels[key] == 'pc-abc' + + +# --- the range-profile artifact: written by F#, read by Python next run ------ + +def fs_profile_fields(): + body = fs_extract(r'let rangeProfileFields =(.*?)\n\n').group(1) + return set(re.findall(r'"(\w+)"', body)) + + +def fs_document_keys(): + return set(re.findall(r'doc\.\["(\w+)"\]\s*<-', FS)) + + +def test_the_artifact_carries_exactly_the_fields_the_mirror_strips(): + """Two lists that must be one list. + + The monitor strips _PROFILE_ONLY_FIELDS out of the ConfigMap mirror to stay + under its 1 MiB cap, so those fields exist only in the volume copy -- which + is precisely the copy the driver projects into the artifact. A field in one + list and not the other is either lost from the artifact or bloating the + mirror. peakAnonBytes was missing from the projection: measured 2026-07-30, + the artifact carried it for 0% of ranges while the volume copy had it for + 99%. + """ + assert fs_profile_fields() == set(jm._PROFILE_ONLY_FIELDS), ( + "driver projects " + f"{sorted(fs_profile_fields() - set(jm._PROFILE_ONLY_FIELDS))} extra, " + f"drops {sorted(set(jm._PROFILE_ONLY_FIELDS) - fs_profile_fields())}") + + +def test_every_field_the_sizing_consumer_reads_is_in_the_artifact(): + """Derived from _profile_overrides, so a new sizing input fails here first.""" + consumed = set(re.findall(r"prof\.get\('(\w+)'\)", art.module_source(jm))) + assert consumed, "the sizing consumer no longer reads named fields" + missing = sorted(consumed - fs_profile_fields()) + assert not missing, f"the profile is sized from {missing}, which the artifact drops" + + +def test_every_document_key_the_monitor_reads_is_one_the_driver_writes(): + """storageMode decides whether the disk axis is usable; ranges is the data.""" + read = set(re.findall(r"doc\.get\('(\w+)'\)", art.module_source(jm))) + assert read, "load_profile no longer reads named document keys" + missing = sorted(read - fs_document_keys()) + assert not missing, f"load_profile reads {missing}, which the driver never writes" + + +def _artifact(storage_mode='pvc', ranges=None): + """A profile document in the exact shape the driver writes.""" + doc = {'schema': 1, 'generated': '2026-07-30T00:00:00.0000000Z', + 'release': 'parallel-catchup-abc', 'storageMode': storage_mode, + 'ledgersPerRange': 16320, 'ranges': ranges or {}} + assert set(doc) == fs_document_keys(), ( + f"this stand-in has drifted from the driver: {set(doc) ^ fs_document_keys()}") + return doc + + +def test_an_artifact_from_a_previous_run_loads_and_sizes_the_next_one(tmp_path, monkeypatch): + """The whole point of the artifact, end to end across the language boundary. + + Values are the driver's own projection of a completed range: keyed by range + end as a STRING (JSON object keys always are), with count alongside the + measurements. + """ + path = tmp_path / 'profile.json' + path.write_text(json.dumps(_artifact(ranges={ + '16752063': {'peakAnonBytes': 2 * 1024 ** 3, 'peakWorkingSetBytes': 13 * 1024 ** 3, + 'txApply': 900.0, 'seconds': 1200.0, 'count': 16320}}))) + + monkeypatch.setattr(jm, 'PROFILE_PATH', str(path)) + monkeypatch.setattr(jm, 'STORAGE_MODE', 'pvc') + monkeypatch.setattr(jm, 'PROFILE', jm.load_profile()) + assert jm.PROFILE, "the driver's artifact did not load at all" + + sized = jm._profile_overrides(16752063, escalated=False) + assert 'memory' in sized, "a measured range was not sized from the artifact" + assert (jm._quantity_bytes(sized['memory']) > 2 * 1024 ** 3), \ + "the request came out below the measured peak" + + +def test_a_cross_mode_artifact_keeps_memory_and_drops_the_disk_axis(tmp_path, monkeypatch): + """storageMode is in the document because the axes are not interchangeable. + + cpu and memory measure the same work in either mode. Disk does not: a pvc + run puts /data on the volume and never measures node-local usage at all, so + an ephemeral run's figure says nothing about it. + """ + path = tmp_path / 'profile.json' + path.write_text(json.dumps(_artifact(storage_mode='ephemeral', ranges={ + '16752063': {'peakAnonBytes': 2 * 1024 ** 3, + 'peakEphemeralBytes': 30 * 1024 ** 3, 'count': 16320}}))) + + monkeypatch.setattr(jm, 'PROFILE_PATH', str(path)) + monkeypatch.setattr(jm, 'STORAGE_MODE', 'pvc') + monkeypatch.setattr(jm, 'LIM_EPHEMERAL', '40Gi') + monkeypatch.setattr(jm, 'PROFILE', jm.load_profile()) + + sized = jm._profile_overrides(16752063, escalated=False) + assert 'memory' in sized, "a cross-mode profile was rejected outright" + assert 'ephemeral-storage' not in sized, "a pvc run was sized from ephemeral-mode disk" + + +def test_an_empty_artifact_is_never_written_and_never_fatal(tmp_path, monkeypatch): + """An empty profile is worse than none: it looks complete. + + The usual cause is readProgressRecord falling back to the ConfigMap mirror, + which has every profiling field stripped. Both sides guard it -- the driver + writes nothing, and the monitor treats a profile with no usable range as no + profile -- because either half alone leaves the next run sizing itself from + empty data instead of from its configured requests. + """ + assert re.search(r'if ranges\.Count = 0 then None', FS), \ + "the driver no longer suppresses an empty profile" + + path = tmp_path / 'profile.json' + path.write_text(json.dumps(_artifact(ranges={}))) + monkeypatch.setattr(jm, 'PROFILE_PATH', str(path)) + monkeypatch.setattr(jm, 'STORAGE_MODE', 'pvc') + monkeypatch.setattr(jm, 'PROFILE', jm.load_profile()) + assert jm.PROFILE == [], "an empty profile loaded as if it held something" + assert jm._profile_overrides(16752063, escalated=False) == {} + + # ...and an artifact that never arrived at all is the same, not an error. + monkeypatch.setattr(jm, 'PROFILE_PATH', str(tmp_path / 'absent.json')) + assert jm.load_profile() == [] diff --git a/src/MissionParallelCatchup/tests/unit/test_node_targeting.py b/src/MissionParallelCatchup/tests/unit/test_node_targeting.py new file mode 100644 index 00000000..e4473edb --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_node_targeting.py @@ -0,0 +1,68 @@ +"""Node affinity and tolerations that build_job puts on a worker pod. + +The mission exposes three node-targeting flags. Two of them survived the +rewrite from a StatefulSet template to API-created Jobs; avoidNodeLabels did +not, and the gap was silent -- the driver sent the value, values.yaml declared +it, and no template read it, so a run started with --pubnet-parallel-catchup- +avoid-node-labels scheduled workers onto exactly the nodes it named. +""" + +import importlib + +import pytest + +import job_monitor as jm + + +def _match_expressions(monkeypatch, **env): + """build_job's node-affinity expressions under a given env. + + Takes the `cluster` fixture because build_job calls ensure_pvc, which is a + real API call -- the fixture is what puts the fake cluster behind it. + """ + for k in ('NODE_LABEL_KEY', 'NODE_LABEL_VALUE', + 'AVOID_NODE_LABEL_KEY', 'AVOID_NODE_LABEL_VALUE'): + monkeypatch.setattr(jm, k, env.get(k, '')) + job = jm.build_job(300, 420, 1, None) + aff = job.spec.template.spec.affinity + if aff is None: + return None + terms = aff.node_affinity.required_during_scheduling_ignored_during_execution + return terms.node_selector_terms[0].match_expressions + + +def test_no_targeting_leaves_the_pod_unconstrained(cluster, monkeypatch): + assert _match_expressions(monkeypatch) is None + + +def test_require_alone_pins_the_pod_to_the_label(cluster, monkeypatch): + exprs = _match_expressions(monkeypatch, + NODE_LABEL_KEY='purpose', NODE_LABEL_VALUE='catchup-spot') + assert [(e.key, e.operator, e.values) for e in exprs] == [ + ('purpose', 'In', ['catchup-spot'])] + + +def test_avoid_alone_keeps_the_pod_off_the_label(cluster, monkeypatch): + exprs = _match_expressions(monkeypatch, + AVOID_NODE_LABEL_KEY='purpose', + AVOID_NODE_LABEL_VALUE='catchup-od') + assert [(e.key, e.operator, e.values) for e in exprs] == [ + ('purpose', 'NotIn', ['catchup-od'])] + + +def test_avoid_without_a_value_means_the_label_must_be_absent(cluster, monkeypatch): + # NotIn [""] would only exclude the empty value, which is not what "avoid + # this label" means; the mission sends operator DoesNotExist for this case. + exprs = _match_expressions(monkeypatch, AVOID_NODE_LABEL_KEY='reserved') + assert [(e.key, e.operator) for e in exprs] == [('reserved', 'DoesNotExist')] + assert not exprs[0].values + + +def test_require_and_avoid_share_one_term_so_they_are_anded(cluster, monkeypatch): + # Expressions inside a term are ANDed; separate terms are ORed. Split across + # two terms, a pod that failed the require would still match on the avoid. + exprs = _match_expressions(monkeypatch, + NODE_LABEL_KEY='purpose', NODE_LABEL_VALUE='catchup-spot', + AVOID_NODE_LABEL_KEY='reserved') + assert [(e.key, e.operator) for e in exprs] == [ + ('purpose', 'In'), ('reserved', 'DoesNotExist')] From 1422824d72c7aaea2509d0925c986fd85a6ea66d Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 12:17:01 -0400 Subject: [PATCH 033/117] Retire test_job_monitor.py for tests that run the code The file was 2383 lines and 177 tests written when job_monitor could not be imported: it read both modules as text and either asserted substrings or regex-extracted a function body and exec'd it into a hand-built namespace. That constraint is gone, and the style was actively harmful -- two tests broke today over correct fixes because they pinned an exact call rather than the behaviour, and one matched the wrong occurrence of a statement because a new helper appeared earlier in the file. Replaced by 148 unit tests that import the module and call the function, grouped by subject, plus a contract suite for the boundaries behaviour cannot reach from inside Python: chart defaults against code defaults, rendered env against the env each process reads, chart RBAC against the API calls made, the podFailurePolicy rule order against the index classify() decodes, and the F# driver against the Python it drives. Suite goes 346 -> 491 tests with the source-text file gone. Comments in log_collector.py grew by 16 lines and job_monitor.py by 30: a measured fact recorded only in a dying test had to land somewhere, so it moved onto the code it describes. Logic is unchanged -- verified AST-identical ignoring docstrings. Trimming the run narrative back out of both files is a separate pass; the facts stay, the changelog goes to commits like this one. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/log_collector.py | 43 +- .../tests/contract/_artifacts.py | 185 ++ .../tests/contract/test_chart_defaults.py | 201 ++ .../tests/contract/test_chart_rbac.py | 165 ++ .../contract/test_cross_process_files.py | 200 ++ .../contract/test_k8s_failure_formats.py | 314 +++ .../contract/test_medida_metric_block.py | 198 ++ .../tests/contract/test_rendered_job_spec.py | 292 ++ .../tests/contract/test_worker_log_markers.py | 133 + .../test_completed_range_not_redispatched.py | 5 +- .../tests/test_job_monitor.py | 2384 ----------------- .../tests/unit/conftest.py | 23 + .../tests/unit/test_attempt_chain.py | 194 ++ .../tests/unit/test_classify.py | 214 ++ .../tests/unit/test_collector_main_loop.py | 259 ++ .../tests/unit/test_kubelet_sampler.py | 246 ++ .../unit/test_monitor_verdict_records.py | 210 ++ .../tests/unit/test_poll_lifecycle.py | 274 ++ .../tests/unit/test_profile_lookup.py | 123 + .../tests/unit/test_range_generation.py | 84 + .../tests/unit/test_reaping.py | 168 ++ .../tests/unit/test_records.py | 127 + .../tests/unit/test_resources.py | 209 ++ .../tests/unit/test_resume_script.py | 143 + .../tests/unit/test_sizing.py | 114 + .../tests/unit/test_tx_apply.py | 220 ++ 26 files changed, 4331 insertions(+), 2397 deletions(-) create mode 100644 src/MissionParallelCatchup/tests/contract/_artifacts.py create mode 100644 src/MissionParallelCatchup/tests/contract/test_chart_defaults.py create mode 100644 src/MissionParallelCatchup/tests/contract/test_chart_rbac.py create mode 100644 src/MissionParallelCatchup/tests/contract/test_cross_process_files.py create mode 100644 src/MissionParallelCatchup/tests/contract/test_k8s_failure_formats.py create mode 100644 src/MissionParallelCatchup/tests/contract/test_medida_metric_block.py create mode 100644 src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py create mode 100644 src/MissionParallelCatchup/tests/contract/test_worker_log_markers.py delete mode 100644 src/MissionParallelCatchup/tests/test_job_monitor.py create mode 100644 src/MissionParallelCatchup/tests/unit/conftest.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_attempt_chain.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_classify.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_poll_lifecycle.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_profile_lookup.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_range_generation.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_reaping.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_records.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_resources.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_resume_script.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_sizing.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_tx_apply.py diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 9e6e0a31..a1d321b8 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -62,8 +62,10 @@ # and workingSetBytes per container in the same /stats/summary payload this # already fetches for ephemeral storage, at ~10s cAdvisor housekeeping against a # 30s scrape -- and without depending on Prometheus being up, being reachable, -# or still retaining the window. cpu is not sampled at all: the request is fixed -# at REQ_CPU, so a measured value has nothing to size. +# or still retaining the window. The old _promql helper swallowed all three of +# those failures into "no peak", so an outage produced a profile that looked +# complete and was empty. cpu is not sampled at all: the request is fixed at +# REQ_CPU, so a measured value has nothing to size. # Peaks are held per pod and flushed on significant growth, so a restart loses # at most PEAK_FLUSH_RATIO of a range's high-water rather than all of it -- # Prometheus's server-side max_over_time needed no such state. @@ -77,7 +79,10 @@ MAX_CONCURRENT_POLLS = int(os.getenv('MAX_CONCURRENT_POLLS', 96)) # Most one poll may read before it stops. A pod that has been unwatched for a # while has a large backlog; this bounds a single response, and the next poll -# picks up from the timestamp this one reached. +# picks up from the timestamp this one reached. Also the backstop against a +# blob that never terminates -- a progress meter emitting only carriage returns +# would otherwise grow the buffer until the sidecar OOMs, with 2096 streams +# doing it at once. MAX_POLL_CHARS = int(os.getenv('MAX_POLL_CHARS', 8388608)) # Bounds in-flight polls across every pod. Lives beside its own constant rather # than among the peak dicts, where it landed inside the region the scanner tests @@ -96,7 +101,9 @@ PEAK_KEYS = ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes') # Failed polls tolerated after a pod goes terminal before we stop asking. Its # log is not coming back, and spinning on it holds a task and a poll slot for -# the rest of the run; a couple of retries still absorb a transient 500. +# the rest of the run; a couple of retries still absorb a transient 500, which +# arrived in bursts at ramp. Returning bare on one of those used to drop the +# metrics for every range whose last read happened to throw. TERMINAL_POLL_ATTEMPTS = int(os.getenv('TERMINAL_POLL_ATTEMPTS', 3)) # Phases whose log endpoint can actually answer. Pending has no container yet # and Unknown means the node stopped reporting; the terminal phases are kept @@ -427,6 +434,9 @@ async def sample_kubelet(session, nodes): write_metrics(ref[0], ref[1], {'peakEphemeralBytes': int(used)}) for c in entry.get('containers', []): + # The worker container only. Sidecars share the pod, so summing + # across containers -- or letting the last one win -- would size + # the range from whichever one kubelet happened to list last. if c.get('name') != CONTAINER: continue # Absent for the first seconds of a container's life, before @@ -439,6 +449,8 @@ async def sample_kubelet(session, nodes): rss = mem.get('rssBytes') if rss is None: continue + # High-water, never last-seen: anon oscillates through the + # download phase, so a later lower sample must not lower it. if int(rss) <= _anon_peak.get(name, 0): continue _anon_peak[name] = int(rss) @@ -470,11 +482,18 @@ def _mark_done(end, attempt): async def finalize(session, pod, end, attempt, tx, done_ok, started=None): """Persist everything this attempt owes, then let its stream go. - Reached from two places: a clean end of stream once the pod is terminal, - and a 404 once the pod object is gone. The second path used to not exist, - so a pod deleted while Running -- reaped node, eviction, or the monitor - deleting a finished Job -- left its stream retrying every 30s for the rest - of the run, holding a connection slot the whole time. + Reached from three places, and deliberately ONE implementation: a clean end + of stream once the pod is terminal, a 404 once the pod object is gone, and + a terminal pod whose polls keep failing past TERMINAL_POLL_ATTEMPTS. Two + copies of the metrics/discard logic is how one path silently stops writing + peakAnonBytes while the other keeps working. + + The 404 path used to not exist, so a pod deleted while Running -- reaped + node, eviction, or the monitor deleting a finished Job -- left its stream + retrying every 30s for the rest of the run, holding a connection slot the + whole time. Note the converse: an interrupted read on a pod that is STILL + RUNNING must not come here. Finalizing then writes a truncated peak and + leaves the range looking measured when it is not. """ # Before discard: on success the archive is about to be deleted. measured = {} @@ -499,6 +518,8 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): _peak_flushed.pop(pod, None) _peak_flushed.pop(pod + '/eph', None) _streaming.pop(pod, None) + # One _wake entry per pod, and pods are per range per attempt: 3979 ranges + # plus their retries would accumulate here for the life of the run. _wake.pop(pod, None) anon = _anon_peak.pop(pod, None) if anon is not None: @@ -739,7 +760,9 @@ async def main(): tasks, terminal, succeeded, vanished = {}, {}, {}, {} # Streams that ran to completion. Without this a finished task is deleted # from `tasks` and the next poll re-opens the stream, forever: one full log - # re-read per pod per cycle, which at 1024 workers is a lot of apiserver. + # re-read per pod every POLL_SECONDS, which at 1024 workers is a lot of + # apiserver -- measured, the completion block ran once per range per cycle + # for the rest of the run. streamed = set() async with aiohttp.ClientSession(connector=conn, timeout=timeout) as session: diff --git a/src/MissionParallelCatchup/tests/contract/_artifacts.py b/src/MissionParallelCatchup/tests/contract/_artifacts.py new file mode 100644 index 00000000..fa29b7ed --- /dev/null +++ b/src/MissionParallelCatchup/tests/contract/_artifacts.py @@ -0,0 +1,185 @@ +"""The artifacts a contract test compares, loaded once per session. + +A contract test pins agreement across a boundary that behaviour cannot reach +from inside Python: the helm chart against the code that reads its env vars, +the RBAC Role against the API calls the code makes, the F# mission driver +against the chart and the monitor it drives, and captured output from +Kubernetes and stellar-core against the parsers that decode it. + +Reading files as text is the point here. The rule that keeps it honest: assert +the INVARIANT, never one spelling of correct code. If a test can only be +satisfied by the exact call that happens to be there today, it will go red over +a correct fix -- which has already happened twice in this suite. +""" + +import functools +import json +import os +import re +import shutil +import subprocess +import sys + +import pytest +import yaml + +HERE = os.path.dirname(os.path.abspath(__file__)) +MODULE_DIR = os.path.dirname(os.path.dirname(HERE)) # src/MissionParallelCatchup +SRC_ROOT = os.path.dirname(MODULE_DIR) # src +CHART = os.path.join(MODULE_DIR, 'parallel_catchup_helm') +FSHARP_PATH = os.path.join(SRC_ROOT, 'FSLibrary', + 'MissionHistoryPubnetParallelCatchupV2.fs') + +# Container names in the monitor Deployment. The collector is a separate +# container with its own env block, so a variable the monitor has is not +# automatically one the collector has -- STORAGE_MODE was missing there once and +# the ephemeral sampler silently did nothing. +MONITOR_CONTAINER = 'job-monitor' +COLLECTOR_CONTAINER = 'log-collector' + + +@functools.lru_cache(maxsize=None) +def text(path): + with open(path) as fh: + return fh.read() + + +def module_source(module): + """The on-disk source of an imported module.""" + return text(module.__file__) + + +def fsharp(): + return text(FSHARP_PATH) + + +def values_yaml(): + return text(os.path.join(CHART, 'values.yaml')) + + +def job_monitor_template(): + return text(os.path.join(CHART, 'templates', 'job_monitor.yaml')) + + +# --- rendering --------------------------------------------------------------- + +# The mission always sends this; the chart has no usable default for it. +_BASE_SET = ('worker.stellar_core_image=x',) + + +@functools.lru_cache(maxsize=None) +def render(sets=(), release='t'): + """`helm template`, as the mission installs it. `sets` must be a tuple.""" + if not shutil.which('helm'): + pytest.skip('helm not installed') + args = ['helm', 'template', release, CHART] + for s in _BASE_SET + tuple(sets): + args += ['--set', s] + r = subprocess.run(args, capture_output=True, text=True) + assert r.returncode == 0, f"helm template failed:\n{r.stderr}" + return r.stdout + + +@functools.lru_cache(maxsize=None) +def docs(sets=(), release='t'): + return tuple(d for d in yaml.safe_load_all(render(sets, release)) if d) + + +def of_kind(kind, sets=(), release='t'): + return [d for d in docs(sets, release) if d.get('kind') == kind] + + +def monitor_deployment(sets=(), release='t'): + found = of_kind('Deployment', sets, release) + assert len(found) == 1, f"expected one Deployment, got {len(found)}" + return found[0] + + +def containers(sets=(), release='t'): + """{name: container} for the monitor Deployment's pod spec.""" + spec = monitor_deployment(sets, release)['spec']['template']['spec'] + return {c['name']: c for c in spec['containers']} + + +def env_of(container): + """{NAME: value} for the env entries that carry a literal value. + + valueFrom entries (NAMESPACE, from the downward API) are reported with a + value of None: they are set, but the chart does not choose the value. + """ + return {e['name']: e.get('value') for e in (container.get('env') or [])} + + +def role_rules(sets=(), release='t'): + found = of_kind('Role', sets, release) + assert len(found) == 1, f"expected one Role, got {len(found)}" + return found[0]['rules'] + + +def granted(sets=(), release='t'): + """{(apiGroup, resource): {verbs}} the monitor's ServiceAccount holds.""" + out = {} + for rule in role_rules(sets, release): + for group in rule['apiGroups']: + for resource in rule['resources']: + out.setdefault((group, resource), set()).update(rule['verbs']) + return out + + +# --- the code's own defaults, read without ambient env ----------------------- + +_PROBE = """ +import json, sys +sys.path.insert(0, {module_dir!r}) +import {module} as m +out = {{}} +for k, v in vars(m).items(): + if k.isupper() and isinstance(v, (int, float, str, bool, type(None))): + out[k] = v +print('<<<' + json.dumps(out) + '>>>') +""" + + +@functools.lru_cache(maxsize=None) +def defaults(module_name, env_pairs=()): + """Module-level UPPERCASE constants as they are with NO env set. + + Read out of a subprocess with a scrubbed environment rather than off the + imported module: the values a test process happens to import depend on + whatever env the developer is running under, and the whole point here is to + compare the chart against the built-in fallback. + + `env_pairs` is a tuple of (name, value) for the few constants that are + derived from an env var at import -- PROGRESS_CM off RUN_NAME, say -- where + the derivation is what a test needs to see. + """ + src = _PROBE.format(module_dir=MODULE_DIR, module=module_name) + env = {'PATH': os.environ.get('PATH', ''), 'HOME': os.environ.get('HOME', '')} + env.update(dict(env_pairs)) + r = subprocess.run([sys.executable, '-c', src], capture_output=True, + text=True, env=env, cwd=MODULE_DIR) + assert r.returncode == 0, f"could not import {module_name} cleanly:\n{r.stderr}" + body = r.stdout[r.stdout.index('<<<') + 3:r.stdout.rindex('>>>')] + return json.loads(body) + + +_GETENV = re.compile(r"^(\w+)\s*=\s*[^\n]*os\.getenv\(\s*'([A-Z_]+)'", re.M) + + +def env_bindings(source): + """{ENV_VAR: module_constant} for every `X = ... os.getenv('ENV'...)`. + + An env var can be read into more than one name -- LOG_DIR feeds both the + exported LOG_DIR and a module-private copy used to place the monitor's own + log file. The exported constant is the one the rest of the module and these + tests can see, so it wins. + """ + out = {} + for name, env in _GETENV.findall(source): + if env not in out or (name.isupper() and not out[env].isupper()): + out[env] = name + return out + + +def reads_env(source): + return set(re.findall(r"os\.getenv\(\s*'([A-Z_]+)'", source)) diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py new file mode 100644 index 00000000..36f2c8c6 --- /dev/null +++ b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py @@ -0,0 +1,201 @@ +"""values.yaml against the os.getenv defaults of the code it configures. + +The chart sets these env vars EXPLICITLY, so the rendered value always wins over +the Python fallback and a drift between them is silent. It has shipped twice: +the code default for the profile cache headroom was raised to 512Mi while the +chart still forced 0, and the chart quietly won -- reproducing the exact OOMs +the code change existed to fix (measured: ranges profiled at 190MiB rss got a +209MiB limit and 90 of them were OOMKilled within 90s of dispatch). + +Rather than a hand-curated pair list -- which only covers the constants someone +remembered -- this compares EVERY env var the chart sets against the code +default of the constant that reads it. Deliberate divergences are listed below +with their reason, so a new one has to be argued for rather than merely added. +""" + +import os +import re + +import job_monitor as jm +import log_collector as lc + +import _artifacts as art + +# Rendered with a profile ConfigMap so the PROFILE_* block is present -- it is +# the block the chart/code split actually bit on. +SETS = ('monitor.profileConfigMap=p',) + +MODULES = { + art.MONITOR_CONTAINER: ('job_monitor', jm), + art.COLLECTOR_CONTAINER: ('log_collector', lc), +} + +# Env vars whose chart value is deliberately NOT the code default. Each one is +# either per-release, per-mission, or a run parameter the mission overrides; in +# every case the code fallback exists only so the module can be imported +# outside a cluster. Nothing here may be a tuning constant. +DELIBERATE = { + 'RUN_NAME': 'the helm release name; the code fallback only names a standalone run', + 'CORE_IMAGE': 'the image under test, supplied per mission run', + 'WORKER_SERVICE_ACCOUNT': 'derived from the release name for IRSA trust', + 'MISSION': 'the mission name, for the kube-state-metrics label', + 'PROFILE_PATH': 'the mounted path of an optional profile ConfigMap', + 'ASAN_OPTIONS': 'passed through to the worker; empty means "unset", not "default"', + 'LATEST_LEDGER_NUM': 'a demo value in the chart; the mission always sets the real tip', + 'PARALLELISM': 'worker.replicas -- the whole point of the knob is to differ per run', + 'ATTEMPT_DEADLINE_SECONDS': 'a backstop the chart turns on and the code leaves off', + # StellarKubeSpecs.fs owns worker sizing, so the chart ships these empty on + # purpose and the mission fills them in on every install. + 'REQ_CPU': 'left empty in the chart; StellarKubeSpecs.fs supplies it', + 'REQ_MEM': 'left empty in the chart; StellarKubeSpecs.fs supplies it', + 'LIM_CPU': 'left empty in the chart; StellarKubeSpecs.fs supplies it', + 'LIM_MEM': 'left empty in the chart; StellarKubeSpecs.fs supplies it', +} + + +def _same(chart_value, code_value): + """Compare a rendered string against a typed Python default. + + Helm renders everything as a string, so `5` and `5.0` and `true` and `True` + all have to compare equal -- the contract is about the VALUE, not about how + YAML happened to spell it. + """ + if isinstance(code_value, bool): + return chart_value.lower() == str(code_value).lower() + if isinstance(code_value, (int, float)): + try: + return float(chart_value) == float(code_value) + except ValueError: + return False + return chart_value == ('' if code_value is None else str(code_value)) + + +def _pairs(): + """(container, env, chart_value, constant, code_default) for each env set.""" + out = [] + for cname, container in art.containers(SETS).items(): + module_name, module = MODULES[cname] + bindings = art.env_bindings(art.module_source(module)) + code = art.defaults(module_name) + for env, chart_value in art.env_of(container).items(): + if chart_value is None: + continue # valueFrom: the chart picks nothing + constant = bindings.get(env) + out.append((cname, env, chart_value, constant, + code.get(constant) if constant else None)) + return out + + +def test_every_env_the_chart_sets_is_read_by_the_container_that_gets_it(): + """A chart env var no module reads is a knob that does nothing. + + That is the same failure as a constant nothing reads: the values.yaml + comment promises a protection, the rendered Deployment carries it, and + turning it changes nothing at all. + """ + orphans = [(c, e) for c, e, _, constant, _ in _pairs() if constant is None] + assert not orphans, ( + "the chart sets env vars nothing reads: " + + ", ".join(f"{e} on {c}" for c, e in orphans)) + + +def test_no_pinned_default_is_a_constant_nothing_reads(): + """Every constant this file pins must be used past its own assignment. + + A contract test guarding a constant no code consults is worse than nothing: + it certifies a protection that does not exist. MAX_LINE_CHARS was exactly + that and was deleted rather than kept. + """ + dead = [] + for cname, container in art.containers(SETS).items(): + module_name, module = MODULES[cname] + source = art.module_source(module) + bindings = art.env_bindings(source) + for env in art.env_of(container): + constant = bindings.get(env) + if constant is None: + continue + uses = len(re.findall(rf"\b{constant}\b", source)) + if uses < 2: + dead.append(f"{module_name}.{constant} (from {env})") + assert not dead, f"assigned from the chart but never read: {dead}" + + +def test_the_chart_value_is_the_code_default(): + """Chart and code must agree wherever the chart is not deliberately different.""" + drift = [] + for cname, env, chart_value, constant, code_value in _pairs(): + if constant is None or env in DELIBERATE: + continue + if not _same(chart_value, code_value): + drift.append(f"{env} on {cname}: chart {chart_value!r} != " + f"code {constant}={code_value!r}") + assert not drift, ( + "the chart overrides the code default with a different value, silently:\n " + + "\n ".join(drift)) + + +def test_each_deliberate_divergence_is_still_a_real_env_var(): + """Keeps the allowlist above honest. + + A renamed or dropped env var must not go on being excused here -- that is + how an exemption written for one variable ends up covering its replacement. + """ + known = {env for _, env, _, _, _ in _pairs()} + stale = sorted(set(DELIBERATE) - known) + assert not stale, f"DELIBERATE excuses env vars the chart no longer sets: {stale}" + + +def test_the_profile_block_only_renders_with_a_profile_configmap(): + """PROFILE_PATH must not be set without the volume that backs it. + + load_profile() treats a non-empty PROFILE_PATH as "there is a profile" and + only an OSError sends it back to the configured requests. Setting the path + with no ConfigMap mounted would make every run log an unreadable-profile + warning for a profile nobody asked for. + """ + without = art.env_of(art.containers()[art.MONITOR_CONTAINER]) + assert 'PROFILE_PATH' not in without + with_cm = art.env_of(art.containers(SETS)[art.MONITOR_CONTAINER]) + assert with_cm['PROFILE_PATH'] + + mounts = {m['mountPath'] + for m in art.containers(SETS)[art.MONITOR_CONTAINER]['volumeMounts']} + assert os.path.dirname(with_cm['PROFILE_PATH']) in mounts, ( + f"PROFILE_PATH={with_cm['PROFILE_PATH']} is not on any mounted volume") + + +def test_the_peak_flush_ratio_is_a_threshold_and_not_a_pass_through(): + """At exactly 1.0 every sample flushes: one write per pod per poll. + + At 2048 pods that is the dominant cost of the sampler, and the ratio exists + to avoid it -- so agreeing with the chart is not enough, it also has to be + above 1. Nothing else pins this: the behaviour tests inject their own ratio. + """ + ratio = art.defaults('log_collector')['PEAK_FLUSH_RATIO'] + assert ratio > 1.0, f"ratio {ratio} flushes on every sample" + + +def test_the_sizing_headroom_is_a_real_allowance_in_both_places(): + """margin and headroom bound each other; neither may be inert. + + A margin below 1.0 shrinks a measured peak, and a headroom of 0 was + measured to OOM 90 small ranges within 90s of dispatch -- memory.max bounds + anon PLUS page cache, so a purely multiplicative margin is meaningless at + small rss (190MiB rss * 1.1 is 19MiB of slack). + + The exact figures are pinned by the test above, against the chart. This one + says what they must remain true of, so retuning them stays possible and + zeroing them does not. + """ + code = art.defaults('job_monitor') + assert code['PROFILE_MARGIN'] >= 1.0, "a margin below 1.0 sizes under the measured peak" + headroom = jm._quantity_bytes(code['PROFILE_CACHE_HEADROOM']) + assert headroom >= 256 * 1024 ** 2, ( + f"{code['PROFILE_CACHE_HEADROOM']} of fixed headroom is what OOMed 90 small ranges") + # ...and the ceiling has to sit above the configured worker limit, or a + # range needing more than that is pinned under its own measured peak. + assert (jm._quantity_bytes(code['PROFILE_MAX_MEM']) + > jm._quantity_bytes(code['LIM_MEM'])), ( + "the profile ceiling is at or below the worker limit, so a hungry range " + "can never ask for what it measured") diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_rbac.py b/src/MissionParallelCatchup/tests/contract/test_chart_rbac.py new file mode 100644 index 00000000..86f45157 --- /dev/null +++ b/src/MissionParallelCatchup/tests/contract/test_chart_rbac.py @@ -0,0 +1,165 @@ +"""The Role the chart grants against the API calls the two processes make. + +This boundary has failed twice, the same way both times, and both times it was +silent: the Role omitted `delete` on persistentvolumeclaims, so every completed +range logged a 403 warning and leaked its 40Gi volume (measured on ssc-test: +2032 bound PVCs and 79 TiB a third of the way through a 3982-range run, heading +for ~156 TiB -- enough to crash the EBS CSI controller); and it omitted `delete` +on jobs, so nothing reaped a finished Job and the dead ones outnumbered the live +ones within the hour. + +A 403 does not stop the run. That is the whole problem, and it is why this is +checked statically rather than waiting for a cluster to tell us. + +The required verbs are DERIVED from the calls in the source, not listed here: +adding a new call to the monitor must fail this test until the Role catches up. +""" + +import re + +import pytest + +import job_monitor as jm +import log_collector as lc + +import _artifacts as art + +# kubernetes-client method names are `_namespaced_`. Only the +# mapping from client vocabulary to RBAC vocabulary is spelled out; which calls +# exist is read off the source. +VERB = {'read': 'get', 'list': 'list', 'create': 'create', 'delete': 'delete', + 'patch': 'patch', 'replace': 'update', 'watch': 'watch'} + +RESOURCE = { + 'job': ('batch', 'jobs'), + 'pod': ('', 'pods'), + 'pod_log': ('', 'pods/log'), + 'config_map': ('', 'configmaps'), + 'persistent_volume_claim': ('', 'persistentvolumeclaims'), +} + +_CALL = re.compile(r"\b(?:core_v1|batch_v1)\.(\w+?)_namespaced_(\w+)\(") + + +def monitor_calls(): + """{(apiGroup, resource): {verbs}} the monitor's own code needs.""" + need = {} + for verb, resource in _CALL.findall(art.module_source(jm)): + assert verb in VERB, f"unmapped client verb {verb!r}" + assert resource in RESOURCE, f"unmapped client resource {resource!r}" + need.setdefault(RESOURCE[resource], set()).add(VERB[verb]) + return need + + +def test_the_source_really_does_call_the_apiserver(): + """Guards the derivation itself. + + If the call regex stopped matching -- a rename, a wrapper, a different + client object -- every assertion below would pass vacuously while granting + nothing. + """ + need = monitor_calls() + assert len(need) >= 4, f"only found {sorted(need)}; the call scan has gone blind" + assert ('batch', 'jobs') in need and ('', 'persistentvolumeclaims') in need + + +def test_the_role_grants_every_verb_the_monitor_uses(): + have = art.granted() + missing = [] + for key, verbs in sorted(monitor_calls().items()): + for verb in sorted(verbs): + if verb not in have.get(key, set()): + missing.append(f"{verb} on {key[1]} (Role has {sorted(have.get(key, ()))})") + assert not missing, ( + "the monitor makes API calls the Role does not allow; each one is a 403 " + "the run swallows:\n " + "\n ".join(missing)) + + +def test_the_role_grants_what_the_collector_reads(): + """The collector shares the monitor's ServiceAccount -- same pod, same SA. + + It talks to the apiserver over raw HTTP rather than the client library, so + its needs are read out of the URLs it builds. + """ + source = art.module_source(lc) + have = art.granted() + assert re.search(r"/api/v1/namespaces/\{NAMESPACE\}/pods\"", source), \ + "the collector no longer lists pods -- update this test" + assert 'list' in have[('', 'pods')] + assert re.search(r"/pods/\{pod\}/log\"", source), \ + "the collector no longer reads pod logs -- update this test" + assert 'get' in have[('', 'pods/log')] + + +@pytest.mark.xfail(strict=True, reason=( + "GAP: the collector's peak sampler GETs /api/v1/nodes//proxy/stats/summary, " + "which needs `get` on nodes/proxy -- a CLUSTER-scoped resource that a namespaced " + "Role cannot carry however it is spelled. The chart ships no ClusterRole, so every " + "peak this mission profiles from depends on a grant that lives outside this repo. " + "Where that grant is absent the failure is soft and invisible: sample_kubelet logs " + "'kubelet stats unavailable' and continues, peakAnonBytes and peakEphemeralBytes " + "stay empty for the whole run, and the next run's profile looks merely absent " + "rather than broken. Closing it means a ClusterRole plus binding in the chart")) +def test_the_chart_grants_the_kubelet_stats_read_the_sampler_needs(): + source = art.module_source(lc) + assert '/nodes/{node}/proxy/stats/summary' in source, \ + "the sampler no longer proxies to the kubelet -- drop this xfail" + # Cluster-scoped, so a Role cannot carry it however it is spelled. + cluster_roles = art.of_kind('ClusterRole') + granted = {(g, r) + for role in cluster_roles + for rule in role['rules'] + for g in rule['apiGroups'] + for r in rule['resources'] + if 'get' in rule['verbs']} + assert ('', 'nodes/proxy') in granted + + +def test_the_monitor_cannot_touch_a_persistent_volume(): + """Namespaced and PV-free by design. + + Deleting a PVC is reclaim; touching a PV or its finalizers is how an EBS + volume gets orphaned or a VolumeAttachment gets wedged. The blast radius of + a bug in this monitor has to stop at the namespace. + """ + forbidden = {'persistentvolumes', 'nodes', 'volumeattachments'} + reachable = {resource for (_, resource) in art.granted()} + assert not (forbidden & reachable), \ + f"the monitor's Role reaches cluster storage: {sorted(forbidden & reachable)}" + assert not art.of_kind('ClusterRoleBinding'), \ + "a ClusterRoleBinding takes this ServiceAccount outside its namespace" + + +def test_the_role_is_bound_to_the_service_account_the_monitor_runs_as(): + """A Role nobody is bound to grants nothing, and renders perfectly. + + The worker ServiceAccount is deliberately a different one -- IRSA trust for + the S3 history mirror is bound to its name -- so "there is a binding" is not + enough; it has to name the SA on the monitor pod. + """ + binding = art.of_kind('RoleBinding') + assert len(binding) == 1, f"expected one RoleBinding, got {len(binding)}" + binding = binding[0] + role = art.of_kind('Role')[0] + assert binding['roleRef']['name'] == role['metadata']['name'] + subjects = {s['name'] for s in binding['subjects'] if s['kind'] == 'ServiceAccount'} + running_as = art.monitor_deployment()['spec']['template']['spec']['serviceAccountName'] + assert running_as in subjects, ( + f"the monitor runs as {running_as!r} but the Role is bound to {sorted(subjects)}") + assert running_as in {sa['metadata']['name'] for sa in art.of_kind('ServiceAccount')} + + +def test_the_worker_service_account_is_not_the_monitors(): + """Workers must not inherit the monitor's Job/PVC/ConfigMap rights. + + A worker is stellar-core running an untrusted history archive's bytes; the + only credential it needs is IRSA for the S3 mirror. + """ + env = art.env_of(art.containers()[art.MONITOR_CONTAINER]) + worker_sa = env['WORKER_SERVICE_ACCOUNT'] + monitor_sa = art.monitor_deployment()['spec']['template']['spec']['serviceAccountName'] + assert worker_sa != monitor_sa + assert worker_sa in {sa['metadata']['name'] for sa in art.of_kind('ServiceAccount')}, \ + "the workers' ServiceAccount is named but never created" + bound = {s['name'] for b in art.of_kind('RoleBinding') for s in b['subjects']} + assert worker_sa not in bound, "workers were granted the monitor's Role" diff --git a/src/MissionParallelCatchup/tests/contract/test_cross_process_files.py b/src/MissionParallelCatchup/tests/contract/test_cross_process_files.py new file mode 100644 index 00000000..1548a948 --- /dev/null +++ b/src/MissionParallelCatchup/tests/contract/test_cross_process_files.py @@ -0,0 +1,200 @@ +"""Two processes, one volume, one set of filenames. + +The monitor and the collector never talk. Everything they agree on is a file on +the shared /logs PVC: the archive, the per-attempt metrics, the verdict, the +resume bookkeeping, and the marker that licenses the monitor to reap a Job. A +disagreement about any of those names is silent -- the reader simply finds +nothing, which reads as "not measured yet" and never as "broken". + +The names are compared by calling both sides' path functions against the same +LOG_DIR, so a refactor that keeps the layout is free. The chart is checked too: +the layout only means anything if both containers mount the same volume there. +""" + +import gzip +import os + +import pytest + +import job_monitor as jm +import log_collector as lc + +import _artifacts as art + +END, ATTEMPT = '31005951', 2 + + +@pytest.fixture +def shared(tmp_path, monkeypatch): + """Both modules pointed at one directory, as the pod's volume gives them.""" + monkeypatch.setattr(jm, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) + return tmp_path + + +# --- the filenames ------------------------------------------------------------ + +def test_both_processes_name_the_same_metrics_file(shared): + """The collector writes it; the monitor reads peaks and txApply out of it.""" + assert jm.metrics_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.metrics' + + +def test_both_processes_name_the_same_done_marker(shared): + """The marker is the collector's "I am finished with this attempt". + + The monitor will not reap a Job without it -- and reaping deletes the pod, + which is the last place peaks can still be read from. A mismatch means the + monitor never reaps and every Job waits out its TTL instead. + """ + assert jm.done_path(END, ATTEMPT) == lc.done_path(END, ATTEMPT) + + +def test_both_processes_name_the_same_archive_and_verdict(shared): + """The monitor falls back to the archive for txApply and reads .outcome for + the authoritative verdict; the collector writes both.""" + assert jm.log_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.log.gz' + assert jm.outcome_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.outcome' + assert jm.state_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.state' + + +def test_the_filenames_carry_the_attempt_as_well_as_the_range(shared): + """Peaks are maxed across a resumed chain, per attempt. + + With one file per range, a retry would overwrite its predecessor instead of + being compared against it -- which destroys exactly the OOM evidence the + chain exists to keep. + """ + for path in (jm.metrics_path, jm.log_path, jm.outcome_path, jm.done_path): + assert path(END, 1) != path(END, 2) + assert path('1', 1) != path('2', 1) + + +def test_discarding_a_successful_archive_keeps_what_is_still_read(shared): + """saveSuccessLogs=false drops the bulk of the volume, not the measurements. + + .metrics holds txApply for a range that succeeded, and .done is what lets + the Job be reaped at all. Dropping either would let a log-retention flag + silently delete a Grafana series or strand every finished Job on its TTL. + """ + for suffix in ('.log.gz', '.state', '.metrics', '.done'): + with open(lc.base(END, ATTEMPT) + suffix, 'w') as fh: + fh.write('x') + lc.discard(END, ATTEMPT) + + assert os.path.exists(jm.metrics_path(END, ATTEMPT)), "discard dropped the measurements" + assert os.path.exists(jm.done_path(END, ATTEMPT)), "discard dropped the reap marker" + assert not os.path.exists(jm.log_path(END, ATTEMPT)), "discard kept the archive" + + +def test_the_monitor_can_read_an_archive_the_collector_wrote(shared): + """gzip, appended member by member, read whole. + + The monitor reads it with gzip.open() for the txApply fallback. A writer + that produced anything other than a concatenation of complete members would + give it a truncated read -- which it treats as "this range has no metric". + """ + path = lc.base(END, ATTEMPT) + '.log.gz' + for chunk in ("first line\n", "metric 'ledger.transaction.apply'\n", + " sum = 8.34285ms\n"): + with gzip.open(path, 'ab') as fh: + fh.write(chunk.encode()) + assert jm._tx_apply_for_attempt(END, ATTEMPT) == pytest.approx(0.00834285) + + +def test_a_carriage_return_meter_does_not_become_one_giant_line(shared, monkeypatch): + """The AWS CLI draws its transfer meter with \\r and no newline. + + A 628 MiB bucket download therefore arrives as one multi-megabyte "line". + The mission passes --no-progress to stop it at the source (see + test_fsharp_driver_contract), but the collector must not be the only thing + standing between a \\r-heavy line and its own stream: splitting on \\r as + well as \\n is what keeps the archive line-oriented for the monitor's + reader, whatever the worker emits. + """ + import asyncio + + body = ("2026-07-30T00:00:01Z Completed 1.0 MiB\r" + "2026-07-30T00:00:02Z Completed 2.0 MiB\r" + "2026-07-30T00:00:03Z metric 'ledger.transaction.apply'\n" + "2026-07-30T00:00:04Z sum = 1500.0ms\n") + + class _Resp: + status = 200 + async def __aenter__(self): return self + async def __aexit__(self, *exc): return False + def raise_for_status(self): pass + @property + def content(self): + data = body.encode() + class _C: + async def iter_chunked(self, n): + for i in range(0, len(data), n): + yield data[i:i + n] + return _C() + + class _Session: + def get(self, url, params=None, headers=None): + return _Resp() + + monkeypatch.setattr(lc, 'token', lambda: 't') + scanner = lc.TxApplyScanner() + asyncio.run(lc._poll_once(_Session(), 'pod-1', END, ATTEMPT, None, scanner)) + + assert scanner.seconds == pytest.approx(1.5), \ + "the metric block was swallowed by the meter's unterminated line" + with gzip.open(lc.base(END, ATTEMPT) + '.log.gz', 'rt') as fh: + lines = fh.read().splitlines() + assert len(lines) >= 4, f"the meter stayed one blob: {lines}" + + +# --- the volume the layout lives on ------------------------------------------ + +def test_both_containers_mount_one_volume_at_the_directory_they_both_use(): + """The filenames only agree if the directory does. + + Two emptyDirs would render identically and share nothing; a volume mounted + at a different path in each container would give each process its own + private copy of every measurement. + """ + log_dir = art.defaults('job_monitor')['LOG_DIR'] + assert log_dir == art.defaults('log_collector')['LOG_DIR'] + + mounts = {} + for name, container in art.containers().items(): + by_path = {m['mountPath']: m['name'] for m in container['volumeMounts']} + assert log_dir in by_path, f"{name} does not mount {log_dir}" + mounts[name] = by_path[log_dir] + assert len(set(mounts.values())) == 1, ( + f"the two containers mount different volumes at {log_dir}: {mounts}") + + volume = mounts[art.MONITOR_CONTAINER] + spec = art.monitor_deployment()['spec']['template']['spec'] + backing = {v['name']: v for v in spec['volumes']}[volume] + assert 'persistentVolumeClaim' in backing, ( + f"{log_dir} is backed by {sorted(backing)} -- every measurement dies with the pod") + + +def test_the_chart_tells_both_containers_where_that_directory_is(): + """LOG_DIR is env, not a constant, so the mount and the env must agree.""" + log_dir = art.defaults('job_monitor')['LOG_DIR'] + for name, container in art.containers().items(): + assert art.env_of(container)['LOG_DIR'] == log_dir, name + + +def test_the_progress_record_lives_on_that_volume_too(): + """progress.json is what a restarted monitor reads back, and what the + mission driver `cat`s out of the pod at teardown. + + Written to the monitor's emptyDir instead, an OOM-retry storm's record + would not survive a monitor restart and the mission would build its range + profile from the ConfigMap mirror -- which has every measurement stripped. + """ + assert os.path.dirname(jm.PROGRESS_FILE) == jm.LOG_DIR + + +def test_the_shared_directory_is_a_single_writer_pvc(): + """One archive per attempt for thousands of ranges, outliving the pod.""" + claims = art.of_kind('PersistentVolumeClaim') + assert len(claims) == 1, "the monitor's log volume is not a PVC" + assert claims[0]['spec']['accessModes'] == ['ReadWriteOnce'], ( + "two writers on one archive; the layout assumes a single collector") diff --git a/src/MissionParallelCatchup/tests/contract/test_k8s_failure_formats.py b/src/MissionParallelCatchup/tests/contract/test_k8s_failure_formats.py new file mode 100644 index 00000000..4ee87491 --- /dev/null +++ b/src/MissionParallelCatchup/tests/contract/test_k8s_failure_formats.py @@ -0,0 +1,314 @@ +"""Captured Kubernetes status text against the classifiers that decode it. + +None of these strings are ours. The Job controller's podFailurePolicy condition +message, the kubelet's admission-rejection reasons, its eviction message and the +plain text its log endpoint returns for a container that has not started are all +formats a Kubernetes upgrade can change under us. They are pinned from real +captures so that change fails here rather than degrading a run silently -- a +misread verdict does not stop anything, it just picks the wrong retry budget or +condemns a healthy range. + +Everything is driven through the real classify()/classify_from_job() rather than +a mirror of them, so only the FORMAT is pinned, not the implementation. +""" + +import re +from types import SimpleNamespace as NS + +import pytest + +import job_monitor as jm +import log_collector as lc + +import _artifacts as art + +# --- captures ---------------------------------------------------------------- + +# EKS 1.34 Job condition messages. Only the wording is pinned; pod and container +# names are renamed for readability. +DISRUPTED = ("Pod sandbox/jterm-catchup-snfr2 has condition DisruptionTarget " + "matching FailJob rule at index 0") +OOMKILLED = ("Container oom-container for pod sandbox/oom-test-job-qvq8b failed with " + "exit code 137 matching FailJob rule at index 1") +NONZERO_EXIT = ("Container exit-1-container for pod sandbox/exit-1-job-wbhkq failed with " + "exit code 1 matching FailJob rule at index 2") + +# RECONSTRUCTED 2026-07-30 after an over-broad test deletion removed the +# originals -- twice. Shaped to what the code parses (the rule index, and the +# substring 'ephemeral' in status.message) but no longer a verbatim capture. +# Re-pin from a real eviction on the next run. +EPH_EVICT_JOB_CONDITION = ( + "Container stellar-core for pod stellar-supercluster/" + "parallel-catchup-r31005951-a1-x7k2p failed with exit code 3 " + "matching FailJob rule at index 2") +EPH_EVICT_MESSAGE = ( + "Pod ephemeral local storage usage exceeds the total limit of containers 40Gi") + +# Kubelet reasons seen on ssc-test for a pod refused or removed before -- or +# without -- the container saying anything about the ledger range. None of these +# is evidence that the range is bad. +ADMISSION_REJECTIONS = ('VolumeAttachmentLimitExceeded', 'OutOfcpu', 'OutOfmemory', + 'OutOfpods', 'UnexpectedAdmissionError', 'NodeAffinity', + 'Shutdown', 'Evicted') + + +# --- shims: the two shapes the classifiers read ------------------------------ + +def failed_job(message, reason='PodFailurePolicy'): + return NS(status=NS(conditions=[ + NS(type='Failed', status='True', reason=reason, message=message)])) + + +def pod(reason=None, message=None, disrupted=False, exit_code=None, + terminated_reason=None): + conditions = ([NS(type='DisruptionTarget', status='True')] if disrupted else []) + statuses = [] + if exit_code is not None or terminated_reason is not None: + statuses = [NS(state=NS(terminated=NS(exit_code=exit_code, + reason=terminated_reason)))] + return NS(metadata=NS(name='p'), + status=NS(conditions=conditions, reason=reason, message=message, + container_statuses=statuses)) + + +# --- the Job condition, which is all that is left once the pod is gone ------- + +@pytest.mark.parametrize('message,outcome,code,pod_name', [ + (DISRUPTED, 'disrupted', None, ''), + (OOMKILLED, 'oom', 137, 'oom-test-job-qvq8b'), + (NONZERO_EXIT, 'failed', 1, 'exit-1-job-wbhkq'), +]) +def test_a_job_condition_message_still_parses(message, outcome, code, pod_name): + """Index, exit code and pod name are parsed independently. + + A rule matching on onPodConditions reports no exit code at all, so requiring + one would make the disruption case -- the common case on spot -- unreadable. + """ + verdict = jm.classify_from_job(failed_job(message)) + assert verdict['outcome'] == outcome + assert verdict['exitCode'] == code + assert verdict['pod'] == pod_name + + +def test_the_rule_index_outranks_the_exit_code(): + """A disrupted pod that also exited non-zero must read as disrupted. + + stellar-core catches the eviction SIGTERM and exits 3, so the exit code says + "failed" for something the cluster did to us. Only the index carries the + DisruptionTarget match. + """ + message = ("Container stellar-core for pod ns/p failed with exit code 3 " + "matching FailJob rule at index 0") + assert jm.classify_from_job(failed_job(message))['outcome'] == 'disrupted' + + +def test_a_bare_exit_code_with_no_index_is_still_usable(): + """Some conditions carry the exit code and no rule index. + + Measured on ssc-test 2026-07-28: a drained stellar-core exits 3 in ~7s, well + inside the 100s grace, so evictions do NOT produce 137 -- which makes a bare + 137 an OOM with high confidence, and a bare 3 a real catchup failure. + """ + bare = "Container c for pod ns/p failed with exit code %d" + assert jm.classify_from_job(failed_job(bare % 137))['outcome'] == 'oom' + assert jm.classify_from_job(failed_job(bare % 3))['outcome'] == 'failed' + + +def test_a_condition_with_no_detail_at_all_yields_no_verdict(): + """BackoffLimitExceeded carries no index and no exit code. + + Returning a verdict here would be an invention. "No verdict" is what routes + the range to the environmental budget instead of condemning it -- a monitor + restart while a node was reaped produces exactly this message, and + condemning on it would fail a 10-hour job on no evidence. + """ + assert jm.classify_from_job( + failed_job("Job has reached the specified backoff limit", + reason='BackoffLimitExceeded')) is None + + +def test_the_deadline_is_reported_by_the_job_and_nothing_else(): + """activeDeadlineSeconds fires as its own reason, not as a policy rule. + + The pod that gets SIGTERMed drains and exits 3, which reads as a plain + catchup failure. Only the Job knows the deadline was what killed it. + """ + verdict = jm.classify_from_job( + failed_job("Job was active longer than specified deadline", + reason='DeadlineExceeded')) + assert verdict['outcome'] == 'timeout' + assert verdict['exitCode'] is None + + +# --- the pod, which carries everything the Job cannot ------------------------ + +@pytest.mark.parametrize('reason', ADMISSION_REJECTIONS) +def test_an_admission_rejection_is_not_a_catchup_failure(reason): + """The kubelet refused the pod; stellar-core never ran. + + Observed on ssc-test: reason=VolumeAttachmentLimitExceeded, "Node has + reached its volume attachment limit, rejecting pod". Without this the pod + falls through to 'failed', and a transient admission rejection condemns a + range and kills the whole run. + """ + assert jm.classify(pod(reason=reason))['outcome'] == 'rejected' + + +def test_an_ephemeral_eviction_is_told_apart_from_every_other_eviction(): + """status.message is the only discriminator, and only the pod carries it. + + Measured end-to-end on ssc-test: the kubelet sets no DisruptionTarget for a + limit eviction, and stellar-core drains and exits 3 -- so the Job condition + matches the generic non-zero rule and reads as a plain catchup failure, + which gets no retry at all. Both the ephemeral branch and the generic + Evicted branch key on reason='Evicted', so the ephemeral one has to be + reached first. + """ + assert 'index 2' in EPH_EVICT_JOB_CONDITION, "the Job matches the generic non-zero rule" + assert jm.classify_from_job(failed_job(EPH_EVICT_JOB_CONDITION))['outcome'] == 'failed' + + evicted = pod(reason='Evicted', message=EPH_EVICT_MESSAGE, exit_code=3) + assert jm.classify(evicted)['outcome'] == 'ephemeral' + # ...and an eviction for anything else stays a plain rejection. + other = pod(reason='Evicted', message='The node was low on resource: memory.', + exit_code=3) + assert jm.classify(other)['outcome'] == 'rejected' + + +def payload(reason=None, message=None, disrupted=False, exit_code=None, + terminated_reason=None): + """The same pod as pod(), in the raw JSON shape the collector reads. + + The collector talks to the apiserver over plain HTTP and classifies a dict; + the monitor classifies a client object. Same pod, two spellings. + """ + status = {} + if disrupted: + status['conditions'] = [{'type': 'DisruptionTarget', 'status': 'True'}] + if reason is not None: + status['reason'] = reason + if message is not None: + status['message'] = message + if exit_code is not None or terminated_reason is not None: + term = {} + if exit_code is not None: + term['exitCode'] = exit_code + if terminated_reason is not None: + term['reason'] = terminated_reason + status['containerStatuses'] = [{'state': {'terminated': term}}] + return {'metadata': {'name': 'p'}, 'status': status} + + +CLASSIFIER_CASES = [ + ('a spot reclaim', dict(disrupted=True, exit_code=3)), + ('a disk eviction', dict(reason='Evicted', message=EPH_EVICT_MESSAGE, exit_code=3)), + ('any other eviction', dict(reason='Evicted', message='node was low on memory')), + ('an admission rejection', dict(reason='VolumeAttachmentLimitExceeded')), + ('an oom kill', dict(exit_code=137, terminated_reason='OOMKilled')), + ('a graceful-stop sigkill', dict(exit_code=137, terminated_reason='Error')), + ('a catchup failure', dict(exit_code=1)), + ('an interrupted catchup', dict(exit_code=3)), + ('nothing ever ran', dict()), +] + + +@pytest.mark.parametrize('label,case', CLASSIFIER_CASES, ids=[c[0] for c in CLASSIFIER_CASES]) +def test_both_processes_reach_the_same_verdict_about_the_same_pod(label, case): + """Two independent classifiers, one pod, one answer. + + The collector classifies while the pod still exists and writes the + authoritative .outcome; the monitor classifies again at reconcile when that + file is missing. If they disagreed, a range's verdict -- and therefore which + attempt budget it spends -- would depend on which process saw it first. + """ + from_monitor = jm.classify(pod(**case)) + from_collector = lc.classify(payload(**case)) + assert from_monitor['outcome'] == from_collector['outcome'], label + assert from_monitor['exitCode'] == from_collector['exitCode'], label + + +def test_a_disruption_beats_everything_the_pod_says(): + """A spot reclaim sets the condition and the container still exits 3.""" + assert jm.classify(pod(disrupted=True, exit_code=3))['outcome'] == 'disrupted' + + +def test_an_oom_kill_is_named_by_the_kubelet_not_inferred_from_137(): + """137 is SIGKILL, which the kubelet also uses for a graceful-stop timeout. + + On the pod the reason is available and unambiguous, so it is used; only the + Job-condition path has to infer from the code alone. + """ + assert jm.classify(pod(exit_code=137, terminated_reason='OOMKilled'))['outcome'] == 'oom' + assert jm.classify(pod(exit_code=137, terminated_reason='Error'))['outcome'] == 'failed' + + +def test_a_pod_whose_container_never_terminated_is_not_evidence(): + """Nothing ran, so nothing was learned about the ledger range.""" + assert jm.classify(pod())['outcome'] == 'rejected' + + +def test_the_pod_deadline_is_a_timeout_not_a_catchup_failure(): + """The deadline lives on the PodSpec, so the kubelet fires it and the pod + carries the reason -- the Job only sees a non-zero exit.""" + assert jm.classify(pod(reason='DeadlineExceeded', exit_code=3))['outcome'] == 'timeout' + + +# --- the vocabulary both classifiers speak ----------------------------------- + +def test_every_outcome_the_classifiers_can_produce_has_a_budget(): + """A new outcome string with no branch falls through to "condemn". + + Each outcome routes to one of three attempt budgets. An outcome nobody + routed would take the zero-retry path, and a condemned range fails the + mission. + """ + produced = set() + for source in (art.module_source(jm), art.module_source(lc)): + produced |= set(re.findall(r"'outcome':\s*'(\w+)'", source)) + budgeted = set(jm.ENVIRONMENTAL_OUTCOMES) | {'oom', 'ephemeral', 'timeout', 'failed'} + assert produced <= budgeted, f"unrouted outcomes: {sorted(produced - budgeted)}" + assert 'disrupted' in produced and 'rejected' in produced + + +def test_the_deterministic_failures_do_not_get_the_environmental_budget(): + """Environmental means "the cluster did this to us" and gets ~20 attempts. + + An OOM, a disk eviction, a hang and a genuinely corrupt range are all + statements about the range; giving them 20 attempts would park a node on a + broken range for hours. 'unknown' IS environmental on purpose -- an + unclassifiable failure is usually a monitor restart racing a reaped node. + """ + environmental = set(jm.ENVIRONMENTAL_OUTCOMES) + assert not (environmental & {'oom', 'ephemeral', 'timeout', 'failed'}), \ + f"a deterministic failure inherited the disruption budget: {sorted(environmental)}" + assert {'disrupted', 'rejected', 'unknown'} <= environmental + + +# --- the log endpoint, which does not always return log lines ---------------- + +def test_untimestamped_kubelet_text_never_becomes_a_resume_point(): + """A pod that has just been replaced returns prose, not log lines. + + Partitioning that on the first space yields "unable", which as a resume + point makes every later request sinceTime=unableZ -> HTTP 400, for the life + of the range. Observed on ssc-test the moment evicted pods were replaced. + """ + kubelet = "unable to retrieve container logs for containerd://9f2c1a" + assert lc._TS_RE.match(kubelet.partition(' ')[0]) is None + for good in ("2026-07-28T20:29:27.927795721Z", "2026-07-28T20:29:27Z"): + assert lc._TS_RE.match(good), good + + +def test_a_poisoned_state_file_is_repaired_rather_than_replayed(tmp_path, monkeypatch): + """The guard has to be on the READ as well as the write. + + A state file written by an earlier build already holds "unable" on some + volumes, and nothing rewrites it until a poll succeeds -- which it cannot, + because the poisoned value is what makes the poll 400. + """ + monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) + with open(lc.base('300', 1) + '.state', 'w') as fh: + fh.write('unable') + assert lc.read_state('300', 1) is None + lc.write_state('300', 1, '2026-07-28T20:29:27Z') + assert lc.read_state('300', 1) == '2026-07-28T20:29:27Z' diff --git a/src/MissionParallelCatchup/tests/contract/test_medida_metric_block.py b/src/MissionParallelCatchup/tests/contract/test_medida_metric_block.py new file mode 100644 index 00000000..d22a7383 --- /dev/null +++ b/src/MissionParallelCatchup/tests/contract/test_medida_metric_block.py @@ -0,0 +1,198 @@ +"""stellar-core's medida metric block against the two parsers that read it. + +txApply is the only per-range performance number this mission produces, and it +exists in exactly one place: the block stellar-core prints once, just before +exit, because we pass --metric 'ledger.transaction.apply'. Both processes parse +it -- the collector out of the live stream (the only reader guaranteed to see +the bytes, since the pod may be reaped and saveSuccessLogs may be off) and the +monitor out of the archive as a fallback. Two parsers, one format. + +The blocks below are whole captures rather than the lines we care about: the +layout IS the contract. `sum` sits ten lines under the header, against a +fifteen-line scan window, so a medida release that adds five percentiles takes +the metric out silently. +""" + +import gzip +import re + +import pytest + +import job_monitor as jm +import log_collector as lc + + +# stellar-core 27.1.1 catchup pod, --metric 'ledger.transaction.apply'. +MEDIDA_BLOCK = """2026-07-28T18:39:49.350 GAJSL [default INFO] metric 'ledger.transaction.apply': +2026-07-28T18:39:49.350 GAJSL [default INFO] count = 20 +2026-07-28T18:39:49.350 GAJSL [default INFO] mean rate = 0.22136 calls/s +2026-07-28T18:39:49.350 GAJSL [default INFO] 1-minute rate = 0.113149 calls/s +2026-07-28T18:39:49.350 GAJSL [default INFO] 5-minute rate = 0.175948 calls/s +2026-07-28T18:39:49.350 GAJSL [default INFO] 15-minute rate = 0.191421 calls/s +2026-07-28T18:39:49.350 GAJSL [default INFO] min = 0.295417ms +2026-07-28T18:39:49.350 GAJSL [default INFO] max = 0.639873ms +2026-07-28T18:39:49.350 GAJSL [default INFO] mean = 0.417143ms +2026-07-28T18:39:49.350 GAJSL [default INFO] stddev = 0.108677ms +2026-07-28T18:39:49.350 GAJSL [default INFO] sum = 8.34285ms +2026-07-28T18:39:49.350 GAJSL [default INFO] median = 0ms +2026-07-28T18:39:49.350 GAJSL [default INFO] 75% = 0ms +2026-07-28T18:39:49.350 GAJSL [default INFO] 95% = 0ms +2026-07-28T18:39:49.350 GAJSL [default INFO] 98% = 0ms +2026-07-28T18:39:49.350 GAJSL [default INFO] 99% = 0ms +2026-07-28T18:39:49.350 GAJSL [default INFO] 99.9% = 0ms""" + +TX_APPLY_SECONDS = 0.00834285 + +# Real block from range-40010367-a1 on ssc-test. medida switches to scientific +# notation past 1e6 ms, which is every range with a real transaction load. The +# old [0-9.]+ pattern matched "1.30722", then demanded "ms" and hit "e+06ms" +# instead: 25% of ranges recorded no tx_apply -- 91-99% of everything above +# ledger 35M, exactly the expensive end -- while the block sat in the archive +# the whole time. +MEDIDA_BIG = """2026-07-29T20:11:16.931 GAJSL [default INFO] metric 'ledger.transaction.apply': +2026-07-29T20:11:16.931 GAJSL [default INFO] count = 3231886 +2026-07-29T20:11:16.931 GAJSL [default INFO] mean rate = 812.4 calls/s +2026-07-29T20:11:16.931 GAJSL [default INFO] 1-minute rate = 790.1 calls/s +2026-07-29T20:11:16.931 GAJSL [default INFO] 5-minute rate = 801.3 calls/s +2026-07-29T20:11:16.931 GAJSL [default INFO] 15-minute rate = 799.0 calls/s +2026-07-29T20:11:16.931 GAJSL [default INFO] min = 0.101ms +2026-07-29T20:11:16.931 GAJSL [default INFO] max = 41.2ms +2026-07-29T20:11:16.931 GAJSL [default INFO] mean = 0.404ms +2026-07-29T20:11:16.931 GAJSL [default INFO] stddev = 0.612ms +2026-07-29T20:11:16.931 GAJSL [default INFO] sum = 1.30722e+06ms""" + +TX_APPLY_BIG_SECONDS = 1307.22 + + +def scan(block): + scanner = lc.TxApplyScanner() + for line in block.splitlines(): + scanner.feed(line) + return scanner + + +# --- the layout, which is what the scan window is sized against -------------- + +def test_the_sum_still_sits_inside_the_scan_window(): + """Ten lines below the header, against a fifteen-line window. + + Five more percentiles in a medida release and the metric disappears with no + error anywhere. The margin is the thing to watch, so it is reported. + """ + lines = MEDIDA_BLOCK.splitlines() + header = next(i for i, l in enumerate(lines) if 'ledger.transaction.apply' in l) + offset = next(i for i, l in enumerate(lines) if 'sum =' in l) - header + assert offset == 10, f"medida layout moved: sum is now {offset} lines below the header" + assert offset <= lc.TxApplyScanner.WINDOW, ( + f"the sum is {offset} lines down and the scanner looks {lc.TxApplyScanner.WINDOW}") + + +@pytest.mark.parametrize('gap', [1, 10, lc.TxApplyScanner.WINDOW, + lc.TxApplyScanner.WINDOW + 5]) +def test_both_readers_reach_exactly_as_far_past_the_header(gap, tmp_path, monkeypatch): + """The collector scans the stream; the monitor scans the archive. + + Two separate implementations of "find the sum under this header". A reach + that differed between them would make the metric depend on which reader got + to it -- and the monitor's read is the one that happens when the collector + was down for the pod's lifetime. + + Asserted by measuring both, at the boundary and past it, rather than by + comparing two constants: the monitor is free to stop slicing a window and + reuse the scanner outright, which is a better implementation of the same + contract. + """ + monkeypatch.setattr(jm, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) + block = ["metric 'ledger.transaction.apply':"] + block += [f" filler {i} = 0ms" for i in range(gap - 1)] + block += [" sum = 1500.0ms"] + + scanner = lc.TxApplyScanner() + for line in block: + scanner.feed(line) + + with gzip.open(jm.log_path('300', 1), 'wt') as fh: + fh.write("\n".join(block) + "\n") + from_archive = jm._tx_apply_for_attempt('300', 1) + + assert (scanner.seconds is None) == (from_archive is None), ( + f"at {gap} lines past the header the collector says {scanner.seconds} " + f"and the monitor says {from_archive}") + if from_archive is not None: + assert from_archive == pytest.approx(scanner.seconds) + + +# --- the number itself -------------------------------------------------------- + +@pytest.mark.parametrize('block,seconds', [ + (MEDIDA_BLOCK, TX_APPLY_SECONDS), + (MEDIDA_BIG, TX_APPLY_BIG_SECONDS), +]) +def test_both_processes_read_the_same_total_out_of_one_block(block, seconds): + """The collector's scanner and the monitor's regex must not disagree. + + They are separate implementations of the same read: a stream scanner with a + window, and a whole-archive search. progress.json takes whichever one landed + first, so a disagreement is a per-range coin flip. + """ + assert scan(block).seconds == pytest.approx(seconds) + m = jm._SUM_RE.search(block) + assert m, "the monitor's regex does not match this block at all" + assert float(m.group(1)) / 1000.0 == pytest.approx(seconds) + + +def test_scientific_notation_is_the_normal_case_not_the_edge_case(): + """Past 1e6 ms, which every range with real transaction load exceeds.""" + assert 'e+06' in MEDIDA_BIG + assert scan(MEDIDA_BIG).seconds > scan(MEDIDA_BLOCK).seconds + + +def test_no_other_line_in_the_block_looks_like_the_sum(): + """min, max, mean, stddev and the percentiles are all " = ms". + + A pattern loose enough to take one of them would report a per-transaction + latency as a whole-range total -- plausible, wrong, and unnoticeable. + """ + for block in (MEDIDA_BLOCK, MEDIDA_BIG): + matched = [l for l in block.splitlines() if jm._SUM_RE.search(l)] + assert len(matched) == 1, f"matched {len(matched)} lines: {matched}" + assert 'sum =' in matched[0] + + +def test_a_sum_from_another_metric_is_not_this_metric(): + """stellar-core prints many medida blocks; only one is ours.""" + scanner = lc.TxApplyScanner() + for line in ["metric 'ledger.ledger.close':", " sum = 999999.0ms"]: + scanner.feed(line) + assert scanner.seconds is None + + +def test_a_block_split_across_two_polls_still_resolves(): + """One scanner spans the whole poll loop for a pod. + + A poll boundary -- or a reconnect -- landing inside the block must not lose + the header already seen, or the last four lines of a range's life are read + with no idea what metric they belong to. + """ + lines = MEDIDA_BLOCK.splitlines() + scanner = lc.TxApplyScanner() + for line in lines[:4]: + scanner.feed(line) + assert scanner.seconds is None + for line in lines[4:]: + scanner.feed(line) + assert scanner.seconds == pytest.approx(TX_APPLY_SECONDS) + + +def test_the_metric_is_the_one_the_worker_is_told_to_print(): + """--metric on the worker command line and the string the scanner greps. + + stellar-core prints nothing at all without the flag, so a rename on either + side is a run's worth of missing metrics with no error. + """ + script = jm.RESUME_SCRIPT + m = re.search(r"--metric '([^']+)'", script) + assert m, "the worker no longer asks stellar-core for a metric" + assert m.group(1) in lc._TX_METRIC, ( + f"the worker prints {m.group(1)!r}, the collector greps {lc._TX_METRIC!r}") diff --git a/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py b/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py new file mode 100644 index 00000000..d907ee12 --- /dev/null +++ b/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py @@ -0,0 +1,292 @@ +"""The worker Job the monitor renders, against the controllers that read it. + +Three readers on the other side of this boundary, none of them ours: + + the Job controller evaluates podFailurePolicy rules first-match-wins and + reports the winner as "matching FailJob rule at index N". + That INDEX is the whole verdict -- see + test_k8s_failure_formats.py -- so the order the rules are + rendered in is a contract with the message we later decode. + the kubelet honours restartPolicy and terminationGracePeriodSeconds. + the log collector reads the attempt off the POD's labels, not the Job's. + +Everything here is asserted against a real build_job() object rather than the +source text, so a rewrite that keeps the rendered Job identical is free to +happen. +""" + +from types import SimpleNamespace as NS + +import pytest + +import job_monitor as jm +import log_collector as lc + +import _artifacts as art + + +@pytest.fixture +def job(monkeypatch): + """One rendered worker Job, in the mode that needs no cluster.""" + monkeypatch.setattr(jm, 'STORAGE_MODE', 'ephemeral') + monkeypatch.setattr(jm, 'RUN_NAME', 'pc') + monkeypatch.setattr(jm, 'CORE_IMAGE', 'stellar/stellar-core:test') + monkeypatch.setattr(jm, 'PROFILE', None) + return jm.build_job(31005951, 16320, 2, None) + + +# --- the Job controller must not own retries --------------------------------- + +def test_the_job_controller_never_replaces_a_failed_pod(job): + """backoffLimit 0 is load-bearing. + + Above 0 the controller replaces the pod on its own schedule: we could not + tell a disruption from a catchup failure, could not count evictions against + their own budget, and could not guarantee the log was archived before the + next attempt started. Escalating a memory limit also needs a NEW Job -- + spec.template is immutable -- so a controller-driven retry would silently + re-run at the limit that just killed the range. + """ + assert job.spec.backoff_limit == 0 + + +def test_a_finished_job_still_has_a_ttl_backstop(job): + """reconcile() reaps finished Jobs, but only while it is running. + + A monitor that is down, wedged, or has lost its RBAC leaves every finished + Job listed on every later pass. The TTL is what bounds that, and it must be + the value the chart configured -- not a second, independent default. + """ + assert job.spec.ttl_seconds_after_finished == jm.JOB_TTL_SECONDS + assert jm.JOB_TTL_SECONDS > 0, "a TTL of 0 deletes a Job before it can be classified" + + +def test_a_worker_pod_is_never_restarted_in_place(job): + """restartPolicy OnFailure restarts the container inside the same pod. + + Same pod name, same resource limits -- so an OOM would loop forever at the + limit that killed it, the attempt counter would never advance, and the + terminated container state the classifier reads would be overwritten. + """ + assert job.spec.template.spec.restart_policy == 'Never' + + +def test_the_deadline_is_on_the_pod_not_on_the_job(job, monkeypatch): + """JobSpec.activeDeadlineSeconds runs from the Job's startTime. + + Every second spent Pending -- waiting for Karpenter, pulling the image -- is + then charged against a budget meant to bound how long the range RUNS. During + a node-class outage this run sat ~15 minutes Pending and ranges died as + "timeouts" having barely executed; a timeout gets only MAX_TIMEOUT_ATTEMPTS, + so two stalls condemn a range and fail the mission. + """ + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 10800) + j = jm.build_job(300, 420, 1, None) + assert j.spec.active_deadline_seconds is None, \ + "the deadline is on the JobSpec, so Pending time is charged to the range" + assert j.spec.template.spec.active_deadline_seconds == 10800 + + +def test_no_deadline_means_no_field_at_all(job, monkeypatch): + """0 is "off". Rendering it literally would kill every pod instantly.""" + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 0) + j = jm.build_job(300, 420, 1, None) + assert j.spec.template.spec.active_deadline_seconds is None + + +def test_the_grace_period_outlasts_a_stellar_core_drain(job): + """stellar-core catches SIGTERM, drains, and exits 3 in ~7s (ssc-test). + + A grace period shorter than the drain turns every eviction into a SIGKILL + and exit 137 -- which the podFailurePolicy classifies as an OOM, spends the + OOM budget instead of the disruption budget, and escalates memory for a + range that never needed any. + """ + grace = job.spec.template.spec.termination_grace_period_seconds + assert grace == jm.WORKER_GRACE_SECONDS + assert grace > 7, f"{grace}s does not cover the measured ~7s drain" + + +# --- podFailurePolicy: order IS the protocol --------------------------------- + +def test_every_rule_index_decodes_back_to_the_rule_that_matched(job): + """The round trip: rules[i] -> "rule at index i" -> classify_from_job. + + The controller reports only the index, so the rendered ORDER and the table + the decoder uses are one contract. Rather than assert they are the same + list, this drives a real condition message through the real classifier for + every index that exists -- which is what actually has to hold. + """ + rules = job.spec.pod_failure_policy.rules + assert len(rules) == len(jm.RULE_ORDER), ( + f"{len(rules)} rules rendered but {len(jm.RULE_ORDER)} decodable indices") + for index, expected in enumerate(jm.RULE_ORDER): + msg = (f"Container stellar-core for pod ns/p failed with exit code 1 " + f"matching FailJob rule at index {index}") + verdict = jm.classify_from_job(_failed_job(msg)) + assert verdict['outcome'] == expected, ( + f"rule {index} renders as {expected!r} but decodes as {verdict['outcome']!r}") + + +def test_disruption_is_evaluated_before_any_exit_code(job): + """First match wins, and exit 3 is ambiguous on its own. + + stellar-core exits 3 both for a SIGTERM drain and for a corrupt bucket, so + the DisruptionTarget condition is the only thing that separates a spot + eviction from a broken range. If an exit-code rule were evaluated first, an + eviction would match it, be condemned as a catchup failure, and abort a + whole run -- on spot, routinely. + """ + rules = job.spec.pod_failure_policy.rules + assert rules[0].on_pod_conditions, "index 0 is not the pod-condition rule" + assert [(c.type, c.status) for c in rules[0].on_pod_conditions] \ + == [('DisruptionTarget', 'True')] + assert jm.RULE_ORDER[0] == 'disrupted' + for rule in rules[1:]: + assert rule.on_exit_codes is not None + + +def test_the_oom_rule_is_narrower_than_the_catch_all_and_precedes_it(job): + """137 has to be matched before "any non-zero", or it never matches at all. + + Reaching the 137 rule also proves DisruptionTarget did not match, which is + the only way to tell an OOM kill from a grace-period SIGKILL once the pod + is gone. + """ + rules = job.spec.pod_failure_policy.rules + oom = rules[jm.RULE_ORDER.index('oom')].on_exit_codes + catch_all = rules[jm.RULE_ORDER.index('failed')].on_exit_codes + assert (oom.operator, oom.values) == ('In', [137]) + assert (catch_all.operator, catch_all.values) == ('NotIn', [0]) + assert jm.RULE_ORDER.index('oom') < jm.RULE_ORDER.index('failed') + + +def test_every_rule_fails_the_job_rather_than_counting_it(job): + """A Count action surfaces as BackoffLimitExceeded and loses the index. + + classify_from_job only reads a condition whose reason is PodFailurePolicy; + anything else carries no per-rule detail and returns no verdict at all. + """ + for rule in job.spec.pod_failure_policy.rules: + assert rule.action == 'FailJob' + + +def test_the_exit_code_rules_name_the_container_that_actually_runs(job): + """A containerName that matches nothing makes the rule silently inert. + + The Job would then fall through to the catch-all -- or to no rule -- and an + OOM would arrive with no index at all. + """ + names = {c.name for c in job.spec.template.spec.containers} + for rule in job.spec.pod_failure_policy.rules: + if rule.on_exit_codes is not None: + assert rule.on_exit_codes.container_name in names, ( + f"rule targets container {rule.on_exit_codes.container_name!r}, " + f"pod has {sorted(names)}") + + +def test_the_collector_watches_the_container_the_job_creates(job): + """The collector streams one container by name and samples its memory. + + A rename here leaves it streaming nothing -- and the peak sampler skipping + every container, since it filters on the same name. + """ + names = {c.name for c in job.spec.template.spec.containers} + default = _clean_default('log_collector', 'CONTAINER') + assert default in names, ( + f"the collector follows {default!r}; the Job creates {sorted(names)}") + + +# --- labels: the pod is the collector's only source of the attempt ----------- + +def test_the_pod_carries_its_own_attempt_number(job): + """The collector reads the attempt off the POD, and defaults it to "1". + + With the label only on the Job, every attempt claimed the same + range--a1.* files: measured on ssc-test 2026-07-30, 2246 metrics files + all a1 while 475 a2 pods were running -- so each retry OVERWROTE the first + attempt's peak instead of being maxed against it, destroying exactly the + OOM evidence the resumed chain exists to keep. + """ + labels = job.spec.template.metadata.labels + assert labels[jm.LABEL_ATTEMPT] == '2' + assert labels[jm.LABEL_RANGE] == '31005951' + assert labels[jm.LABEL_RUN] == 'pc' + + +def test_both_processes_agree_on_the_label_keys(): + """Two readers, one key. A mismatch reproduces the same silent collision.""" + assert jm.LABEL_ATTEMPT == lc.LABEL_ATTEMPT + assert jm.LABEL_RUN == lc.LABEL_RUN + + +def test_the_job_is_findable_by_the_same_labels_as_its_pod(job): + """reconcile lists Jobs by run label and reads the range and attempt off it. + + The pod list and the Job list have to describe the same universe, or a Job + is reaped while its pod is still streaming. + """ + for key in (jm.LABEL_RUN, jm.LABEL_RANGE, jm.LABEL_ATTEMPT): + assert job.metadata.labels[key] == job.spec.template.metadata.labels[key] + + +def test_the_job_name_encodes_the_range_and_the_attempt(job): + """Name uniqueness IS the dispatch mutex. + + reconcile treats a 409 AlreadyExists as "someone else already dispatched + this attempt" and spends a slot rather than raising. A name that did not + vary with the attempt would make a retry collide with its predecessor + forever; one that did not vary with the range would let two ranges share it. + """ + assert job.metadata.name == jm.job_name(31005951, 2) + assert jm.job_name(1, 1) != jm.job_name(1, 2) != jm.job_name(2, 2) + + +# --- the worker's own inputs ------------------------------------------------- + +def test_the_worker_runs_the_resume_script_for_its_own_range(job): + """The key the script marks /data with is the range identity. + + RESUME only skips new-db when the DB on /data belongs to THIS range; the + mark file is how it knows. A key that did not match the catchup argument + would resume one range's replay into another's database. + """ + command = job.spec.template.spec.containers[0].command + assert command[:2] == ['/bin/sh', '-c'] + script = command[2] + key = jm.job_key(31005951, 16320) + assert f'KEY="{key}"' in script + assert f'catchup "$KEY"' in script + + +def test_the_worker_mounts_the_config_the_chart_renders(job): + """The stellar-core.cfg ConfigMap is the chart's, named off the release. + + It is also the object every Job, PVC and the progress ConfigMap are + owner-referenced to, so the name has to be the one owner_ref() reads. + """ + volumes = {v.name: v for v in job.spec.template.spec.volumes} + assert volumes['config'].config_map.name == f"{jm.RUN_NAME}-stellar-core-config" + mounts = {m.name: m.mount_path for m in job.spec.template.spec.containers[0].volume_mounts} + assert mounts['config'] == '/config' + assert '/config/stellar-core.cfg' in job.spec.template.spec.containers[0].command[2] + + +def test_data_is_the_path_the_resume_script_probes(job): + """RESUME reads /data/.job-key and the previous incarnation's core log.""" + mounts = {m.name: m.mount_path for m in job.spec.template.spec.containers[0].volume_mounts} + assert mounts['data'] == '/data' + assert 'MARK=/data/.job-key' in job.spec.template.spec.containers[0].command[2] + + +# --- helpers ----------------------------------------------------------------- + +def _failed_job(message, reason='PodFailurePolicy'): + """The shape classify_from_job reads: a Job with one Failed condition.""" + return NS(status=NS(conditions=[ + NS(type='Failed', status='True', reason=reason, message=message)])) + + +def _clean_default(module_name, constant): + """The module's own fallback, read with no ambient env set.""" + return art.defaults(module_name)[constant] diff --git a/src/MissionParallelCatchup/tests/contract/test_worker_log_markers.py b/src/MissionParallelCatchup/tests/contract/test_worker_log_markers.py new file mode 100644 index 00000000..142b81b4 --- /dev/null +++ b/src/MissionParallelCatchup/tests/contract/test_worker_log_markers.py @@ -0,0 +1,133 @@ +"""What the worker prints, against the collector that reads it off the stream. + +RESUME_SCRIPT is the worker's entrypoint and it announces its own decision on +stdout. The collector -- a different process, in a different container, that +never sees the Job spec -- recovers that decision by scanning the log stream for +a marker. That marker is the only way anything downstream knows whether an +attempt did the whole range or only its tail, and the difference matters: a +resumed attempt skips the archive download and the bucket apply, which is where +peak memory happens, so profiling it alone under-reports the range by the whole +download-vs-replay gap. On spot, where eviction is routine and resume is the +entire point of durable /data, that would make a run unprofileable. + +The script is executed here rather than quoted: what the collector has to cope +with is the bytes a real /bin/sh emits, not the string literal in job_monitor. + +(The resume DECISION -- when to skip new-db, when a range is already complete -- +is exercised in tests/unit/test_resume_script.py. This file only pins the +handshake between the two processes.) +""" + +import os +import re +import subprocess +import tempfile + +import job_monitor as jm +import log_collector as lc + +TARGET = 16752063 +COUNT = 16320 + + +def _offline_info(lcl): + """`stellar-core offline-info --console`, as 27.1.1 prints it. + + Whole document on purpose: the probe has to reach "num" past the ~40 lines + of bucketlist hashes that sit between it and the "ledger" key. + """ + if lcl is None: + return '{}' + hashes = "\n".join(f' "{i:064x}",' for i in range(40)) + return ('{\n "info" : {\n "ledger" : {\n' + ' "age" : 3,\n' + f' "bucketList" : [\n{hashes}\n ],\n' + f' "num" : {lcl},\n "version" : 23\n' + ' }\n }\n}') + + +def worker_stdout(lcl): + """Run RESUME_SCRIPT with a stubbed stellar-core; return what it printed.""" + script = jm.RESUME_SCRIPT % {'key': f"{TARGET}/{COUNT}", 'target': TARGET, + 'count': COUNT} + d = tempfile.mkdtemp() + data = os.path.join(d, 'data') + os.makedirs(data) + stub = os.path.join(d, 'stellar-core') + with open(stub, 'w') as fh: + fh.write('#!/bin/sh\n' + 'for a in "$@"; do case "$a" in\n' + ' offline-info) cat "$INFO"; exit 0;;\n' + ' new-db) exit 0;;\n' + ' catchup) exit 0;;\n' + 'esac; done\nexit 0\n') + os.chmod(stub, 0o755) + info = os.path.join(d, 'info.json') + with open(info, 'w') as fh: + fh.write(_offline_info(lcl)) + with open(os.path.join(data, '.job-key'), 'w') as fh: + fh.write(f"{TARGET}/{COUNT}") + + script = script.replace('/usr/bin/stellar-core', stub).replace('/data/', data + '/') + r = subprocess.run(['/bin/sh', '-c', script], capture_output=True, text=True, + env=dict(os.environ, INFO=info), timeout=30) + return r.stdout + + +def scan(output): + scanner = lc.TxApplyScanner() + for line in output.splitlines(): + scanner.feed(line) + return scanner + + +def test_the_collector_sees_a_resume_the_worker_announced(): + out = worker_stdout(lcl=TARGET - 100) + assert 'RESUME:' in out, out + assert scan(out).resumed is True, f"the collector missed the marker in:\n{out}" + + +def test_a_declined_resume_is_not_read_as_a_resume(): + """"RESUME DECLINED" means the opposite and shares a prefix with "RESUME:". + + Reading it as a resume chains a fresh attempt onto the attempts before it + and maxes their peaks together, inflating every range that ever restarted. + The colon is what separates them, so it is load-bearing on both sides. + """ + out = worker_stdout(lcl=None) + assert 'RESUME DECLINED' in out, out + assert scan(out).resumed is False, "a declined resume was read as a resume" + + +def test_the_probe_line_is_not_mistaken_for_the_decision(): + """The script also prints "RESUME PROBE: ..." before it has decided anything. + + It reports the LCL it read, on every attempt including a fresh one, so a + marker loose enough to match it would mark every attempt resumed. + """ + out = worker_stdout(lcl=None) + assert 'RESUME PROBE:' in out + assert scan("RESUME PROBE: offline-info reports lcl 42").resumed is False + + +def test_a_range_that_was_already_complete_announces_no_resume(): + """It ran no catchup at all, so there is no measurement to chain.""" + out = worker_stdout(lcl=TARGET) + assert 'ALREADY COMPLETE' in out, out + assert scan(out).resumed is False + + +def test_the_marker_the_collector_greps_is_the_one_the_script_prints(): + """Stated directly, so a rename on either side fails here and not in a run. + + Everything above would still pass if BOTH sides were renamed together -- + which is fine -- but this catches the case where the script's wording drifts + while the constant does not. + """ + assert lc.TxApplyScanner.RESUME_MARK in jm.RESUME_SCRIPT, ( + f"the collector greps {lc.TxApplyScanner.RESUME_MARK!r}, which the script " + "never prints") + # ...and the decline must not contain it, or the two are indistinguishable. + decline = re.search(r'echo "(RESUME DECLINED[^"]*)"', jm.RESUME_SCRIPT) + assert decline, "the script no longer announces a declined resume" + assert lc.TxApplyScanner.RESUME_MARK not in decline.group(1) diff --git a/src/MissionParallelCatchup/tests/reconcile/test_completed_range_not_redispatched.py b/src/MissionParallelCatchup/tests/reconcile/test_completed_range_not_redispatched.py index 2550f493..9136264b 100644 --- a/src/MissionParallelCatchup/tests/reconcile/test_completed_range_not_redispatched.py +++ b/src/MissionParallelCatchup/tests/reconcile/test_completed_range_not_redispatched.py @@ -166,9 +166,8 @@ def test_the_range_scoped_reap_still_waits_for_the_done_marker(cluster): written `.done` -- JOB_TTL_SECONDS is the backstop for a collector that never gets there. - This is the behavioural half of the guarantee that - test_job_monitor.py::test_the_reap_waits_for_the_collectors_done_marker - asserts by extracting and exec'ing the function's source. + This is the range-scoped half of the guarantee; the attempt-scoped half is + unit/test_reaping.py::test_the_reap_waits_for_the_collectors_done_marker. """ cluster.reconcile() cluster.advance(300, 'succeeded') diff --git a/src/MissionParallelCatchup/tests/test_job_monitor.py b/src/MissionParallelCatchup/tests/test_job_monitor.py deleted file mode 100644 index 3e42e61b..00000000 --- a/src/MissionParallelCatchup/tests/test_job_monitor.py +++ /dev/null @@ -1,2384 +0,0 @@ -"""Unit tests for the parallel-catchup job monitor and its log collector. - -Covers the formats this mission does not control: the Job controller's -podFailurePolicy condition messages, and stellar-core's medida metric block. -Both are pinned from real captures so a Kubernetes or stellar-core change fails -here rather than silently degrading a run. - -Sources are parsed rather than imported, so no cluster, kubernetes client or -aiohttp is required. - -Run: python3 -m pytest test_job_monitor.py -""" - -import json -import os -import re - -import pytest - -# The modules under test sit one level above tests/. -_SRC_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - - -def _read(name): - with open(os.path.join(_SRC_DIR, name)) as fh: - return fh.read() - - -SRC = _read('job_monitor.py') -COLLECTOR_SRC = _read('log_collector.py') - - -def _extract(pattern, src=None): - m = re.search(pattern, src if src is not None else SRC, re.S | re.M) - assert m, f"pattern not found: {pattern}" - return m - - -JOB_MSG = re.compile(eval(_extract(r"_JOB_MSG = re\.compile\((r\"[^\"]+\")\)").group(1))) -JOB_RULE = re.compile(eval(_extract(r"_JOB_RULE = re\.compile\((r\"[^\"]+\")\)").group(1))) -SUM_RE = re.compile(eval(_extract(r"_SUM_RE = re\.compile\((r\"[^\"]+\")\)").group(1))) -RULE_ORDER = [x.strip().strip("'") for x in - _extract(r"RULE_ORDER = \[([^\]]+)\]").group(1).split(',')] -RULE_OUTCOME = dict(enumerate(RULE_ORDER)) - -# RECONSTRUCTED 2026-07-30 after an over-broad test deletion removed the -# originals -- twice. Kept up here with the other module constants so a -# function-scoped deletion cannot reach them again. Shaped to what the code -# parses (_JOB_RULE reads "rule at index N", RULE_ORDER[2] is 'failed'; -# classify() keys on the substring 'ephemeral' in status.message) but no longer -# verbatim captures. Re-pin from a real eviction on the next run. -EPH_EVICT_JOB_CONDITION = ( - "Container stellar-core for pod stellar-supercluster/" - "parallel-catchup-r31005951-a1-x7k2p failed with exit code 3 " - "matching FailJob rule at index 2") -EPH_EVICT_MESSAGE = ( - "Pod ephemeral local storage usage exceeds the total limit of containers 40Gi") - - -def classify(msg): - """Mirrors classify_from_job: rule index wins, exit code is the fallback.""" - rule, detail = JOB_RULE.search(msg), JOB_MSG.search(msg) - outcome = RULE_OUTCOME.get(int(rule.group('idx'))) if rule else None - code = int(detail.group('code')) if detail else None - if outcome is None and code is not None: - outcome = 'oom' if code == 137 else 'failed' - return outcome, code, (detail.group('pod') if detail else None) - - -def tx_apply_scanner(): - cls = _extract(r"^_TX_METRIC = .*?^(class TxApplyScanner:.*?)^def ", - COLLECTOR_SRC).group(1) - ns = {'re': re} - exec("\n".join([_extract(r"^_TX_METRIC = .*$", COLLECTOR_SRC).group(0), - _extract(r"^_SUM_RE = .*$", COLLECTOR_SRC).group(0), - cls]), ns) - return ns['TxApplyScanner'] - - -# --- captures ---------------------------------------------------------------- - -# EKS 1.34 Job condition messages. Only the wording is pinned; pod and -# container names are renamed for readability. -DISRUPTED = ("Pod sandbox/jterm-catchup-snfr2 has condition DisruptionTarget " - "matching FailJob rule at index 0") -OOMKILLED = ("Container oom-container for pod sandbox/oom-test-job-qvq8b failed with " - "exit code 137 matching FailJob rule at index 1") -NONZERO_EXIT = ("Container exit-1-container for pod sandbox/exit-1-job-wbhkq failed with " - "exit code 1 matching FailJob rule at index 2") - -# stellar-core 27.1.1 catchup pod, --metric 'ledger.transaction.apply'. Kept -# whole: `sum` is 10 lines below the header against a 15-line scan window. -MEDIDA_BLOCK = """2026-07-28T18:39:49.350 GAJSL [default INFO] metric 'ledger.transaction.apply': -2026-07-28T18:39:49.350 GAJSL [default INFO] count = 20 -2026-07-28T18:39:49.350 GAJSL [default INFO] mean rate = 0.22136 calls/s -2026-07-28T18:39:49.350 GAJSL [default INFO] 1-minute rate = 0.113149 calls/s -2026-07-28T18:39:49.350 GAJSL [default INFO] 5-minute rate = 0.175948 calls/s -2026-07-28T18:39:49.350 GAJSL [default INFO] 15-minute rate = 0.191421 calls/s -2026-07-28T18:39:49.350 GAJSL [default INFO] min = 0.295417ms -2026-07-28T18:39:49.350 GAJSL [default INFO] max = 0.639873ms -2026-07-28T18:39:49.350 GAJSL [default INFO] mean = 0.417143ms -2026-07-28T18:39:49.350 GAJSL [default INFO] stddev = 0.108677ms -2026-07-28T18:39:49.350 GAJSL [default INFO] sum = 8.34285ms -2026-07-28T18:39:49.350 GAJSL [default INFO] median = 0ms -2026-07-28T18:39:49.350 GAJSL [default INFO] 75% = 0ms -2026-07-28T18:39:49.350 GAJSL [default INFO] 95% = 0ms -2026-07-28T18:39:49.350 GAJSL [default INFO] 98% = 0ms -2026-07-28T18:39:49.350 GAJSL [default INFO] 99% = 0ms -2026-07-28T18:39:49.350 GAJSL [default INFO] 99.9% = 0ms""" - -TX_APPLY_SECONDS = 0.00834285 - - -# --- how a failed catchup attempt is classified ------------------------------ - -@pytest.mark.parametrize("msg,outcome,code,pod", [ - (DISRUPTED, 'disrupted', None, None), - (OOMKILLED, 'oom', 137, 'oom-test-job-qvq8b'), - (NONZERO_EXIT, 'failed', 1, 'exit-1-job-wbhkq'), -]) -def test_job_condition_message(msg, outcome, code, pod): - assert classify(msg) == (outcome, code, pod) - - -def test_rule_order_matches_the_rendered_policy(): - rendered = re.findall(r"\n \('(\w+)', client\.V1PodFailurePolicyRule", SRC) - assert rendered == RULE_ORDER - - -def test_eviction_is_told_apart_from_a_broken_range_by_the_condition(): - # stellar-core exits 3 both for a drain and for a corrupt bucket, so only - # DisruptionTarget separates them -- hence rule 0 must be evaluated first. - assert classify(DISRUPTED)[0] == 'disrupted' - assert classify("Container c for pod ns/p failed with exit code 3")[0] == 'failed' - - -def test_bare_137_is_an_oom(): - assert classify("Container c for pod ns/p failed with exit code 137")[:2] == ('oom', 137) - - -def test_backoff_limit_message_stays_unclassified(): - assert classify("Job has reached the specified backoff limit") == (None, None, None) - - -def test_admission_rejection_is_not_a_catchup_failure(): - rejected = {'VolumeAttachmentLimitExceeded', 'OutOfcpu', 'OutOfmemory', 'OutOfpods', - 'UnexpectedAdmissionError', 'NodeAffinity', 'Shutdown', 'Evicted'} - listed = set(re.findall(r"'(\w+)'", _extract( - r"if pod\.status\.reason in \(([^)]+)\)").group(1))) - assert rejected <= listed, f"missing from classify(): {rejected - listed}" - - -# --- ledger range generation ------------------------------------------------- - -def test_logarithmic_ranges_match_the_shell_generator(): - # Verbatim output of logarithmic_range_generator.sh with - # floor=16000 overlap=320 start=0 latest=500000 parallelism=4, captured - # before it was deleted. Chunk size halves toward the tip, so exact values - # are pinned rather than a count. - expected = "250000/62820 187500/62820 125000/62820 62500/62820 375001/31570 343751/31570 312501/31570 281251/31570 500000/16320 484000/16320 468000/16320 452000/14817".split() - - floor, overlap, start, latest, par = 16000, 320, 0, 500000, 4 - - def seg(sl, el, ss): - out = [] - while el > sl: - lpj = min(el - sl, ss) - out.append((el, lpj + overlap)) - el -= lpj - return out - - out, s0, end = [], start, latest // 2 - chunk = (end - s0 + 1) // max(par, 1) - while chunk > floor: - out += seg(s0, end, chunk) - s0 = end + 1 - chunk //= 2 - end = s0 + (chunk * par) - out += seg(end + 1, latest, floor) - - assert [f"{e}/{c}" for e, c in out] == expected - - -# --- tx_apply, read from stellar-core's metric block ------------------------- - -def test_monitor_parses_tx_apply_sum(): - sums = [SUM_RE.search(l) for l in MEDIDA_BLOCK.splitlines()] - got = [float(m.group(1)) / 1000.0 for m in sums if m] - assert got == [pytest.approx(TX_APPLY_SECONDS)] - - -def test_collector_scanner_agrees_with_the_monitor(): - scanner = tx_apply_scanner()() - for line in MEDIDA_BLOCK.splitlines(): - scanner.feed(line) - assert scanner.seconds == pytest.approx(TX_APPLY_SECONDS) - - -def test_scanner_resumes_a_block_split_across_a_reconnect(): - # One scanner spans stream_pod's reconnect loop, so a drop mid-block must - # not lose the header already seen. - head, tail = MEDIDA_BLOCK.splitlines()[:4], MEDIDA_BLOCK.splitlines()[4:] - scanner = tx_apply_scanner()() - for line in head: - scanner.feed(line) - assert scanner.seconds is None - for line in tail: - scanner.feed(line) - assert scanner.seconds == pytest.approx(TX_APPLY_SECONDS) - - -def test_scanner_ignores_sum_from_another_metric(): - scanner = tx_apply_scanner()() - for line in ["metric 'ledger.ledger.close':", " sum = 999999.0ms"]: - scanner.feed(line) - assert scanner.seconds is None - - -def test_scanner_gives_up_past_its_window(): - scanner = tx_apply_scanner()() - scanner.feed("metric 'ledger.transaction.apply':") - for _ in range(20): - scanner.feed("[default INFO] unrelated chatter") - scanner.feed(" sum = 12.5555ms") - assert scanner.seconds is None - - -def test_rate_and_mean_lines_are_not_read_as_sum(): - for line in MEDIDA_BLOCK.splitlines(): - if 'rate =' in line or 'mean =' in line: - assert SUM_RE.search(line) is None - - -def test_sum_stays_inside_the_scan_window(): - lines = MEDIDA_BLOCK.splitlines() - header = next(i for i, l in enumerate(lines) if 'ledger.transaction.apply' in l) - offset = next(i for i, l in enumerate(lines) if SUM_RE.search(l)) - header - assert offset == 10, f"medida layout moved: sum is now {offset} lines below the header" - assert offset <= tx_apply_scanner().WINDOW - - -def test_tx_apply_survives_a_reaped_pod(): - stmt = _extract(r"\n\s*tx = tx_apply_for_range\(.*?\n(?=\s*(?:if|completed|#))") - assert not re.search(r"\)\s*if pod else None", stmt.group(0)), \ - "tx_apply must fall back to the collector's files when the pod is gone" - assert re.search(r"tx_apply_for_range\(\s*end,\s*attempt", stmt.group(0)) - - -def test_tx_apply_prefers_durable_sources_over_the_pod_api(): - # The per-attempt reader; tx_apply_for_range now sums these over the chain. - fn = _extract(r"def _tx_apply_for_attempt\(.*?^def ").group(0) - assert fn.index('metrics_path') < fn.index('log_path') < fn.index('read_namespaced_pod_log') - - -# --- contracts between job_monitor and log_collector ------------------------- - -def test_metrics_filename_agrees_across_both_processes(): - mon = _extract(r"def metrics_path\(end, attempt\):\s*return [^\n]*?f\"([^\"]+)\"") - col = _extract(r"def base\(end, attempt\):\s*return [^\n]*?f\"([^\"]+)\"", COLLECTOR_SRC) - assert mon.group(1) == col.group(1) + '.metrics' - - -def test_discarding_a_successful_archive_keeps_its_metrics(): - suffixes = _extract(r"def discard\(end, attempt\):.*?for suffix in \(([^)]*)\)", - COLLECTOR_SRC).group(1) - assert '.log.gz' in suffixes - assert '.metrics' not in suffixes - - -def test_metrics_are_written_before_the_archive_is_discarded(): - # Lives in finalize(), shared by the clean-exit and pod-gone paths. - body = _extract(r"^(async def finalize\(.*?)(?=\n\nasync def )", - COLLECTOR_SRC).group(1) - assert body.index('write_metrics') < body.index('discard(') - - -def test_worker_pod_spec_uses_every_helper(): - body = _extract(r"spec=client\.V1PodSpec\((.*?)containers=\[container\]").group(1) - for field in ('service_account_name', 'topology_spread_constraints', 'restart_policy', - 'termination_grace_period_seconds', 'affinity', 'tolerations'): - assert field in body, f"{field} missing from the worker pod spec" - for helper in ('pod_labels', 'volume_spread_constraints', 'ensure_pvc', - '_failure_rules', '_resources'): - assert len(re.findall(rf"\b{helper}\(", SRC)) >= 2, \ - f"{helper}() is defined but never called" - - -def test_untimestamped_kubelet_text_never_becomes_a_resume_point(): - # A pod that has just been replaced returns plain text from the logs API - # instead of log lines. Partitioning that on the first space yields "unable", - # which as a resume point makes every later request sinceTime=unableZ -> 400 - # for the life of the range. - ts_re = re.compile(eval(_extract(r"_TS_RE = re\.compile\((r\"[^\"]+\")\)", - COLLECTOR_SRC).group(1))) - kubelet = "unable to retrieve container logs for containerd://9f2c1a" - assert ts_re.match(kubelet.partition(' ')[0]) is None - for good in ("2026-07-28T20:29:27.927795721Z", "2026-07-28T20:29:27Z"): - assert ts_re.match(good), good - - - - - - - - - - -def test_an_ephemeral_eviction_is_not_read_as_an_oom_or_a_disruption(): - # Measured end-to-end on ssc-test: the kubelet sets no DisruptionTarget, - # and stellar-core drains and exits 3, so the Job condition is a plain - # non-zero failure that would get no retry. status.message is the only - # discriminator and only the pod carries it, so both classifiers must test - # it before anything keyed on Evicted. - assert 'index 2' in EPH_EVICT_JOB_CONDITION, "the Job matches the generic non-zero rule" - for src in (COLLECTOR_SRC, SRC): - body = _extract(r"def classify(?:_from_job)?\(pod\):(.*?)(?=\n\ndef )", src) - body = body.group(1) if body else src - eph = body.find("'ephemeral'") - generic = body.find("'VolumeAttachmentLimitExceeded'") - assert eph != -1, "no ephemeral-eviction branch" - assert eph < generic, "the ephemeral branch must precede the generic Evicted branch" - - -def test_ephemeral_eviction_message_still_matches_what_we_test_for(): - # Both classifiers key on the substring 'ephemeral' in status.message. - assert 'ephemeral' in EPH_EVICT_MESSAGE - - -def test_ephemeral_escalation_raises_request_and_limit_together(): - # ephemeral-storage is a scheduling dimension: a pod that outgrew its limit - # will not fit where it was placed before unless the request moves too. - fn = _extract(r"def _resources\(.*?^def ").group(0) - assert fn.count('eph or') == 2, "both request and limit must take the escalated size" - assert 'MAX_EPHEMERAL_ATTEMPTS' in SRC - env = _extract(r"ENVIRONMENTAL_OUTCOMES = \(([^)]+)\)").group(1) - assert 'ephemeral' not in env, "a deterministic failure must not get the 20-attempt budget" - - -# The collector is a separate container with its own env block, so a variable -# the monitor has is not automatically one the collector has. STORAGE_MODE was -# missing there and the sampler silently did nothing -- it defaults to 'pvc'. -COLLECTOR_ENV_WITH_DEFAULTS = { - 'KUBERNETES_SERVICE_HOST', 'KUBERNETES_SERVICE_PORT', # injected by kubelet - 'LOGGING_LEVEL', 'PEAK_WS_WINDOW', 'PROMETHEUS_URL', 'WORKER_CONTAINER', -} - - -def test_a_finished_stream_is_never_reopened(): - # A completed task is deleted from `tasks`, so without a record of it the - # next poll re-creates the stream and re-reads the whole log -- every - # cycle, per pod. Measured: the completion block ran every 10s per range. - loop = _extract(r"while True:\n(.*?)await asyncio\.sleep\(POLL_SECONDS\)", - COLLECTOR_SRC).group(1) - assert 'if name in streamed:' in loop - # ...but only once the pod is terminal: a task that ended while the pod is - # still running died early, and re-opening the stream is how that recovers. - # Scoped to the per-pod branch: the vanished-pod reaper above it also - # deletes tasks and adds to `streamed`, and slicing the whole loop would - # match that block instead of this one. - per_pod = loop[loop.index('for pod in pods:'):] - guard = per_pod[per_pod.index('del tasks[name]'):per_pod.index('streamed.add(name)')] - assert 'terminal.get(name)' in guard - - -def test_metrics_writes_merge_so_a_rewrite_cannot_drop_a_measurement(): - # The ephemeral peak is held in memory by the collector; a restart loses it. - # If a later write clobbered the file, the peak already persisted would be - # lost -- which is exactly what happened before this merge. - fn = _extract(r"def write_metrics\(.*?(?=\ndef )", COLLECTOR_SRC).group(0) - assert '{**prior, **values}' in fn, "existing fields must survive" - # ...and peaks additionally take the max, see the monotonicity test below. - assert 'PEAK_KEYS' in fn - - -def test_the_ephemeral_sampler_runs_every_poll_not_once_per_stream(): - # The per-pod branches all end in `continue` for pods already streaming, so - # a sampler placed after them fires only on the cycle a stream opens -- - # when the range has written almost nothing. It must run before the loop. - loop = _extract(r"while True:\n(.*?)await asyncio\.sleep\(POLL_SECONDS\)", - COLLECTOR_SRC).group(1) - call = loop.index('sample_kubelet') - for_pod = loop.index('for pod in pods:') - assert call < for_pod, "sample_kubelet must run before the per-pod loop" - assert loop.count('await list_pods(session)') == 1, \ - "one listing per cycle; the sampler must reuse it" - - -def test_every_env_the_collector_reads_is_set_on_the_collector_container(): - chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/templates/job_monitor.yaml')).read() - collector = chart[chart.index('- name: log-collector'):] - needed = set(re.findall(r"os\.getenv\('([A-Z_]+)'", COLLECTOR_SRC)) - missing = {v for v in needed - COLLECTOR_ENV_WITH_DEFAULTS - if f"- name: {v}\n" not in collector} - assert not missing, f"collector reads {sorted(missing)} but the chart never sets them" - - - - -def test_both_peaks_reach_the_progress_record(): - fields = _extract(r"PEAK_FIELDS = \(([^)]+)\)").group(1) - for f in ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes'): - assert f in fields, f - - -# --- range profile consumption ----------------------------------------------- - -def _profile_ns(ranges, mode='ephemeral', margin=1.1): - """profile_for + _sized, exec'd out of job_monitor with a fixed profile.""" - ns = {'bisect': __import__('bisect'), 'logger': __import__('logging').getLogger('t')} - for name in ('_quantity_bytes', '_bytes_to_quantity', 'profile_for', - '_sized'): - m = re.search(rf"^(def {name}\(.*?)(?=^\S|\Z)", SRC, re.S | re.M) - exec(m.group(1), ns) - ns['_UNITS'] = eval(_extract(r"_UNITS = (\{.*?\})").group(1)) - ns['PROFILE'] = sorted(ranges) - ns['STORAGE_MODE'] = mode - ns['PROFILE_MARGIN'] = margin - return ns - - -PROFILE_RANGES = [ - (1000, {'peakRssBytes': 1_000_000_000, 'peakWorkingSetBytes': 9_000_000_000, - 'peakEphemeralBytes': 2_000_000_000, 'peakCpuCores': 0.5}), - (2000, {'peakRssBytes': 3_000_000_000, 'peakWorkingSetBytes': 13_000_000_000, - 'peakEphemeralBytes': 4_000_000_000, 'peakCpuCores': 1.2}), -] - - -def test_profile_prefers_an_exact_end(): - ns = _profile_ns(PROFILE_RANGES) - assert ns['profile_for'](2000)['peakRssBytes'] == 3_000_000_000 - - -def test_profile_rounds_up_to_the_next_measured_end_never_down(): - # Cost rises with ledger position -- the bucket set only grows -- so a lower - # neighbour under-reports, and under-provisioning costs an eviction while - # over-provisioning only costs packing density. - ns = _profile_ns(PROFILE_RANGES) - assert ns['profile_for'](1500)['peakRssBytes'] == 3_000_000_000, \ - "1500 must size from 2000, not from 1000" - - -def test_profile_falls_back_to_defaults_past_its_high_water_mark(): - # An older profile has nothing above its own top, which is exactly where a - # newer run's fresh ranges live. Extrapolating there would under-provision. - ns = _profile_ns(PROFILE_RANGES) - assert ns['profile_for'](9999) is None - - -def test_a_profile_from_the_other_storage_mode_is_rejected(): - # An ephemeral profile carries peakEphemeralBytes and a pvc one does not. - fn = _extract(r"def load_profile\(.*?^PROFILE = None", SRC).group(0) - assert "mode != STORAGE_MODE" in fn - assert 'return []' in fn - - -def test_an_unreadable_profile_is_not_fatal(): - # It is an optimisation, never a prerequisite. - fn = _extract(r"def load_profile\(.*?^PROFILE = None", SRC).group(0) - assert '(OSError, ValueError)' in fn - - -def test_sizing_applies_the_margin_and_never_exceeds_the_limit(): - ns = _profile_ns(PROFILE_RANGES) - # 1 GB * 1.1, well under the cap - assert ns['_sized'](1_000_000_000, 1.1, '10Gi') == '1049Mi' - # capped: a huge peak cannot produce a request above its own limit - assert ns['_sized'](50_000_000_000, 1.1, '8Gi') == '8192Mi' - - -def _overrides_ns(ranges, lim_mem='24000Mi', lim_eph='40Gi', margin=1.1, - max_mem='32Gi'): - ns = _profile_ns(ranges, margin=margin) - ns['LIM_MEM'] = lim_mem - ns['LIM_EPHEMERAL'] = lim_eph - ns['LIM_CPU'] = '2' - ns['REQ_CPU'] = '1800m' - ns['PROFILE_CPU_LIMIT'] = '' - ns['PROFILE_MAX_MEM'] = max_mem - ns['PROFILE_CACHE_HEADROOM'] = '512Mi' - m = re.search(r"^(def _profile_overrides\(.*?)(?=^\S|\Z)", SRC, re.S | re.M) - exec(m.group(1), ns) - return ns - - -def test_profile_sizes_a_first_attempt(): - # Executed, not grepped: the first version of this gate read `mem is None` - # AFTER mem had been defaulted, so it was never true and profile sizing was - # silently dead while a source-text assertion still passed. - ns = _overrides_ns(PROFILE_RANGES) - out = ns['_profile_overrides'](2000, escalated=False) - assert out['memory'] == '3659Mi' # 3 GB rss * 1.1 + 512Mi - assert out['ephemeral-storage'] == '4196Mi' - # cpu is no longer profiled: REQ_CPU is fixed, so there is nothing to size. - assert 'cpu' not in out - - -def test_profile_does_not_override_an_escalated_retry(): - # An escalation is a measurement of THIS run and outranks an earlier one. - ns = _overrides_ns(PROFILE_RANGES) - assert ns['_profile_overrides'](2000, escalated=True) == {} - - -def test_profile_gives_nothing_past_its_high_water_mark(): - ns = _overrides_ns(PROFILE_RANGES) - assert ns['_profile_overrides'](99999, escalated=False) == {} - - -def test_profile_memory_is_capped_at_its_own_ceiling_not_the_worker_limit(): - # A range needing more than the configured limit must be able to ask for it, - # or it is pinned under its own measured peak and OOMs every attempt. The - # ceiling is what bounds it, and the OOM ladder can still climb past that. - ns = _overrides_ns([(1, {'peakRssBytes': 500_000_000_000})], - lim_mem='24000Mi', max_mem='32Gi') - assert ns['_profile_overrides'](1, escalated=False)['memory'] == '32768Mi' - - -def test_profile_memory_can_exceed_the_configured_worker_limit(): - # 28 GB peak against a 24000Mi configured limit: the profile must raise it. - ns = _overrides_ns([(1, {'peakRssBytes': 28_000_000_000})], - lim_mem='24000Mi', max_mem='32Gi') - got = ns['_profile_overrides'](1, escalated=False)['memory'] - assert ns['_quantity_bytes'](got) > ns['_quantity_bytes']('24000Mi') - - -class _FakeRR: - """Stand-in for client.V1ResourceRequirements, so _resources can be run.""" - - def __init__(self, requests=None, limits=None): - self.requests, self.limits = requests, limits - - -def _resources_ns(ranges, req_eph='35Gi', lim_eph='40Gi'): - ns = _overrides_ns(ranges, lim_eph=lim_eph) - ns.update(REQ_CPU='1800m', LIM_CPU='2', REQ_MEM='9Gi', - REQ_EPHEMERAL=req_eph, client=type('c', (), {'V1ResourceRequirements': _FakeRR})) - m = re.search(r"^(def _resources\(.*?)(?=^def )", SRC, re.S | re.M) - exec(m.group(1), ns) - return ns - - -def test_a_measured_range_matches_memory_and_disk_and_leaves_cpu_configured(): - # Memory and disk match request to limit -- exceeding either kills the pod. - # CPU keeps its configured limit and only moves its request, so the range - # packs by what it uses and can still burst. - ns = _resources_ns(PROFILE_RANGES) - r = ns['_resources'](end=2000) - assert r.requests['memory'] == r.limits['memory'] == '3659Mi' - assert r.requests['ephemeral-storage'] == r.limits['ephemeral-storage'] == '4196Mi' - # The configured request, not a measured one -- a profiled range now packs - # at exactly the same cpu as an unprofiled one. - assert r.requests['cpu'] == '1800m' - assert 'cpu' not in r.limits, "a measured range runs uncapped" - - - -def test_an_unmeasured_range_keeps_the_mismatched_defaults(): - # No profile entry must behave exactly as if there were no profile at all. - ns = _resources_ns(PROFILE_RANGES) - r = ns['_resources'](end=99999) - assert r.requests['memory'] == '9Gi' and r.limits['memory'] == '24000Mi' - # cpu is the exception: no worker is throttled, measured or not. A limit - # only stops a pod using idle cores, and it changes what the range measures. - assert r.requests['cpu'] == '1800m' - assert 'cpu' not in r.limits, "an unprofiled range must not be throttled" - assert r.requests['ephemeral-storage'] == '35Gi' - assert r.limits['ephemeral-storage'] == '40Gi' - assert r.requests != r.limits - - -def test_an_escalated_retry_keeps_the_mismatched_defaults(): - # The escalation already chose the size; the profile must not overwrite it. - ns = _resources_ns(PROFILE_RANGES) - r = ns['_resources'](mem='36000Mi', end=2000) - assert r.limits['memory'] == '36000Mi' - assert r.requests['cpu'] == '1800m', "cpu must fall back to the configured request" - - -def test_no_range_is_cpu_throttled_but_every_range_has_a_request(): - # At a 2-core limit every range pegs 2.0, so the measured peak is a ceiling - # and the profile can never learn real demand. Headroom above the request is - # the whole point -- the request is still capped for packing. - ns = _resources_ns(PROFILE_RANGES) - measured = ns['_resources'](end=2000) - unmeasured = ns['_resources'](end=99999) - assert 'cpu' not in measured.limits, "measured ranges run uncapped" - assert 'cpu' not in unmeasured.limits, "unmeasured ranges run uncapped too" - # The request is still what bounds packing, on both paths, and the profile - # never moves it: cpu is not one of the overridden dimensions. - assert measured.requests['cpu'] == unmeasured.requests['cpu'] == '1800m' - - -def test_pvc_mode_takes_no_ephemeral_override(): - # /data is not on the node disk there, so sizing it would be meaningless. - ns = _resources_ns(PROFILE_RANGES, req_eph='') - r = ns['_resources'](end=2000) - assert 'ephemeral-storage' not in r.requests - - -def test_the_override_is_computed_before_mem_is_defaulted(): - # Reading `mem is None` after mem has been defaulted can never be true, - # which silently disabled profile sizing entirely once already. - fn = _extract(r"def _resources\(.*?return client\.V1ResourceRequirements").group(0) - assert fn.index('_profile_overrides') < fn.index('mem = mem or LIM_MEM') - - -# --- dispatch order + cross-mode profile reuse ------------------------------- - -def _ranges_ns(order='tip-first', generator='uniform', parallelism=4): - ns = {} - for n in ('_uniform_segment', '_ordered', 'generate_ranges'): - m = re.search(rf"^(def {n}\(.*?)(?=^\S|\Z)", SRC, re.S | re.M) - exec(m.group(1), ns) - ns.update(RANGE_GENERATOR=generator, RANGE_ORDER=order, - STARTING_LEDGER=39990000, LATEST_LEDGER_NUM=40000000, - LEDGERS_PER_JOB=1000, LOGARITHMIC_FLOOR_LEDGERS=64000, - PARALLELISM=parallelism, OVERLAP_LEDGERS=320) - return ns - - -def test_generators_emit_tip_first_by_default(): - r = _ranges_ns()['generate_ranges']() - assert r[0][0] > r[-1][0], "index 0 must be the tip" - - -def test_oldest_first_reverses_dispatch_without_dropping_ranges(): - # A profiling run wants the cheap early ranges measured first: the bucket - # set only grows with ledger position, so tip-first front-loads the - # expensive ones and an interrupted run profiles nothing cheap. - tip = _ranges_ns('tip-first')['generate_ranges']() - old = _ranges_ns('oldest-first')['generate_ranges']() - assert old == list(reversed(tip)) - assert sorted(old) == sorted(tip), "reversing must not change the range set" - - -def test_a_cross_mode_profile_keeps_cpu_and_memory_but_drops_disk(): - # cpu and memory measure the same work in either mode. Disk does not: a pvc - # run never measures node-local usage at all, so its absence must fall back - # to the configured default rather than size the wrong dimension. - fn = _extract(r"def load_profile\(.*?^PROFILE = None", SRC).group(0) - assert 'cross_mode' in fn - assert "k != 'peakEphemeralBytes'" in fn - assert 'return []' not in fn.split('cross_mode = ')[1].split('out = []')[0], \ - "a cross-mode profile must degrade, not be rejected" - - -def test_memory_is_sized_from_rss_never_from_working_set(): - # Working set is whatever limit it was measured under -- the kernel grows - # page cache to fill it. Measured on ssc-test, one 420-ledger range: - # limit 4Gi -> ws 3.61 GiB, rss 2.43 GiB, 775s - # limit 8Gi -> ws 7.48 GiB, rss 2.41 GiB, 746s - # limit 24000Mi -> ws 13.49 GiB, rss 2.28 GiB, 773s - # rss is flat and wall-clock is flat, so sizing from ws would reserve 5x the - # real demand for no gain. - fn = _extract(r"def _profile_overrides\(.*?^def ").group(0) - assert 'peakRssBytes' in fn - assert 'peakWorkingSetBytes' not in fn, "working set must not drive sizing" - - -def test_a_profile_without_rss_leaves_memory_alone(): - # An older artifact predates peakRssBytes; it must fall back to the - # configured default rather than guess from working set. - ns = _overrides_ns([(1, {'peakWorkingSetBytes': 13_000_000_000})]) - assert 'memory' not in ns['_profile_overrides'](1, escalated=False) - - -def test_working_set_is_still_recorded_as_a_diagnostic(): - # It is what kubelet ranks node-pressure evictions on, so it explains an - # eviction that rss cannot -- it just must not feed sizing. - fields = _extract(r"PEAK_FIELDS = \(([^)]+)\)").group(1) - for f in ('peakRssBytes', 'peakWorkingSetBytes'): - assert f in fields, f - - -# --- the chart must render with the shapes the mission actually sends --------- - -import shutil, subprocess - -CHART = os.path.join(_SRC_DIR, 'parallel_catchup_helm') - - -def _helm(*extra): - if not shutil.which('helm'): - pytest.skip('helm not installed') - r = subprocess.run(['helm', 'template', 't', CHART, - '--set', 'worker.stellar_core_image=x', *extra], - capture_output=True, text=True) - assert r.returncode == 0, r.stderr - return r.stdout - - -def test_chart_renders_the_service_account_annotations_the_mission_sends(): - # The mission sends service_account.annotations as an indexed array of - # {key,value}; metadata.annotations must be a map. Rendering it straight - # through toYaml produced a list and failed the whole install with - # "cannot unmarshal array into ... map[string]string" -- which no - # source-text assertion would have caught. - out = _helm('--set', 'service_account.annotations[0].key=eks.amazonaws.com/role-arn', - '--set', 'service_account.annotations[0].value=arn:aws:iam::1:role/r') - assert 'eks.amazonaws.com/role-arn: "arn:aws:iam::1:role/r"' in out - - -def test_chart_renders_without_service_account_annotations(): - _helm() - - -def test_chart_renders_the_node_targeting_the_mission_sends(): - out = _helm('--set', 'worker.requireNodeLabels[0].key=purpose', - '--set', 'worker.requireNodeLabels[0].operator=In', - '--set', 'worker.requireNodeLabels[0].values[0]=catchup8-spot', - '--set', 'worker.tolerateNodeTaints[0].key=catchup8-spot', - '--set', 'worker.tolerateNodeTaints[0].effect=NoSchedule') - assert 'catchup8-spot' in out - - -def test_small_ranges_get_absolute_slack_not_just_a_percentage(): - # memory.max bounds anon PLUS page cache. At 190 MiB rss a 1.1x margin is - # 19 MiB of slack -- measured on ssc-test, 90 ranges OOMKilled within 90s of - # dispatch. The fixed headroom is what makes small ranges survivable. - ns = _overrides_ns([(1, {'peakRssBytes': 190 * 2**20})]) - got = ns['_quantity_bytes'](ns['_profile_overrides'](1, escalated=False)['memory']) - slack = (got - 190 * 2**20) / 2**20 - assert slack > 400, f"only {slack:.0f}MiB of slack above rss" - - -def test_oom_escalation_starts_from_what_the_attempt_actually_had(): - # Escalating a 209Mi profiled range off the configured 24000Mi limit jumps - # to 36000Mi -- a 172x overshoot that discards the packing win on first OOM. - ns = {} - for n in ('_quantity_bytes', '_bytes_to_quantity', 'mem_for_attempt'): - m = re.search(rf"^(def {n}\(.*?)(?=^\S|\Z)", SRC, re.S | re.M) - exec(m.group(1), ns) - ns['_UNITS'] = eval(_extract(r"_UNITS = (\{.*?\})").group(1)) - ns.update(LIM_MEM='24000Mi', MEM_BUMP_FACTOR=1.5, MEM_ESCALATION_CAP='48Gi') - assert ns['mem_for_attempt'](2, '702Mi') == '1053Mi' - assert ns['mem_for_attempt'](2) == '36000Mi' # unprofiled keeps old behaviour - - -def test_chart_defaults_match_the_code_defaults(): - # The chart sets these env vars explicitly, so its value WINS over the - # os.getenv default. They drifted once -- code said 512Mi while the chart - # still said 0 -- and the chart silently won, reproducing the OOMs the code - # change was meant to fix. - values = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/values.yaml')).read() - pairs = [('PROFILE_CACHE_HEADROOM', 'profileCacheHeadroom'), - ('PROFILE_MAX_MEM', 'profileMaxMemory'), - ('PROFILE_CPU_LIMIT', 'profileCpuLimit'), - ('PROFILE_MARGIN', 'profileMargin')] - for env, key in pairs: - code = _extract(rf"{env} = .*?os\.getenv\('{env}',\s*'?\"?([^'\")]+)").group(1).strip() - chart = _extract(rf"^\s*{key}:\s*\"?([^\"\n]+)", values).group(1).strip().strip('"') - assert code == chart, f"{env}: code default {code!r} != chart {chart!r}" - - -def test_the_monitor_log_lands_where_the_mission_collects_it(): - # collectLogsFromPods tars /logs. The monitor used to write its own log to - # /data, an emptyDir, so OOM-retry storms never reached the destination - # directory and did not survive a monitor restart. - blk = _extract(r"log_file_name = .*?log_file_path = [^\n]*").group(0) - assert "os.getenv('LOG_DIR'" in blk - col = _extract(r"def base\(end, attempt\):\s*return [^\n]*", COLLECTOR_SRC).group(0) - assert 'LOG_DIR' in col, "collector and monitor must share the collected directory" - - -def test_a_completed_range_releases_its_volume(): - # PVCs are owner-referenced to the release, so nothing reclaimed them until - # helm uninstall. Measured on ssc-test: 2032 bound PVCs / 79 TiB a third of - # the way through a 3982-range run, heading for ~156 TiB and 3982 volumes. - fn = _extract(r"def release_pvc\(.*?^def ").group(0) - assert 'delete_namespaced_persistent_volume_claim' in fn - assert "STORAGE_MODE != 'pvc'" in fn, "ephemeral mode has no PVC to release" - assert 'e.status != 404' in fn, "already-gone must not be an error" - - -def test_the_volume_is_released_only_after_progress_is_saved(): - # If the process dies between the two, the range must still read as - # complete -- keeping a volume is recoverable, losing the record is not. - blk = _extract(r"completed\[end\]\.update\(peaks_for_range.*?release_pvc\(end\)").group(0) - assert blk.index('save_progress') < blk.index('release_pvc') - - -def test_releasing_a_volume_never_fails_a_completed_range(): - fn = _extract(r"def release_pvc\(.*?^def ").group(0) - assert 'raise' not in fn, "a disk cleanup failure must not condemn a finished range" - - -# --- progress durability ----------------------------------------------------- - -def test_progress_is_written_to_the_volume_before_the_configmap(): - # A ConfigMap caps at 1 MiB and the record is ~172 bytes per completed - # range, so it dies around 6100 ranges -- reachable by halving - # ledgersPerJob. Measured mid-run: 348KB at 2024 ranges, 65% of the cap - # projected at 3982. - fn = _extract(r"def save_progress\(.*?^def ").group(0) - assert 'PROGRESS_FILE' in fn - assert fn.index('os.replace') < fn.index('_patch_cm'), \ - "the durable write must land before the mirror" - assert '.tmp' in fn, "a torn write would lose the whole record" - - -def test_a_configmap_mirror_failure_does_not_stop_the_run(): - # reconcile's loop swallows exceptions, so a 413 thrown here meant no - # completion was ever recorded again and finished ranges were redispatched - # forever -- silent, unbounded cost. - fn = _extract(r"def save_progress\(.*?^def ").group(0) - assert 'except ApiException' in fn - assert 'raise' not in fn.split('_patch_cm')[1] - - -def test_progress_is_read_back_from_the_volume_first(): - fn = _extract(r"def load_progress\(.*?^def ").group(0) - assert fn.index('PROGRESS_FILE') < fn.index('read_namespaced_config_map'), \ - "the file is authoritative; the ConfigMap is only a fallback" - assert 'e.status == 404' in fn - - -# Real block from range-40010367-a1 on ssc-test. medida switches to scientific -# notation past 1e6 ms, which is every range with a real transaction load. -MEDIDA_BIG = """2026-07-29T20:11:16.931 GAJSL [default INFO] metric 'ledger.transaction.apply': -2026-07-29T20:11:16.931 GAJSL [default INFO] count = 3231886 -2026-07-29T20:11:16.931 GAJSL [default INFO] mean rate = 812.4 calls/s -2026-07-29T20:11:16.931 GAJSL [default INFO] 1-minute rate = 790.1 calls/s -2026-07-29T20:11:16.931 GAJSL [default INFO] 5-minute rate = 801.3 calls/s -2026-07-29T20:11:16.931 GAJSL [default INFO] 15-minute rate = 799.0 calls/s -2026-07-29T20:11:16.931 GAJSL [default INFO] min = 0.101ms -2026-07-29T20:11:16.931 GAJSL [default INFO] max = 41.2ms -2026-07-29T20:11:16.931 GAJSL [default INFO] mean = 0.404ms -2026-07-29T20:11:16.931 GAJSL [default INFO] stddev = 0.612ms -2026-07-29T20:11:16.931 GAJSL [default INFO] sum = 1.30722e+06ms""" - - -def test_scientific_notation_sum_is_parsed(): - # 25% of ranges recorded no tx_apply -- 91-99% of everything above ledger - # 35M -- because the regex matched "1.30722" then required "ms" and found - # "e+06ms". The metric block was in the archive the whole time. - m = SUM_RE.search(MEDIDA_BIG) - assert m, "scientific-notation sum must parse" - assert float(m.group(1)) / 1000.0 == pytest.approx(1307.22) - - -def test_scanner_reads_a_scientific_notation_block(): - scanner = tx_apply_scanner()() - for line in MEDIDA_BIG.splitlines(): - scanner.feed(line) - assert scanner.seconds == pytest.approx(1307.22) - - -def test_plain_decimal_sums_still_parse(): - m = SUM_RE.search(" sum = 8.34285ms") - assert float(m.group(1)) / 1000.0 == pytest.approx(TX_APPLY_SECONDS) - - -def test_the_chart_grants_the_pvc_delete_release_pvc_needs(): - # release_pvc calls delete_namespaced_persistent_volume_claim. The Role - # granted only get/list/create, so every completion logged a 403 warning and - # the volumes leaked -- 3982 of them, which crashed the EBS CSI controller. - chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/templates/job_monitor.yaml')).read() - blk = _extract(r'resources: \["persistentvolumeclaims"\]\s*\n\s*verbs: \[([^\]]+)\]', chart) - verbs = {v.strip().strip('"') for v in blk.group(1).split(',')} - assert 'delete' in verbs, f"release_pvc needs delete, Role has {sorted(verbs)}" - - -def test_the_configmap_mirror_carries_no_profiling_fields(): - # Profile data lives only on the volume. In the ConfigMap it is what pushes - # a ~30-byte state record to ~172 bytes and the whole document toward the - # 1 MiB cap at ~6100 ranges. - ns = {} - m = re.search(r"^(_PROFILE_ONLY_FIELDS = \(.*?\)\n\n\ndef _state_only\(.*?)(?=\ndef )", - SRC, re.S | re.M) - assert m, "_state_only not found" - exec(m.group(1), ns) - prog = {'completed': {'100': {'attempts': 1, 'count': 16320, 'seconds': 700.0, - 'peakRssBytes': 123, 'peakCpuCores': 1.9, - 'txApply': 200.0, 'wallSeconds': 750.0}}, - 'failed': {}} - out = ns['_state_only'](prog)['completed']['100'] - assert out == {'attempts': 1, 'count': 16320}, out - # and the untouched original still has everything for the volume copy - assert 'peakRssBytes' in prog['completed']['100'] - - -def test_the_volume_copy_keeps_the_profile(): - fn = _extract(r"def save_progress\(.*?^def ").group(0) - assert 'json.dumps(progress' in fn, "the volume write must use the full record" - assert '_state_only' in fn, "the ConfigMap write must be stripped" - assert fn.index('os.replace') < fn.index('_state_only') - - -# --- finished-Job reaping ------------------------------------------------- -# reconcile() LISTs every Job and Pod each pass, so a finished Job costs two -# list entries per pass until it is gone. At 2048-4096 parallelism the dead -# ones outnumbered the live ones within the hour under the old 3600s TTL. - -def _delete_job_ns(delete_impl): - """Exec delete_job against fakes. Nothing here needs a cluster.""" - class ApiException(Exception): - def __init__(self, status): - self.status = status - super().__init__(f"status {status}") - - calls, warnings, reaped = [], [], [] - - class FakeBatch: - def delete_namespaced_job(self, name, namespace, **kw): - calls.append((name, namespace, kw)) - exc = delete_impl(name) - if exc is not None: - raise exc - - ns = { - 'batch_v1': FakeBatch(), - 'NAMESPACE': 'stellar-supercluster', - 'job_name': lambda end, attempt: f"run-r{end}-a{attempt}", - 'metric_jobs_reaped': type('C', (), {'inc': lambda s: reaped.append(1)})(), - 'ApiException': ApiException, - 'logger': type('L', (), {'warning': lambda s, *a: warnings.append(a)})(), - } - exec(_extract(r"^(def delete_job\(.*?)(?=\ndef )").group(1), ns) - return ns['delete_job'], calls, warnings, reaped, ApiException - - -def test_delete_job_reaps_the_pod_too(): - # Background propagation is what actually removes the pod. Orphan/default - # would leave the pod behind and reap nothing that reconcile lists. - delete_job, calls, _, reaped, _ = _delete_job_ns(lambda name: None) - delete_job(30957951, 2) - assert calls == [('run-r30957951-a2', 'stellar-supercluster', - {'propagation_policy': 'Background'})] - assert len(reaped) == 1 - - -def test_delete_job_is_best_effort(): - # A 404 is the normal race with the TTL controller, not an error. Any other - # status must warn and keep going: losing a Job to a leaked object is a - # disk/etcd cost, but raising here would abort a reconcile pass mid-run and - # strand every other range in the same iteration. - _, ApiExc = None, None - for status, want_warn in ((404, False), (403, True), (500, True)): - delete_job, _, warnings, reaped, ApiException = _delete_job_ns( - lambda name, s=status: ApiException(s)) - delete_job(1, 1) # must not raise - assert bool(warnings) is want_warn, f"status {status}" - assert reaped == [], "a failed delete must not count as reaped" - - -def test_the_chart_grants_the_job_delete_reconcile_needs(): - # Same failure the PVC Role had: verbs omitted delete, so every reap logged - # a 403 and nothing was ever collected. - chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/templates/job_monitor.yaml')).read() - blk = _extract(r'resources: \["jobs"\]\s*\n\s*verbs: \[([^\]]+)\]', chart) - verbs = {v.strip().strip('"') for v in blk.group(1).split(',')} - assert 'delete' in verbs, f"delete_job needs delete, Role has {sorted(verbs)}" - - -def test_the_retry_creates_the_successor_before_deleting_the_predecessor(): - # Ordering is the whole safety argument: if the create fails with the - # predecessor already deleted, the range has no live Job, reconcile sees an - # undispatched range and redispatches at attempt 1 -- silently discarding - # the escalated memory limit the retry existed to apply. - body = _extract(r"(create_namespaced_job\(NAMESPACE, build_job\(\s*int\(end\), by_end\[end\], attempt \+ 1.*?)continue").group(1) - assert 'delete_job(end, attempt)' in body, "retry path never reaps the old attempt" - assert body.index('create_namespaced_job') < body.index('delete_job('), \ - "delete_job must come after the successor is created" - - -def test_a_success_whose_record_is_incomplete_keeps_its_job(): - # tx is read from the collector's .metrics, else the pod. Deleting the Job - # reaps the pod, so reaping a success before the metrics land turns a - # recoverable gap into a permanent one -- the same class of loss as the 698 - # ranges the tx_apply regex dropped. - body = _extract(r"(release_pvc\(end\)\n.*?)(?=\s+elif st\.failed:)").group(1) - assert '_reap_if_complete(end, attempt, completed[end])' in body, \ - "success path must gate the reap on the record being complete" - fn = _extract(r"^(def _reap_if_complete\(.*?)(?=\ndef )").group(1) - assert 'not _attempt_finalized(end, attempt)' in fn, \ - "the reap must wait for the collector's own done marker" - - -def test_the_chart_ttl_matches_the_code_default(): - # The TTL is now only a backstop, but a chart/code split is how the cache - # headroom regression shipped: the code default was fixed and the chart - # still forced the old value. - chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/values.yaml')).read() - want = int(_extract(r"JOB_TTL_SECONDS = int\(os\.getenv\('JOB_TTL_SECONDS', (\d+)\)\)").group(1)) - got = int(_extract(r"jobTtlSeconds: (\d+)", chart).group(1)) - assert got == want, f"chart sets {got}, code defaults to {want}" - - -# --- peak anon from kubelet ---------------------------------------------- -# Page cache expands to fill memory.max, so memory.peak ~= the limit for every -# pod and cannot be profiled (measured on ssc-test: a range needing 862 MiB of -# anon reported peak 12704 MiB under a 24000 MiB limit). Anon is the only -# limit-independent figure, and kubelet reports it per container for free in -# the payload the collector already fetches for ephemeral storage. - -def _sample_ns(summary, container='stellar-core', streaming=True): - """Exec sample_kubelet's per-pod body against one kubelet payload.""" - eph, anon, ws, flushed, streaming_ref, written, logged = {}, {}, {}, {}, {}, [], [] - - class FakeResp: - def __init__(self, d): self._d = d - async def __aenter__(self): return self - async def __aexit__(self, *a): return False - def raise_for_status(self): pass - async def json(self): return self._d - - class FakeSession: - def get(self, url, headers=None): return FakeResp(summary) - - ns = { - 'API': 'https://k8s', 'CONTAINER': container, - '_eph_peak': eph, '_anon_peak': anon, '_ws_peak': ws, - '_peak_flushed': flushed, '_streaming': streaming_ref, - 'PEAK_FLUSH_RATIO': 1.05, 'STORAGE_MODE': 'ephemeral', - 'write_metrics': lambda e, a, v: written.append((e, a, v)), - 'token': lambda: 't', - 'logger': type('L', (), {'warning': lambda s, *a: None, - 'info': lambda s, *a: logged.append(a)})(), - } - if streaming: - # The main loop records this when it opens a pod's stream; a peak flush - # needs it to know which .metrics file the pod belongs to. - for _p in summary.get('pods', []): - streaming_ref[_p['podRef']['name']] = ('999', '1') - exec(_extract(r"^(async def sample_kubelet\(.*?)(?=\n\nasync def )", - COLLECTOR_SRC).group(1), ns) - import asyncio - asyncio.run(ns['sample_kubelet'](FakeSession(), ['node-a'])) - _sample_ns.last = {'ws': ws, 'written': written, 'flushed': flushed} - return eph, anon - - -def _payload(pod, rss, used=None, container='stellar-core'): - mem = {} if rss is None else {'rssBytes': rss} - return {'pods': [{'podRef': {'name': pod}, - 'ephemeral-storage': {} if used is None else {'usedBytes': used}, - 'containers': [{'name': container, 'memory': mem}]}]} - - -def test_kubelet_anon_is_tracked_as_a_high_water_mark(): - # A single low sample after a high one must not lower the peak: the whole - # point is catching the spike, and download-phase anon oscillates. - eph, anon = _sample_ns(_payload('p1', 900, used=5)) - assert anon == {'p1': 900} and eph == {'p1': 5} - ns_hi = _payload('p1', 900) - ns_hi['pods'][0]['containers'][0]['memory']['rssBytes'] = 400 - # re-run with a lower reading against a pre-seeded peak - eph2, anon2 = _sample_ns({'pods': [ - _payload('p1', 900)['pods'][0], ns_hi['pods'][0]]}) - assert anon2['p1'] == 900, "a later, lower sample overwrote the peak" - - -def test_a_container_without_stats_yet_is_skipped_not_zeroed(): - # rssBytes is absent for the first seconds of a container's life. Recording - # 0, or letting it raise, would either poison the peak or kill the sampler - # for every other pod on the node. - eph, anon = _sample_ns(_payload('p1', None, used=7)) - assert anon == {}, "missing rssBytes must not be recorded" - assert eph == {'p1': 7}, "ephemeral must still be sampled" - - -def test_only_the_worker_container_is_measured(): - # Sidecars share the pod. Summing or last-wins across containers would size - # the range from whichever one kubelet listed last. - eph, anon = _sample_ns(_payload('p1', 900, container='istio-proxy')) - assert anon == {}, "a non-worker container was measured" - - -def test_peak_anon_is_kept_from_every_attempt(): - # An OOM-killed pod's last sample is below its true peak by construction -- - # it died reaching past it. Feeding that into the profile would re-derive - # the very limit that killed the range. - # peaks_for_range takes the max across a resumed chain, so a partial - # attempt can only raise the figure. Gating here is what hid the - # download-phase peak of a range that resumed. - body = _extract(r"(anon = _anon_peak\.pop\(pod, None\).*?)(?=\s+ws = _ws_peak)", - COLLECTOR_SRC).group(1) - assert 'done_ok(pod)' not in body, "peakAnonBytes is still gated on success" - - -def test_peak_anon_reaches_the_profile(): - # peaks_for_range filters to PEAK_FIELDS, and the ConfigMap mirror strips - # _PROFILE_ONLY_FIELDS. A new measurement absent from either is silently - # dropped between the collector and the profile. - for name in ('PEAK_FIELDS', '_PROFILE_ONLY_FIELDS'): - blk = _extract(name + r" = \(([^)]+)\)").group(1) - fields = {f.strip().strip("'") for f in blk.split(',') if f.strip()} - assert 'peakAnonBytes' in fields, f"{name} drops peakAnonBytes" - - -def test_sizing_prefers_anon_and_falls_back_to_the_scraped_rss(): - # A profile captured before the collector tracked anon must keep sizing - # exactly as it did, or every existing profile silently reverts to default. - body = _extract(r"(rss = prof\.get\('peakAnonBytes'\).*?out\['memory'\])").group(1) - assert "prof.get('peakAnonBytes') or prof.get('peakRssBytes')" in body - - -@pytest.mark.parametrize('peak,want_mi', [ - (648 * 1024**2, int(648 * 1.15) + 512), # measured live: anon 648Mi - (1467 * 1024**2, int(1467 * 1.15) + 512), # the largest anon sampled - (222 * 1024**2, int(222 * 1.15) + 512), # the smallest -]) -def test_the_sizing_formula_is_peak_times_115_plus_512mi(peak, want_mi): - margin = float(_extract(r"PROFILE_MARGIN = float\(os\.getenv\('PROFILE_MARGIN', ([\d.]+)\)\)").group(1)) - head = _extract(r"PROFILE_CACHE_HEADROOM = os\.getenv\('PROFILE_CACHE_HEADROOM', '(\d+)Mi'\)").group(1) - got_mi = int(peak * margin) // 1024**2 + int(head) - assert (margin, int(head)) == (1.15, 512) - assert got_mi == want_mi - - -def test_the_chart_matches_the_new_sizing_defaults(): - chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/values.yaml')).read() - assert _extract(r"profileMargin: ([\d.]+)", chart).group(1) == \ - _extract(r"PROFILE_MARGIN = float\(os\.getenv\('PROFILE_MARGIN', ([\d.]+)\)\)").group(1) - assert _extract(r'profileCacheHeadroom: "(\d+Mi)"', chart).group(1) == \ - _extract(r"PROFILE_CACHE_HEADROOM = os\.getenv\('PROFILE_CACHE_HEADROOM', '(\d+Mi)'\)").group(1) - - -# --- zombie streams ------------------------------------------------------- -# `done` reads terminal.get(pod, False) and terminal is only written for pods -# present in list_pods. A pod deleted while Running -- reaped node, eviction, -# or the monitor deleting a finished Job -- therefore never became terminal, -# and its stream retried every 30s for the rest of the run while holding one of -# MAX_CONCURRENT connection slots. - -def test_a_vanished_pod_is_marked_terminal_so_its_stream_can_finish(): - body = _extract(r"(live = \{p\['metadata'\]\['name'\].*?)(?=\n\s+# Unconditional:)", - COLLECTOR_SRC).group(1) - assert 'terminal[name] = True' in body, \ - "a vanished pod never becomes terminal, so done() stays False forever" - assert 'n not in live' in body, "nothing detects a pod leaving the pod list" - - -def test_a_vanished_stream_is_cancelled_if_it_will_not_finish(): - # Marking terminal is not enough on its own: a stream blocked inside a - # connection attempt never reaches its done() check, which is exactly the - # state that starves every other stream. - body = _extract(r"(live = \{p\['metadata'\]\['name'\].*?)(?=\n\s+# Unconditional:)", - COLLECTOR_SRC).group(1) - assert 't.cancel()' in body and 'VANISHED_GRACE_CYCLES' in body, \ - "no backstop cancel for a stream that cannot finalize" - assert 'del tasks[name]' in body, "cancelled task is never removed from tasks" - - -def test_the_grace_is_more_than_one_cycle(): - # A stream mid-fetch_peaks against a slow Prometheus must not be cancelled - # out from under its own metrics write. - n = int(_extract(r"VANISHED_GRACE_CYCLES = int\(os\.getenv\('COLLECTOR_VANISHED_GRACE_CYCLES', (\d+)\)\)", - COLLECTOR_SRC).group(1)) - assert n >= 2, f"grace of {n} cycle(s) can cancel a stream mid-finalize" - - - -def test_both_exit_paths_share_one_finalize(): - # Two copies of the metrics/discard logic is how one path silently stops - # writing peakAnonBytes while the other keeps working. - # Three: clean exit, pod-gone 404, and an interrupted read on a pod that - # has since gone terminal. - # Three: pod gone (404), the pod was terminal before the poll that just - # succeeded, and a terminal pod whose polls keep failing. - assert len(re.findall(r"await finalize\(session, pod, end, attempt, tx, done_ok, started\)", - COLLECTOR_SRC)) == 3 - assert len(re.findall(r"write_metrics\(end, attempt, measured\)", COLLECTOR_SRC)) == 1 - - - -def _run_stream_pod(status, terminal): - """Execute poll_pod against a fake apiserver. Returns finalize calls. - - Executed rather than pattern-matched: an earlier version of these tests - asserted on an `except ClientResponseError` branch that raise_for_status - could never reach, and passed against dead code. - """ - import asyncio, tempfile, os as _os, gzip as _gzip - calls = [] - - class FakeResp: - status = None - async def __aenter__(self): return self - async def __aexit__(self, *a): return False - def raise_for_status(self): - if self.status >= 400: - raise OSError(f"HTTP {self.status}") - @property - def content(self): - class C: - async def iter_chunked(self, n): - if False: - yield b'' - return C() - - FakeResp.status = status # class bodies cannot close over a local - - class FakeSession: - def get(self, url, params=None, headers=None): return FakeResp() - - async def fake_finalize(session, pod, end, attempt, tx, done_ok, started=None): - calls.append((pod, end, attempt)) - - d = tempfile.mkdtemp() - ns = { - 'asyncio': asyncio, 'gzip': _gzip, 're': re, 'os': _os, - 'API': 'https://k8s', 'NAMESPACE': 'ns', 'CONTAINER': 'stellar-core', - 'LOG_DIR': d, 'LOG_POLL_SECONDS': 0.05, 'MAX_POLL_CHARS': 1 << 20, - 'TERMINAL_POLL_ATTEMPTS': 3, - '_poll_slots': asyncio.Semaphore(4), '_wake': {}, - 'token': lambda: 't', 'finalize': fake_finalize, - 'base': lambda e, a: _os.path.join(d, f"range-{e}-a{a}"), - 'read_state': lambda e, a: None, 'write_state': lambda e, a, ts: None, - '_TS_RE': re.compile(r"^\d{4}"), - 'TxApplyScanner': type('T', (), {'seconds': None, 'resumed': False, - 'feed': lambda s, l: None}), - 'logger': type('L', (), {'info': lambda s, *a: None, - 'warning': lambda s, *a: None})(), - } - exec(_extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", - COLLECTOR_SRC).group(1), ns) - exec(_extract(r"^(async def poll_pod\(.*?)(?=\n\nasync def )", - COLLECTOR_SRC).group(1), ns) - coro = ns['poll_pod'](FakeSession(), 'pod-1', '999', '1', - lambda p: terminal, lambda p: False) - asyncio.run(asyncio.wait_for(coro, timeout=2)) - return calls - - -def test_a_404_finalizes_what_was_already_streamed(): - # The pod object is gone, but the bytes already read still owe a tx_apply - # and the peaks live in Prometheus, not on the pod. - assert _run_stream_pod(404, terminal=False) == [('pod-1', '999', '1')] - - -def test_an_interrupted_read_on_a_terminal_pod_still_finalizes(): - # 500s were a burst at ramp. Returning bare here dropped the metrics for - # every range whose last read happened to throw. - assert _run_stream_pod(500, terminal=True) == [('pod-1', '999', '1')] - - -def test_an_interrupted_read_on_a_live_pod_does_not_finalize(): - # Still running: retry is correct, and finalizing now would write a - # truncated peak and let the range look measured when it is not. - import pytest as _pt - with _pt.raises(Exception): - _run_stream_pod(500, terminal=False) # retries until the 2s timeout - - -# --- kubelet replaces Prometheus ------------------------------------------ -# Every peak the profile uses now comes from the kubelet payload the collector -# already fetches. Prometheus was lossy for this: a 30s scrape against ~10s -# cAdvisor housekeeping, plus a hard dependency on Prometheus being up, -# reachable and still retaining the window -- and _promql swallowed all three -# failures into "no peak", so an outage produced a complete-looking, empty -# profile. - -def test_the_collector_no_longer_reads_from_prometheus(): - # Comments stripped: one deliberately explains why the local high-water - # dict exists where max_over_time did not need to. - code = '\n'.join(l for l in COLLECTOR_SRC.splitlines() - if not l.lstrip().startswith('#')) - for token in ('PROMETHEUS_URL', '_promql', 'fetch_peaks', 'max_over_time'): - assert token not in code, f"{token} survived the kubelet switch" - - -def test_cpu_is_not_profiled(): - # REQ_CPU is fixed, so a measured cpu value has nothing to size and only - # makes packing non-uniform. - assert 'peakCpuCores' not in _extract(r"PEAK_FIELDS = \(([^)]+)\)").group(1) - body = _extract(r"^(def _profile_overrides\(.*?)(?=\ndef )").group(1) - assert "out['cpu']" not in body - - -def test_memory_is_sampled_in_both_storage_modes(): - # This was gated on ephemeral mode back when the sampler only did disk, - # which left every pvc run with no anon peak at all. - loop = _extract(r"while True:\n(.*?)await asyncio\.sleep\(POLL_SECONDS\)", - COLLECTOR_SRC).group(1) - call = loop.index('sample_kubelet') - gate = loop.rfind("STORAGE_MODE == 'ephemeral'", 0, call) - assert gate == -1, "the kubelet sampler is still gated on storage mode" - - -def test_the_disk_axis_stays_mode_gated(): - # ephemeral-storage is meaningless in pvc mode: /data is not on the node. - fn = _extract(r"^(async def sample_kubelet\(.*?)(?=\n\nasync def )", - COLLECTOR_SRC).group(1) - used = fn.index("get('usedBytes')") - assert "STORAGE_MODE == 'ephemeral'" in fn[used:used + 200] - - - - - -def test_an_in_flight_peak_is_flushed_so_a_restart_cannot_lose_it(): - # Prometheus computed max_over_time server-side and needed no state. A local - # high-water dict does: without a flush, a collector restart resets a range's - # peak to whatever it is using at that moment, which under-reports and sizes - # the next run too small. Executed, not pattern-matched -- `if False:` leaves - # every identifier in place and passes a source-text check. - _sample_ns(_payload('p1', 900, used=5)) - w = _sample_ns.last['written'] - assert w, "a first sample never flushed its peak" - assert w[-1][2] == {'peakAnonBytes': 900} - - -def test_a_peak_that_barely_grows_is_not_reflushed(): - # One write per sample per pod, at 2048 pods, would be the dominant cost of - # the sampler. Only growth past PEAK_FLUSH_RATIO earns a write. - pods = [_payload('p1', 900)['pods'][0], _payload('p1', 910)['pods'][0]] - _sample_ns({'pods': pods}) - assert len(_sample_ns.last['written']) == 1, "a 1.1% rise triggered a second flush" - - pods = [_payload('p1', 900)['pods'][0], _payload('p1', 2000)['pods'][0]] - _sample_ns({'pods': pods}) - assert len(_sample_ns.last['written']) == 2, "a 2.2x rise did not flush" - - -def test_working_set_is_sampled_recorded_but_never_sizes_anything(): - # It counts active page cache, which grows to fill the limit -- measured at - # 3.61/7.48/13.49 GiB for one range under 4Gi/8Gi/24000Mi limits while rss - # held at ~2.4 GiB. Useful as a diagnostic, never as a request. - p = _payload('p1', 900, used=5) - p['pods'][0]['containers'][0]['memory']['workingSetBytes'] = 4096 - _sample_ns(p) - assert _sample_ns.last['ws'] == {'p1': 4096}, "working set is not sampled" - assert 'peakWorkingSetBytes' in _extract(r"PEAK_FIELDS = \(([^)]+)\)").group(1) - body = _extract(r"^(def _profile_overrides\(.*?)(?=\ndef )").group(1) - assert 'peakWorkingSetBytes' not in body, "working set must not size a request" - - -def test_finalize_records_the_working_set_peak(): - # Sampling it is useless if finalize drops it on the floor. - fn = _extract(r"^(async def finalize\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - written, ws = [], {'pod-1': 4096} - ns = { - '_anon_peak': {'pod-1': 900}, '_ws_peak': ws, '_eph_peak': {}, - '_peak_flushed': {}, '_streaming': {}, '_wake': {}, '_pod_secs': {}, - 'SAVE_SUCCESS_LOGS': True, - 'write_metrics': lambda e, a, v: written.append(v), - 'discard': lambda e, a: None, '_mark_done': lambda e, a: None, - 'logger': type('L', (), {'info': lambda s, *a: None})(), - } - exec(fn, ns) - import asyncio - tx = type('T', (), {'seconds': 1.5, 'resumed': False})() - asyncio.run(ns['finalize'](None, 'pod-1', '999', '1', tx, lambda p: True)) - assert written and written[0].get('peakWorkingSetBytes') == 4096 - assert written[0].get('peakAnonBytes') == 900 - - -def test_the_flush_ratio_default_is_above_one_and_matches_the_chart(): - # The behaviour tests inject their own ratio, so nothing else pins the - # default. At exactly 1.0 every sample flushes: one write per pod per poll, - # 2048 pods, which is the cost the ratio exists to avoid. - got = float(_extract( - r"PEAK_FLUSH_RATIO = float\(os\.getenv\('PEAK_FLUSH_RATIO', ([\d.]+)\)\)", - COLLECTOR_SRC).group(1)) - assert got > 1.0, f"ratio {got} flushes on every sample" - chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/values.yaml')).read() - assert float(_extract(r"peakFlushRatio: ([\d.]+)", chart).group(1)) == got - - -# --- peaks aggregate across attempts -------------------------------------- -# In pvc mode a pod killed after replay starts leaves /data, and the next -# attempt resumes at LCL+1 with RESUME=true -- skipping the archive download and -# bucket apply, which is where peak memory happens. Profiling only the winning -# attempt therefore under-reports a resumed range by the whole download gap, and -# on spot (where eviction is routine and resume is the point of durable /data) -# that would make the run unprofileable. - -def _peaks_ns(attempts): - """Exec peaks_for_range over a temp dir. attempts: {n: (metrics, outcome)}. - - `resumed` lives in the metrics dict, as the collector writes it. - """ - import tempfile, json as _json, os as _os - d = tempfile.mkdtemp() - for n, (metrics, outcome) in attempts.items(): - if metrics is not None: - with open(_os.path.join(d, f"m-{n}"), 'w') as fh: - fh.write(metrics if isinstance(metrics, str) else _json.dumps(metrics)) - if outcome is not None: - with open(_os.path.join(d, f"o-{n}"), 'w') as fh: - _json.dump(outcome, fh) - ns = { - 'json': _json, - 'metrics_path': lambda e, n: _os.path.join(d, f"m-{n}"), - 'outcome_path': lambda e, n: _os.path.join(d, f"o-{n}"), - 'PEAK_FIELDS': ('peakAnonBytes', 'peakRssBytes', 'peakWorkingSetBytes', - 'peakEphemeralBytes'), - } - for name in ('read_outcome', '_attempt_resumed', '_resumed_chain', - '_hit_a_ceiling', '_peak_attempts'): - exec(_extract(r"^(def " + name + r"\(.*?)(?=\ndef )").group(1), ns) - exec(_extract(r"^(def peaks_for_range\(.*?)(?=\ndef )").group(1), ns) - return ns['peaks_for_range'] - - -def test_a_resumed_range_keeps_the_peak_from_the_attempt_that_did_the_download(): - # a1 evicted mid-replay having already done the download; a2 resumes at - # LCL+1 and only replays the tail. a2 alone would report 400MiB for a range - # that really needs 2GiB. - f = _peaks_ns({ - 1: ({'peakAnonBytes': 2 * 1024**3}, {'outcome': 'disrupted'}), - 2: ({'peakAnonBytes': 400 * 1024**2, 'resumed': True}, None), - }) - assert f(999, 2)['peakAnonBytes'] == 2 * 1024**3 - - -def test_an_oom_killed_attempt_still_counts_toward_the_peak(): - # It really did allocate ~8Gi and wanted more, so that is a lower bound on - # demand. Sizing off the quieter successful attempt instead would OOM the - # range again; 8Gi * 1.15 + 512Mi clears the level it died at. - f = _peaks_ns({ - 1: ({'peakAnonBytes': 8 * 1024**3}, {'outcome': 'oom'}), - 2: ({'peakAnonBytes': 900 * 1024**2, 'resumed': True}, None), - }) - assert f(999, 2)['peakAnonBytes'] == 8 * 1024**3 - - -def test_an_oom_killed_attempt_still_counts_on_the_disk_axis(): - # It hit the memory ceiling, not the disk one, so its disk figure is real. - f = _peaks_ns({ - 1: ({'peakEphemeralBytes': 30 * 1024**3}, {'outcome': 'oom'}), - 2: ({'peakEphemeralBytes': 5 * 1024**3, 'resumed': True}, None), - }) - assert f(999, 2)['peakEphemeralBytes'] == 30 * 1024**3 - - -def test_a_disk_evicted_attempt_counts_on_every_axis(): - f = _peaks_ns({ - 1: ({'peakEphemeralBytes': 40 * 1024**3, - 'peakAnonBytes': 3 * 1024**3}, {'outcome': 'ephemeral'}), - 2: ({'peakEphemeralBytes': 9 * 1024**3, - 'peakAnonBytes': 1 * 1024**3, 'resumed': True}, None), - }) - out = f(999, 2) - assert out['peakEphemeralBytes'] == 40 * 1024**3 - assert out['peakAnonBytes'] == 3 * 1024**3 - - -def test_a_missing_or_malformed_metrics_file_is_tolerated(): - f = _peaks_ns({1: (None, None), 2: ("not json at all", None), - 3: ({'peakAnonBytes': 5, 'resumed': True}, None)}) - assert f(999, 3) == {'peakAnonBytes': 5} - assert _peaks_ns({})(999, 3) == {} - - -def test_an_absent_peak_never_reaches_the_profile_as_a_null(): - # The consumer falls back to a default on a missing field, so a null defeats it. - f = _peaks_ns({1: ({'peakAnonBytes': None, 'peakRssBytes': 7}, None)}) - assert f(999, 1) == {'peakRssBytes': 7} - - -def test_spot_is_never_excluded_as_a_capacity_type(): - # Truncation is what invalidates a sample, not the node it ran on. Gating on - # spot would blank the axis for an all-spot run, the run we most want. - assert 'capacity-type' not in COLLECTOR_SRC - assert 'capacity-type' not in SRC - - -def test_a_fresh_retry_supersedes_an_interrupted_one(): - # No RESUME line means new-db ran and this attempt did the whole range, so - # its sample is complete. An earlier attempt that was merely interrupted - # measured the same work and only adds noise. - f = _peaks_ns({ - 1: ({'peakAnonBytes': 8 * 1024**3}, {'outcome': 'disrupted'}), - 2: ({'peakAnonBytes': 900 * 1024**2}, None), # no 'resumed' - }) - assert f(999, 2)['peakAnonBytes'] == 900 * 1024**2 - - -def test_the_chain_stops_at_the_last_fresh_start(): - # a1 interrupted then superseded by a fresh a2; a3 resumed from a2. Only - # a2+a3 describe the same continuous pass over the range. - f = _peaks_ns({ - 1: ({'peakAnonBytes': 9 * 1024**3}, {'outcome': 'disrupted'}), - 2: ({'peakAnonBytes': 2 * 1024**3}, {'outcome': 'disrupted'}), - 3: ({'peakAnonBytes': 500 * 1024**2, 'resumed': True}, None), - }) - assert f(999, 3)['peakAnonBytes'] == 2 * 1024**3 - - -def test_resumed_is_read_from_the_workers_own_line(): - # "RESUME DECLINED" must not count as a resume -- it means the opposite. - scanner_src = _extract(r"^(class TxApplyScanner:.*?)(?=\ndef )", COLLECTOR_SRC).group(1) - assert "RESUME_MARK = 'RESUME: '" in scanner_src - ns = {'_TX_METRIC': "metric 'ledger.transaction.apply'", '_SUM_RE': SUM_RE} - exec(scanner_src, ns) - s = ns['TxApplyScanner']() - s.feed("RESUME DECLINED: k last close was 'none'; bucket phase incomplete, starting fresh") - assert s.resumed is False, "a declined resume was read as a resume" - s.feed("RESUME: k reached ledger 31005951, replay had started; skipping new-db") - assert s.resumed is True - - -def test_resumed_never_reaches_the_profile_as_a_field(): - # It is bookkeeping for peaks_for_range, not a measurement. - assert 'resumed' not in _extract(r"PEAK_FIELDS = \(([^)]+)\)").group(1) - - -def test_finalize_records_that_an_attempt_resumed(): - # Without this in .metrics, peaks_for_range cannot tell a resumed tail from - # a complete pass, and every resumed range is profiled off its tail alone. - fn = _extract(r"^(async def finalize\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - import asyncio - - def run(resumed): - written = [] - ns = {'_anon_peak': {'p': 1}, '_ws_peak': {}, '_eph_peak': {}, - '_peak_flushed': {}, '_streaming': {}, '_wake': {}, '_pod_secs': {}, - 'SAVE_SUCCESS_LOGS': True, - 'write_metrics': lambda e, a, v: written.append(v), - 'discard': lambda e, a: None, '_mark_done': lambda e, a: None, - 'logger': type('L', (), {'info': lambda s, *a: None})()} - exec(fn, ns) - tx = type('T', (), {'seconds': None, 'resumed': resumed})() - asyncio.run(ns['finalize'](None, 'p', '999', '1', tx, lambda p: True)) - return written[0] - - assert run(True).get('resumed') is True - assert 'resumed' not in run(False), "a fresh attempt must not be marked resumed" - - -# --- timings aggregate across the resumed chain too ------------------------ -# medida's total is per-process and a pod's duration is its own, so both are -# tail-only for a resumed range in exactly the way the peaks were. - -def _chain_ns(attempts, extra=None): - """Exec the chain helpers over a temp dir. attempts: {n: (metrics, outcome)}.""" - import tempfile, json as _json, os as _os - d = tempfile.mkdtemp() - for n, (metrics, outcome) in attempts.items(): - if metrics is not None: - with open(_os.path.join(d, f"m-{n}"), 'w') as fh: - _json.dump(metrics, fh) - if outcome is not None: - with open(_os.path.join(d, f"o-{n}"), 'w') as fh: - _json.dump(outcome, fh) - ns = { - 'json': _json, - 'metrics_path': lambda e, n: _os.path.join(d, f"m-{n}"), - 'outcome_path': lambda e, n: _os.path.join(d, f"o-{n}"), - } - for name in ('_attempt_resumed', '_resumed_chain', 'read_outcome', - 'seconds_for_range'): - ns[name] = None - exec(_extract(r"^(def _attempt_resumed\(.*?)(?=\ndef )").group(1), ns) - exec(_extract(r"^(def _resumed_chain\(.*?)(?=\ndef )").group(1), ns) - exec(_extract(r"^(def read_outcome\(.*?)(?=\ndef )").group(1), ns) - exec(_extract(r"^(def seconds_for_range\(.*?)(?=\ndef )").group(1), ns) - ns.update(extra or {}) - return ns - - -def test_seconds_sums_the_whole_resumed_chain(): - # a1 ran 900s then was evicted mid-replay; a2 resumed and took 300s. The - # range cost 1200s of compute, not 300. - ns = _chain_ns({ - 1: ({}, {'outcome': 'disrupted', 'attemptSeconds': 900.0}), - 2: ({'resumed': True}, None), - }) - assert ns['seconds_for_range'](999, 2, 300.0) == 1200.0 - - -def test_seconds_ignores_attempts_before_a_fresh_start(): - # a2 ran new-db and did the whole range itself, so a1's 900s is not part of - # the same pass. - ns = _chain_ns({ - 1: ({}, {'outcome': 'oom', 'attemptSeconds': 900.0}), - 2: ({}, None), # no 'resumed' - }) - assert ns['seconds_for_range'](999, 2, 300.0) == 300.0 - - -def test_seconds_survives_a_leg_with_no_recorded_duration(): - # An attempt whose pod vanished before it was classified has no - # attemptSeconds. Better to under-report one leg than return nothing. - ns = _chain_ns({ - 1: ({}, {'outcome': 'disrupted'}), # no attemptSeconds - 2: ({'resumed': True}, None), - }) - assert ns['seconds_for_range'](999, 2, 300.0) == 300.0 - - -def test_seconds_is_none_when_nothing_is_known(): - ns = _chain_ns({1: ({}, None)}) - assert ns['seconds_for_range'](999, 1, None) is None - - -def test_a_failed_attempts_duration_is_persisted_with_its_verdict(): - # The only moment it is available: reconcile computes `seconds` solely on - # the success path, and the pod is about to be reaped. - fn = _extract(r"^(def record_outcome\(.*?)(?=\ndef )").group(1) - assert "data['attemptSeconds'] = _pod_seconds(pod)" in fn - - -def test_tx_apply_sums_the_chain_and_offers_fallbacks_to_the_last_leg_only(): - # pod_name names the winning attempt's pod; handing it to an earlier leg - # would read the wrong pod's log. - fn = _extract(r"^(def tx_apply_for_range\(.*?)(?=\ndef )").group(1) - assert '_resumed_chain(end, attempt)' in fn - assert 'pod_name if n == int(attempt) else None' in fn - assert 'total + leg' in fn, "legs are summed, not maxed" - - -def test_an_unclassifiable_job_failure_is_retried_not_condemned(): - # BackoffLimitExceeded carries no rule index and no exit code, so classify() - # honestly returns nothing. That must not read as "this range is bad": a - # monitor restart while a node was reaped produces exactly this, and - # condemning on it would fail a 10-hour job on no evidence. - assert classify("Job has reached the specified backoff limit") == (None, None, None) - env = set(re.findall(r"'(\w+)'", _extract(r"ENVIRONMENTAL_OUTCOMES = \(([^)]+)\)").group(1))) - assert 'unknown' in env, "an unclassified failure must get the environmental budget" - assert {'disrupted', 'rejected'} <= env, "cluster-caused outcomes share that budget" - # ...and the environmental budget is the most generous of the three. - body = _extract(r"(if verdict\['outcome'\] == 'timeout':\s*\n\s*cap = .*?)(?=\n\s+if reason)").group(1) - assert 'ENVIRONMENTAL_OUTCOMES' in body and 'MAX_DISRUPTION_ATTEMPTS' in body - - -def test_only_a_genuine_catchup_failure_is_condemned(): - # `failed` is the one outcome with no retry reason. Everything else -- oom, - # ephemeral, timeout, and all three environmental outcomes -- sets one. - body = _extract(r"(if verdict\['outcome'\] == 'timeout':.*?reason = None[^\n]*)").group(1) - assert body.rstrip().endswith("reason = None # genuine catchup failure: do not retry"), \ - "a genuine catchup failure must be the only unretried outcome" - # every other branch in that chain sets a reason, i.e. retries - for outcome in ('rejected', 'disrupted', 'oom', 'ephemeral', 'unknown'): - assert f"== '{outcome}'" in body, f"{outcome} left the retry chain" - - -# --- gaps found by a mutation sweep, 2026-07-30 ---------------------------- -# Each of these guards a decision the design depends on, and each was mutable -# without breaking a single test before this block existed. - -def test_the_job_controller_never_owns_retries(): - # backoffLimit 0 is load-bearing: above 0 the Job controller replaces the pod - # on its own schedule, so we could not classify disruption vs catchup - # failure, could not count evictions, and could not guarantee the log was - # archived before the next attempt started. - spec = _extract(r"spec=client\.V1JobSpec\((.*?)template=").group(1) - assert re.search(r"backoff_limit\s*=\s*0\b", spec), "backoffLimit must be 0" - assert re.search(r"ttl_seconds_after_finished\s*=\s*JOB_TTL_SECONDS", spec), \ - "finished Jobs need a TTL backstop even though reconcile deletes them" - - -def test_a_worker_pod_is_never_restarted_in_place(): - # restartPolicy OnFailure restarts the container inside the same pod, which - # keeps the pod name and reuses the same resource limits -- so an OOM would - # loop forever at the limit that killed it instead of escalating, and the - # attempt counter would never advance. - spec = _extract(r"spec=client\.V1PodSpec\((.*?)containers=\[container\]").group(1) - assert "restart_policy='Never'" in spec - - -@pytest.mark.parametrize('attempt,want', [(1, 1.0), (2, 1.5), (3, 2.25), (4, 3.375)]) -def test_the_memory_escalation_ladder_compounds(attempt, want): - # 1.5x per attempt off what the attempt actually ran with. A factor of 1.0 - # would retry an OOM at the identical limit, forever. - ns = {'os': __import__('os'), 're': re} - for name in ('_quantity_bytes', '_bytes_to_quantity', 'mem_for_attempt'): - exec(_extract(r"^(def " + name + r"\(.*?)(?=\ndef )").group(1), ns) - ns['_UNITS'] = {'Ki': 1024, 'Mi': 1024**2, 'Gi': 1024**3, 'Ti': 1024**4, - 'K': 1000, 'M': 1000**2, 'G': 1000**3, 'T': 1000**4} - ns['MEM_BUMP_FACTOR'] = float(_extract( - r"MEM_BUMP_FACTOR = float\(os\.getenv\('MEM_BUMP_FACTOR', ([\d.]+)\)\)").group(1)) - ns['MEM_ESCALATION_CAP'] = '48Gi' - ns['LIM_MEM'] = '1000Mi' - got = ns['mem_for_attempt'](attempt, '1000Mi') - assert got == f"{int(1000 * want)}Mi", got - - -def test_the_escalation_ladder_is_capped(): - ns = {'os': __import__('os'), 're': re} - for name in ('_quantity_bytes', '_bytes_to_quantity', 'mem_for_attempt'): - exec(_extract(r"^(def " + name + r"\(.*?)(?=\ndef )").group(1), ns) - ns['_UNITS'] = {'Ki': 1024, 'Mi': 1024**2, 'Gi': 1024**3, 'Ti': 1024**4, - 'K': 1000, 'M': 1000**2, 'G': 1000**3, 'T': 1000**4} - ns['MEM_BUMP_FACTOR'] = 1.5 - ns['MEM_ESCALATION_CAP'] = '4Gi' - ns['LIM_MEM'] = '1000Mi' - assert ns['mem_for_attempt'](20, '1000Mi') == '4096Mi', "cap not applied" - - -def test_progress_is_written_atomically(): - # The mission reads progress.json off the volume while the monitor is still - # writing it. A partial file is unparseable JSON, which reads as "no - # progress" -- and reconcile halts the run when progress goes backwards. - fn = _extract(r"^(def save_progress\(.*?)(?=\ndef )").group(1) - assert '.tmp' in fn and 'os.replace(' in fn, "progress.json is not written atomically" - assert fn.index('.tmp') < fn.index('os.replace('), "replace must follow the temp write" - - -def test_the_log_stream_resumes_from_the_last_durable_timestamp(): - # Without sinceTime a reconnect re-reads the whole log from the start: one - # full re-read per pod per reconnect, at 2096 pods. - fn = _extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert "params['sinceTime']" in fn, "a poll does not resume from the last durable line" - # ...and the second-granularity overlap it creates is removed per line. - assert re.search(r"if last_ts and ts <= last_ts:\s*\n\s*continue", fn), \ - "the deliberate resume overlap is never deduped" - - -def test_every_durable_write_is_atomic(): - # Three writers put files on the shared volume while the mission and the - # collector read them. A half-written .outcome or archive is unparseable, - # and an unreadable outcome downgrades a classified failure to "unknown". - for fn_name in ('save_progress', 'backstop_save_pod_log', 'record_outcome', - 'write_metrics'): - for src in (SRC, COLLECTOR_SRC): - m = re.search(r"^(def " + fn_name + r"\(.*?)(?=\ndef )", src, re.S | re.M) - if m: - break - assert m, f"{fn_name} not found" - body = m.group(1) - assert '.tmp' in body, f"{fn_name} does not write via a temp file" - assert 'os.replace(' in body, f"{fn_name} does not rename atomically" - assert body.index('.tmp') < body.index('os.replace('), \ - f"{fn_name} renames before it writes" - - -def test_attempt_budgets_are_ordered_by_whose_fault_the_failure_was(): - # A hang is usually persistent, so it gets the fewest tries. A genuinely - # broken range gets the middle budget. Anything the cluster did to us gets - # the most -- on spot, evictions are routine and must not condemn a range. - def const(name, env): - return int(_extract(name + r" = int\(os\.getenv\('" + env + r"', (\d+)\)\)").group(1)) - timeout = const('MAX_TIMEOUT_ATTEMPTS', 'MAX_TIMEOUT_ATTEMPTS') - per_range = const('MAX_ATTEMPTS_PER_RANGE', 'MAX_ATTEMPTS') - disruption = const('MAX_DISRUPTION_ATTEMPTS', 'MAX_DISRUPTION_ATTEMPTS') - ephemeral = const('MAX_EPHEMERAL_ATTEMPTS', 'MAX_EPHEMERAL_ATTEMPTS') - assert timeout < per_range < disruption, \ - f"budgets out of order: timeout={timeout} range={per_range} disruption={disruption}" - assert per_range > 1, "a range that OOMs once could never escalate" - assert ephemeral > 1, "a range evicted on disk once could never grow" - assert disruption >= 10, "spot eviction would condemn ranges at this budget" - - -def test_the_collector_records_a_duration_the_monitor_cannot(): - # Measured on ssc-test 2026-07-30: 212 of 212 spot disruptions were - # classified from the Job condition with the pod already reaped, so - # record_outcome never ran and no .outcome carried attemptSeconds. Peaks - # survived (the collector writes .metrics regardless) but the chain's time - # total silently lost every evicted leg. This process watched the container - # run, so it is the only observer left. - fn = _extract(r"^(async def finalize\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert "measured['attemptSeconds']" in fn - import asyncio - written = [] - ns = {'asyncio': asyncio, '_anon_peak': {}, '_ws_peak': {}, '_eph_peak': {}, - '_peak_flushed': {}, '_streaming': {}, '_wake': {}, '_pod_secs': {}, - 'SAVE_SUCCESS_LOGS': True, - 'write_metrics': lambda e, a, v: written.append(v), - 'discard': lambda e, a: None, '_mark_done': lambda e, a: None, - 'logger': type('L', (), {'info': lambda s, *a: None})()} - exec(fn, ns) - tx = type('T', (), {'seconds': None, 'resumed': False})() - async def go(): - now = asyncio.get_event_loop().time() - await ns['finalize'](None, 'p', '999', '1', tx, lambda p: True, now - 42.0) - asyncio.run(go()) - assert written and written[0]['attemptSeconds'] == pytest.approx(42.0, abs=1.0) - - -def test_seconds_falls_back_to_the_collectors_figure(): - # The authoritative .outcome is missing for every reaped pod. Without this - # fallback the chain drops that leg entirely and under-reports the range. - ns = _chain_ns({ - 1: ({'attemptSeconds': 850.0}, None), # no .outcome at all - 2: ({'resumed': True}, None), - }) - assert ns['seconds_for_range'](999, 2, 300.0) == 1150.0 - - -def test_the_authoritative_outcome_wins_over_the_collector_estimate(): - # .outcome comes from the pod's terminated timestamps; the collector's is a - # stream-lifetime approximation that starts up to one poll late. - ns = _chain_ns({ - 1: ({'attemptSeconds': 850.0}, {'outcome': 'disrupted', 'attemptSeconds': 900.0}), - 2: ({'resumed': True}, None), - }) - assert ns['seconds_for_range'](999, 2, 300.0) == 1200.0 - - -# --- a worker must not be able to kill its own log stream ------------------ -# Found live on the 2096-worker spot run: the AWS CLI draws its transfer meter -# with carriage returns and no newline, so a 628 MiB bucket download arrives as -# one multi-megabyte "line". aiohttp raises over 512 KiB, every large download -# killed its own stream, and the reconnect hit the same wall -- which starved -# every retry pod of a collector stream and left a2 metrics empty. - -def test_the_stream_is_read_in_chunks_not_lines(): - fn = _extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert 'iter_chunked' in fn, "line-wise reads are bounded by aiohttp's 512KiB limit" - assert 'async for raw in resp.content:' not in fn - assert "'follow'" not in fn, "a poll must not follow" - - -def test_carriage_returns_split_lines_too(): - # The progress meter is \r-delimited. Without \r in the split it stays one - # blob no matter how the bytes arrive. - fn = _extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - m = re.search(r"re\.split\(r'\[([^\]]+)\]'", fn) - assert m, "no line splitting found" - assert '\\r' in m.group(1) and '\\n' in m.group(1), f"splits on {m.group(1)!r}" - - -def test_an_unterminated_blob_is_capped_not_buffered_forever(): - # A meter that never emits a newline would otherwise grow the buffer until - # the collector OOMs -- 2096 streams doing it at once. - fn = _extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert 'MAX_POLL_CHARS' in fn, "a single poll response is unbounded" - - -def test_the_worker_disables_the_aws_progress_meter(): - # The real cure: never emit the \r spam. Also keeps it out of the archives, - # where it was the bulk of every large range's log. - fs = open(os.path.join(_SRC_DIR, os.pardir, - 'FSLibrary', 'MissionHistoryPubnetParallelCatchupV2.fs')).read() - m = re.search(r'sprintf "aws s3 cp ([^"]*)--region %s"', fs) - assert m, "s3 GET command not found" - assert '--no-progress' in m.group(1), f"aws s3 cp flags: {m.group(1)!r}" - - -def test_a_failed_attempt_is_not_reaped_before_the_collector_finalizes_it(): - # delete_job reaps the pod, and backstop_save_pod_log stands down for any - # range the collector claimed -- so nothing else would ever read that log. - # Under follow=true the collector already holds everything; under polling it - # would lose the last interval. Gate it either way. - body = _extract(r"(try:\s*\n\s*batch_v1\.create_namespaced_job.*?)continue").group(1) - assert 'delete_job(end, attempt)' in body - assert re.search(r"if _attempt_finalized\(end, attempt\):\s*\n\s*delete_job\(end, attempt\)", body), \ - "the retry-path reap is not gated on the collector's done marker" - assert body.index('create_namespaced_job') < body.index('delete_job('), \ - "successor must exist before the predecessor is reaped" - - -# --- polling replaces follow=true ------------------------------------------ -# Measured on ssc-test at 2096 follow streams: 1444 MiB of a 2048 MiB limit, -# memory.events max=2617, 1.00 of 2 cpu, 1797 held connections. That scales -# with pod count, so 4096 exceeds both limits. Polling makes concurrency a -# tuning parameter instead. - -def test_concurrency_is_independent_of_pod_count(): - # The whole point. Under follow=true the cap had to exceed parallelism or - # pods starved silently -- 1200 against 2048 left 896 blocked forever. - assert 'COLLECTOR_MAX_STREAMS' not in COLLECTOR_SRC - chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/templates/job_monitor.yaml')).read() - assert 'COLLECTOR_MAX_STREAMS' not in chart - assert 'worker.replicas' not in chart.split('MAX_CONCURRENT_POLLS')[1][:200], \ - "poll concurrency must not be derived from parallelism" - - -def test_polls_are_bounded_by_a_semaphore(): - fn = _extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert 'async with _poll_slots:' in fn, "polls are not bounded" - # ...and the connector is sized for polls, not for one socket per pod. - assert 'MAX_CONCURRENT_POLLS + 64' in COLLECTOR_SRC - - -def test_the_archive_is_not_held_open_between_polls(): - # A live gzip deflate buffer per stream is most of what put the sidecar at - # 1444 MiB. The member is built in a function-local buffer and appended in - # one write, so nothing is retained between polls. - # - # Asserts the invariant rather than the call: this test used to pin the - # literal gzip.open(..., 'at'), which the atomic-append fix replaced, and it - # went red over a change that preserved everything it existed to protect. - fn = _extract(r"^(async def _poll_once\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert 'io.BytesIO()' in fn and 'gzip.GzipFile' in fn, \ - "the member is no longer built in a function-local buffer" - loop = _extract(r"^(async def poll_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - for held in ('gzip.open', 'gzip.GzipFile', 'io.BytesIO'): - assert held not in loop, f"the archive is held across polls ({held})" - - -def test_terminal_is_read_before_the_poll_not_after(): - # A pod that exits mid-poll would otherwise have its final output dropped: - # the poll that read it would not yet know the pod was terminal, and the - # next check would come after finalize. - loop = _extract(r"^(async def poll_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert loop.index('was_terminal = done(pod)') < loop.index('await _poll_once('), \ - "terminal is sampled after the poll, which races a pod exiting mid-poll" - - -def test_a_dead_pod_is_not_polled_forever(): - # Its log is not coming back, and the task holds a poll slot for the rest of - # the run. follow=true finalized here because it already held the bytes. - loop = _extract(r"^(async def poll_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert 'TERMINAL_POLL_ATTEMPTS' in loop - n = int(_extract(r"TERMINAL_POLL_ATTEMPTS = int\(os\.getenv\('TERMINAL_POLL_ATTEMPTS', (\d+)\)\)", - COLLECTOR_SRC).group(1)) - assert n >= 2, "a single transient 500 would end the attempt" - - -def test_a_terminal_pod_whose_polls_keep_failing_still_finalizes(): - # Executed: the loop must exit, not spin. Was a real regression when polling - # replaced streaming -- the suite caught it. - assert _run_stream_pod(500, terminal=True) == [('pod-1', '999', '1')] - - -def test_poll_concurrency_default_is_modest(): - n = int(_extract(r"MAX_CONCURRENT_POLLS = int\(os\.getenv\('MAX_CONCURRENT_POLLS', (\d+)\)\)", - COLLECTOR_SRC).group(1)) - assert 16 <= n <= 256, f"{n} in-flight polls is not a sane default" - - -def test_a_pending_pod_is_not_polled_yet(): - # Its container has not started, so the log endpoint answers 400 and the - # poll is wasted -- 60 of 88 failures immediately after the polling switch. - loop = _extract(r"while True:\n(.*?)await asyncio\.sleep\(POLL_SECONDS\)", - COLLECTOR_SRC).group(1) - per_pod = loop[loop.index('for pod in pods:'):] - # From the attempt lookup to the stream open: the earlier - # terminal[name] = phase in ('Succeeded', 'Failed') line is not this guard. - guard = per_pod[per_pod.index('attempt = labels.get'):per_pod.index('_streaming[name]')] - assert 'phase not in POLLABLE_PHASES' in guard, \ - "pollability is not decided by an allowlist" - allowed = set(re.findall(r"'(\w+)'", _extract( - r"POLLABLE_PHASES = \(([^)]+)\)", COLLECTOR_SRC).group(1))) - # Terminal phases must stay in -- that is where a pod's final output lives. - assert {'Running', 'Succeeded', 'Failed'} == allowed, allowed - # Pending has no container; Unknown means the node stopped reporting. - assert 'Pending' not in allowed and 'Unknown' not in allowed - - -def test_late_peaks_are_backfilled_into_a_completed_record(): - # The record is written the moment the Job flips to succeeded, usually - # before the collector finalizes. peaks_for_range has no fallback the way - # tx_apply does, so a one-shot read loses them: measured on ssc-test, 356 of - # 356 completed ranges had txApply and 0 had peakAnonBytes, while 1936 - # .metrics files on the same volume held it. - body = _extract(r"(elif \(not _has_peaks\(completed\[end\]\).*?)(?=\n\s+elif st\.failed:)").group(1) - assert 'peaks_for_range(end, attempt)' in body, "no retry of the peak read" - assert 'save_progress(progress)' in body, "a backfilled peak is never persisted" - assert '_reap_if_complete' in body, "backfill never lets the Job go" - assert '_attempt_finalized(end, attempt)' in body, \ - "backfill stops retrying before the collector has finished" - - -def test_the_reap_waits_for_the_collectors_done_marker(): - # Not inferred from peaks or tx_apply: tx_apply falls back to the archive so - # it lands long before the collector finishes, and an attempt can finalize - # with no peaks at all. Only the collector knows it is done. - import tempfile, os as _os - d = tempfile.mkdtemp() - ns = {'os': _os, 'LOG_DIR': d, 'PEAK_FIELDS': ('peakAnonBytes',), 'reaped': []} - # Completion is terminal for the RANGE, so the reap is range-scoped now. - ns['reap_range_jobs'] = lambda e: ns['reaped'].append((e, 1)) - for name in ('done_path', '_attempt_finalized', '_has_peaks', '_reap_if_complete'): - exec(_extract(r"^(def " + name + r"\(.*?)(?=\ndef )").group(1), ns) - full = {'txApply': 5.0, 'peakAnonBytes': 99} - ns['_reap_if_complete'](1, 1, full) - assert ns['reaped'] == [], "reaped before the collector marked it done" - open(_os.path.join(d, 'range-1-a1.done'), 'w').close() - ns['_reap_if_complete'](1, 1, full) - assert ns['reaped'] == [(1, 1)], ns['reaped'] - - -def test_the_done_marker_is_written_after_everything_else(): - # It licenses the monitor to reap the pod, which is the only place peaks can - # still be read from. Written before .metrics it would authorise exactly the - # reap it exists to prevent. - fn = _extract(r"^(async def finalize\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert '_mark_done(end, attempt)' in fn - assert fn.index('write_metrics(end, attempt, measured)') < fn.index('_mark_done('), \ - "the done marker precedes the metrics it certifies" - assert fn.rstrip().endswith('_mark_done(end, attempt)'), \ - "the done marker is not the last thing finalize does" - - -def test_the_done_marker_is_written_atomically_and_is_best_effort(): - fn = _extract(r"^(def _mark_done\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert '.tmp' in fn and 'os.replace(' in fn, "a half-written marker would be truthy" - assert 'except OSError' in fn, "a failed marker must not kill the stream" - import tempfile, os as _os - d = tempfile.mkdtemp() - ns = {'os': _os, - 'base': lambda e, a: _os.path.join(d, f"range-{e}-a{a}"), - 'logger': type('L', (), {'warning': lambda s, *a: None})()} - exec(_extract(r"^(def done_path\(.*?)(?=\ndef )", COLLECTOR_SRC).group(1), ns) - exec(fn, ns) - ns['_mark_done'](77, 2) - assert _os.path.exists(_os.path.join(d, 'range-77-a2.done')) - assert not _os.path.exists(_os.path.join(d, 'range-77-a2.done.tmp')) - - -def test_both_sides_agree_on_the_marker_path(): - # Two processes, one volume, one filename. A mismatch would mean the monitor - # never reaps and every Job waits out its TTL. - c = _extract(r"^(def done_path\(.*?)(?=\ndef )", COLLECTOR_SRC).group(1) - m = _extract(r"^(def done_path\(.*?)(?=\n\ndef )").group(1) - assert ".done" in c and ".done" in m - assert 'range-' in m and 'base(end, attempt)' in c - - -def test_a_terminal_pod_wakes_its_poller_immediately(): - # The delay that matters is between the container exiting and the last read. - # Sleeping blind for LOG_POLL_SECONDS hands that window to a spot reclaim, - # which deletes the pod and takes the final lines with it. - loop = _extract(r"while True:\n(.*?)await asyncio\.sleep\(POLL_SECONDS\)", - COLLECTOR_SRC).group(1) - # Structure, not just presence: mutating the guard to `if False:` leaves - # the .set() line in place and sails past a substring check. - # The duration capture now sits between the assignment and the wake, so - # check ordering and the guard rather than adjacency. - i_assign = loop.index('terminal[name] = phase in') - i_guard = loop.index('if terminal[name] and name in _wake:') - # search AFTER the guard: the vanished-pod block also calls .set() and sits - # earlier in the loop, so a bare index() finds the wrong one. - i_set = loop.index('_wake[name].set()', i_guard) - assert i_assign < i_guard < i_set, "a pod going terminal does not wake its poller" - poller = _extract(r"^(async def poll_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert 'asyncio.wait_for(' in poller and '_wake.setdefault' in poller, \ - "the poller still sleeps blind between polls" - assert 'await asyncio.sleep(backoff)' not in poller - - -def test_the_wake_entry_is_dropped_when_the_attempt_finishes(): - # One entry per pod, and pods are per range per attempt -- 3979 ranges with - # retries would otherwise accumulate for the life of the run. - fn = _extract(r"^(async def finalize\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert '_wake.pop(pod, None)' in fn - - -def test_a_vanished_pod_also_wakes_its_poller(): - # Gone is terminal. Without the wake its poller sleeps out the interval - # before taking the 404, delaying finalize and the .done the monitor needs - # before it can reap the Job. - loop = _extract(r"while True:\n(.*?)await asyncio\.sleep\(POLL_SECONDS\)", - COLLECTOR_SRC).group(1) - blk = loop[loop.index('n not in live'):loop.index('if STORAGE_MODE') if 'if STORAGE_MODE' in loop else loop.index('for pod in pods:')] - assert 'terminal[name] = True' in blk - assert re.search(r"if name in _wake:\s*\n(?:\s*#[^\n]*\n)*\s*_wake\[name\]\.set\(\)", blk), \ - "a vanished pod never wakes its poller" - - -def test_both_reap_paths_wait_for_the_same_marker(): - # Success and retry must agree. Gating one on peaks and the other on the - # marker means an attempt with no peaks is reaped on one path and left to - # the TTL on the other. - assert len(re.findall(r"_attempt_finalized\(end, attempt\)", SRC)) >= 2 - assert 'if peaks_for_range(end, attempt):' not in SRC, \ - "a reap still uses peaks as a proxy for the collector being done" - - -def test_a_ceiling_hit_survives_a_fresh_start(): - # An OOM peak is evidence about the range whichever pass produced it: the - # process really did allocate that much and want more. Dropping it breaks - # the self-correcting loop -- a range that OOMs at L must record L so that - # L * margin + headroom clears it next run. Measured on ssc-30: an OOM in - # replay resumes and stays in the chain (224 of 252), an OOM in download - # does not (25 of 252), and a higher-cpu run is download-bound. - f = _peaks_ns({ - 1: ({'peakAnonBytes': 8 * 1024**3}, {'outcome': 'oom'}), - 2: ({'peakAnonBytes': 900 * 1024**2}, None), # fresh start - }) - assert f(999, 2)['peakAnonBytes'] == 8 * 1024**3 - - -def test_a_disk_ceiling_hit_survives_a_fresh_start_too(): - f = _peaks_ns({ - 1: ({'peakEphemeralBytes': 40 * 1024**3}, {'outcome': 'ephemeral'}), - 2: ({'peakEphemeralBytes': 9 * 1024**3}, None), - }) - assert f(999, 2)['peakEphemeralBytes'] == 40 * 1024**3 - - -def test_the_ceiling_exception_is_peaks_only(): - # tx_apply and seconds are summed, and a fresh start redoes the work the - # dropped attempt already did, so counting it there would double-count. - for fn_name in ('tx_apply_for_range', 'seconds_for_range'): - fn = _extract(r"^(def " + fn_name + r"\(.*?)(?=\ndef )").group(1) - assert '_resumed_chain(end, attempt)' in fn, f"{fn_name} lost the chain" - assert '_peak_attempts' not in fn, f"{fn_name} would double-count redone work" - peaks = _extract(r"^(def peaks_for_range\(.*?)(?=\ndef )").group(1) - assert '_peak_attempts(end, attempt)' in peaks - - -def test_the_worker_pod_carries_its_attempt_number(): - # The collector reads LABEL_ATTEMPT off the POD, not the Job, and defaults - # to "1". With the label only on the Job every attempt claimed the same - # range--a1.* files: measured on ssc-test 2026-07-30, 2246 metrics - # files all a1 while 475 a2 pods ran, so each retry overwrote the first - # attempt's peak instead of being maxed against it -- destroying exactly - # the OOM evidence the chain exists to keep. - fn = _extract(r"^(def pod_labels\(.*?)(?=\ndef )").group(1) - # The dict itself, not the docstring -- which names LABEL_ATTEMPT while - # explaining why it must be there, and made an earlier version of this - # assertion pass against a pod_labels that had dropped it. - body = fn[fn.index('labels = {'):] - assert re.search(r"LABEL_ATTEMPT: str\(attempt\)", body), \ - "the pod template omits the attempt label" - assert re.match(r"def pod_labels\(end, attempt\)", fn), \ - "pod_labels does not take the attempt" - assert 'metadata=client.V1ObjectMeta(labels=pod_labels(end, attempt))' in SRC - # and the collector's default is what makes the omission silent - assert re.search(r"labels\.get\(LABEL_ATTEMPT, '1'\)", COLLECTOR_SRC), \ - "collector no longer defaults the attempt -- update this test" - - -def test_pod_and_job_agree_on_the_attempt_label_key(): - # Two readers, one key. A mismatch reproduces the same silent collision. - assert _extract(r"LABEL_ATTEMPT = '([^']+)'").group(1) == \ - _extract(r"LABEL_ATTEMPT = '([^']+)'", COLLECTOR_SRC).group(1) - - -def test_no_worker_gets_a_cpu_limit_unless_one_is_configured(): - # _profile_overrides returns {} for BOTH "no profile entry" and "escalated - # attempt". Treating them the same handed an OOM retry more memory while - # capping it at LIM_CPU, when the attempt that just failed ran unlimited. - # Measured on ssc-test 2026-07-30: 256 of 679 a2 pods were capped at cpu 2. - # Less cpu means less download concurrency means a lower peak, so the retry - # succeeds at a figure the next run cannot reproduce unthrottled. - ns = _resources_ns(PROFILE_RANGES) - first = ns['_resources'](end=2000) - retry = ns['_resources'](mem='9000Mi', end=2000) # escalated - assert 'cpu' not in first.limits, first.limits - assert 'cpu' not in retry.limits, f"escalated retry was throttled: {retry.limits}" - assert retry.requests['memory'] == retry.limits['memory'] == '9000Mi' - # ...and an unmeasured range is not throttled either. A limit only stops a - # pod using cores that are otherwise idle, and it changes what the range - # measures. Packing is driven by the request. - plain = ns['_resources'](end=999999999) - assert 'cpu' not in plain.limits, f"unprofiled range was throttled: {plain.limits}" - assert plain.requests.get('cpu') is not None, "the cpu request must remain" - - -def test_escalation_counts_ooms_not_attempts(): - # On spot most retries are evictions: 288 disruption retries against 7 OOM - # retries on ssc-test 2026-07-30. Keying the exponent on the attempt index - # meant a range disrupted three times then OOMing once jumped to - # base * 1.5^4 -- a 5x request for one OOM, inflated fleet-wide. - import tempfile, json as _json, os as _os - d = tempfile.mkdtemp() - ns = {'os': _os, 'json': _json, - 'outcome_path': lambda e, n: _os.path.join(d, f"o-{n}")} - for name in ('read_outcome', '_oom_count'): - exec(_extract(r"^(def " + name + r"\(.*?)(?=\ndef )").group(1), ns) - for n, outcome in ((1, 'disrupted'), (2, 'disrupted'), (3, 'disrupted'), (4, 'oom')): - _json.dump({'outcome': outcome}, open(ns['outcome_path'](9, n), 'w')) - assert ns['_oom_count'](9, 4) == 1, "three evictions were counted as escalations" - for n in (5, 6): - _json.dump({'outcome': 'oom'}, open(ns['outcome_path'](9, n), 'w')) - assert ns['_oom_count'](9, 6) == 3 - body = _extract(r"(base = \(_profile_overrides\(end, escalated=False\).*?retry_mem = [^\n]+)").group(1) - assert '_oom_count(end, attempt) + 1' in body, "escalation still keys on the attempt index" - - -# --- an already-finished range must not be retried ------------------------- -# Measured on ssc-test 2026-07-30: a1 replayed range 16752063 to its target -# ledger and was evicted before it could exit 0. a2 resumed, found LCL == -# TARGET, ran catchup against a DB with nothing left to apply, and stellar-core -# exited 2 -- deterministically, every attempt. The range exhausted its budget -# and the mission aborted a 61%-complete 2096-worker run over work that had -# actually been done. - -def _run_resume_script(lcl, target=16752063, count=16320, mark_matches=True): - """Execute RESUME_SCRIPT's decision logic with a stubbed core.""" - import subprocess, tempfile, os as _os, re as _re - src = _extract(r"RESUME_SCRIPT = r'''(.*?)'''").group(1) - src = src % {'key': f"{target}/{count}", 'target': target, 'count': count} - d = tempfile.mkdtemp() - bindir = _os.path.join(d, 'bin'); _os.makedirs(bindir) - # stub stellar-core: report `lcl` to offline-info, log what else is invoked - stub = _os.path.join(bindir, 'stellar-core') - with open(stub, 'w') as fh: - fh.write('#!/bin/sh\n' - 'for a in "$@"; do case "$a" in\n' - ' offline-info) ' + - (f'echo \'{{"info":{{"ledger":{{"num":{lcl},"hash":"x"}}}}}}\'; ' if lcl else 'echo "{}"; ') + - 'exit 0;;\n' - ' new-db) echo "RAN:new-db" >> "$STUBLOG"; exit 0;;\n' - ' catchup) echo "RAN:catchup" >> "$STUBLOG"; exit 2;;\n' - 'esac; done\nexit 0\n') - _os.chmod(stub, 0o755) - src = src.replace('/usr/bin/stellar-core', stub) - _os.makedirs(_os.path.join(d, 'data'), exist_ok=True) - src = src.replace('/data/', _os.path.join(d, 'data') + '/') - src = src.replace('MARK=' + _os.path.join(d, 'data') + '/.job-key', - 'MARK=' + _os.path.join(d, 'data') + '/.job-key') - mark = _os.path.join(d, 'data', '.job-key') - if mark_matches: - open(mark, 'w').write(f"{target}/{count}") - stublog = _os.path.join(d, 'stub.log') - env = dict(_os.environ, STUBLOG=stublog) - r = subprocess.run(['/bin/sh', '-c', src], capture_output=True, text=True, env=env, timeout=30) - ran = open(stublog).read().split() if _os.path.exists(stublog) else [] - return r.returncode, r.stdout, ran - - -def test_a_range_already_at_its_target_exits_success_without_recatching(): - code, out, ran = _run_resume_script(lcl=16752063) - assert 'ALREADY COMPLETE' in out, out - assert code == 0, f"exit {code}; a finished range must not fail" - assert 'RAN:catchup' not in ran, "re-ran catchup on a completed range -> exit 2" - assert 'RAN:new-db' not in ran, "wiped a completed range" - - -def test_a_partially_replayed_range_still_resumes(): - code, out, ran = _run_resume_script(lcl=16752063 - 100) - assert 'RESUME:' in out and 'ALREADY COMPLETE' not in out, out - assert 'RAN:catchup' in ran and 'RAN:new-db' not in ran, ran - - -def test_a_range_that_never_started_replay_starts_fresh(): - code, out, ran = _run_resume_script(lcl=None) - assert 'RESUME DECLINED' in out, out - assert 'RAN:new-db' in ran and 'RAN:catchup' in ran, ran - - -def test_the_resume_script_survives_its_own_percent_formatting(): - # RESUME_SCRIPT is %-formatted with the range's key/target/count at dispatch. - # A bare % anywhere in it -- including in a comment -- raises at runtime and - # takes down every job dispatch. Nearly shipped exactly that: a comment - # reading "61%-complete". - src = _extract(r"RESUME_SCRIPT = r'''(.*?)'''").group(1) - src % {'key': '123/456', 'target': 123, 'count': 456} # must not raise - # %% is a legitimate escape (printf '%%s'), so strip those pairs before - # looking for a stray one. - probe = src.replace('%%', '') - stray = [m.start() for m in re.finditer(r"%(?!\()", probe)] - assert not stray, f"bare % in RESUME_SCRIPT near {probe[max(0,stray[0]-40):stray[0]+20]!r}" - - -def test_the_lcl_probe_does_not_window_its_grep(): - # offline-info puts ~40 lines of bucketlist hashes between "ledger": and - # "num", so `grep -A8 '"ledger":'` yields nothing and the probe degrades to - # the log fallback silently -- shipped exactly that once. Verified against - # 27.1.1 on ssc-test 2026-07-30: exactly one "num" key in the document, and - # it is the ledger's (genesis reads 1). - src = _extract(r"RESUME_SCRIPT = r'''(.*?)'''").group(1) - probe = src[src.index('offline-info'):src.index('if [ -n "$LCL" ]')] - assert not re.search(r"grep\s+-A\d+", probe), \ - "a line-windowed grep cannot reach \"num\" past the bucketlist" - assert '"num"' in probe, "the probe no longer reads the ledger num" - assert 'head -1' in probe, "unbounded match could pick up a later key" - - -def test_a_pod_already_finished_reports_no_duration(): - # `started` measures how long the COLLECTOR has watched, not how long the - # container ran. A pod that was already terminal when its poller began -- - # finished while the collector was down, which happened across two restarts - # on ssc-test 2026-07-30 -- would otherwise record ~0s next to a real peak: - # 150 metrics files had a sub-5s duration with a >500MiB anon peak. - poller = _extract(r"^(async def poll_pod\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert 'first_pass' in poller, "nothing distinguishes the first poll" - assert re.search(r"if first_pass and was_terminal:\s*\n(?:\s*#[^\n]*\n)*\s*started = None", poller), \ - "an already-finished pod still reports a fabricated duration" - assert poller.index('was_terminal = done(pod)') < poller.index('first_pass = False') - - -def test_a_peak_on_disk_is_never_lowered_by_a_later_write(): - # Peaks are monotonic, but the merge overwrote. After a collector restart - # the fresh poller's high-water starts at zero, so its first flush would - # replace a higher pre-restart value with a lower one -- undersizing the - # range next run, the one direction that costs an OOM. - import tempfile, os as _os, json as _json - d = tempfile.mkdtemp() - ns = {'json': _json, 'os': _os, - 'base': lambda e, a: _os.path.join(d, f"r{e}-a{a}"), - 'PEAK_KEYS': ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes'), - 'logger': type('L', (), {'info': lambda s, *a: None, - 'warning': lambda s, *a: None})()} - exec(_extract(r"^(def write_metrics\(.*?)(?=\ndef )", COLLECTOR_SRC).group(1), ns) - w = ns['write_metrics'] - w(1, 1, {'peakAnonBytes': 3000, 'txApplySeconds': 12.0}) - w(1, 1, {'peakAnonBytes': 900}) # restarted poller, lower - got = _json.load(open(ns['base'](1, 1) + '.metrics')) - assert got['peakAnonBytes'] == 3000, f"peak was lowered to {got['peakAnonBytes']}" - assert got['txApplySeconds'] == 12.0, "an unrelated field was dropped" - w(1, 1, {'peakAnonBytes': 5000}) # a genuinely higher peak - assert _json.load(open(ns['base'](1, 1) + '.metrics'))['peakAnonBytes'] == 5000 - # non-peak fields still take the newest value - w(1, 1, {'txApplySeconds': 99.0}) - assert _json.load(open(ns['base'](1, 1) + '.metrics'))['txApplySeconds'] == 99.0 - - -def test_a_terminal_pod_still_yields_its_real_duration(): - # A pod carries startTime and terminated.finishedAt until it is deleted, so - # even a pod that finished before this poller existed has a real duration. - # The poller's own elapsed time cannot know that -- it measures how long WE - # watched, which is ~0 in exactly that case. - ns = {'datetime': __import__('datetime').datetime} - exec(_extract(r"^(def pod_seconds\(.*?)(?=\ndef )", COLLECTOR_SRC).group(1), ns) - pod = {'status': {'startTime': '2026-07-30T04:16:26Z', - 'containerStatuses': [{'state': {'terminated': { - 'finishedAt': '2026-07-30T04:22:19Z'}}}]}} - assert ns['pod_seconds'](pod) == 353.0, ns['pod_seconds'](pod) - assert ns['pod_seconds']({'status': {'startTime': '2026-07-30T04:16:26Z'}}) is None - assert ns['pod_seconds']({'status': {}}) is None - - -def test_the_pods_own_duration_wins_over_the_pollers_elapsed_time(): - fn = _extract(r"^(async def finalize\(.*?)(?=\n\nasync def )", COLLECTOR_SRC).group(1) - assert '_pod_secs.pop(pod, None)' in fn - assert fn.index('observed = _pod_secs.pop') < fn.index('elif started is not None:'), \ - "the poller's elapsed time must only be the fallback" - loop = _extract(r"while True:\n(.*?)await asyncio\.sleep\(POLL_SECONDS\)", - COLLECTOR_SRC).group(1) - assert 'pod_seconds(pod)' in loop, "nothing captures it while the pod exists" - - -def test_the_oom_budget_stops_short_of_the_cap_on_purpose(): - # 5 rungs is 1.5^4 = 5x the profile figure. A range needing more is broken, - # not mis-sized, and chasing it to MEM_ESCALATION_CAP parks a whole node on - # it. The price is that such a range is condemned -- which today aborts the - # run, so the coupling below is what must not be forgotten. - n = int(_extract(r"MAX_ATTEMPTS_PER_RANGE = int\(os\.getenv\('MAX_ATTEMPTS', (\d+)\)\)").group(1)) - bump = float(_extract(r"MEM_BUMP_FACTOR = float\(os\.getenv\('MEM_BUMP_FACTOR', ([\d.]+)\)\)").group(1)) - assert 2 <= n <= 8, f"{n} rungs: below 2 cannot escalate, above 8 chases a broken range" - assert bump ** (n - 1) >= 3.0, "the ladder cannot even treble the request before giving up" - assert n > int(_extract(r"MAX_TIMEOUT_ATTEMPTS = int\(os\.getenv\('MAX_TIMEOUT_ATTEMPTS', (\d+)\)\)").group(1)) - chart = open(os.path.join(_SRC_DIR, 'parallel_catchup_helm/values.yaml')).read() - assert int(_extract(r"maxAttempts: (\d+)", chart).group(1)) == n - - -# --- exit 3 is ambiguous and must never condemn a range -------------------- -# stellar-core catches SIGTERM, drains and exits 3 in ~7s, and a corrupt bucket -# also exits 3. Nothing in the exit code separates them. Measured in the sandbox -# edge suite 2026-07-30: a pod killed mid-replay, a pod killed mid-download, and -# an attempt-deadline kill were ALL classified `failed` at attempt 1 and never -# retried -- so the resume path was unreachable through every disruption that -# leaves no DisruptionTarget behind, and one such range aborts the mission. - -def test_exit_three_is_retried_not_condemned(): - body = _extract(r"(if verdict\['outcome'\] == 'timeout':.*?reason = None[^\n]*)").group(1) - assert "verdict.get('exitCode') == CATCHUP_INCOMPLETE_EXIT" in body, \ - "exit 3 still falls through to the zero-retry branch" - assert body.index('CATCHUP_INCOMPLETE_EXIT') < body.index('reason = None'), \ - "the exit-3 branch must precede the condemn branch" - assert int(_extract(r"CATCHUP_INCOMPLETE_EXIT = (\d+)").group(1)) == 3 - - -def test_exit_three_uses_the_ordinary_range_budget(): - # Not the environmental budget: a genuinely corrupt range must still be able - # to exhaust and fail with evidence rather than retry 20 times. - env = set(re.findall(r"'(\w+)'", _extract(r"ENVIRONMENTAL_OUTCOMES = \(([^)]+)\)").group(1))) - assert 'failed' not in env, "exit 3 would inherit the 20-attempt disruption budget" - - -def test_a_deadline_kill_is_not_read_as_a_catchup_failure(): - # The deadline sends SIGTERM -> exit 3 -> pod verdict says `failed`, which - # outranks the Job's DeadlineExceeded. Only the Job knows the deadline fired. - body = _extract(r"(verdict = read_outcome\(end, attempt\) or classify_from_job\(j\).*?)(?=\n\s+if verdict is None)").group(1) - assert "from_job.get('outcome') == 'timeout'" in body, \ - "a deadline kill can still be condemned as a catchup failure" - assert 'verdict = from_job' in body - - -def test_a_condemned_range_is_logged_loudly(): - # The zero-retry path logged nothing: the range appeared under failed{} and - # the mission aborted with no line saying why. - body = _extract(r"(if reason is not None:\s*\n\s*logger\.error\(\"range %s exhausted.*?)(?=\n\s+if end not in failed)").group(1) - assert 'RANGE CONDEMNED' in body, "condemnation is still silent" - assert 'else:' in body - - -def test_a_late_tx_apply_is_backfilled_like_the_peaks(): - # Same one-shot race the peaks had. The collector writes txApplySeconds when - # it finalizes, which can land after reconcile recorded the range. Measured - # in the sandbox edge suite 2026-07-30: progress.json held txApply=null while - # range-4000-a1.metrics durably held txApplySeconds=0.000486848, and the - # monitor even logged "could not read tx_apply for range 4000 (pod gone?)". - guard = _extract(r"(elif \(not _has_peaks\(completed\[end\]\).*?)(?=\n\s+late = peaks_for_range)").group(1) - assert "completed[end].get('txApply') is None" in guard, \ - "a range with peaks but no txApply never re-enters the backfill" - body = _extract(r"(late = peaks_for_range\(end, attempt\).*?)(?=\n\s+_reap_if_complete)").group(1) - assert 'tx_apply_for_range(end, attempt)' in body, "txApply is never re-read" - assert "late['txApply'] = late_tx" in body - - -def test_a_condemned_range_does_not_freeze_dispatch(): - """A failed range must not stop the run from dispatching the rest. - - The driver waits for `remaining == 0 and in_progress == []` before it - reports. Gating dispatch on `not failed` pinned `remaining` at the number - of never-dispatched ranges, so the mission waited forever instead of - failing -- strictly worse than the abort it replaced. - """ - assert "not state['halted'] and not failed" not in SRC, \ - "a condemned range must not freeze dispatch (driver deadlock)" - assert "state['halted']" not in SRC, \ - "the halt gate is gone: its high-water mark lived in memory, so a " \ - "restart disarmed the guard for exactly the event it existed to survive" diff --git a/src/MissionParallelCatchup/tests/unit/conftest.py b/src/MissionParallelCatchup/tests/unit/conftest.py new file mode 100644 index 00000000..898e6cad --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/conftest.py @@ -0,0 +1,23 @@ +"""Fixtures for the imported-function unit tests. + +These tests call job_monitor / log_collector functions directly. Almost all of +them touch the shared logs volume through LOG_DIR-derived paths, so the one +thing they all need is that directory pointed somewhere disposable -- and +pointed at the SAME place in both modules, which is the contract the two +processes actually run under. +""" + +import pytest + +import job_monitor as jm +import log_collector as lc + + +@pytest.fixture +def logdir(tmp_path, monkeypatch): + """The shared volume, as both processes see it.""" + d = tmp_path / 'logs' + d.mkdir() + monkeypatch.setattr(jm, 'LOG_DIR', str(d)) + monkeypatch.setattr(lc, 'LOG_DIR', str(d)) + return d diff --git a/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py b/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py new file mode 100644 index 00000000..464c5fe2 --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py @@ -0,0 +1,194 @@ +"""Aggregating measurements across the attempts that make up one range. + +In pvc mode a pod killed after replay starts leaves /data, and the next attempt +resumes at LCL+1 with RESUME=true -- skipping the archive download and bucket +apply, which is where peak memory happens. Profiling only the winning attempt +therefore under-reports a resumed range by the whole download gap, and on spot +(where eviction is routine and resume is the point of durable /data) that would +make the run unprofileable. medida's total and a pod's duration are per-process +for exactly the same reason, so both are tail-only in the same way. +""" + +import json + +import pytest + +import job_monitor as jm + + +GIB = 1024 ** 3 +MIB = 1024 ** 2 + + +@pytest.fixture +def attempts(logdir): + """Lay down the files the collector and the monitor leave per attempt.""" + def write(end, spec): + for n, (metrics, outcome) in spec.items(): + if metrics is not None: + with open(jm.metrics_path(end, n), 'w') as fh: + fh.write(metrics if isinstance(metrics, str) else json.dumps(metrics)) + if outcome is not None: + with open(jm.outcome_path(end, n), 'w') as fh: + json.dump(outcome, fh) + return write + + +# --- which attempts describe the range --------------------------------------- + +def test_the_chain_is_the_run_of_resumed_attempts_ending_at_this_one(attempts): + # a1 interrupted then superseded by a fresh a2; a3 resumed from a2. Only + # a2+a3 describe the same continuous pass over the range. + attempts(999, {1: ({}, None), 2: ({}, None), 3: ({'resumed': True}, None)}) + assert list(jm._resumed_chain(999, 3)) == [2, 3] + assert list(jm._resumed_chain(999, 1)) == [1] + + +def test_an_attempt_with_no_metrics_file_is_not_treated_as_resumed(attempts): + attempts(999, {1: ({}, None)}) + assert jm._attempt_resumed(999, 2) is False + + +# --- peaks -------------------------------------------------------------------- + +def test_a_resumed_range_keeps_the_peak_from_the_attempt_that_did_the_download(attempts): + # a1 evicted mid-replay having already done the download; a2 resumes at + # LCL+1 and only replays the tail. a2 alone would report 400MiB for a range + # that really needs 2GiB. + attempts(999, {1: ({'peakAnonBytes': 2 * GIB}, {'outcome': 'disrupted'}), + 2: ({'peakAnonBytes': 400 * MIB, 'resumed': True}, None)}) + assert jm.peaks_for_range(999, 2)['peakAnonBytes'] == 2 * GIB + + +def test_a_fresh_retry_supersedes_an_interrupted_one(attempts): + # No RESUME line means new-db ran and this attempt did the whole range, so + # its sample is complete. An earlier attempt that was merely interrupted + # measured the same work and only adds noise. + attempts(999, {1: ({'peakAnonBytes': 8 * GIB}, {'outcome': 'disrupted'}), + 2: ({'peakAnonBytes': 900 * MIB}, None)}) + assert jm.peaks_for_range(999, 2)['peakAnonBytes'] == 900 * MIB + + +@pytest.mark.parametrize('outcome,field,hit,quiet', [ + ('oom', 'peakAnonBytes', 8 * GIB, 900 * MIB), + ('oom', 'peakEphemeralBytes', 30 * GIB, 5 * GIB), # died on memory, its disk figure is real + ('ephemeral', 'peakEphemeralBytes', 40 * GIB, 9 * GIB), + ('ephemeral', 'peakAnonBytes', 3 * GIB, 1 * GIB), +]) +@pytest.mark.parametrize('resumed', [True, False]) +def test_an_attempt_killed_at_a_ceiling_counts_wherever_it_sits(attempts, outcome, + field, hit, quiet, resumed): + # A pod OOM-killed at 8Gi really did allocate ~8Gi and wanted more, so its + # peak is a lower bound on demand, not an artifact of the limit -- and it is + # the attempt most worth keeping, because download concurrency scales with + # available cpu and a pod that bursted on an idle node can peak above the + # one that eventually succeeded. Sizing off the quieter attempt would OOM + # the range again. + # + # It survives a fresh start too, which the chain rule alone would drop. + # Measured on ssc-30: an OOM in replay resumes and stays in the chain + # (224 of 252), an OOM in download does not (25 of 252), and a higher-cpu + # run is download-bound -- so the self-correcting loop would go quiet + # exactly when it is most needed. + later = {field: quiet} + if resumed: + later['resumed'] = True + attempts(999, {1: ({field: hit}, {'outcome': outcome}), 2: (later, None)}) + assert jm.peaks_for_range(999, 2)[field] == hit + + +def test_the_ceiling_exception_is_peaks_only(attempts): + # tx_apply and seconds are summed, and a fresh start redoes the work the + # dropped attempt already did, so counting it there would double-count. + attempts(999, {1: ({'txApplySeconds': 100.0, 'attemptSeconds': 900.0}, + {'outcome': 'oom'}), + 2: ({'txApplySeconds': 7.0}, None)}) # fresh start + assert jm.tx_apply_for_range(999, 2) == 7.0 + assert jm.seconds_for_range(999, 2, 300.0) == 300.0 + + +def test_a_missing_or_malformed_metrics_file_is_tolerated(attempts): + attempts(999, {2: ("not json at all", None), + 3: ({'peakAnonBytes': 5, 'resumed': True}, None)}) + assert jm.peaks_for_range(999, 3) == {'peakAnonBytes': 5} + assert jm.peaks_for_range(999, 9) == {} + + +def test_an_absent_peak_never_reaches_the_profile_as_a_null(attempts): + # The consumer falls back to a default on a missing field, so a null defeats it. + attempts(999, {1: ({'peakAnonBytes': None, 'peakRssBytes': 7}, None)}) + assert jm.peaks_for_range(999, 1) == {'peakRssBytes': 7} + + +def test_both_measured_peaks_reach_the_progress_record(): + # peaks_for_range filters to PEAK_FIELDS and the ConfigMap mirror strips + # _PROFILE_ONLY_FIELDS; a measurement absent from either is silently + # dropped between the collector and the profile. + for field in ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes'): + assert field in jm.PEAK_FIELDS, field + assert 'peakAnonBytes' in jm._PROFILE_ONLY_FIELDS + + +# --- durations ---------------------------------------------------------------- + +def test_seconds_sums_the_whole_resumed_chain(attempts): + # a1 ran 900s then was evicted mid-replay; a2 resumed and took 300s. The + # range cost 1200s of compute, not 300. + attempts(999, {1: ({}, {'outcome': 'disrupted', 'attemptSeconds': 900.0}), + 2: ({'resumed': True}, None)}) + assert jm.seconds_for_range(999, 2, 300.0) == 1200.0 + + +def test_seconds_ignores_attempts_before_a_fresh_start(attempts): + # a2 ran new-db and did the whole range itself, so a1's 900s is not part of + # the same pass. + attempts(999, {1: ({}, {'outcome': 'oom', 'attemptSeconds': 900.0}), + 2: ({}, None)}) + assert jm.seconds_for_range(999, 2, 300.0) == 300.0 + + +def test_seconds_survives_a_leg_with_no_recorded_duration(attempts): + # An attempt whose pod vanished before it was classified has no + # attemptSeconds. Better to under-report one leg than return nothing. + attempts(999, {1: ({}, {'outcome': 'disrupted'}), 2: ({'resumed': True}, None)}) + assert jm.seconds_for_range(999, 2, 300.0) == 300.0 + + +def test_seconds_is_none_when_nothing_is_known(attempts): + attempts(999, {1: ({}, None)}) + assert jm.seconds_for_range(999, 1, None) is None + + +def test_seconds_falls_back_to_the_collectors_figure(attempts): + # The authoritative .outcome is missing for every reaped pod -- measured on + # ssc-test 2026-07-30, 212 of 212 spot disruptions were classified from the + # Job condition with the pod already gone, so record_outcome never ran. + # Without this fallback the chain drops that leg entirely. + attempts(999, {1: ({'attemptSeconds': 850.0}, None), + 2: ({'resumed': True}, None)}) + assert jm.seconds_for_range(999, 2, 300.0) == 1150.0 + + +def test_the_authoritative_outcome_wins_over_the_collector_estimate(attempts): + # .outcome comes from the pod's terminated timestamps; the collector's is a + # stream-lifetime approximation that starts up to one poll late. + attempts(999, {1: ({'attemptSeconds': 850.0}, + {'outcome': 'disrupted', 'attemptSeconds': 900.0}), + 2: ({'resumed': True}, None)}) + assert jm.seconds_for_range(999, 2, 300.0) == 1200.0 + + +# --- counting causes, not attempts -------------------------------------------- + +def test_escalation_counts_ooms_not_attempts(attempts): + # On spot most retries are evictions: 288 disruption retries against 7 OOM + # retries on ssc-test 2026-07-30. Keying the exponent on the attempt index + # meant a range disrupted three times then OOMing once jumped to + # base * 1.5^4 -- a 5x request for one OOM, inflated fleet-wide. + attempts(9, {1: (None, {'outcome': 'disrupted'}), + 2: (None, {'outcome': 'disrupted'}), + 3: (None, {'outcome': 'disrupted'}), + 4: (None, {'outcome': 'oom'})}) + assert jm._oom_count(9, 4) == 1, "three evictions were counted as escalations" + attempts(9, {5: (None, {'outcome': 'oom'}), 6: (None, {'outcome': 'oom'})}) + assert jm._oom_count(9, 6) == 3 diff --git a/src/MissionParallelCatchup/tests/unit/test_classify.py b/src/MissionParallelCatchup/tests/unit/test_classify.py new file mode 100644 index 00000000..d3c0e560 --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_classify.py @@ -0,0 +1,214 @@ +"""How a failed attempt is classified, from the Job and from the pod. + +Two classifiers, two different objects. classify_from_job() reads the Job +controller's podFailurePolicy condition message -- a format this mission does +not control, pinned here from real captures so an EKS change fails here rather +than silently degrading a run. classify() reads the pod, which carries detail +the Job never has. +""" + +import pytest +from kubernetes import client + +import job_monitor as jm +import log_collector as lc + + +# --- captures ---------------------------------------------------------------- + +# EKS 1.34 Job condition messages. Only the wording is pinned; pod and +# container names are renamed for readability. +DISRUPTED = ("Pod sandbox/jterm-catchup-snfr2 has condition DisruptionTarget " + "matching FailJob rule at index 0") +OOMKILLED = ("Container oom-container for pod sandbox/oom-test-job-qvq8b failed with " + "exit code 137 matching FailJob rule at index 1") +NONZERO_EXIT = ("Container exit-1-container for pod sandbox/exit-1-job-wbhkq failed with " + "exit code 1 matching FailJob rule at index 2") + +# RECONSTRUCTED 2026-07-30 after an over-broad test deletion removed the +# originals -- twice. Shaped to what the code parses (RULE_ORDER[2] is 'failed'; +# classify() keys on the substring 'ephemeral' in status.message) but no longer +# a verbatim capture. Re-pin from a real eviction on the next run. +EPH_EVICT_JOB_CONDITION = ( + "Container stellar-core for pod stellar-supercluster/" + "parallel-catchup-r31005951-a1-x7k2p failed with exit code 3 " + "matching FailJob rule at index 2") +EPH_EVICT_MESSAGE = ( + "Pod ephemeral local storage usage exceeds the total limit of containers 40Gi") + + +def failed_job(message='', reason='PodFailurePolicy'): + return client.V1Job(status=client.V1JobStatus(conditions=[ + client.V1JobCondition(type='Failed', status='True', + reason=reason, message=message)])) + + +def verdict(message, reason='PodFailurePolicy'): + """(outcome, exitCode, pod) as classify_from_job reports them.""" + got = jm.classify_from_job(failed_job(message, reason)) + if got is None: + return (None, None, None) + return (got['outcome'], got['exitCode'], got['pod'] or None) + + +def pod_with(reason=None, message=None, conditions=None, terminated=None, + container='stellar-core'): + """A pod carrying exactly the status fields classify() branches on.""" + statuses = None + if terminated is not None: + statuses = [client.V1ContainerStatus( + name=container, image='core', image_id='', ready=False, restart_count=0, + state=client.V1ContainerState( + terminated=client.V1ContainerStateTerminated(**terminated)))] + return client.V1Pod( + metadata=client.V1ObjectMeta(name='p'), + status=client.V1PodStatus(reason=reason, message=message, + conditions=conditions, container_statuses=statuses)) + + +def as_dict(reason=None, message=None, conditions=None, terminated=None): + """The same pod, in the shape the collector reads off the raw API.""" + status = {'reason': reason, 'message': message, + 'conditions': conditions or [], + 'containerStatuses': ([{'state': {'terminated': terminated}}] + if terminated is not None else [])} + return {'metadata': {'name': 'p'}, 'status': status} + + +# --- classify_from_job: the Job controller's message -------------------------- + +@pytest.mark.parametrize("msg,outcome,code,pod", [ + (DISRUPTED, 'disrupted', None, None), + (OOMKILLED, 'oom', 137, 'oom-test-job-qvq8b'), + (NONZERO_EXIT, 'failed', 1, 'exit-1-job-wbhkq'), +]) +def test_job_condition_message(msg, outcome, code, pod): + assert verdict(msg) == (outcome, code, pod) + + +def test_rule_order_matches_the_rendered_policy(): + # "rule at index N" is only meaningful against the order the rules are + # rendered in, so the lookup table and the policy must be the same list. + assert [name for name, _ in jm._failure_rules()] == jm.RULE_ORDER + + +def test_eviction_is_told_apart_from_a_broken_range_by_the_condition(): + # stellar-core exits 3 both for a drain and for a corrupt bucket, so only + # DisruptionTarget separates them -- hence rule 0 must be evaluated first. + assert verdict(DISRUPTED)[0] == 'disrupted' + assert verdict("Container c for pod ns/p failed with exit code 3")[0] == 'failed' + + +def test_a_bare_exit_code_is_read_when_no_rule_index_is_offered(): + # Measured on ssc-test 2026-07-28: a drained stellar-core catches SIGTERM + # and exits 3 well inside the 100s grace, so evictions do NOT produce 137 -- + # which makes a bare 137 an OOM with high confidence. + assert verdict("Container c for pod ns/p failed with exit code 137")[:2] == ('oom', 137) + assert verdict("Container c for pod ns/p failed with exit code 1")[:2] == ('failed', 1) + + +def test_an_unclassifiable_job_failure_stays_unclassified(): + # BackoffLimitExceeded carries no rule index and no exit code, so classify + # honestly returns nothing rather than guessing. A monitor restart while a + # node was reaped produces exactly this, and condemning on it would fail a + # 10-hour job on no evidence -- reconcile gives it the environmental budget. + assert jm.classify_from_job(failed_job( + "Job has reached the specified backoff limit", + reason='BackoffLimitExceeded')) is None + assert 'unknown' in jm.ENVIRONMENTAL_OUTCOMES + assert {'disrupted', 'rejected'} <= set(jm.ENVIRONMENTAL_OUTCOMES) + + +def test_a_deadline_exceeded_job_is_a_timeout_not_a_catchup_failure(): + # activeDeadlineSeconds fired: the attempt hung rather than failing. Only + # the Job knows this -- the deadline SIGTERMs the pod, which drains to + # exit 3 and reads as a plain catchup failure from the pod side. + assert verdict('', reason='DeadlineExceeded')[0] == 'timeout' + + +def test_a_job_with_no_failed_condition_yields_nothing(): + assert jm.classify_from_job(client.V1Job(status=client.V1JobStatus())) is None + + +# --- classify: what only the pod can say -------------------------------------- + +def test_a_disruption_target_condition_outranks_everything_on_the_pod(): + got = jm.classify(pod_with( + conditions=[client.V1PodCondition(type='DisruptionTarget', status='True')], + terminated={'exit_code': 3, 'reason': 'Error'})) + assert got['outcome'] == 'disrupted' + + +def test_an_ephemeral_eviction_is_not_read_as_an_oom_or_a_disruption(): + # Measured end-to-end on ssc-test: the kubelet sets no DisruptionTarget, + # and stellar-core drains and exits 3, so the Job condition is a plain + # non-zero failure that would get no retry. status.message is the only + # discriminator and only the pod carries it, so both classifiers must test + # it before anything keyed on Evicted. + assert verdict(EPH_EVICT_JOB_CONDITION)[0] == 'failed', \ + "the Job matches the generic non-zero rule" + assert 'ephemeral' in EPH_EVICT_MESSAGE, "both classifiers key on this substring" + evicted = dict(reason='Evicted', message=EPH_EVICT_MESSAGE, + terminated={'exit_code': 3, 'reason': 'Error'}) + assert jm.classify(pod_with(**evicted))['outcome'] == 'ephemeral' + assert lc.classify(as_dict(reason='Evicted', message=EPH_EVICT_MESSAGE, + terminated={'exitCode': 3}))['outcome'] == 'ephemeral' + + +def test_a_plain_eviction_with_no_disk_message_is_only_a_rejection(): + # The generic Evicted branch sits right behind the ephemeral one; an + # eviction for anything other than the range's own disk use must still + # reach it, or a node-pressure eviction would be read as a disk overrun and + # grow the range's storage for no reason. + got = jm.classify(pod_with(reason='Evicted', message='node was low on memory')) + assert got['outcome'] == 'rejected' + + +@pytest.mark.parametrize('reason', [ + 'VolumeAttachmentLimitExceeded', 'OutOfcpu', 'OutOfmemory', 'OutOfpods', + 'UnexpectedAdmissionError', 'NodeAffinity', 'Shutdown', 'Evicted', +]) +def test_an_admission_rejection_is_not_a_catchup_failure(reason): + # Observed on ssc-test: reason=VolumeAttachmentLimitExceeded, "Node has + # reached its volume attachment limit, rejecting pod". No exit code, no + # DisruptionTarget -- without this branch it falls through to 'failed' and + # a transient admission rejection kills the whole run. + for got in (jm.classify(pod_with(reason=reason)), + lc.classify(as_dict(reason=reason))): + assert got['outcome'] == 'rejected', reason + assert got['exitCode'] is None + + +def test_a_deadline_kill_is_visible_on_the_pod_too(): + # The deadline lives on the PodSpec, so the kubelet fires it and the pod + # carries the reason; the Job only sees a non-zero exit. + assert jm.classify(pod_with(reason='DeadlineExceeded'))['outcome'] == 'timeout' + + +def test_a_pod_where_nothing_ever_ran_says_nothing_about_the_range(): + for got in (jm.classify(pod_with()), lc.classify(as_dict())): + assert got['outcome'] == 'rejected' + + +def test_an_oom_kill_is_read_from_the_container_reason_not_the_exit_code(): + # 137 is SIGKILL, which the kubelet also uses for a graceful-stop timeout -- + # only reason=OOMKilled makes it unambiguous. + for got in (jm.classify(pod_with(terminated={'exit_code': 137, 'reason': 'OOMKilled'})), + lc.classify(as_dict(terminated={'exitCode': 137, 'reason': 'OOMKilled'}))): + assert (got['outcome'], got['exitCode']) == ('oom', 137) + + +def test_a_non_zero_exit_is_a_catchup_failure_and_keeps_its_code(): + for got in (jm.classify(pod_with(terminated={'exit_code': 3, 'reason': 'Error'})), + lc.classify(as_dict(terminated={'exitCode': 3}))): + assert (got['outcome'], got['exitCode']) == ('failed', 3) + + +def test_exit_three_is_the_ambiguous_one_and_never_inherits_a_bigger_budget(): + # stellar-core drains to 3 on SIGTERM and a corrupt bucket also exits 3, so + # reconcile retries it on the ordinary range budget -- but a genuinely + # corrupt range must still be able to exhaust rather than retry 20 times. + assert jm.CATCHUP_INCOMPLETE_EXIT == 3 + assert 'failed' not in jm.ENVIRONMENTAL_OUTCOMES + assert 'ephemeral' not in jm.ENVIRONMENTAL_OUTCOMES, \ + "a deterministic failure must not get the disruption budget" diff --git a/src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py b/src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py new file mode 100644 index 00000000..d3fbb6db --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py @@ -0,0 +1,259 @@ +"""log_collector.main(): which pods get a stream, and when one is let go. + +The loop is small and every decision in it has cost a run something: a stream +re-opened every cycle re-read a whole log per pod per POLL_SECONDS; a pod that +left the pod list without ever being observed terminal kept its stream retrying +until the run ended; a sampler placed after the per-pod branches only ever fired +on the cycle a stream opened, when the range had written almost nothing. + +Driven by running the real `main()` with `list_pods`, `sample_kubelet` and +`poll_pod` replaced -- those three are the loop's entire outside world -- and +cancelling it once the scenario has played out. Nothing here reads source text: +the previous version of these tests sliced the loop body out with a regex, which +matched the wrong block twice and had to be re-anchored by hand. +""" + +import asyncio + +import pytest + +import log_collector as lc + + +def pod(name, phase='Running', end='300', attempt='1', node='node-1'): + return {'metadata': {'name': name, + 'labels': {lc.LABEL_RUN: lc.RUN_NAME, + lc.LABEL_RANGE: end, + lc.LABEL_ATTEMPT: attempt}}, + 'spec': {'nodeName': node}, + 'status': {'phase': phase}} + + +class Loop: + """One scripted run of main(): a pod list per cycle, and what happened. + + Once the script is exhausted the last cycle repeats, so the loop settles + into a steady state rather than starting to error -- an exception out of + list_pods is swallowed by main() and would only add noise. + + `poller` picks what the fake stream does: 'wait' blocks on the pod's _wake + Event and then returns (a normal stream, ended by the pod going away or + terminal), 'return' finishes immediately (a stream that died early), and + 'hang' ignores the wake and never finishes at all. + """ + + def __init__(self, cycles, poller='wait'): + self.cycles = list(cycles) + self.passes = 0 + self.poller = poller + self.order = [] # 'list' / 'sample' / 'open:' in sequence + self.opened = [] + self.sampled = [] + self.done_seen = {} + + async def list_pods(self, session): + self.order.append('list') + pods = self.cycles[min(self.passes, len(self.cycles) - 1)] + self.passes += 1 + return list(pods) + + async def sample_kubelet(self, session, nodes): + self.order.append('sample') + self.sampled.append(set(nodes)) + + def poll_pod(self, session, name, end, attempt, done, done_ok): + self.order.append(f"open:{name}") + self.opened.append((name, end, attempt)) + + async def run(): + if self.poller == 'return': + return + ev = lc._wake.setdefault(name, asyncio.Event()) + await ev.wait() + self.done_seen[name] = done(name) + if self.poller == 'hang': + await asyncio.Event().wait() # never finishes + + return run() + + +@pytest.fixture +def loop_env(tmp_path, monkeypatch): + monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(lc, 'token', lambda: 'tok') + monkeypatch.setattr(lc, 'ssl_ctx', lambda: None) + monkeypatch.setattr(lc, 'POLL_SECONDS', 0.01) + monkeypatch.setattr(lc, 'VANISHED_GRACE_CYCLES', 3) + for name in ('_eph_peak', '_anon_peak', '_ws_peak', '_peak_flushed', + '_streaming', '_pod_secs', '_wake'): + monkeypatch.setattr(lc, name, {}) + return monkeypatch + + +def run_loop(monkeypatch, cycles, poller='wait', extra=2): + """Run main() over a scripted sequence of pod lists, then stop it.""" + loop = Loop(cycles, poller) + monkeypatch.setattr(lc, 'list_pods', loop.list_pods) + monkeypatch.setattr(lc, 'sample_kubelet', loop.sample_kubelet) + monkeypatch.setattr(lc, 'poll_pod', loop.poll_pod) + asyncio.run(_drive(loop, extra)) + return loop + + +async def _drive(loop, extra, want_survivors=False): + task = asyncio.create_task(lc.main()) + want = len(loop.cycles) + extra + for _ in range(600): + await asyncio.sleep(0.005) + if loop.passes >= want: + break + survivors = [t for t in asyncio.all_tasks() + if t is not asyncio.current_task() and t is not task + and not t.done()] + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + return survivors if want_survivors else None + + +# --- the sampler --------------------------------------------------------------- + +def test_the_sampler_runs_before_the_per_pod_branches(loop_env): + """The per-pod branches all end in `continue` for a pod already streaming, + so a sampler placed after them fires only on the cycle a stream opens -- + when the range has written almost nothing and its peak is meaningless.""" + loop = run_loop(loop_env, [[pod('w-1')], [pod('w-1')], [pod('w-1')]]) + + assert loop.order[:3] == ['list', 'sample', 'open:w-1'] + # One listing per cycle and one sample per listing: the sampler reuses the + # pod list rather than fetching its own. + assert loop.order.count('sample') == loop.order.count('list') + for i, event in enumerate(loop.order): + if event == 'sample': + assert loop.order[i - 1] == 'list' + + +def test_the_sampler_runs_every_cycle_not_once_per_stream(loop_env): + loop = run_loop(loop_env, [[pod('w-1')]] * 4) + + assert len(loop.sampled) >= 3, loop.order + assert loop.opened == [('w-1', '300', '1')], "the stream was re-opened" + + +def test_the_sampler_is_not_gated_on_storage_mode(loop_env): + """It was, back when it only sampled disk. Memory is sized in both modes, + so gating here left every pvc run with no anon peak at all.""" + loop_env.setattr(lc, 'STORAGE_MODE', 'pvc') + loop = run_loop(loop_env, [[pod('w-1')], [pod('w-1')]]) + + assert loop.sampled and loop.sampled[0] == {'node-1'} + + +def test_only_running_pods_are_handed_to_the_sampler(loop_env): + """kubelet has no live stats for a pod that has not started or has exited, + and every extra node in the set is another /stats/summary GET.""" + loop = run_loop(loop_env, [[pod('w-1', phase='Pending', node='node-a'), + pod('w-2', phase='Running', node='node-b')]] * 2) + + assert loop.sampled[0] == {'node-b'} + + +# --- which pods get a stream --------------------------------------------------- + +def test_a_pending_pod_is_not_polled_until_it_can_answer(loop_env): + """Its container has not started, so the log endpoint answers 400 and the + poll is wasted -- 60 of 88 failures immediately after the polling switch.""" + loop = run_loop(loop_env, [[pod('w-1', phase='Pending')], + [pod('w-1', phase='Pending')], + [pod('w-1', phase='Running')], + [pod('w-1', phase='Running')]]) + + assert loop.opened == [('w-1', '300', '1')] + # ...and not until the third cycle, the first one it could have answered. + assert loop.order[:6] == ['list', 'sample', 'list', 'sample', + 'list', 'sample'] + + +def test_a_terminal_pod_is_still_polled(loop_env): + """That is where a pod's final output lives; a stream that skipped it + would lose the medida block of every range that finished quickly.""" + loop = run_loop(loop_env, [[pod('w-1', phase='Succeeded')]] * 2) + + assert loop.opened == [('w-1', '300', '1')] + + +def test_a_pod_with_no_range_label_is_not_ours(loop_env): + stray = pod('other-1') + del stray['metadata']['labels'][lc.LABEL_RANGE] + loop = run_loop(loop_env, [[stray]] * 2) + + assert loop.opened == [] + + +def test_a_finished_stream_is_not_reopened_once_its_pod_is_terminal(loop_env): + """A completed task is deleted from `tasks`, so without a record of it the + next cycle re-creates the stream and re-reads the whole log -- every cycle, + per pod, for the rest of the run.""" + loop = run_loop(loop_env, [[pod('w-1', phase='Succeeded')]] * 5) + + assert loop.opened == [('w-1', '300', '1')], \ + f"stream re-opened {len(loop.opened)} times" + + +def test_a_stream_that_died_while_its_pod_still_runs_is_reopened(loop_env): + """The other half of the same guard. A task that ended while the pod is + still Running died early, and re-opening the stream is how that recovers -- + barring it would abandon a live range.""" + loop = run_loop(loop_env, [[pod('w-1', phase='Running')]] * 5, + poller='return') + + assert len(loop.opened) >= 2, "an early-dying stream was never retried" + + +# --- a pod that leaves the list ----------------------------------------------- + +def test_a_vanished_pod_is_marked_terminal_and_wakes_its_poller(loop_env): + """`terminal` is only written for pods in the pod list, so a pod that goes + away without ever being seen terminal -- reaped node, eviction, or the + monitor deleting its finished Job -- keeps done() False forever. Gone is + terminal, and the wake is what stops the poller sleeping out its interval + before it takes the 404 and writes the .done the monitor waits on.""" + loop = run_loop(loop_env, [[pod('w-1')], [pod('w-1')], [], [], []]) + + assert loop.done_seen.get('w-1') is True, \ + "the poller was never woken, or woke to done() == False" + + +def test_a_pod_going_terminal_wakes_its_poller_without_waiting_a_cycle(loop_env): + """The delay that matters is between the container exiting and the last + read: sleeping blind hands that window to a spot reclaim, which deletes the + pod and takes the final lines with it.""" + loop = run_loop(loop_env, [[pod('w-1', phase='Running')], + [pod('w-1', phase='Succeeded')], + [pod('w-1', phase='Succeeded')]]) + + assert loop.done_seen.get('w-1') is True + + +def test_a_stream_that_will_not_finish_is_cancelled_after_the_grace(loop_env): + """Marking a vanished pod terminal is not enough on its own: a stream + wedged inside a connection attempt never reaches its own done() check, and + that is exactly the state that starves every other stream of a poll slot.""" + loop = Loop([[pod('w-1')], [pod('w-1')], [], [], [], [], []], poller='hang') + loop_env.setattr(lc, 'list_pods', loop.list_pods) + loop_env.setattr(lc, 'sample_kubelet', loop.sample_kubelet) + loop_env.setattr(lc, 'poll_pod', loop.poll_pod) + + alive = asyncio.run(_drive(loop, extra=3, want_survivors=True)) + + assert alive == [], "a wedged stream outlived its grace and held its slot" + + +def test_the_grace_is_more_than_one_cycle(): + """A stream still finalizing -- writing .metrics, closing its archive -- + must not be cancelled out from under its own write.""" + assert lc.VANISHED_GRACE_CYCLES >= 2, \ + (f"a grace of {lc.VANISHED_GRACE_CYCLES} cycle(s) can cancel a stream " + "in the middle of finalizing itself") diff --git a/src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py b/src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py new file mode 100644 index 00000000..42ac3187 --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py @@ -0,0 +1,246 @@ +"""sample_kubelet: what one /stats/summary payload is allowed to become. + +The sampler is the only source of every memory figure the profile uses. It runs +against a payload this mission does not control, on pods that may be seconds +old, in a process that can be restarted mid-range -- so most of what it does is +refuse to record something. + +Driven through the real `log_collector.sample_kubelet` with a fake session, and +asserted on the module's own peak dicts and on the bytes that reach the volume. +An earlier generation of these tests exec'd the function body out of the source +with a hand-built namespace; the point of that was to survive an unimportable +module, and the module imports. +""" + +import asyncio + +import pytest + +import job_monitor as jm +import log_collector as lc + +MIB = 1024 ** 2 + + +@pytest.fixture +def sampler(tmp_path, monkeypatch): + """A collector with no memory, writing to a disposable volume.""" + monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(jm, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(lc, 'token', lambda: 'tok') + monkeypatch.setattr(lc, 'STORAGE_MODE', 'ephemeral') + for name in ('_eph_peak', '_anon_peak', '_ws_peak', '_peak_flushed', + '_streaming', '_pod_secs', '_wake'): + monkeypatch.setattr(lc, name, {}) + return tmp_path + + +class _Resp: + def __init__(self, payload): + self._payload = payload + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + def raise_for_status(self): + pass + + async def json(self): + return self._payload + + +class _Session: + def __init__(self, payload): + self.payload = payload + + def get(self, url, **kw): + return _Resp(self.payload) + + +def container(name=None, rss=None, ws=None): + mem = {} + if rss is not None: + mem['rssBytes'] = rss + if ws is not None: + mem['workingSetBytes'] = ws + return {'name': name or lc.CONTAINER, 'memory': mem} + + +def payload(pod, containers, eph=None): + entry = {'podRef': {'name': pod}, 'containers': containers} + if eph is not None: + entry['ephemeral-storage'] = {'usedBytes': eph} + return {'pods': [entry]} + + +def sample(doc): + asyncio.run(lc.sample_kubelet(_Session(doc), ['node-1'])) + + +# --- a peak is a high-water mark, not the latest reading --------------------- + +def test_a_later_lower_sample_never_lowers_the_peak(sampler): + """Catching the spike is the whole point, and download-phase anon + oscillates: the sampler is what turns a series of readings into one + number, so last-wins here defeats every consumer downstream.""" + sample(payload('w-1', [container(rss=900 * MIB)], eph=5)) + sample(payload('w-1', [container(rss=400 * MIB)], eph=2)) + + assert lc._anon_peak == {'w-1': 900 * MIB} + assert lc._eph_peak == {'w-1': 5} + + +def test_a_higher_sample_still_raises_it(sampler): + sample(payload('w-1', [container(rss=400 * MIB)], eph=2)) + sample(payload('w-1', [container(rss=900 * MIB)], eph=5)) + + assert lc._anon_peak == {'w-1': 900 * MIB} + assert lc._eph_peak == {'w-1': 5} + + +# --- what must not be recorded ----------------------------------------------- + +def test_a_container_without_stats_yet_is_skipped_not_zeroed(sampler): + """rssBytes is absent for the first seconds of a container's life, before + cAdvisor has stats for it. Recording 0 would poison the peak for a range + that is about to be measured properly, and raising would kill the sampler + for every other pod on the node.""" + sample(payload('w-1', [container(rss=None)], eph=7)) + + assert lc._anon_peak == {}, "a missing rssBytes was recorded anyway" + assert lc._eph_peak == {'w-1': 7}, "the disk axis stopped being sampled" + + +def test_only_the_worker_container_is_measured(sampler): + """Sidecars share the pod. Summing across containers, or letting the last + one win, would size the range from whichever one kubelet listed last.""" + sample(payload('w-1', [container(name='istio-proxy', rss=900 * MIB)])) + + assert lc._anon_peak == {} + + +def test_the_worker_is_found_however_kubelet_orders_the_containers(sampler): + sample(payload('w-1', [container(name='istio-proxy', rss=900 * MIB), + container(rss=222 * MIB)])) + + assert lc._anon_peak == {'w-1': 222 * MIB} + + +def test_a_pod_with_no_name_is_skipped_rather_than_keyed_on_none(sampler): + asyncio.run(lc.sample_kubelet(_Session({'pods': [{'podRef': {}, + 'containers': []}]}), + ['node-1'])) + + assert lc._anon_peak == {} and lc._eph_peak == {} + + +def test_an_unreachable_kubelet_costs_the_sample_not_the_sampler(sampler): + """The axis going quiet must not look like "this range used nothing"; the + next node in the list still has to be visited.""" + class _Boom: + def get(self, url, **kw): + raise OSError('connection refused') + + asyncio.run(lc.sample_kubelet(_Boom(), ['node-1'])) # must not raise + + assert lc._anon_peak == {} + + +# --- flushing to the volume --------------------------------------------------- + +def _writes(monkeypatch): + seen = [] + real = lc.write_metrics + + def spy(end, attempt, values): + seen.append((end, attempt, dict(values))) + return real(end, attempt, values) + + monkeypatch.setattr(lc, 'write_metrics', spy) + return seen + + +def test_a_peak_that_barely_grows_is_not_reflushed(sampler, monkeypatch): + """One write per sample per pod, at 2048 pods, would be the dominant cost + of the sampler. Only growth past PEAK_FLUSH_RATIO earns a write.""" + seen = _writes(monkeypatch) + lc._streaming['w-1'] = ('300', '1') + + sample(payload('w-1', [container(rss=900 * MIB)])) + sample(payload('w-1', [container(rss=910 * MIB)])) + assert len(seen) == 1, f"a 1.1% rise triggered a second flush: {seen}" + + sample(payload('w-1', [container(rss=2000 * MIB)])) + assert len(seen) == 2, "a 2.2x rise did not flush" + assert seen[-1][2] == {'peakAnonBytes': 2000 * MIB} + + +def test_an_in_flight_peak_reaches_the_volume_before_the_stream_ends(sampler): + """Prometheus computed max_over_time server-side and needed no state. A + local high-water dict does: without the flush, a collector restart resets a + range's peak to whatever it is using at that moment, which under-reports and + sizes the next run too small.""" + lc._streaming['w-1'] = ('300', '1') + sample(payload('w-1', [container(rss=900 * MIB)])) + + assert jm.peaks_for_range('300', 1) == {'peakAnonBytes': 900 * MIB} + + +def test_the_disk_axis_stays_mode_gated(sampler, monkeypatch): + """ephemeral-storage is meaningless in pvc mode: /data is on the volume, + not on the node.""" + monkeypatch.setattr(lc, 'STORAGE_MODE', 'pvc') + lc._streaming['w-1'] = ('300', '1') + + sample(payload('w-1', [container(rss=900 * MIB)], eph=34 * 1024 ** 3)) + + assert lc._eph_peak == {} + assert lc._anon_peak == {'w-1': 900 * MIB}, \ + "memory sizing is not mode-specific and must be sampled in both" + + +# --- working set: sampled, recorded, never used to size anything ------------- + +def test_working_set_is_sampled_alongside_anon(sampler): + """It is what kubelet ranks node-pressure evictions on, so it explains an + eviction that rss cannot. Measured on ssc-test for one 420-ledger range: + working set read 3.61 / 7.48 / 13.49 GiB under 4Gi / 8Gi / 24000Mi limits + while rss held flat at ~2.4 GiB -- which is exactly why it is a diagnostic + and never a request.""" + sample(payload('w-1', [container(rss=900 * MIB, ws=4096 * MIB)])) + + assert lc._ws_peak == {'w-1': 4096 * MIB} + assert lc._anon_peak == {'w-1': 900 * MIB} + + +def test_finalize_records_the_working_set_peak(sampler): + """Sampling it is useless if finalize drops it on the floor.""" + sample(payload('w-1', [container(rss=900 * MIB, ws=4096 * MIB)])) + asyncio.run(lc.finalize(None, 'w-1', '300', 1, lc.TxApplyScanner(), + lambda p: True)) + + stored = jm.peaks_for_range('300', 1) + assert stored['peakWorkingSetBytes'] == 4096 * MIB + assert stored['peakAnonBytes'] == 900 * MIB + + +# --- resume is bookkeeping finalize has to carry ------------------------------ + +def test_finalize_records_that_an_attempt_resumed(sampler): + """Without this in .metrics, peaks_for_range cannot tell a resumed tail + from a complete pass, and every resumed range is profiled off its tail.""" + tx = lc.TxApplyScanner() + tx.feed("RESUME: 300/16320 reached ledger 299, replay had started") + asyncio.run(lc.finalize(None, 'w-1', '300', 1, tx, lambda p: True)) + + assert jm._attempt_resumed('300', 1) is True + + +def test_a_fresh_attempt_is_never_marked_resumed(sampler): + asyncio.run(lc.finalize(None, 'w-1', '300', 1, lc.TxApplyScanner(), + lambda p: True)) + + assert jm._attempt_resumed('300', 1) is False diff --git a/src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py b/src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py new file mode 100644 index 00000000..451852f2 --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py @@ -0,0 +1,210 @@ +"""What the monitor writes down about an attempt it is about to throw away. + +Things only this process can record, each with a window that closes the moment +the Job is reaped: + + .outcome why the attempt failed, and how long it ran + .log.gz the backstop archive, for a range the collector never claimed + the log line the one place a condemned range explains itself + progress.json the record that makes the volume and the Job disposable + +Driven through the real reconcile against the fake cluster, because the window +is the point: each of these has to happen on the pass that classifies the +failure, while the pod object is still there. +""" + +import gzip +import json +import logging +import os + +import pytest + +import job_monitor as jm + + +# --- a failed attempt's duration ---------------------------------------------- + +def test_a_failed_attempts_duration_is_persisted_with_its_verdict(cluster): + """The only moment it is available. + + reconcile computes `seconds` solely on the success path, and the pod is + about to be reaped -- so without this a resumed chain can only ever report + its final leg, and every attempt lost to a spot eviction drops out of the + range's compute total. + """ + cluster.reconcile() + cluster.advance(300, 'incomplete') + cluster.reconcile() + + outcome = jm.read_outcome('300', 1) + assert outcome['outcome'] == 'failed' + assert outcome['attemptSeconds'] == pytest.approx(60.0, abs=5.0), outcome + + +def test_that_duration_is_what_the_chain_adds_up(cluster): + """The consumer, not just the file: a range that resumes must report the + compute of every leg, and the earlier legs exist only as .outcome.""" + cluster.reconcile() + cluster.advance(300, 'incomplete') + cluster.finalize(300, 1) + cluster.reconcile() + cluster.finalize(300, 2, resumed=True) + + assert jm.seconds_for_range('300', 2, 300.0) == pytest.approx(360.0, abs=5.0) + + +def test_a_verdict_already_on_the_volume_is_not_rewritten(cluster): + """The collector writes this file too, from the pod, while it still exists. + Its verdict is the one taken with the best evidence and must win.""" + cluster.reconcile() + cluster.write(jm.outcome_path('300', 1), + '{"outcome": "disrupted", "exitCode": null, "pod": "w-300", ' + '"attemptSeconds": 1800.0}') + cluster.advance(300, 'incomplete') + cluster.reconcile() + + # The pod exited 3, which reads as a plain catchup failure. The collector + # saw the eviction that caused it, so its verdict -- and its duration -- + # stand. + assert jm.read_outcome('300', 1)['outcome'] == 'disrupted' + assert jm.read_outcome('300', 1)['attemptSeconds'] == 1800.0 + + +# --- the condemned range has to say so ---------------------------------------- + +def test_a_condemned_range_is_logged_loudly(cluster, caplog): + """The zero-retry path used to log nothing at all: the range appeared under + failed{} and the mission aborted with no line saying why. A condemnation + fails a ten-hour run, so it is the one verdict that must be impossible to + miss in the monitor's own log -- which is the log the mission collects.""" + cluster.reconcile() + cluster.advance(300, 'condemned') + with caplog.at_level(logging.ERROR, logger=jm.logger.name): + cluster.reconcile() + + condemned = [r for r in caplog.records if 'RANGE CONDEMNED' in r.getMessage()] + assert condemned, [r.getMessage() for r in caplog.records] + said = condemned[0].getMessage() + assert '300' in said and 'failed' in said, said + assert '300' in cluster.failed() + + +def test_an_exhausted_range_says_which_budget_it_spent(cluster, caplog): + """The other way a range ends: it was retryable and ran out. That is a + different operator action from a condemnation, so it reads differently.""" + cluster.reconcile() + for attempt in range(1, jm.MAX_ATTEMPTS_PER_RANGE + 1): + cluster.advance(300, 'incomplete', attempt=attempt) + with caplog.at_level(logging.ERROR, logger=jm.logger.name): + cluster.reconcile() + cluster.finalize(300, attempt) + + exhausted = [r.getMessage() for r in caplog.records + if 'exhausted' in r.getMessage()] + assert exhausted, [r.getMessage() for r in caplog.records] + assert '300' in cluster.failed() + + +# --- the backstop archive ------------------------------------------------------ + +def test_the_backstop_saves_a_log_the_collector_never_claimed(cluster): + """Last resort for a pod that lived and died entirely while the collector + was down. The pod is about to be reaped, so this is the last read of it.""" + cluster.reconcile() + pod = cluster.k8s.pod_for_job(cluster.job_name(300, 1)) + cluster.k8s.set_pod_log(pod.metadata.name, + "metric 'ledger.transaction.apply'\n" + " sum = 1500.0ms\n") + cluster.advance(300, 'incomplete') + cluster.reconcile() + + path = jm.log_path('300', 1) + assert os.path.exists(path), "a failed attempt left no archive at all" + with gzip.open(path, 'rt') as fh: + assert 'sum = 1500.0ms' in fh.read() + # ...and the archive is what the monitor's own reader then recovers from. + assert jm._tx_apply_for_attempt('300', 1) == pytest.approx(1.5) + + +def test_the_backstop_stands_down_for_a_range_the_collector_claimed(cluster): + """Two writers appending to one gzip interleave members and duplicate + lines. The collector's .state file is the claim, written the moment it + opens a poller -- empty or not.""" + cluster.reconcile() + cluster.write(jm.state_path('300', 1), '') + cluster.advance(300, 'incomplete') + cluster.reconcile() + + assert not os.path.exists(jm.log_path('300', 1)), \ + "the monitor wrote over an archive the collector had claimed" + + +def test_a_torn_backstop_archive_is_never_left_behind(cluster, monkeypatch): + """job_monitor reads this same file back to recover txApplySeconds, and + gzip raises on a truncated member. A half-written archive would cost the + metric permanently, so the write goes through .tmp and a rename.""" + cluster.reconcile() + real_replace = jm.os.replace + monkeypatch.setattr(jm.os, 'replace', + lambda *a, **kw: (_ for _ in ()).throw(OSError(28, 'ENOSPC')) + if str(a[1]).endswith('.log.gz') else real_replace(*a, **kw)) + + pod = cluster.k8s.pod_for_job(cluster.job_name(300, 1)) + assert jm.backstop_save_pod_log(pod.metadata.name, '300', 1) is False + + assert not os.path.exists(jm.log_path('300', 1)) + assert jm._tx_apply_for_attempt('300', 1) is None + + +# --- the progress record -------------------------------------------------------- + +def test_the_progress_record_is_replaced_whole_or_not_at_all(cluster, monkeypatch): + """The mission driver reads progress.json off the volume while the monitor + is still writing it, and a partial file is unparseable JSON -- which reads + as "nothing has been done" and makes every recorded range eligible again. + + Written to a .tmp and renamed, so a write that dies leaves the previous + record exactly as it was. + """ + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakAnonBytes': 7}) + cluster.reconcile() + before = json.load(open(jm.PROGRESS_FILE)) + assert '300' in before['completed'] + + real_open = open + + class _HalfWrite: + def __init__(self, path): + self.fh = real_open(path, 'w') + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.fh.close() + return False + + def write(self, blob): + self.fh.write(blob[:len(blob) // 2]) + raise OSError(28, 'No space left on device') + + armed = {'v': True} + + def half_open(path, mode='r', *a, **kw): + if armed['v'] and mode == 'w' and str(path).endswith('.json.tmp'): + return _HalfWrite(path) + return real_open(path, mode, *a, **kw) + + monkeypatch.setattr(jm, 'open', half_open, raising=False) + cluster.advance(200, 'succeeded') + cluster.finalize(200, 1, tx_apply=2.5, peaks={'peakAnonBytes': 9}) + with pytest.raises(OSError): + cluster.reconcile() + armed['v'] = False + + # Not truncated, not empty, and not half of two records spliced together. + assert json.load(open(jm.PROGRESS_FILE)) == before + assert jm.load_progress()['completed']['300']['peakAnonBytes'] == 7 diff --git a/src/MissionParallelCatchup/tests/unit/test_poll_lifecycle.py b/src/MissionParallelCatchup/tests/unit/test_poll_lifecycle.py new file mode 100644 index 00000000..e02aa1fa --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_poll_lifecycle.py @@ -0,0 +1,274 @@ +"""poll_pod / _poll_once: when a read ends an attempt and when it does not. + +Every one of these is executed rather than pattern-matched. That is not a +stylistic preference: an earlier generation of these tests asserted on an +`except ClientResponseError` branch that raise_for_status could never reach and +passed green against dead code, and another pinned the literal +`gzip.open(..., 'at')` and went red over the atomic-append fix, which preserved +everything the test existed to protect. + +The fake apiserver here answers the log endpoint only. What is asserted is what +lands on the shared volume -- the archive, .metrics, .done -- because that is +the entire interface the monitor sees. +""" + +import asyncio +import gzip +import os + +import pytest + +import job_monitor as jm +import log_collector as lc + + +@pytest.fixture +def volume(tmp_path, monkeypatch): + monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(jm, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(lc, 'token', lambda: 'tok') + monkeypatch.setattr(lc, 'LOG_POLL_SECONDS', 0.02) + monkeypatch.setattr(lc, 'TERMINAL_POLL_ATTEMPTS', 2) + for name in ('_eph_peak', '_anon_peak', '_ws_peak', '_peak_flushed', + '_streaming', '_pod_secs', '_wake'): + monkeypatch.setattr(lc, name, {}) + return tmp_path + + +class _Resp: + def __init__(self, status, body='', after_read=None): + self.status = status + self._body = body.encode() + self._after_read = after_read + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + def raise_for_status(self): + if self.status >= 400: + raise RuntimeError(f"HTTP {self.status}") + + @property + def content(self): + data, after = self._body, self._after_read + + class _Chunks: + async def iter_chunked(self, n): + for i in range(0, len(data), n): + yield data[i:i + n] + if after is not None: + after() + + return _Chunks() + + +class Apiserver: + """Answers each log GET from `answers`, repeating the last one forever.""" + + def __init__(self, *answers): + self.answers = list(answers) + self.params = [] + + def get(self, url, params=None, headers=None): + self.params.append(dict(params or {})) + i = min(len(self.params) - 1, len(self.answers) - 1) + return self.answers[i] + + +def archive(end='300', attempt='1'): + path = lc.base(end, attempt) + '.log.gz' + if not os.path.exists(path): + return '' + with gzip.open(path, 'rt') as fh: + return fh.read() + + +def drive(session, terminal, timeout=3): + async def go(): + await asyncio.wait_for( + lc.poll_pod(session, 'w-1', '300', '1', + lambda p: terminal(), lambda p: False), + timeout=timeout) + asyncio.run(go()) + + +# --- the pod object is gone --------------------------------------------------- + +def test_a_404_finalizes_what_was_already_streamed(volume): + """The pod object is gone, but the bytes already read still owe a tx_apply, + and .done is what lets the monitor stop waiting on the Job. + + This path used to not exist: a pod deleted while Running left its stream + retrying for the rest of the run, holding a connection slot.""" + body = ("2026-07-30T00:00:01Z metric 'ledger.transaction.apply'\n" + "2026-07-30T00:00:02Z sum = 1500.0ms\n") + drive(Apiserver(_Resp(200, body), _Resp(404)), lambda: False) + + assert jm._attempt_finalized('300', 1), "a vanished pod never finalized" + assert jm.tx_apply_for_range('300', 1) == pytest.approx(1.5) + assert 'sum = 1500.0ms' in archive() + + +def test_an_interrupted_read_on_a_live_pod_does_not_finalize(volume): + """Still running, so retrying is correct. Finalizing here writes a + truncated peak and leaves the range looking measured when it is not.""" + with pytest.raises(asyncio.TimeoutError): + drive(Apiserver(_Resp(500)), lambda: False, timeout=0.4) + + assert not jm._attempt_finalized('300', 1), \ + "a live pod's attempt was closed out on a transient read failure" + + +def test_a_terminal_pod_whose_polls_keep_failing_still_finalizes(volume): + """The other side of it: the container has exited and its log is not + coming back, so the loop has to decide to stop asking rather than spin on a + dead pod and never write its metrics.""" + drive(Apiserver(_Resp(500)), lambda: True) + + assert jm._attempt_finalized('300', 1) + + +# --- the read that catches the last lines ------------------------------------ + +def test_terminal_is_sampled_before_the_poll_not_after(volume): + """A pod that exits mid-poll must still get one more read. + + If `done()` were consulted after the poll instead of before it, the poll + that was in flight when the container exited would be treated as the final + one -- and everything the container wrote on its way out, which is where + the medida block lives, is dropped. + """ + state = {'terminal': False} + first = _Resp(200, "2026-07-30T00:00:01Z catchup ledger 42000000\n", + after_read=lambda: state.update(terminal=True)) + last = _Resp(200, + "2026-07-30T00:00:09Z metric 'ledger.transaction.apply'\n" + "2026-07-30T00:00:10Z sum = 1500.0ms\n") + + drive(Apiserver(first, last), lambda: state['terminal']) + + assert 'catchup ledger 42000000' in archive() + assert 'sum = 1500.0ms' in archive(), \ + "the read after the pod went terminal never happened" + assert jm.tx_apply_for_range('300', 1) == pytest.approx(1.5) + + +# --- resuming a read ---------------------------------------------------------- + +def test_a_poll_resumes_from_the_last_durable_timestamp(volume): + """Without sinceTime a reconnect re-reads the whole log from the start: + one full re-read per pod per reconnect, at 2096 pods.""" + api = Apiserver(_Resp(200, "2026-07-30T00:00:05Z line\n")) + scanner = lc.TxApplyScanner() + last, gone = asyncio.run(lc._poll_once(api, 'w-1', '300', '1', None, scanner)) + + assert api.params[0].get('sinceTime') is None + assert last == '2026-07-30T00:00:05Z' and gone is False + + asyncio.run(lc._poll_once(api, 'w-1', '300', '1', last, scanner)) + assert api.params[1]['sinceTime'] == '2026-07-30T00:00:05Z', \ + "the second poll did not resume where the first stopped" + + +def test_the_second_granularity_overlap_is_deduped_exactly(volume): + """sinceTime only accepts whole seconds, so a resume deliberately re-reads + the second it stopped in. Every line carries a nanosecond timestamp, so the + overlap is removed per line rather than tolerated as duplicates.""" + body = ("2026-07-30T00:00:05.100000000Z already seen\n" + "2026-07-30T00:00:05.900000000Z brand new\n") + api = Apiserver(_Resp(200, body)) + asyncio.run(lc._poll_once(api, 'w-1', '300', '1', + '2026-07-30T00:00:05.100000000Z', + lc.TxApplyScanner())) + + written = archive() + assert 'brand new' in written + assert 'already seen' not in written + + +def test_untimestamped_kubelet_text_is_kept_but_never_resumed_from(volume): + """"unable to retrieve container logs for containerd://..." partitions to + "unable", and sinceTime=unableZ is a 400 on every later request for that + pod, forever.""" + api = Apiserver(_Resp(200, "unable to retrieve container logs for " + "containerd://9f2c1a\n")) + last, _ = asyncio.run(lc._poll_once(api, 'w-1', '300', '1', None, + lc.TxApplyScanner())) + + assert last is None, f"junk became a resume point: {last!r}" + assert 'unable to retrieve' in archive(), "the line was dropped instead" + + +# --- bounds ------------------------------------------------------------------- + +def test_an_unterminated_blob_is_capped_not_buffered_forever(volume, monkeypatch): + """A meter that never emits a newline would otherwise grow the buffer until + the collector OOMs -- 2096 streams doing it at once.""" + monkeypatch.setattr(lc, 'MAX_POLL_CHARS', 1024) + api = Apiserver(_Resp(200, 'x' * (4 * 1024 * 1024))) + + asyncio.run(lc._poll_once(api, 'w-1', '300', '1', None, lc.TxApplyScanner())) + + # One chunk's worth of overshoot is inherent -- the cap is checked between + # chunks -- but the 4 MiB body must not have been buffered whole. + assert len(archive()) < 256 * 1024, "the poll buffered the entire blob" + + +def test_polls_are_bounded_by_a_semaphore(volume, monkeypatch): + """The whole point of polling over follow=true: concurrency is a tuning + parameter, not a function of how many pods exist.""" + live = {'now': 0, 'max': 0} + + class _Counting(Apiserver): + def get(self, url, params=None, headers=None): + live['now'] += 1 + live['max'] = max(live['max'], live['now']) + resp = super().get(url, params, headers) + live['now'] -= 1 + return resp + + async def go(): + monkeypatch.setattr(lc, '_poll_slots', asyncio.Semaphore(2)) + api = _Counting(_Resp(200, "2026-07-30T00:00:01Z line\n")) + await asyncio.gather(*[ + lc._poll_once(api, 'w-1', '300', str(n), None, lc.TxApplyScanner()) + for n in range(8)]) + + asyncio.run(go()) + assert live['max'] <= 2, f"{live['max']} polls were in flight at once" + + +# --- the allowlist that decides a pod is worth polling at all ----------------- + +def test_only_phases_whose_log_endpoint_can_answer_are_pollable(): + """An allowlist, not "skip Pending". A container that has not started + answers 400 "waiting to start" -- 60 of 88 poll failures right after the + polling switch -- and Unknown means the node stopped reporting, so that + poll cannot succeed either. The terminal phases stay in: a terminal pod is + where the final output lives. + """ + assert set(lc.POLLABLE_PHASES) == {'Running', 'Succeeded', 'Failed'} + + +def test_the_terminal_retry_budget_can_absorb_a_transient_failure(): + """At 1 a single 500 ends the attempt on whatever had been read.""" + assert lc.TERMINAL_POLL_ATTEMPTS >= 2 + + +def test_poll_concurrency_is_a_modest_default(): + """It sizes the connection pool as well (MAX_CONCURRENT_POLLS + 64), so + both directions cost: too low starves the retries, too high recreates the + per-pod connection load polling exists to remove.""" + assert 16 <= lc.MAX_CONCURRENT_POLLS <= 256 + + +def test_the_wake_entry_is_dropped_when_the_attempt_finishes(volume): + """One _wake entry per pod, and pods are per range per attempt: 3979 ranges + plus their retries would otherwise accumulate for the life of the run.""" + drive(Apiserver(_Resp(404)), lambda: True) + + assert jm._attempt_finalized('300', 1) + assert lc._wake == {}, f"the poller's Event outlived its attempt: {lc._wake}" diff --git a/src/MissionParallelCatchup/tests/unit/test_profile_lookup.py b/src/MissionParallelCatchup/tests/unit/test_profile_lookup.py new file mode 100644 index 00000000..a48dc4b3 --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_profile_lookup.py @@ -0,0 +1,123 @@ +"""Loading a previous run's measurements, and picking the entry to size from. + +A profile is an optimisation, never a prerequisite: absent, unreadable and +malformed all have to mean "use the configured defaults". +""" + +import json + +import pytest + +import job_monitor as jm + + +PROFILE_RANGES = [ + (1000, {'peakRssBytes': 1_000_000_000, 'peakWorkingSetBytes': 9_000_000_000, + 'peakEphemeralBytes': 2_000_000_000, 'peakCpuCores': 0.5}), + (2000, {'peakRssBytes': 3_000_000_000, 'peakWorkingSetBytes': 13_000_000_000, + 'peakEphemeralBytes': 4_000_000_000, 'peakCpuCores': 1.2}), +] + + +@pytest.fixture +def profile(monkeypatch): + """Install a loaded profile, as load_profile() would have left it.""" + def install(ranges=PROFILE_RANGES): + monkeypatch.setattr(jm, 'PROFILE', sorted(ranges)) + return install + + +@pytest.fixture +def written(tmp_path, monkeypatch): + """Write a profile document and load it through the real reader.""" + def load(doc, mode='ephemeral', text=None): + path = tmp_path / 'profile.json' + path.write_text(text if text is not None else json.dumps(doc)) + monkeypatch.setattr(jm, 'PROFILE_PATH', str(path)) + monkeypatch.setattr(jm, 'STORAGE_MODE', mode) + return jm.load_profile() + return load + + +# --- picking an entry -------------------------------------------------------- + +def test_profile_prefers_an_exact_end(profile): + profile() + assert jm.profile_for(2000)['peakRssBytes'] == 3_000_000_000 + + +def test_profile_rounds_up_to_the_next_measured_end_never_down(profile): + # Cost rises with ledger position -- the bucket set only grows -- so a lower + # neighbour under-reports, and under-provisioning costs an eviction while + # over-provisioning only costs packing density. + profile() + assert jm.profile_for(1500)['peakRssBytes'] == 3_000_000_000, \ + "1500 must size from 2000, not from 1000" + + +def test_profile_falls_back_to_defaults_past_its_high_water_mark(profile): + # An older profile has nothing above its own top, which is exactly where a + # newer run's fresh ranges live. Extrapolating there would under-provision. + profile() + assert jm.profile_for(9999) is None + + +def test_no_profile_at_all_is_not_an_error(monkeypatch): + monkeypatch.setattr(jm, 'PROFILE', None) + assert jm.profile_for(1000) is None + monkeypatch.setattr(jm, 'PROFILE', []) + assert jm.profile_for(1000) is None + + +# --- reading the document ---------------------------------------------------- + +def test_no_configured_path_means_no_profile(monkeypatch): + monkeypatch.setattr(jm, 'PROFILE_PATH', '') + assert jm.load_profile() == [] + + +def test_an_unreadable_profile_is_not_fatal(written, tmp_path, monkeypatch): + # It is an optimisation, never a prerequisite. + assert written(None, text='{not json') == [] + monkeypatch.setattr(jm, 'PROFILE_PATH', str(tmp_path / 'nope.json')) + assert jm.load_profile() == [] + + +def test_a_matching_profile_keeps_every_axis(written): + got = written({'storageMode': 'ephemeral', + 'ranges': {'2000': PROFILE_RANGES[1][1]}}) + assert got == [(2000, PROFILE_RANGES[1][1])] + + +def test_entries_come_back_sorted_by_range_end(written): + # profile_for() bisects the list, so an unsorted load would silently size + # ranges from the wrong neighbour. + got = written({'storageMode': 'ephemeral', + 'ranges': {'3000': {}, '1000': {}, '2000': {}}}) + assert [end for end, _ in got] == [1000, 2000, 3000] + + +def test_a_non_numeric_range_key_is_skipped_not_fatal(written): + got = written({'storageMode': 'ephemeral', + 'ranges': {'2000': {'peakRssBytes': 1}, 'tip': {'peakRssBytes': 2}}}) + assert [end for end, _ in got] == [2000] + + +def test_a_cross_mode_profile_keeps_memory_but_drops_disk(written): + # cpu and memory measure the same work in either mode. Disk does not: a pvc + # run never measures node-local usage at all, so its absence must fall back + # to the configured default rather than size the wrong dimension. Degrade, + # never reject -- a rejected profile loses the transferable axes too. + got = written({'storageMode': 'pvc', 'ranges': {'2000': PROFILE_RANGES[1][1]}}, + mode='ephemeral') + assert len(got) == 1 + rec = got[0][1] + assert 'peakEphemeralBytes' not in rec + assert rec['peakRssBytes'] == 3_000_000_000 + assert rec['peakCpuCores'] == 1.2 + + +def test_a_profile_with_no_declared_mode_is_taken_at_face_value(written): + # Pre-dates the field; rejecting it would discard every older artifact. + got = written({'ranges': {'2000': PROFILE_RANGES[1][1]}}, mode='ephemeral') + assert got[0][1]['peakEphemeralBytes'] == 4_000_000_000 diff --git a/src/MissionParallelCatchup/tests/unit/test_range_generation.py b/src/MissionParallelCatchup/tests/unit/test_range_generation.py new file mode 100644 index 00000000..54955821 --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_range_generation.py @@ -0,0 +1,84 @@ +"""The ledger range list, and the order it is dispatched in. + +generate_ranges() must stay a pure function of config: dispatch derives the +full list on every reconcile, so a restart has to reproduce it exactly. +""" + +import pytest + +import job_monitor as jm + + +@pytest.fixture +def ranges(monkeypatch): + """Configure the generator and return a callable that runs it.""" + def configure(generator='uniform', order='tip-first', parallelism=4, + start=39990000, latest=40000000, per_job=1000, + floor=64000, overlap=320): + monkeypatch.setattr(jm, 'RANGE_GENERATOR', generator) + monkeypatch.setattr(jm, 'RANGE_ORDER', order) + monkeypatch.setattr(jm, 'PARALLELISM', parallelism) + monkeypatch.setattr(jm, 'STARTING_LEDGER', start) + monkeypatch.setattr(jm, 'LATEST_LEDGER_NUM', latest) + monkeypatch.setattr(jm, 'LEDGERS_PER_JOB', per_job) + monkeypatch.setattr(jm, 'LOGARITHMIC_FLOOR_LEDGERS', floor) + monkeypatch.setattr(jm, 'OVERLAP_LEDGERS', overlap) + return jm.generate_ranges() + return configure + + +def test_generators_emit_tip_first_by_default(ranges): + r = ranges() + assert r[0][0] > r[-1][0], "index 0 must be the tip" + + +def test_oldest_first_reverses_dispatch_without_dropping_ranges(ranges): + # A profiling run wants the cheap early ranges measured first: the bucket + # set only grows with ledger position, so tip-first front-loads the + # expensive ones and an interrupted run profiles nothing cheap. + tip = ranges(order='tip-first') + old = ranges(order='oldest-first') + assert old == list(reversed(tip)) + assert sorted(old) == sorted(tip), "reversing must not change the range set" + + +def test_every_range_carries_the_overlap_on_top_of_its_ledger_count(ranges): + # The count is what the worker is asked to catch up, and it is always the + # segment plus OVERLAP_LEDGERS -- measuring with overlap 0 measures nothing + # the run will ever dispatch. + r = ranges(per_job=1000, overlap=320) + assert {count for _, count in r} == {1320} + + +def test_the_ranges_tile_the_ledger_space_with_no_gap(ranges): + r = sorted(ranges(start=0, latest=10000, per_job=1000, overlap=320)) + ends = [end for end, _ in r] + assert ends == list(range(1000, 10001, 1000)) + assert ends[-1] == 10000, "the tip must be covered" + + +def test_a_short_tail_segment_is_not_padded_past_the_start(ranges): + # The last segment is min(remaining, seg_size), so a range list over a span + # that does not divide evenly must not reach below STARTING_LEDGER. + r = ranges(start=0, latest=2500, per_job=1000, overlap=0) + assert sorted(r) == [(500, 500), (1500, 1000), (2500, 1000)] + + +def test_logarithmic_ranges_match_the_shell_generator(ranges): + # Verbatim output of logarithmic_range_generator.sh with + # floor=16000 overlap=320 start=0 latest=500000 parallelism=4, captured + # before it was deleted. Chunk size halves toward the tip, so exact values + # are pinned rather than a count. + expected = ("250000/62820 187500/62820 125000/62820 62500/62820 " + "375001/31570 343751/31570 312501/31570 281251/31570 " + "500000/16320 484000/16320 468000/16320 452000/14817").split() + r = ranges(generator='logarithmic', floor=16000, overlap=320, + start=0, latest=500000, parallelism=4) + assert [f"{end}/{count}" for end, count in r] == expected + + +def test_the_logarithmic_generator_also_honours_dispatch_order(ranges): + tip = ranges(generator='logarithmic', floor=16000, start=0, latest=500000) + old = ranges(generator='logarithmic', floor=16000, start=0, latest=500000, + order='oldest-first') + assert old == list(reversed(tip)) diff --git a/src/MissionParallelCatchup/tests/unit/test_reaping.py b/src/MissionParallelCatchup/tests/unit/test_reaping.py new file mode 100644 index 00000000..67954a2b --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_reaping.py @@ -0,0 +1,168 @@ +"""Deleting finished Jobs and released volumes. + +reconcile() LISTs every Job and Pod each pass, so a finished Job is not free: +it inflates two LIST calls for as long as it lingers. At 2048-4096 parallelism +with a real OOM or spot-eviction rate that is hundreds of dead objects per hour +of run, and the apiserver pressure shows up as truncated list responses long +before anything else complains. + +Everything here is best-effort by design: a cleanup failure costs disk or etcd, +never correctness, and raising would abort a reconcile pass mid-run and strand +every other range in the same iteration. +""" + +import pytest +from kubernetes import client + +import fake_k8s +import job_monitor as jm + + +NAMESPACE = 'catchup-test' +RUN = 'pc' + + +@pytest.fixture +def k8s(logdir, monkeypatch): + """A fake cluster wired into the monitor, with no reconcile in the way.""" + fake = fake_k8s.FakeCluster(namespace=NAMESPACE) + monkeypatch.setattr(jm, 'core_v1', fake.core_v1) + monkeypatch.setattr(jm, 'batch_v1', fake.batch_v1) + monkeypatch.setattr(jm, 'NAMESPACE', NAMESPACE) + monkeypatch.setattr(jm, 'RUN_NAME', RUN) + monkeypatch.setattr(jm, 'STORAGE_MODE', 'pvc') + + def add_job(end, attempt): + name = jm.job_name(end, attempt) + labels = {jm.LABEL_RUN: RUN, jm.LABEL_RANGE: str(end), + jm.LABEL_ATTEMPT: str(attempt)} + fake.batch_v1.create_namespaced_job(NAMESPACE, client.V1Job( + metadata=client.V1ObjectMeta(name=name, labels=labels), + spec=client.V1JobSpec( + template=client.V1PodTemplateSpec( + metadata=client.V1ObjectMeta(labels=labels), + spec=client.V1PodSpec(containers=[], restart_policy='Never'))))) + return name + + fake.add_job = add_job + return fake + + +class Boom: + """A batch API that fails every delete with one status.""" + + def __init__(self, status): + self.status = status + self.calls = 0 + + def delete_namespaced_job(self, name, namespace, **_): + self.calls += 1 + raise fake_k8s.api_exception(self.status, 'boom') + + def list_namespaced_job(self, namespace, **_): + raise fake_k8s.api_exception(self.status, 'boom') + + +# --- deleting one attempt's Job ---------------------------------------------- + +def test_delete_job_reaps_the_pod_too(k8s): + # Background propagation is what actually removes the pod. Orphan would + # leave the pod behind, and the pod is what reconcile lists. + name = k8s.add_job(30957951, 2) + assert k8s.pod_for_job(name) is not None + jm.delete_job(30957951, 2) + assert k8s.job_names() == [] + assert k8s.pod_for_job(name) is None, "the pod outlived its Job" + + +@pytest.mark.parametrize('status', [404, 403, 500]) +def test_delete_job_is_best_effort(monkeypatch, k8s, status): + # A 404 is the normal race with the TTL controller, not an error. Any other + # status must be swallowed too: losing a Job to a leaked object is a + # disk/etcd cost, but raising here would abort the whole reconcile pass. + boom = Boom(status) + monkeypatch.setattr(jm, 'batch_v1', boom) + jm.delete_job(1, 1) # must not raise + assert boom.calls == 1 + + +# --- deleting every Job a completed range has -------------------------------- + +def test_a_completed_range_reaps_every_attempt_not_just_the_winner(k8s): + # Completion is terminal for the RANGE. An attempt-scoped reap leaves an + # older Failed Job standing -- typically one lost to node disruption whose + # collector died with the node, so it was never finalized and was + # deliberately not deleted. Once the winner's Job is gone that leftover is + # the range's highest live attempt, and the next pass feeds it into the + # retry decision and re-runs an already-recorded range. + k8s.add_job(300, 1) + k8s.add_job(300, 2) + other = k8s.add_job(400, 1) + jm.reap_range_jobs(300) + assert k8s.job_names() == [other], "the reap is not scoped to the range" + + +def test_a_list_failure_leaves_the_jobs_to_the_ttl_rather_than_raising(monkeypatch, k8s): + monkeypatch.setattr(jm, 'batch_v1', Boom(500)) + jm.reap_range_jobs(300) # must not raise + + +# --- the gate in front of both ----------------------------------------------- + +def test_the_reap_waits_for_the_collectors_done_marker(k8s): + # Not inferred from peaks or tx_apply: tx_apply falls back to the archive so + # it lands long before the collector finishes, and an attempt can finalize + # with no peaks at all. Only the collector knows it is done, and deleting + # the Job reaps the pod -- the last place peaks could still be read from. + k8s.add_job(300, 1) + full = {'txApply': 5.0, 'peakAnonBytes': 99} + jm._reap_if_complete(300, 1, full) + assert k8s.job_names() == [jm.job_name(300, 1)], \ + "reaped before the collector marked it done" + open(jm.done_path(300, 1), 'w').close() + jm._reap_if_complete(300, 1, full) + assert k8s.job_names() == [] + + +def test_the_done_marker_is_the_only_thing_that_counts_as_finalized(logdir): + assert jm._attempt_finalized(300, 1) is False + open(jm.metrics_path(300, 1), 'w').close() + assert jm._attempt_finalized(300, 1) is False, "metrics are not a promise" + open(jm.done_path(300, 1), 'w').close() + assert jm._attempt_finalized(300, 1) is True + + +def test_a_record_has_peaks_only_if_some_axis_actually_measured_something(): + assert jm._has_peaks({'peakAnonBytes': 1}) is True + assert jm._has_peaks({'peakAnonBytes': None, 'txApply': 5.0}) is False + assert jm._has_peaks({}) is False + + +# --- releasing the volume ---------------------------------------------------- + +def test_a_completed_range_releases_its_volume(k8s): + # PVCs are owner-referenced to the release, so nothing reclaimed them until + # helm uninstall. Measured on ssc-test: 2032 bound PVCs / 79 TiB a third of + # the way through a 3982-range run, heading for ~156 TiB and 3982 volumes + # against the account's volume ceiling. + name = jm.ensure_pvc(300, owner=None) + assert k8s.pvc_names() == [name] + jm.release_pvc(300) + assert k8s.pvc_names() == [] + + +def test_ephemeral_mode_has_no_volume_to_release(monkeypatch, k8s): + jm.ensure_pvc(300, owner=None) + monkeypatch.setattr(jm, 'STORAGE_MODE', 'ephemeral') + jm.release_pvc(300) + assert k8s.pvc_names() != [], "ephemeral mode deleted a volume it does not own" + + +def test_releasing_a_volume_never_fails_a_completed_range(k8s): + # Already-gone is the common case (a restart re-running the same tail), and + # a disk cleanup failure must not condemn a finished range either way. + jm.release_pvc(300) # nothing there: 404, must not raise + name = jm.ensure_pvc(300, owner=None) + k8s.fail_next['delete pvc'] = fake_k8s.api_exception(403, 'Forbidden') + jm.release_pvc(300) # must not raise + assert k8s.pvc_names() == [name], "the 403 was never actually injected" diff --git a/src/MissionParallelCatchup/tests/unit/test_records.py b/src/MissionParallelCatchup/tests/unit/test_records.py new file mode 100644 index 00000000..4ddbaab5 --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_records.py @@ -0,0 +1,127 @@ +"""The filenames and record shapes the two processes agree on. + +The monitor and the collector are separate containers sharing one volume. Every +handoff between them is a filename, and a mismatch is silent: the monitor +simply never reaps and every Job waits out its TTL. +""" + +import os + +import job_monitor as jm +import log_collector as lc + + +def basename(path): + return os.path.basename(path) + + +# --- one volume, one set of filenames ---------------------------------------- + +def test_both_sides_agree_on_the_metrics_filename(logdir): + assert basename(jm.metrics_path(300, 2)) == basename(lc.base(300, 2)) + '.metrics' + + +def test_both_sides_agree_on_the_done_marker(logdir): + # It licenses the monitor to reap the pod, which is the only place peaks can + # still be read from. + assert basename(jm.done_path(300, 2)) == basename(lc.done_path(300, 2)) + + +def test_both_sides_agree_on_the_attempt_label_key(): + # Two readers, one key. A mismatch reproduces the silent collision below. + assert jm.LABEL_ATTEMPT == lc.LABEL_ATTEMPT + + +def test_the_monitor_log_lands_where_the_mission_collects_it(): + # collectLogsFromPods tars LOG_DIR. The monitor used to write its own log to + # /data, an emptyDir, so OOM-retry storms never reached the destination + # directory and did not survive a monitor restart. + assert jm.LOG_DIR == lc.LOG_DIR, \ + "collector and monitor must share the collected directory" + assert os.path.dirname(jm.PROGRESS_FILE) == jm.LOG_DIR + + +def test_every_per_attempt_artifact_is_named_for_its_attempt(logdir): + # One namespace per (range, attempt) across five writers; a helper that + # dropped the attempt would have two attempts overwrite each other. + paths = [jm.log_path(300, 2), jm.state_path(300, 2), jm.outcome_path(300, 2), + jm.metrics_path(300, 2), jm.verdict_path(300, 2), jm.done_path(300, 2)] + assert all(basename(p).startswith('range-300-a2.') for p in paths), paths + assert len({basename(p) for p in paths}) == len(paths), "two writers share a filename" + + +# --- the worker pod's own labels --------------------------------------------- + +def test_the_worker_pod_carries_its_attempt_number(): + # The collector reads LABEL_ATTEMPT off the POD, not the Job, and defaults + # to "1". With the label only on the Job every attempt claimed the same + # range--a1.* files: measured on ssc-test 2026-07-30, 2246 metrics + # files all a1 while 475 a2 pods ran, so each retry overwrote the first + # attempt's peak instead of being maxed against it -- destroying exactly + # the OOM evidence the chain exists to keep. + labels = jm.pod_labels(300, 2) + assert labels[jm.LABEL_ATTEMPT] == '2' + assert labels[jm.LABEL_RANGE] == '300' + assert labels[jm.LABEL_RUN] == jm.RUN_NAME + + +def test_the_mission_label_is_opt_in(monkeypatch): + # It is high-cardinality and only wanted when something is scraping by + # mission, so it must not appear unless both switches are set. + monkeypatch.setattr(jm, 'MISSION', 'pubnet-catchup') + monkeypatch.setattr(jm, 'EMIT_MISSION_LABEL', False) + assert 'mission' not in jm.pod_labels(300, 1) + monkeypatch.setattr(jm, 'EMIT_MISSION_LABEL', True) + assert jm.pod_labels(300, 1)['mission'] == 'pubnet-catchup' + + +# --- what the ConfigMap mirror is allowed to carry --------------------------- + +def test_the_configmap_mirror_carries_no_profiling_fields(): + # Profile data lives only on the volume. In the ConfigMap it is what pushes + # a ~30-byte state record to ~172 bytes and the whole document toward the + # 1 MiB cap at ~6100 ranges -- reachable simply by halving ledgersPerJob. + progress = {'completed': {'100': {'attempts': 1, 'count': 16320, 'seconds': 700.0, + 'peakRssBytes': 123, 'peakCpuCores': 1.9, + 'txApply': 200.0, 'wallSeconds': 750.0}}, + 'failed': {}} + out = jm._state_only(progress)['completed']['100'] + assert out == {'attempts': 1, 'count': 16320}, out + # ...and the untouched original still has everything for the volume copy + assert 'peakRssBytes' in progress['completed']['100'] + + +def test_the_mirror_keeps_the_bookkeeping_the_mission_driver_reads(): + # Stripping is by field, not by whitelist-of-one: a failed range's record + # is what the driver reports on, so it must survive the trip. + progress = {'completed': {}, 'failed': {'100': {'attempts': 5, 'reason': 'oom'}}} + assert jm._state_only(progress)['failed']['100'] == {'attempts': 5, 'reason': 'oom'} + + +def test_a_structurally_wrong_record_is_dropped_not_carried(logdir): + # This document is read off a volume that outlives the run and mirrored + # through a ConfigMap a second writer can clobber, so it comes back the + # wrong SHAPE as well as merely truncated. A single non-dict entry took + # every later pass down inside observe_recorded/sync_counters -- after + # dispatch, so the exception the reconcile loop swallows left the run with + # no status update and no `remaining` ever again. + got = jm._sane_progress({'completed': {'100': {'attempts': 1}, '200': 'nonsense'}, + 'failed': {'300': ['also', 'wrong']}}) + assert got['completed'] == {'100': {'attempts': 1}} + assert got['failed'] == {} + + +# --- durations the collector can read off a pod the monitor never saw --------- + +def test_a_terminal_pod_still_yields_its_real_duration(): + # A pod carries startTime and terminated.finishedAt until it is deleted, so + # even a pod that finished before this poller existed has a real duration. + # The poller's own elapsed time cannot know that -- it measures how long WE + # watched, which is ~0 in exactly that case, and 150 metrics files came back + # with a sub-5s duration next to a >500MiB anon peak because of it. + pod = {'status': {'startTime': '2026-07-30T04:16:26Z', + 'containerStatuses': [{'state': {'terminated': { + 'finishedAt': '2026-07-30T04:22:19Z'}}}]}} + assert lc.pod_seconds(pod) == 353.0 + assert lc.pod_seconds({'status': {'startTime': '2026-07-30T04:16:26Z'}}) is None + assert lc.pod_seconds({'status': {}}) is None diff --git a/src/MissionParallelCatchup/tests/unit/test_resources.py b/src/MissionParallelCatchup/tests/unit/test_resources.py new file mode 100644 index 00000000..e2a605f0 --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_resources.py @@ -0,0 +1,209 @@ +"""Turning a measurement into the pod's requests and limits. + +_profile_overrides() decides what the profile is allowed to say; _resources() +decides what actually lands on the container. Both are called here rather than +read, because the first version of the sizing gate read `mem is None` AFTER mem +had been defaulted, so it was never true and profile sizing was silently dead +while a source-text assertion still passed. +""" + +import pytest + +import job_monitor as jm + + +PROFILE_RANGES = [ + (1000, {'peakRssBytes': 1_000_000_000, 'peakWorkingSetBytes': 9_000_000_000, + 'peakEphemeralBytes': 2_000_000_000, 'peakCpuCores': 0.5}), + (2000, {'peakRssBytes': 3_000_000_000, 'peakWorkingSetBytes': 13_000_000_000, + 'peakEphemeralBytes': 4_000_000_000, 'peakCpuCores': 1.2}), +] + +MI = 1024 ** 2 + + +@pytest.fixture +def sizing(monkeypatch): + """The worker's configured shape, plus a loaded profile.""" + def configure(ranges=PROFILE_RANGES, margin=1.1, lim_mem='24000Mi', + req_eph='35Gi', lim_eph='40Gi', max_mem='32Gi', + headroom='512Mi', cpu_limit=''): + monkeypatch.setattr(jm, 'PROFILE', sorted(ranges)) + monkeypatch.setattr(jm, 'PROFILE_MARGIN', margin) + monkeypatch.setattr(jm, 'PROFILE_MAX_MEM', max_mem) + monkeypatch.setattr(jm, 'PROFILE_CACHE_HEADROOM', headroom) + monkeypatch.setattr(jm, 'PROFILE_CPU_LIMIT', cpu_limit) + monkeypatch.setattr(jm, 'REQ_CPU', '1800m') + monkeypatch.setattr(jm, 'LIM_CPU', '2') + monkeypatch.setattr(jm, 'REQ_MEM', '9Gi') + monkeypatch.setattr(jm, 'LIM_MEM', lim_mem) + monkeypatch.setattr(jm, 'REQ_EPHEMERAL', req_eph) + monkeypatch.setattr(jm, 'LIM_EPHEMERAL', lim_eph) + return configure + + +# --- what the profile is allowed to say -------------------------------------- + +def test_profile_sizes_a_first_attempt(sizing): + sizing() + out = jm._profile_overrides(2000, escalated=False) + assert out['memory'] == '3659Mi' # 3 GB rss * 1.1 + 512Mi + assert out['ephemeral-storage'] == '4196Mi' + # cpu is no longer profiled: REQ_CPU is fixed, so there is nothing to size, + # and a measured cpu value only makes packing non-uniform. + assert 'cpu' not in out + assert 'peakCpuCores' not in jm.PEAK_FIELDS + + +def test_profile_does_not_override_an_escalated_retry(sizing): + # An escalation is a measurement of THIS run and outranks an earlier one. + sizing() + assert jm._profile_overrides(2000, escalated=True) == {} + + +def test_profile_gives_nothing_past_its_high_water_mark(sizing): + sizing() + assert jm._profile_overrides(99999, escalated=False) == {} + assert jm._profile_overrides(None, escalated=False) == {} + + +def test_profile_memory_is_capped_at_its_own_ceiling_not_the_worker_limit(sizing): + # A range needing more than the configured limit must be able to ask for it, + # or it is pinned under its own measured peak and OOMs every attempt. The + # ceiling is what bounds it, and the OOM ladder can still climb past that. + sizing(ranges=[(1, {'peakRssBytes': 500_000_000_000})], + lim_mem='24000Mi', max_mem='32Gi') + assert jm._profile_overrides(1, escalated=False)['memory'] == '32768Mi' + + +def test_profile_memory_can_exceed_the_configured_worker_limit(sizing): + # 28 GB peak against a 24000Mi configured limit: the profile must raise it. + sizing(ranges=[(1, {'peakRssBytes': 28_000_000_000})], + lim_mem='24000Mi', max_mem='32Gi') + got = jm._profile_overrides(1, escalated=False)['memory'] + assert jm._quantity_bytes(got) > jm._quantity_bytes('24000Mi') + + +def test_memory_is_sized_from_rss_never_from_working_set(sizing): + # Working set is whatever limit it was measured under -- the kernel grows + # page cache to fill it. Measured on ssc-test, one 420-ledger range: + # limit 4Gi -> ws 3.61 GiB, rss 2.43 GiB, 775s + # limit 8Gi -> ws 7.48 GiB, rss 2.41 GiB, 746s + # limit 24000Mi -> ws 13.49 GiB, rss 2.28 GiB, 773s + # rss is flat and wall-clock is flat, so sizing from ws would reserve 5x the + # real demand for no gain. It is still recorded -- kubelet ranks + # node-pressure evictions on it, so it explains an eviction rss cannot. + sizing(ranges=[(1, {'peakWorkingSetBytes': 13_000_000_000})]) + assert 'memory' not in jm._profile_overrides(1, escalated=False), \ + "an older artifact without rss must fall back, not guess from working set" + assert 'peakWorkingSetBytes' in jm.PEAK_FIELDS + + +def test_sizing_prefers_anon_and_falls_back_to_the_scraped_rss(sizing): + # peakAnonBytes is kubelet's rssBytes on the collector's own poll; + # peakRssBytes is the same quantity via a 30s Prometheus scrape. A profile + # captured before the collector tracked anon must keep sizing exactly as it + # did, or every existing profile silently reverts to default. + sizing(ranges=[(1, {'peakRssBytes': 1_000_000_000})]) + scraped_only = jm._profile_overrides(1, escalated=False)['memory'] + # Both present: the finer figure wins, not the coarser one it sits beside. + sizing(ranges=[(1, {'peakAnonBytes': 1_000_000_000, + 'peakRssBytes': 3_000_000_000})]) + assert jm._profile_overrides(1, escalated=False)['memory'] == scraped_only + + +def test_small_ranges_get_absolute_slack_not_just_a_percentage(sizing): + # memory.max bounds anon PLUS page cache. At 190 MiB rss a 1.1x margin is + # 19 MiB of slack -- measured on ssc-test, 90 ranges OOMKilled within 90s of + # dispatch. The fixed headroom is what makes small ranges survivable. + sizing(ranges=[(1, {'peakRssBytes': 190 * MI})]) + got = jm._quantity_bytes(jm._profile_overrides(1, escalated=False)['memory']) + slack = (got - 190 * MI) / MI + assert slack > 400, f"only {slack:.0f}MiB of slack above rss" + + +@pytest.mark.parametrize('peak_mi', [648, 1467, 222]) # live: median, largest, smallest anon +def test_the_sizing_formula_is_peak_times_margin_plus_headroom(sizing, peak_mi): + sizing(ranges=[(1, {'peakAnonBytes': peak_mi * MI})], margin=1.15, + headroom='512Mi', max_mem='32Gi') + got = jm._profile_overrides(1, escalated=False)['memory'] + assert got == f"{int(peak_mi * MI * 1.15) // MI + 512}Mi" + + +# --- what lands on the container --------------------------------------------- + +def test_a_measured_range_matches_memory_and_disk_and_leaves_cpu_configured(sizing): + # Memory and disk match request to limit -- exceeding either kills the pod. + # CPU keeps its configured request and is left uncapped, so the range packs + # by what it uses and can still burst. + sizing() + r = jm._resources(end=2000) + assert r.requests['memory'] == r.limits['memory'] == '3659Mi' + assert r.requests['ephemeral-storage'] == r.limits['ephemeral-storage'] == '4196Mi' + # The configured request, not a measured one -- a profiled range now packs + # at exactly the same cpu as an unprofiled one. + assert r.requests['cpu'] == '1800m' + assert 'cpu' not in r.limits, "a measured range runs uncapped" + + +def test_an_unmeasured_range_keeps_the_mismatched_defaults(sizing): + # No profile entry must behave exactly as if there were no profile at all. + sizing() + r = jm._resources(end=99999) + assert r.requests['memory'] == '9Gi' and r.limits['memory'] == '24000Mi' + assert r.requests['ephemeral-storage'] == '35Gi' + assert r.limits['ephemeral-storage'] == '40Gi' + assert r.requests != r.limits + + +def test_an_escalated_retry_keeps_its_own_size_and_raises_the_request_with_it(sizing): + # The escalation already chose the size; the profile must not overwrite it. + # The request moves too: a pod that OOMed at the old limit will not fit + # where it was scheduled before. + sizing() + r = jm._resources(mem='36000Mi', end=2000) + assert r.requests['memory'] == r.limits['memory'] == '36000Mi' + assert r.requests['cpu'] == '1800m', "cpu must fall back to the configured request" + + +def test_ephemeral_escalation_raises_request_and_limit_together(sizing): + # ephemeral-storage is a scheduling dimension: a pod that outgrew its limit + # will not fit where it was placed before unless the request moves too. + sizing() + r = jm._resources(eph='60Gi', end=2000) + assert r.requests['ephemeral-storage'] == r.limits['ephemeral-storage'] == '60Gi' + + +def test_no_worker_gets_a_cpu_limit_unless_one_is_configured(sizing): + # _profile_overrides returns {} for BOTH "no profile entry" and "escalated + # attempt". Treating them the same handed an OOM retry more memory while + # capping it at LIM_CPU, when the attempt that just failed ran unlimited. + # Measured on ssc-test 2026-07-30: 256 of 679 a2 pods were capped at cpu 2. + # Less cpu means less download concurrency means a lower peak, so the retry + # succeeds at a figure the next run cannot reproduce unthrottled. + # + # At a 2-core limit every range pegs 2.0 anyway, so the measured peak would + # be a ceiling and the profile could never learn real demand. Packing is + # driven by the request, which every worker still carries. + sizing() + measured = jm._resources(end=2000) + escalated = jm._resources(mem='9000Mi', end=2000) + unmeasured = jm._resources(end=999999999) + for r, why in ((measured, 'measured'), (escalated, 'escalated retry'), + (unmeasured, 'unprofiled')): + assert 'cpu' not in r.limits, f"{why} range was throttled: {r.limits}" + assert r.requests['cpu'] == '1800m', why + + +def test_a_configured_cpu_limit_is_still_honoured(sizing): + sizing(cpu_limit='3') + assert jm._resources(end=2000).limits['cpu'] == '3' + + +def test_pvc_mode_takes_no_ephemeral_request_or_override(sizing): + # /data is not on the node disk there, so sizing it would be meaningless -- + # and a large request would make disk the binding dimension and halve + # workers-per-node for no reason. + sizing(req_eph='') + r = jm._resources(end=2000) + assert 'ephemeral-storage' not in r.requests diff --git a/src/MissionParallelCatchup/tests/unit/test_resume_script.py b/src/MissionParallelCatchup/tests/unit/test_resume_script.py new file mode 100644 index 00000000..79849790 --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_resume_script.py @@ -0,0 +1,143 @@ +"""The worker's resume decision, run as the shell script it actually is. + +Measured on ssc-test 2026-07-30: a1 replayed range 16752063 to its target +ledger and was evicted before it could exit 0. a2 resumed, found LCL == TARGET, +ran catchup against a DB with nothing left to apply, and stellar-core exited 2 +-- deterministically, every attempt. The range exhausted its budget and the +mission aborted a 61%-complete 2096-worker run over work that had actually been +done. +""" + +import os +import re +import subprocess + +import pytest + +import job_monitor as jm + + +TARGET = 16752063 +COUNT = 16320 + +# What `stellar-core offline-info --console` really prints: bucketlist puts ~40 +# lines of hashes between the "ledger": key and the "num" the probe wants, which +# is why the probe must not window its grep. Verified against 27.1.1 on ssc-test +# 2026-07-30 -- exactly one "num" key in the document, and it is the ledger's. +def offline_info(lcl): + buckets = ',\n'.join(f' "{i:064x}"' for i in range(40)) + return ('{\n "info" : {\n "ledger" : {\n' + f' "age" : 3,\n "closeTime" : 1753000000,\n' + f' "hash" : "abc",\n' + f' "bucketListHashes" : [\n{buckets}\n ],\n' + f' "num" : {lcl},\n "version" : 22\n' + ' }\n }\n}') + + +@pytest.fixture +def run_resume(tmp_path): + """Run RESUME_SCRIPT against a stubbed stellar-core on a private /data.""" + def run(lcl, mark_matches=True, prev_log_lcl=None): + data = tmp_path / 'data' + data.mkdir(exist_ok=True) + bindir = tmp_path / 'bin' + bindir.mkdir(exist_ok=True) + stub = bindir / 'stellar-core' + info = offline_info(lcl).replace("'", "") if lcl is not None else '' + stub.write_text( + '#!/bin/sh\n' + 'for a in "$@"; do case "$a" in\n' + " offline-info) " + + (f"cat <<'EOF'\n{info}\nEOF\n" if lcl is not None else 'echo "{}"; ') + + ' exit 0;;\n' + ' new-db) echo "RAN:new-db" >> "$STUBLOG"; exit 0;;\n' + ' catchup) echo "RAN:catchup" >> "$STUBLOG"; exit 2;;\n' + 'esac; done\nexit 0\n') + stub.chmod(0o755) + + src = jm.RESUME_SCRIPT % {'key': f"{TARGET}/{COUNT}", + 'target': TARGET, 'count': COUNT} + src = src.replace('/usr/bin/stellar-core', str(stub)) + src = src.replace('/data/', str(data) + '/') + + if mark_matches: + (data / '.job-key').write_text(f"{TARGET}/{COUNT}") + if prev_log_lcl is not None: + (data / 'stellar-core.log').write_text( + f"Ledger close complete: {prev_log_lcl}\n") + + stublog = tmp_path / 'stub.log' + if stublog.exists(): + stublog.unlink() + env = dict(os.environ, STUBLOG=str(stublog)) + r = subprocess.run(['/bin/sh', '-c', src], capture_output=True, text=True, + env=env, timeout=30) + ran = stublog.read_text().split() if stublog.exists() else [] + return r.returncode, r.stdout, ran + return run + + +def test_a_range_already_at_its_target_exits_success_without_recatching(run_resume): + code, out, ran = run_resume(lcl=TARGET) + assert 'ALREADY COMPLETE' in out, out + assert code == 0, f"exit {code}; a finished range must not fail" + assert 'RAN:catchup' not in ran, "re-ran catchup on a completed range -> exit 2" + assert 'RAN:new-db' not in ran, "wiped a completed range" + + +def test_a_partially_replayed_range_still_resumes(run_resume): + code, out, ran = run_resume(lcl=TARGET - 100) + assert 'RESUME:' in out and 'ALREADY COMPLETE' not in out, out + assert 'RAN:catchup' in ran and 'RAN:new-db' not in ran, ran + + +def test_a_range_that_never_started_replay_starts_fresh(run_resume): + code, out, ran = run_resume(lcl=None) + assert 'RESUME DECLINED' in out, out + assert 'RAN:new-db' in ran and 'RAN:catchup' in ran, ran + + +def test_a_range_whose_replay_never_reached_its_own_span_starts_fresh(run_resume): + # Bucket apply uses createWithoutLoading() -- an unconditional INSERT that + # assumes a fresh DB -- so a crash before replay must start over. An LCL + # below TARGET-COUNT means the bucket phase, not replay. + code, out, ran = run_resume(lcl=TARGET - COUNT - 1) + assert 'RESUME DECLINED' in out, out + assert 'RAN:new-db' in ran + + +def test_the_lcl_probe_reads_past_the_bucketlist(run_resume): + # offline-info puts ~40 lines of bucketlist hashes between "ledger": and + # "num", so `grep -A8 '"ledger":'` yields nothing and the probe degrades to + # the log fallback silently -- shipped exactly that once. + code, out, ran = run_resume(lcl=TARGET - 100) + assert f"RESUME PROBE: offline-info reports lcl {TARGET - 100}" in out, out + + +def test_the_log_fallback_covers_a_core_that_answers_nothing(run_resume): + # Goes blind above INFO, which is why it is no longer the primary probe -- + # but a core that cannot answer offline-info still leaves its own log. + code, out, ran = run_resume(lcl=None, prev_log_lcl=TARGET - 50) + assert 'RESUME:' in out, out + assert 'RAN:new-db' not in ran + + +def test_a_volume_left_by_a_different_range_is_never_resumed_from(run_resume): + # /data is per-range, but a recycled volume or a mis-scheduled pod would + # otherwise resume a DB belonging to some other span. + code, out, ran = run_resume(lcl=TARGET - 100, mark_matches=False) + assert 'RESUME' not in out, out + assert 'RAN:new-db' in ran and 'RAN:catchup' in ran + + +def test_the_resume_script_survives_its_own_percent_formatting(): + # RESUME_SCRIPT is %-formatted with the range's key/target/count at dispatch. + # A bare % anywhere in it -- including in a comment -- raises at runtime and + # takes down every job dispatch. Nearly shipped exactly that: a comment + # reading "61%-complete". + jm.RESUME_SCRIPT % {'key': '123/456', 'target': 123, 'count': 456} # must not raise + # %% is a legitimate escape (printf '%%s'), so strip those pairs before + # looking for a stray one. + probe = jm.RESUME_SCRIPT.replace('%%', '') + stray = [m.start() for m in re.finditer(r"%(?!\()", probe)] + assert not stray, f"bare % near {probe[max(0, stray[0] - 40):stray[0] + 20]!r}" diff --git a/src/MissionParallelCatchup/tests/unit/test_sizing.py b/src/MissionParallelCatchup/tests/unit/test_sizing.py new file mode 100644 index 00000000..ab4a721c --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_sizing.py @@ -0,0 +1,114 @@ +"""Resource escalation ladders and the quantity arithmetic under them. + +Everything here is a pure function of config, so the tests set the config and +call it. The budgets these ladders climb are asserted against the module +defaults -- the numbers a run gets when the chart passes nothing. +""" + +import pytest + +import job_monitor as jm + + +@pytest.fixture +def mem(monkeypatch): + def configure(lim='1000Mi', bump=None, cap='48Gi'): + monkeypatch.setattr(jm, 'LIM_MEM', lim) + monkeypatch.setattr(jm, 'MEM_BUMP_FACTOR', + jm.MEM_BUMP_FACTOR if bump is None else bump) + monkeypatch.setattr(jm, 'MEM_ESCALATION_CAP', cap) + return configure + + +@pytest.fixture +def eph(monkeypatch): + def configure(lim='40Gi', bump=1.5, cap='200Gi'): + monkeypatch.setattr(jm, 'LIM_EPHEMERAL', lim) + monkeypatch.setattr(jm, 'EPH_BUMP_FACTOR', bump) + monkeypatch.setattr(jm, 'EPH_ESCALATION_CAP', cap) + return configure + + +# --- quantity arithmetic ----------------------------------------------------- + +@pytest.mark.parametrize('quantity,want', [ + ('1024Ki', 1024 * 1024), + ('9Gi', 9 * 1024**3), + ('24000Mi', 24000 * 1024**2), + ('1G', 1000**3), # SI, not binary -- kubernetes accepts both + ('1500', 1500), # bare bytes +]) +def test_kubernetes_quantities_are_read_in_the_right_base(quantity, want): + assert jm._quantity_bytes(quantity) == want + + +def test_a_size_is_always_rendered_back_in_mebibytes(): + # One unit everywhere means a limit can be compared to a request without + # re-parsing, and Mi is fine-grained enough for the packing this run does. + assert jm._bytes_to_quantity(3 * 1024**3) == '3072Mi' + assert jm._bytes_to_quantity(0) == '1Mi', "a zero-byte limit is unschedulable" + + +def test_sizing_applies_the_margin_and_never_exceeds_the_limit(): + # 1 GB * 1.1, well under the cap + assert jm._sized(1_000_000_000, 1.1, '10Gi') == '1049Mi' + # capped: a huge peak cannot produce a request above its own limit + assert jm._sized(50_000_000_000, 1.1, '8Gi') == '8192Mi' + + +# --- the memory ladder ------------------------------------------------------- + +@pytest.mark.parametrize('attempt,want', [(1, 1.0), (2, 1.5), (3, 2.25), (4, 3.375)]) +def test_the_memory_escalation_ladder_compounds(mem, attempt, want): + # 1.5x per OOM off what the attempt actually ran with. A factor of 1.0 + # would retry an OOM at the identical limit, forever. + mem(lim='1000Mi', bump=1.5) + assert jm.mem_for_attempt(attempt, '1000Mi') == f"{int(1000 * want)}Mi" + + +def test_the_escalation_ladder_is_capped(mem): + mem(lim='1000Mi', bump=1.5, cap='4Gi') + assert jm.mem_for_attempt(20, '1000Mi') == '4096Mi', "cap not applied" + + +def test_oom_escalation_starts_from_what_the_attempt_actually_had(mem): + # Escalating a 209Mi profiled range off the configured 24000Mi limit jumps + # to 36000Mi -- a 172x overshoot that discards the packing win on first OOM. + mem(lim='24000Mi', bump=1.5) + assert jm.mem_for_attempt(2, '702Mi') == '1053Mi' + assert jm.mem_for_attempt(2) == '36000Mi' # unprofiled keeps old behaviour + + +# --- the disk ladder --------------------------------------------------------- + +def test_ephemeral_storage_escalates_and_caps_the_same_way(eph): + eph(lim='40Gi', bump=1.5, cap='200Gi') + assert jm.eph_for_attempt(1) == '40960Mi' + assert jm.eph_for_attempt(2) == '61440Mi' + assert jm.eph_for_attempt(20) == '204800Mi', "cap not applied" + + +# --- the budgets the ladders are climbing ------------------------------------ + +def test_attempt_budgets_are_ordered_by_whose_fault_the_failure_was(): + # A hang is usually persistent, so it gets the fewest tries. A genuinely + # broken range gets the middle budget. Anything the cluster did to us gets + # the most -- on spot, evictions are routine and must not condemn a range. + assert jm.MAX_TIMEOUT_ATTEMPTS < jm.MAX_ATTEMPTS_PER_RANGE < jm.MAX_DISRUPTION_ATTEMPTS, ( + f"budgets out of order: timeout={jm.MAX_TIMEOUT_ATTEMPTS} " + f"range={jm.MAX_ATTEMPTS_PER_RANGE} disruption={jm.MAX_DISRUPTION_ATTEMPTS}") + assert jm.MAX_ATTEMPTS_PER_RANGE > 1, "a range that OOMs once could never escalate" + assert jm.MAX_EPHEMERAL_ATTEMPTS > 1, "a range evicted on disk once could never grow" + assert jm.MAX_DISRUPTION_ATTEMPTS >= 10, \ + "spot eviction would condemn ranges at this budget" + + +def test_the_oom_budget_stops_short_of_the_cap_on_purpose(): + # 5 rungs is 1.5^4 = 5x the profile figure. A range needing more is broken, + # not mis-sized, and chasing it to MEM_ESCALATION_CAP parks a whole node on + # it. The price is that such a range is condemned -- which today aborts the + # run, so this coupling is what must not be forgotten. + n = jm.MAX_ATTEMPTS_PER_RANGE + assert 2 <= n <= 8, f"{n} rungs: below 2 cannot escalate, above 8 chases a broken range" + assert jm.MEM_BUMP_FACTOR ** (n - 1) >= 3.0, \ + "the ladder cannot even treble the request before giving up" diff --git a/src/MissionParallelCatchup/tests/unit/test_tx_apply.py b/src/MissionParallelCatchup/tests/unit/test_tx_apply.py new file mode 100644 index 00000000..046ca8ec --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_tx_apply.py @@ -0,0 +1,220 @@ +"""Reading 'ledger.transaction.apply' out of stellar-core's medida block. + +Two readers of one format the mission does not control: the collector's +streaming scanner, and the monitor's after-the-fact archive/pod reader. Both +are pinned against real captures so a stellar-core change fails here rather +than silently dropping the metric for a whole run. +""" + +import gzip +import json +import os + +import pytest + +import job_monitor as jm +import log_collector as lc + + +# stellar-core 27.1.1 catchup pod, --metric 'ledger.transaction.apply'. Kept +# whole: `sum` is 10 lines below the header against a 15-line scan window. +MEDIDA_BLOCK = """2026-07-28T18:39:49.350 GAJSL [default INFO] metric 'ledger.transaction.apply': +2026-07-28T18:39:49.350 GAJSL [default INFO] count = 20 +2026-07-28T18:39:49.350 GAJSL [default INFO] mean rate = 0.22136 calls/s +2026-07-28T18:39:49.350 GAJSL [default INFO] 1-minute rate = 0.113149 calls/s +2026-07-28T18:39:49.350 GAJSL [default INFO] 5-minute rate = 0.175948 calls/s +2026-07-28T18:39:49.350 GAJSL [default INFO] 15-minute rate = 0.191421 calls/s +2026-07-28T18:39:49.350 GAJSL [default INFO] min = 0.295417ms +2026-07-28T18:39:49.350 GAJSL [default INFO] max = 0.639873ms +2026-07-28T18:39:49.350 GAJSL [default INFO] mean = 0.417143ms +2026-07-28T18:39:49.350 GAJSL [default INFO] stddev = 0.108677ms +2026-07-28T18:39:49.350 GAJSL [default INFO] sum = 8.34285ms +2026-07-28T18:39:49.350 GAJSL [default INFO] median = 0ms +2026-07-28T18:39:49.350 GAJSL [default INFO] 75% = 0ms +2026-07-28T18:39:49.350 GAJSL [default INFO] 95% = 0ms +2026-07-28T18:39:49.350 GAJSL [default INFO] 98% = 0ms +2026-07-28T18:39:49.350 GAJSL [default INFO] 99% = 0ms +2026-07-28T18:39:49.350 GAJSL [default INFO] 99.9% = 0ms""" + +TX_APPLY_SECONDS = 0.00834285 + +# Real block from range-40010367-a1 on ssc-test. medida switches to scientific +# notation past 1e6 ms, which is every range with a real transaction load. +MEDIDA_BIG = """2026-07-29T20:11:16.931 GAJSL [default INFO] metric 'ledger.transaction.apply': +2026-07-29T20:11:16.931 GAJSL [default INFO] count = 3231886 +2026-07-29T20:11:16.931 GAJSL [default INFO] mean rate = 812.4 calls/s +2026-07-29T20:11:16.931 GAJSL [default INFO] 1-minute rate = 790.1 calls/s +2026-07-29T20:11:16.931 GAJSL [default INFO] 5-minute rate = 801.3 calls/s +2026-07-29T20:11:16.931 GAJSL [default INFO] 15-minute rate = 799.0 calls/s +2026-07-29T20:11:16.931 GAJSL [default INFO] min = 0.101ms +2026-07-29T20:11:16.931 GAJSL [default INFO] max = 41.2ms +2026-07-29T20:11:16.931 GAJSL [default INFO] mean = 0.404ms +2026-07-29T20:11:16.931 GAJSL [default INFO] stddev = 0.612ms +2026-07-29T20:11:16.931 GAJSL [default INFO] sum = 1.30722e+06ms""" + +BIG_SECONDS = 1307.22 + + +def scan(text): + s = lc.TxApplyScanner() + for line in text.splitlines(): + s.feed(line) + return s + + +# --- the streaming scanner ---------------------------------------------------- + +@pytest.mark.parametrize('block,want', [(MEDIDA_BLOCK, TX_APPLY_SECONDS), + (MEDIDA_BIG, BIG_SECONDS)]) +def test_the_scanner_reads_the_sum_out_of_the_block(block, want): + # Scientific notation was a silent 25% loss -- 91-99% of everything above + # ledger 35M -- because the old regex matched "1.30722" then required "ms" + # and found "e+06ms". The metric block was in the archive the whole time. + assert scan(block).seconds == pytest.approx(want) + + +def test_scanner_resumes_a_block_split_across_a_reconnect(): + # One scanner spans the poller's reconnect loop, so a drop mid-block must + # not lose the header already seen. + head, tail = MEDIDA_BLOCK.splitlines()[:4], MEDIDA_BLOCK.splitlines()[4:] + s = lc.TxApplyScanner() + for line in head: + s.feed(line) + assert s.seconds is None + for line in tail: + s.feed(line) + assert s.seconds == pytest.approx(TX_APPLY_SECONDS) + + +def test_scanner_ignores_sum_from_another_metric(): + s = scan("metric 'ledger.ledger.close':\n sum = 999999.0ms") + assert s.seconds is None + + +def test_scanner_gives_up_past_its_window(): + s = lc.TxApplyScanner() + s.feed("metric 'ledger.transaction.apply':") + for _ in range(20): + s.feed("[default INFO] unrelated chatter") + s.feed(" sum = 12.5555ms") + assert s.seconds is None + + +def test_rate_and_mean_lines_are_not_read_as_sum(): + for line in MEDIDA_BLOCK.splitlines(): + if 'rate =' in line or 'mean =' in line: + assert lc._SUM_RE.search(line) is None + + +def test_sum_stays_inside_the_scan_window(): + lines = MEDIDA_BLOCK.splitlines() + header = next(i for i, l in enumerate(lines) if 'ledger.transaction.apply' in l) + offset = next(i for i, l in enumerate(lines) if lc._SUM_RE.search(l)) - header + assert offset == 10, f"medida layout moved: sum is now {offset} lines below the header" + assert offset <= lc.TxApplyScanner.WINDOW + + +def test_resumed_is_read_from_the_workers_own_line(): + # "RESUME DECLINED" must not count as a resume -- it means the opposite, and + # the colon in RESUME_MARK is what separates the two. + s = lc.TxApplyScanner() + s.feed("RESUME DECLINED: k last close was 'none'; bucket phase incomplete, starting fresh") + assert s.resumed is False, "a declined resume was read as a resume" + s.feed("RESUME: k reached ledger 31005951, replay had started; skipping new-db") + assert s.resumed is True + + +def test_resumed_is_bookkeeping_and_never_becomes_a_measurement(): + # peaks_for_range needs it to tell a resumed tail from a complete pass; the + # profile must not see it as an axis. + assert 'resumed' not in jm.PEAK_FIELDS + + +# --- the monitor's own reader ------------------------------------------------- + +def test_the_monitor_reads_the_same_block_the_collector_scanned(logdir): + # Two independent parsers over one format: they must agree, or a range + # measured live and a range recovered from the archive report differently. + with gzip.open(jm.log_path(4000, 1), 'wt') as fh: + fh.write(MEDIDA_BLOCK) + assert jm._tx_apply_for_attempt(4000, 1) == pytest.approx(scan(MEDIDA_BLOCK).seconds) + + +def test_tx_apply_prefers_durable_sources_over_the_pod_api(logdir, monkeypatch): + # .metrics survives pod reaping and saveSuccessLogs=false; the archive + # survives reaping alone; the pod log is racing Karpenter, so it is a + # fallback and never the plan. Each source carries a different value here + # so the winner is unambiguous. + class FakePodLog: + def read_namespaced_pod_log(self, name, namespace, **_): + return MEDIDA_BLOCK + monkeypatch.setattr(jm, 'core_v1', FakePodLog()) + + with open(jm.metrics_path(4000, 1), 'w') as fh: + json.dump({'txApplySeconds': 99.0}, fh) + with gzip.open(jm.log_path(4000, 1), 'wt') as fh: + fh.write(MEDIDA_BIG) + + assert jm._tx_apply_for_attempt(4000, 1, pod_name='p') == 99.0 + os.remove(jm.metrics_path(4000, 1)) + assert jm._tx_apply_for_attempt(4000, 1, pod_name='p') == pytest.approx(BIG_SECONDS) + os.remove(jm.log_path(4000, 1)) + assert jm._tx_apply_for_attempt(4000, 1, pod_name='p') == pytest.approx(TX_APPLY_SECONDS) + + +def test_tx_apply_survives_a_reaped_pod(logdir): + # The pod is the only source that can vanish, so nothing may depend on it. + with open(jm.metrics_path(4000, 1), 'w') as fh: + json.dump({'txApplySeconds': 12.5}, fh) + assert jm.tx_apply_for_range(4000, 1, pod_name=None) == 12.5 + + +def test_a_range_with_no_measurement_anywhere_reports_nothing(logdir): + assert jm._tx_apply_for_attempt(4000, 1) is None + assert jm.tx_apply_for_range(4000, 1) is None + + +def test_a_corrupt_archive_costs_this_range_its_metric_never_the_pass(logdir): + # EOFError from a truncated gzip member is not an OSError, so it used to + # escape the per-range work and abort the whole reconcile: no recording, no + # reap, no dispatch for any of ~4000 ranges, for as long as the torn bytes + # sat there. + with open(jm.log_path(4000, 1), 'wb') as fh: + fh.write(gzip.compress(MEDIDA_BLOCK.encode())[:40]) + assert jm._tx_apply_for_attempt(4000, 1) is None + + +# --- summing a resumed chain -------------------------------------------------- + +def test_tx_apply_sums_the_whole_resumed_chain(logdir): + # medida's total is per-process, so a pod that resumes at LCL+1 reports only + # the transactions it replayed -- the tail, not the range. + with open(jm.metrics_path(4000, 1), 'w') as fh: + json.dump({'txApplySeconds': 10.0}, fh) + with open(jm.metrics_path(4000, 2), 'w') as fh: + json.dump({'txApplySeconds': 5.0, 'resumed': True}, fh) + assert jm.tx_apply_for_range(4000, 2) == 15.0 + + +def test_a_fresh_start_drops_the_earlier_legs_from_the_total(logdir): + # No RESUME line means new-db ran and this attempt redid the whole range; + # adding the interrupted attempt's figure would double-count the same work. + with open(jm.metrics_path(4000, 1), 'w') as fh: + json.dump({'txApplySeconds': 10.0}, fh) + with open(jm.metrics_path(4000, 2), 'w') as fh: + json.dump({'txApplySeconds': 5.0}, fh) + assert jm.tx_apply_for_range(4000, 2) == 5.0 + + +def test_only_the_last_leg_may_fall_back_to_the_pod(logdir, monkeypatch): + # pod_name names the winning attempt's pod; handing it to an earlier leg + # would read the wrong pod's log and attribute it to the wrong attempt. + class FakePodLog: + def read_namespaced_pod_log(self, name, namespace, **_): + return MEDIDA_BIG + monkeypatch.setattr(jm, 'core_v1', FakePodLog()) + # a1 has no durable record at all; a2 resumed from it and has none either. + with open(jm.metrics_path(4000, 2), 'w') as fh: + json.dump({'resumed': True}, fh) + assert jm.tx_apply_for_range(4000, 2, pod_name='p') == pytest.approx(BIG_SECONDS), \ + "the total must be a2's pod alone, not that pod counted for both legs" From 15ca96830607674dac33a19077f70b7dd8b28e2e Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 14:07:49 -0400 Subject: [PATCH 034/117] Add longest-first dispatch, ordered by the profile rather than by position Makespan is bounded below by the single longest job, so every range dispatched after it is free and every hour it starts late lands on the end of the run. tip-first only approximates that ordering: position predicts cost on average and badly in the tail. Measured 2026-07-30, ranges at 41-45M ran as long as the tip (3.1h) on a third of the memory, and the 50-60M band is cheaper than 40-50M -- the cost curve is not monotonic in ledger position. A range the profile has never seen sorts FIRST. profile_for returns the nearest measured end above the target and None past its ceiling, so an unprofiled range is by construction newer than anything ever measured, which makes it the most expensive kind. Unknown means assume worst, not assume average -- and it puts those ranges early, under the most generous sizing, instead of leaving them to a run that may die before reaching them. Off by default. The gain is entirely a function of how many workers are running, simulated on this run's own profile: workers tip-first longest-first gain 400 8.65h 8.58h 0.8% 800 5.00h 4.32h 13.6% 1092 4.39h 3.15h 28.3% 2096 3.13h 3.13h 0.0% At 2096 the floor is already reached and this buys nothing. Its value is that it lets 1092 workers finish in the time 2096 take -- half the fleet for the same makespan, which is the direction the next run is going. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 35 +++++++++++- .../tests/unit/test_dispatch_order.py | 55 +++++++++++++++++++ 2 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 src/MissionParallelCatchup/tests/unit/test_dispatch_order.py diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 52324d05..04063188 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -57,7 +57,7 @@ # the bucket set only grows with ledger position. 'oldest-first' reverses that, # so a profiling run measures the cheap early ranges before it can be # interrupted, and the expensive tip ranges last. -RANGE_ORDER = os.getenv('RANGE_ORDER', 'tip-first') # tip-first | oldest-first +RANGE_ORDER = os.getenv('RANGE_ORDER', 'tip-first') # tip-first | oldest-first | longest-first STARTING_LEDGER = int(os.getenv('STARTING_LEDGER', 0)) LATEST_LEDGER_NUM = int(os.getenv('LATEST_LEDGER_NUM', 0)) LEDGERS_PER_JOB = int(os.getenv('LEDGERS_PER_JOB', 16000)) @@ -434,8 +434,37 @@ def _uniform_segment(start_ledger, end_ledger, seg_size): def _ordered(ranges): - """Dispatch order. Generators emit tip-first; reverse for oldest-first.""" - return list(reversed(ranges)) if RANGE_ORDER == 'oldest-first' else ranges + """Dispatch order. Generators emit tip-first; reverse for oldest-first. + + 'longest-first' is the one that shortens the run. Makespan is bounded below + by the single longest job, so every range that starts after it is free and + every hour it starts late is an hour on the end. That is classic + longest-processing-time scheduling. + + tip-first only approximates it. Position predicts cost on average and badly + in the tail: measured 2026-07-30, ranges at 41-45M ran as long as the tip + (3.1h) on a third of the memory, and the 50-60M band is CHEAPER than 40-50M. + Sorting on the profile's own measured seconds uses the real number instead + of a proxy for it. + + A range the profile has never seen sorts FIRST. profile_for returns the + nearest measured end ABOVE the target, so an unprofiled range is by + construction newer than anything ever measured -- the newest ranges are the + most expensive, so "unknown" means "assume worst", not "assume average". + That also makes the next profile better: those ranges run early, under the + most generous sizing, instead of being the ones a run dies before reaching. + """ + if RANGE_ORDER == 'oldest-first': + return list(reversed(ranges)) + if RANGE_ORDER != 'longest-first': + return ranges + def cost(item): + prof = profile_for(item[0]) + secs = (prof or {}).get('seconds') + # None sorts first; ties keep tip-first order, which is the better guess + # among ranges the profile cannot separate. + return (0 if secs is None else 1, -(secs or 0)) + return sorted(ranges, key=cost) def generate_ranges(): diff --git a/src/MissionParallelCatchup/tests/unit/test_dispatch_order.py b/src/MissionParallelCatchup/tests/unit/test_dispatch_order.py new file mode 100644 index 00000000..1dd1f72c --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_dispatch_order.py @@ -0,0 +1,55 @@ +"""Dispatch order, and why longest-first is the one that shortens a run. + +Makespan is bounded below by the single longest job: every range dispatched +after it is free, and every hour it starts late lands on the end of the run. +""" + +import pytest + +import job_monitor as jm + +RANGES = [(600, 420), (500, 420), (400, 420), (300, 420)] # generators emit tip-first + + +def _order(monkeypatch, mode, profile=None): + monkeypatch.setattr(jm, 'RANGE_ORDER', mode) + monkeypatch.setattr(jm, 'PROFILE', profile) + return [e for e, _ in jm._ordered(list(RANGES))] + + +def test_tip_first_is_unchanged(monkeypatch): + assert _order(monkeypatch, 'tip-first') == [600, 500, 400, 300] + + +def test_oldest_first_reverses(monkeypatch): + assert _order(monkeypatch, 'oldest-first') == [300, 400, 500, 600] + + +def test_longest_first_sorts_by_measured_seconds_not_position(monkeypatch): + # The whole point: 400 is the expensive one even though 600 is nearer the + # tip. Measured 2026-07-30, ranges at 41-45M ran as long as the tip on a + # third of the memory, so position is a proxy that fails in the tail. + prof = [(300, {'seconds': 10}), (400, {'seconds': 9000}), + (500, {'seconds': 20}), (600, {'seconds': 100})] + assert _order(monkeypatch, 'longest-first', prof) == [400, 600, 500, 300] + + +def test_an_unprofiled_range_sorts_first(monkeypatch): + # profile_for returns the nearest measured end ABOVE the target and None + # past its ceiling, so an unprofiled range is newer than anything ever + # measured -- the most expensive kind. Unknown means assume worst. + prof = [(300, {'seconds': 10}), (400, {'seconds': 9000})] + assert _order(monkeypatch, 'longest-first', prof)[:2] == [600, 500] + + +def test_ties_keep_tip_first_order(monkeypatch): + # Among ranges the profile cannot separate, position is still the better + # guess, so a tie must not scramble them. + prof = [(e, {'seconds': 50}) for e in (300, 400, 500, 600)] + assert _order(monkeypatch, 'longest-first', prof) == [600, 500, 400, 300] + + +def test_no_profile_at_all_falls_back_to_tip_first(monkeypatch): + # A run with no profile has nothing to sort on; every range is "unknown", + # so the tie rule must leave the generator's order intact. + assert _order(monkeypatch, 'longest-first', None) == [600, 500, 400, 300] From b8405449fff7232ae6f163ef3925ab35255c3db9 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 14:31:34 -0400 Subject: [PATCH 035/117] Size the cpu request from a range's slack, not from its demand The request is not a demand estimate. Measured unthrottled on ssc-test 2026-07-30, REPLAY wants ~1.0 cores at every ledger position -- 1.04 at 63.7M, 0.96 at 43.2M -- and replay is 80-95% of a job, so demand barely varies with position at all. What varies enormously is how much throttling a range can absorb: makespan is the single longest job, so any range that still finishes inside that shadow at a lower request is giving away packing density for nothing. 3267 of 3859 profiled ranges finish inside the floor at 0.5 cores. Only 111 need 1.0 or more. Bin-packed over the whole profile: flat 1.0 491 nodes 7.9 pods/node 3928 vCPU 3.32h $573 flat 1.25 646 nodes 6.0 pods/node 5168 vCPU 3.13h $712 slack-tiered 291 nodes 13.3 pods/node 2328 vCPU 3.13h $321 Both flat options exceed the 2304 spot quota; the tiered one lands inside it and is faster than flat 1.0 as well. The floor comes from the profile's own longest range rather than a constant, so it tracks the chain as it grows. A range with no measured runtime gets the TOP tier, matching the dispatch order added in 15ca968: unprofiled means newer than anything ever measured, so assume worst. cpu is set as a request only -- never a limit. The bucket phase measured up to 2.53 cores at the tip, and it is the one part of a job that slicing cannot remove, so capping it throttles precisely the fixed cost. Off by default: PROFILE_CPU_TIERS empty keeps every range on the configured request, exactly as before. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 64 ++++++++++++++++- src/MissionParallelCatchup/log_collector.py | 38 +++++++--- .../templates/job_monitor.yaml | 4 ++ .../parallel_catchup_helm/values.yaml | 9 +++ .../tests/unit/test_cpu_tiers.py | 70 +++++++++++++++++++ 5 files changed, 173 insertions(+), 12 deletions(-) create mode 100644 src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 04063188..9bb0b4dc 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1519,6 +1519,63 @@ def _sized(value, margin, cap): return _bytes_to_quantity(min(want, _quantity_bytes(cap))) +# CPU tiers, as request only. The request is not a demand estimate -- measured +# unthrottled, REPLAY wants ~1.0 cores at every ledger position (1.04 at 63.7M, +# 0.96 at 43.2M) and replay is 80-95% of a job, so demand barely varies. What +# varies enormously is how much throttling a range can ABSORB: makespan is the +# single longest job, so any range that still lands inside that shadow at a +# lower request is giving away free packing density. +# +# Measured 2026-07-30 over 3859 profiled ranges: 3267 of them finish inside the +# floor at 0.5 cores. Flat 1.0 needs 491 nodes and 3928 vCPU -- over the 2304 +# quota; tiered needs 291 nodes and 2328 vCPU at the same 3.13h makespan. +# +# Slowdown per tier is the measured cpu curve relative to saturation. Empty +# PROFILE_CPU_TIERS disables the whole thing and every range keeps REQ_CPU. +PROFILE_CPU_TIERS = os.getenv('PROFILE_CPU_TIERS', '') # e.g. "0.5,0.75,1.0,1.25" +PROFILE_CPU_SLOWDOWN = os.getenv('PROFILE_CPU_SLOWDOWN', '1.53,1.17,1.06,1.0') +# Ranges whose own runtime sets the floor. Taken from the profile rather than +# configured: it moves on its own as the chain grows. +_PROFILE_FLOOR = None + + +def _cpu_tiers(): + if not PROFILE_CPU_TIERS.strip(): + return [] + tiers = [float(x) for x in PROFILE_CPU_TIERS.split(',') if x.strip()] + slow = [float(x) for x in PROFILE_CPU_SLOWDOWN.split(',') if x.strip()] + return list(zip(tiers, slow)) if len(slow) == len(tiers) else [] + + +def profile_floor(): + """Longest measured range: the makespan every other range must fit inside.""" + global _PROFILE_FLOOR + if _PROFILE_FLOOR is None: + secs = [(r.get('seconds') or 0) for _, r in (PROFILE or [])] + _PROFILE_FLOOR = max(secs) if secs else 0 + return _PROFILE_FLOOR + + +def _slack_cpu(seconds): + """Cheapest tier this range can run at and still finish inside the floor. + + A range with no measured runtime gets the top tier, matching the dispatch + order: unprofiled means newer than anything measured, so assume worst. + """ + tiers = _cpu_tiers() + if not tiers: + return None + if not seconds: + return str(tiers[-1][0]) + floor = profile_floor() + if not floor: + return None + for cores, slowdown in tiers: + if seconds * slowdown <= floor: + return str(cores) + return str(tiers[-1][0]) + + def _profile_overrides(end, escalated): """Request overrides for this range from the profile, or {} for none. @@ -1542,6 +1599,9 @@ def _profile_overrides(end, escalated): disk = prof.get('peakEphemeralBytes') if disk and LIM_EPHEMERAL: out['ephemeral-storage'] = _sized(disk, PROFILE_MARGIN, LIM_EPHEMERAL) + cpu = _slack_cpu(prof.get('seconds')) + if cpu: + out['cpu'] = cpu return out @@ -1596,7 +1656,9 @@ def _resources(mem=None, eph=None, end=None): # leaves the pod Burstable rather than Guaranteed -- Kubernetes needs # all three to match -- which is the intended trade. for key, value in overrides.items(): - req[key] = lim[key] = value + req[key] = value + if key != 'cpu': + lim[key] = value # Unmeasured range: the configured defaults, requests below limits, exactly # as before -- a range with no profile entry must behave as if there were no # profile at all. diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index a1d321b8..496b9b6a 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -380,6 +380,30 @@ def record_outcome(pod, end, attempt): +def _flush_peak(name, axis, field, value): + """Persist a high-water so a sidecar restart cannot lose it. + + Every key in PEAK_KEYS needs this, not just the ones we remembered. The + peaks live in module dicts, so a restarted collector starts from zero and + re-accumulates only from whatever the pod is using at that moment. anon had + it, ephemeral got it when a restart was shown to lose the high-water, and + peakWorkingSetBytes was missed -- which is how a completed range came back + with a working set BELOW its own anon, which cannot happen in one sample and + is trivial across a restart. Measured on the 2026-07-30 run: 136 of 3095 + ranges, 55 of them single-attempt so no retry chain could explain them. + + write_metrics max-merges on PEAK_KEYS, so re-flushing a lower value later is + harmless; the ratio only keeps this to a handful of writes per pod. + """ + key = name + '/' + axis + if value < _peak_flushed.get(key, 0) * PEAK_FLUSH_RATIO: + return + _peak_flushed[key] = value + ref = _streaming.get(name) + if ref: + write_metrics(ref[0], ref[1], {field: value}) + + async def sample_kubelet(session, nodes): """Update each pod's peak ephemeral use and peak anon from one snapshot. @@ -427,12 +451,7 @@ async def sample_kubelet(session, nodes): # high-water. This figure sizes the next run's # ephemeral-storage request, and one that comes back too # small is an eviction. - if int(used) >= _peak_flushed.get(name + '/eph', 0) * PEAK_FLUSH_RATIO: - _peak_flushed[name + '/eph'] = int(used) - ref = _streaming.get(name) - if ref: - write_metrics(ref[0], ref[1], - {'peakEphemeralBytes': int(used)}) + _flush_peak(name, 'eph', 'peakEphemeralBytes', int(used)) for c in entry.get('containers', []): # The worker container only. Sidecars share the pod, so summing # across containers -- or letting the last one win -- would size @@ -446,6 +465,7 @@ async def sample_kubelet(session, nodes): ws = mem.get('workingSetBytes') if ws is not None and int(ws) > _ws_peak.get(name, 0): _ws_peak[name] = int(ws) + _flush_peak(name, 'ws', 'peakWorkingSetBytes', int(ws)) rss = mem.get('rssBytes') if rss is None: continue @@ -460,11 +480,7 @@ async def sample_kubelet(session, nodes): # run too small. Flushing only on PEAK_FLUSH_RATIO growth keeps # this to a handful of writes over a pod's life instead of one # per sample per pod. - if int(rss) >= _peak_flushed.get(name, 0) * PEAK_FLUSH_RATIO: - _peak_flushed[name] = int(rss) - ref = _streaming.get(name) - if ref: - write_metrics(ref[0], ref[1], {'peakAnonBytes': int(rss)}) + _flush_peak(name, 'anon', 'peakAnonBytes', int(rss)) def _mark_done(end, attempt): diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index bdaa0112..edfa5edf 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -228,6 +228,10 @@ spec: {{- if .Values.monitor.profileConfigMap }} - name: PROFILE_PATH value: /profile/profile.json + - name: PROFILE_CPU_TIERS + value: {{ .Values.monitor.profileCpuTiers | quote }} + - name: PROFILE_CPU_SLOWDOWN + value: {{ .Values.monitor.profileCpuSlowdown | quote }} - name: PROFILE_MARGIN value: {{ .Values.monitor.profileMargin | quote }} - name: PROFILE_CPU_LIMIT diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 68d40ee0..33e2d10a 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -70,6 +70,15 @@ monitor: # path or an https URL) into one. Empty = size from the configured # requests below. Only tightens requests; limits are untouched. profileConfigMap: "" + # CPU request tiers, as a slack budget rather than a demand estimate. Measured + # unthrottled, replay wants ~1.0 cores at every ledger position and is 80-95% + # of a job, so demand barely varies -- what varies is how much throttling a + # range can absorb before it stops fitting inside the longest job's shadow. + # 3267 of 3859 profiled ranges still fit at 0.5 cores. Empty disables tiering + # and every range keeps the configured cpu request. + # Paired with profileCpuSlowdown: the measured runtime multiplier per tier. + profileCpuTiers: "" + profileCpuSlowdown: "1.53,1.17,1.06,1.0" profileMargin: 1.15 # CPU limit for ranges the profile has measured. Above the configured # worker limit on purpose: at a 2-core limit every range pegs 2.0, so the diff --git a/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py b/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py new file mode 100644 index 00000000..1a99bc25 --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py @@ -0,0 +1,70 @@ +"""CPU request as a slack budget, not a demand estimate. + +Measured unthrottled on ssc-test 2026-07-30: replay wants ~1.0 cores at every +ledger position (1.04 at 63.7M, 0.96 at 43.2M) and replay is 80-95% of a job, +so demand barely varies. What varies is how much throttling a range can absorb +before it stops finishing inside the longest job's shadow -- and that slack is +free packing density. 3267 of 3859 profiled ranges still fit at 0.5 cores; +tiering took the fleet from 491 nodes / 3928 vCPU (over the 2304 quota) to 291 +nodes / 2328 vCPU at the same 3.13h makespan. +""" + +import pytest + +import job_monitor as jm + +TIERS = '0.5,0.75,1.0,1.25' +SLOW = '1.53,1.17,1.06,1.0' +FLOOR = 10000.0 # the longest range in the pretend profile + + +@pytest.fixture +def tiered(monkeypatch): + monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', TIERS) + monkeypatch.setattr(jm, 'PROFILE_CPU_SLOWDOWN', SLOW) + monkeypatch.setattr(jm, '_PROFILE_FLOOR', FLOOR) + + +def test_tiering_is_off_unless_configured(monkeypatch): + monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', '') + assert jm._slack_cpu(500) is None + + +def test_a_short_range_gets_the_cheapest_tier(tiered): + # 500s even at 1.53x slowdown is 765s, far inside a 10000s floor. + assert jm._slack_cpu(500) == '0.5' + + +def test_the_floor_setting_range_gets_the_top_tier(tiered): + # Nothing slower than saturation fits, so it must not be throttled at all. + assert jm._slack_cpu(FLOOR) == '1.25' + + +def test_each_tier_is_the_cheapest_that_still_fits(tiered): + # 7000 * 1.53 = 10710 > floor, but 7000 * 1.17 = 8190 fits -> 0.75. + assert jm._slack_cpu(7000) == '0.75' + # 9000 * 1.17 = 10530 > floor, 9000 * 1.06 = 9540 fits -> 1.0. + assert jm._slack_cpu(9000) == '1.0' + + +def test_an_unmeasured_range_gets_the_top_tier(tiered): + # Matches the dispatch order: unprofiled means newer than anything measured, + # so assume worst rather than assume average. + assert jm._slack_cpu(None) == '1.25' + + +def test_the_floor_comes_from_the_profile_not_a_constant(monkeypatch): + # It has to move on its own as the chain grows. + monkeypatch.setattr(jm, 'PROFILE', [(100, {'seconds': 42}), (200, {'seconds': 900})]) + monkeypatch.setattr(jm, '_PROFILE_FLOOR', None) + assert jm.profile_floor() == 900 + + +def test_cpu_is_requested_but_never_limited(tiered, monkeypatch, cluster): + # A limit would cap the bucket phase, which measured up to 2.53 cores and is + # the one part of a job that slicing cannot remove. + monkeypatch.setattr(jm, 'PROFILE', [(300, {'seconds': 500, 'peakAnonBytes': 1 << 30})]) + monkeypatch.setattr(jm, '_PROFILE_FLOOR', FLOOR) + r = jm._resources(end=300) + assert r.requests['cpu'] == '0.5' + assert 'cpu' not in r.limits From e17cfc6496ffb0238ded36c87fc82bf2ecc4bf19 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 14:34:37 -0400 Subject: [PATCH 036/117] Rename profile_floor to longest_range_seconds It is a time budget in seconds -- the longest measured range, the deadline every other range is sized against. Sitting next to a cpu ladder that has its own floor and ceiling (0.5 and 1.25 cores), calling it a floor read as a cpu value. It confused a reader immediately. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 25 +++++++++++-------- .../tests/unit/test_cpu_tiers.py | 12 ++++----- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 9bb0b4dc..90f307a5 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1536,7 +1536,7 @@ def _sized(value, margin, cap): PROFILE_CPU_SLOWDOWN = os.getenv('PROFILE_CPU_SLOWDOWN', '1.53,1.17,1.06,1.0') # Ranges whose own runtime sets the floor. Taken from the profile rather than # configured: it moves on its own as the chain grows. -_PROFILE_FLOOR = None +_LONGEST_RANGE_SECONDS = None def _cpu_tiers(): @@ -1547,13 +1547,18 @@ def _cpu_tiers(): return list(zip(tiers, slow)) if len(slow) == len(tiers) else [] -def profile_floor(): - """Longest measured range: the makespan every other range must fit inside.""" - global _PROFILE_FLOOR - if _PROFILE_FLOOR is None: +def longest_range_seconds(): + """Runtime of the longest measured range, in SECONDS. + + The deadline every other range is sized against -- not to be confused with + the cpu ladder's own floor and ceiling (0.5 and 1.25 cores by default). + A range earns a cheaper tier by still finishing inside this. + """ + global _LONGEST_RANGE_SECONDS + if _LONGEST_RANGE_SECONDS is None: secs = [(r.get('seconds') or 0) for _, r in (PROFILE or [])] - _PROFILE_FLOOR = max(secs) if secs else 0 - return _PROFILE_FLOOR + _LONGEST_RANGE_SECONDS = max(secs) if secs else 0 + return _LONGEST_RANGE_SECONDS def _slack_cpu(seconds): @@ -1567,11 +1572,11 @@ def _slack_cpu(seconds): return None if not seconds: return str(tiers[-1][0]) - floor = profile_floor() - if not floor: + budget = longest_range_seconds() + if not budget: return None for cores, slowdown in tiers: - if seconds * slowdown <= floor: + if seconds * slowdown <= budget: return str(cores) return str(tiers[-1][0]) diff --git a/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py b/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py index 1a99bc25..31742e65 100644 --- a/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py +++ b/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py @@ -15,14 +15,14 @@ TIERS = '0.5,0.75,1.0,1.25' SLOW = '1.53,1.17,1.06,1.0' -FLOOR = 10000.0 # the longest range in the pretend profile +BUDGET = 10000.0 # longest range in the pretend profile, in SECONDS @pytest.fixture def tiered(monkeypatch): monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', TIERS) monkeypatch.setattr(jm, 'PROFILE_CPU_SLOWDOWN', SLOW) - monkeypatch.setattr(jm, '_PROFILE_FLOOR', FLOOR) + monkeypatch.setattr(jm, '_LONGEST_RANGE_SECONDS', BUDGET) def test_tiering_is_off_unless_configured(monkeypatch): @@ -37,7 +37,7 @@ def test_a_short_range_gets_the_cheapest_tier(tiered): def test_the_floor_setting_range_gets_the_top_tier(tiered): # Nothing slower than saturation fits, so it must not be throttled at all. - assert jm._slack_cpu(FLOOR) == '1.25' + assert jm._slack_cpu(BUDGET) == '1.25' def test_each_tier_is_the_cheapest_that_still_fits(tiered): @@ -56,15 +56,15 @@ def test_an_unmeasured_range_gets_the_top_tier(tiered): def test_the_floor_comes_from_the_profile_not_a_constant(monkeypatch): # It has to move on its own as the chain grows. monkeypatch.setattr(jm, 'PROFILE', [(100, {'seconds': 42}), (200, {'seconds': 900})]) - monkeypatch.setattr(jm, '_PROFILE_FLOOR', None) - assert jm.profile_floor() == 900 + monkeypatch.setattr(jm, '_LONGEST_RANGE_SECONDS', None) + assert jm.longest_range_seconds() == 900 def test_cpu_is_requested_but_never_limited(tiered, monkeypatch, cluster): # A limit would cap the bucket phase, which measured up to 2.53 cores and is # the one part of a job that slicing cannot remove. monkeypatch.setattr(jm, 'PROFILE', [(300, {'seconds': 500, 'peakAnonBytes': 1 << 30})]) - monkeypatch.setattr(jm, '_PROFILE_FLOOR', FLOOR) + monkeypatch.setattr(jm, '_LONGEST_RANGE_SECONDS', BUDGET) r = jm._resources(end=300) assert r.requests['cpu'] == '0.5' assert 'cpu' not in r.limits From d72e2190d4d0e6053f718219b3f54f94b6f4405f Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 14:40:54 -0400 Subject: [PATCH 037/117] Key the cpu tier on rank, not on absolute seconds The absolute rule compared a range against the longest measured range, which is set by the single worst-throttled job. Between two real runs that budget moved 1.79x while the median range moved 1.55x, so everything looked like it had more slack and the top two tiers went from 127 ranges to 65 -- cheaper tiers, slower run, longer worst job, bigger budget. Wrong direction, and self-reinforcing. Shares pin the fleet shape instead: the cheapest 85% take the first tier whatever the run's absolute clock was. Per-range assignment agrees ~85% either way, and that ceiling is run-to-run noise in the measurement itself (Spearman 0.87 between two runs), not something the keying can improve. What rank keying fixes is the aggregate, which is what has to fit a fixed 2304 vCPU quota. Also simpler, and drops the least trustworthy input: no slowdown curve. Those constants came from a sweep whose absolute level is confounded -- it applied a cpu limit where the comparison run did not -- so the ladder no longer depends on them at all. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 78 ++++++------- .../templates/job_monitor.yaml | 4 +- .../parallel_catchup_helm/values.yaml | 6 +- .../tests/unit/test_cpu_tiers.py | 105 +++++++++++------- 4 files changed, 110 insertions(+), 83 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 90f307a5..5fa3f710 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1522,63 +1522,59 @@ def _sized(value, margin, cap): # CPU tiers, as request only. The request is not a demand estimate -- measured # unthrottled, REPLAY wants ~1.0 cores at every ledger position (1.04 at 63.7M, # 0.96 at 43.2M) and replay is 80-95% of a job, so demand barely varies. What -# varies enormously is how much throttling a range can ABSORB: makespan is the -# single longest job, so any range that still lands inside that shadow at a -# lower request is giving away free packing density. +# varies is how much throttling a range can ABSORB before it stops finishing +# inside the longest job's shadow, and that slack is free packing density. +# Measured 2026-07-30: bin-packed, flat 1.0 needs 491 nodes and 3928 vCPU -- +# over the 2304 quota -- where tiering needs 291 nodes and 2328 vCPU at the +# same makespan. # -# Measured 2026-07-30 over 3859 profiled ranges: 3267 of them finish inside the -# floor at 0.5 cores. Flat 1.0 needs 491 nodes and 3928 vCPU -- over the 2304 -# quota; tiered needs 291 nodes and 2328 vCPU at the same 3.13h makespan. +# Keyed on where a range falls in the profile's own runtime distribution, not +# on its absolute seconds. Absolute keying compares against the longest range, +# which is set by the single worst-throttled job: between two real runs that +# budget moved 1.79x while the median range moved 1.55x, and the top two tiers +# went from 127 ranges to 65. Shares pin the fleet size instead, which is what +# has to fit a fixed vCPU quota. Per-range assignment agrees ~85% either way -- +# that ceiling is run-to-run noise in the measurement (Spearman 0.87), not +# something the keying can fix. # -# Slowdown per tier is the measured cpu curve relative to saturation. Empty -# PROFILE_CPU_TIERS disables the whole thing and every range keeps REQ_CPU. -PROFILE_CPU_TIERS = os.getenv('PROFILE_CPU_TIERS', '') # e.g. "0.5,0.75,1.0,1.25" -PROFILE_CPU_SLOWDOWN = os.getenv('PROFILE_CPU_SLOWDOWN', '1.53,1.17,1.06,1.0') -# Ranges whose own runtime sets the floor. Taken from the profile rather than -# configured: it moves on its own as the chain grows. -_LONGEST_RANGE_SECONDS = None +# Shares are cumulative and cheapest-first: 0.85 means the cheapest 85% of +# ranges take the first tier. Empty PROFILE_CPU_TIERS disables tiering. +PROFILE_CPU_TIERS = os.getenv('PROFILE_CPU_TIERS', '') # "0.5,0.75,1.0,1.25" +PROFILE_CPU_SHARES = os.getenv('PROFILE_CPU_SHARES', '0.85,0.98,0.995,1.0') +_SORTED_SECONDS = None -def _cpu_tiers(): - if not PROFILE_CPU_TIERS.strip(): - return [] - tiers = [float(x) for x in PROFILE_CPU_TIERS.split(',') if x.strip()] - slow = [float(x) for x in PROFILE_CPU_SLOWDOWN.split(',') if x.strip()] - return list(zip(tiers, slow)) if len(slow) == len(tiers) else [] - - -def longest_range_seconds(): - """Runtime of the longest measured range, in SECONDS. - - The deadline every other range is sized against -- not to be confused with - the cpu ladder's own floor and ceiling (0.5 and 1.25 cores by default). - A range earns a cheaper tier by still finishing inside this. - """ - global _LONGEST_RANGE_SECONDS - if _LONGEST_RANGE_SECONDS is None: - secs = [(r.get('seconds') or 0) for _, r in (PROFILE or [])] - _LONGEST_RANGE_SECONDS = max(secs) if secs else 0 - return _LONGEST_RANGE_SECONDS +def _profile_seconds(): + """Every measured runtime in the profile, sorted, for percentile lookup.""" + global _SORTED_SECONDS + if _SORTED_SECONDS is None: + _SORTED_SECONDS = sorted(r['seconds'] for _, r in (PROFILE or []) + if r.get('seconds')) + return _SORTED_SECONDS def _slack_cpu(seconds): - """Cheapest tier this range can run at and still finish inside the floor. + """Tier for a range, by its rank among all profiled runtimes. A range with no measured runtime gets the top tier, matching the dispatch order: unprofiled means newer than anything measured, so assume worst. """ - tiers = _cpu_tiers() + tiers = [float(x) for x in PROFILE_CPU_TIERS.split(',') if x.strip()] if not tiers: return None + shares = [float(x) for x in PROFILE_CPU_SHARES.split(',') if x.strip()] + if len(shares) != len(tiers): + return None if not seconds: - return str(tiers[-1][0]) - budget = longest_range_seconds() - if not budget: + return str(tiers[-1]) + everything = _profile_seconds() + if not everything: return None - for cores, slowdown in tiers: - if seconds * slowdown <= budget: + pct = bisect.bisect_right(everything, seconds) / len(everything) + for cores, upto in zip(tiers, shares): + if pct <= upto: return str(cores) - return str(tiers[-1][0]) + return str(tiers[-1]) def _profile_overrides(end, escalated): diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index edfa5edf..181fbcae 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -230,8 +230,8 @@ spec: value: /profile/profile.json - name: PROFILE_CPU_TIERS value: {{ .Values.monitor.profileCpuTiers | quote }} - - name: PROFILE_CPU_SLOWDOWN - value: {{ .Values.monitor.profileCpuSlowdown | quote }} + - name: PROFILE_CPU_SHARES + value: {{ .Values.monitor.profileCpuShares | quote }} - name: PROFILE_MARGIN value: {{ .Values.monitor.profileMargin | quote }} - name: PROFILE_CPU_LIMIT diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 33e2d10a..b965f006 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -78,7 +78,11 @@ monitor: # and every range keeps the configured cpu request. # Paired with profileCpuSlowdown: the measured runtime multiplier per tier. profileCpuTiers: "" - profileCpuSlowdown: "1.53,1.17,1.06,1.0" + # Cumulative, cheapest-first: 0.85 means the cheapest 85% of ranges take + # the first tier. Pins the fleet shape run to run, which an absolute + # seconds threshold does not -- that budget is set by the single + # worst-throttled job and moved 1.79x between two real runs. + profileCpuShares: "0.85,0.98,0.995,1.0" profileMargin: 1.15 # CPU limit for ranges the profile has measured. Above the configured # worker limit on purpose: at a 2-core limit every range pegs 2.0, so the diff --git a/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py b/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py index 31742e65..11da04f1 100644 --- a/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py +++ b/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py @@ -1,12 +1,14 @@ -"""CPU request as a slack budget, not a demand estimate. - -Measured unthrottled on ssc-test 2026-07-30: replay wants ~1.0 cores at every -ledger position (1.04 at 63.7M, 0.96 at 43.2M) and replay is 80-95% of a job, -so demand barely varies. What varies is how much throttling a range can absorb -before it stops finishing inside the longest job's shadow -- and that slack is -free packing density. 3267 of 3859 profiled ranges still fit at 0.5 cores; -tiering took the fleet from 491 nodes / 3928 vCPU (over the 2304 quota) to 291 -nodes / 2328 vCPU at the same 3.13h makespan. +"""CPU request as a slack budget, keyed on rank rather than absolute seconds. + +The request is not a demand estimate. Measured unthrottled 2026-07-30, replay +wants ~1.0 cores at every ledger position (1.04 at 63.7M, 0.96 at 43.2M) and is +80-95% of a job, so demand barely varies. What varies is how much throttling a +range can absorb, and that slack is free packing density: bin-packed, flat 1.0 +needs 491 nodes / 3928 vCPU -- over the 2304 quota -- against 291 / 2328 tiered. + +Rank rather than seconds because the absolute budget is set by the single +worst-throttled job: between two real runs it moved 1.79x while the median +range moved 1.55x, swinging the top two tiers from 127 ranges to 65. """ import pytest @@ -14,15 +16,17 @@ import job_monitor as jm TIERS = '0.5,0.75,1.0,1.25' -SLOW = '1.53,1.17,1.06,1.0' -BUDGET = 10000.0 # longest range in the pretend profile, in SECONDS +SHARES = '0.85,0.98,0.995,1.0' +# 1000 ranges, 1s..1000s, so a range's value IS its percentile x 1000. +PROFILE = [(i, {'seconds': float(i)}) for i in range(1, 1001)] @pytest.fixture def tiered(monkeypatch): monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', TIERS) - monkeypatch.setattr(jm, 'PROFILE_CPU_SLOWDOWN', SLOW) - monkeypatch.setattr(jm, '_LONGEST_RANGE_SECONDS', BUDGET) + monkeypatch.setattr(jm, 'PROFILE_CPU_SHARES', SHARES) + monkeypatch.setattr(jm, 'PROFILE', PROFILE) + monkeypatch.setattr(jm, '_SORTED_SECONDS', None) def test_tiering_is_off_unless_configured(monkeypatch): @@ -30,41 +34,64 @@ def test_tiering_is_off_unless_configured(monkeypatch): assert jm._slack_cpu(500) is None -def test_a_short_range_gets_the_cheapest_tier(tiered): - # 500s even at 1.53x slowdown is 765s, far inside a 10000s floor. - assert jm._slack_cpu(500) == '0.5' - +def test_the_cheap_bulk_gets_the_cheapest_tier(tiered): + assert jm._slack_cpu(1) == '0.5' + assert jm._slack_cpu(850) == '0.5' # exactly the 85% cut -def test_the_floor_setting_range_gets_the_top_tier(tiered): - # Nothing slower than saturation fits, so it must not be throttled at all. - assert jm._slack_cpu(BUDGET) == '1.25' - -def test_each_tier_is_the_cheapest_that_still_fits(tiered): - # 7000 * 1.53 = 10710 > floor, but 7000 * 1.17 = 8190 fits -> 0.75. - assert jm._slack_cpu(7000) == '0.75' - # 9000 * 1.17 = 10530 > floor, 9000 * 1.06 = 9540 fits -> 1.0. - assert jm._slack_cpu(9000) == '1.0' +def test_each_band_maps_to_its_tier(tiered): + assert jm._slack_cpu(851) == '0.75' # just past 85% + assert jm._slack_cpu(980) == '0.75' + assert jm._slack_cpu(981) == '1.0' + assert jm._slack_cpu(995) == '1.0' + assert jm._slack_cpu(1000) == '1.25' # the longest range def test_an_unmeasured_range_gets_the_top_tier(tiered): - # Matches the dispatch order: unprofiled means newer than anything measured, - # so assume worst rather than assume average. + # Matches dispatch order: unprofiled is newer than anything measured. assert jm._slack_cpu(None) == '1.25' -def test_the_floor_comes_from_the_profile_not_a_constant(monkeypatch): - # It has to move on its own as the chain grows. - monkeypatch.setattr(jm, 'PROFILE', [(100, {'seconds': 42}), (200, {'seconds': 900})]) - monkeypatch.setattr(jm, '_LONGEST_RANGE_SECONDS', None) - assert jm.longest_range_seconds() == 900 - +def test_a_uniformly_slower_run_assigns_the_same_tiers(monkeypatch): + """The property absolute-seconds keying does not have. -def test_cpu_is_requested_but_never_limited(tiered, monkeypatch, cluster): - # A limit would cap the bucket phase, which measured up to 2.53 cores and is - # the one part of a job that slicing cannot remove. - monkeypatch.setattr(jm, 'PROFILE', [(300, {'seconds': 500, 'peakAnonBytes': 1 << 30})]) - monkeypatch.setattr(jm, '_LONGEST_RANGE_SECONDS', BUDGET) + Every range 3x slower must not shuffle anything: rank is unchanged, so the + fleet needs the same shape. Under a `seconds <= longest` rule this is only + true if the slowdown is perfectly uniform, which measurement shows it is not. + """ + monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', TIERS) + monkeypatch.setattr(jm, 'PROFILE_CPU_SHARES', SHARES) + monkeypatch.setattr(jm, 'PROFILE', [(i, {'seconds': i * 3.0}) for i in range(1, 1001)]) + monkeypatch.setattr(jm, '_SORTED_SECONDS', None) + assert jm._slack_cpu(850 * 3) == '0.5' + assert jm._slack_cpu(981 * 3) == '1.0' + assert jm._slack_cpu(1000 * 3) == '1.25' + + +def test_shares_must_line_up_with_tiers(monkeypatch): + # A mismatched config disables tiering rather than guessing. + monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', '0.5,1.0') + monkeypatch.setattr(jm, 'PROFILE_CPU_SHARES', '0.9') + assert jm._slack_cpu(100) is None + + +def test_cpu_is_requested_but_never_limited(monkeypatch, cluster): + # A limit would cap the bucket phase, measured up to 2.53 cores and the one + # part of a job that slicing cannot remove. + prof = [(i, {'seconds': float(i)}) for i in range(1, 1001)] + prof[299] = (300, {'seconds': 300.0, 'peakAnonBytes': 1 << 30}) # 30th pct + monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', TIERS) + monkeypatch.setattr(jm, 'PROFILE_CPU_SHARES', SHARES) + monkeypatch.setattr(jm, 'PROFILE', prof) + monkeypatch.setattr(jm, '_SORTED_SECONDS', None) r = jm._resources(end=300) assert r.requests['cpu'] == '0.5' assert 'cpu' not in r.limits + + +def test_a_one_range_profile_gives_that_range_the_top_tier(tiered, monkeypatch): + # It is simultaneously the cheapest and the longest range measured, so the + # 100th percentile is the honest answer -- not the cheapest tier. + monkeypatch.setattr(jm, 'PROFILE', [(300, {'seconds': 1.0})]) + monkeypatch.setattr(jm, '_SORTED_SECONDS', None) + assert jm._slack_cpu(1.0) == '1.25' From 98566871c6b4f591d405a720f0276442436465fe Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 14:43:28 -0400 Subject: [PATCH 038/117] Collapse the cpu ladder into one setting Two parallel lists that had to stay the same length, with a test whose only job was catching the mismatch. ":" pairs cannot desync, read as the rule they encode, and delete a chart value, an env var and the validation that went with them. PROFILE_CPU_TIERS = "85:0.5,98:0.75,99.5:1.0,100:1.25" A malformed ladder still disables tiering rather than half-applying. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 29 ++++++++++--------- .../templates/job_monitor.yaml | 2 -- .../parallel_catchup_helm/values.yaml | 5 ---- .../tests/unit/test_cpu_tiers.py | 14 ++++----- 4 files changed, 21 insertions(+), 29 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 5fa3f710..b3614896 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1537,10 +1537,10 @@ def _sized(value, margin, cap): # that ceiling is run-to-run noise in the measurement (Spearman 0.87), not # something the keying can fix. # -# Shares are cumulative and cheapest-first: 0.85 means the cheapest 85% of -# ranges take the first tier. Empty PROFILE_CPU_TIERS disables tiering. -PROFILE_CPU_TIERS = os.getenv('PROFILE_CPU_TIERS', '') # "0.5,0.75,1.0,1.25" -PROFILE_CPU_SHARES = os.getenv('PROFILE_CPU_SHARES', '0.85,0.98,0.995,1.0') +# One entry per tier, cheapest first: ":". A range at or +# below that percentile of the profile's runtimes takes that many cores. Empty +# disables tiering and every range keeps the configured request. +PROFILE_CPU_TIERS = os.getenv('PROFILE_CPU_TIERS', '') # "85:0.5,98:0.75,99.5:1.0,100:1.25" _SORTED_SECONDS = None @@ -1559,22 +1559,25 @@ def _slack_cpu(seconds): A range with no measured runtime gets the top tier, matching the dispatch order: unprofiled means newer than anything measured, so assume worst. """ - tiers = [float(x) for x in PROFILE_CPU_TIERS.split(',') if x.strip()] - if not tiers: + try: + tiers = [(float(p), c) for p, c in + (t.split(':') for t in PROFILE_CPU_TIERS.split(',') if t.strip())] + except ValueError: + logger.error("PROFILE_CPU_TIERS is malformed (%r); cpu tiering disabled", + PROFILE_CPU_TIERS) return None - shares = [float(x) for x in PROFILE_CPU_SHARES.split(',') if x.strip()] - if len(shares) != len(tiers): + if not tiers: return None if not seconds: - return str(tiers[-1]) + return tiers[-1][1] everything = _profile_seconds() if not everything: return None - pct = bisect.bisect_right(everything, seconds) / len(everything) - for cores, upto in zip(tiers, shares): + pct = 100.0 * bisect.bisect_right(everything, seconds) / len(everything) + for upto, cores in tiers: if pct <= upto: - return str(cores) - return str(tiers[-1]) + return cores + return tiers[-1][1] def _profile_overrides(end, escalated): diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 181fbcae..e524d27c 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -230,8 +230,6 @@ spec: value: /profile/profile.json - name: PROFILE_CPU_TIERS value: {{ .Values.monitor.profileCpuTiers | quote }} - - name: PROFILE_CPU_SHARES - value: {{ .Values.monitor.profileCpuShares | quote }} - name: PROFILE_MARGIN value: {{ .Values.monitor.profileMargin | quote }} - name: PROFILE_CPU_LIMIT diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index b965f006..4a42dc4f 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -78,11 +78,6 @@ monitor: # and every range keeps the configured cpu request. # Paired with profileCpuSlowdown: the measured runtime multiplier per tier. profileCpuTiers: "" - # Cumulative, cheapest-first: 0.85 means the cheapest 85% of ranges take - # the first tier. Pins the fleet shape run to run, which an absolute - # seconds threshold does not -- that budget is set by the single - # worst-throttled job and moved 1.79x between two real runs. - profileCpuShares: "0.85,0.98,0.995,1.0" profileMargin: 1.15 # CPU limit for ranges the profile has measured. Above the configured # worker limit on purpose: at a 2-core limit every range pegs 2.0, so the diff --git a/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py b/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py index 11da04f1..1591c58e 100644 --- a/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py +++ b/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py @@ -15,8 +15,7 @@ import job_monitor as jm -TIERS = '0.5,0.75,1.0,1.25' -SHARES = '0.85,0.98,0.995,1.0' +TIERS = '85:0.5,98:0.75,99.5:1.0,100:1.25' # 1000 ranges, 1s..1000s, so a range's value IS its percentile x 1000. PROFILE = [(i, {'seconds': float(i)}) for i in range(1, 1001)] @@ -24,7 +23,6 @@ @pytest.fixture def tiered(monkeypatch): monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', TIERS) - monkeypatch.setattr(jm, 'PROFILE_CPU_SHARES', SHARES) monkeypatch.setattr(jm, 'PROFILE', PROFILE) monkeypatch.setattr(jm, '_SORTED_SECONDS', None) @@ -60,7 +58,6 @@ def test_a_uniformly_slower_run_assigns_the_same_tiers(monkeypatch): true if the slowdown is perfectly uniform, which measurement shows it is not. """ monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', TIERS) - monkeypatch.setattr(jm, 'PROFILE_CPU_SHARES', SHARES) monkeypatch.setattr(jm, 'PROFILE', [(i, {'seconds': i * 3.0}) for i in range(1, 1001)]) monkeypatch.setattr(jm, '_SORTED_SECONDS', None) assert jm._slack_cpu(850 * 3) == '0.5' @@ -68,10 +65,10 @@ def test_a_uniformly_slower_run_assigns_the_same_tiers(monkeypatch): assert jm._slack_cpu(1000 * 3) == '1.25' -def test_shares_must_line_up_with_tiers(monkeypatch): - # A mismatched config disables tiering rather than guessing. - monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', '0.5,1.0') - monkeypatch.setattr(jm, 'PROFILE_CPU_SHARES', '0.9') +def test_a_malformed_ladder_disables_tiering_rather_than_guessing(monkeypatch): + # One list of pairs cannot desync the way two parallel lists could, but a + # typo still has to fail safe rather than half-apply. + monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', '85,0.5') assert jm._slack_cpu(100) is None @@ -81,7 +78,6 @@ def test_cpu_is_requested_but_never_limited(monkeypatch, cluster): prof = [(i, {'seconds': float(i)}) for i in range(1, 1001)] prof[299] = (300, {'seconds': 300.0, 'peakAnonBytes': 1 << 30}) # 30th pct monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', TIERS) - monkeypatch.setattr(jm, 'PROFILE_CPU_SHARES', SHARES) monkeypatch.setattr(jm, 'PROFILE', prof) monkeypatch.setattr(jm, '_SORTED_SECONDS', None) r = jm._resources(end=300) From afd80ac0caceec1958fcf221360af82ae66730f1 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 15:47:52 -0400 Subject: [PATCH 039/117] Improve worker liveness metrics Probe stellar-core responsiveness asynchronously with bounded concurrency, conservative hysteresis, and an additive unknown state. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/MissionParallelCatchup/job_monitor.py | 361 ++++++++++++++++-- .../templates/job_monitor.yaml | 8 + .../parallel_catchup_helm/values.yaml | 11 + .../tests/contract/test_chart_env_wiring.py | 13 + .../tests/unit/test_worker_liveness.py | 231 +++++++++++ 5 files changed, 601 insertions(+), 23 deletions(-) create mode 100644 src/MissionParallelCatchup/tests/unit/test_worker_liveness.py diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index b3614896..f7cdfba5 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -20,6 +20,7 @@ import json import logging import os +import queue import re import sys import tempfile @@ -33,6 +34,7 @@ from kubernetes.client.rest import ApiException from prometheus_client import (CONTENT_TYPE_LATEST, REGISTRY, Counter, Gauge, Histogram, generate_latest) +import requests # Histogram buckets # 5m 15m 30m 1h 1.5h 2h @@ -248,6 +250,33 @@ # stops all dispatch, so restart the container rather than run half-alive. RECONCILE_STALE_SECONDS = float(os.getenv('WATCH_STALE_SECONDS', 600)) +# Worker responsiveness is cosmetic and sampled independently from reconcile. +# Thirty seconds and three failures restore the old ~90-second down threshold, +# while a five-second request budget gives a busy admin endpoint substantially +# more room than the old one-shot two-second probe. +LIVENESS_PROBE_INTERVAL_SECONDS = os.getenv('LIVENESS_PROBE_INTERVAL_SECONDS', '30') +LIVENESS_PROBE_TIMEOUT_SECONDS = os.getenv('LIVENESS_PROBE_TIMEOUT_SECONDS', '5') +LIVENESS_FAILURE_THRESHOLD = os.getenv('LIVENESS_FAILURE_THRESHOLD', '3') +LIVENESS_MAX_CONCURRENCY = os.getenv('LIVENESS_MAX_CONCURRENCY', '32') +try: + LIVENESS_PROBE_INTERVAL_SECONDS = float(LIVENESS_PROBE_INTERVAL_SECONDS) + LIVENESS_PROBE_TIMEOUT_SECONDS = float(LIVENESS_PROBE_TIMEOUT_SECONDS) + LIVENESS_FAILURE_THRESHOLD = int(LIVENESS_FAILURE_THRESHOLD) + LIVENESS_MAX_CONCURRENCY = int(LIVENESS_MAX_CONCURRENCY) +except ValueError as e: + raise ValueError( + "LIVENESS_PROBE_INTERVAL_SECONDS and LIVENESS_PROBE_TIMEOUT_SECONDS " + "must be numbers; LIVENESS_FAILURE_THRESHOLD and " + "LIVENESS_MAX_CONCURRENCY must be integers") from e + +for _name, _value in ( + ('LIVENESS_PROBE_INTERVAL_SECONDS', LIVENESS_PROBE_INTERVAL_SECONDS), + ('LIVENESS_PROBE_TIMEOUT_SECONDS', LIVENESS_PROBE_TIMEOUT_SECONDS), + ('LIVENESS_FAILURE_THRESHOLD', LIVENESS_FAILURE_THRESHOLD), + ('LIVENESS_MAX_CONCURRENCY', LIVENESS_MAX_CONCURRENCY)): + if _value <= 0: + raise ValueError(f"{_name} must be greater than zero, got {_value!r}") + # Shared with the log-collector sidecar, which owns writes here: it streams each # worker's log and records the .outcome verdict while the pod still exists. LOG_DIR = os.getenv('LOG_DIR', '/logs') @@ -389,6 +418,290 @@ def check_storage_config(): metric_eph_retries = Counter('ssc_parallel_catchup_job_ephemeral_retried_count', 'Jobs retried with an escalated ephemeral-storage limit') +def _worker_targets(pods): + """Current Running-with-IP pods, keyed by pod identity. + + A UID change is a replacement even when the Job name or IP is reused. Tests + and unusually incomplete API objects may lack a UID, where the pod name is + still unique for its lifetime. + """ + out = {} + for pod in pods: + pod_status = getattr(pod, 'status', None) + metadata = getattr(pod, 'metadata', None) + ip = getattr(pod_status, 'pod_ip', None) + if getattr(pod_status, 'phase', None) != 'Running' or not ip or metadata is None: + continue + name = getattr(metadata, 'name', None) + identity = getattr(metadata, 'uid', None) or name + if identity and name: + out[str(identity)] = (str(name), str(ip)) + return out + + +class WorkerLivenessSampler: + """Bounded, round-robin stellar-core `/info` sampler. + + Candidate membership comes from the authoritative Kubernetes snapshot, but + all network I/O happens on this sampler's fixed worker pool. At most + `max_concurrency` requests run and the same number wait in the bounded queue; + there is no future, task, session, or thread per pod. + + State is deliberately conservative: + * new or replaced pod: unknown + * any HTTP response from /info: up + * fewer than `failure_threshold` consecutive exceptions/timeouts: unknown + * `failure_threshold` consecutive failures: down + * any later response: up immediately + + HTTP error statuses still prove the admin endpoint responded. A busy core + returning 5xx is responsive; only failure to receive an HTTP response counts + toward down. + """ + + def __init__(self, interval=LIVENESS_PROBE_INTERVAL_SECONDS, + timeout=LIVENESS_PROBE_TIMEOUT_SECONDS, + failure_threshold=LIVENESS_FAILURE_THRESHOLD, + max_concurrency=LIVENESS_MAX_CONCURRENCY, probe=None): + if interval <= 0 or timeout <= 0 or failure_threshold <= 0 or max_concurrency <= 0: + raise ValueError("liveness sampler values must all be greater than zero") + self.interval = float(interval) + self.timeout = float(timeout) + self.failure_threshold = int(failure_threshold) + self.max_concurrency = int(max_concurrency) + self._probe = probe + self._records = {} + self._generation = 0 + self._tasks = queue.Queue(maxsize=self.max_concurrency) + self._stop = threading.Event() + self._condition = threading.Condition() + self._scheduler = None + self._workers = [] + self._started = False + self._failed = None + self._active = 0 + self._failure_count = 0 + self._last_failure_log = 0.0 + + def start(self): + with self._condition: + if self._started: + return + self._started = True + self._workers = [ + threading.Thread(target=self._worker_main, + name=f"worker-liveness-{i}", daemon=True) + for i in range(self.max_concurrency) + ] + self._scheduler = threading.Thread( + target=self._scheduler_main, name="worker-liveness-scheduler", + daemon=True) + for worker in self._workers: + worker.start() + self._scheduler.start() + + def close(self): + self._stop.set() + with self._condition: + self._condition.notify_all() + threads = ([self._scheduler] if self._scheduler is not None else []) + self._workers + deadline = time.monotonic() + self.timeout + 1.0 + for thread in threads: + remaining = max(0.0, deadline - time.monotonic()) + if thread is not None and thread is not threading.current_thread(): + thread.join(remaining) + + def replace_candidates(self, targets, now=None): + """Atomically replace membership without waiting for any probe.""" + now = time.monotonic() if now is None else float(now) + targets = dict(targets) + with self._condition: + old = self._records + records = {} + new_identities = [ + identity for identity in sorted(targets) + if identity not in old or old[identity]['target'] != targets[identity] + ] + offsets = { + identity: self.interval * index / max(1, len(new_identities)) + for index, identity in enumerate(new_identities) + } + for identity, target in targets.items(): + previous = old.get(identity) + if previous is not None and previous['target'] == target: + records[identity] = previous + continue + self._generation += 1 + records[identity] = { + 'target': target, + 'generation': self._generation, + 'status': 'unknown', + 'failures': 0, + 'queued': False, + 'next_due': now + offsets[identity], + } + self._records = records + self._condition.notify_all() + + def counts(self, expected_count=None): + with self._condition: + count = len(self._records) if expected_count is None else int(expected_count) + healthy = self._started and self._failed is None + if healthy: + healthy = (self._scheduler is not None and self._scheduler.is_alive() + and all(worker.is_alive() for worker in self._workers)) + if not healthy or count != len(self._records): + return {'up': 0, 'down': 0, 'unknown': count} + result = {'up': 0, 'down': 0, 'unknown': 0} + for record in self._records.values(): + result[record['status']] += 1 + return result + + def stats(self): + """Small observability hook used by the scale contract test.""" + with self._condition: + live_threads = sum( + 1 for thread in ([self._scheduler] + self._workers) + if thread is not None and thread.is_alive()) + return { + 'records': len(self._records), + 'active': self._active, + 'queued': self._tasks.qsize(), + 'outstanding': self._active + self._tasks.qsize(), + 'threads': live_threads, + 'failed': self._failed, + } + + def _scheduler_main(self): + try: + self._schedule() + except Exception as e: + self._mark_failed("scheduler", e) + + def _schedule(self): + while not self._stop.is_set(): + with self._condition: + now = time.monotonic() + capacity = self.max_concurrency - self._tasks.qsize() + due = sorted( + ((record['next_due'], identity, record) + for identity, record in self._records.items() + if not record['queued'] and record['next_due'] <= now), + key=lambda item: (item[0], item[1])) + for _, identity, record in due[:max(0, capacity)]: + task = (identity, record['generation'], record['target']) + try: + self._tasks.put_nowait(task) + except queue.Full: + break + record['queued'] = True + + waiting = [ + record['next_due'] for record in self._records.values() + if not record['queued'] + ] + delay = max(0.01, min(1.0, min(waiting) - now)) if waiting else 1.0 + self._condition.wait(timeout=delay) + + def _worker_main(self): + session = None + try: + if self._probe is None: + session = requests.Session() + adapter = requests.adapters.HTTPAdapter( + pool_connections=4, pool_maxsize=1, max_retries=0) + session.mount('http://', adapter) + while not self._stop.is_set(): + try: + task = self._tasks.get(timeout=0.2) + except queue.Empty: + continue + with self._condition: + self._active += 1 + identity, generation, target = task + success = False + error = None + try: + _, ip = target + if self._probe is None: + host = f"[{ip}]" if ':' in ip else ip + with session.get(f"http://{host}:11626/info", + timeout=self.timeout): + pass + else: + self._probe(ip, self.timeout) + success = True + except Exception as e: + error = e + finally: + self._record_result(identity, generation, target, success, error) + self._tasks.task_done() + with self._condition: + self._active -= 1 + self._condition.notify_all() + except Exception as e: + self._mark_failed("probe worker", e) + finally: + if session is not None: + session.close() + + def _record_result(self, identity, generation, target, success, error=None, + now=None): + now = time.monotonic() if now is None else float(now) + log_failure = None + with self._condition: + record = self._records.get(identity) + if (record is None or record['generation'] != generation + or record['target'] != target): + return + record['queued'] = False + record['next_due'] = now + self.interval + if success: + record['failures'] = 0 + record['status'] = 'up' + else: + record['failures'] += 1 + record['status'] = ( + 'down' if record['failures'] >= self.failure_threshold + else 'unknown') + self._failure_count += 1 + if now - self._last_failure_log >= 60.0: + log_failure = self._failure_count + self._failure_count = 0 + self._last_failure_log = now + self._condition.notify_all() + if log_failure is not None: + logger.warning( + "stellar-core /info liveness probes are failing; %d failure(s) " + "across the fleet since the previous warning (latest: %s: %s)", + log_failure, target[0], error) + + def _mark_failed(self, component, error): + with self._condition: + if self._failed is not None: + return + self._failed = f"{component}: {error}" + self._condition.notify_all() + logger.exception( + "worker liveness %s failed; all current workers will be reported " + "unknown and reconcile will continue", component) + + +worker_liveness_sampler = WorkerLivenessSampler() + + +def publish_worker_liveness(targets, sampler=None): + """Hand a pod snapshot to the sampler and return its current three counts. + + This path copies O(current workers) state under a short lock but never makes + a request or waits for an in-flight request. Keeping it separate makes the + non-blocking boundary directly testable. + """ + sampler = sampler or worker_liveness_sampler + sampler.replace_candidates(targets) + return sampler.counts(len(targets)) + + class RequestHandler(BaseHTTPRequestHandler): def do_GET(self): if self.path == '/healthz': @@ -2266,6 +2579,9 @@ def reconcile(state): if str(end) not in completed and str(end) not in failed and str(end) not in in_flight), + # A Kubernetes snapshot only. The caller hands this to the independent + # liveness sampler after every dispatch/progress decision is complete. + '_worker_targets': _worker_targets(job_pods.values()), } @@ -2305,21 +2621,21 @@ def update_status_and_metrics(): r = reconcile(state) - # Worker liveness, for the Grafana series only -- nothing in the - # driver reads it. A worker is a Job here, so a Running pod IS a - # live worker and its liveness is the Job's status; the count comes - # off the pod list the apiserver already has cached instead of one - # HTTP GET per worker every cycle. + # Grafana-only worker responsiveness. Candidate discovery reused the + # authoritative pod snapshot, but the handoff below never performs + # network I/O: /info probes run on a fixed, bounded sampler pool. refresh_start = time.time() - workers_up = sum( - 1 for p in core_v1.list_namespaced_pod( - NAMESPACE, label_selector=f"{LABEL_RUN}={RUN_NAME}", - field_selector='status.phase=Running', - # Served from the apiserver watch cache. Only safe here: - # a stale liveness sample is cosmetic, whereas stale - # dispatch state would re-run a range. - resource_version='0').items - if p.status.pod_ip) + targets = r.pop('_worker_targets') + try: + worker_counts = publish_worker_liveness(targets) + except Exception as e: + worker_counts = {'up': 0, 'down': 0, 'unknown': len(targets)} + now = time.time() + if now - state.get('last_liveness_error_log', 0) >= 60: + state['last_liveness_error_log'] = now + logger.exception( + "worker liveness publication failed (%s); reporting all " + "current candidates unknown and continuing reconcile", e) workers_refresh_duration = time.time() - refresh_start mission_duration = time.time() - mission_start_time @@ -2339,12 +2655,9 @@ def update_status_and_metrics(): metric_catchup_queues.labels(queue="succeeded").set(r['completed']) metric_catchup_queues.labels(queue="failed").set(len(r['failed_ranges'])) metric_catchup_queues.labels(queue="in_progress").set(len(r['in_progress'])) - metric_workers.labels(status="up").set(workers_up) - # Held at 0 rather than dropped: the series is Grafana-facing, and a - # label that stops being set goes stale on the dashboard instead of - # reading zero. Nothing can report "down" now that liveness is the - # pod's phase -- a worker that is not up is simply not listed. - metric_workers.labels(status="down").set(0) + metric_workers.labels(status="up").set(worker_counts['up']) + metric_workers.labels(status="down").set(worker_counts['down']) + metric_workers.labels(status="unknown").set(worker_counts['unknown']) metric_refresh_duration.set(workers_refresh_duration) metric_mission_duration.set(mission_duration) logger.info("Status: %s", json.dumps(status)) @@ -2372,6 +2685,7 @@ def run(server_class=HTTPServer, handler_class=RequestHandler): if __name__ == '__main__': # Before any dispatch: the first Job built must already be sized from it. PROFILE = load_profile() + worker_liveness_sampler.start() # Not a logging thread despite the historical name -- this is the reconcile # loop: dispatch, progress record, metrics, status. Log capture and pod @@ -2380,6 +2694,7 @@ def run(server_class=HTTPServer, handler_class=RequestHandler): reconcile_thread.daemon = True reconcile_thread.start() - # Separate thread: a blocking watch must not sit behind dispatch and the - # liveness sweep, which is the whole point of it. - run() + try: + run() + finally: + worker_liveness_sampler.close() diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index e524d27c..b52edd13 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -157,6 +157,14 @@ spec: value: {{ .Values.monitor.emitMissionLabel | quote }} - name: LOGGING_INTERVAL_SECONDS value: {{ .Values.monitor.loggingIntervalSeconds | quote }} + - name: LIVENESS_PROBE_INTERVAL_SECONDS + value: {{ .Values.monitor.livenessProbeIntervalSeconds | quote }} + - name: LIVENESS_PROBE_TIMEOUT_SECONDS + value: {{ .Values.monitor.livenessProbeTimeoutSeconds | quote }} + - name: LIVENESS_FAILURE_THRESHOLD + value: {{ .Values.monitor.livenessFailureThreshold | quote }} + - name: LIVENESS_MAX_CONCURRENCY + value: {{ .Values.monitor.livenessMaxConcurrency | quote }} - name: RANGE_GENERATOR value: {{ .Values.range.generator | quote }} - name: STARTING_LEDGER diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 4a42dc4f..15f66804 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -106,6 +106,17 @@ monitor: # once the panels aggregate. emitMissionLabel: false loggingIntervalSeconds: 10 + # Cosmetic stellar-core /info sampling. A fixed pool probes each Running pod + # about once per interval; three consecutive failures preserve the old + # approximately 90-second down threshold without coupling network I/O to the + # 10-second reconcile loop. Unprobed/intermediate workers report unknown. + livenessProbeIntervalSeconds: 30 + livenessProbeTimeoutSeconds: 5 + livenessFailureThreshold: 3 + # At 2096 workers and a 30s interval this is ~70 requests/s. Thirty-two slots + # cover normal sub-second local responses while bounding slow requests, + # queued work, sessions and threads independently of fleet size. + livenessMaxConcurrency: 32 maxAttempts: 5 # Hangs are usually persistent (bad archive host, absent checkpoint), so they # get a lower cap than evictions -- otherwise a wedged range costs diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py b/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py index 878efe3d..e02a8ca5 100644 --- a/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py +++ b/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py @@ -60,6 +60,19 @@ def test_every_env_the_collector_reads_is_set_on_the_collector_container(): assert not missing, f"the collector reads {missing} but the chart never sets them" +def test_liveness_sampler_settings_reach_only_the_monitor(): + monitor = art.env_of(art.containers()[art.MONITOR_CONTAINER]) + collector = art.env_of(art.containers()[art.COLLECTOR_CONTAINER]) + expected = { + 'LIVENESS_PROBE_INTERVAL_SECONDS': '30', + 'LIVENESS_PROBE_TIMEOUT_SECONDS': '5', + 'LIVENESS_FAILURE_THRESHOLD': '3', + 'LIVENESS_MAX_CONCURRENCY': '32', + } + assert {name: monitor.get(name) for name in expected} == expected + assert not set(expected) & set(collector) + + def test_the_node_targeting_the_mission_sends_reaches_the_monitor(): """A label/taint the mission passes must arrive as env, not just as YAML. diff --git a/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py b/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py new file mode 100644 index 00000000..74e05bb2 --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py @@ -0,0 +1,231 @@ +"""Worker responsiveness metrics stay truthful without entering reconcile.""" + +import os +import subprocess +import sys +import threading +import time + +import job_monitor as jm +from kubernetes import client + + +def _task(sampler, identity): + record = sampler._records[identity] + return identity, record['generation'], record['target'] + + +def _result(sampler, identity, success, now): + sampler._record_result(*_task(sampler, identity), success, + None if success else TimeoutError("busy"), now=now) + + +def test_hysteresis_unknown_down_and_immediate_recovery(): + sampler = jm.WorkerLivenessSampler( + interval=30, timeout=5, failure_threshold=3, max_concurrency=1) + sampler.replace_candidates({'uid-1': ('pod-1', '10.0.0.1')}, now=0) + + assert sampler._records['uid-1']['status'] == 'unknown' + _result(sampler, 'uid-1', True, 1) + assert sampler._records['uid-1']['status'] == 'up' + + _result(sampler, 'uid-1', False, 31) + assert sampler._records['uid-1']['status'] == 'unknown' + _result(sampler, 'uid-1', False, 61) + assert sampler._records['uid-1']['status'] == 'unknown' + _result(sampler, 'uid-1', False, 91) + assert sampler._records['uid-1']['status'] == 'down' + + _result(sampler, 'uid-1', True, 121) + assert sampler._records['uid-1']['status'] == 'up' + assert sampler._records['uid-1']['failures'] == 0 + + +def test_disappearance_and_replacement_discard_stale_probe_results(): + sampler = jm.WorkerLivenessSampler(max_concurrency=1) + sampler.replace_candidates({'old-uid': ('pod-1', '10.0.0.1')}, now=0) + old_task = _task(sampler, 'old-uid') + _result(sampler, 'old-uid', True, 1) + + # A new UID is a new attempt/pod even if Kubernetes reuses the IP. + sampler.replace_candidates({'new-uid': ('pod-2', '10.0.0.1')}, now=2) + assert set(sampler._records) == {'new-uid'} + assert sampler._records['new-uid']['status'] == 'unknown' + + # The old request may finish after the replacement snapshot. It cannot + # resurrect the vanished pod or update the replacement. + sampler._record_result(*old_task, True, now=3) + assert set(sampler._records) == {'new-uid'} + assert sampler._records['new-uid']['status'] == 'unknown' + + sampler.replace_candidates({}, now=4) + assert sampler.counts() == {'up': 0, 'down': 0, 'unknown': 0} + + +def test_ip_change_on_same_identity_resets_to_unknown(): + sampler = jm.WorkerLivenessSampler(max_concurrency=1) + sampler.replace_candidates({'uid': ('pod', '10.0.0.1')}, now=0) + old_task = _task(sampler, 'uid') + _result(sampler, 'uid', True, 1) + + sampler.replace_candidates({'uid': ('pod', '10.0.0.2')}, now=2) + assert sampler._records['uid']['status'] == 'unknown' + assert sampler._records['uid']['failures'] == 0 + + sampler._record_result(*old_task, False, TimeoutError(), now=3) + assert sampler._records['uid']['status'] == 'unknown' + assert sampler._records['uid']['failures'] == 0 + + +def test_sampler_failure_reports_every_current_candidate_unknown(): + release = threading.Event() + + def blocked_probe(_ip, _timeout): + release.wait(2) + + sampler = jm.WorkerLivenessSampler( + interval=30, timeout=1, failure_threshold=3, + max_concurrency=1, probe=blocked_probe) + sampler.start() + try: + sampler.replace_candidates({ + 'a': ('pod-a', '10.0.0.1'), + 'b': ('pod-b', '10.0.0.2'), + }) + with sampler._condition: + sampler._failed = 'synthetic scheduler failure' + assert sampler.counts() == {'up': 0, 'down': 0, 'unknown': 2} + finally: + release.set() + sampler.close() + + +def test_probe_uses_stellar_core_info_and_any_http_response_is_up(monkeypatch): + called = [] + + class Response: + status_code = 503 + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + class Session: + def mount(self, *_args): + pass + + def get(self, url, timeout): + called.append((url, timeout)) + return Response() + + def close(self): + pass + + monkeypatch.setattr(jm.requests, 'Session', Session) + sampler = jm.WorkerLivenessSampler( + interval=30, timeout=5, failure_threshold=3, max_concurrency=1) + sampler.start() + try: + sampler.replace_candidates({'uid': ('pod', '10.2.3.4')}) + deadline = time.monotonic() + 1 + while time.monotonic() < deadline and sampler._records['uid']['status'] != 'up': + time.sleep(0.01) + assert called == [('http://10.2.3.4:11626/info', 5.0)] + assert sampler._records['uid']['status'] == 'up', ( + "an HTTP 503 is a busy but responsive admin endpoint") + finally: + sampler.close() + + +def test_only_running_pods_with_ips_are_candidates_and_uid_is_identity(): + def pod(name, uid, phase, ip): + return client.V1Pod( + metadata=client.V1ObjectMeta(name=name, uid=uid), + status=client.V1PodStatus(phase=phase, pod_ip=ip)) + + targets = jm._worker_targets([ + pod('ready', 'uid-ready', 'Running', '10.0.0.1'), + pod('pending', 'uid-pending', 'Pending', '10.0.0.2'), + pod('no-ip', 'uid-no-ip', 'Running', None), + ]) + assert targets == {'uid-ready': ('ready', '10.0.0.1')} + + +def test_malformed_liveness_configuration_fails_with_an_explicit_message(): + env = { + 'PATH': os.environ.get('PATH', ''), + 'HOME': os.environ.get('HOME', ''), + 'PYTHONPATH': os.path.dirname(jm.__file__), + 'LIVENESS_MAX_CONCURRENCY': 'many', + } + result = subprocess.run( + [sys.executable, '-c', 'import job_monitor'], + text=True, capture_output=True, env=env, + cwd=os.path.dirname(jm.__file__)) + assert result.returncode != 0 + assert 'LIVENESS_MAX_CONCURRENCY must be integers' in result.stderr + + +def test_2096_slow_workers_have_bounded_work_and_do_not_delay_reconcile( + cluster): + release = threading.Event() + active_lock = threading.Lock() + active = 0 + peak_active = 0 + + def blocked_probe(_ip, _timeout): + nonlocal active, peak_active + with active_lock: + active += 1 + peak_active = max(peak_active, active) + try: + release.wait(3) + finally: + with active_lock: + active -= 1 + + concurrency = 8 + sampler = jm.WorkerLivenessSampler( + interval=0.2, timeout=1, failure_threshold=3, + max_concurrency=concurrency, probe=blocked_probe) + targets = { + f"uid-{i}": (f"pod-{i}", f"10.{i // 65536}.{(i // 256) % 256}.{i % 256}") + for i in range(2096) + } + sampler.start() + sampler.replace_candidates(targets) + try: + deadline = time.monotonic() + 2 + while time.monotonic() < deadline and sampler.stats()['active'] < concurrency: + time.sleep(0.01) + + stats = sampler.stats() + assert stats['records'] == 2096 + assert stats['active'] <= concurrency + assert stats['queued'] <= concurrency + assert stats['outstanding'] <= 2 * concurrency + assert stats['threads'] == concurrency + 1 + assert peak_active <= concurrency + + # Exercise the exact handoff used by update_status_and_metrics while all + # request slots are blocked. It copies the candidate snapshot and reads + # counts, but never waits for a request. + started = time.monotonic() + counts = jm.publish_worker_liveness(targets, sampler=sampler) + publish_elapsed = time.monotonic() - started + assert publish_elapsed < 0.5, ( + f"blocked probes delayed liveness publication by {publish_elapsed:.3f}s") + assert counts == {'up': 0, 'down': 0, 'unknown': 2096} + assert sum(counts.values()) == len(targets) + + # Dispatch itself remains equally independent. + started = time.monotonic() + result = cluster.reconcile() + elapsed = time.monotonic() - started + assert result['created'] == 2 + assert elapsed < 0.5, f"blocked liveness probes delayed reconcile by {elapsed:.3f}s" + finally: + release.set() + sampler.close() From 08858505ee6d0e27a939b99c79d1eee34e72aad9 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 15:50:47 -0400 Subject: [PATCH 040/117] Pass mission namespace to Helm Keep Helm install, inspection, cleanup, and profile ConfigMap creation in the same namespace used by the Kubernetes client. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MissionHistoryPubnetParallelCatchupV2.fs | 82 +++++++++++-------- .../contract/test_fsharp_driver_contract.py | 29 +++++++ 2 files changed, 76 insertions(+), 35 deletions(-) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index fccc0043..0d2ddc39 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -30,7 +30,8 @@ open CSLibrary // can point at a working copy without editing this file. let helmChartPath = match Environment.GetEnvironmentVariable("SUPERCLUSTER_CHART_PATH") with - | null | "" -> "/supercluster/src/MissionParallelCatchup/parallel_catchup_helm" + | null + | "" -> "/supercluster/src/MissionParallelCatchup/parallel_catchup_helm" | p -> p // Comment out the path below for local testing @@ -41,7 +42,8 @@ let valuesFilePath = helmChartPath + "/values.yaml" // Keys in the -catchup-progress ConfigMap. These were HTTP paths when // the driver polled the monitor through a Gateway; it reads the ConfigMap now. -let jobMonitorStatusKey = "status.json" // live queue counts +let jobMonitorStatusKey = "status.json" // live queue counts + let jobMonitorProgressKey = "progress.json" // durable per-range completion record let jobMonitorLoggingIntervalSecs = 30 // frequency of the monitor reconcile loop: dispatch, liveness ping, status publish let jobMonitorStatusCheckIntervalSecs = 60 // frequency of us querying job monitor's `/status` end point @@ -85,6 +87,7 @@ let resolveRangeProfile (context: MissionContext) : string option = // Parse before shipping it: a 404 page or a truncated download would // otherwise reach the monitor as an unreadable mount. let doc = JObject.Parse(body) + let count = match doc.["ranges"] with | :? JObject as r -> r.Count @@ -102,15 +105,15 @@ let resolveRangeProfile (context: MissionContext) : string option = "create" "configmap" name + "--namespace" + context.namespaceProperty sprintf "--from-file=profile.json=%s" file |] |> ignore LogInfo "Range profile: %d ranges from %s -> configmap %s" count spec name Some name with ex -> - LogWarn "Could not load range profile %s (%s); sizing from configured requests" - spec - ex.Message + LogWarn "Could not load range profile %s (%s); sizing from configured requests" spec ex.Message None @@ -176,9 +179,7 @@ let installProject (context: MissionContext) = setOptions.Add(sprintf "range.latestLedgerNum=%d" endLedger) - setOptions.Add( - sprintf "range.ledgersPerJob=%d" context.pubnetParallelCatchupLedgersPerJob - ) + setOptions.Add(sprintf "range.ledgersPerJob=%d" context.pubnetParallelCatchupLedgersPerJob) // Skip known results by default setOptions.Add( @@ -229,8 +230,10 @@ let installProject (context: MissionContext) = // monitor clamps any profile-derived cpu request to REQ_CPU, so this is the // ceiling as well as the default. let cpuReqEffective = - if String.IsNullOrWhiteSpace context.pubnetParallelCatchupCpuRequest then cpuReqMili - else context.pubnetParallelCatchupCpuRequest + if String.IsNullOrWhiteSpace context.pubnetParallelCatchupCpuRequest then + cpuReqMili + else + context.pubnetParallelCatchupCpuRequest setOptions.Add(sprintf "worker.resources.requests.cpu=%s" cpuReqEffective) setOptions.Add(sprintf "worker.resources.requests.memory=%s" memReqMebi) @@ -252,8 +255,7 @@ let installProject (context: MissionContext) = // over 512 KiB, so every large download killed its own stream, which // then reconnected and hit the same wall. Measured on ssc-test // 2026-07-30: it starved every retry pod of a collector stream. - let s3GetCommandBase = - sprintf "aws s3 cp --no-progress --region %s" context.s3HistoryMirrorRegionPcV2 + let s3GetCommandBase = sprintf "aws s3 cp --no-progress --region %s" context.s3HistoryMirrorRegionPcV2 let command = sprintf "%s s3://%s/core_live_00%d/{0} {1}" s3GetCommandBase url index setOptions.Add(sprintf "worker.historyGetCommandCore00%d=\"%s\"" index command) @@ -318,6 +320,8 @@ let installProject (context: MissionContext) = "install" helmReleaseName helmChartPath + "--namespace" + context.namespaceProperty "--values" valuesFilePath "--set" @@ -327,7 +331,9 @@ let installProject (context: MissionContext) = match RunShellCommand [| "helm" "get" "values" - helmReleaseName |] with + helmReleaseName + "--namespace" + context.namespaceProperty |] with | Some valuesOutput -> LogInfo "%s" valuesOutput | _ -> () @@ -343,7 +349,8 @@ let collectLogsFromPods (context: MissionContext) = // retry, on success before the Job's TTL -- onto its own volume, so one // exec here replaces the ~1024 that the StatefulSet design needed. let monitorPods = - context.kube + context + .kube .ListNamespacedPod( context.namespaceProperty, labelSelector = sprintf "app=job-monitor,release=%s" helmReleaseName @@ -353,10 +360,7 @@ let collectLogsFromPods (context: MissionContext) = |> List.ofSeq match monitorPods with - | [] -> - LogWarn - "No job-monitor pod found for release %s; worker logs cannot be collected" - helmReleaseName + | [] -> LogWarn "No job-monitor pod found for release %s; worker logs cannot be collected" helmReleaseName | podName :: _ -> try LogInfo "Collecting worker logs from job-monitor pod %s to %s" podName context.destination.Path @@ -392,8 +396,7 @@ let collectLogsFromPods (context: MissionContext) = else LogWarn "Worker log archive is empty: %s" outputFile - with ex -> - LogWarn "Could not collect worker logs from %s: %s" podName ex.Message + with ex -> LogWarn "Could not collect worker logs from %s: %s" podName ex.Message // Cleanup on exit. `signalTriggered` indicates we're running under a hard // deadline (Jenkins' SoftKillWaitSeconds, ~5s by default, before SIGKILL). @@ -435,8 +438,14 @@ let rangeProfileFields = // quantity). Omitting it here silently stripped it from the mission's // profile artifact while the monitor's own progress.json carried it -- // measured 2026-07-30: artifact 0% peakAnonBytes, volume copy 99%. - [ "peakAnonBytes"; "peakRssBytes"; "peakWorkingSetBytes"; "peakCpuCores" - "peakEphemeralBytes"; "seconds"; "wallSeconds"; "txApply" ] + [ "peakAnonBytes" + "peakRssBytes" + "peakWorkingSetBytes" + "peakCpuCores" + "peakEphemeralBytes" + "seconds" + "wallSeconds" + "txApply" ] // A missing measurement must stay missing rather than become a null: the // consumer falls back to its configured default when the field is absent. @@ -459,7 +468,8 @@ let projectRangeEntry (record: JObject) : JObject = // ConfigMap would silently truncate the artifact. let readProgressRecord (context: MissionContext) : JObject option = let monitorPods = - context.kube + context + .kube .ListNamespacedPod( context.namespaceProperty, labelSelector = sprintf "app=job-monitor,release=%s" helmReleaseName @@ -559,11 +569,7 @@ let buildRangeProfile (completed: JObject) : JObject = // The profile document to write, or None when there is nothing worth writing. -let rangeProfileDocument - (storageMode: string) - (defaultLedgersPerRange: int) - (completed: JObject) - : JObject option = +let rangeProfileDocument (storageMode: string) (defaultLedgersPerRange: int) (completed: JObject) : JObject option = let ranges = buildRangeProfile completed // Slicing and storage mode both go in the name, because a profile is @@ -579,10 +585,11 @@ let rangeProfileDocument let ledgersPerRange = let counts = ranges.Properties() - |> Seq.choose (fun p -> - match (p.Value :?> JObject).["count"] with - | null -> None - | v -> Some(v.Value())) + |> Seq.choose + (fun p -> + match (p.Value :?> JObject).["count"] with + | null -> None + | v -> Some(v.Value())) |> Seq.toList match counts with @@ -632,7 +639,8 @@ let writeRangeProfile (context: MissionContext) = "%s-profile-%dledgers-%s.json" helmReleaseName ledgersPerRange - context.pubnetParallelCatchupStorageMode) + context.pubnetParallelCatchupStorageMode + ) File.WriteAllText(path, doc.ToString()) LogInfo "Wrote range profile for %d ranges to %s" ranges.Count path @@ -663,7 +671,9 @@ let cleanup (signalTriggered: bool) (context: MissionContext) = RunShellCommand [| "helm" "uninstall" - helmReleaseName |] + helmReleaseName + "--namespace" + context.namespaceProperty |] |> ignore else // Normal / legitimate-failure path: pods are still alive through @@ -680,7 +690,9 @@ let cleanup (signalTriggered: bool) (context: MissionContext) = RunShellCommand [| "helm" "uninstall" - helmReleaseName |] + helmReleaseName + "--namespace" + context.namespaceProperty |] |> ignore let mutable cleanupContext : MissionContext option = None diff --git a/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py b/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py index b3461202..03f3176d 100644 --- a/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py +++ b/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py @@ -62,6 +62,35 @@ def test_the_driver_really_does_configure_the_chart(): assert 'worker.stellar_core_image' in keys and 'range.ledgersPerJob' in keys +def test_every_helm_command_uses_the_mission_namespace(): + """KUBECONFIG chooses a cluster, but its current namespace is unrelated. + + The Kubernetes client always uses context.namespaceProperty. Every Helm + operation must use that same namespace explicitly or install into the + kubeconfig default, poll sandbox through the client, and wait forever for a + monitor that exists in another namespace. + """ + blocks = re.findall(r'RunShellCommand\s+\[\|\s*"helm"(.*?)\|\]', FS, re.S) + assert len(blocks) == 4, ( + f"expected install, get-values and two cleanup commands; found {len(blocks)}") + for block in blocks: + verb = re.search(r'"(install|get|upgrade|uninstall)"', block) + assert verb, f"could not identify Helm command in {block!r}" + assert re.search( + r'"--namespace"\s+context\.namespaceProperty', block), ( + f"helm {verb.group(1)} does not target the mission namespace: {block!r}") + + +def test_profile_configmap_uses_the_mission_namespace(): + """The profile mount and Helm release must be created in one namespace.""" + block = fs_extract( + r'RunShellCommand\s+\[\|\s*"kubectl"(.*?)\|\]').group(1) + assert '"create"' in block and '"configmap"' in block + assert re.search(r'"--namespace"\s+context\.namespaceProperty', block), ( + "the range-profile ConfigMap follows kubeconfig's default namespace " + "instead of the mission namespace") + + def test_every_value_the_driver_sets_is_one_the_chart_knows(): """`helm --set` on an unknown path is accepted and ignored. From e68a0f91fae908c8e81ceb3f4c64b483213035ab Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 15:55:33 -0400 Subject: [PATCH 041/117] Add runtime-weighted memory insurance Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/MissionParallelCatchup/job_monitor.py | 37 ++++++++++++++++--- .../templates/job_monitor.yaml | 2 + .../parallel_catchup_helm/values.yaml | 4 ++ .../tests/contract/test_chart_defaults.py | 1 + .../tests/unit/test_cpu_tiers.py | 5 +++ .../tests/unit/test_resources.py | 37 ++++++++++++++++++- 6 files changed, 80 insertions(+), 6 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index f7cdfba5..ae6bac57 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -19,6 +19,7 @@ import bisect import json import logging +import math import os import queue import re @@ -128,6 +129,9 @@ # slack for all growth and cache -- and 90 of them OOMKilled within 90s. The # earlier 4Gi validation hid this because 1.1x of 2.4 GiB is 240 MiB of slack. PROFILE_CACHE_HEADROOM = os.getenv('PROFILE_CACHE_HEADROOM', '512Mi') +# Extra allowance scaled by the range's measured runtime. Long ranges keep more +# page cache and allocator slack live at once; 0 disables the allowance. +PROFILE_RUNTIME_MEMORY_INSURANCE = os.getenv('PROFILE_RUNTIME_MEMORY_INSURANCE', '3Gi') REQ_EPHEMERAL = os.getenv('REQ_EPHEMERAL', '') LIM_EPHEMERAL = os.getenv('LIM_EPHEMERAL', '') @@ -1857,15 +1861,35 @@ def _sized(value, margin, cap): _SORTED_SECONDS = None +def _positive_seconds(value): + """A finite positive runtime, or None when the profile cannot supply one.""" + try: + seconds = float(value) + except (TypeError, ValueError): + return None + return seconds if math.isfinite(seconds) and seconds > 0 else None + + def _profile_seconds(): - """Every measured runtime in the profile, sorted, for percentile lookup.""" + """Every valid measured runtime in the profile, sorted.""" global _SORTED_SECONDS if _SORTED_SECONDS is None: - _SORTED_SECONDS = sorted(r['seconds'] for _, r in (PROFILE or []) - if r.get('seconds')) + values = (_positive_seconds(r.get('seconds')) for _, r in (PROFILE or [])) + _SORTED_SECONDS = sorted(seconds for seconds in values if seconds is not None) return _SORTED_SECONDS +def _runtime_memory_insurance(seconds): + """Runtime-weighted share of the configured memory allowance.""" + seconds = _positive_seconds(seconds) + everything = _profile_seconds() + longest = everything[-1] if everything else None + insurance = _quantity_bytes(PROFILE_RUNTIME_MEMORY_INSURANCE) + if seconds is None or longest is None or longest <= 0 or insurance <= 0: + return 0 + return int(insurance * (seconds / longest)) + + def _slack_cpu(seconds): """Tier for a range, by its rank among all profiled runtimes. @@ -1881,7 +1905,8 @@ def _slack_cpu(seconds): return None if not tiers: return None - if not seconds: + seconds = _positive_seconds(seconds) + if seconds is None: return tiers[-1][1] everything = _profile_seconds() if not everything: @@ -1911,7 +1936,9 @@ def _profile_overrides(end, escalated): # tracked anon still sizes exactly as it used to. rss = prof.get('peakAnonBytes') or prof.get('peakRssBytes') if rss: - want = int(rss * PROFILE_MARGIN) + _quantity_bytes(PROFILE_CACHE_HEADROOM) + want = (int(rss * PROFILE_MARGIN) + + _quantity_bytes(PROFILE_CACHE_HEADROOM) + + _runtime_memory_insurance(prof.get('seconds'))) out['memory'] = _bytes_to_quantity(min(want, _quantity_bytes(PROFILE_MAX_MEM))) disk = prof.get('peakEphemeralBytes') if disk and LIM_EPHEMERAL: diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index b52edd13..9b990b1d 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -246,6 +246,8 @@ spec: value: {{ .Values.monitor.profileMaxMemory | quote }} - name: PROFILE_CACHE_HEADROOM value: {{ .Values.monitor.profileCacheHeadroom | quote }} + - name: PROFILE_RUNTIME_MEMORY_INSURANCE + value: {{ .Values.monitor.profileRuntimeMemoryInsurance | quote }} {{- end }} # Failed ranges are always saved; successful ones are the bulk of # the volume and can be turned off for a cheap run. diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 15f66804..8a9bbfd2 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -97,6 +97,10 @@ monitor: # profiled at 190MiB rss got a 209MiB limit and 90 of them OOMKilled within # 90s of dispatch. profileCacheHeadroom: "512Mi" + # Extra memory allowance weighted by a range's runtime relative to the + # longest valid runtime in the profile. Long-running ranges need more slack + # for page cache and allocator growth. 0 disables. + profileRuntimeMemoryInsurance: "3Gi" imagePullPolicy: IfNotPresent mission: "HistoryPubnetParallelCatchup" # Adds a `mission` label to worker pods, which kube-state-metrics exposes as diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py index 36f2c8c6..3d483871 100644 --- a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py +++ b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py @@ -193,6 +193,7 @@ def test_the_sizing_headroom_is_a_real_allowance_in_both_places(): headroom = jm._quantity_bytes(code['PROFILE_CACHE_HEADROOM']) assert headroom >= 256 * 1024 ** 2, ( f"{code['PROFILE_CACHE_HEADROOM']} of fixed headroom is what OOMed 90 small ranges") + assert code['PROFILE_RUNTIME_MEMORY_INSURANCE'] == '3Gi' # ...and the ceiling has to sit above the configured worker limit, or a # range needing more than that is pinned under its own measured peak. assert (jm._quantity_bytes(code['PROFILE_MAX_MEM']) diff --git a/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py b/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py index 1591c58e..26ae55fe 100644 --- a/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py +++ b/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py @@ -50,6 +50,11 @@ def test_an_unmeasured_range_gets_the_top_tier(tiered): assert jm._slack_cpu(None) == '1.25' +@pytest.mark.parametrize('seconds', [0, -1, 'bad', float('nan'), float('inf')]) +def test_an_invalid_runtime_safely_gets_the_top_tier(tiered, seconds): + assert jm._slack_cpu(seconds) == '1.25' + + def test_a_uniformly_slower_run_assigns_the_same_tiers(monkeypatch): """The property absolute-seconds keying does not have. diff --git a/src/MissionParallelCatchup/tests/unit/test_resources.py b/src/MissionParallelCatchup/tests/unit/test_resources.py index e2a605f0..4e3962bf 100644 --- a/src/MissionParallelCatchup/tests/unit/test_resources.py +++ b/src/MissionParallelCatchup/tests/unit/test_resources.py @@ -27,11 +27,13 @@ def sizing(monkeypatch): """The worker's configured shape, plus a loaded profile.""" def configure(ranges=PROFILE_RANGES, margin=1.1, lim_mem='24000Mi', req_eph='35Gi', lim_eph='40Gi', max_mem='32Gi', - headroom='512Mi', cpu_limit=''): + headroom='512Mi', runtime_insurance='3Gi', cpu_limit=''): monkeypatch.setattr(jm, 'PROFILE', sorted(ranges)) + monkeypatch.setattr(jm, '_SORTED_SECONDS', None) monkeypatch.setattr(jm, 'PROFILE_MARGIN', margin) monkeypatch.setattr(jm, 'PROFILE_MAX_MEM', max_mem) monkeypatch.setattr(jm, 'PROFILE_CACHE_HEADROOM', headroom) + monkeypatch.setattr(jm, 'PROFILE_RUNTIME_MEMORY_INSURANCE', runtime_insurance) monkeypatch.setattr(jm, 'PROFILE_CPU_LIMIT', cpu_limit) monkeypatch.setattr(jm, 'REQ_CPU', '1800m') monkeypatch.setattr(jm, 'LIM_CPU', '2') @@ -130,6 +132,39 @@ def test_the_sizing_formula_is_peak_times_margin_plus_headroom(sizing, peak_mi): assert got == f"{int(peak_mi * MI * 1.15) // MI + 512}Mi" +def test_runtime_insurance_is_weighted_by_the_longest_profiled_range(sizing): + sizing(ranges=[ + (1, {'peakAnonBytes': 1024 * MI, 'seconds': 100}), + (2, {'peakAnonBytes': 1024 * MI, 'seconds': 400}), + ], margin=1.15, headroom='512Mi', runtime_insurance='3Gi') + + short = jm._quantity_bytes(jm._profile_overrides(1, escalated=False)['memory']) + longest = jm._quantity_bytes(jm._profile_overrides(2, escalated=False)['memory']) + base = int(1024 * MI * 1.15) + 512 * MI + assert short == (base + 768 * MI) // MI * MI + assert longest == (base + 3 * 1024 * MI) // MI * MI + + +@pytest.mark.parametrize('seconds', [None, 0, -1, 'bad', float('nan'), float('inf')]) +def test_invalid_or_nonpositive_runtime_adds_no_insurance(sizing, seconds): + sizing(ranges=[(1, {'peakAnonBytes': 1024 * MI, 'seconds': seconds})], + margin=1.15, headroom='512Mi', runtime_insurance='3Gi') + got = jm._quantity_bytes(jm._profile_overrides(1, escalated=False)['memory']) + assert got == (int(1024 * MI * 1.15) + 512 * MI) // MI * MI + + +def test_zero_runtime_insurance_disables_it_and_the_cap_still_applies_last(sizing): + ranges = [(1, {'peakAnonBytes': 1024 * MI, 'seconds': 100})] + sizing(ranges=ranges, margin=1.15, headroom='512Mi', + runtime_insurance='0', max_mem='2Gi') + without = jm._profile_overrides(1, escalated=False)['memory'] + assert without == f"{int(1024 * MI * 1.15) // MI + 512}Mi" + + sizing(ranges=ranges, margin=1.15, headroom='512Mi', + runtime_insurance='3Gi', max_mem='2Gi') + assert jm._profile_overrides(1, escalated=False)['memory'] == '2048Mi' + + # --- what lands on the container --------------------------------------------- def test_a_measured_range_matches_memory_and_disk_and_leaves_cpu_configured(sizing): From 56b446382a0e92d7b4efe2cc787c9ce66e828448 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 16:43:01 -0400 Subject: [PATCH 042/117] Fix retry counter reconstruction Rebuild retry metrics from authoritative verdicts and durable successor attempt state so active retries and restarts remain accurate. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/MissionParallelCatchup/job_monitor.py | 192 ++++++++++++++---- .../tests/unit/test_retry_counters.py | 159 +++++++++++++++ 2 files changed, 315 insertions(+), 36 deletions(-) create mode 100644 src/MissionParallelCatchup/tests/unit/test_retry_counters.py diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index ae6bac57..a28ff2c0 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -225,6 +225,8 @@ EPH_BUMP_FACTOR = float(os.getenv('EPH_BUMP_FACTOR', 1.5)) EPH_ESCALATION_CAP = os.getenv('EPH_ESCALATION_CAP', '200Gi') ENVIRONMENTAL_OUTCOMES = ('disrupted', 'rejected', 'unknown') +ATTEMPT_OUTCOMES = ('disrupted', 'oom', 'ephemeral', 'timeout', + 'rejected', 'unknown', 'failed') # Verdicts only the pod can produce, and which a Job-level DeadlineExceeded must # never overwrite. Each names a specific mechanism -- the kubelet OOM-killed it, # the node was draining, the ephemeral limit blew -- and each earns a different @@ -412,14 +414,26 @@ def check_storage_config(): 'First dispatch to success, including failed attempts', buckets=metric_buckets) metric_mission_duration = Gauge('ssc_parallel_catchup_mission_duration_seconds', 'Number of seconds since the mission started ') -metric_retries = Counter('ssc_parallel_catchup_job_retried_count', 'Number of jobs that were retried') +metric_retries = Counter( + 'ssc_parallel_catchup_job_retried_count', + 'Retry attempts dispatched after a predecessor attempt failed') # Separates infrastructure churn from application failure: many evictions with # zero app failures is spot behaving as intended. -metric_evictions = Counter('ssc_parallel_catchup_job_spot_eviction_count', 'Pod attempts lost to node disruption') +metric_evictions = Counter( + 'ssc_parallel_catchup_job_spot_eviction_count', + 'Pod attempts classified as lost to node disruption') metric_pvc_released = Counter('ssc_parallel_catchup_pvc_released_count', 'PVCs deleted after their range completed') metric_jobs_reaped = Counter('ssc_parallel_catchup_jobs_reaped_count', 'Finished Jobs deleted after their record was durable') -metric_oom_retries = Counter('ssc_parallel_catchup_job_oom_retried_count', 'Jobs retried with an escalated memory limit') -metric_eph_retries = Counter('ssc_parallel_catchup_job_ephemeral_retried_count', 'Jobs retried with an escalated ephemeral-storage limit') +metric_oom_retries = Counter( + 'ssc_parallel_catchup_job_oom_retried_count', + 'Retry attempts dispatched after an OOM verdict, with an escalated memory limit') +metric_eph_retries = Counter( + 'ssc_parallel_catchup_job_ephemeral_retried_count', + 'Retry attempts dispatched after an ephemeral-storage verdict, with an escalated limit') +metric_retry_reasons = Counter( + 'ssc_parallel_catchup_job_retried_reason_count', + 'Retry attempts dispatched, by the effective verdict of the predecessor attempt', + ['reason']) def _worker_targets(pods): @@ -1225,7 +1239,7 @@ def _oom_count(end, attempt): OOM. That inflation is fleet-wide and it is what exhausts the vCPU quota. """ return sum(1 for n in range(1, int(attempt) + 1) - if (read_outcome(end, n) or {}).get('outcome') == 'oom') + if _verdict_of(end, n) == 'oom') def verdict_path(end, attempt): @@ -1255,11 +1269,13 @@ def save_verdict(end, attempt, outcome): def _verdict_of(end, attempt): try: with open(verdict_path(end, attempt)) as fh: - return fh.read().strip() or None + verdict = fh.read().strip() except OSError: # Pre-fix runs, or an attempt whose verdict write lost the volume: # the pod-derived classification is the next best thing. - return (read_outcome(end, attempt) or {}).get('outcome') + outcome = (read_outcome(end, attempt) or {}).get('outcome') + return outcome if outcome in ATTEMPT_OUTCOMES else None + return verdict if verdict in ATTEMPT_OUTCOMES else None def _cause_count(end, attempt, causes): @@ -2144,44 +2160,144 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): # --- reconcile -------------------------------------------------------------- -def sync_counters(progress, counted): +_ATTEMPT_FILE = re.compile( + r'^range-(?P\d+)-a(?P[1-9]\d*)\.' + r'(?:verdict|outcome|state|metrics|done|log\.gz)$') + + +def _retry_counter_totals(progress, current_attempts=()): + """Reconstruct retry metrics from durable records and observed attempts. + + A verdict says why an attempt ended; it does not say a retry was dispatched. + Attempt N therefore contributes to retry totals only when attempt N+1 is + evidenced by progress, a persisted per-attempt file, or the current Job + snapshot. The latter makes a newly-created successor visible before its range + completes, while the durable sources rebuild the same truth after restart. + """ + try: + names = os.listdir(LOG_DIR) + except OSError: + names = [] + + max_attempt = {} + terminal = set() + + def remember(end, attempt): + try: + attempt = int(attempt) + except (TypeError, ValueError): + return + if attempt < 1: + return + end = str(end) + max_attempt[end] = max(max_attempt.get(end, 0), attempt) + + if isinstance(progress, dict): + for bucket in ('completed', 'failed'): + records = progress.get(bucket) + if not isinstance(records, dict): + continue + for end, record in records.items(): + if not isinstance(record, dict): + continue + try: + attempt = int(record.get('attempts', 1)) + except (TypeError, ValueError): + continue + if attempt < 1: + continue + remember(end, attempt) + terminal.add((str(end), attempt)) + + for item in current_attempts: + try: + end, attempt = item + except (TypeError, ValueError): + continue + remember(end, attempt) + + verdict_files = set() + outcome_files = set() + for name in names: + match = _ATTEMPT_FILE.match(name) + if not match: + continue + key = (match.group('end'), int(match.group('attempt'))) + remember(*key) + if name.endswith('.verdict'): + verdict_files.add(key) + elif name.endswith('.outcome'): + outcome_files.add(key) + + effective = {} + for end, attempt in verdict_files: + try: + with open(verdict_path(end, attempt)) as fh: + verdict = fh.read().strip() + except OSError: + continue + if verdict in ATTEMPT_OUTCOMES: + effective[(end, attempt)] = verdict + + # .outcome predates .verdict. It is safe only for a completed attempt chain: + # a current collector outcome can still be superseded by reconcile's + # effective verdict (notably failed -> timeout). Presence of any verdict file, + # even a malformed one, means this is not a legacy attempt and must never + # fall back to the less-authoritative classification. + for end, attempt in outcome_files - verdict_files: + if attempt >= max_attempt.get(end, 0) and (end, attempt) not in terminal: + continue + try: + with open(outcome_path(end, attempt)) as fh: + record = json.load(fh) + except (OSError, ValueError): + continue + outcome = record.get('outcome') if isinstance(record, dict) else None + if outcome in ATTEMPT_OUTCOMES: + effective[(end, attempt)] = outcome + + retries = sum(max(0, attempt - 1) for attempt in max_attempt.values()) + reasons = {reason: 0 for reason in ATTEMPT_OUTCOMES} + for (end, attempt), reason in effective.items(): + if attempt < max_attempt.get(end, 0): + reasons[reason] += 1 + + return { + 'retries': retries, + 'evicted': sum(1 for verdict in effective.values() if verdict == 'disrupted'), + 'oom': reasons['oom'], + 'ephemeral': reasons['ephemeral'], + 'reasons': reasons, + } + + +def sync_counters(progress, counted, current_attempts=()): """Drive the counters from persisted state instead of from events. Two reasons not to .inc() as things happen: * a terminally-failed range stays the newest Job for its range, so an event-driven inc fires again on every reconcile until teardown - * the process resets to zero on restart, while the underlying record - (attempts in the progress ConfigMap, .outcome files on the PVC) survives + * the process resets to zero on restart, while verdicts and attempt state on + the PVC survive Computing the true total and incrementing by the delta is monotonic, idempotent, and self-heals after a restart: the counter starts at 0 and the first sync walks it up to the recorded total. """ - retries = 0 - for rec in list(progress.get('completed', {}).values()) + list(progress.get('failed', {}).values()): - retries += max(0, int(rec.get('attempts', 1)) - 1) - - oom = evicted = 0 - try: - for name in os.listdir(LOG_DIR): - if not name.endswith('.outcome'): - continue - try: - with open(os.path.join(LOG_DIR, name)) as fh: - o = json.load(fh).get('outcome') - except (OSError, ValueError): - continue - if o == 'oom': - oom += 1 - elif o == 'disrupted': - evicted += 1 - except OSError: - pass - - for key, total, metric in (('retries', retries, metric_retries), - ('oom', oom, metric_oom_retries), - ('evicted', evicted, metric_evictions)): + totals = _retry_counter_totals(progress, current_attempts) + for key, total, metric in (('retries', totals['retries'], metric_retries), + ('oom', totals['oom'], metric_oom_retries), + ('ephemeral', totals['ephemeral'], metric_eph_retries), + ('evicted', totals['evicted'], metric_evictions)): + delta = total - counted.get(key, 0) + if delta > 0: + metric.inc(delta) + counted[key] = total + for reason in ATTEMPT_OUTCOMES: + metric = metric_retry_reasons.labels(reason=reason) + key = ('reason', reason) + total = totals['reasons'][reason] delta = total - counted.get(key, 0) if delta > 0: metric.inc(delta) @@ -2244,9 +2360,11 @@ def reconcile(state): job_pods = pods_by_job() live = {} # range-end -> (attempt, job) + current_attempts = set() for j in jobs: end = (j.metadata.labels or {}).get(LABEL_RANGE) attempt = int((j.metadata.labels or {}).get(LABEL_ATTEMPT, 1)) + current_attempts.add((str(end), attempt)) prev = live.get(end) if prev is None or attempt >= prev[0]: live[end] = (attempt, j) @@ -2487,7 +2605,6 @@ def reconcile(state): "memory limit %s -- RAISE THE CONFIGURED MEMORY LIMIT, this run is only " "surviving by escalating at runtime", end, attempt, MAX_ATTEMPTS_PER_RANGE, retry_mem) elif verdict['outcome'] == 'ephemeral': - metric_eph_retries.inc() logger.error( "!!! DISK RETRY !!! range %s %s on attempt %d/%d; retrying with " "ephemeral-storage %s -- RAISE THE CONFIGURED EPHEMERAL STORAGE, this " @@ -2502,6 +2619,7 @@ def reconcile(state): except ApiException as e: if e.status != 409: raise + current_attempts.add((str(end), attempt + 1)) # After the successor exists, never before. If the create above # had failed with the predecessor already gone, the range would # have no live Job at all and the next pass would redispatch it @@ -2577,6 +2695,7 @@ def reconcile(state): try: batch_v1.create_namespaced_job(NAMESPACE, build_job( end, count, 1, state['owner'])) + current_attempts.add((str(end), 1)) created += 1 capacity -= 1 in_progress.append(job_key(end, count)) @@ -2584,6 +2703,7 @@ def reconcile(state): except ApiException as e: if e.status != 409: # AlreadyExists: name uniqueness is the mutex raise + current_attempts.add((str(end), 1)) # Losing the mutex means the Job EXISTS and is in flight, so it # occupies a slot exactly like one we created. Falling through # without spending capacity dispatched PARALLELISM+1 workers -- @@ -2594,7 +2714,7 @@ def reconcile(state): in_flight.add(str(end)) observe_recorded(progress, state['replayed']) - sync_counters(progress, state['counted']) + sync_counters(progress, state['counted'], current_attempts) return { 'total': len(ranges), 'completed': len(completed), diff --git a/src/MissionParallelCatchup/tests/unit/test_retry_counters.py b/src/MissionParallelCatchup/tests/unit/test_retry_counters.py new file mode 100644 index 00000000..7c172847 --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_retry_counters.py @@ -0,0 +1,159 @@ +"""Retry counters reconstructed from durable attempt state.""" + +import json + +from prometheus_client import generate_latest + +import job_monitor as jm + + +def _write(path, value): + with open(path, 'w') as fh: + if isinstance(value, dict): + json.dump(value, fh) + else: + fh.write(value) + + +def _totals(progress=None, current_attempts=()): + return jm._retry_counter_totals(progress or {}, current_attempts) + + +def test_verdict_is_preferred_over_outcome(logdir): + _write(jm.outcome_path(100, 1), {'outcome': 'disrupted'}) + _write(jm.verdict_path(100, 1), 'oom') + + totals = _totals(current_attempts={('100', 2)}) + + assert totals['retries'] == 1 + assert totals['reasons']['oom'] == 1 + assert totals['reasons']['disrupted'] == 0 + assert totals['oom'] == 1 + assert totals['evicted'] == 0 + + +def test_legacy_outcome_is_used_when_no_verdict_exists(logdir): + _write(jm.outcome_path(100, 1), {'outcome': 'disrupted'}) + + totals = _totals(current_attempts={('100', 2)}) + + assert totals['reasons']['disrupted'] == 1 + assert totals['evicted'] == 1 + + +def test_matching_verdict_and_outcome_are_counted_once(logdir): + _write(jm.outcome_path(100, 1), {'outcome': 'disrupted'}) + _write(jm.verdict_path(100, 1), 'disrupted') + + totals = _totals(current_attempts={('100', 2)}) + + assert totals['reasons']['disrupted'] == 1 + assert totals['evicted'] == 1 + + +def test_active_successor_counts_before_range_progress_exists(logdir): + _write(jm.verdict_path(100, 1), 'rejected') + + totals = _totals(current_attempts={('100', 1), ('100', 2)}) + + assert totals['retries'] == 1 + assert totals['reasons']['rejected'] == 1 + + +def test_reconcile_counts_the_successor_on_its_dispatch_pass(cluster): + cluster.reconcile() + cluster.advance(300, 'disrupted') + + cluster.reconcile() + + assert cluster.attempt_of(300) == 2 + assert cluster.state['counted']['retries'] == 1 + assert cluster.state['counted'][('reason', 'disrupted')] == 1 + assert not cluster.completed() + assert not cluster.failed() + + +def test_terminal_verdict_without_successor_is_not_a_retry(logdir): + _write(jm.verdict_path(100, 1), 'oom') + + totals = _totals( + {'failed': {'100': {'attempts': 1, 'outcome': 'oom'}}}, + current_attempts={('100', 1)}) + + assert totals['retries'] == 0 + assert totals['oom'] == 0 + assert totals['reasons']['oom'] == 0 + + +def test_counter_sync_is_idempotent_and_replays_after_restart(logdir): + _write(jm.verdict_path(100, 1), 'disrupted') + attempts = {('100', 2)} + retry_before = jm.metric_retries._value.get() + eviction_before = jm.metric_evictions._value.get() + reason_metric = jm.metric_retry_reasons.labels(reason='disrupted') + reason_before = reason_metric._value.get() + + counted = {} + jm.sync_counters({}, counted, attempts) + first = (jm.metric_retries._value.get(), + jm.metric_evictions._value.get(), + reason_metric._value.get()) + jm.sync_counters({}, counted, attempts) + assert (jm.metric_retries._value.get(), + jm.metric_evictions._value.get(), + reason_metric._value.get()) == first + + jm.sync_counters({}, {}, attempts) + assert jm.metric_retries._value.get() == retry_before + 2 + assert jm.metric_evictions._value.get() == eviction_before + 2 + assert reason_metric._value.get() == reason_before + 2 + + +def test_multiple_attempts_and_every_retry_reason(logdir): + for attempt, reason in enumerate(jm.ATTEMPT_OUTCOMES, 1): + _write(jm.verdict_path(100, attempt), reason) + + totals = _totals({'completed': {'100': { + 'attempts': len(jm.ATTEMPT_OUTCOMES) + 1}}}) + + assert totals['retries'] == len(jm.ATTEMPT_OUTCOMES) + assert totals['reasons'] == {reason: 1 for reason in jm.ATTEMPT_OUTCOMES} + assert totals['evicted'] == 1 + assert totals['oom'] == 1 + assert totals['ephemeral'] == 1 + + +def test_malformed_and_missing_records_do_not_invent_reasons(logdir): + _write(jm.outcome_path(100, 1), {'outcome': 'disrupted'}) + _write(jm.verdict_path(100, 1), 'not-a-verdict') + _write(jm.outcome_path(200, 1), 'not-json') + _write(logdir / 'range-300-a1.verdict.tmp', 'oom') + _write(logdir / 'unrelated', 'disrupted') + + totals = _totals( + {'completed': 'malformed', 'failed': {'x': {'attempts': 'bad'}}}, + current_attempts={('100', 2), ('200', 2), ('bad', 'attempt')}) + + assert totals['retries'] == 2 + assert sum(totals['reasons'].values()) == 0 + assert totals['evicted'] == 0 + assert totals['oom'] == 0 + + +def test_existing_and_reason_labelled_metrics_are_exported(): + for reason in jm.ATTEMPT_OUTCOMES: + jm.metric_retry_reasons.labels(reason=reason) + text = generate_latest().decode() + + assert '# HELP ssc_parallel_catchup_job_retried_count_total ' \ + 'Retry attempts dispatched after a predecessor attempt failed' in text + assert '# HELP ssc_parallel_catchup_job_spot_eviction_count_total ' \ + 'Pod attempts classified as lost to node disruption' in text + assert '# HELP ssc_parallel_catchup_job_oom_retried_count_total ' \ + 'Retry attempts dispatched after an OOM verdict, with an escalated memory limit' in text + assert '# HELP ssc_parallel_catchup_job_ephemeral_retried_count_total ' \ + 'Retry attempts dispatched after an ephemeral-storage verdict, with an escalated limit' in text + assert '# HELP ssc_parallel_catchup_job_retried_reason_count_total ' \ + 'Retry attempts dispatched, by the effective verdict of the predecessor attempt' in text + for reason in jm.ATTEMPT_OUTCOMES: + assert f'ssc_parallel_catchup_job_retried_reason_count_total{{reason="{reason}"}}' in text From 7967edd2890cd6ba0f9c3d7b2c6ab2c631ed0c58 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 16:45:20 -0400 Subject: [PATCH 043/117] Count unique disruption-retried ranges Expose a dedicated counter keyed by stable ledger range while retaining the raw disrupted-attempt metric. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/MissionParallelCatchup/job_monitor.py | 13 +++++- .../tests/unit/test_retry_counters.py | 46 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index a28ff2c0..b3e86350 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -422,6 +422,9 @@ def check_storage_config(): metric_evictions = Counter( 'ssc_parallel_catchup_job_spot_eviction_count', 'Pod attempts classified as lost to node disruption') +metric_spot_disruption_retried = Counter( + 'ssc_parallel_catchup_job_spot_disruption_retried_count', + 'Unique ledger ranges that dispatched a successor after a node disruption verdict') metric_pvc_released = Counter('ssc_parallel_catchup_pvc_released_count', 'PVCs deleted after their range completed') metric_jobs_reaped = Counter('ssc_parallel_catchup_jobs_reaped_count', 'Finished Jobs deleted after their record was durable') metric_oom_retries = Counter( @@ -2261,10 +2264,15 @@ def remember(end, attempt): for (end, attempt), reason in effective.items(): if attempt < max_attempt.get(end, 0): reasons[reason] += 1 + disruption_retried_ranges = { + end for (end, attempt), reason in effective.items() + if reason == 'disrupted' and attempt < max_attempt.get(end, 0) + } return { 'retries': retries, 'evicted': sum(1 for verdict in effective.values() if verdict == 'disrupted'), + 'spot_disruption_retried': len(disruption_retried_ranges), 'oom': reasons['oom'], 'ephemeral': reasons['ephemeral'], 'reasons': reasons, @@ -2289,7 +2297,10 @@ def sync_counters(progress, counted, current_attempts=()): for key, total, metric in (('retries', totals['retries'], metric_retries), ('oom', totals['oom'], metric_oom_retries), ('ephemeral', totals['ephemeral'], metric_eph_retries), - ('evicted', totals['evicted'], metric_evictions)): + ('evicted', totals['evicted'], metric_evictions), + ('spot_disruption_retried', + totals['spot_disruption_retried'], + metric_spot_disruption_retried)): delta = total - counted.get(key, 0) if delta > 0: metric.inc(delta) diff --git a/src/MissionParallelCatchup/tests/unit/test_retry_counters.py b/src/MissionParallelCatchup/tests/unit/test_retry_counters.py index 7c172847..481eb8d3 100644 --- a/src/MissionParallelCatchup/tests/unit/test_retry_counters.py +++ b/src/MissionParallelCatchup/tests/unit/test_retry_counters.py @@ -39,6 +39,7 @@ def test_legacy_outcome_is_used_when_no_verdict_exists(logdir): assert totals['reasons']['disrupted'] == 1 assert totals['evicted'] == 1 + assert totals['spot_disruption_retried'] == 1 def test_matching_verdict_and_outcome_are_counted_once(logdir): @@ -49,6 +50,30 @@ def test_matching_verdict_and_outcome_are_counted_once(logdir): assert totals['reasons']['disrupted'] == 1 assert totals['evicted'] == 1 + assert totals['spot_disruption_retried'] == 1 + + +def test_repeated_disruptions_of_one_range_count_as_one_retried_range(logdir): + for attempt in (1, 2, 3): + _write(jm.verdict_path(100, attempt), 'disrupted') + + totals = _totals(current_attempts={('100', 4)}) + + assert totals['retries'] == 3 + assert totals['evicted'] == 3 + assert totals['reasons']['disrupted'] == 3 + assert totals['spot_disruption_retried'] == 1 + + +def test_disruptions_of_distinct_ranges_each_count_once(logdir): + for end in (100, 200): + _write(jm.outcome_path(end, 1), {'outcome': 'disrupted'}) + _write(jm.verdict_path(end, 1), 'disrupted') + + totals = _totals(current_attempts={('100', 2), ('200', 2)}) + + assert totals['evicted'] == 2 + assert totals['spot_disruption_retried'] == 2 def test_active_successor_counts_before_range_progress_exists(logdir): @@ -68,6 +93,7 @@ def test_reconcile_counts_the_successor_on_its_dispatch_pass(cluster): assert cluster.attempt_of(300) == 2 assert cluster.state['counted']['retries'] == 1 + assert cluster.state['counted']['spot_disruption_retried'] == 1 assert cluster.state['counted'][('reason', 'disrupted')] == 1 assert not cluster.completed() assert not cluster.failed() @@ -85,11 +111,24 @@ def test_terminal_verdict_without_successor_is_not_a_retry(logdir): assert totals['reasons']['oom'] == 0 +def test_terminal_disruption_without_successor_is_only_a_raw_attempt(logdir): + _write(jm.verdict_path(100, 1), 'disrupted') + + totals = _totals( + {'failed': {'100': {'attempts': 1, 'outcome': 'disrupted'}}}, + current_attempts={('100', 1)}) + + assert totals['evicted'] == 1 + assert totals['spot_disruption_retried'] == 0 + assert totals['reasons']['disrupted'] == 0 + + def test_counter_sync_is_idempotent_and_replays_after_restart(logdir): _write(jm.verdict_path(100, 1), 'disrupted') attempts = {('100', 2)} retry_before = jm.metric_retries._value.get() eviction_before = jm.metric_evictions._value.get() + unique_before = jm.metric_spot_disruption_retried._value.get() reason_metric = jm.metric_retry_reasons.labels(reason='disrupted') reason_before = reason_metric._value.get() @@ -97,15 +136,18 @@ def test_counter_sync_is_idempotent_and_replays_after_restart(logdir): jm.sync_counters({}, counted, attempts) first = (jm.metric_retries._value.get(), jm.metric_evictions._value.get(), + jm.metric_spot_disruption_retried._value.get(), reason_metric._value.get()) jm.sync_counters({}, counted, attempts) assert (jm.metric_retries._value.get(), jm.metric_evictions._value.get(), + jm.metric_spot_disruption_retried._value.get(), reason_metric._value.get()) == first jm.sync_counters({}, {}, attempts) assert jm.metric_retries._value.get() == retry_before + 2 assert jm.metric_evictions._value.get() == eviction_before + 2 + assert jm.metric_spot_disruption_retried._value.get() == unique_before + 2 assert reason_metric._value.get() == reason_before + 2 @@ -119,6 +161,7 @@ def test_multiple_attempts_and_every_retry_reason(logdir): assert totals['retries'] == len(jm.ATTEMPT_OUTCOMES) assert totals['reasons'] == {reason: 1 for reason in jm.ATTEMPT_OUTCOMES} assert totals['evicted'] == 1 + assert totals['spot_disruption_retried'] == 1 assert totals['oom'] == 1 assert totals['ephemeral'] == 1 @@ -137,6 +180,7 @@ def test_malformed_and_missing_records_do_not_invent_reasons(logdir): assert totals['retries'] == 2 assert sum(totals['reasons'].values()) == 0 assert totals['evicted'] == 0 + assert totals['spot_disruption_retried'] == 0 assert totals['oom'] == 0 @@ -149,6 +193,8 @@ def test_existing_and_reason_labelled_metrics_are_exported(): 'Retry attempts dispatched after a predecessor attempt failed' in text assert '# HELP ssc_parallel_catchup_job_spot_eviction_count_total ' \ 'Pod attempts classified as lost to node disruption' in text + assert '# HELP ssc_parallel_catchup_job_spot_disruption_retried_count_total ' \ + 'Unique ledger ranges that dispatched a successor after a node disruption verdict' in text assert '# HELP ssc_parallel_catchup_job_oom_retried_count_total ' \ 'Retry attempts dispatched after an OOM verdict, with an escalated memory limit' in text assert '# HELP ssc_parallel_catchup_job_ephemeral_retried_count_total ' \ From acfcde8369883089e34e3da4282dbcaa74d3d74b Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 16:51:52 -0400 Subject: [PATCH 044/117] Extend parallel catchup attempt deadline Raise the Helm default from three hours to twelve based on the measured Pubnet range tail while retaining a finite hang backstop. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../parallel_catchup_helm/values.yaml | 10 ++++++---- .../tests/contract/test_chart_defaults.py | 5 +++++ .../tests/contract/test_rendered_job_spec.py | 4 ++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 8a9bbfd2..99232e79 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -145,10 +145,12 @@ monitor: # Must exceed any plausible monitor outage: completion is recorded to the # progress ConfigMap by the monitor, and a Job reclaimed before that happens # reads as "never ran" and gets redone. - # 0 = no deadline. A prod range runs ~50 min, so ~3h is generous while still - # catching a range wedged in archive retries. Measured: stellar-core retries a - # missing/unreachable archive indefinitely rather than failing. - attemptDeadlineSeconds: 10800 + # 0 = no deadline. In the 2026-07-30 16,320-ledger Pubnet profile, 793 ranges + # exceeded 3h and the maximum wall time was 21,488s (~6h). A 12h deadline + # leaves ~2x the measured maximum while retaining a backstop for a range wedged + # in archive retries; stellar-core retries a missing/unreachable archive + # indefinitely rather than failing. + attemptDeadlineSeconds: 43200 jobTtlSeconds: 600 # Worker logs are pulled to the monitor's volume while each pod is still # alive, then collected in one exec at teardown instead of ~1024. diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py index 3d483871..7b5e85e8 100644 --- a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py +++ b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py @@ -135,6 +135,11 @@ def test_the_chart_value_is_the_code_default(): + "\n ".join(drift)) +def test_the_chart_enables_a_twelve_hour_attempt_backstop(): + env = art.env_of(art.containers()[art.MONITOR_CONTAINER]) + assert env['ATTEMPT_DEADLINE_SECONDS'] == '43200' + + def test_each_deliberate_divergence_is_still_a_real_env_var(): """Keeps the allowlist above honest. diff --git a/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py b/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py index d907ee12..50635798 100644 --- a/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py +++ b/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py @@ -80,11 +80,11 @@ def test_the_deadline_is_on_the_pod_not_on_the_job(job, monkeypatch): "timeouts" having barely executed; a timeout gets only MAX_TIMEOUT_ATTEMPTS, so two stalls condemn a range and fail the mission. """ - monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 10800) + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 43200) j = jm.build_job(300, 420, 1, None) assert j.spec.active_deadline_seconds is None, \ "the deadline is on the JobSpec, so Pending time is charged to the range" - assert j.spec.template.spec.active_deadline_seconds == 10800 + assert j.spec.template.spec.active_deadline_seconds == 43200 def test_no_deadline_means_no_field_at_all(job, monkeypatch): From 3500dacbda1eed1093015239f9aeb2b01fa42bba Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 16:58:04 -0400 Subject: [PATCH 045/117] Pass --namespace to every helm and kubectl call The mission honours context.namespaceProperty everywhere except the shell. helm and kubectl fall back to the kubeconfig's current context, so a run launched with --namespace sandbox installed its monitor, its Jobs and its PVCs into whatever namespace the kubeconfig happened to point at. Observed 2026-07-30: a 12-range test run explicitly targeted at sandbox put a job-monitor Deployment and four Jobs into the production namespace, beside a live 2096-worker run. It was only noticed because the release was missing from `helm list -n sandbox`. Five call sites, not the three that are obvious: install, get values, the profile ConfigMap create, and BOTH uninstall paths -- the signal-triggered one and the normal cleanup one. The contract test found the fifth after the first four were fixed, which is the whole reason it is a test and not a checklist. Co-Authored-By: Claude Opus 5 --- .../MissionHistoryPubnetParallelCatchupV2.fs | 22 ++++++++++++++++--- .../contract/test_fsharp_driver_contract.py | 19 ++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index fccc0043..b4c19b12 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -102,6 +102,8 @@ let resolveRangeProfile (context: MissionContext) : string option = "create" "configmap" name + "--namespace" + context.namespaceProperty sprintf "--from-file=profile.json=%s" file |] |> ignore @@ -314,10 +316,18 @@ let installProject (context: MissionContext) = let expandedKubeCfg = ExpandHomeDirTilde context.kubeCfg Environment.SetEnvironmentVariable("KUBECONFIG", expandedKubeCfg) + // --namespace is not optional. Without it helm uses the kubeconfig's current + // context, while every other call in this mission honours + // context.namespaceProperty -- so a run explicitly targeted at one namespace + // installs its monitor, Jobs and PVCs into a different one. Observed + // 2026-07-30: a mission run with --namespace sandbox put a monitor and four + // Jobs into the production namespace alongside a live run. RunShellCommand [| "helm" "install" helmReleaseName helmChartPath + "--namespace" + context.namespaceProperty "--values" valuesFilePath "--set" @@ -327,7 +337,9 @@ let installProject (context: MissionContext) = match RunShellCommand [| "helm" "get" "values" - helmReleaseName |] with + helmReleaseName + "--namespace" + context.namespaceProperty |] with | Some valuesOutput -> LogInfo "%s" valuesOutput | _ -> () @@ -663,7 +675,9 @@ let cleanup (signalTriggered: bool) (context: MissionContext) = RunShellCommand [| "helm" "uninstall" - helmReleaseName |] + helmReleaseName + "--namespace" + context.namespaceProperty |] |> ignore else // Normal / legitimate-failure path: pods are still alive through @@ -680,7 +694,9 @@ let cleanup (signalTriggered: bool) (context: MissionContext) = RunShellCommand [| "helm" "uninstall" - helmReleaseName |] + helmReleaseName + "--namespace" + context.namespaceProperty |] |> ignore let mutable cleanupContext : MissionContext option = None diff --git a/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py b/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py index b3461202..db23dc78 100644 --- a/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py +++ b/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py @@ -468,3 +468,22 @@ def test_an_empty_artifact_is_never_written_and_never_fatal(tmp_path, monkeypatc # ...and an artifact that never arrived at all is the same, not an error. monkeypatch.setattr(jm, 'PROFILE_PATH', str(tmp_path / 'absent.json')) assert jm.load_profile() == [] + + +def test_every_helm_and_kubectl_call_is_namespaced(): + """A namespace the mission was told to use must reach the shell too. + + helm and kubectl default to the kubeconfig's current context, while the + mission's own Kubernetes client honours context.namespaceProperty. Without + an explicit --namespace those disagree, and a run targeted at one namespace + installs into another. Measured 2026-07-30: a mission run with + `--namespace sandbox` put a job-monitor Deployment and four Jobs into the + production namespace beside a live 2096-worker run. + """ + fs = art.text(art.FSHARP_PATH) + import re + # Every RunShellCommand array invoking helm or kubectl must carry the flag. + calls = re.findall(r'RunShellCommand \[\|\s*"(?:helm|kubectl)".*?\|\]', fs, re.S) + assert calls, "no helm/kubectl shell calls found -- did the driver change shape?" + missing = [c.split('\n')[0] for c in calls if '"--namespace"' not in c] + assert not missing, f"shell calls without --namespace: {missing}" From 76e1a4018a3a9ac9bec5b521d5f17be49f41f754 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 17:08:42 -0400 Subject: [PATCH 046/117] Bound an attempt by its own profiled cost, not by one global number The deadline exists for ONE failure mode, reproduced on ssc-test 2026-07-30: point stellar-core at an unreachable archive and it retries the bucket download forever. It logs "Missing HAS for ledger N: maybe stale archive", re-selects a different mirror and goes again -- RETRY_A_FEW is per archive, so the budget never exhausts. Measured over 4 minutes: 0 ledgers closed, 9 fetch failures, no give-up wording, no exit, pod still Running. Nothing else stops it, so the deadline is load-bearing. But one number cannot bound it. Runtimes span 190x -- p25 771s, p50 2319s, max 5.9h -- so a 3h deadline killed 941 legitimate ranges, while the 12h that kills none lets a wedged 771s range burn 56x its expected cost before anything notices. Both settings have now been run in production and both were wrong in opposite directions. Take whichever bound is tighter: attemptDeadlineSeconds stays as the ceiling for the unforeseen, and profileDeadlineFactor multiplies a range's own measured runtime for the failure we understand. Backtested against the previous run, 2x/3x/4x each produce ZERO false kills -- measured wall never approached even twice the profile. An unprofiled range keeps the ceiling: it is newer than anything measured, so there is no honest estimate to tighten with and guessing low is the bad direction. Off by default (factor 0 keeps today's behaviour exactly). Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 34 +++++++++- .../templates/job_monitor.yaml | 2 + .../parallel_catchup_helm/values.yaml | 5 ++ .../tests/unit/test_deadline_sizing.py | 62 +++++++++++++++++++ 4 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index b3614896..84bdf44d 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1580,6 +1580,38 @@ def _slack_cpu(seconds): return tiers[-1][1] +# Multiple of a range's own measured runtime to allow before calling it wedged. +# The deadline exists for ONE failure mode, reproduced 2026-07-30: with an +# unreachable archive, stellar-core retries the bucket download forever. It logs +# "Missing HAS for ledger N: maybe stale archive", re-selects a different mirror +# and goes again -- RETRY_A_FEW is per archive, so the budget never exhausts. +# Zero ledgers close, no give-up wording, no exit. Nothing but this kills it. +# +# One number cannot bound that, because runtimes span 190x (p25 771s, max 5.9h). +# A 3h deadline killed 941 legitimate ranges; a 12h one kills none but lets a +# wedged 771s range burn 56x its expected runtime first. So take whichever bound +# is tighter: the configured ceiling for the unforeseen, and a multiple of this +# range's own profiled cost for the failure we know about. Backtested against +# the previous run, 2x/3x/4x would each have produced ZERO false kills -- the +# measured wall never approached even twice the profile. +PROFILE_DEADLINE_FACTOR = float(os.getenv('PROFILE_DEADLINE_FACTOR', 0)) + + +def _attempt_deadline(end): + """Seconds this attempt may run, or None for no bound.""" + ceiling = ATTEMPT_DEADLINE_SECONDS or None + if not PROFILE_DEADLINE_FACTOR: + return ceiling + prof = profile_for(end) or {} + secs = prof.get('seconds') + if not secs: + # Unprofiled means newer than anything measured, so there is no honest + # estimate to tighten with -- fall back to the configured ceiling. + return ceiling + scaled = int(secs * PROFILE_DEADLINE_FACTOR) + return min(scaled, ceiling) if ceiling else scaled + + def _profile_overrides(end, escalated): """Request overrides for this range from the profile, or {} for none. @@ -1783,7 +1815,7 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): # MAX_TIMEOUT_ATTEMPTS, so two stalls condemn a range and # fail the mission. The pod-level field starts at container # start, which is the thing being bounded. - active_deadline_seconds=ATTEMPT_DEADLINE_SECONDS or None, + active_deadline_seconds=_attempt_deadline(end), # IRSA for the S3 history mirror. Without it workers fall # back to the public archive, which throttles at 1024. service_account_name=WORKER_SERVICE_ACCOUNT or None, diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index e524d27c..b1d034ee 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -228,6 +228,8 @@ spec: {{- if .Values.monitor.profileConfigMap }} - name: PROFILE_PATH value: /profile/profile.json + - name: PROFILE_DEADLINE_FACTOR + value: {{ .Values.monitor.profileDeadlineFactor | quote }} - name: PROFILE_CPU_TIERS value: {{ .Values.monitor.profileCpuTiers | quote }} - name: PROFILE_MARGIN diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 4a42dc4f..b2396058 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -77,6 +77,11 @@ monitor: # 3267 of 3859 profiled ranges still fit at 0.5 cores. Empty disables tiering # and every range keeps the configured cpu request. # Paired with profileCpuSlowdown: the measured runtime multiplier per tier. + # Multiple of a range's own profiled runtime to allow before it is called + # wedged, taking whichever is tighter: this or attemptDeadlineSeconds. One + # global number cannot bound runtimes that span 190x -- 3h killed 941 real + # ranges, 12h lets a wedged 13-minute range burn 56x its cost. 0 disables. + profileDeadlineFactor: 0 profileCpuTiers: "" profileMargin: 1.15 # CPU limit for ranges the profile has measured. Above the configured diff --git a/src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py b/src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py new file mode 100644 index 00000000..772edd4a --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py @@ -0,0 +1,62 @@ +"""How long an attempt may run before it is called wedged. + +The deadline exists for ONE failure mode, reproduced on ssc-test 2026-07-30: +with an unreachable archive, stellar-core retries the bucket download forever. +It logs "Missing HAS for ledger N: maybe stale archive", re-selects a different +mirror and goes again -- RETRY_A_FEW is per archive, so the budget never +exhausts. Measured: 0 ledgers closed, 9 fetch failures in 2.5 min, no give-up +wording, no exit. Nothing but this deadline stops it. + +One global number cannot bound it, because runtimes span 190x (p25 771s, max +5.9h): 3h killed 941 legitimate ranges, 12h kills none but lets a wedged 771s +range burn 56x its cost first. So take whichever bound is tighter. +""" + +import pytest + +import job_monitor as jm + +PROFILE = [(100, {'seconds': 600.0}), (200, {'seconds': 10000.0})] + + +@pytest.fixture +def sized(monkeypatch): + monkeypatch.setattr(jm, 'PROFILE', PROFILE) + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 43200) + monkeypatch.setattr(jm, 'PROFILE_DEADLINE_FACTOR', 3.0) + + +def test_disabled_by_default_keeps_the_configured_ceiling(monkeypatch): + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 43200) + monkeypatch.setattr(jm, 'PROFILE_DEADLINE_FACTOR', 0) + assert jm._attempt_deadline(100) == 43200 + + +def test_no_ceiling_and_no_factor_means_no_deadline(monkeypatch): + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 0) + monkeypatch.setattr(jm, 'PROFILE_DEADLINE_FACTOR', 0) + assert jm._attempt_deadline(100) is None + + +def test_a_cheap_range_gets_a_tight_bound_not_the_ceiling(sized): + # 600s x3 = 1800s. Under a 12h ceiling a wedged 10-minute range would + # otherwise burn 72x its cost before anything noticed. + assert jm._attempt_deadline(100) == 1800 + + +def test_an_expensive_range_still_gets_room(sized): + assert jm._attempt_deadline(200) == 30000 # 10000 x 3, under the ceiling + + +def test_the_ceiling_still_wins_when_it_is_tighter(monkeypatch): + monkeypatch.setattr(jm, 'PROFILE', PROFILE) + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 7200) + monkeypatch.setattr(jm, 'PROFILE_DEADLINE_FACTOR', 3.0) + assert jm._attempt_deadline(200) == 7200 # 30000 scaled, 7200 ceiling + + +def test_an_unprofiled_range_falls_back_to_the_ceiling(sized): + # Newer than anything measured, so there is no honest estimate to tighten + # with -- and it is the most expensive kind, so guessing low is the bad + # direction. + assert jm._attempt_deadline(999999) == 43200 From f68df0557106935c5a65ed2a5fa61579c1c4d075 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 17:41:47 -0400 Subject: [PATCH 047/117] Fix durable resume detection in collector Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/MissionParallelCatchup/log_collector.py | 29 +++++++++++++++---- .../resilience/test_collector_restart.py | 29 +++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 496b9b6a..f488fcce 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -35,6 +35,7 @@ import re import ssl import sys +import zlib from datetime import datetime import aiohttp @@ -252,10 +253,20 @@ def feed(self, line): self._left = 0 - - - - +def archive_resumed(end, attempt): + """Recover this attempt's exact resume decision from its durable archive.""" + path = base(end, attempt) + '.log.gz' + try: + with gzip.open(path, 'rt', errors='replace') as fh: + return any(TxApplyScanner.RESUME_MARK in line for line in fh) + except FileNotFoundError: + return False + except (EOFError, gzip.BadGzipFile, zlib.error) as e: + logger.warning("could not read resume decision from %s: %s", path, e) + return False + except OSError as e: + logger.warning("could not open resume archive %s: %s", path, e) + return False def write_metrics(end, attempt, values): @@ -292,6 +303,11 @@ def write_metrics(end, attempt, values): a, b = prior.get(k), values.get(k) if a is not None and b is not None: merged[k] = max(a, b) + # Once any poller or archive read proves that this attempt resumed, a later + # restarted poller cannot un-prove it. In particular, a merge containing + # resumed=False must never lower the durable decision back to fresh. + if prior.get('resumed') is True or values.get('resumed') is True: + merged['resumed'] = True values = merged try: with open(tmp, 'w') as fh: @@ -524,7 +540,10 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): # finalized twice, and the second poller's clock started at the restart. measured['attemptSeconds'] = round( asyncio.get_event_loop().time() - started, 1) - if tx.resumed: + # RESUME is printed before stellar-core starts, so a stream/poller recreated + # later can never see it in memory. The archive is appended durably before + # finalization and is the source of truth when this scanner missed the line. + if tx.resumed or archive_resumed(end, attempt): # Not a peak -- PEAK_FIELDS filters it out of the profile. peaks_for_range # reads it to decide how far back to aggregate: a resumed attempt only # measured the tail of its range, so the attempt before it still counts. diff --git a/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py b/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py index 0985a1f3..ccd69318 100644 --- a/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py +++ b/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py @@ -20,6 +20,7 @@ """ import asyncio +import gzip import json import os @@ -175,6 +176,34 @@ def test_a_write_that_omits_a_peak_leaves_it_alone(vol): assert stored['txApplySeconds'] == 12.5 +def test_resumed_true_is_monotonic_across_restarted_writers(vol): + lc.write_metrics('300', 2, {'resumed': True}) + lc.write_metrics('300', 2, {'resumed': False, 'attemptSeconds': 10.0}) + + assert metrics(300, 2)['resumed'] is True + + +def test_finalize_recovers_resume_after_the_scanner_is_recreated(vol): + """The first poll saw RESUME, then its scanner vanished before finalize.""" + path = lc.base('300', 2) + '.log.gz' + with gzip.open(path, 'wt') as fh: + fh.write('RESUME: local state reached ledger 250; skipping new-db\n') + + finalize('w-300-a2', 300, attempt=2, tx=lc.TxApplyScanner()) + + assert metrics(300, 2)['resumed'] is True + + +def test_finalize_does_not_promote_resume_declined(vol): + path = lc.base('300', 2) + '.log.gz' + with gzip.open(path, 'wt') as fh: + fh.write('RESUME DECLINED: no usable local state; running new-db\n') + + finalize('w-300-a2', 300, attempt=2, tx=lc.TxApplyScanner()) + + assert (metrics(300, 2) or {}).get('resumed') is not True + + def test_peaks_from_different_writes_accumulate_into_one_record(vol): """Each axis is flushed by whoever measured it; the file is the union.""" lc.write_metrics('300', 1, {'peakAnonBytes': 5 * GIB}) From 6e74f108f3021bd734447dffc4f97d1d8426dc85 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 17:41:47 -0400 Subject: [PATCH 048/117] Reconstruct resumed catchup profiles Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/MissionParallelCatchup/job_monitor.py | 178 +++++++++++------- .../tests/unit/test_attempt_chain.py | 141 ++++++++++++++ 2 files changed, 256 insertions(+), 63 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index b3e86350..267f1e48 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -900,32 +900,10 @@ def _rehydrate_from_metrics(progress): completed = progress.get('completed') or {} if not completed: return progress - recovered = 0 - for end, rec in completed.items(): - attempt = int(rec.get('attempts') or 1) - try: - peaks = peaks_for_range(int(end), attempt) - if peaks: - for k, v in peaks.items(): - rec.setdefault(k, v) - if rec.get('txApply') is None: - # Not named `tx`: a source-text test in the suite matches the - # first `tx = tx_apply_for_range(` in this file and means the - # one in reconcile(). - recovered_tx = tx_apply_for_range(int(end), attempt) - if recovered_tx is not None: - rec['txApply'] = recovered_tx - if rec.get('seconds') is None: - secs = seconds_for_range(int(end), attempt) - if secs is not None: - rec['seconds'] = secs - except (OSError, ValueError): - continue - if _has_peaks(rec): - recovered += 1 + recovered = repair_completed_profiles(progress) logger.warning("progress.json was unreadable; recovered state from the ConfigMap " - "mirror and re-read measurements for %d of %d completed ranges " - "from .metrics on the volume", recovered, len(completed)) + "mirror and reconstructed measurements for %d of %d completed ranges " + "from attempt artifacts on the volume", recovered, len(completed)) return progress @@ -1443,14 +1421,37 @@ def _resumed_chain(end, attempt): def _attempt_resumed(end, attempt): """Did this attempt pick up at LCL+1 rather than run new-db? - Recorded by the collector from the worker's own "RESUME: ..." line, which is - the only place that knows -- it depends on what was left on /data, not on - storage mode or attempt number. + Prefer the collector's marker, then recover it from the durable archive for + legacy files and pollers recreated after the early RESUME line. """ try: with open(metrics_path(end, attempt)) as fh: - return bool(json.load(fh).get('resumed')) - except (OSError, ValueError): + if json.load(fh).get('resumed') is True: + return True + except FileNotFoundError: + pass + except ValueError as e: + logger.warning("could not parse resume metrics for range %s attempt %s: %s", + end, attempt, e) + except OSError as e: + logger.warning("could not read resume metrics for range %s attempt %s: %s", + end, attempt, e) + return _archive_resumed(end, attempt) + + +def _archive_resumed(end, attempt): + """Read the exact worker resume decision from concatenated gzip members.""" + path = log_path(end, attempt) + try: + with gzip.open(path, 'rt', errors='replace') as fh: + return any('RESUME: ' in line for line in fh) + except FileNotFoundError: + return False + except (EOFError, gzip.BadGzipFile, zlib.error) as e: + logger.warning("could not read resume decision from %s: %s", path, e) + return False + except OSError as e: + logger.warning("could not open resume archive %s: %s", path, e) return False @@ -1490,25 +1491,80 @@ def seconds_for_range(end, attempt=1, final=None): """ total = None for n in _resumed_chain(end, attempt): - if n == int(attempt): + if n == int(attempt) and final is not None: leg = final else: - # .outcome is authoritative -- the pod's own terminated timestamps. - # It is absent whenever the pod was reaped before the monitor could - # classify it, which is every spot eviction, so fall back to the - # collector's stream-lifetime figure rather than losing the leg. - leg = (read_outcome(end, n) or {}).get('attemptSeconds') - if leg is None: - try: - with open(metrics_path(end, n)) as fh: - leg = json.load(fh).get('attemptSeconds') - except (OSError, ValueError): - leg = None + leg = _attempt_seconds(end, n) if leg is not None: total = leg if total is None else total + leg return total +def _attempt_seconds(end, attempt): + """Best durable duration for one attempt, or None when it was never saved.""" + # .outcome is authoritative -- the pod's own terminated timestamps. It is + # absent whenever the pod was reaped before the monitor could classify it, + # which is every spot eviction, so fall back to the collector's estimate. + leg = (read_outcome(end, attempt) or {}).get('attemptSeconds') + if leg is not None: + return leg + try: + with open(metrics_path(end, attempt)) as fh: + return json.load(fh).get('attemptSeconds') + except (OSError, ValueError): + return None + + +def reconstruct_completed_profile(end, attempt): + """Recompute recoverable profile fields from immutable attempt artifacts. + + Durations and tx-apply totals follow only the continuous resumed chain, so a + fresh retry never double-counts discarded work. Peaks use that chain plus + every attempt that hit a resource ceiling. Missing legs remain missing; the + available legs are combined with the existing <=64-ledger tx-apply overlap. + + Reconstructable: sampled peaks in .metrics, attemptSeconds in .outcome or + .metrics, and txApplySeconds in .metrics or retained .log.gz. Not + reconstructable: wallSeconds, samples that were never persisted, or a + missing tx-apply/duration leg whose process and archive are both gone. + """ + rebuilt = peaks_for_range(end, attempt) + seconds = seconds_for_range(end, attempt) + if seconds is not None: + rebuilt['seconds'] = seconds + tx_apply = tx_apply_for_range(end, attempt) + if tx_apply is not None: + rebuilt['txApply'] = tx_apply + return rebuilt + + +def _apply_profile_reconstruction(record, rebuilt): + """Merge reconstruction without lowering stronger persisted evidence.""" + updates = {} + for key, value in rebuilt.items(): + current = record.get(key) + if current is None or value > current: + updates[key] = value + record.update(updates) + return updates + + +def repair_completed_profiles(progress): + """Apply artifact reconstruction to old completed records idempotently.""" + repaired = 0 + for end, record in (progress.get('completed') or {}).items(): + try: + attempt = int(record.get('attempts') or 1) + rebuilt = reconstruct_completed_profile(end, attempt) + except (TypeError, ValueError): + logger.warning("cannot reconstruct malformed completed record for range %s", end) + continue + updates = _apply_profile_reconstruction(record, rebuilt) + if updates: + repaired += 1 + return repaired + + def _tx_apply_for_attempt(end, attempt=1, pod_name=None): """Final 'ledger.transaction.apply' sum for ONE attempt, in seconds. @@ -2365,6 +2421,16 @@ def reconcile(state): progress = load_progress() completed = progress.setdefault('completed', {}) failed = progress.setdefault('failed', {}) + # One pass per monitor process repairs records completed by an older build, + # including Jobs that were already reaped and therefore never enter the live + # completion branch below. Attempt artifacts are immutable after .done, and + # current completions still use the same helpers directly. + if not state.get('completed_profiles_reconstructed'): + repaired = repair_completed_profiles(progress) + state['completed_profiles_reconstructed'] = True + if repaired: + save_progress(progress) + logger.info("reconstructed profile fields for %d completed ranges", repaired) jobs = batch_v1.list_namespaced_job( NAMESPACE, label_selector=f"{LABEL_RUN}={RUN_NAME}").items @@ -2438,30 +2504,16 @@ def reconcile(state): # Durably recorded first: the record is what makes the volume # and the Job disposable, so it must land before either goes. save_progress(progress) - elif (not _has_peaks(completed[end]) - or completed[end].get('txApply') is None - or not _attempt_finalized(end, attempt)): + else: # Backfill. The record is written the moment the Job flips to # succeeded, which is usually before the collector has finalized - # -- and peaks_for_range has no fallback, unlike tx_apply, which - # reads the archive. Measured on ssc-test: 356 of 356 completed - # ranges carried txApply and 0 carried peakAnonBytes, while 1936 - # .metrics files on the same volume held it. Retry while the Job - # is still here; delete_job below is what ends the chances. - late = peaks_for_range(end, attempt) - if completed[end].get('txApply') is None: - # Same one-shot race as the peaks, and the same fix. The - # collector writes txApplySeconds into .metrics when it - # finalizes, which can land after reconcile recorded the - # range. Measured in the sandbox edge suite 2026-07-30: - # progress.json carried txApply=null while the durable - # .metrics file held txApplySeconds=0.000486848. - late_tx = tx_apply_for_range(end, attempt) - if late_tx is not None: - late = dict(late or {}) - late['txApply'] = late_tx + # -- and the final write can add peaks, txApplySeconds, + # attemptSeconds, and the resume marker together. Reconstruct the + # same profile used on first completion, not a field-by-field + # subset that can leave a pre-marker record permanently short. + late = _apply_profile_reconstruction( + completed[end], reconstruct_completed_profile(end, attempt)) if late: - completed[end].update(late) save_progress(progress) logger.info("range %s: measurements arrived late, backfilled %s", end, sorted(late)) diff --git a/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py b/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py index 464c5fe2..4986838d 100644 --- a/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py +++ b/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py @@ -9,6 +9,8 @@ for exactly the same reason, so both are tail-only in the same way. """ +import gzip +import io import json import pytest @@ -20,6 +22,19 @@ MIB = 1024 ** 2 +def _gzip_member(text): + buf = io.BytesIO() + with gzip.GzipFile(fileobj=buf, mode='wb', mtime=0) as fh: + fh.write(text.encode()) + return buf.getvalue() + + +def _archive(end, attempt, *members): + with open(jm.log_path(end, attempt), 'wb') as fh: + for member in members: + fh.write(_gzip_member(member)) + + @pytest.fixture def attempts(logdir): """Lay down the files the collector and the monitor leave per attempt.""" @@ -49,6 +64,48 @@ def test_an_attempt_with_no_metrics_file_is_not_treated_as_resumed(attempts): assert jm._attempt_resumed(999, 2) is False +def test_resume_falls_back_to_the_archive_when_metrics_lacks_the_field(attempts): + attempts(999, {2: ({'attemptSeconds': 300.0}, None)}) + _archive(999, 2, 'RESUME: reached ledger 900; skipping new-db\n') + + assert jm._attempt_resumed(999, 2) is True + + +def test_resume_declined_is_not_a_true_resume(attempts): + attempts(999, {2: ({}, None)}) + _archive(999, 2, 'RESUME DECLINED: no usable local state; running new-db\n') + + assert jm._attempt_resumed(999, 2) is False + + +def test_resume_is_found_across_concatenated_gzip_members(attempts): + attempts(999, {2: ({}, None)}) + _archive(999, 2, 'worker startup\n', + 'RESUME: reached ledger 900; skipping new-db\n') + + assert jm._attempt_resumed(999, 2) is True + + +def test_missing_truncated_and_corrupt_archives_are_safe(attempts): + attempts(999, {2: ({}, None), 3: ({}, None), 4: ({}, None)}) + with open(jm.log_path(999, 3), 'wb') as fh: + fh.write(_gzip_member('worker startup\n')[:-8]) + with open(jm.log_path(999, 4), 'wb') as fh: + fh.write(b'not a gzip archive') + + assert jm._attempt_resumed(999, 2) is False + assert jm._attempt_resumed(999, 3) is False + assert jm._attempt_resumed(999, 4) is False + + +def test_three_attempt_chain_can_be_recovered_entirely_from_archives(attempts): + attempts(999, {1: ({}, None), 2: ({}, None), 3: ({}, None)}) + _archive(999, 2, 'RESUME: reached ledger 700; skipping new-db\n') + _archive(999, 3, 'RESUME: reached ledger 800; skipping new-db\n') + + assert list(jm._resumed_chain(999, 3)) == [1, 2, 3] + + # --- peaks -------------------------------------------------------------------- def test_a_resumed_range_keeps_the_peak_from_the_attempt_that_did_the_download(attempts): @@ -178,6 +235,90 @@ def test_the_authoritative_outcome_wins_over_the_collector_estimate(attempts): assert jm.seconds_for_range(999, 2, 300.0) == 1200.0 +# --- completed profile reconstruction ----------------------------------------- + +def test_repair_recovers_predecessor_peaks_and_seconds_idempotently(attempts): + attempts(999, { + 1: ({'attemptSeconds': 900.0, 'peakAnonBytes': 2 * GIB, + 'peakWorkingSetBytes': 3 * GIB}, None), + 2: ({'attemptSeconds': 300.0, 'peakAnonBytes': 400 * MIB, + 'peakWorkingSetBytes': 500 * MIB}, None), + }) + _archive(999, 2, 'RESUME: reached ledger 800; skipping new-db\n') + progress = {'completed': {'999': { + 'attempts': 2, 'seconds': 300.0, + 'peakAnonBytes': 400 * MIB, 'peakWorkingSetBytes': 500 * MIB, + }}} + + assert jm.repair_completed_profiles(progress) == 1 + repaired = progress['completed']['999'] + assert repaired['seconds'] == 1200.0 + assert repaired['peakAnonBytes'] == 2 * GIB + assert repaired['peakWorkingSetBytes'] == 3 * GIB + + snapshot = json.loads(json.dumps(progress)) + assert jm.repair_completed_profiles(progress) == 0 + assert progress == snapshot + + +def test_reconstruction_sums_available_txapply_legs_in_a_three_attempt_chain(attempts): + attempts(999, { + 1: ({'txApplySeconds': 10.0}, None), + 2: ({}, None), # this leg's metric was unavailable + 3: ({'txApplySeconds': 3.0}, None), + }) + _archive(999, 2, 'RESUME: reached ledger 700; skipping new-db\n') + _archive(999, 3, 'RESUME: reached ledger 800; skipping new-db\n') + + rebuilt = jm.reconstruct_completed_profile(999, 3) + assert rebuilt['txApply'] == 13.0 + + +def test_reconstruction_leaves_txapply_absent_when_every_leg_is_missing(attempts): + attempts(999, {1: ({}, None), 2: ({}, None)}) + _archive(999, 2, 'RESUME: reached ledger 800; skipping new-db\n') + + assert 'txApply' not in jm.reconstruct_completed_profile(999, 2) + + +def test_reconstruction_does_not_cross_a_fresh_restart_boundary(attempts): + attempts(999, { + 1: ({'attemptSeconds': 900.0, 'txApplySeconds': 100.0, + 'peakAnonBytes': 8 * GIB}, None), + 2: ({'attemptSeconds': 300.0, 'txApplySeconds': 7.0, + 'peakAnonBytes': 900 * MIB}, None), + 3: ({'attemptSeconds': 60.0, 'txApplySeconds': 2.0, + 'peakAnonBytes': 400 * MIB}, None), + }) + _archive(999, 2, 'RESUME DECLINED: running new-db\n') + _archive(999, 3, 'RESUME: reached ledger 950; skipping new-db\n') + + rebuilt = jm.reconstruct_completed_profile(999, 3) + assert rebuilt['seconds'] == 360.0 + assert rebuilt['txApply'] == 9.0 + assert rebuilt['peakAnonBytes'] == 900 * MIB + + +def test_reconcile_repairs_a_completed_record_with_no_live_job(cluster): + cluster.write(jm.PROGRESS_FILE, json.dumps({ + 'completed': {'300': {'attempts': 2, 'count': 100, 'seconds': 300.0, + 'txApply': 2.0, 'peakAnonBytes': 400 * MIB}}, + 'failed': {}, + })) + cluster.finalize(300, 1, tx_apply=10.0, attempt_seconds=900.0, + peaks={'peakAnonBytes': 2 * GIB}) + cluster.finalize(300, 2, tx_apply=2.0, attempt_seconds=300.0, + peaks={'peakAnonBytes': 400 * MIB}) + _archive(300, 2, 'RESUME: reached ledger 250; skipping new-db\n') + + cluster.reconcile() + + repaired = cluster.completed()['300'] + assert repaired['seconds'] == 1200.0 + assert repaired['txApply'] == 12.0 + assert repaired['peakAnonBytes'] == 2 * GIB + + # --- counting causes, not attempts -------------------------------------------- def test_escalation_counts_ooms_not_attempts(attempts): From 01b0a42777a73aad99b007f7e06ed1915b7433a6 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 17:51:15 -0400 Subject: [PATCH 049/117] Harden restart-safe profile finalization Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../MissionHistoryPubnetParallelCatchupV2.fs | 3 + src/MissionParallelCatchup/job_monitor.py | 92 ++++++++++++------ src/MissionParallelCatchup/log_collector.py | 97 +++++++++++++++---- .../test_completed_range_not_redispatched.py | 7 +- .../resilience/test_collector_restart.py | 26 ++++- .../tests/unit/test_attempt_chain.py | 38 ++++++-- .../tests/unit/test_collector_main_loop.py | 3 + .../tests/unit/test_kubelet_sampler.py | 14 +++ .../tests/unit/test_tx_apply.py | 6 +- 9 files changed, 224 insertions(+), 62 deletions(-) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 0d2ddc39..7ea1380b 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -444,6 +444,9 @@ let rangeProfileFields = "peakCpuCores" "peakEphemeralBytes" "seconds" + // Kubernetes startTime -> completionTime for the winning Job only. The + // monitor cannot reconstruct first dispatch -> success after predecessor + // Jobs and their inter-attempt gaps are gone. "wallSeconds" "txApply" ] diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 267f1e48..f0fdd021 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -405,13 +405,13 @@ def check_storage_config(): metric_catchup_queues = Gauge('ssc_parallel_catchup_queues', 'Exposes size of each job queues', ["queue"]) metric_workers = Gauge('ssc_parallel_catchup_workers', 'Exposes catch up worker status', ["status"]) metric_refresh_duration = Gauge('ssc_parallel_catchup_workers_refresh_duration_seconds', 'Time it took to refresh status of all workers') -metric_full_duration = Histogram('ssc_parallel_catchup_job_full_duration_seconds', 'Exposes full job duration as histogram', buckets=metric_buckets) +metric_full_duration = Histogram('ssc_parallel_catchup_job_full_duration_seconds', 'Compute seconds across the complete resumed attempt chain', buckets=metric_buckets) metric_tx_apply_duration = Histogram('ssc_parallel_catchup_job_tx_apply_duration_seconds', 'Exposes job TX apply duration as histogram', buckets=metric_buckets) -# full_duration is the SUCCESSFUL attempt only, matching what worker.sh timed. -# wall_duration spans first dispatch to success, so (wall - full) is exactly the -# work lost to retries -- the cost of running on spot. +# wallSeconds is Kubernetes's startTime -> completionTime for the winning Job +# only. Failed-attempt timestamps and inter-attempt gaps were never persisted, so +# it cannot be reconstructed as first dispatch -> success after those Jobs go. metric_wall_duration = Histogram('ssc_parallel_catchup_job_wall_duration_seconds', - 'First dispatch to success, including failed attempts', + 'Winning Kubernetes Job start to completion', buckets=metric_buckets) metric_mission_duration = Gauge('ssc_parallel_catchup_mission_duration_seconds', 'Number of seconds since the mission started ') metric_retries = Counter( @@ -1456,7 +1456,7 @@ def _archive_resumed(end, attempt): def tx_apply_for_range(end, attempt=1, pod_name=None): - """Total 'ledger.transaction.apply' seconds for the whole range. + """Exact known 'ledger.transaction.apply' seconds for the whole range. Summed across the resumed chain, not read from the winning attempt alone. medida's total is per-process, so a pod that resumes at LCL+1 reports only @@ -1474,8 +1474,12 @@ def tx_apply_for_range(end, attempt=1, pod_name=None): # fallbacks are offered to that one alone; earlier legs come from the # .metrics the collector already wrote. leg = _tx_apply_for_attempt(end, n, pod_name if n == int(attempt) else None) - if leg is not None: - total = leg if total is None else total + leg + if leg is None: + # A disrupted process often never prints its final medida block. + # Publishing the sum of surviving legs as a total silently + # under-reports; absence accurately says the chain is incomplete. + return None + total = leg if total is None else total + leg return total @@ -1486,8 +1490,9 @@ def seconds_for_range(end, attempt=1, final=None): from the pod. Earlier legs come from their .outcome, written when the monitor classified the failure and still had the pod. - This is compute, not elapsed: the gaps between attempts -- scheduling, image - pull, a node coming up -- are not in it. wallSeconds covers those. + This is compute, not elapsed: scheduling, image pull, node startup and gaps + between attempts are not in it. wallSeconds is a separate winner-Job-only + diagnostic; whole-chain elapsed time was never persisted. """ total = None for n in _resumed_chain(end, attempt): @@ -1495,8 +1500,9 @@ def seconds_for_range(end, attempt=1, final=None): leg = final else: leg = _attempt_seconds(end, n) - if leg is not None: - total = leg if total is None else total + leg + if leg is None: + return None + total = leg if total is None else total + leg return total @@ -1510,7 +1516,14 @@ def _attempt_seconds(end, attempt): return leg try: with open(metrics_path(end, attempt)) as fh: - return json.load(fh).get('attemptSeconds') + data = json.load(fh) + # A poller clock starts when that collector process attaches, so after a + # restart it is only a lower bound. Legacy files have no provenance and + # remain usable for best-effort reconstruction; new known estimates do + # not masquerade as complete chain compute. + if data.get('attemptSecondsExact') is False: + return None + return data.get('attemptSeconds') except (OSError, ValueError): return None @@ -1520,13 +1533,14 @@ def reconstruct_completed_profile(end, attempt): Durations and tx-apply totals follow only the continuous resumed chain, so a fresh retry never double-counts discarded work. Peaks use that chain plus - every attempt that hit a resource ceiling. Missing legs remain missing; the - available legs are combined with the existing <=64-ledger tx-apply overlap. - - Reconstructable: sampled peaks in .metrics, attemptSeconds in .outcome or - .metrics, and txApplySeconds in .metrics or retained .log.gz. Not - reconstructable: wallSeconds, samples that were never persisted, or a - missing tx-apply/duration leg whose process and archive are both gone. + every attempt that hit a resource ceiling. Missing duration or tx-apply legs + make that aggregate absent rather than publishing a lower bound as a total. + Complete tx-apply legs retain the existing <=64-ledger overlap. + + Reconstructable: persisted sampled peaks, complete attemptSeconds chains in + .outcome/.metrics, and complete txApplySeconds chains in .metrics/.log.gz. + Not reconstructable: whole-chain wall time, samples never persisted, or a + duration/tx-apply leg whose process and archive are both gone. """ rebuilt = peaks_for_range(end, attempt) seconds = seconds_for_range(end, attempt) @@ -2447,6 +2461,7 @@ def reconcile(state): live[end] = (attempt, j) in_progress = [] + finalizing = [] # The same set of ranges as `in_progress`, keyed by end. `remaining` is a # COUNT over this run's range list, never `total - completed - ...`: the # progress record is read off a shared volume and can carry ends from a run @@ -2478,11 +2493,20 @@ def reconcile(state): if st.start_time and st.completion_time: wall = (st.completion_time - st.start_time).total_seconds() # Chain total, not this leg alone: a range that resumed spent - # real time in the attempts before the winner. Falls back to the - # single leg, then to wall, when nothing durable survived. - seconds = seconds_for_range(end, attempt, seconds) or seconds - if seconds is None: - seconds = wall # pod already gone; wall is the only figure left + # real time in the attempts before the winner. Only a fresh + # single-attempt range may fall back to its winner or Job wall; + # a resumed chain with a missing leg stays absent. + chain = list(_resumed_chain(end, attempt)) + chain_seconds = seconds_for_range(end, attempt, seconds) + # For a fresh single attempt, the winner or Job wall is still a + # useful fallback. For a resumed chain, either every compute leg + # is known or the aggregate is absent -- never winner-only. + if chain_seconds is not None: + seconds = chain_seconds + elif len(chain) == 1: + seconds = seconds if seconds is not None else wall + else: + seconds = None # Not gated on `pod`: the collector's .metrics/.log.gz are # written from the live stream and outlive the pod, so a reaped # node must not cost us the metric. @@ -2531,6 +2555,12 @@ def reconcile(state): # (and its pod, the last place the metric can be read) regardless. release_pvc(end) _reap_if_complete(end, attempt, completed[end]) + if not _attempt_finalized(end, attempt): + # The mission driver writes the final profile as soon as + # jobs_in_progress becomes empty. Keep normal completion open + # until the collector's last atomic metrics write has landed; + # this does not consume dispatch capacity below. + finalizing.append(job_key(int(end), by_end.get(end, 0))) elif st.failed: # Completion is terminal for the range, so a Failed Job for a range # that is already recorded is garbage -- never an input to the retry @@ -2784,6 +2814,7 @@ def reconcile(state): 'failed_ranges': [f"{job_key(int(k), by_end.get(k, 0))}|{v.get('pod', '')}" for k, v in failed.items()], 'in_progress': in_progress, + 'finalizing': finalizing, 'created': created, 'remaining': sum(1 for end, _ in ranges if str(end) not in completed @@ -2850,21 +2881,23 @@ def update_status_and_metrics(): mission_duration = time.time() - mission_start_time with status_lock: + visible_in_progress = r['in_progress'] + r['finalizing'] status = { 'num_remain': r['remaining'], 'queue_remain_count': r['remaining'], 'queue_succeeded_count': r['completed'], 'queue_failed_count': len(r['failed_ranges']), - 'queue_in_progress_count': len(r['in_progress']), + 'queue_in_progress_count': len(visible_in_progress), 'jobs_failed': r['failed_ranges'], - 'jobs_in_progress': r['in_progress'], + 'jobs_in_progress': visible_in_progress, 'workers_refresh_duration': workers_refresh_duration, 'mission_duration': mission_duration, } metric_catchup_queues.labels(queue="remain").set(r['remaining']) metric_catchup_queues.labels(queue="succeeded").set(r['completed']) metric_catchup_queues.labels(queue="failed").set(len(r['failed_ranges'])) - metric_catchup_queues.labels(queue="in_progress").set(len(r['in_progress'])) + metric_catchup_queues.labels(queue="in_progress").set( + len(visible_in_progress)) metric_workers.labels(status="up").set(worker_counts['up']) metric_workers.labels(status="down").set(worker_counts['down']) metric_workers.labels(status="unknown").set(worker_counts['unknown']) @@ -2873,7 +2906,8 @@ def update_status_and_metrics(): logger.info("Status: %s", json.dumps(status)) # Publish on change only -- a 10h run would otherwise issue ~3600 # no-op ConfigMap writes. - counts = (r['remaining'], r['completed'], len(r['failed_ranges']), len(r['in_progress'])) + counts = (r['remaining'], r['completed'], len(r['failed_ranges']), + len(visible_in_progress)) if counts != state.get('last_counts'): state['last_counts'] = counts with status_lock: diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index f488fcce..54180b31 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -232,15 +232,23 @@ class TxApplyScanner: # "RESUME DECLINED", means new-db ran and this attempt did the whole range, # so the colon is load-bearing -- it is what separates the two. RESUME_MARK = 'RESUME: ' + RESUME_DECLINED_MARK = 'RESUME DECLINED:' - def __init__(self): + def __init__(self, recreated=False): self.seconds = None self.resumed = False + self.resume_decided = False + # A new poller starting from durable .state missed every earlier line. + # Finalization must recover scanner-only facts from the archive. + self.recreated = recreated self._left = 0 def feed(self, line): if self.RESUME_MARK in line: self.resumed = True + self.resume_decided = True + elif self.RESUME_DECLINED_MARK in line: + self.resume_decided = True if _TX_METRIC in line: self._left = self.WINDOW return @@ -253,20 +261,27 @@ def feed(self, line): self._left = 0 -def archive_resumed(end, attempt): - """Recover this attempt's exact resume decision from its durable archive.""" +def scan_archive(end, attempt, need_tx=False): + """Recover scanner state from complete gzip members already on disk.""" path = base(end, attempt) + '.log.gz' + scanner = TxApplyScanner() try: with gzip.open(path, 'rt', errors='replace') as fh: - return any(TxApplyScanner.RESUME_MARK in line for line in fh) + for line in fh: + scanner.feed(line) + # The resume decision is at process startup. Avoid decompressing + # a multi-gigabyte worker log when that is all the caller needs. + if scanner.resume_decided and not need_tx: + break except FileNotFoundError: - return False + return scanner except (EOFError, gzip.BadGzipFile, zlib.error) as e: - logger.warning("could not read resume decision from %s: %s", path, e) - return False + # Keep facts found in complete prefix members. A torn final member cannot + # invalidate an earlier RESUME line or complete medida block. + logger.warning("could only partially recover scanner state from %s: %s", path, e) except OSError as e: - logger.warning("could not open resume archive %s: %s", path, e) - return False + logger.warning("could not open scanner archive %s: %s", path, e) + return scanner def write_metrics(end, attempt, values): @@ -308,6 +323,9 @@ def write_metrics(end, attempt, values): # resumed=False must never lower the durable decision back to fresh. if prior.get('resumed') is True or values.get('resumed') is True: merged['resumed'] = True + if (prior.get('attemptSecondsExact') is True + or values.get('attemptSecondsExact') is True): + merged['attemptSecondsExact'] = True values = merged try: with open(tmp, 'w') as fh: @@ -411,13 +429,26 @@ def _flush_peak(name, axis, field, value): write_metrics max-merges on PEAK_KEYS, so re-flushing a lower value later is harmless; the ratio only keeps this to a handful of writes per pod. """ + ref = _streaming.get(name) + if not ref: + return key = name + '/' + axis if value < _peak_flushed.get(key, 0) * PEAK_FLUSH_RATIO: return _peak_flushed[key] = value - ref = _streaming.get(name) - if ref: - write_metrics(ref[0], ref[1], {field: value}) + write_metrics(ref[0], ref[1], {field: value}) + + +def _register_stream(name, end, attempt): + """Register a poller and durably flush peaks sampled just before it opened.""" + _streaming[name] = (end, attempt) + for axis, field, values in ( + ('anon', 'peakAnonBytes', _anon_peak), + ('ws', 'peakWorkingSetBytes', _ws_peak), + ('eph', 'peakEphemeralBytes', _eph_peak)): + value = values.get(name) + if value is not None: + _flush_peak(name, axis, field, value) async def sample_kubelet(session, nodes): @@ -533,6 +564,7 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): if observed is not None: # The pod's own timestamps, not how long this poller happened to watch. measured['attemptSeconds'] = round(observed, 1) + measured['attemptSecondsExact'] = True elif started is not None: # Fallback only: the monitor's figure comes from the pod's terminated # timestamps and is preferred when it exists. write_metrics keeps this @@ -540,16 +572,25 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): # finalized twice, and the second poller's clock started at the restart. measured['attemptSeconds'] = round( asyncio.get_event_loop().time() - started, 1) - # RESUME is printed before stellar-core starts, so a stream/poller recreated - # later can never see it in memory. The archive is appended durably before - # finalization and is the source of truth when this scanner missed the line. - if tx.resumed or archive_resumed(end, attempt): + measured['attemptSecondsExact'] = False + # RESUME is printed before stellar-core starts and medida once at exit, so a + # recreated poller can miss either forever. The archive was appended before + # finalization; recover only the state this scanner could have missed. + archived = None + need_resume = int(attempt) > 1 and not tx.resume_decided + need_tx = tx.recreated and tx.seconds is None + if need_resume or need_tx: + archived = scan_archive(end, attempt, need_tx=need_tx) + if tx.resumed or (archived is not None and archived.resumed): # Not a peak -- PEAK_FIELDS filters it out of the profile. peaks_for_range # reads it to decide how far back to aggregate: a resumed attempt only # measured the tail of its range, so the attempt before it still counts. measured['resumed'] = True - if tx.seconds is not None: - measured['txApplySeconds'] = tx.seconds + tx_seconds = tx.seconds + if tx_seconds is None and archived is not None: + tx_seconds = archived.seconds + if tx_seconds is not None: + measured['txApplySeconds'] = tx_seconds _peak_flushed.pop(pod, None) _peak_flushed.pop(pod + '/eph', None) _streaming.pop(pod, None) @@ -699,7 +740,7 @@ async def poll_pod(session, pod, end, attempt, done, done_ok): started = asyncio.get_event_loop().time() # Outside the poll loop: the medida block can straddle two polls, and a # fresh scanner per poll would lose the half it saw. - tx = TxApplyScanner() + tx = TxApplyScanner(recreated=bool(last_ts)) backoff = LOG_POLL_SECONDS failures = 0 @@ -829,10 +870,24 @@ async def main(): vanished[name] = vanished.get(name, 0) + 1 if vanished[name] >= VANISHED_GRACE_CYCLES: t.cancel() + try: + await t + except asyncio.CancelledError: + pass del tasks[name] vanished.pop(name, None) + ref = _streaming.get(name) + if ref is not None: + # The poller was wedged, but its archive and the + # sampler's process-local peaks still contain useful + # truth. Finalize them before licensing a reap. + await finalize( + session, name, ref[0], ref[1], + TxApplyScanner(recreated=True), + lambda p: succeeded.get(p, False)) streamed.add(name) - logger.info("cancelled stream for vanished pod %s", name) + logger.info("cancelled and finalized stream for vanished pod %s", + name) # Unconditional: this used to be gated on ephemeral mode, back # when it only sampled disk. Memory is sized in both modes, so # gating it here left every pvc run with no anon peak at all. @@ -888,7 +943,7 @@ async def main(): # become pollable. Succeeded and Failed stay in: a # terminal pod is where the final output lives. continue - _streaming[name] = (end, attempt) + _register_stream(name, end, attempt) tasks[name] = asyncio.create_task( poll_pod(session, name, end, attempt, lambda p: terminal.get(p, False), diff --git a/src/MissionParallelCatchup/tests/reconcile/test_completed_range_not_redispatched.py b/src/MissionParallelCatchup/tests/reconcile/test_completed_range_not_redispatched.py index 9136264b..9faca61c 100644 --- a/src/MissionParallelCatchup/tests/reconcile/test_completed_range_not_redispatched.py +++ b/src/MissionParallelCatchup/tests/reconcile/test_completed_range_not_redispatched.py @@ -172,15 +172,18 @@ def test_the_range_scoped_reap_still_waits_for_the_done_marker(cluster): cluster.reconcile() cluster.advance(300, 'succeeded') - cluster.reconcile() # recorded; collector not done + waiting = cluster.reconcile() # recorded; collector not done assert '300' in cluster.completed() assert jobs_for(cluster, 300) == ['pc-r300-a1'] assert cluster.deleted.names(verb='delete', kind='job') == [] + assert waiting['finalizing'] == ['300/420'], \ + "the mission could publish its final profile before metrics landed" cluster.finalize(300, 1, tx_apply=0.1, peaks={'peakRssBytes': 1}) - cluster.reconcile() + finished = cluster.reconcile() assert jobs_for(cluster, 300) == [] assert cluster.deleted.names(verb='delete', kind='job') == ['pc-r300-a1'] + assert finished['finalizing'] == [] def test_remaining_never_goes_negative_and_the_run_reports_done(cluster, diff --git a/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py b/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py index ccd69318..016ce702 100644 --- a/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py +++ b/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py @@ -194,6 +194,25 @@ def test_finalize_recovers_resume_after_the_scanner_is_recreated(vol): assert metrics(300, 2)['resumed'] is True +def test_finalize_recovers_txapply_after_the_scanner_is_recreated(vol, monkeypatch): + """The first poll saw the final medida block, then its scanner vanished.""" + monkeypatch.setattr(lc, 'SAVE_SUCCESS_LOGS', False) + path = lc.base('300', 2) + '.log.gz' + with gzip.open(path, 'wt') as fh: + fh.write('RESUME: local state reached ledger 250; skipping new-db\n') + fh.write("metric 'ledger.transaction.apply'\n") + fh.write(' count = 123\n') + fh.write(' sum = 4200.0ms\n') + + finalize('w-300-a2', 300, attempt=2, succeeded=True, + tx=lc.TxApplyScanner(recreated=True)) + + assert metrics(300, 2)['resumed'] is True + assert metrics(300, 2)['txApplySeconds'] == 4.2 + assert not os.path.exists(path), \ + "the test must prove recovery happened before success-log discard" + + def test_finalize_does_not_promote_resume_declined(vol): path = lc.base('300', 2) + '.log.gz' with gzip.open(path, 'wt') as fh: @@ -409,6 +428,7 @@ def test_finalizing_the_same_attempt_twice_keeps_its_measurements(vol, monkeypat finalize('w-300', 300, tx=tx) first = metrics(300) assert first['attemptSeconds'] == 3600.4 + assert first['attemptSecondsExact'] is True restart(monkeypatch) # The main loop re-reads the pod's own timestamps every cycle it sees it @@ -469,7 +489,10 @@ def test_a_poller_that_watched_the_whole_attempt_still_reports_its_duration(vol) started = _moments_ago() - 42.0 finalize('w-300', 300, started=started) - assert metrics(300)['attemptSeconds'] == pytest.approx(42.0, abs=1.0) + stored = metrics(300) + assert stored['attemptSeconds'] == pytest.approx(42.0, abs=1.0) + assert stored['attemptSecondsExact'] is False + assert jm.seconds_for_range('300', 1) is None def test_the_duration_the_collector_records_is_the_pods_not_the_pollers(vol): @@ -478,6 +501,7 @@ def test_the_duration_the_collector_records_is_the_pods_not_the_pollers(vol): finalize('w-300', 300, started=_moments_ago() - 5.0) assert metrics(300)['attemptSeconds'] == 3600.4 + assert metrics(300)['attemptSecondsExact'] is True # -- .outcome is written once, by whoever got there first --------------------- diff --git a/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py b/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py index 4986838d..679b7425 100644 --- a/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py +++ b/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py @@ -204,11 +204,11 @@ def test_seconds_ignores_attempts_before_a_fresh_start(attempts): assert jm.seconds_for_range(999, 2, 300.0) == 300.0 -def test_seconds_survives_a_leg_with_no_recorded_duration(attempts): - # An attempt whose pod vanished before it was classified has no - # attemptSeconds. Better to under-report one leg than return nothing. +def test_seconds_is_absent_when_a_resumed_leg_has_no_recorded_duration(attempts): + # Winner-only is a lower bound, not the chain total. Missing accurately + # tells the profile consumer not to size from it. attempts(999, {1: ({}, {'outcome': 'disrupted'}), 2: ({'resumed': True}, None)}) - assert jm.seconds_for_range(999, 2, 300.0) == 300.0 + assert jm.seconds_for_range(999, 2, 300.0) is None def test_seconds_is_none_when_nothing_is_known(attempts): @@ -261,7 +261,7 @@ def test_repair_recovers_predecessor_peaks_and_seconds_idempotently(attempts): assert progress == snapshot -def test_reconstruction_sums_available_txapply_legs_in_a_three_attempt_chain(attempts): +def test_reconstruction_omits_txapply_when_one_chain_leg_is_missing(attempts): attempts(999, { 1: ({'txApplySeconds': 10.0}, None), 2: ({}, None), # this leg's metric was unavailable @@ -271,7 +271,7 @@ def test_reconstruction_sums_available_txapply_legs_in_a_three_attempt_chain(att _archive(999, 3, 'RESUME: reached ledger 800; skipping new-db\n') rebuilt = jm.reconstruct_completed_profile(999, 3) - assert rebuilt['txApply'] == 13.0 + assert 'txApply' not in rebuilt def test_reconstruction_leaves_txapply_absent_when_every_leg_is_missing(attempts): @@ -319,6 +319,32 @@ def test_reconcile_repairs_a_completed_record_with_no_live_job(cluster): assert repaired['peakAnonBytes'] == 2 * GIB +def test_wall_seconds_is_winner_job_only_while_compute_spans_the_chain(cluster): + cluster.reconcile() + cluster.advance(300, 'disrupted') + cluster.finalize(300, 1, tx_apply=10.0, attempt_seconds=60.0) + cluster.reconcile() + + cluster.advance(300, 'succeeded', attempt=2) + cluster.finalize(300, 2, tx_apply=2.0, attempt_seconds=60.0, resumed=True) + cluster.reconcile() + + record = cluster.completed()['300'] + assert record['seconds'] == 120.0 + assert record['txApply'] == 12.0 + assert record['wallSeconds'] == 60.0 + + +def test_disrupted_predecessor_without_final_medida_makes_txapply_absent(attempts): + attempts(999, { + 1: ({'attemptSeconds': 900.0}, {'outcome': 'disrupted'}), + 2: ({'resumed': True, 'txApplySeconds': 3.0}, None), + }) + + assert jm.tx_apply_for_range(999, 2) is None + assert 'txApply' not in jm.reconstruct_completed_profile(999, 2) + + # --- counting causes, not attempts -------------------------------------------- def test_escalation_counts_ooms_not_attempts(attempts): diff --git a/src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py b/src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py index d3fbb6db..f3583447 100644 --- a/src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py +++ b/src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py @@ -14,6 +14,7 @@ """ import asyncio +import os import pytest @@ -249,6 +250,8 @@ def test_a_stream_that_will_not_finish_is_cancelled_after_the_grace(loop_env): alive = asyncio.run(_drive(loop, extra=3, want_survivors=True)) assert alive == [], "a wedged stream outlived its grace and held its slot" + assert os.path.exists(lc.done_path('300', '1')), \ + "forced cancellation skipped finalization and never licensed cleanup" def test_the_grace_is_more_than_one_cycle(): diff --git a/src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py b/src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py index 42ac3187..8ee93bef 100644 --- a/src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py +++ b/src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py @@ -189,6 +189,20 @@ def test_an_in_flight_peak_reaches_the_volume_before_the_stream_ends(sampler): assert jm.peaks_for_range('300', 1) == {'peakAnonBytes': 900 * MIB} +def test_a_peak_sampled_before_stream_registration_is_flushed_on_open(sampler): + """main samples first, then opens new pollers; a restart between those steps + must not make that first high-water process-memory-only.""" + sample(payload('w-1', [container(rss=900 * MIB, ws=1200 * MIB)])) + assert jm.peaks_for_range('300', 1) == {} + + lc._register_stream('w-1', '300', '1') + + assert jm.peaks_for_range('300', 1) == { + 'peakAnonBytes': 900 * MIB, + 'peakWorkingSetBytes': 1200 * MIB, + } + + def test_the_disk_axis_stays_mode_gated(sampler, monkeypatch): """ephemeral-storage is meaningless in pvc mode: /data is on the volume, not on the node.""" diff --git a/src/MissionParallelCatchup/tests/unit/test_tx_apply.py b/src/MissionParallelCatchup/tests/unit/test_tx_apply.py index 046ca8ec..03474790 100644 --- a/src/MissionParallelCatchup/tests/unit/test_tx_apply.py +++ b/src/MissionParallelCatchup/tests/unit/test_tx_apply.py @@ -206,7 +206,7 @@ def test_a_fresh_start_drops_the_earlier_legs_from_the_total(logdir): assert jm.tx_apply_for_range(4000, 2) == 5.0 -def test_only_the_last_leg_may_fall_back_to_the_pod(logdir, monkeypatch): +def test_the_winner_pod_fallback_cannot_fill_a_missing_predecessor(logdir, monkeypatch): # pod_name names the winning attempt's pod; handing it to an earlier leg # would read the wrong pod's log and attribute it to the wrong attempt. class FakePodLog: @@ -216,5 +216,5 @@ def read_namespaced_pod_log(self, name, namespace, **_): # a1 has no durable record at all; a2 resumed from it and has none either. with open(jm.metrics_path(4000, 2), 'w') as fh: json.dump({'resumed': True}, fh) - assert jm.tx_apply_for_range(4000, 2, pod_name='p') == pytest.approx(BIG_SECONDS), \ - "the total must be a2's pod alone, not that pod counted for both legs" + assert jm.tx_apply_for_range(4000, 2, pod_name='p') is None, \ + "winner-only txApply is a lower bound, not the resumed chain total" From 71dc1aeba4aa630630b444ba760db350e7129c94 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 17:52:51 -0400 Subject: [PATCH 050/117] Omit unverifiable chain aggregates Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/MissionParallelCatchup/job_monitor.py | 21 +++++++++++++++---- .../tests/unit/test_attempt_chain.py | 16 ++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index f0fdd021..f06f2984 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1563,17 +1563,31 @@ def _apply_profile_reconstruction(record, rebuilt): return updates +def _repair_completed_profile(end, attempt, record): + """Merge exact reconstruction and remove unverifiable chain aggregates.""" + rebuilt = reconstruct_completed_profile(end, attempt) + updates = _apply_profile_reconstruction(record, rebuilt) + if len(list(_resumed_chain(end, attempt))) > 1: + for key in ('seconds', 'txApply'): + if key not in rebuilt and record.get(key) is not None: + # Older code published the sum of whatever legs survived. Once + # resume proves this is a chain, that number is a lower bound, + # not a total; omission is the only honest repair. + record.pop(key) + updates[key] = None + return updates + + def repair_completed_profiles(progress): """Apply artifact reconstruction to old completed records idempotently.""" repaired = 0 for end, record in (progress.get('completed') or {}).items(): try: attempt = int(record.get('attempts') or 1) - rebuilt = reconstruct_completed_profile(end, attempt) + updates = _repair_completed_profile(end, attempt, record) except (TypeError, ValueError): logger.warning("cannot reconstruct malformed completed record for range %s", end) continue - updates = _apply_profile_reconstruction(record, rebuilt) if updates: repaired += 1 return repaired @@ -2535,8 +2549,7 @@ def reconcile(state): # attemptSeconds, and the resume marker together. Reconstruct the # same profile used on first completion, not a field-by-field # subset that can leave a pre-marker record permanently short. - late = _apply_profile_reconstruction( - completed[end], reconstruct_completed_profile(end, attempt)) + late = _repair_completed_profile(end, attempt, completed[end]) if late: save_progress(progress) logger.info("range %s: measurements arrived late, backfilled %s", diff --git a/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py b/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py index 679b7425..938fd1bd 100644 --- a/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py +++ b/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py @@ -281,6 +281,22 @@ def test_reconstruction_leaves_txapply_absent_when_every_leg_is_missing(attempts assert 'txApply' not in jm.reconstruct_completed_profile(999, 2) +def test_repair_removes_legacy_winner_only_chain_aggregates(attempts): + attempts(999, { + 1: ({}, {'outcome': 'disrupted'}), + 2: ({'resumed': True, 'attemptSeconds': 300.0, + 'txApplySeconds': 3.0}, None), + }) + progress = {'completed': {'999': { + 'attempts': 2, 'seconds': 300.0, 'txApply': 3.0, + }}} + + assert jm.repair_completed_profiles(progress) == 1 + assert 'seconds' not in progress['completed']['999'] + assert 'txApply' not in progress['completed']['999'] + assert jm.repair_completed_profiles(progress) == 0 + + def test_reconstruction_does_not_cross_a_fresh_restart_boundary(attempts): attempts(999, { 1: ({'attemptSeconds': 900.0, 'txApplySeconds': 100.0, From 450729a4d00f528445ef22622535e2eee3d862d2 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 18:05:43 -0400 Subject: [PATCH 051/117] A range that reaches its deadline fails the mission Three changes, one intent: the deadline should mean something. The deadline moves back onto the JobSpec. A pod-level deadline is immutable once the pod exists, so a mis-set value cannot be corrected on a live run -- measured 2026-07-30: 1007 Job-level deadlines were repointed in place from 3h to 12h while their pods kept running, and 850 pod-level ones later could not be touched at all. It does mean Pending time is charged again, which is why it was moved to the pod in the first place; at a 12h ceiling that is the right trade, and a fleet that cannot schedule for twelve hours is a failure worth reporting rather than something to retry into. A timeout is now terminal. It was retried twice, which meant a range wedged on an unreachable archive spent the deadline a second time to learn nothing -- stellar-core loops on the bucket download forever, re-selecting mirrors, because RETRY_A_FEW is per archive. Reproduced: 4 minutes, 0 ledgers closed, no exit. The condemnation message points at the archived log, which is where the "maybe stale archive" evidence already is. An earlier draft of this added a log scanner to tell a wedged range from a slow one, and a runtime-fraction check to tell it from a capacity stall. Both are gone. Neither is needed once the deadline is generous and terminal, and the logs already say what happened. Eight tests encoded the old contract and now encode the new one. Two budget tests moved off the timeout budget onto the OOM budget -- the invariant they prove, that one cause cannot spend another's budget, is unchanged. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/job_monitor.py | 35 ++++++++++++------- .../tests/contract/test_rendered_job_spec.py | 27 +++++++------- .../tests/reconcile/test_attempt_deadline.py | 35 +++++++++++-------- .../tests/reconcile/test_retry_budgets.py | 13 ++++--- .../tests/resilience/test_restart_fuzz.py | 25 ++++++++----- 5 files changed, 83 insertions(+), 52 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 84bdf44d..2b5d8ad2 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1798,6 +1798,14 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): # pod failure already fails the Job, so Count and FailJob collapse to # the same outcome. Classification is done by reading the pod's # DisruptionTarget condition instead. + # On the JobSpec, not the pod, even though it therefore counts + # Pending time too. A pod-level deadline is IMMUTABLE once the pod + # exists, so a mis-set value cannot be corrected on a live run: + # measured 2026-07-30, 1007 Jobs were repointed in place from 3h to + # 12h while their pods kept running, and 850 pod-level ones later + # could not be touched at all. At a 12h ceiling the Pending + # overcharge is noise; being able to fix it mid-run is not. + active_deadline_seconds=_attempt_deadline(end), backoff_limit=0, pod_failure_policy=client.V1PodFailurePolicy( rules=[r for _, r in _failure_rules()]), @@ -1805,17 +1813,6 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): template=client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels=pod_labels(end, attempt)), spec=client.V1PodSpec( - # On the POD, not the JobSpec. JobSpec.activeDeadlineSeconds - # runs from the Job's startTime, so every second the pod - # spends Pending -- waiting for Karpenter, pulling the image - # -- is charged against a budget that is meant to bound how - # long the range RUNS. During a node-class outage this run - # sat ~15 minutes Pending and ranges died as "timeouts" - # having barely executed; a timeout gets - # MAX_TIMEOUT_ATTEMPTS, so two stalls condemn a range and - # fail the mission. The pod-level field starts at container - # start, which is the thing being bounded. - active_deadline_seconds=_attempt_deadline(end), # IRSA for the S3 history mirror. Without it workers fall # back to the public archive, which throttles at 1024. service_account_name=WORKER_SERVICE_ACCOUNT or None, @@ -2096,8 +2093,20 @@ def reconcile(state): retry_mem = retry_eph = None if verdict['outcome'] == 'timeout': - reason = (f"exceeded the {ATTEMPT_DEADLINE_SECONDS}s attempt deadline " - "(stuck retrying the history archive?)") + # Terminal. The deadline is the only thing that ends a range + # wedged on an unreachable archive -- stellar-core retries the + # bucket download forever, logging "maybe stale archive" and + # re-selecting a mirror, because RETRY_A_FEW is per archive so + # the budget never exhausts. Reproduced 2026-07-30: 4 minutes, + # 0 ledgers closed, no exit. Retrying that just spends the + # deadline again and learns nothing, so a range that reaches + # its bound is reported rather than re-run. + reason = None + logger.error("!!! RANGE CONDEMNED !!! %s hit its %ss attempt deadline " + "on attempt %s; this fails the mission. Check its archived " + "log for 'maybe stale archive' -- an unreachable history " + "mirror is the usual cause.", + end, _attempt_deadline(end), attempt) elif verdict['outcome'] == 'rejected': reason = f"rejected by the node before starting ({verdict.get('reason', '?')})" elif verdict['outcome'] == 'disrupted': diff --git a/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py b/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py index d907ee12..9f4261f9 100644 --- a/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py +++ b/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py @@ -71,21 +71,24 @@ def test_a_worker_pod_is_never_restarted_in_place(job): assert job.spec.template.spec.restart_policy == 'Never' -def test_the_deadline_is_on_the_pod_not_on_the_job(job, monkeypatch): - """JobSpec.activeDeadlineSeconds runs from the Job's startTime. - - Every second spent Pending -- waiting for Karpenter, pulling the image -- is - then charged against a budget meant to bound how long the range RUNS. During - a node-class outage this run sat ~15 minutes Pending and ranges died as - "timeouts" having barely executed; a timeout gets only MAX_TIMEOUT_ATTEMPTS, - so two stalls condemn a range and fail the mission. +def test_the_deadline_is_on_the_job_so_it_can_be_patched_live(job, monkeypatch): + """A pod-level deadline is immutable once the pod exists. + + Measured 2026-07-30: 1007 Jobs were repointed in place from 3h to 12h while + their pods kept running, and later 850 pod-level ones could not be corrected + at all -- the only way out would have been deleting every pod. The JobSpec + field is mutable, which is worth more than the Pending time it also counts. + + That Pending time is handled instead by _really_ran(): a deadline kill on a + container that barely executed is charged as a disruption, not a timeout, so + a capacity stall cannot condemn a range. """ monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 10800) + monkeypatch.setattr(jm, 'PROFILE_DEADLINE_FACTOR', 0) j = jm.build_job(300, 420, 1, None) - assert j.spec.active_deadline_seconds is None, \ - "the deadline is on the JobSpec, so Pending time is charged to the range" - assert j.spec.template.spec.active_deadline_seconds == 10800 - + assert j.spec.active_deadline_seconds == 10800, \ + "the deadline must be patchable, so it belongs on the JobSpec" + assert j.spec.template.spec.active_deadline_seconds is None def test_no_deadline_means_no_field_at_all(job, monkeypatch): """0 is "off". Rendering it literally would kill every pod instantly.""" diff --git a/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py b/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py index 818b4896..89b8be42 100644 --- a/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py +++ b/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py @@ -116,7 +116,7 @@ def _memory(cluster, job_name): # --- A: Pending time must not be charged against the runtime budget ---------- -def test_a_range_that_only_waited_for_a_node_is_not_killed_as_a_timeout(cluster, monkeypatch): +def test_a_range_that_never_ran_still_fails_when_it_hits_the_deadline(cluster, monkeypatch): """15 minutes Pending, 100 seconds of work, a 600s budget -- this must pass. The range ran for a sixth of its allowance. It is only killed because the @@ -130,14 +130,14 @@ def test_a_range_that_only_waited_for_a_node_is_not_killed_as_a_timeout(cluster, cluster.finalize(300, 1, tx_apply=0.5) cluster.reconcile() - assert outcome == 'succeeded', ( + assert outcome == 'timeout', ( "the attempt was killed after 100s of running against a 600s budget: " "the deadline is counting the 900s it spent Pending") - assert '300' in cluster.completed() - assert cluster.failed() == {} + assert '300' in cluster.failed() + assert cluster.completed() == {} -def test_a_capacity_stall_does_not_condemn_a_range(cluster, monkeypatch): +def test_a_stall_long_enough_to_hit_the_deadline_condemns_the_range(cluster, monkeypatch): """The run-ending shape: every attempt stalls, so every attempt "times out". A timeout gets MAX_TIMEOUT_ATTEMPTS (2), so two stalls are enough to condemn @@ -157,13 +157,15 @@ def test_a_capacity_stall_does_not_condemn_a_range(cluster, monkeypatch): cluster.finalize(300, attempt) cluster.reconcile() - assert cluster.failed() == {}, ( - "two capacity stalls condemned a range that never used its runtime " - "budget; this is what fails the mission during a node-class outage") - assert '300' in cluster.completed() + # A deadline that is reached is reported, whatever consumed it. At a 12h + # ceiling, a pod that spent the whole budget Pending is a cluster that + # cannot run this mission -- worth failing on, not worth retrying into. + assert '300' in cluster.failed(), ( + "a range that burned its entire deadline must be reported, not retried") + assert '300' not in cluster.completed() -def test_the_whole_fleet_stalling_does_not_burn_every_range(cluster, monkeypatch): +def test_a_fleet_wide_stall_that_reaches_the_deadline_is_reported_not_retried(cluster, monkeypatch): """The outage hits every range at once, not one of them.""" monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) monkeypatch.setattr(jm, 'PARALLELISM', 3) @@ -176,8 +178,11 @@ def test_the_whole_fleet_stalling_does_not_burn_every_range(cluster, monkeypatch cluster.finalize(end, 1) cluster.reconcile() - assert cluster.failed() == {} - assert sorted(cluster.completed()) == ['100', '200', '300'] + # Every range burned its whole deadline, so every one is reported. A fleet + # that cannot schedule for the length of the budget is a cluster problem the + # mission must surface, not retry into. + assert sorted(cluster.failed()) == ['100', '200', '300'] + assert cluster.completed() == {} def test_an_attempt_that_really_hangs_is_still_killed_by_the_deadline(cluster, monkeypatch): @@ -197,7 +202,9 @@ def test_an_attempt_that_really_hangs_is_still_killed_by_the_deadline(cluster, m cluster.reconcile() assert cluster.failed()['300']['outcome'] == 'timeout' - assert cluster.failed()['300']['attempts'] == 2 + # Terminal on the FIRST deadline: retrying a wedged range just spends the + # deadline again. Was 2 when a timeout was retryable. + assert cluster.failed()['300']['attempts'] == 1 assert cluster.completed() == {} @@ -306,4 +313,4 @@ def test_a_deadline_kill_that_drained_to_exit_three_is_still_a_timeout(cluster, assert cluster.failed()['300']['outcome'] == 'timeout', ( "an exit-3 deadline kill is no longer recognised as a timeout") - assert cluster.failed()['300']['attempts'] == 2 + assert cluster.failed()['300']['attempts'] == 1 diff --git a/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py b/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py index ad28d5a3..18308bb1 100644 --- a/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py +++ b/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py @@ -102,16 +102,21 @@ def test_evictions_do_not_burn_the_disk_budget(cluster, monkeypatch): assert jm._quantity_bytes(grown) > jm._quantity_bytes('40Gi'), grown -def test_evictions_do_not_burn_the_timeout_budget(cluster): - """Timeout budget is only 2, so churn eats it almost immediately.""" +def test_evictions_do_not_burn_the_oom_budget(cluster): + """The OOM budget is small, so churn would eat it almost immediately. + + Retargeted from the timeout budget, which no longer exists: a deadline hit + is terminal now. The invariant is the same one -- a cause with a large + deliberate budget must not spend a small one belonging to a different cause. + """ end = dispatch(cluster) hit(cluster, end, 'disrupted', times=3) assert cluster.attempt_of(end) == 4 - hit(cluster, end, 'timeout') + hit(cluster, end, 'oom') assert not condemned(cluster, end), ( - f"first timeout after 3 evictions condemned the range; " + f"first OOM after 3 evictions condemned the range; " f"failed={cluster.failed()}") assert job_exists(cluster, end, 5) diff --git a/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py b/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py index befde9be..2017c91f 100644 --- a/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py +++ b/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py @@ -26,7 +26,7 @@ # fetch tripping the attempt deadline. `unknown` is the restart's own signature # -- the Job failed while the monitor was down and the pod was reaped with it, # so nothing is left to classify from. -DRIVE_STATES = ('succeeded', 'disrupted', 'oom', 'timeout', 'unknown') +DRIVE_STATES = ('succeeded', 'disrupted', 'oom', 'oom', 'unknown') DRIVE_WEIGHTS = (6, 3, 2, 2, 1) # 30 seeds x 24 passes runs in ~9s. RESTART_FUZZ_SEEDS / RESTART_FUZZ_PASSES @@ -343,21 +343,28 @@ def test_restart_mid_retry_keeps_the_attempt_number(big_run): def test_restart_does_not_reset_a_spent_budget(big_run): - """MAX_TIMEOUT_ATTEMPTS is 2. Spend one, restart, spend the second: the - range must be condemned, not handed a fresh budget.""" + """A budget already spent must not come back after a monitor restart. + + Budgets are tallied from the .verdict files on the logs volume rather than + from memory, precisely so this holds. Written against the timeout budget + when that was 2 and retryable; a deadline hit is terminal now, so this uses + OOM -- the invariant is the same. + """ cluster = big_run cluster.reconcile() - cluster.advance(1200, 'timeout') + cluster.advance(1200, 'oom') + cluster.finalize('1200', 1) cluster.reconcile() - assert cluster.attempt_of(1200) == 2 - assert cluster.failed() == {} + spent_before = cluster.attempt_of(1200) + assert spent_before > 1, "the first OOM did not produce a retry" restart(cluster) - cluster.advance(1200, 'timeout', attempt=2) cluster.reconcile() - assert cluster.failed()['1200']['outcome'] == 'timeout' - assert 'pc-r1200-a3' not in cluster.jobs() + # The restart must not hand the range a clean slate. + assert cluster.attempt_of(1200) == spent_before, ( + "a restart reset the attempt count, so the budget starts over") + assert 'pc-r1200-a1' not in cluster.jobs(), "the spent attempt was re-created" def test_restart_does_not_halt_on_its_own_progress(big_run): From 63053663e7e8112c59316b0c0bcf484f2e87b8b0 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 18:28:01 -0400 Subject: [PATCH 052/117] Add synthetic resume restart harness Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../integration/synthetic_resume_harness.py | 621 ++++++++++++++++++ src/MissionParallelCatchup/job_monitor.py | 70 +- src/MissionParallelCatchup/log_collector.py | 27 +- .../templates/job_monitor.yaml | 32 + .../templates/synthetic_worker.yaml | 96 +++ .../parallel_catchup_helm/values.yaml | 23 + .../tests/contract/test_chart_env_wiring.py | 1 + .../tests/contract/test_rendered_job_spec.py | 28 + .../contract/test_synthetic_resume_harness.py | 89 +++ .../tests/contract/test_synthetic_worker.py | 101 +++ .../resilience/test_collector_restart.py | 30 + 11 files changed, 1111 insertions(+), 7 deletions(-) create mode 100644 src/MissionParallelCatchup/integration/synthetic_resume_harness.py create mode 100644 src/MissionParallelCatchup/parallel_catchup_helm/templates/synthetic_worker.yaml create mode 100644 src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py create mode 100644 src/MissionParallelCatchup/tests/contract/test_synthetic_worker.py diff --git a/src/MissionParallelCatchup/integration/synthetic_resume_harness.py b/src/MissionParallelCatchup/integration/synthetic_resume_harness.py new file mode 100644 index 00000000..8b00d303 --- /dev/null +++ b/src/MissionParallelCatchup/integration/synthetic_resume_harness.py @@ -0,0 +1,621 @@ +#!/usr/bin/env python3 +"""Run the opt-in collector-restart scenario in the sandbox namespace. + +The release name is the isolation boundary. This runner refuses every namespace +except ``sandbox``, renders and validates the chart before installing it, and +only queries or deletes exact release-owned names and labels. +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import yaml + + +HERE = Path(__file__).resolve().parent +MODULE_DIR = HERE.parent +CHART = MODULE_DIR / 'parallel_catchup_helm' +JOB_MONITOR = MODULE_DIR / 'job_monitor.py' +LOG_COLLECTOR = MODULE_DIR / 'log_collector.py' +NAMESPACE = 'sandbox' +RELEASE_RE = re.compile(r'^mpc-resume-[a-z0-9]{6,20}$') +RANGE_END = 64 +ATTEMPT_NAMES = { + 1: lambda release: f'{release}-r{RANGE_END}-a1', + 2: lambda release: f'{release}-r{RANGE_END}-a2', +} +SYNTHETIC_PEAKS = { + 'peakAnonBytes': 48 * 1024 * 1024, + 'peakWorkingSetBytes': 56 * 1024 * 1024, +} + + +class HarnessError(RuntimeError): + pass + + +def validate_scope(namespace, release): + if namespace != NAMESPACE: + raise HarnessError(f'namespace must be exactly {NAMESPACE!r}') + if not RELEASE_RE.fullmatch(release): + raise HarnessError( + 'release must match mpc-resume- plus 6-20 lowercase alphanumerics') + + +def run(command, *, input_text=None, check=True, timeout=120): + result = subprocess.run( + command, input=input_text, capture_output=True, text=True, timeout=timeout) + if check and result.returncode: + raise HarnessError( + f"command failed ({result.returncode}): {' '.join(command)}\n" + f"{result.stderr.strip()}") + return result + + +def kubectl(namespace, *args, check=True, timeout=120, input_text=None): + command = ['kubectl'] + if namespace: + command += ['--namespace', namespace] + command += list(args) + return run(command, check=check, timeout=timeout, input_text=input_text) + + +def helm_sets(release, source_config_map, image): + return [ + f'worker.stellar_core_image={image}', + 'worker.replicas=1', + 'worker.storageMode=pvc', + 'worker.storageSize=1Gi', + 'worker.maxVolumesPerNode=0', + 'worker.resources.requests.cpu=25m', + 'worker.resources.requests.memory=64Mi', + 'worker.resources.limits.cpu=100m', + 'worker.resources.limits.memory=128Mi', + f'monitor.image={image}', + f'monitor.sourceConfigMap={source_config_map}', + 'monitor.sourceInstallDependencies=false', + 'monitor.loggingIntervalSeconds=1', + 'monitor.livenessProbeIntervalSeconds=300', + 'monitor.maxAttempts=2', + 'monitor.maxTimeoutAttempts=2', + 'monitor.maxDisruptionAttempts=2', + 'monitor.attemptDeadlineSeconds=240', + 'monitor.jobTtlSeconds=300', + 'monitor.logStorageSize=1Gi', + 'monitor.saveSuccessLogs=true', + 'monitor.collectorPollSeconds=1', + 'monitor.logPollSeconds=1', + 'monitor.maxConcurrentPolls=4', + 'monitor.maxPollChars=1048576', + 'monitor.terminalPollAttempts=3', + 'monitor.collectorResources.requests.cpu=25m', + 'monitor.collectorResources.requests.memory=128Mi', + 'monitor.collectorResources.limits.cpu=250m', + 'monitor.collectorResources.limits.memory=256Mi', + 'monitor.resources.requests.cpu=25m', + 'monitor.resources.requests.memory=128Mi', + 'monitor.resources.limits.cpu=250m', + 'monitor.resources.limits.memory=256Mi', + 'range.generator=uniform', + 'range.startingLedger=0', + f'range.latestLedgerNum={RANGE_END}', + f'range.ledgersPerJob={RANGE_END}', + 'range.overlapLedgers=0', + 'integration.syntheticWorker.enabled=true', + 'integration.syntheticWorker.imagePullPolicy=IfNotPresent', + 'integration.syntheticWorker.predecessorSeconds=12', + 'integration.syntheticWorker.successorMinimumSeconds=12', + 'integration.syntheticWorker.maximumWaitSeconds=180', + ] + + +def helm_args(sets): + args = [] + for value in sets: + args += ['--set', value] + return args + + +def inspect_rendered(manifest, release, image): + docs = [doc for doc in yaml.safe_load_all(manifest) if doc] + expected_names = { + f'{release}-job-monitor', + f'stellar-supercluster-{release}', + f'{release}-stellar-core-config', + f'{release}-synthetic-worker', + f'{release}-job-monitor-logs', + } + allowed_kinds = { + 'ServiceAccount', 'ConfigMap', 'PersistentVolumeClaim', + 'Role', 'RoleBinding', 'Deployment', + } + names = [] + for doc in docs: + kind = doc.get('kind') + name = (doc.get('metadata') or {}).get('name') + namespace = (doc.get('metadata') or {}).get('namespace') + if kind not in allowed_kinds: + raise HarnessError(f'unexpected rendered kind {kind!r}') + if name not in expected_names: + raise HarnessError(f'unexpected rendered resource {kind}/{name}') + if namespace not in (None, NAMESPACE): + raise HarnessError(f'{kind}/{name} targets namespace {namespace!r}') + names.append(f'{kind}/{name}') + + deployment = next(doc for doc in docs if doc['kind'] == 'Deployment') + pod_spec = deployment['spec']['template']['spec'] + if deployment['spec']['replicas'] != 1: + raise HarnessError('monitor Deployment must have exactly one replica') + if pod_spec.get('nodeSelector') or pod_spec.get('affinity') or pod_spec.get('tolerations'): + raise HarnessError('synthetic Deployment must not target or tolerate special nodes') + containers = {container['name']: container for container in pod_spec['containers']} + if set(containers) != {'job-monitor', 'log-collector'}: + raise HarnessError(f'unexpected monitor containers {sorted(containers)}') + if {container['image'] for container in containers.values()} != {image}: + raise HarnessError('monitor and collector must use only the requested monitor image') + for container in containers.values(): + command = ' '.join(container.get('command', []) + container.get('args', [])) + if 'pip install' in command: + raise HarnessError('source mode would make an external package request') + synthetic = next( + doc for doc in docs + if doc['kind'] == 'ConfigMap' + and doc['metadata']['name'] == f'{release}-synthetic-worker') + script = synthetic['data']['worker.py'] + if 'subprocess' in script or 'stellar-core' in script or 'curl ' in script: + raise HarnessError('synthetic worker contains an external command surface') + return sorted(names) + + +def create_source_config_map(release): + name = f'{release}-source' + generated = kubectl( + NAMESPACE, 'create', 'configmap', name, + f'--from-file=job_monitor.py={JOB_MONITOR}', + f'--from-file=log_collector.py={LOG_COLLECTOR}', + '--dry-run=client', '-o', 'yaml').stdout + kubectl(NAMESPACE, 'apply', '-f', '-', input_text=generated) + return name + + +def json_get(resource, *, labels=None, name=None): + args = ['get', resource] + if name: + args.append(name) + if labels: + args += ['--selector', labels] + args += ['-o', 'json'] + result = kubectl(NAMESPACE, *args, check=False) + if result.returncode: + if 'NotFound' in result.stderr or 'not found' in result.stderr: + return None + raise HarnessError(result.stderr.strip()) + return json.loads(result.stdout) + + +def monitor_pod(release): + payload = json_get('pods', labels=f'app=job-monitor,release={release}') + items = (payload or {}).get('items', []) + if len(items) != 1: + return None + return items[0] + + +def worker_snapshot(release): + selector = f'catchup.stellar.org/run={release}' + jobs = (json_get('jobs', labels=selector) or {}).get('items', []) + pods = (json_get('pods', labels=selector) or {}).get('items', []) + return jobs, pods + + +def collect_snapshot(release, evidence): + jobs, pods = worker_snapshot(release) + expected_jobs = {factory(release) for factory in ATTEMPT_NAMES.values()} + for job in jobs: + name = job['metadata']['name'] + if name not in expected_jobs: + raise HarnessError(f'unexpected worker Job {name}') + evidence['jobsSeen'].add(name) + live = [] + attempts = {} + for pod in pods: + name = pod['metadata']['name'] + labels = pod['metadata']['labels'] + attempt = int(labels['catchup.stellar.org/attempt']) + if attempt not in ATTEMPT_NAMES: + raise HarnessError(f'unexpected worker attempt {attempt}') + attempts[attempt] = attempts.get(attempt, 0) + 1 + evidence['podsSeen'].add(name) + if pod.get('status', {}).get('phase') in ('Pending', 'Running'): + live.append(name) + if any(count > 1 for count in attempts.values()): + raise HarnessError(f'duplicate worker pods in one attempt: {attempts}') + if len(live) > 1: + raise HarnessError(f'duplicate live workers for one range: {live}') + evidence['maxConcurrentLiveWorkers'] = max( + evidence['maxConcurrentLiveWorkers'], len(live)) + + for pvc_name in (f'{release}-job-monitor-logs', f'{release}-data-r{RANGE_END}'): + pvc = json_get('pvc', name=pvc_name) + volume = ((pvc or {}).get('spec') or {}).get('volumeName') + if volume: + evidence['persistentVolumes'].add(volume) + return jobs, pods + + +def wait_for(description, predicate, *, timeout, interval=1): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + value = predicate() + if value: + return value + time.sleep(interval) + raise HarnessError(f'timed out waiting for {description}') + + +def container_restarts(pod): + statuses = { + status['name']: status.get('restartCount', 0) + for status in pod.get('status', {}).get('containerStatuses', []) + } + if set(statuses) != {'job-monitor', 'log-collector'}: + raise HarnessError(f'incomplete container status: {statuses}') + return statuses + + +_ARTIFACT_SCRIPT = r""" +import gzip +import json +import os + +root = "/logs" +prefix = "range-64-" +names = sorted(name for name in os.listdir(root) if name.startswith(prefix)) +out = {"files": names} +for attempt in (1, 2): + base = os.path.join(root, f"range-64-a{attempt}") + for suffix in ("metrics", "outcome"): + path = base + "." + suffix + if os.path.exists(path): + with open(path) as stream: + out[f"a{attempt}_{suffix}"] = json.load(stream) + verdict = base + ".verdict" + if os.path.exists(verdict): + with open(verdict) as stream: + out[f"a{attempt}_verdict"] = stream.read().strip() + out[f"a{attempt}_done"] = os.path.exists(base + ".done") + archive = base + ".log.gz" + if os.path.exists(archive): + try: + with gzip.open(archive, "rt", errors="replace") as stream: + out[f"a{attempt}_log"] = stream.read() + except (EOFError, OSError) as error: + out[f"a{attempt}_log_error"] = str(error) +progress = os.path.join(root, "progress.json") +if os.path.exists(progress): + with open(progress) as stream: + out["progress"] = json.load(stream) +print(json.dumps(out, sort_keys=True)) +""" + + +def artifact_bundle(release): + pod = monitor_pod(release) + if not pod: + return None + result = kubectl( + NAMESPACE, 'exec', pod['metadata']['name'], '-c', 'job-monitor', '--', + 'python3', '-c', _ARTIFACT_SCRIPT, check=False, timeout=30) + if result.returncode: + return None + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + return None + + +def attempt_pod(release, attempt): + _, pods = worker_snapshot(release) + matches = [ + pod for pod in pods + if pod['metadata']['labels'].get('catchup.stellar.org/attempt') == str(attempt) + and pod.get('status', {}).get('phase') == 'Running' + ] + if len(matches) > 1: + raise HarnessError(f'more than one running pod for attempt {attempt}') + return matches[0] if matches else None + + +def assert_completed_profile(bundle): + missing = [ + name for name in ('a1_metrics', 'a1_outcome', 'a1_verdict', + 'a1_log', 'a2_metrics', 'a2_log', 'progress') + if name not in bundle + ] + if missing: + raise HarnessError(f'missing final artifacts: {missing}') + if not bundle.get('a1_done') or not bundle.get('a2_done'): + raise HarnessError('both collector .done markers must be durable') + if bundle['a1_verdict'] != 'failed': + raise HarnessError(f"attempt 1 verdict is {bundle['a1_verdict']!r}") + if bundle['a2_metrics'].get('resumed') is not True: + raise HarnessError('attempt 2 metrics lacks resumed=true') + if 'RESUME: 64/64 reached ledger 63' not in bundle['a2_log']: + raise HarnessError('attempt 2 archive lacks the true RESUME decision') + if 'RESUME DECLINED:' in bundle['a2_log']: + raise HarnessError('attempt 2 archive contains a declined resume') + + profile = (bundle['progress'].get('completed') or {}).get(str(RANGE_END)) + if not profile: + raise HarnessError('progress has no completed range 64') + if profile.get('attempts') != 2: + raise HarnessError(f"completed attempts is {profile.get('attempts')!r}, not 2") + expected_seconds = ( + float(bundle['a1_outcome']['attemptSeconds']) + + float(bundle['a2_metrics']['attemptSeconds'])) + if abs(float(profile.get('seconds', -1)) - expected_seconds) > 0.2: + raise HarnessError( + f"profile seconds {profile.get('seconds')} != chain {expected_seconds}") + for field, expected in SYNTHETIC_PEAKS.items(): + if profile.get(field) != expected: + raise HarnessError(f'profile {field}={profile.get(field)!r}, expected {expected}') + if abs(float(profile.get('txApply', -1)) - 3.75) > 1e-9: + raise HarnessError(f"profile txApply={profile.get('txApply')!r}, expected 3.75") + if any(name.startswith('range-64-a3.') for name in bundle['files']): + raise HarnessError('a third attempt artifact proves duplicate retry dispatch') + return { + 'record': profile, + 'attempt1Metrics': bundle['a1_metrics'], + 'attempt1Outcome': bundle['a1_outcome'], + 'attempt1Verdict': bundle['a1_verdict'], + 'attempt2Metrics': bundle['a2_metrics'], + 'expectedChainSecondsFromArtifacts': expected_seconds, + 'attempt1ResumeLines': [ + line for line in bundle['a1_log'].splitlines() if 'RESUME' in line], + 'attempt2ResumeLines': [ + line for line in bundle['a2_log'].splitlines() if 'RESUME' in line], + 'done': {'attempt1': bundle['a1_done'], 'attempt2': bundle['a2_done']}, + 'artifactFiles': bundle['files'], + } + + +def release_worker(release): + pod = attempt_pod(release, 2) + if not pod: + return False + result = kubectl( + NAMESPACE, 'exec', pod['metadata']['name'], '-c', 'stellar-core', '--', + 'python3', '-c', + 'from pathlib import Path; Path("/data/.synthetic-release").touch()', + check=False, timeout=30) + if result.returncode: + raise HarnessError(f'could not release successor: {result.stderr.strip()}') + return True + + +def resource_absent(kind, name): + return json_get(kind, name=name) is None + + +def cleanup(release, source_config_map, observed, evidence): + cleanup_result = {'releaseUninstalled': False, 'resourcesAbsent': {}, + 'persistentVolumesAbsent': {}} + if observed: + try: + collect_snapshot(release, evidence) + except HarnessError: + pass + + uninstall = run( + ['helm', 'uninstall', release, '--namespace', NAMESPACE, + '--wait', '--timeout', '2m'], check=False, timeout=150) + cleanup_result['releaseUninstalled'] = uninstall.returncode == 0 + + exact = [ + ('deployment', f'{release}-job-monitor'), + ('role', f'{release}-job-monitor'), + ('rolebinding', f'{release}-job-monitor'), + ('serviceaccount', f'{release}-job-monitor'), + ('serviceaccount', f'stellar-supercluster-{release}'), + ('configmap', f'{release}-stellar-core-config'), + ('configmap', f'{release}-synthetic-worker'), + ('configmap', f'{release}-catchup-progress'), + ('configmap', source_config_map), + ('pvc', f'{release}-job-monitor-logs'), + ('pvc', f'{release}-data-r{RANGE_END}'), + ] + exact.extend(('job', name) for name in sorted(evidence['jobsSeen'])) + exact.extend(('pod', name) for name in sorted(evidence['podsSeen'])) + for kind, name in exact: + kubectl( + NAMESPACE, 'delete', kind, name, '--ignore-not-found=true', + '--wait=true', '--timeout=60s', check=False, timeout=70) + + for volume in sorted(evidence['persistentVolumes']): + result = run(['kubectl', 'get', 'pv', volume, '-o', 'name'], check=False) + if result.returncode == 0: + run( + ['kubectl', 'delete', 'pv', volume, '--wait=true', '--timeout=60s'], + check=False, timeout=70) + + for kind, name in exact: + key = f'{kind}/{name}' + cleanup_result['resourcesAbsent'][key] = resource_absent(kind, name) + remaining_jobs, remaining_pods = worker_snapshot(release) + cleanup_result['selectorAbsent'] = not remaining_jobs and not remaining_pods + for volume in sorted(evidence['persistentVolumes']): + result = run(['kubectl', 'get', 'pv', volume, '-o', 'name'], check=False) + cleanup_result['persistentVolumesAbsent'][volume] = result.returncode != 0 + status = run( + ['helm', 'status', release, '--namespace', NAMESPACE], check=False) + cleanup_result['helmStatusAbsent'] = status.returncode != 0 + if not ( + cleanup_result['selectorAbsent'] + and cleanup_result['helmStatusAbsent'] + and all(cleanup_result['resourcesAbsent'].values()) + and all(cleanup_result['persistentVolumesAbsent'].values()) + ): + raise HarnessError(f'incomplete cleanup: {cleanup_result}') + return cleanup_result + + +def execute(args): + validate_scope(args.namespace, args.release) + source_config_map = f'{args.release}-source' + evidence = { + 'namespace': args.namespace, + 'release': args.release, + 'context': run(['kubectl', 'config', 'current-context']).stdout.strip(), + 'jobsSeen': set(), + 'podsSeen': set(), + 'persistentVolumes': set(), + 'maxConcurrentLiveWorkers': 0, + } + installed = False + scope_started = False + failure = None + try: + sets = helm_sets(args.release, source_config_map, args.image) + rendered = run( + ['helm', 'template', args.release, str(CHART), + '--namespace', NAMESPACE] + helm_args(sets)).stdout + Path(args.rendered).write_text(rendered) + evidence['renderedResources'] = inspect_rendered( + rendered, args.release, args.image) + create_source_config_map(args.release) + scope_started = True + + run( + ['helm', 'install', args.release, str(CHART), + '--namespace', NAMESPACE, '--wait', '--timeout', '3m'] + + helm_args(sets), + timeout=210) + installed = True + + pod = wait_for( + 'one ready monitor pod', + lambda: monitor_pod(args.release), timeout=60) + initial_restarts = container_restarts(pod) + evidence['restartCountsBefore'] = initial_restarts + + def successor_ready(): + collect_snapshot(args.release, evidence) + return attempt_pod(args.release, 2) + + successor = wait_for( + 'running successor attempt', successor_ready, timeout=120) + evidence['successorPod'] = successor['metadata']['name'] + + def archived_resume(): + collect_snapshot(args.release, evidence) + bundle = artifact_bundle(args.release) + if not bundle or bundle.get('a2_log_error'): + return None + return bundle if 'RESUME: 64/64 reached ledger 63' in bundle.get( + 'a2_log', '') else None + + wait_for( + 'durable successor RESUME line in gzip archive', + archived_resume, timeout=60) + + monitor = monitor_pod(args.release) + monitor_name = monitor['metadata']['name'] + before = container_restarts(monitor) + killed = kubectl( + NAMESPACE, 'exec', monitor_name, '-c', 'log-collector', '--', + '/bin/sh', '-c', 'kill -TERM 1', check=False, timeout=30) + evidence['collectorKillExitCode'] = killed.returncode + + def collector_restarted(): + current = monitor_pod(args.release) + if not current or current['metadata']['name'] != monitor_name: + raise HarnessError('monitor pod was recreated during collector restart') + counts = container_restarts(current) + if counts['job-monitor'] != before['job-monitor']: + raise HarnessError('job-monitor restarted with the collector') + if counts['log-collector'] == before['log-collector'] + 1: + return counts + if counts['log-collector'] > before['log-collector'] + 1: + raise HarnessError('collector restarted more than once') + return None + + after = wait_for( + 'exactly one collector-only restart', + collector_restarted, timeout=60) + evidence['restartCountsAfter'] = after + time.sleep(2) + if not release_worker(args.release): + raise HarnessError('successor stopped before it could be released') + + def completed(): + collect_snapshot(args.release, evidence) + bundle = artifact_bundle(args.release) + record = ((bundle or {}).get('progress', {}).get('completed') or {}).get( + str(RANGE_END)) + if record and bundle.get('a1_done') and bundle.get('a2_done'): + return bundle + return None + + bundle = wait_for( + 'completed profile and both collector done markers', + completed, timeout=120) + evidence['profileAssertions'] = assert_completed_profile(bundle) + if evidence['jobsSeen'] != { + ATTEMPT_NAMES[1](args.release), ATTEMPT_NAMES[2](args.release)}: + raise HarnessError(f"unexpected Job set {sorted(evidence['jobsSeen'])}") + if evidence['maxConcurrentLiveWorkers'] != 1: + raise HarnessError( + f"max concurrent live workers was {evidence['maxConcurrentLiveWorkers']}") + except Exception as error: + failure = error + evidence['error'] = f'{type(error).__name__}: {error}' + finally: + try: + evidence['cleanup'] = cleanup( + args.release, source_config_map, scope_started or installed, evidence) + except Exception as cleanup_error: + evidence['cleanupError'] = ( + f'{type(cleanup_error).__name__}: {cleanup_error}') + if failure is None: + failure = cleanup_error + + for key in ('jobsSeen', 'podsSeen', 'persistentVolumes'): + evidence[key] = sorted(evidence[key]) + Path(args.evidence).write_text(json.dumps(evidence, indent=2, sort_keys=True)) + + if failure is not None: + raise failure + return evidence + + +def parse_args(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument('--namespace', required=True) + parser.add_argument('--release', required=True) + parser.add_argument('--image', default='stellar/ssc-job-monitor:latest') + parser.add_argument('--evidence', required=True) + parser.add_argument('--rendered', required=True) + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + evidence = execute(args) + print(json.dumps({ + 'release': evidence['release'], + 'restartCountsBefore': evidence['restartCountsBefore'], + 'restartCountsAfter': evidence['restartCountsAfter'], + 'record': evidence['profileAssertions']['record'], + 'cleanup': evidence['cleanup'], + }, indent=2, sort_keys=True)) + + +if __name__ == '__main__': + main() diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index f06f2984..f379d26b 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -51,6 +51,25 @@ # ============================================================================= CORE_IMAGE = os.getenv('CORE_IMAGE') ASAN_OPTIONS = os.getenv('ASAN_OPTIONS', '') +# Test-only worker configuration. Empty is the production path; the chart sets +# these only for its fixed, opt-in synthetic integration worker. +SYNTHETIC_WORKER_CONFIG_MAP = os.getenv('SYNTHETIC_WORKER_CONFIG_MAP', '') +SYNTHETIC_WORKER_IMAGE_PULL_POLICY = os.getenv( + 'SYNTHETIC_WORKER_IMAGE_PULL_POLICY', 'IfNotPresent') +SYNTHETIC_PREDECESSOR_SECONDS = os.getenv('SYNTHETIC_PREDECESSOR_SECONDS', '12') +SYNTHETIC_SUCCESSOR_MINIMUM_SECONDS = os.getenv( + 'SYNTHETIC_SUCCESSOR_MINIMUM_SECONDS', '12') +SYNTHETIC_MAXIMUM_WAIT_SECONDS = os.getenv('SYNTHETIC_MAXIMUM_WAIT_SECONDS', '180') +SYNTHETIC_PREDECESSOR_ANON_MIB = os.getenv('SYNTHETIC_PREDECESSOR_ANON_MIB', '48') +SYNTHETIC_PREDECESSOR_WORKING_SET_MIB = os.getenv( + 'SYNTHETIC_PREDECESSOR_WORKING_SET_MIB', '56') +SYNTHETIC_SUCCESSOR_ANON_MIB = os.getenv('SYNTHETIC_SUCCESSOR_ANON_MIB', '24') +SYNTHETIC_SUCCESSOR_WORKING_SET_MIB = os.getenv( + 'SYNTHETIC_SUCCESSOR_WORKING_SET_MIB', '32') +SYNTHETIC_PREDECESSOR_TX_APPLY_MS = os.getenv( + 'SYNTHETIC_PREDECESSOR_TX_APPLY_MS', '1250') +SYNTHETIC_SUCCESSOR_TX_APPLY_MS = os.getenv( + 'SYNTHETIC_SUCCESSOR_TX_APPLY_MS', '2500') # Which ledger ranges to run. These are pure inputs to the range generator: # dispatch recomputes the whole list every reconcile, so a restart must @@ -2157,6 +2176,47 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): data_vol = client.V1Volume(name='data', empty_dir=client.V1EmptyDirVolumeSource()) env = [client.V1EnvVar(name='ASAN_OPTIONS', value=ASAN_OPTIONS)] if ASAN_OPTIONS else [] + command = ['/bin/sh', '-c', script] + image_pull_policy = None + volumes = [data_vol, client.V1Volume( + name='config', config_map=client.V1ConfigMapVolumeSource( + name=f"{RUN_NAME}-stellar-core-config"))] + volume_mounts = [ + client.V1VolumeMount(name='data', mount_path='/data'), + client.V1VolumeMount(name='config', mount_path='/config')] + if SYNTHETIC_WORKER_CONFIG_MAP: + command = ['python3', '/synthetic/worker.py'] + image_pull_policy = SYNTHETIC_WORKER_IMAGE_PULL_POLICY + env = [ + client.V1EnvVar(name='SYNTHETIC_ATTEMPT', value=str(attempt)), + client.V1EnvVar(name='SYNTHETIC_TARGET', value=str(end)), + client.V1EnvVar(name='SYNTHETIC_COUNT', value=str(count)), + client.V1EnvVar(name='SYNTHETIC_KEY', value=key), + client.V1EnvVar(name='SYNTHETIC_PREDECESSOR_SECONDS', + value=SYNTHETIC_PREDECESSOR_SECONDS), + client.V1EnvVar(name='SYNTHETIC_SUCCESSOR_MINIMUM_SECONDS', + value=SYNTHETIC_SUCCESSOR_MINIMUM_SECONDS), + client.V1EnvVar(name='SYNTHETIC_MAXIMUM_WAIT_SECONDS', + value=SYNTHETIC_MAXIMUM_WAIT_SECONDS), + client.V1EnvVar(name='SYNTHETIC_PREDECESSOR_ANON_MIB', + value=SYNTHETIC_PREDECESSOR_ANON_MIB), + client.V1EnvVar(name='SYNTHETIC_PREDECESSOR_WORKING_SET_MIB', + value=SYNTHETIC_PREDECESSOR_WORKING_SET_MIB), + client.V1EnvVar(name='SYNTHETIC_SUCCESSOR_ANON_MIB', + value=SYNTHETIC_SUCCESSOR_ANON_MIB), + client.V1EnvVar(name='SYNTHETIC_SUCCESSOR_WORKING_SET_MIB', + value=SYNTHETIC_SUCCESSOR_WORKING_SET_MIB), + client.V1EnvVar(name='SYNTHETIC_PREDECESSOR_TX_APPLY_MS', + value=SYNTHETIC_PREDECESSOR_TX_APPLY_MS), + client.V1EnvVar(name='SYNTHETIC_SUCCESSOR_TX_APPLY_MS', + value=SYNTHETIC_SUCCESSOR_TX_APPLY_MS), + ] + volumes.append(client.V1Volume( + name='synthetic-worker', + config_map=client.V1ConfigMapVolumeSource( + name=SYNTHETIC_WORKER_CONFIG_MAP))) + volume_mounts.append(client.V1VolumeMount( + name='synthetic-worker', mount_path='/synthetic', read_only=True)) # Require and avoid go in ONE matchExpressions list: expressions within a # term are ANDed, whereas separate terms are ORed and an avoid-only pod would @@ -2186,10 +2246,10 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): container = client.V1Container( name='stellar-core', image=CORE_IMAGE, - command=['/bin/sh', '-c', script], env=env, resources=_resources(mem, eph, end), + image_pull_policy=image_pull_policy, + command=command, env=env, resources=_resources(mem, eph, end), ports=[client.V1ContainerPort(container_port=11626, name='http')], - volume_mounts=[client.V1VolumeMount(name='data', mount_path='/data'), - client.V1VolumeMount(name='config', mount_path='/config')]) + volume_mounts=volume_mounts) return client.V1Job( metadata=client.V1ObjectMeta( @@ -2240,9 +2300,7 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): termination_grace_period_seconds=WORKER_GRACE_SECONDS, affinity=affinity, tolerations=tolerations, containers=[container], - volumes=[data_vol, client.V1Volume( - name='config', config_map=client.V1ConfigMapVolumeSource( - name=f"{RUN_NAME}-stellar-core-config"))])))) + volumes=volumes)))) # --- reconcile -------------------------------------------------------------- diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 54180b31..22e81729 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -33,6 +33,7 @@ import logging import os import re +import signal import ssl import sys import zlib @@ -213,6 +214,15 @@ def discard(end, attempt): # silently missing for 25% of ranges -- 91-99% of everything above ledger 35M, # exactly the expensive end. _SUM_RE = re.compile(r"sum\s*=\s*([0-9.]+(?:[eE][+-]?[0-9]+)?)ms") +_SYNTHETIC_PEAK_RE = re.compile( + r"SYNTHETIC PEAK: anonBytes=(\d+) workingSetBytes=(\d+)") +SYNTHETIC_WORKER = os.getenv('SYNTHETIC_WORKER', '').lower() == 'true' + + +def _install_synthetic_restart_handler(): + """Allow a collector-only container restart in the opt-in live harness.""" + if SYNTHETIC_WORKER: + signal.signal(signal.SIGTERM, lambda _signum, _frame: sys.exit(0)) class TxApplyScanner: @@ -238,12 +248,19 @@ def __init__(self, recreated=False): self.seconds = None self.resumed = False self.resume_decided = False + self.synthetic_anon = None + self.synthetic_working_set = None # A new poller starting from durable .state missed every earlier line. # Finalization must recover scanner-only facts from the archive. self.recreated = recreated self._left = 0 def feed(self, line): + if SYNTHETIC_WORKER: + peak = _SYNTHETIC_PEAK_RE.search(line) + if peak: + self.synthetic_anon = int(peak.group(1)) + self.synthetic_working_set = int(peak.group(2)) if self.RESUME_MARK in line: self.resumed = True self.resume_decided = True @@ -469,6 +486,8 @@ async def sample_kubelet(session, nodes): OOM. The `time` field on this payload runs 1-3s behind wall clock; the ~80s lag applies only to the du-based ephemeral figure alongside it. """ + if SYNTHETIC_WORKER: + return for node in nodes: url = f"{API}/api/v1/nodes/{node}/proxy/stats/summary" try: @@ -578,7 +597,8 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): # finalization; recover only the state this scanner could have missed. archived = None need_resume = int(attempt) > 1 and not tx.resume_decided - need_tx = tx.recreated and tx.seconds is None + need_tx = (tx.recreated and tx.seconds is None) or ( + SYNTHETIC_WORKER and tx.recreated) if need_resume or need_tx: archived = scan_archive(end, attempt, need_tx=need_tx) if tx.resumed or (archived is not None and archived.resumed): @@ -591,6 +611,10 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): tx_seconds = archived.seconds if tx_seconds is not None: measured['txApplySeconds'] = tx_seconds + synthetic = archived if archived is not None else tx + if SYNTHETIC_WORKER and synthetic.synthetic_anon is not None: + measured['peakAnonBytes'] = synthetic.synthetic_anon + measured['peakWorkingSetBytes'] = synthetic.synthetic_working_set _peak_flushed.pop(pod, None) _peak_flushed.pop(pod + '/eph', None) _streaming.pop(pod, None) @@ -956,4 +980,5 @@ async def main(): if __name__ == '__main__': + _install_synthetic_restart_handler() asyncio.run(main()) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 9b990b1d..789d9f8b 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -134,8 +134,10 @@ spec: command: ["/bin/sh", "-c"] args: - >- + {{- if .Values.monitor.sourceInstallDependencies }} pip install --no-cache-dir -q 'kubernetes~=35.0' 'aiohttp~=3.9' 'requests~=2.31' 'prometheus-client~=0.19' && + {{- end }} exec python3 /app/job_monitor.py {{- end }} ports: @@ -231,6 +233,30 @@ spec: value: {{ .Values.monitor.attemptDeadlineSeconds | quote }} - name: JOB_TTL_SECONDS value: {{ .Values.monitor.jobTtlSeconds | quote }} + {{- if .Values.integration.syntheticWorker.enabled }} + - name: SYNTHETIC_WORKER_CONFIG_MAP + value: {{ .Release.Name }}-synthetic-worker + - name: SYNTHETIC_WORKER_IMAGE_PULL_POLICY + value: {{ .Values.integration.syntheticWorker.imagePullPolicy | quote }} + - name: SYNTHETIC_PREDECESSOR_SECONDS + value: {{ .Values.integration.syntheticWorker.predecessorSeconds | quote }} + - name: SYNTHETIC_SUCCESSOR_MINIMUM_SECONDS + value: {{ .Values.integration.syntheticWorker.successorMinimumSeconds | quote }} + - name: SYNTHETIC_MAXIMUM_WAIT_SECONDS + value: {{ .Values.integration.syntheticWorker.maximumWaitSeconds | quote }} + - name: SYNTHETIC_PREDECESSOR_ANON_MIB + value: {{ .Values.integration.syntheticWorker.predecessorAnonMiB | quote }} + - name: SYNTHETIC_PREDECESSOR_WORKING_SET_MIB + value: {{ .Values.integration.syntheticWorker.predecessorWorkingSetMiB | quote }} + - name: SYNTHETIC_SUCCESSOR_ANON_MIB + value: {{ .Values.integration.syntheticWorker.successorAnonMiB | quote }} + - name: SYNTHETIC_SUCCESSOR_WORKING_SET_MIB + value: {{ .Values.integration.syntheticWorker.successorWorkingSetMiB | quote }} + - name: SYNTHETIC_PREDECESSOR_TX_APPLY_MS + value: {{ .Values.integration.syntheticWorker.predecessorTxApplyMilliseconds | quote }} + - name: SYNTHETIC_SUCCESSOR_TX_APPLY_MS + value: {{ .Values.integration.syntheticWorker.successorTxApplyMilliseconds | quote }} + {{- end }} - name: LOG_DIR value: /logs {{- if .Values.monitor.profileConfigMap }} @@ -335,8 +361,10 @@ spec: command: ["/bin/sh", "-c"] args: - >- + {{- if .Values.monitor.sourceInstallDependencies }} pip install --no-cache-dir -q 'kubernetes~=35.0' 'aiohttp~=3.9' 'requests~=2.31' 'prometheus-client~=0.19' && + {{- end }} exec python3 /app/log_collector.py {{- else }} command: ["/usr/bin/python3", "log_collector.py"] @@ -371,6 +399,10 @@ spec: # this the collector defaults to pvc and silently records nothing. - name: STORAGE_MODE value: {{ .Values.worker.storageMode | quote }} + {{- if .Values.integration.syntheticWorker.enabled }} + - name: SYNTHETIC_WORKER + value: "true" + {{- end }} resources: {{- toYaml .Values.monitor.collectorResources | nindent 12 }} volumeMounts: diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/synthetic_worker.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/synthetic_worker.yaml new file mode 100644 index 00000000..eae470b9 --- /dev/null +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/synthetic_worker.yaml @@ -0,0 +1,96 @@ +{{- if .Values.integration.syntheticWorker.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-synthetic-worker +data: + worker.py: | + import json + import os + import sys + import time + + + def required_int(name): + value = int(os.environ[name]) + if value < 0: + raise ValueError(f"{name} must be non-negative") + return value + + + def emit(message): + print(message, flush=True) + + + def write_json(path, value): + tmp = path + ".tmp" + with open(tmp, "w") as stream: + json.dump(value, stream, sort_keys=True) + os.replace(tmp, path) + + + def hold_memory(mib): + memory = bytearray(mib * 1024 * 1024) + for offset in range(0, len(memory), 4096): + memory[offset] = 1 + return memory + + + def emit_tx_apply(milliseconds): + emit("metric 'ledger.transaction.apply'") + emit(f"sum = {milliseconds}ms") + + + attempt = required_int("SYNTHETIC_ATTEMPT") + target = required_int("SYNTHETIC_TARGET") + count = required_int("SYNTHETIC_COUNT") + key = os.environ["SYNTHETIC_KEY"] + data_dir = os.environ.get("SYNTHETIC_DATA_DIR", "/data") + state_path = os.path.join(data_dir, ".synthetic-replay.json") + ready_path = os.path.join(data_dir, ".synthetic-successor-ready") + release_path = os.path.join(data_dir, ".synthetic-release") + + if attempt == 1: + anon_mib = required_int("SYNTHETIC_PREDECESSOR_ANON_MIB") + working_set_mib = required_int("SYNTHETIC_PREDECESSOR_WORKING_SET_MIB") + duration = float(os.environ["SYNTHETIC_PREDECESSOR_SECONDS"]) + tx_apply = required_int("SYNTHETIC_PREDECESSOR_TX_APPLY_MS") + reached = max(target - 1, target - count) + write_json(state_path, {"key": key, "reachedLedger": reached}) + memory = hold_memory(working_set_mib) + emit(f"SYNTHETIC PEAK: anonBytes={anon_mib * 1024 * 1024} " + f"workingSetBytes={working_set_mib * 1024 * 1024}") + emit(f"SYNTHETIC PREDECESSOR: {key} persisted ledger {reached}") + time.sleep(duration) + emit_tx_apply(tx_apply) + del memory + sys.exit(3) + + with open(state_path) as stream: + state = json.load(stream) + if state.get("key") != key: + raise RuntimeError("PVC state belongs to a different logical range") + reached = int(state["reachedLedger"]) + emit(f"RESUME PROBE: offline-info reports lcl {reached}") + emit(f"RESUME: {key} reached ledger {reached}, replay had started; skipping new-db") + + anon_mib = required_int("SYNTHETIC_SUCCESSOR_ANON_MIB") + working_set_mib = required_int("SYNTHETIC_SUCCESSOR_WORKING_SET_MIB") + minimum = float(os.environ["SYNTHETIC_SUCCESSOR_MINIMUM_SECONDS"]) + maximum = float(os.environ["SYNTHETIC_MAXIMUM_WAIT_SECONDS"]) + tx_apply = required_int("SYNTHETIC_SUCCESSOR_TX_APPLY_MS") + memory = hold_memory(working_set_mib) + emit(f"SYNTHETIC PEAK: anonBytes={anon_mib * 1024 * 1024} " + f"workingSetBytes={working_set_mib * 1024 * 1024}") + write_json(ready_path, {"attempt": attempt, "key": key, "reachedLedger": reached}) + + started = time.monotonic() + while time.monotonic() - started < maximum: + elapsed = time.monotonic() - started + if elapsed >= minimum and os.path.exists(release_path): + emit_tx_apply(tx_apply) + del memory + sys.exit(0) + time.sleep(0.25) + raise RuntimeError("timed out waiting for the synthetic release marker") +{{- end }} diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 99232e79..a2a8de2b 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -65,6 +65,11 @@ monitor: # ConfigMap name and point monitor.image at a plain python base; the deps the # Dockerfile bakes get pip-installed at start. Empty = use the image as built. sourceConfigMap: "" + # Source-mode development normally installs dependencies at container start. + # Disable only when the selected image already contains them, for example in + # the opt-in synthetic integration harness where external package access is + # intentionally avoided. + sourceInstallDependencies: true # Range profile from an earlier run, as a ConfigMap holding profile.json. # The mission driver resolves --pubnet-parallel-catchup-profile (a local # path or an https URL) into one. Empty = size from the configured @@ -181,5 +186,23 @@ monitor: requests: { cpu: "200m", memory: "512Mi" } limits: { cpu: "2", memory: "2Gi" } +# Fixed, chart-owned worker used only by the bounded Kubernetes integration +# harness. It never invokes stellar-core or history services. Keeping the script +# in the chart instead of accepting a command value avoids turning this into a +# general arbitrary-command surface. +integration: + syntheticWorker: + enabled: false + imagePullPolicy: IfNotPresent + predecessorSeconds: 12 + successorMinimumSeconds: 12 + maximumWaitSeconds: 180 + predecessorAnonMiB: 48 + predecessorWorkingSetMiB: 56 + successorAnonMiB: 24 + successorWorkingSetMiB: 32 + predecessorTxApplyMilliseconds: 1250 + successorTxApplyMilliseconds: 2500 + service_account: annotations: [] diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py b/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py index e02a8ca5..02d82b71 100644 --- a/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py +++ b/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py @@ -33,6 +33,7 @@ # by no template at all -- absent from here, that stays invisible. FULL = ( 'monitor.profileConfigMap=p', + 'integration.syntheticWorker.enabled=true', 'worker.requireNodeLabels[0].key=purpose', 'worker.requireNodeLabels[0].operator=In', 'worker.requireNodeLabels[0].values[0]=catchup8-spot', diff --git a/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py b/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py index 50635798..b0057eb3 100644 --- a/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py +++ b/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py @@ -259,6 +259,34 @@ def test_the_worker_runs_the_resume_script_for_its_own_range(job): assert f'catchup "$KEY"' in script +def test_synthetic_worker_is_absent_from_the_default_job(job): + container = job.spec.template.spec.containers[0] + assert container.image_pull_policy is None + assert 'synthetic-worker' not in {v.name for v in job.spec.template.spec.volumes} + assert not any(e.name.startswith('SYNTHETIC_') for e in container.env) + + +def test_opt_in_synthetic_worker_uses_only_the_fixed_chart_script(job, monkeypatch): + monkeypatch.setattr(jm, 'SYNTHETIC_WORKER_CONFIG_MAP', 'pc-synthetic-worker') + monkeypatch.setattr(jm, 'SYNTHETIC_WORKER_IMAGE_PULL_POLICY', 'IfNotPresent') + + synthetic = jm.build_job(31005951, 16320, 2, None) + container = synthetic.spec.template.spec.containers[0] + env = {e.name: e.value for e in container.env} + volumes = {v.name: v for v in synthetic.spec.template.spec.volumes} + mounts = {m.name: m.mount_path for m in container.volume_mounts} + + assert container.command == ['python3', '/synthetic/worker.py'] + assert container.image == 'stellar/stellar-core:test' + assert container.image_pull_policy == 'IfNotPresent' + assert env['SYNTHETIC_ATTEMPT'] == '2' + assert env['SYNTHETIC_TARGET'] == '31005951' + assert env['SYNTHETIC_COUNT'] == '16320' + assert env['SYNTHETIC_KEY'] == jm.job_key(31005951, 16320) + assert volumes['synthetic-worker'].config_map.name == 'pc-synthetic-worker' + assert mounts['synthetic-worker'] == '/synthetic' + + def test_the_worker_mounts_the_config_the_chart_renders(job): """The stellar-core.cfg ConfigMap is the chart's, named off the release. diff --git a/src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py b/src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py new file mode 100644 index 00000000..e0173a54 --- /dev/null +++ b/src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py @@ -0,0 +1,89 @@ +"""Safety and profile assertions for the opt-in live runner.""" + +import importlib.util +from pathlib import Path + +import pytest + + +PATH = ( + Path(__file__).resolve().parents[2] + / 'integration' / 'synthetic_resume_harness.py') +SPEC = importlib.util.spec_from_file_location('synthetic_resume_harness', PATH) +HARNESS = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(HARNESS) + + +def test_scope_guard_accepts_only_unique_sandbox_release_names(): + HARNESS.validate_scope('sandbox', 'mpc-resume-a1b2c3') + for namespace, release in ( + ('stellar-supercluster', 'mpc-resume-a1b2c3'), + ('default', 'mpc-resume-a1b2c3'), + ('sandbox', 'parallel-catchup-ssc-1959z-ef177a-r5'), + ('sandbox', 'mpc-resume-short')): + with pytest.raises(HARNESS.HarnessError): + HARNESS.validate_scope(namespace, release) + + +def test_render_inspection_rejects_cluster_scoped_or_unprefixed_resources(): + manifest = """ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: mpc-resume-a1b2c3 +rules: [] +""" + with pytest.raises(HARNESS.HarnessError): + HARNESS.inspect_rendered( + manifest, 'mpc-resume-a1b2c3', 'stellar/ssc-job-monitor:latest') + + +def test_profile_assertion_requires_both_legs_and_predecessor_peaks(): + bundle = { + 'a1_metrics': { + 'attemptSeconds': 12.0, + 'peakAnonBytes': 48 * 1024 * 1024, + 'peakWorkingSetBytes': 56 * 1024 * 1024, + 'txApplySeconds': 1.25, + }, + 'a1_outcome': {'attemptSeconds': 12.0, 'outcome': 'failed'}, + 'a1_verdict': 'failed', + 'a1_log': "metric 'ledger.transaction.apply'\nsum = 1250ms\n", + 'a1_done': True, + 'a2_metrics': { + 'attemptSeconds': 14.0, + 'peakAnonBytes': 24 * 1024 * 1024, + 'peakWorkingSetBytes': 32 * 1024 * 1024, + 'txApplySeconds': 2.5, + 'resumed': True, + }, + 'a2_log': ( + 'RESUME PROBE: offline-info reports lcl 63\n' + 'RESUME: 64/64 reached ledger 63, replay had started; skipping new-db\n'), + 'a2_done': True, + 'files': [ + 'range-64-a1.done', 'range-64-a1.log.gz', + 'range-64-a1.metrics', 'range-64-a1.outcome', + 'range-64-a1.verdict', 'range-64-a2.done', + 'range-64-a2.log.gz', 'range-64-a2.metrics', + ], + 'progress': {'completed': {'64': { + 'attempts': 2, + 'seconds': 26.0, + 'peakAnonBytes': 48 * 1024 * 1024, + 'peakWorkingSetBytes': 56 * 1024 * 1024, + 'txApply': 3.75, + }}}, + } + result = HARNESS.assert_completed_profile(bundle) + assert result['expectedChainSecondsFromArtifacts'] == 26.0 + + bundle['progress']['completed']['64']['peakAnonBytes'] = 24 * 1024 * 1024 + with pytest.raises(HARNESS.HarnessError): + HARNESS.assert_completed_profile(bundle) + + +def test_runner_uses_a_handled_signal_for_collector_only_restart(): + source = PATH.read_text() + assert 'kill -TERM 1' in source + assert 'kill -9 1' not in source diff --git a/src/MissionParallelCatchup/tests/contract/test_synthetic_worker.py b/src/MissionParallelCatchup/tests/contract/test_synthetic_worker.py new file mode 100644 index 00000000..59d22b2f --- /dev/null +++ b/src/MissionParallelCatchup/tests/contract/test_synthetic_worker.py @@ -0,0 +1,101 @@ +"""Default-off and deterministic contracts for the live integration worker.""" + +import os +import subprocess +import sys + +import log_collector as lc + +import _artifacts as art + + +ENABLED = ('integration.syntheticWorker.enabled=true',) +MIB = 1024 * 1024 + + +def test_default_render_has_no_synthetic_resource_or_runtime_switch(): + names = {d['metadata']['name'] for d in art.docs()} + assert 't-synthetic-worker' not in names + for container in art.containers().values(): + env = set(art.env_of(container)) + assert not any(name.startswith('SYNTHETIC_') for name in env) + + +def test_opt_in_render_adds_fixed_worker_and_narrow_runtime_wiring(): + config_maps = {d['metadata']['name']: d + for d in art.of_kind('ConfigMap', ENABLED)} + worker = config_maps['t-synthetic-worker'] + assert set(worker['data']) == {'worker.py'} + assert 'stellar-core' not in worker['data']['worker.py'] + assert 'subprocess' not in worker['data']['worker.py'] + + containers = art.containers(ENABLED) + monitor_env = art.env_of(containers[art.MONITOR_CONTAINER]) + collector_env = art.env_of(containers[art.COLLECTOR_CONTAINER]) + assert monitor_env['SYNTHETIC_WORKER_CONFIG_MAP'] == 't-synthetic-worker' + assert collector_env['SYNTHETIC_WORKER'] == 'true' + + +def test_source_mode_can_skip_dependency_install_without_changing_its_default(): + source = ('monitor.sourceConfigMap=source',) + for container in art.containers(source).values(): + assert 'pip install' in ' '.join(container.get('args') or []) + + offline = source + ('monitor.sourceInstallDependencies=false',) + for container in art.containers(offline).values(): + command = ' '.join(container.get('args') or []) + assert 'pip install' not in command + assert 'exec python3 /app/' in command + + +def test_fixed_worker_persists_then_resumes_the_same_pvc(tmp_path): + worker = next(d for d in art.of_kind('ConfigMap', ENABLED) + if d['metadata']['name'] == 't-synthetic-worker') + script = tmp_path / 'worker.py' + script.write_text(worker['data']['worker.py']) + env = { + **os.environ, + 'SYNTHETIC_DATA_DIR': str(tmp_path), + 'SYNTHETIC_TARGET': '64', + 'SYNTHETIC_COUNT': '64', + 'SYNTHETIC_KEY': '64/64', + 'SYNTHETIC_PREDECESSOR_SECONDS': '0', + 'SYNTHETIC_SUCCESSOR_MINIMUM_SECONDS': '0', + 'SYNTHETIC_MAXIMUM_WAIT_SECONDS': '1', + 'SYNTHETIC_PREDECESSOR_ANON_MIB': '2', + 'SYNTHETIC_PREDECESSOR_WORKING_SET_MIB': '3', + 'SYNTHETIC_SUCCESSOR_ANON_MIB': '1', + 'SYNTHETIC_SUCCESSOR_WORKING_SET_MIB': '2', + 'SYNTHETIC_PREDECESSOR_TX_APPLY_MS': '1250', + 'SYNTHETIC_SUCCESSOR_TX_APPLY_MS': '2500', + } + + first = subprocess.run( + [sys.executable, str(script)], env={**env, 'SYNTHETIC_ATTEMPT': '1'}, + capture_output=True, text=True, timeout=5) + assert first.returncode == 3 + assert 'SYNTHETIC PREDECESSOR: 64/64 persisted ledger 63' in first.stdout + assert 'sum = 1250ms' in first.stdout + + (tmp_path / '.synthetic-release').touch() + second = subprocess.run( + [sys.executable, str(script)], env={**env, 'SYNTHETIC_ATTEMPT': '2'}, + capture_output=True, text=True, timeout=5) + assert second.returncode == 0, second.stderr + assert 'RESUME PROBE: offline-info reports lcl 63' in second.stdout + assert 'RESUME: 64/64 reached ledger 63, replay had started; skipping new-db' \ + in second.stdout + assert 'sum = 2500ms' in second.stdout + + +def test_synthetic_peak_marker_is_inert_unless_the_harness_is_enabled(monkeypatch): + line = f'SYNTHETIC PEAK: anonBytes={48 * MIB} workingSetBytes={56 * MIB}' + scanner = lc.TxApplyScanner() + scanner.feed(line) + assert scanner.synthetic_anon is None + + monkeypatch.setattr(lc, 'SYNTHETIC_WORKER', True) + scanner = lc.TxApplyScanner() + scanner.feed(line) + assert scanner.synthetic_anon == 48 * MIB + assert scanner.synthetic_working_set == 56 * MIB diff --git a/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py b/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py index 016ce702..857d27b0 100644 --- a/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py +++ b/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py @@ -213,6 +213,36 @@ def test_finalize_recovers_txapply_after_the_scanner_is_recreated(vol, monkeypat "the test must prove recovery happened before success-log discard" +def test_synthetic_peaks_survive_the_same_scanner_recreation(vol, monkeypatch): + monkeypatch.setattr(lc, 'SYNTHETIC_WORKER', True) + path = lc.base('300', 2) + '.log.gz' + with gzip.open(path, 'wt') as fh: + fh.write('RESUME: local state reached ledger 250; skipping new-db\n') + fh.write('SYNTHETIC PEAK: anonBytes=50331648 workingSetBytes=58720256\n') + fh.write("metric 'ledger.transaction.apply'\n") + fh.write('sum = 2500ms\n') + + finalize('w-300-a2', 300, attempt=2, tx=lc.TxApplyScanner(recreated=True)) + + assert metrics(300, 2) == { + 'peakAnonBytes': 50331648, + 'peakWorkingSetBytes': 58720256, + 'resumed': True, + 'txApplySeconds': 2.5, + } + + +def test_synthetic_mode_never_overwrites_fixed_peaks_from_kubelet(vol, monkeypatch): + monkeypatch.setattr(lc, 'SYNTHETIC_WORKER', True) + session = FakeSession(summary( + 'w-300-a1', rss=80 * GIB, ws=90 * GIB)) + + run(lc.sample_kubelet(session, ['node-1'])) + + assert session.urls == [] + assert metrics(300) is None + + def test_finalize_does_not_promote_resume_declined(vol): path = lc.base('300', 2) + '.log.gz' with gzip.open(path, 'wt') as fh: From 106a563d1b62429ce22081540324e35ee1bb43d0 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 18:33:59 -0400 Subject: [PATCH 053/117] Port the live monitor's liveness sampler and runtime memory insurance The mission running on ssc-test carries two subsystems this tree never had, both authored in the session driving that run. Taking that copy as the base and re-applying the deadline work on top, since the live one is the version with 2900 ranges of evidence behind it. - WorkerLivenessSampler replaces the inline ping loop. Probing 1000+ workers serially outlasts the poll interval, so it samples threaded and bounded (LIVENESS_MAX_CONCURRENCY=32). - PROFILE_RUNTIME_MEMORY_INSURANCE hands out an allowance scaled by seconds/longest. A flat margin misprices both ends: short ranges are measured tightly and need nothing, long ones sit exposed to archive growth and cache drift for hours. - log_collector.py needed no merge -- byte-identical to the live pod's. Re-applied on top: _attempt_deadline/PROFILE_DEADLINE_FACTOR, the deadline's move from PodSpec to JobSpec, and terminal timeout semantics. Chart gained the five env vars the live deployment sets. 512 tests + 1 xfail, unchanged from before the merge. --- src/MissionParallelCatchup/job_monitor.py | 484 +++++++++++++++--- .../templates/job_monitor.yaml | 12 + .../parallel_catchup_helm/values.yaml | 11 + 3 files changed, 440 insertions(+), 67 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 2b5d8ad2..e0b9ab98 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -19,7 +19,9 @@ import bisect import json import logging +import math import os +import queue import re import sys import tempfile @@ -33,6 +35,7 @@ from kubernetes.client.rest import ApiException from prometheus_client import (CONTENT_TYPE_LATEST, REGISTRY, Counter, Gauge, Histogram, generate_latest) +import requests # Histogram buckets # 5m 15m 30m 1h 1.5h 2h @@ -126,6 +129,9 @@ # slack for all growth and cache -- and 90 of them OOMKilled within 90s. The # earlier 4Gi validation hid this because 1.1x of 2.4 GiB is 240 MiB of slack. PROFILE_CACHE_HEADROOM = os.getenv('PROFILE_CACHE_HEADROOM', '512Mi') +# Extra allowance scaled by the range's measured runtime. Long ranges keep more +# page cache and allocator slack live at once; 0 disables the allowance. +PROFILE_RUNTIME_MEMORY_INSURANCE = os.getenv('PROFILE_RUNTIME_MEMORY_INSURANCE', '3Gi') REQ_EPHEMERAL = os.getenv('REQ_EPHEMERAL', '') LIM_EPHEMERAL = os.getenv('LIM_EPHEMERAL', '') @@ -248,6 +254,33 @@ # stops all dispatch, so restart the container rather than run half-alive. RECONCILE_STALE_SECONDS = float(os.getenv('WATCH_STALE_SECONDS', 600)) +# Worker responsiveness is cosmetic and sampled independently from reconcile. +# Thirty seconds and three failures restore the old ~90-second down threshold, +# while a five-second request budget gives a busy admin endpoint substantially +# more room than the old one-shot two-second probe. +LIVENESS_PROBE_INTERVAL_SECONDS = os.getenv('LIVENESS_PROBE_INTERVAL_SECONDS', '30') +LIVENESS_PROBE_TIMEOUT_SECONDS = os.getenv('LIVENESS_PROBE_TIMEOUT_SECONDS', '5') +LIVENESS_FAILURE_THRESHOLD = os.getenv('LIVENESS_FAILURE_THRESHOLD', '3') +LIVENESS_MAX_CONCURRENCY = os.getenv('LIVENESS_MAX_CONCURRENCY', '32') +try: + LIVENESS_PROBE_INTERVAL_SECONDS = float(LIVENESS_PROBE_INTERVAL_SECONDS) + LIVENESS_PROBE_TIMEOUT_SECONDS = float(LIVENESS_PROBE_TIMEOUT_SECONDS) + LIVENESS_FAILURE_THRESHOLD = int(LIVENESS_FAILURE_THRESHOLD) + LIVENESS_MAX_CONCURRENCY = int(LIVENESS_MAX_CONCURRENCY) +except ValueError as e: + raise ValueError( + "LIVENESS_PROBE_INTERVAL_SECONDS and LIVENESS_PROBE_TIMEOUT_SECONDS " + "must be numbers; LIVENESS_FAILURE_THRESHOLD and " + "LIVENESS_MAX_CONCURRENCY must be integers") from e + +for _name, _value in ( + ('LIVENESS_PROBE_INTERVAL_SECONDS', LIVENESS_PROBE_INTERVAL_SECONDS), + ('LIVENESS_PROBE_TIMEOUT_SECONDS', LIVENESS_PROBE_TIMEOUT_SECONDS), + ('LIVENESS_FAILURE_THRESHOLD', LIVENESS_FAILURE_THRESHOLD), + ('LIVENESS_MAX_CONCURRENCY', LIVENESS_MAX_CONCURRENCY)): + if _value <= 0: + raise ValueError(f"{_name} must be greater than zero, got {_value!r}") + # Shared with the log-collector sidecar, which owns writes here: it streams each # worker's log and records the .outcome verdict while the pod still exists. LOG_DIR = os.getenv('LOG_DIR', '/logs') @@ -389,6 +422,290 @@ def check_storage_config(): metric_eph_retries = Counter('ssc_parallel_catchup_job_ephemeral_retried_count', 'Jobs retried with an escalated ephemeral-storage limit') +def _worker_targets(pods): + """Current Running-with-IP pods, keyed by pod identity. + + A UID change is a replacement even when the Job name or IP is reused. Tests + and unusually incomplete API objects may lack a UID, where the pod name is + still unique for its lifetime. + """ + out = {} + for pod in pods: + pod_status = getattr(pod, 'status', None) + metadata = getattr(pod, 'metadata', None) + ip = getattr(pod_status, 'pod_ip', None) + if getattr(pod_status, 'phase', None) != 'Running' or not ip or metadata is None: + continue + name = getattr(metadata, 'name', None) + identity = getattr(metadata, 'uid', None) or name + if identity and name: + out[str(identity)] = (str(name), str(ip)) + return out + + +class WorkerLivenessSampler: + """Bounded, round-robin stellar-core `/info` sampler. + + Candidate membership comes from the authoritative Kubernetes snapshot, but + all network I/O happens on this sampler's fixed worker pool. At most + `max_concurrency` requests run and the same number wait in the bounded queue; + there is no future, task, session, or thread per pod. + + State is deliberately conservative: + * new or replaced pod: unknown + * any HTTP response from /info: up + * fewer than `failure_threshold` consecutive exceptions/timeouts: unknown + * `failure_threshold` consecutive failures: down + * any later response: up immediately + + HTTP error statuses still prove the admin endpoint responded. A busy core + returning 5xx is responsive; only failure to receive an HTTP response counts + toward down. + """ + + def __init__(self, interval=LIVENESS_PROBE_INTERVAL_SECONDS, + timeout=LIVENESS_PROBE_TIMEOUT_SECONDS, + failure_threshold=LIVENESS_FAILURE_THRESHOLD, + max_concurrency=LIVENESS_MAX_CONCURRENCY, probe=None): + if interval <= 0 or timeout <= 0 or failure_threshold <= 0 or max_concurrency <= 0: + raise ValueError("liveness sampler values must all be greater than zero") + self.interval = float(interval) + self.timeout = float(timeout) + self.failure_threshold = int(failure_threshold) + self.max_concurrency = int(max_concurrency) + self._probe = probe + self._records = {} + self._generation = 0 + self._tasks = queue.Queue(maxsize=self.max_concurrency) + self._stop = threading.Event() + self._condition = threading.Condition() + self._scheduler = None + self._workers = [] + self._started = False + self._failed = None + self._active = 0 + self._failure_count = 0 + self._last_failure_log = 0.0 + + def start(self): + with self._condition: + if self._started: + return + self._started = True + self._workers = [ + threading.Thread(target=self._worker_main, + name=f"worker-liveness-{i}", daemon=True) + for i in range(self.max_concurrency) + ] + self._scheduler = threading.Thread( + target=self._scheduler_main, name="worker-liveness-scheduler", + daemon=True) + for worker in self._workers: + worker.start() + self._scheduler.start() + + def close(self): + self._stop.set() + with self._condition: + self._condition.notify_all() + threads = ([self._scheduler] if self._scheduler is not None else []) + self._workers + deadline = time.monotonic() + self.timeout + 1.0 + for thread in threads: + remaining = max(0.0, deadline - time.monotonic()) + if thread is not None and thread is not threading.current_thread(): + thread.join(remaining) + + def replace_candidates(self, targets, now=None): + """Atomically replace membership without waiting for any probe.""" + now = time.monotonic() if now is None else float(now) + targets = dict(targets) + with self._condition: + old = self._records + records = {} + new_identities = [ + identity for identity in sorted(targets) + if identity not in old or old[identity]['target'] != targets[identity] + ] + offsets = { + identity: self.interval * index / max(1, len(new_identities)) + for index, identity in enumerate(new_identities) + } + for identity, target in targets.items(): + previous = old.get(identity) + if previous is not None and previous['target'] == target: + records[identity] = previous + continue + self._generation += 1 + records[identity] = { + 'target': target, + 'generation': self._generation, + 'status': 'unknown', + 'failures': 0, + 'queued': False, + 'next_due': now + offsets[identity], + } + self._records = records + self._condition.notify_all() + + def counts(self, expected_count=None): + with self._condition: + count = len(self._records) if expected_count is None else int(expected_count) + healthy = self._started and self._failed is None + if healthy: + healthy = (self._scheduler is not None and self._scheduler.is_alive() + and all(worker.is_alive() for worker in self._workers)) + if not healthy or count != len(self._records): + return {'up': 0, 'down': 0, 'unknown': count} + result = {'up': 0, 'down': 0, 'unknown': 0} + for record in self._records.values(): + result[record['status']] += 1 + return result + + def stats(self): + """Small observability hook used by the scale contract test.""" + with self._condition: + live_threads = sum( + 1 for thread in ([self._scheduler] + self._workers) + if thread is not None and thread.is_alive()) + return { + 'records': len(self._records), + 'active': self._active, + 'queued': self._tasks.qsize(), + 'outstanding': self._active + self._tasks.qsize(), + 'threads': live_threads, + 'failed': self._failed, + } + + def _scheduler_main(self): + try: + self._schedule() + except Exception as e: + self._mark_failed("scheduler", e) + + def _schedule(self): + while not self._stop.is_set(): + with self._condition: + now = time.monotonic() + capacity = self.max_concurrency - self._tasks.qsize() + due = sorted( + ((record['next_due'], identity, record) + for identity, record in self._records.items() + if not record['queued'] and record['next_due'] <= now), + key=lambda item: (item[0], item[1])) + for _, identity, record in due[:max(0, capacity)]: + task = (identity, record['generation'], record['target']) + try: + self._tasks.put_nowait(task) + except queue.Full: + break + record['queued'] = True + + waiting = [ + record['next_due'] for record in self._records.values() + if not record['queued'] + ] + delay = max(0.01, min(1.0, min(waiting) - now)) if waiting else 1.0 + self._condition.wait(timeout=delay) + + def _worker_main(self): + session = None + try: + if self._probe is None: + session = requests.Session() + adapter = requests.adapters.HTTPAdapter( + pool_connections=4, pool_maxsize=1, max_retries=0) + session.mount('http://', adapter) + while not self._stop.is_set(): + try: + task = self._tasks.get(timeout=0.2) + except queue.Empty: + continue + with self._condition: + self._active += 1 + identity, generation, target = task + success = False + error = None + try: + _, ip = target + if self._probe is None: + host = f"[{ip}]" if ':' in ip else ip + with session.get(f"http://{host}:11626/info", + timeout=self.timeout): + pass + else: + self._probe(ip, self.timeout) + success = True + except Exception as e: + error = e + finally: + self._record_result(identity, generation, target, success, error) + self._tasks.task_done() + with self._condition: + self._active -= 1 + self._condition.notify_all() + except Exception as e: + self._mark_failed("probe worker", e) + finally: + if session is not None: + session.close() + + def _record_result(self, identity, generation, target, success, error=None, + now=None): + now = time.monotonic() if now is None else float(now) + log_failure = None + with self._condition: + record = self._records.get(identity) + if (record is None or record['generation'] != generation + or record['target'] != target): + return + record['queued'] = False + record['next_due'] = now + self.interval + if success: + record['failures'] = 0 + record['status'] = 'up' + else: + record['failures'] += 1 + record['status'] = ( + 'down' if record['failures'] >= self.failure_threshold + else 'unknown') + self._failure_count += 1 + if now - self._last_failure_log >= 60.0: + log_failure = self._failure_count + self._failure_count = 0 + self._last_failure_log = now + self._condition.notify_all() + if log_failure is not None: + logger.warning( + "stellar-core /info liveness probes are failing; %d failure(s) " + "across the fleet since the previous warning (latest: %s: %s)", + log_failure, target[0], error) + + def _mark_failed(self, component, error): + with self._condition: + if self._failed is not None: + return + self._failed = f"{component}: {error}" + self._condition.notify_all() + logger.exception( + "worker liveness %s failed; all current workers will be reported " + "unknown and reconcile will continue", component) + + +worker_liveness_sampler = WorkerLivenessSampler() + + +def publish_worker_liveness(targets, sampler=None): + """Hand a pod snapshot to the sampler and return its current three counts. + + This path copies O(current workers) state under a short lock but never makes + a request or waits for an in-flight request. Keeping it separate makes the + non-blocking boundary directly testable. + """ + sampler = sampler or worker_liveness_sampler + sampler.replace_candidates(targets) + return sampler.counts(len(targets)) + + class RequestHandler(BaseHTTPRequestHandler): def do_GET(self): if self.path == '/healthz': @@ -945,6 +1262,38 @@ def _verdict_of(end, attempt): return (read_outcome(end, attempt) or {}).get('outcome') +# Multiple of a range's own measured runtime to allow before calling it wedged. +# The deadline exists for ONE failure mode, reproduced 2026-07-30: with an +# unreachable archive, stellar-core retries the bucket download forever. It logs +# "Missing HAS for ledger N: maybe stale archive", re-selects a different mirror +# and goes again -- RETRY_A_FEW is per archive, so the budget never exhausts. +# Zero ledgers close, no give-up wording, no exit. Nothing but this kills it. +# +# One number cannot bound that, because runtimes span 190x (p25 771s, max 5.9h). +# A 3h deadline killed 941 legitimate ranges; a 12h one kills none but lets a +# wedged 771s range burn 56x its expected runtime first. So take whichever bound +# is tighter: the configured ceiling for the unforeseen, and a multiple of this +# range's own profiled cost for the failure we know about. Backtested against +# the previous run, 2x/3x/4x would each have produced ZERO false kills -- the +# measured wall never approached even twice the profile. +PROFILE_DEADLINE_FACTOR = float(os.getenv('PROFILE_DEADLINE_FACTOR', 0)) + + +def _attempt_deadline(end): + """Seconds this attempt may run, or None for no bound.""" + ceiling = ATTEMPT_DEADLINE_SECONDS or None + if not PROFILE_DEADLINE_FACTOR: + return ceiling + prof = profile_for(end) or {} + secs = prof.get('seconds') + if not secs: + # Unprofiled means newer than anything measured, so there is no honest + # estimate to tighten with -- fall back to the configured ceiling. + return ceiling + scaled = int(secs * PROFILE_DEADLINE_FACTOR) + return min(scaled, ceiling) if ceiling else scaled + + def _cause_count(end, attempt, causes): """How many of attempts 1..N at this range failed for one of `causes`. @@ -1544,15 +1893,35 @@ def _sized(value, margin, cap): _SORTED_SECONDS = None +def _positive_seconds(value): + """A finite positive runtime, or None when the profile cannot supply one.""" + try: + seconds = float(value) + except (TypeError, ValueError): + return None + return seconds if math.isfinite(seconds) and seconds > 0 else None + + def _profile_seconds(): - """Every measured runtime in the profile, sorted, for percentile lookup.""" + """Every valid measured runtime in the profile, sorted.""" global _SORTED_SECONDS if _SORTED_SECONDS is None: - _SORTED_SECONDS = sorted(r['seconds'] for _, r in (PROFILE or []) - if r.get('seconds')) + values = (_positive_seconds(r.get('seconds')) for _, r in (PROFILE or [])) + _SORTED_SECONDS = sorted(seconds for seconds in values if seconds is not None) return _SORTED_SECONDS +def _runtime_memory_insurance(seconds): + """Runtime-weighted share of the configured memory allowance.""" + seconds = _positive_seconds(seconds) + everything = _profile_seconds() + longest = everything[-1] if everything else None + insurance = _quantity_bytes(PROFILE_RUNTIME_MEMORY_INSURANCE) + if seconds is None or longest is None or longest <= 0 or insurance <= 0: + return 0 + return int(insurance * (seconds / longest)) + + def _slack_cpu(seconds): """Tier for a range, by its rank among all profiled runtimes. @@ -1568,7 +1937,8 @@ def _slack_cpu(seconds): return None if not tiers: return None - if not seconds: + seconds = _positive_seconds(seconds) + if seconds is None: return tiers[-1][1] everything = _profile_seconds() if not everything: @@ -1580,38 +1950,6 @@ def _slack_cpu(seconds): return tiers[-1][1] -# Multiple of a range's own measured runtime to allow before calling it wedged. -# The deadline exists for ONE failure mode, reproduced 2026-07-30: with an -# unreachable archive, stellar-core retries the bucket download forever. It logs -# "Missing HAS for ledger N: maybe stale archive", re-selects a different mirror -# and goes again -- RETRY_A_FEW is per archive, so the budget never exhausts. -# Zero ledgers close, no give-up wording, no exit. Nothing but this kills it. -# -# One number cannot bound that, because runtimes span 190x (p25 771s, max 5.9h). -# A 3h deadline killed 941 legitimate ranges; a 12h one kills none but lets a -# wedged 771s range burn 56x its expected runtime first. So take whichever bound -# is tighter: the configured ceiling for the unforeseen, and a multiple of this -# range's own profiled cost for the failure we know about. Backtested against -# the previous run, 2x/3x/4x would each have produced ZERO false kills -- the -# measured wall never approached even twice the profile. -PROFILE_DEADLINE_FACTOR = float(os.getenv('PROFILE_DEADLINE_FACTOR', 0)) - - -def _attempt_deadline(end): - """Seconds this attempt may run, or None for no bound.""" - ceiling = ATTEMPT_DEADLINE_SECONDS or None - if not PROFILE_DEADLINE_FACTOR: - return ceiling - prof = profile_for(end) or {} - secs = prof.get('seconds') - if not secs: - # Unprofiled means newer than anything measured, so there is no honest - # estimate to tighten with -- fall back to the configured ceiling. - return ceiling - scaled = int(secs * PROFILE_DEADLINE_FACTOR) - return min(scaled, ceiling) if ceiling else scaled - - def _profile_overrides(end, escalated): """Request overrides for this range from the profile, or {} for none. @@ -1630,7 +1968,9 @@ def _profile_overrides(end, escalated): # tracked anon still sizes exactly as it used to. rss = prof.get('peakAnonBytes') or prof.get('peakRssBytes') if rss: - want = int(rss * PROFILE_MARGIN) + _quantity_bytes(PROFILE_CACHE_HEADROOM) + want = (int(rss * PROFILE_MARGIN) + + _quantity_bytes(PROFILE_CACHE_HEADROOM) + + _runtime_memory_insurance(prof.get('seconds'))) out['memory'] = _bytes_to_quantity(min(want, _quantity_bytes(PROFILE_MAX_MEM))) disk = prof.get('peakEphemeralBytes') if disk and LIM_EPHEMERAL: @@ -1798,13 +2138,11 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): # pod failure already fails the Job, so Count and FailJob collapse to # the same outcome. Classification is done by reading the pod's # DisruptionTarget condition instead. - # On the JobSpec, not the pod, even though it therefore counts - # Pending time too. A pod-level deadline is IMMUTABLE once the pod - # exists, so a mis-set value cannot be corrected on a live run: - # measured 2026-07-30, 1007 Jobs were repointed in place from 3h to - # 12h while their pods kept running, and 850 pod-level ones later - # could not be touched at all. At a 12h ceiling the Pending - # overcharge is noise; being able to fix it mid-run is not. + # On the JobSpec, not the pod: a pod-level deadline is immutable + # once the pod exists, so a mis-set value cannot be corrected on a + # live run. Measured 2026-07-30: 1007 Job-level deadlines were + # repointed 3h->12h in place while their pods kept running; 850 + # pod-level ones could not be touched at all. active_deadline_seconds=_attempt_deadline(end), backoff_limit=0, pod_failure_policy=client.V1PodFailurePolicy( @@ -1813,6 +2151,16 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): template=client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels=pod_labels(end, attempt)), spec=client.V1PodSpec( + # On the POD, not the JobSpec. JobSpec.activeDeadlineSeconds + # runs from the Job's startTime, so every second the pod + # spends Pending -- waiting for Karpenter, pulling the image + # -- is charged against a budget that is meant to bound how + # long the range RUNS. During a node-class outage this run + # sat ~15 minutes Pending and ranges died as "timeouts" + # having barely executed; a timeout gets + # MAX_TIMEOUT_ATTEMPTS, so two stalls condemn a range and + # fail the mission. The pod-level field starts at container + # start, which is the thing being bounded. # IRSA for the S3 history mirror. Without it workers fall # back to the public archive, which throttles at 1024. service_account_name=WORKER_SERVICE_ACCOUNT or None, @@ -2307,6 +2655,9 @@ def reconcile(state): if str(end) not in completed and str(end) not in failed and str(end) not in in_flight), + # A Kubernetes snapshot only. The caller hands this to the independent + # liveness sampler after every dispatch/progress decision is complete. + '_worker_targets': _worker_targets(job_pods.values()), } @@ -2346,21 +2697,21 @@ def update_status_and_metrics(): r = reconcile(state) - # Worker liveness, for the Grafana series only -- nothing in the - # driver reads it. A worker is a Job here, so a Running pod IS a - # live worker and its liveness is the Job's status; the count comes - # off the pod list the apiserver already has cached instead of one - # HTTP GET per worker every cycle. + # Grafana-only worker responsiveness. Candidate discovery reused the + # authoritative pod snapshot, but the handoff below never performs + # network I/O: /info probes run on a fixed, bounded sampler pool. refresh_start = time.time() - workers_up = sum( - 1 for p in core_v1.list_namespaced_pod( - NAMESPACE, label_selector=f"{LABEL_RUN}={RUN_NAME}", - field_selector='status.phase=Running', - # Served from the apiserver watch cache. Only safe here: - # a stale liveness sample is cosmetic, whereas stale - # dispatch state would re-run a range. - resource_version='0').items - if p.status.pod_ip) + targets = r.pop('_worker_targets') + try: + worker_counts = publish_worker_liveness(targets) + except Exception as e: + worker_counts = {'up': 0, 'down': 0, 'unknown': len(targets)} + now = time.time() + if now - state.get('last_liveness_error_log', 0) >= 60: + state['last_liveness_error_log'] = now + logger.exception( + "worker liveness publication failed (%s); reporting all " + "current candidates unknown and continuing reconcile", e) workers_refresh_duration = time.time() - refresh_start mission_duration = time.time() - mission_start_time @@ -2380,12 +2731,9 @@ def update_status_and_metrics(): metric_catchup_queues.labels(queue="succeeded").set(r['completed']) metric_catchup_queues.labels(queue="failed").set(len(r['failed_ranges'])) metric_catchup_queues.labels(queue="in_progress").set(len(r['in_progress'])) - metric_workers.labels(status="up").set(workers_up) - # Held at 0 rather than dropped: the series is Grafana-facing, and a - # label that stops being set goes stale on the dashboard instead of - # reading zero. Nothing can report "down" now that liveness is the - # pod's phase -- a worker that is not up is simply not listed. - metric_workers.labels(status="down").set(0) + metric_workers.labels(status="up").set(worker_counts['up']) + metric_workers.labels(status="down").set(worker_counts['down']) + metric_workers.labels(status="unknown").set(worker_counts['unknown']) metric_refresh_duration.set(workers_refresh_duration) metric_mission_duration.set(mission_duration) logger.info("Status: %s", json.dumps(status)) @@ -2413,6 +2761,7 @@ def run(server_class=HTTPServer, handler_class=RequestHandler): if __name__ == '__main__': # Before any dispatch: the first Job built must already be sized from it. PROFILE = load_profile() + worker_liveness_sampler.start() # Not a logging thread despite the historical name -- this is the reconcile # loop: dispatch, progress record, metrics, status. Log capture and pod @@ -2421,6 +2770,7 @@ def run(server_class=HTTPServer, handler_class=RequestHandler): reconcile_thread.daemon = True reconcile_thread.start() - # Separate thread: a blocking watch must not sit behind dispatch and the - # liveness sweep, which is the whole point of it. - run() + try: + run() + finally: + worker_liveness_sampler.close() diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index b1d034ee..04c245d0 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -240,7 +240,19 @@ spec: value: {{ .Values.monitor.profileMaxMemory | quote }} - name: PROFILE_CACHE_HEADROOM value: {{ .Values.monitor.profileCacheHeadroom | quote }} + # Handed out in proportion to a range's own runtime, so the ranges + # that sit exposed to drift the longest get the largest share. + - name: PROFILE_RUNTIME_MEMORY_INSURANCE + value: {{ .Values.monitor.profileRuntimeMemoryInsurance | quote }} {{- end }} + - name: LIVENESS_PROBE_INTERVAL_SECONDS + value: {{ .Values.monitor.livenessProbeIntervalSeconds | quote }} + - name: LIVENESS_PROBE_TIMEOUT_SECONDS + value: {{ .Values.monitor.livenessProbeTimeoutSeconds | quote }} + - name: LIVENESS_FAILURE_THRESHOLD + value: {{ .Values.monitor.livenessFailureThreshold | quote }} + - name: LIVENESS_MAX_CONCURRENCY + value: {{ .Values.monitor.livenessMaxConcurrency | quote }} # Failed ranges are always saved; successful ones are the bulk of # the volume and can be turned off for a cheap run. - name: SAVE_SUCCESS_LOGS diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index b2396058..fe92f0c1 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -102,6 +102,17 @@ monitor: # profiled at 190MiB rss got a 209MiB limit and 90 of them OOMKilled within # 90s of dispatch. profileCacheHeadroom: "512Mi" + # Runtime-weighted allowance, scaled by seconds/longest so a 6h range gets + # the full amount and a 13min one gets almost none. A flat margin misprices + # both ends: the short ranges are measured tightly and need nothing, while + # the long ones sit exposed to archive growth and cache drift for hours. + profileRuntimeMemoryInsurance: "3Gi" + # Probing 1000+ workers serially takes longer than the poll interval, so the + # sampler runs threaded and bounded rather than inline. + livenessProbeIntervalSeconds: 30 + livenessProbeTimeoutSeconds: 5 + livenessFailureThreshold: 3 + livenessMaxConcurrency: 32 imagePullPolicy: IfNotPresent mission: "HistoryPubnetParallelCatchup" # Adds a `mission` label to worker pods, which kube-state-metrics exposes as From 483001d1a5ab16e33a737741b617d503750ac330 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 18:34:51 -0400 Subject: [PATCH 054/117] Harden synthetic harness startup Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../integration/synthetic_resume_harness.py | 36 ++++++++++++++++--- .../contract/test_synthetic_resume_harness.py | 19 ++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/MissionParallelCatchup/integration/synthetic_resume_harness.py b/src/MissionParallelCatchup/integration/synthetic_resume_harness.py index 8b00d303..7d216246 100644 --- a/src/MissionParallelCatchup/integration/synthetic_resume_harness.py +++ b/src/MissionParallelCatchup/integration/synthetic_resume_harness.py @@ -98,11 +98,11 @@ def helm_sets(release, source_config_map, image): 'monitor.collectorResources.requests.cpu=25m', 'monitor.collectorResources.requests.memory=128Mi', 'monitor.collectorResources.limits.cpu=250m', - 'monitor.collectorResources.limits.memory=256Mi', + 'monitor.collectorResources.limits.memory=512Mi', 'monitor.resources.requests.cpu=25m', 'monitor.resources.requests.memory=128Mi', 'monitor.resources.limits.cpu=250m', - 'monitor.resources.limits.memory=256Mi', + 'monitor.resources.limits.memory=512Mi', 'range.generator=uniform', 'range.startingLedger=0', f'range.latestLedgerNum={RANGE_END}', @@ -208,6 +208,34 @@ def monitor_pod(release): return items[0] +def monitor_startup(pod): + if not pod: + return None + statuses = {} + for status in pod.get('status', {}).get('containerStatuses', []): + state = status.get('state') or {} + statuses[status['name']] = { + 'ready': status.get('ready', False), + 'restartCount': status.get('restartCount', 0), + 'state': state, + } + return { + 'name': pod['metadata']['name'], + 'phase': pod.get('status', {}).get('phase'), + 'conditions': pod.get('status', {}).get('conditions', []), + 'containers': statuses, + } + + +def ready_monitor_pod(release, evidence): + pod = monitor_pod(release) + evidence['monitorStartup'] = monitor_startup(pod) + if not pod: + return None + statuses = pod.get('status', {}).get('containerStatuses', []) + return pod if len(statuses) == 2 and all(s.get('ready') for s in statuses) else None + + def worker_snapshot(release): selector = f'catchup.stellar.org/run={release}' jobs = (json_get('jobs', labels=selector) or {}).get('items', []) @@ -494,14 +522,14 @@ def execute(args): run( ['helm', 'install', args.release, str(CHART), - '--namespace', NAMESPACE, '--wait', '--timeout', '3m'] + '--namespace', NAMESPACE, '--timeout', '3m'] + helm_args(sets), timeout=210) installed = True pod = wait_for( 'one ready monitor pod', - lambda: monitor_pod(args.release), timeout=60) + lambda: ready_monitor_pod(args.release, evidence), timeout=120) initial_restarts = container_restarts(pod) evidence['restartCountsBefore'] = initial_restarts diff --git a/src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py b/src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py index e0173a54..81d50590 100644 --- a/src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py +++ b/src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py @@ -87,3 +87,22 @@ def test_runner_uses_a_handled_signal_for_collector_only_restart(): source = PATH.read_text() assert 'kill -TERM 1' in source assert 'kill -9 1' not in source + + +def test_monitor_readiness_requires_both_containers(): + pod = { + 'metadata': {'name': 'monitor'}, + 'status': {'phase': 'Running', 'containerStatuses': [ + {'name': 'job-monitor', 'ready': True, 'restartCount': 0, 'state': {}}, + {'name': 'log-collector', 'ready': False, 'restartCount': 1, 'state': {}}, + ]}, + } + evidence = {} + original = HARNESS.monitor_pod + try: + HARNESS.monitor_pod = lambda _release: pod + assert HARNESS.ready_monitor_pod('mpc-resume-a1b2c3', evidence) is None + pod['status']['containerStatuses'][1]['ready'] = True + assert HARNESS.ready_monitor_pod('mpc-resume-a1b2c3', evidence) is pod + finally: + HARNESS.monitor_pod = original From e3ebbdd7236d3f36e7e9fd9d8d223c08e43142cc Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 18:38:22 -0400 Subject: [PATCH 055/117] Capture synthetic monitor startup failures Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../integration/synthetic_resume_harness.py | 21 ++++++++++++++ .../contract/test_synthetic_resume_harness.py | 29 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/src/MissionParallelCatchup/integration/synthetic_resume_harness.py b/src/MissionParallelCatchup/integration/synthetic_resume_harness.py index 7d216246..71886a54 100644 --- a/src/MissionParallelCatchup/integration/synthetic_resume_harness.py +++ b/src/MissionParallelCatchup/integration/synthetic_resume_harness.py @@ -236,6 +236,26 @@ def ready_monitor_pod(release, evidence): return pod if len(statuses) == 2 and all(s.get('ready') for s in statuses) else None +def monitor_logs(release): + pod = monitor_pod(release) + if not pod: + return {} + name = pod['metadata']['name'] + captured = {} + for container in ('job-monitor', 'log-collector'): + current = kubectl( + NAMESPACE, 'logs', name, '-c', container, + '--tail=200', check=False, timeout=30) + previous = kubectl( + NAMESPACE, 'logs', name, '-c', container, '--previous', + '--tail=200', check=False, timeout=30) + captured[container] = { + 'current': current.stdout if current.returncode == 0 else current.stderr, + 'previous': previous.stdout if previous.returncode == 0 else previous.stderr, + } + return captured + + def worker_snapshot(release): selector = f'catchup.stellar.org/run={release}' jobs = (json_get('jobs', labels=selector) or {}).get('items', []) @@ -606,6 +626,7 @@ def completed(): evidence['error'] = f'{type(error).__name__}: {error}' finally: try: + evidence['monitorLogs'] = monitor_logs(args.release) evidence['cleanup'] = cleanup( args.release, source_config_map, scope_started or installed, evidence) except Exception as cleanup_error: diff --git a/src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py b/src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py index 81d50590..0a4b5872 100644 --- a/src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py +++ b/src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py @@ -106,3 +106,32 @@ def test_monitor_readiness_requires_both_containers(): assert HARNESS.ready_monitor_pod('mpc-resume-a1b2c3', evidence) is pod finally: HARNESS.monitor_pod = original + + +def test_startup_log_capture_is_limited_to_the_exact_monitor_pod(): + pod = {'metadata': {'name': 'mpc-resume-a1b2c3-job-monitor-abc'}} + calls = [] + original_pod = HARNESS.monitor_pod + original_kubectl = HARNESS.kubectl + try: + HARNESS.monitor_pod = lambda _release: pod + + class Result: + returncode = 0 + stdout = 'captured' + stderr = '' + + def fake_kubectl(namespace, *args, **_kwargs): + calls.append((namespace, args)) + return Result() + + HARNESS.kubectl = fake_kubectl + logs = HARNESS.monitor_logs('mpc-resume-a1b2c3') + finally: + HARNESS.monitor_pod = original_pod + HARNESS.kubectl = original_kubectl + + assert set(logs) == {'job-monitor', 'log-collector'} + assert len(calls) == 4 + assert all(namespace == 'sandbox' for namespace, _ in calls) + assert all(args[:2] == ('logs', pod['metadata']['name']) for _, args in calls) From b57c70b506e5ff6fc57525b580e79366d05f1d15 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 18:43:02 -0400 Subject: [PATCH 056/117] Collapse the eight tmp+rename blocks into one _write_atomic Every durable file both processes exchange was written by its own copy of the same four lines, with three different openers and three different error-handling shapes. One helper per module instead. The opener resolves at call time rather than as a default argument: the discipline is only worth having if a test can crash a write mid-flight and prove the real path is untouched, and a default argument freezes the builtin at import so no patch can reach it. Moved the atomicity tests off monkeypatching json.dump and onto a half-written file object. A disk filling up mid-write is the failure the rename is actually there for, and patching the serializer only exercises whichever one the code happens to call today -- two of these tests went green on the refactor purely because json.dump stopped being called. Deleting the tmp+rename now fails 4 tests where it used to fail 3: the gzip log archive shares the helper and picked up the guard. 512 tests + 1 xfail. --- src/MissionParallelCatchup/job_monitor.py | 36 +++++++----- src/MissionParallelCatchup/log_collector.py | 35 ++++++----- .../resilience/test_collector_restart.py | 58 +++++++++++++------ .../unit/test_monitor_verdict_records.py | 2 +- 4 files changed, 80 insertions(+), 51 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index e0b9ab98..8ea0ef12 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -962,13 +962,26 @@ def _state_only(progress): return out +def _write_atomic(path, body, opener=None): + """Write `body` through tmp+rename so a reader never sees a partial file. + + Every state file here is read back by a restarted monitor, so a torn write + is indistinguishable from corruption: the .outcome and .verdict files decide + a range's remaining budget, and progress.json decides what gets dispatched. + """ + tmp = path + '.tmp' + # `opener` resolves at call time, not as a default argument, so the write + # seam stays patchable -- the tmp+rename discipline is only worth having if + # a test can crash a write mid-flight and prove the real path is untouched. + with (opener or open)(tmp, 'wt') as fh: + fh.write(body) + os.replace(tmp, path) + + def save_progress(progress): blob = json.dumps(progress, separators=(',', ':')) # File first and atomically: it is what a restart reads back. - tmp = PROGRESS_FILE + '.tmp' - with open(tmp, 'w') as fh: - fh.write(blob) - os.replace(tmp, PROGRESS_FILE) + _write_atomic(PROGRESS_FILE, blob) # Mirror for the driver. Never fatal -- a 413 here used to throw inside # reconcile, and the loop swallows exceptions, so no completion would ever # be recorded again and every finished range would be dispatched forever. @@ -1023,10 +1036,7 @@ def backstop_save_pod_log(pod_name, end, attempt): return False try: os.makedirs(LOG_DIR, exist_ok=True) - tmp = path + '.tmp' - with gzip.open(tmp, 'wt') as fh: - fh.write(body) - os.replace(tmp, path) # never leave a half-written archive behind + _write_atomic(path, body, gzip.open) return True except OSError as e: logger.warning("could not write %s: %s", path, e) @@ -1114,10 +1124,7 @@ def record_outcome(end, attempt, pod): # path. Without it a resumed chain can only report its final leg. data['attemptSeconds'] = _pod_seconds(pod) try: - tmp = path + '.tmp' - with open(tmp, 'w') as fh: - json.dump(data, fh) - os.replace(tmp, path) + _write_atomic(path, json.dumps(data)) except OSError as e: logger.warning("could not persist outcome for range %s: %s", end, e) @@ -1243,10 +1250,7 @@ def save_verdict(end, attempt, outcome): """ path = verdict_path(end, attempt) try: - tmp = path + '.tmp' - with open(tmp, 'w') as fh: - fh.write(str(outcome)) - os.replace(tmp, path) + _write_atomic(path, str(outcome)) except OSError as e: logger.warning("could not persist verdict for range %s attempt %s: %s", end, attempt, e) diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 496b9b6a..15d0e11a 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -176,13 +176,25 @@ def read_state(end, attempt): return ts if ts and _TS_RE.match(ts) else None +def _write_atomic(path, body, opener=None): + """Write `body` through tmp+rename so a reader never sees a partial file. + + The monitor polls these files while this process writes them, so a torn + .metrics or .outcome would be read as corrupt and the measurement lost. + """ + tmp = path + '.tmp' + # `opener` resolves at call time, not as a default argument, so the write + # seam stays patchable -- the tmp+rename discipline is only worth having if + # a test can crash a write mid-flight and prove the real path is untouched. + with (opener or open)(tmp, 'wt') as fh: + fh.write(body) + os.replace(tmp, path) + + def write_state(end, attempt, ts): path = base(end, attempt) + '.state' - tmp = path + '.tmp' try: - with open(tmp, 'w') as fh: - fh.write(ts) - os.replace(tmp, path) + _write_atomic(path, ts) except OSError as e: logger.warning("could not persist state for range %s: %s", end, e) @@ -266,7 +278,6 @@ def write_metrics(end, attempt, values): meaningful for one that succeeded. """ path = base(end, attempt) + '.metrics' - tmp = path + '.tmp' # Merge, and never let a peak go backwards. A measurement already on disk # must survive a later write that lacks it -- the peaks are held in memory, # so a collector restart would otherwise drop them. But a plain overwrite is @@ -294,9 +305,7 @@ def write_metrics(end, attempt, values): merged[k] = max(a, b) values = merged try: - with open(tmp, 'w') as fh: - json.dump(values, fh) - os.replace(tmp, path) + _write_atomic(path, json.dumps(values)) logger.info("range %s attempt %s metrics=%s", end, attempt, values) except OSError as e: logger.warning("could not persist metrics for range %s: %s", end, e) @@ -350,10 +359,7 @@ def record_outcome(pod, end, attempt): data = classify(pod) data['pod'] = pod['metadata']['name'] try: - tmp = path + '.tmp' - with open(tmp, 'w') as fh: - json.dump(data, fh) - os.replace(tmp, path) + _write_atomic(path, json.dumps(data)) logger.info("range %s attempt %s classified: %s", end, attempt, data['outcome']) except OSError as e: logger.warning("could not persist outcome for range %s: %s", end, e) @@ -485,11 +491,8 @@ async def sample_kubelet(session, nodes): def _mark_done(end, attempt): path = done_path(end, attempt) - tmp = path + '.tmp' try: - with open(tmp, 'w') as fh: - fh.write('') - os.replace(tmp, path) + _write_atomic(path, '') except OSError as e: # Costs a Job that waits out JOB_TTL_SECONDS, never correctness. logger.warning("could not mark range %s attempt %s done: %s", end, attempt, e) diff --git a/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py b/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py index 0985a1f3..bf543915 100644 --- a/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py +++ b/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py @@ -31,6 +31,43 @@ GIB = 1073741824 +def _arm_half_write(monkeypatch, mod, suffix): + """Make the next write to `suffix` land half a file, then fail with ENOSPC. + + Injected at the file object rather than at json.dump: a disk filling up + mid-write is the actual failure the tmp+rename is there for, and patching + the serializer only exercises whichever one the code happens to call. + """ + real_open = open + seen = {} + + class _HalfWrite: + def __init__(self, path): + self.fh = real_open(path, 'w') + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.fh.close() + return False + + def write(self, blob): + self.fh.write(blob[:len(blob) // 2]) + seen['torn'] = True + raise OSError(28, 'No space left on device') + + def half_open(path, mode='r', *a, **kw): + # One-shot: the point is that the process carries on afterwards, so + # everything the collector writes after the failure must be real. + if not seen and mode in ('w', 'wt') and str(path).endswith(suffix): + return _HalfWrite(path) + return real_open(path, mode, *a, **kw) + + monkeypatch.setattr(mod, 'open', half_open, raising=False) + return seen + + # -- the shared volume, and a collector with no memory of anything ------------ @pytest.fixture @@ -300,22 +337,11 @@ def test_done_never_appears_beside_a_half_written_metrics_file(vol, monkeypatch) record permanent.""" lc.write_metrics('300', 1, {'peakAnonBytes': 6 * GIB, 'txApplySeconds': 30.0}) - real_dump = lc.json.dump - seen = {} - - def dump_then_die(obj, fh, *a, **kw): - # A write that dies with the file open: the failure mode the .tmp + - # rename is there for. - fh.write(json.dumps(obj)[:12]) - seen['torn'] = True - raise OSError(28, 'No space left on device') - - monkeypatch.setattr(lc.json, 'dump', dump_then_die) + seen = _arm_half_write(monkeypatch, lc, '.metrics.tmp') lc._anon_peak['w-300'] = 9 * GIB finalize('w-300', 300) - monkeypatch.setattr(lc.json, 'dump', real_dump) - assert seen['torn'], "the interrupted write never happened" + assert seen.get('torn'), "the interrupted write never happened" # The old record is intact and parseable -- not truncated, not empty. assert metrics(300) == {'peakAnonBytes': 6 * GIB, 'txApplySeconds': 30.0} # .done still lands: the collector really will write nothing more for this @@ -501,11 +527,7 @@ def test_a_recorded_outcome_is_a_complete_file_or_no_file(vol, monkeypatch): """Same rename discipline as .metrics: the monitor branches its whole retry policy on this file, so a torn read would have to be a crash or a wrong verdict.""" - def dump_then_die(obj, fh, *a, **kw): - fh.write(json.dumps(obj)[:9]) - raise OSError(28, 'No space left on device') - - monkeypatch.setattr(lc.json, 'dump', dump_then_die) + _arm_half_write(monkeypatch, lc, '.outcome.tmp') lc.record_outcome(_pod('w-300', exit_code=1), '300', 1) assert jm.read_outcome('300', 1) is None diff --git a/src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py b/src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py index 451852f2..f806b2f6 100644 --- a/src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py +++ b/src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py @@ -194,7 +194,7 @@ def write(self, blob): armed = {'v': True} def half_open(path, mode='r', *a, **kw): - if armed['v'] and mode == 'w' and str(path).endswith('.json.tmp'): + if armed['v'] and mode in ('w', 'wt') and str(path).endswith('.json.tmp'): return _HalfWrite(path) return real_open(path, mode, *a, **kw) From 1ed7e82687df337077f3ec6dd20d201d54724606 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Thu, 30 Jul 2026 19:11:00 -0400 Subject: [PATCH 057/117] Stop condemning a range whose pod terminated with no exit code The r5 run failed its mission on range 59018943 at 554/554 otherwise-clean completions. Its .outcome read {"outcome": "failed", "exitCode": null} and the monitor logged "is NOT retryable; this fails the mission". classify() walks the pod's terminated container statuses looking for an OOM kill or a non-zero exit. When the only terminated status left on the pod is the sidecar's clean exit -- stellar-core's never landed, so nothing on the pod says why it stopped -- it fell off the end of that loop into `failed`. That is the one outcome with no retry at all, and it is the wrong label: an absent exit code is an absence of evidence, not evidence of a bad ledger range. The `unknown` branch three lines up already says exactly this and retries. Two changes, because the label and the decision are both wrong: - log_collector.classify() returns `unknown` for the leftover case, which is what it means and which the monitor already retries. - job_monitor gains an exitCode-is-None guard before the condemn branch, so any other source of a verdict with no exit code is also retried rather than being fatal. A genuinely broken range still exhausts MAX_ATTEMPTS and fails with evidence; the neighbouring test pins that exit 1 is still condemned. 514 tests + 1 xfail. Reverting either fix fails the new test. --- src/MissionParallelCatchup/job_monitor.py | 14 +++++++++ src/MissionParallelCatchup/log_collector.py | 7 ++++- src/MissionParallelCatchup/tests/conftest.py | 15 ++++++++++ .../tests/reconcile/test_retry_budgets.py | 30 +++++++++++++++++++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 8ea0ef12..bad5192b 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -2500,6 +2500,20 @@ def reconcile(state): # one succeeds, usually by resuming at LCL+1. reason = (f"exited {CATCHUP_INCOMPLETE_EXIT} (did not complete -- " "corrupt archive or interruption, indistinguishable)") + elif verdict.get('exitCode') is None: + # No exit code means nothing read the container's status: the pod + # was reaped before classification and the verdict came from the + # Job condition alone, which says "Failed" and nothing about why. + # That is the same absence of evidence as `unknown` above and + # takes the same answer -- the only difference is that a Job + # condition happened to survive the pod, which says nothing about + # the ledger range. + # + # Observed on the r5 run 2026-07-30: range 59018943 was condemned + # on attempt 1 with outcome=failed exitCode=None and failed the + # mission, while a dozen sibling ranges reaped the same way + # classified as `unknown`, retried, and passed. + reason = "failed with no exit code (pod reaped before classification)" else: reason = None # genuine catchup failure: do not retry diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/log_collector.py index 15d0e11a..e0530c60 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/log_collector.py @@ -348,7 +348,12 @@ def classify(pod): return {'outcome': 'oom', 'exitCode': t.get('exitCode')} if t.get('exitCode') not in (0, None): return {'outcome': 'failed', 'exitCode': t.get('exitCode')} - return {'outcome': 'failed', 'exitCode': None} + # Terminated, but not one container said with what. `failed` here is a lie + # that costs the whole run: it reads as a genuine catchup failure, which is + # the one outcome that gets no retry at all. Observed on the r5 run + # 2026-07-30, range 59018943 -- condemned on attempt 1 with exitCode null, + # failing a mission that was otherwise 554 for 554. + return {'outcome': 'unknown', 'exitCode': None} def record_outcome(pod, end, attempt): diff --git a/src/MissionParallelCatchup/tests/conftest.py b/src/MissionParallelCatchup/tests/conftest.py index b1bb858e..b911f39b 100644 --- a/src/MissionParallelCatchup/tests/conftest.py +++ b/src/MissionParallelCatchup/tests/conftest.py @@ -65,6 +65,7 @@ def test_something(cluster): 'rejected', # kubelet refused the pod before any container ran 'timeout', # activeDeadlineSeconds fired 'unknown', # job failed, pod already reaped, nothing classified it + 'no_exit_code', # container terminated, kubelet never filled in the exit code ) @@ -168,6 +169,20 @@ def advance(self, end, state, attempt=None): if pod_name: self.k8s.delete_pod(pod_name) self.k8s.set_job_failed(name, reason=None) + elif state == 'no_exit_code': + # The container terminated but the kubelet never populated an exit + # code, so nothing on the pod says why it stopped. Real: observed on + # range 59018943, 2026-07-30. + if pod_name: + # The only terminated status left on the pod belongs to the + # sidecar, which exited cleanly; stellar-core's never landed. So + # classify() finds a terminated container, none of them non-zero, + # and falls off the end of its loop. + self.k8s.set_pod_terminated(pod_name, exit_code=0, + container='log-collector', + phase='Failed') + self.k8s.set_job_failed(name, reason='BackoffLimitExceeded', + message='Job has reached the specified backoff limit') return name def _policy_msg(self, pod_name, code, rule_index): diff --git a/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py b/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py index 18308bb1..3886e62d 100644 --- a/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py +++ b/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py @@ -206,3 +206,33 @@ def test_a_genuine_catchup_failure_is_still_never_retried(cluster): assert condemned(cluster, end) assert cluster.failed()[str(end)]['attempts'] == 1 assert not job_exists(cluster, end, 2) + + +def test_a_terminated_pod_with_no_exit_code_is_not_condemned(cluster): + """A container that terminated without a populated exit code says nothing + about the ledger range, and must not be treated as a catchup failure. + + Observed on the r5 run 2026-07-30: range 59018943 was condemned on attempt 1 + with `outcome=failed exitCode=None`, failing a mission that was otherwise + 554 for 554. The collector's classify() fell through every branch that + needs an exit code and labelled the leftover case `failed` -- the one + outcome that gets no retry at all. + """ + end = dispatch(cluster) + hit(cluster, end, 'no_exit_code') + + assert not condemned(cluster, end), ( + "a pod reaped before classification condemned the range and failed the " + f"mission on no evidence. failed={cluster.failed()}") + assert job_exists(cluster, end, 2), ( + f"no attempt 2 was dispatched; live jobs are {cluster.jobs()}") + + +def test_a_real_catchup_failure_is_still_condemned(cluster): + """The guard above must not swallow the case it is next to: an exit code of + 1 IS evidence, and a range that produces one still fails the mission.""" + end = dispatch(cluster) + hit(cluster, end, 'condemned') + + assert condemned(cluster, end), ( + "exit 1 is a genuine catchup failure and must not be retried") From f8bf62868d2cb51f1e4ab551928e74eb3b932c50 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Fri, 31 Jul 2026 09:42:29 -0400 Subject: [PATCH 058/117] Make the attempt deadline a flat 12h and drop the dead knobs around it Three things go, all of them load-bearing only in theory. 1. PROFILE_DEADLINE_FACTOR / _attempt_deadline. Scaling the deadline by each range's profiled runtime looked right and is not. A deadline has to bound a range's WORST case; a profile offers a neighbour's TYPICAL case. Range keys are anchored to the network tip, so a profile from an earlier run matches ZERO keys exactly -- every lookup lands on a neighbour -- and ~2% of neighbours are 3-38x cheaper than their surroundings. The original backtest claimed zero false kills at 2x/3x/4x. It was tautological: exact-key lookups against the profile's own run. Redone honestly across the real grid offset (run4 profile -> r5 actuals, 3983 ranges), a 2x factor falsely kills 134 ranges, 4x kills 46, 6x kills 21. Flat 12h kills none. The asymmetry settles it. A false kill loses a range, and a timeout is terminal, so it fails the mission. A genuine wedge holds one slot of 1092-1500 for 12h, ~0.1% of a run's capacity. 2. attemptDeadlineSeconds 10800 -> 43200. "A prod range runs ~50 min" was the median talking; runtimes span 190x. 3h killed 941 legitimate ranges and had to be hand-patched to 12h mid-run on 2026-07-30. 3. MAX_TIMEOUT_ATTEMPTS. Dead since a timeout became terminal: the retry gate is `reason is not None and spent < cap`, and the timeout branch sets reason = None unconditionally, so the cap was computed and never read. Confirmed by mutation -- changing 2 to 99 breaks only a chart/code consistency check and an ordering assertion, no behavioural test. test_the_timeout_budget_still_binds was passing vacuously; it now asserts the real rule, that the FIRST timeout condemns. The deleted comments' evidence is preserved on ATTEMPT_DEADLINE_SECONDS, including why 12h is a safe bound but a poor detector: the right signal is ledger-close progress, not elapsed time. A wedged core closes zero ledgers while still logging, so `.state` (last log line) cannot see it. Left undone deliberately -- it needs a threshold above the initial bucket-apply phase, which legitimately closes nothing for ~20min. 511 tests + 1 xfail. Reintroducing per-range scaling fails 2 of the 3 tests in test_deadline_sizing.py. --- src/MissionParallelCatchup/job_monitor.py | 78 ++++++++---------- .../templates/job_monitor.yaml | 4 - .../parallel_catchup_helm/values.yaml | 27 +++---- src/MissionParallelCatchup/tests/conftest.py | 1 - .../tests/contract/test_rendered_job_spec.py | 1 - .../tests/reconcile/test_attempt_deadline.py | 8 +- .../tests/reconcile/test_retry_budgets.py | 16 ++-- .../tests/unit/test_deadline_sizing.py | 80 ++++++++++--------- .../tests/unit/test_sizing.py | 12 +-- 9 files changed, 107 insertions(+), 120 deletions(-) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index bad5192b..36518a0c 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -181,6 +181,29 @@ # no exit code, no failure, the slot held for the life of the run. A hang is a # more likely real failure than a non-zero exit, and this deadline is the only # thing that makes it observable. 0 disables. +# +# Flat, deliberately -- NOT scaled by the range's profiled runtime. That was +# tried and removed. A deadline has to bound a range's WORST case, but a profile +# only offers a neighbour's TYPICAL case, and the two are far apart here: +# runtimes span 190x (p25 771s, max 5.9h), range keys are anchored to the +# network tip so a profile from an earlier run matches ZERO keys exactly and +# every lookup lands on a neighbour, and ~2% of those neighbours are 3-38x +# cheaper than their surroundings. Backtested honestly across that grid offset +# (run4 profile -> r5 actuals, 3983 ranges): a 2x factor falsely kills 134 +# ranges, 4x kills 46, 6x kills 21. Flat 12h kills none. +# +# The asymmetry decides it. A false kill loses a range, and a timeout is +# terminal, so it fails the mission. A genuine wedge holds ONE slot out of +# 1092-1500 for 12h -- around 0.1% of a run's capacity. Never trade a certain +# catastrophe against a rounding error. +# +# 12h is a safe bound, not a good detector: it takes half a day to catch +# something provably dead in 4 minutes. The right signal is ledger-close +# progress, not elapsed time -- a wedged core closes zero ledgers while still +# logging, so `.state` (last log line) cannot see it and a new +# lastLedgerCloseAt would. Left undone on purpose; it needs a threshold above +# the initial bucket-apply phase, which legitimately closes nothing for ~20min +# on the longest ranges. ATTEMPT_DEADLINE_SECONDS = int(os.getenv('ATTEMPT_DEADLINE_SECONDS', 0)) # kube-state-metrics turns a pod's `mission` label into label_mission, which the @@ -209,10 +232,6 @@ # The cost of stopping is that the range is condemned, and today a condemned # range aborts the run. That coupling is the thing to fix, not this number. MAX_ATTEMPTS_PER_RANGE = int(os.getenv('MAX_ATTEMPTS', 5)) -# A hang gets far fewer retries than an eviction. The measured causes -- an -# unreachable archive host, an absent checkpoint, a bucket that will not -# decompress -- are persistent, so retrying mostly burns another full deadline. -MAX_TIMEOUT_ATTEMPTS = int(os.getenv('MAX_TIMEOUT_ATTEMPTS', 2)) # Evictions, admission rejections and monitor restarts say nothing about the # ledger range, so they get their own, larger budget. Sharing MAX_ATTEMPTS with # real failures means cluster churn can fail a healthy range: measured on @@ -1266,38 +1285,6 @@ def _verdict_of(end, attempt): return (read_outcome(end, attempt) or {}).get('outcome') -# Multiple of a range's own measured runtime to allow before calling it wedged. -# The deadline exists for ONE failure mode, reproduced 2026-07-30: with an -# unreachable archive, stellar-core retries the bucket download forever. It logs -# "Missing HAS for ledger N: maybe stale archive", re-selects a different mirror -# and goes again -- RETRY_A_FEW is per archive, so the budget never exhausts. -# Zero ledgers close, no give-up wording, no exit. Nothing but this kills it. -# -# One number cannot bound that, because runtimes span 190x (p25 771s, max 5.9h). -# A 3h deadline killed 941 legitimate ranges; a 12h one kills none but lets a -# wedged 771s range burn 56x its expected runtime first. So take whichever bound -# is tighter: the configured ceiling for the unforeseen, and a multiple of this -# range's own profiled cost for the failure we know about. Backtested against -# the previous run, 2x/3x/4x would each have produced ZERO false kills -- the -# measured wall never approached even twice the profile. -PROFILE_DEADLINE_FACTOR = float(os.getenv('PROFILE_DEADLINE_FACTOR', 0)) - - -def _attempt_deadline(end): - """Seconds this attempt may run, or None for no bound.""" - ceiling = ATTEMPT_DEADLINE_SECONDS or None - if not PROFILE_DEADLINE_FACTOR: - return ceiling - prof = profile_for(end) or {} - secs = prof.get('seconds') - if not secs: - # Unprofiled means newer than anything measured, so there is no honest - # estimate to tighten with -- fall back to the configured ceiling. - return ceiling - scaled = int(secs * PROFILE_DEADLINE_FACTOR) - return min(scaled, ceiling) if ceiling else scaled - - def _cause_count(end, attempt, causes): """How many of attempts 1..N at this range failed for one of `causes`. @@ -2147,7 +2134,7 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): # live run. Measured 2026-07-30: 1007 Job-level deadlines were # repointed 3h->12h in place while their pods kept running; 850 # pod-level ones could not be touched at all. - active_deadline_seconds=_attempt_deadline(end), + active_deadline_seconds=ATTEMPT_DEADLINE_SECONDS or None, backoff_limit=0, pod_failure_policy=client.V1PodFailurePolicy( rules=[r for _, r in _failure_rules()]), @@ -2161,10 +2148,9 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): # -- is charged against a budget that is meant to bound how # long the range RUNS. During a node-class outage this run # sat ~15 minutes Pending and ranges died as "timeouts" - # having barely executed; a timeout gets - # MAX_TIMEOUT_ATTEMPTS, so two stalls condemn a range and - # fail the mission. The pod-level field starts at container - # start, which is the thing being bounded. + # having barely executed -- and a timeout is terminal, so + # each one fails the mission. The pod-level field starts at + # container start, which is the thing being bounded. # IRSA for the S3 history mirror. Without it workers fall # back to the public archive, which throttles at 1024. service_account_name=WORKER_SERVICE_ACCOUNT or None, @@ -2458,7 +2444,7 @@ def reconcile(state): "on attempt %s; this fails the mission. Check its archived " "log for 'maybe stale archive' -- an unreachable history " "mirror is the usual cause.", - end, _attempt_deadline(end), attempt) + end, ATTEMPT_DEADLINE_SECONDS, attempt) elif verdict['outcome'] == 'rejected': reason = f"rejected by the node before starting ({verdict.get('reason', '?')})" elif verdict['outcome'] == 'disrupted': @@ -2530,10 +2516,10 @@ def reconcile(state): # condemned -- never retried for an OOM, never escalated, and a # condemned range fails the mission. On spot, where evictions are # routine, that made the OOM and disk budgets effectively zero. - if verdict['outcome'] == 'timeout': - cap = MAX_TIMEOUT_ATTEMPTS - spent = _cause_count(end, attempt, ('timeout',)) - elif verdict['outcome'] == 'ephemeral': + # No timeout branch: a timeout sets reason = None above, which is + # terminal, so it never reaches the retry gate below. It had a budget + # of 2 when it was retryable. + if verdict['outcome'] == 'ephemeral': cap = MAX_EPHEMERAL_ATTEMPTS spent = _cause_count(end, attempt, ('ephemeral',)) elif verdict['outcome'] in ENVIRONMENTAL_OUTCOMES: diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 04c245d0..4f4e18b1 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -199,8 +199,6 @@ spec: value: {{ .Values.worker.resources.limits.ephemeral_storage | quote }} - name: MAX_ATTEMPTS value: {{ .Values.monitor.maxAttempts | quote }} - - name: MAX_TIMEOUT_ATTEMPTS - value: {{ .Values.monitor.maxTimeoutAttempts | quote }} - name: MAX_DISRUPTION_ATTEMPTS value: {{ .Values.monitor.maxDisruptionAttempts | quote }} - name: MEM_BUMP_FACTOR @@ -228,8 +226,6 @@ spec: {{- if .Values.monitor.profileConfigMap }} - name: PROFILE_PATH value: /profile/profile.json - - name: PROFILE_DEADLINE_FACTOR - value: {{ .Values.monitor.profileDeadlineFactor | quote }} - name: PROFILE_CPU_TIERS value: {{ .Values.monitor.profileCpuTiers | quote }} - name: PROFILE_MARGIN diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index fe92f0c1..fe594c28 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -77,11 +77,6 @@ monitor: # 3267 of 3859 profiled ranges still fit at 0.5 cores. Empty disables tiering # and every range keeps the configured cpu request. # Paired with profileCpuSlowdown: the measured runtime multiplier per tier. - # Multiple of a range's own profiled runtime to allow before it is called - # wedged, taking whichever is tighter: this or attemptDeadlineSeconds. One - # global number cannot bound runtimes that span 190x -- 3h killed 941 real - # ranges, 12h lets a wedged 13-minute range burn 56x its cost. 0 disables. - profileDeadlineFactor: 0 profileCpuTiers: "" profileMargin: 1.15 # CPU limit for ranges the profile has measured. Above the configured @@ -123,14 +118,6 @@ monitor: emitMissionLabel: false loggingIntervalSeconds: 10 maxAttempts: 5 - # Hangs are usually persistent (bad archive host, absent checkpoint), so they - # get a lower cap than evictions -- otherwise a wedged range costs - # maxAttempts x attemptDeadlineSeconds before it is reported. - # For a hang caught by attemptDeadlineSeconds. stellar-core bounds its own - # retries (RETRY_A_FEW=5 / RETRY_A_LOT=32, backoff capped at 512s) and exits 3 - # when they are exhausted, so the deadline is only a backstop for the - # pathological tail -- worst case ~3.4h for a RETRY_A_LOT work. - maxTimeoutAttempts: 2 # Evictions, admission rejections and monitor restarts are not the range's # fault, so they do not share the failure budget above. Measured on ssc-test: # ten evictions across 25 workers put four healthy ranges on attempt 3 of 5. @@ -146,10 +133,16 @@ monitor: # Must exceed any plausible monitor outage: completion is recorded to the # progress ConfigMap by the monitor, and a Job reclaimed before that happens # reads as "never ran" and gets redone. - # 0 = no deadline. A prod range runs ~50 min, so ~3h is generous while still - # catching a range wedged in archive retries. Measured: stellar-core retries a - # missing/unreachable archive indefinitely rather than failing. - attemptDeadlineSeconds: 10800 + # 0 = no deadline. Catches a range wedged in archive retries: stellar-core + # retries a missing/unreachable archive indefinitely rather than failing. + # + # 12h, not the 3h this used to be. "A prod range runs ~50 min" was the median + # talking -- runtimes span 190x, p25 771s to a measured max of 5.9h, so 3h + # killed 941 legitimate ranges and had to be hand-patched to 12h mid-run on + # 2026-07-30. A timeout is terminal, so each of those would fail the mission. + # Flat for every range; see ATTEMPT_DEADLINE_SECONDS in job_monitor.py for why + # scaling it per-range by the profile was tried and removed. + attemptDeadlineSeconds: 43200 jobTtlSeconds: 600 # Worker logs are pulled to the monitor's volume while each pod is still # alive, then collected in one exec at teardown instead of ~1024. diff --git a/src/MissionParallelCatchup/tests/conftest.py b/src/MissionParallelCatchup/tests/conftest.py index b911f39b..f3227618 100644 --- a/src/MissionParallelCatchup/tests/conftest.py +++ b/src/MissionParallelCatchup/tests/conftest.py @@ -44,7 +44,6 @@ def test_something(cluster): 'PROFILE_CPU_LIMIT': '', 'ATTEMPT_DEADLINE_SECONDS': 0, 'MAX_ATTEMPTS_PER_RANGE': 5, - 'MAX_TIMEOUT_ATTEMPTS': 2, 'MAX_DISRUPTION_ATTEMPTS': 20, 'MAX_EPHEMERAL_ATTEMPTS': 4, 'LIM_EPHEMERAL': '', diff --git a/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py b/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py index 9f4261f9..33917343 100644 --- a/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py +++ b/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py @@ -84,7 +84,6 @@ def test_the_deadline_is_on_the_job_so_it_can_be_patched_live(job, monkeypatch): a capacity stall cannot condemn a range. """ monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 10800) - monkeypatch.setattr(jm, 'PROFILE_DEADLINE_FACTOR', 0) j = jm.build_job(300, 420, 1, None) assert j.spec.active_deadline_seconds == 10800, \ "the deadline must be patchable, so it belongs on the JobSpec" diff --git a/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py b/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py index 89b8be42..fe4ea3ff 100644 --- a/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py +++ b/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py @@ -13,8 +13,8 @@ B. When the Job reports DeadlineExceeded the monitor takes that verdict unconditionally, over the pod's own terminated reason. A pod the kubelet OOM-killed inside a Job that also tripped its deadline is filed as a timeout: - no memory escalation, and MAX_TIMEOUT_ATTEMPTS (2) instead of the budget the - real cause earns. Two such events condemn the range and fail the mission. + no memory escalation, and no budget at all, because a timeout is terminal. + One such event condemns the range and fails the mission. Nothing here asserts on source text. Facet B is fully drivable with the shipped harness. Facet A needs the one thing the fake cluster does not have -- the piece @@ -140,8 +140,8 @@ def test_a_range_that_never_ran_still_fails_when_it_hits_the_deadline(cluster, m def test_a_stall_long_enough_to_hit_the_deadline_condemns_the_range(cluster, monkeypatch): """The run-ending shape: every attempt stalls, so every attempt "times out". - A timeout gets MAX_TIMEOUT_ATTEMPTS (2), so two stalls are enough to condemn - the range outright -- and a condemned range fails the mission. + A timeout is terminal, so the first stall condemns the range outright -- and + a condemned range fails the mission. """ monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) cluster.reconcile() diff --git a/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py b/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py index 3886e62d..ff47de07 100644 --- a/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py +++ b/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py @@ -176,15 +176,21 @@ def test_the_oom_budget_still_binds(cluster): f"a 6th OOM attempt was dispatched past the budget: {cluster.jobs()}") -def test_the_timeout_budget_still_binds(cluster): - """Two real timeouts exhaust MAX_TIMEOUT_ATTEMPTS.""" +def test_one_timeout_condemns_the_range(cluster): + """A timeout is terminal -- it has no budget to bind. + + The deadline exists only for a range wedged on an unreachable archive, and + retrying that just spends another 12h to learn the same thing. This used to + allow 2 attempts; the assertion is that a SECOND one is never dispatched. + """ end = dispatch(cluster) - hit(cluster, end, 'timeout', times=2) + hit(cluster, end, 'timeout') assert condemned(cluster, end), ( - f"two consecutive timeouts were not condemned; jobs={cluster.jobs()}") + f"the first timeout did not condemn the range; jobs={cluster.jobs()}") assert cluster.failed()[str(end)]['outcome'] == 'timeout' - assert not job_exists(cluster, end, 3) + assert not job_exists(cluster, end, 2), ( + f"a second attempt was dispatched after a terminal timeout: {cluster.jobs()}") def test_the_disruption_budget_still_binds(cluster): diff --git a/src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py b/src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py index 772edd4a..e0340bf6 100644 --- a/src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py +++ b/src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py @@ -1,4 +1,4 @@ -"""How long an attempt may run before it is called wedged. +"""The attempt deadline is flat, and must stay flat. The deadline exists for ONE failure mode, reproduced on ssc-test 2026-07-30: with an unreachable archive, stellar-core retries the bucket download forever. @@ -7,56 +7,64 @@ exhausts. Measured: 0 ledgers closed, 9 fetch failures in 2.5 min, no give-up wording, no exit. Nothing but this deadline stops it. -One global number cannot bound it, because runtimes span 190x (p25 771s, max -5.9h): 3h killed 941 legitimate ranges, 12h kills none but lets a wedged 771s -range burn 56x its cost first. So take whichever bound is tighter. -""" +Scaling it by each range's profiled runtime was tried and removed, and this +file is the guard against it coming back. A deadline has to bound a range's +WORST case; a profile only offers a neighbour's TYPICAL case. Range keys are +anchored to the network tip, so a profile from an earlier run matches ZERO keys +exactly and every lookup lands on a neighbour -- and ~2% of neighbours are +3-38x cheaper than their surroundings. Backtested across that real grid offset +(run4 profile -> r5 actuals, 3983 ranges): a 2x factor falsely kills 134 +ranges, 4x kills 46, 6x kills 21. Flat 12h kills none. + +The asymmetry is what settles it. A false kill loses a range, and a timeout is +terminal, so it fails the whole mission. A genuine wedge holds ONE slot of +1092-1500 for 12h, about 0.1% of a run's capacity. -import pytest +Asserted against the Jobs reconcile actually creates, not against a helper. +""" import job_monitor as jm -PROFILE = [(100, {'seconds': 600.0}), (200, {'seconds': 10000.0})] +DEADLINE = 43200 +# The two ranges the fixture dispatches (PARALLELISM 2, tip-first), given +# measured costs that differ by 17x. Under the removed scaling these produced +# deadlines of 1800s and 30000s; they must now be identical. +PROFILE = [(200, {'seconds': 600.0}), (300, {'seconds': 10000.0})] -@pytest.fixture -def sized(monkeypatch): - monkeypatch.setattr(jm, 'PROFILE', PROFILE) - monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 43200) - monkeypatch.setattr(jm, 'PROFILE_DEADLINE_FACTOR', 3.0) +def _deadline_of(cluster, end, attempt=1): + return cluster.k8s.job(jm.job_name(int(end), attempt)).spec.active_deadline_seconds -def test_disabled_by_default_keeps_the_configured_ceiling(monkeypatch): - monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 43200) - monkeypatch.setattr(jm, 'PROFILE_DEADLINE_FACTOR', 0) - assert jm._attempt_deadline(100) == 43200 +def test_the_cheapest_and_costliest_ranges_get_the_same_deadline(cluster, monkeypatch): + """The regression guard. A 600s range and a 10000s range are bounded alike. -def test_no_ceiling_and_no_factor_means_no_deadline(monkeypatch): - monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 0) - monkeypatch.setattr(jm, 'PROFILE_DEADLINE_FACTOR', 0) - assert jm._attempt_deadline(100) is None + Tightening the cheap one is exactly what killed 134 ranges in the backtest: + its `seconds` came from a neighbour, and the neighbour was wrong. + """ + monkeypatch.setattr(jm, 'PROFILE', PROFILE) + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) + cluster.reconcile() + assert _deadline_of(cluster, 200) == DEADLINE + assert _deadline_of(cluster, 300) == DEADLINE -def test_a_cheap_range_gets_a_tight_bound_not_the_ceiling(sized): - # 600s x3 = 1800s. Under a 12h ceiling a wedged 10-minute range would - # otherwise burn 72x its cost before anything noticed. - assert jm._attempt_deadline(100) == 1800 +def test_an_unprofiled_range_gets_the_same_deadline_too(cluster, monkeypatch): + """No profile at all changes nothing -- there is nothing to scale by.""" + monkeypatch.setattr(jm, 'PROFILE', []) + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) + cluster.reconcile() -def test_an_expensive_range_still_gets_room(sized): - assert jm._attempt_deadline(200) == 30000 # 10000 x 3, under the ceiling + assert _deadline_of(cluster, 300) == DEADLINE -def test_the_ceiling_still_wins_when_it_is_tighter(monkeypatch): +def test_zero_disables_the_deadline_entirely(cluster, monkeypatch): + """0 must mean absent, not 0 -- a zero-second deadline kills every attempt + the moment it is created.""" monkeypatch.setattr(jm, 'PROFILE', PROFILE) - monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 7200) - monkeypatch.setattr(jm, 'PROFILE_DEADLINE_FACTOR', 3.0) - assert jm._attempt_deadline(200) == 7200 # 30000 scaled, 7200 ceiling - + monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 0) + cluster.reconcile() -def test_an_unprofiled_range_falls_back_to_the_ceiling(sized): - # Newer than anything measured, so there is no honest estimate to tighten - # with -- and it is the most expensive kind, so guessing low is the bad - # direction. - assert jm._attempt_deadline(999999) == 43200 + assert _deadline_of(cluster, 300) is None diff --git a/src/MissionParallelCatchup/tests/unit/test_sizing.py b/src/MissionParallelCatchup/tests/unit/test_sizing.py index ab4a721c..14de0d57 100644 --- a/src/MissionParallelCatchup/tests/unit/test_sizing.py +++ b/src/MissionParallelCatchup/tests/unit/test_sizing.py @@ -91,12 +91,12 @@ def test_ephemeral_storage_escalates_and_caps_the_same_way(eph): # --- the budgets the ladders are climbing ------------------------------------ def test_attempt_budgets_are_ordered_by_whose_fault_the_failure_was(): - # A hang is usually persistent, so it gets the fewest tries. A genuinely - # broken range gets the middle budget. Anything the cluster did to us gets - # the most -- on spot, evictions are routine and must not condemn a range. - assert jm.MAX_TIMEOUT_ATTEMPTS < jm.MAX_ATTEMPTS_PER_RANGE < jm.MAX_DISRUPTION_ATTEMPTS, ( - f"budgets out of order: timeout={jm.MAX_TIMEOUT_ATTEMPTS} " - f"range={jm.MAX_ATTEMPTS_PER_RANGE} disruption={jm.MAX_DISRUPTION_ATTEMPTS}") + # A genuinely broken range gets the middle budget. Anything the cluster did + # to us gets the most -- on spot, evictions are routine and must not condemn + # a range. A hang has no budget at all: a timeout is terminal. + assert jm.MAX_ATTEMPTS_PER_RANGE < jm.MAX_DISRUPTION_ATTEMPTS, ( + f"budgets out of order: range={jm.MAX_ATTEMPTS_PER_RANGE} " + f"disruption={jm.MAX_DISRUPTION_ATTEMPTS}") assert jm.MAX_ATTEMPTS_PER_RANGE > 1, "a range that OOMs once could never escalate" assert jm.MAX_EPHEMERAL_ATTEMPTS > 1, "a range evicted on disk once could never grow" assert jm.MAX_DISRUPTION_ATTEMPTS >= 10, \ From 8b36059ceeaf4557ee097d6ef5ec85c20d63d0c8 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Fri, 31 Jul 2026 09:55:26 -0400 Subject: [PATCH 059/117] Stop setting cpu and memory limits on workers Requests only. The only limit a worker still gets is ephemeral-storage, because that is the one dimension where an unbounded pod takes the whole node down with it rather than just itself, and it has its own escalation ladder. Memory is the substantive change. A limit is a hard cap on anon PLUS page cache, and sizing it per-range from a profile got it wrong in the one direction nothing alarms on. Measured 2026-07-31, range 39210943: sized at 1729Mi from a neighbour, genuinely needed 1620Mi of anon, which left ~110Mi for cache. It never OOMed -- it thrashed. 544k major page faults, 0.22 cores used on a node it had entirely to itself, 0.95 ledgers/s against a neighbour norm of 3.3, and it held 1092 idle slots open for three hours at the end of an otherwise finished run. CPU was already unlimited in practice: PROFILE_CPU_LIMIT defaulted empty and _resources popped the cpu limit. LIM_CPU was computed and discarded on every call. Both are gone now, along with PROFILE_CPU_LIMIT, whose only function was to put a cpu limit back. The OOM ladder now escalates the REQUEST. That still buys an OOMing range the two things it needs -- placement where the memory is actually free, and a higher bar before the kubelet picks it as an eviction victim -- and it rebases the ladder off REQ_MEM (9Gi) rather than the old 24000Mi limit, so the rungs are 9216/13824/20736/31104Mi. Removed across the stack: LIM_CPU/LIM_MEM/PROFILE_CPU_LIMIT in the monitor, their env in the chart, worker.resources.limits.cpu/memory in values.yaml, and the two setOptions plus the now-unused Limits reads in MissionHistoryPubnetParallelCatchupV2.fs. 510 tests + 1 xfail. Putting either limit back fails 5 of them. --- .../MissionHistoryPubnetParallelCatchupV2.fs | 11 +- src/MissionParallelCatchup/job_monitor.py | 100 ++++++++---------- .../templates/job_monitor.yaml | 6 -- .../parallel_catchup_helm/values.yaml | 12 +-- src/MissionParallelCatchup/tests/conftest.py | 1 - .../tests/contract/test_chart_defaults.py | 13 ++- .../tests/reconcile/test_attempt_deadline.py | 8 +- .../tests/reconcile/test_retry_budgets.py | 8 +- .../tests/resilience/test_crash_points.py | 2 +- .../tests/resilience/test_restart_fuzz.py | 4 +- .../tests/test_harness_smoke.py | 6 +- .../tests/unit/test_cpu_tiers.py | 2 +- .../tests/unit/test_resources.py | 60 +++++------ .../tests/unit/test_sizing.py | 2 +- 14 files changed, 97 insertions(+), 138 deletions(-) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index b4c19b12..f0c4cdcb 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -199,8 +199,6 @@ let installProject (context: MissionContext) = let resourceRequirements = ParallelCatchupCoreResourceRequirements let cpuReqMili = resourceRequirements.Requests.["cpu"].ToString() let memReqMebi = resourceRequirements.Requests.["memory"].ToString() - let cpuLimMili = resourceRequirements.Limits.["cpu"].ToString() - let memLimMebi = resourceRequirements.Limits.["memory"].ToString() // StellarKubeSpecs sizes ephemeral-storage for ephemeral mode, where /data is // an emptyDir on the node. In pvc mode /data is on the volume and the node // disk only holds logs and tmp, so asking for the full amount reserves disk @@ -215,15 +213,12 @@ let installProject (context: MissionContext) = LogInfo "Resource requirements from StellarKubeCfg:\n\ CPU request: %s\n\ - CPU limit: %s\n\ Memory request: %s\n\ - Memory limit: %s\n\ Storage request: %s\n\ - Storage limit: %s" + Storage limit: %s\n\ + (workers run with no cpu or memory limit)" cpuReqMili - cpuLimMili memReqMebi - memLimMebi storageReqGibi storageLimGibi @@ -236,8 +231,6 @@ let installProject (context: MissionContext) = setOptions.Add(sprintf "worker.resources.requests.cpu=%s" cpuReqEffective) setOptions.Add(sprintf "worker.resources.requests.memory=%s" memReqMebi) - setOptions.Add(sprintf "worker.resources.limits.cpu=%s" cpuLimMili) - setOptions.Add(sprintf "worker.resources.limits.memory=%s" memLimMebi) setOptions.Add(sprintf "worker.resources.requests.ephemeral_storage=%s" storageReqGibi) setOptions.Add(sprintf "worker.resources.limits.ephemeral_storage=%s" storageLimGibi) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 36518a0c..7eb4b999 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -84,30 +84,32 @@ # IRSA trust policies keep matching. WORKER_SERVICE_ACCOUNT = os.getenv('WORKER_SERVICE_ACCOUNT', '') -# Pod resources. +# Pod resources. Requests only: workers are given no cpu limit and no memory +# limit at all. +# +# CPU because a limit only throttles a pod that could otherwise use idle cores, +# and throttling changes what the range measures -- less cpu means less download +# concurrency means a lower peak, so a throttled attempt records a figure an +# unthrottled one cannot reproduce. +# +# Memory because a limit is a hard cap on anon PLUS page cache, and sizing it +# per-range from a profile got it wrong in the one direction that has no alarm +# on it. Measured 2026-07-31, range 39210943: sized at 1729Mi from a neighbour, +# genuinely needed 1620Mi of anon, which left ~110Mi for cache. It never OOMed +# -- it thrashed. 544k major page faults, 0.22 cores used on a node it had +# entirely to itself, 0.95 ledgers/s against a neighbour norm of 3.3, and it +# held 1092 idle slots open for three hours at the end of the run. +# +# Without a limit the request still does the real work: it places the pod and +# it sets eviction order under node pressure. What goes away is the cliff. REQ_CPU = os.getenv('REQ_CPU', '1800m') REQ_MEM = os.getenv('REQ_MEM', '9Gi') -LIM_CPU = os.getenv('LIM_CPU', '2') -LIM_MEM = os.getenv('LIM_MEM', '24000Mi') # Only meaningful in ephemeral storage mode; see check_storage_config(). # Range profile from an earlier run: tightens per-range requests so more # workers fit per node. Requests only -- limits stay as configured, so the # failure semantics and the OOM/disk escalation ladders are unchanged. PROFILE_PATH = os.getenv('PROFILE_PATH', '') PROFILE_MARGIN = float(os.getenv('PROFILE_MARGIN', 1.15)) -# CPU limit for a range the profile has measured. Higher than the unprofiled -# default on purpose: at a 2-core limit every range pegs 2.0, so the measured -# peak is a ceiling and the profile can never learn real demand. Room above the -# request lets each run's peak climb until it finds the true one. -# Empty = no cpu limit at all on a measured range. Measured on ssc-test with -# one pod per node (m8id/NVMe, 16320-ledger range): 168s at limit 2, 111s at 4, -# 99s uncapped. cpu.weight still derives from the request, so a burst only uses -# cycles the neighbours are not using. Set a value to cap it again. -# -# The gain is in bucket-apply and replay, not download: at 65280 ledgers, two -# pods at limit 2 and limit 4 had written 8001 and 8013 MiB after 43 minutes -- -# identical -- because the download phase is storage-bound, not CPU-bound. -PROFILE_CPU_LIMIT = os.getenv('PROFILE_CPU_LIMIT', '') # No safety margin on cpu, unlike memory. Under-requesting cpu costs contention # and the pod can still burst; under-requesting memory gets it OOMKilled. # Ceiling for profile-derived memory, above the unprofiled limit for the same @@ -1300,14 +1302,19 @@ def _cause_count(end, attempt, causes): def mem_for_attempt(attempt, base=None): - """Memory limit after N OOMs, capped at MEM_ESCALATION_CAP. + """Memory REQUEST after N OOMs, capped at MEM_ESCALATION_CAP. `base` is what attempt 1 actually ran with. It matters when a profile sized - the range: escalating a 209Mi profiled range off the configured 24000Mi - limit jumps straight to 36000Mi, a 172x overshoot that throws away the whole + the range: escalating a 209Mi profiled range off the configured default + jumps straight to 36000Mi, a 172x overshoot that throws away the whole packing win on the first OOM. + + Escalating the request, not a limit, because there is no limit any more. It + still buys the same two things an OOMing range needs -- placement somewhere + with the memory actually free, and a higher bar before the kubelet picks it + as an eviction victim. """ - base_q = _quantity_bytes(base or LIM_MEM) + base_q = _quantity_bytes(base or REQ_MEM) want = int(base_q * (MEM_BUMP_FACTOR ** max(0, attempt - 1))) cap = _quantity_bytes(MEM_ESCALATION_CAP) return _bytes_to_quantity(min(want, cap)) @@ -1976,12 +1983,12 @@ def _resources(mem=None, eph=None, end=None): # Before mem is defaulted below -- reading it afterwards can never see None, # which silently disabled profile sizing entirely. overrides = _profile_overrides(end, escalated=(mem is not None or eph is not None)) - mem = mem or LIM_MEM - # Raise the request alongside the limit on an escalated retry: a pod that - # OOMed at the old limit will not fit where it was scheduled before. - req_mem = REQ_MEM if mem == LIM_MEM else mem - req = {'cpu': REQ_CPU, 'memory': req_mem} - lim = {'cpu': LIM_CPU, 'memory': mem} + # `mem` is the escalated request on an OOM retry, else the configured one. + req = {'cpu': REQ_CPU, 'memory': mem or REQ_MEM} + # Nothing but ephemeral-storage is ever limited. That one stays: it is the + # only dimension where an unbounded pod takes the whole NODE down with it + # rather than just itself, and it has its own escalation ladder. + lim = {} # Only meaningful in ephemeral mode. In PVC mode a large request makes disk # the binding dimension and halves workers-per-node for no reason. @@ -1997,39 +2004,16 @@ def _resources(mem=None, eph=None, end=None): if LIM_EPHEMERAL: lim['ephemeral-storage'] = eph or LIM_EPHEMERAL - # No cpu limit on any worker unless one is configured explicitly. Packing is - # driven by the request; a limit only throttles a pod that could otherwise - # use idle cores, and throttling changes what the range measures -- less cpu - # means less download concurrency means a lower peak, so a throttled attempt - # records a figure an unthrottled one cannot reproduce. - # - # This used to be applied only when _profile_overrides returned something, - # which silently excluded two populations: unmeasured ranges, and escalated - # retries (escalated returns {} as well). Measured on ssc-test 2026-07-30, - # 214 a1 and 256 a2 pods were capped at cpu 2 while their peers ran free -- - # and for the retries that meant more memory and less cpu at the same time, - # right after an OOM. - if PROFILE_CPU_LIMIT: - lim['cpu'] = PROFILE_CPU_LIMIT - else: - lim.pop('cpu', None) - if overrides: - # Memory and disk match request to limit: those are the dimensions worth - # pinning, since exceeding either kills the pod outright. - # - # CPU is deliberately not matched. Its limit stays where it is - # configured and only the request follows the measurement, so a range - # packs by what it actually uses while keeping headroom to burst. That - # leaves the pod Burstable rather than Guaranteed -- Kubernetes needs - # all three to match -- which is the intended trade. - for key, value in overrides.items(): - req[key] = value - if key != 'cpu': - lim[key] = value - # Unmeasured range: the configured defaults, requests below limits, exactly - # as before -- a range with no profile entry must behave as if there were no + # The profile only ever moves requests now. Disk is the one exception, and + # only because its limit is what the kubelet enforces -- match it so a range + # measured to need more disk is actually allowed to use it. + for key, value in overrides.items(): + req[key] = value + if key == 'ephemeral-storage' and LIM_EPHEMERAL: + lim[key] = value + # Unmeasured range: the configured requests, exactly as if there were no # profile at all. - return client.V1ResourceRequirements(requests=req, limits=lim) + return client.V1ResourceRequirements(requests=req, limits=lim or None) def volume_spread_constraints(): diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 4f4e18b1..ee038326 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -191,10 +191,6 @@ spec: # makes disk the binding dimension and halves workers-per-node. - name: REQ_EPHEMERAL value: {{ .Values.worker.resources.requests.ephemeral_storage | quote }} - - name: LIM_CPU - value: {{ .Values.worker.resources.limits.cpu | quote }} - - name: LIM_MEM - value: {{ .Values.worker.resources.limits.memory | quote }} - name: LIM_EPHEMERAL value: {{ .Values.worker.resources.limits.ephemeral_storage | quote }} - name: MAX_ATTEMPTS @@ -230,8 +226,6 @@ spec: value: {{ .Values.monitor.profileCpuTiers | quote }} - name: PROFILE_MARGIN value: {{ .Values.monitor.profileMargin | quote }} - - name: PROFILE_CPU_LIMIT - value: {{ .Values.monitor.profileCpuLimit | quote }} - name: PROFILE_MAX_MEM value: {{ .Values.monitor.profileMaxMemory | quote }} - name: PROFILE_CACHE_HEADROOM diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index fe594c28..a15440c8 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -31,8 +31,9 @@ worker: memory: "" ephemeral_storage: "" limits: - cpu: "" - memory: "" + # cpu and memory are deliberately absent -- workers run unlimited on both. + # See REQ_CPU/REQ_MEM in job_monitor.py. Only disk is capped, because that + # is the one dimension where an unbounded pod takes the node with it. ephemeral_storage: "" historyGetCommandCore001: "curl -sf http://history.stellar.org/prd/core-live/core_live_001/{0} -o {1}" @@ -79,13 +80,6 @@ monitor: # Paired with profileCpuSlowdown: the measured runtime multiplier per tier. profileCpuTiers: "" profileMargin: 1.15 - # CPU limit for ranges the profile has measured. Above the configured - # worker limit on purpose: at a 2-core limit every range pegs 2.0, so the - # peak is a ceiling and the profile never learns real demand. Unprofiled - # ranges keep worker.resources.limits.cpu unchanged. - # Empty = measured ranges run with no cpu limit, which is fastest on an - # otherwise-free node (168s/111s/99s at limit 2/4/none). Set a value to cap. - profileCpuLimit: "" # No margin on cpu: it is compressible, so under-requesting costs contention # Ceiling for profile-derived memory. Above the configured worker limit on # purpose: a range needing more than that must be able to ask for it diff --git a/src/MissionParallelCatchup/tests/conftest.py b/src/MissionParallelCatchup/tests/conftest.py index f3227618..c168c626 100644 --- a/src/MissionParallelCatchup/tests/conftest.py +++ b/src/MissionParallelCatchup/tests/conftest.py @@ -41,7 +41,6 @@ def test_something(cluster): 'STORAGE_CLASS': 'gp3', 'SAVE_SUCCESS_LOGS': True, 'PROFILE_PATH': '', - 'PROFILE_CPU_LIMIT': '', 'ATTEMPT_DEADLINE_SECONDS': 0, 'MAX_ATTEMPTS_PER_RANGE': 5, 'MAX_DISRUPTION_ATTEMPTS': 20, diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py index 36f2c8c6..ef1b2542 100644 --- a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py +++ b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py @@ -48,8 +48,6 @@ # purpose and the mission fills them in on every install. 'REQ_CPU': 'left empty in the chart; StellarKubeSpecs.fs supplies it', 'REQ_MEM': 'left empty in the chart; StellarKubeSpecs.fs supplies it', - 'LIM_CPU': 'left empty in the chart; StellarKubeSpecs.fs supplies it', - 'LIM_MEM': 'left empty in the chart; StellarKubeSpecs.fs supplies it', } @@ -193,9 +191,10 @@ def test_the_sizing_headroom_is_a_real_allowance_in_both_places(): headroom = jm._quantity_bytes(code['PROFILE_CACHE_HEADROOM']) assert headroom >= 256 * 1024 ** 2, ( f"{code['PROFILE_CACHE_HEADROOM']} of fixed headroom is what OOMed 90 small ranges") - # ...and the ceiling has to sit above the configured worker limit, or a - # range needing more than that is pinned under its own measured peak. + # ...and the ceiling has to sit above the configured request, or a range + # measured above it can never ask for what it actually uses and will pack as + # though it were small. assert (jm._quantity_bytes(code['PROFILE_MAX_MEM']) - > jm._quantity_bytes(code['LIM_MEM'])), ( - "the profile ceiling is at or below the worker limit, so a hungry range " - "can never ask for what it measured") + > jm._quantity_bytes(code['REQ_MEM'])), ( + "the profile ceiling is at or below the configured request, so a hungry " + "range can never ask for what it measured") diff --git a/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py b/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py index fe4ea3ff..d06849ea 100644 --- a/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py +++ b/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py @@ -228,10 +228,10 @@ def test_an_oom_inside_a_deadline_exceeded_job_escalates_memory(cluster, monkeyp assert jm.read_outcome('300', 1)['outcome'] == 'oom' assert 'pc-r300-a2' in cluster.jobs() resources = _memory(cluster, 'pc-r300-a2') - assert resources.limits['memory'] == '36000Mi', ( + assert resources.requests['memory'] == '13824Mi', ( "the retry went out at the same limit that OOM-killed it: the Job's " "DeadlineExceeded overwrote the kubelet's OOMKilled") - assert resources.requests['memory'] == '36000Mi' + assert resources.requests['memory'] == '13824Mi' def test_two_ooms_inside_deadline_exceeded_jobs_do_not_condemn_the_range(cluster, monkeypatch): @@ -250,7 +250,7 @@ def test_two_ooms_inside_deadline_exceeded_jobs_do_not_condemn_the_range(cluster "of retrying on the 5-attempt range budget") assert 'pc-r300-a3' in cluster.jobs() # Two rungs climbed, capped at MAX_MEM (48Gi). - assert _memory(cluster, 'pc-r300-a3').limits['memory'] == '49152Mi' + assert _memory(cluster, 'pc-r300-a3').requests['memory'] == '20736Mi' def test_a_disruption_inside_a_deadline_exceeded_job_keeps_its_own_budget(cluster, monkeypatch): @@ -274,7 +274,7 @@ def test_a_disruption_inside_a_deadline_exceeded_job_keeps_its_own_budget(cluste "downgraded them to the 2-attempt timeout budget") assert 'pc-r300-a3' in cluster.jobs() # An eviction says nothing about how much memory the range wants. - assert _memory(cluster, 'pc-r300-a3').limits['memory'] == jm.LIM_MEM + assert _memory(cluster, 'pc-r300-a3').requests['memory'] == jm.REQ_MEM def test_an_ephemeral_eviction_inside_a_deadline_exceeded_job_still_grows_the_disk(cluster, monkeypatch): diff --git a/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py b/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py index ff47de07..cd89da9c 100644 --- a/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py +++ b/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py @@ -77,8 +77,8 @@ def test_five_evictions_do_not_burn_the_whole_oom_budget(cluster): # And the whole point of an OOM retry: more memory. One OOM = one rung. res = mem_of(cluster, end, 7) - assert res.limits['memory'] == '36000Mi' - assert res.requests['memory'] == '36000Mi' + assert res.requests['memory'] == '13824Mi' + assert res.requests['memory'] == '13824Mi' def test_evictions_do_not_burn_the_disk_budget(cluster, monkeypatch): @@ -151,7 +151,7 @@ def test_memory_ladder_follows_ooms_not_evictions(cluster, monkeypatch): hit(cluster, end, 'disrupted', times=3) # attempts 1-3, now on 4 hit(cluster, end, 'oom') # OOM #1 on attempt 4 -> a5 assert job_exists(cluster, end, 5) - assert mem_of(cluster, end, 5).limits['memory'] == '36000Mi' + assert mem_of(cluster, end, 5).requests['memory'] == '13824Mi' hit(cluster, end, 'disrupted', times=2) # attempts 5-6, now on 7 hit(cluster, end, 'oom') # OOM #2 on attempt 7 -> a8 @@ -159,7 +159,7 @@ def test_memory_ladder_follows_ooms_not_evictions(cluster, monkeypatch): assert not condemned(cluster, end), f"failed={cluster.failed()}" assert job_exists(cluster, end, 8) # 24000Mi * 1.5^2 -- two OOMs, six evictions, two rungs. - assert mem_of(cluster, end, 8).limits['memory'] == '54000Mi' + assert mem_of(cluster, end, 8).requests['memory'] == '20736Mi' # --- the caps must still bind (a fix that just removes them is not a fix) ---- diff --git a/src/MissionParallelCatchup/tests/resilience/test_crash_points.py b/src/MissionParallelCatchup/tests/resilience/test_crash_points.py index f0012683..5e065001 100644 --- a/src/MissionParallelCatchup/tests/resilience/test_crash_points.py +++ b/src/MissionParallelCatchup/tests/resilience/test_crash_points.py @@ -442,7 +442,7 @@ def test_crash_between_the_verdict_and_the_retry_create(cluster, monkeypatch): .spec.template.spec.containers[0].resources) # One OOM seen, so exactly one rung: 24000Mi * 1.5. Two would mean the # replayed attempt was counted twice. - assert resources.limits['memory'] == '36000Mi' + assert resources.requests['memory'] == '13824Mi' assert jm._cause_count('300', 1, ('oom', 'failed')) == 1 assert cluster.failed() == {} diff --git a/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py b/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py index 2017c91f..711e3483 100644 --- a/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py +++ b/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py @@ -327,7 +327,7 @@ def test_restart_mid_retry_keeps_the_attempt_number(big_run): cluster.reconcile() assert cluster.attempt_of(1200) == 2 limit = (cluster.k8s.job('pc-r1200-a2') - .spec.template.spec.containers[0].resources.limits['memory']) + .spec.template.spec.containers[0].resources.requests['memory']) restart(cluster) cluster.advance(1200, 'oom', attempt=2) @@ -337,7 +337,7 @@ def test_restart_mid_retry_keeps_the_attempt_number(big_run): # so the restart must not reset it to the first rung. assert cluster.attempt_of(1200) == 3 escalated = (cluster.k8s.job('pc-r1200-a3') - .spec.template.spec.containers[0].resources.limits['memory']) + .spec.template.spec.containers[0].resources.requests['memory']) assert jm._quantity_bytes(escalated) > jm._quantity_bytes(limit) assert cluster.failed() == {} diff --git a/src/MissionParallelCatchup/tests/test_harness_smoke.py b/src/MissionParallelCatchup/tests/test_harness_smoke.py index e40d9512..903ad5a2 100644 --- a/src/MissionParallelCatchup/tests/test_harness_smoke.py +++ b/src/MissionParallelCatchup/tests/test_harness_smoke.py @@ -115,8 +115,8 @@ def test_an_oom_retry_escalates_the_memory_limit(cluster): .spec.template.spec.containers[0].resources) # One OOM = one rung: 24000Mi * 1.5. The request follows the limit, because # a pod that OOMed will not fit where it was scheduled before. - assert resources.limits['memory'] == '36000Mi' - assert resources.requests['memory'] == '36000Mi' + assert resources.requests['memory'] == '13824Mi' + assert resources.requests['memory'] == '13824Mi' assert cluster.failed() == {} @@ -132,7 +132,7 @@ def test_a_disruption_does_not_spend_the_range_budget(cluster): # Memory is untouched: an eviction says nothing about how much the range wants. resources = (cluster.k8s.job('pc-r300-a2') .spec.template.spec.containers[0].resources) - assert resources.limits['memory'] == jm.LIM_MEM + assert resources.requests['memory'] == jm.REQ_MEM def test_progress_going_backwards_redispatches_rather_than_halting(cluster): diff --git a/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py b/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py index 1591c58e..83ab0c8a 100644 --- a/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py +++ b/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py @@ -82,7 +82,7 @@ def test_cpu_is_requested_but_never_limited(monkeypatch, cluster): monkeypatch.setattr(jm, '_SORTED_SECONDS', None) r = jm._resources(end=300) assert r.requests['cpu'] == '0.5' - assert 'cpu' not in r.limits + assert not r.limits or 'cpu' not in r.limits def test_a_one_range_profile_gives_that_range_the_top_tier(tiered, monkeypatch): diff --git a/src/MissionParallelCatchup/tests/unit/test_resources.py b/src/MissionParallelCatchup/tests/unit/test_resources.py index e2a605f0..91cc6ec2 100644 --- a/src/MissionParallelCatchup/tests/unit/test_resources.py +++ b/src/MissionParallelCatchup/tests/unit/test_resources.py @@ -25,18 +25,15 @@ @pytest.fixture def sizing(monkeypatch): """The worker's configured shape, plus a loaded profile.""" - def configure(ranges=PROFILE_RANGES, margin=1.1, lim_mem='24000Mi', + def configure(ranges=PROFILE_RANGES, margin=1.1, req_mem='9Gi', req_eph='35Gi', lim_eph='40Gi', max_mem='32Gi', - headroom='512Mi', cpu_limit=''): + headroom='512Mi'): monkeypatch.setattr(jm, 'PROFILE', sorted(ranges)) monkeypatch.setattr(jm, 'PROFILE_MARGIN', margin) monkeypatch.setattr(jm, 'PROFILE_MAX_MEM', max_mem) monkeypatch.setattr(jm, 'PROFILE_CACHE_HEADROOM', headroom) - monkeypatch.setattr(jm, 'PROFILE_CPU_LIMIT', cpu_limit) monkeypatch.setattr(jm, 'REQ_CPU', '1800m') - monkeypatch.setattr(jm, 'LIM_CPU', '2') - monkeypatch.setattr(jm, 'REQ_MEM', '9Gi') - monkeypatch.setattr(jm, 'LIM_MEM', lim_mem) + monkeypatch.setattr(jm, 'REQ_MEM', req_mem) monkeypatch.setattr(jm, 'REQ_EPHEMERAL', req_eph) monkeypatch.setattr(jm, 'LIM_EPHEMERAL', lim_eph) return configure @@ -67,21 +64,21 @@ def test_profile_gives_nothing_past_its_high_water_mark(sizing): assert jm._profile_overrides(None, escalated=False) == {} -def test_profile_memory_is_capped_at_its_own_ceiling_not_the_worker_limit(sizing): - # A range needing more than the configured limit must be able to ask for it, - # or it is pinned under its own measured peak and OOMs every attempt. The +def test_profile_memory_is_capped_at_its_own_ceiling_not_the_configured_request(sizing): + # A range measured above the configured request must be able to ask for more, + # or it packs as though it were small and lands somewhere it cannot fit. The # ceiling is what bounds it, and the OOM ladder can still climb past that. sizing(ranges=[(1, {'peakRssBytes': 500_000_000_000})], - lim_mem='24000Mi', max_mem='32Gi') + req_mem='9Gi', max_mem='32Gi') assert jm._profile_overrides(1, escalated=False)['memory'] == '32768Mi' -def test_profile_memory_can_exceed_the_configured_worker_limit(sizing): - # 28 GB peak against a 24000Mi configured limit: the profile must raise it. +def test_profile_memory_can_exceed_the_configured_request(sizing): + # 28 GB peak against a 9Gi configured request: the profile must raise it. sizing(ranges=[(1, {'peakRssBytes': 28_000_000_000})], - lim_mem='24000Mi', max_mem='32Gi') + req_mem='9Gi', max_mem='32Gi') got = jm._profile_overrides(1, escalated=False)['memory'] - assert jm._quantity_bytes(got) > jm._quantity_bytes('24000Mi') + assert jm._quantity_bytes(got) > jm._quantity_bytes('9Gi') def test_memory_is_sized_from_rss_never_from_working_set(sizing): @@ -132,37 +129,39 @@ def test_the_sizing_formula_is_peak_times_margin_plus_headroom(sizing, peak_mi): # --- what lands on the container --------------------------------------------- -def test_a_measured_range_matches_memory_and_disk_and_leaves_cpu_configured(sizing): - # Memory and disk match request to limit -- exceeding either kills the pod. - # CPU keeps its configured request and is left uncapped, so the range packs - # by what it uses and can still burst. +def test_a_measured_range_requests_its_measurement_and_limits_only_disk(sizing): + # The profile moves requests. Disk is the one dimension still limited, and + # its limit is matched so a range measured to need more is allowed to use it. sizing() r = jm._resources(end=2000) - assert r.requests['memory'] == r.limits['memory'] == '3659Mi' + assert r.requests['memory'] == '3659Mi' assert r.requests['ephemeral-storage'] == r.limits['ephemeral-storage'] == '4196Mi' # The configured request, not a measured one -- a profiled range now packs # at exactly the same cpu as an unprofiled one. assert r.requests['cpu'] == '1800m' - assert 'cpu' not in r.limits, "a measured range runs uncapped" + assert set(r.limits) == {'ephemeral-storage'}, \ + f"a worker may only ever be limited on disk, got {sorted(r.limits)}" -def test_an_unmeasured_range_keeps_the_mismatched_defaults(sizing): +def test_an_unmeasured_range_keeps_the_configured_requests(sizing): # No profile entry must behave exactly as if there were no profile at all. sizing() r = jm._resources(end=99999) - assert r.requests['memory'] == '9Gi' and r.limits['memory'] == '24000Mi' + assert r.requests['memory'] == '9Gi' + assert 'memory' not in r.limits assert r.requests['ephemeral-storage'] == '35Gi' assert r.limits['ephemeral-storage'] == '40Gi' - assert r.requests != r.limits -def test_an_escalated_retry_keeps_its_own_size_and_raises_the_request_with_it(sizing): +def test_an_escalated_retry_keeps_its_own_size(sizing): # The escalation already chose the size; the profile must not overwrite it. - # The request moves too: a pod that OOMed at the old limit will not fit - # where it was scheduled before. + # It lands on the request, which is the whole mechanism now: a bigger request + # places the pod where the memory is actually free, and raises the bar before + # the kubelet picks it as an eviction victim. sizing() r = jm._resources(mem='36000Mi', end=2000) - assert r.requests['memory'] == r.limits['memory'] == '36000Mi' + assert r.requests['memory'] == '36000Mi' + assert 'memory' not in r.limits, "an escalated retry must not be capped either" assert r.requests['cpu'] == '1800m', "cpu must fall back to the configured request" @@ -174,7 +173,7 @@ def test_ephemeral_escalation_raises_request_and_limit_together(sizing): assert r.requests['ephemeral-storage'] == r.limits['ephemeral-storage'] == '60Gi' -def test_no_worker_gets_a_cpu_limit_unless_one_is_configured(sizing): +def test_no_worker_gets_a_cpu_or_memory_limit(sizing): # _profile_overrides returns {} for BOTH "no profile entry" and "escalated # attempt". Treating them the same handed an OOM retry more memory while # capping it at LIM_CPU, when the attempt that just failed ran unlimited. @@ -192,13 +191,10 @@ def test_no_worker_gets_a_cpu_limit_unless_one_is_configured(sizing): for r, why in ((measured, 'measured'), (escalated, 'escalated retry'), (unmeasured, 'unprofiled')): assert 'cpu' not in r.limits, f"{why} range was throttled: {r.limits}" + assert 'memory' not in r.limits, f"{why} range was capped: {r.limits}" assert r.requests['cpu'] == '1800m', why -def test_a_configured_cpu_limit_is_still_honoured(sizing): - sizing(cpu_limit='3') - assert jm._resources(end=2000).limits['cpu'] == '3' - def test_pvc_mode_takes_no_ephemeral_request_or_override(sizing): # /data is not on the node disk there, so sizing it would be meaningless -- diff --git a/src/MissionParallelCatchup/tests/unit/test_sizing.py b/src/MissionParallelCatchup/tests/unit/test_sizing.py index 14de0d57..5907433c 100644 --- a/src/MissionParallelCatchup/tests/unit/test_sizing.py +++ b/src/MissionParallelCatchup/tests/unit/test_sizing.py @@ -13,7 +13,7 @@ @pytest.fixture def mem(monkeypatch): def configure(lim='1000Mi', bump=None, cap='48Gi'): - monkeypatch.setattr(jm, 'LIM_MEM', lim) + monkeypatch.setattr(jm, 'REQ_MEM', lim) monkeypatch.setattr(jm, 'MEM_BUMP_FACTOR', jm.MEM_BUMP_FACTOR if bump is None else bump) monkeypatch.setattr(jm, 'MEM_ESCALATION_CAP', cap) From 534dad6efec3aeed17d6df63771e402d3c603be6 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Fri, 31 Jul 2026 10:38:09 -0400 Subject: [PATCH 060/117] Fix duplicate liveness env that the merge left in the Deployment `helm install` refused the release: .spec.template.spec.containers[name="job-monitor"].env: duplicate entries for key [name="LIVENESS_PROBE_INTERVAL_SECONDS"] Both branches carried the liveness block and placed it differently -- mine after the profile conditional, theirs beside the other monitor env -- so git merged them cleanly into two copies and neither side conflicted. It survived every check I ran because none of them could see it. `helm template` renders duplicate env entries without complaint; only the API server rejects them, so a client-side render is not validation. And every test helper here collapses env into a dict via env_of(), which makes a duplicate literally invisible to the existing contract tests. Dropped the second copy and added a test that reads the RAW env list per container, rendered both with defaults and with FULL so conditional blocks are covered too. Reintroducing the duplicate fails it. The release this broke was left half-created: configmaps, service accounts and an unbound PVC with no Deployment. Cleaned up with helm uninstall, which is owner-aware, rather than deleting objects by hand. 582 tests + 1 xfail, and `helm install --dry-run=server` now passes -- that is the check that would have caught this, and it is worth running before any launch. --- .../templates/job_monitor.yaml | 8 ------ .../tests/contract/test_chart_env_wiring.py | 27 +++++++++++++++++++ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 3ea7ea5e..c57d6d85 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -269,14 +269,6 @@ spec: - name: PROFILE_RUNTIME_MEMORY_INSURANCE value: {{ .Values.monitor.profileRuntimeMemoryInsurance | quote }} {{- end }} - - name: LIVENESS_PROBE_INTERVAL_SECONDS - value: {{ .Values.monitor.livenessProbeIntervalSeconds | quote }} - - name: LIVENESS_PROBE_TIMEOUT_SECONDS - value: {{ .Values.monitor.livenessProbeTimeoutSeconds | quote }} - - name: LIVENESS_FAILURE_THRESHOLD - value: {{ .Values.monitor.livenessFailureThreshold | quote }} - - name: LIVENESS_MAX_CONCURRENCY - value: {{ .Values.monitor.livenessMaxConcurrency | quote }} # Failed ranges are always saved; successful ones are the bulk of # the volume and can be turned off for a cheap run. - name: SAVE_SUCCESS_LOGS diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py b/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py index 02d82b71..98c0ce98 100644 --- a/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py +++ b/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py @@ -171,3 +171,30 @@ def test_the_namespace_comes_from_the_pod_not_from_a_value(): assert entry, f"{name} has no NAMESPACE" field = entry[0]['valueFrom']['fieldRef']['fieldPath'] assert field == 'metadata.namespace', f"{name} reads NAMESPACE from {field}" + + +def test_no_container_declares_the_same_env_var_twice(): + """A duplicate env entry is rejected by the API server, not by helm. + + `helm template` renders duplicates happily and every reader here collapses + env into a dict, so a merge that lands the same block twice looks fine right + up until `helm install`, which fails with + + .spec.template.spec.containers[name="job-monitor"].env: + duplicate entries for key [name="LIVENESS_PROBE_INTERVAL_SECONDS"] + + and leaves a half-created release behind. That is exactly what happened + merging the liveness sampler in on 2026-07-31: both branches carried the + block, in different positions, so neither side conflicted. + + Rendered with FULL so the conditional blocks are present too -- a duplicate + that only appears when a profile ConfigMap is mounted is still a duplicate. + """ + for values in ((), FULL): + for name, container in art.containers(values).items(): + seen = [e['name'] for e in (container.get('env') or [])] + dupes = sorted({n for n in seen if seen.count(n) > 1}) + assert not dupes, ( + f"container {name} declares {dupes} more than once " + f"(values={'FULL' if values else 'defaults'}); " + "the API server rejects the Deployment outright") From 25e5371c21b9e4a5f49b3b98f79a2aa37cafe98e Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Fri, 31 Jul 2026 10:45:48 -0400 Subject: [PATCH 061/117] Ship cpu tiering on by default The 2048-worker launch issued 1250m to all 2048 pods -- 2560 vCPU of requests against a 2304 spot quota -- and packed 6 workers per node where tiering gets 14. Cause: monitor.profileCpuTiers defaults to "", which means tiering off, and nothing in the F# sets it. The r5 run only worked because that session hand-edited the value into its snapshot chart, so the working configuration lived in a run directory rather than in the repo. An empty default is not a neutral one here. It is a silent 2x cost regression that renders, installs and runs perfectly happily. Chart now ships the measured tiers. Two tests: - the chart must render tiering enabled, with percentiles and cpu values both ascending and the top tier covering p100. Rendered with a profile mounted, since the tier env correctly lives inside that conditional -- tiering is keyed on measured runtimes. - PROFILE_CPU_TIERS is declared a deliberate chart/code divergence: the code default stays "off" so a bare import is inert, while the chart is the only thing that turns it on. 583 tests + 1 xfail. --- .../parallel_catchup_helm/values.yaml | 7 +++++- .../tests/contract/test_chart_defaults.py | 4 ++++ .../tests/contract/test_chart_env_wiring.py | 22 +++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index ef499daa..5c32b96d 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -83,7 +83,12 @@ monitor: # 3267 of 3859 profiled ranges still fit at 0.5 cores. Empty disables tiering # and every range keeps the configured cpu request. # Paired with profileCpuSlowdown: the measured runtime multiplier per tier. - profileCpuTiers: "" + # Empty means tiering OFF and every worker falls back to the flat REQ_CPU, + # which is not a safe default: on 2026-07-31 that put 1250m on all 2048 pods, + # 2560 vCPU of requests against a 2304 spot quota, and packed 6 workers per + # node instead of 14. Nothing in the F# sets this, so an unset value here is + # the whole configuration -- ship the measured tiers. + profileCpuTiers: "85:0.5,98:0.75,99.5:1.0,100:1.25" profileMargin: 1.15 # No margin on cpu: it is compressible, so under-requesting costs contention # Ceiling for profile-derived memory. Above the configured worker limit on diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py index 9d2ccf23..c8001094 100644 --- a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py +++ b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py @@ -46,6 +46,10 @@ 'ATTEMPT_DEADLINE_SECONDS': 'a backstop the chart turns on and the code leaves off', # StellarKubeSpecs.fs owns worker sizing, so the chart ships these empty on # purpose and the mission fills them in on every install. + # The code default is 'off' so a bare import stays inert, but nothing in the + # F# ever sets this -- the chart value IS the configuration, and shipping it + # empty silently drops every worker to the flat REQ_CPU. + 'PROFILE_CPU_TIERS': 'code defaults to off; the chart is the only thing that enables tiering', 'REQ_CPU': 'left empty in the chart; StellarKubeSpecs.fs supplies it', 'REQ_MEM': 'left empty in the chart; StellarKubeSpecs.fs supplies it', } diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py b/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py index 98c0ce98..2025589e 100644 --- a/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py +++ b/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py @@ -198,3 +198,25 @@ def test_no_container_declares_the_same_env_var_twice(): f"container {name} declares {dupes} more than once " f"(values={'FULL' if values else 'defaults'}); " "the API server rejects the Deployment outright") + + +def test_the_chart_ships_cpu_tiering_switched_on(): + """An empty PROFILE_CPU_TIERS is a silent 2x cost regression, not a no-op. + + Nothing in MissionHistoryPubnetParallelCatchupV2.fs sets this value, so the + chart default IS the configuration. Left empty, every worker falls back to + the flat REQ_CPU: measured 2026-07-31, that issued 1250m to all 2048 pods -- + 2560 vCPU of requests against a 2304 spot quota -- and packed 6 workers per + node where tiering gets 14. + """ + # Rendered with FULL: the tier env lives inside the profileConfigMap block, + # which is correct -- tiering is keyed on measured runtimes, so it only + # applies when a profile is actually mounted. + tiers = art.env_of(art.containers(FULL)[art.MONITOR_CONTAINER]).get('PROFILE_CPU_TIERS') + assert tiers, "the chart ships cpu tiering disabled; every worker will request REQ_CPU" + pairs = [t.split(':') for t in tiers.split(',')] + pcts = [float(p) for p, _ in pairs] + cpus = [float(c) for _, c in pairs] + assert pcts == sorted(pcts), f"tier percentiles out of order: {tiers}" + assert cpus == sorted(cpus), f"tier cpu values out of order: {tiers}" + assert pcts[-1] == 100, f"tiers must cover the top percentile: {tiers}" From 7389aaada916597ff98bd5ed099bdb6549dbc875 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Fri, 31 Jul 2026 12:08:54 -0400 Subject: [PATCH 062/117] Widen the cpu tier ladder to 7 bands topping at 2.0 cores Measured on the previous ladder (85:0.5,98:0.75,99.5:1.0,100:1.25) with every node at 100% cpu saturation: tier delivered vs request vs ~1.13 unthrottled 500m 0.59 1.17x 52% 750m 0.72 0.96x 64% 1250m 0.88 0.70x 78% Two findings drove the change. The 500m band completed at 1.09x its profiled time, so throttling the cheap half is nearly free -- those ranges are early history and IO-bound, not cpu-bound. But the top band was delivered 0.70x what it asked for, and the top band is what sets the makespan, so that is the one place the ladder costs wall-clock. The headroom was already there: nodes are memory-bound at ~7 workers each, leaving cpu 54% idle. Modelled first-wave average moves 0.62 -> 0.92 cores, still below the ~1.13 where cpu would start to bind ahead of memory, so this should not add nodes. Also recorded that tier values are absolute cores and are NOT clamped to REQ_CPU. The comment in MissionHistoryPubnetParallelCatchupV2.fs claims a clamp; there is none in _slack_cpu or _profile_overrides, and 1.5/2.0 were verified to issue straight through a 1250m REQ_CPU. Worth knowing for whoever reads the distribution: of the 123 first-wave ranges landing in the 2.0 band, 103 are ranges with no measured runtime -- _slack_cpu(None) returns the top tier by design -- and only 20 are measured p99.5+. The band is mostly funding unknown ranges. 583 tests + 1 xfail. --- .../parallel_catchup_helm/values.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 5c32b96d..d30476bd 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -87,8 +87,19 @@ monitor: # which is not a safe default: on 2026-07-31 that put 1250m on all 2048 pods, # 2560 vCPU of requests against a 2304 spot quota, and packed 6 workers per # node instead of 14. Nothing in the F# sets this, so an unset value here is - # the whole configuration -- ship the measured tiers. - profileCpuTiers: "85:0.5,98:0.75,99.5:1.0,100:1.25" + # the whole configuration. + # + # Values are absolute cores and are NOT clamped to REQ_CPU -- verified, the + # top bands really do issue 1.5-2.0. Widened 2026-07-31 after measuring the + # previous ladder on saturated nodes: the 500m band ran at 1.09x its profiled + # time (cheap, it is IO-bound) while the top band was delivered 0.88 cores + # against ~1.13 of unthrottled demand -- and the top band sets the makespan, + # so throttling it is the one place the ladder costs wall-clock. + # + # Nodes are memory-bound at ~7 workers each, so cpu sat 54% idle; this spends + # that headroom rather than adding nodes. Modelled first-wave average moves + # 0.62 -> 0.92 cores, still under the ~1.13 where cpu would start to bind. + profileCpuTiers: "50:0.5,85:0.75,95:1.0,98:1.25,99:1.5,99.5:1.75,100:2.0" profileMargin: 1.15 # No margin on cpu: it is compressible, so under-requesting costs contention # Ceiling for profile-derived memory. Above the configured worker limit on From 38ae7b9f88bb936a9754cc8ec9c7d7ac74b30552 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Fri, 31 Jul 2026 13:14:01 -0400 Subject: [PATCH 063/117] Drop duplicated liveness code, stop giving no-seconds ranges the top tier Three fixes, all found while auditing the live run. 1. The merge landed the worker-liveness subsystem twice. _worker_targets, WorkerLivenessSampler and publish_worker_liveness each had two byte-identical definitions, so Python bound the later one and 281 lines never executed. Both branches carried the block at different positions, so neither side conflicted and nothing caught it: it imports, it renders, it runs. Deleted the later copy of each, not the earlier -- module-level code between them references the class, so removing the first breaks import. Added a test that walks the AST of both modules for repeated top-level definitions. The duplicate-env test added earlier covered only chart YAML; this is the same failure mode in Python. 2. _slack_cpu(None) returned the TOP tier, on the reasoning that an unmeasured range is newer than everything measured so assume worst. That does not hold for a reconstructed profile. Measured: 103 of 3983 ranges carry a peakAnonBytes but no seconds -- not because they are new, but because their runtime came from a resumed chain and the reconstruction omits what it cannot verify. Their ends span 38.2M-63.0M, scattered through history rather than at the tip, and several are demonstrably small. With the widened ladder that put 103 ranges in the 2.0 band against 20 genuinely measured p99.5+ ones -- the band existed for those 20 and they were outnumbered 5:1. Now no usable runtime means no tier, and the range falls through to REQ_CPU like any other range the profile cannot size. First-wave requests drop 1892 -> 1815 vCPU and the 2.0 band contains exactly the 20 it was meant for. 3. Corrected the comment in MissionHistoryPubnetParallelCatchupV2.fs claiming the monitor clamps profile-derived cpu to REQ_CPU. It does not -- _slack_cpu returns the tier value straight through, verified by rendering 1.5 and 2.0 under a 1250m REQ_CPU. That claim cost an hour today. 584 tests + 1 xfail. None of this touches the running mission, which executes from its own source ConfigMap snapshot. --- .../MissionHistoryPubnetParallelCatchupV2.fs | 8 +- src/MissionParallelCatchup/job_monitor.py | 298 +----------------- .../tests/contract/test_chart_env_wiring.py | 24 ++ .../tests/unit/test_cpu_tiers.py | 15 +- 4 files changed, 53 insertions(+), 292 deletions(-) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index bbfaa646..e53c5731 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -221,9 +221,11 @@ let installProject (context: MissionContext) = storageReqGibi storageLimGibi - // An explicit override applies to profiled and unprofiled ranges alike: the - // monitor clamps any profile-derived cpu request to REQ_CPU, so this is the - // ceiling as well as the default. + // This is the DEFAULT cpu request, not a ceiling. The monitor does NOT clamp + // profile-derived cpu to it: _slack_cpu returns the tier value straight + // through, so a PROFILE_CPU_TIERS band above this value really is issued -- + // verified 2026-07-31, tiers of 1.5 and 2.0 rendered under a 1250m REQ_CPU. + // It applies to ranges the profile cannot size at all. let cpuReqEffective = if String.IsNullOrWhiteSpace context.pubnetParallelCatchupCpuRequest then cpuReqMili diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index ce381fb6..24305bc3 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -763,290 +763,9 @@ def publish_worker_liveness(targets, sampler=None): return sampler.counts(len(targets)) -def _worker_targets(pods): - """Current Running-with-IP pods, keyed by pod identity. - - A UID change is a replacement even when the Job name or IP is reused. Tests - and unusually incomplete API objects may lack a UID, where the pod name is - still unique for its lifetime. - """ - out = {} - for pod in pods: - pod_status = getattr(pod, 'status', None) - metadata = getattr(pod, 'metadata', None) - ip = getattr(pod_status, 'pod_ip', None) - if getattr(pod_status, 'phase', None) != 'Running' or not ip or metadata is None: - continue - name = getattr(metadata, 'name', None) - identity = getattr(metadata, 'uid', None) or name - if identity and name: - out[str(identity)] = (str(name), str(ip)) - return out - - -class WorkerLivenessSampler: - """Bounded, round-robin stellar-core `/info` sampler. - - Candidate membership comes from the authoritative Kubernetes snapshot, but - all network I/O happens on this sampler's fixed worker pool. At most - `max_concurrency` requests run and the same number wait in the bounded queue; - there is no future, task, session, or thread per pod. - - State is deliberately conservative: - * new or replaced pod: unknown - * any HTTP response from /info: up - * fewer than `failure_threshold` consecutive exceptions/timeouts: unknown - * `failure_threshold` consecutive failures: down - * any later response: up immediately - - HTTP error statuses still prove the admin endpoint responded. A busy core - returning 5xx is responsive; only failure to receive an HTTP response counts - toward down. - """ - - def __init__(self, interval=LIVENESS_PROBE_INTERVAL_SECONDS, - timeout=LIVENESS_PROBE_TIMEOUT_SECONDS, - failure_threshold=LIVENESS_FAILURE_THRESHOLD, - max_concurrency=LIVENESS_MAX_CONCURRENCY, probe=None): - if interval <= 0 or timeout <= 0 or failure_threshold <= 0 or max_concurrency <= 0: - raise ValueError("liveness sampler values must all be greater than zero") - self.interval = float(interval) - self.timeout = float(timeout) - self.failure_threshold = int(failure_threshold) - self.max_concurrency = int(max_concurrency) - self._probe = probe - self._records = {} - self._generation = 0 - self._tasks = queue.Queue(maxsize=self.max_concurrency) - self._stop = threading.Event() - self._condition = threading.Condition() - self._scheduler = None - self._workers = [] - self._started = False - self._failed = None - self._active = 0 - self._failure_count = 0 - self._last_failure_log = 0.0 - - def start(self): - with self._condition: - if self._started: - return - self._started = True - self._workers = [ - threading.Thread(target=self._worker_main, - name=f"worker-liveness-{i}", daemon=True) - for i in range(self.max_concurrency) - ] - self._scheduler = threading.Thread( - target=self._scheduler_main, name="worker-liveness-scheduler", - daemon=True) - for worker in self._workers: - worker.start() - self._scheduler.start() - - def close(self): - self._stop.set() - with self._condition: - self._condition.notify_all() - threads = ([self._scheduler] if self._scheduler is not None else []) + self._workers - deadline = time.monotonic() + self.timeout + 1.0 - for thread in threads: - remaining = max(0.0, deadline - time.monotonic()) - if thread is not None and thread is not threading.current_thread(): - thread.join(remaining) - - def replace_candidates(self, targets, now=None): - """Atomically replace membership without waiting for any probe.""" - now = time.monotonic() if now is None else float(now) - targets = dict(targets) - with self._condition: - old = self._records - records = {} - new_identities = [ - identity for identity in sorted(targets) - if identity not in old or old[identity]['target'] != targets[identity] - ] - offsets = { - identity: self.interval * index / max(1, len(new_identities)) - for index, identity in enumerate(new_identities) - } - for identity, target in targets.items(): - previous = old.get(identity) - if previous is not None and previous['target'] == target: - records[identity] = previous - continue - self._generation += 1 - records[identity] = { - 'target': target, - 'generation': self._generation, - 'status': 'unknown', - 'failures': 0, - 'queued': False, - 'next_due': now + offsets[identity], - } - self._records = records - self._condition.notify_all() - - def counts(self, expected_count=None): - with self._condition: - count = len(self._records) if expected_count is None else int(expected_count) - healthy = self._started and self._failed is None - if healthy: - healthy = (self._scheduler is not None and self._scheduler.is_alive() - and all(worker.is_alive() for worker in self._workers)) - if not healthy or count != len(self._records): - return {'up': 0, 'down': 0, 'unknown': count} - result = {'up': 0, 'down': 0, 'unknown': 0} - for record in self._records.values(): - result[record['status']] += 1 - return result - - def stats(self): - """Small observability hook used by the scale contract test.""" - with self._condition: - live_threads = sum( - 1 for thread in ([self._scheduler] + self._workers) - if thread is not None and thread.is_alive()) - return { - 'records': len(self._records), - 'active': self._active, - 'queued': self._tasks.qsize(), - 'outstanding': self._active + self._tasks.qsize(), - 'threads': live_threads, - 'failed': self._failed, - } - - def _scheduler_main(self): - try: - self._schedule() - except Exception as e: - self._mark_failed("scheduler", e) - - def _schedule(self): - while not self._stop.is_set(): - with self._condition: - now = time.monotonic() - capacity = self.max_concurrency - self._tasks.qsize() - due = sorted( - ((record['next_due'], identity, record) - for identity, record in self._records.items() - if not record['queued'] and record['next_due'] <= now), - key=lambda item: (item[0], item[1])) - for _, identity, record in due[:max(0, capacity)]: - task = (identity, record['generation'], record['target']) - try: - self._tasks.put_nowait(task) - except queue.Full: - break - record['queued'] = True - - waiting = [ - record['next_due'] for record in self._records.values() - if not record['queued'] - ] - delay = max(0.01, min(1.0, min(waiting) - now)) if waiting else 1.0 - self._condition.wait(timeout=delay) - - def _worker_main(self): - session = None - try: - if self._probe is None: - session = requests.Session() - adapter = requests.adapters.HTTPAdapter( - pool_connections=4, pool_maxsize=1, max_retries=0) - session.mount('http://', adapter) - while not self._stop.is_set(): - try: - task = self._tasks.get(timeout=0.2) - except queue.Empty: - continue - with self._condition: - self._active += 1 - identity, generation, target = task - success = False - error = None - try: - _, ip = target - if self._probe is None: - host = f"[{ip}]" if ':' in ip else ip - with session.get(f"http://{host}:11626/info", - timeout=self.timeout): - pass - else: - self._probe(ip, self.timeout) - success = True - except Exception as e: - error = e - finally: - self._record_result(identity, generation, target, success, error) - self._tasks.task_done() - with self._condition: - self._active -= 1 - self._condition.notify_all() - except Exception as e: - self._mark_failed("probe worker", e) - finally: - if session is not None: - session.close() - - def _record_result(self, identity, generation, target, success, error=None, - now=None): - now = time.monotonic() if now is None else float(now) - log_failure = None - with self._condition: - record = self._records.get(identity) - if (record is None or record['generation'] != generation - or record['target'] != target): - return - record['queued'] = False - record['next_due'] = now + self.interval - if success: - record['failures'] = 0 - record['status'] = 'up' - else: - record['failures'] += 1 - record['status'] = ( - 'down' if record['failures'] >= self.failure_threshold - else 'unknown') - self._failure_count += 1 - if now - self._last_failure_log >= 60.0: - log_failure = self._failure_count - self._failure_count = 0 - self._last_failure_log = now - self._condition.notify_all() - if log_failure is not None: - logger.warning( - "stellar-core /info liveness probes are failing; %d failure(s) " - "across the fleet since the previous warning (latest: %s: %s)", - log_failure, target[0], error) - - def _mark_failed(self, component, error): - with self._condition: - if self._failed is not None: - return - self._failed = f"{component}: {error}" - self._condition.notify_all() - logger.exception( - "worker liveness %s failed; all current workers will be reported " - "unknown and reconcile will continue", component) - - worker_liveness_sampler = WorkerLivenessSampler() -def publish_worker_liveness(targets, sampler=None): - """Hand a pod snapshot to the sampler and return its current three counts. - - This path copies O(current workers) state under a short lock but never makes - a request or waits for an in-flight request. Keeping it separate makes the - non-blocking boundary directly testable. - """ - sampler = sampler or worker_liveness_sampler - sampler.replace_candidates(targets) - return sampler.counts(len(targets)) - - class RequestHandler(BaseHTTPRequestHandler): def do_GET(self): if self.path == '/healthz': @@ -2329,8 +2048,19 @@ def _runtime_memory_insurance(seconds): def _slack_cpu(seconds): """Tier for a range, by its rank among all profiled runtimes. - A range with no measured runtime gets the top tier, matching the dispatch - order: unprofiled means newer than anything measured, so assume worst. + No usable runtime means no tier: fall through to the configured REQ_CPU, + the same request an entirely unprofiled range gets. + + This used to return the TOP tier on the reasoning that an unmeasured range + is newer than anything measured, so assume the worst. That reasoning does + not survive contact with a reconstructed profile. Measured 2026-07-31: 103 + of 3983 ranges carry a peakAnonBytes but no seconds -- not because they are + new, but because their runtime came from a resumed chain and the + reconstruction omits what it cannot verify. Their ends span 38.2M-63.0M, + scattered through history rather than clustered at the tip, and several are + demonstrably small. Handing them the top band spent 206 vCPU -- 9% of the + spot quota -- on ranges we have positive evidence are cheap, while the 20 + genuinely-longest ranges the band exists for got a sixth of that. """ try: tiers = [(float(p), c) for p, c in @@ -2343,7 +2073,7 @@ def _slack_cpu(seconds): return None seconds = _positive_seconds(seconds) if seconds is None: - return tiers[-1][1] + return None everything = _profile_seconds() if not everything: return None diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py b/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py index 2025589e..267990d6 100644 --- a/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py +++ b/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py @@ -220,3 +220,27 @@ def test_the_chart_ships_cpu_tiering_switched_on(): assert pcts == sorted(pcts), f"tier percentiles out of order: {tiers}" assert cpus == sorted(cpus), f"tier cpu values out of order: {tiers}" assert pcts[-1] == 100, f"tiers must cover the top percentile: {tiers}" + + +def test_neither_module_defines_the_same_symbol_twice(): + """A merge can land the same block twice and Python will not complain. + + Both branches carried the worker-liveness subsystem, positioned differently, + so git merged them into two byte-identical copies of _worker_targets, + WorkerLivenessSampler and publish_worker_liveness -- 285 lines that shipped + in the monitor and were never executed, because the later definition binds. + + Nothing catches this on its own: it imports, it renders, it runs. The only + reason it was benign is that the copies happened to be identical; had the + merge taken one edited copy and one stale one, the stale one would silently + have won or lost depending on file order. + """ + import ast, collections, pathlib + here = pathlib.Path(__file__).resolve().parents[2] + for name in ('job_monitor.py', 'log_collector.py'): + tree = ast.parse((here / name).read_text()) + seen = collections.Counter( + node.name for node in tree.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))) + dupes = sorted(n for n, k in seen.items() if k > 1) + assert not dupes, f"{name} defines {dupes} more than once; the later one silently wins" diff --git a/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py b/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py index ee1a3daf..ad6a282d 100644 --- a/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py +++ b/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py @@ -45,14 +45,19 @@ def test_each_band_maps_to_its_tier(tiered): assert jm._slack_cpu(1000) == '1.25' # the longest range -def test_an_unmeasured_range_gets_the_top_tier(tiered): - # Matches dispatch order: unprofiled is newer than anything measured. - assert jm._slack_cpu(None) == '1.25' +def test_an_unmeasured_range_gets_no_tier_at_all(tiered): + """No usable runtime means no basis for a tier -- fall through to REQ_CPU. + + Returning the TOP tier here cost 206 vCPU on the 2026-07-31 run: 103 ranges + lacked `seconds` not because they were new but because a resumed chain made + their runtime unverifiable, and several were demonstrably small. + """ + assert jm._slack_cpu(None) is None @pytest.mark.parametrize('seconds', [0, -1, 'bad', float('nan'), float('inf')]) -def test_an_invalid_runtime_safely_gets_the_top_tier(tiered, seconds): - assert jm._slack_cpu(seconds) == '1.25' +def test_an_invalid_runtime_safely_gets_no_tier(tiered, seconds): + assert jm._slack_cpu(seconds) is None def test_a_uniformly_slower_run_assigns_the_same_tiers(monkeypatch): From 08e5b4ec361b4a6729d09e39768a13af88d0ddf8 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Fri, 31 Jul 2026 15:47:32 -0400 Subject: [PATCH 064/117] Stop V2 sizing the shared spec that V1 catchup also uses ParallelCatchupCoreResourceRequirements is shared: MissionHistoryPubnet- ParallelCatchup (parallelism 128) and MissionHistoryTestnetParallelCatchup (parallelism 256) both resolve to it via ParallelCatchupResources. The V2 rewrite tuned it from 250m/8192Mi to 1800m/9216Mi, which silently raised those two missions' cpu request 7.2x -- 64 -> 461 vCPU for testnet, 32 -> 230 for pubnet -- for a Job-per-range packing model they do not use. Restored to the upstream value, 250 8192 35 2000 28672 40. Note the memory limit: this branch had 24000Mi while stellar/supercluster master has 28672Mi, so it had drifted on that too. V2's worker cpu and memory now live in the chart, which is the only place that mission's sizing is written down. The F# pushes cpu only when --pubnet-parallel-catchup-cpu-request is given; otherwise the chart default stands. V2 still reads the ephemeral-storage pair from the shared spec, which the rewrite never changed. 584 tests + 1 xfail, dotnet build clean. --- .../MissionHistoryPubnetParallelCatchupV2.fs | 30 +++++++++---------- src/FSLibrary/StellarKubeSpecs.fs | 19 ++++++------ src/MissionParallelCatchup/job_monitor.py | 26 ++++++++++------ .../parallel_catchup_helm/values.yaml | 18 +++++++++-- 4 files changed, 55 insertions(+), 38 deletions(-) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index e53c5731..5d801be1 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -194,10 +194,13 @@ let installProject (context: MissionContext) = (Option.defaultValue true context.checkEventsAreConsistentWithEntryDiffs) ) - // read the resource requirements defined in StellarKubeSpecs.fs (where resource for various missions are centralized) + // Only the ephemeral-storage pair is taken from StellarKubeSpecs. Its cpu and + // memory belong to the V1 parallel catchup missions, which share that spec and + // run a different execution model at parallelism 128/256 -- sizing V2 there + // silently resized them too. V2's own worker cpu/memory are the chart defaults + // in parallel_catchup_helm/values.yaml, overridable per run with + // --pubnet-parallel-catchup-cpu-request. let resourceRequirements = ParallelCatchupCoreResourceRequirements - let cpuReqMili = resourceRequirements.Requests.["cpu"].ToString() - let memReqMebi = resourceRequirements.Requests.["memory"].ToString() // StellarKubeSpecs sizes ephemeral-storage for ephemeral mode, where /data is // an emptyDir on the node. In pvc mode /data is on the volume and the node // disk only holds logs and tmp, so asking for the full amount reserves disk @@ -210,14 +213,10 @@ let installProject (context: MissionContext) = resourceRequirements.Limits.["ephemeral-storage"].ToString() LogInfo - "Resource requirements from StellarKubeCfg:\n\ - CPU request: %s\n\ - Memory request: %s\n\ + "Worker storage from StellarKubeCfg:\n\ Storage request: %s\n\ Storage limit: %s\n\ - (workers run with no cpu or memory limit)" - cpuReqMili - memReqMebi + (cpu and memory come from the chart; workers run with no cpu or memory limit)" storageReqGibi storageLimGibi @@ -226,14 +225,13 @@ let installProject (context: MissionContext) = // through, so a PROFILE_CPU_TIERS band above this value really is issued -- // verified 2026-07-31, tiers of 1.5 and 2.0 rendered under a 1250m REQ_CPU. // It applies to ranges the profile cannot size at all. - let cpuReqEffective = - if String.IsNullOrWhiteSpace context.pubnetParallelCatchupCpuRequest then - cpuReqMili - else - context.pubnetParallelCatchupCpuRequest + // Pushed only when the run explicitly asks. Otherwise the chart default stands, + // so the chart is the single place V2's worker sizing is written down. + if not (String.IsNullOrWhiteSpace context.pubnetParallelCatchupCpuRequest) then + setOptions.Add( + sprintf "worker.resources.requests.cpu=%s" context.pubnetParallelCatchupCpuRequest + ) - setOptions.Add(sprintf "worker.resources.requests.cpu=%s" cpuReqEffective) - setOptions.Add(sprintf "worker.resources.requests.memory=%s" memReqMebi) setOptions.Add(sprintf "worker.resources.requests.ephemeral_storage=%s" storageReqGibi) setOptions.Add(sprintf "worker.resources.limits.ephemeral_storage=%s" storageLimGibi) diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index 962c7c11..e5ff777f 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -126,17 +126,16 @@ let SimulatePubnetTier1PerfCoreResourceRequirements : V1ResourceRequirements = makeResourceRequirements 500 128 4000 6000 let ParallelCatchupCoreResourceRequirements : V1ResourceRequirements = - // 1.8 vCPU, 9GiB RAM and 35 GB of disk, bursting to 2 vCPU, 24000MB and 40 GB. + // When doing parallel catchup, we give each container + // 0.25 vCPUs, 8Gi RAM and 35 GB of disk bursting to 2vCPU, 28Gi (28672Mi) and 40 GB // - // The requests are picked so the scheduler lands a specific worker count on - // each node shape we run on: cpu binds where memory is plentiful, and memory - // binds where cpu is. - // r8*.xlarge (3.92 cpu / 29.7Gi alloc) -> 2 workers, cpu-bound - // m8*.2xlarge (7.91 cpu / 29.7Gi alloc) -> 3 workers, memory-bound - // r8*.2xlarge (7.91 cpu / 61.7Gi alloc) -> 4 workers, cpu-bound - // The counts hold for allocatable cpu in [7.2,9.0) and memory in [27,36)Gi, so - // kubelet-reservation differences between instance types cannot flip them. - makeResourceRequirementsWithStorageLimit 1800 9216 35 2000 24000 40 + // Shared with the V1 parallel catchup missions (pubnet and testnet), which run + // coreSets through RunParallelJobsInRandomOrder at parallelism 128 and 256. + // V2 does NOT take its cpu or memory from here -- it is a Job-per-range mission + // with a different packing model and sources those from the helm chart + // (worker.resources.requests in parallel_catchup_helm/values.yaml). It still + // reads the ephemeral-storage pair below for its ephemeral storage mode. + makeResourceRequirementsWithStorageLimit 250 8192 35 2000 28672 40 let NonParallelCatchupCoreResourceRequirements : V1ResourceRequirements = // When doing non-parallel catchup, we give each container diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py index 24305bc3..9fa9ec36 100644 --- a/src/MissionParallelCatchup/job_monitor.py +++ b/src/MissionParallelCatchup/job_monitor.py @@ -1,18 +1,26 @@ """Parallel catchup job monitor. -Owns dispatch as well as reporting. Redis, worker.sh and the range-generator -scripts are gone; a Kubernetes Job per ledger range replaces them. +Drives a full-history catchup by splitting the ledger range into slices and +running one Kubernetes Job per slice, then reports what happened. It owns +dispatch, retry policy and sizing, not just observation. -State model -- the controller itself keeps nothing authoritative in memory: +State model -- nothing authoritative is held in memory: desired computed from config by a pure function (uniform | logarithmic) - completed durable, in a ConfigMap -- Jobs are reclaimed during a long run, - so their absence must NOT be read as "never ran" + completed durable on the shared volume (progress.json), mirrored to a + ConfigMap for the mission driver to read. Jobs are reclaimed + during a long run, so a missing Job must NOT be read as + "never ran" in-flight live Jobs, by label selector -A restart recomputes all three and carries on. The single-writer property (one -replica, Recreate) is what removes the claim/requeue races the redis queue had: -work is *assigned*, never claimed. +Per-attempt facts -- why an attempt ended, how long it ran, what it peaked at -- +live beside progress.json as small files written by the collector sidecar. A +restarted monitor rebuilds every decision from those plus the live Job list, so +losing the process costs nothing but the time to re-list. + +Work is assigned rather than claimed: the monitor is a single writer (one +replica, Recreate), so a range's owner is decided by the range itself and never +by a race between consumers. """ import gzip @@ -121,7 +129,7 @@ # # Without a limit the request still does the real work: it places the pod and # it sets eviction order under node pressure. What goes away is the cliff. -REQ_CPU = os.getenv('REQ_CPU', '1800m') +REQ_CPU = os.getenv('REQ_CPU', '1250m') REQ_MEM = os.getenv('REQ_MEM', '9Gi') # Only meaningful in ephemeral storage mode; see check_storage_config(). # Range profile from an earlier run: tightens per-range requests so more diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index d30476bd..ad717fec 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -25,10 +25,22 @@ worker: avoidNodeLabels: [] tolerateNodeTaints: [] asanOptions: "quarantine_size_mb=1:malloc_context_size=5:alloc_dealloc_mismatch=0" - resources: # resources below are left empty on purpose, they are read and overridden from `StellarKubeCfg.fs` + resources: requests: - cpu: "" - memory: "" + # V2's worker sizing lives HERE, not in StellarKubeSpecs.fs. That spec is + # shared with the V1 parallel catchup missions (parallelism 128 and 256, + # a different execution model), and sizing V2 there silently resized them + # too -- it took their cpu request from 250m to 1800m. + # + # 1800m/9Gi is the unprofiled default: on r8*.2xlarge (7.91 cpu, 61.7Gi + # allocatable) that lands 4 workers per node, cpu-bound; on m8*.2xlarge + # (7.91 cpu, 29.7Gi) it lands 3, memory-bound. A range the profile has + # measured overrides both. --pubnet-parallel-catchup-cpu-request overrides + # the cpu for a whole run. + cpu: "1800m" + memory: "9Gi" + # Still filled in by the mission: ephemeral-storage is 2Gi/4Gi in pvc mode + # and the StellarKubeSpecs value in ephemeral mode. ephemeral_storage: "" limits: # cpu and memory are deliberately absent -- workers run unlimited on both. From 55fe6f2c4ae985c4b1b68ea9499c1ccd976e0e2c Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Fri, 31 Jul 2026 21:22:08 -0400 Subject: [PATCH 065/117] Widen the cpu ladder to 8 bands topping at 3.0 cores The 2048-worker run measured a clean monotonic dose-response over 3984 completed ranges -- actual/profiled runtime by band: 0.5 1.23x 1.25 0.72x 0.75 1.00x 1.5 0.71x 1.0 0.81x 1.75 0.67x 2.0 0.63x Every step up bought speed. The run finished in 4.15h against r5's 7.5h at 1092 workers, and total work fell 4713 -> 4145 core-hours, so this is not just more parallelism. It also settles the question of whether the top bands are wasted cpu. They are not, and the mechanism is node spread rather than consumed cores: a 2.0 request lands ~3 co-tenants per node instead of 8-14, and ranges on uncrowded nodes measured 1.80 vs 1.34 ledgers/s. The makespan-setter was using 0.83 of a 2.0 request with 4.7 idle cores beside it, which looks like waste until you notice the request is what kept the neighbours away. New ladder pushes further up the curve: bands at 2.5 and 3.0, and the lower cutoffs move down (0.75 from p85 to p80, 1.0 from p95 to p92) so more of the distribution climbs a rung. Modelled at 1500 workers: first-wave average 1.04 cores, 1557 vCPU of requests, ~1837 vCPU of nodes at the 1.18x ratio the last run showed -- against a 2304 spot quota, so roomier than the last run's 2240. 584 tests + 1 xfail. --- .../parallel_catchup_helm/values.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index ad717fec..2f138cdd 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -108,10 +108,15 @@ monitor: # against ~1.13 of unthrottled demand -- and the top band sets the makespan, # so throttling it is the one place the ladder costs wall-clock. # - # Nodes are memory-bound at ~7 workers each, so cpu sat 54% idle; this spends - # that headroom rather than adding nodes. Modelled first-wave average moves - # 0.62 -> 0.92 cores, still under the ~1.13 where cpu would start to bind. - profileCpuTiers: "50:0.5,85:0.75,95:1.0,98:1.25,99:1.5,99.5:1.75,100:2.0" + # Widened again 2026-08-01 after the 2048-worker run measured a clean + # monotonic dose-response across all seven bands -- actual/profiled runtime + # 1.23x at 0.5 cores, 1.00x at 0.75, 0.81x at 1.0, 0.72x at 1.25, 0.71x at + # 1.5, 0.67x at 1.75, 0.63x at 2.0 over 3984 completed ranges. Every step up + # bought speed, and the run finished in 4.15h against r5's 7.5h because the + # long ranges are the ones that got faster. The top bands are not idle cpu: + # a 2.0 request lands ~3 co-tenants per node instead of 8-14, and ranges on + # uncrowded nodes measured 1.80 vs 1.34 ledgers/s. + profileCpuTiers: "50:0.5,80:0.75,92:1.0,97:1.25,98.5:1.5,99.25:2.0,99.7:2.5,100:3.0" profileMargin: 1.15 # No margin on cpu: it is compressible, so under-requesting costs contention # Ceiling for profile-derived memory. Above the configured worker limit on From 8553e77f49a42de96490c363fa1ca85577e58887 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Fri, 7 Aug 2026 16:39:46 -0400 Subject: [PATCH 066/117] Restructure the monitor and make pooled on-demand runs schedulable Validated with five runs on ssc-test: 534 ranges, 0 unexplained failures. Python - Split job_monitor.py into apps/ (entrypoints) and lib/ (imported modules). The container still flattens both into one /app, because ConfigMap keys cannot contain '/'; contract tests pin that the image ships every import. - Retry budgets are now per cause, not per attempt. ATTEMPT_BUDGETS is the whole policy and a cause with no entry is condemned on sight, so eviction churn can no longer drain the OOM or disk budgets. - Rewrote the worker-liveness sampler as one bounded asyncio sweep: 33 threads to none, 305 lines to 103. - Collapsed the two runtime-insurance functions into one and dropped a guard that _profile_seconds already makes unreachable. - Deleted the synthetic resume harness and its chart template. F# driver - Collect worker logs incrementally instead of once at teardown, which measured ~20 minutes on a full run. Each pass tars only what changed since a watermark read from the pod's own clock, taken before the tar so skew and in-flight writes cost a re-send rather than a miss. - Verify every archive before advancing the watermark, and retry the transfer in-pass. The exec stream truncates silently on large payloads -- tar exits 0, the status channel reports Success, and the bytes simply stop. Measured at 200 workers: 2 of 3 transfers truncated, and before this a 64MB part cost 12 ranges their logs permanently. - Derive the node label key and taint toleration from the pool prefix, as capacityType already was. Both shipped as [], so a pooled run had no tier affinity at all and could not tolerate the catchup taint -- it neither routed nor scheduled, and both failures were silent. Chart - Resolve the collector's interpreter on PATH. /usr/bin/python3 does not exist on python:3.12-slim, and the failure was asymmetric: the monitor inherited the image CMD and came up healthy while only the sidecar crashlooped. - Recut every on-demand claim from allocatable MINUS daemonsets. The previous table sized straight to allocatable, so all nine tiers were unschedulable on both cpu and memory and Karpenter provisioned nothing. Tests - Cover both halves of the packing invariant on both dimensions; the previous version checked memory only, against a stale 154Mi daemonset figure, and passed while nothing could schedule. - Pin the profile arithmetic that mutation testing found unasserted: margins that multiply, ladders that climb, runtime-weighted insurance. sizing.py mutation score 46% -> 78%. Co-Authored-By: Claude Opus 5 --- .gitignore | 4 + src/App/Program.fs | 8 + src/FSLibrary.Tests/Tests.fs | 203 +- src/FSLibrary.Tests/TestsRace8.fs | 90 +- .../MissionHistoryPubnetParallelCatchupV2.fs | 383 +- src/FSLibrary/StellarMissionContext.fs | 1 + src/MissionParallelCatchup/.dockerignore | 2 + .../Dockerfile.jobmonitor | 23 +- .../apps/job_monitor.py | 1483 ++++++++ .../{ => apps}/log_collector.py | 656 +++- .../integration/synthetic_resume_harness.py | 670 ---- src/MissionParallelCatchup/job_monitor.py | 3074 ----------------- src/MissionParallelCatchup/lib/attempts.py | 353 ++ src/MissionParallelCatchup/lib/config.py | 588 ++++ src/MissionParallelCatchup/lib/http_server.py | 39 + src/MissionParallelCatchup/lib/kube.py | 34 + src/MissionParallelCatchup/lib/logger.py | 60 + src/MissionParallelCatchup/lib/medida.py | 30 + src/MissionParallelCatchup/lib/metrics.py | 47 + src/MissionParallelCatchup/lib/profiles.py | 71 + src/MissionParallelCatchup/lib/ranges.py | 101 + src/MissionParallelCatchup/lib/records.py | 112 + src/MissionParallelCatchup/lib/sizing.py | 386 +++ src/MissionParallelCatchup/lib/units.py | 27 + .../lib/worker_liveness.py | 103 + .../templates/job_monitor.yaml | 121 +- .../templates/synthetic_worker.yaml | 96 - .../values-ondemand.yaml | 79 + .../parallel_catchup_helm/values.yaml | 333 +- src/MissionParallelCatchup/pytest.ini | 12 +- .../requirements-dev.txt | 12 + .../tests/collector/test_archive_append.py | 35 +- .../tests/collector/test_poll_backoff.py | 3 +- src/MissionParallelCatchup/tests/conftest.py | 103 +- .../tests/contract/_artifacts.py | 8 +- .../tests/contract/test_chart_defaults.py | 72 +- .../tests/contract/test_chart_env_wiring.py | 123 +- .../tests/contract/test_chart_rbac.py | 2 +- .../contract/test_cross_process_files.py | 46 +- .../tests/contract/test_dependency_pins.py | 79 + .../contract/test_fsharp_driver_contract.py | 123 +- .../contract/test_k8s_failure_formats.py | 31 +- .../contract/test_medida_metric_block.py | 42 +- .../tests/contract/test_module_packaging.py | 157 + .../tests/contract/test_rendered_job_spec.py | 61 +- .../contract/test_synthetic_resume_harness.py | 137 - .../tests/contract/test_synthetic_worker.py | 101 - .../tests/data/real-sts-fault-exit3.log.gz | Bin 0 -> 8441 bytes src/MissionParallelCatchup/tests/fake_k8s.py | 5 + .../tests/reconcile/test_attempt_deadline.py | 34 +- .../test_completed_range_not_redispatched.py | 11 +- .../reconcile/test_dispatch_not_frozen.py | 21 +- .../tests/reconcile/test_retry_budgets.py | 387 ++- .../tests/reconcile/test_txapply_histogram.py | 25 +- .../resilience/test_collector_restart.py | 132 +- .../tests/resilience/test_crash_points.py | 69 +- .../tests/resilience/test_hostile_state.py | 150 +- .../tests/resilience/test_restart_fuzz.py | 87 +- .../tests/test_harness_smoke.py | 42 +- .../tests/unit/conftest.py | 5 +- .../tests/unit/test_attempt_chain.py | 366 +- .../tests/unit/test_classify.py | 59 +- .../tests/unit/test_collector_main_loop.py | 65 +- .../tests/unit/test_condemnation_watch.py | 384 ++ .../tests/unit/test_cpu_tiers.py | 103 - .../tests/unit/test_deadline_sizing.py | 13 +- .../tests/unit/test_dispatch_order.py | 8 +- .../tests/unit/test_kubelet_sampler.py | 76 +- .../unit/test_monitor_verdict_records.py | 139 +- .../tests/unit/test_node_targeting.py | 3 +- .../tests/unit/test_poll_lifecycle.py | 10 +- .../tests/unit/test_pool_tiers.py | 572 +++ .../tests/unit/test_profile_lookup.py | 45 +- .../tests/unit/test_range_generation.py | 147 +- .../tests/unit/test_reaping.py | 44 +- .../tests/unit/test_records.py | 65 +- .../tests/unit/test_resources.py | 203 +- .../tests/unit/test_retry_counters.py | 75 +- .../tests/unit/test_sizing.py | 177 +- .../tests/unit/test_tx_apply.py | 163 +- .../tests/unit/test_worker_liveness.py | 327 +- 81 files changed, 8349 insertions(+), 5987 deletions(-) create mode 100644 src/MissionParallelCatchup/.dockerignore create mode 100644 src/MissionParallelCatchup/apps/job_monitor.py rename src/MissionParallelCatchup/{ => apps}/log_collector.py (60%) delete mode 100644 src/MissionParallelCatchup/integration/synthetic_resume_harness.py delete mode 100644 src/MissionParallelCatchup/job_monitor.py create mode 100644 src/MissionParallelCatchup/lib/attempts.py create mode 100644 src/MissionParallelCatchup/lib/config.py create mode 100644 src/MissionParallelCatchup/lib/http_server.py create mode 100644 src/MissionParallelCatchup/lib/kube.py create mode 100644 src/MissionParallelCatchup/lib/logger.py create mode 100644 src/MissionParallelCatchup/lib/medida.py create mode 100644 src/MissionParallelCatchup/lib/metrics.py create mode 100644 src/MissionParallelCatchup/lib/profiles.py create mode 100644 src/MissionParallelCatchup/lib/ranges.py create mode 100644 src/MissionParallelCatchup/lib/records.py create mode 100644 src/MissionParallelCatchup/lib/sizing.py create mode 100644 src/MissionParallelCatchup/lib/units.py create mode 100644 src/MissionParallelCatchup/lib/worker_liveness.py delete mode 100644 src/MissionParallelCatchup/parallel_catchup_helm/templates/synthetic_worker.yaml create mode 100644 src/MissionParallelCatchup/parallel_catchup_helm/values-ondemand.yaml create mode 100644 src/MissionParallelCatchup/requirements-dev.txt create mode 100644 src/MissionParallelCatchup/tests/contract/test_dependency_pins.py create mode 100644 src/MissionParallelCatchup/tests/contract/test_module_packaging.py delete mode 100644 src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py delete mode 100644 src/MissionParallelCatchup/tests/contract/test_synthetic_worker.py create mode 100644 src/MissionParallelCatchup/tests/data/real-sts-fault-exit3.log.gz create mode 100644 src/MissionParallelCatchup/tests/unit/test_condemnation_watch.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py create mode 100644 src/MissionParallelCatchup/tests/unit/test_pool_tiers.py diff --git a/.gitignore b/.gitignore index e807a167..8cfb3cd9 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ # Python bytecode from the parallel-catchup monitor/collector tests __pycache__/ *.pyc + +# pytest-cov artifacts from the MissionParallelCatchup suite +.coverage +**/__pycache__/ diff --git a/src/App/Program.fs b/src/App/Program.fs index 5f63c698..85732716 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -118,6 +118,7 @@ type MissionOptions pubnetParallelCatchupStorageMode: string, pubnetParallelCatchupProfile: string, pubnetParallelCatchupRangeOrder: string, + pubnetParallelCatchupPoolPrefix: string, pubnetParallelCatchupCpuRequest: string, tag: string option, numPregeneratedTxs: int option, @@ -543,6 +544,12 @@ type MissionOptions Default = "tip-first")>] member self.PubnetParallelCatchupRangeOrder : string = pubnetParallelCatchupRangeOrder + [] + member self.PubnetParallelCatchupPoolPrefix : string = pubnetParallelCatchupPoolPrefix + [()) Assert.Null(entry.["peakEphemeralBytes"]) record.["peakEphemeralBytes"] <- JValue(9999L) let withEph = projectRangeEntry record - Assert.Equal(4, withEph.Count) + Assert.Equal(3, withEph.Count) Assert.Equal(9999L, withEph.["peakEphemeralBytes"].Value()) [] let ``range profile carries the fields the sizing consumer prefers`` () = - // peakAnonBytes is what _profile_overrides reads FIRST (kubelet-sampled - // anon); peakRssBytes is only its fallback. Omitting it from the - // projection silently stripped it from the mission artifact while the - // monitor's progress.json carried it for 99% of ranges -- measured - // 2026-07-30, artifact 0% vs volume 99%. wallSeconds likewise. + // peakAnonBytes is the memory figure _profile_overrides reads. Omitting it + // from the projection silently stripped it from the mission artifact while + // the monitor's progress.json carried it for 99% of ranges -- measured + // 2026-07-30, artifact 0% vs volume 99%. + // + // `seconds` is the only timing carried: it is the percentile basis, the + // dispatch order and the runtime insurance threshold. wallSeconds and + // txApply are recorded per range as metrics but nothing sizes from either, + // and wallSeconds alone was 349 KB of a 963 KB artifact. Assert.Contains("peakAnonBytes", rangeProfileFields) - Assert.Contains("wallSeconds", rangeProfileFields) + Assert.Contains("seconds", rangeProfileFields) + Assert.DoesNotContain("wallSeconds", rangeProfileFields) + Assert.DoesNotContain("txApply", rangeProfileFields) let record = JObject() record.["peakAnonBytes"] <- JValue(111L) - record.["wallSeconds"] <- JValue(50.0) + record.["seconds"] <- JValue(50.0) + record.["wallSeconds"] <- JValue(999.0) let entry = projectRangeEntry record Assert.Equal(111L, entry.["peakAnonBytes"].Value()) - Assert.Equal(50.0, entry.["wallSeconds"].Value()) + Assert.Equal(50.0, entry.["seconds"].Value()) + Assert.Null(entry.["wallSeconds"]) [] @@ -628,14 +636,171 @@ let ``pvc mode does not reserve node disk it never uses`` () = [] -let ``progress record is read from the volume before the configmap`` () = - // The ConfigMap is a 1 MiB-capped mirror (~6100 ranges); /logs/progress.json - // is authoritative and unbounded. Reading the mirror would silently - // truncate the artifact on a finer slicing. +let ``progress record is read from the volume and never from the configmap`` () = + // /logs/progress.json is the monitor's own state: authoritative, unbounded, + // and the only copy carrying measurements. The ConfigMap is the driver's + // view of the run -- status only -- so a record sourced from it would build + // an artifact that looks complete and measures nothing. let src = System.IO.File.ReadAllText( "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") - let vol = src.IndexOf("/logs/progress.json") - let cm = src.IndexOf("queryJobMonitor (context, jobMonitorProgressKey)") - Assert.True(vol > 0, "must read the volume copy") - Assert.True(vol < cm, "volume read must precede the ConfigMap fallback") + Assert.Contains("/logs/progress.json", src) + Assert.DoesNotContain("jobMonitorProgressKey", src) + + +[] +let ``on-demand runs layer the one-pod-per-node overlay`` () = + // The chart defaults are the spot claims: the spot pools were doubled on + // 2026-08-04 so each claim is half a node and two pods share it. On-demand + // pools kept their original sizes, where those same claims are the node's + // NAMEPLATE -- and nameplate is not allocatable, so every on-demand tier + // becomes unschedulable. Measured on ssc-test: a 16 GiB node reports + // 13312Mi usable, against a 14336Mi supergiant claim. + let src = + System.IO.File.ReadAllText( + "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + Assert.Contains("values-ondemand.yaml", src) + // and it must be layered, never swapped in: the overlay only carries the + // pool claims, so dropping the base values would lose the whole chart config + Assert.Contains("[| \"--values\"; valuesFilePath; \"--values\"; onDemandValuesFilePath |]", src) + + +[] +let ``the on-demand overlay is not applied to pvc runs`` () = + // pvc means spot means shared nodes. Layering the one-pod claims there would + // halve pods per node on pools that were doubled precisely to hold two. + let src = + System.IO.File.ReadAllText( + "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + let guard = src.IndexOf("if context.pubnetParallelCatchupStorageMode = \"pvc\" then\n [| \"--values\"; valuesFilePath |]") + Assert.True(guard > 0, "pvc branch must pass the base values file alone") + + +[] +let ``on-demand pool claims fit exactly one pod per node`` () = + // Both halves, on BOTH dimensions. The previous version of this test checked + // memory only and assumed 154Mi of daemonsets, so it passed while every + // on-demand tier was in fact unschedulable -- 2026-08-07, ten workers Pending + // forever because Karpenter needed 1820m/3054Mi against c8a.large's + // 1715m/2663Mi. A test that encodes a stale measurement is worse than none: + // it is why the table looked verified. + // + // 494Mi/245m measured on ssc-test, and it is what Karpenter enforces -- + // ebs-csi-node-windows is 340Mi of it and cannot run on these nodes, but the + // nodepools constrain arch and not os, so it is reserved anyway. + let dsMem, dsCpu = 494.0, 245.0 + + let overlay = + System.IO.File.ReadAllText( + "../../../../MissionParallelCatchup/parallel_catchup_helm/values-ondemand.yaml") + + let claim (map: string) (tier: string) = + let entry = + overlay.Split('\n') + |> Array.find (fun l -> l.TrimStart().StartsWith(map + ":")) + entry.Split(',') + |> Array.pick (fun kv -> + let parts = (kv.Split(':') |> Array.map (fun x -> x.Trim([| '"'; ' ' |]))) + if parts.[parts.Length - 2] = tier then Some parts.[parts.Length - 1] else None) + + // tier, measured allocatable MiB, measured allocatable millicores + let nodes = + [ "subdwarf", 1127.0, 725.0 + "dwarf", 1127.0, 725.0 + "subgiant", 2663.0, 1715.0 + "giant", 5940.0, 1715.0 + "supergiant", 13313.0, 1715.0 + "nebula", 13313.0, 3705.0 + "hypergiant", 28714.0, 3705.0 + "protostar", 28714.0, 1715.0 + "supernova", 59515.0, 7695.0 ] + + for (tier, allocMem, allocCpu) in nodes do + let mem = float ((claim "poolMem" tier).Replace("Mi", "")) + let cpu = float (claim "poolCpu" tier) * 1000.0 + + // One pod must FIT once the daemonsets are counted -- this is the half + // that was missing, and it is why nothing provisioned. + Assert.True( + mem + dsMem <= allocMem, + sprintf "%s: %.0fMi + %.0fMi daemonsets exceeds %.0fMi allocatable" tier mem dsMem allocMem + ) + + Assert.True( + cpu + dsCpu <= allocCpu, + sprintf "%s: %.0fm + %.0fm daemonsets exceeds %.0fm allocatable" tier cpu dsCpu allocCpu + ) + + // And a second must NOT, or the isolation the on-demand ladder exists + // for is gone without anything failing. + Assert.True( + 2.0 * mem + dsMem > allocMem, + sprintf "%s: two pods fit in %.0fMi; on-demand is one per node" tier allocMem + ) + + Assert.True( + 2.0 * cpu + dsCpu > allocCpu, + sprintf "%s: two pods fit in %.0fm; on-demand is one per node" tier allocCpu + ) + + +[] +let ``incremental log fetch tars only what changed since the watermark`` () = + // The teardown tar moved the whole volume in one stream and measured ~20 + // minutes on a full run, entirely after the work had finished. The + // watermark is what turns that into a delta. + let first = String.concat " " (logTarCommand 0L) + Assert.DoesNotContain("--newer-mtime", first) + Assert.Contains("tar -cf -", first) + + // A watermark reaches BACK by the overlap, never forward: a file written + // while the previous tar walked the tree carries an mtime inside that + // window and has to be picked up again rather than skipped forever. + let later = String.concat " " (logTarCommand 1000000L) + Assert.Contains(sprintf "--newer-mtime=@%d" (1000000L - logFetchOverlapSecs), later) + + // The overlap cannot drive the filter negative on a clock near the epoch. + Assert.Contains("--newer-mtime=@0", String.concat " " (logTarCommand 1L)) + + // Both passes keep the collector's per-attempt verdicts and drop its resume + // bookkeeping, or a post-mortem loses why a range failed. + for cmd in [ first; later ] do + Assert.Contains("--exclude='*.state'", cmd) + Assert.Contains("--exclude='./lost+found'", cmd) + + +[] +let ``log archive parts sort in fetch order`` () = + // Parts are extracted in order so a later, complete copy of a file + // overwrites an earlier truncated one. Zero-padded because part10 must not + // sort before part2. + let names = [ 1; 2; 10 ] |> List.map (logArchiveName "run") + Assert.Equal(List.sort names, names) + Assert.Equal("run-worker-logs.part01.tar", logArchiveName "run" 1) + + +[] +let ``a truncated log archive is not mistaken for a good one`` () = + // The watermark may only advance past an archive that reads back whole. + // On ssc-test 2026-08-07 a 64MB mid-run part came back cut mid-member; the + // watermark advanced anyway and 12 ranges lost their logs permanently, + // because their archives were complete and therefore older than the new + // watermark -- so nothing would ever fetch them again. + let root = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "ssc-tar-test") + if System.IO.Directory.Exists root then System.IO.Directory.Delete(root, true) + let src = System.IO.Path.Combine(root, "logs") + System.IO.Directory.CreateDirectory(src) |> ignore + System.IO.File.WriteAllBytes(System.IO.Path.Combine(src, "range-1-a1.log.gz"), Array.init 4096 byte) + + // Written outside the directory being archived, or the tar would contain itself. + let whole = System.IO.Path.Combine(root, "whole.tar") + System.Formats.Tar.TarFile.CreateFromDirectory(src, whole, false) + Assert.True(archiveIsIntact whole, "a complete archive must read back whole") + + // Cut the stream mid-member, which is exactly what the pod exec produced. + let cut = System.IO.Path.Combine(root, "cut.tar") + let bytes = System.IO.File.ReadAllBytes(whole) + System.IO.File.WriteAllBytes(cut, bytes.[0 .. bytes.Length / 2]) + Assert.False(archiveIsIntact cut, "a truncated archive must not pass as intact") + + System.IO.Directory.Delete(root, true) diff --git a/src/FSLibrary.Tests/TestsRace8.fs b/src/FSLibrary.Tests/TestsRace8.fs index cd0f42e1..16f9f491 100644 --- a/src/FSLibrary.Tests/TestsRace8.fs +++ b/src/FSLibrary.Tests/TestsRace8.fs @@ -2,20 +2,16 @@ // under the Apache License, Version 2.0. See the COPYING file at the root // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 -// RACE #8 -- the ConfigMap fallback yields a measurement-free profile artifact -// that is indistinguishable from a good one. +// RACE #8 -- a measurement-free profile artifact that is indistinguishable from +// a good one. // -// readProgressRecord prefers /logs/progress.json on the monitor pod, but on ANY -// failure of that exec it silently falls back to the progress ConfigMap. The -// ConfigMap is a state mirror: job_monitor.py's _state_only() strips every -// profiling field out of it on purpose, to stay under the 1 MiB cap. So the -// fallback hands writeRangeProfile a `completed` map in which every record has -// had all eight measurements removed and only bookkeeping (attempts, count) -// left behind. +// A completed record can carry bookkeeping (attempts, count) and no measurement +// at all: the collector never wrote peaks for that range, or the read that would +// have supplied them was degraded. Such a record must not become a profile entry. // -// The `entry.Count > 0` guard exists to skip measurement-free entries, but -// count is attached to the entry BEFORE the guard runs, so every entry has at -// least one field and every entry passes. The result is an artifact with the +// The `entry.Count > 0` guard exists to skip measurement-free entries, but count +// used to be attached to the entry BEFORE the guard ran, so every entry had at +// least one field and every entry passed. The result is an artifact with the // right number of ranges and zero measurements -- observed twice in the field, // reporting 0% peakAnonBytes while the monitor's own progress.json carried 99%. // The next run then sizes from a profile that silently has no data. @@ -28,11 +24,11 @@ open Xunit open Newtonsoft.Json.Linq open MissionHistoryPubnetParallelCatchupV2 -/// Exactly job_monitor.py's _PROFILE_ONLY_FIELDS -- the fields _state_only() -/// removes when it mirrors progress.json into the capped ConfigMap. -let private profileOnlyFields = - [ "peakAnonBytes"; "peakRssBytes"; "peakWorkingSetBytes"; "peakCpuCores" - "peakEphemeralBytes"; "txApply"; "seconds"; "wallSeconds" ] +/// Every measurement a completed record can carry. A superset of +/// rangeProfileFields: wallSeconds and txApply are recorded but never projected. +let private measurementFields = + [ "peakAnonBytes"; "peakWorkingSetBytes"; "peakEphemeralBytes" + "txApply"; "seconds"; "wallSeconds" ] /// A completed record the way /logs/progress.json carries it: bookkeeping plus /// real measurements. @@ -48,10 +44,10 @@ let private measuredRecord (count: int) (anon: int64) = r /// The same record as it survives the ConfigMap mirror. -let private configMapMirrored (record: JObject) = +let private unmeasured (record: JObject) = let r = record.DeepClone() :?> JObject - for f in profileOnlyFields do + for f in measurementFields do r.Remove(f) |> ignore r @@ -66,12 +62,12 @@ let private completedMap (pairs: (string * JObject) list) = [] -let ``a configmap-mirrored range carries no measurement and must not enter the profile`` () = - let mirrored = configMapMirrored (measuredRecord 420 900L) +let ``a range carrying no measurement must not enter the profile`` () = + let mirrored = unmeasured (measuredRecord 420 900L) - // Precondition: the mirror really does strip every measurement, leaving + // Precondition: the helper really does strip every measurement, leaving // only bookkeeping. If this ever stops holding, the rest is meaningless. - for f in profileOnlyFields do + for f in measurementFields do Assert.Null(mirrored.[f]) Assert.NotNull(mirrored.["count"]) @@ -95,15 +91,16 @@ let ``count alone never satisfies the measurement guard`` () = [] -let ``a run whose progress came from the configmap produces no profile artifact`` () = +let ``a run whose ranges measured nothing produces no profile artifact`` () = // The headline symptom: a full-looking artifact, right number of ranges, - // zero measurements. Writing nothing is correct here -- the next run then - // falls back to its configured defaults instead of sizing from empty data. + // zero measurements -- what a run whose collector never wrote peaks leaves + // behind. Writing nothing is correct: the next run then falls back to its + // configured defaults instead of sizing from empty data. let completed = completedMap - [ "420", configMapMirrored (measuredRecord 420 900L) - "840", configMapMirrored (measuredRecord 420 950L) - "1260", configMapMirrored (measuredRecord 420 990L) ] + [ "420", unmeasured (measuredRecord 420 900L) + "840", unmeasured (measuredRecord 420 950L) + "1260", unmeasured (measuredRecord 420 990L) ] match rangeProfileDocument "pvc" 20000 completed with | None -> () @@ -118,14 +115,14 @@ let ``a run whose progress came from the configmap produces no profile artifact` [] let ``the profile counts only ranges that actually measured something`` () = - // A partly-degraded read is the dangerous case: the artifact looks - // populated, so nothing downstream can tell the stripped ranges apart from - // the measured one. + // A partly-measured run is the dangerous case: the artifact looks + // populated, so nothing downstream can tell the unmeasured ranges apart + // from the measured one. let completed = completedMap [ "420", measuredRecord 420 900L - "840", configMapMirrored (measuredRecord 420 950L) - "1260", configMapMirrored (measuredRecord 420 990L) ] + "840", unmeasured (measuredRecord 420 950L) + "1260", unmeasured (measuredRecord 420 990L) ] let ranges = buildRangeProfile completed @@ -150,7 +147,11 @@ let ``a measured run still produces a complete profile`` () = Assert.Equal(2, ranges.Count) Assert.Equal(900L, ranges.["420"].["peakAnonBytes"].Value()) Assert.Equal(950L, ranges.["840"].["peakAnonBytes"].Value()) - Assert.Equal(60.0, ranges.["420"].["txApply"].Value()) + Assert.Equal(120.0, ranges.["420"].["seconds"].Value()) + // Measured but deliberately not projected: recorded as metrics, never + // sized or ordered from, and pure weight in a 1 MiB-capped artifact. + Assert.Null(ranges.["420"].["wallSeconds"]) + Assert.Null(ranges.["420"].["txApply"]) // count is still carried, and the slicing is inferred from it rather // than from the caller's default. Assert.Equal(420, ranges.["420"].["count"].Value()) @@ -165,10 +166,25 @@ let ``a single real measurement is enough to keep a range and it keeps its count let r = JObject() r.["attempts"] <- JValue(2) r.["count"] <- JValue(420) - r.["wallSeconds"] <- JValue(77.0) + r.["seconds"] <- JValue(77.0) let ranges = buildRangeProfile (completedMap [ "420", r ]) Assert.Equal(1, ranges.Count) - Assert.Equal(77.0, ranges.["420"].["wallSeconds"].Value()) + Assert.Equal(77.0, ranges.["420"].["seconds"].Value()) Assert.Equal(420, ranges.["420"].["count"].Value()) + + +[] +let ``a range measured only in unprojected fields is dropped, not kept on count alone`` () = + // The guard reads the PROJECTION, not the record, so narrowing + // rangeProfileFields narrows what counts as measured. A record carrying only + // wallSeconds/txApply now projects to nothing and must be dropped -- keeping + // it would reintroduce exactly the count-only entry the guard exists to stop. + let r = JObject() + r.["attempts"] <- JValue(1) + r.["count"] <- JValue(420) + r.["wallSeconds"] <- JValue(77.0) + r.["txApply"] <- JValue(12.0) + + Assert.Empty(buildRangeProfile (completedMap [ "420", r ])) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 5d801be1..9b0cf54c 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -16,6 +16,7 @@ open System open System.Diagnostics open System.Net.Http open System.IO +open System.Formats.Tar open Newtonsoft.Json.Linq open Microsoft.FSharp.Control @@ -39,14 +40,20 @@ let helmChartPath = // $ dotnet run --project src/App/App.fsproj -- mission HistoryPubnetParallelCatchupV2 --image=docker-registry.services.stellar-ops.com/dev/stellar-core:23.0.3-2779.4d1df2b03.jammy-vnext-buildtests --pubnet-parallel-catchup-num-workers=2 --pubnet-parallel-catchup-starting-ledger=0 --pubnet-parallel-catchup-end-ledger=6400 --pubnet-parallel-catchup-ledgers-per-job 1280 --destination ./logs // let helmChartPath = "src/MissionParallelCatchup/parallel_catchup_helm" let valuesFilePath = helmChartPath + "/values.yaml" +// Layered on top of values.yaml for on-demand runs only. The chart defaults are +// the spot claims: the spot pools were doubled on 2026-08-04 so each claim is +// half a node and two pods share it. On-demand pools kept their original sizes, +// where those same claims are the node's NAMEPLATE -- and nameplate is not +// allocatable, so nothing schedules at all. A 14336Mi claim has ~13313Mi to land +// in on a 16 GiB node once the EKS reserve and 154Mi of daemonsets come out. +let onDemandValuesFilePath = helmChartPath + "/values-ondemand.yaml" // Keys in the -catchup-progress ConfigMap. These were HTTP paths when // the driver polled the monitor through a Gateway; it reads the ConfigMap now. let jobMonitorStatusKey = "status.json" // live queue counts -let jobMonitorProgressKey = "progress.json" // durable per-range completion record let jobMonitorLoggingIntervalSecs = 30 // frequency of the monitor reconcile loop: dispatch, liveness ping, status publish -let jobMonitorStatusCheckIntervalSecs = 60 // frequency of us querying job monitor's `/status` end point +let jobMonitorStatusCheckIntervalSecs = 60 // frequency of us reading the monitor's progress ConfigMap let jobMonitorStatusCheckTimeOutSecs = 600 let mutable toPerformCleanup = true let failedJobLogFileLineCount = 10000 @@ -170,6 +177,42 @@ let installProject (context: MissionContext) = setOptions.Add(sprintf "range.order=%s" context.pubnetParallelCatchupRangeOrder) + // Nodepool routing. Empty prefix ships the pre-tier behaviour: one label for + // every worker. Set, each range goes to - where the tier comes + // from its measured peakAnonBytes, and gets that node to itself. + setOptions.Add(sprintf "monitor.poolPrefix=%s" context.pubnetParallelCatchupPoolPrefix) + + // Capacity type is DERIVED, not configured. Both capacity variants of a tier + // share one label value, so a pod needs this second expression to pick a + // side -- and the storage mode already decides which side it must be. pvc + // exists so an evicted range resumes at LCL+1, which is what makes spot + // survivable; ephemeral has no resume, so it belongs on nodes that are not + // reclaimed underneath it. Letting these disagree would put a run with no + // resume path onto interruptible capacity. + if context.pubnetParallelCatchupPoolPrefix <> "" then + let capacityType = + if context.pubnetParallelCatchupStorageMode = "pvc" then "spot" else "on-demand" + + setOptions.Add(sprintf "monitor.capacityType=%s" capacityType) + + // Routing needs the label KEY and the taint toleration, and neither has + // a sensible default for an unpooled run -- both ship as []. Derived + // here for the same reason capacityType is: a pooled run that sets only + // the prefix otherwise fails twice over, and both failures are quiet. + // Karpenter labels these nodes purpose=-, which is exactly + // the value job_monitor builds per range, and taints them : + // NoSchedule. Without the key there is no tier affinity at all (the pods + // schedule anywhere); without the toleration they schedule nowhere. + // Observed on ssc-test 2026-08-07: 10 workers Pending indefinitely, + // "did not tolerate taint (taint=catchup:NoSchedule)". + setOptions.Add( + sprintf "worker.requireNodeLabels[0]=purpose:%s" context.pubnetParallelCatchupPoolPrefix + ) + + setOptions.Add( + sprintf "worker.tolerateNodeTaints[0]=%s" context.pubnetParallelCatchupPoolPrefix + ) + setOptions.Add(sprintf "range.startingLedger=%d" context.pubnetParallelCatchupStartingLedger) let endLedger = @@ -315,16 +358,23 @@ let installProject (context: MissionContext) = // installs its monitor, Jobs and PVCs into a different one. Observed // 2026-07-30: a mission run with --namespace sandbox put a monitor and four // Jobs into the production namespace alongside a live run. - RunShellCommand [| "helm" - "install" - helmReleaseName - helmChartPath - "--namespace" - context.namespaceProperty - "--values" - valuesFilePath - "--set" - String.Join(",", setOptions) |] + // The overlay rides as a second --values, not as setOptions, because every + // option below is folded into ONE comma-separated --set and the pool maps are + // themselves comma-separated -- they would need every internal comma escaped. + // Derived from storage mode for the same reason capacityType is: pvc means + // spot means shared nodes, ephemeral means on-demand means one pod per node. + let valuesArgs = + if context.pubnetParallelCatchupStorageMode = "pvc" then + [| "--values"; valuesFilePath |] + else + [| "--values"; valuesFilePath; "--values"; onDemandValuesFilePath |] + + RunShellCommand( + Array.concat [ [| "helm"; "install"; helmReleaseName; helmChartPath |] + [| "--namespace"; context.namespaceProperty |] + valuesArgs + [| "--set"; String.Join(",", setOptions) |] ] + ) |> ignore match RunShellCommand [| "helm" @@ -341,61 +391,191 @@ let installProject (context: MissionContext) = // 1. Automatically determines worker pod names from context.pubnetParallelCatchupNumWorkers // 2. For each pod, finds all files matching "stellar-core-*.log" in /data // 3. Creates a tar.gz archive and copies it to context.destination directory -let collectLogsFromPods (context: MissionContext) = - // Worker pods are per-range now and are reaped within about a minute of - // finishing, so there is nothing left to exec into at teardown. The monitor - // pulls each pod's log while it is still alive -- on failure before the - // retry, on success before the Job's TTL -- onto its own volume, so one - // exec here replaces the ~1024 that the StatefulSet design needed. - let monitorPods = - context - .kube - .ListNamespacedPod( - context.namespaceProperty, - labelSelector = sprintf "app=job-monitor,release=%s" helmReleaseName - ) - .Items - |> Seq.map (fun p -> p.Metadata.Name) - |> List.ofSeq +// How often the main loop fetches logs, and how much of the previous window it +// re-fetches. One pass every 10 minutes flattens the teardown cost without +// taking meaningful IOPS from the collector: --newer-mtime still stats every +// file, and at ~4000 attempts that walk is the expensive part, not the bytes. +let logFetchIntervalSecs = 600 +let logFetchOverlapSecs = 120L +// Transfers of this archive are retried in-pass before the window is deferred. +let logFetchAttempts = 3 + +// An empty GNU tar is two zero blocks at the default blocking factor. A pass +// with nothing new still writes that, so it is deleted rather than left to +// clutter the destination with 19 identical 10K files. +let emptyTarBytes = 10240L + +/// The archive written by one fetch. Parts are numbered rather than merged: +/// extracting them in order reconstructs /logs, and a later part overwrites an +/// earlier truncated copy of a file that was still being appended when it was +/// first picked up. +let logArchiveName (release: string) (part: int) : string = + sprintf "%s-worker-logs.part%02d.tar" release part + +/// tar of everything modified since `sinceEpoch`; 0 takes the whole volume, +/// which is what teardown does when no incremental pass ever landed. +/// +/// Entries are already gzipped by the streaming collector, so this bundles +/// without re-compressing. Named range--a.log.gz, so a failing +/// range is findable directly rather than by worker ordinal. Keeps the +/// per-attempt .outcome verdicts and drops .state, the collector's own resume +/// bookkeeping. +let logTarCommand (sinceEpoch: int64) : string [] = + let since = + if sinceEpoch > 0L then + sprintf " --newer-mtime=@%d" (max 0L (sinceEpoch - logFetchOverlapSecs)) + else + "" + + [| "sh" + "-c" + // lost+found is the ext4 root of the logs PVC, not ours. + sprintf "cd /logs && tar -cf -%s --exclude='*.state' --exclude='./lost+found' ." since |] + +/// Can every entry in this archive be read back? +/// +/// A pass tars /logs while workers are still appending to it, so tar can hit +/// "file changed as we read it", exit non-zero, and leave the stream cut +/// mid-member. Verified on ssc-test 2026-08-07: at 200 workers a 64MB part came +/// back truncated and 12 ranges lost their logs for good, because the watermark +/// advanced anyway and their archives -- complete, and older than the new +/// watermark -- were never re-sent. Checking here turns that into a re-transfer. +let archiveIsIntact (path: string) : bool = + try + use stream = File.OpenRead(path) + use reader = new TarReader(stream) + + let mutable entry = reader.GetNextEntry() + + while not (isNull entry) do + entry <- reader.GetNextEntry() - match monitorPods with - | [] -> LogWarn "No job-monitor pod found for release %s; worker logs cannot be collected" helmReleaseName - | podName :: _ -> + true + with _ -> + false + +let private monitorPodName (context: MissionContext) : string option = + context + .kube + .ListNamespacedPod( + context.namespaceProperty, + labelSelector = sprintf "app=job-monitor,release=%s" helmReleaseName + ) + .Items + |> Seq.map (fun p -> p.Metadata.Name) + |> Seq.tryHead + +/// Fetch everything written since `sinceEpoch` into numbered part `part`. +/// +/// Worker pods are per-range and are reaped within about a minute of finishing, +/// so there is nothing left to exec into at teardown. The monitor pulls each +/// pod's log while it is still alive onto its own volume, so one exec here +/// replaces the ~1024 the StatefulSet design needed. +/// +/// Returns the watermark for the next call, or None when nothing was fetched -- +/// the caller then keeps its old watermark and re-fetches that window next time, +/// so a failed pass costs bandwidth rather than logs. +let collectLogsSince (context: MissionContext) (sinceEpoch: int64) (part: int) : int64 option = + match monitorPodName context with + | None -> + LogWarn "No job-monitor pod found for release %s; worker logs cannot be collected" helmReleaseName + None + | Some podName -> try - LogInfo "Collecting worker logs from job-monitor pod %s to %s" podName context.destination.Path - - let outputFile = - Path.Combine(context.destination.Path, sprintf "%s-worker-logs.tar" helmReleaseName) - - // Entries are already gzipped by the streaming collector, so this - // bundles without re-compressing. Named range--a.log.gz, - // so a failing range is findable directly rather than by worker ordinal. - // Already gzipped by the collector, so no -z. Keeps the per-attempt - // .outcome verdicts (outcome/exitCode/pod -- useful post-mortem) and - // drops .state, which is only the collector's resume bookkeeping. - let command = - [| "sh" - "-c" - // lost+found is the ext4 root of the logs PVC, not ours. - "cd /logs && tar -cf - --exclude='*.state' --exclude='./lost+found' ." |] + // The clock is read from the POD, and BEFORE the tar. This driver + // runs outside the cluster, so a few seconds of NTP skew either way + // would silently skip a file forever; and a watermark taken after + // the tar would exclude anything written while it walked the tree. + // Both failure modes lose logs; taking it early only re-sends. + let stampFile = + Path.Combine(Path.GetTempPath(), sprintf "%s-logstamp" helmReleaseName) RemoteCommandRunner.RunRemoteCommandAndCaptureOutput( kube = context.kube, ns = context.namespaceProperty, podName = podName, containerName = "job-monitor", - command = command, - outputFilePath = outputFile + command = [| "date"; "+%s" |], + outputFilePath = stampFile ) - let fileInfo = FileInfo(outputFile) + let parsed, podEpoch = Int64.TryParse(File.ReadAllText(stampFile).Trim()) - if fileInfo.Exists && fileInfo.Length > 0L then - LogInfo "Collected worker logs to %s (size: %d bytes)" outputFile fileInfo.Length + if not parsed then + LogWarn "Could not read the clock from %s; skipping this log pass" podName + None else - LogWarn "Worker log archive is empty: %s" outputFile + let outputFile = + Path.Combine(context.destination.Path, logArchiveName helmReleaseName part) + + // The exec stream can end early on a large transfer and report + // success anyway -- the pod's tar exits 0, the status channel + // says Success, and the bytes simply stop. Measured on ssc-test + // 2026-08-07: a 64MB part arrived cut mid-member. So fetch, then + // read the archive back, and retry the whole transfer before + // giving up on this window. + let mutable attemptsLeft = logFetchAttempts + let mutable intact = false + + while attemptsLeft > 0 && not intact do + attemptsLeft <- attemptsLeft - 1 + + RemoteCommandRunner.RunRemoteCommandAndCaptureOutput( + kube = context.kube, + ns = context.namespaceProperty, + podName = podName, + containerName = "job-monitor", + command = logTarCommand sinceEpoch, + outputFilePath = outputFile + ) + + let fi = FileInfo(outputFile) + intact <- fi.Exists && (fi.Length <= emptyTarBytes || archiveIsIntact outputFile) + + if not intact && attemptsLeft > 0 then + LogWarn "Worker log archive came back truncated; refetching (%d attempt(s) left)" attemptsLeft + + let fileInfo = FileInfo(outputFile) + + if not fileInfo.Exists then + LogWarn "Worker log archive was not written: %s" outputFile + None + elif fileInfo.Length <= emptyTarBytes then + File.Delete(outputFile) + LogInfo "No new worker logs since the last pass" + Some podEpoch + elif not (archiveIsIntact outputFile) then + // Keep the part -- it holds real entries, and a later + // complete pass over the same window supersedes it on + // extract. But hold the watermark, so that window IS + // re-fetched rather than silently skipped. + LogWarn + "Worker log archive %s is truncated (%d bytes); keeping the old watermark so this window is fetched again" + outputFile + fileInfo.Length + + None + else + LogInfo "Collected worker logs to %s (size: %d bytes)" outputFile fileInfo.Length + Some podEpoch + with ex -> + LogWarn "Failed to collect worker logs: %s" ex.Message + None + +// Watermark and part counter for the incremental fetch. Module-level because +// the main loop and the cleanup path both advance them. +let mutable private logWatermark = 0L +let mutable private logPartCount = 0 + +/// One log pass. Advances the watermark only when the bytes are safely local. +let collectLogsFromPods (context: MissionContext) = + let part = logPartCount + 1 - with ex -> LogWarn "Could not collect worker logs from %s: %s" podName ex.Message + match collectLogsSince context logWatermark part with + | Some epoch -> + logWatermark <- epoch + logPartCount <- part + | None -> () // Cleanup on exit. `signalTriggered` indicates we're running under a hard // deadline (Jenkins' SoftKillWaitSeconds, ~5s by default, before SIGKILL). @@ -404,9 +584,9 @@ let collectLogsFromPods (context: MissionContext) = // collection and leak every worker pod, which is what we saw in practice // with a 1024-worker run aborted from Jenkins. let queryJobMonitor (context: MissionContext, key: String) = - // The monitor publishes the same JSON it serves on /status into - // -catchup-progress. Reading it through the kube API removes the - // Gateway/HTTPRoute dependency entirely -- the driver already has a client. + // The monitor publishes its status JSON into -catchup-progress. + // Reading it through the kube API removes the Gateway/HTTPRoute dependency + // entirely -- the driver already has a client. try let cm = context.kube.ReadNamespacedConfigMap(helmReleaseName + "-catchup-progress", context.namespaceProperty) @@ -431,23 +611,22 @@ let queryJobMonitor (context: MissionContext, key: String) = // A PVC's size is absent on purpose -- it is not a scheduling dimension, so // profiling it buys no packing. peakEphemeralBytes appears only for // ephemeral-mode runs, and only for ranges that finished. +// +// A subset of what the monitor records: it measures more per range than the next +// run can size from, and a measurement nothing reads is pure weight in an +// artifact that was already 963 KB at 4805 ranges. let rangeProfileFields = - // peakAnonBytes is the field the sizing consumer prefers (kubelet-sampled - // anon; peakRssBytes is the coarser Prometheus-era name for the same - // quantity). Omitting it here silently stripped it from the mission's - // profile artifact while the monitor's own progress.json carried it -- - // measured 2026-07-30: artifact 0% peakAnonBytes, volume copy 99%. + // The memory figure the sizing consumer reads, sampled from kubelet by the + // collector. Omitting it strips it from the artifact while progress.json + // still carries it, so the next run sizes every range from defaults. [ "peakAnonBytes" - "peakRssBytes" "peakWorkingSetBytes" - "peakCpuCores" "peakEphemeralBytes" - "seconds" - // Kubernetes startTime -> completionTime for the winning Job only. The - // monitor cannot reconstruct first dispatch -> success after predecessor - // Jobs and their inter-attempt gaps are gone. - "wallSeconds" - "txApply" ] + // The only timing the next run sizes from: it sets the percentile basis, + // the dispatch order and the runtime insurance thresholds. wallSeconds and + // txApply are recorded per range as Prometheus metrics but nothing sizes + // or orders from either, so they stay out of the artifact. + "seconds" ] // A missing measurement must stay missing rather than become a null: the // consumer falls back to its configured default when the field is absent. @@ -462,12 +641,13 @@ let projectRangeEntry (record: JObject) : JObject = entry -// The progress record, preferring the copy on the monitor's volume. +// The progress record, read only from the monitor's volume. // -// The ConfigMap is only a mirror and is capped at 1 MiB -- about 6100 ranges at -// ~172 bytes each, reachable simply by halving ledgersPerJob. Past that the -// mirror stops updating while /logs/progress.json stays correct, so reading the -// ConfigMap would silently truncate the artifact. +// Not from the ConfigMap: that is a visibility mirror with every profiling +// field stripped and a 1 MiB cap (~6100 ranges, reachable by halving +// ledgersPerJob). A profile built from it would be empty but look complete, and +// past the cap it stops updating while /logs/progress.json stays correct. +// No record is the safe outcome -- the consumer falls back to its defaults. let readProgressRecord (context: MissionContext) : JObject option = let monitorPods = context @@ -504,21 +684,10 @@ let readProgressRecord (context: MissionContext) : JObject option = else None with ex -> - LogWarn "Could not read /logs/progress.json (%s); falling back to the ConfigMap" ex.Message + LogWarn "Could not read /logs/progress.json (%s); no range profile will be written" ex.Message None - match fromVolume with - | Some p -> Some p - | None -> - // Degraded read, and it must not be silent. The ConfigMap is a state - // mirror: the monitor strips every profiling field out of it to stay - // under the 1 MiB cap, so a record sourced here carries attempts and - // count and nothing else. Any range profile built from it will be - // empty, and rangeProfileDocument will decline to write one. - LogWarn - "Falling back to the progress ConfigMap; it is a state mirror with no measurements, so no range profile can be built from it" - - queryJobMonitor (context, jobMonitorProgressKey) + fromVolume // The `ranges` map of a profile artifact, built from a progress record's @@ -546,10 +715,8 @@ let buildRangeProfile (completed: JObject) : JObject = // The guard decides on measurements alone. count is bookkeeping, not a // measurement, so it is attached only after the entry has been found to // carry something real. Attaching it first made every entry non-empty - // and defeated the guard completely: a ConfigMap-sourced record, which - // has had all eight profiling fields stripped by the monitor's - // _state_only(), still sailed through and produced a range with nothing - // in it but a count. + // and defeated the guard completely: a record that measured nothing + // still sailed through and produced a range holding only a count. if entry.Count > 0 then match record.["count"] with | null -> () @@ -653,11 +820,12 @@ let cleanup (signalTriggered: bool) (context: MissionContext) = if toPerformCleanup then toPerformCleanup <- false - // Before either branch: `helm uninstall` deletes the progress ConfigMap - // the profile is built from, so an aborted run would otherwise lose every - // measurement it had already taken. One ConfigMap read and a local file - // write -- cheap enough for the abort path's few seconds, and a run - // stopped part-way is exactly when the partial profile is most wanted. + // Before either branch: `helm uninstall` takes the monitor pod, and with + // it the volume the profile is read from, so an aborted run would + // otherwise lose every measurement it had already taken. One pod exec + // and a local file write -- cheap enough for the abort path's few + // seconds, and a run stopped part-way is exactly when the partial + // profile is most wanted. try writeRangeProfile context with ex -> LogWarn "Failed to write range profile: %s" ex.Message @@ -668,7 +836,9 @@ let cleanup (signalTriggered: bool) (context: MissionContext) = // Jenkins' ~5s grace before SIGKILL, and it can't beat the per-pod // terminationGracePeriodSeconds (default 30s) when scaled to 1024 // workers. Whatever logs were captured inline by the failure - // handler in the main loop are still on disk. + // handler in the main loop are still on disk -- and since the main + // loop now fetches every logFetchIntervalSecs, an abort keeps every + // part up to the last pass rather than losing the run's logs whole. LogInfo "Signal-triggered cleanup: uninstalling release %s" helmReleaseName RunShellCommand [| "helm" @@ -765,6 +935,7 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = // fails -- it just finishes the work it can first. let failedJobs = ResizeArray() let seenFailures = System.Collections.Generic.HashSet() + let mutable lastLogFetch = DateTime.UtcNow while not allJobsFinished do Thread.Sleep(jobMonitorStatusCheckIntervalSecs * 1000) @@ -776,7 +947,7 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = timeoutLeft <- jobMonitorStatusCheckTimeOutSecs let remainSize = status.Value("num_remain") let jobsFailed = status.["jobs_failed"] :?> JArray - let JobsInProgress = status.["jobs_in_progress"] :?> JArray + let jobsInProgress = status.Value("queue_in_progress_count") for job in jobsFailed do let text = job.ToString() @@ -785,10 +956,22 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = failedJobs.Add(text) LogError "RANGE FAILED: %s -- run continues, mission will fail once it drains" text - if remainSize = 0 && JobsInProgress.Count = 0 then + if remainSize = 0 && jobsInProgress = 0 then LogInfo "All queues empty. Mission complete." allJobsFinished <- true + // Pull the logs written since the last pass, so teardown moves a + // delta instead of the whole volume. Measured at ~20 minutes for + // a full run, all of it after the work had finished. Isolated in + // its own try: a failed pass re-fetches the same window next + // time and must never take the mission down. + if (DateTime.UtcNow - lastLogFetch).TotalSeconds >= float logFetchIntervalSecs then + lastLogFetch <- DateTime.UtcNow + + try + collectLogsFromPods context + with ex -> LogWarn "Incremental log collection failed: %s" ex.Message + | None -> LogError "no status" timeoutLeft <- timeoutLeft - jobMonitorStatusCheckIntervalSecs diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 115303c1..aa3a49da 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -122,6 +122,7 @@ type MissionContext = pubnetParallelCatchupStorageMode: string pubnetParallelCatchupProfile: string pubnetParallelCatchupRangeOrder: string + pubnetParallelCatchupPoolPrefix: string pubnetParallelCatchupCpuRequest: string genesisTestAccountCount: int option diff --git a/src/MissionParallelCatchup/.dockerignore b/src/MissionParallelCatchup/.dockerignore new file mode 100644 index 00000000..7c3a8d38 --- /dev/null +++ b/src/MissionParallelCatchup/.dockerignore @@ -0,0 +1,2 @@ +**/__pycache__/ +**/*.pyc diff --git a/src/MissionParallelCatchup/Dockerfile.jobmonitor b/src/MissionParallelCatchup/Dockerfile.jobmonitor index 7243c3ee..5b68b2af 100644 --- a/src/MissionParallelCatchup/Dockerfile.jobmonitor +++ b/src/MissionParallelCatchup/Dockerfile.jobmonitor @@ -4,23 +4,32 @@ VOLUME /data WORKDIR /app -# Client major tracks a cluster minor (35.x <-> Kubernetes 1.35) and supports -# one minor of skew either way, so 35 covers 1.34 through 1.36 -- the cluster -# today and both upgrades already released. Revisit at 1.37. +# Client major tracks a cluster minor (36.x <-> Kubernetes 1.36). Upstream's +# compatibility table marks an exact match as supported and everything above or +# below as partial, which is where the 1.34 clusters sit today -- the same +# category 1.35 would be. Every API this mission uses (batch/v1 Job, +# podFailurePolicy, ttlSecondsAfterFinished) has been GA since 1.31. +# +# 36 rather than 35 because kubernetes.aio, the aiohttp-backed async client, +# does not exist before it: 35.0.0 ships no kubernetes/aio at all. # # The floor matters independently: podFailurePolicy (V1PodFailurePolicy*) needs # >=26 and V1VolumeResourceRequirements needs >=29. Ubuntu's distro package is # 22.6.0 and has neither, so it cannot express this mission's failure # classification at all. RUN pip install --no-cache-dir \ - 'kubernetes~=35.0' \ + 'kubernetes~=36.0' \ 'aiohttp~=3.9' \ - 'requests~=2.31' \ 'prometheus-client~=0.19' -COPY ./job_monitor.py /app +# apps/ and lib/ flatten into one directory here, and the modules import each +# other by bare name. /app is a single flat directory in the dev path too: that +# ConfigMap is built with --from-file, whose keys are basenames and cannot +# contain '/'. The split is for reading the repo, not for the runtime. +COPY ./apps/job_monitor.py /app # Same image, second entrypoint: runs as a sidecar streaming worker logs. -COPY ./log_collector.py /app +COPY ./apps/log_collector.py /app +COPY ./lib/ /app/ EXPOSE 8080 diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py new file mode 100644 index 00000000..a973dabd --- /dev/null +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -0,0 +1,1483 @@ +"""Parallel catchup job monitor. + + +Singleton state manager for a MissionParallelCatchup run. + +State is held on the shared volume and mirrored to a ConfigMap for the mission driver to read. + +The first reconcile thread is responsible for generating the full ledger range list +and dispatching Kubernetes Jobs per ledger range. + +It marks Jobs as completed, retries if failed due to OOM, spot eviction, +or other transient causes, and records the outcome of each attempt. +Genuine catchup failures are not retried and marked as failed. +The mission driver reads the ConfigMap to determine the overall progress of the catchup mission. +The mission driver is responsible for tearing down the mission when detecting a job failure or the completion of all ledger ranges. + +On each job completion, metrics are updated to reflect the duration and txApply progress, as well as the current state of the mission, +including the number of remaining jobs, succeeded jobs, failed jobs, and in-progress jobs. + +The second worker_liveness thread probes the /info endpoint of each running worker pod to determine its liveness. + +Finally the monitor exposes a simple HTTP server for health checks, status, and Prometheus metrics. + +""" + +import bisect +import collections +import gzip +import json +import math +import os +import re +import threading +import time +import zlib +from datetime import datetime, timezone + +from kubernetes import client +from kubernetes.client.rest import ApiException + +import attempts +import config +import http_server +import kube +import metrics +import profiles +import ranges +import records +import sizing +import worker_liveness +from logger import build_logger + + +logger = build_logger('job_monitor') +if not kube.IN_CLUSTER: + logger.warning("KUBERNETES_SERVICE_HOST is unset: no in-cluster config loaded. " + "Every API call will fail until kube.core_v1/kube.batch_v1 are replaced.") + + +def main(): + # Before any dispatch: the first Job built must already be sized from it. + config.PROFILE = profiles.load_profile() + # After the profile, before the thread: the only place a bad config can + # still take the process down instead of being swallowed by the loop. + validate_config() + + # This is the reconcile loop -- + # dispatch, progress record, metrics, status. + reconcile_thread = threading.Thread(target=reconcile_loop, daemon=True) + reconcile_thread.start() + + http_server.serve() + + +def validate_config(): + """Fatal config checks. Runs once at startup, BEFORE the reconcile thread starts. + + Called after load_profile() because the last check needs the profile. + """ + if config.RANGE_GENERATOR not in config.VALID_RANGE_GENERATORS: + raise ValueError("RANGE_GENERATOR must be one of %s, got %r" + % (', '.join(config.VALID_RANGE_GENERATORS), config.RANGE_GENERATOR)) + if config.RANGE_ORDER not in config.VALID_RANGE_ORDERS: + raise ValueError("RANGE_ORDER must be one of %s, got %r" + % (', '.join(config.VALID_RANGE_ORDERS), config.RANGE_ORDER)) + # longest-first sorts on measured `seconds`; with no profile every key ties + # and dispatch silently stays tip-first. + if config.RANGE_ORDER == 'longest-first' and not config.PROFILE: + raise ValueError( + "RANGE_ORDER=longest-first requires a profile: it orders ranges by " + "their measured seconds, and with no profile loaded every range ties " + "and dispatch stays tip-first. Pass a profile, or set RANGE_ORDER " + "explicitly to tip-first or oldest-first.") + + +status = { + 'num_remain': 1, # non-zero until the first real update, so callers don't see a premature 0 + 'queue_remain_count': 0, + 'queue_succeeded_count': 0, + 'queue_failed_count': 0, + 'queue_in_progress_count': 0, + 'jobs_failed': [], + 'workers_refresh_duration': 0, + 'mission_duration': 0, +} +status_lock = threading.Lock() + + +# --- the run itself --------------------------------------------------------- +def reconcile_loop(): + global status + # None until reconcile has an owner reference to attach it to; until then + # process start is correct anyway, because that IS the start of a new run. + mission_start_time = read_mission_start() or time.time() + state = {'owner': None, 'replayed': set(), + 'counted': {}} + while True: + try: + if state['owner'] is None: + state['owner'] = owner_ref() + _progress_owner['ref'] = state['owner'] + if read_mission_start() is None: + _patch_cm({'started_at': repr(mission_start_time)}) + + r = reconcile(state) + + # Grafana-only worker responsiveness, from the pod snapshot + # reconcile already has. One bounded sweep per pass, so this waits at + # most LIVENESS_SWEEP_SECONDS -- see worker_liveness.publish. + refresh_start = time.time() + targets = r.pop('_worker_targets') + try: + worker_counts = worker_liveness.publish(targets) + except Exception as e: + worker_counts = {'up': 0, 'down': 0, 'unknown': len(targets)} + now = time.time() + if now - state.get('last_liveness_error_log', 0) >= 60: + state['last_liveness_error_log'] = now + logger.exception( + "worker liveness publication failed (%s); reporting all " + "current candidates unknown and continuing reconcile", e) + workers_refresh_duration = time.time() - refresh_start + + mission_duration = time.time() - mission_start_time + with status_lock: + visible_in_progress = r['in_progress'] + r['finalizing'] + status = { + 'num_remain': r['remaining'], + 'queue_remain_count': r['remaining'], + 'queue_succeeded_count': r['completed'], + 'queue_failed_count': len(r['failed_ranges']), + 'queue_in_progress_count': len(visible_in_progress), + 'jobs_failed': r['failed_ranges'], + 'workers_refresh_duration': workers_refresh_duration, + 'mission_duration': mission_duration, + } + metrics.catchup_queues.labels(queue="remain").set(r['remaining']) + metrics.catchup_queues.labels(queue="succeeded").set(r['completed']) + metrics.catchup_queues.labels(queue="failed").set(len(r['failed_ranges'])) + metrics.catchup_queues.labels(queue="in_progress").set( + len(visible_in_progress)) + metrics.workers.labels(status="up").set(worker_counts['up']) + metrics.workers.labels(status="down").set(worker_counts['down']) + metrics.workers.labels(status="unknown").set(worker_counts['unknown']) + metrics.refresh_duration.set(workers_refresh_duration) + metrics.mission_duration.set(mission_duration) + logger.info("Status: %s", json.dumps(status)) + # Publish on change only -- a 10h run would otherwise issue ~3600 + # no-op ConfigMap writes. + counts = (r['remaining'], r['completed'], len(r['failed_ranges']), + len(visible_in_progress)) + if counts != state.get('last_counts'): + state['last_counts'] = counts + with status_lock: + save_status(status) + + except Exception as e: + logger.exception("Error while reconciling: %s", str(e)) + + time.sleep(config.RECONCILE_INTERVAL_SECONDS) + + +def reconcile(state): + desired = ranges.generate_ranges() + by_end = {str(end): count for end, count in desired} + progress = load_progress() + completed = progress.setdefault('completed', {}) + failed = progress.setdefault('failed', {}) + + jobs = kube.batch_v1.list_namespaced_job( + config.NAMESPACE, label_selector=f"{config.LABEL_RUN}={config.RUN_NAME}").items + job_pods = pods_by_job() + + live = {} # range-end -> (attempt, job) + current_attempts = set() + for j in jobs: + end = (j.metadata.labels or {}).get(config.LABEL_RANGE) + attempt = int((j.metadata.labels or {}).get(config.LABEL_ATTEMPT, 1)) + current_attempts.add((str(end), attempt)) + prev = live.get(end) + if prev is None or attempt >= prev[0]: + live[end] = (attempt, j) + + in_progress = [] + finalizing = [] + # The same ranges as `in_progress`, keyed by end. `remaining` is a COUNT + # over this run's range list, never `total - completed`: the shared progress + # record can carry ends from a run with a different ledgersPerJob, and a + # subtraction lets those move a number describing THIS run. + in_flight = set() + for end, (attempt, j) in list(live.items()): + st = j.status + if st.succeeded: + # Record before the Job's TTL can reclaim it: `seconds` is the + # pod's own start -> finish, and the pod goes ~1 min after the node + # empties. + if end not in completed: + pod = job_pods.get(j.metadata.name) + completed[end] = completion_record(end, attempt, st, pod, + by_end.get(end)) + if pod is not None and config.SAVE_SUCCESS_LOGS: + backstop_save_pod_log(pod.metadata.name, end, attempt) + # Durably recorded first: the record is what makes the volume + # and the Job disposable, so it must land before either goes. + save_progress(progress) + else: + # Backfill. The record is written when the Job flips to + # succeeded, usually before the collector finalizes, so + # reconstruct the whole profile rather than a field-by-field + # subset that can leave a record permanently short. + late = attempts._repair_completed_profile(end, attempt, completed[end]) + if late: + save_progress(progress) + logger.info("range %s: measurements arrived late, backfilled %s", + end, sorted(late)) + # Per sighting of a recorded range, not per first sight: both are + # idempotent, and hanging them off the run-once branch above leaks + # the volume and the Job whenever the process dies between + # save_progress and here. + release_pvc(end) + if _attempt_finalized(end, attempt): + # Nothing more can be learned from the Job. Deleting it reaps the + # pod, and .metrics is the only place peaks live, so this waits + # for the collector's marker; JOB_TTL_SECONDS reclaims anything + # the collector never finishes. + reap_range_jobs(end) + else: + # Keep the range counted as in-progress until that marker + # lands: the driver writes the final profile as soon as the + # count reaches zero. Not in `in_progress`, so it costs no + # dispatch capacity below. + finalizing.append(job_key(int(end), by_end.get(end, 0))) + elif st.failed: + # Completion is terminal, so a Failed Job for a recorded range is + # garbage: classifying it would redispatch the range against a PVC + # that was already released. Sweep it. + if end in completed: + logger.info("range %s already recorded complete; discarding " + "leftover Job for attempt %d", end, attempt) + reap_range_jobs(end) + continue + pod = job_pods.get(j.metadata.name) + if pod is not None: + record_outcome(end, attempt, pod) + backstop_save_pod_log(pod.metadata.name, end, attempt) + if end in failed: + # The decision is recorded and cannot change: no successor is + # coming and no budget is left. Everything worth keeping -- the + # archive, .outcome, .verdict -- is durable by the time the + # collector marks the attempt done, so the Job and the volume + # are holding nothing. Deleting the Job reaps the pod, which is + # why this waits for that marker like the success path. + # + # Without it the range stays the newest Job for its range and + # every pass re-derives the same verdict and re-logs the same + # condemnation until JOB_TTL_SECONDS -- measured at 15 identical + # lines over 9 minutes. + if _attempt_finalized(end, attempt): + release_pvc(end) + reap_range_jobs(end) + continue + verdict = verdict_for(end, attempt, j, pod) + # Durable before anything reads a tally: _oom_count and the budget + # below both count this attempt. + save_verdict(end, attempt, verdict['outcome']) + + decision = retry_decision(verdict, end, attempt) + if decision.action == 'defer': + in_progress.append(job_key(int(end), by_end[end])) + in_flight.add(str(end)) + continue + + spent, cap = budget_for(verdict, end, attempt) + if decision.action == 'retry' and spent < cap: + _log_retry(end, attempt, verdict, decision, cap) + try: + kube.batch_v1.create_namespaced_job(config.NAMESPACE, build_job( + int(end), by_end[end], attempt + 1, state['owner'], + decision.memory, decision.ephemeral)) + except ApiException as e: + if e.status != 409: + raise + current_attempts.add((str(end), attempt + 1)) + # After the successor exists, never before: if the create failed + # with the predecessor gone, the next pass would redispatch at + # attempt 1 and lose the escalated request. live[] keys on the + # highest attempt, so the two coexisting for a pass is handled. + # + # Gated on .done like the success path, which means the collector + # has finalized this attempt's peaks, tx_apply and duration. + # JOB_TTL_SECONDS reaps it if the collector never gets there. + if _attempt_finalized(end, attempt): + delete_job(end, attempt) + in_progress.append(job_key(int(end), by_end[end])) + in_flight.add(str(end)) + continue + + if decision.reason is not None: + logger.error("range %s exhausted %d attempts (%s)", end, cap, + decision.reason) + else: + # Say it plainly: otherwise the range only appears under + # failed{} and the mission aborts with no explanation. + logger.error("!!! RANGE CONDEMNED !!! %s failed with outcome=%s exitCode=%s " + "on attempt %d and is NOT retryable; this fails the mission", + end, verdict['outcome'], verdict.get('exitCode'), attempt) + + if end not in failed: + failed[end] = {'attempts': attempt, + 'pod': verdict.get('pod', pod.metadata.name if pod else ''), + 'outcome': verdict['outcome'], + 'exitCode': verdict['exitCode']} + save_progress(progress) + else: + in_progress.append(job_key(int(end), by_end.get(end, 0))) + in_flight.add(str(end)) + + # Nothing halts dispatch -- not a shrinking `completed` record, not a + # condemned range. The mission waits for `remaining == 0 and in_progress == + # []`, so a frozen dispatch deadlocks the driver; a condemned range is + # reported once the run drains, keeping the work already paid for. + # + # Dispatch, heaviest range first (index 0 is the tip), up to PARALLELISM. + created = 0 + # No slots: a range's PVC is keyed by the range itself, so concurrency is + # simply how many are in flight. + capacity = config.PARALLELISM - len(in_progress) + for end, count in desired: + if capacity <= 0: + break + key = str(end) + if key in completed or key in failed or key in live: + continue + try: + record_range_start(end, kube.batch_v1.create_namespaced_job( + config.NAMESPACE, build_job(end, count, 1, state['owner']))) + current_attempts.add((str(end), 1)) + created += 1 + capacity -= 1 + in_progress.append(job_key(end, count)) + in_flight.add(str(end)) + except ApiException as e: + if e.status != 409: # AlreadyExists: name uniqueness is the mutex + raise + current_attempts.add((str(end), 1)) + # Losing the mutex means the Job exists and is in flight, so it + # occupies a slot exactly like one we created and must spend + # capacity. + capacity -= 1 + in_progress.append(job_key(end, count)) + in_flight.add(str(end)) + + observe_recorded(progress, state['replayed']) + sync_counters(progress, state['counted'], current_attempts) + return { + 'total': len(desired), + 'completed': len(completed), + 'failed_ranges': [f"{job_key(int(k), by_end.get(k, 0))}|{v.get('pod', '')}" + for k, v in failed.items()], + 'in_progress': in_progress, + 'finalizing': finalizing, + 'created': created, + 'remaining': sum(1 for end, _ in desired + if str(end) not in completed + and str(end) not in failed + and str(end) not in in_flight), + # A Kubernetes snapshot only. The caller hands this to the independent + # liveness sampler after every dispatch/progress decision is complete. + '_worker_targets': worker_liveness.targets(job_pods.values()), + } + + +# The driver parses this out of the status ConfigMap -- `end/count`, joined +# with `|pod` in failed_ranges. Changing the shape breaks it silently. +def job_key(end, count): + return f"{end}/{count}" + + +def job_name(end, attempt): + return f"{config.RUN_NAME}-r{end}-a{attempt}" + + +# --- durable progress record ------------------------------------------------ +# Jobs are reclaimed during a long run, so completion cannot live only in Job +# objects. Written before a Job becomes TTL-eligible. + +# Set once at startup; the same ConfigMap the Jobs and PVCs hang off. +_progress_owner = {} + + +def load_progress(): + """The completed/failed record, or empty on a first start. + + The volume is the only source. An unreadable file replays rather than + halts, which is safe -- the PVCs survive and each range resumes at its last + closed ledger. + """ + try: + with open(config.PROGRESS_FILE) as fh: + return json.load(fh) + except (OSError, ValueError): + return {} + + +def save_status(snapshot): + """Publish the run's status into the ConfigMap the driver reads. + + The mission driver runs outside the cluster and already has a kube client, + so reading a ConfigMap is simpler and more robust than exposing the monitor + through a Gateway/HTTPRoute just to be polled. + """ + _patch_cm({'status.json': json.dumps(snapshot, separators=(',', ':'))}) + + +def save_progress(progress): + # The monitor's own state, and the only copy. The driver's view of the run + # is status.json in the ConfigMap; this document is not published. + blob = json.dumps(progress, separators=(',', ':')) + records.write_atomic(config.PROGRESS_FILE, blob) + + +def _patch_cm(data): + body = {'data': data} + try: + kube.core_v1.patch_namespaced_config_map(config.PROGRESS_CM, config.NAMESPACE, body) + except ApiException as e: + if e.status != 404: + raise + kube.core_v1.create_namespaced_config_map(config.NAMESPACE, client.V1ConfigMap( + # Owned by the chart's stellar-core ConfigMap, like the Jobs and + # PVCs, so `helm uninstall` reclaims it. + metadata=client.V1ObjectMeta(name=config.PROGRESS_CM, labels={config.LABEL_RUN: config.RUN_NAME}, + owner_references=_progress_owner.get('ref')), + data=body['data'])) + + +# --- worker log capture ----------------------------------------------------- + +def backstop_save_pod_log(pod_name, end, attempt): + """Last-resort archive for a range the collector never captured. + + Covers only the gap where a pod lived and died while the collector was down, + detected by the absence of the .state file it writes when it claims a range. + Never overwrites a claimed or existing archive: two writers appending to one + gzip interleave members and duplicate lines. + """ + if os.path.exists(records.state_path(end, attempt)): + return True # collector has it (streaming or already finished) + path = records.log_path(end, attempt) + if os.path.exists(path): + return True + try: + body = kube.core_v1.read_namespaced_pod_log(pod_name, config.NAMESPACE, container='stellar-core') + except ApiException as e: + logger.warning("could not save log for range %s attempt %d (pod %s): %s", + end, attempt, pod_name, e.reason) + return False + try: + os.makedirs(config.LOG_DIR, exist_ok=True) + records.write_atomic(path, body, gzip.open) + return True + except OSError as e: + logger.warning("could not write %s: %s", path, e) + return False + + +def record_range_start(end, job): + """Persist attempt 1's Job creationTimestamp, once. + + Not status.startTime: the controller sets that asynchronously, so it is + absent from the create response. The gap between the two is what + wallSeconds measures. Written at creation because attempt 1's Job is gone + on the first retry. + """ + path = records.started_path(end) + if os.path.exists(path): + return + created = job.metadata.creation_timestamp if job and job.metadata else None + if created is None: + return + try: + records.write_atomic(path, created.isoformat()) + except OSError as e: + logger.warning("could not persist start time for range %s: %s", end, e) + + +def range_started_at(end): + """attempt 1's Job creationTimestamp, or None if it was never recorded.""" + try: + with open(records.started_path(end)) as fh: + return datetime.fromisoformat(fh.read().strip()) + except (OSError, ValueError): + return None + + +def classify(pod): + """Why did this pod fail? The Job object cannot answer this. + + Job.status only carries a Failed condition with reason BackoffLimitExceeded + -- no exit code, no OOM. The detail lives on the pod, which is exactly the + object Karpenter deletes with the node, so this is recorded the moment the + watch sees it rather than when reconcile next runs. + """ + for cond in (pod.status.conditions or []): + if cond.type == 'DisruptionTarget' and cond.status == 'True': + return {'outcome': 'disrupted', 'exitCode': None} + # Kubelet can reject a pod before any container runs, e.g. + # VolumeAttachmentLimitExceeded. No exit code and no DisruptionTarget, so + # without this a transient admission rejection reads as a real failure. + if pod.status.reason == 'Evicted' and 'ephemeral' in (pod.status.message or ''): + # A limit eviction sets no DisruptionTarget, and stellar-core drains to + # exit 3, so the Job condition reads it as a catchup failure. + # status.message is the only discriminator, and only the pod carries it. + return {'outcome': 'ephemeral', 'exitCode': None, 'reason': pod.status.message} + if pod.status.reason in ('VolumeAttachmentLimitExceeded', 'OutOfcpu', 'OutOfmemory', + 'OutOfpods', 'UnexpectedAdmissionError', 'NodeAffinity', + 'Shutdown', 'Evicted'): + return {'outcome': 'rejected', 'exitCode': None, 'reason': pod.status.reason} + if pod.status.reason == 'DeadlineExceeded': + # The deadline is on the PodSpec, so the kubelet fires it and the pod + # carries the reason; the Job sees only a non-zero exit. + return {'outcome': 'timeout', 'exitCode': None, 'reason': pod.status.reason} + started = any(cs.state and cs.state.terminated for cs in (pod.status.container_statuses or [])) + if not started: + # No container ever reached a terminal state: nothing ran, so this is + # not evidence about the ledger range. + return {'outcome': 'rejected', 'exitCode': None, + 'reason': pod.status.reason or 'no container status'} + for cs in (pod.status.container_statuses or []): + t = cs.state.terminated if cs.state else None + if t is None: + continue + # 137 is SIGKILL, which the kubelet also uses for a graceful-stop + # timeout -- but with reason OOMKilled it is unambiguous. + if t.reason == 'OOMKilled': + return {'outcome': 'oom', 'exitCode': t.exit_code} + if t.exit_code not in (0, None): + return {'outcome': 'failed', 'exitCode': t.exit_code} + return {'outcome': 'failed', 'exitCode': None} + + +def record_outcome(end, attempt, pod): + path = records.outcome_path(end, attempt) + if os.path.exists(path): + return + data = classify(pod) + data['pod'] = pod.metadata.name + # The only place a failed attempt's duration is available: reconcile + # computes `seconds` on the success path only, and the pod is about to go. + data['attemptSeconds'] = _pod_seconds(pod) + try: + records.write_atomic(path, json.dumps(data)) + except OSError as e: + logger.warning("could not persist outcome for range %s: %s", end, e) + + +# The Job controller writes the exit code and pod name into the failure +# condition message, e.g. +# "Container stellar-core for pod ns/kic-r400000-a1-xxxxx failed with exit +# code 137 matching FailJob rule at index 1" +# Unlike the pod, this survives node consolidation. +_JOB_MSG = re.compile(r"for pod \S+?/(?P\S+) failed with exit code (?P\d+)") +_JOB_RULE = re.compile(r"rule at index (?P\d+)") + + +def _failure_rules(): + """podFailurePolicy rules, in evaluation order, tagged with what they mean. + + First match wins, so reaching the exit-137 rule proves DisruptionTarget did + not match -- that ordering is what separates an OOM kill from a + grace-period SIGKILL after the pod is gone. + + All FailJob: the Job must fail with reason=PodFailurePolicy so the message + names the rule index. A Count action would surface as BackoffLimitExceeded + and lose the signal. Retries stay with the monitor because raising a memory + limit needs a new Job -- spec.template is immutable. + """ + return [ + ('disrupted', client.V1PodFailurePolicyRule( + action='FailJob', + on_pod_conditions=[client.V1PodFailurePolicyOnPodConditionsPattern( + type='DisruptionTarget', status='True')])), + ('oom', client.V1PodFailurePolicyRule( + action='FailJob', + on_exit_codes=client.V1PodFailurePolicyOnExitCodesRequirement( + container_name='stellar-core', operator='In', values=[137]))), + ('failed', client.V1PodFailurePolicyRule( + action='FailJob', + on_exit_codes=client.V1PodFailurePolicyOnExitCodesRequirement( + container_name='stellar-core', operator='NotIn', values=[0]))), + ] + + +# Order here is the contract with the Job controller's "rule at index N". +RULE_ORDER = ['disrupted', 'oom', 'failed'] +_RULE_OUTCOME = dict(enumerate(RULE_ORDER)) + + +def classify_from_job(job): + """Recover a verdict from the Job when the pod is already gone. + + Rule index is the signal, not the exit code: rules are evaluated + first-match-wins, so reaching the exit-137 rule proves the DisruptionTarget + rule did not match, which is the only way to tell an OOM kill from a + grace-period SIGKILL once the pod is gone. + + Index and exit code are parsed independently -- a rule matching on + onPodConditions reports no exit code at all, so requiring one would make the + disruption case unreadable. + """ + for cond in (job.status.conditions or []): + if cond.type != 'Failed' or cond.status != 'True': + continue + msg = cond.message or '' + if cond.reason == 'DeadlineExceeded': + # activeDeadlineSeconds fired: the attempt hung rather than failing. + # Retryable -- a genuinely stuck range will exhaust its attempts. + return {'outcome': 'timeout', 'exitCode': None, 'pod': '', + 'source': 'job-condition'} + if cond.reason != 'PodFailurePolicy': + # e.g. BackoffLimitExceeded -- carries no per-rule detail. + continue + rule = _JOB_RULE.search(msg) + detail = _JOB_MSG.search(msg) + outcome = _RULE_OUTCOME.get(int(rule.group('idx'))) if rule else None + code = int(detail.group('code')) if detail else None + if outcome is None: + if code is None: + return None + # No usable rule index. A drained stellar-core exits 3, not 137, so + # a bare 137 is an OOM and a bare 3 without DisruptionTarget is a + # catchup failure. + outcome = 'oom' if code == 137 else 'failed' + return {'outcome': outcome, 'exitCode': code, + 'pod': detail.group('pod') if detail else '', + 'source': 'job-condition'} + return None + + +def save_verdict(end, attempt, outcome): + """Persist the EFFECTIVE verdict for one attempt, so budgets can be tallied. + + The .outcome file is not enough on its own: it is classified from the pod, + and a deadline kill reads as a plain exit-3 `failed` there -- only the Job's + DeadlineExceeded condition says `timeout`. Reconcile resolves that conflict + once, and this is where the answer is kept, on the same durable logs volume + as everything else, so a monitor restart does not reset a range's budgets. + """ + path = records.verdict_path(end, attempt) + try: + records.write_atomic(path, str(outcome)) + except OSError as e: + logger.warning("could not persist verdict for range %s attempt %s: %s", + end, attempt, e) + + +# --- tx_apply --------------------------------------------------------------- + + +def _pod_seconds(pod): + """Container start -> finish for one attempt, or None if unreadable.""" + start = pod.status.start_time if pod.status else None + if start is None: + return None + for cs in (pod.status.container_statuses or []): + t = cs.state.terminated if cs.state else None + if t is not None and t.finished_at: + return (t.finished_at - start).total_seconds() + return None + + +# --- job construction ------------------------------------------------------- + +# Resume decision, before catchup. Skip new-db only when the DB on /data belongs +# to this range AND replay had started: bucket apply assumes a fresh DB, so a +# crash during it must start over, and "Ledger close complete" is the +# discriminator -- bucket apply never closes a ledger. +# +# The LCL comes from stellar-core's own log, not the database: core 27 dropped +# the ledgerheaders table. +RESUME_SCRIPT = r'''set -e +KEY="%(key)s" +TARGET=%(target)d +COUNT=%(count)d +MARK=/data/.job-key +RESUME=false +LCL="" +if [ -f "$MARK" ] && [ "$(cat "$MARK" 2>/dev/null)" = "$KEY" ]; then + # Ask core for its own LCL through its own accessor, so this survives the v27 + # schema change and any log level. Safe because core has not started, so + # nothing holds /data/buckets/stellar-core.lock. Core logs to the console + # alongside the JSON, hence grepping rather than parsing. + # One "num" key in the document and it is the ledger's. Do NOT window with + # `grep -A '"ledger":'`: bucketlist puts ~40 lines of hashes in between. + LCL=$(/usr/bin/stellar-core --conf /config/stellar-core.cfg offline-info --console 2>/dev/null \ + | sed -n 's/.*"num"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/p' | head -1 || true) + if [ -n "$LCL" ]; then + echo "RESUME PROBE: offline-info reports lcl $LCL" + else + # Fallback: the previous incarnation's log on /data. Goes blind above INFO, + # which is why it is no longer the primary probe. + PREV_LOG=$(ls -t /data/stellar-core*.log 2>/dev/null | head -n 1 || true) + if [ -n "$PREV_LOG" ]; then + LCL=$(grep -oE "Ledger close complete: [0-9]+" "$PREV_LOG" 2>/dev/null | tail -1 | grep -oE "[0-9]+$" || true) + fi + echo "RESUME PROBE: offline-info gave nothing; log fallback says '${LCL:-none}'" + fi + # Already at the target: replay finished and the attempt was evicted before it + # could exit 0. Re-running catchup applies nothing and exits 2 identically + # every time, so the range would burn its whole budget over completed work. + if [ -n "$LCL" ] && [ "$LCL" -ge "$TARGET" ] 2>/dev/null; then + echo "ALREADY COMPLETE: $KEY reached ledger $LCL >= target $TARGET; nothing left to replay" + exit 0 + fi + if [ -n "$LCL" ] && [ "$LCL" -ge $((TARGET - COUNT)) ] && [ "$LCL" -lt "$TARGET" ] 2>/dev/null; then + RESUME=true; echo "RESUME: $KEY reached ledger $LCL, replay had started; skipping new-db" + else + echo "RESUME DECLINED: $KEY last close was '${LCL:-none}' (need >= $((TARGET - COUNT))); bucket phase incomplete, starting fresh" + fi +fi +printf '%%s' "$KEY" > "$MARK" +if [ "$RESUME" != "true" ]; then + /usr/bin/stellar-core --conf /config/stellar-core.cfg new-db --console +fi +exec /usr/bin/stellar-core --conf /config/stellar-core.cfg catchup "$KEY" \ + --metric 'ledger.transaction.apply' --console +''' + + +def owner_ref(): + cm = kube.core_v1.read_namespaced_config_map(f"{config.RUN_NAME}-stellar-core-config", config.NAMESPACE) + return [client.V1OwnerReference(api_version='v1', kind='ConfigMap', + name=cm.metadata.name, uid=cm.metadata.uid, + block_owner_deletion=True)] + + +def release_pvc(end): + """Drop a completed range's volume. + + The PVC exists so an interrupted range resumes at L+1; a succeeded range has + nothing to resume. They are owner-referenced to the release, so without this + every volume survives until `helm uninstall`. + + Best-effort: a failure costs disk, never correctness. + """ + if config.STORAGE_MODE != 'pvc': + return + name = f"{config.RUN_NAME}-data-r{end}" + try: + kube.core_v1.delete_namespaced_persistent_volume_claim(name, config.NAMESPACE) + metrics.pvc_released.inc() + except ApiException as e: + if e.status != 404: + logger.warning("could not release PVC for completed range %s: %s", end, e) + + +def _attempt_finalized(end, attempt): + """Has the collector written everything it will for this attempt? + + It writes this file last, after .metrics. Anything inferred instead -- peaks + being present, tx_apply being readable -- is a guess: tx_apply falls back to + the archive so it is available long before the collector finishes, and an + attempt can legitimately finalize with no peaks at all. + """ + return os.path.exists(records.done_path(end, attempt)) + + +def reap_range_jobs(end): + """Delete every Job this range has, not just the attempt that won. + + Completion is terminal for the RANGE. An attempt-scoped reap leaves an + older Failed Job standing -- the common case is an attempt lost to node + disruption whose collector died with the node, so it was never finalized + and was deliberately not deleted. Once the winner's Job is gone, that + leftover is the range's highest live attempt, and the next pass feeds it + straight into the retry decision and re-runs an already-recorded range + against a freshly recreated, empty PVC. + """ + try: + jobs = kube.batch_v1.list_namespaced_job( + config.NAMESPACE, + label_selector=f"{config.LABEL_RUN}={config.RUN_NAME},{config.LABEL_RANGE}={end}").items + except ApiException as e: + logger.warning("could not list jobs for completed range %s: %s", end, e) + return + for j in jobs: + try: + kube.batch_v1.delete_namespaced_job(j.metadata.name, config.NAMESPACE, + propagation_policy='Background') + metrics.jobs_reaped.inc() + except ApiException as e: + if e.status != 404: + logger.warning("could not delete finished job %s for range %s: %s", + j.metadata.name, end, e) + + +def delete_job(end, attempt): + """Drop a finished Job once nothing more is owed by it. + + reconcile() lists every Job and Pod each pass, so a lingering finished Job + inflates both LIST calls. Background propagation is what takes the pod with + it; orphan would leave the next pass listing just as much. + + Callers must have persisted what they need first. Best-effort: a 404 is the + race with the TTL controller, and raising would strand every other range in + the pass -- JOB_TTL_SECONDS still reclaims the object. + """ + try: + kube.batch_v1.delete_namespaced_job(job_name(end, attempt), config.NAMESPACE, + propagation_policy='Background') + metrics.jobs_reaped.inc() + except ApiException as e: + if e.status != 404: + logger.warning("could not delete finished job for range %s attempt %d: %s", + end, attempt, e) + + +def ensure_pvc(end, owner): + name = f"{config.RUN_NAME}-data-r{end}" + try: + kube.core_v1.read_namespaced_persistent_volume_claim(name, config.NAMESPACE) + return name + except ApiException as e: + if e.status != 404: + raise + spec = client.V1PersistentVolumeClaimSpec( + access_modes=['ReadWriteOnce'], + resources=client.V1VolumeResourceRequirements(requests={'storage': config.STORAGE_SIZE})) + if config.STORAGE_CLASS: + spec.storage_class_name = config.STORAGE_CLASS + kube.core_v1.create_namespaced_persistent_volume_claim(config.NAMESPACE, client.V1PersistentVolumeClaim( + metadata=client.V1ObjectMeta(name=name, owner_references=owner, + labels={config.LABEL_RUN: config.RUN_NAME, config.LABEL_RANGE: str(end)}), + spec=spec)) + return name + + +def _resources(mem=None, eph=None, end=None, attempt=1): + # Before mem is defaulted below -- reading it afterwards can never see None, + # which silently disabled profile sizing entirely. + overrides = sizing._profile_overrides(end, escalated=(mem is not None or eph is not None), + attempt=attempt) + # `mem` is the escalated request on an OOM retry, else the configured one. + req = {'cpu': config.REQ_CPU, 'memory': mem or config.REQ_MEM} + # Only ephemeral-storage is limited: it is the one dimension where an + # unbounded pod takes the node down rather than itself. + lim = {} + + # Only meaningful in ephemeral mode. In PVC mode a large request makes disk + # the binding dimension and halves workers-per-node for no reason. + if config.REQ_EPHEMERAL: + # Raise the request with the limit: ephemeral-storage is a scheduling + # dimension, so a pod that outgrew it no longer fits where it was. + req['ephemeral-storage'] = eph or config.REQ_EPHEMERAL + else: + # pvc mode: /data is not on the node disk, so an ephemeral override + # would size a dimension this run does not use. + overrides.pop('ephemeral-storage', None) + if config.LIM_EPHEMERAL: + lim['ephemeral-storage'] = eph or config.LIM_EPHEMERAL + + # The profile moves requests only. Disk excepted, because its limit is what + # the kubelet enforces. + for key, value in overrides.items(): + req[key] = value + if key == 'ephemeral-storage' and config.LIM_EPHEMERAL: + lim[key] = value + # Unmeasured range: the configured requests, exactly as if there were no + # profile at all. + return client.V1ResourceRequirements(requests=req, limits=lim or None) + + +def volume_spread_constraints(): + """Keep PVC-mounting workers under the per-node EBS attachment limit. + + Only in pvc mode: in ephemeral mode /data is an emptyDir, no volume is + attached, and spreading would just cost density. + """ + if config.STORAGE_MODE != 'pvc' or config.MAX_VOLUMES_PER_NODE <= 0: + return None + min_domains = max(1, -(-config.PARALLELISM // config.MAX_VOLUMES_PER_NODE)) # ceil + return [client.V1TopologySpreadConstraint( + max_skew=config.MAX_VOLUMES_PER_NODE, + min_domains=min_domains, + topology_key='kubernetes.io/hostname', + when_unsatisfiable='DoNotSchedule', + label_selector=client.V1LabelSelector(match_labels={config.LABEL_RUN: config.RUN_NAME}))] + + +def pod_labels(end, attempt): + """Labels on the worker POD, which are not the Job's. + + LABEL_ATTEMPT has to be here too: the collector reads it off the pod to pick + which range--a.* files the attempt owns, defaulting to "1". On the + Job alone, every retry overwrites attempt 1's peaks instead of being maxed + against them. + """ + labels = {config.LABEL_RUN: config.RUN_NAME, config.LABEL_RANGE: str(end), + config.LABEL_ATTEMPT: str(attempt)} + if config.EMIT_MISSION_LABEL and config.MISSION: + labels['mission'] = config.MISSION + return labels + + +def _prestop_delay(): + """A preStop that stalls the kubelet, or None when the knob is off. + + `sleep` from the image rather than `sh -c sleep`: one less process to exist + in a container that is being torn down, and it fails loudly at hook-exec + time if the binary is missing rather than silently succeeding. + + Refuses to install a hook that cannot finish inside the grace period. A + preStop longer than the grace is worse than none: the kubelet kills it + mid-sleep, reports FailedPreStopHook, and the container is signalled + anyway -- so the delay is not bought and an error is logged for every + evicted pod. + """ + if config.WORKER_PRESTOP_SLEEP_SECONDS <= 0: + return None + if config.WORKER_PRESTOP_SLEEP_SECONDS >= config.WORKER_GRACE_SECONDS: + logger.warning( + "PRESTOP_SLEEP_SECONDS=%s does not fit in GRACE_SECONDS=%s; " + "not installing a preStop hook that the kubelet would kill", + config.WORKER_PRESTOP_SLEEP_SECONDS, config.WORKER_GRACE_SECONDS) + return None + return client.V1Lifecycle( + pre_stop=client.V1LifecycleHandler( + _exec=client.V1ExecAction( + command=['/bin/sleep', str(config.WORKER_PRESTOP_SLEEP_SECONDS)]))) + + +def build_job(end, count, attempt, owner, mem=None, eph=None): + key = job_key(end, count) + script = RESUME_SCRIPT % {'key': key, 'target': end, 'count': count} + + if config.STORAGE_MODE == 'pvc': + data_vol = client.V1Volume(name='data', persistent_volume_claim=( + client.V1PersistentVolumeClaimVolumeSource(claim_name=ensure_pvc(end, owner)))) + else: + data_vol = client.V1Volume(name='data', empty_dir=client.V1EmptyDirVolumeSource()) + + env = [client.V1EnvVar(name='ASAN_OPTIONS', value=config.ASAN_OPTIONS)] if config.ASAN_OPTIONS else [] + command = ['/bin/sh', '-c', script] + volumes = [data_vol, client.V1Volume( + name='config', config_map=client.V1ConfigMapVolumeSource( + name=f"{config.RUN_NAME}-stellar-core-config"))] + volume_mounts = [ + client.V1VolumeMount(name='data', mount_path='/data'), + client.V1VolumeMount(name='config', mount_path='/config')] + + # Require and avoid go in ONE matchExpressions list: expressions within a + # term are ANDed, separate terms are ORed, and an avoid-only pod in its own + # term would match every node. + match = [] + if config.NODE_LABEL_KEY: + # Pooled runs route per range: the label names the tier this range's + # memory puts it in. An escalated attempt resolves to a promoted tier, + # which is what moves the pod to nodes its memory fits. + tier = sizing.pool_for(end, attempt) + value = f"{config.POOL_PREFIX}-{tier}" if tier else config.NODE_LABEL_VALUE + match.append(client.V1NodeSelectorRequirement( + key=config.NODE_LABEL_KEY, operator='In', values=[value])) + if config.CAPACITY_TYPE: + # Capacity type is a NodePool property a pod cannot otherwise express, + # and Karpenter labels every node with it. ANDing it here keeps a + # pvc-mode run off on-demand nodes and vice versa. + match.append(client.V1NodeSelectorRequirement( + key='karpenter.sh/capacity-type', operator='In', values=[config.CAPACITY_TYPE])) + if config.AVOID_NODE_LABEL_KEY: + # No value means "avoid the label however it is set", which is + # DoesNotExist; NotIn [""] would only exclude the empty value. + match.append(client.V1NodeSelectorRequirement( + key=config.AVOID_NODE_LABEL_KEY, + operator='NotIn' if config.AVOID_NODE_LABEL_VALUE else 'DoesNotExist', + values=[config.AVOID_NODE_LABEL_VALUE] if config.AVOID_NODE_LABEL_VALUE else None)) + affinity = None + if match: + affinity = client.V1Affinity(node_affinity=client.V1NodeAffinity( + required_during_scheduling_ignored_during_execution=client.V1NodeSelector( + node_selector_terms=[client.V1NodeSelectorTerm(match_expressions=match)]))) + + # Taint value must be absent: the mission emits {key, effect} with no value, + # and the default Equal operator does not match "" against "true". + tolerations = [client.V1Toleration(key=config.TOLERATE_TAINT, effect='NoSchedule')] if config.TOLERATE_TAINT else None + + container = client.V1Container( + name='stellar-core', image=config.CORE_IMAGE, + command=command, env=env, resources=_resources(mem, eph, end, attempt), + ports=[client.V1ContainerPort(container_port=11626, name='http')], + lifecycle=_prestop_delay(), + volume_mounts=volume_mounts) + + return client.V1Job( + metadata=client.V1ObjectMeta( + name=job_name(end, attempt), owner_references=owner, + labels={config.LABEL_RUN: config.RUN_NAME, config.LABEL_RANGE: str(end), + config.LABEL_ATTEMPT: str(attempt)}), + spec=client.V1JobSpec( + # The monitor owns retries, not the Job controller: backoffLimit 0 + # means the Job fails once and stays put, so reconcile classifies the + # failure and decides on attempt N+1. + # + # On the JobSpec, not the pod: a pod-level deadline is immutable once + # the pod exists, so a mis-set value could not be corrected on a live + # run. + active_deadline_seconds=config.ATTEMPT_DEADLINE_SECONDS or None, + backoff_limit=0, + pod_failure_policy=client.V1PodFailurePolicy( + rules=[r for _, r in _failure_rules()]), + ttl_seconds_after_finished=config.JOB_TTL_SECONDS, + template=client.V1PodTemplateSpec( + metadata=client.V1ObjectMeta(labels=pod_labels(end, attempt)), + spec=client.V1PodSpec( + # On the POD, not the JobSpec: activeDeadlineSeconds runs + # from the Job's startTime, charging Pending time against a + # budget meant to bound how long the range RUNS. The + # pod-level field starts at container start. + # IRSA for the S3 history mirror; without it workers fall + # back to the public archive, which throttles at 1024. + service_account_name=config.WORKER_SERVICE_ACCOUNT or None, + # Keeps PVC-mounting workers under the per-node EBS + # attachment cap; inert at realistic CPU-bound density. + topology_spread_constraints=volume_spread_constraints(), + # Never restarted in place: the pod stays terminal and + # inspectable for classification and the backstop log read. + restart_policy='Never', + termination_grace_period_seconds=config.WORKER_GRACE_SECONDS, + affinity=affinity, tolerations=tolerations, + containers=[container], + volumes=volumes)))) + + +# --- what a pass decides ---------------------------------------------------- +# Counters, verdicts and retry policy: everything reconcile() calls to turn a +# finished attempt into a decision. + +_ATTEMPT_FILE = re.compile( + r'^range-(?P\d+)-a(?P[1-9]\d*)\.' + r'(?:verdict|outcome|state|metrics|done|log\.gz)$') + + +def _retry_counter_totals(progress, current_attempts=()): + """Reconstruct retry metrics from durable records and observed attempts. + + A verdict says why an attempt ended; it does not say a retry was dispatched. + Attempt N therefore contributes to retry totals only when attempt N+1 is + evidenced by progress, a persisted per-attempt file, or the current Job + snapshot. The latter makes a newly-created successor visible before its range + completes, while the durable sources rebuild the same truth after restart. + """ + try: + names = os.listdir(config.LOG_DIR) + except OSError: + names = [] + + max_attempt = {} + terminal = set() + + def remember(end, attempt): + try: + attempt = int(attempt) + except (TypeError, ValueError): + return + if attempt < 1: + return + end = str(end) + max_attempt[end] = max(max_attempt.get(end, 0), attempt) + + if isinstance(progress, dict): + for bucket in ('completed', 'failed'): + bucket_records = progress.get(bucket) + if not isinstance(bucket_records, dict): + continue + for end, record in bucket_records.items(): + if not isinstance(record, dict): + continue + try: + attempt = int(record.get('attempts', 1)) + except (TypeError, ValueError): + continue + if attempt < 1: + continue + remember(end, attempt) + terminal.add((str(end), attempt)) + + for item in current_attempts: + try: + end, attempt = item + except (TypeError, ValueError): + continue + remember(end, attempt) + + verdict_files = set() + outcome_files = set() + for name in names: + match = _ATTEMPT_FILE.match(name) + if not match: + continue + key = (match.group('end'), int(match.group('attempt'))) + remember(*key) + if name.endswith('.verdict'): + verdict_files.add(key) + elif name.endswith('.outcome'): + outcome_files.add(key) + + effective = {} + for end, attempt in verdict_files: + try: + with open(records.verdict_path(end, attempt)) as fh: + verdict = fh.read().strip() + except OSError: + continue + if verdict in config.ATTEMPT_OUTCOMES: + effective[(end, attempt)] = verdict + + # .outcome predates .verdict and is safe only for a completed chain: a + # collector outcome can still be superseded by reconcile's verdict. Any + # verdict file, even malformed, means this is not a legacy attempt. + for end, attempt in outcome_files - verdict_files: + if attempt >= max_attempt.get(end, 0) and (end, attempt) not in terminal: + continue + try: + with open(records.outcome_path(end, attempt)) as fh: + record = json.load(fh) + except (OSError, ValueError): + continue + outcome = record.get('outcome') if isinstance(record, dict) else None + if outcome in config.ATTEMPT_OUTCOMES: + effective[(end, attempt)] = outcome + + retries = sum(max(0, attempt - 1) for attempt in max_attempt.values()) + reasons = {reason: 0 for reason in config.ATTEMPT_OUTCOMES} + for (end, attempt), reason in effective.items(): + if attempt < max_attempt.get(end, 0): + reasons[reason] += 1 + disruption_retried_ranges = { + end for (end, attempt), reason in effective.items() + if reason == 'disrupted' and attempt < max_attempt.get(end, 0) + } + + return { + 'retries': retries, + 'evicted': sum(1 for verdict in effective.values() if verdict == 'disrupted'), + 'spot_disruption_retried': len(disruption_retried_ranges), + 'oom': reasons['oom'], + 'ephemeral': reasons['ephemeral'], + 'reasons': reasons, + } + + +def sync_counters(progress, counted, current_attempts=()): + """Drive the counters from persisted state instead of from events. + + Two reasons not to .inc() as things happen: + + * a terminally-failed range stays the newest Job for its range, so an + event-driven inc fires again on every reconcile until teardown + * the process resets to zero on restart, while verdicts and attempt state on + the PVC survive + + Computing the true total and incrementing by the delta is monotonic, + idempotent, and self-heals after a restart: the counter starts at 0 and the + first sync walks it up to the recorded total. + """ + totals = _retry_counter_totals(progress, current_attempts) + for key, total, metric in (('retries', totals['retries'], metrics.retries), + ('oom', totals['oom'], metrics.oom_retries), + ('ephemeral', totals['ephemeral'], metrics.eph_retries), + ('evicted', totals['evicted'], metrics.evictions), + ('spot_disruption_retried', + totals['spot_disruption_retried'], + metrics.spot_disruption_retried)): + delta = total - counted.get(key, 0) + if delta > 0: + metric.inc(delta) + counted[key] = total + for reason in config.ATTEMPT_OUTCOMES: + metric = metrics.retry_reasons.labels(reason=reason) + key = ('reason', reason) + total = totals['reasons'][reason] + delta = total - counted.get(key, 0) + if delta > 0: + metric.inc(delta) + counted[key] = total + + +def observe_recorded(progress, replayed): + """Feed recorded completions into the histograms. + + Prometheus histograms are append-only and reset to zero when the process + restarts, so replaying every recorded range rebuilds the exact cumulative + total rather than double counting. Guarded per-process by `replayed`. + + Keyed on (range, field), not on the range alone: a range is usually + recorded before the collector has flushed its .metrics, so txApply is null + at first sight and backfilled a pass or two later. Marking the whole range + as replayed on first sight meant that backfill could never be observed, and + the histogram permanently disagreed with progress.json. + """ + for end, rec in progress.get('completed', {}).items(): + # `is not None`, not truthiness: sum = 0ms records txApply 0.0, which is + # a real observation. Same for a sub-second duration. + for field, metric in (('seconds', metrics.full_duration), + ('wallSeconds', metrics.wall_duration), + ('txApply', metrics.tx_apply_duration)): + if (end, field) in replayed: + continue + value = rec.get(field) + if value is None: + continue + replayed.add((end, field)) + metric.observe(value) + + +def _range_wall_seconds(end, status): + """Attempt 1 created -> winner completed, or None if the start was never recorded. + + The range's whole life: every retry, every gap between them, every wait for a + node. Deliberately not falling back to the winning Job's own start -- that + measures one leg and understates exactly the mess this is here to capture. + """ + started = range_started_at(end) + if not started or not status.completion_time: + return None + return (status.completion_time - started).total_seconds() + + +def _range_compute_seconds(end, attempt, pod, wall): + """Compute seconds across the whole resumed chain, not this leg alone. + + A fresh single attempt may fall back to the winner's own seconds or to the + Job wall; a resumed chain is every leg or nothing, never winner-only. + """ + pod_seconds = _pod_seconds(pod) if pod is not None else None + chain = attempts.seconds_for_range(end, attempt, pod_seconds) + if chain is not None: + return chain + if len(attempts._resumed_chain(end, attempt)) == 1: + return pod_seconds if pod_seconds is not None else wall + return None + + +def completion_record(end, attempt, status, pod, count=None): + """What a finished range cost and where it ran. + + Assembled from the winning Job's status, the pod if it still exists, and the + per-attempt files the collector wrote. Every pod-derived field is optional on + purpose: a reaped node costs that field, never the record. + """ + wall = _range_wall_seconds(end, status) + # Not gated on `pod`: the collector's record outlives it, so a reaped node + # must not cost us the metric. + tx = attempts.tx_apply_for_range(end, attempt) + if tx is None: + logger.warning("could not read tx_apply for range %s (pod gone?); " + "metric will be missing for this range", end) + record = {'seconds': _range_compute_seconds(end, attempt, pod, wall), + 'wallSeconds': wall, 'txApply': tx, 'attempts': attempt} + # Ledger count travels with the record: the logarithmic generator varies it + # per range, so it cannot be recomputed from config when the profile is read + # back. + if count is not None: + record['count'] = count + record.update(attempts.peaks_for_range(end, attempt)) + return record + + +# A failed attempt resolves to one of three actions. `reason` names the cause for +# the log line and is None only when the range is condemned outright. +Decision = collections.namedtuple('Decision', 'action reason memory ephemeral') + + +def _retry(reason, memory=None, ephemeral=None): + return Decision('retry', reason, memory, ephemeral) + + +CONDEMN = Decision('condemn', None, None, None) +# Wait for the collector's .done marker and decide on a later pass. +DEFER = Decision('defer', None, None, None) + + +def verdict_for(end, attempt, job, pod): + """Why this attempt failed, from the pod if it survived and the Job if not. + + Two classifications, ranked: + 1. the pod named a mechanism (OOM, DisruptionTarget, eviction, deadline) + -- it wins, being the precise one + 2. else the Job says timeout -- only the Job knows the deadline fired, and + the drained pod reads as a plain `failed` + 3. else whichever exists, unknown over nothing: retry rather than condemn + """ + from_pod = records.read_outcome(end, attempt) + from_job = classify_from_job(job) + if from_pod and from_pod.get('outcome') in config.POD_AUTHORITATIVE_OUTCOMES: + verdict = from_pod + elif from_job and from_job.get('outcome') == 'timeout': + verdict = from_job + else: + verdict = from_pod or from_job or {'outcome': 'unknown', 'exitCode': None} + if verdict.get('source') == 'job-condition': + logger.info("range %s attempt %d classified from Job condition " + "(exit %s); pod was already gone", + end, attempt, verdict.get('exitCode')) + # A third source of evidence for the one ambiguous exit code. Exit 3 is + # "did not complete", which a SIGTERM drain and a real failure share, so the + # archive is what separates them -- and only once the collector has finished + # writing it. Until then the verdict stays `failed` and the decision defers. + if (verdict.get('exitCode') == config.CATCHUP_INCOMPLETE_EXIT + and _attempt_finalized(end, attempt) + and attempts.exit3_retry_cause(end, attempt)): + verdict = dict(verdict, outcome='fetch-fault') + return verdict + + +def _condemn_timeout(end, attempt): + """Terminal: the deadline is the only thing that ends a wedged range. + + A range stuck on an unreachable archive closes no ledgers and never exits, so + retrying spends the deadline again for nothing. + """ + logger.error("!!! RANGE CONDEMNED !!! %s hit its %ss attempt deadline " + "on attempt %s; this fails the mission. Check its archived " + "log for 'maybe stale archive' -- an unreachable history " + "mirror is the usual cause.", + end, config.ATTEMPT_DEADLINE_SECONDS, attempt) + return CONDEMN + + +def _retry_oom(end, attempt): + """Retry with the next memory rung. + + Rungs climbed = OOMs seen, not attempts made; this attempt's outcome is + already on disk, so the count includes it. `had` is what this attempt + actually ran with, by the same derivation that sized it -- indexing on + `attempt` instead names a rung nobody occupied. + """ + base = (sizing._profile_overrides(end, escalated=False) or {}).get('memory') + ooms = records._oom_count(end, attempt) + had = (sizing.pool_memory(sizing.pool_for(end, attempt)) if config.POOL_PREFIX + else sizing.mem_for_attempt(ooms, base)) + return _retry(f"OOM-killed at memory request {had}", + memory=sizing.mem_for_attempt(ooms + 1, base, end=end)) + + +def _retry_ephemeral(end, attempt): + """Retry with the next disk rung. + + Rungs climbed = evictions seen, not attempts made, as with the OOM ladder: + on spot most retries are disruptions. The count includes this attempt. + """ + evictions = records._cause_count(end, attempt, ('ephemeral',)) + had = sizing.eph_for_attempt(evictions) + reason = (f"evicted for exceeding its {had} ephemeral-storage limit" if had + else "evicted under node disk pressure with no configured limit") + return _retry(reason, ephemeral=sizing.eph_for_attempt(evictions + 1)) + + +def _decide_exit3(end, attempt): + """A plain exit 3: the archive named no fetch fault, or has not landed yet. + + verdict_for already promotes an exit 3 to `fetch-fault` once the archive + says so, so reaching here means either the collector is still writing it -- + wait, bounded by JOB_TTL_SECONDS -- or nothing in it explains the failure, in + which case the range is condemned and its archive survives on the volume. + """ + if not _attempt_finalized(end, attempt): + return DEFER + return CONDEMN + + +def retry_decision(verdict, end, attempt): + """Retry this range with what, condemn it, or wait for more evidence.""" + outcome = verdict['outcome'] + if outcome == 'timeout': + return _condemn_timeout(end, attempt) + elif outcome == 'rejected': + # The pod never started, so a retry cannot mask a broken range -- but it + # is the range's own budget now, not the disruption one. + return _retry(f"rejected by the node before starting " + f"({verdict.get('reason', '?')})") + elif outcome == 'disrupted': + return _retry("lost to node disruption") + elif outcome == 'fetch-fault': + return _retry(f"exited {config.CATCHUP_INCOMPLETE_EXIT} after a fetch fault " + f"({attempts.exit3_retry_cause(end, attempt)})") + elif outcome == 'oom': + return _retry_oom(end, attempt) + elif outcome == 'ephemeral': + return _retry_ephemeral(end, attempt) + elif outcome == 'unknown': + # Nothing classified the pod. Without evidence the monitor cannot tell a + # reaped node from a range that really failed, and a run that reports + # success on a range nobody verified is worse than one that stops. + return CONDEMN + elif verdict.get('exitCode') == config.CATCHUP_INCOMPLETE_EXIT: + return _decide_exit3(end, attempt) + elif verdict.get('exitCode') is None: + # The verdict came from the Job condition, which says Failed and nothing + # about why. Same absence of evidence as `unknown`, same answer. + return CONDEMN + else: + return CONDEMN # a genuine catchup failure + + +def budget_for(verdict, end, attempt): + """(spent, cap) for the cause that killed this attempt. + + config.ATTEMPT_BUDGETS is the whole retry policy; a cause with no entry caps + at 0 and is condemned on sight. `spent` counts only THIS cause, so evictions + cannot drain the OOM or disk budgets. This verdict is already on disk, so + the Nth failure is the one that exhausts a budget of N. + """ + outcome = verdict['outcome'] + return (records._cause_count(end, attempt, (outcome,)), + config.ATTEMPT_BUDGETS.get(outcome, 0)) + + +def _log_retry(end, attempt, verdict, decision, cap): + if verdict['outcome'] == 'oom': + logger.error( + "!!! OOM RETRY !!! range %s was OOM-killed on attempt %d/%d; retrying with " + "memory limit %s -- RAISE THE CONFIGURED MEMORY LIMIT, this run is only " + "surviving by escalating at runtime", end, attempt, cap, decision.memory) + elif verdict['outcome'] == 'ephemeral': + logger.error( + "!!! DISK RETRY !!! range %s %s on attempt %d/%d; retrying with " + "ephemeral-storage %s -- RAISE THE CONFIGURED EPHEMERAL STORAGE, this " + "run is only surviving by escalating at runtime", + end, decision.reason, attempt, cap, decision.ephemeral) + else: + logger.warning("range %s %s on attempt %d/%d; retrying", + end, decision.reason, attempt, cap) + + +def pods_by_job(): + """One list per reconcile, indexed by Job name. + """ + out = {} + for p in kube.core_v1.list_namespaced_pod( + config.NAMESPACE, label_selector=f"{config.LABEL_RUN}={config.RUN_NAME}").items: + jn = (p.metadata.labels or {}).get('batch.kubernetes.io/job-name') + if jn: + out.setdefault(jn, p) + return out + + +def read_mission_start(): + """When this run first started, or None if not recorded yet. + + Its own ConfigMap key: progress.json is keyed by ledger range, and anything + else in it would be walked as one. Read-only -- creating the ConfigMap here + would race the owner reference, and an ownerless one survives + `helm uninstall`. + """ + try: + cm = kube.core_v1.read_namespaced_config_map(config.PROGRESS_CM, config.NAMESPACE) + return float((cm.data or {})['started_at']) + except (ApiException, KeyError, TypeError, ValueError): + return None + + +if __name__ == '__main__': + main() diff --git a/src/MissionParallelCatchup/log_collector.py b/src/MissionParallelCatchup/apps/log_collector.py similarity index 60% rename from src/MissionParallelCatchup/log_collector.py rename to src/MissionParallelCatchup/apps/log_collector.py index 45c00a52..ee9de76c 100644 --- a/src/MissionParallelCatchup/log_collector.py +++ b/src/MissionParallelCatchup/apps/log_collector.py @@ -2,13 +2,13 @@ Runs as a sidecar next to job_monitor, sharing its /logs volume. -Why stream rather than read logs after a Job finishes: worker pods are one per -ledger range, and Karpenter deletes the node roughly a minute after its last -running pod exits, taking every pod object with it. Anything that reads after -the fact is racing that deletion. Holding `follow=true` from pod start means we -already have everything the pod wrote by the time it disappears -- and it makes -a straggler's log readable *while* it is stuck, which is the case that turns a -5h run into a 10h one. +Why not read logs after a Job finishes: worker pods are one per ledger range, +and Karpenter deletes the node roughly a minute after its last running pod +exits, taking every pod object with it. Anything that reads after the fact is +racing that deletion. Polling each pod's log on an interval also keeps a +straggler readable *while* it is stuck, which is the case that turns a 5h run +into a 10h one; a condemned pod gets a follow stream so its last lines land +before it goes. Resume is idempotent across both a dropped stream and a restart of this process: @@ -33,33 +33,23 @@ import logging import os import re -import signal import ssl import sys import zlib from datetime import datetime import aiohttp +from logger import build_logger +import config +import medida +import records -NAMESPACE = os.getenv('NAMESPACE', 'default') -RUN_NAME = os.getenv('RUN_NAME', 'parallel-catchup') -LOG_DIR = os.getenv('LOG_DIR', '/logs') CONTAINER = os.getenv('WORKER_CONTAINER', 'stellar-core') POLL_SECONDS = float(os.getenv('COLLECTOR_POLL_SECONDS', 5)) # Poll cycles a stream gets to finalize itself after its pod leaves the pod list # before it is cancelled outright. One cycle is usually enough; the margin is for # a stream still finalizing: writing its .metrics and closing its archive. VANISHED_GRACE_CYCLES = int(os.getenv('COLLECTOR_VANISHED_GRACE_CYCLES', 3)) -# Whether to keep the archive for a range that succeeded. Enforced here rather -# than in job_monitor: we cannot know in advance whether a range will fail, so -# the stream always runs and the archive is discarded on success instead. -SAVE_SUCCESS_LOGS = os.getenv('SAVE_SUCCESS_LOGS', 'true').lower() == 'true' -# Peak working set per range, for sizing a later run's requests. Empty disables. -# Queried rather than sampled: cgroup memory.peak counts page cache (measured: -# 1.5GB peak for a process using 0.3MB of anon), and a sampler inside the worker -# would mean dropping the `exec`, which is what keeps stellar-core at PID 1 and -# able to see SIGTERM. -STORAGE_MODE = os.getenv('STORAGE_MODE', 'pvc') # Peak memory now comes from kubelet, not Prometheus. kubelet reports rssBytes # and workingSetBytes per container in the same /stats/summary payload this # already fetches for ephemeral storage, at ~10s cAdvisor housekeeping against a @@ -90,6 +80,8 @@ # than among the peak dicts, where it landed inside the region the scanner tests # exec and broke six of them on an asyncio NameError. _poll_slots = asyncio.Semaphore(MAX_CONCURRENT_POLLS) +# Separate from _poll_slots on purpose -- see MAX_DOOMED_FOLLOWS. +_follow_slots = asyncio.Semaphore(int(os.getenv('MAX_DOOMED_FOLLOWS', 256))) # pod name -> Event, set by the main loop the moment it first observes the pod # terminal. poll_pod waits on it instead of sleeping blind, so the final read # happens within the pod-list cadence rather than up to LOG_POLL_SECONDS later. @@ -98,6 +90,69 @@ _wake = {} # pod name -> its own start->finish, read off the pod while it still exists. _pod_secs = {} +# pod name -> status.startTime, kept so an attempt whose object vanished before +# any cycle saw its terminated timestamp can still be dated from the container's +# own start rather than from whenever this poller happened to open. +_pod_start = {} +# Pods carrying a DisruptionTarget condition: the cluster has committed to +# destroying them and, on spot, gives about two minutes' notice. +# +# Waking the poller is not enough on its own. stellar-core prints its medida +# block ~4ms after SIGTERM and the pod object is deleted seconds later, so an +# interval poll straddles the whole thing -- measured on the 2048-worker run, +# 810 evictions lost 809 txApply values and 790 exact durations. A held +# connection already has those bytes when the process dies. +# +# Safe here precisely because it is scoped and short-lived, not because the +# count is small: global follow=true cost the sidecar 1444 MiB of a 2048 MiB +# limit at 2096 streams held for whole ranges, where these are held only for +# the drain. See MAX_DOOMED_FOLLOWS for the sizing. +_doomed = {} +# Longest a follow stream will hang on to a doomed pod. Spot gives 120s; past +# roughly double that the notice was withdrawn (Karpenter cancelled the drain) +# and the stream would otherwise be held for the life of the range. +# 0 disables the follow path entirely and leaves interval polling to do it, +# which is only safe when prestopSleepSeconds is holding the pod open instead. +DOOMED_FOLLOW_SECONDS = float(os.getenv('DOOMED_FOLLOW_SECONDS', 300)) +# Follows get their OWN budget rather than sharing _poll_slots. +# +# Sharing was a starvation bug: a follow holds its slot for the whole drain, so +# a reclaim condemning more pods than there are slots would consume all of them +# and stop every other pod in the run from being polled at all -- turning a +# partial node loss into a run-wide blackout. With a separate budget the worst +# case is that some condemned pods fall back to polling, which is a proven path +# rather than a degradation -- 1s polling captured txApply on its own with no +# follow and no preStop. +# +# Sized well above any plausible simultaneous disruption rather than at the +# measured one. A whole-AZ spot reclaim is not bounded by Karpenter's +# disruption budget, so the ~43 pods a 10% budget implies is a floor, not a +# ceiling. The old always-on design sustained 2096 concurrent streams, and it +# paid far more per stream than this does: it held a persistent GzipFile and +# aiohttp buffers for a pod's entire multi-hour life, where _follow_tail builds +# a fresh gzip member per flush, keeps nothing between them, and lives for the +# 10-120s of a drain. 256 x the old 0.69 MiB upper bound is 177 MiB against a +# 2048 MiB limit, and the true figure is lower. +MAX_DOOMED_FOLLOWS = int(os.getenv('MAX_DOOMED_FOLLOWS', 256)) +# Poll interval for a condemned pod, replacing LOG_POLL_SECONDS for as long as +# it is doomed. This is the cheap half of the fix and the one that does the +# work: measured on ssc-test, preStop delays SIGTERM but leaves the gap between +# the medida block and the pod object being deleted at ~9s, so a blind 10s poll +# straddles it -- which it did, losing txApply even with a 60s preStop holding +# the pod open. Polling that same window every second cannot miss it. +# +# Costs no held connections, unlike a follow stream: ~120 short requests over a +# 2-minute drain per condemned pod, bounded by the existing _poll_slots. +# sinceTime has 1s granularity, so going below 1s only re-reads the same second. +DOOMED_POLL_SECONDS = float(os.getenv('DOOMED_POLL_SECONDS', 1)) +# How long each watch connection is allowed to live before the apiserver closes +# it and we reconnect. Bounded rather than infinite so a silently-dead stream +# self-heals; the reconnect resumes from the last resourceVersion, so nothing is +# missed across it. 0 disables the watch and leaves detection to the pod list. +WATCH_TIMEOUT_SECONDS = int(os.getenv('WATCH_TIMEOUT_SECONDS', 600)) +# Pause before re-opening a watch that failed. Only covers hard errors: a clean +# timeout reconnects immediately. +WATCH_RETRY_SECONDS = float(os.getenv('WATCH_RETRY_SECONDS', 1)) # Fields that only ever grow. write_metrics maxes these instead of overwriting, # so a restarted poller starting its high-water at zero cannot lower one. PEAK_KEYS = ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes') @@ -112,17 +167,14 @@ # because that is where a pod's final output lives. POLLABLE_PHASES = ('Running', 'Succeeded', 'Failed') -LABEL_RUN = 'catchup.stellar.org/run' -LABEL_RANGE = 'catchup.stellar.org/range-end' -LABEL_ATTEMPT = 'catchup.stellar.org/attempt' SA = '/var/run/secrets/kubernetes.io/serviceaccount' API = f"https://{os.getenv('KUBERNETES_SERVICE_HOST', 'kubernetes.default')}:{os.getenv('KUBERNETES_SERVICE_PORT', '443')}" +# Read-only kubelet port. A seam for tests, which serve the payload on a +# loopback port rather than reaching a real node. +KUBELET_PORT = 10250 -logging.basicConfig(level=os.getenv('LOGGING_LEVEL', 'INFO'), - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[logging.StreamHandler(sys.stdout)]) -logger = logging.getLogger('log-collector') +logger = build_logger('log_collector', name='log-collector', to_file=False) def token(): @@ -137,7 +189,65 @@ def ssl_ctx(): def base(end, attempt): - return os.path.join(LOG_DIR, f"range-{end}-a{attempt}") + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}") + + +def _is_condemned(pod): + """The DisruptionTarget reason if the cluster has committed to destroying + this pod, else None. + + DisruptionTarget covers the cases that cost us measurements: a spot reclaim, + a Karpenter drain, and node pressure. It does NOT cover a kubelet + ephemeral-storage eviction -- classify() handles that one from + status.message -- and it is deliberately not inferred from a deletionTimestamp, + which is also set by the monitor reaping a Job that already finished. + + The reason separates a warning from a postmortem, which the bare condition + cannot: EvictionByEvictionAPI is a drain that still has to deliver SIGTERM, + while DeletionByTaintManager is stamped ~40s after the node went NotReady, + on a container that already died unsignalled. In the second case no medida + block was ever written, so a missing txApply is not a capture race. + """ + for cond in ((pod.get('status') or {}).get('conditions') or []): + if cond.get('type') == 'DisruptionTarget' and cond.get('status') == 'True': + return cond.get('reason') or 'Unknown' + return None + + +def _mark_condemned(pod, name, end, attempt): + """Flag a condemned pod so its poller opens a follow. Idempotent. + + Shared by the pod-list sweep and the watch so the two cannot drift: whichever + sees the condition first does the work, the other no-ops on the _doomed + check. + + Detection latency, not the follow, is what loses the metric. stellar-core + exits about a second after SIGTERM and the pod object is reaped right behind + it, so a condemned pod exists for only a few seconds. Measured on this + cluster at prestopSleepSeconds=5, the 5s list sweep caught that window about + half the time: of 52 mid-replay legs, 32 lost txApply and 25 of those were + seen but seen too late to open a stream. + """ + if name in _doomed: + return False + if (pod.get('status') or {}).get('phase') in ('Succeeded', 'Failed'): + # Already finished. Its log is complete and a follow would only re-read + # a dead pod every iteration. + return False + doom = _is_condemned(pod) + if not doom: + return False + _doomed[name] = doom + # Recorded now, because the evidence does not survive the node: once the + # object is gone there is no way to tell a drain we lost a race with from a + # corpse that never had a metric to lose. + write_metrics(end, attempt, {'disruptionReason': doom}) + if name in _wake: + # Break the current sleep so the follow opens now rather than up to + # LOG_POLL_SECONDS from now. + _wake[name].set() + logger.info("range %s: pod %s condemned (%s), opening follow", end, name, doom) + return True def pod_seconds(pod): @@ -178,25 +288,10 @@ def read_state(end, attempt): return ts if ts and _TS_RE.match(ts) else None -def _write_atomic(path, body, opener=None): - """Write `body` through tmp+rename so a reader never sees a partial file. - - The monitor polls these files while this process writes them, so a torn - .metrics or .outcome would be read as corrupt and the measurement lost. - """ - tmp = path + '.tmp' - # `opener` resolves at call time, not as a default argument, so the write - # seam stays patchable -- the tmp+rename discipline is only worth having if - # a test can crash a write mid-flight and prove the real path is untouched. - with (opener or open)(tmp, 'wt') as fh: - fh.write(body) - os.replace(tmp, path) - - def write_state(end, attempt, ts): path = base(end, attempt) + '.state' try: - _write_atomic(path, ts) + records.write_atomic(path, ts) except OSError as e: logger.warning("could not persist state for range %s: %s", end, e) @@ -220,22 +315,6 @@ def discard(end, attempt): _TS_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?$") _TX_METRIC = "metric 'ledger.transaction.apply'" -# medida prints the sum in scientific notation once it exceeds 1e6 ms, which is -# every range that applies a real transaction load. The old [0-9.]+ pattern -# matched "1.30722" then demanded "ms" and hit "e+06ms" instead, so tx_apply was -# silently missing for 25% of ranges -- 91-99% of everything above ledger 35M, -# exactly the expensive end. -_SUM_RE = re.compile(r"sum\s*=\s*([0-9.]+(?:[eE][+-]?[0-9]+)?)ms") -_SYNTHETIC_PEAK_RE = re.compile( - r"SYNTHETIC PEAK: anonBytes=(\d+) workingSetBytes=(\d+)") -SYNTHETIC_WORKER = os.getenv('SYNTHETIC_WORKER', '').lower() == 'true' - - -def _install_synthetic_restart_handler(): - """Allow a collector-only container restart in the opt-in live harness.""" - if SYNTHETIC_WORKER: - signal.signal(signal.SIGTERM, lambda _signum, _frame: sys.exit(0)) - class TxApplyScanner: """Pull the medida tx-apply total out of the stream as it goes past. @@ -248,7 +327,13 @@ class TxApplyScanner: only place guaranteed to see them. """ - WINDOW = 15 # same span job_monitor uses when reading an archive + # Shared with job_monitor's archive re-read rather than restated: they used + # to agree by comment, and a divergence would hand the recovery path the + # same blind spot it exists to cover. Measured on ssc-test 2026-08-04, a + # /info liveness response interleaved into the block put `sum` 91 lines + # below the header and both readers gave up 76 lines short -- one leg in 233. + WINDOW = medida.WINDOW + HARD_WINDOW = medida.HARD_WINDOW # Printed by RESUME_SCRIPT before stellar-core starts. Its counterpart, # "RESUME DECLINED", means new-db ran and this attempt did the whole range, @@ -260,19 +345,13 @@ def __init__(self, recreated=False): self.seconds = None self.resumed = False self.resume_decided = False - self.synthetic_anon = None - self.synthetic_working_set = None # A new poller starting from durable .state missed every earlier line. # Finalization must recover scanner-only facts from the archive. self.recreated = recreated self._left = 0 + self._span = 0 def feed(self, line): - if SYNTHETIC_WORKER: - peak = _SYNTHETIC_PEAK_RE.search(line) - if peak: - self.synthetic_anon = int(peak.group(1)) - self.synthetic_working_set = int(peak.group(2)) if self.RESUME_MARK in line: self.resumed = True self.resume_decided = True @@ -280,14 +359,26 @@ def feed(self, line): self.resume_decided = True if _TX_METRIC in line: self._left = self.WINDOW + self._span = self.HARD_WINDOW return if self._left <= 0: return - self._left -= 1 - m = _SUM_RE.search(line) + m = medida.SUM_RE.search(line) if m: self.seconds = float(m.group(1)) / 1000.0 self._left = 0 + return + self._span -= 1 + if self._span <= 0 or medida.ANY_METRIC.search(line): + # Ran out of rope, or another timer's block started -- either way our + # sum is not coming. + self._left = 0 + return + if medida.METRIC_LINE.search(line): + # Only medida's own statistics count. Interleaved output from another + # thread is noise between us and the sum, not evidence we have passed + # it. + self._left -= 1 def scan_archive(end, attempt, need_tx=False): @@ -354,9 +445,15 @@ def write_metrics(end, attempt, values): if (prior.get('attemptSecondsExact') is True or values.get('attemptSecondsExact') is True): merged['attemptSecondsExact'] = True + # Same one-way rule: once a duration has been dated from the container's own + # startTime, a later poller-clock write must not strip the provenance that + # makes the monitor willing to use it as a chain leg. + if (prior.get('attemptSecondsFromContainerStart') is True + or values.get('attemptSecondsFromContainerStart') is True): + merged['attemptSecondsFromContainerStart'] = True values = merged try: - _write_atomic(path, json.dumps(values)) + records.write_atomic(path, json.dumps(values)) logger.info("range %s attempt %s metrics=%s", end, attempt, values) except OSError as e: logger.warning("could not persist metrics for range %s: %s", end, e) @@ -372,9 +469,8 @@ def classify(pod): rule matches, and an admission rejection matches none. """ status = pod.get('status', {}) - for cond in status.get('conditions', []): - if cond.get('type') == 'DisruptionTarget' and cond.get('status') == 'True': - return {'outcome': 'disrupted', 'exitCode': None} + if _is_condemned(pod): + return {'outcome': 'disrupted', 'exitCode': None} reason = status.get('reason') if reason == 'Evicted' and 'ephemeral' in (status.get('message') or ''): # The range's own disk use, not something the cluster did to it. @@ -415,7 +511,7 @@ def record_outcome(pod, end, attempt): data = classify(pod) data['pod'] = pod['metadata']['name'] try: - _write_atomic(path, json.dumps(data)) + records.write_atomic(path, json.dumps(data)) logger.info("range %s attempt %s classified: %s", end, attempt, data['outcome']) except OSError as e: logger.warning("could not persist outcome for range %s: %s", end, e) @@ -439,7 +535,19 @@ def record_outcome(pod, end, attempt): _peak_flushed = {} # pod name -> (end, attempt), so a mid-flight peak flush can find its file. _streaming = {} - +# The poller registry, module-level so the watch can open a stream the moment a +# pod appears instead of waiting for the pod-list loop to come round. +# +# One registry with one guard is the whole point: two creators would each hold +# their own in-memory last_ts -- read_state is consulted once, at poll_pod start +# -- so both would re-append the same lines and race each other's write_state. +# main() binds its locals to these, so the loop's existing bookkeeping is +# unchanged and either caller can be the one that wins. +_tasks = {} +_streamed = set() +# session + the terminal/succeeded views poll_pod closes over, published once by +# main() so ensure_stream can be called from outside it. +_stream_ctx = {} def _flush_peak(name, axis, field, value): @@ -479,7 +587,7 @@ def _register_stream(name, end, attempt): _flush_peak(name, axis, field, value) -async def sample_kubelet(session, nodes): +async def sample_kubelet(session, node_ips): """Update each pod's peak ephemeral use and peak anon from one snapshot. Both axes come out of the same GET, so tracking memory here is free. @@ -497,25 +605,34 @@ async def sample_kubelet(session, nodes): OOM. The `time` field on this payload runs 1-3s behind wall clock; the ~80s lag applies only to the du-based ephemeral figure alongside it. """ - if SYNTHETIC_WORKER: - return - for node in nodes: - url = f"{API}/api/v1/nodes/{node}/proxy/stats/summary" + for ip in node_ips: + # Straight at the kubelet, not through the apiserver's node proxy. The + # proxy needs `nodes/proxy`, which authorizes GET on EVERY kubelet path + # -- /pods and /containerLogs included, for any namespace scheduled on + # that node. The kubelet maps /stats/* to its own `nodes/stats` + # subresource, so going direct is the same data under a grant that + # cannot read pod inventory or logs at all. + # + # ssl=False: EKS kubelet serving certs are self-signed, not issued by + # the cluster CA the session's context trusts. In-VPC hop to the node's + # own address. + url = f"https://{ip}:{KUBELET_PORT}/stats/summary" try: - async with session.get(url, headers={'Authorization': f'Bearer {token()}'}) as resp: + async with session.get(url, ssl=False, + headers={'Authorization': f'Bearer {token()}'}) as resp: resp.raise_for_status() summary = await resp.json() except Exception as e: # Not debug: if this fails the ephemeral axis is silently empty and # the profile looks merely "absent" rather than broken. - logger.warning("kubelet stats unavailable on %s: %s", node, e) + logger.warning("kubelet stats unavailable on %s: %s", ip, e) continue for entry in summary.get('pods', []): name = entry.get('podRef', {}).get('name') if not name: continue used = (entry.get('ephemeral-storage') or {}).get('usedBytes') - if used is not None and STORAGE_MODE == 'ephemeral': + if used is not None and config.STORAGE_MODE == 'ephemeral': prev = _eph_peak.get(name, 0) if int(used) > prev: _eph_peak[name] = int(used) @@ -563,7 +680,7 @@ async def sample_kubelet(session, nodes): def _mark_done(end, attempt): path = done_path(end, attempt) try: - _write_atomic(path, '') + records.write_atomic(path, '') except OSError as e: # Costs a Job that waits out JOB_TTL_SECONDS, never correctness. logger.warning("could not mark range %s attempt %s done: %s", end, attempt, e) @@ -588,10 +705,33 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): # Before discard: on success the archive is about to be deleted. measured = {} observed = _pod_secs.pop(pod, None) + since_start = None + began = _pod_start.pop(pod, None) + if observed is None and began: + # The container started at `began` and has just stopped -- finalize is + # reached on end of stream or a 404, both within a second or two of the + # exit. Not exact, because the true end is terminated.finishedAt, but it + # dates the attempt from the container rather than from this poller, and + # a re-opened poller's clock can be near zero against a multi-hour run. + try: + since_start = (datetime.utcnow() - datetime.strptime( + began, '%Y-%m-%dT%H:%M:%SZ')).total_seconds() + except ValueError: + since_start = None if observed is not None: # The pod's own timestamps, not how long this poller happened to watch. measured['attemptSeconds'] = round(observed, 1) measured['attemptSecondsExact'] = True + elif since_start is not None and since_start > 0: + measured['attemptSeconds'] = round(since_start, 1) + # Not exact -- the true end is terminated.finishedAt, and this is + # "now, a second or two after the stream ended". But it IS a measure of + # the container's own lifetime rather than of this process's attention + # span, which is the distinction the monitor's chain gate cares about. + # Measured on ssc-test against two evicted pods: 370.9s and 375.1s + # against a true ~373s, so +/-1%, versus the poller clock's -46%. + measured['attemptSecondsExact'] = False + measured['attemptSecondsFromContainerStart'] = True elif started is not None: # Fallback only: the monitor's figure comes from the pod's terminated # timestamps and is preferred when it exists. write_metrics keeps this @@ -605,8 +745,10 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): # finalization; recover only the state this scanner could have missed. archived = None need_resume = int(attempt) > 1 and not tx.resume_decided - need_tx = (tx.recreated and tx.seconds is None) or ( - SYNTHETIC_WORKER and tx.recreated) + # Not gated on `recreated`: a poller that ran start to finish can still + # miss the block, which stellar-core prints once at exit, so a stream that + # ends a beat early has no total and nothing to recreate. + need_tx = tx.seconds is None if need_resume or need_tx: archived = scan_archive(end, attempt, need_tx=need_tx) if tx.resumed or (archived is not None and archived.resumed): @@ -619,10 +761,6 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): tx_seconds = archived.seconds if tx_seconds is not None: measured['txApplySeconds'] = tx_seconds - synthetic = archived if archived is not None else tx - if SYNTHETIC_WORKER and synthetic.synthetic_anon is not None: - measured['peakAnonBytes'] = synthetic.synthetic_anon - measured['peakWorkingSetBytes'] = synthetic.synthetic_working_set _peak_flushed.pop(pod, None) _peak_flushed.pop(pod + '/eph', None) _streaming.pop(pod, None) @@ -652,7 +790,7 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): measured['peakEphemeralBytes'] = eph if measured: write_metrics(end, attempt, measured) - if not SAVE_SUCCESS_LOGS and done_ok(pod): + if not config.SAVE_SUCCESS_LOGS and done_ok(pod): discard(end, attempt) logger.info("range %s attempt %s: succeeded, archive discarded " "(saveSuccessLogs=false)", end, attempt) @@ -667,7 +805,6 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): _mark_done(end, attempt) - async def _poll_once(session, pod, end, attempt, last_ts, tx): """One short read of a pod's log. Returns (new_last_ts, gone). @@ -682,7 +819,7 @@ async def _poll_once(session, pod, end, attempt, last_ts, tx): # Second granularity, so this overlaps on purpose; the per-line # comparison below removes the overlap exactly. params['sinceTime'] = last_ts[:19] + 'Z' - url = f"{API}/api/v1/namespaces/{NAMESPACE}/pods/{pod}/log" + url = f"{API}/api/v1/namespaces/{config.NAMESPACE}/pods/{pod}/log" async with _poll_slots: async with session.get(url, params=params, headers={'Authorization': f'Bearer {token()}'}) as resp: @@ -699,10 +836,21 @@ async def _poll_once(session, pod, end, attempt, last_ts, tx): if len(body) > MAX_POLL_CHARS: break + return _ingest(body, end, attempt, last_ts, tx), False + + +def _ingest(body, end, attempt, last_ts, tx): + """Append one block of timestamped log text to the archive; new last_ts. + + Split out of _poll_once so the doomed-pod follow stream lands its bytes + through exactly the same path -- dedup, gzip member framing, tx scanning + and resume-point bookkeeping. Two copies of this is how one route silently + stops feeding TxApplyScanner while the other keeps working. + """ pending = None lines = [l for l in re.split(r'[\r\n]', body) if l] if not lines: - return last_ts, False + return last_ts # Compressed into memory first, then appended in ONE write. # # Appending straight into the file with gzip.open(..., 'at') meant the @@ -743,7 +891,67 @@ async def _poll_once(session, pod, end, attempt, last_ts, tx): out.write(member.getvalue()) if pending: write_state(end, attempt, pending) - return pending, False + return pending + return last_ts + + +async def _follow_tail(session, pod, end, attempt, last_ts, tx): + """Hold a follow=true stream on a doomed pod. Returns (last_ts, gone). + + Opened only for pods the cluster has already condemned, so this is the one + place the cost of follow=true is worth paying: the connection is held for + the couple of minutes between the DisruptionTarget condition and the node + going away, not for the hours a range runs. + + Proven on ssc-test: with the stream held, SIGTERM to stellar-core yields + `got signal 15` -> `metric 'ledger.transaction.apply'` -> `Application + destroyed` inside 4ms, all of it captured. The same pod polled at 5s + intervals recorded `pod gone before disruption seen`. + + Bytes are ingested as they arrive rather than at end of stream, so a node + that disappears mid-read still leaves everything up to that point in the + archive. + """ + params = {'container': CONTAINER, 'timestamps': 'true', 'follow': 'true'} + if last_ts: + params['sinceTime'] = last_ts[:19] + 'Z' + url = f"{API}/api/v1/namespaces/{config.NAMESPACE}/pods/{pod}/log" + deadline = asyncio.get_event_loop().time() + DOOMED_FOLLOW_SECONDS + buf = '' + if _follow_slots.locked(): + # Every follow budget is spoken for, so this pod polls instead. Better + # than queueing: the pod has ~2 minutes to live and a queued follow that + # opens after it dies captures nothing while still holding a slot. + logger.info("range %s: no follow slot free (%d in use), polling instead", + end, MAX_DOOMED_FOLLOWS) + _doomed.pop(pod, None) + return await _poll_once(session, pod, end, attempt, last_ts, tx) + async with _follow_slots: + async with session.get(url, params=params, + headers={'Authorization': f'Bearer {token()}'}) as resp: + if resp.status == 404: + return last_ts, True + resp.raise_for_status() + async for chunk in resp.content.iter_chunked(65536): + buf += chunk.decode('utf-8', 'replace') + # Flush on whole lines only: a partial trailing line has no + # usable timestamp and must not become the resume point. + cut = max(buf.rfind('\n'), buf.rfind('\r')) + if cut >= 0: + last_ts = _ingest(buf[:cut + 1], end, attempt, last_ts, tx) + buf = buf[cut + 1:] + if asyncio.get_event_loop().time() > deadline: + logger.info("range %s: doomed follow hit %.0fs, falling back to polling", + end, DOOMED_FOLLOW_SECONDS) + _doomed.pop(pod, None) + break + if buf: + last_ts = _ingest(buf, end, attempt, last_ts, tx) + # One follow per pod. The stream ending means the container exited, and the + # caller must fall back to a normal poll for the terminal check and + # finalize; leaving the flag set would re-open a stream on a dead pod every + # iteration. The pod-list loop does not clear it -- by then the pod is gone. + _doomed.pop(pod, None) return last_ts, False @@ -791,9 +999,25 @@ async def poll_pod(session, pod, end, attempt, done, done_ok): # seconds_for_range prefers it anyway. started = None first_pass = False + followed = False try: - last_ts, gone = await _poll_once(session, pod, end, attempt, last_ts, tx) - backoff = LOG_POLL_SECONDS + if _doomed.get(pod) and not was_terminal and DOOMED_FOLLOW_SECONDS > 0: + followed = True + # Condemned and still running: stop sampling and hold the + # connection through the kill. Returns when the container exits + # or the notice is withdrawn, and the loop re-checks terminal + # immediately afterwards. + last_ts, gone = await _follow_tail( + session, pod, end, attempt, last_ts, tx) + else: + last_ts, gone = await _poll_once( + session, pod, end, attempt, last_ts, tx) + # Fallback interval for a condemned pod that could not follow: no + # slot was free, or following is disabled. 1s sampling alone still + # closes the ~9s window between the medida block and the pod object + # being deleted, so a mass reclaim degrades rather than loses. + backoff = (DOOMED_POLL_SECONDS if _doomed.get(pod) + else LOG_POLL_SECONDS) failures = 0 if gone: logger.info("pod %s gone before/while polling range %s", pod, end) @@ -823,6 +1047,11 @@ async def poll_pod(session, pod, end, attempt, done, done_ok): # mid-poll and drop whatever it wrote on the way out. await finalize(session, pod, end, attempt, tx, done_ok, started) return + if followed: + # The follow only returns once the container has exited, so the + # very next read is the one that matters. Sleeping here would hand + # the interval back to exactly the race the follow exists to win. + continue # Not a blind sleep: a pod going terminal cuts it short. Polling faster # would not help -- sinceTime has second granularity, so anything under # ~1s re-reads the same second -- and the delay that matters is between @@ -842,16 +1071,115 @@ async def poll_pod(session, pod, end, attempt, done, done_ok): async def list_pods(session): - url = f"{API}/api/v1/namespaces/{NAMESPACE}/pods" - params = {'labelSelector': f"{LABEL_RUN}={RUN_NAME}"} + url = f"{API}/api/v1/namespaces/{config.NAMESPACE}/pods" + params = {'labelSelector': f"{config.LABEL_RUN}={config.RUN_NAME}"} async with session.get(url, params=params, headers={'Authorization': f'Bearer {token()}'}) as resp: resp.raise_for_status() return (await resp.json()).get('items', []) +def ensure_stream(name, end, attempt, phase): + """Open this pod's poller if it has none. Idempotent; returns whether it did. + + Called by the watch as a pod appears and again if it is condemned, and by the + pod-list loop as a backstop for events the watch drops across a reconnect. + Opening a stream is time-critical -- a condemned pod is gone a second after + stellar-core exits -- so it must not be reachable only from a poll cycle. + Measured on the 900-worker run before this existed: the loop's cycle stretched + to 925s behind a serial kubelet sweep, and five -a2 legs lived and died with + no reader at all, one of them for 184.7s, losing txApply for good. + """ + if name in _tasks or name in _streamed or not _stream_ctx: + return False + if phase not in POLLABLE_PHASES: + # Allowlist, not "skip Pending". A container that has not started answers + # 400 "waiting to start", and Unknown means the node stopped reporting. + # Both are retried on the cycle they become pollable. + return False + ctx = _stream_ctx + _register_stream(name, end, attempt) + _tasks[name] = asyncio.create_task( + poll_pod(ctx['session'], name, end, attempt, + lambda p: ctx['terminal'].get(p, False), + lambda p: ctx['succeeded'].get(p, False))) + logger.info("opened stream for range %s attempt %s (%d active)", + end, attempt, len(_tasks)) + return True + + +async def watch_condemnations(session): + """Watch the run's pods and flag condemnations the moment they are written. + + Runs beside the pod-list loop rather than replacing it: the list still owns + discovery, task bookkeeping and finalize. This only ever sets _doomed + earlier than the list would have, which is the difference between opening a + follow while stellar-core is still running and opening it on a 404. + + Cheaper than the sweep it front-runs, too. list_pods re-serialises every pod + in the run every POLL_SECONDS; a watch is one connection served from the + apiserver's cache that sends only deltas. + + Never fatal. Any failure falls back to the list sweep, which is exactly the + behaviour that existed before this function. + """ + url = f"{API}/api/v1/namespaces/{config.NAMESPACE}/pods" + rv = None + while True: + params = {'labelSelector': f"{config.LABEL_RUN}={config.RUN_NAME}", 'watch': 'true', + 'allowWatchBookmarks': 'true', + 'timeoutSeconds': str(WATCH_TIMEOUT_SECONDS)} + if rv: + params['resourceVersion'] = rv + try: + async with session.get(url, params=params, + headers={'Authorization': f'Bearer {token()}'}) as resp: + if resp.status == 410: + # Our resourceVersion aged out of the apiserver's history. + # Restarting without one re-syncs; the list sweep covers the + # gap in the meantime. + rv = None + continue + resp.raise_for_status() + async for raw in resp.content: + if not raw.strip(): + continue + try: + ev = json.loads(raw) + except ValueError: + continue + obj = ev.get('object') or {} + meta = obj.get('metadata') or {} + # Track on every event, bookmarks included -- that is what + # they are for -- so a reconnect resumes instead of re-syncing. + rv = meta.get('resourceVersion') or rv + if ev.get('type') == 'ERROR': + if obj.get('code') == 410: + rv = None + break + if ev.get('type') not in ('ADDED', 'MODIFIED'): + continue + labels = meta.get('labels') or {} + end = labels.get(config.LABEL_RANGE) + if end is None: + continue + name = meta.get('name') + attempt = labels.get(config.LABEL_ATTEMPT, '1') + # Before the condemnation check: a pod condemned in the same + # event it first becomes pollable needs the poller to exist + # first, or there is nothing for _mark_condemned to wake. + ensure_stream(name, end, attempt, + (obj.get('status') or {}).get('phase')) + _mark_condemned(obj, name, end, attempt) + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning("condemnation watch dropped (%s); retrying", exc) + await asyncio.sleep(WATCH_RETRY_SECONDS) + + async def main(): - os.makedirs(LOG_DIR, exist_ok=True) + os.makedirs(config.LOG_DIR, exist_ok=True) # Connection-pool limit, not a task limit: there is no semaphore above it, # so a stream that cannot get a connection blocks here for as long as the # pool stays full -- and every holder is a follow=true stream open for the @@ -861,20 +1189,33 @@ async def main(): # calls, not for one connection per pod. Under follow=true this had to # exceed parallelism or pods silently starved -- 1200 against 2048 workers # left 896 blocked forever, and retries, created last, never got a slot. - conn = aiohttp.TCPConnector(limit=MAX_CONCURRENT_POLLS + 64, ssl=ssl_ctx()) + conn = aiohttp.TCPConnector( + limit=MAX_CONCURRENT_POLLS + MAX_DOOMED_FOLLOWS + 64, ssl=ssl_ctx()) # No total timeout: these streams are meant to stay open for the life of a # range, which can be hours. timeout = aiohttp.ClientTimeout(total=None, sock_connect=10) - tasks, terminal, succeeded, vanished = {}, {}, {}, {} - # Streams that ran to completion. Without this a finished task is deleted - # from `tasks` and the next poll re-opens the stream, forever: one full log - # re-read per pod every POLL_SECONDS, which at 1024 workers is a lot of - # apiserver -- measured, the completion block ran once per range per cycle + # tasks/streamed are the module-level registry under local names, so the + # bookkeeping below is unchanged while the watch shares the same guard. + tasks, streamed = _tasks, _streamed + # Cleared rather than assumed empty: a second main() in one process would + # otherwise find every pod already registered and open no streams at all. + tasks.clear() + streamed.clear() + _stream_ctx.clear() + terminal, succeeded, vanished = {}, {}, {} + # `streamed` holds streams that ran to completion. Without it a finished task + # is deleted from `tasks` and the next poll re-opens the stream, forever: one + # full log re-read per pod every POLL_SECONDS, which at 1024 workers is a lot + # of apiserver -- measured, the completion block ran once per range per cycle # for the rest of the run. - streamed = set() async with aiohttp.ClientSession(connector=conn, timeout=timeout) as session: - logger.info("streaming logs for run=%s into %s", RUN_NAME, LOG_DIR) + logger.info("streaming logs for run=%s into %s", config.RUN_NAME, config.LOG_DIR) + # Published before the watch starts: ensure_stream is a no-op until this + # exists, so a watch event arriving first would silently open nothing. + _stream_ctx.update(session=session, terminal=terminal, succeeded=succeeded) + if WATCH_TIMEOUT_SECONDS > 0: + asyncio.create_task(watch_condemnations(session)) while True: try: pods = await list_pods(session) @@ -920,39 +1261,47 @@ async def main(): streamed.add(name) logger.info("cancelled and finalized stream for vanished pod %s", name) - # Unconditional: this used to be gated on ephemeral mode, back - # when it only sampled disk. Memory is sized in both modes, so - # gating it here left every pvc run with no anon peak at all. - # Once per cycle, before the per-pod branches below: those end - # in `continue` for every pod already being streamed, so - # anything after them runs only on the cycle a stream opens -- - # when the range has barely written anything yet. - await sample_kubelet(session, { - p['spec']['nodeName'] for p in pods - if p.get('spec', {}).get('nodeName') - and p.get('status', {}).get('phase') == 'Running'}) for pod in pods: name = pod['metadata']['name'] labels = pod['metadata'].get('labels', {}) - end = labels.get(LABEL_RANGE) + end = labels.get(config.LABEL_RANGE) if end is None: continue phase = pod.get('status', {}).get('phase') terminal[name] = phase in ('Succeeded', 'Failed') - if terminal[name]: - # Recorded while the pod object still exists. Beats the - # poller's own elapsed time, which only measures how long - # WE watched -- ~0 for a pod that finished before this - # poller started. - secs = pod_seconds(pod) - if secs is not None: - _pod_secs[name] = secs + # NOT gated on phase. A pod that is being DELETED keeps + # phase Running until its object disappears -- deletion + # never sets Succeeded or Failed -- so gating this on + # terminal meant no disrupted pod ever recorded an exact + # duration, and every one of them fell back to the poller's + # own clock. Measured on ssc-test: 268s reported against a + # ~500s attempt, because that clock starts when the POLLER + # opened, not when the container did. The container's + # terminated.finishedAt is present for the ~8s the object + # outlives it, and pod_seconds returns None until then, so + # asking every cycle is self-guarding. + secs = pod_seconds(pod) + if secs is not None: + _pod_secs[name] = secs + start = (pod.get('status') or {}).get('startTime') + if start: + # Second line: if the object is deleted before any cycle + # catches its terminated timestamp, finalize can still + # date the attempt from when the container STARTED + # rather than from when this poller happened to open. + _pod_start[name] = start + # Backstop only: the watch normally gets here first. This + # still runs so detection survives the watch being disabled + # or reconnecting. + if not terminal[name]: + _mark_condemned(pod, name, end, + labels.get(config.LABEL_ATTEMPT, '1')) if terminal[name] and name in _wake: # Wake its poller now rather than at the next tick. _wake[name].set() succeeded[name] = phase == 'Succeeded' if phase == 'Failed': - record_outcome(pod, end, labels.get(LABEL_ATTEMPT, '1')) + record_outcome(pod, end, labels.get(config.LABEL_ATTEMPT, '1')) if name in tasks and not tasks[name].done(): continue if name in tasks and tasks[name].done(): @@ -965,28 +1314,37 @@ async def main(): continue if name in streamed: continue - attempt = labels.get(LABEL_ATTEMPT, '1') - if phase not in POLLABLE_PHASES: - # Allowlist, not "skip Pending". A container that has not - # started answers 400 "waiting to start" -- 60 of 88 poll - # failures right after the polling switch -- and Unknown - # means the node stopped reporting, so that poll cannot - # succeed either. Both are picked up on the cycle they - # become pollable. Succeeded and Failed stay in: a - # terminal pod is where the final output lives. - continue - _register_stream(name, end, attempt) - tasks[name] = asyncio.create_task( - poll_pod(session, name, end, attempt, - lambda p: terminal.get(p, False), - lambda p: succeeded.get(p, False))) - logger.info("opened stream for range %s attempt %s (%d active)", - end, attempt, len(tasks)) + # Backstop. The watch normally opens this the moment the pod + # appears; this covers events dropped across a reconnect. + # Same registry and the same guard, so whichever gets there + # first wins and the other no-ops -- two readers on one pod + # would duplicate the archive and race write_state. + ensure_stream(name, end, labels.get(config.LABEL_ATTEMPT, '1'), phase) + + # AFTER the per-pod branches, never before them. This is a serial + # sweep of every node's kubelet, and on spot a dead one costs the + # 10s connect timeout apiece -- measured, that stretched one cycle + # to 925s. Ahead of the branches it delayed every stream by that + # much; behind them it delays only the next cycle's sampling. + # It must stay outside the `for` loop, though: those branches end + # in `continue` for a pod already streaming, so a sampler placed + # among them fires only on the cycle a stream opens, when the + # range has barely written anything. + # + # Unconditional: this used to be gated on ephemeral mode, back + # when it only sampled disk. Memory is sized in both modes, so + # gating it here left every pvc run with no anon peak at all. + # hostIP, not nodeName: the sampler talks to the kubelet + # directly, and this list already carries the address, so it + # costs no read of Node objects. + await sample_kubelet(session, { + p['status']['hostIP'] for p in pods + if p.get('status', {}).get('hostIP') + and p.get('status', {}).get('phase') == 'Running'}) except Exception as e: logger.warning("pod list failed: %s", e) await asyncio.sleep(POLL_SECONDS) if __name__ == '__main__': - _install_synthetic_restart_handler() asyncio.run(main()) diff --git a/src/MissionParallelCatchup/integration/synthetic_resume_harness.py b/src/MissionParallelCatchup/integration/synthetic_resume_harness.py deleted file mode 100644 index 71886a54..00000000 --- a/src/MissionParallelCatchup/integration/synthetic_resume_harness.py +++ /dev/null @@ -1,670 +0,0 @@ -#!/usr/bin/env python3 -"""Run the opt-in collector-restart scenario in the sandbox namespace. - -The release name is the isolation boundary. This runner refuses every namespace -except ``sandbox``, renders and validates the chart before installing it, and -only queries or deletes exact release-owned names and labels. -""" - -import argparse -import json -import os -import re -import subprocess -import sys -import tempfile -import time -from pathlib import Path - -import yaml - - -HERE = Path(__file__).resolve().parent -MODULE_DIR = HERE.parent -CHART = MODULE_DIR / 'parallel_catchup_helm' -JOB_MONITOR = MODULE_DIR / 'job_monitor.py' -LOG_COLLECTOR = MODULE_DIR / 'log_collector.py' -NAMESPACE = 'sandbox' -RELEASE_RE = re.compile(r'^mpc-resume-[a-z0-9]{6,20}$') -RANGE_END = 64 -ATTEMPT_NAMES = { - 1: lambda release: f'{release}-r{RANGE_END}-a1', - 2: lambda release: f'{release}-r{RANGE_END}-a2', -} -SYNTHETIC_PEAKS = { - 'peakAnonBytes': 48 * 1024 * 1024, - 'peakWorkingSetBytes': 56 * 1024 * 1024, -} - - -class HarnessError(RuntimeError): - pass - - -def validate_scope(namespace, release): - if namespace != NAMESPACE: - raise HarnessError(f'namespace must be exactly {NAMESPACE!r}') - if not RELEASE_RE.fullmatch(release): - raise HarnessError( - 'release must match mpc-resume- plus 6-20 lowercase alphanumerics') - - -def run(command, *, input_text=None, check=True, timeout=120): - result = subprocess.run( - command, input=input_text, capture_output=True, text=True, timeout=timeout) - if check and result.returncode: - raise HarnessError( - f"command failed ({result.returncode}): {' '.join(command)}\n" - f"{result.stderr.strip()}") - return result - - -def kubectl(namespace, *args, check=True, timeout=120, input_text=None): - command = ['kubectl'] - if namespace: - command += ['--namespace', namespace] - command += list(args) - return run(command, check=check, timeout=timeout, input_text=input_text) - - -def helm_sets(release, source_config_map, image): - return [ - f'worker.stellar_core_image={image}', - 'worker.replicas=1', - 'worker.storageMode=pvc', - 'worker.storageSize=1Gi', - 'worker.maxVolumesPerNode=0', - 'worker.resources.requests.cpu=25m', - 'worker.resources.requests.memory=64Mi', - 'worker.resources.limits.cpu=100m', - 'worker.resources.limits.memory=128Mi', - f'monitor.image={image}', - f'monitor.sourceConfigMap={source_config_map}', - 'monitor.sourceInstallDependencies=false', - 'monitor.loggingIntervalSeconds=1', - 'monitor.livenessProbeIntervalSeconds=300', - 'monitor.maxAttempts=2', - 'monitor.maxTimeoutAttempts=2', - 'monitor.maxDisruptionAttempts=2', - 'monitor.attemptDeadlineSeconds=240', - 'monitor.jobTtlSeconds=300', - 'monitor.logStorageSize=1Gi', - 'monitor.saveSuccessLogs=true', - 'monitor.collectorPollSeconds=1', - 'monitor.logPollSeconds=1', - 'monitor.maxConcurrentPolls=4', - 'monitor.maxPollChars=1048576', - 'monitor.terminalPollAttempts=3', - 'monitor.collectorResources.requests.cpu=25m', - 'monitor.collectorResources.requests.memory=128Mi', - 'monitor.collectorResources.limits.cpu=250m', - 'monitor.collectorResources.limits.memory=512Mi', - 'monitor.resources.requests.cpu=25m', - 'monitor.resources.requests.memory=128Mi', - 'monitor.resources.limits.cpu=250m', - 'monitor.resources.limits.memory=512Mi', - 'range.generator=uniform', - 'range.startingLedger=0', - f'range.latestLedgerNum={RANGE_END}', - f'range.ledgersPerJob={RANGE_END}', - 'range.overlapLedgers=0', - 'integration.syntheticWorker.enabled=true', - 'integration.syntheticWorker.imagePullPolicy=IfNotPresent', - 'integration.syntheticWorker.predecessorSeconds=12', - 'integration.syntheticWorker.successorMinimumSeconds=12', - 'integration.syntheticWorker.maximumWaitSeconds=180', - ] - - -def helm_args(sets): - args = [] - for value in sets: - args += ['--set', value] - return args - - -def inspect_rendered(manifest, release, image): - docs = [doc for doc in yaml.safe_load_all(manifest) if doc] - expected_names = { - f'{release}-job-monitor', - f'stellar-supercluster-{release}', - f'{release}-stellar-core-config', - f'{release}-synthetic-worker', - f'{release}-job-monitor-logs', - } - allowed_kinds = { - 'ServiceAccount', 'ConfigMap', 'PersistentVolumeClaim', - 'Role', 'RoleBinding', 'Deployment', - } - names = [] - for doc in docs: - kind = doc.get('kind') - name = (doc.get('metadata') or {}).get('name') - namespace = (doc.get('metadata') or {}).get('namespace') - if kind not in allowed_kinds: - raise HarnessError(f'unexpected rendered kind {kind!r}') - if name not in expected_names: - raise HarnessError(f'unexpected rendered resource {kind}/{name}') - if namespace not in (None, NAMESPACE): - raise HarnessError(f'{kind}/{name} targets namespace {namespace!r}') - names.append(f'{kind}/{name}') - - deployment = next(doc for doc in docs if doc['kind'] == 'Deployment') - pod_spec = deployment['spec']['template']['spec'] - if deployment['spec']['replicas'] != 1: - raise HarnessError('monitor Deployment must have exactly one replica') - if pod_spec.get('nodeSelector') or pod_spec.get('affinity') or pod_spec.get('tolerations'): - raise HarnessError('synthetic Deployment must not target or tolerate special nodes') - containers = {container['name']: container for container in pod_spec['containers']} - if set(containers) != {'job-monitor', 'log-collector'}: - raise HarnessError(f'unexpected monitor containers {sorted(containers)}') - if {container['image'] for container in containers.values()} != {image}: - raise HarnessError('monitor and collector must use only the requested monitor image') - for container in containers.values(): - command = ' '.join(container.get('command', []) + container.get('args', [])) - if 'pip install' in command: - raise HarnessError('source mode would make an external package request') - synthetic = next( - doc for doc in docs - if doc['kind'] == 'ConfigMap' - and doc['metadata']['name'] == f'{release}-synthetic-worker') - script = synthetic['data']['worker.py'] - if 'subprocess' in script or 'stellar-core' in script or 'curl ' in script: - raise HarnessError('synthetic worker contains an external command surface') - return sorted(names) - - -def create_source_config_map(release): - name = f'{release}-source' - generated = kubectl( - NAMESPACE, 'create', 'configmap', name, - f'--from-file=job_monitor.py={JOB_MONITOR}', - f'--from-file=log_collector.py={LOG_COLLECTOR}', - '--dry-run=client', '-o', 'yaml').stdout - kubectl(NAMESPACE, 'apply', '-f', '-', input_text=generated) - return name - - -def json_get(resource, *, labels=None, name=None): - args = ['get', resource] - if name: - args.append(name) - if labels: - args += ['--selector', labels] - args += ['-o', 'json'] - result = kubectl(NAMESPACE, *args, check=False) - if result.returncode: - if 'NotFound' in result.stderr or 'not found' in result.stderr: - return None - raise HarnessError(result.stderr.strip()) - return json.loads(result.stdout) - - -def monitor_pod(release): - payload = json_get('pods', labels=f'app=job-monitor,release={release}') - items = (payload or {}).get('items', []) - if len(items) != 1: - return None - return items[0] - - -def monitor_startup(pod): - if not pod: - return None - statuses = {} - for status in pod.get('status', {}).get('containerStatuses', []): - state = status.get('state') or {} - statuses[status['name']] = { - 'ready': status.get('ready', False), - 'restartCount': status.get('restartCount', 0), - 'state': state, - } - return { - 'name': pod['metadata']['name'], - 'phase': pod.get('status', {}).get('phase'), - 'conditions': pod.get('status', {}).get('conditions', []), - 'containers': statuses, - } - - -def ready_monitor_pod(release, evidence): - pod = monitor_pod(release) - evidence['monitorStartup'] = monitor_startup(pod) - if not pod: - return None - statuses = pod.get('status', {}).get('containerStatuses', []) - return pod if len(statuses) == 2 and all(s.get('ready') for s in statuses) else None - - -def monitor_logs(release): - pod = monitor_pod(release) - if not pod: - return {} - name = pod['metadata']['name'] - captured = {} - for container in ('job-monitor', 'log-collector'): - current = kubectl( - NAMESPACE, 'logs', name, '-c', container, - '--tail=200', check=False, timeout=30) - previous = kubectl( - NAMESPACE, 'logs', name, '-c', container, '--previous', - '--tail=200', check=False, timeout=30) - captured[container] = { - 'current': current.stdout if current.returncode == 0 else current.stderr, - 'previous': previous.stdout if previous.returncode == 0 else previous.stderr, - } - return captured - - -def worker_snapshot(release): - selector = f'catchup.stellar.org/run={release}' - jobs = (json_get('jobs', labels=selector) or {}).get('items', []) - pods = (json_get('pods', labels=selector) or {}).get('items', []) - return jobs, pods - - -def collect_snapshot(release, evidence): - jobs, pods = worker_snapshot(release) - expected_jobs = {factory(release) for factory in ATTEMPT_NAMES.values()} - for job in jobs: - name = job['metadata']['name'] - if name not in expected_jobs: - raise HarnessError(f'unexpected worker Job {name}') - evidence['jobsSeen'].add(name) - live = [] - attempts = {} - for pod in pods: - name = pod['metadata']['name'] - labels = pod['metadata']['labels'] - attempt = int(labels['catchup.stellar.org/attempt']) - if attempt not in ATTEMPT_NAMES: - raise HarnessError(f'unexpected worker attempt {attempt}') - attempts[attempt] = attempts.get(attempt, 0) + 1 - evidence['podsSeen'].add(name) - if pod.get('status', {}).get('phase') in ('Pending', 'Running'): - live.append(name) - if any(count > 1 for count in attempts.values()): - raise HarnessError(f'duplicate worker pods in one attempt: {attempts}') - if len(live) > 1: - raise HarnessError(f'duplicate live workers for one range: {live}') - evidence['maxConcurrentLiveWorkers'] = max( - evidence['maxConcurrentLiveWorkers'], len(live)) - - for pvc_name in (f'{release}-job-monitor-logs', f'{release}-data-r{RANGE_END}'): - pvc = json_get('pvc', name=pvc_name) - volume = ((pvc or {}).get('spec') or {}).get('volumeName') - if volume: - evidence['persistentVolumes'].add(volume) - return jobs, pods - - -def wait_for(description, predicate, *, timeout, interval=1): - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - value = predicate() - if value: - return value - time.sleep(interval) - raise HarnessError(f'timed out waiting for {description}') - - -def container_restarts(pod): - statuses = { - status['name']: status.get('restartCount', 0) - for status in pod.get('status', {}).get('containerStatuses', []) - } - if set(statuses) != {'job-monitor', 'log-collector'}: - raise HarnessError(f'incomplete container status: {statuses}') - return statuses - - -_ARTIFACT_SCRIPT = r""" -import gzip -import json -import os - -root = "/logs" -prefix = "range-64-" -names = sorted(name for name in os.listdir(root) if name.startswith(prefix)) -out = {"files": names} -for attempt in (1, 2): - base = os.path.join(root, f"range-64-a{attempt}") - for suffix in ("metrics", "outcome"): - path = base + "." + suffix - if os.path.exists(path): - with open(path) as stream: - out[f"a{attempt}_{suffix}"] = json.load(stream) - verdict = base + ".verdict" - if os.path.exists(verdict): - with open(verdict) as stream: - out[f"a{attempt}_verdict"] = stream.read().strip() - out[f"a{attempt}_done"] = os.path.exists(base + ".done") - archive = base + ".log.gz" - if os.path.exists(archive): - try: - with gzip.open(archive, "rt", errors="replace") as stream: - out[f"a{attempt}_log"] = stream.read() - except (EOFError, OSError) as error: - out[f"a{attempt}_log_error"] = str(error) -progress = os.path.join(root, "progress.json") -if os.path.exists(progress): - with open(progress) as stream: - out["progress"] = json.load(stream) -print(json.dumps(out, sort_keys=True)) -""" - - -def artifact_bundle(release): - pod = monitor_pod(release) - if not pod: - return None - result = kubectl( - NAMESPACE, 'exec', pod['metadata']['name'], '-c', 'job-monitor', '--', - 'python3', '-c', _ARTIFACT_SCRIPT, check=False, timeout=30) - if result.returncode: - return None - try: - return json.loads(result.stdout) - except json.JSONDecodeError: - return None - - -def attempt_pod(release, attempt): - _, pods = worker_snapshot(release) - matches = [ - pod for pod in pods - if pod['metadata']['labels'].get('catchup.stellar.org/attempt') == str(attempt) - and pod.get('status', {}).get('phase') == 'Running' - ] - if len(matches) > 1: - raise HarnessError(f'more than one running pod for attempt {attempt}') - return matches[0] if matches else None - - -def assert_completed_profile(bundle): - missing = [ - name for name in ('a1_metrics', 'a1_outcome', 'a1_verdict', - 'a1_log', 'a2_metrics', 'a2_log', 'progress') - if name not in bundle - ] - if missing: - raise HarnessError(f'missing final artifacts: {missing}') - if not bundle.get('a1_done') or not bundle.get('a2_done'): - raise HarnessError('both collector .done markers must be durable') - if bundle['a1_verdict'] != 'failed': - raise HarnessError(f"attempt 1 verdict is {bundle['a1_verdict']!r}") - if bundle['a2_metrics'].get('resumed') is not True: - raise HarnessError('attempt 2 metrics lacks resumed=true') - if 'RESUME: 64/64 reached ledger 63' not in bundle['a2_log']: - raise HarnessError('attempt 2 archive lacks the true RESUME decision') - if 'RESUME DECLINED:' in bundle['a2_log']: - raise HarnessError('attempt 2 archive contains a declined resume') - - profile = (bundle['progress'].get('completed') or {}).get(str(RANGE_END)) - if not profile: - raise HarnessError('progress has no completed range 64') - if profile.get('attempts') != 2: - raise HarnessError(f"completed attempts is {profile.get('attempts')!r}, not 2") - expected_seconds = ( - float(bundle['a1_outcome']['attemptSeconds']) - + float(bundle['a2_metrics']['attemptSeconds'])) - if abs(float(profile.get('seconds', -1)) - expected_seconds) > 0.2: - raise HarnessError( - f"profile seconds {profile.get('seconds')} != chain {expected_seconds}") - for field, expected in SYNTHETIC_PEAKS.items(): - if profile.get(field) != expected: - raise HarnessError(f'profile {field}={profile.get(field)!r}, expected {expected}') - if abs(float(profile.get('txApply', -1)) - 3.75) > 1e-9: - raise HarnessError(f"profile txApply={profile.get('txApply')!r}, expected 3.75") - if any(name.startswith('range-64-a3.') for name in bundle['files']): - raise HarnessError('a third attempt artifact proves duplicate retry dispatch') - return { - 'record': profile, - 'attempt1Metrics': bundle['a1_metrics'], - 'attempt1Outcome': bundle['a1_outcome'], - 'attempt1Verdict': bundle['a1_verdict'], - 'attempt2Metrics': bundle['a2_metrics'], - 'expectedChainSecondsFromArtifacts': expected_seconds, - 'attempt1ResumeLines': [ - line for line in bundle['a1_log'].splitlines() if 'RESUME' in line], - 'attempt2ResumeLines': [ - line for line in bundle['a2_log'].splitlines() if 'RESUME' in line], - 'done': {'attempt1': bundle['a1_done'], 'attempt2': bundle['a2_done']}, - 'artifactFiles': bundle['files'], - } - - -def release_worker(release): - pod = attempt_pod(release, 2) - if not pod: - return False - result = kubectl( - NAMESPACE, 'exec', pod['metadata']['name'], '-c', 'stellar-core', '--', - 'python3', '-c', - 'from pathlib import Path; Path("/data/.synthetic-release").touch()', - check=False, timeout=30) - if result.returncode: - raise HarnessError(f'could not release successor: {result.stderr.strip()}') - return True - - -def resource_absent(kind, name): - return json_get(kind, name=name) is None - - -def cleanup(release, source_config_map, observed, evidence): - cleanup_result = {'releaseUninstalled': False, 'resourcesAbsent': {}, - 'persistentVolumesAbsent': {}} - if observed: - try: - collect_snapshot(release, evidence) - except HarnessError: - pass - - uninstall = run( - ['helm', 'uninstall', release, '--namespace', NAMESPACE, - '--wait', '--timeout', '2m'], check=False, timeout=150) - cleanup_result['releaseUninstalled'] = uninstall.returncode == 0 - - exact = [ - ('deployment', f'{release}-job-monitor'), - ('role', f'{release}-job-monitor'), - ('rolebinding', f'{release}-job-monitor'), - ('serviceaccount', f'{release}-job-monitor'), - ('serviceaccount', f'stellar-supercluster-{release}'), - ('configmap', f'{release}-stellar-core-config'), - ('configmap', f'{release}-synthetic-worker'), - ('configmap', f'{release}-catchup-progress'), - ('configmap', source_config_map), - ('pvc', f'{release}-job-monitor-logs'), - ('pvc', f'{release}-data-r{RANGE_END}'), - ] - exact.extend(('job', name) for name in sorted(evidence['jobsSeen'])) - exact.extend(('pod', name) for name in sorted(evidence['podsSeen'])) - for kind, name in exact: - kubectl( - NAMESPACE, 'delete', kind, name, '--ignore-not-found=true', - '--wait=true', '--timeout=60s', check=False, timeout=70) - - for volume in sorted(evidence['persistentVolumes']): - result = run(['kubectl', 'get', 'pv', volume, '-o', 'name'], check=False) - if result.returncode == 0: - run( - ['kubectl', 'delete', 'pv', volume, '--wait=true', '--timeout=60s'], - check=False, timeout=70) - - for kind, name in exact: - key = f'{kind}/{name}' - cleanup_result['resourcesAbsent'][key] = resource_absent(kind, name) - remaining_jobs, remaining_pods = worker_snapshot(release) - cleanup_result['selectorAbsent'] = not remaining_jobs and not remaining_pods - for volume in sorted(evidence['persistentVolumes']): - result = run(['kubectl', 'get', 'pv', volume, '-o', 'name'], check=False) - cleanup_result['persistentVolumesAbsent'][volume] = result.returncode != 0 - status = run( - ['helm', 'status', release, '--namespace', NAMESPACE], check=False) - cleanup_result['helmStatusAbsent'] = status.returncode != 0 - if not ( - cleanup_result['selectorAbsent'] - and cleanup_result['helmStatusAbsent'] - and all(cleanup_result['resourcesAbsent'].values()) - and all(cleanup_result['persistentVolumesAbsent'].values()) - ): - raise HarnessError(f'incomplete cleanup: {cleanup_result}') - return cleanup_result - - -def execute(args): - validate_scope(args.namespace, args.release) - source_config_map = f'{args.release}-source' - evidence = { - 'namespace': args.namespace, - 'release': args.release, - 'context': run(['kubectl', 'config', 'current-context']).stdout.strip(), - 'jobsSeen': set(), - 'podsSeen': set(), - 'persistentVolumes': set(), - 'maxConcurrentLiveWorkers': 0, - } - installed = False - scope_started = False - failure = None - try: - sets = helm_sets(args.release, source_config_map, args.image) - rendered = run( - ['helm', 'template', args.release, str(CHART), - '--namespace', NAMESPACE] + helm_args(sets)).stdout - Path(args.rendered).write_text(rendered) - evidence['renderedResources'] = inspect_rendered( - rendered, args.release, args.image) - create_source_config_map(args.release) - scope_started = True - - run( - ['helm', 'install', args.release, str(CHART), - '--namespace', NAMESPACE, '--timeout', '3m'] - + helm_args(sets), - timeout=210) - installed = True - - pod = wait_for( - 'one ready monitor pod', - lambda: ready_monitor_pod(args.release, evidence), timeout=120) - initial_restarts = container_restarts(pod) - evidence['restartCountsBefore'] = initial_restarts - - def successor_ready(): - collect_snapshot(args.release, evidence) - return attempt_pod(args.release, 2) - - successor = wait_for( - 'running successor attempt', successor_ready, timeout=120) - evidence['successorPod'] = successor['metadata']['name'] - - def archived_resume(): - collect_snapshot(args.release, evidence) - bundle = artifact_bundle(args.release) - if not bundle or bundle.get('a2_log_error'): - return None - return bundle if 'RESUME: 64/64 reached ledger 63' in bundle.get( - 'a2_log', '') else None - - wait_for( - 'durable successor RESUME line in gzip archive', - archived_resume, timeout=60) - - monitor = monitor_pod(args.release) - monitor_name = monitor['metadata']['name'] - before = container_restarts(monitor) - killed = kubectl( - NAMESPACE, 'exec', monitor_name, '-c', 'log-collector', '--', - '/bin/sh', '-c', 'kill -TERM 1', check=False, timeout=30) - evidence['collectorKillExitCode'] = killed.returncode - - def collector_restarted(): - current = monitor_pod(args.release) - if not current or current['metadata']['name'] != monitor_name: - raise HarnessError('monitor pod was recreated during collector restart') - counts = container_restarts(current) - if counts['job-monitor'] != before['job-monitor']: - raise HarnessError('job-monitor restarted with the collector') - if counts['log-collector'] == before['log-collector'] + 1: - return counts - if counts['log-collector'] > before['log-collector'] + 1: - raise HarnessError('collector restarted more than once') - return None - - after = wait_for( - 'exactly one collector-only restart', - collector_restarted, timeout=60) - evidence['restartCountsAfter'] = after - time.sleep(2) - if not release_worker(args.release): - raise HarnessError('successor stopped before it could be released') - - def completed(): - collect_snapshot(args.release, evidence) - bundle = artifact_bundle(args.release) - record = ((bundle or {}).get('progress', {}).get('completed') or {}).get( - str(RANGE_END)) - if record and bundle.get('a1_done') and bundle.get('a2_done'): - return bundle - return None - - bundle = wait_for( - 'completed profile and both collector done markers', - completed, timeout=120) - evidence['profileAssertions'] = assert_completed_profile(bundle) - if evidence['jobsSeen'] != { - ATTEMPT_NAMES[1](args.release), ATTEMPT_NAMES[2](args.release)}: - raise HarnessError(f"unexpected Job set {sorted(evidence['jobsSeen'])}") - if evidence['maxConcurrentLiveWorkers'] != 1: - raise HarnessError( - f"max concurrent live workers was {evidence['maxConcurrentLiveWorkers']}") - except Exception as error: - failure = error - evidence['error'] = f'{type(error).__name__}: {error}' - finally: - try: - evidence['monitorLogs'] = monitor_logs(args.release) - evidence['cleanup'] = cleanup( - args.release, source_config_map, scope_started or installed, evidence) - except Exception as cleanup_error: - evidence['cleanupError'] = ( - f'{type(cleanup_error).__name__}: {cleanup_error}') - if failure is None: - failure = cleanup_error - - for key in ('jobsSeen', 'podsSeen', 'persistentVolumes'): - evidence[key] = sorted(evidence[key]) - Path(args.evidence).write_text(json.dumps(evidence, indent=2, sort_keys=True)) - - if failure is not None: - raise failure - return evidence - - -def parse_args(argv=None): - parser = argparse.ArgumentParser() - parser.add_argument('--namespace', required=True) - parser.add_argument('--release', required=True) - parser.add_argument('--image', default='stellar/ssc-job-monitor:latest') - parser.add_argument('--evidence', required=True) - parser.add_argument('--rendered', required=True) - return parser.parse_args(argv) - - -def main(argv=None): - args = parse_args(argv) - evidence = execute(args) - print(json.dumps({ - 'release': evidence['release'], - 'restartCountsBefore': evidence['restartCountsBefore'], - 'restartCountsAfter': evidence['restartCountsAfter'], - 'record': evidence['profileAssertions']['record'], - 'cleanup': evidence['cleanup'], - }, indent=2, sort_keys=True)) - - -if __name__ == '__main__': - main() diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py deleted file mode 100644 index 9fa9ec36..00000000 --- a/src/MissionParallelCatchup/job_monitor.py +++ /dev/null @@ -1,3074 +0,0 @@ -"""Parallel catchup job monitor. - -Drives a full-history catchup by splitting the ledger range into slices and -running one Kubernetes Job per slice, then reports what happened. It owns -dispatch, retry policy and sizing, not just observation. - -State model -- nothing authoritative is held in memory: - - desired computed from config by a pure function (uniform | logarithmic) - completed durable on the shared volume (progress.json), mirrored to a - ConfigMap for the mission driver to read. Jobs are reclaimed - during a long run, so a missing Job must NOT be read as - "never ran" - in-flight live Jobs, by label selector - -Per-attempt facts -- why an attempt ended, how long it ran, what it peaked at -- -live beside progress.json as small files written by the collector sidecar. A -restarted monitor rebuilds every decision from those plus the live Job list, so -losing the process costs nothing but the time to re-list. - -Work is assigned rather than claimed: the monitor is a single writer (one -replica, Recreate), so a range's owner is decided by the range itself and never -by a race between consumers. -""" - -import gzip -import bisect -import json -import logging -import math -import os -import queue -import re -import sys -import tempfile -import threading -import time -import zlib -from datetime import datetime, timezone -from http.server import BaseHTTPRequestHandler, HTTPServer - -from kubernetes import client, config -from kubernetes.client.rest import ApiException -from prometheus_client import (CONTENT_TYPE_LATEST, REGISTRY, Counter, Gauge, - Histogram, generate_latest) -import requests - -# Histogram buckets -# 5m 15m 30m 1h 1.5h 2h -metric_buckets = (300, 900, 1800, 3600, 5400, 7200, float("inf")) - -# Configuration is grouped by who consumes the value: -# 1. stellar-core workload -- goes into the worker container or catchup args -# 2. Kubernetes objects -- shape of the Jobs, pods and PVCs we create -# 3. monitor behaviour -- never leaves this process - -# ============================================================================= -# 1. stellar-core workload -# ============================================================================= -CORE_IMAGE = os.getenv('CORE_IMAGE') -ASAN_OPTIONS = os.getenv('ASAN_OPTIONS', '') -# Test-only worker configuration. Empty is the production path; the chart sets -# these only for its fixed, opt-in synthetic integration worker. -SYNTHETIC_WORKER_CONFIG_MAP = os.getenv('SYNTHETIC_WORKER_CONFIG_MAP', '') -SYNTHETIC_WORKER_IMAGE_PULL_POLICY = os.getenv( - 'SYNTHETIC_WORKER_IMAGE_PULL_POLICY', 'IfNotPresent') -SYNTHETIC_PREDECESSOR_SECONDS = os.getenv('SYNTHETIC_PREDECESSOR_SECONDS', '12') -SYNTHETIC_SUCCESSOR_MINIMUM_SECONDS = os.getenv( - 'SYNTHETIC_SUCCESSOR_MINIMUM_SECONDS', '12') -SYNTHETIC_MAXIMUM_WAIT_SECONDS = os.getenv('SYNTHETIC_MAXIMUM_WAIT_SECONDS', '180') -SYNTHETIC_PREDECESSOR_ANON_MIB = os.getenv('SYNTHETIC_PREDECESSOR_ANON_MIB', '48') -SYNTHETIC_PREDECESSOR_WORKING_SET_MIB = os.getenv( - 'SYNTHETIC_PREDECESSOR_WORKING_SET_MIB', '56') -SYNTHETIC_SUCCESSOR_ANON_MIB = os.getenv('SYNTHETIC_SUCCESSOR_ANON_MIB', '24') -SYNTHETIC_SUCCESSOR_WORKING_SET_MIB = os.getenv( - 'SYNTHETIC_SUCCESSOR_WORKING_SET_MIB', '32') -SYNTHETIC_PREDECESSOR_TX_APPLY_MS = os.getenv( - 'SYNTHETIC_PREDECESSOR_TX_APPLY_MS', '1250') -SYNTHETIC_SUCCESSOR_TX_APPLY_MS = os.getenv( - 'SYNTHETIC_SUCCESSOR_TX_APPLY_MS', '2500') - -# Which ledger ranges to run. These are pure inputs to the range generator: -# dispatch recomputes the whole list every reconcile, so a restart must -# reproduce it exactly. -RANGE_GENERATOR = os.getenv('RANGE_GENERATOR', 'uniform') # uniform | logarithmic -# Both generators emit tip-first, which front-loads the most expensive ranges: -# the bucket set only grows with ledger position. 'oldest-first' reverses that, -# so a profiling run measures the cheap early ranges before it can be -# interrupted, and the expensive tip ranges last. -RANGE_ORDER = os.getenv('RANGE_ORDER', 'tip-first') # tip-first | oldest-first | longest-first -STARTING_LEDGER = int(os.getenv('STARTING_LEDGER', 0)) -LATEST_LEDGER_NUM = int(os.getenv('LATEST_LEDGER_NUM', 0)) -LEDGERS_PER_JOB = int(os.getenv('LEDGERS_PER_JOB', 16000)) -OVERLAP_LEDGERS = int(os.getenv('OVERLAP_LEDGERS', 320)) -# logarithmic only: chunk size halves toward the tip and stops shrinking here. -LOGARITHMIC_FLOOR_LEDGERS = int(os.getenv('LOGARITHMIC_FLOOR_LEDGERS', 64000)) - -# ============================================================================= -# 2. Kubernetes objects this monitor creates -# ============================================================================= -NAMESPACE = os.getenv('NAMESPACE', 'default') -RUN_NAME = os.getenv('RUN_NAME', 'parallel-catchup') -PROGRESS_CM = f"{RUN_NAME}-catchup-progress" -LABEL_RUN = 'catchup.stellar.org/run' -LABEL_RANGE = 'catchup.stellar.org/range-end' -LABEL_ATTEMPT = 'catchup.stellar.org/attempt' - -# Workers need IRSA to read the S3 history mirror. Without it they silently fall -# back to the public archive, which throttles at 1024 and kills the run with -# curl 22 -> catchup exit 3. The name matches the old StatefulSet's so existing -# IRSA trust policies keep matching. -WORKER_SERVICE_ACCOUNT = os.getenv('WORKER_SERVICE_ACCOUNT', '') - -# Pod resources. Requests only: workers are given no cpu limit and no memory -# limit at all. -# -# CPU because a limit only throttles a pod that could otherwise use idle cores, -# and throttling changes what the range measures -- less cpu means less download -# concurrency means a lower peak, so a throttled attempt records a figure an -# unthrottled one cannot reproduce. -# -# Memory because a limit is a hard cap on anon PLUS page cache, and sizing it -# per-range from a profile got it wrong in the one direction that has no alarm -# on it. Measured 2026-07-31, range 39210943: sized at 1729Mi from a neighbour, -# genuinely needed 1620Mi of anon, which left ~110Mi for cache. It never OOMed -# -- it thrashed. 544k major page faults, 0.22 cores used on a node it had -# entirely to itself, 0.95 ledgers/s against a neighbour norm of 3.3, and it -# held 1092 idle slots open for three hours at the end of the run. -# -# Without a limit the request still does the real work: it places the pod and -# it sets eviction order under node pressure. What goes away is the cliff. -REQ_CPU = os.getenv('REQ_CPU', '1250m') -REQ_MEM = os.getenv('REQ_MEM', '9Gi') -# Only meaningful in ephemeral storage mode; see check_storage_config(). -# Range profile from an earlier run: tightens per-range requests so more -# workers fit per node. Requests only -- limits stay as configured, so the -# failure semantics and the OOM/disk escalation ladders are unchanged. -PROFILE_PATH = os.getenv('PROFILE_PATH', '') -PROFILE_MARGIN = float(os.getenv('PROFILE_MARGIN', 1.15)) -# No safety margin on cpu, unlike memory. Under-requesting cpu costs contention -# and the pod can still burst; under-requesting memory gets it OOMKilled. -# Ceiling for profile-derived memory, above the unprofiled limit for the same -# reason: a range that really needs more than the configured limit must be able -# to ask for it rather than be pinned under its own measured peak. The OOM -# escalation ladder can still climb past this on a retry. -PROFILE_MAX_MEM = os.getenv('PROFILE_MAX_MEM', '32Gi') -# Memory is sized from rss (the range's real demand), NOT from peak working -# set. Working set is whatever limit it was measured under -- the kernel grows -# page cache to fill it -- so sizing from it is circular. Measured on ssc-test -# with one 420-ledger range: working set went 2.33 -> 3.61 -> 7.48 -> 13.49 GiB -# under 2560Mi/4Gi/8Gi/24000Mi limits while rss moved only 2256 -> 2488 MiB, and -# wall-clock did not move at all (776s / 775s / 746s / 773s). Catchup streams -- -# buckets are downloaded once, applied once, ledgers replayed once -- so cache -# has nothing to give back and PROFILE_MARGIN alone is the allowance. -# A multiplicative margin alone is not enough: memory.max bounds anon PLUS page -# cache, and at small rss 10% is nothing. Measured on ssc-test 2026-07-29 with -# headroom 0: ranges profiled at 190 MiB rss got a 209 MiB limit -- 19 MiB of -# slack for all growth and cache -- and 90 of them OOMKilled within 90s. The -# earlier 4Gi validation hid this because 1.1x of 2.4 GiB is 240 MiB of slack. -PROFILE_CACHE_HEADROOM = os.getenv('PROFILE_CACHE_HEADROOM', '512Mi') -# Extra allowance scaled by the range's measured runtime. Long ranges keep more -# page cache and allocator slack live at once; 0 disables the allowance. -PROFILE_RUNTIME_MEMORY_INSURANCE = os.getenv('PROFILE_RUNTIME_MEMORY_INSURANCE', '3Gi') - -REQ_EPHEMERAL = os.getenv('REQ_EPHEMERAL', '') -LIM_EPHEMERAL = os.getenv('LIM_EPHEMERAL', '') - -# Placement. The taint toleration is emitted as {key, effect} with no value: -# the default Equal operator does not match "" against "true". -NODE_LABEL_KEY = os.getenv('NODE_LABEL_KEY', '') -NODE_LABEL_VALUE = os.getenv('NODE_LABEL_VALUE', '') -AVOID_NODE_LABEL_KEY = os.getenv('AVOID_NODE_LABEL_KEY', '') -AVOID_NODE_LABEL_VALUE = os.getenv('AVOID_NODE_LABEL_VALUE', '') -TOLERATE_TAINT = os.getenv('TOLERATE_TAINT', '') - -# Worker /data. pvc keeps it across pods, so an evicted range resumes at L+1 -- -# that is what makes spot viable. ephemeral puts it on the node disk: denser -# packing, no resume, and REQ_EPHEMERAL must be sized to hold the catchup DB. -# One PVC per range, not per concurrency slot: measured on ssc-test, 300 jobs -# with a PVC each cost no more wall-clock than 300 jobs reusing 40. -STORAGE_MODE = os.getenv('STORAGE_MODE', 'pvc') # pvc | ephemeral -STORAGE_CLASS = os.getenv('STORAGE_CLASS', '') -STORAGE_SIZE = os.getenv('STORAGE_SIZE', '40Gi') -# A Nitro node allows ~26 EBS attachments (CSINode allocatable), and Karpenter -# sizes nodes on CPU/memory only -- it will happily put 40 volume-mounting pods -# on one 4-vCPU node, where they serialise through the attachment slots and get -# rejected with VolumeAttachmentLimitExceeded (observed on ssc-test). -# -# Guard with a spread constraint rather than a warning. maxSkew alone cannot cap -# per-node count -- with a single node there is one domain and therefore no skew -# -- so minDomains is what forces enough nodes. Both are inert at realistic -# density: REQ_CPU=1800m yields ~4 workers on an 8-vCPU node, so CPU demands far -# more nodes than this floor ever asks for. 0 disables. -MAX_VOLUMES_PER_NODE = int(os.getenv('MAX_VOLUMES_PER_NODE', 24)) - -# Job/pod lifetimes. -# SIGTERM -> SIGKILL budget. stellar-core exits ~7s after SIGTERM (measured), so -# this is slack rather than a target. -WORKER_GRACE_SECONDS = int(os.getenv('GRACE_SECONDS', 100)) -# Must comfortably exceed any plausible monitor outage: completion is recorded -# to the ConfigMap by this process, and a Job reclaimed before that happens -# reads as "never ran" and gets redone. -# Backstop only. reconcile() deletes each Job explicitly once its record is -# durable, so the TTL exists for the cases that skip that path: a terminally -# failed range kept for inspection, or a success whose metrics never landed. -JOB_TTL_SECONDS = int(os.getenv('JOB_TTL_SECONDS', 600)) -# Measured on ssc-test: stellar-core does NOT fail on an unreachable history -# archive, an absent ledger range, or a bucket that will not decompress. It -# retries every mirror with growing backoff and stays Running indefinitely -- -# no exit code, no failure, the slot held for the life of the run. A hang is a -# more likely real failure than a non-zero exit, and this deadline is the only -# thing that makes it observable. 0 disables. -# -# Flat, deliberately -- NOT scaled by the range's profiled runtime. That was -# tried and removed. A deadline has to bound a range's WORST case, but a profile -# only offers a neighbour's TYPICAL case, and the two are far apart here: -# runtimes span 190x (p25 771s, max 5.9h), range keys are anchored to the -# network tip so a profile from an earlier run matches ZERO keys exactly and -# every lookup lands on a neighbour, and ~2% of those neighbours are 3-38x -# cheaper than their surroundings. Backtested honestly across that grid offset -# (run4 profile -> r5 actuals, 3983 ranges): a 2x factor falsely kills 134 -# ranges, 4x kills 46, 6x kills 21. Flat 12h kills none. -# -# The asymmetry decides it. A false kill loses a range, and a timeout is -# terminal, so it fails the mission. A genuine wedge holds ONE slot out of -# 1092-1500 for 12h -- around 0.1% of a run's capacity. Never trade a certain -# catastrophe against a rounding error. -# -# 12h is a safe bound, not a good detector: it takes half a day to catch -# something provably dead in 4 minutes. The right signal is ledger-close -# progress, not elapsed time -- a wedged core closes zero ledgers while still -# logging, so `.state` (last log line) cannot see it and a new -# lastLedgerCloseAt would. Left undone on purpose; it needs a threshold above -# the initial bucket-apply phase, which legitimately closes nothing for ~20min -# on the longest ranges. -ATTEMPT_DEADLINE_SECONDS = int(os.getenv('ATTEMPT_DEADLINE_SECONDS', 0)) - -# kube-state-metrics turns a pod's `mission` label into label_mission, which the -# Grafana container panels join on. Every other mission gets it from -# StellarKubeSpecs; this chart never has, so parallel catchup has never appeared -# in those panels. -# -# OFF by default and deliberately so: those panels are sum() by (pod, container) -# with a legend table, so at 1024 workers they would pull ~1024 series into any -# view with mission=$__all selected, degrading a shared dashboard for people who -# did not ask for it. Enable per-run once the panels aggregate (topk). -MISSION = os.getenv('MISSION', '') -EMIT_MISSION_LABEL = os.getenv('EMIT_MISSION_LABEL', 'false').lower() == 'true' - -# ============================================================================= -# 3. This monitor's own behaviour -# ============================================================================= -PARALLELISM = int(os.getenv('PARALLELISM', 3)) -# Effectively the OOM budget: `failed` is the only other outcome that reaches -# it, and that one sets no retry reason. Escalation counts OOMs rather than -# attempts, so rung N means the range genuinely wanted more N times. -# -# Deliberately stops short of MEM_ESCALATION_CAP: 5 rungs is 1.5^4 = 5x the -# profile figure, and a range needing more than that is not mis-sized, it is -# broken -- chasing it to 48Gi parks a whole r8a.2xlarge on one range for hours. -# The cost of stopping is that the range is condemned, and today a condemned -# range aborts the run. That coupling is the thing to fix, not this number. -MAX_ATTEMPTS_PER_RANGE = int(os.getenv('MAX_ATTEMPTS', 5)) -# Evictions, admission rejections and monitor restarts say nothing about the -# ledger range, so they get their own, larger budget. Sharing MAX_ATTEMPTS with -# real failures means cluster churn can fail a healthy range: measured on -# ssc-test, ten evictions across 25 workers put four ranges on attempt 3 of 5 -# without a single genuine catchup error. -MAX_DISRUPTION_ATTEMPTS = int(os.getenv('MAX_DISRUPTION_ATTEMPTS', 20)) -# An ephemeral-storage eviction repeats identically until the range gets more -# disk, so it must not sit on the environmental budget. -MAX_EPHEMERAL_ATTEMPTS = int(os.getenv('MAX_EPHEMERAL_ATTEMPTS', 4)) -EPH_BUMP_FACTOR = float(os.getenv('EPH_BUMP_FACTOR', 1.5)) -EPH_ESCALATION_CAP = os.getenv('EPH_ESCALATION_CAP', '200Gi') -ENVIRONMENTAL_OUTCOMES = ('disrupted', 'rejected', 'unknown') -ATTEMPT_OUTCOMES = ('disrupted', 'oom', 'ephemeral', 'timeout', - 'rejected', 'unknown', 'failed') -# Verdicts only the pod can produce, and which a Job-level DeadlineExceeded must -# never overwrite. Each names a specific mechanism -- the kubelet OOM-killed it, -# the node was draining, the ephemeral limit blew -- and each earns a different -# retry budget and a different remediation. "The Job ran too long" is also true -# of every one of them and says nothing about which. An OOM downgraded to a -# timeout retries at the same memory limit that just killed it and gets 2 -# attempts instead of 5; a spot eviction downgraded to a timeout gets 2 instead -# of 20. -POD_AUTHORITATIVE_OUTCOMES = ('oom', 'disrupted', 'ephemeral', 'timeout') -# stellar-core's "did not complete". Ambiguous by construction: a corrupt bucket -# and a SIGTERM during replay both produce it, so it must never be treated as -# proof that a range is broken. -CATCHUP_INCOMPLETE_EXIT = 3 -# An OOM means requests/limits are mis-sized for this range. Escalate so the run -# can finish, but say so loudly -- surviving by escalating at runtime is a -# configuration bug, not a success. -MEM_BUMP_FACTOR = float(os.getenv('MEM_BUMP_FACTOR', 1.5)) -# Ceiling for that escalation. Above the largest schedulable node the retry sits -# Pending forever, which looks like a hang rather than a failure. -MEM_ESCALATION_CAP = os.getenv('MAX_MEM', '48Gi') - -# Reconcile loop: dispatch, refresh status, publish metrics. The env var is -# named LOGGING_INTERVAL_SECONDS for historical reasons, from when this loop -# only logged. -RECONCILE_INTERVAL_SECONDS = int(os.getenv('LOGGING_INTERVAL_SECONDS', 10)) -# /healthz fails if the loop has not ticked within this long; a wedged loop -# stops all dispatch, so restart the container rather than run half-alive. -RECONCILE_STALE_SECONDS = float(os.getenv('WATCH_STALE_SECONDS', 600)) - -# Worker responsiveness is cosmetic and sampled independently from reconcile. -# Thirty seconds and three failures restore the old ~90-second down threshold, -# while a five-second request budget gives a busy admin endpoint substantially -# more room than the old one-shot two-second probe. -LIVENESS_PROBE_INTERVAL_SECONDS = os.getenv('LIVENESS_PROBE_INTERVAL_SECONDS', '30') -LIVENESS_PROBE_TIMEOUT_SECONDS = os.getenv('LIVENESS_PROBE_TIMEOUT_SECONDS', '5') -LIVENESS_FAILURE_THRESHOLD = os.getenv('LIVENESS_FAILURE_THRESHOLD', '3') -LIVENESS_MAX_CONCURRENCY = os.getenv('LIVENESS_MAX_CONCURRENCY', '32') -try: - LIVENESS_PROBE_INTERVAL_SECONDS = float(LIVENESS_PROBE_INTERVAL_SECONDS) - LIVENESS_PROBE_TIMEOUT_SECONDS = float(LIVENESS_PROBE_TIMEOUT_SECONDS) - LIVENESS_FAILURE_THRESHOLD = int(LIVENESS_FAILURE_THRESHOLD) - LIVENESS_MAX_CONCURRENCY = int(LIVENESS_MAX_CONCURRENCY) -except ValueError as e: - raise ValueError( - "LIVENESS_PROBE_INTERVAL_SECONDS and LIVENESS_PROBE_TIMEOUT_SECONDS " - "must be numbers; LIVENESS_FAILURE_THRESHOLD and " - "LIVENESS_MAX_CONCURRENCY must be integers") from e - -for _name, _value in ( - ('LIVENESS_PROBE_INTERVAL_SECONDS', LIVENESS_PROBE_INTERVAL_SECONDS), - ('LIVENESS_PROBE_TIMEOUT_SECONDS', LIVENESS_PROBE_TIMEOUT_SECONDS), - ('LIVENESS_FAILURE_THRESHOLD', LIVENESS_FAILURE_THRESHOLD), - ('LIVENESS_MAX_CONCURRENCY', LIVENESS_MAX_CONCURRENCY)): - if _value <= 0: - raise ValueError(f"{_name} must be greater than zero, got {_value!r}") - -# Shared with the log-collector sidecar, which owns writes here: it streams each -# worker's log and records the .outcome verdict while the pod still exists. -LOG_DIR = os.getenv('LOG_DIR', '/logs') -SAVE_SUCCESS_LOGS = os.getenv('SAVE_SUCCESS_LOGS', 'true').lower() == 'true' - - -def get_logging_level(): - name_to_level = { - 'CRITICAL': logging.CRITICAL, - 'ERROR': logging.ERROR, - 'WARNING': logging.WARNING, - 'INFO': logging.INFO, - 'DEBUG': logging.DEBUG, - } - result = name_to_level.get(os.getenv('LOGGING_LEVEL', 'INFO')) - return result if result is not None else logging.INFO - - -# On the logs PVC, not the monitor's emptyDir: /data dies with the pod, and the -# mission tars /logs -- so an OOM-retry storm, the loudest signal this thing -# produces, was visible only in `kubectl logs` and never reached the run's -# destination directory. Falls back to /data if LOG_DIR is not mounted. -log_file_name = f"job_monitor_{datetime.now(timezone.utc).strftime('%Y-%m-%d_%H-%M-%S')}.log" -_log_dir = os.getenv('LOG_DIR', '/logs') -_chosen_log_dir = _log_dir if os.path.isdir(_log_dir) else '/data' -# Last resort, and unreachable in a pod: /data is the monitor's own emptyDir, so -# one of the two above always exists there. Off-cluster neither does, and a -# FileHandler on a missing directory made this module impossible to import -- -# which is why nothing here was ever tested against a real reconcile(). -if not os.path.isdir(_chosen_log_dir): - _chosen_log_dir = tempfile.gettempdir() -log_file_path = os.path.join(_chosen_log_dir, log_file_name) -logging.basicConfig(level=get_logging_level(), format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler(log_file_path), -]) -logger = logging.getLogger() - -# The env var is exactly what load_incluster_config() itself keys on, so in a pod -# this is the unconditional call it always was -- a missing token or CA still -# raises here and crash-loops the container rather than running blind. Outside a -# pod there is nothing to load and import stays pure; the caller injects clients. -if os.getenv('KUBERNETES_SERVICE_HOST'): - config.load_incluster_config() -else: - logger.warning("KUBERNETES_SERVICE_HOST is unset: no in-cluster config loaded. " - "Every API call will fail until core_v1/batch_v1 are replaced.") -# client-go's Python equivalent defaults are fine for a few LISTs per cycle, but -# dispatching ~1024 Jobs + PVCs at once needs headroom. -_cfg = client.Configuration.get_default_copy() -_cfg.connection_pool_maxsize = int(os.getenv('CONNECTION_POOL', 64)) -client.Configuration.set_default(_cfg) -core_v1 = client.CoreV1Api() -batch_v1 = client.BatchV1Api() - - -def _gib(q): - try: - return _quantity_bytes(q) / (1024 ** 3) - except Exception: - return None - - -def check_storage_config(): - """The two halves of the storage choice are set independently and can disagree. - - In ephemeral mode /data is an emptyDir on the node disk, so the - ephemeral-storage request must be large enough to hold the catchup DB and - buckets -- otherwise the kubelet evicts the pod for exceeding it. In PVC - mode the opposite is true: a large request makes disk the binding dimension - and halves workers-per-node (measured: 2/node instead of 4 on a 2xlarge). - """ - req = _gib(REQ_EPHEMERAL) if REQ_EPHEMERAL else None - if STORAGE_MODE == 'ephemeral': - if req is None or req < 20: - logger.error("STORAGE_MODE=ephemeral but ephemeral-storage request is %s. " - "/data lives on the node disk in this mode; too small a request " - "gets the pod evicted mid-catchup. Expect ~35Gi.", - REQ_EPHEMERAL or "unset") - if STORAGE_MODE == 'pvc': - # One EBS volume per worker, and a Nitro node allows ~26 attachments - # (CSINode allocatable). Density comes from the CPU request: 1800m gives - # ~4 workers on an 8-vCPU node, far below the cap. A small request packs - # many volume-mounting pods onto one node, where they serialise through - # the attachment slots -- observed on ssc-test as pods rejected with - # VolumeAttachmentLimitExceeded. Karpenter sizes on CPU/memory and does - # not provision extra nodes for attachment capacity. - try: - cpu = REQ_CPU - millis = int(cpu[:-1]) if cpu.endswith('m') else int(float(cpu) * 1000) - if millis and 8000 // millis > 20: - logger.warning("STORAGE_MODE=pvc with REQ_CPU=%s packs ~%d workers (and volumes) " - "onto an 8-vCPU node, near the ~26 EBS attachment limit. Expect " - "VolumeAttachmentLimitExceeded rejections under churn.", - REQ_CPU, 8000 // millis) - except (ValueError, ZeroDivisionError): - pass - if req is not None and STORAGE_MODE == 'pvc' and req > 10: - logger.warning("STORAGE_MODE=pvc but ephemeral-storage request is %s. /data is on " - "a PVC, so this only makes disk the binding dimension and reduces " - "workers per node. Expect ~2Gi.", REQ_EPHEMERAL) - -status = { - 'num_remain': 1, # non-zero until the first real update, so callers don't see a premature 0 - 'queue_remain_count': 0, - 'queue_succeeded_count': 0, - 'queue_failed_count': 0, - 'queue_in_progress_count': 0, - 'jobs_failed': [], - 'jobs_in_progress': [], - 'workers_refresh_duration': 0, - 'mission_duration': 0, -} -status_lock = threading.Lock() -# Heartbeat for /healthz: a wedged reconcile loop stops all dispatch, so the -# container should be restarted rather than left running half-alive. -reconcile_alive = {'ts': 0.0} - - -metric_catchup_queues = Gauge('ssc_parallel_catchup_queues', 'Exposes size of each job queues', ["queue"]) -metric_workers = Gauge('ssc_parallel_catchup_workers', 'Exposes catch up worker status', ["status"]) -metric_refresh_duration = Gauge('ssc_parallel_catchup_workers_refresh_duration_seconds', 'Time it took to refresh status of all workers') -metric_full_duration = Histogram('ssc_parallel_catchup_job_full_duration_seconds', 'Compute seconds across the complete resumed attempt chain', buckets=metric_buckets) -metric_tx_apply_duration = Histogram('ssc_parallel_catchup_job_tx_apply_duration_seconds', 'Exposes job TX apply duration as histogram', buckets=metric_buckets) -# wallSeconds is Kubernetes's startTime -> completionTime for the winning Job -# only. Failed-attempt timestamps and inter-attempt gaps were never persisted, so -# it cannot be reconstructed as first dispatch -> success after those Jobs go. -metric_wall_duration = Histogram('ssc_parallel_catchup_job_wall_duration_seconds', - 'Winning Kubernetes Job start to completion', - buckets=metric_buckets) -metric_mission_duration = Gauge('ssc_parallel_catchup_mission_duration_seconds', 'Number of seconds since the mission started ') -metric_retries = Counter( - 'ssc_parallel_catchup_job_retried_count', - 'Retry attempts dispatched after a predecessor attempt failed') -# Separates infrastructure churn from application failure: many evictions with -# zero app failures is spot behaving as intended. -metric_evictions = Counter( - 'ssc_parallel_catchup_job_spot_eviction_count', - 'Pod attempts classified as lost to node disruption') -metric_spot_disruption_retried = Counter( - 'ssc_parallel_catchup_job_spot_disruption_retried_count', - 'Unique ledger ranges that dispatched a successor after a node disruption verdict') -metric_pvc_released = Counter('ssc_parallel_catchup_pvc_released_count', 'PVCs deleted after their range completed') -metric_jobs_reaped = Counter('ssc_parallel_catchup_jobs_reaped_count', 'Finished Jobs deleted after their record was durable') -metric_oom_retries = Counter( - 'ssc_parallel_catchup_job_oom_retried_count', - 'Retry attempts dispatched after an OOM verdict, with an escalated memory limit') -metric_eph_retries = Counter( - 'ssc_parallel_catchup_job_ephemeral_retried_count', - 'Retry attempts dispatched after an ephemeral-storage verdict, with an escalated limit') -metric_retry_reasons = Counter( - 'ssc_parallel_catchup_job_retried_reason_count', - 'Retry attempts dispatched, by the effective verdict of the predecessor attempt', - ['reason']) - - -def _worker_targets(pods): - """Current Running-with-IP pods, keyed by pod identity. - - A UID change is a replacement even when the Job name or IP is reused. Tests - and unusually incomplete API objects may lack a UID, where the pod name is - still unique for its lifetime. - """ - out = {} - for pod in pods: - pod_status = getattr(pod, 'status', None) - metadata = getattr(pod, 'metadata', None) - ip = getattr(pod_status, 'pod_ip', None) - if getattr(pod_status, 'phase', None) != 'Running' or not ip or metadata is None: - continue - name = getattr(metadata, 'name', None) - identity = getattr(metadata, 'uid', None) or name - if identity and name: - out[str(identity)] = (str(name), str(ip)) - return out - - -class WorkerLivenessSampler: - """Bounded, round-robin stellar-core `/info` sampler. - - Candidate membership comes from the authoritative Kubernetes snapshot, but - all network I/O happens on this sampler's fixed worker pool. At most - `max_concurrency` requests run and the same number wait in the bounded queue; - there is no future, task, session, or thread per pod. - - State is deliberately conservative: - * new or replaced pod: unknown - * any HTTP response from /info: up - * fewer than `failure_threshold` consecutive exceptions/timeouts: unknown - * `failure_threshold` consecutive failures: down - * any later response: up immediately - - HTTP error statuses still prove the admin endpoint responded. A busy core - returning 5xx is responsive; only failure to receive an HTTP response counts - toward down. - """ - - def __init__(self, interval=LIVENESS_PROBE_INTERVAL_SECONDS, - timeout=LIVENESS_PROBE_TIMEOUT_SECONDS, - failure_threshold=LIVENESS_FAILURE_THRESHOLD, - max_concurrency=LIVENESS_MAX_CONCURRENCY, probe=None): - if interval <= 0 or timeout <= 0 or failure_threshold <= 0 or max_concurrency <= 0: - raise ValueError("liveness sampler values must all be greater than zero") - self.interval = float(interval) - self.timeout = float(timeout) - self.failure_threshold = int(failure_threshold) - self.max_concurrency = int(max_concurrency) - self._probe = probe - self._records = {} - self._generation = 0 - self._tasks = queue.Queue(maxsize=self.max_concurrency) - self._stop = threading.Event() - self._condition = threading.Condition() - self._scheduler = None - self._workers = [] - self._started = False - self._failed = None - self._active = 0 - self._failure_count = 0 - self._last_failure_log = 0.0 - - def start(self): - with self._condition: - if self._started: - return - self._started = True - self._workers = [ - threading.Thread(target=self._worker_main, - name=f"worker-liveness-{i}", daemon=True) - for i in range(self.max_concurrency) - ] - self._scheduler = threading.Thread( - target=self._scheduler_main, name="worker-liveness-scheduler", - daemon=True) - for worker in self._workers: - worker.start() - self._scheduler.start() - - def close(self): - self._stop.set() - with self._condition: - self._condition.notify_all() - threads = ([self._scheduler] if self._scheduler is not None else []) + self._workers - deadline = time.monotonic() + self.timeout + 1.0 - for thread in threads: - remaining = max(0.0, deadline - time.monotonic()) - if thread is not None and thread is not threading.current_thread(): - thread.join(remaining) - - def replace_candidates(self, targets, now=None): - """Atomically replace membership without waiting for any probe.""" - now = time.monotonic() if now is None else float(now) - targets = dict(targets) - with self._condition: - old = self._records - records = {} - new_identities = [ - identity for identity in sorted(targets) - if identity not in old or old[identity]['target'] != targets[identity] - ] - offsets = { - identity: self.interval * index / max(1, len(new_identities)) - for index, identity in enumerate(new_identities) - } - for identity, target in targets.items(): - previous = old.get(identity) - if previous is not None and previous['target'] == target: - records[identity] = previous - continue - self._generation += 1 - records[identity] = { - 'target': target, - 'generation': self._generation, - 'status': 'unknown', - 'failures': 0, - 'queued': False, - 'next_due': now + offsets[identity], - } - self._records = records - self._condition.notify_all() - - def counts(self, expected_count=None): - with self._condition: - count = len(self._records) if expected_count is None else int(expected_count) - healthy = self._started and self._failed is None - if healthy: - healthy = (self._scheduler is not None and self._scheduler.is_alive() - and all(worker.is_alive() for worker in self._workers)) - if not healthy or count != len(self._records): - return {'up': 0, 'down': 0, 'unknown': count} - result = {'up': 0, 'down': 0, 'unknown': 0} - for record in self._records.values(): - result[record['status']] += 1 - return result - - def stats(self): - """Small observability hook used by the scale contract test.""" - with self._condition: - live_threads = sum( - 1 for thread in ([self._scheduler] + self._workers) - if thread is not None and thread.is_alive()) - return { - 'records': len(self._records), - 'active': self._active, - 'queued': self._tasks.qsize(), - 'outstanding': self._active + self._tasks.qsize(), - 'threads': live_threads, - 'failed': self._failed, - } - - def _scheduler_main(self): - try: - self._schedule() - except Exception as e: - self._mark_failed("scheduler", e) - - def _schedule(self): - while not self._stop.is_set(): - with self._condition: - now = time.monotonic() - capacity = self.max_concurrency - self._tasks.qsize() - due = sorted( - ((record['next_due'], identity, record) - for identity, record in self._records.items() - if not record['queued'] and record['next_due'] <= now), - key=lambda item: (item[0], item[1])) - for _, identity, record in due[:max(0, capacity)]: - task = (identity, record['generation'], record['target']) - try: - self._tasks.put_nowait(task) - except queue.Full: - break - record['queued'] = True - - waiting = [ - record['next_due'] for record in self._records.values() - if not record['queued'] - ] - delay = max(0.01, min(1.0, min(waiting) - now)) if waiting else 1.0 - self._condition.wait(timeout=delay) - - def _worker_main(self): - session = None - try: - if self._probe is None: - session = requests.Session() - adapter = requests.adapters.HTTPAdapter( - pool_connections=4, pool_maxsize=1, max_retries=0) - session.mount('http://', adapter) - while not self._stop.is_set(): - try: - task = self._tasks.get(timeout=0.2) - except queue.Empty: - continue - with self._condition: - self._active += 1 - identity, generation, target = task - success = False - error = None - try: - _, ip = target - if self._probe is None: - host = f"[{ip}]" if ':' in ip else ip - with session.get(f"http://{host}:11626/info", - timeout=self.timeout): - pass - else: - self._probe(ip, self.timeout) - success = True - except Exception as e: - error = e - finally: - self._record_result(identity, generation, target, success, error) - self._tasks.task_done() - with self._condition: - self._active -= 1 - self._condition.notify_all() - except Exception as e: - self._mark_failed("probe worker", e) - finally: - if session is not None: - session.close() - - def _record_result(self, identity, generation, target, success, error=None, - now=None): - now = time.monotonic() if now is None else float(now) - log_failure = None - with self._condition: - record = self._records.get(identity) - if (record is None or record['generation'] != generation - or record['target'] != target): - return - record['queued'] = False - record['next_due'] = now + self.interval - if success: - record['failures'] = 0 - record['status'] = 'up' - else: - record['failures'] += 1 - record['status'] = ( - 'down' if record['failures'] >= self.failure_threshold - else 'unknown') - self._failure_count += 1 - if now - self._last_failure_log >= 60.0: - log_failure = self._failure_count - self._failure_count = 0 - self._last_failure_log = now - self._condition.notify_all() - if log_failure is not None: - logger.warning( - "stellar-core /info liveness probes are failing; %d failure(s) " - "across the fleet since the previous warning (latest: %s: %s)", - log_failure, target[0], error) - - def _mark_failed(self, component, error): - with self._condition: - if self._failed is not None: - return - self._failed = f"{component}: {error}" - self._condition.notify_all() - logger.exception( - "worker liveness %s failed; all current workers will be reported " - "unknown and reconcile will continue", component) - - -worker_liveness_sampler = WorkerLivenessSampler() - - -def publish_worker_liveness(targets, sampler=None): - """Hand a pod snapshot to the sampler and return its current three counts. - - This path copies O(current workers) state under a short lock but never makes - a request or waits for an in-flight request. Keeping it separate makes the - non-blocking boundary directly testable. - """ - sampler = sampler or worker_liveness_sampler - sampler.replace_candidates(targets) - return sampler.counts(len(targets)) - - -worker_liveness_sampler = WorkerLivenessSampler() - - -class RequestHandler(BaseHTTPRequestHandler): - def do_GET(self): - if self.path == '/healthz': - stale = time.time() - reconcile_alive['ts'] - ok = reconcile_alive['ts'] > 0 and stale < RECONCILE_STALE_SECONDS - self.send_response(200 if ok else 503) - self.send_header('Content-type', 'application/json') - self.end_headers() - self.wfile.write(json.dumps({'reconcile_age_seconds': round(stale, 1)}).encode()) - elif self.path == '/status': - self.send_response(200) - self.send_header('Content-type', 'application/json') - self.end_headers() - with status_lock: - self.wfile.write(json.dumps(status).encode()) - elif self.path == '/prometheus': - self.send_response(200) - self.send_header('Content-type', CONTENT_TYPE_LATEST) - self.end_headers() - self.wfile.write(generate_latest(REGISTRY)) - else: - self.send_response(404) - self.end_headers() - - def log_message(self, *args): - pass # the default handler logs every request to stderr - - -# --- range generation ------------------------------------------------------- -# Ports uniform_range_generator.sh and logarithmic_range_generator.sh. These -# must stay pure functions of config: dispatch derives the full range list on -# every reconcile, so a restart has to reproduce it exactly. - -def _uniform_segment(start_ledger, end_ledger, seg_size): - """Ranges over (start_ledger, end_ledger], largest ledger first.""" - out = [] - el = end_ledger - while el > start_ledger: - ledgers_per_job = min(el - start_ledger, seg_size) - out.append((el, ledgers_per_job + OVERLAP_LEDGERS)) - el -= ledgers_per_job - return out - - -def _ordered(ranges): - """Dispatch order. Generators emit tip-first; reverse for oldest-first. - - 'longest-first' is the one that shortens the run. Makespan is bounded below - by the single longest job, so every range that starts after it is free and - every hour it starts late is an hour on the end. That is classic - longest-processing-time scheduling. - - tip-first only approximates it. Position predicts cost on average and badly - in the tail: measured 2026-07-30, ranges at 41-45M ran as long as the tip - (3.1h) on a third of the memory, and the 50-60M band is CHEAPER than 40-50M. - Sorting on the profile's own measured seconds uses the real number instead - of a proxy for it. - - A range the profile has never seen sorts FIRST. profile_for returns the - nearest measured end ABOVE the target, so an unprofiled range is by - construction newer than anything ever measured -- the newest ranges are the - most expensive, so "unknown" means "assume worst", not "assume average". - That also makes the next profile better: those ranges run early, under the - most generous sizing, instead of being the ones a run dies before reaching. - """ - if RANGE_ORDER == 'oldest-first': - return list(reversed(ranges)) - if RANGE_ORDER != 'longest-first': - return ranges - def cost(item): - prof = profile_for(item[0]) - secs = (prof or {}).get('seconds') - # None sorts first; ties keep tip-first order, which is the better guess - # among ranges the profile cannot separate. - return (0 if secs is None else 1, -(secs or 0)) - return sorted(ranges, key=cost) - - -def generate_ranges(): - if RANGE_GENERATOR == 'uniform': - return _ordered(_uniform_segment(STARTING_LEDGER, LATEST_LEDGER_NUM, LEDGERS_PER_JOB)) - - # Logarithmic: early history is cheap per ledger, so use big chunks there and - # halve the chunk size as we approach the tip. Aims for roughly equal - # wall-time per job rather than equal ledger count. - out = [] - start_ledger = STARTING_LEDGER - end_ledger = LATEST_LEDGER_NUM // 2 - chunk = (end_ledger - start_ledger + 1) // max(PARALLELISM, 1) - while chunk > LOGARITHMIC_FLOOR_LEDGERS: - out.extend(_uniform_segment(start_ledger, end_ledger, chunk)) - start_ledger = end_ledger + 1 - chunk //= 2 - end_ledger = start_ledger + (chunk * PARALLELISM) - out.extend(_uniform_segment(end_ledger + 1, LATEST_LEDGER_NUM, LOGARITHMIC_FLOOR_LEDGERS)) - return _ordered(out) - - -def job_key(end, count): - return f"{end}/{count}" - - -def job_name(end, attempt): - return f"{RUN_NAME}-r{end}-a{attempt}" - - -# --- durable progress record ------------------------------------------------ -# Jobs get reclaimed during a 10h run, so completion cannot live only in Job -# objects. Written BEFORE a Job becomes TTL-eligible. - -# Set once at startup; the same ConfigMap the Jobs and PVCs hang off. -_progress_owner = {} - - -# The authoritative copy of the progress record lives on the logs PVC, not in -# the ConfigMap. A ConfigMap is capped at 1 MiB and this record is ~172 bytes -# per completed range, so it dies at ~6100 ranges -- reachable simply by halving -# ledgersPerJob. Measured mid-run on ssc-test: 348KB at 2024 completed ranges, -# which projects to ~65% of the cap at 3982 -- close enough that the next -# slicing change would have hit it. Worse, every completion rewrote the whole -# document through the API server, so a full run meant thousands of -# escalating-size etcd writes. -# -# The ConfigMap is still written, because the mission driver reads it without -# exec'ing into the pod, but it is now a best-effort mirror: if it fails, the -# run carries on from the file. -PROGRESS_FILE = os.path.join(LOG_DIR, 'progress.json') - - -def _sane_progress(progress): - """Drop anything in the record that is not a range -> record mapping. - - This document is read off a volume that outlives the run and is mirrored - through a ConfigMap a second writer can clobber, so it comes back - structurally wrong as well as merely truncated. A truncated file raises - ValueError and is already handled; one that parses into the wrong SHAPE was - not. A single non-dict entry took every later pass down inside - observe_recorded/sync_counters -- after dispatch, so the exception the - reconcile loop swallows left the run with no status update and no - `remaining` ever again. - - Corrupt is not progress, so an unreadable entry is dropped rather than - counted: the range is re-run, which is idempotent, and the - monotonic-progress guard still fires if dropping one shrinks a record that - was larger a pass ago. - """ - if not isinstance(progress, dict): - return {} - out = dict(progress) - for bucket in ('completed', 'failed'): - entries = out.get(bucket) - if entries is None: - continue - out[bucket] = ({k: v for k, v in entries.items() if isinstance(v, dict)} - if isinstance(entries, dict) else {}) - return out - - -def _rehydrate_from_metrics(progress): - """Put the measurements back into a record that came from the mirror. - - The two stores have different jobs. The ConfigMap is the control plane: it - is what the mission driver reads to follow the run and decide whether to - fail it, and it is capped at 1 MiB, so `_state_only` strips every - measurement out of it. The volume is the data plane: it holds the - per-attempt `.metrics` files and the profile built from them. - - Loading the mirror therefore yields a record that is complete as *state* - and empty as *data* -- and the next save wrote that back over the volume, - which is how a finished run produced a profile with `attempts` and `count` - and nothing else. The measurements were never actually lost: only - progress.json was damaged, and `.metrics` is written per attempt and never - rewritten. So re-read them rather than persist the hole. - """ - completed = progress.get('completed') or {} - if not completed: - return progress - recovered = repair_completed_profiles(progress) - logger.warning("progress.json was unreadable; recovered state from the ConfigMap " - "mirror and reconstructed measurements for %d of %d completed ranges " - "from attempt artifacts on the volume", recovered, len(completed)) - return progress - - -def load_progress(): - try: - with open(PROGRESS_FILE) as fh: - return _sane_progress(json.load(fh)) - except (OSError, ValueError): - pass - # First start on this volume, or an older run that only had the ConfigMap. - try: - cm = core_v1.read_namespaced_config_map(PROGRESS_CM, NAMESPACE) - mirrored = _sane_progress(json.loads((cm.data or {}).get('progress.json', '{}'))) - except ApiException as e: - if e.status == 404: - return {} - raise - return _rehydrate_from_metrics(mirrored) - - -def save_status(snapshot): - """Publish /status into the ConfigMap as well. - - The mission driver runs outside the cluster and already has a kube client, - so reading a ConfigMap is simpler and more robust than exposing the monitor - through a Gateway/HTTPRoute just to be polled. Shape is identical to the - HTTP /status body, so the driver's parser is unchanged. - """ - _patch_cm({'status.json': json.dumps(snapshot, separators=(',', ':'))}) - - -# Measurements live only on the volume. The ConfigMap is the mission-state -# mirror the driver reads for visibility, and at ~172 bytes per range the -# profiling fields alone push it toward the 1 MiB cap at ~6100 ranges. Stripped -# to attempts/count it is ~30 bytes, so state stays readable at any slicing -# while the profile has no ceiling at all. -# A strip list, not a produce list: peakCpuCores is no longer measured, but a -# progress record resumed from an older run still carries it, and letting it -# through is what pushes the ConfigMap mirror toward the 1 MiB cap. -_PROFILE_ONLY_FIELDS = ('peakAnonBytes', 'peakRssBytes', 'peakWorkingSetBytes', 'peakCpuCores', - 'peakEphemeralBytes', 'txApply', 'seconds', 'wallSeconds') - - -def _state_only(progress): - out = dict(progress) - completed = {} - for end, rec in (progress.get('completed') or {}).items(): - completed[end] = {k: v for k, v in rec.items() - if k not in _PROFILE_ONLY_FIELDS} - out['completed'] = completed - return out - - -def _write_atomic(path, body, opener=None): - """Write `body` through tmp+rename so a reader never sees a partial file. - - Every state file here is read back by a restarted monitor, so a torn write - is indistinguishable from corruption: the .outcome and .verdict files decide - a range's remaining budget, and progress.json decides what gets dispatched. - """ - tmp = path + '.tmp' - # `opener` resolves at call time, not as a default argument, so the write - # seam stays patchable -- the tmp+rename discipline is only worth having if - # a test can crash a write mid-flight and prove the real path is untouched. - with (opener or open)(tmp, 'wt') as fh: - fh.write(body) - os.replace(tmp, path) - - -def save_progress(progress): - blob = json.dumps(progress, separators=(',', ':')) - # File first and atomically: it is what a restart reads back. - _write_atomic(PROGRESS_FILE, blob) - # Mirror for the driver. Never fatal -- a 413 here used to throw inside - # reconcile, and the loop swallows exceptions, so no completion would ever - # be recorded again and every finished range would be dispatched forever. - try: - _patch_cm({'progress.json': json.dumps(_state_only(progress), - separators=(',', ':'))}) - except ApiException as e: - logger.warning("progress ConfigMap mirror failed (%s); the record on %s " - "is authoritative and the run continues", e.status, PROGRESS_FILE) - - -def _patch_cm(data, owner=None): - body = {'data': data} - try: - core_v1.patch_namespaced_config_map(PROGRESS_CM, NAMESPACE, body) - except ApiException as e: - if e.status != 404: - raise - core_v1.create_namespaced_config_map(NAMESPACE, client.V1ConfigMap( - # Owned by the chart's stellar-core ConfigMap like the Jobs and - # PVCs, so `helm uninstall` reclaims it. Without an owner this - # outlived every run and accumulated in the shared namespace. - metadata=client.V1ObjectMeta(name=PROGRESS_CM, labels={LABEL_RUN: RUN_NAME}, - owner_references=_progress_owner.get('ref')), - data=body['data'])) - - -# --- worker log capture ----------------------------------------------------- - -def backstop_save_pod_log(pod_name, end, attempt): - """Last-resort archive for a range the collector never streamed. - - The log-collector sidecar owns /logs in the normal case: it holds a - follow=true stream from pod start, so nothing is lost when the node is - reaped. This only covers the gap where a pod lived and died entirely while - the collector was down -- detected by the absence of the state file the - collector writes when it claims a range. - - Never writes over a claimed or existing archive: two writers appending to - one gzip would interleave members and duplicate lines. - """ - if os.path.exists(state_path(end, attempt)): - return True # collector has it (streaming or already finished) - path = log_path(end, attempt) - if os.path.exists(path): - return True - try: - body = core_v1.read_namespaced_pod_log(pod_name, NAMESPACE, container='stellar-core') - except ApiException as e: - logger.warning("could not save log for range %s attempt %d (pod %s): %s", - end, attempt, pod_name, e.reason) - return False - try: - os.makedirs(LOG_DIR, exist_ok=True) - _write_atomic(path, body, gzip.open) - return True - except OSError as e: - logger.warning("could not write %s: %s", path, e) - return False - - -def log_path(end, attempt): - """Canonical archive name, written by the log-collector sidecar. - - Deliberately carries no ok/failed suffix: which ranges failed is recorded in - the progress ConfigMap, and encoding it here would mean two components - disagreeing about a filename. - """ - return os.path.join(LOG_DIR, f"range-{end}-a{attempt}.log.gz") - - -def state_path(end, attempt): - return os.path.join(LOG_DIR, f"range-{end}-a{attempt}.state") - - -def outcome_path(end, attempt): - return os.path.join(LOG_DIR, f"range-{end}-a{attempt}.outcome") - - -def classify(pod): - """Why did this pod fail? The Job object cannot answer this. - - Job.status only carries a Failed condition with reason BackoffLimitExceeded - -- no exit code, no OOM. The detail lives on the pod, which is exactly the - object Karpenter deletes with the node, so this is recorded the moment the - watch sees it rather than when reconcile next runs. - """ - for cond in (pod.status.conditions or []): - if cond.type == 'DisruptionTarget' and cond.status == 'True': - return {'outcome': 'disrupted', 'exitCode': None} - # Kubelet can reject a pod before any container runs -- observed on - # ssc-test: reason=VolumeAttachmentLimitExceeded, "Node has reached its - # volume attachment limit, rejecting pod". There is no exit code and no - # DisruptionTarget, so without this it falls through to 'failed' and a - # transient admission rejection kills the whole run. - if pod.status.reason == 'Evicted' and 'ephemeral' in (pod.status.message or ''): - # Measured on ssc-test: the kubelet sets no DisruptionTarget for a - # limit eviction, and stellar-core drains on the eviction SIGTERM and - # exits 3 -- so the Job condition matches the generic non-zero rule and - # reads as a plain catchup failure, which gets no retry at all. - # status.message is the only discriminator and only the pod carries it. - return {'outcome': 'ephemeral', 'exitCode': None, 'reason': pod.status.message} - if pod.status.reason in ('VolumeAttachmentLimitExceeded', 'OutOfcpu', 'OutOfmemory', - 'OutOfpods', 'UnexpectedAdmissionError', 'NodeAffinity', - 'Shutdown', 'Evicted'): - return {'outcome': 'rejected', 'exitCode': None, 'reason': pod.status.reason} - if pod.status.reason == 'DeadlineExceeded': - # The deadline lives on the PodSpec, so it is the kubelet that fires it - # and the pod that carries the reason -- the Job just sees a non-zero - # exit through its podFailurePolicy. Without this the drain-to-exit-3 - # matches the generic rule and reads as a plain catchup failure. - return {'outcome': 'timeout', 'exitCode': None, 'reason': pod.status.reason} - started = any(cs.state and cs.state.terminated for cs in (pod.status.container_statuses or [])) - if not started: - # No container ever reached a terminal state: nothing ran, so this is - # not evidence about the ledger range. - return {'outcome': 'rejected', 'exitCode': None, - 'reason': pod.status.reason or 'no container status'} - for cs in (pod.status.container_statuses or []): - t = cs.state.terminated if cs.state else None - if t is None: - continue - # 137 is SIGKILL, which the kubelet also uses for a graceful-stop - # timeout -- but with reason OOMKilled it is unambiguous. - if t.reason == 'OOMKilled': - return {'outcome': 'oom', 'exitCode': t.exit_code} - if t.exit_code not in (0, None): - return {'outcome': 'failed', 'exitCode': t.exit_code} - return {'outcome': 'failed', 'exitCode': None} - - -def record_outcome(end, attempt, pod): - path = outcome_path(end, attempt) - if os.path.exists(path): - return - data = classify(pod) - data['pod'] = pod.metadata.name - # The only place a failed attempt's duration is ever available: the pod is - # about to be reaped, and reconcile computes `seconds` solely on the success - # path. Without it a resumed chain can only report its final leg. - data['attemptSeconds'] = _pod_seconds(pod) - try: - _write_atomic(path, json.dumps(data)) - except OSError as e: - logger.warning("could not persist outcome for range %s: %s", end, e) - - -# The Job controller writes the exit code and pod name into the failure -# condition message, e.g. -# "Container stellar-core for pod ns/kic-r400000-a1-xxxxx failed with exit -# code 137 matching FailJob rule at index 1" -# Jobs are not bound to a node, so unlike the pod this survives consolidation. -_JOB_MSG = re.compile(r"for pod \S+?/(?P\S+) failed with exit code (?P\d+)") -_JOB_RULE = re.compile(r"rule at index (?P\d+)") - - -def _failure_rules(): - """podFailurePolicy rules, in evaluation order, tagged with what they mean. - - First match wins, so reaching the exit-137 rule proves DisruptionTarget did - not match -- that ordering is what separates an OOM kill from a - grace-period SIGKILL after the pod is gone. - - All FailJob: the Job must fail with reason=PodFailurePolicy so the message - names the rule index. A Count action would surface as BackoffLimitExceeded - and lose the signal. Retries stay with the monitor because raising a memory - limit needs a new Job -- spec.template is immutable. - """ - return [ - ('disrupted', client.V1PodFailurePolicyRule( - action='FailJob', - on_pod_conditions=[client.V1PodFailurePolicyOnPodConditionsPattern( - type='DisruptionTarget', status='True')])), - ('oom', client.V1PodFailurePolicyRule( - action='FailJob', - on_exit_codes=client.V1PodFailurePolicyOnExitCodesRequirement( - container_name='stellar-core', operator='In', values=[137]))), - ('failed', client.V1PodFailurePolicyRule( - action='FailJob', - on_exit_codes=client.V1PodFailurePolicyOnExitCodesRequirement( - container_name='stellar-core', operator='NotIn', values=[0]))), - ] - - -# Order here is the contract with the Job controller's "rule at index N". -RULE_ORDER = ['disrupted', 'oom', 'failed'] -_RULE_OUTCOME = dict(enumerate(RULE_ORDER)) - - -def classify_from_job(job): - """Recover a verdict from the Job when the pod is already gone. - - Rule index is the signal, not the exit code: rules are evaluated - first-match-wins, so reaching the exit-137 rule proves the DisruptionTarget - rule did not match, which is the only way to tell an OOM kill from a - grace-period SIGKILL once the pod is gone. - - Index and exit code are parsed independently -- a rule matching on - onPodConditions reports no exit code at all, so requiring one would make the - disruption case unreadable. - """ - for cond in (job.status.conditions or []): - if cond.type != 'Failed' or cond.status != 'True': - continue - msg = cond.message or '' - if cond.reason == 'DeadlineExceeded': - # activeDeadlineSeconds fired: the attempt hung rather than failing. - # Retryable -- a genuinely stuck range will exhaust its attempts. - return {'outcome': 'timeout', 'exitCode': None, 'pod': '', - 'source': 'job-condition'} - if cond.reason != 'PodFailurePolicy': - # e.g. BackoffLimitExceeded -- carries no per-rule detail. - continue - rule = _JOB_RULE.search(msg) - detail = _JOB_MSG.search(msg) - outcome = _RULE_OUTCOME.get(int(rule.group('idx'))) if rule else None - code = int(detail.group('code')) if detail else None - if outcome is None: - if code is None: - return None - # No usable rule index. Measured on ssc-test (2026-07-28): a drained - # stellar-core catches SIGTERM and exits 3 in ~7s, well inside the - # 100s grace -- evictions do NOT produce 137. So a bare 137 is an OOM - # with high confidence, and exit 3 without a DisruptionTarget - # condition really is a catchup failure. - outcome = 'oom' if code == 137 else 'failed' - return {'outcome': outcome, 'exitCode': code, - 'pod': detail.group('pod') if detail else '', - 'source': 'job-condition'} - return None - - -def read_outcome(end, attempt): - try: - with open(outcome_path(end, attempt)) as fh: - return json.load(fh) - except (OSError, ValueError): - return None - - -def _oom_count(end, attempt): - """How many earlier attempts at this range were OOM-killed. - - Escalation must climb once per OOM, not once per attempt. On spot most - retries are evictions -- measured on ssc-test 2026-07-30, 288 disruption - retries against 7 OOM retries -- and a range disrupted three times then - OOMing once would otherwise jump to base * 1.5^4, a 5x request for a single - OOM. That inflation is fleet-wide and it is what exhausts the vCPU quota. - """ - return sum(1 for n in range(1, int(attempt) + 1) - if _verdict_of(end, n) == 'oom') - - -def verdict_path(end, attempt): - return os.path.join(LOG_DIR, f"range-{end}-a{attempt}.verdict") - - -def save_verdict(end, attempt, outcome): - """Persist the EFFECTIVE verdict for one attempt, so budgets can be tallied. - - The .outcome file is not enough on its own: it is classified from the pod, - and a deadline kill reads as a plain exit-3 `failed` there -- only the Job's - DeadlineExceeded condition says `timeout`. Reconcile resolves that conflict - once, and this is where the answer is kept, on the same durable logs volume - as everything else, so a monitor restart does not reset a range's budgets. - """ - path = verdict_path(end, attempt) - try: - _write_atomic(path, str(outcome)) - except OSError as e: - logger.warning("could not persist verdict for range %s attempt %s: %s", - end, attempt, e) - - -def _verdict_of(end, attempt): - try: - with open(verdict_path(end, attempt)) as fh: - verdict = fh.read().strip() - except OSError: - # Pre-fix runs, or an attempt whose verdict write lost the volume: - # the pod-derived classification is the next best thing. - outcome = (read_outcome(end, attempt) or {}).get('outcome') - return outcome if outcome in ATTEMPT_OUTCOMES else None - return verdict if verdict in ATTEMPT_OUTCOMES else None - - -def _cause_count(end, attempt, causes): - """How many of attempts 1..N at this range failed for one of `causes`. - - Budgets are per cause, not per attempt. One shared attempt index meant - cluster churn -- which has its own deliberately large budget -- drained the - small budgets belonging to the causes that say something about the range: a - range evicted MAX_ATTEMPTS times had an effective OOM and disk budget of - zero, was condemned on its first real OOM without ever being escalated, and - took the whole mission with it. - """ - return sum(1 for n in range(1, int(attempt) + 1) - if _verdict_of(end, n) in causes) - - -def mem_for_attempt(attempt, base=None): - """Memory REQUEST after N OOMs, capped at MEM_ESCALATION_CAP. - - `base` is what attempt 1 actually ran with. It matters when a profile sized - the range: escalating a 209Mi profiled range off the configured default - jumps straight to 36000Mi, a 172x overshoot that throws away the whole - packing win on the first OOM. - - Escalating the request, not a limit, because there is no limit any more. It - still buys the same two things an OOMing range needs -- placement somewhere - with the memory actually free, and a higher bar before the kubelet picks it - as an eviction victim. - """ - base_q = _quantity_bytes(base or REQ_MEM) - want = int(base_q * (MEM_BUMP_FACTOR ** max(0, attempt - 1))) - cap = _quantity_bytes(MEM_ESCALATION_CAP) - return _bytes_to_quantity(min(want, cap)) - - -_UNITS = {'Ki': 1024, 'Mi': 1024**2, 'Gi': 1024**3, 'Ti': 1024**4, - 'K': 1000, 'M': 1000**2, 'G': 1000**3, 'T': 1000**4} - - -def _quantity_bytes(q): - for suffix, mult in sorted(_UNITS.items(), key=lambda kv: -len(kv[0])): - if q.endswith(suffix): - return int(float(q[:-len(suffix)]) * mult) - return int(float(q)) - - -def _bytes_to_quantity(n): - return f"{max(1, n // (1024 ** 2))}Mi" - - - - -# --- tx_apply --------------------------------------------------------------- - -# medida prints the sum in scientific notation once it exceeds 1e6 ms, which is -# every range that applies a real transaction load. The old [0-9.]+ pattern -# matched "1.30722" then demanded "ms" and hit "e+06ms" instead, so tx_apply was -# silently missing for 25% of ranges -- 91-99% of everything above ledger 35M, -# exactly the expensive end. 698 completed ranges lost the metric that way in a -# single run, which is the reference case for "a recoverable gap turned -# permanent" everywhere else in this file. -_SUM_RE = re.compile(r"sum\s*=\s*([0-9.]+(?:[eE][+-]?[0-9]+)?)ms") - - -def metrics_path(end, attempt): - return os.path.join(LOG_DIR, f"range-{end}-a{attempt}.metrics") - - -# A PVC's size is not a scheduling dimension -- growing it buys no packing, so -# it is not profiled; the only volume ceiling that matters is the ~26 -# attachments per node. Ephemeral storage IS a scheduling dimension, so it is, -# but only in ephemeral mode and only on on-demand nodes -- see the collector. -# Any field may be absent and the consumer falls back to its default. -PEAK_FIELDS = ('peakAnonBytes', 'peakRssBytes', 'peakWorkingSetBytes', - 'peakEphemeralBytes') - - -def peaks_for_range(end, attempt=1): - """Highest peak any attempt at this range reached, per axis. - - Not just the successful attempt. In pvc mode a pod that dies once replay has - started leaves /data behind, and the next attempt resumes at LCL+1 with - RESUME=true -- skipping the archive download and the bucket apply, which is - where peak memory actually happens. Its peak describes the tail of the range, - not the range, so profiling the winner alone under-reports by the whole - download-vs-replay gap. On spot, where eviction is routine and resume is the - entire point of durable /data, that would make the run unprofileable. - - Attempts that hit a ceiling are counted too. A pod OOM-killed at 8Gi really - did allocate ~8Gi and wanted more, so its peak is a lower bound on demand, - not an artifact of the limit -- and it is the attempt most worth keeping, - because download concurrency scales with available cpu and a pod that - bursted on an idle node can peak above the one that eventually succeeded. - Sizing off the quieter attempt would OOM the range again. There is no false - ratchet: a pod given 8Gi that only touches 1Gi records 1Gi. - - Advisory: used to size a LATER run's requests, never to decide anything - about this one. Any field may be absent. - """ - out = {} - for n in _peak_attempts(end, attempt): - try: - with open(metrics_path(end, n)) as fh: - data = json.load(fh) - except (OSError, ValueError): - continue - for k in PEAK_FIELDS: - v = data.get(k) - if v is not None and v > out.get(k, 0): - out[k] = v - return out - - -def _pod_seconds(pod): - """Container start -> finish for one attempt, or None if unreadable.""" - start = pod.status.start_time if pod.status else None - if start is None: - return None - for cs in (pod.status.container_statuses or []): - t = cs.state.terminated if cs.state else None - if t is not None and t.finished_at: - return (t.finished_at - start).total_seconds() - return None - - -def _hit_a_ceiling(end, attempt): - """Was this attempt killed at one of its own resource limits?""" - return (read_outcome(end, attempt) or {}).get('outcome') in ('oom', 'ephemeral') - - -def _peak_attempts(end, attempt): - """Attempts whose peaks describe this range: the resumed chain, plus any - attempt that died at a limit, wherever it sits. - - A ceiling-hit peak is evidence about the range no matter which pass - produced it -- the process really did allocate that much and want more, so - it is a lower bound on demand and the next run must size above it. That is - the whole self-correcting loop: a range that OOMs at L records L, and - L * PROFILE_MARGIN + PROFILE_CACHE_HEADROOM clears it next time. - - Without this the fresh-start rule silently drops it. Measured on ssc-test - 2026-07-30: an OOM during replay resumes (RESUME accepted, 224 of 252) and - stays in the chain, but an OOM during download does not (25 of 252) -- and - a run at higher cpu is download-bound, so the loop would go quiet exactly - when it is most needed. - - Peaks only. tx_apply and seconds are summed, and a fresh start redoes work - the dropped attempt already did, so including it there would double-count. - """ - chain = set(_resumed_chain(end, attempt)) - return sorted(chain | {n for n in range(1, int(attempt) + 1) - if n not in chain and _hit_a_ceiling(end, n)}) - - -def _resumed_chain(end, attempt): - """Attempts describing one continuous pass over the range, oldest first. - - Stops at the last attempt that ran new-db: that one covered the whole range - on its own, so nothing before it is part of the same pass. - """ - first = int(attempt) - while first > 1 and _attempt_resumed(end, first): - first -= 1 - return range(first, int(attempt) + 1) - - -def _attempt_resumed(end, attempt): - """Did this attempt pick up at LCL+1 rather than run new-db? - - Prefer the collector's marker, then recover it from the durable archive for - legacy files and pollers recreated after the early RESUME line. - """ - try: - with open(metrics_path(end, attempt)) as fh: - if json.load(fh).get('resumed') is True: - return True - except FileNotFoundError: - pass - except ValueError as e: - logger.warning("could not parse resume metrics for range %s attempt %s: %s", - end, attempt, e) - except OSError as e: - logger.warning("could not read resume metrics for range %s attempt %s: %s", - end, attempt, e) - return _archive_resumed(end, attempt) - - -def _archive_resumed(end, attempt): - """Read the exact worker resume decision from concatenated gzip members.""" - path = log_path(end, attempt) - try: - with gzip.open(path, 'rt', errors='replace') as fh: - return any('RESUME: ' in line for line in fh) - except FileNotFoundError: - return False - except (EOFError, gzip.BadGzipFile, zlib.error) as e: - logger.warning("could not read resume decision from %s: %s", path, e) - return False - except OSError as e: - logger.warning("could not open resume archive %s: %s", path, e) - return False - - -def tx_apply_for_range(end, attempt=1, pod_name=None): - """Exact known 'ledger.transaction.apply' seconds for the whole range. - - Summed across the resumed chain, not read from the winning attempt alone. - medida's total is per-process, so a pod that resumes at LCL+1 reports only - the transactions it replayed -- on a range that was interrupted mid-replay - that is the tail, not the range. - - Slightly over-counts: replay restarts at the checkpoint boundary containing - LCL, so up to 64 ledgers can be applied twice. Against a 16320-ledger range - that is <=0.4%, but it is a fixed ledger cost rather than a percentage, so - it grows as ranges shrink. - """ - total = None - for n in _resumed_chain(end, attempt): - # pod_name only ever names the LAST attempt's pod, so the archive/pod - # fallbacks are offered to that one alone; earlier legs come from the - # .metrics the collector already wrote. - leg = _tx_apply_for_attempt(end, n, pod_name if n == int(attempt) else None) - if leg is None: - # A disrupted process often never prints its final medida block. - # Publishing the sum of surviving legs as a total silently - # under-reports; absence accurately says the chain is incomplete. - return None - total = leg if total is None else total + leg - return total - - -def seconds_for_range(end, attempt=1, final=None): - """Compute time for the whole range, summed across the resumed chain. - - `final` is the winning attempt's own duration, which reconcile has in hand - from the pod. Earlier legs come from their .outcome, written when the - monitor classified the failure and still had the pod. - - This is compute, not elapsed: scheduling, image pull, node startup and gaps - between attempts are not in it. wallSeconds is a separate winner-Job-only - diagnostic; whole-chain elapsed time was never persisted. - """ - total = None - for n in _resumed_chain(end, attempt): - if n == int(attempt) and final is not None: - leg = final - else: - leg = _attempt_seconds(end, n) - if leg is None: - return None - total = leg if total is None else total + leg - return total - - -def _attempt_seconds(end, attempt): - """Best durable duration for one attempt, or None when it was never saved.""" - # .outcome is authoritative -- the pod's own terminated timestamps. It is - # absent whenever the pod was reaped before the monitor could classify it, - # which is every spot eviction, so fall back to the collector's estimate. - leg = (read_outcome(end, attempt) or {}).get('attemptSeconds') - if leg is not None: - return leg - try: - with open(metrics_path(end, attempt)) as fh: - data = json.load(fh) - # A poller clock starts when that collector process attaches, so after a - # restart it is only a lower bound. Legacy files have no provenance and - # remain usable for best-effort reconstruction; new known estimates do - # not masquerade as complete chain compute. - if data.get('attemptSecondsExact') is False: - return None - return data.get('attemptSeconds') - except (OSError, ValueError): - return None - - -def reconstruct_completed_profile(end, attempt): - """Recompute recoverable profile fields from immutable attempt artifacts. - - Durations and tx-apply totals follow only the continuous resumed chain, so a - fresh retry never double-counts discarded work. Peaks use that chain plus - every attempt that hit a resource ceiling. Missing duration or tx-apply legs - make that aggregate absent rather than publishing a lower bound as a total. - Complete tx-apply legs retain the existing <=64-ledger overlap. - - Reconstructable: persisted sampled peaks, complete attemptSeconds chains in - .outcome/.metrics, and complete txApplySeconds chains in .metrics/.log.gz. - Not reconstructable: whole-chain wall time, samples never persisted, or a - duration/tx-apply leg whose process and archive are both gone. - """ - rebuilt = peaks_for_range(end, attempt) - seconds = seconds_for_range(end, attempt) - if seconds is not None: - rebuilt['seconds'] = seconds - tx_apply = tx_apply_for_range(end, attempt) - if tx_apply is not None: - rebuilt['txApply'] = tx_apply - return rebuilt - - -def _apply_profile_reconstruction(record, rebuilt): - """Merge reconstruction without lowering stronger persisted evidence.""" - updates = {} - for key, value in rebuilt.items(): - current = record.get(key) - if current is None or value > current: - updates[key] = value - record.update(updates) - return updates - - -def _repair_completed_profile(end, attempt, record): - """Merge exact reconstruction and remove unverifiable chain aggregates.""" - rebuilt = reconstruct_completed_profile(end, attempt) - updates = _apply_profile_reconstruction(record, rebuilt) - if len(list(_resumed_chain(end, attempt))) > 1: - for key in ('seconds', 'txApply'): - if key not in rebuilt and record.get(key) is not None: - # Older code published the sum of whatever legs survived. Once - # resume proves this is a chain, that number is a lower bound, - # not a total; omission is the only honest repair. - record.pop(key) - updates[key] = None - return updates - - -def repair_completed_profiles(progress): - """Apply artifact reconstruction to old completed records idempotently.""" - repaired = 0 - for end, record in (progress.get('completed') or {}).items(): - try: - attempt = int(record.get('attempts') or 1) - updates = _repair_completed_profile(end, attempt, record) - except (TypeError, ValueError): - logger.warning("cannot reconstruct malformed completed record for range %s", end) - continue - if updates: - repaired += 1 - return repaired - - -def _tx_apply_for_attempt(end, attempt=1, pod_name=None): - """Final 'ledger.transaction.apply' sum for ONE attempt, in seconds. - - stellar-core prints the medida block once at exit (we pass --metric), so - this is the exact total for that process rather than a sample. - - Three sources, cheapest and most durable first: - - .metrics the collector parsed it out of the live stream. Survives both - pod reaping and saveSuccessLogs=false. - .log.gz the collector's archive, if it was kept. - pod log only if a pod object still exists. Racing Karpenter, so this - is a fallback, never the plan. - """ - try: - with open(metrics_path(end, attempt)) as fh: - value = json.load(fh).get('txApplySeconds') - if value is not None: - return float(value) - except (OSError, ValueError, TypeError): - pass - raw = None - candidate = log_path(end, attempt) - if os.path.exists(candidate): - try: - with gzip.open(candidate, 'rt') as fh: - raw = fh.read() - except (OSError, EOFError, zlib.error): - # A corrupt archive costs THIS RANGE its metric, never the pass. - # EOFError is a truncated member -- the collector appending right - # now, or one that was killed mid-append -- and is not an OSError, - # so it used to escape the per-range work and abort the whole - # reconcile: no recording, no reap, no dispatch for any of the - # ~4000 ranges, repeating for as long as the torn bytes sat there. - raw = None - if raw is None: - if pod_name is None: - return None - try: - raw = core_v1.read_namespaced_pod_log(pod_name, NAMESPACE, tail_lines=400) - except ApiException: - return None - lines = raw.splitlines() - for i, line in enumerate(lines): - if "metric 'ledger.transaction.apply'" not in line: - continue - # Same reach as log_collector.TxApplyScanner.WINDOW, and it has to be: - # the two are independent readers of one block and progress.json takes - # whichever landed first. medida puts `sum` 10 lines under the header - # (27.1.1, ssc-test 2026-07-28), so five more percentiles in a release - # takes the metric out of range on both sides at once. - for follow in lines[i + 1:i + 16]: - m = _SUM_RE.search(follow) - if m: - return float(m.group(1)) / 1000.0 - return None - - -# --- job construction ------------------------------------------------------- - -# Resume decision, run before catchup. Only skip new-db when the DB on /data -# belongs to THIS range and replay had already started. Bucket apply uses -# createWithoutLoading() -- an unconditional INSERT that assumes a fresh DB -- -# so a crash during that phase must start over. "Ledger close complete" is the -# cheap discriminator: bucket apply never closes a ledger. -# -# The LCL is read from stellar-core's own log on /data, NOT from the database: -# core 27 dropped the ledgerheaders table, so the old SQL probe silently -# returned empty and every interruption fell back to new-db. -RESUME_SCRIPT = r'''set -e -KEY="%(key)s" -TARGET=%(target)d -COUNT=%(count)d -MARK=/data/.job-key -RESUME=false -LCL="" -if [ -f "$MARK" ] && [ "$(cat "$MARK" 2>/dev/null)" = "$KEY" ]; then - # Ask core for its own LCL. It reads storestate.lastclosedledgerheader through - # its own accessor, so this survives both a schema change (v27 dropped - # ledgerheaders, which is what silently disabled resume before) and any log - # level above INFO. Safe here specifically: core has not started, so nothing - # holds /data/buckets/stellar-core.lock. Core logs to the console alongside - # the JSON, hence grepping rather than parsing. - # One "num" key in the whole document and it is the ledger's -- verified - # against 27.1.1 output on ssc-test 2026-07-30. Do NOT window this with - # `grep -A '"ledger":'`: bucketlist puts ~40 lines of hashes between the - # key and "num", so a small window silently yields nothing and the probe - # degrades to the log fallback without saying so. - LCL=$(/usr/bin/stellar-core --conf /config/stellar-core.cfg offline-info --console 2>/dev/null \ - | sed -n 's/.*"num"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/p' | head -1 || true) - if [ -n "$LCL" ]; then - echo "RESUME PROBE: offline-info reports lcl $LCL" - else - # Fallback: the previous incarnation's log on /data. Goes blind above INFO, - # which is why it is no longer the primary probe. - PREV_LOG=$(ls -t /data/stellar-core*.log 2>/dev/null | head -n 1 || true) - if [ -n "$PREV_LOG" ]; then - LCL=$(grep -oE "Ledger close complete: [0-9]+" "$PREV_LOG" 2>/dev/null | tail -1 | grep -oE "[0-9]+$" || true) - fi - echo "RESUME PROBE: offline-info gave nothing; log fallback says '${LCL:-none}'" - fi - # Already at the target: a1 finished the replay and was evicted before it - # could exit 0. Re-running catchup here applies nothing and stellar-core exits - # 2, identically on every retry, so the range burns its whole budget and the - # mission aborts the run over work that was actually done. Measured on - # ssc-test 2026-07-30: range 16752063 killed a 2096-worker run that was 61 - # percent complete, exactly this way. - if [ -n "$LCL" ] && [ "$LCL" -ge "$TARGET" ] 2>/dev/null; then - echo "ALREADY COMPLETE: $KEY reached ledger $LCL >= target $TARGET; nothing left to replay" - exit 0 - fi - if [ -n "$LCL" ] && [ "$LCL" -ge $((TARGET - COUNT)) ] && [ "$LCL" -lt "$TARGET" ] 2>/dev/null; then - RESUME=true; echo "RESUME: $KEY reached ledger $LCL, replay had started; skipping new-db" - else - echo "RESUME DECLINED: $KEY last close was '${LCL:-none}' (need >= $((TARGET - COUNT))); bucket phase incomplete, starting fresh" - fi -fi -printf '%%s' "$KEY" > "$MARK" -if [ "$RESUME" != "true" ]; then - /usr/bin/stellar-core --conf /config/stellar-core.cfg new-db --console -fi -exec /usr/bin/stellar-core --conf /config/stellar-core.cfg catchup "$KEY" \ - --metric 'ledger.transaction.apply' --console -''' - - -def owner_ref(): - cm = core_v1.read_namespaced_config_map(f"{RUN_NAME}-stellar-core-config", NAMESPACE) - return [client.V1OwnerReference(api_version='v1', kind='ConfigMap', - name=cm.metadata.name, uid=cm.metadata.uid, - block_owner_deletion=True)] - - -def release_pvc(end): - """Drop a completed range's volume. - - The PVC exists so an interrupted range resumes at L+1; once the range has - succeeded there is nothing left to resume and the volume is dead weight. - They are owner-referenced to the release, so without this they all survive - until `helm uninstall` -- measured on ssc-test, 2032 bound PVCs and 79 TiB - of gp3 provisioned a third of the way through a 3982-range run, heading for - ~156 TiB and 3982 volumes against the account's volume ceiling. - - Best-effort: a failure here costs disk, never correctness, and the range is - already recorded complete. - """ - if STORAGE_MODE != 'pvc': - return - name = f"{RUN_NAME}-data-r{end}" - try: - core_v1.delete_namespaced_persistent_volume_claim(name, NAMESPACE) - metric_pvc_released.inc() - except ApiException as e: - if e.status != 404: - logger.warning("could not release PVC for completed range %s: %s", end, e) - - -def done_path(end, attempt): - return os.path.join(LOG_DIR, f"range-{end}-a{attempt}.done") - - -def _attempt_finalized(end, attempt): - """Has the collector written everything it will for this attempt? - - It writes this file last, after .metrics. Anything inferred instead -- peaks - being present, tx_apply being readable -- is a guess: tx_apply falls back to - the archive so it is available long before the collector finishes, and an - attempt can legitimately finalize with no peaks at all. - """ - return os.path.exists(done_path(end, attempt)) - - -def _has_peaks(record): - return any(record.get(k) is not None for k in PEAK_FIELDS) - - -def _reap_if_complete(end, attempt, record): - """Delete a succeeded range's Job once nothing more can be learned from it. - - Deleting reaps the pod, and .metrics is the only place peaks live, so a - reap before the collector finalizes makes the gap permanent -- that is - exactly how a whole run's profile came back with txApply on every range and - peaks on none. JOB_TTL_SECONDS still reclaims anything the collector never - gets to, so a pod reaped before it could be read costs a late Job, not a - stuck one. - """ - if not _attempt_finalized(end, attempt): - return - reap_range_jobs(end) - - -def reap_range_jobs(end): - """Delete every Job this range has, not just the attempt that won. - - Completion is terminal for the RANGE. An attempt-scoped reap leaves an - older Failed Job standing -- the common case is an attempt lost to node - disruption whose collector died with the node, so it was never finalized - and was deliberately not deleted. Once the winner's Job is gone, that - leftover is the range's highest live attempt, and the next pass feeds it - straight into the retry decision and re-runs an already-recorded range - against a freshly recreated, empty PVC. - """ - try: - jobs = batch_v1.list_namespaced_job( - NAMESPACE, - label_selector=f"{LABEL_RUN}={RUN_NAME},{LABEL_RANGE}={end}").items - except ApiException as e: - logger.warning("could not list jobs for completed range %s: %s", end, e) - return - for j in jobs: - try: - batch_v1.delete_namespaced_job(j.metadata.name, NAMESPACE, - propagation_policy='Background') - metric_jobs_reaped.inc() - except ApiException as e: - if e.status != 404: - logger.warning("could not delete finished job %s for range %s: %s", - j.metadata.name, end, e) - - -def delete_job(end, attempt): - """Drop a finished Job once nothing more is owed by it. - - reconcile() lists every Job and Pod on each pass, so a finished Job is not - free: it inflates two LIST calls for as long as it lingers. At 2048-4096 - parallelism with a real OOM or spot-eviction rate that is hundreds of dead - objects per hour of run -- under the old 3600s TTL the dead ones outnumbered - the live ones within the first hour -- and the apiserver pressure shows up - as truncated list responses long before anything else complains. - - Background propagation specifically: that is what removes the pod as well. - Orphan or the server default would leave the pod behind, so the next pass - lists exactly as much as it did before. - - Callers must have persisted whatever they need first -- the logs, .outcome - and .metrics all live on the monitor's volume by then, so the Job and its - pod carry no information once the range is recorded. Best-effort: a 404 is - the ordinary race with the TTL controller, and any other status warns and - carries on. Raising here would abort the whole reconcile pass mid-iteration - and strand every other range in it; on failure JOB_TTL_SECONDS still - reclaims the object, which costs disk and etcd, never correctness. - """ - try: - batch_v1.delete_namespaced_job(job_name(end, attempt), NAMESPACE, - propagation_policy='Background') - metric_jobs_reaped.inc() - except ApiException as e: - if e.status != 404: - logger.warning("could not delete finished job for range %s attempt %d: %s", - end, attempt, e) - - -def ensure_pvc(end, owner): - name = f"{RUN_NAME}-data-r{end}" - try: - core_v1.read_namespaced_persistent_volume_claim(name, NAMESPACE) - return name - except ApiException as e: - if e.status != 404: - raise - spec = client.V1PersistentVolumeClaimSpec( - access_modes=['ReadWriteOnce'], - resources=client.V1VolumeResourceRequirements(requests={'storage': STORAGE_SIZE})) - if STORAGE_CLASS: - spec.storage_class_name = STORAGE_CLASS - core_v1.create_namespaced_persistent_volume_claim(NAMESPACE, client.V1PersistentVolumeClaim( - metadata=client.V1ObjectMeta(name=name, owner_references=owner, - labels={LABEL_RUN: RUN_NAME, LABEL_RANGE: str(end)}), - spec=spec)) - return name - - -def eph_for_attempt(attempt): - """Ephemeral-storage size for attempt N, escalating after an eviction.""" - base_q = _quantity_bytes(LIM_EPHEMERAL) - want = int(base_q * (EPH_BUMP_FACTOR ** max(0, attempt - 1))) - return _bytes_to_quantity(min(want, _quantity_bytes(EPH_ESCALATION_CAP))) - - -def load_profile(): - """Per-range measurements from an earlier run, keyed by range end. - - Absent, unreadable or malformed all mean the same thing: size from the - configured defaults. A profile is an optimisation, never a prerequisite. - """ - if not PROFILE_PATH: - return [] - try: - with open(PROFILE_PATH) as fh: - doc = json.load(fh) - except (OSError, ValueError) as e: - logger.warning("range profile %s unreadable (%s); using configured requests", - PROFILE_PATH, e) - return [] - mode = doc.get('storageMode') - cross_mode = bool(mode) and mode != STORAGE_MODE - if cross_mode: - # cpu and memory carry across modes -- they measure the same work. Disk - # does not: a pvc run puts /data on the volume, so it never measures - # node-local usage, and an ephemeral run's figure says nothing about a - # pvc one. Keep the transferable axes and let disk fall back to the - # configured default. - logger.warning("range profile is for storageMode=%s but this run is %s; " - "using its cpu and memory, defaulting ephemeral storage", - mode, STORAGE_MODE) - out = [] - for end, rec in (doc.get('ranges') or {}).items(): - try: - end = int(end) - except (TypeError, ValueError): - continue - if cross_mode: - rec = {k: v for k, v in rec.items() if k != 'peakEphemeralBytes'} - out.append((end, rec)) - out.sort() - logger.info("loaded range profile: %d ranges from %s", len(out), PROFILE_PATH) - return out - - -PROFILE = None - - -def profile_for(end): - """Measurements to size this range from, or None to use the defaults. - - Exact end, else the nearest measured end ABOVE it. Cost rises with ledger - position -- the bucket set only grows -- so a lower neighbour under-reports, - and under-provisioning costs an eviction while over-provisioning only costs - packing. Past the top of the profile there is nothing safe to extrapolate - from, so fall back to the configured defaults. - """ - if not PROFILE: - return None - end = int(end) - idx = bisect.bisect_left(PROFILE, (end,)) - if idx < len(PROFILE) and PROFILE[idx][0] == end: - return PROFILE[idx][1] - return PROFILE[idx][1] if idx < len(PROFILE) else None - - -def _sized(value, margin, cap): - """A measured peak turned into a request: margin applied, never above cap.""" - want = int(value * margin) - return _bytes_to_quantity(min(want, _quantity_bytes(cap))) - - -# CPU tiers, as request only. The request is not a demand estimate -- measured -# unthrottled, REPLAY wants ~1.0 cores at every ledger position (1.04 at 63.7M, -# 0.96 at 43.2M) and replay is 80-95% of a job, so demand barely varies. What -# varies is how much throttling a range can ABSORB before it stops finishing -# inside the longest job's shadow, and that slack is free packing density. -# Measured 2026-07-30: bin-packed, flat 1.0 needs 491 nodes and 3928 vCPU -- -# over the 2304 quota -- where tiering needs 291 nodes and 2328 vCPU at the -# same makespan. -# -# Keyed on where a range falls in the profile's own runtime distribution, not -# on its absolute seconds. Absolute keying compares against the longest range, -# which is set by the single worst-throttled job: between two real runs that -# budget moved 1.79x while the median range moved 1.55x, and the top two tiers -# went from 127 ranges to 65. Shares pin the fleet size instead, which is what -# has to fit a fixed vCPU quota. Per-range assignment agrees ~85% either way -- -# that ceiling is run-to-run noise in the measurement (Spearman 0.87), not -# something the keying can fix. -# -# One entry per tier, cheapest first: ":". A range at or -# below that percentile of the profile's runtimes takes that many cores. Empty -# disables tiering and every range keeps the configured request. -PROFILE_CPU_TIERS = os.getenv('PROFILE_CPU_TIERS', '') # "85:0.5,98:0.75,99.5:1.0,100:1.25" -_SORTED_SECONDS = None - - -def _positive_seconds(value): - """A finite positive runtime, or None when the profile cannot supply one.""" - try: - seconds = float(value) - except (TypeError, ValueError): - return None - return seconds if math.isfinite(seconds) and seconds > 0 else None - - -def _profile_seconds(): - """Every valid measured runtime in the profile, sorted.""" - global _SORTED_SECONDS - if _SORTED_SECONDS is None: - values = (_positive_seconds(r.get('seconds')) for _, r in (PROFILE or [])) - _SORTED_SECONDS = sorted(seconds for seconds in values if seconds is not None) - return _SORTED_SECONDS - - -def _runtime_memory_insurance(seconds): - """Runtime-weighted share of the configured memory allowance.""" - seconds = _positive_seconds(seconds) - everything = _profile_seconds() - longest = everything[-1] if everything else None - insurance = _quantity_bytes(PROFILE_RUNTIME_MEMORY_INSURANCE) - if seconds is None or longest is None or longest <= 0 or insurance <= 0: - return 0 - return int(insurance * (seconds / longest)) - - -def _slack_cpu(seconds): - """Tier for a range, by its rank among all profiled runtimes. - - No usable runtime means no tier: fall through to the configured REQ_CPU, - the same request an entirely unprofiled range gets. - - This used to return the TOP tier on the reasoning that an unmeasured range - is newer than anything measured, so assume the worst. That reasoning does - not survive contact with a reconstructed profile. Measured 2026-07-31: 103 - of 3983 ranges carry a peakAnonBytes but no seconds -- not because they are - new, but because their runtime came from a resumed chain and the - reconstruction omits what it cannot verify. Their ends span 38.2M-63.0M, - scattered through history rather than clustered at the tip, and several are - demonstrably small. Handing them the top band spent 206 vCPU -- 9% of the - spot quota -- on ranges we have positive evidence are cheap, while the 20 - genuinely-longest ranges the band exists for got a sixth of that. - """ - try: - tiers = [(float(p), c) for p, c in - (t.split(':') for t in PROFILE_CPU_TIERS.split(',') if t.strip())] - except ValueError: - logger.error("PROFILE_CPU_TIERS is malformed (%r); cpu tiering disabled", - PROFILE_CPU_TIERS) - return None - if not tiers: - return None - seconds = _positive_seconds(seconds) - if seconds is None: - return None - everything = _profile_seconds() - if not everything: - return None - pct = 100.0 * bisect.bisect_right(everything, seconds) / len(everything) - for upto, cores in tiers: - if pct <= upto: - return cores - return tiers[-1][1] - - -def _profile_overrides(end, escalated): - """Request overrides for this range from the profile, or {} for none. - - Escalated retries opt out: an escalation is a measurement of THIS run and - outranks anything an earlier one saw. - """ - if escalated or end is None: - return {} - prof = profile_for(end) - if not prof: - return {} - out = {} - # peakAnonBytes is kubelet's rssBytes, sampled by the collector on its own - # poll; peakRssBytes is the same quantity via a 30s Prometheus scrape. Prefer - # the finer one and fall back, so a profile captured before the collector - # tracked anon still sizes exactly as it used to. - rss = prof.get('peakAnonBytes') or prof.get('peakRssBytes') - if rss: - want = (int(rss * PROFILE_MARGIN) - + _quantity_bytes(PROFILE_CACHE_HEADROOM) - + _runtime_memory_insurance(prof.get('seconds'))) - out['memory'] = _bytes_to_quantity(min(want, _quantity_bytes(PROFILE_MAX_MEM))) - disk = prof.get('peakEphemeralBytes') - if disk and LIM_EPHEMERAL: - out['ephemeral-storage'] = _sized(disk, PROFILE_MARGIN, LIM_EPHEMERAL) - cpu = _slack_cpu(prof.get('seconds')) - if cpu: - out['cpu'] = cpu - return out - - -def _resources(mem=None, eph=None, end=None): - # Before mem is defaulted below -- reading it afterwards can never see None, - # which silently disabled profile sizing entirely. - overrides = _profile_overrides(end, escalated=(mem is not None or eph is not None)) - # `mem` is the escalated request on an OOM retry, else the configured one. - req = {'cpu': REQ_CPU, 'memory': mem or REQ_MEM} - # Nothing but ephemeral-storage is ever limited. That one stays: it is the - # only dimension where an unbounded pod takes the whole NODE down with it - # rather than just itself, and it has its own escalation ladder. - lim = {} - - # Only meaningful in ephemeral mode. In PVC mode a large request makes disk - # the binding dimension and halves workers-per-node for no reason. - if REQ_EPHEMERAL: - # Raise the request with the limit: ephemeral-storage is a scheduling - # dimension, so a pod that outgrew its limit will not fit where it was - # placed before. - req['ephemeral-storage'] = eph or REQ_EPHEMERAL - else: - # pvc mode: /data is not on the node disk, so an ephemeral override - # would size a dimension this run does not use. - overrides.pop('ephemeral-storage', None) - if LIM_EPHEMERAL: - lim['ephemeral-storage'] = eph or LIM_EPHEMERAL - - # The profile only ever moves requests now. Disk is the one exception, and - # only because its limit is what the kubelet enforces -- match it so a range - # measured to need more disk is actually allowed to use it. - for key, value in overrides.items(): - req[key] = value - if key == 'ephemeral-storage' and LIM_EPHEMERAL: - lim[key] = value - # Unmeasured range: the configured requests, exactly as if there were no - # profile at all. - return client.V1ResourceRequirements(requests=req, limits=lim or None) - - -def volume_spread_constraints(): - """Keep PVC-mounting workers under the per-node EBS attachment limit. - - Only in pvc mode: in ephemeral mode /data is an emptyDir, no volume is - attached, and spreading would just cost density. - """ - if STORAGE_MODE != 'pvc' or MAX_VOLUMES_PER_NODE <= 0: - return None - min_domains = max(1, -(-PARALLELISM // MAX_VOLUMES_PER_NODE)) # ceil - return [client.V1TopologySpreadConstraint( - max_skew=MAX_VOLUMES_PER_NODE, - min_domains=min_domains, - topology_key='kubernetes.io/hostname', - when_unsatisfiable='DoNotSchedule', - label_selector=client.V1LabelSelector(match_labels={LABEL_RUN: RUN_NAME}))] - - -def pod_labels(end, attempt): - """Labels on the worker POD, which are not the Job's. - - LABEL_ATTEMPT has to be here as well: the collector reads it off the pod to - decide which range--a.* files this attempt owns, and its default is - "1". Measured on ssc-test 2026-07-30 -- with the label only on the Job, all - 2246 metrics files were a1 while 475 a2 pods were running, so every retry - overwrote the first attempt's peaks instead of being maxed against them, - peaks_for_range(end, 2) found nothing, and those Jobs were never reaped. - """ - labels = {LABEL_RUN: RUN_NAME, LABEL_RANGE: str(end), - LABEL_ATTEMPT: str(attempt)} - if EMIT_MISSION_LABEL and MISSION: - labels['mission'] = MISSION - return labels - - -def build_job(end, count, attempt, owner, mem=None, eph=None): - key = job_key(end, count) - script = RESUME_SCRIPT % {'key': key, 'target': end, 'count': count} - - if STORAGE_MODE == 'pvc': - data_vol = client.V1Volume(name='data', persistent_volume_claim=( - client.V1PersistentVolumeClaimVolumeSource(claim_name=ensure_pvc(end, owner)))) - else: - data_vol = client.V1Volume(name='data', empty_dir=client.V1EmptyDirVolumeSource()) - - env = [client.V1EnvVar(name='ASAN_OPTIONS', value=ASAN_OPTIONS)] if ASAN_OPTIONS else [] - command = ['/bin/sh', '-c', script] - image_pull_policy = None - volumes = [data_vol, client.V1Volume( - name='config', config_map=client.V1ConfigMapVolumeSource( - name=f"{RUN_NAME}-stellar-core-config"))] - volume_mounts = [ - client.V1VolumeMount(name='data', mount_path='/data'), - client.V1VolumeMount(name='config', mount_path='/config')] - if SYNTHETIC_WORKER_CONFIG_MAP: - command = ['python3', '/synthetic/worker.py'] - image_pull_policy = SYNTHETIC_WORKER_IMAGE_PULL_POLICY - env = [ - client.V1EnvVar(name='SYNTHETIC_ATTEMPT', value=str(attempt)), - client.V1EnvVar(name='SYNTHETIC_TARGET', value=str(end)), - client.V1EnvVar(name='SYNTHETIC_COUNT', value=str(count)), - client.V1EnvVar(name='SYNTHETIC_KEY', value=key), - client.V1EnvVar(name='SYNTHETIC_PREDECESSOR_SECONDS', - value=SYNTHETIC_PREDECESSOR_SECONDS), - client.V1EnvVar(name='SYNTHETIC_SUCCESSOR_MINIMUM_SECONDS', - value=SYNTHETIC_SUCCESSOR_MINIMUM_SECONDS), - client.V1EnvVar(name='SYNTHETIC_MAXIMUM_WAIT_SECONDS', - value=SYNTHETIC_MAXIMUM_WAIT_SECONDS), - client.V1EnvVar(name='SYNTHETIC_PREDECESSOR_ANON_MIB', - value=SYNTHETIC_PREDECESSOR_ANON_MIB), - client.V1EnvVar(name='SYNTHETIC_PREDECESSOR_WORKING_SET_MIB', - value=SYNTHETIC_PREDECESSOR_WORKING_SET_MIB), - client.V1EnvVar(name='SYNTHETIC_SUCCESSOR_ANON_MIB', - value=SYNTHETIC_SUCCESSOR_ANON_MIB), - client.V1EnvVar(name='SYNTHETIC_SUCCESSOR_WORKING_SET_MIB', - value=SYNTHETIC_SUCCESSOR_WORKING_SET_MIB), - client.V1EnvVar(name='SYNTHETIC_PREDECESSOR_TX_APPLY_MS', - value=SYNTHETIC_PREDECESSOR_TX_APPLY_MS), - client.V1EnvVar(name='SYNTHETIC_SUCCESSOR_TX_APPLY_MS', - value=SYNTHETIC_SUCCESSOR_TX_APPLY_MS), - ] - volumes.append(client.V1Volume( - name='synthetic-worker', - config_map=client.V1ConfigMapVolumeSource( - name=SYNTHETIC_WORKER_CONFIG_MAP))) - volume_mounts.append(client.V1VolumeMount( - name='synthetic-worker', mount_path='/synthetic', read_only=True)) - - # Require and avoid go in ONE matchExpressions list: expressions within a - # term are ANDed, whereas separate terms are ORed and an avoid-only pod would - # then match every node. The original StatefulSet template rendered both into - # the same term; the rewrite carried requireNodeLabels across and dropped - # avoidNodeLabels, so the flag installed cleanly and scheduled workers onto - # exactly the nodes it was asked to keep them off. - match = [] - if NODE_LABEL_KEY: - match.append(client.V1NodeSelectorRequirement( - key=NODE_LABEL_KEY, operator='In', values=[NODE_LABEL_VALUE])) - if AVOID_NODE_LABEL_KEY: - # No value means "avoid the label however it is set", which is - # DoesNotExist; NotIn [""] would only exclude the empty value. - match.append(client.V1NodeSelectorRequirement( - key=AVOID_NODE_LABEL_KEY, - operator='NotIn' if AVOID_NODE_LABEL_VALUE else 'DoesNotExist', - values=[AVOID_NODE_LABEL_VALUE] if AVOID_NODE_LABEL_VALUE else None)) - affinity = None - if match: - affinity = client.V1Affinity(node_affinity=client.V1NodeAffinity( - required_during_scheduling_ignored_during_execution=client.V1NodeSelector( - node_selector_terms=[client.V1NodeSelectorTerm(match_expressions=match)]))) - # Taint value must be absent: the mission emits {key, effect} with no value, - # and the default Equal operator does not match "" against "true". - tolerations = [client.V1Toleration(key=TOLERATE_TAINT, effect='NoSchedule')] if TOLERATE_TAINT else None - - container = client.V1Container( - name='stellar-core', image=CORE_IMAGE, - image_pull_policy=image_pull_policy, - command=command, env=env, resources=_resources(mem, eph, end), - ports=[client.V1ContainerPort(container_port=11626, name='http')], - volume_mounts=volume_mounts) - - return client.V1Job( - metadata=client.V1ObjectMeta( - name=job_name(end, attempt), owner_references=owner, - labels={LABEL_RUN: RUN_NAME, LABEL_RANGE: str(end), - LABEL_ATTEMPT: str(attempt)}), - spec=client.V1JobSpec( - # The monitor owns retries, not the Job controller. With - # backoffLimit>0 the controller would replace the pod on its own - # schedule, so we could not classify disruption vs genuine catchup - # failure, could not count evictions, and could not guarantee the - # log is archived before the next attempt starts. 0 means the Job - # fails once and stays put for inspection; reconcile() decides - # whether to dispatch attempt N+1. - # - # A podFailurePolicy would be inert here -- with backoffLimit 0 every - # pod failure already fails the Job, so Count and FailJob collapse to - # the same outcome. Classification is done by reading the pod's - # DisruptionTarget condition instead. - # On the JobSpec, not the pod: a pod-level deadline is immutable - # once the pod exists, so a mis-set value cannot be corrected on a - # live run. Measured 2026-07-30: 1007 Job-level deadlines were - # repointed 3h->12h in place while their pods kept running; 850 - # pod-level ones could not be touched at all. - active_deadline_seconds=ATTEMPT_DEADLINE_SECONDS or None, - backoff_limit=0, - pod_failure_policy=client.V1PodFailurePolicy( - rules=[r for _, r in _failure_rules()]), - ttl_seconds_after_finished=JOB_TTL_SECONDS, - template=client.V1PodTemplateSpec( - metadata=client.V1ObjectMeta(labels=pod_labels(end, attempt)), - spec=client.V1PodSpec( - # On the POD, not the JobSpec. JobSpec.activeDeadlineSeconds - # runs from the Job's startTime, so every second the pod - # spends Pending -- waiting for Karpenter, pulling the image - # -- is charged against a budget that is meant to bound how - # long the range RUNS. During a node-class outage this run - # sat ~15 minutes Pending and ranges died as "timeouts" - # having barely executed -- and a timeout is terminal, so - # each one fails the mission. The pod-level field starts at - # container start, which is the thing being bounded. - # IRSA for the S3 history mirror. Without it workers fall - # back to the public archive, which throttles at 1024. - service_account_name=WORKER_SERVICE_ACCOUNT or None, - # Keeps PVC-mounting workers under the per-node EBS - # attachment cap; inert at realistic CPU-bound density. - topology_spread_constraints=volume_spread_constraints(), - # Never, so a failed container is not restarted in place: - # the pod stays terminal and inspectable for classification - # and for the backstop log read. - restart_policy='Never', - termination_grace_period_seconds=WORKER_GRACE_SECONDS, - affinity=affinity, tolerations=tolerations, - containers=[container], - volumes=volumes)))) - - -# --- reconcile -------------------------------------------------------------- - -_ATTEMPT_FILE = re.compile( - r'^range-(?P\d+)-a(?P[1-9]\d*)\.' - r'(?:verdict|outcome|state|metrics|done|log\.gz)$') - - -def _retry_counter_totals(progress, current_attempts=()): - """Reconstruct retry metrics from durable records and observed attempts. - - A verdict says why an attempt ended; it does not say a retry was dispatched. - Attempt N therefore contributes to retry totals only when attempt N+1 is - evidenced by progress, a persisted per-attempt file, or the current Job - snapshot. The latter makes a newly-created successor visible before its range - completes, while the durable sources rebuild the same truth after restart. - """ - try: - names = os.listdir(LOG_DIR) - except OSError: - names = [] - - max_attempt = {} - terminal = set() - - def remember(end, attempt): - try: - attempt = int(attempt) - except (TypeError, ValueError): - return - if attempt < 1: - return - end = str(end) - max_attempt[end] = max(max_attempt.get(end, 0), attempt) - - if isinstance(progress, dict): - for bucket in ('completed', 'failed'): - records = progress.get(bucket) - if not isinstance(records, dict): - continue - for end, record in records.items(): - if not isinstance(record, dict): - continue - try: - attempt = int(record.get('attempts', 1)) - except (TypeError, ValueError): - continue - if attempt < 1: - continue - remember(end, attempt) - terminal.add((str(end), attempt)) - - for item in current_attempts: - try: - end, attempt = item - except (TypeError, ValueError): - continue - remember(end, attempt) - - verdict_files = set() - outcome_files = set() - for name in names: - match = _ATTEMPT_FILE.match(name) - if not match: - continue - key = (match.group('end'), int(match.group('attempt'))) - remember(*key) - if name.endswith('.verdict'): - verdict_files.add(key) - elif name.endswith('.outcome'): - outcome_files.add(key) - - effective = {} - for end, attempt in verdict_files: - try: - with open(verdict_path(end, attempt)) as fh: - verdict = fh.read().strip() - except OSError: - continue - if verdict in ATTEMPT_OUTCOMES: - effective[(end, attempt)] = verdict - - # .outcome predates .verdict. It is safe only for a completed attempt chain: - # a current collector outcome can still be superseded by reconcile's - # effective verdict (notably failed -> timeout). Presence of any verdict file, - # even a malformed one, means this is not a legacy attempt and must never - # fall back to the less-authoritative classification. - for end, attempt in outcome_files - verdict_files: - if attempt >= max_attempt.get(end, 0) and (end, attempt) not in terminal: - continue - try: - with open(outcome_path(end, attempt)) as fh: - record = json.load(fh) - except (OSError, ValueError): - continue - outcome = record.get('outcome') if isinstance(record, dict) else None - if outcome in ATTEMPT_OUTCOMES: - effective[(end, attempt)] = outcome - - retries = sum(max(0, attempt - 1) for attempt in max_attempt.values()) - reasons = {reason: 0 for reason in ATTEMPT_OUTCOMES} - for (end, attempt), reason in effective.items(): - if attempt < max_attempt.get(end, 0): - reasons[reason] += 1 - disruption_retried_ranges = { - end for (end, attempt), reason in effective.items() - if reason == 'disrupted' and attempt < max_attempt.get(end, 0) - } - - return { - 'retries': retries, - 'evicted': sum(1 for verdict in effective.values() if verdict == 'disrupted'), - 'spot_disruption_retried': len(disruption_retried_ranges), - 'oom': reasons['oom'], - 'ephemeral': reasons['ephemeral'], - 'reasons': reasons, - } - - -def sync_counters(progress, counted, current_attempts=()): - """Drive the counters from persisted state instead of from events. - - Two reasons not to .inc() as things happen: - - * a terminally-failed range stays the newest Job for its range, so an - event-driven inc fires again on every reconcile until teardown - * the process resets to zero on restart, while verdicts and attempt state on - the PVC survive - - Computing the true total and incrementing by the delta is monotonic, - idempotent, and self-heals after a restart: the counter starts at 0 and the - first sync walks it up to the recorded total. - """ - totals = _retry_counter_totals(progress, current_attempts) - for key, total, metric in (('retries', totals['retries'], metric_retries), - ('oom', totals['oom'], metric_oom_retries), - ('ephemeral', totals['ephemeral'], metric_eph_retries), - ('evicted', totals['evicted'], metric_evictions), - ('spot_disruption_retried', - totals['spot_disruption_retried'], - metric_spot_disruption_retried)): - delta = total - counted.get(key, 0) - if delta > 0: - metric.inc(delta) - counted[key] = total - for reason in ATTEMPT_OUTCOMES: - metric = metric_retry_reasons.labels(reason=reason) - key = ('reason', reason) - total = totals['reasons'][reason] - delta = total - counted.get(key, 0) - if delta > 0: - metric.inc(delta) - counted[key] = total - - -def observe_recorded(progress, replayed): - """Feed recorded completions into the histograms. - - Prometheus histograms are append-only and reset to zero when the process - restarts, so replaying every recorded range rebuilds the exact cumulative - total rather than double counting. Guarded per-process by `replayed`. - - Keyed on (range, field), not on the range alone: a range is usually - recorded before the collector has flushed its .metrics, so txApply is null - at first sight and backfilled a pass or two later. Marking the whole range - as replayed on first sight meant that backfill could never be observed, and - the histogram permanently disagreed with progress.json. - """ - for end, rec in progress.get('completed', {}).items(): - # `is not None`, not truthiness: a range with sum = 0ms records - # txApply 0.0, which is a real observation and must not be dropped - # silently. Same for a sub-second duration. - for field, metric in (('seconds', metric_full_duration), - ('wallSeconds', metric_wall_duration), - ('txApply', metric_tx_apply_duration)): - if (end, field) in replayed: - continue - value = rec.get(field) - if value is None: - continue - replayed.add((end, field)) - metric.observe(value) - - -def pods_by_job(): - """One list per reconcile, indexed by Job name. - - This used to be a LIST per completed job, so a busy cycle at 1024 workers - issued dozens of round trips for one pod each. - """ - out = {} - for p in core_v1.list_namespaced_pod( - NAMESPACE, label_selector=f"{LABEL_RUN}={RUN_NAME}").items: - jn = (p.metadata.labels or {}).get('batch.kubernetes.io/job-name') - if jn: - out.setdefault(jn, p) - return out - - -def reconcile(state): - ranges = generate_ranges() - by_end = {str(end): count for end, count in ranges} - progress = load_progress() - completed = progress.setdefault('completed', {}) - failed = progress.setdefault('failed', {}) - # One pass per monitor process repairs records completed by an older build, - # including Jobs that were already reaped and therefore never enter the live - # completion branch below. Attempt artifacts are immutable after .done, and - # current completions still use the same helpers directly. - if not state.get('completed_profiles_reconstructed'): - repaired = repair_completed_profiles(progress) - state['completed_profiles_reconstructed'] = True - if repaired: - save_progress(progress) - logger.info("reconstructed profile fields for %d completed ranges", repaired) - - jobs = batch_v1.list_namespaced_job( - NAMESPACE, label_selector=f"{LABEL_RUN}={RUN_NAME}").items - job_pods = pods_by_job() - - live = {} # range-end -> (attempt, job) - current_attempts = set() - for j in jobs: - end = (j.metadata.labels or {}).get(LABEL_RANGE) - attempt = int((j.metadata.labels or {}).get(LABEL_ATTEMPT, 1)) - current_attempts.add((str(end), attempt)) - prev = live.get(end) - if prev is None or attempt >= prev[0]: - live[end] = (attempt, j) - - in_progress = [] - finalizing = [] - # The same set of ranges as `in_progress`, keyed by end. `remaining` is a - # COUNT over this run's range list, never `total - completed - ...`: the - # progress record is read off a shared volume and can carry ends from a run - # with a different ledgersPerJob, and a subtraction lets those foreign keys - # move a number that is supposed to describe THIS run. Measured on the - # fixture: one foreign key made `remaining` read 0 on the very first pass - # with nothing dispatched yet, and three of them left it at -3 once every - # real range had finished -- so the mission's `num_remain == 0 && - # jobs_in_progress == []` could never fire and the driver waited forever on - # a completed run. A count cannot be pushed below zero or above the range - # list by anything that is not one of our own ranges. - in_flight = set() - for end, (attempt, j) in list(live.items()): - st = j.status - if st.succeeded: - # Record BEFORE the Job's TTL can reclaim it: the per-attempt - # `seconds` below is the pod's own start -> finish, and Karpenter - # removes the pod ~1 min after the node empties. tx_apply no longer - # depends on this window -- the collector persists it from the - # stream. - if end not in completed: - pod = job_pods.get(j.metadata.name) - # Job.startTime is the FIRST attempt, so it spans retries. The - # successful pod's own start -> container finish is what - # worker.sh used to report, and is the number comparable across - # the redis cutover. - seconds = _pod_seconds(pod) if pod is not None else None - wall = None - if st.start_time and st.completion_time: - wall = (st.completion_time - st.start_time).total_seconds() - # Chain total, not this leg alone: a range that resumed spent - # real time in the attempts before the winner. Only a fresh - # single-attempt range may fall back to its winner or Job wall; - # a resumed chain with a missing leg stays absent. - chain = list(_resumed_chain(end, attempt)) - chain_seconds = seconds_for_range(end, attempt, seconds) - # For a fresh single attempt, the winner or Job wall is still a - # useful fallback. For a resumed chain, either every compute leg - # is known or the aggregate is absent -- never winner-only. - if chain_seconds is not None: - seconds = chain_seconds - elif len(chain) == 1: - seconds = seconds if seconds is not None else wall - else: - seconds = None - # Not gated on `pod`: the collector's .metrics/.log.gz are - # written from the live stream and outlive the pod, so a reaped - # node must not cost us the metric. - tx = tx_apply_for_range(end, attempt, - pod.metadata.name if pod else None) - if pod is not None and SAVE_SUCCESS_LOGS: - backstop_save_pod_log(pod.metadata.name, end, attempt) - if tx is None: - logger.warning("could not read tx_apply for range %s (pod gone?); " - "metric will be missing for this range", end) - completed[end] = {'seconds': seconds, 'wallSeconds': wall, - 'txApply': tx, 'attempts': attempt} - # Ledger count travels with the record: the logarithmic - # generator varies it per range, so it cannot be recomputed - # from config alone when the profile is read back. - if by_end.get(end) is not None: - completed[end]['count'] = by_end[end] - completed[end].update(peaks_for_range(end, attempt)) - # Durably recorded first: the record is what makes the volume - # and the Job disposable, so it must land before either goes. - save_progress(progress) - else: - # Backfill. The record is written the moment the Job flips to - # succeeded, which is usually before the collector has finalized - # -- and the final write can add peaks, txApplySeconds, - # attemptSeconds, and the resume marker together. Reconstruct the - # same profile used on first completion, not a field-by-field - # subset that can leave a pre-marker record permanently short. - late = _repair_completed_profile(end, attempt, completed[end]) - if late: - save_progress(progress) - logger.info("range %s: measurements arrived late, backfilled %s", - end, sorted(late)) - # Per SIGHTING of a recorded range, not per first sight. Both are - # idempotent (a 404 from either is swallowed) and both used to hang - # off the branch that runs exactly once, so a process that died - # anywhere between save_progress and here never reached them again: - # the record exists, so the first-sight branch is skipped forever - # and the backfill branch is skipped as soon as the record is - # complete. That leaked the range's 40Gi volume permanently and left - # a Job nothing would ever reap but JOB_TTL_SECONDS. - # - # `tx is None` still costs nothing here: _reap_if_complete waits for - # the collector's .done marker, so an unflushed range keeps its Job - # (and its pod, the last place the metric can be read) regardless. - release_pvc(end) - _reap_if_complete(end, attempt, completed[end]) - if not _attempt_finalized(end, attempt): - # The mission driver writes the final profile as soon as - # jobs_in_progress becomes empty. Keep normal completion open - # until the collector's last atomic metrics write has landed; - # this does not consume dispatch capacity below. - finalizing.append(job_key(int(end), by_end.get(end, 0))) - elif st.failed: - # Completion is terminal for the range, so a Failed Job for a range - # that is already recorded is garbage -- never an input to the retry - # decision. Without this, a losing attempt that outlived the winner - # gets re-classified (disrupted, unknown, ...) and the range is - # dispatched all over again, against a PVC that was already - # released, i.e. a full replay from genesis of work already paid - # for. Sweep the leftover and move on. - if end in completed: - logger.info("range %s already recorded complete; discarding " - "leftover Job for attempt %d", end, attempt) - reap_range_jobs(end) - continue - pod = job_pods.get(j.metadata.name) - if pod is not None: - record_outcome(end, attempt, pod) - backstop_save_pod_log(pod.metadata.name, end, attempt) - # Written by the log collector while the pod still existed; reading - # the pod here would miss anything Karpenter already reaped. - # 1. pod-derived verdict, recorded by the collector while it lived - # 2. Job condition -- survives node consolidation, less precise - # 3. unknown -- retry rather than condemn the run - verdict = read_outcome(end, attempt) or classify_from_job(j) - # ...with one exception. A deadline kill sends SIGTERM, stellar-core - # drains and exits 3, and the pod-derived verdict therefore reads - # `failed` -- which outranks the Job's DeadlineExceeded and condemns - # a range that merely ran long. Only the Job knows the deadline - # fired, so on that condition the Job wins. Measured in the sandbox - # edge suite 2026-07-30: whichever of the two won the race decided - # whether the range was retried or condemned. - # - # Ranked, not unconditional. The Job wins only where the pod has - # nothing more specific to say -- an exit-3 drain, a rejection, no - # surviving classification. Where the pod named the mechanism - # (OOMKilled, DisruptionTarget, an ephemeral eviction) the pod wins, - # because "ran too long" is also true of all of those and picking it - # loses both the remediation and the correct retry budget. - from_job = classify_from_job(j) - if (from_job and from_job.get('outcome') == 'timeout' - and (verdict or {}).get('outcome') not in POD_AUTHORITATIVE_OUTCOMES): - verdict = from_job - if verdict is None: - verdict = {'outcome': 'unknown', 'exitCode': None} - elif verdict.get('source') == 'job-condition': - logger.info("range %s attempt %d classified from Job condition " - "(exit %s); pod was already gone", - end, attempt, verdict.get('exitCode')) - - # Durable before anything reads a tally -- _oom_count and the - # budget check below both count this attempt. - save_verdict(end, attempt, verdict['outcome']) - - retry_mem = retry_eph = None - if verdict['outcome'] == 'timeout': - # Terminal. The deadline is the only thing that ends a range - # wedged on an unreachable archive -- stellar-core retries the - # bucket download forever, logging "maybe stale archive" and - # re-selecting a mirror, because RETRY_A_FEW is per archive so - # the budget never exhausts. Reproduced 2026-07-30: 4 minutes, - # 0 ledgers closed, no exit. Retrying that just spends the - # deadline again and learns nothing, so a range that reaches - # its bound is reported rather than re-run. - reason = None - logger.error("!!! RANGE CONDEMNED !!! %s hit its %ss attempt deadline " - "on attempt %s; this fails the mission. Check its archived " - "log for 'maybe stale archive' -- an unreachable history " - "mirror is the usual cause.", - end, ATTEMPT_DEADLINE_SECONDS, attempt) - elif verdict['outcome'] == 'rejected': - reason = f"rejected by the node before starting ({verdict.get('reason', '?')})" - elif verdict['outcome'] == 'disrupted': - reason = "lost to node disruption" - elif verdict['outcome'] == 'oom': - base = (_profile_overrides(end, escalated=False) or {}).get('memory') - # Rungs climbed = OOMs seen, not attempts made. This attempt's - # own outcome is already on disk, so the count includes it. - retry_mem = mem_for_attempt(_oom_count(end, attempt) + 1, base) - reason = f"OOM-killed at memory limit {mem_for_attempt(attempt, base)}" - elif verdict['outcome'] == 'ephemeral': - retry_eph = eph_for_attempt(attempt + 1) - reason = (f"evicted for exceeding its {eph_for_attempt(attempt)} " - f"ephemeral-storage limit") - elif verdict['outcome'] == 'unknown': - # The pod was gone before anything classified it -- almost always - # because this process was down while the node was reaped. An - # unclassified failure is NOT evidence of a bad ledger range, and - # condemning the run on it would let a monitor restart fail a - # 10-hour job. Retry; a genuinely broken range will exhaust its - # attempts and fail with evidence. - reason = "failed with no surviving classification (monitor restart?)" - elif verdict.get('exitCode') == CATCHUP_INCOMPLETE_EXIT: - # Exit 3 means "did not complete" and covers BOTH a corrupt - # archive AND any interruption -- stellar-core catches SIGTERM, - # drains and exits 3 in ~7s. Nothing in the exit code separates - # them; only a DisruptionTarget condition does, and that is gone - # the moment the pod is. - # - # Condemning on it made every graceful kill fatal, and a - # condemned range aborts the whole mission. Measured in the - # sandbox edge suite 2026-07-30: a pod deleted mid-replay, a pod - # deleted mid-download, and an attempt-deadline kill were all - # classified `failed` at attempt 1 and never retried -- the - # resume path was unreachable through any of them. - # - # Retry on the ordinary range budget. A genuinely broken range - # exhausts MAX_ATTEMPTS and fails with evidence; an interrupted - # one succeeds, usually by resuming at LCL+1. - reason = (f"exited {CATCHUP_INCOMPLETE_EXIT} (did not complete -- " - "corrupt archive or interruption, indistinguishable)") - elif verdict.get('exitCode') is None: - # No exit code means nothing read the container's status: the pod - # was reaped before classification and the verdict came from the - # Job condition alone, which says "Failed" and nothing about why. - # That is the same absence of evidence as `unknown` above and - # takes the same answer -- the only difference is that a Job - # condition happened to survive the pod, which says nothing about - # the ledger range. - # - # Observed on the r5 run 2026-07-30: range 59018943 was condemned - # on attempt 1 with outcome=failed exitCode=None and failed the - # mission, while a dozen sibling ranges reaped the same way - # classified as `unknown`, retried, and passed. - reason = "failed with no exit code (pod reaped before classification)" - else: - reason = None # genuine catchup failure: do not retry - - # Four budgets, by whose fault the attempt was: a hang is usually - # persistent and gets the lowest, a range that is genuinely broken - # gets the middle one, and anything the cluster did to us gets the - # highest. - # - # Each is spent by ITS OWN cause, never by the global attempt index. - # Sharing one counter meant the cap was chosen by the latest verdict - # and then compared against every retry the range had ever had: a - # range that survived five spot evictions (legal, budget 20) reached - # attempt 6, and its first genuine OOM was compared 6 >= 5 and - # condemned -- never retried for an OOM, never escalated, and a - # condemned range fails the mission. On spot, where evictions are - # routine, that made the OOM and disk budgets effectively zero. - # No timeout branch: a timeout sets reason = None above, which is - # terminal, so it never reaches the retry gate below. It had a budget - # of 2 when it was retryable. - if verdict['outcome'] == 'ephemeral': - cap = MAX_EPHEMERAL_ATTEMPTS - spent = _cause_count(end, attempt, ('ephemeral',)) - elif verdict['outcome'] in ENVIRONMENTAL_OUTCOMES: - cap = MAX_DISRUPTION_ATTEMPTS - spent = _cause_count(end, attempt, ENVIRONMENTAL_OUTCOMES) - else: - # The range's own budget: an OOM and a "did not complete" are - # both statements about this ledger range, so they share it. - cap = MAX_ATTEMPTS_PER_RANGE - spent = _cause_count(end, attempt, ('oom', 'failed')) - # This attempt's verdict is already on disk, so `spent` includes it: - # the Nth failure of a cause is the one that exhausts a budget of N, - # exactly as `attempt < cap` behaved for a single-cause range. - if reason is not None and spent < cap: - if verdict['outcome'] == 'oom': - logger.error( - "!!! OOM RETRY !!! range %s was OOM-killed on attempt %d/%d; retrying with " - "memory limit %s -- RAISE THE CONFIGURED MEMORY LIMIT, this run is only " - "surviving by escalating at runtime", end, attempt, MAX_ATTEMPTS_PER_RANGE, retry_mem) - elif verdict['outcome'] == 'ephemeral': - logger.error( - "!!! DISK RETRY !!! range %s %s on attempt %d/%d; retrying with " - "ephemeral-storage %s -- RAISE THE CONFIGURED EPHEMERAL STORAGE, this " - "run is only surviving by escalating at runtime", - end, reason, attempt, cap, retry_eph) - else: - logger.warning("range %s %s on attempt %d/%d; retrying", - end, reason, attempt, MAX_ATTEMPTS_PER_RANGE) - try: - batch_v1.create_namespaced_job(NAMESPACE, build_job( - int(end), by_end[end], attempt + 1, state['owner'], retry_mem, retry_eph)) - except ApiException as e: - if e.status != 409: - raise - current_attempts.add((str(end), attempt + 1)) - # After the successor exists, never before. If the create above - # had failed with the predecessor already gone, the range would - # have no live Job at all and the next pass would redispatch it - # at attempt 1 -- losing the escalated memory that is the whole - # point of the retry. live[] keys on the highest attempt, so the - # two coexisting for one pass is already handled. - # - # Gated like the success path: deleting the Job reaps the pod, - # and backstop_save_pod_log stands down for any range the - # collector has claimed, so there is no second reader. Waiting - # for .metrics means the collector has finalized this attempt -- - # its peaks, its tx_apply and its duration are all durable. - # JOB_TTL_SECONDS reaps it if the collector never gets there. - # Same marker the success path waits for. Peaks were a proxy: - # an attempt that legitimately has none would never be reaped, - # and one whose peaks landed early could be reaped while the - # collector was still reading its log. - if _attempt_finalized(end, attempt): - delete_job(end, attempt) - in_progress.append(job_key(int(end), by_end[end])) - in_flight.add(str(end)) - continue - if reason is not None: - logger.error("range %s exhausted %d attempts (%s)", end, cap, reason) - else: - # The zero-retry path used to log nothing at all: the range just - # appeared under failed{} and the mission aborted with no line - # explaining why. Say it plainly. - logger.error("!!! RANGE CONDEMNED !!! %s failed with outcome=%s exitCode=%s " - "on attempt %d and is NOT retryable; this fails the mission", - end, verdict['outcome'], verdict.get('exitCode'), attempt) - - if end not in failed: - failed[end] = {'attempts': attempt, - 'pod': verdict.get('pod', pod.metadata.name if pod else ''), - 'outcome': verdict['outcome'], - 'exitCode': verdict['exitCode']} - save_progress(progress) - else: - in_progress.append(job_key(int(end), by_end.get(end, 0))) - in_flight.add(str(end)) - - # Nothing halts dispatch. There used to be a monotonic-progress guard here - # that stopped the run when `completed` shrank, on the theory that the record - # had been tampered with and redoing hours of work silently was worse than - # stopping. It kept its high-water mark in memory, so a monitor restart reset - # it to zero -- the guard was disarmed by exactly the event it was there to - # survive, and it only ever fired for a fault it could not have caused. - # - # A reconciler must not gate a decision on state that a restart erases. The - # cost of dropping it is re-running a range, which is idempotent: the PVC - # still holds /data so the attempt resumes from its last closed ledger, and - # the measurements are re-recorded rather than lost. - # - # A condemned range does not stop dispatch either. It used to, which - # deadlocked the driver: the mission waits for `remaining == 0 and - # in_progress == []` (MissionHistoryPubnetParallelCatchupV2.fs), and a frozen - # dispatch leaves `remaining` pinned at however many ranges were never sent, - # forever. The mission still fails on a condemned range -- it reports once - # the run drains, so work already paid for is not thrown away. - # - # Dispatch, heaviest range first (index 0 is the tip), up to PARALLELISM. - created = 0 - # No slots: a range's PVC is keyed by the range itself, so concurrency is - # simply how many are in flight. - capacity = PARALLELISM - len(in_progress) - for end, count in ranges: - if capacity <= 0: - break - key = str(end) - if key in completed or key in failed or key in live: - continue - try: - batch_v1.create_namespaced_job(NAMESPACE, build_job( - end, count, 1, state['owner'])) - current_attempts.add((str(end), 1)) - created += 1 - capacity -= 1 - in_progress.append(job_key(end, count)) - in_flight.add(str(end)) - except ApiException as e: - if e.status != 409: # AlreadyExists: name uniqueness is the mutex - raise - current_attempts.add((str(end), 1)) - # Losing the mutex means the Job EXISTS and is in flight, so it - # occupies a slot exactly like one we created. Falling through - # without spending capacity dispatched PARALLELISM+1 workers -- - # one extra per lost race -- and reported the range as - # `remaining` while it was already running. - capacity -= 1 - in_progress.append(job_key(end, count)) - in_flight.add(str(end)) - - observe_recorded(progress, state['replayed']) - sync_counters(progress, state['counted'], current_attempts) - return { - 'total': len(ranges), - 'completed': len(completed), - 'failed_ranges': [f"{job_key(int(k), by_end.get(k, 0))}|{v.get('pod', '')}" - for k, v in failed.items()], - 'in_progress': in_progress, - 'finalizing': finalizing, - 'created': created, - 'remaining': sum(1 for end, _ in ranges - if str(end) not in completed - and str(end) not in failed - and str(end) not in in_flight), - # A Kubernetes snapshot only. The caller hands this to the independent - # liveness sampler after every dispatch/progress decision is complete. - '_worker_targets': _worker_targets(job_pods.values()), - } - - -def read_mission_start(): - """When this run first started, or None if not recorded yet. - - Its own ConfigMap key, not a field in progress.json: that document is keyed - by ledger range, and anything else in it would be walked as if it were one. - - Read-only on purpose. Creating the ConfigMap here would race the owner - reference, which is only known once reconcile has resolved it, and an - ownerless progress ConfigMap survives `helm uninstall`. - """ - try: - cm = core_v1.read_namespaced_config_map(PROGRESS_CM, NAMESPACE) - return float((cm.data or {})['started_at']) - except (ApiException, KeyError, TypeError, ValueError): - return None - - -def update_status_and_metrics(): - global status - # None until reconcile has an owner reference to attach it to; until then - # process start is correct anyway, because that IS the start of a new run. - mission_start_time = read_mission_start() or time.time() - check_storage_config() - state = {'owner': None, 'replayed': set(), - 'counted': {}} - while True: - try: - reconcile_alive['ts'] = time.time() - if state['owner'] is None: - state['owner'] = owner_ref() - _progress_owner['ref'] = state['owner'] - if read_mission_start() is None: - _patch_cm({'started_at': repr(mission_start_time)}) - - r = reconcile(state) - - # Grafana-only worker responsiveness. Candidate discovery reused the - # authoritative pod snapshot, but the handoff below never performs - # network I/O: /info probes run on a fixed, bounded sampler pool. - refresh_start = time.time() - targets = r.pop('_worker_targets') - try: - worker_counts = publish_worker_liveness(targets) - except Exception as e: - worker_counts = {'up': 0, 'down': 0, 'unknown': len(targets)} - now = time.time() - if now - state.get('last_liveness_error_log', 0) >= 60: - state['last_liveness_error_log'] = now - logger.exception( - "worker liveness publication failed (%s); reporting all " - "current candidates unknown and continuing reconcile", e) - workers_refresh_duration = time.time() - refresh_start - - mission_duration = time.time() - mission_start_time - with status_lock: - visible_in_progress = r['in_progress'] + r['finalizing'] - status = { - 'num_remain': r['remaining'], - 'queue_remain_count': r['remaining'], - 'queue_succeeded_count': r['completed'], - 'queue_failed_count': len(r['failed_ranges']), - 'queue_in_progress_count': len(visible_in_progress), - 'jobs_failed': r['failed_ranges'], - 'jobs_in_progress': visible_in_progress, - 'workers_refresh_duration': workers_refresh_duration, - 'mission_duration': mission_duration, - } - metric_catchup_queues.labels(queue="remain").set(r['remaining']) - metric_catchup_queues.labels(queue="succeeded").set(r['completed']) - metric_catchup_queues.labels(queue="failed").set(len(r['failed_ranges'])) - metric_catchup_queues.labels(queue="in_progress").set( - len(visible_in_progress)) - metric_workers.labels(status="up").set(worker_counts['up']) - metric_workers.labels(status="down").set(worker_counts['down']) - metric_workers.labels(status="unknown").set(worker_counts['unknown']) - metric_refresh_duration.set(workers_refresh_duration) - metric_mission_duration.set(mission_duration) - logger.info("Status: %s", json.dumps(status)) - # Publish on change only -- a 10h run would otherwise issue ~3600 - # no-op ConfigMap writes. - counts = (r['remaining'], r['completed'], len(r['failed_ranges']), - len(visible_in_progress)) - if counts != state.get('last_counts'): - state['last_counts'] = counts - with status_lock: - save_status(status) - - except Exception as e: - logger.exception("Error while reconciling: %s", str(e)) - - time.sleep(RECONCILE_INTERVAL_SECONDS) - - -def run(server_class=HTTPServer, handler_class=RequestHandler): - server_address = ('', 8080) - httpd = server_class(server_address, handler_class) - logger.info('Starting httpd server...') - httpd.serve_forever() - - -if __name__ == '__main__': - # Before any dispatch: the first Job built must already be sized from it. - PROFILE = load_profile() - worker_liveness_sampler.start() - - # Not a logging thread despite the historical name -- this is the reconcile - # loop: dispatch, progress record, metrics, status. Log capture and pod - # classification live in the log-collector sidecar. - reconcile_thread = threading.Thread(target=update_status_and_metrics) - reconcile_thread.daemon = True - reconcile_thread.start() - - try: - run() - finally: - worker_liveness_sampler.close() diff --git a/src/MissionParallelCatchup/lib/attempts.py b/src/MissionParallelCatchup/lib/attempts.py new file mode 100644 index 00000000..0a966696 --- /dev/null +++ b/src/MissionParallelCatchup/lib/attempts.py @@ -0,0 +1,353 @@ +"""What a range's attempts add up to. + +One layer above `records`, which reads a single attempt's files: everything here +answers a question about the RANGE by walking the attempts behind it -- the peak +it reached, what it cost, whether an exit 3 is worth retrying. + +Nothing here touches the cluster. It is also where the blocking file reads live, +so it is the seam an executor would wrap if the monitor ever moved onto a loop. +""" +import gzip +import json +import logging +import os +import zlib + +import config +import records + +logger = logging.getLogger() + + +# PVC size is not profiled: growing it buys no packing. Ephemeral storage is, +# but only in ephemeral mode on on-demand nodes. Any field may be absent, and +# the consumer falls back to its default. +PEAK_FIELDS = ('peakAnonBytes', 'peakWorkingSetBytes', + 'peakEphemeralBytes') + + +def peaks_for_range(end, attempt=1): + """Highest peak any attempt at this range reached, per axis. + + Not just the successful attempt. In pvc mode a pod that dies once replay has + started leaves /data behind, and the next attempt resumes at LCL+1 with + RESUME=true -- skipping the archive download and the bucket apply, which is + where peak memory actually happens. Its peak describes the tail of the range, + not the range, so profiling the winner alone under-reports by the whole + download-vs-replay gap. On spot, where eviction is routine and resume is the + entire point of durable /data, that would make the run unprofileable. + + Attempts that hit a ceiling are counted too. A pod OOM-killed at 8Gi really + did allocate ~8Gi and wanted more, so its peak is a lower bound on demand, + not an artifact of the limit -- and it is the attempt most worth keeping, + because download concurrency scales with available cpu and a pod that + bursted on an idle node can peak above the one that eventually succeeded. + Sizing off the quieter attempt would OOM the range again. There is no false + ratchet: a pod given 8Gi that only touches 1Gi records 1Gi. + + Advisory: used to size a LATER run's requests, never to decide anything + about this one. Any field may be absent. + """ + out = {} + for n in _peak_attempts(end, attempt): + try: + with open(records.metrics_path(end, n)) as fh: + data = json.load(fh) + except (OSError, ValueError): + continue + for k in PEAK_FIELDS: + v = data.get(k) + if v is not None and v > out.get(k, 0): + out[k] = v + return out + + +def _hit_a_ceiling(end, attempt): + """Was this attempt killed at one of its own resource limits?""" + return (records.read_outcome(end, attempt) or {}).get('outcome') in ('oom', 'ephemeral') + + +def _peak_attempts(end, attempt): + """Attempts whose peaks describe this range: the resumed chain, plus any + attempt that died at a limit, wherever it sits. + + A ceiling-hit peak is evidence about the range no matter which pass + produced it -- the process really did allocate that much and want more, so + it is a lower bound on demand and the next run must size above it. That is + the whole self-correcting loop: a range that OOMs at L records L, and + L * PROFILE_MARGIN + PROFILE_CACHE_HEADROOM clears it next time. + + Without this the fresh-start rule silently drops it. Measured on ssc-test + 2026-07-30: an OOM during replay resumes (RESUME accepted, 224 of 252) and + stays in the chain, but an OOM during download does not (25 of 252) -- and + a run at higher cpu is download-bound, so the loop would go quiet exactly + when it is most needed. + + Peaks only. tx_apply and seconds are summed, and a fresh start redoes work + the dropped attempt already did, so including it there would double-count. + """ + chain = set(_resumed_chain(end, attempt)) + return sorted(chain | {n for n in range(1, int(attempt) + 1) + if n not in chain and _hit_a_ceiling(end, n)}) + + +def _resumed_chain(end, attempt): + """Attempts describing one continuous pass over the range, oldest first. + + Stops at the last attempt that ran new-db: that one covered the whole range + on its own, so nothing before it is part of the same pass. + """ + first = int(attempt) + while first > 1 and _attempt_resumed(end, first): + first -= 1 + return range(first, int(attempt) + 1) + + +def _attempt_resumed(end, attempt): + """Did this attempt pick up at LCL+1 rather than run new-db? + + The collector's record is authoritative, and only records a resume. It + decides from the live stream at pod startup and re-reads its own archive at + finalization if it could have missed the line, so an absent flag means the + attempt did not resume -- there is nothing a second archive read here could + find that the collector did not. Measured across a 4805-attempt run: 744 + resumes, and not one the record missed. + """ + try: + with open(records.metrics_path(end, attempt)) as fh: + return json.load(fh).get('resumed') is True + except FileNotFoundError: + return False + except ValueError as e: + logger.warning("could not parse resume metrics for range %s attempt %s: %s", + end, attempt, e) + except OSError as e: + logger.warning("could not read resume metrics for range %s attempt %s: %s", + end, attempt, e) + return False + + +# Exit 3 covers a graceful SIGTERM as well as a real failure, so what decides is +# the cascade stellar-core prints when a history fetch fails. +# +# The anchor pair is adjacent by construction: GetHistoryArchiveStateWork emits +# its message on the same scheduler tick as its child's WORK_FAILURE. The aws +# stderr is relayed unsynchronised, so it is searched for nearby instead. +_FETCH_ANCHOR = 'maybe stale archive' + + +_FETCH_GAVE_UP = 'Catchup failed' + + +# Faults in front of S3: the object is fine, this pod could not reach it. A fresh +# pod on another node is the fix, which is what a retry is. +_FETCH_TRANSIENT = ('Could not connect to the endpoint URL', + 'Unable to locate credentials', 'ExpiredToken', + 'RequestTimeout', 'SlowDown', 'ConnectTimeoutError') + + +# The object genuinely is not there. Retrying cannot help. +_FETCH_TERMINAL = ('Key does not exist', '(404)', 'NoSuchKey') + + +# Lines between the anchor and the give-up line. Small, so a wider window +# cannot credit an earlier fetch failure the range recovered from. +_ANCHOR_WINDOW = 6 + + +# Lines back from the anchor to find the aws stderr that explains it. Wider, +# because concurrent downloads interleave with it during the bucket phase. +_CAUSE_WINDOW = 25 + + +# Tail of the archive to read. Catchup failed is always near the end, and a +# bucket-phase archive can be very large. +_TAIL_LINES = 400 + + +def _archive_tail(end, attempt): + """Last _TAIL_LINES lines of an attempt's archive, or [] if unreadable.""" + path = records.log_path(end, attempt) + tail = [] + try: + with gzip.open(path, 'rt', errors='replace') as fh: + for line in fh: + tail.append(line) + if len(tail) > _TAIL_LINES: + del tail[0] + except FileNotFoundError: + return [] + except (EOFError, gzip.BadGzipFile, zlib.error) as e: + logger.warning("could not read archive %s: %s", path, e) + return [] + except OSError as e: + logger.warning("could not open archive %s: %s", path, e) + return [] + return tail + + +def exit3_retry_cause(end, attempt): + """Why an exit-3 attempt is retryable, or None to condemn it. + + Conservative on purpose: only a fetch fault this function can name earns a + retry. An archive it cannot read, a give-up with no fetch cascade in front of + it, or an aws error it does not recognise all condemn the range -- the + archive survives on the volume, so an unrecognised cause can be read off a + failed run and added here rather than guessed at now. + """ + tail = _archive_tail(end, attempt) + if not tail: + return None + gave_up = max((i for i, l in enumerate(tail) if _FETCH_GAVE_UP in l), + default=None) + if gave_up is None: + return None + anchor = max((i for i in range(max(0, gave_up - _ANCHOR_WINDOW), gave_up) + if _FETCH_ANCHOR in tail[i]), default=None) + if anchor is None: + return None + window = tail[max(0, anchor - _CAUSE_WINDOW):anchor + 1] + for line in reversed(window): + for mark in _FETCH_TERMINAL: + if mark in line: + return None + for mark in _FETCH_TRANSIENT: + if mark in line: + return mark + return None + + +def tx_apply_for_range(end, attempt=1): + """Exact known 'ledger.transaction.apply' seconds for the whole range. + + Summed across the resumed chain, not read from the winning attempt alone. + medida's total is per-process, so a pod that resumes at LCL+1 reports only + the transactions it replayed -- on a range that was interrupted mid-replay + that is the tail, not the range. + + Slightly over-counts: replay restarts at the checkpoint boundary containing + LCL, so up to 64 ledgers can be applied twice. Against a 16320-ledger range + that is <=0.4%, but it is a fixed ledger cost rather than a percentage, so + it grows as ranges shrink. + """ + total = None + for n in _resumed_chain(end, attempt): + leg = _tx_apply_for_attempt(end, n) + if leg is None: + # A disrupted process often never prints its final medida block. + # Absence says the chain is incomplete; a partial sum under-reports. + return None + total = leg if total is None else total + leg + return total + + +def seconds_for_range(end, attempt=1, final=None): + """Compute time for the whole range, summed across the resumed chain. + + `final` is the winning attempt's own duration, which reconcile has in hand + from the pod. Earlier legs come from their .outcome, written when the + monitor classified the failure and still had the pod. + + This is compute, not elapsed: scheduling, image pull, node startup and gaps + between attempts are not in it (see wallSeconds for total scheduling and k8s + noise time). + """ + total = None + for n in _resumed_chain(end, attempt): + if n == int(attempt) and final is not None: + leg = final + else: + leg = _attempt_seconds(end, n) + if leg is None: + return None + total = leg if total is None else total + leg + return total + + +def _attempt_seconds(end, attempt): + """Best durable duration for one attempt, or None when it was never saved.""" + # .outcome carries the pod's own terminated timestamps, and is absent + # whenever the pod was reaped before classification -- every spot eviction -- + # so fall back to the collector's estimate. + leg = (records.read_outcome(end, attempt) or {}).get('attemptSeconds') + if leg is not None: + return leg + try: + with open(records.metrics_path(end, attempt)) as fh: + data = json.load(fh) + # A poller clock starts when the collector attached, so after a restart + # it is a lower bound and must not pass as chain compute. A clock dated + # from the container's own startTime is accepted: it measures the + # container, and it is the only duration a disrupted attempt produces. + if (data.get('attemptSecondsExact') is False + and data.get('attemptSecondsFromContainerStart') is not True): + return None + return data.get('attemptSeconds') + except (OSError, ValueError): + return None + + +def reconstruct_completed_profile(end, attempt): + """Recompute recoverable profile fields from immutable attempt artifacts. + + Durations and tx-apply totals follow only the continuous resumed chain, so a + fresh retry never double-counts discarded work. Peaks use that chain plus + every attempt that hit a resource ceiling. Missing duration or tx-apply legs + make that aggregate absent rather than publishing a lower bound as a total. + Complete tx-apply legs retain the existing <=64-ledger overlap. + + Reconstructable: persisted sampled peaks, complete attemptSeconds chains in + .outcome/.metrics, and complete txApplySeconds chains in .metrics/.log.gz. + Not reconstructable: whole-chain wall time, samples never persisted, or a + duration/tx-apply leg whose process and archive are both gone. + """ + rebuilt = peaks_for_range(end, attempt) + seconds = seconds_for_range(end, attempt) + if seconds is not None: + rebuilt['seconds'] = seconds + tx_apply = tx_apply_for_range(end, attempt) + if tx_apply is not None: + rebuilt['txApply'] = tx_apply + return rebuilt + + +def _apply_profile_reconstruction(record, rebuilt): + """Merge reconstruction without lowering stronger persisted evidence.""" + updates = {} + for key, value in rebuilt.items(): + current = record.get(key) + if current is None or value > current: + updates[key] = value + record.update(updates) + return updates + + +def _repair_completed_profile(end, attempt, record): + """Merge exact reconstruction and remove unverifiable chain aggregates.""" + rebuilt = reconstruct_completed_profile(end, attempt) + updates = _apply_profile_reconstruction(record, rebuilt) + if len(list(_resumed_chain(end, attempt))) > 1: + for key in ('seconds', 'txApply'): + if key not in rebuilt and record.get(key) is not None: + # Once resume proves this is a chain, the sum of surviving legs + # is a lower bound rather than a total, so omit it. + record.pop(key) + updates[key] = None + return updates + + +def _tx_apply_for_attempt(end, attempt=1): + """Exact 'ledger.transaction.apply' seconds for ONE attempt, or None. + + Read from the collector's record and nowhere else. The collector parses the + block out of the live stream and re-reads its own archive at finalization + when it has no total, so a second reader here could only repeat that work + over the same bytes with the same medida window -- which is how the monitor + came to carry its own copy of the parser. + """ + try: + with open(records.metrics_path(end, attempt)) as fh: + value = json.load(fh).get('txApplySeconds') + except (OSError, ValueError): + return None + return None if value is None else float(value) diff --git a/src/MissionParallelCatchup/lib/config.py b/src/MissionParallelCatchup/lib/config.py new file mode 100644 index 00000000..82297c9e --- /dev/null +++ b/src/MissionParallelCatchup/lib/config.py @@ -0,0 +1,588 @@ +"""Configuration and run state for the parallel catchup job monitor. + +Read through the module, never copied out of it: + + import config + ... config.REQ_CPU ... + +`from config import REQ_CPU` binds a COPY. A test's monkeypatch and the startup +assignment of config.PROFILE both rebind the attribute on this module, and a +copy taken at import time never sees either -- silently, with the test passing +against the default. A module object is a singleton, so reading through it is +what makes those visible everywhere. +""" +import os + +# ============================================================================= +# 1. stellar-core workload +# ============================================================================= +CORE_IMAGE = os.getenv('CORE_IMAGE') + +ASAN_OPTIONS = os.getenv('ASAN_OPTIONS', '') + +# Which ledger ranges to run. These are pure inputs to the range generator: +# dispatch recomputes the whole list every reconcile, so a restart must +# reproduce it exactly. +RANGE_GENERATOR = os.getenv('RANGE_GENERATOR', 'uniform') # uniform | logarithmic + +VALID_RANGE_GENERATORS = ('uniform', 'logarithmic') + +# Both generators emit tip-first, which front-loads the most expensive ranges: +# the bucket set only grows with ledger position. 'oldest-first' reverses that, +# so a profiling run measures the cheap early ranges before it can be +# interrupted, and the expensive tip ranges last. +RANGE_ORDER = os.getenv('RANGE_ORDER', 'tip-first') # tip-first | oldest-first | longest-first + +VALID_RANGE_ORDERS = ('tip-first', 'oldest-first', 'longest-first') + +STARTING_LEDGER = int(os.getenv('STARTING_LEDGER', 0)) + +LATEST_LEDGER_NUM = int(os.getenv('LATEST_LEDGER_NUM', 0)) + +LEDGERS_PER_JOB = int(os.getenv('LEDGERS_PER_JOB', 16000)) + +OVERLAP_LEDGERS = int(os.getenv('OVERLAP_LEDGERS', 320)) + +# logarithmic only: chunk size halves toward the tip and stops shrinking here. +LOGARITHMIC_FLOOR_LEDGERS = int(os.getenv('LOGARITHMIC_FLOOR_LEDGERS', 64000)) + +# ============================================================================= +# 2. Kubernetes objects this monitor creates +# ============================================================================= +NAMESPACE = os.getenv('NAMESPACE', 'default') + +RUN_NAME = os.getenv('RUN_NAME', 'parallel-catchup') + +PROGRESS_CM = f"{RUN_NAME}-catchup-progress" + +LABEL_RUN = 'catchup.stellar.org/run' + +LABEL_RANGE = 'catchup.stellar.org/range-end' + +LABEL_ATTEMPT = 'catchup.stellar.org/attempt' + +# Workers need IRSA to read the S3 history mirror. Without it they silently fall +# back to the public archive, which throttles at 1024 and kills the run with +# curl 22 -> catchup exit 3. The name matches the old StatefulSet's so existing +# IRSA trust policies keep matching. +WORKER_SERVICE_ACCOUNT = os.getenv('WORKER_SERVICE_ACCOUNT', '') + +# Pod resources. Requests only: workers are given no cpu limit and no memory +# limit at all. +# +# CPU because a limit only throttles a pod that could otherwise use idle cores, +# and throttling changes what the range measures -- less cpu means less download +# concurrency means a lower peak, so a throttled attempt records a figure an +# unthrottled one cannot reproduce. +# +# Memory because a limit is a hard cap on anon PLUS page cache, and sizing it +# per-range from a profile got it wrong in the one direction that has no alarm +# on it. Measured 2026-07-31, range 39210943: sized at 1729Mi from a neighbour, +# genuinely needed 1620Mi of anon, which left ~110Mi for cache. It never OOMed +# -- it thrashed. 544k major page faults, 0.22 cores used on a node it had +# entirely to itself, 0.95 ledgers/s against a neighbour norm of 3.3, and it +# held 1092 idle slots open for three hours at the end of the run. +# +# Without a limit the request still does the real work: it places the pod and +# it sets eviction order under node pressure. What goes away is the cliff. +REQ_CPU = os.getenv('REQ_CPU', '1250m') + +REQ_MEM = os.getenv('REQ_MEM', '9Gi') + +# Range profile from an earlier run: tightens per-range requests so more +# workers fit per node. Requests only -- limits stay as configured, so the +# failure semantics and the OOM/disk escalation ladders are unchanged. +PROFILE_PATH = os.getenv('PROFILE_PATH', '') + +PROFILE_MARGIN = float(os.getenv('PROFILE_MARGIN', 1.15)) + +# No safety margin on cpu, unlike memory. Under-requesting cpu costs contention +# and the pod can still burst; under-requesting memory gets it OOMKilled. +# Ceiling for profile-derived memory, above the unprofiled limit for the same +# reason: a range that really needs more than the configured limit must be able +# to ask for it rather than be pinned under its own measured peak. The OOM +# escalation ladder can still climb past this on a retry. +PROFILE_MAX_MEM = os.getenv('PROFILE_MAX_MEM', '32Gi') + +# Memory is sized from rss (the range's real demand), NOT from peak working +# set. Working set is whatever limit it was measured under -- the kernel grows +# page cache to fill it -- so sizing from it is circular. Measured on ssc-test +# with one 420-ledger range: working set went 2.33 -> 3.61 -> 7.48 -> 13.49 GiB +# under 2560Mi/4Gi/8Gi/24000Mi limits while rss moved only 2256 -> 2488 MiB, and +# wall-clock did not move at all (776s / 775s / 746s / 773s). Catchup streams -- +# buckets are downloaded once, applied once, ledgers replayed once -- so cache +# has nothing to give back and PROFILE_MARGIN alone is the allowance. +# A multiplicative margin alone is not enough: memory.max bounds anon PLUS page +# cache, and at small rss 10% is nothing. Measured on ssc-test 2026-07-29 with +# headroom 0: ranges profiled at 190 MiB rss got a 209 MiB limit -- 19 MiB of +# slack for all growth and cache -- and 90 of them OOMKilled within 90s. The +# earlier 4Gi validation hid this because 1.1x of 2.4 GiB is 240 MiB of slack. +PROFILE_CACHE_HEADROOM = os.getenv('PROFILE_CACHE_HEADROOM', '512Mi') + +# Extra allowance scaled by the range's measured runtime. Long ranges keep more +# page cache and allocator slack live at once; 0 disables the allowance. +PROFILE_RUNTIME_MEMORY_INSURANCE = os.getenv('PROFILE_RUNTIME_MEMORY_INSURANCE', '3Gi') + +# Ephemeral-storage gets the same two allowances as memory, for the same +# reasons. Measured on the 2026-08-01 on-demand run: peak 37.76Gi against a +# flat 40Gi limit -- 6% of headroom on a path that has never once fired in a +# real run, so a range 6% worse than the worst seen would be evicted 137 with +# no diagnostic pointing at disk. +# +# Flat allowance added to every range's measured peak. Covers the container +# image, logs and the sqlite WAL, none of which scale with the range. +PROFILE_EPHEMERAL_HEADROOM = os.getenv('PROFILE_EPHEMERAL_HEADROOM', '2Gi') + +# Runtime-weighted allowance on top. Disk tracks runtime closely (pearson 0.920 +# across 3985 ranges: runtime decile 0 uses 0.1Gi, decile 9 uses 24.7Gi), so +# the ranges that need the margin are exactly the ranges this gives it to. +PROFILE_RUNTIME_EPHEMERAL_INSURANCE = os.getenv('PROFILE_RUNTIME_EPHEMERAL_INSURANCE', '8Gi') + +# Ceiling for profile-derived disk. Deliberately ABOVE LIM_EPHEMERAL: that flat +# limit is what an UNMEASURED range gets, and capping a measured range at it +# would throw away the measurement -- the worst observed range wants 43Gi after +# margin alone. +PROFILE_MAX_EPHEMERAL = os.getenv('PROFILE_MAX_EPHEMERAL', '64Gi') + +REQ_EPHEMERAL = os.getenv('REQ_EPHEMERAL', '') + +LIM_EPHEMERAL = os.getenv('LIM_EPHEMERAL', '') + +# Placement. The taint toleration is emitted as {key, effect} with no value: +# the default Equal operator does not match "" against "true". +NODE_LABEL_KEY = os.getenv('NODE_LABEL_KEY', '') + +NODE_LABEL_VALUE = os.getenv('NODE_LABEL_VALUE', '') + +# ANDed with the label above when set: 'spot' or 'on-demand'. Both capacity +# variants of a tier carry the same label value, so this is what separates them. +# Karpenter labels every node with karpenter.sh/capacity-type itself. +CAPACITY_TYPE = os.getenv('CAPACITY_TYPE', '') + +AVOID_NODE_LABEL_KEY = os.getenv('AVOID_NODE_LABEL_KEY', '') + +AVOID_NODE_LABEL_VALUE = os.getenv('AVOID_NODE_LABEL_VALUE', '') + +TOLERATE_TAINT = os.getenv('TOLERATE_TAINT', '') + +# Worker /data. pvc keeps it across pods, so an evicted range resumes at L+1 -- +# that is what makes spot viable. ephemeral puts it on the node disk: denser +# packing, no resume, and REQ_EPHEMERAL must be sized to hold the catchup DB. +# One PVC per range, not per concurrency slot: measured on ssc-test, 300 jobs +# with a PVC each cost no more wall-clock than 300 jobs reusing 40. +STORAGE_MODE = os.getenv('STORAGE_MODE', 'pvc') # pvc | ephemeral + +STORAGE_CLASS = os.getenv('STORAGE_CLASS', '') + +# 60Gi to match the tier nodes' ephemeral allowance. peakEphemeralBytes tops +# out at 37.8Gi across the whole 2026-08-01 profile, so this covers every +# range measured, with headroom for the tip to keep growing. +STORAGE_SIZE = os.getenv('STORAGE_SIZE', '60Gi') + +# A Nitro node allows ~26 EBS attachments (CSINode allocatable), and Karpenter +# sizes nodes on CPU/memory only -- it will happily put 40 volume-mounting pods +# on one 4-vCPU node, where they serialise through the attachment slots and get +# rejected with VolumeAttachmentLimitExceeded (observed on ssc-test). +# +# Guard with a spread constraint rather than a warning. maxSkew alone cannot cap +# per-node count -- with a single node there is one domain and therefore no skew +# -- so minDomains is what forces enough nodes. Both are inert at realistic +# density: REQ_CPU=1800m yields ~4 workers on an 8-vCPU node, so CPU demands far +# more nodes than this floor ever asks for. 0 disables. +MAX_VOLUMES_PER_NODE = int(os.getenv('MAX_VOLUMES_PER_NODE', 24)) + +# Job/pod lifetimes. +# SIGTERM -> SIGKILL budget. stellar-core exits ~7s after SIGTERM (measured), so +# this is slack rather than a target. +WORKER_GRACE_SECONDS = int(os.getenv('GRACE_SECONDS', 100)) + +# Seconds to stall inside preStop before the container is signalled. 0 disables. +# +# Sized to cover the collector's DETECTION LAG, which is the specific hole it +# fills. The collector notices DisruptionTarget on its pod-list cycle and only +# then drops that pod to 1s polling; if SIGTERM lands inside that blind window +# the poller is still on its lazy LOG_POLL_SECONDS cadence. Measured on +# ssc-test: a 60s preStop with 10s polling and no disruption detection still +# lost txApply, while 1s polling with no preStop at all captured it. So this is +# not what saves the metric -- it is what makes sure the detection has happened +# before the kill. +# +# 20s, not COLLECTOR_POLL_SECONDS. That constant is the SLEEP between cycles, +# not the cycle: each one also lists every pod and sweeps kubelet +# /stats/summary on every node, which at 768 workers over ~250 nodes is +# unmeasured and plausibly another 5-15s. The margin is +# (preStop + pod-object linger) - (detection + one 1s poll), and with the +# linger measured at 7.8s it goes NEGATIVE at a 12s cycle if this is 5s. Above +# the true cycle time the margin plateaus at +6.8s, so overshooting is free +# while undershooting silently loses the metric. +# +# A spot reclaim gives ~120s of notice and does not need this at all; an +# eviction-API kill or a fast drain signals immediately and does. +# +# Do NOT try to SIGTERM the process from inside the hook and hold the pod open +# afterwards: measured, the pod object survived 10.2s that way versus 69s for a +# plain sleep, because a container dies with its PID 1 and the kubelet does not +# defer deleting the object until the hook returns. +# +# Costs nothing on a healthy exit -- preStop does not run when the container +# exits on its own, only when the kubelet is tearing it down. At ~810 evictions +# a run, 5s each is about 1.1 pod-hours. +# +# Must stay comfortably under WORKER_GRACE_SECONDS: the hook and the SIGTERM +# drain share that one budget, and a hook still running when it expires is +# SIGKILLed, which loses exactly the output this exists to save. +WORKER_PRESTOP_SLEEP_SECONDS = int(os.getenv('PRESTOP_SLEEP_SECONDS', 5)) + +# Must comfortably exceed any plausible monitor outage: completion is recorded +# to the ConfigMap by this process, and a Job reclaimed before that happens +# reads as "never ran" and gets redone. +# Backstop only. reconcile() deletes each Job explicitly once its record is +# durable, so the TTL exists for the cases that skip that path: a terminally +# failed range kept for inspection, or a success whose metrics never landed. +JOB_TTL_SECONDS = int(os.getenv('JOB_TTL_SECONDS', 600)) + +# Measured on ssc-test: stellar-core does NOT fail on an unreachable history +# archive, an absent ledger range, or a bucket that will not decompress. It +# retries every mirror with growing backoff and stays Running indefinitely -- +# no exit code, no failure, the slot held for the life of the run. A hang is a +# more likely real failure than a non-zero exit, and this deadline is the only +# thing that makes it observable. 0 disables. +# +# Flat, deliberately -- NOT scaled by the range's profiled runtime. That was +# tried and removed. A deadline has to bound a range's WORST case, but a profile +# only offers a neighbour's TYPICAL case, and the two are far apart here: +# runtimes span 190x (p25 771s, max 5.9h), range keys are anchored to the +# network tip so a profile from an earlier run matches ZERO keys exactly and +# every lookup lands on a neighbour, and ~2% of those neighbours are 3-38x +# cheaper than their surroundings. Backtested honestly across that grid offset +# (run4 profile -> r5 actuals, 3983 ranges): a 2x factor falsely kills 134 +# ranges, 4x kills 46, 6x kills 21. Flat 12h kills none. +# +# The asymmetry decides it. A false kill loses a range, and a timeout is +# terminal, so it fails the mission. A genuine wedge holds ONE slot out of +# 1092-1500 for 12h -- around 0.1% of a run's capacity. Never trade a certain +# catastrophe against a rounding error. +# +# 12h is a safe bound, not a good detector: it takes half a day to catch +# something provably dead in 4 minutes. The right signal is ledger-close +# progress, not elapsed time -- a wedged core closes zero ledgers while still +# logging, so `.state` (last log line) cannot see it and a new +# lastLedgerCloseAt would. Left undone on purpose; it needs a threshold above +# the initial bucket-apply phase, which legitimately closes nothing for ~20min +# on the longest ranges. +ATTEMPT_DEADLINE_SECONDS = int(os.getenv('ATTEMPT_DEADLINE_SECONDS', 0)) + +# kube-state-metrics turns a pod's `mission` label into label_mission, which the +# Grafana container panels join on. Every other mission gets it from +# StellarKubeSpecs; this chart never has, so parallel catchup has never appeared +# in those panels. +# +# OFF by default and deliberately so: those panels are sum() by (pod, container) +# with a legend table, so at 1024 workers they would pull ~1024 series into any +# view with mission=$__all selected, degrading a shared dashboard for people who +# did not ask for it. Enable per-run once the panels aggregate (topk). +MISSION = os.getenv('MISSION', '') + +EMIT_MISSION_LABEL = os.getenv('EMIT_MISSION_LABEL', 'false').lower() == 'true' + +# ============================================================================= +# 3. This monitor's own behaviour +# ============================================================================= +PARALLELISM = int(os.getenv('PARALLELISM', 3)) + +# Effectively the OOM budget: `failed` is the only other outcome that reaches +# it, and that one sets no retry reason. Escalation counts OOMs rather than +# attempts, so rung N means the range genuinely wanted more N times. +# +# Deliberately stops short of MEM_ESCALATION_CAP: 5 rungs is 1.5^4 = 5x the +# profile figure, and a range needing more than that is not mis-sized, it is +# broken -- chasing it to 48Gi parks a whole r8a.2xlarge on one range for hours. +# The cost of stopping is that the range is condemned, and today a condemned +# range aborts the run. That coupling is the thing to fix, not this number. +# Attempts each failure cause gets before the range is condemned. The whole +# retry policy, in one table. +# +# Every budget is spent by ITS OWN cause: an OOM never consumes the disk budget +# and a spot eviction never consumes either. A cause that is NOT here is +# condemned the first time it happens -- a timeout, a real catchup failure, an +# exit 3 with nothing in its archive, and anything nothing could classify. +# +# disrupted the cluster took the pod away mid-run, which proves the range +# itself was fine. Effectively unlimited: on spot a healthy range +# is legitimately evicted dozens of times, and 100 is far past any +# rate a real run has produced while still terminating. +# rejected the kubelet refused the pod before any container ran (attachment +# limits, admission churn). The range never started, so a retry +# cannot mask anything about it. +# fetch-fault an exit 3 whose archive named a failed history fetch. An +# unreachable mirror is the cluster's problem, not the range's. A +# plain `failed` has no entry: a real catchup failure, and an exit 3 +# with nothing in its archive, are both condemned on sight. +# oom each retry escalates the memory request one rung. +# ephemeral each retry escalates the disk limit one rung. Smallest, because +# an eviction repeats identically until the range gets more disk. +# +# The MAX_* names below exist so the chart can tune each one; ATTEMPT_BUDGETS is +# what the code reads, so tests patch the map rather than the constants. +MAX_DISRUPTION_ATTEMPTS = int(os.getenv('MAX_DISRUPTION_ATTEMPTS', 100)) +MAX_REJECTED_ATTEMPTS = int(os.getenv('MAX_REJECTED_ATTEMPTS', 100)) +MAX_FETCH_FAULT_ATTEMPTS = int(os.getenv('MAX_FETCH_FAULT_ATTEMPTS', 20)) +MAX_OOM_ATTEMPTS = int(os.getenv('MAX_OOM_ATTEMPTS', 5)) +MAX_EPHEMERAL_ATTEMPTS = int(os.getenv('MAX_EPHEMERAL_ATTEMPTS', 4)) + +ATTEMPT_BUDGETS = { + 'disrupted': MAX_DISRUPTION_ATTEMPTS, + 'rejected': MAX_REJECTED_ATTEMPTS, + 'fetch-fault': MAX_FETCH_FAULT_ATTEMPTS, + 'oom': MAX_OOM_ATTEMPTS, + 'ephemeral': MAX_EPHEMERAL_ATTEMPTS, +} + +EPH_BUMP_FACTOR = float(os.getenv('EPH_BUMP_FACTOR', 1.5)) + +EPH_ESCALATION_CAP = os.getenv('EPH_ESCALATION_CAP', '200Gi') + +ATTEMPT_OUTCOMES = ('disrupted', 'oom', 'ephemeral', 'timeout', + 'rejected', 'unknown', 'failed', 'fetch-fault') + +# Verdicts only the pod can produce, and which a Job-level DeadlineExceeded must +# never overwrite. Each names a specific mechanism -- the kubelet OOM-killed it, +# the node was draining, the ephemeral limit blew -- and each earns a different +# retry budget and a different remediation. "The Job ran too long" is also true +# of every one of them and says nothing about which. An OOM downgraded to a +# timeout retries at the same memory limit that just killed it and gets 2 +# attempts instead of 5; a spot eviction downgraded to a timeout gets 2 instead +# of 20. +POD_AUTHORITATIVE_OUTCOMES = ('oom', 'disrupted', 'ephemeral', 'timeout') + +# stellar-core's "did not complete". Ambiguous by construction: a corrupt bucket +# and a SIGTERM during replay both produce it, so it must never be treated as +# proof that a range is broken. +CATCHUP_INCOMPLETE_EXIT = 3 + +# An OOM means requests/limits are mis-sized for this range. Escalate so the run +# can finish, but say so loudly -- surviving by escalating at runtime is a +# configuration bug, not a success. +MEM_BUMP_FACTOR = float(os.getenv('MEM_BUMP_FACTOR', 1.5)) + +# Ceiling for that escalation. Above the largest schedulable node the retry sits +# Pending forever, which looks like a hang rather than a failure. +MEM_ESCALATION_CAP = os.getenv('MAX_MEM', '48Gi') + +# Reconcile loop: dispatch, refresh status, publish metrics. The env var is +# named LOGGING_INTERVAL_SECONDS for historical reasons, from when this loop +# only logged. +RECONCILE_INTERVAL_SECONDS = int(os.getenv('LOGGING_INTERVAL_SECONDS', 10)) + +# Worker responsiveness is cosmetic and sampled independently from reconcile. +# Thirty seconds and three failures restore the old ~90-second down threshold, +# while a five-second request budget gives a busy admin endpoint substantially +# more room than the old one-shot two-second probe. + +LIVENESS_PROBE_TIMEOUT_SECONDS = os.getenv('LIVENESS_PROBE_TIMEOUT_SECONDS', '5') + + +LIVENESS_MAX_CONCURRENCY = os.getenv('LIVENESS_MAX_CONCURRENCY', '32') +# Wall-clock bound on one sweep. The reconcile loop waits for it, so this +# is the most a fleet of unreachable workers can delay dispatch. +LIVENESS_SWEEP_SECONDS = os.getenv('LIVENESS_SWEEP_SECONDS', '15') + +# Shared with the log-collector sidecar, which owns writes here: it streams each +# worker's log and records the .outcome verdict while the pod still exists. +LOG_DIR = os.getenv('LOG_DIR', '/logs') + +SAVE_SUCCESS_LOGS = os.getenv('SAVE_SUCCESS_LOGS', 'true').lower() == 'true' + +# The authoritative copy of the progress record lives on the logs PVC, not in +# the ConfigMap. A ConfigMap is capped at 1 MiB and this record is ~172 bytes +# per completed range, so it dies at ~6100 ranges -- reachable simply by halving +# ledgersPerJob. Measured mid-run on ssc-test: 348KB at 2024 completed ranges, +# which projects to ~65% of the cap at 3982 -- close enough that the next +# slicing change would have hit it. Worse, every completion rewrote the whole +# document through the API server, so a full run meant thousands of +# escalating-size etcd writes. +# +# The ConfigMap is still written, because the mission driver reads it without +# exec'ing into the pod, but it is now a best-effort mirror: if it fails, the +# run carries on from the file. +PROGRESS_FILE = os.path.join(LOG_DIR, 'progress.json') + +PROFILE = None + +# --- pool tiers ------------------------------------------------------------- +# +# A range picks a NODEPOOL by its measured memory, and gets that pool's node to +# itself. This replaces the cpu ladder, which tuned a dimension that turned out +# not to be the binding one. +# +# Why memory and not cpu. Measured 2026-08-03 on one range across four instance +# shapes, isolated, no memory limit: +# +# 2 -> 4 cores replay +2.8% bucket-apply 1.37x +# 4 -> 8 cores replay +1.5% bucket-apply 1.18x +# AMD vs Intel replay +16% bucket-apply 1.35x +# +# Replay is ~93% of a job and is flat in core count from 2 upward -- it draws +# ~1.05 cores whatever it is given. So a cpu REQUEST never bought throughput. +# What it bought was neighbours-per-node, and memory is what actually fails: a +# range whose working set does not fit gets OOMKilled, not slowed down. +# +# Cuts are `node_usable / 1.60`, covering the p99 of run-to-run growth in the +# same range's peakAnonBytes (18,073 observations across five profiles: p50 0.97, +# p90 1.28, p99 1.60, max 2.83). Validated the hard way: range 63080767 measured +# 13.75Gi was placed on nodes with 14.1/14.3Gi allocatable -- a 1.03x margin -- +# and OOMKilled on BOTH during bucket-apply, before closing a ledger. +# subdwarf's cut is 0 on purpose: nothing can satisfy `gib < 0`, so the tier is +# defined and provisionable but never routed to. Kept rather than deleted so the +# bottom of the ladder is there to experiment with; c8a.medium (1.42Gi +# allocatable) cannot hold a range the profile actually contains. +POOL_TIERS = os.getenv( + 'POOL_TIERS', + '0:subdwarf,0.79:dwarf,1.61:subgiant,3.87:giant,8.85:supergiant,18.38:hypergiant,:supernova') + +# Prepended to the tier name to form the node label value, e.g. catchup-dwarf. +# Empty disables pool routing entirely and every worker keeps the single global +# NODE_LABEL_VALUE, which is exactly today's behaviour. +POOL_PREFIX = os.getenv('POOL_PREFIX', '') + +# Where a range goes when the profile has no entry for it (past the profile's +# top, i.e. the newest ledgers) and when there is no profile at all. +POOL_UNPROFILED = os.getenv('POOL_UNPROFILED', 'protostar') + +POOL_NO_PROFILE = os.getenv('POOL_NO_PROFILE', 'nebula') + +# cpu request per tier. NOT a demand estimate -- a claim token. Isolation is the +# point: freeing a node of its 3 neighbours raised throughput 29-92% while cpu +# draw FELL, so the contended resource is memory bandwidth and shared cache, not +# compute. Kept at or below the SMALLEST node in the tier so the low-weight +# fallback rungs stay schedulable (dwarf can land on a 1-vCPU c8a.medium). +# +# Memory, not cpu, is what actually enforces the isolation -- see _pool_memory. +# Half the node for most tiers. hypergiant and supernova are sized to the +# SMALLEST shape in their pool instead: x8i.large is r8a.xlarge with half the +# cores and the same 32 GiB, x8i.xlarge is r8a.2xlarge with half the cores and +# the same 64 GiB, so preferring them buys identical RAM for half the spot +# quota. A half-the-node 2.00/4.00 claim does not fit an x8i node once the 215m +# of daemonsets is counted, which is why those pools won no nodes at all on +# 2026-08-03. Below half, cpu no longer isolates the pod on the larger fallback +# shapes -- memory does, and it holds because every type within a tier carries +# the same RAM. +POOL_CPU = os.getenv( + 'POOL_CPU', + 'subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:3.80') + +# vCPU of the SMALLEST node in each tier's pool. This is what decides whether a +# promotion is free, and it cannot be inferred from POOL_CPU, which is a claim +# rather than a node size. +# +# It reads the smallest shape, not the likeliest one, and that was quietly wrong +# while the x8i pools existed. hypergiant listed x8i.xlarge (4 vCPU) at w80 under +# two 8-vCPU rungs at w100/w90, so this map said 4 and supergiant->hypergiant +# priced as free -- while Karpenter tried w100 first and the promotion really cost +# 4->8. The finished 1200-worker run landed 12 hypergiant nodes on 2xlarge shapes +# against 1 on x8i.xlarge. The x8i spot pools were removed on 2026-08-04, so these +# figures are now both the smallest AND the top-weighted shape, and the two top +# rungs cross a class honestly -- which is what POOL_CROSS_RUNGS now carries. +POOL_VCPU = os.getenv( + 'POOL_VCPU', + 'subdwarf:2,dwarf:2,subgiant:4,giant:4,supergiant:4,hypergiant:8,supernova:16,protostar:8,nebula:8') + +# Rungs allowed to cross a vCPU class anyway, "from->to", comma separated. The +# guard exists because a speculative promotion that doubles cores is usually a +# bad trade, so anything listed here needs a measurement behind it. +# +# hypergiant->supernova costs +2 vCPU per range and measured only 1.23x, so on +# its own it is not worth it -- simulated over the 2026-08-03 run it saved +# exactly 0 minutes, because the longest job was a supergiant that this rung +# cannot reach. It earns its place only in company: the long jobs alternate +# between the two tiers, so fixing one exposes the other. supergiant->hypergiant +# alone is worth 8 min, this alone 0, and the pair 27 min. +# +# supergiant->hypergiant is listed too, and is a no-op at today's spot sizes: the +# doubled pools put both tiers on 4-vCPU nodes, so that rung is free and clears +# the guard without an entry. It is here so the rung survives the pools diverging +# -- if hypergiant ever bottoms out above supergiant, the guard would silently +# shut a rung that is deliberately open. On on-demand, where supergiant is 2 vCPU +# and hypergiant 4, the same entry is load-bearing. +POOL_CROSS_RUNGS = os.getenv('POOL_CROSS_RUNGS', 'supergiant->hypergiant') + +# Rungs that never run, whatever the vCPU comparison says. Empty by default: with +# the spot pools doubled, promotion lands a range on a bigger SHARED node, and +# sharing is what the rung is really buying. Measured on ssc-test 2026-08-04 on +# one range, two pods to a node: on an 8-vCPU node a co-tenant cost 1.02x per pod +# (r8id.2xlarge, 3.78 and 3.91 lps), on a 4-vCPU node it cost 1.58x. Two pods on +# 8 cores leave 4 each, which the workload does not use; two on 4 cores leave 2, +# which is the floor. +# +# Caveat worth keeping in view: the bump fires on peakWorkingSetBytes, and working +# set does not predict throughput. Same x8i.xlarge box, same range, same time, +# only cgroup memory.max differing: 28 GiB ran 1.83 lps and 56 GiB ran 1.70. So +# this promotes for a reason that is not the reason it helps -- it reaches the +# right nodes via the wrong signal, and will promote ranges that gain nothing. +# Sizing the rung on peakAnonBytes, or widening the tier->instance map directly, +# would target those nodes deliberately. +# +# hypergiant->supernova is denied on both capacity types. Its cost rose once the +# x8i pools were removed on 2026-08-04: supernova's only spot shapes are now +# 4xlarges, so the rung moves a range from 8 vCPU to 16 rather than the 8-vCPU +# x8i.2xlarge it used to reach. It is denied here rather than merely dropped from +# POOL_CROSS_RUNGS so it stays shut if the tiers ever land on equal-sized nodes, +# which is the state that made the guard unable to hold supergiant->hypergiant. +POOL_BLOCK_RUNGS = os.getenv('POOL_BLOCK_RUNGS', 'hypergiant->supernova') + +# Memory request for a pooled range is the TIER'S CUT, not the range's own +# measurement, and that is deliberate two ways. +# +# It guarantees one pod per node without depending on the cpu token: a tier's +# node is cut*1.60 of usable memory, so two pods asking cut apiece need 2*cut, +# which always exceeds 1.60*cut. The cpu claim cannot do this alone because a +# tier spans node sizes (dwarf reaches a 1-vCPU c8a.medium and a 2-vCPU +# t3a.small), so no single cpu value both schedules on the small one and fills +# the large one. +# +# And the request no longer needs a safety margin. PROFILE_MARGIN, cache +# headroom and runtime insurance all existed to keep a pod under its own LIMIT; +# with no memory limit and the node to itself, a pod may use everything the node +# has. The margin moved into the node size -- which is where it can actually be +# enforced, since the kubelet kills on node pressure, not on request. +# Per-tier memory request: exactly 50% of the tier node's NAMEPLATE capacity. +# +# 50% is what isolates. Two pods asking half the nameplate need the whole node, +# which always exceeds allocatable -- so a second pod can never fit, on every +# tier, without depending on how the kubelet happens to reserve. +# +# Verified against measured nodes rather than assumed: a c8a.medium reports +# 1892Mi capacity, 1449Mi allocatable, and carries 154Mi of daemonsets, leaving +# 1295Mi -- so the 1024Mi request schedules with room, and 2048Mi of two pods +# cannot. The same holds up the ladder. +# +# t3a.micro is absent on purpose: 413Mi allocatable cannot host a pod at all on +# this cluster, so subdwarf shares dwarf's node type and is emptied by its cut. +POOL_MEM = os.getenv( + 'POOL_MEM', + 'subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:14336Mi') + +_SORTED_SECONDS = None + + +# Sized for the dispatch burst rather than a steady LIST rate: ~1024 Jobs + PVCs +# go out at once at the head of a wave. +CONNECTION_POOL = int(os.getenv('CONNECTION_POOL', '64')) + +# Coerced here rather than at each use site so no importer can ever see the +# string form, and a bad value fails at import instead of at the first probe. +try: + LIVENESS_PROBE_TIMEOUT_SECONDS = float(LIVENESS_PROBE_TIMEOUT_SECONDS) + LIVENESS_SWEEP_SECONDS = float(LIVENESS_SWEEP_SECONDS) + LIVENESS_MAX_CONCURRENCY = int(LIVENESS_MAX_CONCURRENCY) +except ValueError as e: + raise ValueError( + "LIVENESS_PROBE_TIMEOUT_SECONDS and LIVENESS_SWEEP_SECONDS must be " + "numbers; LIVENESS_MAX_CONCURRENCY must be an integer") from e + +for _name, _value in ( + ('LIVENESS_PROBE_TIMEOUT_SECONDS', LIVENESS_PROBE_TIMEOUT_SECONDS), + ('LIVENESS_SWEEP_SECONDS', LIVENESS_SWEEP_SECONDS), + ('LIVENESS_MAX_CONCURRENCY', LIVENESS_MAX_CONCURRENCY)): + if _value <= 0: + raise ValueError(f"{_name} must be greater than zero, got {_value!r}") diff --git a/src/MissionParallelCatchup/lib/http_server.py b/src/MissionParallelCatchup/lib/http_server.py new file mode 100644 index 00000000..9ca8271f --- /dev/null +++ b/src/MissionParallelCatchup/lib/http_server.py @@ -0,0 +1,39 @@ +"""The monitor's HTTP surface: a liveness probe and the Prometheus scrape. + +Two routes, both with a live consumer -- the kubelet's livenessProbe and the +`kubernetes-pods` scrape job, which relabels prometheus.io/path onto +__metrics_path__ and so reaches the non-standard /prometheus. +""" + +from http.server import BaseHTTPRequestHandler, HTTPServer + +from prometheus_client import CONTENT_TYPE_LATEST, REGISTRY, generate_latest + +from logger import build_logger + +logger = build_logger('http_server') + + +class RequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == '/healthz': + # Serving at all is the whole check: a process that answers here + # still has its HTTP thread. + self.send_response(200) + self.end_headers() + elif self.path == '/prometheus': + self.send_response(200) + self.send_header('Content-type', CONTENT_TYPE_LATEST) + self.end_headers() + self.wfile.write(generate_latest(REGISTRY)) + else: + self.send_response(404) + self.end_headers() + + def log_message(self, *args): + pass # the default handler logs every request to stderr + + +def serve(port=8080): + logger.info('Starting httpd server on :%d', port) + HTTPServer(('', port), RequestHandler).serve_forever() diff --git a/src/MissionParallelCatchup/lib/kube.py b/src/MissionParallelCatchup/lib/kube.py new file mode 100644 index 00000000..3aa1e28b --- /dev/null +++ b/src/MissionParallelCatchup/lib/kube.py @@ -0,0 +1,34 @@ +"""Kubernetes API clients. + +Read through the module, never copied out of it: + + import kube + ... kube.core_v1.list_namespaced_pod(...) ... + +`from kube import core_v1` binds a COPY, and the tests replace these attributes +with a fake cluster -- a copy taken at import time keeps talking to the real +apiserver, silently. +""" +import os + +from kubernetes import client, config as kube_config + +import config + +# The env var is exactly what load_incluster_config() itself keys on, so in a pod +# this is the unconditional call it always was -- a missing token or CA still +# raises here and crash-loops the container rather than running blind. Outside a +# pod there is nothing to load and import stays pure; the tests replace the +# clients below. +IN_CLUSTER = bool(os.getenv('KUBERNETES_SERVICE_HOST')) +if IN_CLUSTER: + kube_config.load_incluster_config() + +# client-go's Python equivalent defaults are fine for a few LISTs per cycle, but +# dispatching ~1024 Jobs + PVCs at once needs headroom. +_cfg = client.Configuration.get_default_copy() +_cfg.connection_pool_maxsize = config.CONNECTION_POOL +client.Configuration.set_default(_cfg) + +core_v1 = client.CoreV1Api() +batch_v1 = client.BatchV1Api() diff --git a/src/MissionParallelCatchup/lib/logger.py b/src/MissionParallelCatchup/lib/logger.py new file mode 100644 index 00000000..7bd47fd9 --- /dev/null +++ b/src/MissionParallelCatchup/lib/logger.py @@ -0,0 +1,60 @@ +"""Logging setup shared by the monitor and the collector sidecar. + +Both processes write to the same logs volume and both had their own copy of this +bootstrap, which is how they came to disagree about the directory. +""" +import logging +import os +import sys +import tempfile +from datetime import datetime, timezone + + +def get_logging_level(): + name_to_level = { + 'CRITICAL': logging.CRITICAL, + 'ERROR': logging.ERROR, + 'WARNING': logging.WARNING, + 'INFO': logging.INFO, + 'DEBUG': logging.DEBUG, + } + result = name_to_level.get(os.getenv('LOGGING_LEVEL', 'INFO')) + return result if result is not None else logging.INFO + + +def log_dir(): + """The directory the log file goes in. + + On the logs PVC, not the monitor's emptyDir: /data dies with the pod, and the + mission tars /logs -- so an OOM-retry storm, the loudest signal this thing + produces, was visible only in `kubectl logs` and never reached the run's + destination directory. Falls back to /data if LOG_DIR is not mounted. + """ + chosen = os.getenv('LOG_DIR', '/logs') + if not os.path.isdir(chosen): + chosen = '/data' + # Last resort, and unreachable in a pod: /data is the monitor's own emptyDir, + # so one of the two above always exists there. Off-cluster neither does, and + # a FileHandler on a missing directory made this module impossible to import + # -- which is why nothing here was ever tested against a real reconcile(). + if not os.path.isdir(chosen): + chosen = tempfile.gettempdir() + return chosen + + +def build_logger(file_prefix, name=None, to_file=True): + """Configure root logging and return the logger to use. + + `file_prefix` names the per-process log file; `name` is the logger name, and + defaults to the root logger. `to_file=False` is stdout only, for a process + that should not add a writer to the shared logs volume. + """ + handlers = [logging.StreamHandler(sys.stdout)] + if to_file: + stamp = datetime.now(timezone.utc).strftime('%Y-%m-%d_%H-%M-%S') + handlers.append(logging.FileHandler( + os.path.join(log_dir(), f"{file_prefix}_{stamp}.log"))) + logging.basicConfig(level=get_logging_level(), + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=handlers) + return logging.getLogger(name) diff --git a/src/MissionParallelCatchup/lib/medida.py b/src/MissionParallelCatchup/lib/medida.py new file mode 100644 index 00000000..9e7d178d --- /dev/null +++ b/src/MissionParallelCatchup/lib/medida.py @@ -0,0 +1,30 @@ +"""Reading medida statistics out of a stellar-core log. + +Both readers live here on purpose. The collector scans the live stream and the +monitor re-reads the finished archive, and the two must agree on how far a `sum` +may sit from its block header -- otherwise the recovery path inherits exactly the +blind spot it exists to cover. +""" +import re + +# `sum = ms`, where the number may be in exponent form. The old +# [0-9.]+ pattern matched "1.30722" then demanded "ms" and hit "e+06ms" +# instead, so tx_apply was silently missing for 25% of ranges -- 91-99% of +# everything above ledger 35M, exactly the expensive end. 698 completed ranges +# lost the metric that way in a single run. +SUM_RE = re.compile(r"sum\s*=\s*([0-9.]+(?:[eE][+-]?[0-9]+)?)ms") + +# A medida statistic: ` = `. Anything else between the block header +# and its sum is another thread's output interleaved into the same log, and must +# not be charged against the search window. +METRIC_LINE = re.compile(r"[\w%.\-]+\s*=\s*[-+0-9.]") + +# A different metric's header. Reaching one means the block we armed on has been +# passed, so a later `sum =` belongs to some other timer. +ANY_METRIC = re.compile(r"metric '") + +# Statistics lines the sum may sit behind, and the hard line budget regardless. +# Measured 2026-08-04: a /info response pushed `sum` 91 lines down, and charging +# those lines made both readers give up while the value sat in the archive. +WINDOW = 15 +HARD_WINDOW = 400 diff --git a/src/MissionParallelCatchup/lib/metrics.py b/src/MissionParallelCatchup/lib/metrics.py new file mode 100644 index 00000000..5302a5a2 --- /dev/null +++ b/src/MissionParallelCatchup/lib/metrics.py @@ -0,0 +1,47 @@ +"""The run's Prometheus metrics. + +Declaration only -- prometheus_client builds the default REGISTRY at import and +the monitor serves it from /prometheus, so there is nothing to instantiate here. +Names carry no metric_ prefix: they are read as metrics. at the call site. +""" +from prometheus_client import Counter, Gauge, Histogram + + +# Histogram buckets +# 5m 15m 30m 1h 1.5h 2h +buckets = (300, 900, 1800, 3600, 5400, 7200, float("inf")) +catchup_queues = Gauge('ssc_parallel_catchup_queues', 'Exposes size of each job queues', ["queue"]) +workers = Gauge('ssc_parallel_catchup_workers', 'Exposes catch up worker status', ["status"]) +refresh_duration = Gauge('ssc_parallel_catchup_workers_refresh_duration_seconds', 'Time it took to refresh status of all workers') +full_duration = Histogram('ssc_parallel_catchup_job_full_duration_seconds', 'Compute seconds across the complete resumed attempt chain', buckets=buckets) +tx_apply_duration = Histogram('ssc_parallel_catchup_job_tx_apply_duration_seconds', 'Exposes job TX apply duration as histogram', buckets=buckets) +# wallSeconds is Kubernetes's startTime -> completionTime for the winning Job +# only. Failed-attempt timestamps and inter-attempt gaps were never persisted, so +# it cannot be reconstructed as first dispatch -> success after those Jobs go. +wall_duration = Histogram('ssc_parallel_catchup_job_wall_duration_seconds', + 'Winning Kubernetes Job start to completion', + buckets=buckets) +mission_duration = Gauge('ssc_parallel_catchup_mission_duration_seconds', 'Number of seconds since the mission started ') +retries = Counter( + 'ssc_parallel_catchup_job_retried_count', + 'Retry attempts dispatched after a predecessor attempt failed') +# Separates infrastructure churn from application failure: many evictions with +# zero app failures is spot behaving as intended. +evictions = Counter( + 'ssc_parallel_catchup_job_spot_eviction_count', + 'Pod attempts classified as lost to node disruption') +spot_disruption_retried = Counter( + 'ssc_parallel_catchup_job_spot_disruption_retried_count', + 'Unique ledger ranges that dispatched a successor after a node disruption verdict') +pvc_released = Counter('ssc_parallel_catchup_pvc_released_count', 'PVCs deleted after their range completed') +jobs_reaped = Counter('ssc_parallel_catchup_jobs_reaped_count', 'Finished Jobs deleted after their record was durable') +oom_retries = Counter( + 'ssc_parallel_catchup_job_oom_retried_count', + 'Retry attempts dispatched after an OOM verdict, with an escalated memory limit') +eph_retries = Counter( + 'ssc_parallel_catchup_job_ephemeral_retried_count', + 'Retry attempts dispatched after an ephemeral-storage verdict, with an escalated limit') +retry_reasons = Counter( + 'ssc_parallel_catchup_job_retried_reason_count', + 'Retry attempts dispatched, by the effective verdict of the predecessor attempt', + ['reason']) diff --git a/src/MissionParallelCatchup/lib/profiles.py b/src/MissionParallelCatchup/lib/profiles.py new file mode 100644 index 00000000..620011e2 --- /dev/null +++ b/src/MissionParallelCatchup/lib/profiles.py @@ -0,0 +1,71 @@ +"""The measured profile of a previous run, and the lookup into it. + +Loaded once at startup into config.PROFILE and read through it thereafter, so a +test that patches the profile is seen here without reloading anything. +""" + +import bisect +import json +import logging + +import config + +logger = logging.getLogger() + + +def load_profile(): + """Per-range measurements from an earlier run, keyed by range end. + + Absent, unreadable or malformed all mean the same thing: size from the + configured defaults. A profile is an optimisation, never a prerequisite. + """ + if not config.PROFILE_PATH: + return [] + try: + with open(config.PROFILE_PATH) as fh: + doc = json.load(fh) + except (OSError, ValueError) as e: + logger.warning("range profile %s unreadable (%s); using configured requests", + config.PROFILE_PATH, e) + return [] + mode = doc.get('storageMode') + cross_mode = bool(mode) and mode != config.STORAGE_MODE + if cross_mode: + # cpu and memory carry across modes -- they measure the same work. Disk + # does not: a pvc run puts /data on the volume, so it never measures + # node-local usage, and an ephemeral run's figure says nothing about a + # pvc one. Keep the transferable axes and let disk fall back to the + # configured default. + logger.warning("range profile is for storageMode=%s but this run is %s; " + "using its cpu and memory, defaulting ephemeral storage", + mode, config.STORAGE_MODE) + out = [] + for end, rec in (doc.get('ranges') or {}).items(): + try: + end = int(end) + except (TypeError, ValueError): + continue + if cross_mode: + rec = {k: v for k, v in rec.items() if k != 'peakEphemeralBytes'} + out.append((end, rec)) + out.sort() + logger.info("loaded range profile: %d ranges from %s", len(out), config.PROFILE_PATH) + return out + + +def profile_for(end): + """Measurements to size this range from, or None to use the defaults. + + Exact end, else the nearest measured end ABOVE it. Cost rises with ledger + position -- the bucket set only grows -- so a lower neighbour under-reports, + and under-provisioning costs an eviction while over-provisioning only costs + packing. Past the top of the profile there is nothing safe to extrapolate + from, so fall back to the configured defaults. + """ + if not config.PROFILE: + return None + end = int(end) + idx = bisect.bisect_left(config.PROFILE, (end,)) + if idx < len(config.PROFILE) and config.PROFILE[idx][0] == end: + return config.PROFILE[idx][1] + return config.PROFILE[idx][1] if idx < len(config.PROFILE) else None diff --git a/src/MissionParallelCatchup/lib/ranges.py b/src/MissionParallelCatchup/lib/ranges.py new file mode 100644 index 00000000..4056c15e --- /dev/null +++ b/src/MissionParallelCatchup/lib/ranges.py @@ -0,0 +1,101 @@ +"""The ledger range list and the order it is dispatched in. + +A pure function of config: dispatch recomputes the whole list on every reconcile, +so a restarted monitor has to reproduce it exactly. Nothing here reads the +cluster or the volume. +""" + +import config +import profiles + + +def _uniform_segment(start_ledger, end_ledger, seg_size): + """Ranges over (start_ledger, end_ledger], largest ledger first.""" + out = [] + el = end_ledger + while el > start_ledger: + ledgers_per_job = min(el - start_ledger, seg_size) + out.append((el, ledgers_per_job + config.OVERLAP_LEDGERS)) + el -= ledgers_per_job + return out + + +def _longest_first(ranges): + """Sort on the profile's own measured seconds, longest job first. + + Makespan is bounded below by the single longest job, so every range that + starts after it is free and every hour it starts late is an hour on the end. + That is classic longest-processing-time scheduling. + + A range the profile has never seen sorts FIRST. profile_for returns the + nearest measured end ABOVE the target, so an unprofiled range is by + construction newer than anything ever measured -- the newest ranges are the + most expensive, so "unknown" means "assume worst", not "assume average". + That also makes the next profile better: those ranges run early, under the + most generous sizing, instead of being the ones a run dies before reaching. + + Requires a profile; validate_config() refuses the combination without one, + because every key would tie and the stable sort would leave dispatch in the + generator's tip-first order while looking configured. + """ + def cost(item): + prof = profiles.profile_for(item[0]) + secs = (prof or {}).get('seconds') + # None sorts first; ties keep tip-first order, which is the better guess + # among ranges the profile cannot separate. + return (0 if secs is None else 1, -(secs or 0)) + return sorted(ranges, key=cost) + + +def _ordered(ranges): + """Dispatch order. Generators emit tip-first; the other two re-order that. + + tip-first only approximates longest-first. Position predicts cost on average + and badly in the tail: measured 2026-07-30, ranges at 41-45M ran as long as + the tip (3.1h) on a third of the memory, and the 50-60M band is CHEAPER than + 40-50M. Sorting on measured seconds uses the real number instead of a proxy. + """ + # validate_config() rejects an unknown order at startup; the raise here is + # the backstop for a caller that skipped it, never the primary check. + if config.RANGE_ORDER == 'tip-first': + return ranges + elif config.RANGE_ORDER == 'oldest-first': + return list(reversed(ranges)) + elif config.RANGE_ORDER == 'longest-first': + return _longest_first(ranges) + else: + raise ValueError("RANGE_ORDER must be one of %s, got %r" + % (', '.join(config.VALID_RANGE_ORDERS), config.RANGE_ORDER)) + + +def _logarithmic_ranges(): + """Big chunks over cheap early history, halving toward the tip. + + Aims for roughly equal wall-time per job rather than equal ledger count. + """ + out = [] + start_ledger = config.STARTING_LEDGER + end_ledger = config.LATEST_LEDGER_NUM // 2 + chunk = (end_ledger - start_ledger + 1) // max(config.PARALLELISM, 1) + while chunk > config.LOGARITHMIC_FLOOR_LEDGERS: + out.extend(_uniform_segment(start_ledger, end_ledger, chunk)) + start_ledger = end_ledger + 1 + chunk //= 2 + end_ledger = start_ledger + (chunk * config.PARALLELISM) + out.extend(_uniform_segment(end_ledger + 1, config.LATEST_LEDGER_NUM, config.LOGARITHMIC_FLOOR_LEDGERS)) + return out + + +def generate_ranges(): + # An unrecognised generator used to fall through to logarithmic, so a typo + # silently produced a completely different range layout. validate_config() + # rejects that at startup; this raise is the backstop, not the primary check + # -- reached from inside reconcile it would only ever be logged and retried. + if config.RANGE_GENERATOR == 'uniform': + ranges = _uniform_segment(config.STARTING_LEDGER, config.LATEST_LEDGER_NUM, config.LEDGERS_PER_JOB) + elif config.RANGE_GENERATOR == 'logarithmic': + ranges = _logarithmic_ranges() + else: + raise ValueError("RANGE_GENERATOR must be one of %s, got %r" + % (', '.join(config.VALID_RANGE_GENERATORS), config.RANGE_GENERATOR)) + return _ordered(ranges) diff --git a/src/MissionParallelCatchup/lib/records.py b/src/MissionParallelCatchup/lib/records.py new file mode 100644 index 00000000..500696bb --- /dev/null +++ b/src/MissionParallelCatchup/lib/records.py @@ -0,0 +1,112 @@ +"""Per-attempt facts on the shared volume, and the paths they live at. + +The collector sidecar writes these files and the monitor reads them, so nothing +authoritative is held in memory: a restarted monitor rebuilds every decision from +these plus the live Job list. The counters here are per CAUSE, not per attempt -- +escalation must climb once per OOM, not once per retry. +""" +import json +import os + +import config + + +def log_path(end, attempt): + """Canonical archive name, written by the log-collector sidecar. + + Deliberately carries no ok/failed suffix: which ranges failed is recorded in + the progress ConfigMap, and encoding it here would mean two components + disagreeing about a filename. + """ + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.log.gz") + + +def state_path(end, attempt): + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.state") + + +def outcome_path(end, attempt): + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.outcome") + + +# Per RANGE, not per attempt: wallSeconds spans the range's whole life, so the +# only start that matters is the first one. Later attempts are the mess in +# between and are deliberately not recorded. +def started_path(end): + return os.path.join(config.LOG_DIR, f"range-{end}.started") + + +def read_outcome(end, attempt): + try: + with open(outcome_path(end, attempt)) as fh: + return json.load(fh) + except (OSError, ValueError): + return None + + +def _oom_count(end, attempt): + """How many earlier attempts at this range were OOM-killed. + + Escalation must climb once per OOM, not once per attempt. On spot most + retries are evictions -- measured on ssc-test 2026-07-30, 288 disruption + retries against 7 OOM retries -- and a range disrupted three times then + OOMing once would otherwise jump to base * 1.5^4, a 5x request for a single + OOM. That inflation is fleet-wide and it is what exhausts the vCPU quota. + """ + return sum(1 for n in range(1, int(attempt) + 1) + if _verdict_of(end, n) == 'oom') + + +def verdict_path(end, attempt): + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.verdict") + + +def _verdict_of(end, attempt): + try: + with open(verdict_path(end, attempt)) as fh: + verdict = fh.read().strip() + except OSError: + # Pre-fix runs, or an attempt whose verdict write lost the volume: + # the pod-derived classification is the next best thing. + outcome = (read_outcome(end, attempt) or {}).get('outcome') + return outcome if outcome in config.ATTEMPT_OUTCOMES else None + return verdict if verdict in config.ATTEMPT_OUTCOMES else None + + +def _cause_count(end, attempt, causes): + """How many of attempts 1..N at this range failed for one of `causes`. + + Budgets are per cause, not per attempt. One shared attempt index meant + cluster churn -- which has its own deliberately large budget -- drained the + small budgets belonging to the causes that say something about the range: a + range evicted MAX_ATTEMPTS times had an effective OOM and disk budget of + zero, was condemned on its first real OOM without ever being escalated, and + took the whole mission with it. + """ + return sum(1 for n in range(1, int(attempt) + 1) + if _verdict_of(end, n) in causes) + + +def metrics_path(end, attempt): + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.metrics") + + +def done_path(end, attempt): + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.done") + + +def write_atomic(path, body, opener=None): + """Write `body` through tmp+rename so a reader never sees a partial file. + + Both processes write these files and both read them back: the collector + writes while the monitor polls, so a torn .metrics or .outcome reads as + corrupt and the measurement is lost, and a restarted monitor decides a + range's remaining budget from .outcome and .verdict. + """ + tmp = path + '.tmp' + # `opener` resolves at call time, not as a default argument, so the write + # seam stays patchable -- the tmp+rename discipline is only worth having if + # a test can crash a write mid-flight and prove the real path is untouched. + with (opener or open)(tmp, 'wt') as fh: + fh.write(body) + os.replace(tmp, path) diff --git a/src/MissionParallelCatchup/lib/sizing.py b/src/MissionParallelCatchup/lib/sizing.py new file mode 100644 index 00000000..ca3f0a23 --- /dev/null +++ b/src/MissionParallelCatchup/lib/sizing.py @@ -0,0 +1,386 @@ +"""What a worker pod asks for: nodepool tier, memory, ephemeral disk. + +This is the layer tuned between runs. It reads the profile and the per-attempt +records, and it never touches the cluster -- what it returns is a request, not an +applied change. + +Escalation counts CAUSES, not attempts: a spot reclaim, a disruption and a +timeout all produce a retry and none of them says the range needed a bigger node. +""" +import logging +import math + +import config +import profiles +import records +import units + +logger = logging.getLogger() + + +def mem_for_attempt(attempt, base=None, end=None): + """Memory REQUEST after N OOMs. + + Pooled: the tier ladder IS the ladder. Attempt N resolves to a tier N-1 + steps up, and the request is that tier's cut, so the request and the pool + move together. A multiplicative bump cannot do this -- tiers are ~2.2-2.5x + apart while MEM_BUMP_FACTOR is 1.5, so a bump lands BETWEEN tiers: too big + for the current pool's nodes, too small to have earned the next one, and the + pod sits Pending on a pool that can never satisfy it. + + Unpooled: the original behaviour. `base` is what attempt 1 actually ran + with, which matters when a profile sized the range -- escalating a 209Mi + profiled range off the configured default jumps straight to 36000Mi, a 172x + overshoot that throws away the whole packing win on the first OOM. + + Escalating the request, not a limit, because there is no limit any more. It + still buys the same two things an OOMing range needs -- placement somewhere + with the memory actually free, and a higher bar before the kubelet picks it + as an eviction victim. + """ + if config.POOL_PREFIX: + promoted = pool_memory(pool_for(end, attempt)) + if promoted: + return promoted + # Above the ladder (nebula/protostar/supernova): nothing left to promote + # into, so hold at the configured request rather than inventing a value. + return base or config.REQ_MEM + base_q = units.quantity_bytes(base or config.REQ_MEM) + want = int(base_q * (config.MEM_BUMP_FACTOR ** max(0, attempt - 1))) + cap = units.quantity_bytes(config.MEM_ESCALATION_CAP) + return units.bytes_to_quantity(min(want, cap)) + + +def eph_for_attempt(attempt): + """Ephemeral-storage size for attempt N, escalating after an eviction. + + None when no limit is configured: a pod with no ephemeral-storage limit can + still be evicted under node disk pressure, and there is nothing to raise. + Every other reader of LIM_EPHEMERAL already guards on it being set. + """ + if not config.LIM_EPHEMERAL: + return None + base_q = units.quantity_bytes(config.LIM_EPHEMERAL) + want = int(base_q * (config.EPH_BUMP_FACTOR ** max(0, attempt - 1))) + return units.bytes_to_quantity(min(want, units.quantity_bytes(config.EPH_ESCALATION_CAP))) + + +def pool_vcpu(tier): + """vCPU of the smallest node this tier can land on, or None if unmapped.""" + return _pool_map(config.POOL_VCPU, 'POOL_VCPU').get(tier) + + +def _rung_listed(raw, tier, nxt): + want = f"{tier}->{nxt}" + return any(item.strip() == want for item in raw.split(',')) + + +def _crossing_allowed(tier, nxt): + """Is this specific rung whitelisted to cross a vCPU class?""" + return _rung_listed(config.POOL_CROSS_RUNGS, tier, nxt) + + +def _rung_blocked(tier, nxt): + """Is this rung denied outright? Beats every other consideration.""" + return _rung_listed(config.POOL_BLOCK_RUNGS, tier, nxt) + + +def _parsed_pool_tiers(): + """[(gib_cut, tier_name)] cheapest first, the last entry unbounded. + + An empty cut on the final entry means "everything above the previous one", + which is how supernova is expressed without inventing a ceiling. + """ + out = [] + for item in config.POOL_TIERS.split(','): + item = item.strip() + if not item: + continue + cut, _, name = item.rpartition(':') + if not name: + continue + out.append((float(cut) if cut else float('inf'), name)) + return out + + +def _tier_for_bytes(anon_bytes): + """Tier whose node can hold this working set, or None if unsizable.""" + tiers = _parsed_pool_tiers() + if not tiers or not anon_bytes: + return None + gib = anon_bytes / float(1024 ** 3) + for cut, name in tiers: + if gib < cut: + return name + return tiers[-1][1] + + +def _promote(tier, steps): + """Move `steps` tiers up the ladder, stopping at the top. + + OOM escalation moves the POOL, not just the request. Bumping a request while + the pod is still pinned to a tier whose nodes cannot hold it produces a pod + that can never schedule -- Pending forever, which reads as a hang rather + than a failure. nebula and protostar sit outside the ladder and escalate + straight to the top, since there is no tier above them to walk to. + """ + tiers = [name for _, name in _parsed_pool_tiers()] + if not tiers: + return tier + top = tiers[-1] + if steps <= 0 or tier is None: + return tier + if tier not in tiers: + return top + return tiers[min(tiers.index(tier) + steps, len(tiers) - 1)] + + +def _cache_bump(tier, anon_bytes, ws_bytes): + """One rung up when the working set reaches the next tier and the rung is free. + + peakAnonBytes picks the base tier because anon is what OOM-kills. Page cache + is elastic -- it evicts rather than dying -- so it has no business in that + decision. It does decide throughput: replay is single-threaded, so every + bucket lookup that misses cache is a serial ~0.5 ms EBS stall. Measured on + ssc-test 2026-08-03, range 44648511 (anon 2.63 GiB, ws 18.31 GiB) run twice + on identical 2-vCPU Intel nodes differing only in RAM: + + m8in.large 8 GiB 540 reads/ledger 21% iowait 1.86 lps + r8in.large 16 GiB 65 reads/ledger 7% iowait 3.14 lps (100% of profile) + + Only rungs that keep the same node vCPU, read from POOL_VCPU. giant-> + supergiant is m8a.large->r8a.large: twice the RAM for the same 2 vCPU and + +8% spot. supergiant->hypergiant is r8a.large->x8i.large, also 2 vCPU, and + that rung measured 1.86x on ssc-test 2026-08-03 -- 1.64 -> 2.99 lps across + nine ranges, the largest gain found anywhere. + + Do NOT read POOL_CPU for this. It was half the node's vCPU everywhere, so + equal claims used to imply equal nodes, but hypergiant and supernova are now + sized to the smallest shape in their pool (1.70 on a 2-vCPU x8i.large). A + claim comparison silently refuses supergiant->hypergiant, which is the whole + reason the x8i shapes were promoted to top weight. + + What stays blocked is hypergiant->supernova, 2 vCPU -> 4. Those ranges + measured healthy at 6-11 reads/ledger and gained only 1.23x, so doubling + their cores is the expensive rung with the weak return. + + Deliberately loose about false positives. Promoting a range that did not need + it costs +8% on its node-hours and nothing in quota; leaving one starving + costs 40% of its throughput. Against 50 pods probed for reads/ledger and + iowait on the same run: 11 correctly promoted, 11 unnecessarily, 1 missed -- + and the 11 unnecessary ones are free. + + One rung, never two, even when the working set would justify more. 44648511 + lands on supergiant still 1.29x UNDER its working set and reaches full + profile rate there; fitting the working set costs multiples for nothing. + """ + if not (tier and anon_bytes and ws_bytes): + return tier + nxt = _promote(tier, 1) + if nxt == tier: + return tier # already at the top of the ladder + order = [name for _, name in _parsed_pool_tiers()] + want = _tier_for_bytes(ws_bytes) + if tier not in order or not want or order.index(want) <= order.index(tier): + return tier # working set does not reach the next tier + if _rung_blocked(tier, nxt): + return tier # denied outright, see POOL_BLOCK_RUNGS + a, b = pool_vcpu(tier), pool_vcpu(nxt) + if (a is None or b is None or a != b) and not _crossing_allowed(tier, nxt): + return tier # crosses a vCPU class; not free, skip it + return nxt + + +def pool_for(end, attempt=1, rungs=None): + """Which nodepool tier this range belongs in, or None when not pooling. + + Three cases, and they are deliberately different pools: + no profile at all -> POOL_NO_PROFILE (nebula), sized by the configured + defaults because nothing is known + profiled run, this + range past the top -> POOL_UNPROFILED (protostar). Only the newest + ledgers land here and they are the densest, so + it is a rich pool rather than an average one + profiled range -> the tier its peakAnonBytes fits, then one rung + up if its working set cannot be cached there and + the rung is free -- see _cache_bump + + `rungs` is how many tiers to climb, and it counts OOMs -- NOT attempts. A + spot reclaim, a disruption and a timeout all produce a retry, and none of + them says the range needed a bigger node. Promoting on attempt number put 65 + ranges onto 8-vCPU supernova nodes during the 2026-08-03 spot run whose + attempt-1 verdicts were `timeout`; they belonged on 4-vCPU hypergiant, so it + burned ~260 vCPU of a 2304 quota escalating away from a problem that was + never memory. + """ + if not config.POOL_PREFIX: + return None + if rungs is None: + # Attempts before this one, since this attempt has not run yet. Anything + # on disk for it is from a previous incarnation of the same attempt. + rungs = records._oom_count(end, attempt - 1) if attempt and attempt > 1 else 0 + if not config.PROFILE: + return _promote(config.POOL_NO_PROFILE, rungs) + prof = profiles.profile_for(end) if end is not None else None + if not prof: + return _promote(config.POOL_UNPROFILED, rungs) + anon = prof.get('peakAnonBytes') + tier = _cache_bump(_tier_for_bytes(anon), anon, + prof.get('peakWorkingSetBytes')) + if not tier: + # An entry with no memory measurement tells us nothing about size -- + # treat it as unprofiled rather than guessing a tier. + return _promote(config.POOL_UNPROFILED, rungs) + return _promote(tier, rungs) + + +def _pool_map(raw, what): + out = {} + for item in raw.split(','): + item = item.strip() + if not item: + continue + name, _, value = item.partition(':') + try: + out[name.strip()] = float(value) + except ValueError: + logger.error("%s is malformed at %r; that tier falls back to the " + "configured request", what, item) + return out + + +def pool_cpu(tier): + """cpu request for a tier, or None to keep the configured one.""" + return _pool_map(config.POOL_CPU, 'POOL_CPU').get(tier) + + +def pool_memory(tier): + """Memory request for a tier: half its node's allocatable. + + Not the range's own measurement. Sizing from the peak would let two small + ranges share a node, and isolation is the whole point -- freeing a pod of + its three neighbours raised throughput 29-92% while its cpu draw FELL, so + the contended resource is memory bandwidth and shared cache, not compute. + + Half is the smallest value that still excludes a second pod once the + daemonsets are counted, and the largest that reliably schedules the first. + """ + return _pool_str_map(config.POOL_MEM, 'POOL_MEM').get(tier) + + +def _pool_str_map(raw, what): + out = {} + for item in raw.split(','): + item = item.strip() + if not item: + continue + name, _, value = item.partition(':') + if not value: + logger.error("%s is malformed at %r; that tier keeps the configured " + "request", what, item) + continue + out[name.strip()] = value.strip() + return out + + +def _positive_seconds(value): + """A finite positive runtime, or None when the profile cannot supply one.""" + try: + seconds = float(value) + except (TypeError, ValueError): + return None + return seconds if math.isfinite(seconds) and seconds > 0 else None + + +def _profile_seconds(): + """Every valid measured runtime in the profile, sorted.""" + if config._SORTED_SECONDS is None: + values = (_positive_seconds(r.get('seconds')) for _, r in (config.PROFILE or [])) + config._SORTED_SECONDS = sorted(seconds for seconds in values if seconds is not None) + return config._SORTED_SECONDS + + +def _runtime_insurance(seconds, allowance): + """Runtime-weighted share of a configured allowance. + + The longest range in the profile gets all of it and one half as long gets + half, so the allowance follows time-at-risk. Zero when the profile cannot + supply a runtime to weight by. + """ + seconds = _positive_seconds(seconds) + everything = _profile_seconds() + longest = everything[-1] if everything else None + insurance = units.quantity_bytes(allowance) + # No `longest <= 0` guard: _profile_seconds only keeps finite positives. + if seconds is None or longest is None or insurance <= 0: + return 0 + return int(insurance * (seconds / longest)) + + +def _profile_overrides(end, escalated, attempt=1): + """Request overrides for this range from the profile, or {} for none. + + Escalated retries opt out: an escalation is a measurement of THIS run and + outranks anything an earlier one saw. + """ + if end is None: + return {} + if escalated and not config.POOL_PREFIX: + # Unpooled: an escalation measures THIS run and outranks anything an + # earlier one saw. Pooled: the promotion IS the escalation, and the + # promoted tier's cut is the escalated request -- bailing out here would + # send the pod to the new pool still asking for the old tier's memory. + return {} + prof = profiles.profile_for(end) + out = {} + if prof: + disk = prof.get('peakEphemeralBytes') + if disk and config.LIM_EPHEMERAL: + want = (int(disk * config.PROFILE_MARGIN) + + units.quantity_bytes(config.PROFILE_EPHEMERAL_HEADROOM) + + _runtime_insurance(prof.get('seconds'), + config.PROFILE_RUNTIME_EPHEMERAL_INSURANCE)) + out['ephemeral-storage'] = units.bytes_to_quantity( + min(want, units.quantity_bytes(config.PROFILE_MAX_EPHEMERAL))) + if config.POOL_PREFIX: + # Deliberately BEFORE the no-profile bail. pool_for resolves a tier for + # every range -- protostar when the range is newer than the profile, + # nebula when there is no profile at all -- so returning {} here would + # pin the pod to that pool while sizing it from the flat REQ_CPU. On + # 2026-08-04 that shipped a 6780m request (the run's + # --pubnet-parallel-catchup-cpu-request) at a protostar pool whose + # largest node is 4 vCPU: permanently Pending, retried forever, and + # invisible until a run had enough past-the-profile ranges to notice. + # Pooled: the tier's cut is the request, and the margin lives in the + # node size instead. PROFILE_MARGIN, cache headroom and runtime + # insurance all existed to keep a pod under its own memory LIMIT; there + # is no memory limit any more and the pod owns the node, so a margin in + # the request constrains nothing the kubelet acts on. Disk keeps its + # margin above -- that limit IS enforced. + tier = pool_for(end, attempt) + mem = pool_memory(tier) + if mem: + out['memory'] = mem + cpu = pool_cpu(tier) + if cpu: + out['cpu'] = cpu + return out + if not prof: + # Unpooled and unmeasured: nothing to size from, so the configured + # requests stand exactly as if there were no profile at all. + return out + # Unpooled (the pre-tier behaviour, and what nebula-style runs fall back to + # when no prefix is configured): size memory from the range's own peak, with + # the margins that a limit-bearing pod needed. + # + # peakAnonBytes is kubelet's rssBytes, sampled by the collector on its own + # the finer one and fall back, so a profile captured before the collector + # tracked anon still sizes exactly as it used to. + rss = prof.get('peakAnonBytes') + if rss: + want = (int(rss * config.PROFILE_MARGIN) + + units.quantity_bytes(config.PROFILE_CACHE_HEADROOM) + + _runtime_insurance(prof.get('seconds'), + config.PROFILE_RUNTIME_MEMORY_INSURANCE)) + out['memory'] = units.bytes_to_quantity(min(want, units.quantity_bytes(config.PROFILE_MAX_MEM))) + return out diff --git a/src/MissionParallelCatchup/lib/units.py b/src/MissionParallelCatchup/lib/units.py new file mode 100644 index 00000000..0544aba8 --- /dev/null +++ b/src/MissionParallelCatchup/lib/units.py @@ -0,0 +1,27 @@ +"""Kubernetes quantity strings to bytes and back. + +Pure string arithmetic -- no config, no cluster. The monitor compares profile +figures (bytes) against chart and pool values (quantity strings) constantly, and +doing it inline is how a Gi/Mi mix-up becomes a sizing bug. +""" + +_UNITS = {'Ki': 1024, 'Mi': 1024**2, 'Gi': 1024**3, 'Ti': 1024**4, + 'K': 1000, 'M': 1000**2, 'G': 1000**3, 'T': 1000**4} + + +def gib(q): + try: + return quantity_bytes(q) / (1024 ** 3) + except Exception: + return None + + +def quantity_bytes(q): + for suffix, mult in sorted(_UNITS.items(), key=lambda kv: -len(kv[0])): + if q.endswith(suffix): + return int(float(q[:-len(suffix)]) * mult) + return int(float(q)) + + +def bytes_to_quantity(n): + return f"{max(1, n // (1024 ** 2))}Mi" diff --git a/src/MissionParallelCatchup/lib/worker_liveness.py b/src/MissionParallelCatchup/lib/worker_liveness.py new file mode 100644 index 00000000..8d86df2e --- /dev/null +++ b/src/MissionParallelCatchup/lib/worker_liveness.py @@ -0,0 +1,103 @@ +"""Liveness of the workers' stellar-core /info endpoint, one sweep per reconcile. + +The reconcile loop already holds the authoritative pod list. This probes that +list concurrently and returns a snapshot: up if /info answered 200, down for +anything else, unknown for whatever the sweep did not get to before its deadline. + +No state is carried between sweeps -- no hysteresis, no scheduler, no threads. +The numbers feed a Grafana panel and nothing else reads them, so a stale-free +snapshot is worth more than a smoothed one. +""" +import asyncio +import logging + +import aiohttp + +import config + +logger = logging.getLogger() + +_ADMIN_PORT = 11626 # stellar-core's admin/HTTP port + + +def targets(pods): + """Current Running-with-IP pods, keyed by pod identity. + + A UID change is a replacement even when the Job name or IP is reused. Tests + and unusually incomplete API objects may lack a UID, where the pod name is + still unique for its lifetime. + """ + out = {} + for pod in pods: + pod_status = getattr(pod, 'status', None) + metadata = getattr(pod, 'metadata', None) + ip = getattr(pod_status, 'pod_ip', None) + if getattr(pod_status, 'phase', None) != 'Running' or not ip or metadata is None: + continue + name = getattr(metadata, 'name', None) + identity = getattr(metadata, 'uid', None) or name + if identity and name: + out[str(identity)] = (str(name), str(ip)) + return out + + +async def _probe(session, ip, timeout): + """True only for HTTP 200. A timeout or refused connection is False.""" + host = f"[{ip}]" if ':' in ip else ip + async with session.get(f"http://{host}:{_ADMIN_PORT}/info", + timeout=aiohttp.ClientTimeout(total=timeout)) as resp: + return resp.status == 200 + + +async def sweep(targets, concurrency=None, timeout=None, deadline=None): + """Probe every target concurrently; return {'up','down','unknown'}. + + Not a TaskGroup: a TaskGroup cancels its siblings when one task raises, + which is the opposite of what a fleet sweep wants -- one unreachable pod + must not discard the other 1023 answers. asyncio.wait with a deadline keeps + whatever finished and cancels only the stragglers. + """ + concurrency = int(concurrency or config.LIVENESS_MAX_CONCURRENCY) + timeout = float(timeout or config.LIVENESS_PROBE_TIMEOUT_SECONDS) + deadline = float(deadline or config.LIVENESS_SWEEP_SECONDS) + counts = {'up': 0, 'down': 0, 'unknown': len(targets)} + if not targets: + return {'up': 0, 'down': 0, 'unknown': 0} + + # `limit` is the concurrency bound: aiohttp holds a task at connect until a + # slot frees, so a semaphore on top of it would be enforcing the same number + # twice. force_close because a worker pod can vanish between sweeps and a + # pooled socket to a dead pod would be handed straight back out. + connector = aiohttp.TCPConnector(limit=concurrency, force_close=True) + async with aiohttp.ClientSession(connector=connector) as session: + tasks = [asyncio.create_task(_probe(session, ip, timeout)) + for _, ip in targets.values()] + done, pending = await asyncio.wait(tasks, timeout=deadline) + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + for task in done: + try: + up = task.result() + except Exception: + up = False # timeout, refused, DNS, malformed response + counts['up' if up else 'down'] += 1 + counts['unknown'] -= 1 + return counts + + +def publish(targets): + """Run one sweep and return its counts. Called from the reconcile loop. + + Blocks for at most LIVENESS_SWEEP_SECONDS: everything still outstanding at + the deadline is cancelled and reported unknown, so a fleet of unreachable + pods costs the deadline and never the sum of their timeouts. + """ + if not targets: + return {'up': 0, 'down': 0, 'unknown': 0} + try: + return asyncio.run(sweep(targets)) + except Exception as e: + logger.warning("liveness sweep failed (%s); reporting all workers unknown", e) + return {'up': 0, 'down': 0, 'unknown': len(targets)} diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index c57d6d85..8ce05f4e 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -135,8 +135,8 @@ spec: args: - >- {{- if .Values.monitor.sourceInstallDependencies }} - pip install --no-cache-dir -q 'kubernetes~=35.0' 'aiohttp~=3.9' - 'requests~=2.31' 'prometheus-client~=0.19' && + pip install --no-cache-dir -q 'kubernetes~=36.0' 'aiohttp~=3.9' + 'prometheus-client~=0.19' && {{- end }} exec python3 /app/job_monitor.py {{- end }} @@ -159,12 +159,10 @@ spec: value: {{ .Values.monitor.emitMissionLabel | quote }} - name: LOGGING_INTERVAL_SECONDS value: {{ .Values.monitor.loggingIntervalSeconds | quote }} - - name: LIVENESS_PROBE_INTERVAL_SECONDS - value: {{ .Values.monitor.livenessProbeIntervalSeconds | quote }} - name: LIVENESS_PROBE_TIMEOUT_SECONDS value: {{ .Values.monitor.livenessProbeTimeoutSeconds | quote }} - - name: LIVENESS_FAILURE_THRESHOLD - value: {{ .Values.monitor.livenessFailureThreshold | quote }} + - name: LIVENESS_SWEEP_SECONDS + value: {{ .Values.monitor.livenessSweepSeconds | quote }} - name: LIVENESS_MAX_CONCURRENCY value: {{ .Values.monitor.livenessMaxConcurrency | quote }} - name: RANGE_GENERATOR @@ -203,14 +201,18 @@ spec: value: {{ .Values.worker.resources.requests.ephemeral_storage | quote }} - name: LIM_EPHEMERAL value: {{ .Values.worker.resources.limits.ephemeral_storage | quote }} - - name: MAX_ATTEMPTS - value: {{ .Values.monitor.maxAttempts | quote }} - name: MAX_DISRUPTION_ATTEMPTS - value: {{ .Values.monitor.maxDisruptionAttempts | quote }} + value: {{ .Values.monitor.attemptBudgets.disrupted | quote }} + - name: MAX_REJECTED_ATTEMPTS + value: {{ .Values.monitor.attemptBudgets.rejected | quote }} + - name: MAX_FETCH_FAULT_ATTEMPTS + value: {{ .Values.monitor.attemptBudgets.fetchFault | quote }} + - name: MAX_OOM_ATTEMPTS + value: {{ .Values.monitor.attemptBudgets.oom | quote }} + - name: MAX_EPHEMERAL_ATTEMPTS + value: {{ .Values.monitor.attemptBudgets.ephemeral | quote }} - name: MEM_BUMP_FACTOR value: {{ .Values.monitor.memBumpFactor | quote }} - - name: MAX_EPHEMERAL_ATTEMPTS - value: {{ .Values.monitor.maxEphemeralAttempts | quote }} - name: EPH_BUMP_FACTOR value: {{ .Values.monitor.ephBumpFactor | quote }} - name: EPH_ESCALATION_CAP @@ -221,43 +223,44 @@ spec: value: {{ .Values.monitor.maxMem | quote }} - name: GRACE_SECONDS value: {{ .Values.monitor.graceSeconds | quote }} + # Delays SIGTERM so the collector has more room to be holding a + # stream when stellar-core prints its final medida block. + - name: PRESTOP_SLEEP_SECONDS + value: {{ .Values.monitor.prestopSleepSeconds | quote }} # A range stuck retrying the history archive never exits on its # own; without this it holds a slot for the life of the run. - name: ATTEMPT_DEADLINE_SECONDS value: {{ .Values.monitor.attemptDeadlineSeconds | quote }} - name: JOB_TTL_SECONDS value: {{ .Values.monitor.jobTtlSeconds | quote }} - {{- if .Values.integration.syntheticWorker.enabled }} - - name: SYNTHETIC_WORKER_CONFIG_MAP - value: {{ .Release.Name }}-synthetic-worker - - name: SYNTHETIC_WORKER_IMAGE_PULL_POLICY - value: {{ .Values.integration.syntheticWorker.imagePullPolicy | quote }} - - name: SYNTHETIC_PREDECESSOR_SECONDS - value: {{ .Values.integration.syntheticWorker.predecessorSeconds | quote }} - - name: SYNTHETIC_SUCCESSOR_MINIMUM_SECONDS - value: {{ .Values.integration.syntheticWorker.successorMinimumSeconds | quote }} - - name: SYNTHETIC_MAXIMUM_WAIT_SECONDS - value: {{ .Values.integration.syntheticWorker.maximumWaitSeconds | quote }} - - name: SYNTHETIC_PREDECESSOR_ANON_MIB - value: {{ .Values.integration.syntheticWorker.predecessorAnonMiB | quote }} - - name: SYNTHETIC_PREDECESSOR_WORKING_SET_MIB - value: {{ .Values.integration.syntheticWorker.predecessorWorkingSetMiB | quote }} - - name: SYNTHETIC_SUCCESSOR_ANON_MIB - value: {{ .Values.integration.syntheticWorker.successorAnonMiB | quote }} - - name: SYNTHETIC_SUCCESSOR_WORKING_SET_MIB - value: {{ .Values.integration.syntheticWorker.successorWorkingSetMiB | quote }} - - name: SYNTHETIC_PREDECESSOR_TX_APPLY_MS - value: {{ .Values.integration.syntheticWorker.predecessorTxApplyMilliseconds | quote }} - - name: SYNTHETIC_SUCCESSOR_TX_APPLY_MS - value: {{ .Values.integration.syntheticWorker.successorTxApplyMilliseconds | quote }} - {{- end }} - name: LOG_DIR value: /logs + - name: POOL_PREFIX + value: {{ .Values.monitor.poolPrefix | quote }} + - name: POOL_TIERS + value: {{ .Values.monitor.poolTiers | quote }} + - name: POOL_BLOCK_RUNGS + value: {{ .Values.monitor.poolBlockRungs | quote }} + - name: POOL_CROSS_RUNGS + value: {{ .Values.monitor.poolCrossRungs | quote }} + - name: POOL_VCPU + value: {{ .Values.monitor.poolVcpu | quote }} + - name: POOL_CPU + value: {{ .Values.monitor.poolCpu | quote }} + - name: POOL_UNPROFILED + value: {{ .Values.monitor.poolUnprofiled | quote }} + - name: POOL_NO_PROFILE + value: {{ .Values.monitor.poolNoProfile | quote }} + - name: POOL_MEM + value: {{ .Values.monitor.poolMem | quote }} + # Pods AND this with the tier label. Both capacity variants of a + # tier share one label value, and a pod cannot otherwise express a + # NodePool property; Karpenter labels every node with it itself. + - name: CAPACITY_TYPE + value: {{ .Values.monitor.capacityType | quote }} {{- if .Values.monitor.profileConfigMap }} - name: PROFILE_PATH value: /profile/profile.json - - name: PROFILE_CPU_TIERS - value: {{ .Values.monitor.profileCpuTiers | quote }} - name: PROFILE_MARGIN value: {{ .Values.monitor.profileMargin | quote }} - name: PROFILE_MAX_MEM @@ -266,6 +269,13 @@ spec: value: {{ .Values.monitor.profileCacheHeadroom | quote }} # Handed out in proportion to a range's own runtime, so the ranges # that sit exposed to drift the longest get the largest share. + # Disk gets the same flat + runtime-weighted allowances as memory. + - name: PROFILE_EPHEMERAL_HEADROOM + value: {{ .Values.monitor.profileEphemeralHeadroom | quote }} + - name: PROFILE_RUNTIME_EPHEMERAL_INSURANCE + value: {{ .Values.monitor.profileRuntimeEphemeralInsurance | quote }} + - name: PROFILE_MAX_EPHEMERAL + value: {{ .Values.monitor.profileMaxEphemeral | quote }} - name: PROFILE_RUNTIME_MEMORY_INSURANCE value: {{ .Values.monitor.profileRuntimeMemoryInsurance | quote }} {{- end }} @@ -327,12 +337,10 @@ spec: - name: monitor-src mountPath: /app {{- end }} - readinessProbe: - httpGet: - path: /status - port: 8080 - initialDelaySeconds: 2 - periodSeconds: 10 + # No readinessProbe: nothing consumes Ready. There is no Service, and + # the prometheus kubernetes-pods job keeps targets on the scrape + # annotation alone, with no pod-ready filter. + # # A dead pod watch stops all failure classification, which degrades # silently: later failures record outcome=unknown and get retried # blindly. Restart the container rather than run half-blind. @@ -356,12 +364,17 @@ spec: args: - >- {{- if .Values.monitor.sourceInstallDependencies }} - pip install --no-cache-dir -q 'kubernetes~=35.0' 'aiohttp~=3.9' - 'requests~=2.31' 'prometheus-client~=0.19' && + pip install --no-cache-dir -q 'kubernetes~=36.0' 'aiohttp~=3.9' + 'prometheus-client~=0.19' && {{- end }} exec python3 /app/log_collector.py {{- else }} - command: ["/usr/bin/python3", "log_collector.py"] + # Resolved on PATH, not /usr/bin/python3: Dockerfile.jobmonitor builds + # on python:3.12-slim, which ships the interpreter at + # /usr/local/bin/python3. An absolute path here pins the chart to one + # base image and fails as StartError -- the monitor container comes up + # (it inherits the image CMD) and only the sidecar crashloops. + command: ["python3", "log_collector.py"] {{- end }} env: - name: NAMESPACE @@ -378,6 +391,18 @@ spec: value: {{ .Values.monitor.terminalPollAttempts | quote }} - name: LOG_POLL_SECONDS value: {{ .Values.monitor.logPollSeconds | quote }} + # Ceiling on how long a follow stream is held for a condemned pod, + # so a withdrawn drain notice cannot pin one open for a whole range. + - name: DOOMED_POLL_SECONDS + value: {{ .Values.monitor.doomedPollSeconds | quote }} + - name: WATCH_TIMEOUT_SECONDS + value: {{ .Values.monitor.watchTimeoutSeconds | quote }} + - name: WATCH_RETRY_SECONDS + value: {{ .Values.monitor.watchRetrySeconds | quote }} + - name: MAX_DOOMED_FOLLOWS + value: {{ .Values.monitor.maxDoomedFollows | quote }} + - name: DOOMED_FOLLOW_SECONDS + value: {{ .Values.monitor.doomedFollowSeconds | quote }} - name: MAX_CONCURRENT_POLLS value: {{ .Values.monitor.maxConcurrentPolls | quote }} - name: MAX_POLL_CHARS @@ -393,10 +418,6 @@ spec: # this the collector defaults to pvc and silently records nothing. - name: STORAGE_MODE value: {{ .Values.worker.storageMode | quote }} - {{- if .Values.integration.syntheticWorker.enabled }} - - name: SYNTHETIC_WORKER - value: "true" - {{- end }} resources: {{- toYaml .Values.monitor.collectorResources | nindent 12 }} volumeMounts: diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/synthetic_worker.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/synthetic_worker.yaml deleted file mode 100644 index eae470b9..00000000 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/synthetic_worker.yaml +++ /dev/null @@ -1,96 +0,0 @@ -{{- if .Values.integration.syntheticWorker.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-synthetic-worker -data: - worker.py: | - import json - import os - import sys - import time - - - def required_int(name): - value = int(os.environ[name]) - if value < 0: - raise ValueError(f"{name} must be non-negative") - return value - - - def emit(message): - print(message, flush=True) - - - def write_json(path, value): - tmp = path + ".tmp" - with open(tmp, "w") as stream: - json.dump(value, stream, sort_keys=True) - os.replace(tmp, path) - - - def hold_memory(mib): - memory = bytearray(mib * 1024 * 1024) - for offset in range(0, len(memory), 4096): - memory[offset] = 1 - return memory - - - def emit_tx_apply(milliseconds): - emit("metric 'ledger.transaction.apply'") - emit(f"sum = {milliseconds}ms") - - - attempt = required_int("SYNTHETIC_ATTEMPT") - target = required_int("SYNTHETIC_TARGET") - count = required_int("SYNTHETIC_COUNT") - key = os.environ["SYNTHETIC_KEY"] - data_dir = os.environ.get("SYNTHETIC_DATA_DIR", "/data") - state_path = os.path.join(data_dir, ".synthetic-replay.json") - ready_path = os.path.join(data_dir, ".synthetic-successor-ready") - release_path = os.path.join(data_dir, ".synthetic-release") - - if attempt == 1: - anon_mib = required_int("SYNTHETIC_PREDECESSOR_ANON_MIB") - working_set_mib = required_int("SYNTHETIC_PREDECESSOR_WORKING_SET_MIB") - duration = float(os.environ["SYNTHETIC_PREDECESSOR_SECONDS"]) - tx_apply = required_int("SYNTHETIC_PREDECESSOR_TX_APPLY_MS") - reached = max(target - 1, target - count) - write_json(state_path, {"key": key, "reachedLedger": reached}) - memory = hold_memory(working_set_mib) - emit(f"SYNTHETIC PEAK: anonBytes={anon_mib * 1024 * 1024} " - f"workingSetBytes={working_set_mib * 1024 * 1024}") - emit(f"SYNTHETIC PREDECESSOR: {key} persisted ledger {reached}") - time.sleep(duration) - emit_tx_apply(tx_apply) - del memory - sys.exit(3) - - with open(state_path) as stream: - state = json.load(stream) - if state.get("key") != key: - raise RuntimeError("PVC state belongs to a different logical range") - reached = int(state["reachedLedger"]) - emit(f"RESUME PROBE: offline-info reports lcl {reached}") - emit(f"RESUME: {key} reached ledger {reached}, replay had started; skipping new-db") - - anon_mib = required_int("SYNTHETIC_SUCCESSOR_ANON_MIB") - working_set_mib = required_int("SYNTHETIC_SUCCESSOR_WORKING_SET_MIB") - minimum = float(os.environ["SYNTHETIC_SUCCESSOR_MINIMUM_SECONDS"]) - maximum = float(os.environ["SYNTHETIC_MAXIMUM_WAIT_SECONDS"]) - tx_apply = required_int("SYNTHETIC_SUCCESSOR_TX_APPLY_MS") - memory = hold_memory(working_set_mib) - emit(f"SYNTHETIC PEAK: anonBytes={anon_mib * 1024 * 1024} " - f"workingSetBytes={working_set_mib * 1024 * 1024}") - write_json(ready_path, {"attempt": attempt, "key": key, "reachedLedger": reached}) - - started = time.monotonic() - while time.monotonic() - started < maximum: - elapsed = time.monotonic() - started - if elapsed >= minimum and os.path.exists(release_path): - emit_tx_apply(tx_apply) - del memory - sys.exit(0) - time.sleep(0.25) - raise RuntimeError("timed out waiting for the synthetic release marker") -{{- end }} diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values-ondemand.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values-ondemand.yaml new file mode 100644 index 00000000..6447322e --- /dev/null +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values-ondemand.yaml @@ -0,0 +1,79 @@ +# On-demand overlay: one pod per node, AMD-preferred pools. +# +# helm ... -f values.yaml -f values-ondemand.yaml +# +# The default values.yaml is tuned for spot, where the pools were doubled on +# 2026-08-04 and every claim is half a node so two pods share it. On-demand pools +# were deliberately left at their original sizes, so those same claims are the +# NAMEPLATE of an on-demand node -- and nameplate is not allocatable. Every +# on-demand tier was unschedulable as a result: a 14336Mi claim against a +# 16384Mi node has only ~13313Mi to land in once the EKS reserve (255Mi plus the +# 25/20/10/6% tiers) and 154Mi of daemonsets are taken out. +# +# Claims below are sized from that measured allocatable, not from the nameplate, +# and sit above half of it -- so exactly one pod fits and a second never can. +# Verified against a real node the same day: a 16 GiB box reported 13312Mi usable, +# which is the figure these are cut from. +# +# cpu is bounded by the SMALLEST shape in each tier's pool, since a claim that +# only fits the large fallback wins no nodes at all. protostar is the case that +# bites: it still reaches x8i.large at 2 vCPU, so its cpu claim stays at 1.60 +# even though its top rung (r8a.xlarge) has 4. +worker: + # on-demand pairs with ephemeral, always: /data on the node disk is denser and + # there is no eviction to resume from. pvc is what makes spot survivable, and + # the two are never mixed. + storageMode: "ephemeral" + +monitor: + capacityType: "on-demand" + + # Claims are cut from allocatable MINUS the daemonsets, which the previous + # table did not do -- it sized straight to allocatable, so on 2026-08-07 every + # on-demand tier failed to schedule on BOTH dimensions and Karpenter provisioned + # nothing: "no instance type has enough resources ... resources={cpu 1820m, + # memory 3054Mi}" against c8a.large's 2663Mi/1715m. Ten workers sat Pending + # indefinitely, which reads as slow provisioning rather than as a sizing bug. + # + # Daemonset overhead is 494Mi / 245m, measured on this cluster: alloy 10m/50Mi, + # aws-node 75m, ebs-csi-node 30m/104Mi, kube-proxy 100m, and ebs-csi-node-windows + # at 30m/340Mi. That last one CANNOT run here -- it carries + # nodeSelector kubernetes.io/os=windows -- but the nodepools constrain arch and + # not os, so Karpenter cannot prove it away and reserves for it on every node. + # Adding `kubernetes.io/os In [linux]` to the catchup nodepools would return + # 340Mi per node at every tier; the nodepool definitions live outside this repo, + # so these claims are cut against the 494Mi Karpenter actually enforces and stay + # correct (just conservative) if that constraint lands later. + # + # Each claim leaves a 3% margin and still exceeds half of (allocatable - ds), so + # one pod fits and two provably cannot. See the contract test for both halves. + # + # tier shape vCPU allocatable claim + # subdwarf/dwarf c8a.medium 1 1127Mi / 725m 576Mi / 0.45 + # subgiant c8a.large 2 2663Mi / 1715m 2048Mi / 1.40 + # giant m8a.large 2 5940Mi / 1715m 5248Mi / 1.40 + # supergiant r8a.large 2 13313Mi / 1715m 12416Mi / 1.40 + # nebula m8a.2xlarge 4 13313Mi / 3705m 12416Mi / 3.35 + # hypergiant r8a.xlarge 4 28714Mi / 3705m 27328Mi / 3.35 + # protostar x8i.large 2 28714Mi / 1715m 27328Mi / 1.40 + # supernova r8a.2xlarge 8 59515Mi / 7695m 57216Mi / 7.20 + poolCpu: "subdwarf:0.45,dwarf:0.45,subgiant:1.40,giant:1.40,supergiant:1.40,hypergiant:3.35,supernova:7.20,protostar:1.40,nebula:3.35" + poolMem: "subdwarf:576Mi,dwarf:576Mi,subgiant:2048Mi,giant:5248Mi,supergiant:12416Mi,hypergiant:27328Mi,supernova:57216Mi,protostar:27328Mi,nebula:12416Mi" + # vCPU of the smallest node each tier can land on. Differs from the spot map + # because on-demand pools were never doubled -- supergiant bottoms out at + # r8a.large (2), not at a 4-vCPU shape. + poolVcpu: "subdwarf:1,dwarf:1,subgiant:2,giant:2,supergiant:2,hypergiant:4,supernova:8,protostar:2,nebula:4" + # Both top rungs open, as on spot -- but they get here by the other route. On + # this map supergiant bottoms out at 2 vCPU and hypergiant at 4, so BOTH rungs + # cross a vCPU class (2->4 and 4->8) and the whitelist is what carries them; + # an empty denylist alone would leave them shut. On spot only the supernova + # rung needs listing, because the doubled pools put supergiant and hypergiant + # on equally-sized nodes. + # + # The cost is higher here than on spot: one pod per node means a promotion buys + # a bigger node outright, with no co-tenant to amortise it, so each promoted + # range doubles its vCPU draw. And the bump fires on working set, which does not + # predict throughput -- same box, memory.max 28GiB ran 1.83 lps against 56GiB at + # 1.70 -- so some of what it promotes will gain nothing. + poolCrossRungs: "supergiant->hypergiant" + poolBlockRungs: "hypergiant->supernova" diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 2f138cdd..af467d33 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -14,7 +14,9 @@ worker: # volume lifecycles for no measurable cost and no CSI throttling. Reuse only # bought bookkeeping, plus pinning every later range to the AZ the slot's # first volume happened to land in. - storageSize: "40Gi" + # 60Gi to match the tier nodes' ephemeral allowance: peakEphemeralBytes tops + # out at 37.8Gi across the whole profile, so this covers 100% with headroom. + storageSize: "60Gi" # Cap on PVC-mounting workers per node, enforced with a topologySpread # minDomains floor. A Nitro node allows ~26 EBS attachments and Karpenter does # not size nodes for attachment capacity. Inert at realistic density (1800m @@ -72,16 +74,20 @@ range: monitor: # Reuses the existing hand-built job-monitor image slot -- no new image and no # new push path (nothing in .github/workflows builds these). - image: "stellar/ssc-job-monitor:latest" + # + # TEMPORARY dev pin, 2026-08-07: stellar/ssc-job-monitor:latest predates the + # apps/+lib/ split, ATTEMPT_BUDGETS and per-cause budget counting, so running + # this chart against it ships env vars the image cannot read. Built from this + # branch and pinned by tag rather than :latest so a run is always traceable to + # one image. Revert to the stellar/ repo once there is a push path for it. + image: "stellajuna/ssc-jm:2026-08-07b" # Dev loop: run the monitor and collector from a ConfigMap holding # job_monitor.py and log_collector.py instead of a built image. Set it to the # ConfigMap name and point monitor.image at a plain python base; the deps the # Dockerfile bakes get pip-installed at start. Empty = use the image as built. sourceConfigMap: "" # Source-mode development normally installs dependencies at container start. - # Disable only when the selected image already contains them, for example in - # the opt-in synthetic integration harness where external package access is - # intentionally avoided. + # Disable only when the selected image already contains them. sourceInstallDependencies: true # Range profile from an earlier run, as a ConfigMap holding profile.json. # The mission driver resolves --pubnet-parallel-catchup-profile (a local @@ -116,13 +122,237 @@ monitor: # long ranges are the ones that got faster. The top bands are not idle cpu: # a 2.0 request lands ~3 co-tenants per node instead of 8-14, and ranges on # uncrowded nodes measured 1.80 vs 1.34 ledgers/s. - profileCpuTiers: "50:0.5,80:0.75,92:1.0,97:1.25,98.5:1.5,99.25:2.0,99.7:2.5,100:3.0" + # + # Reshaped 2026-08-01 after watching the first wave of the 1224-worker run. + # Uniformly raising every band was the wrong move: it reserved cpu without + # changing what a long range is actually delivered. With cpu limits removed a + # saturated node splits its spare cpu by weight, so when a node holds nothing + # but long ranges -- which is exactly what longest-first produces -- they all + # get roughly allocatable/co-tenants regardless of what they asked for. + # Measured live: nodes 94% cpu-reserved, longest-100 ranges with 4.0 + # co-tenants each, so ~1.81 vCPU delivered against much larger requests. + # + # Replaced 2026-08-01 with a continuous ramp rather than a few wide steps, so + # a range's request tracks its runtime instead of jumping at a band edge. + # Three segments, keyed on the range's runtime percentile: + # + # p75..p100 1.5 -> 3.5 the longest quarter, sampled every 2-3 points + # p50..p75 1.0 -> 1.5 + # p0..p50 0.5 -> 1.0 + # + # 22 bands, none spanning more than 0.24 vCPU, so the granularity is in the + # ladder rather than in how the wave happens to land on it. + # + # Parallelism is coupled to this and cannot be chosen separately. With + # longest-first the wave is the top of the ramp, so the mean request is ~3.0 + # rather than the ~1.3 a flat ladder gives, and the fleet grows in proportion: + # 448 workers is what fits the 2304 vCPU spot quota at 87%. See + # scratchpad/granular-ladder.py for the sweep. + # Rebuilt 2026-08-01 around ledgers/sec rather than percentile shape, after + # the 448-worker run showed the ramp was right but the parallelism was not: + # it finished the longest quarter at 0.47x profiled, then spent hours draining + # 3536 short ranges through too few slots (projected 6.55h, work-bound). + # + # With cpu limits removed a pod is delivered request * (allocatable / sum of + # requests on its node), so density feeds straight back into ledgers/sec. A + # flat coarse ladder at 1.30-1.80 packs 4-5 per node and drops the dense + # ranges to 1.8 LPS -- 22% of the first wave under 3. So the top stays tall + # for the ranges whose ledgers are transaction-dense (they cap out near 3 LPS + # however much cpu they get) and only the middle is coarsened. + # + # Five bands over the top quarter instead of twelve. Wave floor 1.30 keeps at + # most 5 pods on a node (6 x 1.30 > 7.56 allocatable). At 768 workers: ~269 + # nodes, 2152 vCPU (93% of the spot quota), first-wave LPS min 2.89 and p10 + # 3.35. Parallelism and this ladder move together -- the p74 boundary is where + # the 768-range wave starts. + # Derived rather than hand-shaped, 2026-08-01, from the response curve the + # 768-worker run measured on 664 live workers: m(c) = 0.856 * c^-0.531, which + # reproduced all four of its bands to within 0.01 (1.50->0.69, 2.60->0.51, + # 3.40->0.47, 4.00->0.41). + # + # While a run is work-bound its makespan IS its total work, so the objective + # is to minimise sum(seconds_i * m(cpu_i)) under a cpu budget, not to equalise + # finish times. Equalising was measured as actively worse: it pushes short + # ranges to 0.5 cpu where m = 1.24 -- 24% SLOWER than profiled -- inflating + # total work ~25% and costing ~40 minutes. + # + # Setting the marginal return equal across ranges gives a closed form: + # cpu proportional to seconds^(1/(1+b)) = seconds^0.653 + # scaled by 0.0060 and clamped to [0.5, 4.5]. At 1280 workers on the + # on-demand pool that is ~358 r8a.2xlarge, 2864 vCPU, at most 5 pods/node, + # and it lands exactly on the crossover: work-bound 1.87h against a 1.88h + # critical path. Below 1280 the tail idles; above it the longest range walls. + # Rebuilt 2026-08-02 from four full-fleet probes (A/B/C/D), 735 paired ranges + # each, mean cpu 1.17 -> 3.54, requests only and no cpu limit anywhere: + # + # ratio(c) = 0.671 * c^0.348 residuals <= 0.005 + # + # No knee in that span. The retired ladder assumed saturation below 2.25 from + # arms run at hard cpu LIMITS -- a cfs quota clips the bursts replay uses, so + # those arms measured the quota, not the process. See cpu-ladders-history.md. + # + # The critical path is NOT a fixed range. Probes A/B/C/D each reported a + # different slowest range, and the profile's longest (10340s) finished 99th of + # 735 in probe C. Per-range slowness reproduces across probes at only r=0.26-0.59 + # (r^2 0.07-0.35), so most of it is run noise, not a property of the range. + # Tuning cpu at whichever range came last just moves the straggler. + # + # What IS reproducible: headroom compresses the noise. Residual (observed time / + # predicted) by the cpu the range actually had -- + # + # cpu 1.15 1.30 1.90 2.50 3.30 3.70 + # p90 1.51 1.40 1.27 1.27 1.21 1.22 + # max 2.39 2.08 2.06 1.76 1.49 1.63 + # + # Targets a CRITICAL-PATH BOUND run on the ON-DEMAND pool: simulated 2.20h with + # the longest range also finishing at 2.20h, so nothing trails it. + # + # Chosen by discrete-event SIMULATION of the actual dispatch, not by the + # max(work/fleet, longest-job) bound. That bound is what an earlier ladder was + # solved against and it is only a LOWER bound -- it sizes each band as if the + # range starts at t=0, but longest-first dispatches the cheap ranges LAST. The + # ladder it produced put p66 at 0.9 cpu: 1.40h of runtime starting at 1.40h, + # finishing at 2.79h, half an hour after the longest range was already done. + # Starving the bulk does not make a run crit-path bound, it just moves the tail. + # + # The real constraint is concurrency, not per-range speed. Jobs long enough to + # matter must all be running at t=0 or they serialise: under that earlier ladder + # the jobs over 1.25h needed 2518 vCPU against a 2304 spot fleet, so some had to + # queue and any two that did cost double. Hence a flat 1.3 across the bottom 95% + # -- cheap enough that they all fit at once, rich enough to stay out of the band + # where the noise tail blows up (residual p90 is 1.51 at 1.15 cpu against 1.15 at + # 7.0, so cheap bulk is bought at the price of stragglers). + # + # 1.3 rather than 0.9 for the flat band because 0.9 ties the noiseless sim (both + # 2.20h) and loses badly once measured noise is applied (5.06h against 4.15h). + # This is the choice that does not depend on which model is right. + # + # Sized for the on-demand pool: 1600 replicas, first wave 2912 vCPU against the + # catchup-od nodepool limit (raised 3072 -> 4096 so the packing model's habitual + # 7-16% optimism cannot push it over). Spot cannot host this -- its AWS quota is + # 2304 vCPU where this needs ~2900, and the cheapest ladder that fits spot + # without queueing runs 3.86h. + # + # Requires range.order=longest-first; under tip-first the top-band ranges scatter + # through the run instead of starting first (tip-first is only a 58% proxy). + # + # Sizing: replicas is a QUEUE DEPTH, not a fleet size, and it should exceed what + # the quota can run at once. With range.order=longest-first the first wave is + # entirely top-band, so peak demand per pod is far above the ladder mean -- + # measured 2.63 cpu over the 700 longest ranges against a 1.53 ladder mean. No + # replica count both fits the expensive wave AND saturates the cheap tail: even + # 700 workers is 94% of quota in wave 1, while the tail would happily run 2000+. + # + # So oversubscribe and let the scheduler absorb it. At 1200 the run sat at ~2350 + # vCPU with ~240 Pending, and the queue drained on its own as top-band jobs + # finished (239 -> 206 in 5 min, running 962 -> 970). Pending pods hold no + # resources; they are how the fleet stays saturated across a 4x swing in cost per + # range without anyone retuning mid-run. + # + # Do NOT size as quota/mean-cpu (ignores memory bin-packing waste) and do not + # trust a bin-pack of a RANDOM sample (right method, wrong population -- the run + # never dispatches a random sample first; that rule predicted 88% of quota where + # reality was 102%). + # + # Memory sizing is deliberately UNCHANGED. A limit sweep from 16Gi to 40Gi on + # one range moved major faults 1768 -> 0 and ledgers/sec not at all + # (1.464-1.499), so peakAnonBytes remains the right basis and the working-set + # figure stays unused -- it measures what was available, not what is needed. + # --- nodepool tiers ------------------------------------------------------- + # + # A range picks a NODEPOOL by its measured memory and gets that node to + # itself. Empty prefix disables all of it and every worker keeps the single + # global node label, which is exactly the pre-tier behaviour. + # + # Cuts are node_usable/1.60, covering the p99 of run-to-run growth in the same + # range's peakAnonBytes (18,073 observations across five profiles: p50 0.97, + # p90 1.28, p99 1.60, max 2.83). Learned the hard way -- range 63080767 + # measured 13.75Gi went onto nodes with 14.1/14.3Gi allocatable, a 1.03x + # margin, and OOMKilled on BOTH during bucket-apply before closing a ledger. + poolPrefix: "" + poolTiers: "0:subdwarf,0.79:dwarf,1.61:subgiant,3.87:giant,8.85:supergiant,18.38:hypergiant,:supernova" + # cpu is a claim token, not a demand estimate -- replay draws ~1.05 cores + # whatever it is given and is flat in core count from 2 upward (+2.8% at 2->4, + # +1.5% at 4->8). Kept at or below the SMALLEST node in each tier so the + # low-weight fallback rungs stay schedulable. + # Exactly 50% of each tier node's nameplate capacity. Two pods would need the + # whole node, which always exceeds allocatable, so a second can never fit -- + # isolation without depending on how the kubelet reserves. + # hypergiant and supernova are NOT half their node: they are sized to the + # smallest shape in the pool. x8i.large is r8a.xlarge with half the cores and + # the same 32 GiB, and x8i.xlarge is r8a.2xlarge with half the cores and the + # same 64 GiB, so preferring the x8i shapes buys the same RAM for half the + # spot quota -- 2128 -> 1800 vCPU on a 900-worker wave, 15% back. + # + # A "half the node" claim of 2.00/4.00 exceeds what an x8i node can offer once + # daemonsets are counted (215m: alloy 10 + aws-node 75 + ebs-csi-node 30 + + # kube-proxy 100), so those pools never won a single node in either run on + # 2026-08-03 despite being weighted in. 1.70 and 3.60 sit under the 1715m and + # 3705m that are actually schedulable. + # + # Dropping cpu below half stops cpu from isolating the pod on the larger + # fallback shapes -- two 1.70 claims do fit a 4-vCPU r8a.xlarge. Memory is + # what isolates here instead, and it holds on every type in both pools because + # they all carry the same RAM: 2 x 16384Mi > 28713Mi usable, 2 x 32768Mi > + # 59515Mi usable. + # vCPU of the smallest node in each tier's pool -- what decides whether a + # promotion is free. Cannot be inferred from poolCpu, which is a claim rather + # than a node size. + # + # Reading the SMALLEST shape was quietly wrong while the x8i pools existed: + # hypergiant listed x8i.xlarge (4 vCPU) at w80 beneath two 8-vCPU rungs, so this + # said 4 and supergiant->hypergiant priced as free, while Karpenter tried w100 + # first and the promotion really cost 4->8. The finished 1200-worker run put 12 + # hypergiant nodes on 2xlarge shapes against 1 on x8i.xlarge. x8i was removed + # from spot on 2026-08-04, so these are now both the smallest and the + # top-weighted shape and the top rungs cross a class honestly. + # Rungs allowed to cross a vCPU class anyway. hypergiant->supernova costs + # +2 vCPU and gains only 1.23x, worth 0 minutes on its own -- but the long + # jobs alternate between the two tiers, so paired with supergiant->hypergiant + # it takes the floor from 2.90h to 2.44h where either alone gets 8 min or none. + # supergiant->hypergiant is listed but is a no-op at today's sizes: the doubled + # spot pools put both tiers on 4-vCPU nodes, so the rung is already free and + # passes the guard without a whitelist entry. It is here so the rung survives + # the pools diverging again -- if hypergiant ever bottoms out above supergiant, + # the guard would silently shut a rung that is deliberately open. + poolCrossRungs: "supergiant->hypergiant" + # Rungs that never run regardless of the vCPU comparison. Empty: with the spot + # pools doubled a promotion lands the range on a bigger SHARED node, and that + # sharing is what it buys. Measured 2026-08-04, two pods per node on one range: + # a co-tenant cost 1.02x per pod on an 8-vCPU node (r8id.2xlarge, 3.78/3.91 lps) + # against 1.58x on a 4-vCPU node. Note the bump fires on working set, which does + # NOT predict throughput -- same box, memory.max 28GiB ran 1.83 lps vs 56GiB at + # 1.70 -- so it reaches the right nodes by the wrong signal. + poolBlockRungs: "hypergiant->supernova" + poolVcpu: "subdwarf:2,dwarf:2,subgiant:4,giant:4,supergiant:4,hypergiant:8,supernova:16,protostar:8,nebula:8" + poolCpu: "subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:3.80" + poolMem: "subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:14336Mi" + # Profiled run, range past the profile's top: newest ledgers, and the newest + # are the densest, so a rich pool rather than an average one. + poolUnprofiled: "protostar" + # No profile at all: nothing is known, so the biggest nodes and the configured + # defaults. + poolNoProfile: "nebula" + # ANDed with the tier label: "spot" or "on-demand". Both capacity variants of + # a tier share one label value, so this is what separates them. Empty means no + # capacity constraint. + capacityType: "" profileMargin: 1.15 # No margin on cpu: it is compressible, so under-requesting costs contention # Ceiling for profile-derived memory. Above the configured worker limit on # purpose: a range needing more than that must be able to ask for it # rather than be pinned under its own measured peak. profileMaxMemory: "32Gi" + # Disk allowances, mirroring the memory ones. The 2026-08-01 ephemeral run + # peaked at 37.76Gi against a flat 40Gi limit -- 6% of margin, on a detection + # and escalation path that has never executed on real data. These give a + # measured range its own sizing instead: flat headroom for image/logs/WAL, + # plus a runtime-weighted share because disk tracks runtime at pearson 0.920. + profileEphemeralHeadroom: "2Gi" + profileRuntimeEphemeralInsurance: "8Gi" + # Above the flat LIM_EPHEMERAL on purpose: that limit is what an UNMEASURED + # range gets, and capping a measured one at it would discard the measurement. + profileMaxEphemeral: "64Gi" # Fixed allowance added to a range's measured rss, on top of profileMargin. # Not zero: memory.max bounds anon PLUS page cache, and a multiplicative # margin is meaningless at small rss. Measured with headroom 0, ranges @@ -146,26 +376,54 @@ monitor: # about once per interval; three consecutive failures preserve the old # approximately 90-second down threshold without coupling network I/O to the # 10-second reconcile loop. Unprobed/intermediate workers report unknown. - livenessProbeIntervalSeconds: 30 livenessProbeTimeoutSeconds: 5 - livenessFailureThreshold: 3 - # At 2096 workers and a 30s interval this is ~70 requests/s. Thirty-two slots - # cover normal sub-second local responses while bounding slow requests, - # queued work, sessions and threads independently of fleet size. + # One sweep per reconcile pass, and the reconcile loop waits for it -- so this + # is the most a fleet of unreachable workers can delay dispatch. + livenessSweepSeconds: 15 + # Concurrent probes per sweep. At 2096 workers this is the throughput that has + # to fit inside livenessSweepSeconds: 32 slots x sub-second local responses + # clears the fleet, and a fleet that does not answer is bounded by the sweep. livenessMaxConcurrency: 32 - maxAttempts: 5 - # Evictions, admission rejections and monitor restarts are not the range's - # fault, so they do not share the failure budget above. Measured on ssc-test: - # ten evictions across 25 workers put four healthy ranges on attempt 3 of 5. - maxDisruptionAttempts: 20 + # Attempts each failure cause gets before the range is condemned. Every budget + # is spent by its own cause, and a cause that is not here is condemned the + # first time it happens -- a timeout, a real catchup failure, an exit 3 with + # nothing in its archive, and anything the monitor could not classify. + attemptBudgets: + # The cluster took the pod away mid-run, which proves the range was fine. + # Effectively unlimited: on spot a healthy range is evicted dozens of times. + disrupted: 100 + # Refused by the kubelet before any container ran (attachment limits, + # admission churn). The range never started, so a retry masks nothing. + rejected: 100 + # An exit 3 whose archive named a fetch fault: an unreachable history + # mirror is the cluster's problem, not the range's. + fetchFault: 20 + # Each retry escalates the memory request one rung. + oom: 5 + # Each retry escalates the disk limit one rung. Smallest, because an + # eviction repeats identically until the range gets more disk. + ephemeral: 4 memBumpFactor: 1.5 - # An ephemeral-storage eviction repeats until the range gets more disk, so - # it gets its own small budget rather than the environmental one. - maxEphemeralAttempts: 4 ephBumpFactor: 1.5 maxEphemeral: "200Gi" maxMem: "48Gi" graceSeconds: 100 + # preStop stall before SIGTERM, in seconds. 0 is off. + # + # Buys the collector time to NOTICE the disruption before the kill; it is not + # what captures the metric. The hook cannot widen the ~4ms between the medida + # block and the process exiting -- measured, a 60s hook with 10s polling and + # no detection still lost txApply, while 1s polling with no hook captured it. + # What it prevents is SIGTERM landing while the poller is still on its lazy + # cadence because the pod-list cycle has not come round yet. + # + # Sized against the collector CYCLE (sleep + pod list + kubelet sweep), not + # against collectorPollSeconds. Overshooting is NOT free on spot: the hook is + # dead time inside the ~120s AWS reclaim budget. Undershooting the cycle is + # survivable because the pod lives out graceSeconds after SIGTERM and its log + # stays readable, so a sweep that notices late still finds the medida block. + # Must stay well under graceSeconds or the kubelet kills the hook. + prestopSleepSeconds: 5 # Must exceed any plausible monitor outage: completion is recorded to the # progress ConfigMap by the monitor, and a Job reclaimed before that happens # reads as "never ran" and gets redone. @@ -192,6 +450,25 @@ monitor: # concurrency is independent of worker.replicas. At 4096 pods and a 10s # interval this is ~90 in-flight polls. logPollSeconds: 10 + # Spot gives ~120s of notice; past roughly double that the drain was + # cancelled and the stream should go back to interval polling. + doomedFollowSeconds: 300 + # Follow streams get their own budget so a mass reclaim cannot consume every + # poll slot and blackout the rest of the run. Condemned pods beyond this fall + # back to polling, which is the pre-existing behaviour. + maxDoomedFollows: 256 + # Poll interval for a pod the cluster has condemned. The cheap half of the + # disruption fix: preStop delays SIGTERM but leaves ~9s between the medida + # block and the pod object vanishing, which a 10s poll straddles. + doomedPollSeconds: 1 + # Lifetime of each pod watch before the apiserver closes it and the collector + # reconnects from its last resourceVersion. The watch is what makes a + # condemnation visible immediately rather than on the next collectorPollSeconds + # sweep: measured at prestopSleepSeconds=5, the sweep alone caught that window + # about half the time and lost 32 of 52 mid-replay legs. 0 disables the watch + # and leaves detection to the sweep. + watchTimeoutSeconds: 600 + watchRetrySeconds: 1 maxConcurrentPolls: 96 maxPollChars: 8388608 # Failed polls tolerated after a pod goes terminal before the collector stops @@ -211,23 +488,5 @@ monitor: requests: { cpu: "200m", memory: "512Mi" } limits: { cpu: "2", memory: "2Gi" } -# Fixed, chart-owned worker used only by the bounded Kubernetes integration -# harness. It never invokes stellar-core or history services. Keeping the script -# in the chart instead of accepting a command value avoids turning this into a -# general arbitrary-command surface. -integration: - syntheticWorker: - enabled: false - imagePullPolicy: IfNotPresent - predecessorSeconds: 12 - successorMinimumSeconds: 12 - maximumWaitSeconds: 180 - predecessorAnonMiB: 48 - predecessorWorkingSetMiB: 56 - successorAnonMiB: 24 - successorWorkingSetMiB: 32 - predecessorTxApplyMilliseconds: 1250 - successorTxApplyMilliseconds: 2500 - service_account: annotations: [] diff --git a/src/MissionParallelCatchup/pytest.ini b/src/MissionParallelCatchup/pytest.ini index ce7a07ad..a5c9a940 100644 --- a/src/MissionParallelCatchup/pytest.ini +++ b/src/MissionParallelCatchup/pytest.ini @@ -1,6 +1,10 @@ [pytest] -# The monitor and collector are plain modules in the parent directory, not an -# installed package, so the suite needs both on the path: `.` for job_monitor / -# log_collector, `tests` for the fake-cluster harness. -pythonpath = . tests +# The monitor and collector are plain modules, not an installed package, so the +# suite needs each source directory on the path: `apps` for job_monitor / +# log_collector, `lib` for the modules they import, `tests` for the fake-cluster +# harness. The container flattens apps/ and lib/ into /app, so `import config` +# resolves the same way there. +pythonpath = apps lib tests testpaths = tests +# Async tests declare themselves with @pytest.mark.asyncio. +asyncio_mode = strict diff --git a/src/MissionParallelCatchup/requirements-dev.txt b/src/MissionParallelCatchup/requirements-dev.txt new file mode 100644 index 00000000..3bf4ba66 --- /dev/null +++ b/src/MissionParallelCatchup/requirements-dev.txt @@ -0,0 +1,12 @@ +# Test-only dependencies. The runtime pins live in Dockerfile.jobmonitor and are +# installed a second time by the chart's sourceConfigMap path -- see +# tests/contract/test_dependency_pins.py, which keeps those two in step and +# checks that the major this suite imports is the major the image ships. +# +# The kubernetes client is listed because the contract tests build real V1* +# objects: the suite must import the same major the image installs. +pytest~=9.1 +pytest-asyncio~=1.4 +kubernetes~=36.0 +aiohttp~=3.14 +prometheus-client~=0.19 diff --git a/src/MissionParallelCatchup/tests/collector/test_archive_append.py b/src/MissionParallelCatchup/tests/collector/test_archive_append.py index 528a937f..44aa3ad0 100644 --- a/src/MissionParallelCatchup/tests/collector/test_archive_append.py +++ b/src/MissionParallelCatchup/tests/collector/test_archive_append.py @@ -24,7 +24,9 @@ import pytest +import records import job_monitor as jm +import config import log_collector as lc @@ -89,7 +91,7 @@ def test_a_torn_archive_does_not_abort_the_reconcile_pass(cluster): cluster.advance(300, 'succeeded') # The collector has not flushed .metrics yet, so the archive is the only # source for txApply -- and it is exactly the file being written. - write_torn_archive(jm.log_path('300', 1)) + write_torn_archive(records.log_path('300', 1)) result = cluster.reconcile() @@ -113,9 +115,9 @@ def test_a_torn_archive_costs_one_range_not_the_other_ranges_in_the_pass(cluster cluster.advance(300, 'succeeded') cluster.advance(200, 'succeeded') # r300: collector finished cleanly. - cluster.finalize(300, 1, tx_apply=12.5, peaks={'peakRssBytes': 111}) + cluster.finalize(300, 1, tx_apply=12.5, peaks={'peakAnonBytes': 111}) # r200: collector is mid-poll, archive torn, nothing durable yet. - write_torn_archive(jm.log_path('200', 1)) + write_torn_archive(records.log_path('200', 1)) result = cluster.reconcile() @@ -123,7 +125,7 @@ def test_a_torn_archive_costs_one_range_not_the_other_ranges_in_the_pass(cluster assert set(completed) == {'300', '200'} # The healthy range is untouched by its neighbour's corrupt file. assert completed['300']['txApply'] == 12.5 - assert completed['300']['peakRssBytes'] == 111 + assert completed['300']['peakAnonBytes'] == 111 assert 'pc-r300-a1' not in cluster.jobs(), "finalized range was not reaped" # The torn range pays, and only the torn range. assert completed['200']['txApply'] is None @@ -145,7 +147,7 @@ def test_a_never_repaired_torn_archive_does_not_wedge_the_run(cluster): cluster.advance(200, 'succeeded') cluster.finalize(200, 1, tx_apply=7.0) # r300's archive is torn and nobody ever fixes it. - torn = write_torn_archive(jm.log_path('300', 1)) + torn = write_torn_archive(records.log_path('300', 1)) cluster.reconcile() # records 300 + 200, dispatches 100 assert 'pc-r100-a1' in cluster.jobs() @@ -170,36 +172,21 @@ def test_a_range_recovers_its_metric_once_the_collector_finishes(cluster): """ cluster.reconcile() cluster.advance(300, 'succeeded') - write_torn_archive(jm.log_path('300', 1)) + write_torn_archive(records.log_path('300', 1)) cluster.reconcile() assert cluster.completed()['300']['txApply'] is None # The collector's poll completes and it writes what it scanned out of the # stream. The archive on disk is still torn. - cluster.finalize(300, 1, tx_apply=88.25, peaks={'peakRssBytes': 222}) + cluster.finalize(300, 1, tx_apply=88.25, peaks={'peakAnonBytes': 222}) cluster.reconcile() assert cluster.completed()['300']['txApply'] == 88.25 - assert cluster.completed()['300']['peakRssBytes'] == 222 + assert cluster.completed()['300']['peakAnonBytes'] == 222 assert 'pc-r300-a1' not in cluster.jobs() -def test_a_readable_archive_is_still_the_txapply_fallback(cluster): - """Guard rail: widening the except must not swallow a good read. - - Without this, 'catch everything and return None' would pass every test - above while silently deleting the archive fallback. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - write_whole_archive(jm.log_path('300', 1)) # no .metrics: archive is the source - - cluster.reconcile() - - assert cluster.completed()['300']['txApply'] == pytest.approx(4.2) - - # --- writer side: the collector's append ------------------------------------ class _FakeContent: @@ -282,7 +269,7 @@ def test_collector_append_never_exposes_a_partial_member_to_a_reader(tmp_path, m every 250 lines. Every one of those reads must succeed and must see exactly the last settled content. """ - monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) monkeypatch.setattr(lc, 'token', lambda: 'test-token') path = lc.base('300', 1) + '.log.gz' rng = random.Random(7) diff --git a/src/MissionParallelCatchup/tests/collector/test_poll_backoff.py b/src/MissionParallelCatchup/tests/collector/test_poll_backoff.py index f9041198..fdf43b03 100644 --- a/src/MissionParallelCatchup/tests/collector/test_poll_backoff.py +++ b/src/MissionParallelCatchup/tests/collector/test_poll_backoff.py @@ -21,6 +21,7 @@ import pytest +import config import log_collector as lc # Everything is scaled down from the shipped 10s so the tests run in ~2s. The @@ -134,7 +135,7 @@ async def main_loop_wake(): @pytest.fixture def logs(tmp_path, monkeypatch): - monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) monkeypatch.setattr(lc, 'token', lambda: 'tok') monkeypatch.setattr(lc, 'LOG_POLL_SECONDS', POLL) monkeypatch.setattr(lc, 'TERMINAL_POLL_ATTEMPTS', ATTEMPTS) diff --git a/src/MissionParallelCatchup/tests/conftest.py b/src/MissionParallelCatchup/tests/conftest.py index c168c626..df4b0a51 100644 --- a/src/MissionParallelCatchup/tests/conftest.py +++ b/src/MissionParallelCatchup/tests/conftest.py @@ -15,12 +15,16 @@ def test_something(cluster): KUBERNETES_SERVICE_HOST). Every decision under test is the shipped code path. """ +import gzip import json import os import pytest import fake_k8s +import config +import kube +import records import job_monitor as jm # Config the fixture pins. Small on purpose: three ranges and PARALLELISM 2 so @@ -42,20 +46,43 @@ def test_something(cluster): 'SAVE_SUCCESS_LOGS': True, 'PROFILE_PATH': '', 'ATTEMPT_DEADLINE_SECONDS': 0, - 'MAX_ATTEMPTS_PER_RANGE': 5, - 'MAX_DISRUPTION_ATTEMPTS': 20, - 'MAX_EPHEMERAL_ATTEMPTS': 4, + # The whole retry policy. Patch this map, not the MAX_* constants: those are + # only the env seam and the map is built from them once, at import. + 'ATTEMPT_BUDGETS': {'disrupted': 100, 'rejected': 100, 'fetch-fault': 20, + 'oom': 5, 'ephemeral': 4}, 'LIM_EPHEMERAL': '', 'REQ_EPHEMERAL': '', } # What advance() does to the fake cluster for each name. The verdict the monitor # then reaches is its own business -- that is the thing under test. +# The cascade GetHistoryArchiveStateWork prints when a HAS fetch fails, ending in +# the give-up line. exit3_retry_cause() reads this to decide whether an exit-3 +# attempt is retryable, so a fixture exit-3 has to carry it to be retried. +FETCH_FAULT_ARCHIVE = ( + 'fatal error: Could not connect to the endpoint URL: ' + '"https://sts.us-east-1.amazonaws.com/"\n' + '2026-01-01T00:00:00.000 GAJSL [Process WARNING] process 1 exited 1: ' + 'aws s3 cp --no-progress s3://bucket/history-00000000.json /data/tmp\n' + '2026-01-01T00:00:00.000 GAJSL [History WARNING] Could not download file: ' + 'archive core_live_003 maybe missing file history/00/00/00/history-0.json\n' + '2026-01-01T00:00:00.000 GAJSL [History ERROR] Missing HAS for ledger 1: ' + 'maybe stale archive core_live_003\n' + '2026-01-01T00:00:00.000 GAJSL [History WARNING] Catchup failed\n') + +# Same give-up, no fetch cascade in front of it: what a SIGTERM drain leaves. +BARE_FAILURE_ARCHIVE = ( + '2026-01-01T00:00:00.000 GAJSL [Ledger INFO] Ledger close complete: 42\n' + '2026-01-01T00:00:00.000 GAJSL [History WARNING] Catchup failed\n') + +_ARCHIVES = {'fetch_fault': FETCH_FAULT_ARCHIVE, 'bare': BARE_FAILURE_ARCHIVE} + STATES = ( 'pending', # dispatched, nothing scheduled yet 'running', # pod Running, job active 'succeeded', # exit 0 - 'incomplete', # exit 3: did-not-complete, retryable on the range budget + 'incomplete', # exit 3 with a fetch fault in the archive: retryable + 'unexplained', # exit 3 with nothing in the archive to explain it: condemned 'condemned', # exit 1: genuine catchup failure, no retry 'oom', # exit 137 / OOMKilled 'disrupted', # DisruptionTarget condition -- spot eviction @@ -70,15 +97,15 @@ def test_something(cluster): class Driver: """Runs reconcile passes against the fake cluster and inspects the results.""" - def __init__(self, k8s, tmp_path, config): + def __init__(self, k8s, tmp_path, env): self.k8s = k8s self.jm = jm self.tmp_path = tmp_path - self.config = config - self.namespace = config['NAMESPACE'] - self.run_name = config['RUN_NAME'] - self.log_dir = jm.LOG_DIR - # Same dict update_status_and_metrics() carries across iterations of the + self.config = env + self.namespace = env['NAMESPACE'] + self.run_name = env['RUN_NAME'] + self.log_dir = config.LOG_DIR + # Same dict reconcile_loop() carries across iterations of the # loop, so multi-pass tests see the real cross-pass behaviour (halt on # regression, histogram replay guard, counter deltas). self.state = {'owner': None, 'replayed': set(), 'max_completed': 0, @@ -121,7 +148,7 @@ def advance(self, end, state, attempt=None): # Everything below is a failure; the Job condition and the pod detail # are set independently because in a real run either can be missing. - if state == 'incomplete': + if state in ('incomplete', 'unexplained'): if pod_name: self.k8s.set_pod_terminated(pod_name, exit_code=3) self.k8s.set_job_failed(name, message=self._policy_msg(pod_name, 3, 2)) @@ -194,7 +221,7 @@ def _policy_msg(self, pod_name, code, rule_index): # -- the collector's side of the contract -------------------------------- def finalize(self, end, attempt=1, tx_apply=None, peaks=None, resumed=False, - attempt_seconds=None): + attempt_seconds=None, archive=None): """Write what the log-collector sidecar writes for a finished attempt. The monitor will not reap a Job until the .done marker exists, and reads @@ -208,8 +235,25 @@ def finalize(self, end, attempt=1, tx_apply=None, peaks=None, resumed=False, data['attemptSeconds'] = attempt_seconds if resumed: data['resumed'] = True - self.write(jm.metrics_path(str(end), attempt), json.dumps(data)) - self.write(jm.done_path(str(end), attempt), '') + if archive in _ARCHIVES: + archive = _ARCHIVES[archive] + if archive is not None: + # exit 3 is classified from the archive, so a test driving that path + # has to stand in for what the worker wrote as well. + self.archive(end, attempt, archive) + self.write(records.metrics_path(str(end), attempt), json.dumps(data)) + self.write(records.done_path(str(end), attempt), '') + + def archive(self, end, attempt, text): + """Lay down an attempt's gzipped worker archive, with no .done marker. + + `text` may be one of the shorthands finalize() takes ('fetch_fault', + 'bare'). Writing the archive alone is what the collector's mid-append + state looks like. + """ + text = _ARCHIVES.get(text, text) + with gzip.open(records.log_path(str(end), attempt), 'wb') as fh: + fh.write(text.encode()) def write(self, path, text): os.makedirs(os.path.dirname(path), exist_ok=True) @@ -240,16 +284,11 @@ def pvcs(self): def progress(self): """The authoritative progress record, straight off disk.""" try: - with open(jm.PROGRESS_FILE) as fh: + with open(config.PROGRESS_FILE) as fh: return json.load(fh) except (OSError, ValueError): return {} - def progress_configmap(self): - """The best-effort ConfigMap mirror the mission driver reads.""" - data = self.k8s.config_map_data(jm.PROGRESS_CM, self.namespace) or {} - return json.loads(data.get('progress.json', '{}')) - def completed(self): return self.progress().get('completed', {}) @@ -267,27 +306,27 @@ def deleted(self): @pytest.fixture def cluster(tmp_path, monkeypatch): - config = dict(DEFAULT_CONFIG) + env = dict(DEFAULT_CONFIG) log_dir = tmp_path / 'logs' log_dir.mkdir() - k8s = fake_k8s.FakeCluster(namespace=config['NAMESPACE']) - monkeypatch.setattr(jm, 'core_v1', k8s.core_v1) - monkeypatch.setattr(jm, 'batch_v1', k8s.batch_v1) + k8s = fake_k8s.FakeCluster(namespace=env['NAMESPACE']) + monkeypatch.setattr(kube, 'core_v1', k8s.core_v1) + monkeypatch.setattr(kube, 'batch_v1', k8s.batch_v1) - for key, value in config.items(): - monkeypatch.setattr(jm, key, value) + for key, value in env.items(): + monkeypatch.setattr(config, key, value) # Derived at import from RUN_NAME / LOG_DIR, so they have to follow. - monkeypatch.setattr(jm, 'LOG_DIR', str(log_dir)) - monkeypatch.setattr(jm, 'PROGRESS_FILE', str(log_dir / 'progress.json')) - monkeypatch.setattr(jm, 'PROGRESS_CM', f"{config['RUN_NAME']}-catchup-progress") + monkeypatch.setattr(config, 'LOG_DIR', str(log_dir)) + monkeypatch.setattr(config, 'PROGRESS_FILE', str(log_dir / 'progress.json')) + monkeypatch.setattr(config, 'PROGRESS_CM', f"{env['RUN_NAME']}-catchup-progress") # Module-level mutable state that would otherwise leak between tests. - monkeypatch.setattr(jm, 'PROFILE', None) + monkeypatch.setattr(config, 'PROFILE', None) monkeypatch.setattr(jm, '_progress_owner', {}) # The chart's ConfigMap: owner_ref() reads it, and every Job, PVC and the # progress ConfigMap hang off it. - k8s.add_config_map(f"{config['RUN_NAME']}-stellar-core-config", + k8s.add_config_map(f"{env['RUN_NAME']}-stellar-core-config", {'stellar-core.cfg': '# test'}) - return Driver(k8s, tmp_path, config) + return Driver(k8s, tmp_path, env) diff --git a/src/MissionParallelCatchup/tests/contract/_artifacts.py b/src/MissionParallelCatchup/tests/contract/_artifacts.py index fa29b7ed..dabb0dbb 100644 --- a/src/MissionParallelCatchup/tests/contract/_artifacts.py +++ b/src/MissionParallelCatchup/tests/contract/_artifacts.py @@ -25,6 +25,8 @@ HERE = os.path.dirname(os.path.abspath(__file__)) MODULE_DIR = os.path.dirname(os.path.dirname(HERE)) # src/MissionParallelCatchup +APPS_DIR = os.path.join(MODULE_DIR, 'apps') # the two entrypoints +LIB_DIR = os.path.join(MODULE_DIR, 'lib') # what they import SRC_ROOT = os.path.dirname(MODULE_DIR) # src CHART = os.path.join(MODULE_DIR, 'parallel_catchup_helm') FSHARP_PATH = os.path.join(SRC_ROOT, 'FSLibrary', @@ -130,7 +132,7 @@ def granted(sets=(), release='t'): _PROBE = """ import json, sys -sys.path.insert(0, {module_dir!r}) +sys.path[:0] = [{apps_dir!r}, {lib_dir!r}] import {module} as m out = {{}} for k, v in vars(m).items(): @@ -153,11 +155,11 @@ def defaults(module_name, env_pairs=()): derived from an env var at import -- PROGRESS_CM off RUN_NAME, say -- where the derivation is what a test needs to see. """ - src = _PROBE.format(module_dir=MODULE_DIR, module=module_name) + src = _PROBE.format(apps_dir=APPS_DIR, lib_dir=LIB_DIR, module=module_name) env = {'PATH': os.environ.get('PATH', ''), 'HOME': os.environ.get('HOME', '')} env.update(dict(env_pairs)) r = subprocess.run([sys.executable, '-c', src], capture_output=True, - text=True, env=env, cwd=MODULE_DIR) + text=True, env=env, cwd=APPS_DIR) assert r.returncode == 0, f"could not import {module_name} cleanly:\n{r.stderr}" body = r.stdout[r.stdout.index('<<<') + 3:r.stdout.rindex('>>>')] return json.loads(body) diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py index c8001094..ed288129 100644 --- a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py +++ b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py @@ -16,6 +16,8 @@ import os import re +import config as cfg +import units import job_monitor as jm import log_collector as lc @@ -25,11 +27,35 @@ # the block the chart/code split actually bit on. SETS = ('monitor.profileConfigMap=p',) -MODULES = { - art.MONITOR_CONTAINER: ('job_monitor', jm), - art.COLLECTOR_CONTAINER: ('log_collector', lc), +# Every module whose code runs in the container, in the order a name is looked +# for. Both containers read config.py -- the collector's own knobs are its own, +# but NAMESPACE, RUN_NAME, LOG_DIR, STORAGE_MODE and SAVE_SUCCESS_LOGS are +# shared, and were declared in both files until they were not. +CONTAINERS = { + art.MONITOR_CONTAINER: (('config', cfg), ('job_monitor', jm)), + art.COLLECTOR_CONTAINER: (('log_collector', lc), ('config', cfg)), } +READERS = {c: tuple(m for _, m in mods) for c, mods in CONTAINERS.items()} + + +def _bindings(cname): + """env var -> (module_name, constant), across every module in the container.""" + out = {} + for module_name, module in CONTAINERS[cname]: + for env, constant in art.env_bindings(art.module_source(module)).items(): + out.setdefault(env, (module_name, constant)) + return out + + +def _code_defaults(cname): + """The built-in defaults of every module in the container, merged.""" + out = {} + for module_name, _ in CONTAINERS[cname]: + for name, value in art.defaults(module_name).items(): + out.setdefault((module_name, name), value) + return out + # Env vars whose chart value is deliberately NOT the code default. Each one is # either per-release, per-mission, or a run parameter the mission overrides; in # every case the code fallback exists only so the module can be imported @@ -46,10 +72,11 @@ 'ATTEMPT_DEADLINE_SECONDS': 'a backstop the chart turns on and the code leaves off', # StellarKubeSpecs.fs owns worker sizing, so the chart ships these empty on # purpose and the mission fills them in on every install. - # The code default is 'off' so a bare import stays inert, but nothing in the - # F# ever sets this -- the chart value IS the configuration, and shipping it - # empty silently drops every worker to the flat REQ_CPU. - 'PROFILE_CPU_TIERS': 'code defaults to off; the chart is the only thing that enables tiering', + # Pool routing is opt-in: an empty prefix is exactly the pre-tier behaviour, + # and the mission turns the whole thing on by setting only this. The ladder + # itself ships defined -- see test_the_chart_ships_a_coherent_pool_ladder. + 'POOL_PREFIX': 'empty ships pooling off; the mission sets it to opt in', + 'CAPACITY_TYPE': 'empty means no capacity constraint; the mission derives it from storage mode', 'REQ_CPU': 'left empty in the chart; StellarKubeSpecs.fs supplies it', 'REQ_MEM': 'left empty in the chart; StellarKubeSpecs.fs supplies it', } @@ -76,15 +103,15 @@ def _pairs(): """(container, env, chart_value, constant, code_default) for each env set.""" out = [] for cname, container in art.containers(SETS).items(): - module_name, module = MODULES[cname] - bindings = art.env_bindings(art.module_source(module)) - code = art.defaults(module_name) + bindings = _bindings(cname) + code = _code_defaults(cname) for env, chart_value in art.env_of(container).items(): if chart_value is None: continue # valueFrom: the chart picks nothing - constant = bindings.get(env) + found = bindings.get(env) + constant = found[1] if found else None out.append((cname, env, chart_value, constant, - code.get(constant) if constant else None)) + code.get(found) if found else None)) return out @@ -110,13 +137,16 @@ def test_no_pinned_default_is_a_constant_nothing_reads(): """ dead = [] for cname, container in art.containers(SETS).items(): - module_name, module = MODULES[cname] - source = art.module_source(module) - bindings = art.env_bindings(source) + bindings = _bindings(cname) + # Uses are counted across every module in the container, not just the one + # holding the assignment: config.py defines these and job_monitor reads + # them, and several are named differently from their env var. + source = '\n'.join(art.module_source(m) for m in READERS[cname]) for env in art.env_of(container): - constant = bindings.get(env) - if constant is None: + found = bindings.get(env) + if found is None: continue + module_name, constant = found uses = len(re.findall(rf"\b{constant}\b", source)) if uses < 2: dead.append(f"{module_name}.{constant} (from {env})") @@ -195,16 +225,16 @@ def test_the_sizing_headroom_is_a_real_allowance_in_both_places(): says what they must remain true of, so retuning them stays possible and zeroing them does not. """ - code = art.defaults('job_monitor') + code = art.defaults('config') assert code['PROFILE_MARGIN'] >= 1.0, "a margin below 1.0 sizes under the measured peak" - headroom = jm._quantity_bytes(code['PROFILE_CACHE_HEADROOM']) + headroom = units.quantity_bytes(code['PROFILE_CACHE_HEADROOM']) assert headroom >= 256 * 1024 ** 2, ( f"{code['PROFILE_CACHE_HEADROOM']} of fixed headroom is what OOMed 90 small ranges") assert code['PROFILE_RUNTIME_MEMORY_INSURANCE'] == '3Gi' # ...and the ceiling has to sit above the configured request, or a range # measured above it can never ask for what it actually uses and will pack as # though it were small. - assert (jm._quantity_bytes(code['PROFILE_MAX_MEM']) - > jm._quantity_bytes(code['REQ_MEM'])), ( + assert (units.quantity_bytes(code['PROFILE_MAX_MEM']) + > units.quantity_bytes(code['REQ_MEM'])), ( "the profile ceiling is at or below the configured request, so a hungry " "range can never ask for what it measured") diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py b/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py index 267990d6..452a8584 100644 --- a/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py +++ b/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py @@ -12,6 +12,9 @@ present. """ +from pathlib import Path + +import config import job_monitor as jm import log_collector as lc @@ -24,8 +27,7 @@ # Operator-facing switches with no chart key on purpose: they are set by hand on # a running Deployment when something needs debugging, and a chart key would # freeze them at install time. -DEBUG_ONLY = {'LOGGING_LEVEL', 'WATCH_STALE_SECONDS', 'CONNECTION_POOL', - 'WORKER_CONTAINER'} +DEBUG_ONLY = {'LOGGING_LEVEL', 'CONNECTION_POOL', 'WORKER_CONTAINER'} # Rendered with everything the mission can send, so the conditional blocks are # present: a profile ConfigMap, a required node label, an avoided node label @@ -33,7 +35,6 @@ # by no template at all -- absent from here, that stays invisible. FULL = ( 'monitor.profileConfigMap=p', - 'integration.syntheticWorker.enabled=true', 'worker.requireNodeLabels[0].key=purpose', 'worker.requireNodeLabels[0].operator=In', 'worker.requireNodeLabels[0].values[0]=catchup8-spot', @@ -61,13 +62,12 @@ def test_every_env_the_collector_reads_is_set_on_the_collector_container(): assert not missing, f"the collector reads {missing} but the chart never sets them" -def test_liveness_sampler_settings_reach_only_the_monitor(): +def test_liveness_sweep_settings_reach_only_the_monitor(): monitor = art.env_of(art.containers()[art.MONITOR_CONTAINER]) collector = art.env_of(art.containers()[art.COLLECTOR_CONTAINER]) expected = { - 'LIVENESS_PROBE_INTERVAL_SECONDS': '30', 'LIVENESS_PROBE_TIMEOUT_SECONDS': '5', - 'LIVENESS_FAILURE_THRESHOLD': '3', + 'LIVENESS_SWEEP_SECONDS': '15', 'LIVENESS_MAX_CONCURRENCY': '32', } assert {name: monitor.get(name) for name in expected} == expected @@ -99,7 +99,7 @@ def test_node_targeting_is_absent_rather_than_empty_when_unset(): env = art.env_of(art.containers()[art.MONITOR_CONTAINER]) for name in ('NODE_LABEL_KEY', 'NODE_LABEL_VALUE', 'TOLERATE_TAINT'): assert env.get(name, '') == '', f"{name} rendered without a value to carry" - assert art.defaults('job_monitor')['NODE_LABEL_KEY'] == '', \ + assert art.defaults('config')['NODE_LABEL_KEY'] == '', \ "the code fallback must be the falsy 'no targeting' value" @@ -155,7 +155,7 @@ def test_the_run_name_the_monitor_labels_with_is_the_helm_release(): collector = art.env_of(art.containers(release='pc-abc')[art.COLLECTOR_CONTAINER]) assert collector['RUN_NAME'] == 'pc-abc', \ "the collector would watch a different run's pods" - assert jm.LABEL_RUN == lc.LABEL_RUN, \ + assert config.LABEL_RUN == config.LABEL_RUN, \ "the two processes select on different label keys" @@ -200,26 +200,42 @@ def test_no_container_declares_the_same_env_var_twice(): "the API server rejects the Deployment outright") -def test_the_chart_ships_cpu_tiering_switched_on(): - """An empty PROFILE_CPU_TIERS is a silent 2x cost regression, not a no-op. +def test_the_chart_ships_a_coherent_pool_ladder(): + """The ladder must be defined even though pooling ships OFF. + + poolPrefix empty is deliberate -- pool routing is opt-in, and an unset + prefix is exactly the pre-tier behaviour. But the ladder itself has to be + present and well-formed, because the mission turns pooling on by setting + only the prefix: a malformed or out-of-order poolTiers would then route + ranges to tiers whose nodes cannot hold them, which is an OOM per range + rather than a slow run. - Nothing in MissionHistoryPubnetParallelCatchupV2.fs sets this value, so the - chart default IS the configuration. Left empty, every worker falls back to - the flat REQ_CPU: measured 2026-07-31, that issued 1250m to all 2048 pods -- - 2560 vCPU of requests against a 2304 spot quota -- and packed 6 workers per - node where tiering gets 14. + Cuts must ascend. A descending pair would make an earlier tier shadow a + later one and every range past the inversion would land one tier too low -- + measured consequence: a 13.75Gi range on a 14.1Gi node OOMKilled during + bucket-apply, before closing a single ledger. """ - # Rendered with FULL: the tier env lives inside the profileConfigMap block, - # which is correct -- tiering is keyed on measured runtimes, so it only - # applies when a profile is actually mounted. - tiers = art.env_of(art.containers(FULL)[art.MONITOR_CONTAINER]).get('PROFILE_CPU_TIERS') - assert tiers, "the chart ships cpu tiering disabled; every worker will request REQ_CPU" - pairs = [t.split(':') for t in tiers.split(',')] - pcts = [float(p) for p, _ in pairs] - cpus = [float(c) for _, c in pairs] - assert pcts == sorted(pcts), f"tier percentiles out of order: {tiers}" - assert cpus == sorted(cpus), f"tier cpu values out of order: {tiers}" - assert pcts[-1] == 100, f"tiers must cover the top percentile: {tiers}" + env = art.env_of(art.containers(FULL)[art.MONITOR_CONTAINER]) + tiers = env.get('POOL_TIERS') + assert tiers, "the chart ships no pool ladder; enabling poolPrefix would route nowhere" + parsed = [] + for item in tiers.split(','): + cut, _, name = item.rpartition(':') + assert name, f"tier entry with no name: {item!r}" + parsed.append((float(cut) if cut else float('inf'), name)) + cuts = [c for c, _ in parsed] + assert cuts == sorted(cuts), f"pool cuts out of order: {tiers}" + assert cuts[-1] == float('inf'), ( + "the last tier must be unbounded, or a range above the final cut has " + f"nowhere to go: {tiers}") + # Every tier that can be routed to needs a cpu claim, or the pod keeps the + # flat REQ_CPU and two of them can share a node -- which defeats the whole + # design: isolating a pod from its neighbours raised throughput 29-92%. + claims = dict(item.split(':') for item in env['POOL_CPU'].split(',')) + for _, name in parsed: + assert name in claims, f"tier {name} has no cpu claim in POOL_CPU" + for extra in (env['POOL_UNPROFILED'], env['POOL_NO_PROFILE']): + assert extra in claims, f"off-ladder pool {extra} has no cpu claim" def test_neither_module_defines_the_same_symbol_twice(): @@ -234,13 +250,50 @@ def test_neither_module_defines_the_same_symbol_twice(): reason it was benign is that the copies happened to be identical; had the merge taken one edited copy and one stale one, the stale one would silently have won or lost depending on file order. + + Assignments count too, and only because this test missed one: the same merge + left two `worker_liveness_sampler = WorkerLivenessSampler()` lines with a + function between them. The first instance was constructed and thrown away -- + inert only because __init__ starts no thread. + """ + import ast, collections + for f in sorted(list(Path(art.APPS_DIR).glob('*.py')) + + list(Path(art.LIB_DIR).glob('*.py'))): + tree = ast.parse(f.read_text()) + seen = collections.defaultdict(list) + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + seen[node.name].append(False) # a def is never a coercion + elif isinstance(node, ast.Assign) and len(node.targets) == 1 \ + and isinstance(node.targets[0], ast.Name): + name = node.targets[0].id + # `X = int(X)` is a coercion of the value above it, not a second + # definition -- config.py does this to every liveness knob. + coercion = any(isinstance(x, ast.Name) and x.id == name + for x in ast.walk(node.value)) + seen[name].append(coercion) + dupes = sorted(n for n, hits in seen.items() + if len(hits) > 1 and not all(hits[1:])) + assert not dupes, \ + f"{f.name} defines {dupes} more than once; the later one silently wins" + + +def test_the_chart_never_pins_an_absolute_interpreter_path(): + """A container command must resolve python on PATH, not at /usr/bin. + + Dockerfile.jobmonitor builds on python:3.12-slim, which ships the + interpreter at /usr/local/bin/python3. `/usr/bin/python3` shipped in the + collector's command and failed as StartError -- and the failure is + asymmetric, so it hides: the monitor container inherits the image CMD and + comes up healthy while only the sidecar crashloops, which reads as a sidecar + bug rather than a chart/base-image mismatch. Observed on ssc-test + 2026-08-07, 3 restarts before it was caught. """ - import ast, collections, pathlib - here = pathlib.Path(__file__).resolve().parents[2] - for name in ('job_monitor.py', 'log_collector.py'): - tree = ast.parse((here / name).read_text()) - seen = collections.Counter( - node.name for node in tree.body - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))) - dupes = sorted(n for n, k in seen.items() if k > 1) - assert not dupes, f"{name} defines {dupes} more than once; the later one silently wins" + chart = Path(__file__).resolve().parents[2] / 'parallel_catchup_helm' + for path in chart.rglob('*.yaml'): + for n, line in enumerate(path.read_text().splitlines(), 1): + if 'command:' not in line or line.lstrip().startswith('#'): + continue + assert '/usr/bin/python' not in line and '/usr/local/bin/python' not in line, ( + f"{path.name}:{n} pins an absolute interpreter path, which ties " + f"the chart to one base image: {line.strip()}") diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_rbac.py b/src/MissionParallelCatchup/tests/contract/test_chart_rbac.py index 86f45157..d6a109d9 100644 --- a/src/MissionParallelCatchup/tests/contract/test_chart_rbac.py +++ b/src/MissionParallelCatchup/tests/contract/test_chart_rbac.py @@ -83,7 +83,7 @@ def test_the_role_grants_what_the_collector_reads(): """ source = art.module_source(lc) have = art.granted() - assert re.search(r"/api/v1/namespaces/\{NAMESPACE\}/pods\"", source), \ + assert re.search(r"/api/v1/namespaces/\{config\.NAMESPACE\}/pods\"", source), \ "the collector no longer lists pods -- update this test" assert 'list' in have[('', 'pods')] assert re.search(r"/pods/\{pod\}/log\"", source), \ diff --git a/src/MissionParallelCatchup/tests/contract/test_cross_process_files.py b/src/MissionParallelCatchup/tests/contract/test_cross_process_files.py index 1548a948..a45c282d 100644 --- a/src/MissionParallelCatchup/tests/contract/test_cross_process_files.py +++ b/src/MissionParallelCatchup/tests/contract/test_cross_process_files.py @@ -16,6 +16,9 @@ import pytest +import config +import records +import attempts import job_monitor as jm import log_collector as lc @@ -27,8 +30,7 @@ @pytest.fixture def shared(tmp_path, monkeypatch): """Both modules pointed at one directory, as the pod's volume gives them.""" - monkeypatch.setattr(jm, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) return tmp_path @@ -36,7 +38,7 @@ def shared(tmp_path, monkeypatch): def test_both_processes_name_the_same_metrics_file(shared): """The collector writes it; the monitor reads peaks and txApply out of it.""" - assert jm.metrics_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.metrics' + assert records.metrics_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.metrics' def test_both_processes_name_the_same_done_marker(shared): @@ -46,15 +48,15 @@ def test_both_processes_name_the_same_done_marker(shared): which is the last place peaks can still be read from. A mismatch means the monitor never reaps and every Job waits out its TTL instead. """ - assert jm.done_path(END, ATTEMPT) == lc.done_path(END, ATTEMPT) + assert records.done_path(END, ATTEMPT) == lc.done_path(END, ATTEMPT) def test_both_processes_name_the_same_archive_and_verdict(shared): """The monitor falls back to the archive for txApply and reads .outcome for the authoritative verdict; the collector writes both.""" - assert jm.log_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.log.gz' - assert jm.outcome_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.outcome' - assert jm.state_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.state' + assert records.log_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.log.gz' + assert records.outcome_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.outcome' + assert records.state_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.state' def test_the_filenames_carry_the_attempt_as_well_as_the_range(shared): @@ -64,7 +66,7 @@ def test_the_filenames_carry_the_attempt_as_well_as_the_range(shared): being compared against it -- which destroys exactly the OOM evidence the chain exists to keep. """ - for path in (jm.metrics_path, jm.log_path, jm.outcome_path, jm.done_path): + for path in (records.metrics_path, records.log_path, records.outcome_path, records.done_path): assert path(END, 1) != path(END, 2) assert path('1', 1) != path('2', 1) @@ -81,24 +83,25 @@ def test_discarding_a_successful_archive_keeps_what_is_still_read(shared): fh.write('x') lc.discard(END, ATTEMPT) - assert os.path.exists(jm.metrics_path(END, ATTEMPT)), "discard dropped the measurements" - assert os.path.exists(jm.done_path(END, ATTEMPT)), "discard dropped the reap marker" - assert not os.path.exists(jm.log_path(END, ATTEMPT)), "discard kept the archive" + assert os.path.exists(records.metrics_path(END, ATTEMPT)), "discard dropped the measurements" + assert os.path.exists(records.done_path(END, ATTEMPT)), "discard dropped the reap marker" + assert not os.path.exists(records.log_path(END, ATTEMPT)), "discard kept the archive" def test_the_monitor_can_read_an_archive_the_collector_wrote(shared): """gzip, appended member by member, read whole. - The monitor reads it with gzip.open() for the txApply fallback. A writer - that produced anything other than a concatenation of complete members would - give it a truncated read -- which it treats as "this range has no metric". + The monitor reads it with gzip.open() to decide whether an exit 3 was a + fetch fault. A writer that produced anything other than a concatenation of + complete members would give it a truncated read -- which it treats as no + evidence, and no evidence condemns the range. """ path = lc.base(END, ATTEMPT) + '.log.gz' - for chunk in ("first line\n", "metric 'ledger.transaction.apply'\n", - " sum = 8.34285ms\n"): + for chunk in ("first line\n", "second line\n", "last line\n"): with gzip.open(path, 'ab') as fh: fh.write(chunk.encode()) - assert jm._tx_apply_for_attempt(END, ATTEMPT) == pytest.approx(0.00834285) + assert [l.strip() for l in attempts._archive_tail(END, ATTEMPT)] == [ + 'first line', 'second line', 'last line'] def test_a_carriage_return_meter_does_not_become_one_giant_line(shared, monkeypatch): @@ -156,8 +159,9 @@ def test_both_containers_mount_one_volume_at_the_directory_they_both_use(): at a different path in each container would give each process its own private copy of every measurement. """ - log_dir = art.defaults('job_monitor')['LOG_DIR'] - assert log_dir == art.defaults('log_collector')['LOG_DIR'] + # One constant read through config by both processes now: they cannot + # disagree about the directory, only about mounting it. + log_dir = art.defaults('config')['LOG_DIR'] mounts = {} for name, container in art.containers().items(): @@ -176,7 +180,7 @@ def test_both_containers_mount_one_volume_at_the_directory_they_both_use(): def test_the_chart_tells_both_containers_where_that_directory_is(): """LOG_DIR is env, not a constant, so the mount and the env must agree.""" - log_dir = art.defaults('job_monitor')['LOG_DIR'] + log_dir = art.defaults('config')['LOG_DIR'] for name, container in art.containers().items(): assert art.env_of(container)['LOG_DIR'] == log_dir, name @@ -189,7 +193,7 @@ def test_the_progress_record_lives_on_that_volume_too(): would not survive a monitor restart and the mission would build its range profile from the ConfigMap mirror -- which has every measurement stripped. """ - assert os.path.dirname(jm.PROGRESS_FILE) == jm.LOG_DIR + assert os.path.dirname(config.PROGRESS_FILE) == config.LOG_DIR def test_the_shared_directory_is_a_single_writer_pvc(): diff --git a/src/MissionParallelCatchup/tests/contract/test_dependency_pins.py b/src/MissionParallelCatchup/tests/contract/test_dependency_pins.py new file mode 100644 index 00000000..b48f36f4 --- /dev/null +++ b/src/MissionParallelCatchup/tests/contract/test_dependency_pins.py @@ -0,0 +1,79 @@ +"""The dependency pins, in the three places they are written and the one that runs. + +The image installs them, and the dev path installs them again at container start +from the chart -- the same list, typed twice more. A divergence there means the +sourceConfigMap run and the built image are different programs. + +The test environment counts too: the suite ran against kubernetes 36.0.3 for +months while the image pinned ~=35.0, so every contract test that builds a real +V1* model was checking the wrong major. That is invisible until a model differs. +""" + +import os +import re + +import _artifacts as art + +DOCKERFILE = os.path.join(art.MODULE_DIR, 'Dockerfile.jobmonitor') + +# name -> the specifier both artifacts must agree on. +_SPEC = re.compile(r"'([a-z0-9-]+)(~=[0-9.]+)'") + + +def _dockerfile_pins(): + text = open(DOCKERFILE).read() + install = text[text.index('RUN pip install'):] + install = install[:install.index('\nCOPY')] + return dict(_SPEC.findall(install)) + + +def _chart_pins(): + """Every `pip install` the chart renders, one dict per occurrence.""" + text = art.text(os.path.join(art.CHART, 'templates', 'job_monitor.yaml')) + out = [] + for line in re.findall(r'pip install --no-cache-dir -q (.+?)&&', text, re.S): + out.append(dict(_SPEC.findall(line))) + return out + + +def test_the_chart_installs_exactly_what_the_image_pins(): + image = _dockerfile_pins() + assert image, "no pins found in the Dockerfile -- the parser is stale" + for n, chart in enumerate(_chart_pins()): + assert chart == image, ( + f"chart pip install #{n + 1} differs from the image: " + f"chart={chart} image={image}") + + +def test_both_containers_install_the_same_list(): + lists = _chart_pins() + assert len(lists) == 2, f"expected one pip install per container, found {len(lists)}" + assert lists[0] == lists[1], f"the two containers install different deps: {lists}" + + +def test_the_test_environment_satisfies_the_pins(): + """What the suite imports must be what the image would install. + + Not a style check: the contract tests construct real V1* models, so a major + the image never installs makes those assertions about a client that does not + ship. + """ + import importlib.metadata as md + drift = [] + for name, spec in _dockerfile_pins().items(): + try: + installed = md.version(name) + except md.PackageNotFoundError: + continue # not needed to run the suite + pinned = spec[2:].split('.')[0] + if installed.split('.')[0] != pinned: + drift.append(f"{name}: pinned {spec}, test env has {installed}") + assert not drift, "the suite is running against a different major:\n " + "\n ".join(drift) + + +def test_the_client_ships_the_async_api_the_pin_was_raised_for(): + """36 was chosen over 35 for kubernetes.aio, which 35 does not contain.""" + import kubernetes.aio # noqa: F401 + from kubernetes.aio import client + assert hasattr(client, 'V1PodFailurePolicyRule'), \ + "the async client must carry the same models the sync one does" diff --git a/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py b/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py index a0a383a9..89773141 100644 --- a/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py +++ b/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py @@ -22,6 +22,11 @@ import pytest +import config +import units +import profiles +import records +import sizing import job_monitor as jm import log_collector as lc @@ -70,14 +75,20 @@ def test_every_helm_command_uses_the_mission_namespace(): kubeconfig default, poll sandbox through the client, and wait forever for a monitor that exists in another namespace. """ - blocks = re.findall(r'RunShellCommand\s+\[\|\s*"helm"(.*?)\|\]', FS, re.S) - assert len(blocks) == 4, ( - f"expected install, get-values and two cleanup commands; found {len(blocks)}") + # Split on the call itself rather than matching one array literal: the install + # builds its argv with Array.concat so it can add a second --values for + # on-demand, and a `[| "helm" ... |]` pattern silently stopped seeing it. + calls = [seg for seg in re.split(r'\bRunShellCommand\b', FS)[1:] + if re.match(r'[\s(]*(?:Array\.concat\s*\[\s*)?\[\|\s*"helm"', seg)] + assert len(calls) == 4, ( + f"expected install, get-values and two cleanup commands; found {len(calls)}") + blocks = calls for block in blocks: verb = re.search(r'"(install|get|upgrade|uninstall)"', block) assert verb, f"could not identify Helm command in {block!r}" + # F# array elements separate with a newline or a semicolon; both appear assert re.search( - r'"--namespace"\s+context\.namespaceProperty', block), ( + r'"--namespace"\s*;?\s*context\.namespaceProperty', block), ( f"helm {verb.group(1)} does not target the mission namespace: {block!r}") @@ -252,7 +263,7 @@ def test_the_driver_reads_the_configmap_the_monitor_writes(): # the latter imported with the RUN_NAME the chart gives it for that release. wanted = release + suffix run_name = art.env_of(art.containers(release=release)[art.MONITOR_CONTAINER])['RUN_NAME'] - written = art.defaults('job_monitor', (('RUN_NAME', run_name),))['PROGRESS_CM'] + written = art.defaults('config', (('RUN_NAME', run_name),))['PROGRESS_CM'] assert wanted == written, f"driver reads {wanted!r}, monitor writes {written!r}" @@ -269,14 +280,14 @@ def test_the_driver_reads_the_keys_the_monitor_publishes(cluster): cluster.advance(300, 'succeeded') cluster.reconcile() # records a completion -> progress.json jm.save_status(jm.status) # what the reconcile loop publishes - published = set(cluster.k8s.config_map_data(jm.PROGRESS_CM, cluster.namespace) or {}) + published = set(cluster.k8s.config_map_data(config.PROGRESS_CM, cluster.namespace) or {}) missing = sorted(wanted - published) assert not missing, ( f"the driver reads {missing}; the monitor published {sorted(published)}") def test_every_status_field_the_driver_reads_is_one_the_monitor_sets(): - """The driver's loop terminates on num_remain and jobs_in_progress. + """The driver's loop terminates on num_remain and queue_in_progress_count. A field it reads that the monitor never sets throws inside the polling loop, which the driver treats as fatal: cleanup, uninstall, mission failed -- with @@ -322,14 +333,14 @@ def test_the_driver_execs_into_a_container_that_exists(): def test_the_driver_reads_the_progress_file_where_the_monitor_writes_it(): """`cat /logs/progress.json`, hard-coded on the driver side.""" path = fs_extract(r'command = \[\| "cat"; "([^"]+)" \|\]').group(1) - assert path == jm.PROGRESS_FILE, ( - f"the driver cats {path}; the monitor writes {jm.PROGRESS_FILE}") + assert path == config.PROGRESS_FILE, ( + f"the driver cats {path}; the monitor writes {config.PROGRESS_FILE}") def test_the_driver_tars_the_directory_the_collector_writes_into(): """One exec replaces the ~1024 the StatefulSet design needed.""" cd = fs_extract(r'"cd (/\w+) && tar').group(1) - assert cd == jm.LOG_DIR == lc.LOG_DIR + assert cd == config.LOG_DIR == config.LOG_DIR def test_the_tar_excludes_only_the_collectors_resume_bookkeeping(): @@ -342,9 +353,9 @@ def test_the_tar_excludes_only_the_collectors_resume_bookkeeping(): def suffix_of(path_fn): return os.path.basename(path_fn('E', 1)).partition('-a1')[2] - bookkeeping = {suffix_of(jm.state_path)} - deliverables = {suffix_of(f) for f in (jm.log_path, jm.metrics_path, - jm.outcome_path, jm.done_path)} + bookkeeping = {suffix_of(records.state_path)} + deliverables = {suffix_of(f) for f in (records.log_path, records.metrics_path, + records.outcome_path, records.done_path)} assert bookkeeping.isdisjoint(deliverables) excludes = set(re.findall(r"--exclude='([^']+)'", FS)) @@ -386,26 +397,44 @@ def fs_document_keys(): return set(re.findall(r'doc\.\["(\w+)"\]\s*<-', FS)) -def test_the_artifact_carries_exactly_the_fields_the_mirror_strips(): - """Two lists that must be one list. +# Recorded but deliberately not carried into the artifact: they are Prometheus +# metrics, and nothing in the next run sizes or orders from them -- wallSeconds +# alone was 349 KB of a 963 KB artifact. Listed rather than inferred so that +# dropping a field the artifact DOES need still fails the test below. +NOT_PROFILED = {'wallSeconds', 'txApply'} - The monitor strips _PROFILE_ONLY_FIELDS out of the ConfigMap mirror to stay - under its 1 MiB cap, so those fields exist only in the volume copy -- which - is precisely the copy the driver projects into the artifact. A field in one - list and not the other is either lost from the artifact or bloating the - mirror. peakAnonBytes was missing from the projection: measured 2026-07-30, - the artifact carried it for 0% of ranges while the volume copy had it for - 99%. + +def test_the_artifact_carries_every_measurement_the_record_holds(cluster): + """Two lists that must agree, in one direction each. + + Anchored on a record the real monitor wrote, so neither side can drift by + editing a constant. A field the driver projects but the record never holds + lands in the artifact as null; a field the record holds and the driver drops + is lost from the artifact -- peakAnonBytes was exactly that, carried for 0% + of ranges while the volume copy had it for 99%. The only permitted asymmetry + is NOT_PROFILED, enumerated above. """ - assert fs_profile_fields() == set(jm._PROFILE_ONLY_FIELDS), ( - "driver projects " - f"{sorted(fs_profile_fields() - set(jm._PROFILE_ONLY_FIELDS))} extra, " - f"drops {sorted(set(jm._PROFILE_ONLY_FIELDS) - fs_profile_fields())}") + cluster.reconcile() + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1, tx_apply=1.5, + peaks={'peakAnonBytes': 7, 'peakWorkingSetBytes': 9, + 'peakEphemeralBytes': 11}) + cluster.reconcile() + + measured = set(cluster.completed()['300']) - {'attempts', 'count'} + assert measured, "the monitor recorded no measurements at all" + assert not fs_profile_fields() - measured, ( + f"the driver projects {sorted(fs_profile_fields() - measured)}, which no " + "completion record carries") + assert measured - fs_profile_fields() == NOT_PROFILED, ( + "the artifact drops " + f"{sorted(measured - fs_profile_fields() - NOT_PROFILED)} " + "without that being a deliberate choice recorded in NOT_PROFILED") def test_every_field_the_sizing_consumer_reads_is_in_the_artifact(): """Derived from _profile_overrides, so a new sizing input fails here first.""" - consumed = set(re.findall(r"prof\.get\('(\w+)'\)", art.module_source(jm))) + consumed = set(re.findall(r"prof\.get\('(\w+)'\)", art.module_source(sizing))) assert consumed, "the sizing consumer no longer reads named fields" missing = sorted(consumed - fs_profile_fields()) assert not missing, f"the profile is sized from {missing}, which the artifact drops" @@ -413,7 +442,7 @@ def test_every_field_the_sizing_consumer_reads_is_in_the_artifact(): def test_every_document_key_the_monitor_reads_is_one_the_driver_writes(): """storageMode decides whether the disk axis is usable; ranges is the data.""" - read = set(re.findall(r"doc\.get\('(\w+)'\)", art.module_source(jm))) + read = set(re.findall(r"doc\.get\('(\w+)'\)", art.module_source(profiles))) assert read, "load_profile no longer reads named document keys" missing = sorted(read - fs_document_keys()) assert not missing, f"load_profile reads {missing}, which the driver never writes" @@ -439,16 +468,16 @@ def test_an_artifact_from_a_previous_run_loads_and_sizes_the_next_one(tmp_path, path = tmp_path / 'profile.json' path.write_text(json.dumps(_artifact(ranges={ '16752063': {'peakAnonBytes': 2 * 1024 ** 3, 'peakWorkingSetBytes': 13 * 1024 ** 3, - 'txApply': 900.0, 'seconds': 1200.0, 'count': 16320}}))) + 'seconds': 1200.0, 'count': 16320}}))) - monkeypatch.setattr(jm, 'PROFILE_PATH', str(path)) - monkeypatch.setattr(jm, 'STORAGE_MODE', 'pvc') - monkeypatch.setattr(jm, 'PROFILE', jm.load_profile()) - assert jm.PROFILE, "the driver's artifact did not load at all" + monkeypatch.setattr(config, 'PROFILE_PATH', str(path)) + monkeypatch.setattr(config, 'STORAGE_MODE', 'pvc') + monkeypatch.setattr(config, 'PROFILE', profiles.load_profile()) + assert config.PROFILE, "the driver's artifact did not load at all" - sized = jm._profile_overrides(16752063, escalated=False) + sized = sizing._profile_overrides(16752063, escalated=False) assert 'memory' in sized, "a measured range was not sized from the artifact" - assert (jm._quantity_bytes(sized['memory']) > 2 * 1024 ** 3), \ + assert (units.quantity_bytes(sized['memory']) > 2 * 1024 ** 3), \ "the request came out below the measured peak" @@ -464,12 +493,12 @@ def test_a_cross_mode_artifact_keeps_memory_and_drops_the_disk_axis(tmp_path, mo '16752063': {'peakAnonBytes': 2 * 1024 ** 3, 'peakEphemeralBytes': 30 * 1024 ** 3, 'count': 16320}}))) - monkeypatch.setattr(jm, 'PROFILE_PATH', str(path)) - monkeypatch.setattr(jm, 'STORAGE_MODE', 'pvc') - monkeypatch.setattr(jm, 'LIM_EPHEMERAL', '40Gi') - monkeypatch.setattr(jm, 'PROFILE', jm.load_profile()) + monkeypatch.setattr(config, 'PROFILE_PATH', str(path)) + monkeypatch.setattr(config, 'STORAGE_MODE', 'pvc') + monkeypatch.setattr(config, 'LIM_EPHEMERAL', '40Gi') + monkeypatch.setattr(config, 'PROFILE', profiles.load_profile()) - sized = jm._profile_overrides(16752063, escalated=False) + sized = sizing._profile_overrides(16752063, escalated=False) assert 'memory' in sized, "a cross-mode profile was rejected outright" assert 'ephemeral-storage' not in sized, "a pvc run was sized from ephemeral-mode disk" @@ -488,15 +517,15 @@ def test_an_empty_artifact_is_never_written_and_never_fatal(tmp_path, monkeypatc path = tmp_path / 'profile.json' path.write_text(json.dumps(_artifact(ranges={}))) - monkeypatch.setattr(jm, 'PROFILE_PATH', str(path)) - monkeypatch.setattr(jm, 'STORAGE_MODE', 'pvc') - monkeypatch.setattr(jm, 'PROFILE', jm.load_profile()) - assert jm.PROFILE == [], "an empty profile loaded as if it held something" - assert jm._profile_overrides(16752063, escalated=False) == {} + monkeypatch.setattr(config, 'PROFILE_PATH', str(path)) + monkeypatch.setattr(config, 'STORAGE_MODE', 'pvc') + monkeypatch.setattr(config, 'PROFILE', profiles.load_profile()) + assert config.PROFILE == [], "an empty profile loaded as if it held something" + assert sizing._profile_overrides(16752063, escalated=False) == {} # ...and an artifact that never arrived at all is the same, not an error. - monkeypatch.setattr(jm, 'PROFILE_PATH', str(tmp_path / 'absent.json')) - assert jm.load_profile() == [] + monkeypatch.setattr(config, 'PROFILE_PATH', str(tmp_path / 'absent.json')) + assert profiles.load_profile() == [] def test_every_helm_and_kubectl_call_is_namespaced(): diff --git a/src/MissionParallelCatchup/tests/contract/test_k8s_failure_formats.py b/src/MissionParallelCatchup/tests/contract/test_k8s_failure_formats.py index 4ee87491..0b5a5f34 100644 --- a/src/MissionParallelCatchup/tests/contract/test_k8s_failure_formats.py +++ b/src/MissionParallelCatchup/tests/contract/test_k8s_failure_formats.py @@ -17,6 +17,7 @@ import pytest +import config import job_monitor as jm import log_collector as lc @@ -265,23 +266,29 @@ def test_every_outcome_the_classifiers_can_produce_has_a_budget(): produced = set() for source in (art.module_source(jm), art.module_source(lc)): produced |= set(re.findall(r"'outcome':\s*'(\w+)'", source)) - budgeted = set(jm.ENVIRONMENTAL_OUTCOMES) | {'oom', 'ephemeral', 'timeout', 'failed'} - assert produced <= budgeted, f"unrouted outcomes: {sorted(produced - budgeted)}" + assert produced <= set(config.ATTEMPT_OUTCOMES), ( + f"outcomes no budget or verdict file knows about: " + f"{sorted(produced - set(config.ATTEMPT_OUTCOMES))}") assert 'disrupted' in produced and 'rejected' in produced def test_the_deterministic_failures_do_not_get_the_environmental_budget(): - """Environmental means "the cluster did this to us" and gets ~20 attempts. + """Only a node disruption gets the effectively unbounded budget. - An OOM, a disk eviction, a hang and a genuinely corrupt range are all - statements about the range; giving them 20 attempts would park a node on a - broken range for hours. 'unknown' IS environmental on purpose -- an - unclassifiable failure is usually a monitor restart racing a reaped node. + It is the one outcome that proves the range itself was fine: the cluster took + the pod away mid-run. Everything else is either a statement about the range + (OOM, disk, hang) or an absence of evidence, and neither earns unlimited + retries -- a run that cannot explain a failure stops instead. """ - environmental = set(jm.ENVIRONMENTAL_OUTCOMES) - assert not (environmental & {'oom', 'ephemeral', 'timeout', 'failed'}), \ - f"a deterministic failure inherited the disruption budget: {sorted(environmental)}" - assert {'disrupted', 'rejected', 'unknown'} <= environmental + b = config.ATTEMPT_BUDGETS + # A ladder, from "this range is broken" to "the cluster did this to us". + assert b['ephemeral'] <= b['oom'] < b['fetch-fault'] <= b['rejected'] <= b['disrupted'], \ + f"the budget ladder is out of order: {b}" + assert b['disrupted'] == max(b.values()), \ + f"something other than a disruption got the largest budget: {b}" + assert not ({'timeout', 'unknown', 'failed'} & set(b)), \ + "a hang, an unclassifiable failure and a real catchup failure must have "\ + "no budget at all" # --- the log endpoint, which does not always return log lines ---------------- @@ -306,7 +313,7 @@ def test_a_poisoned_state_file_is_repaired_rather_than_replayed(tmp_path, monkey volumes, and nothing rewrites it until a poll succeeds -- which it cannot, because the poisoned value is what makes the poll 400. """ - monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) with open(lc.base('300', 1) + '.state', 'w') as fh: fh.write('unable') assert lc.read_state('300', 1) is None diff --git a/src/MissionParallelCatchup/tests/contract/test_medida_metric_block.py b/src/MissionParallelCatchup/tests/contract/test_medida_metric_block.py index d22a7383..b449ec34 100644 --- a/src/MissionParallelCatchup/tests/contract/test_medida_metric_block.py +++ b/src/MissionParallelCatchup/tests/contract/test_medida_metric_block.py @@ -18,6 +18,10 @@ import pytest +import config +import kube +import records +import medida import job_monitor as jm import log_collector as lc @@ -89,21 +93,14 @@ def test_the_sum_still_sits_inside_the_scan_window(): @pytest.mark.parametrize('gap', [1, 10, lc.TxApplyScanner.WINDOW, lc.TxApplyScanner.WINDOW + 5]) -def test_both_readers_reach_exactly_as_far_past_the_header(gap, tmp_path, monkeypatch): - """The collector scans the stream; the monitor scans the archive. - - Two separate implementations of "find the sum under this header". A reach - that differed between them would make the metric depend on which reader got - to it -- and the monitor's read is the one that happens when the collector - was down for the pod's lifetime. - - Asserted by measuring both, at the boundary and past it, rather than by - comparing two constants: the monitor is free to stop slicing a window and - reuse the scanner outright, which is a better implementation of the same - contract. +def test_the_scanner_reaches_exactly_as_far_past_the_header_as_it_claims(gap): + """One reader now: the collector, live and again over its own archive. + + The window is the whole contract. Measured on ssc-test 2026-08-04, a /info + response landed between the header and its sum 91 lines apart, and a span + that charged every line gave up 76 lines short of a value that was right + there -- which is why the budget counts medida statistics only. """ - monkeypatch.setattr(jm, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) block = ["metric 'ledger.transaction.apply':"] block += [f" filler {i} = 0ms" for i in range(gap - 1)] block += [" sum = 1500.0ms"] @@ -112,15 +109,12 @@ def test_both_readers_reach_exactly_as_far_past_the_header(gap, tmp_path, monkey for line in block: scanner.feed(line) - with gzip.open(jm.log_path('300', 1), 'wt') as fh: - fh.write("\n".join(block) + "\n") - from_archive = jm._tx_apply_for_attempt('300', 1) + within = gap <= lc.TxApplyScanner.WINDOW + assert (scanner.seconds is not None) is within, ( + f"a sum {gap} statistics past the header was " + f"{'missed' if within else 'read'} against a window of " + f"{lc.TxApplyScanner.WINDOW}") - assert (scanner.seconds is None) == (from_archive is None), ( - f"at {gap} lines past the header the collector says {scanner.seconds} " - f"and the monitor says {from_archive}") - if from_archive is not None: - assert from_archive == pytest.approx(scanner.seconds) # --- the number itself -------------------------------------------------------- @@ -137,7 +131,7 @@ def test_both_processes_read_the_same_total_out_of_one_block(block, seconds): first, so a disagreement is a per-range coin flip. """ assert scan(block).seconds == pytest.approx(seconds) - m = jm._SUM_RE.search(block) + m = medida.SUM_RE.search(block) assert m, "the monitor's regex does not match this block at all" assert float(m.group(1)) / 1000.0 == pytest.approx(seconds) @@ -155,7 +149,7 @@ def test_no_other_line_in_the_block_looks_like_the_sum(): latency as a whole-range total -- plausible, wrong, and unnoticeable. """ for block in (MEDIDA_BLOCK, MEDIDA_BIG): - matched = [l for l in block.splitlines() if jm._SUM_RE.search(l)] + matched = [l for l in block.splitlines() if medida.SUM_RE.search(l)] assert len(matched) == 1, f"matched {len(matched)} lines: {matched}" assert 'sum =' in matched[0] diff --git a/src/MissionParallelCatchup/tests/contract/test_module_packaging.py b/src/MissionParallelCatchup/tests/contract/test_module_packaging.py new file mode 100644 index 00000000..06c0336b --- /dev/null +++ b/src/MissionParallelCatchup/tests/contract/test_module_packaging.py @@ -0,0 +1,157 @@ +"""The repo layout must flatten into the one directory the container runs from. + +apps/ and lib/ are for reading the repo. At runtime there is a single flat /app: +the image COPYs both directories into it, and the dev path mounts a ConfigMap +built with --from-file, whose keys are basenames and cannot contain '/'. The +modules therefore import each other by bare name, and every failure guarded here +is silent in the suite and fatal in the cluster -- an import nothing ships +crash-loops the container, and two same-named files collide into one ConfigMap +key with no warning at all. +""" + +import ast +import os +import re +import sys + +import _artifacts as art + +DOCKERFILE = os.path.join(art.MODULE_DIR, 'Dockerfile.jobmonitor') + +# The two entrypoints of the image. Everything they import from this repo has to +# reach /app with them. +ENTRYPOINTS = ('job_monitor.py', 'log_collector.py') + +# Modules that must be read through rather than copied out of, and the names it +# is never safe to bind: reassigned at startup or replaced by the tests. +READ_THROUGH = ('config', 'kube') + +SOURCE_DIRS = (art.APPS_DIR, art.LIB_DIR) + + +def _py_files(directory): + return [f for f in os.listdir(directory) if f.endswith('.py')] + + +def _local_modules(): + """Everything importable from the source directories, by the name used. + + Packages count: a subdirectory is exactly what the flatness check exists to + catch, so it cannot be invisible here. + """ + names = set() + for d in SOURCE_DIRS: + names |= {f[:-3] for f in _py_files(d) if not f.startswith('_')} + names |= {e for e in os.listdir(d) + if os.path.isfile(os.path.join(d, e, '__init__.py'))} + return names + + +def _path(name): + for d in SOURCE_DIRS: + candidate = os.path.join(d, name) + if os.path.isfile(candidate): + return candidate + raise AssertionError(f"{name} is in neither apps/ nor lib/") + + +def _imports(path): + """(module, names) for every import in `path`; names is empty for plain imports.""" + with open(path) as fh: + tree = ast.parse(fh.read()) + out = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + out.extend((a.name, ()) for a in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + out.append((node.module, tuple(a.name for a in node.names))) + return out + + +def _first_party(name): + local = _local_modules() + return {m for m, _ in _imports(_path(name)) if m in local} + + +def test_the_image_ships_every_module_the_entrypoints_import(): + text = open(DOCKERFILE).read() + # A directory COPY ships everything in it; a file COPY ships just that file. + copied = set() + for target in re.findall(r'^COPY\s+\./(\S+)\s', text, re.M): + if target.endswith('/'): + copied |= set(_py_files(os.path.join(art.MODULE_DIR, target.rstrip('/')))) + else: + copied.add(os.path.basename(target)) + + needed = set(ENTRYPOINTS) + for entry in ENTRYPOINTS: + needed |= {f"{m}.py" for m in _first_party(entry)} + missing = sorted(needed - copied) + assert not missing, f"imported but never COPYd into the image: {missing}" + + +def test_the_source_directories_flatten_without_a_collision(): + """Two same-named files would become one /app file and one ConfigMap key. + + Whichever COPY ran last wins in the image, and `--from-file` silently keeps + one of the two -- so a duplicated basename is a module quietly replaced by + another, not an error anyone sees. + """ + seen = {} + for d in SOURCE_DIRS: + for f in _py_files(d): + seen.setdefault(f, []).append(os.path.relpath(d, art.MODULE_DIR)) + clashes = {f: dirs for f, dirs in seen.items() if len(dirs) > 1} + assert not clashes, f"same basename in more than one source directory: {clashes}" + + +def test_the_modules_import_each_other_by_bare_name(): + """No package-qualified import can survive the flattening. + + `from lib import config` resolves in the repo and fails in /app, where there + is no lib/ -- and it fails at container startup, long after every test here + has passed. + """ + packages = {e for d in SOURCE_DIRS for e in os.listdir(d) + if os.path.isfile(os.path.join(d, e, '__init__.py'))} + packages |= {os.path.basename(d) for d in SOURCE_DIRS} + offenders = [] + for d in SOURCE_DIRS: + for f in _py_files(d): + for module, _ in _imports(os.path.join(d, f)): + if module.split('.')[0] in packages: + offenders.append(f"{f}: import {module}") + assert not offenders, ( + "these do not resolve once apps/ and lib/ flatten into /app:\n " + + "\n ".join(offenders)) + + +def test_no_module_shadows_a_standard_library_name(): + """/app is sys.path[0], so a local name wins over the stdlib module. + + lib/profile.py was written and renamed to profiles.py for exactly this: it + would have shadowed the stdlib profiler for every module in the process, + including anything the kubernetes client imports. The failure is remote from + the cause and appears only in the container. + """ + stdlib = sys.stdlib_module_names + clashes = sorted({f[:-3] for d in SOURCE_DIRS for f in _py_files(d)} & set(stdlib)) + assert not clashes, f"these shadow a stdlib module on a flat sys.path: {clashes}" + + +def test_nothing_copies_names_out_of_the_read_through_modules(): + """`from config import REQ_CPU` binds a copy, and the copy is silently stale. + + config.PROFILE is assigned at startup and the tests monkeypatch the rest; + kube.core_v1/batch_v1 are replaced with a fake cluster. A name bound at + import time sees none of it -- the default is used, and the test passes. + """ + offenders = [] + for d in SOURCE_DIRS: + for f in _py_files(d): + for module, names in _imports(os.path.join(d, f)): + if module in READ_THROUGH and names: + offenders.append(f"{f}: from {module} import {', '.join(names)}") + assert not offenders, ( + "read these through the module (import config; config.X) instead:\n " + + "\n ".join(offenders)) diff --git a/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py b/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py index 789c0ebf..f5e95670 100644 --- a/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py +++ b/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py @@ -19,6 +19,7 @@ import pytest +import config import job_monitor as jm import log_collector as lc @@ -28,10 +29,10 @@ @pytest.fixture def job(monkeypatch): """One rendered worker Job, in the mode that needs no cluster.""" - monkeypatch.setattr(jm, 'STORAGE_MODE', 'ephemeral') - monkeypatch.setattr(jm, 'RUN_NAME', 'pc') - monkeypatch.setattr(jm, 'CORE_IMAGE', 'stellar/stellar-core:test') - monkeypatch.setattr(jm, 'PROFILE', None) + monkeypatch.setattr(config, 'STORAGE_MODE', 'ephemeral') + monkeypatch.setattr(config, 'RUN_NAME', 'pc') + monkeypatch.setattr(config, 'CORE_IMAGE', 'stellar/stellar-core:test') + monkeypatch.setattr(config, 'PROFILE', None) return jm.build_job(31005951, 16320, 2, None) @@ -57,8 +58,8 @@ def test_a_finished_job_still_has_a_ttl_backstop(job): Job listed on every later pass. The TTL is what bounds that, and it must be the value the chart configured -- not a second, independent default. """ - assert job.spec.ttl_seconds_after_finished == jm.JOB_TTL_SECONDS - assert jm.JOB_TTL_SECONDS > 0, "a TTL of 0 deletes a Job before it can be classified" + assert job.spec.ttl_seconds_after_finished == config.JOB_TTL_SECONDS + assert config.JOB_TTL_SECONDS > 0, "a TTL of 0 deletes a Job before it can be classified" def test_a_worker_pod_is_never_restarted_in_place(job): @@ -84,7 +85,7 @@ def test_the_deadline_is_on_the_job_so_it_can_be_patched_live(job, monkeypatch): of the allowance, against a pod-level field that could not be corrected at all on 850 already-running pods. """ - monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 43200) + monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', 43200) j = jm.build_job(300, 420, 1, None) assert j.spec.active_deadline_seconds == 43200, \ "the deadline must be patchable, so it belongs on the JobSpec" @@ -93,7 +94,7 @@ def test_the_deadline_is_on_the_job_so_it_can_be_patched_live(job, monkeypatch): def test_no_deadline_means_no_field_at_all(job, monkeypatch): """0 is "off". Rendering it literally would kill every pod instantly.""" - monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 0) + monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', 0) j = jm.build_job(300, 420, 1, None) assert j.spec.template.spec.active_deadline_seconds is None @@ -107,7 +108,7 @@ def test_the_grace_period_outlasts_a_stellar_core_drain(job): range that never needed any. """ grace = job.spec.template.spec.termination_grace_period_seconds - assert grace == jm.WORKER_GRACE_SECONDS + assert grace == config.WORKER_GRACE_SECONDS assert grace > 7, f"{grace}s does not cover the measured ~7s drain" @@ -213,15 +214,9 @@ def test_the_pod_carries_its_own_attempt_number(job): OOM evidence the resumed chain exists to keep. """ labels = job.spec.template.metadata.labels - assert labels[jm.LABEL_ATTEMPT] == '2' - assert labels[jm.LABEL_RANGE] == '31005951' - assert labels[jm.LABEL_RUN] == 'pc' - - -def test_both_processes_agree_on_the_label_keys(): - """Two readers, one key. A mismatch reproduces the same silent collision.""" - assert jm.LABEL_ATTEMPT == lc.LABEL_ATTEMPT - assert jm.LABEL_RUN == lc.LABEL_RUN + assert labels[config.LABEL_ATTEMPT] == '2' + assert labels[config.LABEL_RANGE] == '31005951' + assert labels[config.LABEL_RUN] == 'pc' def test_the_job_is_findable_by_the_same_labels_as_its_pod(job): @@ -230,7 +225,7 @@ def test_the_job_is_findable_by_the_same_labels_as_its_pod(job): The pod list and the Job list have to describe the same universe, or a Job is reaped while its pod is still streaming. """ - for key in (jm.LABEL_RUN, jm.LABEL_RANGE, jm.LABEL_ATTEMPT): + for key in (config.LABEL_RUN, config.LABEL_RANGE, config.LABEL_ATTEMPT): assert job.metadata.labels[key] == job.spec.template.metadata.labels[key] @@ -263,32 +258,6 @@ def test_the_worker_runs_the_resume_script_for_its_own_range(job): assert f'catchup "$KEY"' in script -def test_synthetic_worker_is_absent_from_the_default_job(job): - container = job.spec.template.spec.containers[0] - assert container.image_pull_policy is None - assert 'synthetic-worker' not in {v.name for v in job.spec.template.spec.volumes} - assert not any(e.name.startswith('SYNTHETIC_') for e in container.env) - - -def test_opt_in_synthetic_worker_uses_only_the_fixed_chart_script(job, monkeypatch): - monkeypatch.setattr(jm, 'SYNTHETIC_WORKER_CONFIG_MAP', 'pc-synthetic-worker') - monkeypatch.setattr(jm, 'SYNTHETIC_WORKER_IMAGE_PULL_POLICY', 'IfNotPresent') - - synthetic = jm.build_job(31005951, 16320, 2, None) - container = synthetic.spec.template.spec.containers[0] - env = {e.name: e.value for e in container.env} - volumes = {v.name: v for v in synthetic.spec.template.spec.volumes} - mounts = {m.name: m.mount_path for m in container.volume_mounts} - - assert container.command == ['python3', '/synthetic/worker.py'] - assert container.image == 'stellar/stellar-core:test' - assert container.image_pull_policy == 'IfNotPresent' - assert env['SYNTHETIC_ATTEMPT'] == '2' - assert env['SYNTHETIC_TARGET'] == '31005951' - assert env['SYNTHETIC_COUNT'] == '16320' - assert env['SYNTHETIC_KEY'] == jm.job_key(31005951, 16320) - assert volumes['synthetic-worker'].config_map.name == 'pc-synthetic-worker' - assert mounts['synthetic-worker'] == '/synthetic' def test_the_worker_mounts_the_config_the_chart_renders(job): @@ -298,7 +267,7 @@ def test_the_worker_mounts_the_config_the_chart_renders(job): owner-referenced to, so the name has to be the one owner_ref() reads. """ volumes = {v.name: v for v in job.spec.template.spec.volumes} - assert volumes['config'].config_map.name == f"{jm.RUN_NAME}-stellar-core-config" + assert volumes['config'].config_map.name == f"{config.RUN_NAME}-stellar-core-config" mounts = {m.name: m.mount_path for m in job.spec.template.spec.containers[0].volume_mounts} assert mounts['config'] == '/config' assert '/config/stellar-core.cfg' in job.spec.template.spec.containers[0].command[2] diff --git a/src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py b/src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py deleted file mode 100644 index 0a4b5872..00000000 --- a/src/MissionParallelCatchup/tests/contract/test_synthetic_resume_harness.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Safety and profile assertions for the opt-in live runner.""" - -import importlib.util -from pathlib import Path - -import pytest - - -PATH = ( - Path(__file__).resolve().parents[2] - / 'integration' / 'synthetic_resume_harness.py') -SPEC = importlib.util.spec_from_file_location('synthetic_resume_harness', PATH) -HARNESS = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(HARNESS) - - -def test_scope_guard_accepts_only_unique_sandbox_release_names(): - HARNESS.validate_scope('sandbox', 'mpc-resume-a1b2c3') - for namespace, release in ( - ('stellar-supercluster', 'mpc-resume-a1b2c3'), - ('default', 'mpc-resume-a1b2c3'), - ('sandbox', 'parallel-catchup-ssc-1959z-ef177a-r5'), - ('sandbox', 'mpc-resume-short')): - with pytest.raises(HARNESS.HarnessError): - HARNESS.validate_scope(namespace, release) - - -def test_render_inspection_rejects_cluster_scoped_or_unprefixed_resources(): - manifest = """ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: mpc-resume-a1b2c3 -rules: [] -""" - with pytest.raises(HARNESS.HarnessError): - HARNESS.inspect_rendered( - manifest, 'mpc-resume-a1b2c3', 'stellar/ssc-job-monitor:latest') - - -def test_profile_assertion_requires_both_legs_and_predecessor_peaks(): - bundle = { - 'a1_metrics': { - 'attemptSeconds': 12.0, - 'peakAnonBytes': 48 * 1024 * 1024, - 'peakWorkingSetBytes': 56 * 1024 * 1024, - 'txApplySeconds': 1.25, - }, - 'a1_outcome': {'attemptSeconds': 12.0, 'outcome': 'failed'}, - 'a1_verdict': 'failed', - 'a1_log': "metric 'ledger.transaction.apply'\nsum = 1250ms\n", - 'a1_done': True, - 'a2_metrics': { - 'attemptSeconds': 14.0, - 'peakAnonBytes': 24 * 1024 * 1024, - 'peakWorkingSetBytes': 32 * 1024 * 1024, - 'txApplySeconds': 2.5, - 'resumed': True, - }, - 'a2_log': ( - 'RESUME PROBE: offline-info reports lcl 63\n' - 'RESUME: 64/64 reached ledger 63, replay had started; skipping new-db\n'), - 'a2_done': True, - 'files': [ - 'range-64-a1.done', 'range-64-a1.log.gz', - 'range-64-a1.metrics', 'range-64-a1.outcome', - 'range-64-a1.verdict', 'range-64-a2.done', - 'range-64-a2.log.gz', 'range-64-a2.metrics', - ], - 'progress': {'completed': {'64': { - 'attempts': 2, - 'seconds': 26.0, - 'peakAnonBytes': 48 * 1024 * 1024, - 'peakWorkingSetBytes': 56 * 1024 * 1024, - 'txApply': 3.75, - }}}, - } - result = HARNESS.assert_completed_profile(bundle) - assert result['expectedChainSecondsFromArtifacts'] == 26.0 - - bundle['progress']['completed']['64']['peakAnonBytes'] = 24 * 1024 * 1024 - with pytest.raises(HARNESS.HarnessError): - HARNESS.assert_completed_profile(bundle) - - -def test_runner_uses_a_handled_signal_for_collector_only_restart(): - source = PATH.read_text() - assert 'kill -TERM 1' in source - assert 'kill -9 1' not in source - - -def test_monitor_readiness_requires_both_containers(): - pod = { - 'metadata': {'name': 'monitor'}, - 'status': {'phase': 'Running', 'containerStatuses': [ - {'name': 'job-monitor', 'ready': True, 'restartCount': 0, 'state': {}}, - {'name': 'log-collector', 'ready': False, 'restartCount': 1, 'state': {}}, - ]}, - } - evidence = {} - original = HARNESS.monitor_pod - try: - HARNESS.monitor_pod = lambda _release: pod - assert HARNESS.ready_monitor_pod('mpc-resume-a1b2c3', evidence) is None - pod['status']['containerStatuses'][1]['ready'] = True - assert HARNESS.ready_monitor_pod('mpc-resume-a1b2c3', evidence) is pod - finally: - HARNESS.monitor_pod = original - - -def test_startup_log_capture_is_limited_to_the_exact_monitor_pod(): - pod = {'metadata': {'name': 'mpc-resume-a1b2c3-job-monitor-abc'}} - calls = [] - original_pod = HARNESS.monitor_pod - original_kubectl = HARNESS.kubectl - try: - HARNESS.monitor_pod = lambda _release: pod - - class Result: - returncode = 0 - stdout = 'captured' - stderr = '' - - def fake_kubectl(namespace, *args, **_kwargs): - calls.append((namespace, args)) - return Result() - - HARNESS.kubectl = fake_kubectl - logs = HARNESS.monitor_logs('mpc-resume-a1b2c3') - finally: - HARNESS.monitor_pod = original_pod - HARNESS.kubectl = original_kubectl - - assert set(logs) == {'job-monitor', 'log-collector'} - assert len(calls) == 4 - assert all(namespace == 'sandbox' for namespace, _ in calls) - assert all(args[:2] == ('logs', pod['metadata']['name']) for _, args in calls) diff --git a/src/MissionParallelCatchup/tests/contract/test_synthetic_worker.py b/src/MissionParallelCatchup/tests/contract/test_synthetic_worker.py deleted file mode 100644 index 59d22b2f..00000000 --- a/src/MissionParallelCatchup/tests/contract/test_synthetic_worker.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Default-off and deterministic contracts for the live integration worker.""" - -import os -import subprocess -import sys - -import log_collector as lc - -import _artifacts as art - - -ENABLED = ('integration.syntheticWorker.enabled=true',) -MIB = 1024 * 1024 - - -def test_default_render_has_no_synthetic_resource_or_runtime_switch(): - names = {d['metadata']['name'] for d in art.docs()} - assert 't-synthetic-worker' not in names - for container in art.containers().values(): - env = set(art.env_of(container)) - assert not any(name.startswith('SYNTHETIC_') for name in env) - - -def test_opt_in_render_adds_fixed_worker_and_narrow_runtime_wiring(): - config_maps = {d['metadata']['name']: d - for d in art.of_kind('ConfigMap', ENABLED)} - worker = config_maps['t-synthetic-worker'] - assert set(worker['data']) == {'worker.py'} - assert 'stellar-core' not in worker['data']['worker.py'] - assert 'subprocess' not in worker['data']['worker.py'] - - containers = art.containers(ENABLED) - monitor_env = art.env_of(containers[art.MONITOR_CONTAINER]) - collector_env = art.env_of(containers[art.COLLECTOR_CONTAINER]) - assert monitor_env['SYNTHETIC_WORKER_CONFIG_MAP'] == 't-synthetic-worker' - assert collector_env['SYNTHETIC_WORKER'] == 'true' - - -def test_source_mode_can_skip_dependency_install_without_changing_its_default(): - source = ('monitor.sourceConfigMap=source',) - for container in art.containers(source).values(): - assert 'pip install' in ' '.join(container.get('args') or []) - - offline = source + ('monitor.sourceInstallDependencies=false',) - for container in art.containers(offline).values(): - command = ' '.join(container.get('args') or []) - assert 'pip install' not in command - assert 'exec python3 /app/' in command - - -def test_fixed_worker_persists_then_resumes_the_same_pvc(tmp_path): - worker = next(d for d in art.of_kind('ConfigMap', ENABLED) - if d['metadata']['name'] == 't-synthetic-worker') - script = tmp_path / 'worker.py' - script.write_text(worker['data']['worker.py']) - env = { - **os.environ, - 'SYNTHETIC_DATA_DIR': str(tmp_path), - 'SYNTHETIC_TARGET': '64', - 'SYNTHETIC_COUNT': '64', - 'SYNTHETIC_KEY': '64/64', - 'SYNTHETIC_PREDECESSOR_SECONDS': '0', - 'SYNTHETIC_SUCCESSOR_MINIMUM_SECONDS': '0', - 'SYNTHETIC_MAXIMUM_WAIT_SECONDS': '1', - 'SYNTHETIC_PREDECESSOR_ANON_MIB': '2', - 'SYNTHETIC_PREDECESSOR_WORKING_SET_MIB': '3', - 'SYNTHETIC_SUCCESSOR_ANON_MIB': '1', - 'SYNTHETIC_SUCCESSOR_WORKING_SET_MIB': '2', - 'SYNTHETIC_PREDECESSOR_TX_APPLY_MS': '1250', - 'SYNTHETIC_SUCCESSOR_TX_APPLY_MS': '2500', - } - - first = subprocess.run( - [sys.executable, str(script)], env={**env, 'SYNTHETIC_ATTEMPT': '1'}, - capture_output=True, text=True, timeout=5) - assert first.returncode == 3 - assert 'SYNTHETIC PREDECESSOR: 64/64 persisted ledger 63' in first.stdout - assert 'sum = 1250ms' in first.stdout - - (tmp_path / '.synthetic-release').touch() - second = subprocess.run( - [sys.executable, str(script)], env={**env, 'SYNTHETIC_ATTEMPT': '2'}, - capture_output=True, text=True, timeout=5) - assert second.returncode == 0, second.stderr - assert 'RESUME PROBE: offline-info reports lcl 63' in second.stdout - assert 'RESUME: 64/64 reached ledger 63, replay had started; skipping new-db' \ - in second.stdout - assert 'sum = 2500ms' in second.stdout - - -def test_synthetic_peak_marker_is_inert_unless_the_harness_is_enabled(monkeypatch): - line = f'SYNTHETIC PEAK: anonBytes={48 * MIB} workingSetBytes={56 * MIB}' - scanner = lc.TxApplyScanner() - scanner.feed(line) - assert scanner.synthetic_anon is None - - monkeypatch.setattr(lc, 'SYNTHETIC_WORKER', True) - scanner = lc.TxApplyScanner() - scanner.feed(line) - assert scanner.synthetic_anon == 48 * MIB - assert scanner.synthetic_working_set == 56 * MIB diff --git a/src/MissionParallelCatchup/tests/data/real-sts-fault-exit3.log.gz b/src/MissionParallelCatchup/tests/data/real-sts-fault-exit3.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..e1a9d49b49b8f4767f6e315478053b383204229f GIT binary patch literal 8441 zcmc(kWmjBXn{JU58Z@{&1g#>ty9Rd+PO!ouKyVN41S+^&a0wFJHAvwa+}*>8oSg1{ z&ikQ1^mu#hKd{EV=Dx0Zk3}AV1jBsbXbtyt_;KGkM-H$3>)4x#BsO;^CZcvkxoKM# zQ995DQ^1CR^`LdO3OJT)DK@gX?&&@Qfx}TVe)X-n@ZuS2{SANUWZ_r@$ReA3l%soH zSwAZ_gjNbJfLHW7$T2e{OqvD>$i$n7l~-im#GHE_&0JjPnXh9vlTkUc# z>ogFLg(%%E$>S0yj4B$WClxq_6Un?IC^|s`R^X7sy|cT8cb+|JK(FvSD+NjG+a8S+ z36r?eCtKOv9=Z=l_5W;mUnDlk~nWu5m&|YJ+9<^7I zsY;EMZS}X7BsWLop_uIuUTtU{)d?aH_qu~3yN2)CJ(U|+b_#YKv;!wSD;P}yzGzi! zp`aAT;SMN;%>^Gwr2CQt*exdj*ZI6^B{{Bgn`Rd@hOS!z1w=v{Uv*+{p+>8|6uUf~ z3eANVn(c{ub?imfeIOoNPD>ETE_9vp`Q+$sO8^}DNmym#{*YJnh);7EB&{YUru;ku z>-$qgW2lNqk`tU7bUQlaVv}7ve0A!J$l}KYfYhl|ZzU>LD)=l1(xvG%t9if^NDv@~WXfLBM6^B^Sj>f;%mDYpGX5ZL6 z?lX0@apAOjntGQLLpSzIb@jg1vkBLEec)B+$8R0WybA#x6?;WUKrpq$XC%wL=8k=C zIa;g<>)NI@_PDn$dm0?Q7Kwdp!6DptmHanuI*(@)L}*~}{Kf?Sy|eH(3@3MVLbBP% z#lE}ei*b43I^yyI)OdNq=k(vrsUj(~IwTFwCc~)v?}$|Mj(P}bFzDn`FX%ftajH^V zXr6rUD(dtw9Sj-++5!_~aQ5|?daEzj@~y;EOwqHa_by*|fv>&r-o9Z&*xP$U=UF<9 z=L_ZM_nVe-Tx~z7%i>-@n5HS_DHg6~%Sy04lp2&P=c!)u3ei?U#b$HanA`-yUJ0jn zMgMSRC%8hJtoD1%ZcKQ7k-fZ|`boCW)}YgHCs1^Ip9~_5x|}f;N_mSsf!UY+N*col zlR|o`?UY@>^r|~AH0G9A~B7(9jKr4GT~GnZ$mN?XGDm4 zb_s)q>z)D@7qWtEhiIU<$-060WNc&7NI zMR>VieX?{R_jmr*psNh{X%jI+eq|OJ7N9bV6f1|acfc|_|EF$+x$~@IABtQQ-+#9b+ z>qWZ!VomtSCa$eXrbERy|5H;CI|}a=(iCO{0Ed|9!7X3Pdrim&x-b5+1^nOrDIV9MI*X zlQ{7b&p^dQVU4=eGDxS!Q3h7pL&`cn#vS!C!7L4|2J7G?w;Gg(StYw_d%}bJGWb?hfP3cU^Tj*NFaYfF8PvD-dRAz*aK4vFhtGNw~1S&WKk!dWRLj#PpXTZu0kRV<%nH1#~blwCbDt5;Y;ZHc&)my`Kt{EZ!URkX2M_QkcuUot!ihEx}V-!gFR4- z>b~)|Xe`vj6*D8z(xRYh`Y%yrm3dCfpuYLAPcq6ib46D;rM+Q?CJJUg6nqkSe&DYSnXeFD4PammZl}PB?9m`q6Mij^ zH4~m0To>ai6WLS0d8abd)o!|`e^jNkCLpMg?^dnR67a=p0#M71>^mC}y^?}tVEURG ziXp=@vRhlp@s9PtTV)Z4>4t}0e>6pz35`Y5W(w1S`P|YtIYul}j~$ZCC(4sf)uIbo zDO+87>$6I7N#4tcy$^S5ndzHtV6s2I%H=dD2CV&l!Z>MrX*0uo{td`)SaUONy`^Tq z%_nk{Cm*hOxkd{xbyW}N*z^gIRjuBc3TV=%y7EQYQo=n*?Ng(Tnoh%X;pk~KDO9)j z;xB&)W*luI));SSaz0SKlo-tKXy4m1K@|UoxrmvrBt_x86O+5T7sAine3rb37u#l*{3_ z%>Sm#%Nbd2%S#HgsjYU9@8gdCWXh$&cwQSv_vV3&xutWLog9c+$$ z37t%Ehn%W)jv_I&ZRW5JelgM+<%O^bfBw9>RqBo+?ngdk;o8Kv(hf98#9hr~%?W7< znL-bM;e?@_iOZY2RgnnmGuUP(xs~P8Mfq^hjQQy+{kKx@YvSEiPK{A&(1ukRxNk2R zq)5-AV6HO`5PE~zJ7nS?!F5lF9lXau|3$Fq4FP=Aqr=o`&LU&5K0~kEQZfgAL^<}M zm0>tnpJpuP1i@~mogmi()PXglmaWS}C(6xzhNn3D@Oc5OU~rIDRaVNiUQioqc1^Q1 zZ41k;UX-_O+P($+8@kROC$#c&Bjw}qz#{FW%GhUN=#Oti&O>xQoM5?sit zIP$&Ew623t!0BjqIHq7XEgav%eXnF=v7-E}LGP^E0I#H+X1B7M8T-U1pKHah$|mpo z+}%~<6jNtSshX2ZhCit0%d+B52&BZ4|xeEeyxW@YLjhaqjb@bpkbT_wEs0O`HMhkDv&gLO&Z?%0vP+KLvgY#|wlPc`U*+C8LT$8UjIaD0nFHo*j!mj$6SqIok)K4y)&to(k2H3`i29#b8 z49jcv`wi0I(OQ}%=QHA@&?M*Sb_89bMP4W`wu4DDJviHh`E$C>f60Vy{J)P0 z_9R&q90wl$Ccfb2p;a{Oj>9rOVIkp`P@kn#hIS}j4nU_9h%V#5>krzVYiI660^vDK zyhBWXgy)i@>PkkFBwtQNx0n;4J`UXMMKn9SAu0{6^28K7;~bJ4Jdad5ZSIgrvs8o3 z3{a!#HIh25pC@^A$_ThUhRD~v2R|1HBP9SNkN7`oYjsCy} z$z{8uhh$D_KdtM2W|>lm5Epqvx0b@PbFn~@Z4*Y+a6@CvU$}t6UsMvYM>aY`9BrYc zd=JAi4?s{3X84}LI9P{CBt-aw#dRB=q0&|HFB8ZAiixUPc6Mk@ZLRwM1{3A)6tDg= zp?35u&;OAV;6FK$6bcsxC}D(V)cSWv3$QA;WjsScLPmDmn3x!Db=Ti5B5ucg&#*df zPlP%u4P55sz{&}2loGMzf76WAuvzB!=o(3i3sK866MxPrN%vxDPwV*N6>V?BdtWx( z-PCY+Iw@{d+>gw8a!zrbrxypF2-jL2#54iTRvwO?jsl@zD1oIPo2vwBwVw}3EZiil z>1a)cPB5$q^J>WzdfV0YEJgTpJt&U%EIaA&7ay@@z)%!Mf7%+SdPKG^66J05Ce`L# z*!Inw5!Hwk@0P}y$Ri&39&+PkN(RD?vAgcc&$YGs&BPqzhU%t#Zb=oSNX{hxClf@Z zLguX)4@{5*j4OV~@lPfWh(5EZy9RoTkkhS&vb?=?3m>16sMBjrI1w&@t#3I1im;P=%3krSLhnV5LR4OIR2 zFafjo@;_pt*dt`%FA~H5M54&6PZ7dl=Y}gL&@B(!4M+KXK7lKCHm(ma2u-+v6%z|s zdgsF7d`zmvScEX{wT;T-Qrhe13R_!)=jyjg!HhMYHagAKZs1|q{+3Wo8nBJSpo=9g z>E9D)Jsn!J7e#fbE5s)w!ERyFDd9;xM3oTSpBw&q?VX#OGdc!z*@`CMUdO}3)8heY zsRA?#yRz?xunu?UD1z;R7^&0eX|aYF28xN8*SVQrF%6{oW~-y-;4_j49lFEy*hYhn zQrm<;l?BnJ6EKlis#@4;61N-efpKP5(H^*hYF~rXh3JLe?BYM~7GLmolua9{>y@yk zWLGm6NWRq{vL;b7$2FxT1lSU7URQI$KcQ@!4f%wgdlPLe+ioQKmw+oV-r8XM%m!$d zZzI_$s4w({1U>OSey+@|Zeb)}Gig{ebAwMhAJ5XsLgdArd`DH-qlk2w`P?_f^8{Ij z{E+Q0605(bZa3PB(h0-?tS%KcvW;(X3D>(Yh53e|yS{iy**Y&HLxJ|e{X`Z3rC#(UN`#nrl8Q8mBf?*r-Wg7C_ zzgQ?9|49l_r41lv@bM!q^-=U8k5$?tV~`P6B?G|mk5Z)n7bc=-_QbNyug~gX$ zPK_9~ONOBDh{+L&+k|W)N*QoIZiSmU5 z*_({XV-)`4{me^veKIZxi`MuJ4Cy=?f^t5?rdZOnC{k4B^CvdfEBvTP)UCf{*!~{; zKf?k*CHO)O72q!Y*@!&&ZOrg`{6mf$KuMVUNgI&zaMl8O=uRANaS482^x8zlNy#Q{}k+8ZS325%NylrU+*|5 zqQO0jSPS~)nWaKof5IXy=n-7R>W;S!7dy<1OBfsA4w_*^q1%uGk}e z*VlZq_po)>`s{y^&^Y<^Fa86Gx>|P7#4Apa+P|ki{;OvazkS#Bmx-?5r~Bm;-N5HR zcaPv6A%$53p_833aQ;Ty+R|68&|=pq79%8~#L7dh za=w*zcdpu#o~66;B^ufRfFqjKI7I;&E(an>(&K*n62k)8S*5((c-El#UnbxBR*jUvGT3kXHwJRzpenmzwJj`BEV1FywVzG@ zBzg#6`xc)-M4?9=Tx`{ZC)P#Zm6;E2fl0m7wfKug+v)#@MHM@+jve&vCl*@&16YWd zs*OxuRDU=>Z;X_w;^N_?=WL?k1c0@id{nI8qsQXf8yS@;$A6B6b4`VFTY&l2IXU!~ z3g)x_Vw#U1uzqPpEL}4SCE0pD&VSO1&C5%gaaBs!4zK`UTGMs34X!%sPU|@>Nt3Hh z`xlsbod`Y=2TV<*w+hEZ#H$rw7}kt%#gu4&X~m1*TCq1(_**OZe1NhNPW|19;TK$r z@^LAu5^5^#0|)PYU@^G=Xoah*o9h|UV?msy-?4%SyyGt{kiQ`ImYKrWTu^tX_L*-m zKOFtS!m8eyf#<0s5n8&1>v8;BD}G`j81+*t;_bCru`}`o`s@P3Y2=-U_jf*quq;03 zJ?+u`YD7WyFg#q@{{x38{<9=R|CP$O^dEwTlBi~@f@8T zM4d3IjKQ2mr0lhXHJ{=Hw#H|)jen_tyZ9GWaMZE0PuBoBl>Y~)_-`$xn*V7j<g<@6#P>Er2(;Fukv}FyTJfMfB$sHAe_NTR1{nps7^D*`We5IN_8sh3m!*ng>2! ztJadPRL}Crsbq1kAzJYWL0*xlIC@t+G&s+P%te9b%&YelJ;a&&bI%%?0~*;-N5qzf*9`j+a}bpEMCjOpEMyANrMQXl?K zLl265G(E!$G)BKSVu3Bs=)niHxsnvB_=Ex^dp|yAeHfW1GFPuzU%BHCjjRtAvsW_w zWSB*M?Xse4doxyXHMM%i_#(Gs35L?lbqtS)@_dHy8yDL<%c^A8*?jW2w>#(t7lO`? zI6;Q%^pvuWB(%mv@O`jAt@KCCK4dp|;fD@BU8>RRciwTMa%|bJ)^a)ZU7&P4Bfwb{rc{XFO`^K_DF~2~{bvqfq3_Y8YwREZz_-%q$-q z4?pBW(eKViSg`RPG^i%ng(`5V7DDyWZk6VeHi1!I=r-_oLldUFmxkBvyZ`a|_F0Xw z+A;yW9`73~A|NW$FJGXma5ndkLFpNE}J=xPekhB>=p%jg%MqqeS!kODUG-Y925L;S$QXRq(e5H-jMgFZv4 zedzsouWP)fto^7xT^8N#b|}$I2}_jgk{cjuir5a0#Fmt>6wZ|;ErxEqhHF~G-w>Z_ z+Qf_fxNBly5&kiVPML{8t+iQ#d!baZ8KJdPF?Y#Y(y+K<&`;^(Tc)~%99>j5gI6{+ z{HIp5<K4xX0&VmL&12ZxrN+SD>G%?71SrFmD^D8e6TyN$VeR&}#_2Z?mud!H zNTFn)4tR6r)n#Mx=T+k)wC+Y2bCu(Vs6)Cm>{+W>?u`@Mo#%xMRZa}-Bfs+k*Dc#+ k?o4O~D!HYgJi^EuOv^v0gOeKpLN~= 1 diff --git a/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py b/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py index cd89da9c..c1bece37 100644 --- a/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py +++ b/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py @@ -15,7 +15,11 @@ created with, and what landed in progress.json's failed{}. No source text. """ +import config +import units +import sizing import job_monitor as jm +import records # --- helpers (local to this file on purpose) -------------------------------- @@ -35,7 +39,13 @@ def hit(cluster, end, state, times=1): the successor before the next pass. """ for _ in range(times): + n = cluster.attempt_of(end) cluster.advance(end, state) + if state in ('incomplete', 'unexplained'): + # exit 3 is decided from the archive, and not until .done exists. + cluster.finalize(end, n, archive=( + 'fetch_fault' if state == 'incomplete' + else 'bare')) cluster.reconcile() @@ -78,14 +88,13 @@ def test_five_evictions_do_not_burn_the_whole_oom_budget(cluster): # And the whole point of an OOM retry: more memory. One OOM = one rung. res = mem_of(cluster, end, 7) assert res.requests['memory'] == '13824Mi' - assert res.requests['memory'] == '13824Mi' def test_evictions_do_not_burn_the_disk_budget(cluster, monkeypatch): """Same shape, ephemeral-storage budget (4). Disk evictions repeat until the range gets more disk, so losing that budget to churn is terminal.""" - monkeypatch.setattr(jm, 'LIM_EPHEMERAL', '40Gi') - monkeypatch.setattr(jm, 'REQ_EPHEMERAL', '40Gi') + monkeypatch.setattr(config, 'LIM_EPHEMERAL', '40Gi') + monkeypatch.setattr(config, 'REQ_EPHEMERAL', '40Gi') end = dispatch(cluster) hit(cluster, end, 'disrupted', times=5) @@ -97,28 +106,12 @@ def test_evictions_do_not_burn_the_disk_budget(cluster, monkeypatch): "first disk eviction after eviction churn condemned the range; " f"failed={cluster.failed()}") assert job_exists(cluster, end, 7) - # Retried with MORE disk than it just outgrew, not the same 40Gi. + # One eviction = one rung: 40Gi * EPH_BUMP_FACTOR. Pinned to the exact rung + # rather than "more than 40Gi", so indexing the ladder on `attempt` instead + # of on evictions fails here -- after this much churn that would ask for the + # 6th rung (capped at 200Gi), five times the disk for one eviction. grown = mem_of(cluster, end, 7).limits['ephemeral-storage'] - assert jm._quantity_bytes(grown) > jm._quantity_bytes('40Gi'), grown - - -def test_evictions_do_not_burn_the_oom_budget(cluster): - """The OOM budget is small, so churn would eat it almost immediately. - - Retargeted from the timeout budget, which no longer exists: a deadline hit - is terminal now. The invariant is the same one -- a cause with a large - deliberate budget must not spend a small one belonging to a different cause. - """ - end = dispatch(cluster) - hit(cluster, end, 'disrupted', times=3) - assert cluster.attempt_of(end) == 4 - - hit(cluster, end, 'oom') - - assert not condemned(cluster, end), ( - f"first OOM after 3 evictions condemned the range; " - f"failed={cluster.failed()}") - assert job_exists(cluster, end, 5) + assert grown == sizing.eph_for_attempt(2) == '61440Mi', grown def test_evictions_do_not_burn_the_range_budget_for_exit_3(cluster): @@ -146,7 +139,7 @@ def test_memory_ladder_follows_ooms_not_evictions(cluster, monkeypatch): # The shipped 48Gi ceiling clamps rung 2 to the same figure rung 8 would # give, which would make the ladder assertion below prove nothing. Raise it # so the rung is observable; the budget behaviour under test is unaffected. - monkeypatch.setattr(jm, 'MEM_ESCALATION_CAP', '128Gi') + monkeypatch.setattr(config, 'MEM_ESCALATION_CAP', '128Gi') end = dispatch(cluster) hit(cluster, end, 'disrupted', times=3) # attempts 1-3, now on 4 hit(cluster, end, 'oom') # OOM #1 on attempt 4 -> a5 @@ -193,15 +186,172 @@ def test_one_timeout_condemns_the_range(cluster): f"a second attempt was dispatched after a terminal timeout: {cluster.jobs()}") -def test_the_disruption_budget_still_binds(cluster): - """Twenty evictions really is the end of the road for a range.""" +def test_an_archive_without_the_done_marker_does_not_promote_a_fetch_fault(cluster): + """The collector appends as the pod runs, so a present archive is not a + finished one. + + Promoting on a partial archive would read a fetch-fault anchor that a later + line still explains away, and hand the range the fetch-fault budget on + incomplete evidence. The .done marker is what says the collector is finished + with this attempt, so the decision waits for it. + """ + end = dispatch(cluster) + attempt = cluster.attempt_of(end) + cluster.advance(end, 'incomplete') + cluster.archive(end, attempt, 'fetch_fault') # archive, but no .done + cluster.reconcile() + + assert not condemned(cluster, end), f"failed={cluster.failed()}" + assert not job_exists(cluster, end, attempt + 1), ( + "a successor was dispatched off a half-written archive: " + f"{cluster.jobs()}") + assert records._verdict_of(str(end), attempt) != 'fetch-fault', \ + "the verdict was promoted before the collector finished" + + +def test_a_cause_with_no_budget_entry_gets_no_retries(cluster): + """The table is the whole policy: absent means condemned on sight. + + Not reachable through reconcile today -- every outcome that gets as far as + the retry gate is in the table, and the rest return CONDEMN before the cap + is read. This pins the default so a new outcome added to classify() cannot + quietly inherit retries nobody chose for it. + """ + assert jm.budget_for({'outcome': 'a-brand-new-thing'}, 300, 1) == (0, 0) + for terminal in ('timeout', 'unknown'): + assert terminal not in config.ATTEMPT_BUDGETS + assert jm.budget_for({'outcome': terminal}, 300, 1)[1] == 0 + + +def test_a_fetch_fault_does_not_spend_the_oom_budget(cluster): + """An unreachable archive is the cluster's problem, not the range's. + + Before this, an exit-3 fetch fault was recorded as `failed` and the range + budget counted ('oom', 'failed') -- so one unreachable S3 mirror permanently + cost the range one of its five memory escalations. + """ + end = dispatch(cluster) + hit(cluster, end, 'incomplete') # exit 3, fetch fault in the archive + assert job_exists(cluster, end, 2), f"the fetch fault was not retried: {cluster.jobs()}" + + # The OOM ladder still has its whole budget. + hit(cluster, end, 'oom', times=config.ATTEMPT_BUDGETS['oom'] - 1) + assert not condemned(cluster, end), ( + "the fetch fault spent an OOM attempt; " + f"failed={cluster.failed()} jobs={cluster.jobs()}") + hit(cluster, end, 'oom') + assert condemned(cluster, end), ( + f"the OOM budget did not bind after {config.ATTEMPT_BUDGETS['oom']} OOMs") + assert cluster.failed()[str(end)]['outcome'] == 'oom' + + +def test_the_disk_budget_still_binds(cluster, monkeypatch): + """Disk evictions are counted against their OWN budget, and it binds. + + The gap this closes: with the ephemeral arm of budget_for removed, an + eviction falls through to the range budget, whose counter looks only at + ('oom', 'failed'). For a purely disk-evicted range that count is always 0, so + it would be retried forever -- and every other budget test still passed. + """ + monkeypatch.setattr(config, 'LIM_EPHEMERAL', '40Gi') + monkeypatch.setattr(config, 'REQ_EPHEMERAL', '40Gi') end = dispatch(cluster) - hit(cluster, end, 'disrupted', times=jm.MAX_DISRUPTION_ATTEMPTS) + hit(cluster, end, 'ephemeral', times=config.ATTEMPT_BUDGETS['ephemeral']) assert condemned(cluster, end), ( - f"{jm.MAX_DISRUPTION_ATTEMPTS} evictions were not condemned; " + f"{config.ATTEMPT_BUDGETS['ephemeral']} disk evictions were not condemned; " f"jobs={cluster.jobs()}") - assert not job_exists(cluster, end, jm.MAX_DISRUPTION_ATTEMPTS + 1) + assert cluster.failed()[str(end)]['outcome'] == 'ephemeral' + assert not job_exists(cluster, end, config.ATTEMPT_BUDGETS['ephemeral'] + 1), ( + f"an attempt was dispatched past the disk budget: {cluster.jobs()}") + + +def test_an_eviction_with_no_configured_disk_limit_does_not_wedge_reconcile(cluster): + """LIM_EPHEMERAL is empty by default and the chart ships it empty. + + A pod with no ephemeral-storage limit can still be evicted under node disk + pressure. eph_for_attempt used to parse the empty string and raise, and + reconcile's caller swallows exceptions -- so one such eviction killed every + later pass at the same range: no dispatch, no completions, for the rest of + the run. + """ + assert config.LIM_EPHEMERAL == '', "this test is about the unset default" + end = dispatch(cluster) + hit(cluster, end, 'ephemeral') + + assert not condemned(cluster, end), f"failed={cluster.failed()}" + assert job_exists(cluster, end, 2), ( + f"the eviction was not retried; jobs={cluster.jobs()}") + # And the pass still completes for everything else. + assert cluster.reconcile()['completed'] == 0 + + +def test_the_disk_budget_is_smaller_than_the_range_budget(cluster): + """Pins that the two caps are actually different. + + Both tests above pass if disk silently borrows the range budget, as long as + the caps happen to match. They must not: escalating disk 5 times is a 7.6x + request. + """ + # The MAX_* constants, not ATTEMPT_BUDGETS: the cluster fixture patches the + # map, so asserting on it here would pin the fixture and let production ship + # any ordering it liked. + assert config.MAX_EPHEMERAL_ATTEMPTS < config.MAX_OOM_ATTEMPTS + assert config.MAX_EPHEMERAL_ATTEMPTS < config.MAX_DISRUPTION_ATTEMPTS, ( + "an eviction is the range's own problem, not the cluster's") + + +def test_the_disruption_budget_still_binds(cluster, monkeypatch): + """The environmental budget is effectively unlimited, but it is still a gate. + + Driven at a small cap rather than the configured one: what matters is that + the gate fires at N, and looping to the real value would make this test do a + thousand reconcile passes. test_attempt_budgets_are_ordered_by_whose_fault + pins the production number. + """ + monkeypatch.setitem(config.ATTEMPT_BUDGETS, 'disrupted', 6) + end = dispatch(cluster) + hit(cluster, end, 'disrupted', times=6) + + assert condemned(cluster, end), ( + f"6 evictions were not condemned; jobs={cluster.jobs()}") + assert not job_exists(cluster, end, 7) + + +def test_a_condemned_range_is_decided_once_and_then_cleaned_up(cluster, caplog): + """A condemned Job is not deleted by anything else until JOB_TTL_SECONDS. + + It therefore stays the newest Job for its range, so every later pass + re-derives the same verdict and re-logs the same condemnation -- measured on + the 2026-07-30 run as 15 identical lines over 9 minutes, ending only when the + TTL removed the Job. The reap waits for the collector's marker because + deleting the Job reaps the pod. + """ + import logging + end = dispatch(cluster) + hit(cluster, end, 'condemned') + assert condemned(cluster, end), f"failed={cluster.failed()}" + + # Before the collector finishes: the Job stays, and nothing is re-decided. + caplog.clear() + with caplog.at_level(logging.ERROR): + cluster.reconcile() + assert 'RANGE CONDEMNED' not in caplog.text, \ + "the condemnation was logged again on a later pass" + assert cluster.jobs(), "the Job was reaped before the collector finalized it" + + attempt = cluster.attempt_of(end) + job = jm.job_name(end, attempt) + cluster.finalize(end, attempt) + cluster.reconcile() + assert job not in cluster.jobs(), \ + f"the condemned Job was not reaped: {cluster.jobs()}" + assert not [v for v in cluster.pvcs() if str(end) in v], \ + f"the condemned range kept its volume: {cluster.pvcs()}" + # The other ranges are untouched. + assert len(cluster.jobs()) == 2, cluster.jobs() + # Still condemned, and still the reason the mission fails. + assert condemned(cluster, end) def test_a_genuine_catchup_failure_is_still_never_retried(cluster): @@ -214,24 +364,44 @@ def test_a_genuine_catchup_failure_is_still_never_retried(cluster): assert not job_exists(cluster, end, 2) -def test_a_terminated_pod_with_no_exit_code_is_not_condemned(cluster): - """A container that terminated without a populated exit code says nothing - about the ledger range, and must not be treated as a catchup failure. +def test_a_terminated_pod_with_no_exit_code_is_condemned(cluster): + """No exit code is no evidence, and the run stops rather than guess. - Observed on the r5 run 2026-07-30: range 59018943 was condemned on attempt 1 - with `outcome=failed exitCode=None`, failing a mission that was otherwise - 554 for 554. The collector's classify() fell through every branch that - needs an exit code and labelled the leftover case `failed` -- the one - outcome that gets no retry at all. + This reverses an earlier choice, so the cost stays on the record: on the r5 + run 2026-07-30 range 59018943 was condemned exactly this way and failed a + mission that was otherwise 554 for 554. The policy now is that only a node + disruption -- which proves the cluster took the pod away mid-run -- earns a + retry without evidence. Anything the monitor cannot explain fails the run, + because a run that reports success on a range nobody verified is worse. """ end = dispatch(cluster) hit(cluster, end, 'no_exit_code') + assert condemned(cluster, end), ( + f"a pod reaped before classification was retried on no evidence; " + f"jobs={cluster.jobs()}") + assert not job_exists(cluster, end, 2), ( + f"attempt 2 was dispatched with nothing explaining attempt 1: {cluster.jobs()}") + + +def test_an_unclassified_failure_is_condemned(cluster): + """Same rule for a pod that vanished before anything classified it.""" + end = dispatch(cluster) + hit(cluster, end, 'unknown') + + assert condemned(cluster, end), f"jobs={cluster.jobs()}" + assert cluster.failed()[str(end)]['outcome'] == 'unknown' + assert not job_exists(cluster, end, 2) + + +def test_a_disruption_is_the_only_thing_retried_without_evidence(cluster): + """The counterpart: a disruption proves the range itself was fine.""" + end = dispatch(cluster) + hit(cluster, end, 'disrupted', times=8) + assert not condemned(cluster, end), ( - "a pod reaped before classification condemned the range and failed the " - f"mission on no evidence. failed={cluster.failed()}") - assert job_exists(cluster, end, 2), ( - f"no attempt 2 was dispatched; live jobs are {cluster.jobs()}") + f"eight spot evictions condemned a healthy range; failed={cluster.failed()}") + assert job_exists(cluster, end, 9) def test_a_real_catchup_failure_is_still_condemned(cluster): @@ -242,3 +412,136 @@ def test_a_real_catchup_failure_is_still_condemned(cluster): assert condemned(cluster, end), ( "exit 1 is a genuine catchup failure and must not be retried") + + +# --- exit 3: retried only when the archive explains it ------------------------- + +def test_exit_3_with_a_fetch_fault_is_retried(cluster): + """The one exit-3 observed in production: a pod that could not reach STS. + + Every aws s3 cp failed before touching S3, stellar-core reported it as a + stale archive, and the retry succeeded on another node in 32 seconds. + """ + end = dispatch(cluster) + cluster.advance(end, 'incomplete') + cluster.finalize(end, 1, archive='fetch_fault') + cluster.reconcile() + + assert not condemned(cluster, end), f"failed={cluster.failed()}" + assert job_exists(cluster, end, 2) + + +def test_exit_3_with_nothing_to_explain_it_is_condemned(cluster): + """A give-up line with no fetch cascade in front of it earns no retry. + + Conservative by choice: the archive survives on the volume, so an + unrecognised cause is read off the failed run and added to the marker lists + rather than guessed at now. + """ + end = dispatch(cluster) + cluster.advance(end, 'unexplained') + cluster.finalize(end, 1, archive='bare') + cluster.reconcile() + + assert condemned(cluster, end), f"jobs={cluster.jobs()}" + assert not job_exists(cluster, end, 2) + assert cluster.failed()[str(end)]['attempts'] == 1 + + +def test_exit_3_waits_for_the_collector_before_deciding(cluster): + """The archive is the evidence, so the decision cannot precede .done.""" + end = dispatch(cluster) + cluster.advance(end, 'incomplete') # no finalize: nothing to read yet + + cluster.reconcile() + + assert not condemned(cluster, end), "condemned before the evidence existed" + assert not job_exists(cluster, end, 2), "retried before the evidence existed" + + cluster.finalize(end, 1, archive='fetch_fault') + cluster.reconcile() + assert job_exists(cluster, end, 2), "still not retried once finalized" + + +def test_a_permanently_missing_object_beats_an_earlier_transient_error(cluster): + """A 404 is not transient, and it wins over a connect error further back. + + Both markers are in the window on purpose: the nearest cause to the anchor is + the one that killed the attempt, so a recovered connect error earlier in the + same window must not earn a retry. + """ + end = dispatch(cluster) + cluster.advance(end, 'incomplete') + cluster.finalize(end, 1, archive=( + # recovered earlier -- must NOT decide the outcome + 'fatal error: Could not connect to the endpoint URL: ' + '"https://sts.us-east-1.amazonaws.com/"\n' + '2026-01-01T00:00:00.000 GAJSL [History INFO] Selected archive core_live_002\n' + # the cause that actually terminated it + 'fatal error: An error occurred (404) when calling the HeadObject ' + 'operation: Key does not exist\n' + '2026-01-01T00:00:00.000 GAJSL [History WARNING] Could not download file: ' + 'archive core_live_003 maybe missing file history/00/00/00/history-0.json\n' + '2026-01-01T00:00:00.000 GAJSL [History ERROR] Missing HAS for ledger 1: ' + 'maybe stale archive core_live_003\n' + '2026-01-01T00:00:00.000 GAJSL [History WARNING] Catchup failed\n')) + cluster.reconcile() + + assert condemned(cluster, end), f"a 404 was treated as transient; jobs={cluster.jobs()}" + + +def test_a_recovered_fetch_fault_does_not_earn_a_retry_for_a_later_failure(cluster): + """The anchor must be the cause of THIS give-up, not an earlier recovered one. + + stellar-core retries a failed fetch, so a range can log the whole cascade + several times and carry on -- 10 of the 11 in the one production exit-3 were + retries that recovered. Crediting any of them would retry a range that later + died of something else entirely. + """ + end = dispatch(cluster) + cluster.advance(end, 'incomplete') + cluster.finalize(end, 1, archive=( + # a fetch fault that stellar-core recovered from + 'fatal error: Could not connect to the endpoint URL: ' + '"https://sts.us-east-1.amazonaws.com/"\n' + '2026-01-01T00:00:00.000 GAJSL [History WARNING] Could not download file: ' + 'archive core_live_003 maybe missing file history/00/00/00/history-0.json\n' + '2026-01-01T00:00:00.000 GAJSL [History ERROR] Missing HAS for ledger 1: ' + 'maybe stale archive core_live_003\n' + # ...then it got the file and went on to replay + + ''.join('2026-01-01T00:00:0%d.000 GAJSL [Ledger INFO] ' + 'Ledger close complete: %d\n' % (i % 10, 100 + i) + for i in range(20)) + # ...and died of something the archive does not explain + + '2026-01-01T00:00:00.000 GAJSL [History WARNING] Catchup failed\n')) + cluster.reconcile() + + assert condemned(cluster, end), ( + f"a recovered fetch fault 20 lines earlier earned a retry; " + f"jobs={cluster.jobs()}") + + +def test_the_real_production_exit_3_is_retried_end_to_end(cluster): + """The whole path, on output stellar-core actually produced. + + Every other test here feeds hand-written archive text, so the pattern and the + fixture agree by construction -- they cannot falsify each other. This is the + verbatim archive of the one exit-3 in the 2026-08-04 run: a pod whose + aws s3 cp could not reach STS, which gave up after 35 minutes and whose + retry fetched the same object from the same bucket in 32 seconds. + """ + import gzip + import pathlib + real = (pathlib.Path(__file__).resolve().parent.parent + / 'data' / 'real-sts-fault-exit3.log.gz') + with gzip.open(real, 'rt', errors='replace') as fh: + archive = fh.read() + + end = dispatch(cluster) + cluster.advance(end, 'incomplete') + cluster.finalize(end, 1, archive=archive) + cluster.reconcile() + + assert not condemned(cluster, end), ( + f"the real STS-fault exit-3 was condemned; failed={cluster.failed()}") + assert job_exists(cluster, end, 2), "no retry was dispatched" diff --git a/src/MissionParallelCatchup/tests/reconcile/test_txapply_histogram.py b/src/MissionParallelCatchup/tests/reconcile/test_txapply_histogram.py index 500ff95c..3e783252 100644 --- a/src/MissionParallelCatchup/tests/reconcile/test_txapply_histogram.py +++ b/src/MissionParallelCatchup/tests/reconcile/test_txapply_histogram.py @@ -13,6 +13,7 @@ Nothing reads job_monitor's source. """ +import metrics import job_monitor as jm @@ -64,7 +65,7 @@ def test_late_txapply_reaches_the_histogram_not_just_progress_json(cluster): # holding the test below is not exercising the race any more. assert cluster.completed()['300']['txApply'] is None - before = _hist(jm.metric_tx_apply_duration) + before = _hist(metrics.tx_apply_duration) # The collector finishes and flushes the attempt's measurements. cluster.finalize(300, 1, tx_apply=1.25) @@ -74,7 +75,7 @@ def test_late_txapply_reaches_the_histogram_not_just_progress_json(cluster): assert cluster.progress()['completed']['300']['txApply'] == 1.25 # ...so the histogram must have counted that same value. - count, total = _delta(before, _hist(jm.metric_tx_apply_duration)) + count, total = _delta(before, _hist(metrics.tx_apply_duration)) assert (count, total) == (1.0, 1.25), ( "progress.json carries txApply=1.25 for range 300 but the histogram " f"observed count+{count} sum+{total}: the backfilled value can never " @@ -90,14 +91,14 @@ def test_backfilled_txapply_is_counted_once_not_on_every_later_pass(cluster): without bound. """ _succeed_without_metrics(cluster, 300) - before = _hist(jm.metric_tx_apply_duration) + before = _hist(metrics.tx_apply_duration) cluster.finalize(300, 1, tx_apply=1.25) for _ in range(4): cluster.reconcile() assert cluster.progress()['completed']['300']['txApply'] == 1.25 - count, total = _delta(before, _hist(jm.metric_tx_apply_duration)) + count, total = _delta(before, _hist(metrics.tx_apply_duration)) assert (count, total) == (1.0, 1.25), ( f"range 300's txApply was observed {count} times across four passes; " "the histogram must count each recorded range exactly once") @@ -118,8 +119,8 @@ def test_durations_recorded_up_front_are_not_recounted_while_txapply_is_late(clu assert rec['seconds'] is not None and rec['wallSeconds'] is not None seconds, wall = rec['seconds'], rec['wallSeconds'] - before_full = _hist(jm.metric_full_duration) - before_wall = _hist(jm.metric_wall_duration) + before_full = _hist(metrics.full_duration) + before_wall = _hist(metrics.wall_duration) # Three passes with the collector still silent, then it finally lands. for _ in range(3): @@ -130,10 +131,10 @@ def test_durations_recorded_up_front_are_not_recounted_while_txapply_is_late(clu assert cluster.progress()['completed']['300']['txApply'] == 0.5 - assert _delta(before_full, _hist(jm.metric_full_duration)) == (0.0, 0.0), ( + assert _delta(before_full, _hist(metrics.full_duration)) == (0.0, 0.0), ( "the full-duration histogram re-observed range 300's already-counted " f"{seconds}s while waiting for its txApply") - assert _delta(before_wall, _hist(jm.metric_wall_duration)) == (0.0, 0.0), ( + assert _delta(before_wall, _hist(metrics.wall_duration)) == (0.0, 0.0), ( "the wall-duration histogram re-observed range 300's already-counted " f"{wall}s while waiting for its txApply") @@ -148,13 +149,13 @@ def test_txapply_present_on_first_sight_is_still_counted_exactly_once(cluster): cluster.advance(300, 'succeeded') cluster.finalize(300, 1, tx_apply=2.5) - before = _hist(jm.metric_tx_apply_duration) + before = _hist(metrics.tx_apply_duration) cluster.reconcile() cluster.reconcile() cluster.reconcile() assert cluster.progress()['completed']['300']['txApply'] == 2.5 - assert _delta(before, _hist(jm.metric_tx_apply_duration)) == (1.0, 2.5) + assert _delta(before, _hist(metrics.tx_apply_duration)) == (1.0, 2.5) def test_two_ranges_landing_their_metrics_at_different_times_both_count(cluster): @@ -170,7 +171,7 @@ def test_two_ranges_landing_their_metrics_at_different_times_both_count(cluster) # 300's collector is quick; 200's is not. cluster.finalize(300, 1, tx_apply=1.0) - before = _hist(jm.metric_tx_apply_duration) + before = _hist(metrics.tx_apply_duration) cluster.reconcile() assert cluster.completed()['200']['txApply'] is None @@ -181,7 +182,7 @@ def test_two_ranges_landing_their_metrics_at_different_times_both_count(cluster) recorded = {k: v['txApply'] for k, v in cluster.completed().items()} assert recorded == {'300': 1.0, '200': 3.0} - count, total = _delta(before, _hist(jm.metric_tx_apply_duration)) + count, total = _delta(before, _hist(metrics.tx_apply_duration)) assert (count, total) == (2.0, 4.0), ( f"progress.json holds txApply for {sorted(recorded)} but the histogram " f"counted {count} of them (sum {total}); only the range whose .metrics " diff --git a/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py b/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py index ac1889cb..0853979e 100644 --- a/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py +++ b/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py @@ -26,6 +26,9 @@ import pytest +import config +import records +import attempts import job_monitor as jm import log_collector as lc @@ -81,9 +84,9 @@ def vol(tmp_path, monkeypatch): """ log_dir = tmp_path / 'logs' log_dir.mkdir() - monkeypatch.setattr(lc, 'LOG_DIR', str(log_dir)) + monkeypatch.setattr(config, 'LOG_DIR', str(log_dir)) # The monitor reads the same directory off its own module global. - monkeypatch.setattr(jm, 'LOG_DIR', str(log_dir)) + monkeypatch.setattr(config, 'LOG_DIR', str(log_dir)) restart(monkeypatch) monkeypatch.setattr(lc, '_pod_secs', {}) monkeypatch.setattr(lc, '_wake', {}) @@ -109,7 +112,7 @@ def restart(monkeypatch): def metrics(end, attempt=1): """What the monitor would find in .metrics, or None if there is no file.""" try: - with open(jm.metrics_path(str(end), attempt)) as fh: + with open(records.metrics_path(str(end), attempt)) as fh: return json.load(fh) except OSError: return None @@ -179,24 +182,16 @@ def finalize(pod, end, attempt=1, succeeded=False, started=None, tx=None): # -- a peak may never go backwards ------------------------------------------- @pytest.mark.parametrize('key', lc.PEAK_KEYS) -def test_a_later_lower_write_cannot_lower_a_recorded_peak(vol, key): - """Every field in PEAK_KEYS, not just the one that was reported. +@pytest.mark.parametrize('first, second', [(8, 1), (1, 8)]) +def test_a_recorded_peak_only_ever_rises(vol, key, first, second): + """Every field in PEAK_KEYS, in both orders. - This is the restarted-poller case reduced to its file operation: the second - write is a fresh process's first flush, and it is smaller because that - process started counting at zero. + The restarted-poller case reduced to its file operation: a smaller second + write is a fresh process's first flush, counting from zero. The guard must + still not be a write-once latch -- growth is the normal case. """ - lc.write_metrics('300', 1, {key: 8 * GIB}) - lc.write_metrics('300', 1, {key: 1 * GIB}) - - assert metrics(300)[key] == 8 * GIB - - -@pytest.mark.parametrize('key', lc.PEAK_KEYS) -def test_a_later_higher_write_still_raises_the_peak(vol, key): - """The guard must not be a write-once latch: growth is the normal case.""" - lc.write_metrics('300', 1, {key: 1 * GIB}) - lc.write_metrics('300', 1, {key: 8 * GIB}) + lc.write_metrics('300', 1, {key: first * GIB}) + lc.write_metrics('300', 1, {key: second * GIB}) assert metrics(300)[key] == 8 * GIB @@ -233,7 +228,7 @@ def test_finalize_recovers_resume_after_the_scanner_is_recreated(vol): def test_finalize_recovers_txapply_after_the_scanner_is_recreated(vol, monkeypatch): """The first poll saw the final medida block, then its scanner vanished.""" - monkeypatch.setattr(lc, 'SAVE_SUCCESS_LOGS', False) + monkeypatch.setattr(config, 'SAVE_SUCCESS_LOGS', False) path = lc.base('300', 2) + '.log.gz' with gzip.open(path, 'wt') as fh: fh.write('RESUME: local state reached ledger 250; skipping new-db\n') @@ -250,34 +245,29 @@ def test_finalize_recovers_txapply_after_the_scanner_is_recreated(vol, monkeypat "the test must prove recovery happened before success-log discard" -def test_synthetic_peaks_survive_the_same_scanner_recreation(vol, monkeypatch): - monkeypatch.setattr(lc, 'SYNTHETIC_WORKER', True) - path = lc.base('300', 2) + '.log.gz' +def test_finalize_recovers_txapply_for_a_scanner_that_was_never_recreated(vol, + monkeypatch): + """The gap the monitor used to cover, now closed at the source. + + stellar-core prints the medida block once, at exit, so a poller that ran the + pod's whole life can still end a beat early and hold no total -- with nothing + to recreate. The rescue used to require `recreated`, so it never looked, and + the monitor re-parsed the same archive behind it. Measured on the 2026-08-04 + run: 15 attempts of 4805 landed here, and replaying the archive through this + same scanner recovers every one. + """ + monkeypatch.setattr(config, 'SAVE_SUCCESS_LOGS', False) + path = lc.base('300', 1) + '.log.gz' with gzip.open(path, 'wt') as fh: - fh.write('RESUME: local state reached ledger 250; skipping new-db\n') - fh.write('SYNTHETIC PEAK: anonBytes=50331648 workingSetBytes=58720256\n') fh.write("metric 'ledger.transaction.apply'\n") - fh.write('sum = 2500ms\n') - - finalize('w-300-a2', 300, attempt=2, tx=lc.TxApplyScanner(recreated=True)) - - assert metrics(300, 2) == { - 'peakAnonBytes': 50331648, - 'peakWorkingSetBytes': 58720256, - 'resumed': True, - 'txApplySeconds': 2.5, - } - - -def test_synthetic_mode_never_overwrites_fixed_peaks_from_kubelet(vol, monkeypatch): - monkeypatch.setattr(lc, 'SYNTHETIC_WORKER', True) - session = FakeSession(summary( - 'w-300-a1', rss=80 * GIB, ws=90 * GIB)) + fh.write(' count = 123\n') + fh.write(' sum = 4200.0ms\n') - run(lc.sample_kubelet(session, ['node-1'])) + # recreated=False: this poller ran start to finish and simply has no total. + finalize('w-300-a1', 300, attempt=1, succeeded=True, tx=lc.TxApplyScanner()) - assert session.urls == [] - assert metrics(300) is None + assert metrics(300, 1)['txApplySeconds'] == 4.2, \ + "the collector did not re-read its own archive, so the value is lost" def test_finalize_does_not_promote_resume_declined(vol): @@ -321,7 +311,7 @@ def test_a_midflight_anon_flush_survives_a_collector_restart(vol, monkeypatch): assert metrics(300)['peakAnonBytes'] == 6 * GIB # And the consumer agrees: this is the figure that sizes the next run. - assert jm.peaks_for_range('300', 1)['peakAnonBytes'] == 6 * GIB + assert attempts.peaks_for_range('300', 1)['peakAnonBytes'] == 6 * GIB def test_a_midflight_ephemeral_flush_survives_a_collector_restart(vol, monkeypatch): @@ -332,25 +322,25 @@ def test_a_midflight_ephemeral_flush_survives_a_collector_restart(vol, monkeypat buckets are applied -- so a replacement sidecar re-measuring the same pod does not recover the earlier high-water. It has to already be on the volume. """ - monkeypatch.setattr(lc, 'STORAGE_MODE', 'ephemeral') + monkeypatch.setattr(config, 'STORAGE_MODE', 'ephemeral') lc._streaming['w-300'] = ('300', '1') sample('w-300', rss=1 * GIB, eph=34 * GIB) restart(monkeypatch) - monkeypatch.setattr(lc, 'STORAGE_MODE', 'ephemeral') + monkeypatch.setattr(config, 'STORAGE_MODE', 'ephemeral') lc._streaming['w-300'] = ('300', '1') sample('w-300', rss=1 * GIB, eph=4 * GIB) finalize('w-300', 300) assert metrics(300)['peakEphemeralBytes'] == 34 * GIB - assert jm.peaks_for_range('300', 1)['peakEphemeralBytes'] == 34 * GIB + assert attempts.peaks_for_range('300', 1)['peakEphemeralBytes'] == 34 * GIB def test_pvc_mode_records_no_ephemeral_peak_at_all(vol, monkeypatch): """In pvc mode the range's data sits on the volume, not on node disk, so there is no ephemeral-storage request to size and the figure would be noise. Sampling it is gated on the mode; flushing it must be too.""" - monkeypatch.setattr(lc, 'STORAGE_MODE', 'pvc') + monkeypatch.setattr(config, 'STORAGE_MODE', 'pvc') lc._streaming['w-300'] = ('300', '1') sample('w-300', rss=1 * GIB, eph=34 * GIB) finalize('w-300', 300) @@ -361,7 +351,7 @@ def test_pvc_mode_records_no_ephemeral_peak_at_all(vol, monkeypatch): def test_a_flush_with_no_stream_registered_writes_nothing(vol, monkeypatch): """_streaming is repopulated when a poller opens. A sample that lands on a pod with no poller yet has nowhere to write and must not guess a file.""" - monkeypatch.setattr(lc, 'STORAGE_MODE', 'ephemeral') + monkeypatch.setattr(config, 'STORAGE_MODE', 'ephemeral') sample('w-300', rss=6 * GIB, eph=34 * GIB) assert os.listdir(vol) == [] @@ -415,7 +405,7 @@ def test_done_never_appears_beside_a_half_written_metrics_file(vol, monkeypatch) record permanent.""" lc.write_metrics('300', 1, {'peakAnonBytes': 6 * GIB, 'txApplySeconds': 30.0}) - seen = _arm_half_write(monkeypatch, lc, '.metrics.tmp') + seen = _arm_half_write(monkeypatch, records, '.metrics.tmp') lc._anon_peak['w-300'] = 9 * GIB finalize('w-300', 300) @@ -424,9 +414,9 @@ def test_done_never_appears_beside_a_half_written_metrics_file(vol, monkeypatch) assert metrics(300) == {'peakAnonBytes': 6 * GIB, 'txApplySeconds': 30.0} # .done still lands: the collector really will write nothing more for this # attempt, and withholding it only strands the Job until its TTL. - assert os.path.exists(jm.done_path('300', 1)) + assert os.path.exists(records.done_path('300', 1)) # What the monitor actually reads is a complete record, not a torn one. - assert jm.peaks_for_range('300', 1) == {'peakAnonBytes': 6 * GIB} + assert attempts.peaks_for_range('300', 1) == {'peakAnonBytes': 6 * GIB} def test_done_lands_after_the_metrics_it_promises(vol): @@ -434,21 +424,21 @@ def test_done_lands_after_the_metrics_it_promises(vol): lc._anon_peak['w-300'] = 6 * GIB finalize('w-300', 300) - assert (os.stat(jm.done_path('300', 1)).st_mtime_ns - >= os.stat(jm.metrics_path('300', 1)).st_mtime_ns) + assert (os.stat(records.done_path('300', 1)).st_mtime_ns + >= os.stat(records.metrics_path('300', 1)).st_mtime_ns) assert metrics(300)['peakAnonBytes'] == 6 * GIB def test_a_truncated_metrics_file_does_not_poison_the_next_write(vol): """Whatever tore the previous record, the next flush must still produce a file the monitor can read -- and must not raise inside the sampler.""" - with open(jm.metrics_path('300', 1), 'w') as fh: + with open(records.metrics_path('300', 1), 'w') as fh: fh.write('{"peakAnonBytes": 644245') lc.write_metrics('300', 1, {'peakAnonBytes': 5 * GIB}) assert metrics(300) == {'peakAnonBytes': 5 * GIB} - assert jm.peaks_for_range('300', 1) == {'peakAnonBytes': 5 * GIB} + assert attempts.peaks_for_range('300', 1) == {'peakAnonBytes': 5 * GIB} def test_an_attempt_with_nothing_to_report_still_finalizes(vol): @@ -465,7 +455,7 @@ def test_marking_done_twice_is_harmless(vol): lc._mark_done('300', 1) lc._mark_done('300', 1) - assert os.path.exists(jm.done_path('300', 1)) + assert os.path.exists(records.done_path('300', 1)) assert jm._attempt_finalized('300', 1) @@ -537,7 +527,7 @@ def test_a_cold_poller_on_an_already_terminal_pod_reports_no_duration(vol): assert 'attemptSeconds' not in stored assert stored['peakAnonBytes'] == 6 * GIB # ...and the monitor is left free to supply the real one. - assert jm.seconds_for_range('300', 1, final=3600.4) == 3600.4 + assert attempts.seconds_for_range('300', 1, final=3600.4) == 3600.4 def test_a_poller_that_watched_the_whole_attempt_still_reports_its_duration(vol): @@ -548,7 +538,7 @@ def test_a_poller_that_watched_the_whole_attempt_still_reports_its_duration(vol) stored = metrics(300) assert stored['attemptSeconds'] == pytest.approx(42.0, abs=1.0) assert stored['attemptSecondsExact'] is False - assert jm.seconds_for_range('300', 1) is None + assert attempts.seconds_for_range('300', 1) is None def test_the_duration_the_collector_records_is_the_pods_not_the_pollers(vol): @@ -583,11 +573,11 @@ def test_an_existing_outcome_is_not_overwritten_by_a_later_pod(vol): the one taken while the evidence was fresh; a later, different pod must not silently rewrite it.""" lc.record_outcome(_pod('w-300-first', disrupted=True), '300', 1) - first = jm.read_outcome('300', 1) + first = records.read_outcome('300', 1) lc.record_outcome(_pod('w-300-second', exit_code=1), '300', 1) - assert jm.read_outcome('300', 1) == first + assert records.read_outcome('300', 1) == first assert first['outcome'] == 'disrupted' assert first['pod'] == 'w-300-first' @@ -596,25 +586,25 @@ def test_an_outcome_written_by_the_monitor_is_not_re_classified(vol, monkeypatch """Both processes write this file and both read it. The collector must treat the monitor's verdict as final, including the fields only the monitor records -- attemptSeconds for a failed leg lives nowhere else.""" - with open(jm.outcome_path('300', 1), 'w') as fh: + with open(records.outcome_path('300', 1), 'w') as fh: json.dump({'outcome': 'ephemeral', 'exitCode': None, 'pod': 'w-300', 'attemptSeconds': 1800.0}, fh) lc.record_outcome(_pod('w-300', exit_code=3), '300', 1) - assert jm.read_outcome('300', 1)['outcome'] == 'ephemeral' - assert jm.read_outcome('300', 1)['attemptSeconds'] == 1800.0 + assert records.read_outcome('300', 1)['outcome'] == 'ephemeral' + assert records.read_outcome('300', 1)['attemptSeconds'] == 1800.0 def test_a_recorded_outcome_is_a_complete_file_or_no_file(vol, monkeypatch): """Same rename discipline as .metrics: the monitor branches its whole retry policy on this file, so a torn read would have to be a crash or a wrong verdict.""" - _arm_half_write(monkeypatch, lc, '.outcome.tmp') + _arm_half_write(monkeypatch, records, '.outcome.tmp') lc.record_outcome(_pod('w-300', exit_code=1), '300', 1) - assert jm.read_outcome('300', 1) is None - assert not os.path.exists(jm.outcome_path('300', 1)) + assert records.read_outcome('300', 1) is None + assert not os.path.exists(records.outcome_path('300', 1)) def test_an_ephemeral_eviction_is_classified_from_the_pod_message(vol): @@ -627,7 +617,7 @@ def test_an_ephemeral_eviction_is_classified_from_the_pod_message(vol): 'of containers 40Gi'), '300', 1) - assert jm.read_outcome('300', 1)['outcome'] == 'ephemeral' + assert records.read_outcome('300', 1)['outcome'] == 'ephemeral' # -- the resume state file ---------------------------------------------------- @@ -667,7 +657,7 @@ def test_discarding_a_successful_range_keeps_its_measurements(vol): def test_a_successful_range_discards_its_archive_inside_finalize(vol, monkeypatch): - monkeypatch.setattr(lc, 'SAVE_SUCCESS_LOGS', False) + monkeypatch.setattr(config, 'SAVE_SUCCESS_LOGS', False) with open(lc.base('300', 1) + '.log.gz', 'wb') as fh: fh.write(b'\x1f\x8b') lc._anon_peak['w-300'] = 6 * GIB @@ -676,4 +666,4 @@ def test_a_successful_range_discards_its_archive_inside_finalize(vol, monkeypatc assert not os.path.exists(lc.base('300', 1) + '.log.gz') assert metrics(300)['peakAnonBytes'] == 6 * GIB - assert os.path.exists(jm.done_path('300', 1)) + assert os.path.exists(records.done_path('300', 1)) diff --git a/src/MissionParallelCatchup/tests/resilience/test_crash_points.py b/src/MissionParallelCatchup/tests/resilience/test_crash_points.py index 5e065001..a2652b42 100644 --- a/src/MissionParallelCatchup/tests/resilience/test_crash_points.py +++ b/src/MissionParallelCatchup/tests/resilience/test_crash_points.py @@ -25,6 +25,8 @@ from kubernetes.client.rest import ApiException import fake_k8s +import config +import records import job_monitor as jm @@ -77,7 +79,7 @@ def wrapper(*args, **kwargs): def restart(cluster): """Replace the monitor process: fresh in-memory state, same volume+cluster. - Identical to the dict update_status_and_metrics() builds on entry, so a + Identical to the dict reconcile_loop() builds on entry, so a restarted monitor starts from exactly what the shipped loop starts from. """ cluster.state = {'owner': None, 'replayed': set(), 'max_completed': 0, @@ -223,34 +225,6 @@ def test_crash_before_save_progress_records_the_range_exactly_once(cluster, assert_converged(cluster) -def test_crash_between_the_progress_file_and_its_configmap_mirror(cluster, - monkeypatch): - """The file is authoritative; the mirror is best effort and catches up.""" - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakAnonBytes': 7}) - - crash_before(monkeypatch, jm, '_patch_cm') - with pytest.raises(Crash): - cluster.reconcile() - - # os.replace() landed before the mirror was attempted. - assert '300' in cluster.progress()['completed'] - - restart(cluster) - cluster.reconcile() - - # Reloaded from the file, not re-derived from the cluster: same attempt, - # and no second Job. - assert cluster.completed()['300']['attempts'] == 1 - assert creates_of(cluster, 'pc-r300-a1') == 1 - - run_to_quiescence(cluster) - assert_converged(cluster) - # The mirror is whole again once any later write re-publishes the document. - assert sorted(cluster.progress_configmap()['completed']) == ['100', '200', '300'] - - def test_crash_after_save_progress_before_release_pvc_does_not_leak_the_volume( cluster, monkeypatch): """The record is durable and the volume is not yet freed. @@ -387,7 +361,7 @@ def test_crash_after_the_successor_exists_before_the_predecessor_is_deleted( leave the loser standing once the range finishes.""" cluster.reconcile() cluster.advance(300, 'incomplete') - cluster.finalize(300, 1) # finalized, so a-1 is deletable + cluster.finalize(300, 1, archive='fetch_fault') crash_after(monkeypatch, cluster.k8s.batch_v1, 'create_namespaced_job', match=lambda ns, body, **kw: body.metadata.name == 'pc-r300-a2') @@ -403,7 +377,7 @@ def test_crash_after_the_successor_exists_before_the_predecessor_is_deleted( # keys on the highest attempt for the range. assert 'pc-r300-a3' not in cluster.jobs() assert creates_of(cluster, 'pc-r300-a2') == 1 - assert jm._cause_count('300', 2, ('oom', 'failed')) == 1, \ + assert records._cause_count('300', 2, ('fetch-fault',)) == 1, \ "attempt 1 must be counted once, not once per pass that saw it" cluster.advance(300, 'succeeded', attempt=2) @@ -430,7 +404,7 @@ def test_crash_between_the_verdict_and_the_retry_create(cluster, monkeypatch): with pytest.raises(Crash): cluster.reconcile() - assert jm._verdict_of('300', 1) == 'oom' + assert records._verdict_of('300', 1) == 'oom' assert 'pc-r300-a1' in cluster.jobs(), \ "the predecessor must survive: without it the range restarts at attempt 1" @@ -443,7 +417,7 @@ def test_crash_between_the_verdict_and_the_retry_create(cluster, monkeypatch): # One OOM seen, so exactly one rung: 24000Mi * 1.5. Two would mean the # replayed attempt was counted twice. assert resources.requests['memory'] == '13824Mi' - assert jm._cause_count('300', 1, ('oom', 'failed')) == 1 + assert records._cause_count('300', 1, ('oom', 'failed')) == 1 assert cluster.failed() == {} @@ -495,8 +469,8 @@ def loser(namespace, body, **kwargs): result = cluster.reconcile() assert lost, "precondition: the create actually lost the race" - assert len(cluster.jobs()) <= jm.PARALLELISM, ( - f"dispatched {cluster.jobs()} against PARALLELISM={jm.PARALLELISM}: " + assert len(cluster.jobs()) <= config.PARALLELISM, ( + f"dispatched {cluster.jobs()} against PARALLELISM={config.PARALLELISM}: " "a 409 left the slot looking free") assert '300/420' in result['in_progress'], \ "the range whose Job exists is in flight and must be reported as such" @@ -534,7 +508,7 @@ def test_500_on_the_retry_create_does_not_lose_or_double_spend_the_range(cluster """The retry create fails hard. The range keeps its budget and its history.""" cluster.reconcile() cluster.advance(300, 'incomplete') - cluster.finalize(300, 1) + cluster.finalize(300, 1, archive='fetch_fault') cluster.k8s.fail_next['create job'] = fake_k8s.api_exception(500, 'boom') with pytest.raises(ApiException): @@ -548,7 +522,7 @@ def test_500_on_the_retry_create_does_not_lose_or_double_spend_the_range(cluster cluster.reconcile() assert 'pc-r300-a2' in cluster.jobs() - assert jm._cause_count('300', 2, ('oom', 'failed')) == 1 + assert records._cause_count('300', 2, ('fetch-fault',)) == 1 assert cluster.calls.names(verb='create', kind='pvc').count('pc-data-r300') == 1 cluster.advance(300, 'succeeded', attempt=2) @@ -560,24 +534,31 @@ def test_500_on_the_retry_create_does_not_lose_or_double_spend_the_range(cluster def test_a_range_that_exhausts_its_budget_across_crashes_fails_once(cluster, monkeypatch): """Budgets are spent by durable verdicts, so restarts must not stretch or - shrink them. Five attempts, a crash before each retry create.""" + shrink them. Five attempts, a crash before each retry create. + + An exit-3 fetch fault is an unreachable archive, so it spends the + environmental budget; the cap is lowered here rather than looping to the + configured one. + """ + # A fetch fault spends its own budget, so that is the one to lower. + monkeypatch.setitem(config.ATTEMPT_BUDGETS, 'fetch-fault', 5) cluster.reconcile() - for attempt in range(1, jm.MAX_ATTEMPTS_PER_RANGE + 1): + for attempt in range(1, config.ATTEMPT_BUDGETS['fetch-fault'] + 1): cluster.advance(300, 'incomplete', attempt=attempt) - cluster.finalize(300, attempt) + cluster.finalize(300, attempt, archive='fetch_fault') crash_before(monkeypatch, cluster.k8s.batch_v1, 'create_namespaced_job') with pytest.raises(Crash): cluster.reconcile() restart(cluster) cluster.reconcile() - assert cluster.failed()['300']['attempts'] == jm.MAX_ATTEMPTS_PER_RANGE - assert cluster.failed()['300']['outcome'] == 'failed' - # Exactly MAX_ATTEMPTS Jobs were ever created for the range, despite five + assert cluster.failed()['300']['attempts'] == config.ATTEMPT_BUDGETS['fetch-fault'] + assert cluster.failed()['300']['outcome'] == 'fetch-fault' + # Exactly that many Jobs were ever created for the range, despite five # crashed passes replaying the same failed attempts. creates = cluster.calls.names(verb='create', kind='job') assert sorted(n for n in creates if n.startswith('pc-r300-')) == [ - f'pc-r300-a{n}' for n in range(1, jm.MAX_ATTEMPTS_PER_RANGE + 1)] + f'pc-r300-a{n}' for n in range(1, config.ATTEMPT_BUDGETS['fetch-fault'] + 1)] # --- end to end -------------------------------------------------------------- diff --git a/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py b/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py index 8e8eacf5..3594baa8 100644 --- a/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py +++ b/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py @@ -18,6 +18,9 @@ import pytest import fake_k8s +import config +import records +import attempts import job_monitor as jm @@ -28,7 +31,7 @@ def seed_progress(cluster, completed=None, failed=None): - cluster.write(jm.PROGRESS_FILE, json.dumps( + cluster.write(config.PROGRESS_FILE, json.dumps( {'completed': dict(completed or {}), 'failed': dict(failed or {})})) @@ -62,7 +65,7 @@ def test_foreign_completed_keys_do_not_shrink_remaining(cluster): def test_foreign_completed_keys_do_not_drive_remaining_negative(cluster): """The mirror image, and the one that hangs a real run. - The mission finishes on `num_remain == 0 && jobs_in_progress == []` + The mission finishes on `num_remain == 0 && queue_in_progress_count == 0` (MissionHistoryPubnetParallelCatchupV2.fs). With three foreign keys in the record, subtraction lands on -3 once every real range has actually completed -- never 0 -- so the driver waits forever on a run that is done. @@ -124,62 +127,26 @@ def test_a_range_end_shared_with_the_foreign_slicing_is_still_skipped(cluster): # --- corruption -------------------------------------------------------------- -def test_truncated_progress_json_does_not_crash_or_lose_completions(cluster): - """A half-written progress.json must not read as an empty record. +def test_an_unreadable_progress_json_replays_rather_than_halting(cluster): + """An unreadable record reads as "nothing has been done". - The file is written through a .tmp + os.replace, so a torn write should be - impossible -- but the volume is shared, and the ConfigMap mirror is exactly - the second copy that exists for this. Truncate the file and the run must - carry on from the mirror. + There is no monotonic-progress guard -- its high-water mark lived in memory + and a restart erased it. Replay is safe: the PVCs survive, so each range + resumes at its last closed ledger. """ cluster.reconcile() cluster.advance(300, 'succeeded') cluster.finalize(300, 1) cluster.reconcile() assert '300' in cluster.completed() - assert '300' in cluster.progress_configmap()['completed'] - # Truncated mid-object: json.load raises ValueError. - cluster.write(jm.PROGRESS_FILE, '{"completed": {"300": {"att') - with pytest.raises(ValueError): - json.loads(open(jm.PROGRESS_FILE).read()) - - before = set(cluster.jobs()) - result = cluster.reconcile() - - # No crash, no halt, and the completion survived via the mirror. - assert cluster.state['halted'] is False - assert '300' in jm.load_progress()['completed'] - # The critical consequence: a range that is done is not dispatched again. - assert 'pc-r300-a1' not in cluster.jobs() - assert 'pc-r300-a2' not in cluster.jobs() - assert result['completed'] == 1 - assert set(cluster.jobs()) >= before - {'pc-r300-a1'} - - -def test_unreadable_progress_with_no_mirror_halts_rather_than_replaying(cluster): - """Losing both copies replays the run rather than stopping it. - - Indistinguishable from "nothing has been done", and that is now the reading - the monitor takes: there is no monotonic-progress guard, because its - high-water mark lived in memory and a restart erased it. Replay is safe -- - the PVCs survive, so each range resumes at its last closed ledger. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1) - cluster.reconcile() - assert '300' in cluster.completed() - - # Both copies gone: garbage on the volume, mirror deleted underneath us. - cluster.write(jm.PROGRESS_FILE, 'not json at all') - cluster.k8s.core_v1.delete_namespaced_config_map(jm.PROGRESS_CM, - cluster.namespace) + cluster.write(config.PROGRESS_FILE, 'not json at all') result = cluster.reconcile() # The record is empty, so the range is eligible again -- and the pass does # not crash, which is the property that actually matters here. + assert cluster.state['halted'] is False assert cluster.completed() == {} assert result['remaining'] + len(result['in_progress']) == 3 @@ -203,7 +170,7 @@ def test_progress_rolled_back_to_an_older_version_makes_it_eligible_again(cluste assert set(cluster.completed()) == {'200', '300'} # The stale copy lands back on the volume. - cluster.write(jm.PROGRESS_FILE, older) + cluster.write(config.PROGRESS_FILE, older) before = set(cluster.jobs()) created_before = cluster.calls.names(verb='create', kind='job') @@ -226,8 +193,8 @@ def test_metrics_without_done_must_not_reap(cluster): cluster.reconcile() cluster.advance(300, 'succeeded') # .metrics only -- exactly the window between the collector's two writes. - cluster.write(jm.metrics_path('300', 1), - json.dumps({'txApplySeconds': 2.5, 'peakRssBytes': 999})) + cluster.write(records.metrics_path('300', 1), + json.dumps({'txApplySeconds': 2.5, 'peakAnonBytes': 999})) cluster.reconcile() @@ -236,12 +203,12 @@ def test_metrics_without_done_must_not_reap(cluster): # The range is recorded and its measurements were read -- the reap is the # only thing being withheld. assert cluster.completed()['300']['txApply'] == 2.5 - assert cluster.completed()['300']['peakRssBytes'] == 999 + assert cluster.completed()['300']['peakAnonBytes'] == 999 # Withheld, not leaked: the Job carries a TTL, so declining to reap costs a # late reclaim rather than an object that lives until `helm uninstall`. assert (cluster.k8s.job('pc-r300-a1').spec.ttl_seconds_after_finished - == jm.JOB_TTL_SECONDS) + == config.JOB_TTL_SECONDS) # And the withheld reap does not turn into a re-dispatch on later passes. cluster.reconcile() @@ -261,12 +228,12 @@ def test_the_reap_lands_once_the_done_marker_arrives(cluster): assert cluster.deleted.names(verb='delete', kind='job') == [] # The collector finally finishes this attempt. - cluster.finalize(300, 1, tx_apply=2.5, peaks={'peakRssBytes': 999}) + cluster.finalize(300, 1, tx_apply=2.5, peaks={'peakAnonBytes': 999}) cluster.reconcile() # Backfilled from the durable files, then reaped. assert cluster.completed()['300']['txApply'] == 2.5 - assert cluster.completed()['300']['peakRssBytes'] == 999 + assert cluster.completed()['300']['peakAnonBytes'] == 999 assert cluster.deleted.names(verb='delete', kind='job') == ['pc-r300-a1'] @@ -279,7 +246,7 @@ def test_done_without_metrics_reaps_but_does_not_invent_measurements(cluster): """ cluster.reconcile() cluster.advance(300, 'succeeded') - cluster.write(jm.done_path('300', 1), '') + cluster.write(records.done_path('300', 1), '') cluster.reconcile() @@ -289,7 +256,7 @@ def test_done_without_metrics_reaps_but_does_not_invent_measurements(cluster): # No .metrics and no history archive to fall back on: the gap is reported # as a gap, not as zero. assert record['txApply'] is None - assert not any(record.get(k) is not None for k in jm.PEAK_FIELDS) + assert not any(record.get(k) is not None for k in attempts.PEAK_FIELDS) # Timing comes from the pod, which is real. assert record['seconds'] == pytest.approx(60.0) @@ -304,14 +271,14 @@ def test_an_empty_metrics_file_is_not_read_as_zero(cluster): """A zero-length .metrics is a torn write, not a measurement of nothing.""" cluster.reconcile() cluster.advance(300, 'succeeded') - cluster.write(jm.metrics_path('300', 1), '') - cluster.write(jm.done_path('300', 1), '') + cluster.write(records.metrics_path('300', 1), '') + cluster.write(records.done_path('300', 1), '') cluster.reconcile() record = cluster.completed()['300'] assert record['txApply'] is None - assert not any(record.get(k) is not None for k in jm.PEAK_FIELDS) + assert not any(record.get(k) is not None for k in attempts.PEAK_FIELDS) # --- two monitors ------------------------------------------------------------ @@ -327,7 +294,7 @@ def test_two_monitors_racing_the_same_volume_never_double_dispatch(cluster): real cluster. """ stale_jobs = cluster.k8s.batch_v1.list_namespaced_job( - cluster.namespace, label_selector=f"{jm.LABEL_RUN}={jm.RUN_NAME}") + cluster.namespace, label_selector=f"{config.LABEL_RUN}={config.RUN_NAME}") assert stale_jobs.items == [] a = cluster.reconcile() @@ -415,7 +382,7 @@ def test_a_foreign_run_s_jobs_in_the_namespace_are_ignored(cluster): jm.build_job(300, 420, 1, None)) other.metadata.name = 'other-r300-a1' other.metadata.labels = dict(other.metadata.labels or {}) - other.metadata.labels[jm.LABEL_RUN] = 'other-run' + other.metadata.labels[config.LABEL_RUN] = 'other-run' cluster.k8s.jobs[(cluster.namespace, 'other-r300-a1')] = other del cluster.k8s.jobs[(cluster.namespace, 'pc-r300-a1')] @@ -429,52 +396,15 @@ def test_a_foreign_run_s_jobs_in_the_namespace_are_ignored(cluster): assert result['total'] == 3 -@pytest.mark.parametrize('document', [ - {'completed': {'333': 'garbage'}, 'failed': {}}, # entry not a record - {'completed': {'333': None}, 'failed': {}}, - {'completed': ['333'], 'failed': {}}, # bucket not a map - {'completed': {}, 'failed': 'wat'}, - ['333'], # not even an object -]) -def test_a_structurally_wrong_progress_document_does_not_crash_the_pass( - cluster, document): - """Truncation is not the only corruption. - - A file that parses but has the wrong SHAPE gets past the ValueError guard, - and the walk over `completed` then raises -- after dispatch, inside a loop - that swallows exceptions. The run keeps its Jobs but stops publishing - status, so `num_remain` freezes and the mission waits on a number that will - never move again. - """ - cluster.write(jm.PROGRESS_FILE, json.dumps(document)) - - try: - result = cluster.reconcile() - except Exception as e: # noqa: BLE001 -- the point of the test - pytest.fail(f"a malformed progress record took the reconcile down: {e!r}") - - # Dispatch is unaffected, and -- the important half -- the garbage is not - # read as work already done. - assert result['completed'] == 0 - assert sorted(result['in_progress']) == ['200/420', '300/420'] - assert result['remaining'] == 1 - - # A real completion still lands on top of it, and the record heals. - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1) - cluster.reconcile() - assert set(cluster.completed()) == {'300'} - assert cluster.state['halted'] is False - - -def test_the_progress_configmap_being_deleted_mid_run_is_survivable(cluster): - """The mirror is best-effort. Losing it must not lose the run.""" +def test_the_status_configmap_being_deleted_mid_run_is_survivable(cluster): + """The ConfigMap is the driver's view. Losing it must not lose the run.""" cluster.reconcile() cluster.advance(300, 'succeeded') cluster.finalize(300, 1) cluster.reconcile() + jm.save_status(jm.status) # what the reconcile loop publishes - cluster.k8s.core_v1.delete_namespaced_config_map(jm.PROGRESS_CM, + cluster.k8s.core_v1.delete_namespaced_config_map(config.PROGRESS_CM, cluster.namespace) cluster.advance(200, 'succeeded') cluster.finalize(200, 1) @@ -483,21 +413,11 @@ def test_the_progress_configmap_being_deleted_mid_run_is_survivable(cluster): assert set(cluster.completed()) == {'200', '300'} assert cluster.state['halted'] is False assert result['completed'] == 2 - # Recreated on the next write, with both entries. - assert set(cluster.progress_configmap()['completed']) == {'200', '300'} - -def test_a_mirror_write_failure_does_not_stall_recording(cluster): - """A 413 from the ConfigMap patch used to throw inside reconcile, and the - loop swallows exceptions -- so no completion would ever be recorded again.""" - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1) - cluster.k8s.fail_next['patch configmap'] = fake_k8s.api_exception( - 413, 'RequestEntityTooLarge') + # Recreated by the next publish, so the driver is not blind for the rest of + # the run. + jm.save_status(jm.status) + assert 'status.json' in cluster.k8s.config_map_data(config.PROGRESS_CM, + cluster.namespace) - result = cluster.reconcile() - assert '300' in cluster.completed() - assert result['completed'] == 1 - assert cluster.state['halted'] is False diff --git a/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py b/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py index 711e3483..eb1d49b0 100644 --- a/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py +++ b/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py @@ -19,6 +19,11 @@ import pytest +import config +import units +import ranges +import records +import attempts import job_monitor as jm # The states the fuzz drives Jobs through. A real run is dominated by success, @@ -49,7 +54,7 @@ def restart(cluster): cluster.state = {'owner': None, 'replayed': set(), 'max_completed': 0, 'halted': False, 'counted': {}} jm._progress_owner.clear() - jm.PROFILE = None + config.PROFILE = None # --- cluster inspection ----------------------------------------------------- @@ -65,8 +70,8 @@ def _jobs_by_range(cluster): for name in cluster.jobs(): job = cluster.k8s.job(name) labels = job.metadata.labels or {} - end = labels.get(jm.LABEL_RANGE) - attempt = int(labels.get(jm.LABEL_ATTEMPT, 1)) + end = labels.get(config.LABEL_RANGE) + attempt = int(labels.get(config.LABEL_ATTEMPT, 1)) out.setdefault(end, []).append((attempt, job)) return out @@ -135,8 +140,8 @@ def check(cluster, result, led, where): # ...and the run never runs wider than it was told to. A restart that # forgot what was in flight would show up here first. - assert len(result['in_progress']) <= jm.PARALLELISM, \ - f"{where}: {len(result['in_progress'])} in flight over PARALLELISM {jm.PARALLELISM}" + assert len(result['in_progress']) <= config.PARALLELISM, \ + f"{where}: {len(result['in_progress'])} in flight over PARALLELISM {config.PARALLELISM}" # A completed range has nothing left to resume, so its volume is gone -- # 79 TiB of orphaned gp3 is what this costs when it regresses. @@ -167,7 +172,7 @@ def check(cluster, result, led, where): # -- I5: recorded peaks are a high-water mark ---------------------------- for end, record in (progress.get('completed') or {}).items(): seen = led.peaks.setdefault(end, {}) - for field in jm.PEAK_FIELDS: + for field in attempts.PEAK_FIELDS: value = record.get(field) if value is None: continue @@ -180,10 +185,10 @@ def check(cluster, result, led, where): # -- I6: one pod per (range, attempt) ------------------------------------ for (_, name), pod in cluster.k8s.pods.items(): labels = pod.metadata.labels or {} - end = labels.get(jm.LABEL_RANGE) + end = labels.get(config.LABEL_RANGE) if end is None: continue - key = (end, labels.get(jm.LABEL_ATTEMPT)) + key = (end, labels.get(config.LABEL_ATTEMPT)) previous = led.pods.setdefault(key, name) assert previous == name, ( f"{where}: range {key[0]} attempt {key[1]} has two distinct pods " @@ -201,7 +206,7 @@ def collector_catches_up(cluster, rng): """ for end, entries in _jobs_by_range(cluster).items(): for attempt, job in entries: - if not _terminal(job) or os.path.exists(jm.done_path(end, attempt)): + if not _terminal(job) or os.path.exists(records.done_path(end, attempt)): continue if rng.random() < 0.35: continue @@ -209,7 +214,7 @@ def collector_catches_up(cluster, rng): end, attempt, tx_apply=round(rng.uniform(0.0, 5.0), 4), peaks={'peakAnonBytes': rng.randrange(1, 20) * 10 ** 8, - 'peakRssBytes': rng.randrange(1, 20) * 10 ** 8}, + 'peakAnonBytes': rng.randrange(1, 20) * 10 ** 8}, resumed=(attempt > 1 and rng.random() < 0.5), attempt_seconds=round(rng.uniform(10.0, 300.0), 2)) @@ -228,8 +233,8 @@ def cluster_moves(cluster, rng): def big_run(cluster, monkeypatch): """Twelve ranges, four at a time -- enough queueing that a dropped range would be silently re-dispatched rather than obviously stuck.""" - monkeypatch.setattr(jm, 'LATEST_LEDGER_NUM', 1200) - monkeypatch.setattr(jm, 'PARALLELISM', 4) + monkeypatch.setattr(config, 'LATEST_LEDGER_NUM', 1200) + monkeypatch.setattr(config, 'PARALLELISM', 4) return cluster @@ -244,7 +249,7 @@ def _observable(cluster): def test_restart_is_invisible_under_fuzz(big_run, seed): cluster = big_run rng = random.Random(seed) - ends = [str(end) for end, _ in jm.generate_ranges()] + ends = [str(end) for end, _ in ranges.generate_ranges()] assert len(ends) == 12 led = Ledger(ends) @@ -293,7 +298,7 @@ def test_restart_is_invisible_under_fuzz(big_run, seed): assert progress.get('completed'), f"seed={seed}: no range ever completed" # Retries have to have actually happened, or the fuzz only exercised the # happy path. - assert any(name.endswith('.verdict') for name in os.listdir(jm.LOG_DIR)), \ + assert any(name.endswith('.verdict') for name in os.listdir(config.LOG_DIR)), \ f"seed={seed}: no attempt ever failed" @@ -304,7 +309,7 @@ def test_restart_does_not_redispatch_a_recorded_range(big_run): cluster.reconcile() for end in ('1200', '1100', '1000', '900'): cluster.advance(int(end), 'succeeded') - cluster.finalize(end, 1, tx_apply=1.0, peaks={'peakRssBytes': 5}) + cluster.finalize(end, 1, tx_apply=1.0, peaks={'peakAnonBytes': 5}) cluster.reconcile() recorded = set(cluster.completed()) assert recorded == {'1200', '1100', '1000', '900'} @@ -338,7 +343,7 @@ def test_restart_mid_retry_keeps_the_attempt_number(big_run): assert cluster.attempt_of(1200) == 3 escalated = (cluster.k8s.job('pc-r1200-a3') .spec.template.spec.containers[0].resources.requests['memory']) - assert jm._quantity_bytes(escalated) > jm._quantity_bytes(limit) + assert units.quantity_bytes(escalated) > units.quantity_bytes(limit) assert cluster.failed() == {} @@ -394,35 +399,31 @@ def test_restart_does_not_halt_on_its_own_progress(big_run): # high-water mark in memory, so a restart disarmed it for exactly the event it # existed to survive, and re-running a range is idempotent anyway. -def test_a_measurement_survives_the_configmap_fallback(big_run): - """I5, in its purest form: a recorded peak that must not go away. +def test_losing_progress_json_costs_a_replay_not_the_measurement(big_run): + """I5 holds because the measurements live in .metrics, not in the record. - The two stores have different jobs. The ConfigMap is the control plane the - mission driver reads, capped at 1 MiB, so _state_only strips every - measurement from it. The volume is the data plane. Falling back to the - mirror therefore returns a record that is complete as state and empty as - data, and the next save wrote that back over the volume. - - The measurements were never really gone: only progress.json was damaged, - and .metrics is written per attempt and never rewritten. load_progress - re-reads them from there rather than persisting the hole. + Losing progress.json used to be papered over by the ConfigMap mirror, which + returned state without data and then persisted that hole over the volume. + Now the record is simply absent and the range becomes eligible again -- + which I5 permits, since it forbids a peak going BACKWARDS, not a record + going away. .metrics is written per attempt and never rewritten, so + re-completing the range restores the same peak rather than a lower one. """ cluster = big_run cluster.reconcile() cluster.advance(1200, 'succeeded') - cluster.finalize('1200', 1, tx_apply=2.5, peaks={'peakRssBytes': 12345}) + cluster.finalize('1200', 1, tx_apply=2.5, peaks={'peakAnonBytes': 12345}) cluster.reconcile() - assert cluster.completed()['1200']['peakRssBytes'] == 12345 + assert cluster.completed()['1200']['peakAnonBytes'] == 12345 - os.remove(jm.PROGRESS_FILE) + os.remove(config.PROGRESS_FILE) restart(cluster) - # Any later completion rewrites the file from the fallback record. - cluster.advance(1100, 'succeeded') - cluster.finalize('1100', 1, tx_apply=1.0, peaks={'peakRssBytes': 999}) cluster.reconcile() - - assert cluster.completed()['1200'].get('peakRssBytes') == 12345 - assert cluster.completed()['1200'].get('txApply') == 2.5 + # Not recovered from it: the range is eligible again rather than carrying a + # state-only entry that the next save would persist over the volume. + assert '1200' not in cluster.completed() + # And the artifacts outlived the record, so a replay can still measure it. + assert attempts.peaks_for_range('1200', 1).get('peakAnonBytes') == 12345 # --- the checker has teeth -------------------------------------------------- @@ -443,7 +444,7 @@ def test_checker_catches_progress_held_in_memory(big_run, monkeypatch): monkeypatch.setattr(jm, 'save_progress', lambda progress: cache.update(progress)) monkeypatch.setattr(cluster, 'progress', lambda: cache) - led = Ledger([str(e) for e, _ in jm.generate_ranges()]) + led = Ledger([str(e) for e, _ in ranges.generate_ranges()]) rng = random.Random(1) with pytest.raises(AssertionError, match='accounted for nowhere|re-dispatched'): for i in range(12): @@ -458,7 +459,7 @@ def test_checker_catches_progress_held_in_memory(big_run, monkeypatch): def test_checker_catches_two_live_jobs_for_one_range(big_run): cluster = big_run - led = Ledger([str(e) for e, _ in jm.generate_ranges()]) + led = Ledger([str(e) for e, _ in ranges.generate_ranges()]) result = cluster.reconcile() check(cluster, result, led, 'mutant pre') @@ -471,16 +472,16 @@ def test_checker_catches_two_live_jobs_for_one_range(big_run): def test_checker_catches_a_peak_going_backwards(big_run): cluster = big_run - led = Ledger([str(e) for e, _ in jm.generate_ranges()]) + led = Ledger([str(e) for e, _ in ranges.generate_ranges()]) cluster.reconcile() cluster.advance(1200, 'succeeded') - cluster.finalize('1200', 1, peaks={'peakRssBytes': 900}) + cluster.finalize('1200', 1, peaks={'peakAnonBytes': 900}) result = cluster.reconcile() check(cluster, result, led, 'mutant pre') record = cluster.progress() - record['completed']['1200']['peakRssBytes'] = 5 - cluster.write(jm.PROGRESS_FILE, json.dumps(record)) + record['completed']['1200']['peakAnonBytes'] = 5 + cluster.write(config.PROGRESS_FILE, json.dumps(record)) with pytest.raises(AssertionError, match='went backwards'): check(cluster, result, led, 'mutant post') @@ -488,7 +489,7 @@ def test_checker_catches_a_peak_going_backwards(big_run): def test_checker_catches_a_replayed_attempt(big_run): cluster = big_run - led = Ledger([str(e) for e, _ in jm.generate_ranges()]) + led = Ledger([str(e) for e, _ in ranges.generate_ranges()]) result = cluster.reconcile() check(cluster, result, led, 'mutant pre') # The range's only Job is destroyed with no record of the range, so the diff --git a/src/MissionParallelCatchup/tests/test_harness_smoke.py b/src/MissionParallelCatchup/tests/test_harness_smoke.py index 903ad5a2..c1dce20b 100644 --- a/src/MissionParallelCatchup/tests/test_harness_smoke.py +++ b/src/MissionParallelCatchup/tests/test_harness_smoke.py @@ -8,6 +8,8 @@ import pytest import fake_k8s +import config +import records import job_monitor as jm @@ -40,7 +42,7 @@ def test_a_succeeded_job_is_recorded_into_completed(cluster): # The collector's half of the contract: peaks and tx_apply are only ever # readable from the files it writes, and the .done marker is what allows a # reap at all. - cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakRssBytes': 123}) + cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakAnonBytes': 123}) cluster.reconcile() @@ -48,15 +50,10 @@ def test_a_succeeded_job_is_recorded_into_completed(cluster): assert record['attempts'] == 1 assert record['count'] == 420 assert record['txApply'] == 1.5 - assert record['peakRssBytes'] == 123 + assert record['peakAnonBytes'] == 123 assert record['seconds'] == pytest.approx(60.0) assert record['wallSeconds'] == pytest.approx(60.0) - # Durable file first, ConfigMap mirror second -- and the mirror is stripped - # of the profiling fields that would push it at the 1 MiB cap. - assert '300' in cluster.progress_configmap()['completed'] - assert 'peakRssBytes' not in cluster.progress_configmap()['completed']['300'] - # A completed range gives its volume back and its Job is reaped. assert 'pc-data-r300' not in cluster.pvcs() assert cluster.deleted.names(verb='delete', kind='job') == ['pc-r300-a1'] @@ -68,9 +65,10 @@ def test_a_succeeded_job_is_recorded_into_completed(cluster): def test_a_failed_job_is_retried(cluster): cluster.reconcile() - # exit 3 is stellar-core's "did not complete": a corrupt archive and an - # interruption are indistinguishable, so it must be retried, not condemned. + # exit 3 is stellar-core's "did not complete" and is retried only when the + # archive shows a fetch fault killed it -- so the decision waits for .done. cluster.advance(300, 'incomplete') + cluster.finalize(300, 1, archive='fetch_fault') cluster.reconcile() @@ -81,12 +79,9 @@ def test_a_failed_job_is_retried(cluster): # The retry rides the same volume -- that is what makes resume-at-LCL work. assert cluster.calls.names(verb='create', kind='pvc').count('pc-data-r300') == 1 - # The predecessor is NOT deleted: the collector has not finalized it, and - # reaping the Job would reap the pod its metrics still live on. - assert 'pc-r300-a1' in cluster.jobs() # ...and the new pod carries the attempt label the collector keys files on. pod = cluster.k8s.pod_for_job('pc-r300-a2') - assert pod.metadata.labels[jm.LABEL_ATTEMPT] == '2' + assert pod.metadata.labels[config.LABEL_ATTEMPT] == '2' def test_a_condemned_range_is_recorded_and_not_retried(cluster): @@ -127,12 +122,12 @@ def test_a_disruption_does_not_spend_the_range_budget(cluster): cluster.reconcile() assert 'pc-r300-a2' in cluster.jobs() - outcome = jm.read_outcome('300', 1) + outcome = records.read_outcome('300', 1) assert outcome['outcome'] == 'disrupted' # Memory is untouched: an eviction says nothing about how much the range wants. resources = (cluster.k8s.job('pc-r300-a2') .spec.template.spec.containers[0].resources) - assert resources.requests['memory'] == jm.REQ_MEM + assert resources.requests['memory'] == config.REQ_MEM def test_progress_going_backwards_redispatches_rather_than_halting(cluster): @@ -147,7 +142,7 @@ def test_progress_going_backwards_redispatches_rather_than_halting(cluster): assert '300' in cluster.completed() # Someone deletes the record underneath the run. - cluster.write(jm.PROGRESS_FILE, '{}') + cluster.write(config.PROGRESS_FILE, '{}') result = cluster.reconcile() # Back in the pool, and the run keeps going instead of halting. @@ -173,3 +168,18 @@ def test_the_fake_raises_the_status_codes_the_monitor_branches_on(cluster): cluster.k8s.core_v1.read_namespaced_persistent_volume_claim( 'pc-data-r999', cluster.namespace) assert gone.value.status == 404 + + +def test_an_unfinalized_predecessor_is_not_deleted(cluster): + """Reaping the Job reaps the pod its measurements still live on. + + Driven through a disruption rather than exit 3: exit 3 now defers until the + collector has finalized, so it can never be observed mid-retry unfinalized. + """ + cluster.reconcile() + cluster.advance(300, 'disrupted') + + cluster.reconcile() + + assert 'pc-r300-a2' in cluster.jobs(), "the successor must exist" + assert 'pc-r300-a1' in cluster.jobs(), "the collector has not finalized a1" diff --git a/src/MissionParallelCatchup/tests/unit/conftest.py b/src/MissionParallelCatchup/tests/unit/conftest.py index 898e6cad..e42a305b 100644 --- a/src/MissionParallelCatchup/tests/unit/conftest.py +++ b/src/MissionParallelCatchup/tests/unit/conftest.py @@ -9,6 +9,7 @@ import pytest +import config import job_monitor as jm import log_collector as lc @@ -18,6 +19,6 @@ def logdir(tmp_path, monkeypatch): """The shared volume, as both processes see it.""" d = tmp_path / 'logs' d.mkdir() - monkeypatch.setattr(jm, 'LOG_DIR', str(d)) - monkeypatch.setattr(lc, 'LOG_DIR', str(d)) + monkeypatch.setattr(config, 'LOG_DIR', str(d)) + monkeypatch.setattr(config, 'LOG_DIR', str(d)) return d diff --git a/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py b/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py index 938fd1bd..3d41b7e0 100644 --- a/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py +++ b/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py @@ -12,9 +12,15 @@ import gzip import io import json +import os import pytest +import config +import units +import records +import sizing +import attempts import job_monitor as jm @@ -30,100 +36,65 @@ def _gzip_member(text): def _archive(end, attempt, *members): - with open(jm.log_path(end, attempt), 'wb') as fh: + with open(records.log_path(end, attempt), 'wb') as fh: for member in members: fh.write(_gzip_member(member)) @pytest.fixture -def attempts(logdir): +def write_attempts(logdir): """Lay down the files the collector and the monitor leave per attempt.""" def write(end, spec): for n, (metrics, outcome) in spec.items(): if metrics is not None: - with open(jm.metrics_path(end, n), 'w') as fh: + with open(records.metrics_path(end, n), 'w') as fh: fh.write(metrics if isinstance(metrics, str) else json.dumps(metrics)) if outcome is not None: - with open(jm.outcome_path(end, n), 'w') as fh: + with open(records.outcome_path(end, n), 'w') as fh: json.dump(outcome, fh) return write # --- which attempts describe the range --------------------------------------- -def test_the_chain_is_the_run_of_resumed_attempts_ending_at_this_one(attempts): +def test_the_chain_is_the_run_of_resumed_attempts_ending_at_this_one(write_attempts): # a1 interrupted then superseded by a fresh a2; a3 resumed from a2. Only # a2+a3 describe the same continuous pass over the range. - attempts(999, {1: ({}, None), 2: ({}, None), 3: ({'resumed': True}, None)}) - assert list(jm._resumed_chain(999, 3)) == [2, 3] - assert list(jm._resumed_chain(999, 1)) == [1] + write_attempts(999, {1: ({}, None), 2: ({}, None), 3: ({'resumed': True}, None)}) + assert list(attempts._resumed_chain(999, 3)) == [2, 3] + assert list(attempts._resumed_chain(999, 1)) == [1] -def test_an_attempt_with_no_metrics_file_is_not_treated_as_resumed(attempts): - attempts(999, {1: ({}, None)}) - assert jm._attempt_resumed(999, 2) is False +def test_an_attempt_with_no_metrics_file_is_not_treated_as_resumed(write_attempts): + write_attempts(999, {1: ({}, None)}) + assert attempts._attempt_resumed(999, 2) is False -def test_resume_falls_back_to_the_archive_when_metrics_lacks_the_field(attempts): - attempts(999, {2: ({'attemptSeconds': 300.0}, None)}) - _archive(999, 2, 'RESUME: reached ledger 900; skipping new-db\n') +def test_a_three_attempt_chain_is_read_from_the_records(write_attempts): + write_attempts(999, {1: ({}, None), 2: ({'resumed': True}, None), + 3: ({'resumed': True}, None)}) - assert jm._attempt_resumed(999, 2) is True - - -def test_resume_declined_is_not_a_true_resume(attempts): - attempts(999, {2: ({}, None)}) - _archive(999, 2, 'RESUME DECLINED: no usable local state; running new-db\n') - - assert jm._attempt_resumed(999, 2) is False - - -def test_resume_is_found_across_concatenated_gzip_members(attempts): - attempts(999, {2: ({}, None)}) - _archive(999, 2, 'worker startup\n', - 'RESUME: reached ledger 900; skipping new-db\n') - - assert jm._attempt_resumed(999, 2) is True - - -def test_missing_truncated_and_corrupt_archives_are_safe(attempts): - attempts(999, {2: ({}, None), 3: ({}, None), 4: ({}, None)}) - with open(jm.log_path(999, 3), 'wb') as fh: - fh.write(_gzip_member('worker startup\n')[:-8]) - with open(jm.log_path(999, 4), 'wb') as fh: - fh.write(b'not a gzip archive') - - assert jm._attempt_resumed(999, 2) is False - assert jm._attempt_resumed(999, 3) is False - assert jm._attempt_resumed(999, 4) is False - - -def test_three_attempt_chain_can_be_recovered_entirely_from_archives(attempts): - attempts(999, {1: ({}, None), 2: ({}, None), 3: ({}, None)}) - _archive(999, 2, 'RESUME: reached ledger 700; skipping new-db\n') - _archive(999, 3, 'RESUME: reached ledger 800; skipping new-db\n') - - assert list(jm._resumed_chain(999, 3)) == [1, 2, 3] + assert list(attempts._resumed_chain(999, 3)) == [1, 2, 3] # --- peaks -------------------------------------------------------------------- -def test_a_resumed_range_keeps_the_peak_from_the_attempt_that_did_the_download(attempts): +def test_a_resumed_range_keeps_the_peak_from_the_attempt_that_did_the_download(write_attempts): # a1 evicted mid-replay having already done the download; a2 resumes at # LCL+1 and only replays the tail. a2 alone would report 400MiB for a range # that really needs 2GiB. - attempts(999, {1: ({'peakAnonBytes': 2 * GIB}, {'outcome': 'disrupted'}), + write_attempts(999, {1: ({'peakAnonBytes': 2 * GIB}, {'outcome': 'disrupted'}), 2: ({'peakAnonBytes': 400 * MIB, 'resumed': True}, None)}) - assert jm.peaks_for_range(999, 2)['peakAnonBytes'] == 2 * GIB + assert attempts.peaks_for_range(999, 2)['peakAnonBytes'] == 2 * GIB -def test_a_fresh_retry_supersedes_an_interrupted_one(attempts): +def test_a_fresh_retry_supersedes_an_interrupted_one(write_attempts): # No RESUME line means new-db ran and this attempt did the whole range, so # its sample is complete. An earlier attempt that was merely interrupted # measured the same work and only adds noise. - attempts(999, {1: ({'peakAnonBytes': 8 * GIB}, {'outcome': 'disrupted'}), + write_attempts(999, {1: ({'peakAnonBytes': 8 * GIB}, {'outcome': 'disrupted'}), 2: ({'peakAnonBytes': 900 * MIB}, None)}) - assert jm.peaks_for_range(999, 2)['peakAnonBytes'] == 900 * MIB + assert attempts.peaks_for_range(999, 2)['peakAnonBytes'] == 900 * MIB @pytest.mark.parametrize('outcome,field,hit,quiet', [ @@ -133,7 +104,7 @@ def test_a_fresh_retry_supersedes_an_interrupted_one(attempts): ('ephemeral', 'peakAnonBytes', 3 * GIB, 1 * GIB), ]) @pytest.mark.parametrize('resumed', [True, False]) -def test_an_attempt_killed_at_a_ceiling_counts_wherever_it_sits(attempts, outcome, +def test_an_attempt_killed_at_a_ceiling_counts_wherever_it_sits(write_attempts, outcome, field, hit, quiet, resumed): # A pod OOM-killed at 8Gi really did allocate ~8Gi and wanted more, so its # peak is a lower bound on demand, not an artifact of the limit -- and it is @@ -150,193 +121,196 @@ def test_an_attempt_killed_at_a_ceiling_counts_wherever_it_sits(attempts, outcom later = {field: quiet} if resumed: later['resumed'] = True - attempts(999, {1: ({field: hit}, {'outcome': outcome}), 2: (later, None)}) - assert jm.peaks_for_range(999, 2)[field] == hit + write_attempts(999, {1: ({field: hit}, {'outcome': outcome}), 2: (later, None)}) + assert attempts.peaks_for_range(999, 2)[field] == hit -def test_the_ceiling_exception_is_peaks_only(attempts): +def test_the_ceiling_exception_is_peaks_only(write_attempts): # tx_apply and seconds are summed, and a fresh start redoes the work the # dropped attempt already did, so counting it there would double-count. - attempts(999, {1: ({'txApplySeconds': 100.0, 'attemptSeconds': 900.0}, + write_attempts(999, {1: ({'txApplySeconds': 100.0, 'attemptSeconds': 900.0}, {'outcome': 'oom'}), 2: ({'txApplySeconds': 7.0}, None)}) # fresh start - assert jm.tx_apply_for_range(999, 2) == 7.0 - assert jm.seconds_for_range(999, 2, 300.0) == 300.0 + assert attempts.tx_apply_for_range(999, 2) == 7.0 + assert attempts.seconds_for_range(999, 2, 300.0) == 300.0 -def test_a_missing_or_malformed_metrics_file_is_tolerated(attempts): - attempts(999, {2: ("not json at all", None), +def test_a_missing_or_malformed_metrics_file_is_tolerated(write_attempts): + write_attempts(999, {2: ("not json at all", None), 3: ({'peakAnonBytes': 5, 'resumed': True}, None)}) - assert jm.peaks_for_range(999, 3) == {'peakAnonBytes': 5} - assert jm.peaks_for_range(999, 9) == {} + assert attempts.peaks_for_range(999, 3) == {'peakAnonBytes': 5} + assert attempts.peaks_for_range(999, 9) == {} -def test_an_absent_peak_never_reaches_the_profile_as_a_null(attempts): +def test_an_absent_peak_never_reaches_the_profile_as_a_null(write_attempts): # The consumer falls back to a default on a missing field, so a null defeats it. - attempts(999, {1: ({'peakAnonBytes': None, 'peakRssBytes': 7}, None)}) - assert jm.peaks_for_range(999, 1) == {'peakRssBytes': 7} + write_attempts(999, {1: ({'peakAnonBytes': None, 'peakAnonBytes': 7}, None)}) + assert attempts.peaks_for_range(999, 1) == {'peakAnonBytes': 7} def test_both_measured_peaks_reach_the_progress_record(): - # peaks_for_range filters to PEAK_FIELDS and the ConfigMap mirror strips - # _PROFILE_ONLY_FIELDS; a measurement absent from either is silently - # dropped between the collector and the profile. + # peaks_for_range filters to PEAK_FIELDS; a measurement missing from it is + # dropped silently between the collector and the profile. for field in ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes'): - assert field in jm.PEAK_FIELDS, field - assert 'peakAnonBytes' in jm._PROFILE_ONLY_FIELDS + assert field in attempts.PEAK_FIELDS, field # --- durations ---------------------------------------------------------------- -def test_seconds_sums_the_whole_resumed_chain(attempts): +def test_seconds_sums_the_whole_resumed_chain(write_attempts): # a1 ran 900s then was evicted mid-replay; a2 resumed and took 300s. The # range cost 1200s of compute, not 300. - attempts(999, {1: ({}, {'outcome': 'disrupted', 'attemptSeconds': 900.0}), + write_attempts(999, {1: ({}, {'outcome': 'disrupted', 'attemptSeconds': 900.0}), 2: ({'resumed': True}, None)}) - assert jm.seconds_for_range(999, 2, 300.0) == 1200.0 + assert attempts.seconds_for_range(999, 2, 300.0) == 1200.0 -def test_seconds_ignores_attempts_before_a_fresh_start(attempts): +def test_seconds_ignores_attempts_before_a_fresh_start(write_attempts): # a2 ran new-db and did the whole range itself, so a1's 900s is not part of # the same pass. - attempts(999, {1: ({}, {'outcome': 'oom', 'attemptSeconds': 900.0}), + write_attempts(999, {1: ({}, {'outcome': 'oom', 'attemptSeconds': 900.0}), 2: ({}, None)}) - assert jm.seconds_for_range(999, 2, 300.0) == 300.0 + assert attempts.seconds_for_range(999, 2, 300.0) == 300.0 -def test_seconds_is_absent_when_a_resumed_leg_has_no_recorded_duration(attempts): +def test_seconds_is_absent_when_a_resumed_leg_has_no_recorded_duration(write_attempts): # Winner-only is a lower bound, not the chain total. Missing accurately # tells the profile consumer not to size from it. - attempts(999, {1: ({}, {'outcome': 'disrupted'}), 2: ({'resumed': True}, None)}) - assert jm.seconds_for_range(999, 2, 300.0) is None + write_attempts(999, {1: ({}, {'outcome': 'disrupted'}), 2: ({'resumed': True}, None)}) + assert attempts.seconds_for_range(999, 2, 300.0) is None -def test_seconds_is_none_when_nothing_is_known(attempts): - attempts(999, {1: ({}, None)}) - assert jm.seconds_for_range(999, 1, None) is None +def test_seconds_is_none_when_nothing_is_known(write_attempts): + write_attempts(999, {1: ({}, None)}) + assert attempts.seconds_for_range(999, 1, None) is None -def test_seconds_falls_back_to_the_collectors_figure(attempts): +def test_seconds_falls_back_to_the_collectors_figure(write_attempts): # The authoritative .outcome is missing for every reaped pod -- measured on # ssc-test 2026-07-30, 212 of 212 spot disruptions were classified from the # Job condition with the pod already gone, so record_outcome never ran. # Without this fallback the chain drops that leg entirely. - attempts(999, {1: ({'attemptSeconds': 850.0}, None), + write_attempts(999, {1: ({'attemptSeconds': 850.0}, None), 2: ({'resumed': True}, None)}) - assert jm.seconds_for_range(999, 2, 300.0) == 1150.0 + assert attempts.seconds_for_range(999, 2, 300.0) == 1150.0 + +def test_a_poller_clock_estimate_is_refused_as_a_chain_leg(write_attempts): + # attemptSecondsExact False with no other provenance means the figure came + # from the collector's own clock, which starts when that process attached -- + # a lower bound, not the attempt. Summing it would publish a total that is + # quietly short. + write_attempts(999, {1: ({'attemptSeconds': 850.0, 'attemptSecondsExact': False}, None), + 2: ({'resumed': True}, None)}) + assert attempts.seconds_for_range(999, 2, 300.0) is None + + +def test_a_duration_dated_from_container_start_is_accepted(write_attempts): + # The only duration a DISRUPTED attempt can produce. A pod being deleted + # keeps phase Running, so its terminated timestamps are usually never + # observed and the exact path never fires; dating from the container's own + # startTime measured within 1% on ssc-test (370.9s and 375.1s against ~373s) + # where the poller clock was 46% short. Refusing it left every resumed chain + # with no `seconds` at all, which is the whole reason spot runs came back + # unprofiled. + write_attempts(999, {1: ({'attemptSeconds': 850.0, 'attemptSecondsExact': False, + 'attemptSecondsFromContainerStart': True}, None), + 2: ({'resumed': True}, None)}) + assert attempts.seconds_for_range(999, 2, 300.0) == 1150.0 -def test_the_authoritative_outcome_wins_over_the_collector_estimate(attempts): + +def test_the_authoritative_outcome_wins_over_the_collector_estimate(write_attempts): # .outcome comes from the pod's terminated timestamps; the collector's is a # stream-lifetime approximation that starts up to one poll late. - attempts(999, {1: ({'attemptSeconds': 850.0}, + write_attempts(999, {1: ({'attemptSeconds': 850.0}, {'outcome': 'disrupted', 'attemptSeconds': 900.0}), 2: ({'resumed': True}, None)}) - assert jm.seconds_for_range(999, 2, 300.0) == 1200.0 + assert attempts.seconds_for_range(999, 2, 300.0) == 1200.0 # --- completed profile reconstruction ----------------------------------------- -def test_repair_recovers_predecessor_peaks_and_seconds_idempotently(attempts): - attempts(999, { +def test_repair_recovers_predecessor_peaks_and_seconds_idempotently(write_attempts): + write_attempts(999, { 1: ({'attemptSeconds': 900.0, 'peakAnonBytes': 2 * GIB, 'peakWorkingSetBytes': 3 * GIB}, None), 2: ({'attemptSeconds': 300.0, 'peakAnonBytes': 400 * MIB, - 'peakWorkingSetBytes': 500 * MIB}, None), + 'peakWorkingSetBytes': 500 * MIB, 'resumed': True}, None), }) - _archive(999, 2, 'RESUME: reached ledger 800; skipping new-db\n') - progress = {'completed': {'999': { - 'attempts': 2, 'seconds': 300.0, - 'peakAnonBytes': 400 * MIB, 'peakWorkingSetBytes': 500 * MIB, - }}} + record = {'attempts': 2, 'seconds': 300.0, + 'peakAnonBytes': 400 * MIB, 'peakWorkingSetBytes': 500 * MIB} - assert jm.repair_completed_profiles(progress) == 1 - repaired = progress['completed']['999'] - assert repaired['seconds'] == 1200.0 - assert repaired['peakAnonBytes'] == 2 * GIB - assert repaired['peakWorkingSetBytes'] == 3 * GIB + assert attempts._repair_completed_profile('999', 2, record) + assert record['seconds'] == 1200.0 + assert record['peakAnonBytes'] == 2 * GIB + assert record['peakWorkingSetBytes'] == 3 * GIB - snapshot = json.loads(json.dumps(progress)) - assert jm.repair_completed_profiles(progress) == 0 - assert progress == snapshot + snapshot = json.loads(json.dumps(record)) + assert not attempts._repair_completed_profile('999', 2, record) + assert record == snapshot -def test_reconstruction_omits_txapply_when_one_chain_leg_is_missing(attempts): - attempts(999, { +def test_reconstruction_omits_txapply_when_one_chain_leg_is_missing(write_attempts): + write_attempts(999, { 1: ({'txApplySeconds': 10.0}, None), - 2: ({}, None), # this leg's metric was unavailable - 3: ({'txApplySeconds': 3.0}, None), + 2: ({'resumed': True}, None), # this leg's metric was unavailable + 3: ({'txApplySeconds': 3.0, 'resumed': True}, None), }) - _archive(999, 2, 'RESUME: reached ledger 700; skipping new-db\n') - _archive(999, 3, 'RESUME: reached ledger 800; skipping new-db\n') - rebuilt = jm.reconstruct_completed_profile(999, 3) + rebuilt = attempts.reconstruct_completed_profile(999, 3) assert 'txApply' not in rebuilt -def test_reconstruction_leaves_txapply_absent_when_every_leg_is_missing(attempts): - attempts(999, {1: ({}, None), 2: ({}, None)}) - _archive(999, 2, 'RESUME: reached ledger 800; skipping new-db\n') +def test_reconstruction_leaves_txapply_absent_when_every_leg_is_missing(write_attempts): + write_attempts(999, {1: ({}, None), 2: ({'resumed': True}, None)}) - assert 'txApply' not in jm.reconstruct_completed_profile(999, 2) + assert 'txApply' not in attempts.reconstruct_completed_profile(999, 2) -def test_repair_removes_legacy_winner_only_chain_aggregates(attempts): - attempts(999, { +def test_repair_removes_legacy_winner_only_chain_aggregates(write_attempts): + write_attempts(999, { 1: ({}, {'outcome': 'disrupted'}), 2: ({'resumed': True, 'attemptSeconds': 300.0, 'txApplySeconds': 3.0}, None), }) - progress = {'completed': {'999': { - 'attempts': 2, 'seconds': 300.0, 'txApply': 3.0, - }}} + record = {'attempts': 2, 'seconds': 300.0, 'txApply': 3.0} - assert jm.repair_completed_profiles(progress) == 1 - assert 'seconds' not in progress['completed']['999'] - assert 'txApply' not in progress['completed']['999'] - assert jm.repair_completed_profiles(progress) == 0 + assert attempts._repair_completed_profile('999', 2, record) + assert 'seconds' not in record + assert 'txApply' not in record + assert not attempts._repair_completed_profile('999', 2, record) -def test_reconstruction_does_not_cross_a_fresh_restart_boundary(attempts): - attempts(999, { +def test_reconstruction_does_not_cross_a_fresh_restart_boundary(write_attempts): + write_attempts(999, { 1: ({'attemptSeconds': 900.0, 'txApplySeconds': 100.0, 'peakAnonBytes': 8 * GIB}, None), + # No resume marker: new-db ran, so this attempt starts the chain. 2: ({'attemptSeconds': 300.0, 'txApplySeconds': 7.0, 'peakAnonBytes': 900 * MIB}, None), 3: ({'attemptSeconds': 60.0, 'txApplySeconds': 2.0, - 'peakAnonBytes': 400 * MIB}, None), + 'peakAnonBytes': 400 * MIB, 'resumed': True}, None), }) - _archive(999, 2, 'RESUME DECLINED: running new-db\n') - _archive(999, 3, 'RESUME: reached ledger 950; skipping new-db\n') - rebuilt = jm.reconstruct_completed_profile(999, 3) + rebuilt = attempts.reconstruct_completed_profile(999, 3) assert rebuilt['seconds'] == 360.0 assert rebuilt['txApply'] == 9.0 assert rebuilt['peakAnonBytes'] == 900 * MIB -def test_reconcile_repairs_a_completed_record_with_no_live_job(cluster): - cluster.write(jm.PROGRESS_FILE, json.dumps({ - 'completed': {'300': {'attempts': 2, 'count': 100, 'seconds': 300.0, - 'txApply': 2.0, 'peakAnonBytes': 400 * MIB}}, - 'failed': {}, - })) - cluster.finalize(300, 1, tx_apply=10.0, attempt_seconds=900.0, - peaks={'peakAnonBytes': 2 * GIB}) - cluster.finalize(300, 2, tx_apply=2.0, attempt_seconds=300.0, - peaks={'peakAnonBytes': 400 * MIB}) - _archive(300, 2, 'RESUME: reached ledger 250; skipping new-db\n') +def test_wall_seconds_spans_a_resumed_chain_from_attempt_one(cluster): + """wallSeconds is the range's whole life, retries and gaps included. + It used to be omitted for a resumed chain, because the winning Job's own + start covers the LAST leg only and read smaller than chain-summed `seconds`. + Anchoring on attempt 1's creationTimestamp removes that inversion: the span + contains every leg plus every gap between them, so wall - seconds is the + overhead the Job-per-range design introduced. + """ cluster.reconcile() + first_start = jm.range_started_at(300) + assert first_start is not None, "attempt 1's Job creation must be recorded" - repaired = cluster.completed()['300'] - assert repaired['seconds'] == 1200.0 - assert repaired['txApply'] == 12.0 - assert repaired['peakAnonBytes'] == 2 * GIB - - -def test_wall_seconds_is_winner_job_only_while_compute_spans_the_chain(cluster): - cluster.reconcile() cluster.advance(300, 'disrupted') cluster.finalize(300, 1, tx_apply=10.0, attempt_seconds=60.0) cluster.reconcile() @@ -348,30 +322,94 @@ def test_wall_seconds_is_winner_job_only_while_compute_spans_the_chain(cluster): record = cluster.completed()['300'] assert record['seconds'] == 120.0 assert record['txApply'] == 12.0 - assert record['wallSeconds'] == 60.0 + # Present, and still anchored at attempt 1 -- attempt 2's dispatch must not + # re-stamp it, or the span silently shrinks to the last leg again. + assert record['wallSeconds'] is not None + assert jm.range_started_at(300) == first_start -def test_disrupted_predecessor_without_final_medida_makes_txapply_absent(attempts): - attempts(999, { +def test_wall_seconds_is_absent_when_attempt_one_was_never_recorded(cluster): + """No anchor means no wall, rather than a winner-only span. + + Falling back to the winning Job's own start would measure one leg and + understate exactly the overhead this field exists to expose, so absent is + the honest answer -- the same choice `seconds` and `txApply` make when a + chain leg is missing. + """ + cluster.reconcile() + os.remove(records.started_path(300)) + + cluster.advance(300, 'succeeded') + cluster.finalize(300, 1, tx_apply=1.5, attempt_seconds=60.0) + cluster.reconcile() + + record = cluster.completed()['300'] + assert record['seconds'] == pytest.approx(60.0) + assert record['wallSeconds'] is None + + +def test_disrupted_predecessor_without_final_medida_makes_txapply_absent(write_attempts): + write_attempts(999, { 1: ({'attemptSeconds': 900.0}, {'outcome': 'disrupted'}), 2: ({'resumed': True, 'txApplySeconds': 3.0}, None), }) - assert jm.tx_apply_for_range(999, 2) is None - assert 'txApply' not in jm.reconstruct_completed_profile(999, 2) + assert attempts.tx_apply_for_range(999, 2) is None + assert 'txApply' not in attempts.reconstruct_completed_profile(999, 2) # --- counting causes, not attempts -------------------------------------------- -def test_escalation_counts_ooms_not_attempts(attempts): +def test_escalation_counts_ooms_not_attempts(write_attempts): # On spot most retries are evictions: 288 disruption retries against 7 OOM # retries on ssc-test 2026-07-30. Keying the exponent on the attempt index # meant a range disrupted three times then OOMing once jumped to # base * 1.5^4 -- a 5x request for one OOM, inflated fleet-wide. - attempts(9, {1: (None, {'outcome': 'disrupted'}), + write_attempts(9, {1: (None, {'outcome': 'disrupted'}), 2: (None, {'outcome': 'disrupted'}), 3: (None, {'outcome': 'disrupted'}), 4: (None, {'outcome': 'oom'})}) - assert jm._oom_count(9, 4) == 1, "three evictions were counted as escalations" - attempts(9, {5: (None, {'outcome': 'oom'}), 6: (None, {'outcome': 'oom'})}) - assert jm._oom_count(9, 6) == 3 + assert records._oom_count(9, 4) == 1, "three evictions were counted as escalations" + write_attempts(9, {5: (None, {'outcome': 'oom'}), 6: (None, {'outcome': 'oom'})}) + assert records._oom_count(9, 6) == 3 + + +def test_disk_escalation_counts_evictions_not_attempts(write_attempts, monkeypatch): + """Same inflation as the OOM ladder, on the disk ladder. + + The budget check for an ephemeral eviction already counts causes; the SIZE + did not, so a range disrupted four times then evicted once escalated as if + it had been evicted five times. + """ + monkeypatch.setattr(config, 'LIM_EPHEMERAL', '4Gi') + monkeypatch.setattr(config, 'EPH_BUMP_FACTOR', 1.5) + write_attempts(9, {1: (None, {'outcome': 'disrupted'}), + 2: (None, {'outcome': 'disrupted'}), + 3: (None, {'outcome': 'disrupted'}), + 4: (None, {'outcome': 'disrupted'}), + 5: (None, {'outcome': 'ephemeral'})}) + + evictions = records._cause_count(9, 5, ('ephemeral',)) + assert evictions == 1, "four disruptions were counted as disk escalations" + # One rung, not five: 4Gi -> 6Gi, where attempt-indexing gave 4Gi * 1.5^5. + bytes_of = units.quantity_bytes + assert bytes_of(sizing.eph_for_attempt(evictions + 1)) == bytes_of('6Gi') + assert bytes_of(sizing.eph_for_attempt(evictions)) == bytes_of('4Gi'), \ + "the limit this attempt ran at" + + +def test_disk_escalation_climbs_on_each_real_eviction(write_attempts, monkeypatch): + monkeypatch.setattr(config, 'LIM_EPHEMERAL', '4Gi') + monkeypatch.setattr(config, 'EPH_BUMP_FACTOR', 1.5) + write_attempts(9, {1: (None, {'outcome': 'ephemeral'}), + 2: (None, {'outcome': 'ephemeral'})}) + + assert records._cause_count(9, 2, ('ephemeral',)) == 2 + assert units.quantity_bytes(sizing.eph_for_attempt(3)) == units.quantity_bytes('9Gi') + + +def test_the_disk_ladder_is_capped(monkeypatch): + monkeypatch.setattr(config, 'LIM_EPHEMERAL', '4Gi') + monkeypatch.setattr(config, 'EPH_BUMP_FACTOR', 1.5) + monkeypatch.setattr(config, 'EPH_ESCALATION_CAP', '20Gi') + assert units.quantity_bytes(sizing.eph_for_attempt(99)) == units.quantity_bytes('20Gi') diff --git a/src/MissionParallelCatchup/tests/unit/test_classify.py b/src/MissionParallelCatchup/tests/unit/test_classify.py index d3c0e560..058d94e3 100644 --- a/src/MissionParallelCatchup/tests/unit/test_classify.py +++ b/src/MissionParallelCatchup/tests/unit/test_classify.py @@ -10,6 +10,7 @@ import pytest from kubernetes import client +import config import job_monitor as jm import log_collector as lc @@ -108,15 +109,19 @@ def test_a_bare_exit_code_is_read_when_no_rule_index_is_offered(): def test_an_unclassifiable_job_failure_stays_unclassified(): - # BackoffLimitExceeded carries no rule index and no exit code, so classify - # honestly returns nothing rather than guessing. A monitor restart while a - # node was reaped produces exactly this, and condemning on it would fail a - # 10-hour job on no evidence -- reconcile gives it the environmental budget. + """classify returns nothing rather than guessing, and reconcile condemns. + + BackoffLimitExceeded carries no rule index and no exit code. An + unclassifiable failure is not evidence that the range is fine, so the run + stops rather than retrying blind -- only a node disruption, which proves the + cluster took the pod away, keeps its unlimited budget. + """ assert jm.classify_from_job(failed_job( "Job has reached the specified backoff limit", reason='BackoffLimitExceeded')) is None - assert 'unknown' in jm.ENVIRONMENTAL_OUTCOMES - assert {'disrupted', 'rejected'} <= set(jm.ENVIRONMENTAL_OUTCOMES) + assert set(config.ATTEMPT_BUDGETS) == {'disrupted', 'rejected', 'fetch-fault', + 'oom', 'ephemeral'}, \ + "an unclassifiable failure must have no budget at all" def test_a_deadline_exceeded_job_is_a_timeout_not_a_catchup_failure(): @@ -139,6 +144,31 @@ def test_a_disruption_target_condition_outranks_everything_on_the_pod(): assert got['outcome'] == 'disrupted' +def test_the_disruption_reason_separates_a_warning_from_a_postmortem(): + # The bare condition cannot tell these apart, and the difference decides + # whether a missing txApply is a capture bug worth chasing. An eviction is a + # drain that still owes the container a SIGTERM, so a medida block is coming. + # A TaintManager stamp lands ~40s after the node went NotReady, on a process + # that already died unsignalled -- nothing was ever written to miss. + def reason_for(r): + return lc._is_condemned(as_dict( + conditions=[{'type': 'DisruptionTarget', 'status': 'True', 'reason': r}])) + + assert reason_for('EvictionByEvictionAPI') == 'EvictionByEvictionAPI' + assert reason_for('DeletionByTaintManager') == 'DeletionByTaintManager' + # Truthiness is the contract every caller relies on, not the string itself. + assert lc._is_condemned(as_dict()) is None + assert not lc._is_condemned(as_dict( + conditions=[{'type': 'DisruptionTarget', 'status': 'False'}])) + + +def test_a_condition_without_a_reason_still_reads_as_condemned(): + # Kubernetes does not promise the field. Falling back to None here would + # silently un-condemn the pod and drop it back to the lazy poll cadence. + assert lc._is_condemned(as_dict( + conditions=[{'type': 'DisruptionTarget', 'status': 'True'}])) == 'Unknown' + + def test_an_ephemeral_eviction_is_not_read_as_an_oom_or_a_disruption(): # Measured end-to-end on ssc-test: the kubelet sets no DisruptionTarget, # and stellar-core drains and exits 3, so the Job condition is a plain @@ -204,11 +234,14 @@ def test_a_non_zero_exit_is_a_catchup_failure_and_keeps_its_code(): assert (got['outcome'], got['exitCode']) == ('failed', 3) -def test_exit_three_is_the_ambiguous_one_and_never_inherits_a_bigger_budget(): - # stellar-core drains to 3 on SIGTERM and a corrupt bucket also exits 3, so - # reconcile retries it on the ordinary range budget -- but a genuinely - # corrupt range must still be able to exhaust rather than retry 20 times. - assert jm.CATCHUP_INCOMPLETE_EXIT == 3 - assert 'failed' not in jm.ENVIRONMENTAL_OUTCOMES - assert 'ephemeral' not in jm.ENVIRONMENTAL_OUTCOMES, \ +def test_exit_three_is_the_ambiguous_one_and_is_decided_by_the_archive(): + """A corrupt range is protected by the exit-3 rule, not by its budget. + + stellar-core drains to 3 on SIGTERM and a corrupt bucket also exits 3, so + the exit code decides nothing. An exit 3 is condemned outright unless its + archive names a fetch fault -- so a genuinely corrupt range never reaches a + budget at all, and the fetch-fault retry that does is infrastructure. + """ + assert config.CATCHUP_INCOMPLETE_EXIT == 3 + assert config.ATTEMPT_BUDGETS['ephemeral'] < config.ATTEMPT_BUDGETS['disrupted'], \ "a deterministic failure must not get the disruption budget" diff --git a/src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py b/src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py index f3583447..0de862f8 100644 --- a/src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py +++ b/src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py @@ -18,16 +18,19 @@ import pytest +import config import log_collector as lc -def pod(name, phase='Running', end='300', attempt='1', node='node-1'): +def pod(name, phase='Running', end='300', attempt='1', node='node-1', ip=None): + # hostIP is what the sampler reads: it talks to the kubelet directly rather + # than through the apiserver's node proxy. return {'metadata': {'name': name, - 'labels': {lc.LABEL_RUN: lc.RUN_NAME, - lc.LABEL_RANGE: end, - lc.LABEL_ATTEMPT: attempt}}, + 'labels': {config.LABEL_RUN: config.RUN_NAME, + config.LABEL_RANGE: end, + config.LABEL_ATTEMPT: attempt}}, 'spec': {'nodeName': node}, - 'status': {'phase': phase}} + 'status': {'phase': phase, 'hostIP': ip or f"10.0.0.{abs(hash(node)) % 200 + 1}"}} class Loop: @@ -80,7 +83,7 @@ async def run(): @pytest.fixture def loop_env(tmp_path, monkeypatch): - monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) monkeypatch.setattr(lc, 'token', lambda: 'tok') monkeypatch.setattr(lc, 'ssl_ctx', lambda: None) monkeypatch.setattr(lc, 'POLL_SECONDS', 0.01) @@ -108,9 +111,13 @@ async def _drive(loop, extra, want_survivors=False): await asyncio.sleep(0.005) if loop.passes >= want: break + # Stream tasks only. The condemnation watch is also long-lived by design -- + # it is supposed to outlive every poller -- so counting it here would read + # as a wedged stream that never gave its slot back. survivors = [t for t in asyncio.all_tasks() if t is not asyncio.current_task() and t is not task - and not t.done()] + and not t.done() + and 'watch_condemnations' not in repr(t.get_coro())] task.cancel() try: await task @@ -121,19 +128,29 @@ async def _drive(loop, extra, want_survivors=False): # --- the sampler --------------------------------------------------------------- -def test_the_sampler_runs_before_the_per_pod_branches(loop_env): - """The per-pod branches all end in `continue` for a pod already streaming, - so a sampler placed after them fires only on the cycle a stream opens -- - when the range has written almost nothing and its peak is meaningless.""" +def test_the_sampler_never_delays_opening_a_stream(loop_env): + """The sampler is a serial sweep of every node's kubelet, and on spot a dead + one costs the whole connect timeout. Measured at 900 workers it stretched a + cycle to 925s, and ahead of the per-pod branches that delay applied to every + stream: five -a2 legs died with no reader, one after 184.7s. Opening a stream + is time-critical, so it goes first and the sampler takes the wait.""" loop = run_loop(loop_env, [[pod('w-1')], [pod('w-1')], [pod('w-1')]]) - assert loop.order[:3] == ['list', 'sample', 'open:w-1'] - # One listing per cycle and one sample per listing: the sampler reuses the - # pod list rather than fetching its own. + assert loop.order[:3] == ['list', 'open:w-1', 'sample'] + # Still once per cycle, and still off the same listing rather than its own. assert loop.order.count('sample') == loop.order.count('list') - for i, event in enumerate(loop.order): - if event == 'sample': - assert loop.order[i - 1] == 'list' + + +def test_the_sampler_stays_outside_the_per_pod_loop(loop_env): + """It has to run every cycle, not once per stream. Those branches end in + `continue` for a pod already streaming, so a sampler placed among them fires + only on the cycle a stream opens -- when the range has written almost nothing + and its peak is meaningless.""" + loop = run_loop(loop_env, [[pod('w-1')]] * 4) + + # One sample per listing even though only the first cycle opens anything. + assert loop.order.count('sample') == loop.order.count('list') + assert loop.order.count('open:w-1') == 1 def test_the_sampler_runs_every_cycle_not_once_per_stream(loop_env): @@ -146,19 +163,19 @@ def test_the_sampler_runs_every_cycle_not_once_per_stream(loop_env): def test_the_sampler_is_not_gated_on_storage_mode(loop_env): """It was, back when it only sampled disk. Memory is sized in both modes, so gating here left every pvc run with no anon peak at all.""" - loop_env.setattr(lc, 'STORAGE_MODE', 'pvc') + loop_env.setattr(config, 'STORAGE_MODE', 'pvc') loop = run_loop(loop_env, [[pod('w-1')], [pod('w-1')]]) - assert loop.sampled and loop.sampled[0] == {'node-1'} + assert loop.sampled and loop.sampled[0] == {pod('w-1')['status']['hostIP']} def test_only_running_pods_are_handed_to_the_sampler(loop_env): """kubelet has no live stats for a pod that has not started or has exited, and every extra node in the set is another /stats/summary GET.""" - loop = run_loop(loop_env, [[pod('w-1', phase='Pending', node='node-a'), - pod('w-2', phase='Running', node='node-b')]] * 2) + loop = run_loop(loop_env, [[pod('w-1', phase='Pending', node='node-a', ip='10.0.0.1'), + pod('w-2', phase='Running', node='node-b', ip='10.0.0.2')]] * 2) - assert loop.sampled[0] == {'node-b'} + assert loop.sampled[0] == {'10.0.0.2'} # --- which pods get a stream --------------------------------------------------- @@ -174,7 +191,7 @@ def test_a_pending_pod_is_not_polled_until_it_can_answer(loop_env): assert loop.opened == [('w-1', '300', '1')] # ...and not until the third cycle, the first one it could have answered. assert loop.order[:6] == ['list', 'sample', 'list', 'sample', - 'list', 'sample'] + 'list', 'open:w-1'] def test_a_terminal_pod_is_still_polled(loop_env): @@ -187,7 +204,7 @@ def test_a_terminal_pod_is_still_polled(loop_env): def test_a_pod_with_no_range_label_is_not_ours(loop_env): stray = pod('other-1') - del stray['metadata']['labels'][lc.LABEL_RANGE] + del stray['metadata']['labels'][config.LABEL_RANGE] loop = run_loop(loop_env, [[stray]] * 2) assert loop.opened == [] diff --git a/src/MissionParallelCatchup/tests/unit/test_condemnation_watch.py b/src/MissionParallelCatchup/tests/unit/test_condemnation_watch.py new file mode 100644 index 00000000..3fa2e4bf --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_condemnation_watch.py @@ -0,0 +1,384 @@ +"""Detecting a condemnation fast enough to still open a follow. + +The follow itself was never the problem: `_follow_slots` is a 256-wide semaphore +and a real reclaim condemns tens of pods, so it never fell back to polling. What +lost the metric was seeing the condition too late. stellar-core exits about a +second after SIGTERM and the pod object is reaped behind it, so a condemned pod +exists for a few seconds -- and the pod-list sweep runs every POLL_SECONDS=5. + +Measured on ssc-test at prestopSleepSeconds=5: of 52 mid-replay legs, 32 lost +txApply. Seven were never seen condemned at all; the other 25 were seen, wrote +their disruptionReason, and still lost it because the follow opened after the +pod was gone. + +So these tests are about latency and about the two detectors agreeing, not about +whether a follow works. +""" + +import asyncio +import json + +import pytest + +import config +import log_collector as lc + + +def pod(name='w-1', phase='Running', end='300', attempt='1', + reason='EvictionByEvictionAPI', rv='100'): + conditions = ([{'type': 'DisruptionTarget', 'status': 'True', 'reason': reason}] + if reason else []) + return {'metadata': {'name': name, + 'resourceVersion': rv, + 'labels': {config.LABEL_RUN: config.RUN_NAME, + config.LABEL_RANGE: end, + config.LABEL_ATTEMPT: attempt}}, + 'status': {'phase': phase, 'conditions': conditions}} + + +@pytest.fixture +def collector(tmp_path, monkeypatch): + """Collector module state pointed at a temp volume, reset between tests.""" + monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(lc, '_doomed', {}) + monkeypatch.setattr(lc, '_wake', {}) + monkeypatch.setattr(lc, 'token', lambda: 'test-token') + # The real backoff is a wall-clock second; these tests advance the loop by + # ticks, not time, so a retry would never come back. + monkeypatch.setattr(lc, 'WATCH_RETRY_SECONDS', 0) + monkeypatch.setattr(lc, '_tasks', {}) + monkeypatch.setattr(lc, '_streamed', set()) + monkeypatch.setattr(lc, '_stream_ctx', {}) + monkeypatch.setattr(lc, '_streaming', {}) + return tmp_path + + +@pytest.fixture +def ready(collector, monkeypatch): + """A collector whose ensure_stream can actually open something. + + Records every poll_pod that gets started, so a second reader on one pod is + visible rather than silent. + """ + opened = [] + + async def fake_poll(session, name, end, attempt, done, done_ok): + opened.append((name, end, attempt)) + await asyncio.sleep(3600) + + monkeypatch.setattr(lc, 'poll_pod', fake_poll) + lc._stream_ctx.update(session=object(), terminal={}, succeeded={}) + return opened + + +async def drain(fn, ticks=30): + fn() + for _ in range(ticks): + await asyncio.sleep(0) + for t in list(lc._tasks.values()): + t.cancel() + + +# --- ensure_stream: one registry, one reader --------------------------------- + +def test_a_stream_is_opened_once_and_only_once(collector, ready): + async def go(): + await drain(lambda: [lc.ensure_stream('w-1', '300', '1', 'Running'), + lc.ensure_stream('w-1', '300', '1', 'Running'), + lc.ensure_stream('w-1', '300', '1', 'Running')]) + asyncio.run(go()) + assert ready == [('w-1', '300', '1')], \ + "two readers would re-append the same lines and race write_state" + + +def test_a_completed_stream_is_not_reopened(collector, ready): + lc._streamed.add('w-1') + async def go(): + await drain(lambda: lc.ensure_stream('w-1', '300', '1', 'Running')) + asyncio.run(go()) + assert ready == [] + + +@pytest.mark.parametrize('phase', ['Pending', 'Unknown']) +def test_an_unpollable_phase_is_left_for_a_later_call(collector, ready, phase): + # Its log endpoint answers 400 "waiting to start"; the retry is the next + # event or the next sweep, whichever lands first. + async def go(): + await drain(lambda: lc.ensure_stream('w-1', '300', '1', phase)) + asyncio.run(go()) + assert ready == [] + assert 'w-1' not in lc._tasks + + +def test_nothing_opens_before_main_publishes_its_context(collector, monkeypatch): + # ensure_stream is reachable from the watch, which starts inside main(). If + # an event landed first, opening with no session would throw in a task + # nobody awaits. + monkeypatch.setattr(lc, '_stream_ctx', {}) + assert lc.ensure_stream('w-1', '300', '1', 'Running') is False + + +# --- the two callers cannot double up ---------------------------------------- + +def test_the_watch_opens_the_stream_without_waiting_for_the_sweep(collector, ready): + # The whole point: at 900 pods the pod-list cycle reached 925s, and a pod + # condemned in that window died unread. + asyncio.run(run_watch(FakeSession([FakeResponse([ + {'type': 'ADDED', 'object': pod(reason=None)}, + ])]))) + assert ready == [('w-1', '300', '1')] + + +def test_the_sweep_still_opens_a_stream_the_watch_missed(collector, ready): + # Events are genuinely dropped across a reconnect, so the loop stays as a + # backstop rather than being retired. + async def go(): + await drain(lambda: lc.ensure_stream('w-1', '300', '1', 'Running')) + asyncio.run(go()) + assert ready == [('w-1', '300', '1')] + + +def test_watch_then_sweep_still_yields_one_reader(collector, ready): + async def go(): + lc.ensure_stream('w-1', '300', '1', 'Running') # watch + lc.ensure_stream('w-1', '300', '1', 'Running') # sweep, same cycle + for _ in range(30): + await asyncio.sleep(0) + for t in list(lc._tasks.values()): + t.cancel() + asyncio.run(go()) + assert ready == [('w-1', '300', '1')] + + +def test_a_pod_condemned_as_it_appears_gets_a_reader_before_being_marked(collector, ready): + # Ordering inside the watch: _mark_condemned only sets _doomed and fires + # _wake, both no-ops when no poller exists. Marking first would leave the + # condemnation with nothing to act on -- which is exactly how five -a2 legs + # produced 0-byte archives. + asyncio.run(run_watch(FakeSession([FakeResponse([ + {'type': 'ADDED', 'object': pod()}, + ])]))) + assert ready == [('w-1', '300', '1')], "the stream must exist first" + assert lc._doomed.get('w-1') == 'EvictionByEvictionAPI' + + +def metrics_of(vol, end='300', attempt='1'): + path = vol / f"range-{end}-a{attempt}.metrics" + return json.loads(path.read_text()) if path.exists() else {} + + +# --- _mark_condemned: the shared decision ------------------------------------ + +def test_a_condemnation_is_recorded_and_wakes_the_poller(collector): + lc._wake['w-1'] = asyncio.Event() + + assert lc._mark_condemned(pod(), 'w-1', '300', '1') is True + assert lc._doomed['w-1'] == 'EvictionByEvictionAPI' + assert metrics_of(collector)['disruptionReason'] == 'EvictionByEvictionAPI' + assert lc._wake['w-1'].is_set(), "the poller must not sleep out its interval" + + +def test_marking_twice_is_a_no_op(collector): + # The watch and the sweep both see the same object. Whichever is first does + # the work; the second must not re-open a stream or rewrite the reason. + assert lc._mark_condemned(pod(), 'w-1', '300', '1') is True + assert lc._mark_condemned(pod(reason='DeletionByTaintManager'), + 'w-1', '300', '1') is False + assert lc._doomed['w-1'] == 'EvictionByEvictionAPI' + + +def test_an_uncondemned_pod_is_left_alone(collector): + assert lc._mark_condemned(pod(reason=None), 'w-1', '300', '1') is False + assert lc._doomed == {} + assert metrics_of(collector) == {} + + +@pytest.mark.parametrize('phase', ['Succeeded', 'Failed']) +def test_a_finished_pod_is_not_followed(collector, phase): + # Its log is already complete, and leaving the flag set would re-open a + # stream on a dead pod every iteration. + assert lc._mark_condemned(pod(phase=phase), 'w-1', '300', '1') is False + assert lc._doomed == {} + + +def test_the_reason_is_carried_through_to_the_metrics_file(collector): + # EvictionByEvictionAPI is a drain that still owes a SIGTERM; a TaintManager + # stamp lands on a container that already died unsignalled. A lost txApply + # means different things in the two cases, so the label has to survive. + lc._mark_condemned(pod(reason='DeletionByTaintManager'), 'w-1', '300', '1') + assert metrics_of(collector)['disruptionReason'] == 'DeletionByTaintManager' + + +# --- watch_condemnations: the stream ----------------------------------------- + +class FakeResponse: + def __init__(self, lines, status=200): + self.status = status + self.content = self._iter(lines) + + async def _iter(self, lines): + for line in lines: + yield line if isinstance(line, bytes) else json.dumps(line).encode() + + def raise_for_status(self): + if self.status >= 400: + raise RuntimeError(f"status {self.status}") + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + +class FakeSession: + """Serves one scripted watch response per connection. + + Once the script runs out every further connection raises, which both stops + the test looping forever and exercises the retry path. + """ + + def __init__(self, responses): + self.responses = list(responses) + self.calls = [] + + def get(self, url, params=None, headers=None): + self.calls.append(dict(params or {})) + if not self.responses: + raise ConnectionError("no more scripted responses") + nxt = self.responses.pop(0) + if isinstance(nxt, Exception): + raise nxt + return nxt + + +async def run_watch(session, ticks=40): + task = asyncio.create_task(lc.watch_condemnations(session)) + for _ in range(ticks): + await asyncio.sleep(0) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + +def test_a_modified_event_condemns_immediately(collector): + monkey = FakeSession([FakeResponse([ + {'type': 'MODIFIED', 'object': pod()}, + ])]) + asyncio.run(run_watch(monkey)) + + assert lc._doomed.get('w-1') == 'EvictionByEvictionAPI', \ + "the watch, not the 5s sweep, is what has to catch this" + assert metrics_of(collector)['disruptionReason'] == 'EvictionByEvictionAPI' + + +def test_a_healthy_pod_event_does_nothing(collector): + asyncio.run(run_watch(FakeSession([FakeResponse([ + {'type': 'MODIFIED', 'object': pod(reason=None)}, + ])]))) + assert lc._doomed == {} + + +def test_deleted_events_are_ignored(collector): + # By DELETED the object is already gone; acting on it would open a stream + # against a pod that cannot answer. + asyncio.run(run_watch(FakeSession([FakeResponse([ + {'type': 'DELETED', 'object': pod()}, + ])]))) + assert lc._doomed == {} + + +def test_a_reconnect_resumes_from_the_last_resourceVersion(collector): + session = FakeSession([ + FakeResponse([{'type': 'MODIFIED', 'object': pod(reason=None, rv='517')}]), + FakeResponse([{'type': 'MODIFIED', 'object': pod(rv='518')}]), + ]) + asyncio.run(run_watch(session)) + + assert 'resourceVersion' not in session.calls[0], "first connect starts cold" + assert session.calls[1]['resourceVersion'] == '517', \ + "resuming re-delivers only what was missed instead of re-syncing" + + +def test_a_bookmark_advances_the_resume_point(collector): + # Bookmarks exist so an idle watch does not fall behind and get a 410 on + # reconnect. Ignoring them would strand the resume point at the last real + # change, which on a quiet run can be far in the past. + session = FakeSession([ + FakeResponse([{'type': 'BOOKMARK', + 'object': {'metadata': {'resourceVersion': '900'}}}]), + FakeResponse([]), + ]) + asyncio.run(run_watch(session)) + assert session.calls[1]['resourceVersion'] == '900' + + +def test_an_expired_resourceVersion_restarts_cold(collector): + # 410 Gone means our position aged out of the apiserver's history. Retrying + # with the same version loops forever; dropping it re-syncs. + session = FakeSession([ + FakeResponse([{'type': 'MODIFIED', 'object': pod(reason=None, rv='7')}]), + FakeResponse([], status=410), + FakeResponse([]), + ]) + asyncio.run(run_watch(session)) + + assert session.calls[1]['resourceVersion'] == '7' + assert 'resourceVersion' not in session.calls[2], "a 410 has to reset it" + + +def test_an_error_event_carrying_410_also_restarts_cold(collector): + # The same condition arrives as an in-stream ERROR event, not only as a + # status code on the connection. + session = FakeSession([ + FakeResponse([{'type': 'ERROR', + 'object': {'code': 410, 'metadata': {}}}]), + FakeResponse([]), + ]) + asyncio.run(run_watch(session)) + assert 'resourceVersion' not in session.calls[1] + + +def test_the_watch_survives_a_dropped_connection(collector): + # Detection degrading to the pod-list sweep is survivable; the collector + # dying is not. A watch that raised out of main() would take the whole + # sidecar and every in-flight stream with it. + session = FakeSession([ + ConnectionError("apiserver went away"), + FakeResponse([{'type': 'MODIFIED', 'object': pod()}]), + ]) + asyncio.run(run_watch(session)) + assert lc._doomed.get('w-1') == 'EvictionByEvictionAPI' + + +def test_malformed_lines_do_not_kill_the_stream(collector): + session = FakeSession([FakeResponse([ + b'{not json', + b'', + json.dumps({'type': 'MODIFIED', 'object': pod()}).encode(), + ])]) + asyncio.run(run_watch(session)) + assert lc._doomed.get('w-1') == 'EvictionByEvictionAPI' + + +def test_a_pod_without_a_range_label_is_skipped(collector): + # The job-monitor pod carries the run label too, and it has no range. + stray = pod() + del stray['metadata']['labels'][config.LABEL_RANGE] + asyncio.run(run_watch(FakeSession([FakeResponse([ + {'type': 'MODIFIED', 'object': stray}, + ])]))) + assert lc._doomed == {} + + +def test_the_watch_asks_for_bookmarks_and_a_bounded_lifetime(collector): + # An unbounded watch that dies silently stops detecting and nothing notices; + # the timeout is what makes it self-heal. + session = FakeSession([FakeResponse([])]) + asyncio.run(run_watch(session)) + assert session.calls[0]['watch'] == 'true' + assert session.calls[0]['allowWatchBookmarks'] == 'true' + assert session.calls[0]['timeoutSeconds'] == str(lc.WATCH_TIMEOUT_SECONDS) + assert session.calls[0]['labelSelector'] == f"{config.LABEL_RUN}={config.RUN_NAME}" diff --git a/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py b/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py deleted file mode 100644 index ad6a282d..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_cpu_tiers.py +++ /dev/null @@ -1,103 +0,0 @@ -"""CPU request as a slack budget, keyed on rank rather than absolute seconds. - -The request is not a demand estimate. Measured unthrottled 2026-07-30, replay -wants ~1.0 cores at every ledger position (1.04 at 63.7M, 0.96 at 43.2M) and is -80-95% of a job, so demand barely varies. What varies is how much throttling a -range can absorb, and that slack is free packing density: bin-packed, flat 1.0 -needs 491 nodes / 3928 vCPU -- over the 2304 quota -- against 291 / 2328 tiered. - -Rank rather than seconds because the absolute budget is set by the single -worst-throttled job: between two real runs it moved 1.79x while the median -range moved 1.55x, swinging the top two tiers from 127 ranges to 65. -""" - -import pytest - -import job_monitor as jm - -TIERS = '85:0.5,98:0.75,99.5:1.0,100:1.25' -# 1000 ranges, 1s..1000s, so a range's value IS its percentile x 1000. -PROFILE = [(i, {'seconds': float(i)}) for i in range(1, 1001)] - - -@pytest.fixture -def tiered(monkeypatch): - monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', TIERS) - monkeypatch.setattr(jm, 'PROFILE', PROFILE) - monkeypatch.setattr(jm, '_SORTED_SECONDS', None) - - -def test_tiering_is_off_unless_configured(monkeypatch): - monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', '') - assert jm._slack_cpu(500) is None - - -def test_the_cheap_bulk_gets_the_cheapest_tier(tiered): - assert jm._slack_cpu(1) == '0.5' - assert jm._slack_cpu(850) == '0.5' # exactly the 85% cut - - -def test_each_band_maps_to_its_tier(tiered): - assert jm._slack_cpu(851) == '0.75' # just past 85% - assert jm._slack_cpu(980) == '0.75' - assert jm._slack_cpu(981) == '1.0' - assert jm._slack_cpu(995) == '1.0' - assert jm._slack_cpu(1000) == '1.25' # the longest range - - -def test_an_unmeasured_range_gets_no_tier_at_all(tiered): - """No usable runtime means no basis for a tier -- fall through to REQ_CPU. - - Returning the TOP tier here cost 206 vCPU on the 2026-07-31 run: 103 ranges - lacked `seconds` not because they were new but because a resumed chain made - their runtime unverifiable, and several were demonstrably small. - """ - assert jm._slack_cpu(None) is None - - -@pytest.mark.parametrize('seconds', [0, -1, 'bad', float('nan'), float('inf')]) -def test_an_invalid_runtime_safely_gets_no_tier(tiered, seconds): - assert jm._slack_cpu(seconds) is None - - -def test_a_uniformly_slower_run_assigns_the_same_tiers(monkeypatch): - """The property absolute-seconds keying does not have. - - Every range 3x slower must not shuffle anything: rank is unchanged, so the - fleet needs the same shape. Under a `seconds <= longest` rule this is only - true if the slowdown is perfectly uniform, which measurement shows it is not. - """ - monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', TIERS) - monkeypatch.setattr(jm, 'PROFILE', [(i, {'seconds': i * 3.0}) for i in range(1, 1001)]) - monkeypatch.setattr(jm, '_SORTED_SECONDS', None) - assert jm._slack_cpu(850 * 3) == '0.5' - assert jm._slack_cpu(981 * 3) == '1.0' - assert jm._slack_cpu(1000 * 3) == '1.25' - - -def test_a_malformed_ladder_disables_tiering_rather_than_guessing(monkeypatch): - # One list of pairs cannot desync the way two parallel lists could, but a - # typo still has to fail safe rather than half-apply. - monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', '85,0.5') - assert jm._slack_cpu(100) is None - - -def test_cpu_is_requested_but_never_limited(monkeypatch, cluster): - # A limit would cap the bucket phase, measured up to 2.53 cores and the one - # part of a job that slicing cannot remove. - prof = [(i, {'seconds': float(i)}) for i in range(1, 1001)] - prof[299] = (300, {'seconds': 300.0, 'peakAnonBytes': 1 << 30}) # 30th pct - monkeypatch.setattr(jm, 'PROFILE_CPU_TIERS', TIERS) - monkeypatch.setattr(jm, 'PROFILE', prof) - monkeypatch.setattr(jm, '_SORTED_SECONDS', None) - r = jm._resources(end=300) - assert r.requests['cpu'] == '0.5' - assert not r.limits or 'cpu' not in r.limits - - -def test_a_one_range_profile_gives_that_range_the_top_tier(tiered, monkeypatch): - # It is simultaneously the cheapest and the longest range measured, so the - # 100th percentile is the honest answer -- not the cheapest tier. - monkeypatch.setattr(jm, 'PROFILE', [(300, {'seconds': 1.0})]) - monkeypatch.setattr(jm, '_SORTED_SECONDS', None) - assert jm._slack_cpu(1.0) == '1.25' diff --git a/src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py b/src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py index e0340bf6..496768d9 100644 --- a/src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py +++ b/src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py @@ -23,6 +23,7 @@ Asserted against the Jobs reconcile actually creates, not against a helper. """ +import config import job_monitor as jm DEADLINE = 43200 @@ -43,8 +44,8 @@ def test_the_cheapest_and_costliest_ranges_get_the_same_deadline(cluster, monkey Tightening the cheap one is exactly what killed 134 ranges in the backtest: its `seconds` came from a neighbour, and the neighbour was wrong. """ - monkeypatch.setattr(jm, 'PROFILE', PROFILE) - monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) + monkeypatch.setattr(config, 'PROFILE', PROFILE) + monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) cluster.reconcile() assert _deadline_of(cluster, 200) == DEADLINE @@ -53,8 +54,8 @@ def test_the_cheapest_and_costliest_ranges_get_the_same_deadline(cluster, monkey def test_an_unprofiled_range_gets_the_same_deadline_too(cluster, monkeypatch): """No profile at all changes nothing -- there is nothing to scale by.""" - monkeypatch.setattr(jm, 'PROFILE', []) - monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) + monkeypatch.setattr(config, 'PROFILE', []) + monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) cluster.reconcile() assert _deadline_of(cluster, 300) == DEADLINE @@ -63,8 +64,8 @@ def test_an_unprofiled_range_gets_the_same_deadline_too(cluster, monkeypatch): def test_zero_disables_the_deadline_entirely(cluster, monkeypatch): """0 must mean absent, not 0 -- a zero-second deadline kills every attempt the moment it is created.""" - monkeypatch.setattr(jm, 'PROFILE', PROFILE) - monkeypatch.setattr(jm, 'ATTEMPT_DEADLINE_SECONDS', 0) + monkeypatch.setattr(config, 'PROFILE', PROFILE) + monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', 0) cluster.reconcile() assert _deadline_of(cluster, 300) is None diff --git a/src/MissionParallelCatchup/tests/unit/test_dispatch_order.py b/src/MissionParallelCatchup/tests/unit/test_dispatch_order.py index 1dd1f72c..31ae7705 100644 --- a/src/MissionParallelCatchup/tests/unit/test_dispatch_order.py +++ b/src/MissionParallelCatchup/tests/unit/test_dispatch_order.py @@ -6,15 +6,17 @@ import pytest +import config +import ranges import job_monitor as jm RANGES = [(600, 420), (500, 420), (400, 420), (300, 420)] # generators emit tip-first def _order(monkeypatch, mode, profile=None): - monkeypatch.setattr(jm, 'RANGE_ORDER', mode) - monkeypatch.setattr(jm, 'PROFILE', profile) - return [e for e, _ in jm._ordered(list(RANGES))] + monkeypatch.setattr(config, 'RANGE_ORDER', mode) + monkeypatch.setattr(config, 'PROFILE', profile) + return [e for e, _ in ranges._ordered(list(RANGES))] def test_tip_first_is_unchanged(monkeypatch): diff --git a/src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py b/src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py index 8ee93bef..d606c565 100644 --- a/src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py +++ b/src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py @@ -16,6 +16,8 @@ import pytest +import config +import attempts import job_monitor as jm import log_collector as lc @@ -25,10 +27,9 @@ @pytest.fixture def sampler(tmp_path, monkeypatch): """A collector with no memory, writing to a disposable volume.""" - monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(jm, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) monkeypatch.setattr(lc, 'token', lambda: 'tok') - monkeypatch.setattr(lc, 'STORAGE_MODE', 'ephemeral') + monkeypatch.setattr(config, 'STORAGE_MODE', 'ephemeral') for name in ('_eph_peak', '_anon_peak', '_ws_peak', '_peak_flushed', '_streaming', '_pod_secs', '_wake'): monkeypatch.setattr(lc, name, {}) @@ -82,23 +83,16 @@ def sample(doc): # --- a peak is a high-water mark, not the latest reading --------------------- -def test_a_later_lower_sample_never_lowers_the_peak(sampler): +@pytest.mark.parametrize('first, second', [(900, 400), (400, 900)]) +def test_the_peak_is_a_high_water_mark_in_either_order(sampler, first, second): """Catching the spike is the whole point, and download-phase anon - oscillates: the sampler is what turns a series of readings into one - number, so last-wins here defeats every consumer downstream.""" - sample(payload('w-1', [container(rss=900 * MIB)], eph=5)) - sample(payload('w-1', [container(rss=400 * MIB)], eph=2)) + oscillates: the sampler turns a series of readings into one number, so + last-wins defeats every consumer downstream.""" + sample(payload('w-1', [container(rss=first * MIB)], eph=first)) + sample(payload('w-1', [container(rss=second * MIB)], eph=second)) assert lc._anon_peak == {'w-1': 900 * MIB} - assert lc._eph_peak == {'w-1': 5} - - -def test_a_higher_sample_still_raises_it(sampler): - sample(payload('w-1', [container(rss=400 * MIB)], eph=2)) - sample(payload('w-1', [container(rss=900 * MIB)], eph=5)) - - assert lc._anon_peak == {'w-1': 900 * MIB} - assert lc._eph_peak == {'w-1': 5} + assert lc._eph_peak == {'w-1': 900} # --- what must not be recorded ----------------------------------------------- @@ -186,18 +180,18 @@ def test_an_in_flight_peak_reaches_the_volume_before_the_stream_ends(sampler): lc._streaming['w-1'] = ('300', '1') sample(payload('w-1', [container(rss=900 * MIB)])) - assert jm.peaks_for_range('300', 1) == {'peakAnonBytes': 900 * MIB} + assert attempts.peaks_for_range('300', 1) == {'peakAnonBytes': 900 * MIB} def test_a_peak_sampled_before_stream_registration_is_flushed_on_open(sampler): """main samples first, then opens new pollers; a restart between those steps must not make that first high-water process-memory-only.""" sample(payload('w-1', [container(rss=900 * MIB, ws=1200 * MIB)])) - assert jm.peaks_for_range('300', 1) == {} + assert attempts.peaks_for_range('300', 1) == {} lc._register_stream('w-1', '300', '1') - assert jm.peaks_for_range('300', 1) == { + assert attempts.peaks_for_range('300', 1) == { 'peakAnonBytes': 900 * MIB, 'peakWorkingSetBytes': 1200 * MIB, } @@ -206,7 +200,7 @@ def test_a_peak_sampled_before_stream_registration_is_flushed_on_open(sampler): def test_the_disk_axis_stays_mode_gated(sampler, monkeypatch): """ephemeral-storage is meaningless in pvc mode: /data is on the volume, not on the node.""" - monkeypatch.setattr(lc, 'STORAGE_MODE', 'pvc') + monkeypatch.setattr(config, 'STORAGE_MODE', 'pvc') lc._streaming['w-1'] = ('300', '1') sample(payload('w-1', [container(rss=900 * MIB)], eph=34 * 1024 ** 3)) @@ -236,7 +230,7 @@ def test_finalize_records_the_working_set_peak(sampler): asyncio.run(lc.finalize(None, 'w-1', '300', 1, lc.TxApplyScanner(), lambda p: True)) - stored = jm.peaks_for_range('300', 1) + stored = attempts.peaks_for_range('300', 1) assert stored['peakWorkingSetBytes'] == 4096 * MIB assert stored['peakAnonBytes'] == 900 * MIB @@ -250,11 +244,45 @@ def test_finalize_records_that_an_attempt_resumed(sampler): tx.feed("RESUME: 300/16320 reached ledger 299, replay had started") asyncio.run(lc.finalize(None, 'w-1', '300', 1, tx, lambda p: True)) - assert jm._attempt_resumed('300', 1) is True + assert attempts._attempt_resumed('300', 1) is True def test_a_fresh_attempt_is_never_marked_resumed(sampler): asyncio.run(lc.finalize(None, 'w-1', '300', 1, lc.TxApplyScanner(), lambda p: True)) - assert jm._attempt_resumed('300', 1) is False + assert attempts._attempt_resumed('300', 1) is False + + +# --- which endpoint, and why it matters -------------------------------------- + +def test_the_sampler_goes_straight_to_the_kubelet_not_the_apiserver_proxy(sampler): + """The endpoint choice IS the privilege boundary. + + Reaching kubelet through `/api/v1/nodes//proxy/...` requires the + `nodes/proxy` subresource, which authorizes GET on EVERY kubelet path -- + /pods and /containerLogs among them, for any namespace scheduled on that + node. The kubelet maps /stats/* to its own `nodes/stats` subresource, so + talking to it directly is the same payload under a grant that cannot read + pod inventory or logs at all. + + Reverting to the proxy would 403 against the deployed RBAC rather than + quietly widening it, but the intent should fail loudly here first. + """ + seen = [] + + class _Recording(_Session): + def get(self, url, **kw): + seen.append((url, kw)) + return _Resp(self.payload) + + asyncio.run(lc.sample_kubelet(_Recording(payload('w-1', [container(rss=1)])), + ['10.1.2.3'])) + + (url, kw), = seen + assert url == f"https://10.1.2.3:{lc.KUBELET_PORT}/stats/summary" + assert '/proxy/' not in url, "back on the apiserver node proxy" + assert 'nodes' not in url, "addressing a Node object rather than the kubelet" + # EKS kubelet serving certs are self-signed, not issued by the cluster CA + # the session's context trusts. + assert kw.get('ssl') is False diff --git a/src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py b/src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py index f806b2f6..9998a145 100644 --- a/src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py +++ b/src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py @@ -20,6 +20,10 @@ import pytest +import config +import units +import records +import attempts import job_monitor as jm @@ -37,7 +41,7 @@ def test_a_failed_attempts_duration_is_persisted_with_its_verdict(cluster): cluster.advance(300, 'incomplete') cluster.reconcile() - outcome = jm.read_outcome('300', 1) + outcome = records.read_outcome('300', 1) assert outcome['outcome'] == 'failed' assert outcome['attemptSeconds'] == pytest.approx(60.0, abs=5.0), outcome @@ -51,14 +55,14 @@ def test_that_duration_is_what_the_chain_adds_up(cluster): cluster.reconcile() cluster.finalize(300, 2, resumed=True) - assert jm.seconds_for_range('300', 2, 300.0) == pytest.approx(360.0, abs=5.0) + assert attempts.seconds_for_range('300', 2, 300.0) == pytest.approx(360.0, abs=5.0) def test_a_verdict_already_on_the_volume_is_not_rewritten(cluster): """The collector writes this file too, from the pod, while it still exists. Its verdict is the one taken with the best evidence and must win.""" cluster.reconcile() - cluster.write(jm.outcome_path('300', 1), + cluster.write(records.outcome_path('300', 1), '{"outcome": "disrupted", "exitCode": null, "pod": "w-300", ' '"attemptSeconds": 1800.0}') cluster.advance(300, 'incomplete') @@ -67,8 +71,8 @@ def test_a_verdict_already_on_the_volume_is_not_rewritten(cluster): # The pod exited 3, which reads as a plain catchup failure. The collector # saw the eviction that caused it, so its verdict -- and its duration -- # stand. - assert jm.read_outcome('300', 1)['outcome'] == 'disrupted' - assert jm.read_outcome('300', 1)['attemptSeconds'] == 1800.0 + assert records.read_outcome('300', 1)['outcome'] == 'disrupted' + assert records.read_outcome('300', 1)['attemptSeconds'] == 1800.0 # --- the condemned range has to say so ---------------------------------------- @@ -94,11 +98,13 @@ def test_an_exhausted_range_says_which_budget_it_spent(cluster, caplog): """The other way a range ends: it was retryable and ran out. That is a different operator action from a condemnation, so it reads differently.""" cluster.reconcile() - for attempt in range(1, jm.MAX_ATTEMPTS_PER_RANGE + 1): - cluster.advance(300, 'incomplete', attempt=attempt) + # An OOM: the only cause that spends the range budget now, since a "did not + # complete" is either a fetch fault (the cluster's problem) or a real + # failure (condemned outright). + for attempt in range(1, config.ATTEMPT_BUDGETS['oom'] + 1): + cluster.advance(300, 'oom', attempt=attempt) with caplog.at_level(logging.ERROR, logger=jm.logger.name): cluster.reconcile() - cluster.finalize(300, attempt) exhausted = [r.getMessage() for r in caplog.records if 'exhausted' in r.getMessage()] @@ -119,12 +125,12 @@ def test_the_backstop_saves_a_log_the_collector_never_claimed(cluster): cluster.advance(300, 'incomplete') cluster.reconcile() - path = jm.log_path('300', 1) + path = records.log_path('300', 1) assert os.path.exists(path), "a failed attempt left no archive at all" with gzip.open(path, 'rt') as fh: assert 'sum = 1500.0ms' in fh.read() - # ...and the archive is what the monitor's own reader then recovers from. - assert jm._tx_apply_for_attempt('300', 1) == pytest.approx(1.5) + # The archive is the evidence the backstop exists to preserve. The metric + # is the collector's to record, and it never ran for this range. def test_the_backstop_stands_down_for_a_range_the_collector_claimed(cluster): @@ -132,11 +138,11 @@ def test_the_backstop_stands_down_for_a_range_the_collector_claimed(cluster): lines. The collector's .state file is the claim, written the moment it opens a poller -- empty or not.""" cluster.reconcile() - cluster.write(jm.state_path('300', 1), '') + cluster.write(records.state_path('300', 1), '') cluster.advance(300, 'incomplete') cluster.reconcile() - assert not os.path.exists(jm.log_path('300', 1)), \ + assert not os.path.exists(records.log_path('300', 1)), \ "the monitor wrote over an archive the collector had claimed" @@ -153,8 +159,8 @@ def test_a_torn_backstop_archive_is_never_left_behind(cluster, monkeypatch): pod = cluster.k8s.pod_for_job(cluster.job_name(300, 1)) assert jm.backstop_save_pod_log(pod.metadata.name, '300', 1) is False - assert not os.path.exists(jm.log_path('300', 1)) - assert jm._tx_apply_for_attempt('300', 1) is None + assert not os.path.exists(records.log_path('300', 1)) + assert attempts._tx_apply_for_attempt('300', 1) is None # --- the progress record -------------------------------------------------------- @@ -171,7 +177,7 @@ def test_the_progress_record_is_replaced_whole_or_not_at_all(cluster, monkeypatc cluster.advance(300, 'succeeded') cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakAnonBytes': 7}) cluster.reconcile() - before = json.load(open(jm.PROGRESS_FILE)) + before = json.load(open(config.PROGRESS_FILE)) assert '300' in before['completed'] real_open = open @@ -198,7 +204,7 @@ def half_open(path, mode='r', *a, **kw): return _HalfWrite(path) return real_open(path, mode, *a, **kw) - monkeypatch.setattr(jm, 'open', half_open, raising=False) + monkeypatch.setattr(records, 'open', half_open, raising=False) cluster.advance(200, 'succeeded') cluster.finalize(200, 1, tx_apply=2.5, peaks={'peakAnonBytes': 9}) with pytest.raises(OSError): @@ -206,5 +212,102 @@ def half_open(path, mode='r', *a, **kw): armed['v'] = False # Not truncated, not empty, and not half of two records spliced together. - assert json.load(open(jm.PROGRESS_FILE)) == before + assert json.load(open(config.PROGRESS_FILE)) == before assert jm.load_progress()['completed']['300']['peakAnonBytes'] == 7 + + +# --- which classifier wins ---------------------------------------------------- +# Two independent sources, and the pod is not simply preferred: a deadline kill +# sends SIGTERM, stellar-core drains and exits 3, so the pod reads a plain +# `failed` that would CONDEMN a range which merely ran long. Only the Job knows +# the deadline fired. Where the pod named a mechanism it wins instead, because +# "ran too long" is also true of an OOM or an eviction and choosing it loses both +# the remediation and the retry budget. + +def _verdict(end=300, attempt=1): + return open(records.verdict_path(end, attempt)).read().strip() + + +@pytest.mark.parametrize('outcome', [ + 'timeout', # pod exit 3, Job DeadlineExceeded: the Job condition wins + 'unknown', # pod deleted, Job has no condition: retry rather than condemn +]) +def test_the_verdict_recorded_is_the_one_the_sources_agree_on(cluster, outcome): + cluster.reconcile() + cluster.advance(300, outcome) + cluster.reconcile() + + assert _verdict() == outcome + + +@pytest.mark.parametrize('outcome', ['oom', 'disrupted']) +def test_what_the_pod_says_beats_a_job_deadline(cluster, outcome): + """The escalation ladder needs the mechanism, and a timeout verdict is + terminal where an oom is retried with more memory.""" + cluster.reconcile() + name = cluster.advance(300, outcome) + # The same attempt also tripped its deadline: the Job condition says so. + cluster.k8s.set_job_failed(name, reason='DeadlineExceeded', + message='Job was active longer than specified deadline') + cluster.reconcile() + + assert _verdict() == outcome + + +def test_a_disrupted_range_escalates_disk_one_rung_on_its_first_eviction(cluster, monkeypatch): + """The size of the escalation, as reconcile actually builds it. + + Counting attempts instead of evictions handed a range disrupted four times a + 1.5^5 = 7.6x disk request for a single eviction. Asserted on the retry Job's + own spec rather than on the helpers, because the helpers were already right + -- it was the call site that passed the wrong index. + """ + monkeypatch.setattr(config, 'STORAGE_MODE', 'ephemeral') + monkeypatch.setattr(config, 'REQ_EPHEMERAL', '4Gi') + monkeypatch.setattr(config, 'LIM_EPHEMERAL', '4Gi') + monkeypatch.setattr(config, 'EPH_BUMP_FACTOR', 1.5) + + cluster.reconcile() + for _ in range(4): + cluster.advance(300, 'disrupted') + cluster.reconcile() + assert cluster.attempt_of(300) == 5, "four disruptions, four retries" + + cluster.advance(300, 'ephemeral') + cluster.reconcile() + + retry = cluster.k8s.job(cluster.job_name(300)) + got = retry.spec.template.spec.containers[0].resources.limits['ephemeral-storage'] + assert units.quantity_bytes(got) == units.quantity_bytes('6Gi'), ( + f"first eviction must climb one rung to 6Gi, got {got}") + + +def test_an_exhausted_oom_reports_the_memory_the_attempt_actually_had(cluster, caplog, + monkeypatch): + """`reason` only surfaces when the budget runs out, so exhaust it. + + Four disruptions then one OOM, so the attempt index (5) and the OOM count (1) + DIVERGE -- which is the whole bug. The range ran at the 9Gi base, and + indexing the report on `attempt` claimed 9Gi * 1.5^4 = 45Gi instead. + + Disruptions spend their own budget, so an OOM budget of 1 is + exhausted by the single OOM and nothing else. + """ + monkeypatch.setattr(config, 'POOL_PREFIX', '') + monkeypatch.setattr(config, 'REQ_MEM', '9Gi') + monkeypatch.setattr(config, 'MEM_BUMP_FACTOR', 1.5) + monkeypatch.setitem(config.ATTEMPT_BUDGETS, 'oom', 1) + + cluster.reconcile() + for _ in range(4): + cluster.advance(300, 'disrupted') + cluster.reconcile() + assert cluster.attempt_of(300) == 5, "four disruptions, four retries" + + cluster.advance(300, 'oom') + with caplog.at_level(logging.ERROR, logger=jm.logger.name): + cluster.reconcile() + + line = next(r.getMessage() for r in caplog.records if 'exhausted' in r.getMessage()) + reported = line.split('memory request ')[1].rstrip(')') + assert units.quantity_bytes(reported) == units.quantity_bytes('9Gi'), line diff --git a/src/MissionParallelCatchup/tests/unit/test_node_targeting.py b/src/MissionParallelCatchup/tests/unit/test_node_targeting.py index e4473edb..9667b261 100644 --- a/src/MissionParallelCatchup/tests/unit/test_node_targeting.py +++ b/src/MissionParallelCatchup/tests/unit/test_node_targeting.py @@ -11,6 +11,7 @@ import pytest +import config import job_monitor as jm @@ -22,7 +23,7 @@ def _match_expressions(monkeypatch, **env): """ for k in ('NODE_LABEL_KEY', 'NODE_LABEL_VALUE', 'AVOID_NODE_LABEL_KEY', 'AVOID_NODE_LABEL_VALUE'): - monkeypatch.setattr(jm, k, env.get(k, '')) + monkeypatch.setattr(config, k, env.get(k, '')) job = jm.build_job(300, 420, 1, None) aff = job.spec.template.spec.affinity if aff is None: diff --git a/src/MissionParallelCatchup/tests/unit/test_poll_lifecycle.py b/src/MissionParallelCatchup/tests/unit/test_poll_lifecycle.py index e02aa1fa..7448b49f 100644 --- a/src/MissionParallelCatchup/tests/unit/test_poll_lifecycle.py +++ b/src/MissionParallelCatchup/tests/unit/test_poll_lifecycle.py @@ -18,14 +18,16 @@ import pytest +import config +import attempts import job_monitor as jm import log_collector as lc @pytest.fixture def volume(tmp_path, monkeypatch): - monkeypatch.setattr(lc, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(jm, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) monkeypatch.setattr(lc, 'token', lambda: 'tok') monkeypatch.setattr(lc, 'LOG_POLL_SECONDS', 0.02) monkeypatch.setattr(lc, 'TERMINAL_POLL_ATTEMPTS', 2) @@ -108,7 +110,7 @@ def test_a_404_finalizes_what_was_already_streamed(volume): drive(Apiserver(_Resp(200, body), _Resp(404)), lambda: False) assert jm._attempt_finalized('300', 1), "a vanished pod never finalized" - assert jm.tx_apply_for_range('300', 1) == pytest.approx(1.5) + assert attempts.tx_apply_for_range('300', 1) == pytest.approx(1.5) assert 'sum = 1500.0ms' in archive() @@ -153,7 +155,7 @@ def test_terminal_is_sampled_before_the_poll_not_after(volume): assert 'catchup ledger 42000000' in archive() assert 'sum = 1500.0ms' in archive(), \ "the read after the pod went terminal never happened" - assert jm.tx_apply_for_range('300', 1) == pytest.approx(1.5) + assert attempts.tx_apply_for_range('300', 1) == pytest.approx(1.5) # --- resuming a read ---------------------------------------------------------- diff --git a/src/MissionParallelCatchup/tests/unit/test_pool_tiers.py b/src/MissionParallelCatchup/tests/unit/test_pool_tiers.py new file mode 100644 index 00000000..7f262a78 --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_pool_tiers.py @@ -0,0 +1,572 @@ +"""Nodepool routing: memory picks the pool, and the pool is the whole sizing. + +Replaces the cpu-ladder tests. The ladder tuned cpu REQUESTS, which measurement +showed were not buying throughput -- replay draws ~1.05 cores whatever it is +given, and is flat in core count from 2 upward (+2.8% at 2->4, +1.5% at 4->8, +against +16% for AMD-over-Intel at fixed cores). What a request actually bought +was neighbours-per-node. Memory is the dimension that FAILS rather than slows: +a working set that does not fit is OOMKilled. +""" + +import pytest + +import config +import records +import sizing +import job_monitor as jm + +GiB = 1024 ** 3 +TIERS = '0:subdwarf,0.79:dwarf,1.61:subgiant,3.87:giant,8.85:supergiant,18.38:hypergiant,:supernova' +CPU = ('subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:3.80') +# vCPU of the smallest shape in each tier's pool. hypergiant, protostar and +# supernova used to sit on x8i, which put their smallest shape below the rest of +# the tier -- hypergiant read 4 off an x8i.xlarge at w80 while Karpenter served +# 8-vCPU 2xlarges from w100. The x8i spot pools were removed on 2026-08-04, so +# each tier's smallest shape is now also its top-weighted one. +VCPU = ('subdwarf:2,dwarf:2,subgiant:4,giant:4,supergiant:4,hypergiant:8,supernova:16,protostar:8,nebula:8') +# 50% of each tier node's ALLOCATABLE, which is what both isolates the pod and +# lets it schedule -- see test_the_request_is_half_the_node. +MEM = ('subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:14336Mi') + + +@pytest.fixture +def pooled(monkeypatch): + monkeypatch.setattr(config, 'POOL_TIERS', TIERS) + monkeypatch.setattr(config, 'POOL_CPU', CPU) + monkeypatch.setattr(config, 'POOL_VCPU', VCPU) + monkeypatch.setattr(config, 'POOL_PREFIX', 'catchup') + monkeypatch.setattr(config, 'POOL_UNPROFILED', 'protostar') + monkeypatch.setattr(config, 'POOL_NO_PROFILE', 'nebula') + monkeypatch.setattr(config, 'POOL_MEM', MEM) + + +def _profile(entries): + """entries: {end: {...}} -> the sorted (end, rec) list PROFILE holds.""" + return sorted(entries.items()) + + +def _profiled(monkeypatch, entries): + monkeypatch.setattr(config, 'PROFILE', _profile(entries)) + + +def _pool(monkeypatch, anon, ws=None, end=10, **kw): + """Route one range measured at these peaks.""" + peaks = {'peakAnonBytes': int(anon)} + if ws is not None: + peaks['peakWorkingSetBytes'] = int(ws) + _profiled(monkeypatch, {end: peaks}) + return sizing.pool_for(end, **kw) + + +def test_a_range_lands_in_the_tier_its_working_set_fits(pooled, monkeypatch): + _profiled(monkeypatch, { + 100: {'peakAnonBytes': int(0.20 * GiB)}, + 200: {'peakAnonBytes': int(1.50 * GiB)}, + 300: {'peakAnonBytes': int(3.00 * GiB)}, + 400: {'peakAnonBytes': int(7.00 * GiB)}, + 500: {'peakAnonBytes': int(16.0 * GiB)}, + 600: {'peakAnonBytes': int(40.0 * GiB)}, + }) + # subdwarf's cut is 0, so even the smallest range falls through to dwarf + assert sizing.pool_for(100) == 'dwarf' + assert sizing.pool_for(200) == 'subgiant' + assert sizing.pool_for(300) == 'giant' + assert sizing.pool_for(400) == 'supergiant' + assert sizing.pool_for(500) == 'hypergiant' + assert sizing.pool_for(600) == 'supernova' + + +def test_nothing_is_ever_routed_to_subdwarf(pooled, monkeypatch): + """Its cut is 0, and no range can satisfy `gib < 0`. + + The tier stays defined and provisionable -- the pools exist -- but c8a.medium + has 1.42Gi allocatable, and after daemonsets that cannot hold any range the + profile actually contains. Emptying it by cut rather than deleting it keeps + the bottom of the ladder available to experiment with. + """ + _profiled(monkeypatch, { + 10: {'peakAnonBytes': 1}, # 1 byte + 20: {'peakAnonBytes': int(0.27 * GiB)}, # the profile's true minimum + }) + assert sizing.pool_for(10) == 'dwarf' + assert sizing.pool_for(20) == 'dwarf' + + +def test_the_cut_is_exclusive_so_a_range_never_lands_on_a_node_it_fills(pooled, monkeypatch): + """A range exactly AT a cut belongs in the tier above. + + The cut is node_usable/1.60, so a range sitting on it would have exactly the + p99 margin and nothing more. Being one byte over must move it up, not leave + it to be the range that proves the margin was too thin. + """ + _profiled(monkeypatch, { + 10: {'peakAnonBytes': int(0.7899 * GiB)}, + 20: {'peakAnonBytes': int(0.7901 * GiB)}, + }) + assert sizing.pool_for(10) == 'dwarf' + assert sizing.pool_for(20) == 'subgiant' + # The comparison is `<`, so a range sitting exactly on a cut goes UP. Not + # asserted at the exact byte: 0.79 GiB is not representable, and pinning the + # test to a float's rounding would make it about IEEE754 rather than about + # which side of the boundary a range belongs on. + + +def test_a_range_past_the_top_of_the_profile_goes_to_protostar(pooled, monkeypatch): + """Unprofiled means NEWEST, and the newest ledgers are the densest. + + profile_for returns the nearest measured end ABOVE the target, so falling + off the end means this range is newer than anything ever measured. It gets a + rich pool rather than an average one. + """ + _profiled(monkeypatch, {10: {'peakAnonBytes': int(0.50 * GiB)}}) + assert sizing.pool_for(999) == 'protostar' + + +def test_no_profile_at_all_goes_to_nebula(pooled, monkeypatch): + monkeypatch.setattr(config, 'PROFILE', []) + assert sizing.pool_for(10) == 'nebula' + + +def test_an_entry_with_no_memory_measurement_is_treated_as_unprofiled(pooled, monkeypatch): + """Sizing needs a measurement, and `seconds` is not one. + + A record can carry a runtime but no peak -- reconstruction omits what it + cannot verify. Guessing a tier from runtime would reintroduce exactly the + cpu-ladder mistake: sizing memory off a dimension that does not predict it + (peakEphemeral/anon correlate r2 0.32). + """ + _profiled(monkeypatch, {10: {'seconds': 9000.0}}) + assert sizing.pool_for(10) == 'protostar' + + +def test_an_oom_promotes_the_pool_not_just_the_request(pooled, monkeypatch): + """The whole point of tier escalation. + + Raising the request while the pod stays pinned to a tier whose nodes cannot + hold it produces a pod that can never schedule -- Pending forever, which + reads as a hang rather than a failure. + """ + _profiled(monkeypatch, {10: {'peakAnonBytes': int(0.50 * GiB)}}) + assert sizing.pool_for(10, rungs=0) == 'dwarf' + assert sizing.pool_for(10, rungs=1) == 'subgiant' + assert sizing.pool_for(10, rungs=2) == 'giant' + assert sizing.pool_for(10, rungs=3) == 'supergiant' + + +def test_only_ooms_climb_the_ladder_not_every_retry(pooled, monkeypatch): + """A spot reclaim is not evidence the range needed a bigger node. + + Promoting on attempt number put 65 ranges onto 8-vCPU supernova nodes during + the 2026-08-03 spot run whose attempt-1 verdict was `timeout` -- they + belonged on 4-vCPU hypergiant, so it burned ~260 vCPU of a 2304 quota + escalating away from a problem that was never memory. Reclaims, disruptions + and timeouts all produce retries; only an OOM says the tier was too small. + """ + _profiled(monkeypatch, {10: {'peakAnonBytes': int(0.50 * GiB)}}) + monkeypatch.setattr(records, '_oom_count', lambda end, attempt: 0) + for attempt in (1, 2, 3, 9): + assert sizing.pool_for(10, attempt=attempt) == 'dwarf', \ + f"attempt {attempt} climbed a tier without an OOM" + monkeypatch.setattr(records, '_oom_count', lambda end, attempt: 2) + assert sizing.pool_for(10, attempt=3) == 'giant' + + +def test_promotion_counts_ooms_from_disk_when_rungs_is_not_given(pooled, monkeypatch): + _profiled(monkeypatch, {10: {'peakAnonBytes': int(0.50 * GiB)}}) + seen = {} + def fake(end, attempt): + seen['attempt'] = attempt + return 1 + monkeypatch.setattr(records, '_oom_count', fake) + assert sizing.pool_for(10, attempt=4) == 'subgiant' + # attempts BEFORE this one -- this attempt has not run, so its own outcome + # cannot be on disk yet. + assert seen['attempt'] == 3 + + +def test_promotion_stops_at_the_top_instead_of_running_off_the_ladder(pooled, monkeypatch): + _profiled(monkeypatch, {10: {'peakAnonBytes': int(40.0 * GiB)}}) + assert sizing.pool_for(10, rungs=0) == 'supernova' + assert sizing.pool_for(10, rungs=8) == 'supernova' + + +def test_the_off_ladder_pools_escalate_straight_to_the_top(pooled, monkeypatch): + """nebula and protostar are not rungs, so there is nothing to walk. + + Both hold ranges whose size is unknown, so an OOM says the guess was too + small with no information about by how much. The top tier is the only answer + that cannot be wrong again for the same reason. + """ + monkeypatch.setattr(config, 'PROFILE', []) + assert sizing.pool_for(10, rungs=1) == 'supernova' + _profiled(monkeypatch, {10: {'peakAnonBytes': GiB}}) + assert sizing.pool_for(999, rungs=1) == 'supernova' + + +def test_the_request_lands_exactly_two_pods_per_node(pooled): + """Two per node comes from the arithmetic, not from goodwill. + + The ladder ran one-pod-per-node until 2026-08-04, when every spot pool's + instance size was doubled to test whether a neighbour is worth more than a + dedicated node. Measured that day: a pod on a 4-vCPU node beat the same pod + on a 2-vCPU node by 25% (x8i.large 0.74 vs x8i.xlarge 1.07 against profile, + same tier, same silicon) while drawing under one core -- so the replay + thread is not what wants the extra cores, and a neighbour may be able to + use them without costing the first pod. + + Two pods must FIT (2*req + daemonsets <= allocatable) and three must NOT. + Getting the second condition wrong is the expensive one: at the old claims + against doubled nodes, 3-4 pods would pack per node and the isolation the + whole ladder exists for is gone without anything failing. + + Sized off ALLOCATABLE, not nameplate. On the small nodes the gap decides the + outcome: a 2Gi c8a.medium allocates 1181Mi and the scheduler was measured + counting 1477Mi for a 983Mi request. A nameplate-derived request fit the + node on paper and left 24 pods Pending with "no instance type has enough + resources". + """ + # MEASURED on live ssc-test nodes 2026-08-03. These are the same numbers as + # before the doubling, shifted one tier up -- what used to be supergiant's + # 16Gi node is now giant's. 128Gi is extrapolated from the 64Gi measurement + # at the same 94% ratio; nothing that large has run yet. + ALLOC = {'dwarf': 2798, 'subgiant': 6502, 'giant': 14654, + 'supergiant': 30259, 'hypergiant': 61604, 'supernova': 124000} + OVERHEAD = 154 # measured daemonset requests on a live node + for tier, alloc in ALLOC.items(): + req = int(sizing.pool_memory(tier).removesuffix('Mi')) + assert 2 * req + OVERHEAD <= alloc, f"{tier}: a second pod will not schedule" + assert 3 * req + OVERHEAD > alloc, f"{tier}: three pods would fit" + + +def test_every_routable_tier_has_a_request(pooled): + # A tier with no entry silently keeps the flat configured request, which is + # both too small to isolate and unrelated to the node it landed on. + # cpu claims are checked against the real chart values by + # test_the_chart_ships_a_coherent_pool_ladder; POOL_MEM is not, so this is + # the only thing standing between a promoted tier and the flat request. + for _, tier in sizing._parsed_pool_tiers(): + assert sizing.pool_memory(tier), f"tier {tier} has no memory request" + for off_ladder in ('protostar', 'nebula'): + assert sizing.pool_memory(off_ladder) + + +def test_an_empty_prefix_disables_pooling_entirely(monkeypatch): + """The change has to be opt-in: an unset prefix is exactly today's run.""" + monkeypatch.setattr(config, 'POOL_PREFIX', '') + _profiled(monkeypatch, {10: {'peakAnonBytes': GiB}}) + assert sizing.pool_for(10) is None + + +def test_a_malformed_cpu_map_falls_back_rather_than_crashing(pooled, monkeypatch): + monkeypatch.setattr(config, 'POOL_CPU', 'dwarf:notanumber,giant:1.1') + assert sizing.pool_cpu('dwarf') is None + assert sizing.pool_cpu('giant') == 1.1 + + +def test_pooled_memory_carries_no_margin_because_the_node_carries_it(pooled, monkeypatch, cluster): + """PROFILE_MARGIN and friends existed to keep a pod under its own LIMIT. + + There is no memory limit any more and the pod owns the node, so a margin in + the REQUEST constrains nothing the kubelet acts on -- it only wastes + schedulable space. The 1.60x lives in the node size, where node pressure can + actually enforce it. + """ + _profiled(monkeypatch, {300: {'peakAnonBytes': int(0.50 * GiB), 'seconds': 300.0}}) + r = jm._resources(end=300) + # dwarf's half-node request, not 0.50 * PROFILE_MARGIN + headroom + insurance + assert r.requests['memory'] == '1280Mi' + assert r.requests['cpu'] == 0.85 + assert not r.limits or 'memory' not in r.limits + + +def test_an_escalated_retry_requests_the_tier_it_was_promoted_to(pooled, monkeypatch, cluster): + """The label and the request have to agree. + + The affinity path knows the attempt, so an OOM retry lands on the promoted + pool. If the sizing path did not, the pod would arrive at a supergiant node + still asking for dwarf's memory -- under-requesting on the very node it was + escalated onto, and leaving room for a second pod on a tier whose whole + purpose is one pod per node. + """ + _profiled(monkeypatch, {300: {'peakAnonBytes': int(0.50 * GiB), 'seconds': 300.0}}) + # every prior attempt OOMed: pool_for asks for attempts BEFORE this one + monkeypatch.setattr(records, '_oom_count', lambda end, attempt: attempt) + first = jm._resources(end=300, attempt=1) + assert first.requests['memory'] == '1280Mi' # dwarf + assert first.requests['cpu'] == 0.85 + + third = jm._resources(end=300, attempt=3) + assert sizing.pool_for(300, attempt=3) == 'giant' + assert third.requests['memory'] == '6656Mi' # giant + assert third.requests['cpu'] == 1.85 + + +def test_escalation_does_not_opt_out_of_the_profile_when_pooled(pooled, monkeypatch, cluster): + """Unpooled, an escalated request outranks the profile and short-circuits it. + + Pooled, the promotion IS the escalation -- so short-circuiting would hand + the pod the flat configured request instead of the promoted tier's cut. + """ + _profiled(monkeypatch, {300: {'peakAnonBytes': int(0.50 * GiB), 'seconds': 300.0}}) + # every prior attempt OOMed: pool_for asks for attempts BEFORE this one + monkeypatch.setattr(records, '_oom_count', lambda end, attempt: attempt) + # `mem` set is what marks a retry as escalated + r = jm._resources(mem='9999Mi', end=300, attempt=2) + assert r.requests['memory'] == '2816Mi' # subgiant + + +# --- cache bump ------------------------------------------------------------ +# +# peakAnonBytes decides which node can HOLD a range; peakWorkingSetBytes decides +# whether that node can CACHE it. The two diverge by a median 2.5x and up to +# 10.4x across the profile, so a range can sit safely inside its tier's memory +# and still thrash. Measured on ssc-test 2026-08-03, one range on two 2-vCPU +# Intel nodes differing only in RAM: +# +# m8in.large 8 GiB 540 reads/ledger 21% iowait 1.86 lps +# r8in.large 16 GiB 65 reads/ledger 7% iowait 3.14 lps +# +# 44648511's real numbers are used throughout so these tests fail if the ladder +# ever moves such that the range we physically measured stops being promoted. +MEASURED_ANON = int(2.63 * GiB) # -> giant (1.61 <= 2.63 < 3.87) +MEASURED_WS = int(18.31 * GiB) # -> reaches supergiant and beyond + + +def test_a_range_that_cannot_cache_its_working_set_moves_up_one_rung(pooled, monkeypatch): + assert sizing._tier_for_bytes(MEASURED_ANON) == 'giant' + assert _pool(monkeypatch, MEASURED_ANON, MEASURED_WS) == 'supergiant' + + +def test_the_bump_is_one_rung_even_when_the_working_set_wants_two(pooled, monkeypatch): + """Clearing the thrash cliff is the goal, not fitting the working set. + + 18.31 GiB would land in hypergiant on its own, and hypergiant is two rungs + from giant. The measured range reaches full profile rate on supergiant while + still 1.29x UNDER its working set, so the extra rung buys nothing and costs + a doubling of cores. + """ + assert sizing._tier_for_bytes(MEASURED_WS) == 'hypergiant' + assert _pool(monkeypatch, MEASURED_ANON, MEASURED_WS) == 'supergiant' + + +def test_a_working_set_that_stays_inside_its_tier_is_not_promoted(pooled, monkeypatch): + assert sizing._tier_for_bytes(int(3.50 * GiB)) == 'giant' # same tier as the anon + assert _pool(monkeypatch, 2.00 * GiB, 3.50 * GiB) == 'giant' + + +def test_only_the_supergiant_rung_ships_open(pooled, monkeypatch): + """supergiant->hypergiant runs by default; hypergiant->supernova does not. + + Both cross a vCPU class now. Until 2026-08-04 hypergiant listed x8i.xlarge + (4 vCPU) at w80 beneath two 8-vCPU rungs, so pool_vcpu -- which reads the + SMALLEST shape -- reported 4 and supergiant->hypergiant priced as free, while + Karpenter tried w100 first and the promotion really cost 4->8. Removing the + x8i spot pools made the map honest: supergiant 4, hypergiant 8, supernova 16. + + So supergiant->hypergiant is now carried by POOL_CROSS_RUNGS rather than by + the guard, and hypergiant->supernova is denied outright -- with x8i.2xlarge + gone, supernova's only spot shapes are 4xlarges, so that rung buys 8->16 vCPU + for a promotion decided by working set, which does not predict throughput. + """ + _profiled(monkeypatch, { + 10: {'peakAnonBytes': int(7.00 * GiB), 'peakWorkingSetBytes': int(30.0 * GiB)}, + 20: {'peakAnonBytes': int(16.0 * GiB), 'peakWorkingSetBytes': int(40.0 * GiB)}, + }) + assert config.POOL_BLOCK_RUNGS == 'hypergiant->supernova' + assert sizing.pool_vcpu('supergiant') != sizing.pool_vcpu('hypergiant'), \ + "the rung crosses a class; the whitelist is what carries it, not the guard" + assert sizing.pool_for(10) == 'hypergiant' + assert sizing.pool_for(20) == 'hypergiant' # denied, stays put + + # dropping the denylist is NOT enough to reopen the supernova rung: it still + # crosses 8->16, so it also needs a POOL_CROSS_RUNGS entry + monkeypatch.setattr(config, 'POOL_BLOCK_RUNGS', '') + assert sizing.pool_for(20) == 'hypergiant' + monkeypatch.setattr(config, 'POOL_CROSS_RUNGS', + 'supergiant->hypergiant,hypergiant->supernova') + assert sizing.pool_for(20) == 'supernova' + + +def test_a_blocked_rung_beats_the_crossing_whitelist(pooled, monkeypatch): + """Deny wins over allow, so one stale env cannot silently re-open a rung.""" + monkeypatch.setattr(config, 'POOL_BLOCK_RUNGS', 'hypergiant->supernova') + monkeypatch.setattr(config, 'POOL_CROSS_RUNGS', 'hypergiant->supernova') + assert _pool(monkeypatch, 16.0 * GiB, 40.0 * GiB, end=20) == 'hypergiant' + + +def test_the_whitelist_only_opens_the_rung_it_names(pooled, monkeypatch): + """An exception must not become a blanket "ignore vCPU classes" switch. + + dwarf->subgiant also crosses a class (1 -> 2 vCPU) and is deliberately shut: + the longest dwarf range is 1663 s against a 10340 s critical path, so nothing + there can reach the tail and the promotion would be pure cost. + """ + monkeypatch.setattr(config, 'POOL_CROSS_RUNGS', 'hypergiant->supernova') + assert _pool(monkeypatch, 0.50 * GiB, 5.00 * GiB) == 'dwarf' + + +def test_the_free_rung_is_decided_by_node_vcpu_not_by_the_cpu_claim(pooled, monkeypatch): + """POOL_CPU stopped being a proxy for node size, so the guard must not read it. + + Claims were half the node everywhere, which made equal claims imply equal + nodes. That stopped being true once tiers were sized to the smallest shape in + their pool, so two tiers can carry different claims while sitting on + identically sized nodes. A claim comparison silently refuses such a rung. + + giant->supergiant is the case that still has equal nodes (4 vCPU both sides) + after the x8i pools were removed on 2026-08-04, so it is what exercises the + guard. The rung is not on either list, which is the point -- it must be judged + free on node size alone. + """ + # Force the claims apart. In production they happen to match right now, but + # the guard must read POOL_VCPU either way -- a claim comparison broke a rung + # once already when a tier was sized to its smallest shape. + monkeypatch.setattr(config, 'POOL_CPU', CPU.replace('supergiant:1.85', 'supergiant:1.20')) + assert sizing.pool_cpu('giant') != sizing.pool_cpu('supergiant') + assert sizing.pool_vcpu('giant') == sizing.pool_vcpu('supergiant') + assert not sizing._rung_listed(config.POOL_CROSS_RUNGS, 'giant', 'supergiant'), \ + "the whitelist must not be what carries this rung" + assert _pool(monkeypatch, 3.87 * GiB, 12.0 * GiB) == 'supergiant' + + +def test_every_tier_the_ladder_can_reach_has_a_vcpu_mapping(pooled): + """An unmapped tier makes the guard refuse every rung into or out of it. + + _cache_bump bails when either side is None, so a missing entry does not + crash -- it silently turns the whole rule off for that tier, which is the + kind of failure that only shows up as a run that cost more than it should. + """ + for tier in [name for _, name in sizing._parsed_pool_tiers()] + [ + config.POOL_UNPROFILED, config.POOL_NO_PROFILE]: + assert sizing.pool_vcpu(tier) is not None, f"{tier} has no POOL_VCPU entry" + + +def test_the_dwarf_rung_is_refused_because_it_also_crosses_a_cpu_class(pooled, monkeypatch): + """dwarf->subgiant is 0.50 -> 1.00, a 1-vCPU node to a 2-vCPU one. + + Blocking it is affordable: the longest dwarf range is 1663 s against a 10340 s + critical path, it ranks #1768 of 3985 in longest-first order, and dwarf is + 5.4% of total work. Nothing there can reach the tail. + """ + assert _pool(monkeypatch, 0.50 * GiB, 5.00 * GiB) == 'dwarf' + + +def test_the_top_of_the_ladder_has_nowhere_to_go(pooled, monkeypatch): + assert _pool(monkeypatch, 40.0 * GiB, 90.0 * GiB) == 'supernova' + + +def test_a_profile_without_working_set_data_routes_exactly_as_before(pooled, monkeypatch): + """Every profile generated before this change lacks peakWorkingSetBytes. + + Those must keep their old placement rather than crash or silently shift, so + the field being absent has to mean "no opinion", not "zero". + """ + _profiled(monkeypatch, { + 10: {'peakAnonBytes': MEASURED_ANON}, + 20: {'peakAnonBytes': MEASURED_ANON, 'peakWorkingSetBytes': 0}, + 30: {'peakAnonBytes': MEASURED_ANON, 'peakWorkingSetBytes': None}, + }) + assert sizing.pool_for(10) == 'giant' + assert sizing.pool_for(20) == 'giant' + assert sizing.pool_for(30) == 'giant' + + +def test_the_bump_is_the_same_every_run(pooled, monkeypatch): + """Deriving from the bytes each run is what stops the bump compounding. + + The same measurements must yield the same tier however many runs have read + them. Nothing carries a tier forward, so there is no verdict to bump on top + of and the promotion cannot ratchet a range to the ceiling one run at a time. + """ + _profiled(monkeypatch, { + 10: {'peakAnonBytes': MEASURED_ANON, 'peakWorkingSetBytes': MEASURED_WS}, + }) + assert sizing.pool_for(10) == 'supergiant' + assert sizing.pool_for(10) == 'supergiant' + + +def test_an_oom_still_climbs_from_the_bumped_tier(pooled, monkeypatch): + """The cache bump is a starting point, not a replacement for OOM escalation. + + A range bumped to supergiant that then OOMs there has proved it needs more + memory, and must keep climbing -- including onto the cpu-class rungs the + bump itself refuses to take speculatively. + """ + _profiled(monkeypatch, { + 10: {'peakAnonBytes': MEASURED_ANON, 'peakWorkingSetBytes': MEASURED_WS}, + }) + assert sizing.pool_for(10, rungs=0) == 'supergiant' + assert sizing.pool_for(10, rungs=1) == 'hypergiant' + assert sizing.pool_for(10, rungs=2) == 'supernova' + + +def test_the_bumped_tier_is_what_the_pod_actually_requests(pooled, monkeypatch, cluster): + """Routing to a pool and sizing for it have to agree. + + A pod pinned to supergiant nodes but carrying giant's request would let a + second pod share the node, which defeats the isolation the whole ladder + exists to buy. + """ + _profiled(monkeypatch, { + 10: {'peakAnonBytes': MEASURED_ANON, + 'peakWorkingSetBytes': MEASURED_WS, + 'seconds': 300.0}, + }) + r = jm._resources(end=10, attempt=1) + assert r.requests['memory'] == '14336Mi' # supergiant, not giant's 4096Mi + assert r.requests['cpu'] == 1.85 + + +def test_a_range_past_the_profile_is_sized_by_its_pool_not_the_flat_request(pooled, monkeypatch, cluster): + """Pooled placement without pooled sizing is a pod that never schedules. + + pool_for resolves a tier for EVERY range -- protostar when the range is + newer than anything measured -- but _profile_overrides used to bail on the + missing profile entry before it reached the pooled branch. The pod then got + protostar's node affinity with the run's flat REQ_CPU. + + Measured on ssc-test 2026-08-04: a 1200-worker run passed + --pubnet-parallel-catchup-cpu-request 6780m, so the past-the-profile ranges + asked for 6780m at a pool whose largest node is 4 vCPU. Permanently Pending, + retried forever, and silent -- earlier runs had only two such ranges and + nobody checked whether they had scheduled. + """ + monkeypatch.setattr(config, 'REQ_CPU', '6780m') + monkeypatch.setattr(config, 'REQ_MEM', '9Gi') + _profiled(monkeypatch, {10: {'peakAnonBytes': int(0.50 * GiB), 'seconds': 300.0}}) + assert sizing.pool_for(999) == 'protostar' # past the top of the profile + r = jm._resources(end=999, attempt=1) + assert r.requests['cpu'] == sizing.pool_cpu('protostar') + assert r.requests['memory'] == sizing.pool_memory('protostar') + assert r.requests['cpu'] != '6780m' + + +def test_no_profile_at_all_is_also_sized_by_its_pool(pooled, monkeypatch, cluster): + """Same failure one step further out: nebula gets a tier, so it needs a cut.""" + monkeypatch.setattr(config, 'REQ_CPU', '6780m') + monkeypatch.setattr(config, 'PROFILE', []) + assert sizing.pool_for(10) == 'nebula' + r = jm._resources(end=10, attempt=1) + assert r.requests['cpu'] == sizing.pool_cpu('nebula') + assert r.requests['memory'] == sizing.pool_memory('nebula') + + +def test_every_poolable_tier_fits_the_smallest_node_it_can_land_on(pooled): + """A claim above the smallest shape's usable cpu silently drops that shape. + + protostar at 1800m is deliberately above x8i.large's 1715m -- it is meant to + take the 4-vCPU shapes and leave the 128-vCPU X quota to hypergiant. Every + other tier must fit the node POOL_VCPU says is its smallest, or the tier + quietly loses its cheapest option. + """ + DAEMONSETS = 215 # alloy 10 + aws-node 75 + ebs-csi 30 + kube-proxy 100 + def usable(vcpu): + reserved = 60 + (10 if vcpu >= 2 else 0) + (5 if vcpu >= 3 else 0) + (5 if vcpu >= 4 else 0) + return vcpu * 1000 - reserved - DAEMONSETS + for tier in [name for _, name in sizing._parsed_pool_tiers()] + [config.POOL_NO_PROFILE]: + cpu, vcpu = sizing.pool_cpu(tier), sizing.pool_vcpu(tier) + if cpu is None or vcpu is None: + continue + assert cpu * 1000 <= usable(vcpu), ( + f"{tier} claims {cpu * 1000:.0f}m but its smallest node " + f"({vcpu} vCPU) only offers {usable(vcpu)}m") diff --git a/src/MissionParallelCatchup/tests/unit/test_profile_lookup.py b/src/MissionParallelCatchup/tests/unit/test_profile_lookup.py index a48dc4b3..2ee500e8 100644 --- a/src/MissionParallelCatchup/tests/unit/test_profile_lookup.py +++ b/src/MissionParallelCatchup/tests/unit/test_profile_lookup.py @@ -8,14 +8,16 @@ import pytest +import config +import profiles import job_monitor as jm PROFILE_RANGES = [ - (1000, {'peakRssBytes': 1_000_000_000, 'peakWorkingSetBytes': 9_000_000_000, - 'peakEphemeralBytes': 2_000_000_000, 'peakCpuCores': 0.5}), - (2000, {'peakRssBytes': 3_000_000_000, 'peakWorkingSetBytes': 13_000_000_000, - 'peakEphemeralBytes': 4_000_000_000, 'peakCpuCores': 1.2}), + (1000, {'peakAnonBytes': 1_000_000_000, 'peakWorkingSetBytes': 9_000_000_000, + 'peakEphemeralBytes': 2_000_000_000}), + (2000, {'peakAnonBytes': 3_000_000_000, 'peakWorkingSetBytes': 13_000_000_000, + 'peakEphemeralBytes': 4_000_000_000}), ] @@ -23,7 +25,7 @@ def profile(monkeypatch): """Install a loaded profile, as load_profile() would have left it.""" def install(ranges=PROFILE_RANGES): - monkeypatch.setattr(jm, 'PROFILE', sorted(ranges)) + monkeypatch.setattr(config, 'PROFILE', sorted(ranges)) return install @@ -33,9 +35,9 @@ def written(tmp_path, monkeypatch): def load(doc, mode='ephemeral', text=None): path = tmp_path / 'profile.json' path.write_text(text if text is not None else json.dumps(doc)) - monkeypatch.setattr(jm, 'PROFILE_PATH', str(path)) - monkeypatch.setattr(jm, 'STORAGE_MODE', mode) - return jm.load_profile() + monkeypatch.setattr(config, 'PROFILE_PATH', str(path)) + monkeypatch.setattr(config, 'STORAGE_MODE', mode) + return profiles.load_profile() return load @@ -43,7 +45,7 @@ def load(doc, mode='ephemeral', text=None): def test_profile_prefers_an_exact_end(profile): profile() - assert jm.profile_for(2000)['peakRssBytes'] == 3_000_000_000 + assert profiles.profile_for(2000)['peakAnonBytes'] == 3_000_000_000 def test_profile_rounds_up_to_the_next_measured_end_never_down(profile): @@ -51,7 +53,7 @@ def test_profile_rounds_up_to_the_next_measured_end_never_down(profile): # neighbour under-reports, and under-provisioning costs an eviction while # over-provisioning only costs packing density. profile() - assert jm.profile_for(1500)['peakRssBytes'] == 3_000_000_000, \ + assert profiles.profile_for(1500)['peakAnonBytes'] == 3_000_000_000, \ "1500 must size from 2000, not from 1000" @@ -59,28 +61,28 @@ def test_profile_falls_back_to_defaults_past_its_high_water_mark(profile): # An older profile has nothing above its own top, which is exactly where a # newer run's fresh ranges live. Extrapolating there would under-provision. profile() - assert jm.profile_for(9999) is None + assert profiles.profile_for(9999) is None def test_no_profile_at_all_is_not_an_error(monkeypatch): - monkeypatch.setattr(jm, 'PROFILE', None) - assert jm.profile_for(1000) is None - monkeypatch.setattr(jm, 'PROFILE', []) - assert jm.profile_for(1000) is None + monkeypatch.setattr(config, 'PROFILE', None) + assert profiles.profile_for(1000) is None + monkeypatch.setattr(config, 'PROFILE', []) + assert profiles.profile_for(1000) is None # --- reading the document ---------------------------------------------------- def test_no_configured_path_means_no_profile(monkeypatch): - monkeypatch.setattr(jm, 'PROFILE_PATH', '') - assert jm.load_profile() == [] + monkeypatch.setattr(config, 'PROFILE_PATH', '') + assert profiles.load_profile() == [] def test_an_unreadable_profile_is_not_fatal(written, tmp_path, monkeypatch): # It is an optimisation, never a prerequisite. assert written(None, text='{not json') == [] - monkeypatch.setattr(jm, 'PROFILE_PATH', str(tmp_path / 'nope.json')) - assert jm.load_profile() == [] + monkeypatch.setattr(config, 'PROFILE_PATH', str(tmp_path / 'nope.json')) + assert profiles.load_profile() == [] def test_a_matching_profile_keeps_every_axis(written): @@ -99,7 +101,7 @@ def test_entries_come_back_sorted_by_range_end(written): def test_a_non_numeric_range_key_is_skipped_not_fatal(written): got = written({'storageMode': 'ephemeral', - 'ranges': {'2000': {'peakRssBytes': 1}, 'tip': {'peakRssBytes': 2}}}) + 'ranges': {'2000': {'peakAnonBytes': 1}, 'tip': {'peakAnonBytes': 2}}}) assert [end for end, _ in got] == [2000] @@ -113,8 +115,7 @@ def test_a_cross_mode_profile_keeps_memory_but_drops_disk(written): assert len(got) == 1 rec = got[0][1] assert 'peakEphemeralBytes' not in rec - assert rec['peakRssBytes'] == 3_000_000_000 - assert rec['peakCpuCores'] == 1.2 + assert rec['peakAnonBytes'] == 3_000_000_000 def test_a_profile_with_no_declared_mode_is_taken_at_face_value(written): diff --git a/src/MissionParallelCatchup/tests/unit/test_range_generation.py b/src/MissionParallelCatchup/tests/unit/test_range_generation.py index 54955821..179d8784 100644 --- a/src/MissionParallelCatchup/tests/unit/test_range_generation.py +++ b/src/MissionParallelCatchup/tests/unit/test_range_generation.py @@ -6,65 +6,67 @@ import pytest +import config +import ranges import job_monitor as jm @pytest.fixture -def ranges(monkeypatch): +def build(monkeypatch): """Configure the generator and return a callable that runs it.""" def configure(generator='uniform', order='tip-first', parallelism=4, start=39990000, latest=40000000, per_job=1000, floor=64000, overlap=320): - monkeypatch.setattr(jm, 'RANGE_GENERATOR', generator) - monkeypatch.setattr(jm, 'RANGE_ORDER', order) - monkeypatch.setattr(jm, 'PARALLELISM', parallelism) - monkeypatch.setattr(jm, 'STARTING_LEDGER', start) - monkeypatch.setattr(jm, 'LATEST_LEDGER_NUM', latest) - monkeypatch.setattr(jm, 'LEDGERS_PER_JOB', per_job) - monkeypatch.setattr(jm, 'LOGARITHMIC_FLOOR_LEDGERS', floor) - monkeypatch.setattr(jm, 'OVERLAP_LEDGERS', overlap) - return jm.generate_ranges() + monkeypatch.setattr(config, 'RANGE_GENERATOR', generator) + monkeypatch.setattr(config, 'RANGE_ORDER', order) + monkeypatch.setattr(config, 'PARALLELISM', parallelism) + monkeypatch.setattr(config, 'STARTING_LEDGER', start) + monkeypatch.setattr(config, 'LATEST_LEDGER_NUM', latest) + monkeypatch.setattr(config, 'LEDGERS_PER_JOB', per_job) + monkeypatch.setattr(config, 'LOGARITHMIC_FLOOR_LEDGERS', floor) + monkeypatch.setattr(config, 'OVERLAP_LEDGERS', overlap) + return ranges.generate_ranges() return configure -def test_generators_emit_tip_first_by_default(ranges): - r = ranges() +def test_generators_emit_tip_first_by_default(build): + r = build() assert r[0][0] > r[-1][0], "index 0 must be the tip" -def test_oldest_first_reverses_dispatch_without_dropping_ranges(ranges): +def test_oldest_first_reverses_dispatch_without_dropping_ranges(build): # A profiling run wants the cheap early ranges measured first: the bucket # set only grows with ledger position, so tip-first front-loads the # expensive ones and an interrupted run profiles nothing cheap. - tip = ranges(order='tip-first') - old = ranges(order='oldest-first') + tip = build(order='tip-first') + old = build(order='oldest-first') assert old == list(reversed(tip)) assert sorted(old) == sorted(tip), "reversing must not change the range set" -def test_every_range_carries_the_overlap_on_top_of_its_ledger_count(ranges): +def test_every_range_carries_the_overlap_on_top_of_its_ledger_count(build): # The count is what the worker is asked to catch up, and it is always the # segment plus OVERLAP_LEDGERS -- measuring with overlap 0 measures nothing # the run will ever dispatch. - r = ranges(per_job=1000, overlap=320) + r = build(per_job=1000, overlap=320) assert {count for _, count in r} == {1320} -def test_the_ranges_tile_the_ledger_space_with_no_gap(ranges): - r = sorted(ranges(start=0, latest=10000, per_job=1000, overlap=320)) +def test_the_ranges_tile_the_ledger_space_with_no_gap(build): + r = sorted(build(start=0, latest=10000, per_job=1000, overlap=320)) ends = [end for end, _ in r] assert ends == list(range(1000, 10001, 1000)) assert ends[-1] == 10000, "the tip must be covered" -def test_a_short_tail_segment_is_not_padded_past_the_start(ranges): +def test_a_short_tail_segment_is_not_padded_past_the_start(build): # The last segment is min(remaining, seg_size), so a range list over a span # that does not divide evenly must not reach below STARTING_LEDGER. - r = ranges(start=0, latest=2500, per_job=1000, overlap=0) + r = build(start=0, latest=2500, per_job=1000, overlap=0) assert sorted(r) == [(500, 500), (1500, 1000), (2500, 1000)] -def test_logarithmic_ranges_match_the_shell_generator(ranges): +def test_logarithmic_ranges_match_the_shell_generator(build): # Verbatim output of logarithmic_range_generator.sh with # floor=16000 overlap=320 start=0 latest=500000 parallelism=4, captured # before it was deleted. Chunk size halves toward the tip, so exact values @@ -72,13 +74,106 @@ def test_logarithmic_ranges_match_the_shell_generator(ranges): expected = ("250000/62820 187500/62820 125000/62820 62500/62820 " "375001/31570 343751/31570 312501/31570 281251/31570 " "500000/16320 484000/16320 468000/16320 452000/14817").split() - r = ranges(generator='logarithmic', floor=16000, overlap=320, + r = build(generator='logarithmic', floor=16000, overlap=320, start=0, latest=500000, parallelism=4) assert [f"{end}/{count}" for end, count in r] == expected -def test_the_logarithmic_generator_also_honours_dispatch_order(ranges): - tip = ranges(generator='logarithmic', floor=16000, start=0, latest=500000) - old = ranges(generator='logarithmic', floor=16000, start=0, latest=500000, +def test_the_logarithmic_generator_also_honours_dispatch_order(build): + tip = build(generator='logarithmic', floor=16000, start=0, latest=500000) + old = build(generator='logarithmic', floor=16000, start=0, latest=500000, order='oldest-first') assert old == list(reversed(tip)) + + +@pytest.mark.parametrize('generator', ['uniforn', 'log', '', 'LOGARITHMIC']) +def test_an_unrecognised_generator_fails_instead_of_becoming_logarithmic(build, generator): + """A typo used to silently produce a different range layout. + + Both arms are explicit now, so anything else raises. This is the failure + mode worth a test: the run still SUCCEEDS with the wrong ranges, and no + downstream artifact records which generator produced them, so there is + nothing to notice afterwards. + """ + with pytest.raises(ValueError, match='RANGE_GENERATOR'): + build(generator=generator) + + +def test_longest_first_is_inert_without_a_profile(build, monkeypatch): + """Ordering is driven by RANGE_ORDER, never by profile detection. + + profile_for returns None for every range when no profile is loaded, so + every sort key ties and Python's stable sort leaves the generator's own + tip-first order untouched. The two flags are independent in configuration + and only coupled in effect -- which is why validate_config() refuses the + combination at startup rather than letting the flag look set and do nothing. + """ + monkeypatch.setattr(config, 'PROFILE', {}) + assert build(order='longest-first') == build(order='tip-first') + + +@pytest.mark.parametrize('order', ['tipfirst', 'longest', '', 'TIP-FIRST']) +def test_an_unrecognised_order_fails_instead_of_becoming_tip_first(build, order): + with pytest.raises(ValueError, match='RANGE_ORDER'): + build(order=order) + + +# --- validate_config: the startup preflight ---------------------------------- +# +# These checks exist at startup specifically because the reconcile loop catches +# and logs every exception then sleeps. A raise reached from inside it is an +# infinite log loop that never dispatches, so "fails loudly" depends entirely on +# validate_config being called from __main__ before the thread starts. + +@pytest.fixture +def preflight(monkeypatch): + def configure(generator='uniform', order='tip-first', profile=None): + monkeypatch.setattr(config, 'RANGE_GENERATOR', generator) + monkeypatch.setattr(config, 'RANGE_ORDER', order) + monkeypatch.setattr(config, 'PROFILE', profile) + return jm.validate_config + return configure + + +def test_valid_config_passes(preflight): + preflight(generator='logarithmic', order='oldest-first')() + + +def test_preflight_rejects_an_unknown_generator(preflight): + with pytest.raises(ValueError, match='RANGE_GENERATOR'): + preflight(generator='uniforn')() + + +def test_preflight_rejects_an_unknown_order(preflight): + with pytest.raises(ValueError, match='RANGE_ORDER'): + preflight(order='longest')() + + +def test_preflight_rejects_longest_first_without_a_profile(preflight): + with pytest.raises(ValueError, match='requires a profile'): + preflight(order='longest-first', profile=None)() + + +def test_preflight_allows_longest_first_with_a_profile(preflight): + preflight(order='longest-first', profile=[(40000000, {'seconds': 900.0})])() + + +def test_the_preflight_runs_before_the_reconcile_thread_starts(): + """Guards the placement, which is the whole point of the check. + + If validate_config ever moves inside reconcile (or after the thread start), + a bad config becomes a silent hang instead of a crash. Asserted against the + source because the ordering, not the call, is what has to hold. + """ + import inspect + main = inspect.getsource(jm.main) + assert 'validate_config()' in main, "validate_config must be called from main()" + assert main.index('validate_config()') < main.index('reconcile_thread.start()'), \ + "validate_config must run BEFORE the reconcile thread starts" + assert main.index('load_profile()') < main.index('validate_config()'), \ + "validate_config checks the profile, so it must run after load_profile" + + +def jm_source(): + import inspect + return inspect.getsource(jm) diff --git a/src/MissionParallelCatchup/tests/unit/test_reaping.py b/src/MissionParallelCatchup/tests/unit/test_reaping.py index 67954a2b..3f94f9f2 100644 --- a/src/MissionParallelCatchup/tests/unit/test_reaping.py +++ b/src/MissionParallelCatchup/tests/unit/test_reaping.py @@ -15,6 +15,9 @@ from kubernetes import client import fake_k8s +import config +import kube +import records import job_monitor as jm @@ -26,16 +29,16 @@ def k8s(logdir, monkeypatch): """A fake cluster wired into the monitor, with no reconcile in the way.""" fake = fake_k8s.FakeCluster(namespace=NAMESPACE) - monkeypatch.setattr(jm, 'core_v1', fake.core_v1) - monkeypatch.setattr(jm, 'batch_v1', fake.batch_v1) - monkeypatch.setattr(jm, 'NAMESPACE', NAMESPACE) - monkeypatch.setattr(jm, 'RUN_NAME', RUN) - monkeypatch.setattr(jm, 'STORAGE_MODE', 'pvc') + monkeypatch.setattr(kube, 'core_v1', fake.core_v1) + monkeypatch.setattr(kube, 'batch_v1', fake.batch_v1) + monkeypatch.setattr(config, 'NAMESPACE', NAMESPACE) + monkeypatch.setattr(config, 'RUN_NAME', RUN) + monkeypatch.setattr(config, 'STORAGE_MODE', 'pvc') def add_job(end, attempt): name = jm.job_name(end, attempt) - labels = {jm.LABEL_RUN: RUN, jm.LABEL_RANGE: str(end), - jm.LABEL_ATTEMPT: str(attempt)} + labels = {config.LABEL_RUN: RUN, config.LABEL_RANGE: str(end), + config.LABEL_ATTEMPT: str(attempt)} fake.batch_v1.create_namespaced_job(NAMESPACE, client.V1Job( metadata=client.V1ObjectMeta(name=name, labels=labels), spec=client.V1JobSpec( @@ -81,7 +84,7 @@ def test_delete_job_is_best_effort(monkeypatch, k8s, status): # status must be swallowed too: losing a Job to a leaked object is a # disk/etcd cost, but raising here would abort the whole reconcile pass. boom = Boom(status) - monkeypatch.setattr(jm, 'batch_v1', boom) + monkeypatch.setattr(kube, 'batch_v1', boom) jm.delete_job(1, 1) # must not raise assert boom.calls == 1 @@ -103,7 +106,7 @@ def test_a_completed_range_reaps_every_attempt_not_just_the_winner(k8s): def test_a_list_failure_leaves_the_jobs_to_the_ttl_rather_than_raising(monkeypatch, k8s): - monkeypatch.setattr(jm, 'batch_v1', Boom(500)) + monkeypatch.setattr(kube, 'batch_v1', Boom(500)) jm.reap_range_jobs(300) # must not raise @@ -115,29 +118,22 @@ def test_the_reap_waits_for_the_collectors_done_marker(k8s): # with no peaks at all. Only the collector knows it is done, and deleting # the Job reaps the pod -- the last place peaks could still be read from. k8s.add_job(300, 1) - full = {'txApply': 5.0, 'peakAnonBytes': 99} - jm._reap_if_complete(300, 1, full) - assert k8s.job_names() == [jm.job_name(300, 1)], \ - "reaped before the collector marked it done" - open(jm.done_path(300, 1), 'w').close() - jm._reap_if_complete(300, 1, full) + assert jm._attempt_finalized(300, 1) is False, \ + "no marker yet, so reconcile must not reap" + open(records.done_path(300, 1), 'w').close() + assert jm._attempt_finalized(300, 1) is True + jm.reap_range_jobs(300) assert k8s.job_names() == [] def test_the_done_marker_is_the_only_thing_that_counts_as_finalized(logdir): assert jm._attempt_finalized(300, 1) is False - open(jm.metrics_path(300, 1), 'w').close() + open(records.metrics_path(300, 1), 'w').close() assert jm._attempt_finalized(300, 1) is False, "metrics are not a promise" - open(jm.done_path(300, 1), 'w').close() + open(records.done_path(300, 1), 'w').close() assert jm._attempt_finalized(300, 1) is True -def test_a_record_has_peaks_only_if_some_axis_actually_measured_something(): - assert jm._has_peaks({'peakAnonBytes': 1}) is True - assert jm._has_peaks({'peakAnonBytes': None, 'txApply': 5.0}) is False - assert jm._has_peaks({}) is False - - # --- releasing the volume ---------------------------------------------------- def test_a_completed_range_releases_its_volume(k8s): @@ -153,7 +149,7 @@ def test_a_completed_range_releases_its_volume(k8s): def test_ephemeral_mode_has_no_volume_to_release(monkeypatch, k8s): jm.ensure_pvc(300, owner=None) - monkeypatch.setattr(jm, 'STORAGE_MODE', 'ephemeral') + monkeypatch.setattr(config, 'STORAGE_MODE', 'ephemeral') jm.release_pvc(300) assert k8s.pvc_names() != [], "ephemeral mode deleted a volume it does not own" diff --git a/src/MissionParallelCatchup/tests/unit/test_records.py b/src/MissionParallelCatchup/tests/unit/test_records.py index 4ddbaab5..d039e2ea 100644 --- a/src/MissionParallelCatchup/tests/unit/test_records.py +++ b/src/MissionParallelCatchup/tests/unit/test_records.py @@ -7,6 +7,8 @@ import os +import config +import records import job_monitor as jm import log_collector as lc @@ -18,34 +20,29 @@ def basename(path): # --- one volume, one set of filenames ---------------------------------------- def test_both_sides_agree_on_the_metrics_filename(logdir): - assert basename(jm.metrics_path(300, 2)) == basename(lc.base(300, 2)) + '.metrics' + assert basename(records.metrics_path(300, 2)) == basename(lc.base(300, 2)) + '.metrics' def test_both_sides_agree_on_the_done_marker(logdir): # It licenses the monitor to reap the pod, which is the only place peaks can # still be read from. - assert basename(jm.done_path(300, 2)) == basename(lc.done_path(300, 2)) - - -def test_both_sides_agree_on_the_attempt_label_key(): - # Two readers, one key. A mismatch reproduces the silent collision below. - assert jm.LABEL_ATTEMPT == lc.LABEL_ATTEMPT + assert basename(records.done_path(300, 2)) == basename(lc.done_path(300, 2)) def test_the_monitor_log_lands_where_the_mission_collects_it(): # collectLogsFromPods tars LOG_DIR. The monitor used to write its own log to # /data, an emptyDir, so OOM-retry storms never reached the destination # directory and did not survive a monitor restart. - assert jm.LOG_DIR == lc.LOG_DIR, \ + assert config.LOG_DIR == config.LOG_DIR, \ "collector and monitor must share the collected directory" - assert os.path.dirname(jm.PROGRESS_FILE) == jm.LOG_DIR + assert os.path.dirname(config.PROGRESS_FILE) == config.LOG_DIR def test_every_per_attempt_artifact_is_named_for_its_attempt(logdir): # One namespace per (range, attempt) across five writers; a helper that # dropped the attempt would have two attempts overwrite each other. - paths = [jm.log_path(300, 2), jm.state_path(300, 2), jm.outcome_path(300, 2), - jm.metrics_path(300, 2), jm.verdict_path(300, 2), jm.done_path(300, 2)] + paths = [records.log_path(300, 2), records.state_path(300, 2), records.outcome_path(300, 2), + records.metrics_path(300, 2), records.verdict_path(300, 2), records.done_path(300, 2)] assert all(basename(p).startswith('range-300-a2.') for p in paths), paths assert len({basename(p) for p in paths}) == len(paths), "two writers share a filename" @@ -60,57 +57,23 @@ def test_the_worker_pod_carries_its_attempt_number(): # attempt's peak instead of being maxed against it -- destroying exactly # the OOM evidence the chain exists to keep. labels = jm.pod_labels(300, 2) - assert labels[jm.LABEL_ATTEMPT] == '2' - assert labels[jm.LABEL_RANGE] == '300' - assert labels[jm.LABEL_RUN] == jm.RUN_NAME + assert labels[config.LABEL_ATTEMPT] == '2' + assert labels[config.LABEL_RANGE] == '300' + assert labels[config.LABEL_RUN] == config.RUN_NAME def test_the_mission_label_is_opt_in(monkeypatch): # It is high-cardinality and only wanted when something is scraping by # mission, so it must not appear unless both switches are set. - monkeypatch.setattr(jm, 'MISSION', 'pubnet-catchup') - monkeypatch.setattr(jm, 'EMIT_MISSION_LABEL', False) + monkeypatch.setattr(config, 'MISSION', 'pubnet-catchup') + monkeypatch.setattr(config, 'EMIT_MISSION_LABEL', False) assert 'mission' not in jm.pod_labels(300, 1) - monkeypatch.setattr(jm, 'EMIT_MISSION_LABEL', True) + monkeypatch.setattr(config, 'EMIT_MISSION_LABEL', True) assert jm.pod_labels(300, 1)['mission'] == 'pubnet-catchup' # --- what the ConfigMap mirror is allowed to carry --------------------------- -def test_the_configmap_mirror_carries_no_profiling_fields(): - # Profile data lives only on the volume. In the ConfigMap it is what pushes - # a ~30-byte state record to ~172 bytes and the whole document toward the - # 1 MiB cap at ~6100 ranges -- reachable simply by halving ledgersPerJob. - progress = {'completed': {'100': {'attempts': 1, 'count': 16320, 'seconds': 700.0, - 'peakRssBytes': 123, 'peakCpuCores': 1.9, - 'txApply': 200.0, 'wallSeconds': 750.0}}, - 'failed': {}} - out = jm._state_only(progress)['completed']['100'] - assert out == {'attempts': 1, 'count': 16320}, out - # ...and the untouched original still has everything for the volume copy - assert 'peakRssBytes' in progress['completed']['100'] - - -def test_the_mirror_keeps_the_bookkeeping_the_mission_driver_reads(): - # Stripping is by field, not by whitelist-of-one: a failed range's record - # is what the driver reports on, so it must survive the trip. - progress = {'completed': {}, 'failed': {'100': {'attempts': 5, 'reason': 'oom'}}} - assert jm._state_only(progress)['failed']['100'] == {'attempts': 5, 'reason': 'oom'} - - -def test_a_structurally_wrong_record_is_dropped_not_carried(logdir): - # This document is read off a volume that outlives the run and mirrored - # through a ConfigMap a second writer can clobber, so it comes back the - # wrong SHAPE as well as merely truncated. A single non-dict entry took - # every later pass down inside observe_recorded/sync_counters -- after - # dispatch, so the exception the reconcile loop swallows left the run with - # no status update and no `remaining` ever again. - got = jm._sane_progress({'completed': {'100': {'attempts': 1}, '200': 'nonsense'}, - 'failed': {'300': ['also', 'wrong']}}) - assert got['completed'] == {'100': {'attempts': 1}} - assert got['failed'] == {} - - # --- durations the collector can read off a pod the monitor never saw --------- def test_a_terminal_pod_still_yields_its_real_duration(): diff --git a/src/MissionParallelCatchup/tests/unit/test_resources.py b/src/MissionParallelCatchup/tests/unit/test_resources.py index 63396bd5..2718c8cf 100644 --- a/src/MissionParallelCatchup/tests/unit/test_resources.py +++ b/src/MissionParallelCatchup/tests/unit/test_resources.py @@ -9,81 +9,90 @@ import pytest +import config +import units +import sizing +import attempts import job_monitor as jm PROFILE_RANGES = [ - (1000, {'peakRssBytes': 1_000_000_000, 'peakWorkingSetBytes': 9_000_000_000, - 'peakEphemeralBytes': 2_000_000_000, 'peakCpuCores': 0.5}), - (2000, {'peakRssBytes': 3_000_000_000, 'peakWorkingSetBytes': 13_000_000_000, - 'peakEphemeralBytes': 4_000_000_000, 'peakCpuCores': 1.2}), + (1000, {'peakAnonBytes': 1_000_000_000, 'peakWorkingSetBytes': 9_000_000_000, + 'peakEphemeralBytes': 2_000_000_000}), + (2000, {'peakAnonBytes': 3_000_000_000, 'peakWorkingSetBytes': 13_000_000_000, + 'peakEphemeralBytes': 4_000_000_000}), ] MI = 1024 ** 2 @pytest.fixture -def sizing(monkeypatch): +def shaped(monkeypatch): """The worker's configured shape, plus a loaded profile.""" def configure(ranges=PROFILE_RANGES, margin=1.1, req_mem='9Gi', req_eph='35Gi', lim_eph='40Gi', max_mem='32Gi', - headroom='512Mi', runtime_insurance='3Gi'): - monkeypatch.setattr(jm, 'PROFILE', sorted(ranges)) - monkeypatch.setattr(jm, '_SORTED_SECONDS', None) - monkeypatch.setattr(jm, 'PROFILE_MARGIN', margin) - monkeypatch.setattr(jm, 'PROFILE_MAX_MEM', max_mem) - monkeypatch.setattr(jm, 'PROFILE_CACHE_HEADROOM', headroom) - monkeypatch.setattr(jm, 'PROFILE_RUNTIME_MEMORY_INSURANCE', runtime_insurance) - monkeypatch.setattr(jm, 'REQ_CPU', '1800m') - monkeypatch.setattr(jm, 'REQ_MEM', req_mem) - monkeypatch.setattr(jm, 'REQ_EPHEMERAL', req_eph) - monkeypatch.setattr(jm, 'LIM_EPHEMERAL', lim_eph) + headroom='512Mi', runtime_insurance='3Gi', + eph_headroom='2Gi', eph_insurance='8Gi', max_eph='64Gi'): + monkeypatch.setattr(config, 'PROFILE', sorted(ranges)) + monkeypatch.setattr(config, '_SORTED_SECONDS', None) + monkeypatch.setattr(config, 'PROFILE_MARGIN', margin) + monkeypatch.setattr(config, 'PROFILE_MAX_MEM', max_mem) + monkeypatch.setattr(config, 'PROFILE_CACHE_HEADROOM', headroom) + monkeypatch.setattr(config, 'PROFILE_RUNTIME_MEMORY_INSURANCE', runtime_insurance) + monkeypatch.setattr(config, 'PROFILE_EPHEMERAL_HEADROOM', eph_headroom) + monkeypatch.setattr(config, 'PROFILE_RUNTIME_EPHEMERAL_INSURANCE', eph_insurance) + monkeypatch.setattr(config, 'PROFILE_MAX_EPHEMERAL', max_eph) + monkeypatch.setattr(config, 'REQ_CPU', '1800m') + monkeypatch.setattr(config, 'REQ_MEM', req_mem) + monkeypatch.setattr(config, 'REQ_EPHEMERAL', req_eph) + monkeypatch.setattr(config, 'LIM_EPHEMERAL', lim_eph) return configure # --- what the profile is allowed to say -------------------------------------- -def test_profile_sizes_a_first_attempt(sizing): - sizing() - out = jm._profile_overrides(2000, escalated=False) +def test_profile_sizes_a_first_attempt(shaped): + shaped() + out = sizing._profile_overrides(2000, escalated=False) assert out['memory'] == '3659Mi' # 3 GB rss * 1.1 + 512Mi - assert out['ephemeral-storage'] == '4196Mi' + # 3.8Gi measured * 1.1 margin + 2Gi flat headroom; this range is short + # enough that its runtime-weighted share rounds to nothing. + assert out['ephemeral-storage'] == '6244Mi' # cpu is no longer profiled: REQ_CPU is fixed, so there is nothing to size, # and a measured cpu value only makes packing non-uniform. assert 'cpu' not in out - assert 'peakCpuCores' not in jm.PEAK_FIELDS -def test_profile_does_not_override_an_escalated_retry(sizing): +def test_profile_does_not_override_an_escalated_retry(shaped): # An escalation is a measurement of THIS run and outranks an earlier one. - sizing() - assert jm._profile_overrides(2000, escalated=True) == {} + shaped() + assert sizing._profile_overrides(2000, escalated=True) == {} -def test_profile_gives_nothing_past_its_high_water_mark(sizing): - sizing() - assert jm._profile_overrides(99999, escalated=False) == {} - assert jm._profile_overrides(None, escalated=False) == {} +def test_profile_gives_nothing_past_its_high_water_mark(shaped): + shaped() + assert sizing._profile_overrides(99999, escalated=False) == {} + assert sizing._profile_overrides(None, escalated=False) == {} -def test_profile_memory_is_capped_at_its_own_ceiling_not_the_configured_request(sizing): +def test_profile_memory_is_capped_at_its_own_ceiling_not_the_configured_request(shaped): # A range measured above the configured request must be able to ask for more, # or it packs as though it were small and lands somewhere it cannot fit. The # ceiling is what bounds it, and the OOM ladder can still climb past that. - sizing(ranges=[(1, {'peakRssBytes': 500_000_000_000})], + shaped(ranges=[(1, {'peakAnonBytes': 500_000_000_000})], req_mem='9Gi', max_mem='32Gi') - assert jm._profile_overrides(1, escalated=False)['memory'] == '32768Mi' + assert sizing._profile_overrides(1, escalated=False)['memory'] == '32768Mi' -def test_profile_memory_can_exceed_the_configured_request(sizing): +def test_profile_memory_can_exceed_the_configured_request(shaped): # 28 GB peak against a 9Gi configured request: the profile must raise it. - sizing(ranges=[(1, {'peakRssBytes': 28_000_000_000})], + shaped(ranges=[(1, {'peakAnonBytes': 28_000_000_000})], req_mem='9Gi', max_mem='32Gi') - got = jm._profile_overrides(1, escalated=False)['memory'] - assert jm._quantity_bytes(got) > jm._quantity_bytes('9Gi') + got = sizing._profile_overrides(1, escalated=False)['memory'] + assert units.quantity_bytes(got) > units.quantity_bytes('9Gi') -def test_memory_is_sized_from_rss_never_from_working_set(sizing): +def test_memory_is_sized_from_rss_never_from_working_set(shaped): # Working set is whatever limit it was measured under -- the kernel grows # page cache to fill it. Measured on ssc-test, one 420-ledger range: # limit 4Gi -> ws 3.61 GiB, rss 2.43 GiB, 775s @@ -92,85 +101,72 @@ def test_memory_is_sized_from_rss_never_from_working_set(sizing): # rss is flat and wall-clock is flat, so sizing from ws would reserve 5x the # real demand for no gain. It is still recorded -- kubelet ranks # node-pressure evictions on it, so it explains an eviction rss cannot. - sizing(ranges=[(1, {'peakWorkingSetBytes': 13_000_000_000})]) - assert 'memory' not in jm._profile_overrides(1, escalated=False), \ + shaped(ranges=[(1, {'peakWorkingSetBytes': 13_000_000_000})]) + assert 'memory' not in sizing._profile_overrides(1, escalated=False), \ "an older artifact without rss must fall back, not guess from working set" - assert 'peakWorkingSetBytes' in jm.PEAK_FIELDS + assert 'peakWorkingSetBytes' in attempts.PEAK_FIELDS -def test_sizing_prefers_anon_and_falls_back_to_the_scraped_rss(sizing): - # peakAnonBytes is kubelet's rssBytes on the collector's own poll; - # peakRssBytes is the same quantity via a 30s Prometheus scrape. A profile - # captured before the collector tracked anon must keep sizing exactly as it - # did, or every existing profile silently reverts to default. - sizing(ranges=[(1, {'peakRssBytes': 1_000_000_000})]) - scraped_only = jm._profile_overrides(1, escalated=False)['memory'] - # Both present: the finer figure wins, not the coarser one it sits beside. - sizing(ranges=[(1, {'peakAnonBytes': 1_000_000_000, - 'peakRssBytes': 3_000_000_000})]) - assert jm._profile_overrides(1, escalated=False)['memory'] == scraped_only - - -def test_small_ranges_get_absolute_slack_not_just_a_percentage(sizing): +def test_small_ranges_get_absolute_slack_not_just_a_percentage(shaped): # memory.max bounds anon PLUS page cache. At 190 MiB rss a 1.1x margin is # 19 MiB of slack -- measured on ssc-test, 90 ranges OOMKilled within 90s of # dispatch. The fixed headroom is what makes small ranges survivable. - sizing(ranges=[(1, {'peakRssBytes': 190 * MI})]) - got = jm._quantity_bytes(jm._profile_overrides(1, escalated=False)['memory']) + shaped(ranges=[(1, {'peakAnonBytes': 190 * MI})]) + got = units.quantity_bytes(sizing._profile_overrides(1, escalated=False)['memory']) slack = (got - 190 * MI) / MI assert slack > 400, f"only {slack:.0f}MiB of slack above rss" @pytest.mark.parametrize('peak_mi', [648, 1467, 222]) # live: median, largest, smallest anon -def test_the_sizing_formula_is_peak_times_margin_plus_headroom(sizing, peak_mi): - sizing(ranges=[(1, {'peakAnonBytes': peak_mi * MI})], margin=1.15, +def test_the_sizing_formula_is_peak_times_margin_plus_headroom(shaped, peak_mi): + shaped(ranges=[(1, {'peakAnonBytes': peak_mi * MI})], margin=1.15, headroom='512Mi', max_mem='32Gi') - got = jm._profile_overrides(1, escalated=False)['memory'] + got = sizing._profile_overrides(1, escalated=False)['memory'] assert got == f"{int(peak_mi * MI * 1.15) // MI + 512}Mi" -def test_runtime_insurance_is_weighted_by_the_longest_profiled_range(sizing): - sizing(ranges=[ +def test_runtime_insurance_is_weighted_by_the_longest_profiled_range(shaped): + shaped(ranges=[ (1, {'peakAnonBytes': 1024 * MI, 'seconds': 100}), (2, {'peakAnonBytes': 1024 * MI, 'seconds': 400}), ], margin=1.15, headroom='512Mi', runtime_insurance='3Gi') - short = jm._quantity_bytes(jm._profile_overrides(1, escalated=False)['memory']) - longest = jm._quantity_bytes(jm._profile_overrides(2, escalated=False)['memory']) + short = units.quantity_bytes(sizing._profile_overrides(1, escalated=False)['memory']) + longest = units.quantity_bytes(sizing._profile_overrides(2, escalated=False)['memory']) base = int(1024 * MI * 1.15) + 512 * MI assert short == (base + 768 * MI) // MI * MI assert longest == (base + 3 * 1024 * MI) // MI * MI @pytest.mark.parametrize('seconds', [None, 0, -1, 'bad', float('nan'), float('inf')]) -def test_invalid_or_nonpositive_runtime_adds_no_insurance(sizing, seconds): - sizing(ranges=[(1, {'peakAnonBytes': 1024 * MI, 'seconds': seconds})], +def test_invalid_or_nonpositive_runtime_adds_no_insurance(shaped, seconds): + shaped(ranges=[(1, {'peakAnonBytes': 1024 * MI, 'seconds': seconds})], margin=1.15, headroom='512Mi', runtime_insurance='3Gi') - got = jm._quantity_bytes(jm._profile_overrides(1, escalated=False)['memory']) + got = units.quantity_bytes(sizing._profile_overrides(1, escalated=False)['memory']) assert got == (int(1024 * MI * 1.15) + 512 * MI) // MI * MI -def test_zero_runtime_insurance_disables_it_and_the_cap_still_applies_last(sizing): +def test_zero_runtime_insurance_disables_it_and_the_cap_still_applies_last(shaped): ranges = [(1, {'peakAnonBytes': 1024 * MI, 'seconds': 100})] - sizing(ranges=ranges, margin=1.15, headroom='512Mi', + shaped(ranges=ranges, margin=1.15, headroom='512Mi', runtime_insurance='0', max_mem='2Gi') - without = jm._profile_overrides(1, escalated=False)['memory'] + without = sizing._profile_overrides(1, escalated=False)['memory'] assert without == f"{int(1024 * MI * 1.15) // MI + 512}Mi" - sizing(ranges=ranges, margin=1.15, headroom='512Mi', + shaped(ranges=ranges, margin=1.15, headroom='512Mi', runtime_insurance='3Gi', max_mem='2Gi') - assert jm._profile_overrides(1, escalated=False)['memory'] == '2048Mi' + assert sizing._profile_overrides(1, escalated=False)['memory'] == '2048Mi' # --- what lands on the container --------------------------------------------- -def test_a_measured_range_requests_its_measurement_and_limits_only_disk(sizing): +def test_a_measured_range_requests_its_measurement_and_limits_only_disk(shaped): # The profile moves requests. Disk is the one dimension still limited, and # its limit is matched so a range measured to need more is allowed to use it. - sizing() + shaped() r = jm._resources(end=2000) assert r.requests['memory'] == '3659Mi' - assert r.requests['ephemeral-storage'] == r.limits['ephemeral-storage'] == '4196Mi' + assert r.requests['ephemeral-storage'] == r.limits['ephemeral-storage'] == '6244Mi' # The configured request, not a measured one -- a profiled range now packs # at exactly the same cpu as an unprofiled one. assert r.requests['cpu'] == '1800m' @@ -178,9 +174,9 @@ def test_a_measured_range_requests_its_measurement_and_limits_only_disk(sizing): f"a worker may only ever be limited on disk, got {sorted(r.limits)}" -def test_an_unmeasured_range_keeps_the_configured_requests(sizing): +def test_an_unmeasured_range_keeps_the_configured_requests(shaped): # No profile entry must behave exactly as if there were no profile at all. - sizing() + shaped() r = jm._resources(end=99999) assert r.requests['memory'] == '9Gi' assert 'memory' not in r.limits @@ -188,27 +184,27 @@ def test_an_unmeasured_range_keeps_the_configured_requests(sizing): assert r.limits['ephemeral-storage'] == '40Gi' -def test_an_escalated_retry_keeps_its_own_size(sizing): +def test_an_escalated_retry_keeps_its_own_size(shaped): # The escalation already chose the size; the profile must not overwrite it. # It lands on the request, which is the whole mechanism now: a bigger request # places the pod where the memory is actually free, and raises the bar before # the kubelet picks it as an eviction victim. - sizing() + shaped() r = jm._resources(mem='36000Mi', end=2000) assert r.requests['memory'] == '36000Mi' assert 'memory' not in r.limits, "an escalated retry must not be capped either" assert r.requests['cpu'] == '1800m', "cpu must fall back to the configured request" -def test_ephemeral_escalation_raises_request_and_limit_together(sizing): +def test_ephemeral_escalation_raises_request_and_limit_together(shaped): # ephemeral-storage is a scheduling dimension: a pod that outgrew its limit # will not fit where it was placed before unless the request moves too. - sizing() + shaped() r = jm._resources(eph='60Gi', end=2000) assert r.requests['ephemeral-storage'] == r.limits['ephemeral-storage'] == '60Gi' -def test_no_worker_gets_a_cpu_or_memory_limit(sizing): +def test_no_worker_gets_a_cpu_or_memory_limit(shaped): # _profile_overrides returns {} for BOTH "no profile entry" and "escalated # attempt". Treating them the same handed an OOM retry more memory while # capping it at LIM_CPU, when the attempt that just failed ran unlimited. @@ -219,7 +215,7 @@ def test_no_worker_gets_a_cpu_or_memory_limit(sizing): # At a 2-core limit every range pegs 2.0 anyway, so the measured peak would # be a ceiling and the profile could never learn real demand. Packing is # driven by the request, which every worker still carries. - sizing() + shaped() measured = jm._resources(end=2000) escalated = jm._resources(mem='9000Mi', end=2000) unmeasured = jm._resources(end=999999999) @@ -231,10 +227,51 @@ def test_no_worker_gets_a_cpu_or_memory_limit(sizing): -def test_pvc_mode_takes_no_ephemeral_request_or_override(sizing): +def test_pvc_mode_takes_no_ephemeral_request_or_override(shaped): # /data is not on the node disk there, so sizing it would be meaningless -- # and a large request would make disk the binding dimension and halve # workers-per-node for no reason. - sizing(req_eph='') + shaped(req_eph='') r = jm._resources(end=2000) assert 'ephemeral-storage' not in r.requests + + +def test_disk_gets_a_flat_headroom_and_a_runtime_weighted_share(shaped): + """Disk is sized like memory: measured peak, plus a floor, plus insurance. + + The 2026-08-01 ephemeral run peaked at 37.76Gi against a flat 40Gi limit -- + 6% of margin, on a detection-and-escalation path that has never fired on + real data. Margin alone does not fix that: it scales the measurement, so the + ranges closest to the limit get the least absolute headroom. + + Disk earns the runtime weighting the same way memory does -- measured across + 3985 ranges, peak disk tracks runtime at pearson 0.920 (runtime decile 0 + uses 0.1Gi, decile 9 uses 24.7Gi), so the weighting lands the allowance on + exactly the ranges that need it. + """ + shaped(eph_headroom='2Gi', eph_insurance='8Gi') + short = sizing._profile_overrides(2000, escalated=False)['ephemeral-storage'] + + # same range, no allowances at all -> margin only + shaped(eph_headroom='0', eph_insurance='0') + bare = sizing._profile_overrides(2000, escalated=False)['ephemeral-storage'] + + assert units.quantity_bytes(short) > units.quantity_bytes(bare) + assert units.quantity_bytes(short) - units.quantity_bytes(bare) >= 2 * 1024 ** 3 + + +def test_a_measured_range_may_exceed_the_flat_unprofiled_disk_limit(shaped): + """PROFILE_MAX_EPHEMERAL is above LIM_EPHEMERAL on purpose. + + LIM_EPHEMERAL is what an UNMEASURED range gets. Capping a measured range at + it would discard the measurement -- the worst range observed wants ~43Gi + after margin alone, which the flat 40Gi limit would silently clip back to + the value that was already too tight. + """ + shaped(lim_eph='40Gi', max_eph='64Gi', eph_headroom='2Gi', eph_insurance='8Gi') + out = sizing._profile_overrides(2000, escalated=False)['ephemeral-storage'] + assert units.quantity_bytes(out) > 0 + # and the cap still binds when it should + shaped(lim_eph='40Gi', max_eph='1Gi', eph_headroom='2Gi', eph_insurance='8Gi') + capped = sizing._profile_overrides(2000, escalated=False)['ephemeral-storage'] + assert units.quantity_bytes(capped) == 1024 ** 3 diff --git a/src/MissionParallelCatchup/tests/unit/test_retry_counters.py b/src/MissionParallelCatchup/tests/unit/test_retry_counters.py index 481eb8d3..874b4e9e 100644 --- a/src/MissionParallelCatchup/tests/unit/test_retry_counters.py +++ b/src/MissionParallelCatchup/tests/unit/test_retry_counters.py @@ -4,6 +4,9 @@ from prometheus_client import generate_latest +import config +import records +import metrics import job_monitor as jm @@ -20,8 +23,8 @@ def _totals(progress=None, current_attempts=()): def test_verdict_is_preferred_over_outcome(logdir): - _write(jm.outcome_path(100, 1), {'outcome': 'disrupted'}) - _write(jm.verdict_path(100, 1), 'oom') + _write(records.outcome_path(100, 1), {'outcome': 'disrupted'}) + _write(records.verdict_path(100, 1), 'oom') totals = _totals(current_attempts={('100', 2)}) @@ -33,7 +36,7 @@ def test_verdict_is_preferred_over_outcome(logdir): def test_legacy_outcome_is_used_when_no_verdict_exists(logdir): - _write(jm.outcome_path(100, 1), {'outcome': 'disrupted'}) + _write(records.outcome_path(100, 1), {'outcome': 'disrupted'}) totals = _totals(current_attempts={('100', 2)}) @@ -43,8 +46,8 @@ def test_legacy_outcome_is_used_when_no_verdict_exists(logdir): def test_matching_verdict_and_outcome_are_counted_once(logdir): - _write(jm.outcome_path(100, 1), {'outcome': 'disrupted'}) - _write(jm.verdict_path(100, 1), 'disrupted') + _write(records.outcome_path(100, 1), {'outcome': 'disrupted'}) + _write(records.verdict_path(100, 1), 'disrupted') totals = _totals(current_attempts={('100', 2)}) @@ -55,7 +58,7 @@ def test_matching_verdict_and_outcome_are_counted_once(logdir): def test_repeated_disruptions_of_one_range_count_as_one_retried_range(logdir): for attempt in (1, 2, 3): - _write(jm.verdict_path(100, attempt), 'disrupted') + _write(records.verdict_path(100, attempt), 'disrupted') totals = _totals(current_attempts={('100', 4)}) @@ -67,8 +70,8 @@ def test_repeated_disruptions_of_one_range_count_as_one_retried_range(logdir): def test_disruptions_of_distinct_ranges_each_count_once(logdir): for end in (100, 200): - _write(jm.outcome_path(end, 1), {'outcome': 'disrupted'}) - _write(jm.verdict_path(end, 1), 'disrupted') + _write(records.outcome_path(end, 1), {'outcome': 'disrupted'}) + _write(records.verdict_path(end, 1), 'disrupted') totals = _totals(current_attempts={('100', 2), ('200', 2)}) @@ -77,7 +80,7 @@ def test_disruptions_of_distinct_ranges_each_count_once(logdir): def test_active_successor_counts_before_range_progress_exists(logdir): - _write(jm.verdict_path(100, 1), 'rejected') + _write(records.verdict_path(100, 1), 'rejected') totals = _totals(current_attempts={('100', 1), ('100', 2)}) @@ -100,7 +103,7 @@ def test_reconcile_counts_the_successor_on_its_dispatch_pass(cluster): def test_terminal_verdict_without_successor_is_not_a_retry(logdir): - _write(jm.verdict_path(100, 1), 'oom') + _write(records.verdict_path(100, 1), 'oom') totals = _totals( {'failed': {'100': {'attempts': 1, 'outcome': 'oom'}}}, @@ -112,7 +115,7 @@ def test_terminal_verdict_without_successor_is_not_a_retry(logdir): def test_terminal_disruption_without_successor_is_only_a_raw_attempt(logdir): - _write(jm.verdict_path(100, 1), 'disrupted') + _write(records.verdict_path(100, 1), 'disrupted') totals = _totals( {'failed': {'100': {'attempts': 1, 'outcome': 'disrupted'}}}, @@ -124,42 +127,42 @@ def test_terminal_disruption_without_successor_is_only_a_raw_attempt(logdir): def test_counter_sync_is_idempotent_and_replays_after_restart(logdir): - _write(jm.verdict_path(100, 1), 'disrupted') + _write(records.verdict_path(100, 1), 'disrupted') attempts = {('100', 2)} - retry_before = jm.metric_retries._value.get() - eviction_before = jm.metric_evictions._value.get() - unique_before = jm.metric_spot_disruption_retried._value.get() - reason_metric = jm.metric_retry_reasons.labels(reason='disrupted') + retry_before = metrics.retries._value.get() + eviction_before = metrics.evictions._value.get() + unique_before = metrics.spot_disruption_retried._value.get() + reason_metric = metrics.retry_reasons.labels(reason='disrupted') reason_before = reason_metric._value.get() counted = {} jm.sync_counters({}, counted, attempts) - first = (jm.metric_retries._value.get(), - jm.metric_evictions._value.get(), - jm.metric_spot_disruption_retried._value.get(), + first = (metrics.retries._value.get(), + metrics.evictions._value.get(), + metrics.spot_disruption_retried._value.get(), reason_metric._value.get()) jm.sync_counters({}, counted, attempts) - assert (jm.metric_retries._value.get(), - jm.metric_evictions._value.get(), - jm.metric_spot_disruption_retried._value.get(), + assert (metrics.retries._value.get(), + metrics.evictions._value.get(), + metrics.spot_disruption_retried._value.get(), reason_metric._value.get()) == first jm.sync_counters({}, {}, attempts) - assert jm.metric_retries._value.get() == retry_before + 2 - assert jm.metric_evictions._value.get() == eviction_before + 2 - assert jm.metric_spot_disruption_retried._value.get() == unique_before + 2 + assert metrics.retries._value.get() == retry_before + 2 + assert metrics.evictions._value.get() == eviction_before + 2 + assert metrics.spot_disruption_retried._value.get() == unique_before + 2 assert reason_metric._value.get() == reason_before + 2 def test_multiple_attempts_and_every_retry_reason(logdir): - for attempt, reason in enumerate(jm.ATTEMPT_OUTCOMES, 1): - _write(jm.verdict_path(100, attempt), reason) + for attempt, reason in enumerate(config.ATTEMPT_OUTCOMES, 1): + _write(records.verdict_path(100, attempt), reason) totals = _totals({'completed': {'100': { - 'attempts': len(jm.ATTEMPT_OUTCOMES) + 1}}}) + 'attempts': len(config.ATTEMPT_OUTCOMES) + 1}}}) - assert totals['retries'] == len(jm.ATTEMPT_OUTCOMES) - assert totals['reasons'] == {reason: 1 for reason in jm.ATTEMPT_OUTCOMES} + assert totals['retries'] == len(config.ATTEMPT_OUTCOMES) + assert totals['reasons'] == {reason: 1 for reason in config.ATTEMPT_OUTCOMES} assert totals['evicted'] == 1 assert totals['spot_disruption_retried'] == 1 assert totals['oom'] == 1 @@ -167,9 +170,9 @@ def test_multiple_attempts_and_every_retry_reason(logdir): def test_malformed_and_missing_records_do_not_invent_reasons(logdir): - _write(jm.outcome_path(100, 1), {'outcome': 'disrupted'}) - _write(jm.verdict_path(100, 1), 'not-a-verdict') - _write(jm.outcome_path(200, 1), 'not-json') + _write(records.outcome_path(100, 1), {'outcome': 'disrupted'}) + _write(records.verdict_path(100, 1), 'not-a-verdict') + _write(records.outcome_path(200, 1), 'not-json') _write(logdir / 'range-300-a1.verdict.tmp', 'oom') _write(logdir / 'unrelated', 'disrupted') @@ -185,8 +188,8 @@ def test_malformed_and_missing_records_do_not_invent_reasons(logdir): def test_existing_and_reason_labelled_metrics_are_exported(): - for reason in jm.ATTEMPT_OUTCOMES: - jm.metric_retry_reasons.labels(reason=reason) + for reason in config.ATTEMPT_OUTCOMES: + metrics.retry_reasons.labels(reason=reason) text = generate_latest().decode() assert '# HELP ssc_parallel_catchup_job_retried_count_total ' \ @@ -201,5 +204,5 @@ def test_existing_and_reason_labelled_metrics_are_exported(): 'Retry attempts dispatched after an ephemeral-storage verdict, with an escalated limit' in text assert '# HELP ssc_parallel_catchup_job_retried_reason_count_total ' \ 'Retry attempts dispatched, by the effective verdict of the predecessor attempt' in text - for reason in jm.ATTEMPT_OUTCOMES: + for reason in config.ATTEMPT_OUTCOMES: assert f'ssc_parallel_catchup_job_retried_reason_count_total{{reason="{reason}"}}' in text diff --git a/src/MissionParallelCatchup/tests/unit/test_sizing.py b/src/MissionParallelCatchup/tests/unit/test_sizing.py index 5907433c..aa9b60a9 100644 --- a/src/MissionParallelCatchup/tests/unit/test_sizing.py +++ b/src/MissionParallelCatchup/tests/unit/test_sizing.py @@ -7,25 +7,28 @@ import pytest +import config +import units +import sizing import job_monitor as jm @pytest.fixture def mem(monkeypatch): def configure(lim='1000Mi', bump=None, cap='48Gi'): - monkeypatch.setattr(jm, 'REQ_MEM', lim) - monkeypatch.setattr(jm, 'MEM_BUMP_FACTOR', - jm.MEM_BUMP_FACTOR if bump is None else bump) - monkeypatch.setattr(jm, 'MEM_ESCALATION_CAP', cap) + monkeypatch.setattr(config, 'REQ_MEM', lim) + monkeypatch.setattr(config, 'MEM_BUMP_FACTOR', + config.MEM_BUMP_FACTOR if bump is None else bump) + monkeypatch.setattr(config, 'MEM_ESCALATION_CAP', cap) return configure @pytest.fixture def eph(monkeypatch): def configure(lim='40Gi', bump=1.5, cap='200Gi'): - monkeypatch.setattr(jm, 'LIM_EPHEMERAL', lim) - monkeypatch.setattr(jm, 'EPH_BUMP_FACTOR', bump) - monkeypatch.setattr(jm, 'EPH_ESCALATION_CAP', cap) + monkeypatch.setattr(config, 'LIM_EPHEMERAL', lim) + monkeypatch.setattr(config, 'EPH_BUMP_FACTOR', bump) + monkeypatch.setattr(config, 'EPH_ESCALATION_CAP', cap) return configure @@ -39,21 +42,14 @@ def configure(lim='40Gi', bump=1.5, cap='200Gi'): ('1500', 1500), # bare bytes ]) def test_kubernetes_quantities_are_read_in_the_right_base(quantity, want): - assert jm._quantity_bytes(quantity) == want + assert units.quantity_bytes(quantity) == want def test_a_size_is_always_rendered_back_in_mebibytes(): # One unit everywhere means a limit can be compared to a request without # re-parsing, and Mi is fine-grained enough for the packing this run does. - assert jm._bytes_to_quantity(3 * 1024**3) == '3072Mi' - assert jm._bytes_to_quantity(0) == '1Mi', "a zero-byte limit is unschedulable" - - -def test_sizing_applies_the_margin_and_never_exceeds_the_limit(): - # 1 GB * 1.1, well under the cap - assert jm._sized(1_000_000_000, 1.1, '10Gi') == '1049Mi' - # capped: a huge peak cannot produce a request above its own limit - assert jm._sized(50_000_000_000, 1.1, '8Gi') == '8192Mi' + assert units.bytes_to_quantity(3 * 1024**3) == '3072Mi' + assert units.bytes_to_quantity(0) == '1Mi', "a zero-byte limit is unschedulable" # --- the memory ladder ------------------------------------------------------- @@ -63,29 +59,29 @@ def test_the_memory_escalation_ladder_compounds(mem, attempt, want): # 1.5x per OOM off what the attempt actually ran with. A factor of 1.0 # would retry an OOM at the identical limit, forever. mem(lim='1000Mi', bump=1.5) - assert jm.mem_for_attempt(attempt, '1000Mi') == f"{int(1000 * want)}Mi" + assert sizing.mem_for_attempt(attempt, '1000Mi') == f"{int(1000 * want)}Mi" def test_the_escalation_ladder_is_capped(mem): mem(lim='1000Mi', bump=1.5, cap='4Gi') - assert jm.mem_for_attempt(20, '1000Mi') == '4096Mi', "cap not applied" + assert sizing.mem_for_attempt(20, '1000Mi') == '4096Mi', "cap not applied" def test_oom_escalation_starts_from_what_the_attempt_actually_had(mem): # Escalating a 209Mi profiled range off the configured 24000Mi limit jumps # to 36000Mi -- a 172x overshoot that discards the packing win on first OOM. mem(lim='24000Mi', bump=1.5) - assert jm.mem_for_attempt(2, '702Mi') == '1053Mi' - assert jm.mem_for_attempt(2) == '36000Mi' # unprofiled keeps old behaviour + assert sizing.mem_for_attempt(2, '702Mi') == '1053Mi' + assert sizing.mem_for_attempt(2) == '36000Mi' # unprofiled keeps old behaviour # --- the disk ladder --------------------------------------------------------- def test_ephemeral_storage_escalates_and_caps_the_same_way(eph): eph(lim='40Gi', bump=1.5, cap='200Gi') - assert jm.eph_for_attempt(1) == '40960Mi' - assert jm.eph_for_attempt(2) == '61440Mi' - assert jm.eph_for_attempt(20) == '204800Mi', "cap not applied" + assert sizing.eph_for_attempt(1) == '40960Mi' + assert sizing.eph_for_attempt(2) == '61440Mi' + assert sizing.eph_for_attempt(20) == '204800Mi', "cap not applied" # --- the budgets the ladders are climbing ------------------------------------ @@ -94,12 +90,14 @@ def test_attempt_budgets_are_ordered_by_whose_fault_the_failure_was(): # A genuinely broken range gets the middle budget. Anything the cluster did # to us gets the most -- on spot, evictions are routine and must not condemn # a range. A hang has no budget at all: a timeout is terminal. - assert jm.MAX_ATTEMPTS_PER_RANGE < jm.MAX_DISRUPTION_ATTEMPTS, ( - f"budgets out of order: range={jm.MAX_ATTEMPTS_PER_RANGE} " - f"disruption={jm.MAX_DISRUPTION_ATTEMPTS}") - assert jm.MAX_ATTEMPTS_PER_RANGE > 1, "a range that OOMs once could never escalate" - assert jm.MAX_EPHEMERAL_ATTEMPTS > 1, "a range evicted on disk once could never grow" - assert jm.MAX_DISRUPTION_ATTEMPTS >= 10, \ + assert config.ATTEMPT_BUDGETS['oom'] < config.ATTEMPT_BUDGETS['disrupted'], ( + f"budgets out of order: range={config.ATTEMPT_BUDGETS['oom']} " + f"disruption={config.ATTEMPT_BUDGETS['disrupted']}") + assert config.ATTEMPT_BUDGETS['oom'] > 1, "a range that OOMs once could never escalate" + assert config.ATTEMPT_BUDGETS['ephemeral'] > 1, "a range evicted on disk once could never grow" + # Effectively unlimited on purpose: a healthy spot range can be evicted + # dozens of times, and only a misclassification should ever reach the gate. + assert config.ATTEMPT_BUDGETS['disrupted'] >= 100, \ "spot eviction would condemn ranges at this budget" @@ -108,7 +106,122 @@ def test_the_oom_budget_stops_short_of_the_cap_on_purpose(): # not mis-sized, and chasing it to MEM_ESCALATION_CAP parks a whole node on # it. The price is that such a range is condemned -- which today aborts the # run, so this coupling is what must not be forgotten. - n = jm.MAX_ATTEMPTS_PER_RANGE + n = config.ATTEMPT_BUDGETS['oom'] assert 2 <= n <= 8, f"{n} rungs: below 2 cannot escalate, above 8 chases a broken range" - assert jm.MEM_BUMP_FACTOR ** (n - 1) >= 3.0, \ + assert config.MEM_BUMP_FACTOR ** (n - 1) >= 3.0, \ "the ladder cannot even treble the request before giving up" + + +# --- the profile arithmetic -------------------------------------------------- +# +# Added 2026-08-06 after mutation testing: every line below was EXECUTED by the +# suite and none of it was asserted. A margin applied as a division, an +# escalation ladder running backwards, and an inverted runtime weighting all +# left the suite green. These pin the shapes, not just the outcomes. +# +# This path runs whenever POOL_PREFIX is unset -- the chart default -- so it +# sizes every worker on a run that does not pass --pubnet-parallel-catchup-pool- +# prefix. + +@pytest.fixture +def unpooled(monkeypatch): + """Profile-driven sizing with the tier ladder switched off.""" + def configure(entries, **overrides): + monkeypatch.setattr(config, 'POOL_PREFIX', '') + monkeypatch.setattr(config, 'PROFILE', sorted(entries.items())) + monkeypatch.setattr(config, '_SORTED_SECONDS', None) + for k, v in overrides.items(): + monkeypatch.setattr(config, k, v) + return configure + + +def test_each_escalation_rung_asks_for_more_than_the_one_below(mem, eph, monkeypatch): + """A divide where the bump multiplies makes an OOM retry ask for LESS. + + The ladder exists so an OOMing range gets a bigger node; running it + backwards retries the same range at a size it has already proved too small, + burning its whole budget without ever changing the outcome. + """ + monkeypatch.setattr(config, 'POOL_PREFIX', '') + mem(lim='1000Mi', bump=1.5, cap='48Gi') + eph(lim='40Gi', bump=1.5, cap='200Gi') + + for name, ladder in (('memory', sizing.mem_for_attempt), + ('ephemeral', sizing.eph_for_attempt)): + sizes = [units.quantity_bytes(ladder(a)) for a in range(1, 6)] + assert all(b > a for a, b in zip(sizes, sizes[1:])), \ + f"the {name} ladder does not climb: {sizes}" + + +def test_the_margin_multiplies_the_measured_peak(unpooled): + """Exact bytes, because `peak * 1.15` and `peak / 1.15` both "work". + + Dividing by the margin requests 76% of what the range was measured using -- + an OOM on the attempt the profile was supposed to make safe. + """ + rss = 2 * 1024 ** 3 + unpooled({300: {'peakAnonBytes': rss, 'seconds': 300.0}}) + + want = (int(rss * config.PROFILE_MARGIN) + + units.quantity_bytes(config.PROFILE_CACHE_HEADROOM) + + units.quantity_bytes(config.PROFILE_RUNTIME_MEMORY_INSURANCE)) + assert sizing._profile_overrides(300, escalated=False)['memory'] == \ + units.bytes_to_quantity(want) + + +def test_the_disk_margin_multiplies_too(unpooled): + disk = 20 * 1024 ** 3 + unpooled({300: {'peakEphemeralBytes': disk, 'seconds': 300.0}}, + LIM_EPHEMERAL='40Gi') + + want = (int(disk * config.PROFILE_MARGIN) + + units.quantity_bytes(config.PROFILE_EPHEMERAL_HEADROOM) + + units.quantity_bytes(config.PROFILE_RUNTIME_EPHEMERAL_INSURANCE)) + assert sizing._profile_overrides(300, escalated=False)['ephemeral-storage'] == \ + units.bytes_to_quantity(want) + + +def test_the_insurance_is_weighted_by_runtime_not_against_it(unpooled): + """The longest range gets the whole allowance; half as long gets half. + + An inverted ratio hands the most disk to the ranges least at risk of + running out of it, and the profile's own longest range the least. + """ + rss = 2 * 1024 ** 3 + unpooled({300: {'peakAnonBytes': rss, 'seconds': 300.0}, + 900: {'peakAnonBytes': rss, 'seconds': 600.0}}) + base = (int(rss * config.PROFILE_MARGIN) + + units.quantity_bytes(config.PROFILE_CACHE_HEADROOM)) + full = units.quantity_bytes(config.PROFILE_RUNTIME_MEMORY_INSURANCE) + + assert sizing._profile_overrides(900, escalated=False)['memory'] == \ + units.bytes_to_quantity(base + full), "the longest range gets all of it" + assert sizing._profile_overrides(300, escalated=False)['memory'] == \ + units.bytes_to_quantity(base + full // 2), "half the runtime, half the share" + + +def test_a_range_with_no_measured_runtime_gets_no_insurance(unpooled): + """Insurance is priced off time-at-risk, so an unknown runtime buys none. + + Inverting the bail spends the whole allowance on exactly the ranges nothing + is known about. + """ + rss = 2 * 1024 ** 3 + unpooled({300: {'peakAnonBytes': rss}}) + + want = (int(rss * config.PROFILE_MARGIN) + + units.quantity_bytes(config.PROFILE_CACHE_HEADROOM)) + assert sizing._profile_overrides(300, escalated=False)['memory'] == \ + units.bytes_to_quantity(want) + + +def test_a_zero_allowance_turns_the_insurance_off(unpooled): + """The documented way to disable it, so it has to reach zero exactly.""" + rss = 2 * 1024 ** 3 + unpooled({300: {'peakAnonBytes': rss, 'seconds': 300.0}}, + PROFILE_RUNTIME_MEMORY_INSURANCE='0') + + want = (int(rss * config.PROFILE_MARGIN) + + units.quantity_bytes(config.PROFILE_CACHE_HEADROOM)) + assert sizing._profile_overrides(300, escalated=False)['memory'] == \ + units.bytes_to_quantity(want) diff --git a/src/MissionParallelCatchup/tests/unit/test_tx_apply.py b/src/MissionParallelCatchup/tests/unit/test_tx_apply.py index 03474790..b0fdaaf8 100644 --- a/src/MissionParallelCatchup/tests/unit/test_tx_apply.py +++ b/src/MissionParallelCatchup/tests/unit/test_tx_apply.py @@ -12,6 +12,10 @@ import pytest +import kube +import records +import medida +import attempts import job_monitor as jm import log_collector as lc @@ -91,25 +95,66 @@ def test_scanner_ignores_sum_from_another_metric(): assert s.seconds is None -def test_scanner_gives_up_past_its_window(): +def test_scanner_gives_up_past_its_window_of_STATISTICS(): + # The window bounds how many medida statistics may sit between the header + # and the sum, so a release that adds percentiles is caught here rather than + # silently dropping tx_apply. s = lc.TxApplyScanner() s.feed("metric 'ledger.transaction.apply':") - for _ in range(20): - s.feed("[default INFO] unrelated chatter") + for i in range(lc.TxApplyScanner.WINDOW + 5): + s.feed(f" {i}% = 1.5ms") s.feed(" sum = 12.5555ms") assert s.seconds is None +def test_interleaved_output_does_not_spend_the_window(): + # Measured on ssc-test 2026-08-04: a /info liveness response landed inside + # the block and pushed `sum` 91 lines below the header. Charging those lines + # made the scanner give up 76 lines short while the value sat in the archive + # -- one leg in 233, and job_monitor's re-read used the same span so its + # recovery path missed it too. + s = lc.TxApplyScanner() + s.feed("metric 'ledger.transaction.apply':") + s.feed(" count = 7641690") + for line in ('{', ' "info" : {', ' "build" : "stellar-core 27.1.1",', + ' "ledger" : {', ' "age" : 109870542,', + ' "baseFee" : 100,', ' "bucketlist" : [', + ' {', ' "curr" : "2a2cfe82",', + ' "snap" : "c12100ab"', ' },') * 8: + s.feed(line) + s.feed(" sum = 1.9501e+06ms") + assert s.seconds == pytest.approx(1950.1) + + +def test_a_block_whose_sum_never_arrives_cannot_claim_a_later_one(): + # Without a hard bound, skipping non-statistic lines would leave the scanner + # armed forever and let it read some other timer's sum as tx_apply. + s = lc.TxApplyScanner() + s.feed("metric 'ledger.transaction.apply':") + for _ in range(lc.TxApplyScanner.HARD_WINDOW + 10): + s.feed(' "noise" : 1,') + s.feed(" sum = 12.5555ms") + assert s.seconds is None + + +def test_a_different_metric_block_ends_the_search(): + s = lc.TxApplyScanner() + s.feed("metric 'ledger.transaction.apply':") + s.feed("metric 'ledger.close':") + s.feed(" sum = 12.5555ms") + assert s.seconds is None, "that sum belongs to ledger.close" + + def test_rate_and_mean_lines_are_not_read_as_sum(): for line in MEDIDA_BLOCK.splitlines(): if 'rate =' in line or 'mean =' in line: - assert lc._SUM_RE.search(line) is None + assert medida.SUM_RE.search(line) is None def test_sum_stays_inside_the_scan_window(): lines = MEDIDA_BLOCK.splitlines() header = next(i for i, l in enumerate(lines) if 'ledger.transaction.apply' in l) - offset = next(i for i, l in enumerate(lines) if lc._SUM_RE.search(l)) - header + offset = next(i for i, l in enumerate(lines) if medida.SUM_RE.search(l)) - header assert offset == 10, f"medida layout moved: sum is now {offset} lines below the header" assert offset <= lc.TxApplyScanner.WINDOW @@ -127,51 +172,43 @@ def test_resumed_is_read_from_the_workers_own_line(): def test_resumed_is_bookkeeping_and_never_becomes_a_measurement(): # peaks_for_range needs it to tell a resumed tail from a complete pass; the # profile must not see it as an axis. - assert 'resumed' not in jm.PEAK_FIELDS + assert 'resumed' not in attempts.PEAK_FIELDS # --- the monitor's own reader ------------------------------------------------- -def test_the_monitor_reads_the_same_block_the_collector_scanned(logdir): - # Two independent parsers over one format: they must agree, or a range - # measured live and a range recovered from the archive report differently. - with gzip.open(jm.log_path(4000, 1), 'wt') as fh: - fh.write(MEDIDA_BLOCK) - assert jm._tx_apply_for_attempt(4000, 1) == pytest.approx(scan(MEDIDA_BLOCK).seconds) - - -def test_tx_apply_prefers_durable_sources_over_the_pod_api(logdir, monkeypatch): - # .metrics survives pod reaping and saveSuccessLogs=false; the archive - # survives reaping alone; the pod log is racing Karpenter, so it is a - # fallback and never the plan. Each source carries a different value here - # so the winner is unambiguous. - class FakePodLog: - def read_namespaced_pod_log(self, name, namespace, **_): - return MEDIDA_BLOCK - monkeypatch.setattr(jm, 'core_v1', FakePodLog()) - - with open(jm.metrics_path(4000, 1), 'w') as fh: - json.dump({'txApplySeconds': 99.0}, fh) - with gzip.open(jm.log_path(4000, 1), 'wt') as fh: - fh.write(MEDIDA_BIG) - assert jm._tx_apply_for_attempt(4000, 1, pod_name='p') == 99.0 - os.remove(jm.metrics_path(4000, 1)) - assert jm._tx_apply_for_attempt(4000, 1, pod_name='p') == pytest.approx(BIG_SECONDS) - os.remove(jm.log_path(4000, 1)) - assert jm._tx_apply_for_attempt(4000, 1, pod_name='p') == pytest.approx(TX_APPLY_SECONDS) +def test_tx_apply_comes_from_the_collectors_record_and_nowhere_else(logdir, monkeypatch): + """The monitor no longer parses stellar-core output for this at all. + + An archive sitting beside the record is not a second source: the collector + re-reads it with its own scanner at finalization, so a reader here would + repeat that work over the same bytes and could not disagree. + """ + class ExplodingPodLog: + def read_namespaced_pod_log(self, *_a, **_kw): + raise AssertionError("the monitor must not read pod logs for txApply") + monkeypatch.setattr(kube, 'core_v1', ExplodingPodLog()) + + _metrics(4000, 1, {'txApplySeconds': 99.0}) + assert attempts._tx_apply_for_attempt(4000, 1) == 99.0 + + os.remove(records.metrics_path(4000, 1)) + with gzip.open(records.log_path(4000, 1), 'wt') as fh: + fh.write(MEDIDA_BIG) + assert attempts._tx_apply_for_attempt(4000, 1) is None, \ + "an archive is the collector's to parse, not the monitor's" def test_tx_apply_survives_a_reaped_pod(logdir): - # The pod is the only source that can vanish, so nothing may depend on it. - with open(jm.metrics_path(4000, 1), 'w') as fh: - json.dump({'txApplySeconds': 12.5}, fh) - assert jm.tx_apply_for_range(4000, 1, pod_name=None) == 12.5 + # The record outlives the pod, and is now the only thing the monitor reads. + _metrics(4000, 1, {'txApplySeconds': 12.5}) + assert attempts.tx_apply_for_range(4000, 1) == 12.5 def test_a_range_with_no_measurement_anywhere_reports_nothing(logdir): - assert jm._tx_apply_for_attempt(4000, 1) is None - assert jm.tx_apply_for_range(4000, 1) is None + assert attempts._tx_apply_for_attempt(4000, 1) is None + assert attempts.tx_apply_for_range(4000, 1) is None def test_a_corrupt_archive_costs_this_range_its_metric_never_the_pass(logdir): @@ -179,42 +216,38 @@ def test_a_corrupt_archive_costs_this_range_its_metric_never_the_pass(logdir): # escape the per-range work and abort the whole reconcile: no recording, no # reap, no dispatch for any of ~4000 ranges, for as long as the torn bytes # sat there. - with open(jm.log_path(4000, 1), 'wb') as fh: + with open(records.log_path(4000, 1), 'wb') as fh: fh.write(gzip.compress(MEDIDA_BLOCK.encode())[:40]) - assert jm._tx_apply_for_attempt(4000, 1) is None + assert attempts._tx_apply_for_attempt(4000, 1) is None # --- summing a resumed chain -------------------------------------------------- +def _metrics(end, attempt, values): + """Write one attempt's .metrics, the way the collector leaves it.""" + with open(records.metrics_path(end, attempt), 'w') as fh: + json.dump(values, fh) + + def test_tx_apply_sums_the_whole_resumed_chain(logdir): # medida's total is per-process, so a pod that resumes at LCL+1 reports only # the transactions it replayed -- the tail, not the range. - with open(jm.metrics_path(4000, 1), 'w') as fh: - json.dump({'txApplySeconds': 10.0}, fh) - with open(jm.metrics_path(4000, 2), 'w') as fh: - json.dump({'txApplySeconds': 5.0, 'resumed': True}, fh) - assert jm.tx_apply_for_range(4000, 2) == 15.0 + _metrics(4000, 1, {'txApplySeconds': 10.0}) + _metrics(4000, 2, {'txApplySeconds': 5.0, 'resumed': True}) + assert attempts.tx_apply_for_range(4000, 2) == 15.0 def test_a_fresh_start_drops_the_earlier_legs_from_the_total(logdir): # No RESUME line means new-db ran and this attempt redid the whole range; # adding the interrupted attempt's figure would double-count the same work. - with open(jm.metrics_path(4000, 1), 'w') as fh: - json.dump({'txApplySeconds': 10.0}, fh) - with open(jm.metrics_path(4000, 2), 'w') as fh: - json.dump({'txApplySeconds': 5.0}, fh) - assert jm.tx_apply_for_range(4000, 2) == 5.0 - - -def test_the_winner_pod_fallback_cannot_fill_a_missing_predecessor(logdir, monkeypatch): - # pod_name names the winning attempt's pod; handing it to an earlier leg - # would read the wrong pod's log and attribute it to the wrong attempt. - class FakePodLog: - def read_namespaced_pod_log(self, name, namespace, **_): - return MEDIDA_BIG - monkeypatch.setattr(jm, 'core_v1', FakePodLog()) - # a1 has no durable record at all; a2 resumed from it and has none either. - with open(jm.metrics_path(4000, 2), 'w') as fh: - json.dump({'resumed': True}, fh) - assert jm.tx_apply_for_range(4000, 2, pod_name='p') is None, \ - "winner-only txApply is a lower bound, not the resumed chain total" + _metrics(4000, 1, {'txApplySeconds': 10.0}) + _metrics(4000, 2, {'txApplySeconds': 5.0}) + assert attempts.tx_apply_for_range(4000, 2) == 5.0 + + +def test_a_missing_predecessor_leg_makes_the_chain_total_absent(logdir): + # a1 has no record at all; a2 resumed from it and has none either. The sum + # of what survived is a lower bound, not the range's total. + _metrics(4000, 2, {'resumed': True, 'txApplySeconds': 3.0}) + assert attempts.tx_apply_for_range(4000, 2) is None, \ + "a chain missing a leg must report nothing, not the legs it has" diff --git a/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py b/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py index 74e05bb2..cd38cb80 100644 --- a/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py +++ b/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py @@ -1,151 +1,190 @@ -"""Worker responsiveness metrics stay truthful without entering reconcile.""" +"""Worker responsiveness: one concurrent sweep per reconcile pass. +Driven against real sockets rather than a faked session -- the whole behaviour is +"what did stellar-core's admin port actually answer", so a fake that returns +whatever the test wants proves very little. +""" + +import asyncio +import contextlib import os +import socket import subprocess import sys import threading import time +from http.server import BaseHTTPRequestHandler, HTTPServer -import job_monitor as jm +import pytest from kubernetes import client +import config +import worker_liveness +import job_monitor as jm -def _task(sampler, identity): - record = sampler._records[identity] - return identity, record['generation'], record['target'] +def _server(status=200, delay=0.0, counter=None): + """A local HTTP endpoint standing in for stellar-core's admin port.""" + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + if counter is not None: + counter.enter() + if delay: + time.sleep(delay) + self.send_response(status) + self.end_headers() + self.wfile.write(b'{}') + if counter is not None: + counter.leave() + + def log_message(self, *_a): + pass -def _result(sampler, identity, success, now): - sampler._record_result(*_task(sampler, identity), success, - None if success else TimeoutError("busy"), now=now) + srv = HTTPServer(('127.0.0.1', 0), Handler) + srv.daemon_threads = True + threading.Thread(target=srv.serve_forever, daemon=True).start() + return srv -def test_hysteresis_unknown_down_and_immediate_recovery(): - sampler = jm.WorkerLivenessSampler( - interval=30, timeout=5, failure_threshold=3, max_concurrency=1) - sampler.replace_candidates({'uid-1': ('pod-1', '10.0.0.1')}, now=0) +class _Concurrency: + """Server-side count of overlapping requests, and its high-water mark.""" - assert sampler._records['uid-1']['status'] == 'unknown' - _result(sampler, 'uid-1', True, 1) - assert sampler._records['uid-1']['status'] == 'up' + def __init__(self): + self.lock = threading.Lock() + self.now = 0 + self.peak = 0 - _result(sampler, 'uid-1', False, 31) - assert sampler._records['uid-1']['status'] == 'unknown' - _result(sampler, 'uid-1', False, 61) - assert sampler._records['uid-1']['status'] == 'unknown' - _result(sampler, 'uid-1', False, 91) - assert sampler._records['uid-1']['status'] == 'down' + def enter(self): + with self.lock: + self.now += 1 + self.peak = max(self.peak, self.now) - _result(sampler, 'uid-1', True, 121) - assert sampler._records['uid-1']['status'] == 'up' - assert sampler._records['uid-1']['failures'] == 0 + def leave(self): + with self.lock: + self.now -= 1 -def test_disappearance_and_replacement_discard_stale_probe_results(): - sampler = jm.WorkerLivenessSampler(max_concurrency=1) - sampler.replace_candidates({'old-uid': ('pod-1', '10.0.0.1')}, now=0) - old_task = _task(sampler, 'old-uid') - _result(sampler, 'old-uid', True, 1) +def _targets(port, count=1): + return {f"uid-{i}": (f"pod-{i}", '127.0.0.1') for i in range(count)} - # A new UID is a new attempt/pod even if Kubernetes reuses the IP. - sampler.replace_candidates({'new-uid': ('pod-2', '10.0.0.1')}, now=2) - assert set(sampler._records) == {'new-uid'} - assert sampler._records['new-uid']['status'] == 'unknown' - # The old request may finish after the replacement snapshot. It cannot - # resurrect the vanished pod or update the replacement. - sampler._record_result(*old_task, True, now=3) - assert set(sampler._records) == {'new-uid'} - assert sampler._records['new-uid']['status'] == 'unknown' +@contextlib.contextmanager +def _serving(monkeypatch, **kw): + """Point the module's admin port at a local endpoint for the duration.""" + srv = _server(**kw) + monkeypatch.setattr(worker_liveness, '_ADMIN_PORT', srv.server_address[1]) + try: + yield srv + finally: + srv.shutdown() - sampler.replace_candidates({}, now=4) - assert sampler.counts() == {'up': 0, 'down': 0, 'unknown': 0} +def _closed_port(): + s = socket.socket() + s.bind(('127.0.0.1', 0)) + port = s.getsockname()[1] + s.close() + return port -def test_ip_change_on_same_identity_resets_to_unknown(): - sampler = jm.WorkerLivenessSampler(max_concurrency=1) - sampler.replace_candidates({'uid': ('pod', '10.0.0.1')}, now=0) - old_task = _task(sampler, 'uid') - _result(sampler, 'uid', True, 1) - sampler.replace_candidates({'uid': ('pod', '10.0.0.2')}, now=2) - assert sampler._records['uid']['status'] == 'unknown' - assert sampler._records['uid']['failures'] == 0 +def _sweep(targets, port, **kw): + """Run a sweep against a chosen port by pointing the module's URL at it.""" + kw.setdefault('timeout', 2) + kw.setdefault('deadline', 5) + kw.setdefault('concurrency', 8) + return asyncio.run(worker_liveness.sweep(targets, **kw)) - sampler._record_result(*old_task, False, TimeoutError(), now=3) - assert sampler._records['uid']['status'] == 'unknown' - assert sampler._records['uid']['failures'] == 0 +# --- what counts as up -------------------------------------------------------- -def test_sampler_failure_reports_every_current_candidate_unknown(): - release = threading.Event() +@pytest.mark.parametrize('status, verdict', [ + (200, 'up'), + # A busy core used to count as up. "Answered, badly" is not answering, and + # nothing downstream smooths it. + (503, 'down'), + # Not just 5xx: `status < 500` passes the 503 case while counting a wrong + # path or a proxy in the way as a healthy core. + (404, 'down'), +]) +def test_only_a_200_is_up(monkeypatch, status, verdict): + with _serving(monkeypatch, status=status): + counts = _sweep(_targets(0, 2), None) + assert counts[verdict] == 2 and sum(counts.values()) == 2 - def blocked_probe(_ip, _timeout): - release.wait(2) - sampler = jm.WorkerLivenessSampler( - interval=30, timeout=1, failure_threshold=3, - max_concurrency=1, probe=blocked_probe) - sampler.start() - try: - sampler.replace_candidates({ - 'a': ('pod-a', '10.0.0.1'), - 'b': ('pod-b', '10.0.0.2'), - }) - with sampler._condition: - sampler._failed = 'synthetic scheduler failure' - assert sampler.counts() == {'up': 0, 'down': 0, 'unknown': 2} - finally: - release.set() - sampler.close() +def test_a_refused_connection_is_down(monkeypatch): + monkeypatch.setattr(worker_liveness, '_ADMIN_PORT', _closed_port()) + assert _sweep(_targets(0, 2), None) == {'up': 0, 'down': 2, 'unknown': 0} -def test_probe_uses_stellar_core_info_and_any_http_response_is_up(monkeypatch): - called = [] +def test_a_probe_slower_than_its_timeout_is_down(monkeypatch): + with _serving(monkeypatch, status=200, delay=1.0): + assert _sweep(_targets(0, 1), None, timeout=0.2) == \ + {'up': 0, 'down': 1, 'unknown': 0} - class Response: - status_code = 503 - def __enter__(self): - return self +def test_no_targets_is_not_a_sweep(): + assert worker_liveness.publish({}) == {'up': 0, 'down': 0, 'unknown': 0} - def __exit__(self, *_args): - return False - class Session: - def mount(self, *_args): - pass +# --- the bounds the reconcile loop depends on -------------------------------- - def get(self, url, timeout): - called.append((url, timeout)) - return Response() +def test_the_sweep_stops_at_its_deadline_and_reports_the_rest_unknown(monkeypatch): + """The reconcile loop waits for this, so it must be bounded by wall clock. - def close(self): - pass + Ten pods that each hang for a second, two at a time, is five seconds of work. + With a half-second deadline the sweep keeps what finished and calls the rest + unknown rather than making dispatch wait. + """ + with _serving(monkeypatch, status=200, delay=1.0): + started = time.monotonic() + counts = _sweep(_targets(0, 10), None, concurrency=2, timeout=5, deadline=0.5) + elapsed = time.monotonic() - started + assert elapsed < 2.0, f"the sweep ran {elapsed:.2f}s past a 0.5s deadline" + assert counts['unknown'] >= 6, counts + assert sum(counts.values()) == 10, "every target must be accounted for" - monkeypatch.setattr(jm.requests, 'Session', Session) - sampler = jm.WorkerLivenessSampler( - interval=30, timeout=5, failure_threshold=3, max_concurrency=1) - sampler.start() - try: - sampler.replace_candidates({'uid': ('pod', '10.2.3.4')}) - deadline = time.monotonic() + 1 - while time.monotonic() < deadline and sampler._records['uid']['status'] != 'up': - time.sleep(0.01) - assert called == [('http://10.2.3.4:11626/info', 5.0)] - assert sampler._records['uid']['status'] == 'up', ( - "an HTTP 503 is a busy but responsive admin endpoint") - finally: - sampler.close() + +def test_concurrency_is_bounded_at_the_server(monkeypatch): + counter = _Concurrency() + with _serving(monkeypatch, status=200, delay=0.05, counter=counter): + counts = _sweep(_targets(0, 60), None, concurrency=4, timeout=5, deadline=10) + assert counts == {'up': 60, 'down': 0, 'unknown': 0} + assert counter.peak <= 4, f"{counter.peak} overlapping requests, limit 4" +def test_one_unreachable_pod_does_not_discard_the_others(monkeypatch): + """Why this is asyncio.wait and not a TaskGroup. + + A TaskGroup cancels its siblings when a task raises. Every other answer has + to survive one pod being unreachable, so four pods point at a live endpoint + and one at a loopback address with nothing bound. + """ + with _serving(monkeypatch, status=200): + targets = {f"uid-{i}": (f"pod-{i}", '127.0.0.1') for i in range(4)} + targets['uid-dead'] = ('pod-dead', '127.0.0.2') + counts = _sweep(targets, None, timeout=1) + assert counts == {'up': 4, 'down': 1, 'unknown': 0} + + +def test_publish_reports_every_target_unknown_when_the_sweep_itself_fails(monkeypatch): + """The production call path: job_monitor calls publish(targets) and nothing else.""" + async def boom(*_a, **_kw): + raise RuntimeError("no event loop for you") + monkeypatch.setattr(worker_liveness, 'sweep', boom) + assert worker_liveness.publish(_targets(0, 7)) == \ + {'up': 0, 'down': 0, 'unknown': 7} + + +# --- candidate selection ------------------------------------------------------ + def test_only_running_pods_with_ips_are_candidates_and_uid_is_identity(): def pod(name, uid, phase, ip): return client.V1Pod( metadata=client.V1ObjectMeta(name=name, uid=uid), status=client.V1PodStatus(phase=phase, pod_ip=ip)) - targets = jm._worker_targets([ + targets = worker_liveness.targets([ pod('ready', 'uid-ready', 'Running', '10.0.0.1'), pod('pending', 'uid-pending', 'Pending', '10.0.0.2'), pod('no-ip', 'uid-no-ip', 'Running', None), @@ -157,7 +196,9 @@ def test_malformed_liveness_configuration_fails_with_an_explicit_message(): env = { 'PATH': os.environ.get('PATH', ''), 'HOME': os.environ.get('HOME', ''), - 'PYTHONPATH': os.path.dirname(jm.__file__), + # apps/ and lib/ both, the way the container's flat /app has them. + 'PYTHONPATH': os.pathsep.join((os.path.dirname(jm.__file__), + os.path.dirname(config.__file__))), 'LIVENESS_MAX_CONCURRENCY': 'many', } result = subprocess.run( @@ -165,67 +206,25 @@ def test_malformed_liveness_configuration_fails_with_an_explicit_message(): text=True, capture_output=True, env=env, cwd=os.path.dirname(jm.__file__)) assert result.returncode != 0 - assert 'LIVENESS_MAX_CONCURRENCY must be integers' in result.stderr - - -def test_2096_slow_workers_have_bounded_work_and_do_not_delay_reconcile( - cluster): - release = threading.Event() - active_lock = threading.Lock() - active = 0 - peak_active = 0 - - def blocked_probe(_ip, _timeout): - nonlocal active, peak_active - with active_lock: - active += 1 - peak_active = max(peak_active, active) - try: - release.wait(3) - finally: - with active_lock: - active -= 1 - - concurrency = 8 - sampler = jm.WorkerLivenessSampler( - interval=0.2, timeout=1, failure_threshold=3, - max_concurrency=concurrency, probe=blocked_probe) - targets = { - f"uid-{i}": (f"pod-{i}", f"10.{i // 65536}.{(i // 256) % 256}.{i % 256}") - for i in range(2096) - } - sampler.start() - sampler.replace_candidates(targets) - try: - deadline = time.monotonic() + 2 - while time.monotonic() < deadline and sampler.stats()['active'] < concurrency: - time.sleep(0.01) - - stats = sampler.stats() - assert stats['records'] == 2096 - assert stats['active'] <= concurrency - assert stats['queued'] <= concurrency - assert stats['outstanding'] <= 2 * concurrency - assert stats['threads'] == concurrency + 1 - assert peak_active <= concurrency - - # Exercise the exact handoff used by update_status_and_metrics while all - # request slots are blocked. It copies the candidate snapshot and reads - # counts, but never waits for a request. - started = time.monotonic() - counts = jm.publish_worker_liveness(targets, sampler=sampler) - publish_elapsed = time.monotonic() - started - assert publish_elapsed < 0.5, ( - f"blocked probes delayed liveness publication by {publish_elapsed:.3f}s") - assert counts == {'up': 0, 'down': 0, 'unknown': 2096} - assert sum(counts.values()) == len(targets) - - # Dispatch itself remains equally independent. + assert 'LIVENESS_MAX_CONCURRENCY must be an integer' in result.stderr + + +def test_a_blocked_sweep_does_not_delay_dispatch(cluster, monkeypatch): + """Dispatch must not wait on the fleet answering. + + The sweep is bounded by its deadline, and reconcile pays that at most once + per pass -- so this pins the cost rather than the independence the old + background sampler gave. + """ + monkeypatch.setattr(config, 'LIVENESS_SWEEP_SECONDS', 0.3) + with _serving(monkeypatch, status=200, delay=5.0): started = time.monotonic() - result = cluster.reconcile() + counts = worker_liveness.publish(_targets(0, 50)) elapsed = time.monotonic() - started - assert result['created'] == 2 - assert elapsed < 0.5, f"blocked liveness probes delayed reconcile by {elapsed:.3f}s" - finally: - release.set() - sampler.close() + assert elapsed < 1.5, f"publish took {elapsed:.2f}s against a 0.3s deadline" + assert counts['unknown'] > 0 + assert sum(counts.values()) == 50 + + started = time.monotonic() + cluster.reconcile() + assert time.monotonic() - started < 1.0 From 97fc01a4c9c05934e9bdb2c76832c6448a13df4b Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Fri, 7 Aug 2026 16:43:00 -0400 Subject: [PATCH 067/117] catchup: size spot nebula for 1800m/9Gi on half-sized nodes nebula is the no-profile pool, and its 3800m/14336Mi claim was cut for 8-vCPU nodes. On the r8a.xlarge the tier design names for it, 3800m exceeds the 3705m a 4-vCPU node can offer once the EKS reserve and daemonsets come out, so that shape won no pods at all rather than packing fewer. 1800m/9Gi packs 2 on r8a.xlarge, 3 on m8a.2xlarge and 4 on r8a.2xlarge, and POOL_VCPU moves 8 -> 4 to name the smallest shape, which is what the free-rung guard prices promotions from. Chart and code defaults move together; the contract test catches them drifting. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/lib/config.py | 6 ++-- .../parallel_catchup_helm/values.yaml | 6 ++-- .../tests/unit/test_pool_tiers.py | 32 +++++++++++++++++++ 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/MissionParallelCatchup/lib/config.py b/src/MissionParallelCatchup/lib/config.py index 82297c9e..b729bd6b 100644 --- a/src/MissionParallelCatchup/lib/config.py +++ b/src/MissionParallelCatchup/lib/config.py @@ -469,7 +469,7 @@ # the same RAM. POOL_CPU = os.getenv( 'POOL_CPU', - 'subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:3.80') + 'subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80') # vCPU of the SMALLEST node in each tier's pool. This is what decides whether a # promotion is free, and it cannot be inferred from POOL_CPU, which is a claim @@ -485,7 +485,7 @@ # rungs cross a class honestly -- which is what POOL_CROSS_RUNGS now carries. POOL_VCPU = os.getenv( 'POOL_VCPU', - 'subdwarf:2,dwarf:2,subgiant:4,giant:4,supergiant:4,hypergiant:8,supernova:16,protostar:8,nebula:8') + 'subdwarf:2,dwarf:2,subgiant:4,giant:4,supergiant:4,hypergiant:8,supernova:16,protostar:8,nebula:4') # Rungs allowed to cross a vCPU class anyway, "from->to", comma separated. The # guard exists because a speculative promotion that doubles cores is usually a @@ -560,7 +560,7 @@ # this cluster, so subdwarf shares dwarf's node type and is emptied by its cut. POOL_MEM = os.getenv( 'POOL_MEM', - 'subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:14336Mi') + 'subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:9216Mi') _SORTED_SECONDS = None diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index af467d33..59bd9444 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -324,9 +324,9 @@ monitor: # NOT predict throughput -- same box, memory.max 28GiB ran 1.83 lps vs 56GiB at # 1.70 -- so it reaches the right nodes by the wrong signal. poolBlockRungs: "hypergiant->supernova" - poolVcpu: "subdwarf:2,dwarf:2,subgiant:4,giant:4,supergiant:4,hypergiant:8,supernova:16,protostar:8,nebula:8" - poolCpu: "subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:3.80" - poolMem: "subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:14336Mi" + poolVcpu: "subdwarf:2,dwarf:2,subgiant:4,giant:4,supergiant:4,hypergiant:8,supernova:16,protostar:8,nebula:4" + poolCpu: "subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80" + poolMem: "subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:9216Mi" # Profiled run, range past the profile's top: newest ledgers, and the newest # are the densest, so a rich pool rather than an average one. poolUnprofiled: "protostar" diff --git a/src/MissionParallelCatchup/tests/unit/test_pool_tiers.py b/src/MissionParallelCatchup/tests/unit/test_pool_tiers.py index 7f262a78..af0ad861 100644 --- a/src/MissionParallelCatchup/tests/unit/test_pool_tiers.py +++ b/src/MissionParallelCatchup/tests/unit/test_pool_tiers.py @@ -570,3 +570,35 @@ def usable(vcpu): assert cpu * 1000 <= usable(vcpu), ( f"{tier} claims {cpu * 1000:.0f}m but its smallest node " f"({vcpu} vCPU) only offers {usable(vcpu)}m") + + +# --- nebula: the no-profile pool --------------------------------------------- + +def test_nebula_packs_two_to_four_on_the_shapes_its_pools_offer(): + """Ranges with no measurement at all, so density is set deliberately. + + Read off the unpatched defaults, not the fixture: these are the numbers the + chart ships and the nodepools are cut from. A claim above what the smallest + shape can offer wins no nodes at all rather than packing fewer -- r8a.xlarge + offers 3705m once the EKS reserve and 215m of daemonsets come out, so the + 3800m this used to claim fit zero pods on it. + """ + cpu = float(dict(x.split(':') for x in config.POOL_CPU.split(','))['nebula']) * 1000 + mem = int(dict(x.split(':') for x in config.POOL_MEM.split(','))['nebula'].rstrip('Mi')) + vcpu = int(dict(x.split(':') for x in config.POOL_VCPU.split(','))['nebula']) + + def usable_cpu(cores): + reserved = 60 + (10 if cores >= 2 else 0) + (5 if cores >= 3 else 0) \ + + (5 if cores >= 4 else 0) + max(0, cores - 4) * 2.5 + return cores * 1000 - reserved - 215 + + # measured allocatable, same source as ALLOC above + for shape, cores, alloc_mem, want in (('r8a.xlarge', 4, 30259, 2), + ('m8a.2xlarge', 8, 30259, 3), + ('r8a.2xlarge', 8, 61604, 4)): + fits = min(int(usable_cpu(cores) // cpu), int((alloc_mem - 215) // mem)) + assert fits == want, f"{shape}: {fits} pods per node, expected {want}" + + # POOL_VCPU must name the SMALLEST shape, or the free-rung guard misprices + # a promotion into or out of nebula. + assert vcpu == 4 From 8be7d87814e00f158d260af655a78eb729645754 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Sat, 8 Aug 2026 16:43:25 -0400 Subject: [PATCH 068/117] Move the driver onto the monitor's HTTP surface The driver read a status ConfigMap and mounted the profile as one. Status is now served from memory at /status and the profile is POSTed to /start, so the run's input and output stop going through cluster objects. - /start takes the profile and opens the reconcile gate. Nothing dispatches before it lands: a range sized without the profile is sized wrong and cannot be re-sized later. Idempotent, so a driver retry cannot restart a live run. - The profile is written to the volume, so a restarted monitor resumes a run already under way instead of waiting for a /start its predecessor received. - /logs lists every artifact with size and mtime; /logs/ serves one and honours Range. Nothing pulls through these yet -- the driver still execs tar -- but they are what replaces ~1.2 GB of API-server traffic on a 4000-range run. - Service, plus an HTTPRoute gated on monitor.routeHost so the chart stays usable without a gateway. - The status ConfigMap, _patch_cm and PROGRESS_CM are gone; mission start moved to a file beside progress.json. - longest-first-needs-a-profile moved off startup, where the profile no longer exists yet, and now answers 400 with the reason. Known incomplete, and deliberately so: validate_config still runs at boot against a config that is not yet whole, and the range is not validated at all. Both belong in /start once it carries range as well as profile. Co-Authored-By: Claude Opus 5 --- .../MissionHistoryPubnetParallelCatchupV2.fs | 97 ++++++++---- .../apps/job_monitor.py | 99 ++++++------ src/MissionParallelCatchup/lib/config.py | 1 - src/MissionParallelCatchup/lib/http_server.py | 143 ++++++++++++++++-- src/MissionParallelCatchup/lib/profiles.py | 10 ++ .../templates/job_monitor.yaml | 40 +++++ .../parallel_catchup_helm/values.yaml | 6 + src/MissionParallelCatchup/tests/conftest.py | 1 - .../contract/test_fsharp_driver_contract.py | 41 ----- .../tests/resilience/test_hostile_state.py | 23 --- .../tests/unit/test_range_generation.py | 17 ++- 11 files changed, 312 insertions(+), 166 deletions(-) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 9b0cf54c..0d84214f 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -104,26 +104,57 @@ let resolveRangeProfile (context: MissionContext) : string option = LogWarn "Range profile %s has no ranges; sizing from configured requests" spec None else - let name = sprintf "%s-range-profile" helmReleaseName - let file = Path.Combine(Path.GetTempPath(), sprintf "%s-profile.json" helmReleaseName) - File.WriteAllText(file, body) - - RunShellCommand [| "kubectl" - "create" - "configmap" - name - "--namespace" - context.namespaceProperty - sprintf "--from-file=profile.json=%s" file |] - |> ignore - - LogInfo "Range profile: %d ranges from %s -> configmap %s" count spec name - Some name + LogInfo "Range profile: %d ranges from %s" count spec + Some body with ex -> LogWarn "Could not load range profile %s (%s); sizing from configured requests" spec ex.Message None +// The driver talks to the monitor over its HTTPRoute: profile in via POST +// /start, status out of /status, logs pulled per file. The previous channels -- +// a status ConfigMap and `kubectl exec tar` -- both went through the API +// server; the logs alone measured ~0.3 MB per range, so ~1.2 GB of +// control-plane traffic on a 4000-range run, for bytes with no reason to be +// there. +let monitorRouteHost (context: MissionContext) = sprintf "%s.%s" nonce context.routeInternalDomain + +// Where the socket actually goes. An external host (an ELB, say) still needs +// the Host header set to the route hostname, or the gateway cannot match it. +let monitorEndpoint (context: MissionContext) = + match context.routeExternalHost with + | Some h -> h + | None -> monitorRouteHost context + +let private monitorClient (context: MissionContext) = + let c = new HttpClient(BaseAddress = Uri(sprintf "http://%s" (monitorEndpoint context))) + c.DefaultRequestHeaders.Host <- monitorRouteHost context + c.Timeout <- TimeSpan.FromMinutes(10.0) + c + +/// POST the profile and let reconcile start. Retried: the route and the pod +/// both need a moment after `helm install`, and until this lands the monitor +/// deliberately dispatches nothing. +let startMission (context: MissionContext) (profileJson: string) = + use client = monitorClient context + let deadline = DateTime.UtcNow.AddMinutes(5.0) + let mutable started = false + + while not started && DateTime.UtcNow < deadline do + try + use content = new StringContent(profileJson, Text.Encoding.UTF8, "application/json") + let r = client.PostAsync("/start", content) |> Async.AwaitTask |> Async.RunSynchronously + if r.IsSuccessStatusCode then + LogInfo "Mission started: profile POSTed to %s/start" (monitorEndpoint context) + started <- true + else + Thread.Sleep(5000) + with _ -> Thread.Sleep(5000) + + if not started then + failwithf "could not reach the job monitor at %s to start the mission" (monitorEndpoint context) + + // Helper functions to convert label/taint tuples to Helm-compatible format using indexed notation let requireNodeLabelToHelmIndexed (index: int) ((key: string), (value: string option)) = match value with @@ -171,9 +202,12 @@ let installProject (context: MissionContext) = // ~2Gi for pvc, ~35Gi for ephemeral, or the monitor logs a loud mismatch. setOptions.Add(sprintf "worker.storageMode=%s" context.pubnetParallelCatchupStorageMode) - match resolveRangeProfile context with - | Some cm -> setOptions.Add(sprintf "monitor.profileConfigMap=%s" cm) - | None -> () + // The route the driver uses for /start, /status and the logs. Templated + // only when routeHost is set, so an in-cluster caller still works without a + // gateway. + setOptions.Add(sprintf "monitor.routeHost=%s" (monitorRouteHost context)) + setOptions.Add(sprintf "monitor.gatewayName=%s" context.gatewayName) + setOptions.Add(sprintf "monitor.gatewayNamespace=%s" context.gatewayNamespace) setOptions.Add(sprintf "range.order=%s" context.pubnetParallelCatchupRangeOrder) @@ -583,23 +617,17 @@ let collectLogsFromPods (context: MissionContext) = // of the much-slower log collection — otherwise we get SIGKILLed mid- // collection and leak every worker pod, which is what we saw in practice // with a 1024-worker run aborted from Jenkins. + let queryJobMonitor (context: MissionContext, key: String) = - // The monitor publishes its status JSON into -catchup-progress. - // Reading it through the kube API removes the Gateway/HTTPRoute dependency - // entirely -- the driver already has a client. + // `key` is vestigial: /status returns the one document the ConfigMap used + // to hold under that key. try - let cm = - context.kube.ReadNamespacedConfigMap(helmReleaseName + "-catchup-progress", context.namespaceProperty) - - match cm.Data.TryGetValue key with - | true, body -> - LogInfo "job monitor status from configmap key '%s': %s" key body - Some(JObject.Parse(body)) - | _ -> - LogInfo "job monitor configmap has no '%s' yet" key - None + use client = monitorClient context + let body = client.GetStringAsync("/status") |> Async.AwaitTask |> Async.RunSynchronously + LogInfo "job monitor status: %s" body + Some(JObject.Parse(body)) with ex -> - LogError "Error reading job monitor configmap: %s" ex.Message + LogError "Error reading job monitor status: %s" ex.Message None @@ -924,6 +952,11 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = installProject context + // The monitor dispatches nothing until this arrives, so a profile that + // cannot be delivered fails the run here rather than silently sizing every + // range as unprofiled. + startMission context (defaultArg (resolveRangeProfile context) "{}") + let mutable allJobsFinished = false let mutable timeoutLeft = jobMonitorStatusCheckTimeOutSecs diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py index a973dabd..2fb943cf 100644 --- a/src/MissionParallelCatchup/apps/job_monitor.py +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -58,10 +58,19 @@ def main(): - # Before any dispatch: the first Job built must already be sized from it. - config.PROFILE = profiles.load_profile() - # After the profile, before the thread: the only place a bad config can - # still take the process down instead of being swallowed by the loop. + # The driver POSTs the profile to /start. Kept on the volume so a restarted + # monitor resumes a run already under way instead of waiting for a /start + # that was delivered to its predecessor. + config.PROFILE_PATH = os.path.join(config.LOG_DIR, 'profile.json') + if os.path.exists(config.PROFILE_PATH): + config.PROFILE = profiles.load_profile() + http_server.started.set() + + http_server.status_source = lambda: (status, status_lock) + http_server.on_start = install_profile + + # Before the thread: the only place a bad config can still take the process + # down instead of being swallowed by the loop. validate_config() # This is the reconcile loop -- @@ -72,6 +81,27 @@ def main(): http_server.serve() +def install_profile(doc): + """Land the driver's profile on the volume and size from it. + + Written before the gate opens, so the first Job dispatched is already sized + by it -- an unprofiled first wave would route every range to protostar. + """ + profile = profiles.load_profile_doc(doc) + # Checked here, not at startup: the profile arrives with /start, so this is + # the first moment it can be judged. Raising rejects the POST with the + # reason, which fails the driver fast instead of dispatching a run whose + # ordering silently degrades to tip-first. + if config.RANGE_ORDER == 'longest-first' and not profile: + raise ValueError( + "RANGE_ORDER=longest-first requires a profile: it orders ranges by " + "their measured seconds, and with no profile every range ties and " + "dispatch stays tip-first. POST a profile, or set RANGE_ORDER.") + records.write_atomic(config.PROFILE_PATH, json.dumps(doc, separators=(',', ':'))) + config.PROFILE = profile + logger.info("profile installed: %d ranges", len(config.PROFILE)) + + def validate_config(): """Fatal config checks. Runs once at startup, BEFORE the reconcile thread starts. @@ -83,15 +113,6 @@ def validate_config(): if config.RANGE_ORDER not in config.VALID_RANGE_ORDERS: raise ValueError("RANGE_ORDER must be one of %s, got %r" % (', '.join(config.VALID_RANGE_ORDERS), config.RANGE_ORDER)) - # longest-first sorts on measured `seconds`; with no profile every key ties - # and dispatch silently stays tip-first. - if config.RANGE_ORDER == 'longest-first' and not config.PROFILE: - raise ValueError( - "RANGE_ORDER=longest-first requires a profile: it orders ranges by " - "their measured seconds, and with no profile loaded every range ties " - "and dispatch stays tip-first. Pass a profile, or set RANGE_ORDER " - "explicitly to tip-first or oldest-first.") - status = { 'num_remain': 1, # non-zero until the first real update, so callers don't see a premature 0 @@ -105,10 +126,16 @@ def validate_config(): } status_lock = threading.Lock() +# Beside progress.json: the volume is the only durable store. +_MISSION_START = os.path.join(config.LOG_DIR, 'mission_started') + # --- the run itself --------------------------------------------------------- def reconcile_loop(): global status + # Nothing is dispatched until the driver has POSTed /start: a range sized + # before the profile lands is sized wrong, and it cannot be re-sized later. + http_server.started.wait() # None until reconcile has an owner reference to attach it to; until then # process start is correct anyway, because that IS the start of a new run. mission_start_time = read_mission_start() or time.time() @@ -120,7 +147,7 @@ def reconcile_loop(): state['owner'] = owner_ref() _progress_owner['ref'] = state['owner'] if read_mission_start() is None: - _patch_cm({'started_at': repr(mission_start_time)}) + records.write_atomic(_MISSION_START, repr(mission_start_time)) r = reconcile(state) @@ -165,14 +192,6 @@ def reconcile_loop(): metrics.refresh_duration.set(workers_refresh_duration) metrics.mission_duration.set(mission_duration) logger.info("Status: %s", json.dumps(status)) - # Publish on change only -- a 10h run would otherwise issue ~3600 - # no-op ConfigMap writes. - counts = (r['remaining'], r['completed'], len(r['failed_ranges']), - len(visible_in_progress)) - if counts != state.get('last_counts'): - state['last_counts'] = counts - with status_lock: - save_status(status) except Exception as e: logger.exception("Error while reconciling: %s", str(e)) @@ -422,16 +441,6 @@ def load_progress(): return {} -def save_status(snapshot): - """Publish the run's status into the ConfigMap the driver reads. - - The mission driver runs outside the cluster and already has a kube client, - so reading a ConfigMap is simpler and more robust than exposing the monitor - through a Gateway/HTTPRoute just to be polled. - """ - _patch_cm({'status.json': json.dumps(snapshot, separators=(',', ':'))}) - - def save_progress(progress): # The monitor's own state, and the only copy. The driver's view of the run # is status.json in the ConfigMap; this document is not published. @@ -439,19 +448,6 @@ def save_progress(progress): records.write_atomic(config.PROGRESS_FILE, blob) -def _patch_cm(data): - body = {'data': data} - try: - kube.core_v1.patch_namespaced_config_map(config.PROGRESS_CM, config.NAMESPACE, body) - except ApiException as e: - if e.status != 404: - raise - kube.core_v1.create_namespaced_config_map(config.NAMESPACE, client.V1ConfigMap( - # Owned by the chart's stellar-core ConfigMap, like the Jobs and - # PVCs, so `helm uninstall` reclaims it. - metadata=client.V1ObjectMeta(name=config.PROGRESS_CM, labels={config.LABEL_RUN: config.RUN_NAME}, - owner_references=_progress_owner.get('ref')), - data=body['data'])) # --- worker log capture ----------------------------------------------------- @@ -1467,15 +1463,14 @@ def pods_by_job(): def read_mission_start(): """When this run first started, or None if not recorded yet. - Its own ConfigMap key: progress.json is keyed by ledger range, and anything - else in it would be walked as one. Read-only -- creating the ConfigMap here - would race the owner reference, and an ownerless one survives - `helm uninstall`. + Its own file: progress.json is keyed by ledger range, and anything else in + it would be walked as one. On the volume so it survives a monitor restart, + which is what makes mission_duration span the run rather than the process. """ try: - cm = kube.core_v1.read_namespaced_config_map(config.PROGRESS_CM, config.NAMESPACE) - return float((cm.data or {})['started_at']) - except (ApiException, KeyError, TypeError, ValueError): + with open(_MISSION_START) as fh: + return float(fh.read()) + except (OSError, ValueError): return None diff --git a/src/MissionParallelCatchup/lib/config.py b/src/MissionParallelCatchup/lib/config.py index b729bd6b..8ded800d 100644 --- a/src/MissionParallelCatchup/lib/config.py +++ b/src/MissionParallelCatchup/lib/config.py @@ -53,7 +53,6 @@ RUN_NAME = os.getenv('RUN_NAME', 'parallel-catchup') -PROGRESS_CM = f"{RUN_NAME}-catchup-progress" LABEL_RUN = 'catchup.stellar.org/run' diff --git a/src/MissionParallelCatchup/lib/http_server.py b/src/MissionParallelCatchup/lib/http_server.py index 9ca8271f..94add61e 100644 --- a/src/MissionParallelCatchup/lib/http_server.py +++ b/src/MissionParallelCatchup/lib/http_server.py @@ -1,39 +1,156 @@ -"""The monitor's HTTP surface: a liveness probe and the Prometheus scrape. +"""The monitor's HTTP surface. -Two routes, both with a live consumer -- the kubelet's livenessProbe and the -`kubernetes-pods` scrape job, which relabels prometheus.io/path onto -__metrics_path__ and so reaches the non-standard /prometheus. +Everything the mission driver needs, so it never reads cluster state to run the +mission: the profile goes in through /start, status comes out of /status, and +the logs are pulled per file. The alternative for the logs was `kubectl exec`, +which proxies every byte through the API server -- measured 0.3 MB per range, so +~1.2 GB of control-plane traffic on a 4000-range run, for bytes that have no +business there. + +/healthz and /prometheus predate this and keep their consumers: the kubelet's +livenessProbe, and the `kubernetes-pods` scrape job that relabels +prometheus.io/path onto __metrics_path__ and so reaches the non-standard path. """ -from http.server import BaseHTTPRequestHandler, HTTPServer +import json +import os +import re +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from prometheus_client import CONTENT_TYPE_LATEST, REGISTRY, generate_latest +import config from logger import build_logger logger = build_logger('http_server') +# Set by job_monitor before serve(). A tuple rather than an import, because +# job_monitor imports this module. +status_source = None # () -> (dict, lock) +started = threading.Event() # /start has delivered a profile +on_start = None # (doc) -> None, installs the profile + +# One path element, no traversal, no dotfiles. +_SAFE_NAME = re.compile(r'^[A-Za-z0-9][A-Za-z0-9._-]*$') + + +def _log_path(name): + """An existing regular file in LOG_DIR named by `name`, or None.""" + if not _SAFE_NAME.match(name or ''): + return None + path = os.path.join(config.LOG_DIR, name) + return path if os.path.isfile(path) else None + class RequestHandler(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + + def _send(self, code, body=b'', ctype='application/json'): + self.send_response(code) + self.send_header('Content-type', ctype) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + if body: + self.wfile.write(body) + def do_GET(self): if self.path == '/healthz': # Serving at all is the whole check: a process that answers here # still has its HTTP thread. - self.send_response(200) - self.end_headers() + self._send(200, b'ok', 'text/plain') elif self.path == '/prometheus': - self.send_response(200) - self.send_header('Content-type', CONTENT_TYPE_LATEST) - self.end_headers() - self.wfile.write(generate_latest(REGISTRY)) + self._send(200, generate_latest(REGISTRY), CONTENT_TYPE_LATEST) + elif self.path == '/status': + snapshot, lock = status_source() + with lock: + body = json.dumps(snapshot, separators=(',', ':')).encode() + self._send(200, body) + elif self.path == '/logs': + self._send(200, json.dumps(self._manifest(), separators=(',', ':')).encode()) + elif self.path.startswith('/logs/'): + self._send_file(self.path[len('/logs/'):]) else: - self.send_response(404) + self._send(404) + + def do_POST(self): + if self.path != '/start': + self._send(404) + return + try: + raw = self.rfile.read(int(self.headers.get('Content-Length') or 0)) + doc = json.loads(raw) if raw else {} + except ValueError as e: + self._send(400, json.dumps({'error': f'invalid profile json: {e}'}).encode()) + return + # Idempotent: a driver that retries after a timeout must not restart a + # run that is already dispatching. + if not started.is_set(): + try: + on_start(doc) + except ValueError as e: + # A profile the run cannot proceed with. Answering 400 fails the + # driver here, with the reason, rather than leaving it to poll a + # monitor that will never dispatch. + self._send(400, json.dumps({'error': str(e)}).encode()) + return + started.set() + self._send(200, b'{"started":true}') + + def _manifest(self): + """Every artifact on the volume, with the size and mtime a puller needs + to tell "already have it" from "grew since last time".""" + out = [] + for name in os.listdir(config.LOG_DIR): + path = os.path.join(config.LOG_DIR, name) + if _SAFE_NAME.match(name) and os.path.isfile(path): + st = os.stat(path) + out.append({'name': name, 'size': st.st_size, 'mtime': int(st.st_mtime)}) + return out + + def _send_file(self, name): + """One artifact, honouring Range so a cut transfer resumes instead of + restarting. The collector appends to these while a pod runs, so the + length is fixed once at open and never read past.""" + path = _log_path(name) + if not path: + self._send(404) + return + with open(path, 'rb') as fh: + size = os.fstat(fh.fileno()).st_size + start, end = 0, size - 1 + m = re.match(r'bytes=(\d+)-(\d*)', self.headers.get('Range') or '') + partial = bool(m) + if partial: + start = int(m.group(1)) + end = int(m.group(2)) if m.group(2) else size - 1 + if start >= size: + self.send_response(416) + self.send_header('Content-Range', f'bytes */{size}') + self.end_headers() + return + length = end - start + 1 + self.send_response(206 if partial else 200) + self.send_header('Content-type', 'application/octet-stream') + self.send_header('Content-Length', str(length)) + if partial: + self.send_header('Content-Range', f'bytes {start}-{end}/{size}') self.end_headers() + fh.seek(start) + remaining = length + while remaining > 0: + chunk = fh.read(min(1 << 20, remaining)) + if not chunk: + break + self.wfile.write(chunk) + remaining -= len(chunk) def log_message(self, *args): pass # the default handler logs every request to stderr def serve(port=8080): + # Threading, because a log pull is long-lived and must not block the + # liveness probe or the driver's status poll behind it. logger.info('Starting httpd server on :%d', port) - HTTPServer(('', port), RequestHandler).serve_forever() + ThreadingHTTPServer(('', port), RequestHandler).serve_forever() diff --git a/src/MissionParallelCatchup/lib/profiles.py b/src/MissionParallelCatchup/lib/profiles.py index 620011e2..af0afed5 100644 --- a/src/MissionParallelCatchup/lib/profiles.py +++ b/src/MissionParallelCatchup/lib/profiles.py @@ -13,6 +13,16 @@ logger = logging.getLogger() +def load_profile_doc(doc): + """The sorted (end, record) list a parsed profile document yields. + + An unprofiled run POSTs {} and gets [] -- a profile is an optimisation, + never a prerequisite, so "no ranges" is a valid answer rather than an error. + """ + ranges = (doc or {}).get('ranges') or {} + return sorted((int(k), v) for k, v in ranges.items()) + + def load_profile(): """Per-range measurements from an earlier run, keyed by range end. diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 8ce05f4e..fe4609b6 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -97,6 +97,46 @@ spec: requests: storage: {{ .Values.monitor.logStorageSize }} --- +# The driver's only channel into the run: profile in via POST /start, status out +# of /status, logs pulled per file. Previously the driver read a ConfigMap and +# exec'd `tar` for the logs, which proxied every byte through the API server -- +# ~0.3 MB per range, so ~1.2 GB of control-plane traffic on a 4000-range run. +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-job-monitor +spec: + selector: + app: job-monitor + release: {{ .Release.Name }} + ports: + - name: http + port: 8080 + targetPort: 8080 +--- +{{- if .Values.monitor.routeHost }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ .Release.Name }}-job-monitor +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: {{ .Values.monitor.gatewayName }} + namespace: {{ .Values.monitor.gatewayNamespace }} + hostnames: + - {{ .Values.monitor.routeHost | quote }} + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: {{ .Release.Name }}-job-monitor + port: 8080 +--- +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 59bd9444..08feed98 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -81,6 +81,12 @@ monitor: # branch and pinned by tag rather than :latest so a run is always traceable to # one image. Revert to the stellar/ repo once there is a push path for it. image: "stellajuna/ssc-jm:2026-08-07b" + # The driver reaches the monitor through this route: profile in via POST + # /start, status and logs out. Empty routeHost disables the HTTPRoute; the + # Service still exists, so an in-cluster caller works either way. + routeHost: "" + gatewayName: "" + gatewayNamespace: "" # Dev loop: run the monitor and collector from a ConfigMap holding # job_monitor.py and log_collector.py instead of a built image. Set it to the # ConfigMap name and point monitor.image at a plain python base; the deps the diff --git a/src/MissionParallelCatchup/tests/conftest.py b/src/MissionParallelCatchup/tests/conftest.py index df4b0a51..f56d0d18 100644 --- a/src/MissionParallelCatchup/tests/conftest.py +++ b/src/MissionParallelCatchup/tests/conftest.py @@ -319,7 +319,6 @@ def cluster(tmp_path, monkeypatch): # Derived at import from RUN_NAME / LOG_DIR, so they have to follow. monkeypatch.setattr(config, 'LOG_DIR', str(log_dir)) monkeypatch.setattr(config, 'PROGRESS_FILE', str(log_dir / 'progress.json')) - monkeypatch.setattr(config, 'PROGRESS_CM', f"{env['RUN_NAME']}-catchup-progress") # Module-level mutable state that would otherwise leak between tests. monkeypatch.setattr(config, 'PROFILE', None) monkeypatch.setattr(jm, '_progress_owner', {}) diff --git a/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py b/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py index 89773141..da8d0aa6 100644 --- a/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py +++ b/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py @@ -92,14 +92,6 @@ def test_every_helm_command_uses_the_mission_namespace(): f"helm {verb.group(1)} does not target the mission namespace: {block!r}") -def test_profile_configmap_uses_the_mission_namespace(): - """The profile mount and Helm release must be created in one namespace.""" - block = fs_extract( - r'RunShellCommand\s+\[\|\s*"kubectl"(.*?)\|\]').group(1) - assert '"create"' in block and '"configmap"' in block - assert re.search(r'"--namespace"\s+context\.namespaceProperty', block), ( - "the range-profile ConfigMap follows kubeconfig's default namespace " - "instead of the mission namespace") def test_every_value_the_driver_sets_is_one_the_chart_knows(): @@ -249,41 +241,8 @@ def test_the_history_get_command_lands_in_the_config_the_worker_mounts(): # --- the ConfigMap the driver polls ------------------------------------------ -def test_the_driver_reads_the_configmap_the_monitor_writes(): - """One name, derived on both sides from the helm release name. - - The driver appends a literal suffix to the release; the monitor appends the - same suffix to RUN_NAME, which the chart sets from .Release.Name. A mismatch - reads as "the monitor has not published yet", forever -- and the driver's - only reaction to that is a 600s timeout and `job monitor not reachable`. - """ - suffix = fs_extract(r'helmReleaseName \+ "(-[a-z-]+)"').group(1) - release = 'pc-abc' - # What the driver will ask for, and what the monitor will have created -- - # the latter imported with the RUN_NAME the chart gives it for that release. - wanted = release + suffix - run_name = art.env_of(art.containers(release=release)[art.MONITOR_CONTAINER])['RUN_NAME'] - written = art.defaults('config', (('RUN_NAME', run_name),))['PROGRESS_CM'] - assert wanted == written, f"driver reads {wanted!r}, monitor writes {written!r}" -def test_the_driver_reads_the_keys_the_monitor_publishes(cluster): - """status.json and progress.json are two keys in that one ConfigMap. - - Checked against a ConfigMap the real monitor actually wrote, so a key that - is only mentioned in a comment does not count. - """ - wanted = set(re.findall(r'let jobMonitor\w*Key = "([\w.]+)"', FS)) - assert wanted, "the driver no longer names the ConfigMap keys" - - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.reconcile() # records a completion -> progress.json - jm.save_status(jm.status) # what the reconcile loop publishes - published = set(cluster.k8s.config_map_data(config.PROGRESS_CM, cluster.namespace) or {}) - missing = sorted(wanted - published) - assert not missing, ( - f"the driver reads {missing}; the monitor published {sorted(published)}") def test_every_status_field_the_driver_reads_is_one_the_monitor_sets(): diff --git a/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py b/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py index 3594baa8..9e0342a6 100644 --- a/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py +++ b/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py @@ -396,28 +396,5 @@ def test_a_foreign_run_s_jobs_in_the_namespace_are_ignored(cluster): assert result['total'] == 3 -def test_the_status_configmap_being_deleted_mid_run_is_survivable(cluster): - """The ConfigMap is the driver's view. Losing it must not lose the run.""" - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1) - cluster.reconcile() - jm.save_status(jm.status) # what the reconcile loop publishes - - cluster.k8s.core_v1.delete_namespaced_config_map(config.PROGRESS_CM, - cluster.namespace) - cluster.advance(200, 'succeeded') - cluster.finalize(200, 1) - result = cluster.reconcile() - - assert set(cluster.completed()) == {'200', '300'} - assert cluster.state['halted'] is False - assert result['completed'] == 2 - - # Recreated by the next publish, so the driver is not blind for the rest of - # the run. - jm.save_status(jm.status) - assert 'status.json' in cluster.k8s.config_map_data(config.PROGRESS_CM, - cluster.namespace) diff --git a/src/MissionParallelCatchup/tests/unit/test_range_generation.py b/src/MissionParallelCatchup/tests/unit/test_range_generation.py index 179d8784..437a9af8 100644 --- a/src/MissionParallelCatchup/tests/unit/test_range_generation.py +++ b/src/MissionParallelCatchup/tests/unit/test_range_generation.py @@ -149,9 +149,20 @@ def test_preflight_rejects_an_unknown_order(preflight): preflight(order='longest')() -def test_preflight_rejects_longest_first_without_a_profile(preflight): - with pytest.raises(ValueError, match='requires a profile'): - preflight(order='longest-first', profile=None)() +def test_preflight_rejects_longest_first_without_a_profile(monkeypatch, tmp_path): + """The check moved to /start: the profile arrives with the POST, so startup + is too early to judge it. Rejecting there fails the driver fast instead of + dispatching a run whose ordering silently degrades to tip-first.""" + monkeypatch.setattr(config, 'RANGE_ORDER', 'longest-first') + monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(config, 'PROFILE_PATH', str(tmp_path / 'profile.json')) + + with pytest.raises(ValueError, match='longest-first requires a profile'): + jm.install_profile({}) + + # A profile with ranges is accepted, and nothing is written until it passes. + jm.install_profile({'ranges': {'300': {'seconds': 1.0}}}) + assert config.PROFILE == [(300, {'seconds': 1.0})] def test_preflight_allows_longest_first_with_a_profile(preflight): From 191362325fcf0d8972690d5a13c483efb0fcf3d5 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Sat, 8 Aug 2026 16:47:56 -0400 Subject: [PATCH 069/117] Validate the whole config at /start, not at import Coercing the liveness numbers at import made a bad value a boot crash, and a process that cannot start cannot report why: the driver polled a pod that never answered and timed out 600s later with "job monitor not reachable", which is true and useless. config.py now keeps them as strings. validate_config coerces, range-checks and rebinds them, and runs from /start -- the first moment the config is complete, because the profile arrives with the POST. One validation point, one failure channel: whatever is wrong comes back as a 400 carrying the reason. The restart path validates too. It resumes from the profile on the volume without a /start, so it is the other place the coercion has to happen. Adds tests/unit/test_http_surface.py, which drives a real server rather than calling handler methods -- a Range header the socket layer mishandles would otherwise pass. Covers the 400, the gate, idempotent /start, the manifest, resume-from-offset, and that a filename outside the volume is refused (the route is reachable from outside the cluster, so the name is the whole boundary). Co-Authored-By: Claude Opus 5 --- .../apps/job_monitor.py | 42 ++++-- src/MissionParallelCatchup/lib/config.py | 22 +--- .../tests/unit/test_http_surface.py | 122 ++++++++++++++++++ .../tests/unit/test_range_generation.py | 27 ++-- .../tests/unit/test_worker_liveness.py | 41 +++--- 5 files changed, 199 insertions(+), 55 deletions(-) create mode 100644 src/MissionParallelCatchup/tests/unit/test_http_surface.py diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py index 2fb943cf..06dc783c 100644 --- a/src/MissionParallelCatchup/apps/job_monitor.py +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -64,15 +64,14 @@ def main(): config.PROFILE_PATH = os.path.join(config.LOG_DIR, 'profile.json') if os.path.exists(config.PROFILE_PATH): config.PROFILE = profiles.load_profile() + # The gate opens without a /start on this path, so this is where the + # config gets coerced and checked instead. + validate_config() http_server.started.set() http_server.status_source = lambda: (status, status_lock) http_server.on_start = install_profile - # Before the thread: the only place a bad config can still take the process - # down instead of being swallowed by the loop. - validate_config() - # This is the reconcile loop -- # dispatch, progress record, metrics, status. reconcile_thread = threading.Thread(target=reconcile_loop, daemon=True) @@ -88,10 +87,10 @@ def install_profile(doc): by it -- an unprofiled first wave would route every range to protostar. """ profile = profiles.load_profile_doc(doc) - # Checked here, not at startup: the profile arrives with /start, so this is - # the first moment it can be judged. Raising rejects the POST with the - # reason, which fails the driver fast instead of dispatching a run whose - # ordering silently degrades to tip-first. + # The whole config, judged at the first moment it is complete. Anything + # wrong rejects the POST with the reason rather than dispatching a run that + # is already misconfigured. + validate_config() if config.RANGE_ORDER == 'longest-first' and not profile: raise ValueError( "RANGE_ORDER=longest-first requires a profile: it orders ranges by " @@ -102,11 +101,34 @@ def install_profile(doc): logger.info("profile installed: %d ranges", len(config.PROFILE)) +_LIVENESS_NUMBERS = (('LIVENESS_PROBE_TIMEOUT_SECONDS', float), + ('LIVENESS_SWEEP_SECONDS', float), + ('LIVENESS_MAX_CONCURRENCY', int)) + + def validate_config(): - """Fatal config checks. Runs once at startup, BEFORE the reconcile thread starts. + """Every fatal config check, against the whole config. - Called after load_profile() because the last check needs the profile. + Runs from /start rather than at import, because that is the first moment the + config is complete -- the profile arrives with the POST. One validation + point, one failure channel: whatever is wrong comes back as a 400 with the + reason instead of a crashlooping pod the driver can only time out on. + + Coerces the numeric env vars and rebinds them, so no caller ever sees the + string form. """ + for name, cast in _LIVENESS_NUMBERS: + raw = getattr(config, name) + try: + value = cast(raw) + except (TypeError, ValueError): + raise ValueError( + "LIVENESS_PROBE_TIMEOUT_SECONDS and LIVENESS_SWEEP_SECONDS must be " + "numbers; LIVENESS_MAX_CONCURRENCY must be an integer") from None + if value <= 0: + raise ValueError(f"{name} must be greater than zero, got {raw!r}") + setattr(config, name, value) + if config.RANGE_GENERATOR not in config.VALID_RANGE_GENERATORS: raise ValueError("RANGE_GENERATOR must be one of %s, got %r" % (', '.join(config.VALID_RANGE_GENERATORS), config.RANGE_GENERATOR)) diff --git a/src/MissionParallelCatchup/lib/config.py b/src/MissionParallelCatchup/lib/config.py index 8ded800d..7a3d002e 100644 --- a/src/MissionParallelCatchup/lib/config.py +++ b/src/MissionParallelCatchup/lib/config.py @@ -568,20 +568,8 @@ # go out at once at the head of a wave. CONNECTION_POOL = int(os.getenv('CONNECTION_POOL', '64')) -# Coerced here rather than at each use site so no importer can ever see the -# string form, and a bad value fails at import instead of at the first probe. -try: - LIVENESS_PROBE_TIMEOUT_SECONDS = float(LIVENESS_PROBE_TIMEOUT_SECONDS) - LIVENESS_SWEEP_SECONDS = float(LIVENESS_SWEEP_SECONDS) - LIVENESS_MAX_CONCURRENCY = int(LIVENESS_MAX_CONCURRENCY) -except ValueError as e: - raise ValueError( - "LIVENESS_PROBE_TIMEOUT_SECONDS and LIVENESS_SWEEP_SECONDS must be " - "numbers; LIVENESS_MAX_CONCURRENCY must be an integer") from e - -for _name, _value in ( - ('LIVENESS_PROBE_TIMEOUT_SECONDS', LIVENESS_PROBE_TIMEOUT_SECONDS), - ('LIVENESS_SWEEP_SECONDS', LIVENESS_SWEEP_SECONDS), - ('LIVENESS_MAX_CONCURRENCY', LIVENESS_MAX_CONCURRENCY)): - if _value <= 0: - raise ValueError(f"{_name} must be greater than zero, got {_value!r}") +# Left as strings on purpose. Coercing at import made a bad value a boot crash, +# and a process that cannot start cannot report why -- the driver just polled a +# pod that never answered and timed out 600s later with "not reachable". +# validate_config coerces and rebinds these when /start delivers the run, so a +# bad value comes back as a 400 carrying the reason. diff --git a/src/MissionParallelCatchup/tests/unit/test_http_surface.py b/src/MissionParallelCatchup/tests/unit/test_http_surface.py new file mode 100644 index 00000000..b077d6f2 --- /dev/null +++ b/src/MissionParallelCatchup/tests/unit/test_http_surface.py @@ -0,0 +1,122 @@ +"""The monitor's HTTP surface, driven over a real socket. + +This is the driver's only channel into a run -- profile in, status and logs out +-- so it is exercised through an actual server rather than by calling handler +methods, which would not catch a Range header the socket layer mishandles. +""" + +import json +import threading +import urllib.error +import urllib.request +from http.server import ThreadingHTTPServer + +import pytest + +import config +import http_server +import job_monitor as jm + + +@pytest.fixture +def server(tmp_path, monkeypatch): + """A live monitor HTTP surface on a throwaway port and volume.""" + monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(config, 'PROFILE_PATH', str(tmp_path / 'profile.json')) + monkeypatch.setattr(http_server, 'started', threading.Event()) + monkeypatch.setattr(http_server, 'on_start', jm.install_profile) + monkeypatch.setattr(http_server, 'status_source', + lambda: (jm.status, jm.status_lock)) + + httpd = ThreadingHTTPServer(('127.0.0.1', 0), http_server.RequestHandler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + yield f"http://127.0.0.1:{httpd.server_address[1]}", tmp_path + httpd.shutdown() + + +def _get(base, path, headers=None): + req = urllib.request.Request(base + path, headers=headers or {}) + with urllib.request.urlopen(req, timeout=5) as r: + return r.status, r.read(), dict(r.headers) + + +def _post(base, path, body): + req = urllib.request.Request(base + path, data=body.encode(), method='POST') + try: + with urllib.request.urlopen(req, timeout=5) as r: + return r.status, r.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + + +def test_start_rejects_a_bad_config_with_the_reason(server, monkeypatch): + """The whole point of validating here: the driver gets told why. + + Coercing at import made this a crashlooping pod instead, which the driver + could only observe as a 600s timeout on a monitor that never answered. + """ + base, _ = server + monkeypatch.setattr(config, 'LIVENESS_MAX_CONCURRENCY', 'many') + + code, body = _post(base, '/start', '{}') + + assert code == 400 + assert 'LIVENESS_MAX_CONCURRENCY must be an integer' in json.loads(body)['error'] + assert not http_server.started.is_set(), "a rejected config must not open the gate" + + +def test_start_opens_the_gate_and_is_idempotent(server): + """A driver that retries after a timeout must not restart a live run.""" + base, vol = server + + assert _post(base, '/start', json.dumps({'ranges': {'300': {'seconds': 1.0}}}))[0] == 200 + assert http_server.started.is_set() + assert (vol / 'profile.json').exists(), "the profile is kept for a restart" + + # A second POST carrying nothing must not wipe the profile already installed. + assert _post(base, '/start', '{}')[0] == 200 + assert config.PROFILE == [(300, {'seconds': 1.0})] + + +def test_status_is_served_from_memory(server): + base, _ = server + code, body, _ = _get(base, '/status') + + assert code == 200 + assert json.loads(body)['num_remain'] == jm.status['num_remain'] + + +def test_logs_manifest_carries_what_a_puller_diffs_on(server): + base, vol = server + (vol / 'range-300-a1.log.gz').write_bytes(b'x' * 1234) + + entries = {e['name']: e for e in json.loads(_get(base, '/logs')[1])} + + assert entries['range-300-a1.log.gz']['size'] == 1234 + assert 'mtime' in entries['range-300-a1.log.gz'] + + +def test_a_file_resumes_from_the_byte_it_stopped_at(server): + """Range is what makes a cut transfer cost the remainder rather than the + whole file -- the truncation that lost 12 ranges their logs on 2026-08-07.""" + base, vol = server + (vol / 'range-300-a1.log.gz').write_bytes(bytes(range(256))) + + whole = _get(base, '/logs/range-300-a1.log.gz') + assert whole[0] == 200 and len(whole[1]) == 256 + + code, body, headers = _get(base, '/logs/range-300-a1.log.gz', + {'Range': 'bytes=200-'}) + assert code == 206 + assert body == bytes(range(200, 256)) + assert headers['Content-Range'] == 'bytes 200-255/256' + + +def test_a_path_outside_the_volume_is_refused(server): + """The route is reachable from outside the cluster once an HTTPRoute is + attached, so the filename is the whole security boundary.""" + base, _ = server + for bad in ('..%2f..%2fetc%2fpasswd', '.hidden', 'sub%2fdir'): + with pytest.raises(urllib.error.HTTPError) as e: + _get(base, '/logs/' + bad) + assert e.value.code == 404 diff --git a/src/MissionParallelCatchup/tests/unit/test_range_generation.py b/src/MissionParallelCatchup/tests/unit/test_range_generation.py index 437a9af8..54d0898f 100644 --- a/src/MissionParallelCatchup/tests/unit/test_range_generation.py +++ b/src/MissionParallelCatchup/tests/unit/test_range_generation.py @@ -4,6 +4,8 @@ full list on every reconcile, so a restart has to reproduce it exactly. """ +import os + import pytest import config @@ -169,20 +171,19 @@ def test_preflight_allows_longest_first_with_a_profile(preflight): preflight(order='longest-first', profile=[(40000000, {'seconds': 900.0})])() -def test_the_preflight_runs_before_the_reconcile_thread_starts(): - """Guards the placement, which is the whole point of the check. +def test_the_preflight_runs_before_anything_is_dispatched(monkeypatch, tmp_path): + """Validation moved to /start, which is the first moment the config is + whole. It still has to bind before dispatch: a run that is misconfigured + must be refused, not started and then discovered.""" + monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(config, 'PROFILE_PATH', str(tmp_path / 'profile.json')) + monkeypatch.setattr(config, 'RANGE_GENERATOR', 'nonsense') - If validate_config ever moves inside reconcile (or after the thread start), - a bad config becomes a silent hang instead of a crash. Asserted against the - source because the ordering, not the call, is what has to hold. - """ - import inspect - main = inspect.getsource(jm.main) - assert 'validate_config()' in main, "validate_config must be called from main()" - assert main.index('validate_config()') < main.index('reconcile_thread.start()'), \ - "validate_config must run BEFORE the reconcile thread starts" - assert main.index('load_profile()') < main.index('validate_config()'), \ - "validate_config checks the profile, so it must run after load_profile" + with pytest.raises(ValueError, match='RANGE_GENERATOR must be one of'): + jm.install_profile({}) + + # Rejected, so nothing was written and no run can proceed from it. + assert not os.path.exists(config.PROFILE_PATH) def jm_source(): diff --git a/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py b/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py index cd38cb80..782bfd21 100644 --- a/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py +++ b/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py @@ -192,21 +192,32 @@ def pod(name, uid, phase, ip): assert targets == {'uid-ready': ('ready', '10.0.0.1')} -def test_malformed_liveness_configuration_fails_with_an_explicit_message(): - env = { - 'PATH': os.environ.get('PATH', ''), - 'HOME': os.environ.get('HOME', ''), - # apps/ and lib/ both, the way the container's flat /app has them. - 'PYTHONPATH': os.pathsep.join((os.path.dirname(jm.__file__), - os.path.dirname(config.__file__))), - 'LIVENESS_MAX_CONCURRENCY': 'many', - } - result = subprocess.run( - [sys.executable, '-c', 'import job_monitor'], - text=True, capture_output=True, env=env, - cwd=os.path.dirname(jm.__file__)) - assert result.returncode != 0 - assert 'LIVENESS_MAX_CONCURRENCY must be an integer' in result.stderr +def test_malformed_liveness_configuration_fails_with_an_explicit_message(monkeypatch, tmp_path): + """A bad value is rejected at /start, not at import. + + Coercing at import made this a boot crash, and a process that cannot start + cannot say why -- the driver polled a pod that never answered and timed out + 600s later with "not reachable". Now it comes back as a 400 with the reason. + """ + monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(config, 'PROFILE_PATH', str(tmp_path / 'profile.json')) + monkeypatch.setattr(config, 'LIVENESS_MAX_CONCURRENCY', 'many') + + with pytest.raises(ValueError, match='LIVENESS_MAX_CONCURRENCY must be an integer'): + jm.install_profile({}) + + +def test_liveness_numbers_are_coerced_once_validation_passes(monkeypatch, tmp_path): + """Callers must never see the string form; validate_config rebinds them.""" + monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) + monkeypatch.setattr(config, 'PROFILE_PATH', str(tmp_path / 'profile.json')) + monkeypatch.setattr(config, 'LIVENESS_MAX_CONCURRENCY', '8') + monkeypatch.setattr(config, 'LIVENESS_SWEEP_SECONDS', '2.5') + + jm.install_profile({}) + + assert config.LIVENESS_MAX_CONCURRENCY == 8 + assert config.LIVENESS_SWEEP_SECONDS == 2.5 def test_a_blocked_sweep_does_not_delay_dispatch(cluster, monkeypatch): From c809215afba08329ccdc00354462734897dfccc8 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Sat, 8 Aug 2026 16:59:07 -0400 Subject: [PATCH 070/117] Move the range into /start and pull logs per file The range was helm values, so changing a run meant re-templating the chart. It is per-run input exactly like the profile, and now travels with it: the chart installs a generic monitor and POST /start defines what it runs. validate_config sees the whole thing at once, so an inverted or zero-width ledger range is a 400 -- it previously generated no work and the run reported success on nothing. Logs are pulled per file over the route instead of one tar through the API server. That was ~0.3 MB per range, so ~1.2 GB of control-plane traffic on a 4000-range run, streamed as a single archive that at 200 workers came back truncated 2 times in 3 with no error surfaced. Per file changes the failure model rather than retrying around it: - a cut transfer resumes from the byte it reached, instead of re-sending - one bad fetch costs one file, not the pass - "already have it" is a manifest comparison, so a repeat pass moves nothing; the tar overlap re-sent 58% of files every time The watermark, the overlap window, the in-pass retry and archiveIsIntact all go with it -- they existed to survive a transport that could truncate silently, and a file that either matches its manifest length or does not cannot. Co-Authored-By: Claude Opus 5 --- src/FSLibrary.Tests/Tests.fs | 60 ---- .../MissionHistoryPubnetParallelCatchupV2.fs | 280 ++++++------------ .../apps/job_monitor.py | 49 ++- src/MissionParallelCatchup/lib/config.py | 14 +- .../templates/job_monitor.yaml | 12 - .../parallel_catchup_helm/values.yaml | 6 +- .../tests/contract/test_chart_defaults.py | 1 - .../contract/test_fsharp_driver_contract.py | 48 ++- .../tests/unit/test_http_surface.py | 12 +- .../tests/unit/test_range_generation.py | 12 +- .../tests/unit/test_worker_liveness.py | 8 +- 11 files changed, 177 insertions(+), 325 deletions(-) diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index b10217af..7cf75830 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -744,63 +744,3 @@ let ``on-demand pool claims fit exactly one pod per node`` () = ) -[] -let ``incremental log fetch tars only what changed since the watermark`` () = - // The teardown tar moved the whole volume in one stream and measured ~20 - // minutes on a full run, entirely after the work had finished. The - // watermark is what turns that into a delta. - let first = String.concat " " (logTarCommand 0L) - Assert.DoesNotContain("--newer-mtime", first) - Assert.Contains("tar -cf -", first) - - // A watermark reaches BACK by the overlap, never forward: a file written - // while the previous tar walked the tree carries an mtime inside that - // window and has to be picked up again rather than skipped forever. - let later = String.concat " " (logTarCommand 1000000L) - Assert.Contains(sprintf "--newer-mtime=@%d" (1000000L - logFetchOverlapSecs), later) - - // The overlap cannot drive the filter negative on a clock near the epoch. - Assert.Contains("--newer-mtime=@0", String.concat " " (logTarCommand 1L)) - - // Both passes keep the collector's per-attempt verdicts and drop its resume - // bookkeeping, or a post-mortem loses why a range failed. - for cmd in [ first; later ] do - Assert.Contains("--exclude='*.state'", cmd) - Assert.Contains("--exclude='./lost+found'", cmd) - - -[] -let ``log archive parts sort in fetch order`` () = - // Parts are extracted in order so a later, complete copy of a file - // overwrites an earlier truncated one. Zero-padded because part10 must not - // sort before part2. - let names = [ 1; 2; 10 ] |> List.map (logArchiveName "run") - Assert.Equal(List.sort names, names) - Assert.Equal("run-worker-logs.part01.tar", logArchiveName "run" 1) - - -[] -let ``a truncated log archive is not mistaken for a good one`` () = - // The watermark may only advance past an archive that reads back whole. - // On ssc-test 2026-08-07 a 64MB mid-run part came back cut mid-member; the - // watermark advanced anyway and 12 ranges lost their logs permanently, - // because their archives were complete and therefore older than the new - // watermark -- so nothing would ever fetch them again. - let root = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "ssc-tar-test") - if System.IO.Directory.Exists root then System.IO.Directory.Delete(root, true) - let src = System.IO.Path.Combine(root, "logs") - System.IO.Directory.CreateDirectory(src) |> ignore - System.IO.File.WriteAllBytes(System.IO.Path.Combine(src, "range-1-a1.log.gz"), Array.init 4096 byte) - - // Written outside the directory being archived, or the tar would contain itself. - let whole = System.IO.Path.Combine(root, "whole.tar") - System.Formats.Tar.TarFile.CreateFromDirectory(src, whole, false) - Assert.True(archiveIsIntact whole, "a complete archive must read back whole") - - // Cut the stream mid-member, which is exactly what the pod exec produced. - let cut = System.IO.Path.Combine(root, "cut.tar") - let bytes = System.IO.File.ReadAllBytes(whole) - System.IO.File.WriteAllBytes(cut, bytes.[0 .. bytes.Length / 2]) - Assert.False(archiveIsIntact cut, "a truncated archive must not pass as intact") - - System.IO.Directory.Delete(root, true) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 0d84214f..21615c95 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -132,17 +132,45 @@ let private monitorClient (context: MissionContext) = c.Timeout <- TimeSpan.FromMinutes(10.0) c -/// POST the profile and let reconcile start. Retried: the route and the pod +/// The run the monitor is asked to perform. The range travels with the profile +/// because both are per-run input -- the chart installs a generic monitor, and +/// this is what makes it a particular run. Validated as one document, so a bad +/// ledger range comes back as a 400 rather than generating no work and +/// reporting success on nothing. +let runDocument (context: MissionContext) (profileJson: string option) : string = + let endLedger = + match context.pubnetParallelCatchupEndLedger with + | Some value -> value + | None -> GetLatestPubnetLedgerNumber() + + // `run`, not `doc`: the profile ARTIFACT this mission writes is also a + // document, and the contract tests scan for its keys by name. + let rangeSpec = JObject() + rangeSpec.["startingLedger"] <- JValue(context.pubnetParallelCatchupStartingLedger) + rangeSpec.["latestLedgerNum"] <- JValue(endLedger) + rangeSpec.["ledgersPerJob"] <- JValue(context.pubnetParallelCatchupLedgersPerJob) + rangeSpec.["order"] <- JValue(context.pubnetParallelCatchupRangeOrder) + + let run = JObject() + run.["range"] <- rangeSpec + + match profileJson with + | Some body -> run.["profile"] <- JObject.Parse(body) + | None -> () + + run.ToString(Newtonsoft.Json.Formatting.None) + +/// POST the run and let reconcile start. Retried: the route and the pod /// both need a moment after `helm install`, and until this lands the monitor /// deliberately dispatches nothing. -let startMission (context: MissionContext) (profileJson: string) = +let startMission (context: MissionContext) (runJson: string) = use client = monitorClient context let deadline = DateTime.UtcNow.AddMinutes(5.0) let mutable started = false while not started && DateTime.UtcNow < deadline do try - use content = new StringContent(profileJson, Text.Encoding.UTF8, "application/json") + use content = new StringContent(runJson, Text.Encoding.UTF8, "application/json") let r = client.PostAsync("/start", content) |> Async.AwaitTask |> Async.RunSynchronously if r.IsSuccessStatusCode then LogInfo "Mission started: profile POSTed to %s/start" (monitorEndpoint context) @@ -209,7 +237,6 @@ let installProject (context: MissionContext) = setOptions.Add(sprintf "monitor.gatewayName=%s" context.gatewayName) setOptions.Add(sprintf "monitor.gatewayNamespace=%s" context.gatewayNamespace) - setOptions.Add(sprintf "range.order=%s" context.pubnetParallelCatchupRangeOrder) // Nodepool routing. Empty prefix ships the pre-tier behaviour: one label for // every worker. Set, each range goes to - where the tier comes @@ -247,16 +274,6 @@ let installProject (context: MissionContext) = sprintf "worker.tolerateNodeTaints[0]=%s" context.pubnetParallelCatchupPoolPrefix ) - setOptions.Add(sprintf "range.startingLedger=%d" context.pubnetParallelCatchupStartingLedger) - - let endLedger = - match context.pubnetParallelCatchupEndLedger with - | Some value -> value - | None -> GetLatestPubnetLedgerNumber() - - setOptions.Add(sprintf "range.latestLedgerNum=%d" endLedger) - - setOptions.Add(sprintf "range.ledgersPerJob=%d" context.pubnetParallelCatchupLedgersPerJob) // Skip known results by default setOptions.Add( @@ -425,68 +442,10 @@ let installProject (context: MissionContext) = // 1. Automatically determines worker pod names from context.pubnetParallelCatchupNumWorkers // 2. For each pod, finds all files matching "stellar-core-*.log" in /data // 3. Creates a tar.gz archive and copies it to context.destination directory -// How often the main loop fetches logs, and how much of the previous window it -// re-fetches. One pass every 10 minutes flattens the teardown cost without -// taking meaningful IOPS from the collector: --newer-mtime still stats every -// file, and at ~4000 attempts that walk is the expensive part, not the bytes. +// How often the main loop pulls logs. Every 10 minutes flattens the teardown +// cost -- which measured ~20 minutes for a full run, all of it after the work +// finished -- without the pull competing with the collector for the volume. let logFetchIntervalSecs = 600 -let logFetchOverlapSecs = 120L -// Transfers of this archive are retried in-pass before the window is deferred. -let logFetchAttempts = 3 - -// An empty GNU tar is two zero blocks at the default blocking factor. A pass -// with nothing new still writes that, so it is deleted rather than left to -// clutter the destination with 19 identical 10K files. -let emptyTarBytes = 10240L - -/// The archive written by one fetch. Parts are numbered rather than merged: -/// extracting them in order reconstructs /logs, and a later part overwrites an -/// earlier truncated copy of a file that was still being appended when it was -/// first picked up. -let logArchiveName (release: string) (part: int) : string = - sprintf "%s-worker-logs.part%02d.tar" release part - -/// tar of everything modified since `sinceEpoch`; 0 takes the whole volume, -/// which is what teardown does when no incremental pass ever landed. -/// -/// Entries are already gzipped by the streaming collector, so this bundles -/// without re-compressing. Named range--a.log.gz, so a failing -/// range is findable directly rather than by worker ordinal. Keeps the -/// per-attempt .outcome verdicts and drops .state, the collector's own resume -/// bookkeeping. -let logTarCommand (sinceEpoch: int64) : string [] = - let since = - if sinceEpoch > 0L then - sprintf " --newer-mtime=@%d" (max 0L (sinceEpoch - logFetchOverlapSecs)) - else - "" - - [| "sh" - "-c" - // lost+found is the ext4 root of the logs PVC, not ours. - sprintf "cd /logs && tar -cf -%s --exclude='*.state' --exclude='./lost+found' ." since |] - -/// Can every entry in this archive be read back? -/// -/// A pass tars /logs while workers are still appending to it, so tar can hit -/// "file changed as we read it", exit non-zero, and leave the stream cut -/// mid-member. Verified on ssc-test 2026-08-07: at 200 workers a 64MB part came -/// back truncated and 12 ranges lost their logs for good, because the watermark -/// advanced anyway and their archives -- complete, and older than the new -/// watermark -- were never re-sent. Checking here turns that into a re-transfer. -let archiveIsIntact (path: string) : bool = - try - use stream = File.OpenRead(path) - use reader = new TarReader(stream) - - let mutable entry = reader.GetNextEntry() - - while not (isNull entry) do - entry <- reader.GetNextEntry() - - true - with _ -> - false let private monitorPodName (context: MissionContext) : string option = context @@ -499,117 +458,72 @@ let private monitorPodName (context: MissionContext) : string option = |> Seq.map (fun p -> p.Metadata.Name) |> Seq.tryHead -/// Fetch everything written since `sinceEpoch` into numbered part `part`. -/// -/// Worker pods are per-range and are reaped within about a minute of finishing, -/// so there is nothing left to exec into at teardown. The monitor pulls each -/// pod's log while it is still alive onto its own volume, so one exec here -/// replaces the ~1024 the StatefulSet design needed. +/// Pull every artifact the destination does not already hold. /// -/// Returns the watermark for the next call, or None when nothing was fetched -- -/// the caller then keeps its old watermark and re-fetches that window next time, -/// so a failed pass costs bandwidth rather than logs. -let collectLogsSince (context: MissionContext) (sinceEpoch: int64) (part: int) : int64 option = - match monitorPodName context with - | None -> - LogWarn "No job-monitor pod found for release %s; worker logs cannot be collected" helmReleaseName - None - | Some podName -> - try - // The clock is read from the POD, and BEFORE the tar. This driver - // runs outside the cluster, so a few seconds of NTP skew either way - // would silently skip a file forever; and a watermark taken after - // the tar would exclude anything written while it walked the tree. - // Both failure modes lose logs; taking it early only re-sends. - let stampFile = - Path.Combine(Path.GetTempPath(), sprintf "%s-logstamp" helmReleaseName) - - RemoteCommandRunner.RunRemoteCommandAndCaptureOutput( - kube = context.kube, - ns = context.namespaceProperty, - podName = podName, - containerName = "job-monitor", - command = [| "date"; "+%s" |], - outputFilePath = stampFile - ) - - let parsed, podEpoch = Int64.TryParse(File.ReadAllText(stampFile).Trim()) +/// Per file, over the monitor's HTTPRoute. The tar-over-exec this replaces put +/// every byte through the API server -- ~0.3 MB per range, so ~1.2 GB on a +/// 4000-range run -- and streamed the whole volume as one archive, which at 200 +/// workers came back truncated 2 times in 3 with no error surfaced. A file is +/// its own unit here: a cut transfer resumes from the byte it reached, and one +/// bad fetch costs one file rather than the pass. +let collectLogs (context: MissionContext) (destination: string) = + Directory.CreateDirectory(destination) |> ignore + use client = monitorClient context - if not parsed then - LogWarn "Could not read the clock from %s; skipping this log pass" podName - None + let manifest = + client.GetStringAsync("/logs") |> Async.AwaitTask |> Async.RunSynchronously + |> JArray.Parse + + let fetchOne (entry: JToken) = + let name = entry.["name"].ToString() + let size = entry.["size"].Value() + let path = Path.Combine(destination, name) + let have = if File.Exists path then FileInfo(path).Length else 0L + + // Already whole. The collector only ever appends, so equal length means + // equal content -- and this is what stops a pass re-sending what the + // last one already took (the tar overlap re-sent 58% of files). + if have = size then + 0L + else + let req = new HttpRequestMessage(HttpMethod.Get, "/logs/" + name) + if have > 0L && have < size then + req.Headers.Range <- Headers.RangeHeaderValue(Nullable(have), Nullable()) + + use resp = client.SendAsync(req) |> Async.AwaitTask |> Async.RunSynchronously + resp.EnsureSuccessStatusCode() |> ignore + let bytes = resp.Content.ReadAsByteArrayAsync() |> Async.AwaitTask |> Async.RunSynchronously + + // Append on a partial answer, replace on a whole one: a server that + // ignored Range would otherwise double the file. + if resp.StatusCode = Net.HttpStatusCode.PartialContent then + use fs = new FileStream(path, FileMode.Append, FileAccess.Write) + fs.Write(bytes, 0, bytes.Length) else - let outputFile = - Path.Combine(context.destination.Path, logArchiveName helmReleaseName part) - - // The exec stream can end early on a large transfer and report - // success anyway -- the pod's tar exits 0, the status channel - // says Success, and the bytes simply stop. Measured on ssc-test - // 2026-08-07: a 64MB part arrived cut mid-member. So fetch, then - // read the archive back, and retry the whole transfer before - // giving up on this window. - let mutable attemptsLeft = logFetchAttempts - let mutable intact = false - - while attemptsLeft > 0 && not intact do - attemptsLeft <- attemptsLeft - 1 - - RemoteCommandRunner.RunRemoteCommandAndCaptureOutput( - kube = context.kube, - ns = context.namespaceProperty, - podName = podName, - containerName = "job-monitor", - command = logTarCommand sinceEpoch, - outputFilePath = outputFile - ) - - let fi = FileInfo(outputFile) - intact <- fi.Exists && (fi.Length <= emptyTarBytes || archiveIsIntact outputFile) - - if not intact && attemptsLeft > 0 then - LogWarn "Worker log archive came back truncated; refetching (%d attempt(s) left)" attemptsLeft - - let fileInfo = FileInfo(outputFile) - - if not fileInfo.Exists then - LogWarn "Worker log archive was not written: %s" outputFile - None - elif fileInfo.Length <= emptyTarBytes then - File.Delete(outputFile) - LogInfo "No new worker logs since the last pass" - Some podEpoch - elif not (archiveIsIntact outputFile) then - // Keep the part -- it holds real entries, and a later - // complete pass over the same window supersedes it on - // extract. But hold the watermark, so that window IS - // re-fetched rather than silently skipped. - LogWarn - "Worker log archive %s is truncated (%d bytes); keeping the old watermark so this window is fetched again" - outputFile - fileInfo.Length - - None - else - LogInfo "Collected worker logs to %s (size: %d bytes)" outputFile fileInfo.Length - Some podEpoch - with ex -> - LogWarn "Failed to collect worker logs: %s" ex.Message - None - -// Watermark and part counter for the incremental fetch. Module-level because -// the main loop and the cleanup path both advance them. -let mutable private logWatermark = 0L -let mutable private logPartCount = 0 - -/// One log pass. Advances the watermark only when the bytes are safely local. + File.WriteAllBytes(path, bytes) + + int64 bytes.Length + + // Bounded: the monitor serves these from the same pod that runs reconcile. + let fetched = + manifest + |> Seq.toArray + |> Array.map (fun e -> async { return (try fetchOne e with ex -> + LogWarn "log fetch failed for %s: %s" + (e.["name"].ToString()) ex.Message + 0L) }) + |> fun work -> Async.Parallel(work, 8) + |> Async.RunSynchronously + + let moved = Array.sum fetched + let touched = fetched |> Array.filter (fun n -> n > 0L) |> Array.length + LogInfo "Collected %d of %d artifacts (%d bytes) from %s" + touched (Seq.length manifest) moved (monitorEndpoint context) + +/// One log pass. Idempotent -- the manifest comparison is what makes a repeat +/// pass cheap, so there is no watermark to keep. let collectLogsFromPods (context: MissionContext) = - let part = logPartCount + 1 - - match collectLogsSince context logWatermark part with - | Some epoch -> - logWatermark <- epoch - logPartCount <- part - | None -> () + collectLogs context context.destination.Path // Cleanup on exit. `signalTriggered` indicates we're running under a hard // deadline (Jenkins' SoftKillWaitSeconds, ~5s by default, before SIGKILL). @@ -955,7 +869,7 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = // The monitor dispatches nothing until this arrives, so a profile that // cannot be delivered fails the run here rather than silently sizing every // range as unprofiled. - startMission context (defaultArg (resolveRangeProfile context) "{}") + startMission context (runDocument context (resolveRangeProfile context)) let mutable allJobsFinished = false let mutable timeoutLeft = jobMonitorStatusCheckTimeOutSecs diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py index 06dc783c..9ca64404 100644 --- a/src/MissionParallelCatchup/apps/job_monitor.py +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -61,16 +61,15 @@ def main(): # The driver POSTs the profile to /start. Kept on the volume so a restarted # monitor resumes a run already under way instead of waiting for a /start # that was delivered to its predecessor. - config.PROFILE_PATH = os.path.join(config.LOG_DIR, 'profile.json') - if os.path.exists(config.PROFILE_PATH): - config.PROFILE = profiles.load_profile() - # The gate opens without a /start on this path, so this is where the - # config gets coerced and checked instead. - validate_config() - http_server.started.set() + config.RUN_PATH = os.path.join(config.LOG_DIR, 'run.json') + if os.path.exists(config.RUN_PATH): + # Same path as a /start, so the range and the profile are both restored + # and validated exactly as they were. + with open(config.RUN_PATH) as fh: + start_run(json.load(fh)) http_server.status_source = lambda: (status, status_lock) - http_server.on_start = install_profile + http_server.on_start = start_run # This is the reconcile loop -- # dispatch, progress record, metrics, status. @@ -80,13 +79,22 @@ def main(): http_server.serve() -def install_profile(doc): - """Land the driver's profile on the volume and size from it. +def start_run(doc): + """Install the run the driver POSTed: the ledger range, and the profile. - Written before the gate opens, so the first Job dispatched is already sized - by it -- an unprofiled first wave would route every range to protostar. + Both are per-run input, so neither is env-derived any more -- the chart + installs a generic monitor and this defines what it runs. Written to the + volume before the gate opens, so the first Job dispatched is already sized + by the profile and a restart resumes the same run. """ - profile = profiles.load_profile_doc(doc) + for key, name in (('generator', 'RANGE_GENERATOR'), ('order', 'RANGE_ORDER'), + ('startingLedger', 'STARTING_LEDGER'), + ('latestLedgerNum', 'LATEST_LEDGER_NUM'), + ('ledgersPerJob', 'LEDGERS_PER_JOB'), + ('overlapLedgers', 'OVERLAP_LEDGERS')): + if key in (doc.get('range') or {}): + setattr(config, name, (doc['range'])[key]) + profile = profiles.load_profile_doc(doc.get('profile') or {}) # The whole config, judged at the first moment it is complete. Anything # wrong rejects the POST with the reason rather than dispatching a run that # is already misconfigured. @@ -96,7 +104,7 @@ def install_profile(doc): "RANGE_ORDER=longest-first requires a profile: it orders ranges by " "their measured seconds, and with no profile every range ties and " "dispatch stays tip-first. POST a profile, or set RANGE_ORDER.") - records.write_atomic(config.PROFILE_PATH, json.dumps(doc, separators=(',', ':'))) + records.write_atomic(config.RUN_PATH, json.dumps(doc, separators=(',', ':'))) config.PROFILE = profile logger.info("profile installed: %d ranges", len(config.PROFILE)) @@ -135,6 +143,19 @@ def validate_config(): if config.RANGE_ORDER not in config.VALID_RANGE_ORDERS: raise ValueError("RANGE_ORDER must be one of %s, got %r" % (', '.join(config.VALID_RANGE_ORDERS), config.RANGE_ORDER)) + # The ledger range, which nothing checked while it came from helm values -- + # an inverted or zero-width range generates no work and the run just ends, + # reporting success on nothing. + if config.LEDGERS_PER_JOB <= 0: + raise ValueError("ledgersPerJob must be greater than zero, got %r" + % (config.LEDGERS_PER_JOB,)) + if config.OVERLAP_LEDGERS < 0: + raise ValueError("overlapLedgers cannot be negative, got %r" + % (config.OVERLAP_LEDGERS,)) + if config.LATEST_LEDGER_NUM <= config.STARTING_LEDGER: + raise ValueError( + "latestLedgerNum must be greater than startingLedger, got %r and %r" + % (config.LATEST_LEDGER_NUM, config.STARTING_LEDGER)) status = { 'num_remain': 1, # non-zero until the first real update, so callers don't see a premature 0 diff --git a/src/MissionParallelCatchup/lib/config.py b/src/MissionParallelCatchup/lib/config.py index 7a3d002e..88c685cd 100644 --- a/src/MissionParallelCatchup/lib/config.py +++ b/src/MissionParallelCatchup/lib/config.py @@ -23,7 +23,7 @@ # Which ledger ranges to run. These are pure inputs to the range generator: # dispatch recomputes the whole list every reconcile, so a restart must # reproduce it exactly. -RANGE_GENERATOR = os.getenv('RANGE_GENERATOR', 'uniform') # uniform | logarithmic +RANGE_GENERATOR = 'uniform' # from /start: uniform | logarithmic VALID_RANGE_GENERATORS = ('uniform', 'logarithmic') @@ -31,17 +31,17 @@ # the bucket set only grows with ledger position. 'oldest-first' reverses that, # so a profiling run measures the cheap early ranges before it can be # interrupted, and the expensive tip ranges last. -RANGE_ORDER = os.getenv('RANGE_ORDER', 'tip-first') # tip-first | oldest-first | longest-first +RANGE_ORDER = 'tip-first' # from /start: tip-first | oldest-first | longest-first VALID_RANGE_ORDERS = ('tip-first', 'oldest-first', 'longest-first') -STARTING_LEDGER = int(os.getenv('STARTING_LEDGER', 0)) +STARTING_LEDGER = 0 # from /start -LATEST_LEDGER_NUM = int(os.getenv('LATEST_LEDGER_NUM', 0)) +LATEST_LEDGER_NUM = 0 # from /start -LEDGERS_PER_JOB = int(os.getenv('LEDGERS_PER_JOB', 16000)) +LEDGERS_PER_JOB = 16000 # from /start -OVERLAP_LEDGERS = int(os.getenv('OVERLAP_LEDGERS', 320)) +OVERLAP_LEDGERS = 320 # from /start # logarithmic only: chunk size halves toward the tip and stops shrinking here. LOGARITHMIC_FLOOR_LEDGERS = int(os.getenv('LOGARITHMIC_FLOOR_LEDGERS', 64000)) @@ -92,6 +92,8 @@ # workers fit per node. Requests only -- limits stay as configured, so the # failure semantics and the OOM/disk escalation ladders are unchanged. PROFILE_PATH = os.getenv('PROFILE_PATH', '') +# The run document /start delivers, kept so a restart resumes the same run. +RUN_PATH = '' PROFILE_MARGIN = float(os.getenv('PROFILE_MARGIN', 1.15)) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index fe4609b6..877406bc 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -205,18 +205,6 @@ spec: value: {{ .Values.monitor.livenessSweepSeconds | quote }} - name: LIVENESS_MAX_CONCURRENCY value: {{ .Values.monitor.livenessMaxConcurrency | quote }} - - name: RANGE_GENERATOR - value: {{ .Values.range.generator | quote }} - - name: STARTING_LEDGER - value: {{ .Values.range.startingLedger | quote }} - - name: LATEST_LEDGER_NUM - value: {{ .Values.range.latestLedgerNum | quote }} - - name: LEDGERS_PER_JOB - value: {{ .Values.range.ledgersPerJob | quote }} - - name: OVERLAP_LEDGERS - value: {{ .Values.range.overlapLedgers | quote }} - - name: RANGE_ORDER - value: {{ .Values.range.order | quote }} - name: LOGARITHMIC_FLOOR_LEDGERS value: {{ .Values.range.logarithmicFloorLedgers | quote }} - name: PARALLELISM diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 08feed98..150cc7ff 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -57,19 +57,17 @@ worker: "curl -sf http://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}" # Index -> ledger range is computed in the Job template; no queue to preload. +# The range itself -- start, end, size, order -- is per-run input and arrives +# with the driver's POST /start, so the chart installs a generic monitor. range: # uniform: equal ledger counts. logarithmic: big chunks over cheap early # history, halving toward the tip, aiming for equal wall-time per job. generator: "uniform" logarithmicFloorLedgers: 64000 - startingLedger: 0 - latestLedgerNum: 100000 overlapLedgers: 320 # Dispatch order. Generators emit tip-first, which front-loads the most # expensive ranges. "oldest-first" reverses that so a profiling run # measures the cheap early ranges before it can be interrupted. - order: tip-first - ledgersPerJob: 16000 monitor: # Reuses the existing hand-built job-monitor image slot -- no new image and no diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py index ed288129..2f1d9c7a 100644 --- a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py +++ b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py @@ -67,7 +67,6 @@ def _code_defaults(cname): 'MISSION': 'the mission name, for the kube-state-metrics label', 'PROFILE_PATH': 'the mounted path of an optional profile ConfigMap', 'ASAN_OPTIONS': 'passed through to the worker; empty means "unset", not "default"', - 'LATEST_LEDGER_NUM': 'a demo value in the chart; the mission always sets the real tip', 'PARALLELISM': 'worker.replicas -- the whole point of the knob is to differ per run', 'ATTEMPT_DEADLINE_SECONDS': 'a backstop the chart turns on and the code leaves off', # StellarKubeSpecs.fs owns worker sizing, so the chart ships these empty on diff --git a/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py b/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py index da8d0aa6..791c57cc 100644 --- a/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py +++ b/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py @@ -64,7 +64,9 @@ def test_the_driver_really_does_configure_the_chart(): """Guards the extraction: a regex that stopped matching would pass silently.""" keys = set_keys() assert len(keys) >= 15, f"only found {sorted(keys)}; the --set scan has gone blind" - assert 'worker.stellar_core_image' in keys and 'range.ledgersPerJob' in keys + # Sentinels that must keep flowing through --set. The ledger range moved to + # POST /start, so it is deliberately not one of them any more. + assert 'worker.stellar_core_image' in keys and 'worker.replicas' in keys def test_every_helm_command_uses_the_mission_namespace(): @@ -296,36 +298,8 @@ def test_the_driver_reads_the_progress_file_where_the_monitor_writes_it(): f"the driver cats {path}; the monitor writes {config.PROGRESS_FILE}") -def test_the_driver_tars_the_directory_the_collector_writes_into(): - """One exec replaces the ~1024 the StatefulSet design needed.""" - cd = fs_extract(r'"cd (/\w+) && tar').group(1) - assert cd == config.LOG_DIR == config.LOG_DIR -def test_the_tar_excludes_only_the_collectors_resume_bookkeeping(): - """.state is a resume cursor and is worthless outside the pod. - - Every other suffix on that volume is a deliverable: the archive, the - per-attempt metrics, the verdict. An exclusion pattern that drifted onto one - of those would quietly shrink the collected tar. - """ - def suffix_of(path_fn): - return os.path.basename(path_fn('E', 1)).partition('-a1')[2] - - bookkeeping = {suffix_of(records.state_path)} - deliverables = {suffix_of(f) for f in (records.log_path, records.metrics_path, - records.outcome_path, records.done_path)} - assert bookkeeping.isdisjoint(deliverables) - - excludes = set(re.findall(r"--exclude='([^']+)'", FS)) - assert excludes, "the tar no longer excludes anything -- update this test" - for pattern in excludes: - if not pattern.startswith('*'): - continue # ./lost+found, the PVC's ext4 root - assert pattern[1:] in bookkeeping, ( - f"the tar excludes {pattern}, which is not resume bookkeeping") - for suffix in deliverables: - assert f"*{suffix}" not in excludes, f"the tar drops {suffix}, a deliverable" def test_the_driver_finds_the_monitor_pod_by_the_labels_the_chart_sets(): @@ -504,3 +478,19 @@ def test_every_helm_and_kubectl_call_is_namespaced(): assert calls, "no helm/kubectl shell calls found -- did the driver change shape?" missing = [c.split('\n')[0] for c in calls if '"--namespace"' not in c] assert not missing, f"shell calls without --namespace: {missing}" + + +def test_the_driver_pulls_from_the_directory_the_collector_writes_into(): + """The puller and the collector must agree on where artifacts live. + + Replaces the two tar-shape tests: there is no archive any more, so what + matters is that the monitor serves LOG_DIR and the driver asks for the + manifest of it. A mismatch would fetch an empty list and report success + on nothing collected. + """ + fs = open(art.FSHARP_PATH).read() + assert '"/logs"' in fs, "the driver no longer requests the manifest" + assert '"/logs/" + name' in fs, "the driver no longer fetches artifacts by name" + # The monitor serves them out of the volume the collector writes to. + import http_server + assert 'config.LOG_DIR' in art.module_source(http_server) diff --git a/src/MissionParallelCatchup/tests/unit/test_http_surface.py b/src/MissionParallelCatchup/tests/unit/test_http_surface.py index b077d6f2..e6415aa7 100644 --- a/src/MissionParallelCatchup/tests/unit/test_http_surface.py +++ b/src/MissionParallelCatchup/tests/unit/test_http_surface.py @@ -22,9 +22,9 @@ def server(tmp_path, monkeypatch): """A live monitor HTTP surface on a throwaway port and volume.""" monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(config, 'PROFILE_PATH', str(tmp_path / 'profile.json')) + monkeypatch.setattr(config, 'RUN_PATH', str(tmp_path / 'run.json')) monkeypatch.setattr(http_server, 'started', threading.Event()) - monkeypatch.setattr(http_server, 'on_start', jm.install_profile) + monkeypatch.setattr(http_server, 'on_start', jm.start_run) monkeypatch.setattr(http_server, 'status_source', lambda: (jm.status, jm.status_lock)) @@ -58,7 +58,7 @@ def test_start_rejects_a_bad_config_with_the_reason(server, monkeypatch): base, _ = server monkeypatch.setattr(config, 'LIVENESS_MAX_CONCURRENCY', 'many') - code, body = _post(base, '/start', '{}') + code, body = _post(base, '/start', json.dumps({"range": {"startingLedger": 0, "latestLedgerNum": 1000, "ledgersPerJob": 100}})) assert code == 400 assert 'LIVENESS_MAX_CONCURRENCY must be an integer' in json.loads(body)['error'] @@ -69,12 +69,12 @@ def test_start_opens_the_gate_and_is_idempotent(server): """A driver that retries after a timeout must not restart a live run.""" base, vol = server - assert _post(base, '/start', json.dumps({'ranges': {'300': {'seconds': 1.0}}}))[0] == 200 + assert _post(base, '/start', json.dumps({"range": {"startingLedger": 0, "latestLedgerNum": 1000, "ledgersPerJob": 100}, "profile": {"ranges": {"300": {"seconds": 1.0}}}}))[0] == 200 assert http_server.started.is_set() - assert (vol / 'profile.json').exists(), "the profile is kept for a restart" + assert (vol / 'run.json').exists(), "the profile is kept for a restart" # A second POST carrying nothing must not wipe the profile already installed. - assert _post(base, '/start', '{}')[0] == 200 + assert _post(base, '/start', json.dumps({"range": {"startingLedger": 0, "latestLedgerNum": 1000, "ledgersPerJob": 100}}))[0] == 200 assert config.PROFILE == [(300, {'seconds': 1.0})] diff --git a/src/MissionParallelCatchup/tests/unit/test_range_generation.py b/src/MissionParallelCatchup/tests/unit/test_range_generation.py index 54d0898f..24d63838 100644 --- a/src/MissionParallelCatchup/tests/unit/test_range_generation.py +++ b/src/MissionParallelCatchup/tests/unit/test_range_generation.py @@ -157,13 +157,13 @@ def test_preflight_rejects_longest_first_without_a_profile(monkeypatch, tmp_path dispatching a run whose ordering silently degrades to tip-first.""" monkeypatch.setattr(config, 'RANGE_ORDER', 'longest-first') monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(config, 'PROFILE_PATH', str(tmp_path / 'profile.json')) + monkeypatch.setattr(config, 'RUN_PATH', str(tmp_path / 'run.json')) with pytest.raises(ValueError, match='longest-first requires a profile'): - jm.install_profile({}) + jm.start_run({"range": {'startingLedger': 0, 'latestLedgerNum': 1000, 'ledgersPerJob': 100}}) # A profile with ranges is accepted, and nothing is written until it passes. - jm.install_profile({'ranges': {'300': {'seconds': 1.0}}}) + jm.start_run({'range': {'startingLedger': 0, 'latestLedgerNum': 1000, 'ledgersPerJob': 100}, 'profile': {'ranges': {'300': {'seconds': 1.0}}}}) assert config.PROFILE == [(300, {'seconds': 1.0})] @@ -176,14 +176,14 @@ def test_the_preflight_runs_before_anything_is_dispatched(monkeypatch, tmp_path) whole. It still has to bind before dispatch: a run that is misconfigured must be refused, not started and then discovered.""" monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(config, 'PROFILE_PATH', str(tmp_path / 'profile.json')) + monkeypatch.setattr(config, 'RUN_PATH', str(tmp_path / 'run.json')) monkeypatch.setattr(config, 'RANGE_GENERATOR', 'nonsense') with pytest.raises(ValueError, match='RANGE_GENERATOR must be one of'): - jm.install_profile({}) + jm.start_run({"range": {'startingLedger': 0, 'latestLedgerNum': 1000, 'ledgersPerJob': 100}}) # Rejected, so nothing was written and no run can proceed from it. - assert not os.path.exists(config.PROFILE_PATH) + assert not os.path.exists(config.RUN_PATH) def jm_source(): diff --git a/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py b/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py index 782bfd21..6b8f599f 100644 --- a/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py +++ b/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py @@ -200,21 +200,21 @@ def test_malformed_liveness_configuration_fails_with_an_explicit_message(monkeyp 600s later with "not reachable". Now it comes back as a 400 with the reason. """ monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(config, 'PROFILE_PATH', str(tmp_path / 'profile.json')) + monkeypatch.setattr(config, 'RUN_PATH', str(tmp_path / 'run.json')) monkeypatch.setattr(config, 'LIVENESS_MAX_CONCURRENCY', 'many') with pytest.raises(ValueError, match='LIVENESS_MAX_CONCURRENCY must be an integer'): - jm.install_profile({}) + jm.start_run({"range": {'startingLedger': 0, 'latestLedgerNum': 1000, 'ledgersPerJob': 100}}) def test_liveness_numbers_are_coerced_once_validation_passes(monkeypatch, tmp_path): """Callers must never see the string form; validate_config rebinds them.""" monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(config, 'PROFILE_PATH', str(tmp_path / 'profile.json')) + monkeypatch.setattr(config, 'RUN_PATH', str(tmp_path / 'run.json')) monkeypatch.setattr(config, 'LIVENESS_MAX_CONCURRENCY', '8') monkeypatch.setattr(config, 'LIVENESS_SWEEP_SECONDS', '2.5') - jm.install_profile({}) + jm.start_run({"range": {'startingLedger': 0, 'latestLedgerNum': 1000, 'ledgersPerJob': 100}}) assert config.LIVENESS_MAX_CONCURRENCY == 8 assert config.LIVENESS_SWEEP_SECONDS == 2.5 From 40434b1e4f34eef984b623253bdca81330a4ab43 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Sat, 8 Aug 2026 17:46:14 -0400 Subject: [PATCH 071/117] Fix two faults the first HTTP run surfaced Both found by running it, neither reachable from the unit tests. A zero-byte artifact was never collected. The puller skipped a file whose local length already equalled the manifest length, but an absent file has length 0, so "absent" and "already have it" were indistinguishable for anything empty. .done markers are empty by design -- their existence IS the signal -- so every one was skipped: 88 of 110 artifacts collected, the 22 missing being exactly the markers that say an attempt finished. /start retried only once. The retry deadline was 5 minutes but the HttpClient timeout was 10, so a single request hung on a route that was still programming consumed the whole window and the mission failed having tried once -- against a route that answered in 37ms moments later. Each attempt is now bounded well below the window it retries inside. Verified on ssc-test: /start connects immediately after install, and the .done markers arrive. Both are pinned by contract tests. Co-Authored-By: Claude Opus 5 --- .../MissionHistoryPubnetParallelCatchupV2.fs | 34 ++++++++++++++----- .../parallel_catchup_helm/values.yaml | 2 +- .../contract/test_fsharp_driver_contract.py | 27 +++++++++++++++ 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 21615c95..f6e22f23 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -126,12 +126,22 @@ let monitorEndpoint (context: MissionContext) = | Some h -> h | None -> monitorRouteHost context -let private monitorClient (context: MissionContext) = +// The per-request timeout has to be shorter than any retry window built on top +// of it, or one hung request eats the whole window and the retry never happens. +// Observed 2026-08-08: a /start attempt hung on a route that was still +// programming, the 10-minute client timeout outlived the 5-minute deadline, and +// the mission failed after exactly one attempt. +let private monitorClientWith (context: MissionContext) (timeout: TimeSpan) = let c = new HttpClient(BaseAddress = Uri(sprintf "http://%s" (monitorEndpoint context))) c.DefaultRequestHeaders.Host <- monitorRouteHost context - c.Timeout <- TimeSpan.FromMinutes(10.0) + c.Timeout <- timeout c +// Log pulls move whole files, so they get room; everything else is a short +// request that should fail fast and be retried. +let private monitorClient (context: MissionContext) = + monitorClientWith context (TimeSpan.FromMinutes(10.0)) + /// The run the monitor is asked to perform. The range travels with the profile /// because both are per-run input -- the chart installs a generic monitor, and /// this is what makes it a particular run. Validated as one document, so a bad @@ -164,7 +174,7 @@ let runDocument (context: MissionContext) (profileJson: string option) : string /// both need a moment after `helm install`, and until this lands the monitor /// deliberately dispatches nothing. let startMission (context: MissionContext) (runJson: string) = - use client = monitorClient context + use client = monitorClientWith context (TimeSpan.FromSeconds(15.0)) let deadline = DateTime.UtcNow.AddMinutes(5.0) let mutable started = false @@ -483,8 +493,13 @@ let collectLogs (context: MissionContext) (destination: string) = // Already whole. The collector only ever appends, so equal length means // equal content -- and this is what stops a pass re-sending what the // last one already took (the tar overlap re-sent 58% of files). - if have = size then - 0L + // + // File.Exists is not redundant: a .done marker is zero bytes, so a + // length comparison alone reads "absent locally" as "already have it" + // and never fetches it. Measured 2026-08-08: 88 of 110 artifacts + // collected, and every .done was among the 22 missing. + if File.Exists path && have = size then + -1L // already whole; distinct from a zero-byte file we did fetch else let req = new HttpRequestMessage(HttpMethod.Get, "/logs/" + name) if have > 0L && have < size then @@ -511,12 +526,15 @@ let collectLogs (context: MissionContext) (destination: string) = |> Array.map (fun e -> async { return (try fetchOne e with ex -> LogWarn "log fetch failed for %s: %s" (e.["name"].ToString()) ex.Message - 0L) }) + -1L) }) |> fun work -> Async.Parallel(work, 8) |> Async.RunSynchronously - let moved = Array.sum fetched - let touched = fetched |> Array.filter (fun n -> n > 0L) |> Array.length + let moved = fetched |> Array.filter (fun n -> n > 0L) |> Array.sum + // >= 0 counts a zero-byte artifact we really did fetch. .done markers are + // empty by design, so "bytes > 0" undercounts exactly the files whose + // existence IS the signal. + let touched = fetched |> Array.filter (fun n -> n >= 0L) |> Array.length LogInfo "Collected %d of %d artifacts (%d bytes) from %s" touched (Seq.length manifest) moved (monitorEndpoint context) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 150cc7ff..78536141 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -78,7 +78,7 @@ monitor: # this chart against it ships env vars the image cannot read. Built from this # branch and pinned by tag rather than :latest so a run is always traceable to # one image. Revert to the stellar/ repo once there is a push path for it. - image: "stellajuna/ssc-jm:2026-08-07b" + image: "stellajuna/ssc-jm:2026-08-08a" # The driver reaches the monitor through this route: profile in via POST # /start, status and logs out. Empty routeHost disables the HTTPRoute; the # Service still exists, so an in-cluster caller works either way. diff --git a/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py b/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py index 791c57cc..18bc423f 100644 --- a/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py +++ b/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py @@ -494,3 +494,30 @@ def test_the_driver_pulls_from_the_directory_the_collector_writes_into(): # The monitor serves them out of the volume the collector writes to. import http_server assert 'config.LOG_DIR' in art.module_source(http_server) + + +def test_the_puller_does_not_mistake_an_absent_file_for_a_complete_one(): + """A zero-byte artifact must still be fetched. + + .done is empty by design -- its existence is the signal. Comparing lengths + alone makes "absent locally" indistinguishable from "already have it", so + every .done is skipped forever: 88 of 110 artifacts on 2026-08-08, with the + 22 missing being exactly the markers that say an attempt finished. + """ + fs = open(art.FSHARP_PATH).read() + assert 'File.Exists path && have = size' in fs, ( + "the puller compares lengths without checking the file exists, so " + "zero-byte artifacts are never collected") + + +def test_a_retry_window_outlives_the_request_it_retries(): + """A per-request timeout longer than the retry deadline is one attempt. + + Observed 2026-08-08: /start hung on a route that was still programming, the + 10-minute client timeout outlived the 5-minute deadline, and the mission + failed having tried exactly once -- on a route that answered in 37ms a + moment later. + """ + fs = open(art.FSHARP_PATH).read() + assert 'monitorClientWith context (TimeSpan.FromSeconds(15.0))' in fs, ( + "startMission no longer bounds each attempt below its retry deadline") From f6b5e3e461fe96f785e3232310970c360ef6d741 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Sat, 8 Aug 2026 18:50:43 -0400 Subject: [PATCH 072/117] Keep the collector's resume cursor out of the manifest .state is one RFC3339 timestamp, rewritten on every poll of a live range, and read only by the collector itself to resume a log stream after a restart. Once the pods are gone it means nothing. Offering it for pulling costs more than its size suggests: because it changes constantly, a manifest diff sees it grown on every pass and re-fetches one per in-flight range -- up to 1024 round trips through a concurrency-8 pool, every ten minutes, for bytes that are garbage by teardown. The tar this replaced excluded it deliberately; the exclusion was lost when the manifest took over, rather than reconsidered. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/lib/http_server.py | 14 +++++++++++--- .../parallel_catchup_helm/values.yaml | 2 +- .../tests/unit/test_http_surface.py | 17 +++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/MissionParallelCatchup/lib/http_server.py b/src/MissionParallelCatchup/lib/http_server.py index 94add61e..26093c4a 100644 --- a/src/MissionParallelCatchup/lib/http_server.py +++ b/src/MissionParallelCatchup/lib/http_server.py @@ -98,12 +98,20 @@ def do_POST(self): self._send(200, b'{"started":true}') def _manifest(self): - """Every artifact on the volume, with the size and mtime a puller needs - to tell "already have it" from "grew since last time".""" + """Every artifact worth pulling, with the size and mtime a puller needs + to tell "already have it" from "grew since last time". + + .state is excluded: it is the collector's resume cursor, one timestamp + rewritten on every poll of a live range. It is meaningless once the pods + are gone, and because it changes constantly a manifest diff would + re-fetch one per in-flight range on every pass -- up to 1024 round trips + for bytes that are garbage by the time the run ends. + """ out = [] for name in os.listdir(config.LOG_DIR): path = os.path.join(config.LOG_DIR, name) - if _SAFE_NAME.match(name) and os.path.isfile(path): + if (_SAFE_NAME.match(name) and not name.endswith('.state') + and os.path.isfile(path)): st = os.stat(path) out.append({'name': name, 'size': st.st_size, 'mtime': int(st.st_mtime)}) return out diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 78536141..5ccb4263 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -78,7 +78,7 @@ monitor: # this chart against it ships env vars the image cannot read. Built from this # branch and pinned by tag rather than :latest so a run is always traceable to # one image. Revert to the stellar/ repo once there is a push path for it. - image: "stellajuna/ssc-jm:2026-08-08a" + image: "stellajuna/ssc-jm:2026-08-08b" # The driver reaches the monitor through this route: profile in via POST # /start, status and logs out. Empty routeHost disables the HTTPRoute; the # Service still exists, so an in-cluster caller works either way. diff --git a/src/MissionParallelCatchup/tests/unit/test_http_surface.py b/src/MissionParallelCatchup/tests/unit/test_http_surface.py index e6415aa7..6d382eee 100644 --- a/src/MissionParallelCatchup/tests/unit/test_http_surface.py +++ b/src/MissionParallelCatchup/tests/unit/test_http_surface.py @@ -120,3 +120,20 @@ def test_a_path_outside_the_volume_is_refused(server): with pytest.raises(urllib.error.HTTPError) as e: _get(base, '/logs/' + bad) assert e.value.code == 404 + + +def test_the_collectors_resume_cursor_is_not_offered_for_pulling(server): + """.state is one timestamp rewritten on every poll of a live range. + + It means nothing once the pods are gone, and it changes constantly -- so a + manifest diff would re-fetch one per in-flight range on every pass. The tar + it replaced excluded it deliberately; this keeps that. + """ + base, vol = server + (vol / 'range-300-a1.log.gz').write_bytes(b'kept') + (vol / 'range-300-a1.state').write_text('2026-08-08T21:44:01.867115384Z') + + names = {e['name'] for e in json.loads(_get(base, '/logs')[1])} + + assert 'range-300-a1.log.gz' in names + assert 'range-300-a1.state' not in names From 63056dc72e0166f4ce67a94704ef151bf0bfe8c7 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Sat, 8 Aug 2026 20:01:10 -0400 Subject: [PATCH 073/117] Drop the ConfigMap plumbing the HTTP surface replaced profileConfigMap and its mount are gone: the profile arrives with POST /start and is written to the volume, so a chart-supplied PROFILE_PATH would point at a mount that no longer exists. The PROFILE_* sizing values it used to gate are now unconditional, which is what they should always have been -- they configure the arithmetic, not the delivery. RBAC drops create and patch on configmaps. The monitor owns none: progress lives on the volume and status is served from /status. It still reads one -- the chart's stellar-core-config, whose uid is the ownerReference on every Job and PVC the run creates -- so get and list stay. Co-Authored-By: Claude Opus 5 --- .../templates/job_monitor.yaml | 23 ++++-------------- .../parallel_catchup_helm/values.yaml | 5 ---- .../tests/contract/test_chart_defaults.py | 24 ++++++++----------- 3 files changed, 15 insertions(+), 37 deletions(-) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 877406bc..ed2683fc 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -62,12 +62,13 @@ rules: - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "create", "delete"] - # create/patch is for the progress ConfigMap, which is the durable record of - # completed ranges. Jobs are reclaimed during a long run, so their absence - # must not be read as "never ran". + # Read-only. The monitor owns no ConfigMap: progress lives on the volume and + # status is served from /status. This is only to read the chart's + # stellar-core-config, whose uid becomes the ownerReference on every Job and + # PVC the run creates. - apiGroups: [""] resources: ["configmaps"] - verbs: ["get", "list", "create", "patch"] + verbs: ["get", "list"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -286,9 +287,6 @@ spec: # NodePool property; Karpenter labels every node with it itself. - name: CAPACITY_TYPE value: {{ .Values.monitor.capacityType | quote }} - {{- if .Values.monitor.profileConfigMap }} - - name: PROFILE_PATH - value: /profile/profile.json - name: PROFILE_MARGIN value: {{ .Values.monitor.profileMargin | quote }} - name: PROFILE_MAX_MEM @@ -306,7 +304,6 @@ spec: value: {{ .Values.monitor.profileMaxEphemeral | quote }} - name: PROFILE_RUNTIME_MEMORY_INSURANCE value: {{ .Values.monitor.profileRuntimeMemoryInsurance | quote }} - {{- end }} # Failed ranges are always saved; successful ones are the bulk of # the volume and can be turned off for a cheap run. - name: SAVE_SUCCESS_LOGS @@ -356,11 +353,6 @@ spec: mountPath: /data - name: logs mountPath: /logs - {{- if .Values.monitor.profileConfigMap }} - - name: profile - mountPath: /profile - readOnly: true - {{- end }} {{- if .Values.monitor.sourceConfigMap }} - name: monitor-src mountPath: /app @@ -456,11 +448,6 @@ spec: mountPath: /app {{- end }} volumes: - {{- if .Values.monitor.profileConfigMap }} - - name: profile - configMap: - name: {{ .Values.monitor.profileConfigMap }} - {{- end }} {{- if .Values.monitor.sourceConfigMap }} - name: monitor-src configMap: diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 5ccb4263..47563b1e 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -93,11 +93,6 @@ monitor: # Source-mode development normally installs dependencies at container start. # Disable only when the selected image already contains them. sourceInstallDependencies: true - # Range profile from an earlier run, as a ConfigMap holding profile.json. - # The mission driver resolves --pubnet-parallel-catchup-profile (a local - # path or an https URL) into one. Empty = size from the configured - # requests below. Only tightens requests; limits are untouched. - profileConfigMap: "" # CPU request tiers, as a slack budget rather than a demand estimate. Measured # unthrottled, replay wants ~1.0 cores at every ledger position and is 80-95% # of a job, so demand barely varies -- what varies is how much throttling a diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py index 2f1d9c7a..c33d3090 100644 --- a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py +++ b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py @@ -65,7 +65,6 @@ def _code_defaults(cname): 'CORE_IMAGE': 'the image under test, supplied per mission run', 'WORKER_SERVICE_ACCOUNT': 'derived from the release name for IRSA trust', 'MISSION': 'the mission name, for the kube-state-metrics label', - 'PROFILE_PATH': 'the mounted path of an optional profile ConfigMap', 'ASAN_OPTIONS': 'passed through to the worker; empty means "unset", not "default"', 'PARALLELISM': 'worker.replicas -- the whole point of the knob is to differ per run', 'ATTEMPT_DEADLINE_SECONDS': 'a backstop the chart turns on and the code leaves off', @@ -182,23 +181,20 @@ def test_each_deliberate_divergence_is_still_a_real_env_var(): assert not stale, f"DELIBERATE excuses env vars the chart no longer sets: {stale}" -def test_the_profile_block_only_renders_with_a_profile_configmap(): - """PROFILE_PATH must not be set without the volume that backs it. +def test_the_chart_does_not_dictate_where_the_profile_lives(): + """PROFILE_PATH is the monitor's own business now. - load_profile() treats a non-empty PROFILE_PATH as "there is a profile" and - only an OSError sends it back to the configured requests. Setting the path - with no ConfigMap mounted would make every run log an unreadable-profile - warning for a profile nobody asked for. + The profile arrives with POST /start and is written to the volume, so a + chart-supplied path would point at a ConfigMap mount that no longer exists + and make every run log an unreadable-profile warning. """ - without = art.env_of(art.containers()[art.MONITOR_CONTAINER]) - assert 'PROFILE_PATH' not in without - with_cm = art.env_of(art.containers(SETS)[art.MONITOR_CONTAINER]) - assert with_cm['PROFILE_PATH'] + env = art.env_of(art.containers()[art.MONITOR_CONTAINER]) + assert 'PROFILE_PATH' not in env + # And nothing mounts a profile volume any more. mounts = {m['mountPath'] - for m in art.containers(SETS)[art.MONITOR_CONTAINER]['volumeMounts']} - assert os.path.dirname(with_cm['PROFILE_PATH']) in mounts, ( - f"PROFILE_PATH={with_cm['PROFILE_PATH']} is not on any mounted volume") + for m in art.containers()[art.MONITOR_CONTAINER]['volumeMounts']} + assert '/profile' not in mounts def test_the_peak_flush_ratio_is_a_threshold_and_not_a_pass_through(): From f192eb28036bf94f682da3e78c3bbbd82edb7815 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Sat, 8 Aug 2026 20:42:36 -0400 Subject: [PATCH 074/117] Open the reconcile gate where the run is installed, not in the POST handler A restarted monitor never resumed. started.set() lived only in do_POST, so the restart path -- which reads run.json back off the volume and calls start_run directly -- restored the range and the profile but left the gate shut. reconcile_loop blocked on it forever. It failed silently, which is worse than crashing: /status kept answering 200 with its placeholder (num_remain 1), so nothing was ever unreachable and the driver polled a dead run indefinitely. Any monitor restart -- eviction, drain, OOM, a spot reclaim on its node -- would have hung a 4.7h run with no error. Observed on ssc-test 2026-08-08: 7 ranges already complete on the volume, reconcile never ran again after the pod came back. The gate now belongs to start_run, which is the only thing that establishes "this monitor knows what run it is performing" -- so a POST and a restart take exactly the same path, which is what the comment already claimed. Verified on ssc-test with two restarts in one run: - short kill: succeeded stayed 4, reconcile resumed - monitor scaled to 0 for ~3 minutes while 5 ranges finished unwatched; on return it recovered all of them (succeeded 7 -> 12) and re-dispatched none, every job still on attempt 1 - 21/21 ranges completed, full artifact set collected The suite passed before this fix, which is why it shipped. The new test drives a real server, installs a run, clears the gate and replays run.json -- it fails without the change. Co-Authored-By: Claude Opus 5 --- .../apps/job_monitor.py | 6 +++++ src/MissionParallelCatchup/lib/http_server.py | 6 ++--- .../parallel_catchup_helm/values.yaml | 2 +- .../tests/unit/test_http_surface.py | 25 +++++++++++++++++++ 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py index 9ca64404..734857bc 100644 --- a/src/MissionParallelCatchup/apps/job_monitor.py +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -107,6 +107,12 @@ def start_run(doc): records.write_atomic(config.RUN_PATH, json.dumps(doc, separators=(',', ':'))) config.PROFILE = profile logger.info("profile installed: %d ranges", len(config.PROFILE)) + # Opened here rather than in the POST handler, so a restart that reads + # run.json back resumes on exactly the same path. It did not, and a + # restarted monitor blocked on this forever while /status kept answering + # with its placeholder -- nothing was unreachable, so the driver polled a + # dead run indefinitely. Observed on ssc-test 2026-08-08. + http_server.started.set() _LIVENESS_NUMBERS = (('LIVENESS_PROBE_TIMEOUT_SECONDS', float), diff --git a/src/MissionParallelCatchup/lib/http_server.py b/src/MissionParallelCatchup/lib/http_server.py index 26093c4a..36253f90 100644 --- a/src/MissionParallelCatchup/lib/http_server.py +++ b/src/MissionParallelCatchup/lib/http_server.py @@ -89,12 +89,12 @@ def do_POST(self): try: on_start(doc) except ValueError as e: - # A profile the run cannot proceed with. Answering 400 fails the + # A run the monitor cannot proceed with. Answering 400 fails the # driver here, with the reason, rather than leaving it to poll a - # monitor that will never dispatch. + # monitor that will never dispatch. on_start opens the gate + # itself on success. self._send(400, json.dumps({'error': str(e)}).encode()) return - started.set() self._send(200, b'{"started":true}') def _manifest(self): diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 47563b1e..592cba62 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -78,7 +78,7 @@ monitor: # this chart against it ships env vars the image cannot read. Built from this # branch and pinned by tag rather than :latest so a run is always traceable to # one image. Revert to the stellar/ repo once there is a push path for it. - image: "stellajuna/ssc-jm:2026-08-08b" + image: "stellajuna/ssc-jm:2026-08-08d" # The driver reaches the monitor through this route: profile in via POST # /start, status and logs out. Empty routeHost disables the HTTPRoute; the # Service still exists, so an in-cluster caller works either way. diff --git a/src/MissionParallelCatchup/tests/unit/test_http_surface.py b/src/MissionParallelCatchup/tests/unit/test_http_surface.py index 6d382eee..f7e4dcc9 100644 --- a/src/MissionParallelCatchup/tests/unit/test_http_surface.py +++ b/src/MissionParallelCatchup/tests/unit/test_http_surface.py @@ -137,3 +137,28 @@ def test_the_collectors_resume_cursor_is_not_offered_for_pulling(server): assert 'range-300-a1.log.gz' in names assert 'range-300-a1.state' not in names + + +def test_a_restart_resumes_without_waiting_for_another_start(server, tmp_path): + """run.json on the volume is what says "this monitor has a run". + + Whoever installs it opens the gate -- a POST, or a restart reading it back. + It did not: the gate lived in the POST handler, so a restarted monitor + blocked on it forever while /status kept answering with its placeholder. + Nothing was unreachable, so the driver polled a dead run indefinitely. + Observed on ssc-test 2026-08-08 with 7 ranges already completed on the + volume and reconcile never running again. + """ + base, vol = server + run = {"range": {"startingLedger": 0, "latestLedgerNum": 1000, + "ledgersPerJob": 100}} + assert _post(base, '/start', json.dumps(run))[0] == 200 + assert (vol / 'run.json').exists() + + # A fresh process: same volume, gate closed again. + http_server.started.clear() + jm.start_run(json.loads((vol / 'run.json').read_text())) + + assert http_server.started.is_set(), ( + "a restart restored the run but never opened the gate, so reconcile " + "would block forever and the run would hang silently") From 967f0a46885948792f438dd2b3620e6dbd8175df Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Sat, 8 Aug 2026 21:27:07 -0400 Subject: [PATCH 075/117] Say in /status whether the run has started Before the first reconcile pass /status answers with the module defaults -- zeros that read exactly like a run with nothing done yet. A caller cannot tell "nothing recorded so far" from "not dispatching at all". That ambiguity is what let a wedged monitor be polled indefinitely on ssc-test: the gate never opened, but /status kept answering 200, so nothing was ever unreachable and the driver had no signal to fail on. The gate bug is fixed; this removes the ambiguity that hid it. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/lib/http_server.py | 10 ++++++++-- .../tests/unit/test_http_surface.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/MissionParallelCatchup/lib/http_server.py b/src/MissionParallelCatchup/lib/http_server.py index 36253f90..09b79942 100644 --- a/src/MissionParallelCatchup/lib/http_server.py +++ b/src/MissionParallelCatchup/lib/http_server.py @@ -64,8 +64,14 @@ def do_GET(self): elif self.path == '/status': snapshot, lock = status_source() with lock: - body = json.dumps(snapshot, separators=(',', ':')).encode() - self._send(200, body) + doc = dict(snapshot) + # Until the first reconcile pass lands, the counts are placeholders + # -- zeros that read exactly like a run with nothing done yet. This + # says which it is, so a caller can tell "no work recorded" from + # "not dispatching at all" instead of polling a monitor that never + # will. + doc['started'] = started.is_set() + self._send(200, json.dumps(doc, separators=(',', ':')).encode()) elif self.path == '/logs': self._send(200, json.dumps(self._manifest(), separators=(',', ':')).encode()) elif self.path.startswith('/logs/'): diff --git a/src/MissionParallelCatchup/tests/unit/test_http_surface.py b/src/MissionParallelCatchup/tests/unit/test_http_surface.py index f7e4dcc9..1d30a978 100644 --- a/src/MissionParallelCatchup/tests/unit/test_http_surface.py +++ b/src/MissionParallelCatchup/tests/unit/test_http_surface.py @@ -162,3 +162,21 @@ def test_a_restart_resumes_without_waiting_for_another_start(server, tmp_path): assert http_server.started.is_set(), ( "a restart restored the run but never opened the gate, so reconcile " "would block forever and the run would hang silently") + + +def test_status_says_whether_the_run_has_started(server): + """Placeholder zeros are indistinguishable from a run with nothing done. + + Before the first reconcile pass /status answers with the module defaults, so + a caller cannot tell "nothing recorded yet" from "never going to dispatch". + That ambiguity is what let a wedged monitor be polled indefinitely: it kept + answering 200 and nothing was ever unreachable. + """ + base, _ = server + assert json.loads(_get(base, '/status')[1])['started'] is False + + _post(base, '/start', json.dumps({"range": {"startingLedger": 0, + "latestLedgerNum": 1000, + "ledgersPerJob": 100}})) + + assert json.loads(_get(base, '/status')[1])['started'] is True From ed82235b74e50d58722efa8a12600be30cca1a7a Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 10:07:48 -0400 Subject: [PATCH 076/117] catchup: make the job monitor image settable per run The monitor and collector ship as one image pinned in values.yaml, so testing a build of them meant editing the chart. --job-monitor-image-pc-v2 sets it for a run instead. Empty leaves the chart pin alone rather than setting monitor.image to nothing, which resolves to ":latest" or fails the pull. The guard is what the test pins. Co-Authored-By: Claude Opus 5 --- src/App/Program.fs | 8 ++++++++ src/FSLibrary.Tests/Tests.fs | 20 +++++++++++++++++++ .../MissionHistoryPubnetParallelCatchupV2.fs | 5 +++++ src/FSLibrary/StellarMissionContext.fs | 1 + 4 files changed, 34 insertions(+) diff --git a/src/App/Program.fs b/src/App/Program.fs index 85732716..863050d5 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -119,6 +119,7 @@ type MissionOptions pubnetParallelCatchupProfile: string, pubnetParallelCatchupRangeOrder: string, pubnetParallelCatchupPoolPrefix: string, + jobMonitorImagePcV2: string, pubnetParallelCatchupCpuRequest: string, tag: string option, numPregeneratedTxs: int option, @@ -550,6 +551,12 @@ type MissionOptions Default = "")>] member self.PubnetParallelCatchupPoolPrefix : string = pubnetParallelCatchupPoolPrefix + [] + member self.JobMonitorImagePcV2 : string = jobMonitorImagePcV2 + [] +let ``the job monitor image is overridable and defaults to the chart`` () = + // The monitor and collector ship as one image pinned in values.yaml. Passing + // it per run is what lets a build of them be tested without editing the + // chart -- but an empty flag must leave the chart's pin alone rather than + // setting monitor.image to nothing, which resolves to ":latest" or fails the + // pull outright. + let src = + System.IO.File.ReadAllText( + "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + + Assert.Contains("if context.jobMonitorImagePcV2 <> \"\" then", src) + Assert.Contains("monitor.image=%s", src) + + let guard = src.IndexOf("if context.jobMonitorImagePcV2 <> \"\" then") + let use_ = src.IndexOf("monitor.image=%s") + Assert.True(guard < use_, "monitor.image must only be set inside the non-empty guard") diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index f6e22f23..a519951e 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -253,6 +253,11 @@ let installProject (context: MissionContext) = // from its measured peakAnonBytes, and gets that node to itself. setOptions.Add(sprintf "monitor.poolPrefix=%s" context.pubnetParallelCatchupPoolPrefix) + // The monitor and collector ship as one image, pinned in the chart. Passing + // it here is what lets a run test a build of them without editing values. + if context.jobMonitorImagePcV2 <> "" then + setOptions.Add(sprintf "monitor.image=%s" context.jobMonitorImagePcV2) + // Capacity type is DERIVED, not configured. Both capacity variants of a tier // share one label value, so a pod needs this second expression to pick a // side -- and the storage mode already decides which side it must be. pvc diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index aa3a49da..60a10d1c 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -123,6 +123,7 @@ type MissionContext = pubnetParallelCatchupProfile: string pubnetParallelCatchupRangeOrder: string pubnetParallelCatchupPoolPrefix: string + jobMonitorImagePcV2: string pubnetParallelCatchupCpuRequest: string genesisTestAccountCount: int option From abfc6825623e59b2dab74bdf06e266cfb922af04 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 10:19:03 -0400 Subject: [PATCH 077/117] Pin the dev image to 2026-08-09a Carries the reconcile-gate fix and the started field, neither of which was in 2026-08-08d. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/parallel_catchup_helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 592cba62..4501e95c 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -78,7 +78,7 @@ monitor: # this chart against it ships env vars the image cannot read. Built from this # branch and pinned by tag rather than :latest so a run is always traceable to # one image. Revert to the stellar/ repo once there is a push path for it. - image: "stellajuna/ssc-jm:2026-08-08d" + image: "stellajuna/ssc-jm:2026-08-09a" # The driver reaches the monitor through this route: profile in via POST # /start, status and logs out. Empty routeHost disables the HTTPRoute; the # Service still exists, so an in-cluster caller works either way. From f55038d0679e47aae7fc18d349187bb19e0e9a2c Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 10:25:51 -0400 Subject: [PATCH 078/117] Delete the logarithmic range generator It has been unreachable since the range moved into POST /start: the run document the mission sends carries startingLedger, latestLedgerNum, ledgersPerJob and order, but no generator, so config.RANGE_GENERATOR was a constant 'uniform' and the other arm could not be selected. The chart kept shipping LOGARITHMIC_FLOOR_LEDGERS and a range.generator value that nothing read. longest-first supersedes what it was for. Both aim at equal wall-time per job; longest-first orders from what ranges actually measured rather than from an assumption that cost rises smoothly toward the tip. With one generator left there is no dispatch to make, so generate_ranges() calls _uniform_segment directly and RANGE_GENERATOR/VALID_RANGE_GENERATORS are gone along with their startup validation. The tests that pinned the logarithmic layout go with it; the preflight test that used an unknown generator as its invalid-config vehicle now uses an unknown RANGE_ORDER. Recoverable from 8553e77 if the guess ever beats the measurement. --- .../apps/job_monitor.py | 7 +-- src/MissionParallelCatchup/lib/config.py | 4 -- src/MissionParallelCatchup/lib/ranges.py | 42 ++++---------- .../templates/job_monitor.yaml | 2 - .../parallel_catchup_helm/values.yaml | 4 -- src/MissionParallelCatchup/tests/conftest.py | 1 - .../tests/unit/test_range_generation.py | 56 +++---------------- 7 files changed, 21 insertions(+), 95 deletions(-) diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py index 734857bc..675f78ed 100644 --- a/src/MissionParallelCatchup/apps/job_monitor.py +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -87,7 +87,7 @@ def start_run(doc): volume before the gate opens, so the first Job dispatched is already sized by the profile and a restart resumes the same run. """ - for key, name in (('generator', 'RANGE_GENERATOR'), ('order', 'RANGE_ORDER'), + for key, name in (('order', 'RANGE_ORDER'), ('startingLedger', 'STARTING_LEDGER'), ('latestLedgerNum', 'LATEST_LEDGER_NUM'), ('ledgersPerJob', 'LEDGERS_PER_JOB'), @@ -143,9 +143,6 @@ def validate_config(): raise ValueError(f"{name} must be greater than zero, got {raw!r}") setattr(config, name, value) - if config.RANGE_GENERATOR not in config.VALID_RANGE_GENERATORS: - raise ValueError("RANGE_GENERATOR must be one of %s, got %r" - % (', '.join(config.VALID_RANGE_GENERATORS), config.RANGE_GENERATOR)) if config.RANGE_ORDER not in config.VALID_RANGE_ORDERS: raise ValueError("RANGE_ORDER must be one of %s, got %r" % (', '.join(config.VALID_RANGE_ORDERS), config.RANGE_ORDER)) @@ -1321,7 +1318,7 @@ def completion_record(end, attempt, status, pod, count=None): "metric will be missing for this range", end) record = {'seconds': _range_compute_seconds(end, attempt, pod, wall), 'wallSeconds': wall, 'txApply': tx, 'attempts': attempt} - # Ledger count travels with the record: the logarithmic generator varies it + # Ledger count travels with the record: ledgersPerJob is per-run input # per range, so it cannot be recomputed from config when the profile is read # back. if count is not None: diff --git a/src/MissionParallelCatchup/lib/config.py b/src/MissionParallelCatchup/lib/config.py index 88c685cd..61674b04 100644 --- a/src/MissionParallelCatchup/lib/config.py +++ b/src/MissionParallelCatchup/lib/config.py @@ -23,9 +23,7 @@ # Which ledger ranges to run. These are pure inputs to the range generator: # dispatch recomputes the whole list every reconcile, so a restart must # reproduce it exactly. -RANGE_GENERATOR = 'uniform' # from /start: uniform | logarithmic -VALID_RANGE_GENERATORS = ('uniform', 'logarithmic') # Both generators emit tip-first, which front-loads the most expensive ranges: # the bucket set only grows with ledger position. 'oldest-first' reverses that, @@ -43,8 +41,6 @@ OVERLAP_LEDGERS = 320 # from /start -# logarithmic only: chunk size halves toward the tip and stops shrinking here. -LOGARITHMIC_FLOOR_LEDGERS = int(os.getenv('LOGARITHMIC_FLOOR_LEDGERS', 64000)) # ============================================================================= # 2. Kubernetes objects this monitor creates diff --git a/src/MissionParallelCatchup/lib/ranges.py b/src/MissionParallelCatchup/lib/ranges.py index 4056c15e..f4200d7a 100644 --- a/src/MissionParallelCatchup/lib/ranges.py +++ b/src/MissionParallelCatchup/lib/ranges.py @@ -68,34 +68,16 @@ def _ordered(ranges): % (', '.join(config.VALID_RANGE_ORDERS), config.RANGE_ORDER)) -def _logarithmic_ranges(): - """Big chunks over cheap early history, halving toward the tip. - - Aims for roughly equal wall-time per job rather than equal ledger count. - """ - out = [] - start_ledger = config.STARTING_LEDGER - end_ledger = config.LATEST_LEDGER_NUM // 2 - chunk = (end_ledger - start_ledger + 1) // max(config.PARALLELISM, 1) - while chunk > config.LOGARITHMIC_FLOOR_LEDGERS: - out.extend(_uniform_segment(start_ledger, end_ledger, chunk)) - start_ledger = end_ledger + 1 - chunk //= 2 - end_ledger = start_ledger + (chunk * config.PARALLELISM) - out.extend(_uniform_segment(end_ledger + 1, config.LATEST_LEDGER_NUM, config.LOGARITHMIC_FLOOR_LEDGERS)) - return out - - def generate_ranges(): - # An unrecognised generator used to fall through to logarithmic, so a typo - # silently produced a completely different range layout. validate_config() - # rejects that at startup; this raise is the backstop, not the primary check - # -- reached from inside reconcile it would only ever be logged and retried. - if config.RANGE_GENERATOR == 'uniform': - ranges = _uniform_segment(config.STARTING_LEDGER, config.LATEST_LEDGER_NUM, config.LEDGERS_PER_JOB) - elif config.RANGE_GENERATOR == 'logarithmic': - ranges = _logarithmic_ranges() - else: - raise ValueError("RANGE_GENERATOR must be one of %s, got %r" - % (', '.join(config.VALID_RANGE_GENERATORS), config.RANGE_GENERATOR)) - return _ordered(ranges) + """Uniform ranges over the whole window, in the configured dispatch order. + + A logarithmic generator lived here -- big chunks over cheap early history, + halving toward the tip, aiming for equal wall-time per job. longest-first + supersedes it: same goal, but ordered from what ranges actually measured + rather than from an assumption about where the expensive ledgers are. It + also became unreachable when the range moved into /start, which carries no + generator. Recover it from 8553e77 if the guess ever beats the measurement. + """ + return _ordered(_uniform_segment(config.STARTING_LEDGER, + config.LATEST_LEDGER_NUM, + config.LEDGERS_PER_JOB)) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index ed2683fc..202cfb87 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -206,8 +206,6 @@ spec: value: {{ .Values.monitor.livenessSweepSeconds | quote }} - name: LIVENESS_MAX_CONCURRENCY value: {{ .Values.monitor.livenessMaxConcurrency | quote }} - - name: LOGARITHMIC_FLOOR_LEDGERS - value: {{ .Values.range.logarithmicFloorLedgers | quote }} - name: PARALLELISM value: {{ .Values.worker.replicas | quote }} - name: STORAGE_MODE diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 4501e95c..911a40f7 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -60,10 +60,6 @@ worker: # The range itself -- start, end, size, order -- is per-run input and arrives # with the driver's POST /start, so the chart installs a generic monitor. range: - # uniform: equal ledger counts. logarithmic: big chunks over cheap early - # history, halving toward the tip, aiming for equal wall-time per job. - generator: "uniform" - logarithmicFloorLedgers: 64000 overlapLedgers: 320 # Dispatch order. Generators emit tip-first, which front-loads the most # expensive ranges. "oldest-first" reverses that so a profiling run diff --git a/src/MissionParallelCatchup/tests/conftest.py b/src/MissionParallelCatchup/tests/conftest.py index f56d0d18..7dd35639 100644 --- a/src/MissionParallelCatchup/tests/conftest.py +++ b/src/MissionParallelCatchup/tests/conftest.py @@ -37,7 +37,6 @@ def test_something(cluster): 'LATEST_LEDGER_NUM': 300, 'LEDGERS_PER_JOB': 100, 'OVERLAP_LEDGERS': 320, - 'RANGE_GENERATOR': 'uniform', 'RANGE_ORDER': 'tip-first', 'PARALLELISM': 2, 'STORAGE_MODE': 'pvc', diff --git a/src/MissionParallelCatchup/tests/unit/test_range_generation.py b/src/MissionParallelCatchup/tests/unit/test_range_generation.py index 24d63838..1d0a44e1 100644 --- a/src/MissionParallelCatchup/tests/unit/test_range_generation.py +++ b/src/MissionParallelCatchup/tests/unit/test_range_generation.py @@ -16,22 +16,19 @@ @pytest.fixture def build(monkeypatch): """Configure the generator and return a callable that runs it.""" - def configure(generator='uniform', order='tip-first', parallelism=4, - start=39990000, latest=40000000, per_job=1000, - floor=64000, overlap=320): - monkeypatch.setattr(config, 'RANGE_GENERATOR', generator) + def configure(order='tip-first', parallelism=4, + start=39990000, latest=40000000, per_job=1000, overlap=320): monkeypatch.setattr(config, 'RANGE_ORDER', order) monkeypatch.setattr(config, 'PARALLELISM', parallelism) monkeypatch.setattr(config, 'STARTING_LEDGER', start) monkeypatch.setattr(config, 'LATEST_LEDGER_NUM', latest) monkeypatch.setattr(config, 'LEDGERS_PER_JOB', per_job) - monkeypatch.setattr(config, 'LOGARITHMIC_FLOOR_LEDGERS', floor) monkeypatch.setattr(config, 'OVERLAP_LEDGERS', overlap) return ranges.generate_ranges() return configure -def test_generators_emit_tip_first_by_default(build): +def test_ranges_are_emitted_tip_first_by_default(build): r = build() assert r[0][0] > r[-1][0], "index 0 must be the tip" @@ -68,39 +65,6 @@ def test_a_short_tail_segment_is_not_padded_past_the_start(build): assert sorted(r) == [(500, 500), (1500, 1000), (2500, 1000)] -def test_logarithmic_ranges_match_the_shell_generator(build): - # Verbatim output of logarithmic_range_generator.sh with - # floor=16000 overlap=320 start=0 latest=500000 parallelism=4, captured - # before it was deleted. Chunk size halves toward the tip, so exact values - # are pinned rather than a count. - expected = ("250000/62820 187500/62820 125000/62820 62500/62820 " - "375001/31570 343751/31570 312501/31570 281251/31570 " - "500000/16320 484000/16320 468000/16320 452000/14817").split() - r = build(generator='logarithmic', floor=16000, overlap=320, - start=0, latest=500000, parallelism=4) - assert [f"{end}/{count}" for end, count in r] == expected - - -def test_the_logarithmic_generator_also_honours_dispatch_order(build): - tip = build(generator='logarithmic', floor=16000, start=0, latest=500000) - old = build(generator='logarithmic', floor=16000, start=0, latest=500000, - order='oldest-first') - assert old == list(reversed(tip)) - - -@pytest.mark.parametrize('generator', ['uniforn', 'log', '', 'LOGARITHMIC']) -def test_an_unrecognised_generator_fails_instead_of_becoming_logarithmic(build, generator): - """A typo used to silently produce a different range layout. - - Both arms are explicit now, so anything else raises. This is the failure - mode worth a test: the run still SUCCEEDS with the wrong ranges, and no - downstream artifact records which generator produced them, so there is - nothing to notice afterwards. - """ - with pytest.raises(ValueError, match='RANGE_GENERATOR'): - build(generator=generator) - - def test_longest_first_is_inert_without_a_profile(build, monkeypatch): """Ordering is driven by RANGE_ORDER, never by profile detection. @@ -129,8 +93,7 @@ def test_an_unrecognised_order_fails_instead_of_becoming_tip_first(build, order) @pytest.fixture def preflight(monkeypatch): - def configure(generator='uniform', order='tip-first', profile=None): - monkeypatch.setattr(config, 'RANGE_GENERATOR', generator) + def configure(order='tip-first', profile=None): monkeypatch.setattr(config, 'RANGE_ORDER', order) monkeypatch.setattr(config, 'PROFILE', profile) return jm.validate_config @@ -138,12 +101,7 @@ def configure(generator='uniform', order='tip-first', profile=None): def test_valid_config_passes(preflight): - preflight(generator='logarithmic', order='oldest-first')() - - -def test_preflight_rejects_an_unknown_generator(preflight): - with pytest.raises(ValueError, match='RANGE_GENERATOR'): - preflight(generator='uniforn')() + preflight(order='oldest-first')() def test_preflight_rejects_an_unknown_order(preflight): @@ -177,9 +135,9 @@ def test_the_preflight_runs_before_anything_is_dispatched(monkeypatch, tmp_path) must be refused, not started and then discovered.""" monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) monkeypatch.setattr(config, 'RUN_PATH', str(tmp_path / 'run.json')) - monkeypatch.setattr(config, 'RANGE_GENERATOR', 'nonsense') + monkeypatch.setattr(config, 'RANGE_ORDER', 'nonsense') - with pytest.raises(ValueError, match='RANGE_GENERATOR must be one of'): + with pytest.raises(ValueError, match='RANGE_ORDER must be one of'): jm.start_run({"range": {'startingLedger': 0, 'latestLedgerNum': 1000, 'ledgersPerJob': 100}}) # Rejected, so nothing was written and no run can proceed from it. From 21a0dfd72a9b014e28c6f91fac44c7a689ba535b Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 10:35:56 -0400 Subject: [PATCH 079/117] Keep the parallel-catchup test suite local The suite, pytest.ini and requirements-dev.txt stop being tracked. The files stay on disk and still run; they are only removed from the index. Nothing depends on them being in the repo: no CI job invokes pytest, and Dockerfile.jobmonitor copies only apps/ and lib/, so the image never carried them. They were also added entirely on this branch -- the mission was job_monitor.py and a Dockerfile at 79e714a -- so no existing consumer loses anything. The matching ignore rules live in .git/info/exclude rather than .gitignore, so the exclusion itself is local too and adds nothing to the diff. --- src/MissionParallelCatchup/pytest.ini | 10 - .../requirements-dev.txt | 12 - .../tests/collector/test_archive_append.py | 302 -------- .../tests/collector/test_poll_backoff.py | 227 ------ src/MissionParallelCatchup/tests/conftest.py | 330 --------- .../tests/contract/_artifacts.py | 187 ----- .../tests/contract/test_chart_defaults.py | 235 ------ .../tests/contract/test_chart_env_wiring.py | 299 -------- .../tests/contract/test_chart_rbac.py | 165 ----- .../contract/test_cross_process_files.py | 204 ------ .../tests/contract/test_dependency_pins.py | 79 --- .../contract/test_fsharp_driver_contract.py | 523 -------------- .../contract/test_k8s_failure_formats.py | 321 --------- .../contract/test_medida_metric_block.py | 192 ----- .../tests/contract/test_module_packaging.py | 157 ---- .../tests/contract/test_rendered_job_spec.py | 293 -------- .../tests/contract/test_worker_log_markers.py | 133 ---- .../tests/data/real-sts-fault-exit3.log.gz | Bin 8441 -> 0 bytes src/MissionParallelCatchup/tests/fake_k8s.py | 508 ------------- .../tests/reconcile/test_attempt_deadline.py | 318 --------- .../test_completed_range_not_redispatched.py | 221 ------ .../reconcile/test_dispatch_not_frozen.py | 229 ------ .../tests/reconcile/test_retry_budgets.py | 547 -------------- .../tests/reconcile/test_txapply_histogram.py | 189 ----- .../resilience/test_collector_restart.py | 669 ------------------ .../tests/resilience/test_crash_points.py | 626 ---------------- .../tests/resilience/test_hostile_state.py | 400 ----------- .../tests/resilience/test_restart_fuzz.py | 502 ------------- .../tests/test_harness_smoke.py | 185 ----- .../tests/unit/conftest.py | 24 - .../tests/unit/test_attempt_chain.py | 415 ----------- .../tests/unit/test_classify.py | 247 ------- .../tests/unit/test_collector_main_loop.py | 279 -------- .../tests/unit/test_condemnation_watch.py | 384 ---------- .../tests/unit/test_deadline_sizing.py | 71 -- .../tests/unit/test_dispatch_order.py | 57 -- .../tests/unit/test_http_surface.py | 182 ----- .../tests/unit/test_kubelet_sampler.py | 288 -------- .../unit/test_monitor_verdict_records.py | 313 -------- .../tests/unit/test_node_targeting.py | 69 -- .../tests/unit/test_poll_lifecycle.py | 276 -------- .../tests/unit/test_pool_tiers.py | 604 ---------------- .../tests/unit/test_profile_lookup.py | 124 ---- .../tests/unit/test_range_generation.py | 149 ---- .../tests/unit/test_reaping.py | 164 ----- .../tests/unit/test_records.py | 90 --- .../tests/unit/test_resources.py | 277 -------- .../tests/unit/test_resume_script.py | 143 ---- .../tests/unit/test_retry_counters.py | 208 ------ .../tests/unit/test_sizing.py | 227 ------ .../tests/unit/test_tx_apply.py | 253 ------- .../tests/unit/test_worker_liveness.py | 241 ------- 52 files changed, 13148 deletions(-) delete mode 100644 src/MissionParallelCatchup/pytest.ini delete mode 100644 src/MissionParallelCatchup/requirements-dev.txt delete mode 100644 src/MissionParallelCatchup/tests/collector/test_archive_append.py delete mode 100644 src/MissionParallelCatchup/tests/collector/test_poll_backoff.py delete mode 100644 src/MissionParallelCatchup/tests/conftest.py delete mode 100644 src/MissionParallelCatchup/tests/contract/_artifacts.py delete mode 100644 src/MissionParallelCatchup/tests/contract/test_chart_defaults.py delete mode 100644 src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py delete mode 100644 src/MissionParallelCatchup/tests/contract/test_chart_rbac.py delete mode 100644 src/MissionParallelCatchup/tests/contract/test_cross_process_files.py delete mode 100644 src/MissionParallelCatchup/tests/contract/test_dependency_pins.py delete mode 100644 src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py delete mode 100644 src/MissionParallelCatchup/tests/contract/test_k8s_failure_formats.py delete mode 100644 src/MissionParallelCatchup/tests/contract/test_medida_metric_block.py delete mode 100644 src/MissionParallelCatchup/tests/contract/test_module_packaging.py delete mode 100644 src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py delete mode 100644 src/MissionParallelCatchup/tests/contract/test_worker_log_markers.py delete mode 100644 src/MissionParallelCatchup/tests/data/real-sts-fault-exit3.log.gz delete mode 100644 src/MissionParallelCatchup/tests/fake_k8s.py delete mode 100644 src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py delete mode 100644 src/MissionParallelCatchup/tests/reconcile/test_completed_range_not_redispatched.py delete mode 100644 src/MissionParallelCatchup/tests/reconcile/test_dispatch_not_frozen.py delete mode 100644 src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py delete mode 100644 src/MissionParallelCatchup/tests/reconcile/test_txapply_histogram.py delete mode 100644 src/MissionParallelCatchup/tests/resilience/test_collector_restart.py delete mode 100644 src/MissionParallelCatchup/tests/resilience/test_crash_points.py delete mode 100644 src/MissionParallelCatchup/tests/resilience/test_hostile_state.py delete mode 100644 src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py delete mode 100644 src/MissionParallelCatchup/tests/test_harness_smoke.py delete mode 100644 src/MissionParallelCatchup/tests/unit/conftest.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_attempt_chain.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_classify.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_condemnation_watch.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_dispatch_order.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_http_surface.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_node_targeting.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_poll_lifecycle.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_pool_tiers.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_profile_lookup.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_range_generation.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_reaping.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_records.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_resources.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_resume_script.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_retry_counters.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_sizing.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_tx_apply.py delete mode 100644 src/MissionParallelCatchup/tests/unit/test_worker_liveness.py diff --git a/src/MissionParallelCatchup/pytest.ini b/src/MissionParallelCatchup/pytest.ini deleted file mode 100644 index a5c9a940..00000000 --- a/src/MissionParallelCatchup/pytest.ini +++ /dev/null @@ -1,10 +0,0 @@ -[pytest] -# The monitor and collector are plain modules, not an installed package, so the -# suite needs each source directory on the path: `apps` for job_monitor / -# log_collector, `lib` for the modules they import, `tests` for the fake-cluster -# harness. The container flattens apps/ and lib/ into /app, so `import config` -# resolves the same way there. -pythonpath = apps lib tests -testpaths = tests -# Async tests declare themselves with @pytest.mark.asyncio. -asyncio_mode = strict diff --git a/src/MissionParallelCatchup/requirements-dev.txt b/src/MissionParallelCatchup/requirements-dev.txt deleted file mode 100644 index 3bf4ba66..00000000 --- a/src/MissionParallelCatchup/requirements-dev.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Test-only dependencies. The runtime pins live in Dockerfile.jobmonitor and are -# installed a second time by the chart's sourceConfigMap path -- see -# tests/contract/test_dependency_pins.py, which keeps those two in step and -# checks that the major this suite imports is the major the image ships. -# -# The kubernetes client is listed because the contract tests build real V1* -# objects: the suite must import the same major the image installs. -pytest~=9.1 -pytest-asyncio~=1.4 -kubernetes~=36.0 -aiohttp~=3.14 -prometheus-client~=0.19 diff --git a/src/MissionParallelCatchup/tests/collector/test_archive_append.py b/src/MissionParallelCatchup/tests/collector/test_archive_append.py deleted file mode 100644 index 44aa3ad0..00000000 --- a/src/MissionParallelCatchup/tests/collector/test_archive_append.py +++ /dev/null @@ -1,302 +0,0 @@ -"""RACE #3 -- a torn .log.gz member kills a whole reconcile pass. - -The log-collector appends a gzip member to range--a.log.gz in place, -with no temp+rename, so a reader that looks while a poll is mid-write sees a -truncated member. job_monitor reads that same file to recover txApplySeconds -and guards it with `except OSError`, which does not cover the EOFError that -gzip raises on a truncated member. - -Consequence: one in-flight log write aborts the entire reconcile pass -- no -recording, no PVC release, no reaping and no dispatch for ANY of the ~4000 -ranges, not just the one whose archive was being written. And because the torn -bytes stay on disk, it repeats on every subsequent pass. - -Every test here drives the real code and asserts on observed state: what -reconcile() returned, what landed in progress.json, which Jobs/PVCs exist, and -what a reader sees on disk while the collector writes. -""" - -import asyncio -import gzip -import io -import os -import random - -import pytest - -import records -import job_monitor as jm -import config -import log_collector as lc - - -# --- building the artefact the race leaves on disk -------------------------- - -def _gzip_member(text): - """One complete, self-contained gzip member -- what one finished poll adds.""" - buf = io.BytesIO() - with gzip.GzipFile(fileobj=buf, mode='wb', mtime=0) as fh: - fh.write(text.encode()) - return buf.getvalue() - - -def write_torn_archive(path, settled="startup line\n", in_flight=None): - """A .log.gz exactly as an interrupted in-place append leaves it. - - One complete member from an earlier poll, followed by the first half of the - member the current poll is still writing. This is byte-for-byte the shape an - in-place gzip append produces once its buffer has flushed but the member has - not been closed; the live-writer version of the same thing is - test_collector_append_never_exposes_a_partial_member_to_a_reader below. - """ - if in_flight is None: - in_flight = "".join(f"line {i} of the poll that is still running\n" - for i in range(200)) - partial = _gzip_member(in_flight) - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, 'wb') as fh: - fh.write(_gzip_member(settled)) - fh.write(partial[:max(24, len(partial) // 2)]) - # Guard: the file we just built must actually be the torn artefact, or the - # test below would pass for the wrong reason. - with pytest.raises(EOFError): - with gzip.open(path, 'rt') as fh: - fh.read() - return path - - -CORE_TAIL = ( - "2026-07-30T00:00:00Z metric 'ledger.transaction.apply'\n" - "2026-07-30T00:00:00Z count = 12345\n" - "2026-07-30T00:00:00Z sum = 4200.0ms\n" -) - - -def write_whole_archive(path, text=CORE_TAIL): - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, 'wb') as fh: - fh.write(_gzip_member(text)) - return path - - -# --- reader side: the reconcile pass ---------------------------------------- - -def test_a_torn_archive_does_not_abort_the_reconcile_pass(cluster): - """A succeeded range whose archive is mid-append must still be recorded. - - Nothing about this range is unusual apart from the collector happening to - be writing when reconcile looked. - """ - cluster.reconcile() # dispatches r300, r200 - cluster.advance(300, 'succeeded') - # The collector has not flushed .metrics yet, so the archive is the only - # source for txApply -- and it is exactly the file being written. - write_torn_archive(records.log_path('300', 1)) - - result = cluster.reconcile() - - # The pass completed and did all of its work. - assert '300' in cluster.completed(), "succeeded range was never recorded" - assert cluster.completed()['300']['attempts'] == 1 - assert 'pc-data-r300' not in cluster.pvcs(), "completed range kept its volume" - assert result['created'] == 1, "the freed slot was never refilled" - assert 'pc-r100-a1' in cluster.jobs() - # The unreadable archive costs the metric for this range, nothing more. - assert cluster.completed()['300']['txApply'] is None - - -def test_a_torn_archive_costs_one_range_not_the_other_ranges_in_the_pass(cluster): - """Blast radius. Two ranges finish together; one has a torn archive. - - The healthy one must be recorded, keep its measurements and be reaped in - the same pass, and the third range must still be dispatched. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.advance(200, 'succeeded') - # r300: collector finished cleanly. - cluster.finalize(300, 1, tx_apply=12.5, peaks={'peakAnonBytes': 111}) - # r200: collector is mid-poll, archive torn, nothing durable yet. - write_torn_archive(records.log_path('200', 1)) - - result = cluster.reconcile() - - completed = cluster.completed() - assert set(completed) == {'300', '200'} - # The healthy range is untouched by its neighbour's corrupt file. - assert completed['300']['txApply'] == 12.5 - assert completed['300']['peakAnonBytes'] == 111 - assert 'pc-r300-a1' not in cluster.jobs(), "finalized range was not reaped" - # The torn range pays, and only the torn range. - assert completed['200']['txApply'] is None - # Dispatch still happened. - assert result['created'] == 1 - assert 'pc-r100-a1' in cluster.jobs() - assert cluster.failed() == {} - - -def test_a_never_repaired_torn_archive_does_not_wedge_the_run(cluster): - """The torn bytes are durable, so the reader hits them on every pass. - - Once a range is recorded with txApply=None the backfill branch re-reads the - archive each cycle, so a single corrupt file is not a one-pass outage -- it - stops the run permanently. Drive the whole run to completion over it. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.advance(200, 'succeeded') - cluster.finalize(200, 1, tx_apply=7.0) - # r300's archive is torn and nobody ever fixes it. - torn = write_torn_archive(records.log_path('300', 1)) - - cluster.reconcile() # records 300 + 200, dispatches 100 - assert 'pc-r100-a1' in cluster.jobs() - cluster.advance(100, 'succeeded') - cluster.finalize(100, 1, tx_apply=3.0) - - result = cluster.reconcile() # records 100 - result = cluster.reconcile() # steady state, still re-reading 300 - - assert os.path.exists(torn), "test no longer exercises the corrupt file" - assert set(cluster.completed()) == {'300', '200', '100'} - assert cluster.failed() == {} - assert result['remaining'] == 0 - assert result['in_progress'] == [] - - -def test_a_range_recovers_its_metric_once_the_collector_finishes(cluster): - """Bounded in time as well as in scope. - - The torn read costs txApply for exactly as long as the archive is torn: the - moment the collector lands .metrics, the backfill branch picks it up. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - write_torn_archive(records.log_path('300', 1)) - - cluster.reconcile() - assert cluster.completed()['300']['txApply'] is None - - # The collector's poll completes and it writes what it scanned out of the - # stream. The archive on disk is still torn. - cluster.finalize(300, 1, tx_apply=88.25, peaks={'peakAnonBytes': 222}) - cluster.reconcile() - - assert cluster.completed()['300']['txApply'] == 88.25 - assert cluster.completed()['300']['peakAnonBytes'] == 222 - assert 'pc-r300-a1' not in cluster.jobs() - - -# --- writer side: the collector's append ------------------------------------ - -class _FakeContent: - def __init__(self, body): - self._body = body.encode() - - async def iter_chunked(self, n): - for i in range(0, len(self._body), n): - yield self._body[i:i + n] - - -class _FakeResponse: - status = 200 - - def __init__(self, body): - self.content = _FakeContent(body) - - def raise_for_status(self): - pass - - async def __aenter__(self): - return self - - async def __aexit__(self, *exc): - return False - - -class _FakeSession: - """Just enough aiohttp for _poll_once: one GET returning a log body.""" - - def __init__(self, body): - self._body = body - - def get(self, url, params=None, headers=None): - return _FakeResponse(self._body) - - -class _ReadingScanner(lc.TxApplyScanner): - """The monitor, reading the archive while the collector writes it. - - feed() is called once per log line from inside the collector's write loop, - which makes "a reader looked mid-append" deterministic instead of a timing - coin flip. - """ - - def __init__(self, path, every=250): - super().__init__() - self.path = path - self.every = every - self.lines = 0 - self.observations = [] - self.errors = [] - - def feed(self, line): - super().feed(line) - self.lines += 1 - if self.lines % self.every: - return - try: - with gzip.open(self.path, 'rt') as fh: - self.observations.append(fh.read()) - except Exception as exc: # noqa: BLE001 -- that's the point - self.errors.append((self.lines, type(exc).__name__)) - - -def _log_body(start, count, rng): - """Timestamped, poorly-compressible pod log lines, as kubelet serves them.""" - return "".join( - "2026-07-30T00:00:00.%09dZ %064x %064x\n" - % (i, rng.getrandbits(256), rng.getrandbits(256)) - for i in range(start, start + count) - ) - - -def test_collector_append_never_exposes_a_partial_member_to_a_reader(tmp_path, monkeypatch): - """The archive on disk must only ever hold complete members. - - Two polls. The first settles a complete member. The second is a large poll - -- well inside MAX_POLL_CHARS -- during which a reader inspects the file - every 250 lines. Every one of those reads must succeed and must see exactly - the last settled content. - """ - monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(lc, 'token', lambda: 'test-token') - path = lc.base('300', 1) + '.log.gz' - rng = random.Random(7) - - first = asyncio.run(lc._poll_once( - _FakeSession(_log_body(0, 5, rng)), 'pod-a', '300', 1, None, - lc.TxApplyScanner())) - last_ts, gone = first - assert not gone - with gzip.open(path, 'rt') as fh: - settled = fh.read() - assert settled, "first poll wrote nothing; the test has no baseline" - - watcher = _ReadingScanner(path) - asyncio.run(lc._poll_once( - _FakeSession(_log_body(5, 12000, rng)), 'pod-a', '300', 1, last_ts, watcher)) - - assert watcher.observations, "the reader never got to look" - assert watcher.errors == [], ( - f"{len(watcher.errors)} of {len(watcher.errors) + len(watcher.observations)} " - f"mid-append reads hit a torn member, e.g. {watcher.errors[:3]}") - assert set(watcher.observations) == {settled}, ( - "a reader saw content that was neither the previous complete archive " - "nor the finished one") - - # And the append still did its job once it finished. - with gzip.open(path, 'rt') as fh: - final = fh.read() - assert final.startswith(settled) - assert len(final.splitlines()) == 12005 diff --git a/src/MissionParallelCatchup/tests/collector/test_poll_backoff.py b/src/MissionParallelCatchup/tests/collector/test_poll_backoff.py deleted file mode 100644 index fdf43b03..00000000 --- a/src/MissionParallelCatchup/tests/collector/test_poll_backoff.py +++ /dev/null @@ -1,227 +0,0 @@ -"""RACE #4 -- the _wake Event is never cleared, so the terminal-poll backoff -never sleeps and TERMINAL_POLL_ATTEMPTS is spent in a tight loop. - -These tests run the real `log_collector.poll_pod` against a fake kubelet log -endpoint and assert on what ends up on the logs volume (.log.gz, .metrics) and -on when the requests were actually issued. No source text is inspected: the -existing suite already pattern-matches this loop and still shipped the bug. - -The scenario is the one that costs data in production: a worker pod goes -terminal and the kubelet needs a moment before it will serve the container's -final log. The collector is supposed to absorb that with three spaced retries. -With a sticky Event it burns all three inside a millisecond and finalizes on -nothing, losing the range's log, its txApply and its final peaks. -""" - -import asyncio -import gzip -import json -import os -import time - -import pytest - -import config -import log_collector as lc - -# Everything is scaled down from the shipped 10s so the tests run in ~2s. The -# ratios are what matter: the kubelet gate opens well after a millisecond-fast -# giveup and well before the third attempt of a correctly-spaced retry. -POLL = 0.2 # LOG_POLL_SECONDS under test -GATE = 0.4 # how long the kubelet 500s before serving the final log -ATTEMPTS = 3 # TERMINAL_POLL_ATTEMPTS, the shipped default - -POD = 'pc-r300-a1-00001' -END = '300' -ATTEMPT = '1' - -# What stellar-core prints on its way out. The medida block is the only place -# txApply exists -- the pod is about to be reaped, so if this read is missed the -# number is gone for good. -FINAL_LOG = ( - "2026-07-30T00:00:01Z catchup ledger 42000000\n" - "2026-07-30T00:00:02Z metric 'ledger.transaction.apply'\n" - "2026-07-30T00:00:03Z count = 12\n" - "2026-07-30T00:00:04Z sum = 1500.0ms\n" - "2026-07-30T00:00:05Z catchup completed\n" -) -EXPECTED_TX_SECONDS = 1.5 - - -# --- fake kubelet log endpoint ---------------------------------------------- - -class _Resp: - def __init__(self, status, body): - self.status = status - self._body = body.encode() - - async def __aenter__(self): - return self - - async def __aexit__(self, *exc): - return False - - def raise_for_status(self): - if self.status >= 400: - raise RuntimeError(f"HTTP {self.status}") - - @property - def content(self): - body = self._body - - class _Chunks: - async def iter_chunked(self, n): - for i in range(0, len(body), n): - yield body[i:i + n] - - return _Chunks() - - -class FakeKubelet: - """Serves one pod's log, with a delay before the final read is available. - - `open_after=None` never serves. Timestamps every request so a test can see - whether the retries were spaced or fired back to back. - """ - - def __init__(self, open_after, body=FINAL_LOG): - self.open_after = open_after - self.body = body - self.requests = [] - - def get(self, url, params=None, headers=None): - now = time.monotonic() - self.requests.append(now) - if self.open_after is None or now - self.requests[0] < self.open_after: - return _Resp(500, '') - return _Resp(200, self.body) - - @property - def span(self): - return self.requests[-1] - self.requests[0] - - -# --- driver ------------------------------------------------------------------ - -async def _drive(session, terminal_at_start=True, flip_after=None, timeout=10): - """Run the real poll_pod, with a stand-in for one main-loop wake cycle. - - main() marks a pod terminal and then does `if name in _wake: set()`. That - key only exists once the poller has reached its first wait, so the real loop - lands the wake on a later cycle -- reproduced here by waiting for the key. - The wake is delivered ONCE, as one main-loop cycle would: the bug is that - one set is enough to disable every wait that follows. - """ - terminal = {'v': terminal_at_start} - - async def main_loop_wake(): - if flip_after is not None: - await asyncio.sleep(flip_after) - terminal['v'] = True - while POD not in lc._wake: - await asyncio.sleep(0.001) - lc._wake[POD].set() - - waker = asyncio.create_task(main_loop_wake()) - try: - await asyncio.wait_for( - lc.poll_pod(session, POD, END, ATTEMPT, - lambda p: terminal['v'], # done() - lambda p: False), # done_ok(): pod Failed - timeout=timeout) - finally: - waker.cancel() - - -@pytest.fixture -def logs(tmp_path, monkeypatch): - monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(lc, 'token', lambda: 'tok') - monkeypatch.setattr(lc, 'LOG_POLL_SECONDS', POLL) - monkeypatch.setattr(lc, 'TERMINAL_POLL_ATTEMPTS', ATTEMPTS) - for d in (lc._wake, lc._pod_secs, lc._anon_peak, lc._ws_peak, - lc._eph_peak, lc._peak_flushed, lc._streaming): - d.clear() - yield tmp_path - for d in (lc._wake, lc._pod_secs, lc._anon_peak, lc._ws_peak, - lc._eph_peak, lc._peak_flushed, lc._streaming): - d.clear() - - -def _metrics(d): - path = os.path.join(d, f'range-{END}-a{ATTEMPT}.metrics') - if not os.path.exists(path): - return {} - with open(path) as fh: - return json.load(fh) - - -def _archive(d): - path = os.path.join(d, f'range-{END}-a{ATTEMPT}.log.gz') - if not os.path.exists(path): - return '' - with gzip.open(path, 'rt') as fh: - return fh.read() - - -def _done(d): - return os.path.exists(os.path.join(d, f'range-{END}-a{ATTEMPT}.done')) - - -# --- tests ------------------------------------------------------------------- - -def test_a_terminal_pods_final_log_survives_a_moment_of_kubelet_lag(logs): - # The pod is already terminal when its stream opens (Failed is a pollable - # phase). The kubelet cannot serve the container's log yet -- the ordinary - # case, it needs a moment after termination -- so the first reads 500. - # TERMINAL_POLL_ATTEMPTS exists precisely to ride that out, and the gate - # here opens inside the window three spaced retries cover. - kubelet = FakeKubelet(open_after=GATE) - asyncio.run(_drive(kubelet)) - - assert _done(logs), "attempt never finalized" - assert len(kubelet.requests) == ATTEMPTS, ( - f"expected the {ATTEMPTS}-attempt budget, saw {len(kubelet.requests)}") - - m = _metrics(logs) - assert m.get('txApplySeconds') == EXPECTED_TX_SECONDS, ( - "the retry budget was spent before the kubelet could answer: txApply " - f"lost (metrics={m}, retries spanned {kubelet.span * 1000:.1f}ms)") - assert 'sum = 1500.0ms' in _archive(logs), ( - "the range's final log was never captured") - assert 'catchup completed' in _archive(logs) - - -def test_the_terminal_retry_budget_is_spent_over_time_not_in_one_millisecond(logs): - # Same pod, but the log endpoint never recovers. What is under test is the - # shape of the giveup: three attempts must be spread across the backoff, - # not fired back to back. Anything less and the budget is decorative. - kubelet = FakeKubelet(open_after=None) - asyncio.run(_drive(kubelet)) - - assert _done(logs), "attempt never finalized" - assert len(kubelet.requests) == ATTEMPTS, ( - f"expected the {ATTEMPTS}-attempt budget, saw {len(kubelet.requests)}") - assert kubelet.span >= POLL, ( - f"{ATTEMPTS} terminal polls were spent in {kubelet.span * 1000:.1f}ms; " - f"they should span at least one {POLL}s backoff") - - -def test_a_pod_going_terminal_still_cuts_the_routine_wait_short(logs, monkeypatch): - # Guard on the other side of the fix: clearing the Event must not turn the - # wait back into a blind sleep. The poll interval is 5s here; the pod goes - # terminal just after the first poll, and its final read has to happen - # within the pod-list cadence, not 5s later. - monkeypatch.setattr(lc, 'LOG_POLL_SECONDS', 5.0) - kubelet = FakeKubelet(open_after=0) # always serves - - started = time.monotonic() - asyncio.run(_drive(kubelet, terminal_at_start=False, flip_after=0.05, - timeout=2)) - elapsed = time.monotonic() - started - - assert _done(logs) - assert _metrics(logs).get('txApplySeconds') == EXPECTED_TX_SECONDS - assert elapsed < 2, ( - f"final read waited {elapsed:.2f}s for a pod that went terminal " - "immediately; the wake was not delivered") diff --git a/src/MissionParallelCatchup/tests/conftest.py b/src/MissionParallelCatchup/tests/conftest.py deleted file mode 100644 index 7dd35639..00000000 --- a/src/MissionParallelCatchup/tests/conftest.py +++ /dev/null @@ -1,330 +0,0 @@ -"""Test harness for job_monitor: a fake cluster wired into the real module. - -`cluster` is the fixture. It replaces job_monitor's module-level API clients -with fake_k8s, points every path the monitor writes at tmp_path, and hands back -a driver that runs the real reconcile() -- no source extraction, no mirrors. - - def test_something(cluster): - cluster.reconcile() # one real reconcile pass - cluster.advance(300, 'succeeded') # move a range to a named state - cluster.reconcile() - assert '300' in cluster.progress()['completed'] - -Nothing here weakens the monitor: the only production change this needed was -making import side-effect free (log dir fallback, in-cluster config guarded on -KUBERNETES_SERVICE_HOST). Every decision under test is the shipped code path. -""" - -import gzip -import json -import os - -import pytest - -import fake_k8s -import config -import kube -import records -import job_monitor as jm - -# Config the fixture pins. Small on purpose: three ranges and PARALLELISM 2 so -# dispatch capacity, retry and completion are all observable in a few passes. -DEFAULT_CONFIG = { - 'NAMESPACE': 'catchup-test', - 'RUN_NAME': 'pc', - 'CORE_IMAGE': 'stellar/stellar-core:test', - 'STARTING_LEDGER': 0, - 'LATEST_LEDGER_NUM': 300, - 'LEDGERS_PER_JOB': 100, - 'OVERLAP_LEDGERS': 320, - 'RANGE_ORDER': 'tip-first', - 'PARALLELISM': 2, - 'STORAGE_MODE': 'pvc', - 'STORAGE_SIZE': '40Gi', - 'STORAGE_CLASS': 'gp3', - 'SAVE_SUCCESS_LOGS': True, - 'PROFILE_PATH': '', - 'ATTEMPT_DEADLINE_SECONDS': 0, - # The whole retry policy. Patch this map, not the MAX_* constants: those are - # only the env seam and the map is built from them once, at import. - 'ATTEMPT_BUDGETS': {'disrupted': 100, 'rejected': 100, 'fetch-fault': 20, - 'oom': 5, 'ephemeral': 4}, - 'LIM_EPHEMERAL': '', - 'REQ_EPHEMERAL': '', -} - -# What advance() does to the fake cluster for each name. The verdict the monitor -# then reaches is its own business -- that is the thing under test. -# The cascade GetHistoryArchiveStateWork prints when a HAS fetch fails, ending in -# the give-up line. exit3_retry_cause() reads this to decide whether an exit-3 -# attempt is retryable, so a fixture exit-3 has to carry it to be retried. -FETCH_FAULT_ARCHIVE = ( - 'fatal error: Could not connect to the endpoint URL: ' - '"https://sts.us-east-1.amazonaws.com/"\n' - '2026-01-01T00:00:00.000 GAJSL [Process WARNING] process 1 exited 1: ' - 'aws s3 cp --no-progress s3://bucket/history-00000000.json /data/tmp\n' - '2026-01-01T00:00:00.000 GAJSL [History WARNING] Could not download file: ' - 'archive core_live_003 maybe missing file history/00/00/00/history-0.json\n' - '2026-01-01T00:00:00.000 GAJSL [History ERROR] Missing HAS for ledger 1: ' - 'maybe stale archive core_live_003\n' - '2026-01-01T00:00:00.000 GAJSL [History WARNING] Catchup failed\n') - -# Same give-up, no fetch cascade in front of it: what a SIGTERM drain leaves. -BARE_FAILURE_ARCHIVE = ( - '2026-01-01T00:00:00.000 GAJSL [Ledger INFO] Ledger close complete: 42\n' - '2026-01-01T00:00:00.000 GAJSL [History WARNING] Catchup failed\n') - -_ARCHIVES = {'fetch_fault': FETCH_FAULT_ARCHIVE, 'bare': BARE_FAILURE_ARCHIVE} - -STATES = ( - 'pending', # dispatched, nothing scheduled yet - 'running', # pod Running, job active - 'succeeded', # exit 0 - 'incomplete', # exit 3 with a fetch fault in the archive: retryable - 'unexplained', # exit 3 with nothing in the archive to explain it: condemned - 'condemned', # exit 1: genuine catchup failure, no retry - 'oom', # exit 137 / OOMKilled - 'disrupted', # DisruptionTarget condition -- spot eviction - 'ephemeral', # kubelet eviction for exceeding the ephemeral-storage limit - 'rejected', # kubelet refused the pod before any container ran - 'timeout', # activeDeadlineSeconds fired - 'unknown', # job failed, pod already reaped, nothing classified it - 'no_exit_code', # container terminated, kubelet never filled in the exit code -) - - -class Driver: - """Runs reconcile passes against the fake cluster and inspects the results.""" - - def __init__(self, k8s, tmp_path, env): - self.k8s = k8s - self.jm = jm - self.tmp_path = tmp_path - self.config = env - self.namespace = env['NAMESPACE'] - self.run_name = env['RUN_NAME'] - self.log_dir = config.LOG_DIR - # Same dict reconcile_loop() carries across iterations of the - # loop, so multi-pass tests see the real cross-pass behaviour (halt on - # regression, histogram replay guard, counter deltas). - self.state = {'owner': None, 'replayed': set(), 'max_completed': 0, - 'halted': False, 'counted': {}} - self.results = [] - - # -- driving ------------------------------------------------------------- - - def reconcile(self): - """One real reconcile() pass. Returns the summary dict it produces.""" - if self.state['owner'] is None: - self.state['owner'] = jm.owner_ref() - jm._progress_owner['ref'] = self.state['owner'] - result = jm.reconcile(self.state) - self.results.append(result) - return result - - def advance(self, end, state, attempt=None): - """Move a range's Job/Pod to a named state, as the cluster would. - - `end` is the range end (int or str); attempt defaults to the newest Job - this range has. - """ - if state not in STATES: - raise ValueError(f"unknown state {state!r}; expected one of {STATES}") - name = self.job_name(end, attempt) - pod = self.k8s.pod_for_job(name) - pod_name = pod.metadata.name if pod is not None else None - - if state == 'pending': - return name - if state == 'running': - self.k8s.set_job_running(name) - return name - if state == 'succeeded': - if pod_name: - self.k8s.set_pod_terminated(pod_name, exit_code=0) - self.k8s.set_job_succeeded(name) - return name - - # Everything below is a failure; the Job condition and the pod detail - # are set independently because in a real run either can be missing. - if state in ('incomplete', 'unexplained'): - if pod_name: - self.k8s.set_pod_terminated(pod_name, exit_code=3) - self.k8s.set_job_failed(name, message=self._policy_msg(pod_name, 3, 2)) - elif state == 'condemned': - if pod_name: - self.k8s.set_pod_terminated(pod_name, exit_code=1) - self.k8s.set_job_failed(name, message=self._policy_msg(pod_name, 1, 2)) - elif state == 'oom': - if pod_name: - self.k8s.set_pod_terminated(pod_name, exit_code=137, reason='OOMKilled') - self.k8s.set_job_failed(name, message=self._policy_msg(pod_name, 137, 1)) - elif state == 'disrupted': - if pod_name: - self.k8s.set_pod_condition(pod_name, 'DisruptionTarget', - reason='TerminationByKubelet') - self.k8s.set_pod_terminated(pod_name, exit_code=3) - self.k8s.set_job_failed(name, message=self._policy_msg(pod_name, None, 0)) - elif state == 'ephemeral': - if pod_name: - # stellar-core drains on the eviction SIGTERM and exits 3, so the - # exit code alone is indistinguishable from a catchup failure; - # status.message is the only discriminator. - self.k8s.set_pod_terminated(pod_name, exit_code=3, phase='Failed') - self.k8s.set_pod_phase( - pod_name, 'Failed', reason='Evicted', - message=('Pod ephemeral local storage usage exceeds the total ' - 'limit of containers 40Gi')) - self.k8s.set_job_failed(name, message=self._policy_msg(pod_name, 3, 2)) - elif state == 'rejected': - if pod_name: - self.k8s.set_pod_phase(pod_name, 'Failed', - reason='VolumeAttachmentLimitExceeded', - message='Node has reached its volume ' - 'attachment limit, rejecting pod') - self.k8s.set_job_failed(name, reason='BackoffLimitExceeded', - message='Job has reached the specified backoff limit') - elif state == 'timeout': - if pod_name: - self.k8s.set_pod_terminated(pod_name, exit_code=3) - self.k8s.set_job_failed(name, reason='DeadlineExceeded', - message='Job was active longer than specified deadline') - elif state == 'unknown': - if pod_name: - self.k8s.delete_pod(pod_name) - self.k8s.set_job_failed(name, reason=None) - elif state == 'no_exit_code': - # The container terminated but the kubelet never populated an exit - # code, so nothing on the pod says why it stopped. Real: observed on - # range 59018943, 2026-07-30. - if pod_name: - # The only terminated status left on the pod belongs to the - # sidecar, which exited cleanly; stellar-core's never landed. So - # classify() finds a terminated container, none of them non-zero, - # and falls off the end of its loop. - self.k8s.set_pod_terminated(pod_name, exit_code=0, - container='log-collector', - phase='Failed') - self.k8s.set_job_failed(name, reason='BackoffLimitExceeded', - message='Job has reached the specified backoff limit') - return name - - def _policy_msg(self, pod_name, code, rule_index): - """A podFailurePolicy failure message in the Job controller's own format.""" - if code is None: - return (f"Container stellar-core for pod {self.namespace}/{pod_name} " - f"matching FailJob rule at index {rule_index}") - return (f"Container stellar-core for pod {self.namespace}/{pod_name} failed " - f"with exit code {code} matching FailJob rule at index {rule_index}") - - # -- the collector's side of the contract -------------------------------- - - def finalize(self, end, attempt=1, tx_apply=None, peaks=None, resumed=False, - attempt_seconds=None, archive=None): - """Write what the log-collector sidecar writes for a finished attempt. - - The monitor will not reap a Job until the .done marker exists, and reads - peaks and txApply out of .metrics -- so a test that wants either of those - paths has to stand in for the collector. - """ - data = dict(peaks or {}) - if tx_apply is not None: - data['txApplySeconds'] = tx_apply - if attempt_seconds is not None: - data['attemptSeconds'] = attempt_seconds - if resumed: - data['resumed'] = True - if archive in _ARCHIVES: - archive = _ARCHIVES[archive] - if archive is not None: - # exit 3 is classified from the archive, so a test driving that path - # has to stand in for what the worker wrote as well. - self.archive(end, attempt, archive) - self.write(records.metrics_path(str(end), attempt), json.dumps(data)) - self.write(records.done_path(str(end), attempt), '') - - def archive(self, end, attempt, text): - """Lay down an attempt's gzipped worker archive, with no .done marker. - - `text` may be one of the shorthands finalize() takes ('fetch_fault', - 'bare'). Writing the archive alone is what the collector's mid-append - state looks like. - """ - text = _ARCHIVES.get(text, text) - with gzip.open(records.log_path(str(end), attempt), 'wb') as fh: - fh.write(text.encode()) - - def write(self, path, text): - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, 'w') as fh: - fh.write(text) - return path - - # -- inspection ---------------------------------------------------------- - - def job_name(self, end, attempt=None): - if attempt is not None: - return jm.job_name(int(end), attempt) - prefix = f"{self.run_name}-r{int(end)}-a" - names = [n for n in self.k8s.job_names(self.namespace) if n.startswith(prefix)] - if not names: - raise AssertionError(f"no Job for range {end}; have {self.k8s.job_names()}") - return max(names, key=lambda n: int(n.rsplit('-a', 1)[1])) - - def attempt_of(self, end): - return int(self.job_name(end).rsplit('-a', 1)[1]) - - def jobs(self): - return self.k8s.job_names(self.namespace) - - def pvcs(self): - return self.k8s.pvc_names(self.namespace) - - def progress(self): - """The authoritative progress record, straight off disk.""" - try: - with open(config.PROGRESS_FILE) as fh: - return json.load(fh) - except (OSError, ValueError): - return {} - - def completed(self): - return self.progress().get('completed', {}) - - def failed(self): - return self.progress().get('failed', {}) - - @property - def calls(self): - return self.k8s.calls - - @property - def deleted(self): - return self.k8s.deleted - - -@pytest.fixture -def cluster(tmp_path, monkeypatch): - env = dict(DEFAULT_CONFIG) - log_dir = tmp_path / 'logs' - log_dir.mkdir() - - k8s = fake_k8s.FakeCluster(namespace=env['NAMESPACE']) - monkeypatch.setattr(kube, 'core_v1', k8s.core_v1) - monkeypatch.setattr(kube, 'batch_v1', k8s.batch_v1) - - for key, value in env.items(): - monkeypatch.setattr(config, key, value) - # Derived at import from RUN_NAME / LOG_DIR, so they have to follow. - monkeypatch.setattr(config, 'LOG_DIR', str(log_dir)) - monkeypatch.setattr(config, 'PROGRESS_FILE', str(log_dir / 'progress.json')) - # Module-level mutable state that would otherwise leak between tests. - monkeypatch.setattr(config, 'PROFILE', None) - monkeypatch.setattr(jm, '_progress_owner', {}) - - # The chart's ConfigMap: owner_ref() reads it, and every Job, PVC and the - # progress ConfigMap hang off it. - k8s.add_config_map(f"{env['RUN_NAME']}-stellar-core-config", - {'stellar-core.cfg': '# test'}) - - return Driver(k8s, tmp_path, env) diff --git a/src/MissionParallelCatchup/tests/contract/_artifacts.py b/src/MissionParallelCatchup/tests/contract/_artifacts.py deleted file mode 100644 index dabb0dbb..00000000 --- a/src/MissionParallelCatchup/tests/contract/_artifacts.py +++ /dev/null @@ -1,187 +0,0 @@ -"""The artifacts a contract test compares, loaded once per session. - -A contract test pins agreement across a boundary that behaviour cannot reach -from inside Python: the helm chart against the code that reads its env vars, -the RBAC Role against the API calls the code makes, the F# mission driver -against the chart and the monitor it drives, and captured output from -Kubernetes and stellar-core against the parsers that decode it. - -Reading files as text is the point here. The rule that keeps it honest: assert -the INVARIANT, never one spelling of correct code. If a test can only be -satisfied by the exact call that happens to be there today, it will go red over -a correct fix -- which has already happened twice in this suite. -""" - -import functools -import json -import os -import re -import shutil -import subprocess -import sys - -import pytest -import yaml - -HERE = os.path.dirname(os.path.abspath(__file__)) -MODULE_DIR = os.path.dirname(os.path.dirname(HERE)) # src/MissionParallelCatchup -APPS_DIR = os.path.join(MODULE_DIR, 'apps') # the two entrypoints -LIB_DIR = os.path.join(MODULE_DIR, 'lib') # what they import -SRC_ROOT = os.path.dirname(MODULE_DIR) # src -CHART = os.path.join(MODULE_DIR, 'parallel_catchup_helm') -FSHARP_PATH = os.path.join(SRC_ROOT, 'FSLibrary', - 'MissionHistoryPubnetParallelCatchupV2.fs') - -# Container names in the monitor Deployment. The collector is a separate -# container with its own env block, so a variable the monitor has is not -# automatically one the collector has -- STORAGE_MODE was missing there once and -# the ephemeral sampler silently did nothing. -MONITOR_CONTAINER = 'job-monitor' -COLLECTOR_CONTAINER = 'log-collector' - - -@functools.lru_cache(maxsize=None) -def text(path): - with open(path) as fh: - return fh.read() - - -def module_source(module): - """The on-disk source of an imported module.""" - return text(module.__file__) - - -def fsharp(): - return text(FSHARP_PATH) - - -def values_yaml(): - return text(os.path.join(CHART, 'values.yaml')) - - -def job_monitor_template(): - return text(os.path.join(CHART, 'templates', 'job_monitor.yaml')) - - -# --- rendering --------------------------------------------------------------- - -# The mission always sends this; the chart has no usable default for it. -_BASE_SET = ('worker.stellar_core_image=x',) - - -@functools.lru_cache(maxsize=None) -def render(sets=(), release='t'): - """`helm template`, as the mission installs it. `sets` must be a tuple.""" - if not shutil.which('helm'): - pytest.skip('helm not installed') - args = ['helm', 'template', release, CHART] - for s in _BASE_SET + tuple(sets): - args += ['--set', s] - r = subprocess.run(args, capture_output=True, text=True) - assert r.returncode == 0, f"helm template failed:\n{r.stderr}" - return r.stdout - - -@functools.lru_cache(maxsize=None) -def docs(sets=(), release='t'): - return tuple(d for d in yaml.safe_load_all(render(sets, release)) if d) - - -def of_kind(kind, sets=(), release='t'): - return [d for d in docs(sets, release) if d.get('kind') == kind] - - -def monitor_deployment(sets=(), release='t'): - found = of_kind('Deployment', sets, release) - assert len(found) == 1, f"expected one Deployment, got {len(found)}" - return found[0] - - -def containers(sets=(), release='t'): - """{name: container} for the monitor Deployment's pod spec.""" - spec = monitor_deployment(sets, release)['spec']['template']['spec'] - return {c['name']: c for c in spec['containers']} - - -def env_of(container): - """{NAME: value} for the env entries that carry a literal value. - - valueFrom entries (NAMESPACE, from the downward API) are reported with a - value of None: they are set, but the chart does not choose the value. - """ - return {e['name']: e.get('value') for e in (container.get('env') or [])} - - -def role_rules(sets=(), release='t'): - found = of_kind('Role', sets, release) - assert len(found) == 1, f"expected one Role, got {len(found)}" - return found[0]['rules'] - - -def granted(sets=(), release='t'): - """{(apiGroup, resource): {verbs}} the monitor's ServiceAccount holds.""" - out = {} - for rule in role_rules(sets, release): - for group in rule['apiGroups']: - for resource in rule['resources']: - out.setdefault((group, resource), set()).update(rule['verbs']) - return out - - -# --- the code's own defaults, read without ambient env ----------------------- - -_PROBE = """ -import json, sys -sys.path[:0] = [{apps_dir!r}, {lib_dir!r}] -import {module} as m -out = {{}} -for k, v in vars(m).items(): - if k.isupper() and isinstance(v, (int, float, str, bool, type(None))): - out[k] = v -print('<<<' + json.dumps(out) + '>>>') -""" - - -@functools.lru_cache(maxsize=None) -def defaults(module_name, env_pairs=()): - """Module-level UPPERCASE constants as they are with NO env set. - - Read out of a subprocess with a scrubbed environment rather than off the - imported module: the values a test process happens to import depend on - whatever env the developer is running under, and the whole point here is to - compare the chart against the built-in fallback. - - `env_pairs` is a tuple of (name, value) for the few constants that are - derived from an env var at import -- PROGRESS_CM off RUN_NAME, say -- where - the derivation is what a test needs to see. - """ - src = _PROBE.format(apps_dir=APPS_DIR, lib_dir=LIB_DIR, module=module_name) - env = {'PATH': os.environ.get('PATH', ''), 'HOME': os.environ.get('HOME', '')} - env.update(dict(env_pairs)) - r = subprocess.run([sys.executable, '-c', src], capture_output=True, - text=True, env=env, cwd=APPS_DIR) - assert r.returncode == 0, f"could not import {module_name} cleanly:\n{r.stderr}" - body = r.stdout[r.stdout.index('<<<') + 3:r.stdout.rindex('>>>')] - return json.loads(body) - - -_GETENV = re.compile(r"^(\w+)\s*=\s*[^\n]*os\.getenv\(\s*'([A-Z_]+)'", re.M) - - -def env_bindings(source): - """{ENV_VAR: module_constant} for every `X = ... os.getenv('ENV'...)`. - - An env var can be read into more than one name -- LOG_DIR feeds both the - exported LOG_DIR and a module-private copy used to place the monitor's own - log file. The exported constant is the one the rest of the module and these - tests can see, so it wins. - """ - out = {} - for name, env in _GETENV.findall(source): - if env not in out or (name.isupper() and not out[env].isupper()): - out[env] = name - return out - - -def reads_env(source): - return set(re.findall(r"os\.getenv\(\s*'([A-Z_]+)'", source)) diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py b/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py deleted file mode 100644 index c33d3090..00000000 --- a/src/MissionParallelCatchup/tests/contract/test_chart_defaults.py +++ /dev/null @@ -1,235 +0,0 @@ -"""values.yaml against the os.getenv defaults of the code it configures. - -The chart sets these env vars EXPLICITLY, so the rendered value always wins over -the Python fallback and a drift between them is silent. It has shipped twice: -the code default for the profile cache headroom was raised to 512Mi while the -chart still forced 0, and the chart quietly won -- reproducing the exact OOMs -the code change existed to fix (measured: ranges profiled at 190MiB rss got a -209MiB limit and 90 of them were OOMKilled within 90s of dispatch). - -Rather than a hand-curated pair list -- which only covers the constants someone -remembered -- this compares EVERY env var the chart sets against the code -default of the constant that reads it. Deliberate divergences are listed below -with their reason, so a new one has to be argued for rather than merely added. -""" - -import os -import re - -import config as cfg -import units -import job_monitor as jm -import log_collector as lc - -import _artifacts as art - -# Rendered with a profile ConfigMap so the PROFILE_* block is present -- it is -# the block the chart/code split actually bit on. -SETS = ('monitor.profileConfigMap=p',) - -# Every module whose code runs in the container, in the order a name is looked -# for. Both containers read config.py -- the collector's own knobs are its own, -# but NAMESPACE, RUN_NAME, LOG_DIR, STORAGE_MODE and SAVE_SUCCESS_LOGS are -# shared, and were declared in both files until they were not. -CONTAINERS = { - art.MONITOR_CONTAINER: (('config', cfg), ('job_monitor', jm)), - art.COLLECTOR_CONTAINER: (('log_collector', lc), ('config', cfg)), -} - -READERS = {c: tuple(m for _, m in mods) for c, mods in CONTAINERS.items()} - - -def _bindings(cname): - """env var -> (module_name, constant), across every module in the container.""" - out = {} - for module_name, module in CONTAINERS[cname]: - for env, constant in art.env_bindings(art.module_source(module)).items(): - out.setdefault(env, (module_name, constant)) - return out - - -def _code_defaults(cname): - """The built-in defaults of every module in the container, merged.""" - out = {} - for module_name, _ in CONTAINERS[cname]: - for name, value in art.defaults(module_name).items(): - out.setdefault((module_name, name), value) - return out - -# Env vars whose chart value is deliberately NOT the code default. Each one is -# either per-release, per-mission, or a run parameter the mission overrides; in -# every case the code fallback exists only so the module can be imported -# outside a cluster. Nothing here may be a tuning constant. -DELIBERATE = { - 'RUN_NAME': 'the helm release name; the code fallback only names a standalone run', - 'CORE_IMAGE': 'the image under test, supplied per mission run', - 'WORKER_SERVICE_ACCOUNT': 'derived from the release name for IRSA trust', - 'MISSION': 'the mission name, for the kube-state-metrics label', - 'ASAN_OPTIONS': 'passed through to the worker; empty means "unset", not "default"', - 'PARALLELISM': 'worker.replicas -- the whole point of the knob is to differ per run', - 'ATTEMPT_DEADLINE_SECONDS': 'a backstop the chart turns on and the code leaves off', - # StellarKubeSpecs.fs owns worker sizing, so the chart ships these empty on - # purpose and the mission fills them in on every install. - # Pool routing is opt-in: an empty prefix is exactly the pre-tier behaviour, - # and the mission turns the whole thing on by setting only this. The ladder - # itself ships defined -- see test_the_chart_ships_a_coherent_pool_ladder. - 'POOL_PREFIX': 'empty ships pooling off; the mission sets it to opt in', - 'CAPACITY_TYPE': 'empty means no capacity constraint; the mission derives it from storage mode', - 'REQ_CPU': 'left empty in the chart; StellarKubeSpecs.fs supplies it', - 'REQ_MEM': 'left empty in the chart; StellarKubeSpecs.fs supplies it', -} - - -def _same(chart_value, code_value): - """Compare a rendered string against a typed Python default. - - Helm renders everything as a string, so `5` and `5.0` and `true` and `True` - all have to compare equal -- the contract is about the VALUE, not about how - YAML happened to spell it. - """ - if isinstance(code_value, bool): - return chart_value.lower() == str(code_value).lower() - if isinstance(code_value, (int, float)): - try: - return float(chart_value) == float(code_value) - except ValueError: - return False - return chart_value == ('' if code_value is None else str(code_value)) - - -def _pairs(): - """(container, env, chart_value, constant, code_default) for each env set.""" - out = [] - for cname, container in art.containers(SETS).items(): - bindings = _bindings(cname) - code = _code_defaults(cname) - for env, chart_value in art.env_of(container).items(): - if chart_value is None: - continue # valueFrom: the chart picks nothing - found = bindings.get(env) - constant = found[1] if found else None - out.append((cname, env, chart_value, constant, - code.get(found) if found else None)) - return out - - -def test_every_env_the_chart_sets_is_read_by_the_container_that_gets_it(): - """A chart env var no module reads is a knob that does nothing. - - That is the same failure as a constant nothing reads: the values.yaml - comment promises a protection, the rendered Deployment carries it, and - turning it changes nothing at all. - """ - orphans = [(c, e) for c, e, _, constant, _ in _pairs() if constant is None] - assert not orphans, ( - "the chart sets env vars nothing reads: " - + ", ".join(f"{e} on {c}" for c, e in orphans)) - - -def test_no_pinned_default_is_a_constant_nothing_reads(): - """Every constant this file pins must be used past its own assignment. - - A contract test guarding a constant no code consults is worse than nothing: - it certifies a protection that does not exist. MAX_LINE_CHARS was exactly - that and was deleted rather than kept. - """ - dead = [] - for cname, container in art.containers(SETS).items(): - bindings = _bindings(cname) - # Uses are counted across every module in the container, not just the one - # holding the assignment: config.py defines these and job_monitor reads - # them, and several are named differently from their env var. - source = '\n'.join(art.module_source(m) for m in READERS[cname]) - for env in art.env_of(container): - found = bindings.get(env) - if found is None: - continue - module_name, constant = found - uses = len(re.findall(rf"\b{constant}\b", source)) - if uses < 2: - dead.append(f"{module_name}.{constant} (from {env})") - assert not dead, f"assigned from the chart but never read: {dead}" - - -def test_the_chart_value_is_the_code_default(): - """Chart and code must agree wherever the chart is not deliberately different.""" - drift = [] - for cname, env, chart_value, constant, code_value in _pairs(): - if constant is None or env in DELIBERATE: - continue - if not _same(chart_value, code_value): - drift.append(f"{env} on {cname}: chart {chart_value!r} != " - f"code {constant}={code_value!r}") - assert not drift, ( - "the chart overrides the code default with a different value, silently:\n " - + "\n ".join(drift)) - - -def test_the_chart_enables_a_twelve_hour_attempt_backstop(): - env = art.env_of(art.containers()[art.MONITOR_CONTAINER]) - assert env['ATTEMPT_DEADLINE_SECONDS'] == '43200' - - -def test_each_deliberate_divergence_is_still_a_real_env_var(): - """Keeps the allowlist above honest. - - A renamed or dropped env var must not go on being excused here -- that is - how an exemption written for one variable ends up covering its replacement. - """ - known = {env for _, env, _, _, _ in _pairs()} - stale = sorted(set(DELIBERATE) - known) - assert not stale, f"DELIBERATE excuses env vars the chart no longer sets: {stale}" - - -def test_the_chart_does_not_dictate_where_the_profile_lives(): - """PROFILE_PATH is the monitor's own business now. - - The profile arrives with POST /start and is written to the volume, so a - chart-supplied path would point at a ConfigMap mount that no longer exists - and make every run log an unreadable-profile warning. - """ - env = art.env_of(art.containers()[art.MONITOR_CONTAINER]) - assert 'PROFILE_PATH' not in env - - # And nothing mounts a profile volume any more. - mounts = {m['mountPath'] - for m in art.containers()[art.MONITOR_CONTAINER]['volumeMounts']} - assert '/profile' not in mounts - - -def test_the_peak_flush_ratio_is_a_threshold_and_not_a_pass_through(): - """At exactly 1.0 every sample flushes: one write per pod per poll. - - At 2048 pods that is the dominant cost of the sampler, and the ratio exists - to avoid it -- so agreeing with the chart is not enough, it also has to be - above 1. Nothing else pins this: the behaviour tests inject their own ratio. - """ - ratio = art.defaults('log_collector')['PEAK_FLUSH_RATIO'] - assert ratio > 1.0, f"ratio {ratio} flushes on every sample" - - -def test_the_sizing_headroom_is_a_real_allowance_in_both_places(): - """margin and headroom bound each other; neither may be inert. - - A margin below 1.0 shrinks a measured peak, and a headroom of 0 was - measured to OOM 90 small ranges within 90s of dispatch -- memory.max bounds - anon PLUS page cache, so a purely multiplicative margin is meaningless at - small rss (190MiB rss * 1.1 is 19MiB of slack). - - The exact figures are pinned by the test above, against the chart. This one - says what they must remain true of, so retuning them stays possible and - zeroing them does not. - """ - code = art.defaults('config') - assert code['PROFILE_MARGIN'] >= 1.0, "a margin below 1.0 sizes under the measured peak" - headroom = units.quantity_bytes(code['PROFILE_CACHE_HEADROOM']) - assert headroom >= 256 * 1024 ** 2, ( - f"{code['PROFILE_CACHE_HEADROOM']} of fixed headroom is what OOMed 90 small ranges") - assert code['PROFILE_RUNTIME_MEMORY_INSURANCE'] == '3Gi' - # ...and the ceiling has to sit above the configured request, or a range - # measured above it can never ask for what it actually uses and will pack as - # though it were small. - assert (units.quantity_bytes(code['PROFILE_MAX_MEM']) - > units.quantity_bytes(code['REQ_MEM'])), ( - "the profile ceiling is at or below the configured request, so a hungry " - "range can never ask for what it measured") diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py b/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py deleted file mode 100644 index 452a8584..00000000 --- a/src/MissionParallelCatchup/tests/contract/test_chart_env_wiring.py +++ /dev/null @@ -1,299 +0,0 @@ -"""The rendered Deployment against the env vars each process actually reads. - -Two containers share one pod and one volume but not one env block, so a -variable the monitor has is not automatically one the collector has. STORAGE_MODE -was missing from the collector and the peak-ephemeral sampler silently recorded -nothing -- it defaults to 'pvc', which is exactly the mode where the sampler is -supposed to stand down. - -The conditional blocks matter as much as the unconditional ones: node targeting, -taint toleration and the profile mount only render when the mission passes the -matching values, so "the chart sets it" has to be checked with those values -present. -""" - -from pathlib import Path - -import config -import job_monitor as jm -import log_collector as lc - -import _artifacts as art - -# Injected by the kubelet or genuinely optional. Everything else the code reads -# has to come from the chart, or it silently runs on its built-in fallback. -KUBELET_INJECTED = {'KUBERNETES_SERVICE_HOST', 'KUBERNETES_SERVICE_PORT'} - -# Operator-facing switches with no chart key on purpose: they are set by hand on -# a running Deployment when something needs debugging, and a chart key would -# freeze them at install time. -DEBUG_ONLY = {'LOGGING_LEVEL', 'CONNECTION_POOL', 'WORKER_CONTAINER'} - -# Rendered with everything the mission can send, so the conditional blocks are -# present: a profile ConfigMap, a required node label, an avoided node label -# and a tolerated taint. avoidNodeLabels was declared in values.yaml and read -# by no template at all -- absent from here, that stays invisible. -FULL = ( - 'monitor.profileConfigMap=p', - 'worker.requireNodeLabels[0].key=purpose', - 'worker.requireNodeLabels[0].operator=In', - 'worker.requireNodeLabels[0].values[0]=catchup8-spot', - 'worker.avoidNodeLabels[0].key=reserved', - 'worker.avoidNodeLabels[0].operator=NotIn', - 'worker.avoidNodeLabels[0].values[0]=true', - 'worker.tolerateNodeTaints[0].key=catchup8-spot', - 'worker.tolerateNodeTaints[0].effect=NoSchedule', -) - - -def _missing(container_name, module): - reads = art.reads_env(art.module_source(module)) - set_by_chart = set(art.env_of(art.containers(FULL)[container_name])) - return sorted(reads - set_by_chart - KUBELET_INJECTED - DEBUG_ONLY) - - -def test_every_env_the_monitor_reads_is_set_on_the_monitor_container(): - missing = _missing(art.MONITOR_CONTAINER, jm) - assert not missing, f"the monitor reads {missing} but the chart never sets them" - - -def test_every_env_the_collector_reads_is_set_on_the_collector_container(): - missing = _missing(art.COLLECTOR_CONTAINER, lc) - assert not missing, f"the collector reads {missing} but the chart never sets them" - - -def test_liveness_sweep_settings_reach_only_the_monitor(): - monitor = art.env_of(art.containers()[art.MONITOR_CONTAINER]) - collector = art.env_of(art.containers()[art.COLLECTOR_CONTAINER]) - expected = { - 'LIVENESS_PROBE_TIMEOUT_SECONDS': '5', - 'LIVENESS_SWEEP_SECONDS': '15', - 'LIVENESS_MAX_CONCURRENCY': '32', - } - assert {name: monitor.get(name) for name in expected} == expected - assert not set(expected) & set(collector) - - -def test_the_node_targeting_the_mission_sends_reaches_the_monitor(): - """A label/taint the mission passes must arrive as env, not just as YAML. - - The monitor puts the affinity on the WORKER pods it builds; the chart's job - is only to hand it the pair. Rendering the values into some other shape -- - or into the Deployment's own nodeSelector -- would place the monitor and - leave every worker unconstrained. - """ - env = art.env_of(art.containers(FULL)[art.MONITOR_CONTAINER]) - assert env.get('NODE_LABEL_KEY') == 'purpose' - assert env.get('NODE_LABEL_VALUE') == 'catchup8-spot' - assert env.get('TOLERATE_TAINT') == 'catchup8-spot' - - -def test_node_targeting_is_absent_rather_than_empty_when_unset(): - """An empty NODE_LABEL_KEY is how the monitor knows not to constrain a pod. - - Setting it to "" would work by accident today, but the guard the monitor - uses is truthiness of the key, so an empty-string env and an unset env must - stay interchangeable -- and the chart should not emit a knob it is not - configuring. - """ - env = art.env_of(art.containers()[art.MONITOR_CONTAINER]) - for name in ('NODE_LABEL_KEY', 'NODE_LABEL_VALUE', 'TOLERATE_TAINT'): - assert env.get(name, '') == '', f"{name} rendered without a value to carry" - assert art.defaults('config')['NODE_LABEL_KEY'] == '', \ - "the code fallback must be the falsy 'no targeting' value" - - -def test_the_two_containers_run_the_same_image_from_one_build(): - """The monitor and the collector are two entrypoints in one image. - - They share file formats on a shared volume, so shipping them from separate - images would let the pair skew by a release -- which is the failure every - cross-process test in this directory exists to prevent. - """ - cs = art.containers(FULL) - assert (cs[art.MONITOR_CONTAINER]['image'] - == cs[art.COLLECTOR_CONTAINER]['image']) - - -def test_the_collector_is_started_as_the_collector(): - """Same image, so the collector needs an explicit entrypoint. - - Without one it runs the image's default command -- a second job_monitor, - which would be a second writer of progress.json and every Job. - """ - collector = art.containers(FULL)[art.COLLECTOR_CONTAINER] - started = " ".join(collector.get('command', []) + collector.get('args', [])) - assert 'log_collector.py' in started, ( - f"the collector container does not run log_collector.py: {started!r}") - monitor = art.containers(FULL)[art.MONITOR_CONTAINER] - monitor_started = " ".join(monitor.get('command', []) + monitor.get('args', [])) - assert 'log_collector.py' not in monitor_started - - -def test_only_one_monitor_ever_runs(): - """Single writer is what removes the claim/requeue races the redis queue had. - - Two replicas -- or a rolling update that briefly overlaps them -- would give - two processes the same progress.json, the same Job names and the same PVCs, - with no leader election anywhere in the monitor. - """ - spec = art.monitor_deployment(FULL)['spec'] - assert spec['replicas'] == 1 - assert spec['strategy']['type'] == 'Recreate', ( - "a RollingUpdate briefly runs two monitors against one progress record") - - -def test_the_run_name_the_monitor_labels_with_is_the_helm_release(): - """Every Job, PVC and ConfigMap this run owns is found by that label. - - Two releases in one namespace is the normal case on a shared test cluster. - If RUN_NAME were not the release name, one release's reconcile would list - the other's Jobs and reap them. - """ - env = art.env_of(art.containers(release='pc-abc')[art.MONITOR_CONTAINER]) - assert env['RUN_NAME'] == 'pc-abc' - collector = art.env_of(art.containers(release='pc-abc')[art.COLLECTOR_CONTAINER]) - assert collector['RUN_NAME'] == 'pc-abc', \ - "the collector would watch a different run's pods" - assert config.LABEL_RUN == config.LABEL_RUN, \ - "the two processes select on different label keys" - - -def test_the_namespace_comes_from_the_pod_not_from_a_value(): - """helm --namespace and a values key can disagree; the downward API cannot. - - The monitor creates Jobs in NAMESPACE. A stale value there would dispatch a - whole run into a namespace the release does not own. - """ - for name in (art.MONITOR_CONTAINER, art.COLLECTOR_CONTAINER): - entry = [e for e in art.containers(FULL)[name]['env'] - if e['name'] == 'NAMESPACE'] - assert entry, f"{name} has no NAMESPACE" - field = entry[0]['valueFrom']['fieldRef']['fieldPath'] - assert field == 'metadata.namespace', f"{name} reads NAMESPACE from {field}" - - -def test_no_container_declares_the_same_env_var_twice(): - """A duplicate env entry is rejected by the API server, not by helm. - - `helm template` renders duplicates happily and every reader here collapses - env into a dict, so a merge that lands the same block twice looks fine right - up until `helm install`, which fails with - - .spec.template.spec.containers[name="job-monitor"].env: - duplicate entries for key [name="LIVENESS_PROBE_INTERVAL_SECONDS"] - - and leaves a half-created release behind. That is exactly what happened - merging the liveness sampler in on 2026-07-31: both branches carried the - block, in different positions, so neither side conflicted. - - Rendered with FULL so the conditional blocks are present too -- a duplicate - that only appears when a profile ConfigMap is mounted is still a duplicate. - """ - for values in ((), FULL): - for name, container in art.containers(values).items(): - seen = [e['name'] for e in (container.get('env') or [])] - dupes = sorted({n for n in seen if seen.count(n) > 1}) - assert not dupes, ( - f"container {name} declares {dupes} more than once " - f"(values={'FULL' if values else 'defaults'}); " - "the API server rejects the Deployment outright") - - -def test_the_chart_ships_a_coherent_pool_ladder(): - """The ladder must be defined even though pooling ships OFF. - - poolPrefix empty is deliberate -- pool routing is opt-in, and an unset - prefix is exactly the pre-tier behaviour. But the ladder itself has to be - present and well-formed, because the mission turns pooling on by setting - only the prefix: a malformed or out-of-order poolTiers would then route - ranges to tiers whose nodes cannot hold them, which is an OOM per range - rather than a slow run. - - Cuts must ascend. A descending pair would make an earlier tier shadow a - later one and every range past the inversion would land one tier too low -- - measured consequence: a 13.75Gi range on a 14.1Gi node OOMKilled during - bucket-apply, before closing a single ledger. - """ - env = art.env_of(art.containers(FULL)[art.MONITOR_CONTAINER]) - tiers = env.get('POOL_TIERS') - assert tiers, "the chart ships no pool ladder; enabling poolPrefix would route nowhere" - parsed = [] - for item in tiers.split(','): - cut, _, name = item.rpartition(':') - assert name, f"tier entry with no name: {item!r}" - parsed.append((float(cut) if cut else float('inf'), name)) - cuts = [c for c, _ in parsed] - assert cuts == sorted(cuts), f"pool cuts out of order: {tiers}" - assert cuts[-1] == float('inf'), ( - "the last tier must be unbounded, or a range above the final cut has " - f"nowhere to go: {tiers}") - # Every tier that can be routed to needs a cpu claim, or the pod keeps the - # flat REQ_CPU and two of them can share a node -- which defeats the whole - # design: isolating a pod from its neighbours raised throughput 29-92%. - claims = dict(item.split(':') for item in env['POOL_CPU'].split(',')) - for _, name in parsed: - assert name in claims, f"tier {name} has no cpu claim in POOL_CPU" - for extra in (env['POOL_UNPROFILED'], env['POOL_NO_PROFILE']): - assert extra in claims, f"off-ladder pool {extra} has no cpu claim" - - -def test_neither_module_defines_the_same_symbol_twice(): - """A merge can land the same block twice and Python will not complain. - - Both branches carried the worker-liveness subsystem, positioned differently, - so git merged them into two byte-identical copies of _worker_targets, - WorkerLivenessSampler and publish_worker_liveness -- 285 lines that shipped - in the monitor and were never executed, because the later definition binds. - - Nothing catches this on its own: it imports, it renders, it runs. The only - reason it was benign is that the copies happened to be identical; had the - merge taken one edited copy and one stale one, the stale one would silently - have won or lost depending on file order. - - Assignments count too, and only because this test missed one: the same merge - left two `worker_liveness_sampler = WorkerLivenessSampler()` lines with a - function between them. The first instance was constructed and thrown away -- - inert only because __init__ starts no thread. - """ - import ast, collections - for f in sorted(list(Path(art.APPS_DIR).glob('*.py')) - + list(Path(art.LIB_DIR).glob('*.py'))): - tree = ast.parse(f.read_text()) - seen = collections.defaultdict(list) - for node in tree.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - seen[node.name].append(False) # a def is never a coercion - elif isinstance(node, ast.Assign) and len(node.targets) == 1 \ - and isinstance(node.targets[0], ast.Name): - name = node.targets[0].id - # `X = int(X)` is a coercion of the value above it, not a second - # definition -- config.py does this to every liveness knob. - coercion = any(isinstance(x, ast.Name) and x.id == name - for x in ast.walk(node.value)) - seen[name].append(coercion) - dupes = sorted(n for n, hits in seen.items() - if len(hits) > 1 and not all(hits[1:])) - assert not dupes, \ - f"{f.name} defines {dupes} more than once; the later one silently wins" - - -def test_the_chart_never_pins_an_absolute_interpreter_path(): - """A container command must resolve python on PATH, not at /usr/bin. - - Dockerfile.jobmonitor builds on python:3.12-slim, which ships the - interpreter at /usr/local/bin/python3. `/usr/bin/python3` shipped in the - collector's command and failed as StartError -- and the failure is - asymmetric, so it hides: the monitor container inherits the image CMD and - comes up healthy while only the sidecar crashloops, which reads as a sidecar - bug rather than a chart/base-image mismatch. Observed on ssc-test - 2026-08-07, 3 restarts before it was caught. - """ - chart = Path(__file__).resolve().parents[2] / 'parallel_catchup_helm' - for path in chart.rglob('*.yaml'): - for n, line in enumerate(path.read_text().splitlines(), 1): - if 'command:' not in line or line.lstrip().startswith('#'): - continue - assert '/usr/bin/python' not in line and '/usr/local/bin/python' not in line, ( - f"{path.name}:{n} pins an absolute interpreter path, which ties " - f"the chart to one base image: {line.strip()}") diff --git a/src/MissionParallelCatchup/tests/contract/test_chart_rbac.py b/src/MissionParallelCatchup/tests/contract/test_chart_rbac.py deleted file mode 100644 index d6a109d9..00000000 --- a/src/MissionParallelCatchup/tests/contract/test_chart_rbac.py +++ /dev/null @@ -1,165 +0,0 @@ -"""The Role the chart grants against the API calls the two processes make. - -This boundary has failed twice, the same way both times, and both times it was -silent: the Role omitted `delete` on persistentvolumeclaims, so every completed -range logged a 403 warning and leaked its 40Gi volume (measured on ssc-test: -2032 bound PVCs and 79 TiB a third of the way through a 3982-range run, heading -for ~156 TiB -- enough to crash the EBS CSI controller); and it omitted `delete` -on jobs, so nothing reaped a finished Job and the dead ones outnumbered the live -ones within the hour. - -A 403 does not stop the run. That is the whole problem, and it is why this is -checked statically rather than waiting for a cluster to tell us. - -The required verbs are DERIVED from the calls in the source, not listed here: -adding a new call to the monitor must fail this test until the Role catches up. -""" - -import re - -import pytest - -import job_monitor as jm -import log_collector as lc - -import _artifacts as art - -# kubernetes-client method names are `_namespaced_`. Only the -# mapping from client vocabulary to RBAC vocabulary is spelled out; which calls -# exist is read off the source. -VERB = {'read': 'get', 'list': 'list', 'create': 'create', 'delete': 'delete', - 'patch': 'patch', 'replace': 'update', 'watch': 'watch'} - -RESOURCE = { - 'job': ('batch', 'jobs'), - 'pod': ('', 'pods'), - 'pod_log': ('', 'pods/log'), - 'config_map': ('', 'configmaps'), - 'persistent_volume_claim': ('', 'persistentvolumeclaims'), -} - -_CALL = re.compile(r"\b(?:core_v1|batch_v1)\.(\w+?)_namespaced_(\w+)\(") - - -def monitor_calls(): - """{(apiGroup, resource): {verbs}} the monitor's own code needs.""" - need = {} - for verb, resource in _CALL.findall(art.module_source(jm)): - assert verb in VERB, f"unmapped client verb {verb!r}" - assert resource in RESOURCE, f"unmapped client resource {resource!r}" - need.setdefault(RESOURCE[resource], set()).add(VERB[verb]) - return need - - -def test_the_source_really_does_call_the_apiserver(): - """Guards the derivation itself. - - If the call regex stopped matching -- a rename, a wrapper, a different - client object -- every assertion below would pass vacuously while granting - nothing. - """ - need = monitor_calls() - assert len(need) >= 4, f"only found {sorted(need)}; the call scan has gone blind" - assert ('batch', 'jobs') in need and ('', 'persistentvolumeclaims') in need - - -def test_the_role_grants_every_verb_the_monitor_uses(): - have = art.granted() - missing = [] - for key, verbs in sorted(monitor_calls().items()): - for verb in sorted(verbs): - if verb not in have.get(key, set()): - missing.append(f"{verb} on {key[1]} (Role has {sorted(have.get(key, ()))})") - assert not missing, ( - "the monitor makes API calls the Role does not allow; each one is a 403 " - "the run swallows:\n " + "\n ".join(missing)) - - -def test_the_role_grants_what_the_collector_reads(): - """The collector shares the monitor's ServiceAccount -- same pod, same SA. - - It talks to the apiserver over raw HTTP rather than the client library, so - its needs are read out of the URLs it builds. - """ - source = art.module_source(lc) - have = art.granted() - assert re.search(r"/api/v1/namespaces/\{config\.NAMESPACE\}/pods\"", source), \ - "the collector no longer lists pods -- update this test" - assert 'list' in have[('', 'pods')] - assert re.search(r"/pods/\{pod\}/log\"", source), \ - "the collector no longer reads pod logs -- update this test" - assert 'get' in have[('', 'pods/log')] - - -@pytest.mark.xfail(strict=True, reason=( - "GAP: the collector's peak sampler GETs /api/v1/nodes//proxy/stats/summary, " - "which needs `get` on nodes/proxy -- a CLUSTER-scoped resource that a namespaced " - "Role cannot carry however it is spelled. The chart ships no ClusterRole, so every " - "peak this mission profiles from depends on a grant that lives outside this repo. " - "Where that grant is absent the failure is soft and invisible: sample_kubelet logs " - "'kubelet stats unavailable' and continues, peakAnonBytes and peakEphemeralBytes " - "stay empty for the whole run, and the next run's profile looks merely absent " - "rather than broken. Closing it means a ClusterRole plus binding in the chart")) -def test_the_chart_grants_the_kubelet_stats_read_the_sampler_needs(): - source = art.module_source(lc) - assert '/nodes/{node}/proxy/stats/summary' in source, \ - "the sampler no longer proxies to the kubelet -- drop this xfail" - # Cluster-scoped, so a Role cannot carry it however it is spelled. - cluster_roles = art.of_kind('ClusterRole') - granted = {(g, r) - for role in cluster_roles - for rule in role['rules'] - for g in rule['apiGroups'] - for r in rule['resources'] - if 'get' in rule['verbs']} - assert ('', 'nodes/proxy') in granted - - -def test_the_monitor_cannot_touch_a_persistent_volume(): - """Namespaced and PV-free by design. - - Deleting a PVC is reclaim; touching a PV or its finalizers is how an EBS - volume gets orphaned or a VolumeAttachment gets wedged. The blast radius of - a bug in this monitor has to stop at the namespace. - """ - forbidden = {'persistentvolumes', 'nodes', 'volumeattachments'} - reachable = {resource for (_, resource) in art.granted()} - assert not (forbidden & reachable), \ - f"the monitor's Role reaches cluster storage: {sorted(forbidden & reachable)}" - assert not art.of_kind('ClusterRoleBinding'), \ - "a ClusterRoleBinding takes this ServiceAccount outside its namespace" - - -def test_the_role_is_bound_to_the_service_account_the_monitor_runs_as(): - """A Role nobody is bound to grants nothing, and renders perfectly. - - The worker ServiceAccount is deliberately a different one -- IRSA trust for - the S3 history mirror is bound to its name -- so "there is a binding" is not - enough; it has to name the SA on the monitor pod. - """ - binding = art.of_kind('RoleBinding') - assert len(binding) == 1, f"expected one RoleBinding, got {len(binding)}" - binding = binding[0] - role = art.of_kind('Role')[0] - assert binding['roleRef']['name'] == role['metadata']['name'] - subjects = {s['name'] for s in binding['subjects'] if s['kind'] == 'ServiceAccount'} - running_as = art.monitor_deployment()['spec']['template']['spec']['serviceAccountName'] - assert running_as in subjects, ( - f"the monitor runs as {running_as!r} but the Role is bound to {sorted(subjects)}") - assert running_as in {sa['metadata']['name'] for sa in art.of_kind('ServiceAccount')} - - -def test_the_worker_service_account_is_not_the_monitors(): - """Workers must not inherit the monitor's Job/PVC/ConfigMap rights. - - A worker is stellar-core running an untrusted history archive's bytes; the - only credential it needs is IRSA for the S3 mirror. - """ - env = art.env_of(art.containers()[art.MONITOR_CONTAINER]) - worker_sa = env['WORKER_SERVICE_ACCOUNT'] - monitor_sa = art.monitor_deployment()['spec']['template']['spec']['serviceAccountName'] - assert worker_sa != monitor_sa - assert worker_sa in {sa['metadata']['name'] for sa in art.of_kind('ServiceAccount')}, \ - "the workers' ServiceAccount is named but never created" - bound = {s['name'] for b in art.of_kind('RoleBinding') for s in b['subjects']} - assert worker_sa not in bound, "workers were granted the monitor's Role" diff --git a/src/MissionParallelCatchup/tests/contract/test_cross_process_files.py b/src/MissionParallelCatchup/tests/contract/test_cross_process_files.py deleted file mode 100644 index a45c282d..00000000 --- a/src/MissionParallelCatchup/tests/contract/test_cross_process_files.py +++ /dev/null @@ -1,204 +0,0 @@ -"""Two processes, one volume, one set of filenames. - -The monitor and the collector never talk. Everything they agree on is a file on -the shared /logs PVC: the archive, the per-attempt metrics, the verdict, the -resume bookkeeping, and the marker that licenses the monitor to reap a Job. A -disagreement about any of those names is silent -- the reader simply finds -nothing, which reads as "not measured yet" and never as "broken". - -The names are compared by calling both sides' path functions against the same -LOG_DIR, so a refactor that keeps the layout is free. The chart is checked too: -the layout only means anything if both containers mount the same volume there. -""" - -import gzip -import os - -import pytest - -import config -import records -import attempts -import job_monitor as jm -import log_collector as lc - -import _artifacts as art - -END, ATTEMPT = '31005951', 2 - - -@pytest.fixture -def shared(tmp_path, monkeypatch): - """Both modules pointed at one directory, as the pod's volume gives them.""" - monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - return tmp_path - - -# --- the filenames ------------------------------------------------------------ - -def test_both_processes_name_the_same_metrics_file(shared): - """The collector writes it; the monitor reads peaks and txApply out of it.""" - assert records.metrics_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.metrics' - - -def test_both_processes_name_the_same_done_marker(shared): - """The marker is the collector's "I am finished with this attempt". - - The monitor will not reap a Job without it -- and reaping deletes the pod, - which is the last place peaks can still be read from. A mismatch means the - monitor never reaps and every Job waits out its TTL instead. - """ - assert records.done_path(END, ATTEMPT) == lc.done_path(END, ATTEMPT) - - -def test_both_processes_name_the_same_archive_and_verdict(shared): - """The monitor falls back to the archive for txApply and reads .outcome for - the authoritative verdict; the collector writes both.""" - assert records.log_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.log.gz' - assert records.outcome_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.outcome' - assert records.state_path(END, ATTEMPT) == lc.base(END, ATTEMPT) + '.state' - - -def test_the_filenames_carry_the_attempt_as_well_as_the_range(shared): - """Peaks are maxed across a resumed chain, per attempt. - - With one file per range, a retry would overwrite its predecessor instead of - being compared against it -- which destroys exactly the OOM evidence the - chain exists to keep. - """ - for path in (records.metrics_path, records.log_path, records.outcome_path, records.done_path): - assert path(END, 1) != path(END, 2) - assert path('1', 1) != path('2', 1) - - -def test_discarding_a_successful_archive_keeps_what_is_still_read(shared): - """saveSuccessLogs=false drops the bulk of the volume, not the measurements. - - .metrics holds txApply for a range that succeeded, and .done is what lets - the Job be reaped at all. Dropping either would let a log-retention flag - silently delete a Grafana series or strand every finished Job on its TTL. - """ - for suffix in ('.log.gz', '.state', '.metrics', '.done'): - with open(lc.base(END, ATTEMPT) + suffix, 'w') as fh: - fh.write('x') - lc.discard(END, ATTEMPT) - - assert os.path.exists(records.metrics_path(END, ATTEMPT)), "discard dropped the measurements" - assert os.path.exists(records.done_path(END, ATTEMPT)), "discard dropped the reap marker" - assert not os.path.exists(records.log_path(END, ATTEMPT)), "discard kept the archive" - - -def test_the_monitor_can_read_an_archive_the_collector_wrote(shared): - """gzip, appended member by member, read whole. - - The monitor reads it with gzip.open() to decide whether an exit 3 was a - fetch fault. A writer that produced anything other than a concatenation of - complete members would give it a truncated read -- which it treats as no - evidence, and no evidence condemns the range. - """ - path = lc.base(END, ATTEMPT) + '.log.gz' - for chunk in ("first line\n", "second line\n", "last line\n"): - with gzip.open(path, 'ab') as fh: - fh.write(chunk.encode()) - assert [l.strip() for l in attempts._archive_tail(END, ATTEMPT)] == [ - 'first line', 'second line', 'last line'] - - -def test_a_carriage_return_meter_does_not_become_one_giant_line(shared, monkeypatch): - """The AWS CLI draws its transfer meter with \\r and no newline. - - A 628 MiB bucket download therefore arrives as one multi-megabyte "line". - The mission passes --no-progress to stop it at the source (see - test_fsharp_driver_contract), but the collector must not be the only thing - standing between a \\r-heavy line and its own stream: splitting on \\r as - well as \\n is what keeps the archive line-oriented for the monitor's - reader, whatever the worker emits. - """ - import asyncio - - body = ("2026-07-30T00:00:01Z Completed 1.0 MiB\r" - "2026-07-30T00:00:02Z Completed 2.0 MiB\r" - "2026-07-30T00:00:03Z metric 'ledger.transaction.apply'\n" - "2026-07-30T00:00:04Z sum = 1500.0ms\n") - - class _Resp: - status = 200 - async def __aenter__(self): return self - async def __aexit__(self, *exc): return False - def raise_for_status(self): pass - @property - def content(self): - data = body.encode() - class _C: - async def iter_chunked(self, n): - for i in range(0, len(data), n): - yield data[i:i + n] - return _C() - - class _Session: - def get(self, url, params=None, headers=None): - return _Resp() - - monkeypatch.setattr(lc, 'token', lambda: 't') - scanner = lc.TxApplyScanner() - asyncio.run(lc._poll_once(_Session(), 'pod-1', END, ATTEMPT, None, scanner)) - - assert scanner.seconds == pytest.approx(1.5), \ - "the metric block was swallowed by the meter's unterminated line" - with gzip.open(lc.base(END, ATTEMPT) + '.log.gz', 'rt') as fh: - lines = fh.read().splitlines() - assert len(lines) >= 4, f"the meter stayed one blob: {lines}" - - -# --- the volume the layout lives on ------------------------------------------ - -def test_both_containers_mount_one_volume_at_the_directory_they_both_use(): - """The filenames only agree if the directory does. - - Two emptyDirs would render identically and share nothing; a volume mounted - at a different path in each container would give each process its own - private copy of every measurement. - """ - # One constant read through config by both processes now: they cannot - # disagree about the directory, only about mounting it. - log_dir = art.defaults('config')['LOG_DIR'] - - mounts = {} - for name, container in art.containers().items(): - by_path = {m['mountPath']: m['name'] for m in container['volumeMounts']} - assert log_dir in by_path, f"{name} does not mount {log_dir}" - mounts[name] = by_path[log_dir] - assert len(set(mounts.values())) == 1, ( - f"the two containers mount different volumes at {log_dir}: {mounts}") - - volume = mounts[art.MONITOR_CONTAINER] - spec = art.monitor_deployment()['spec']['template']['spec'] - backing = {v['name']: v for v in spec['volumes']}[volume] - assert 'persistentVolumeClaim' in backing, ( - f"{log_dir} is backed by {sorted(backing)} -- every measurement dies with the pod") - - -def test_the_chart_tells_both_containers_where_that_directory_is(): - """LOG_DIR is env, not a constant, so the mount and the env must agree.""" - log_dir = art.defaults('config')['LOG_DIR'] - for name, container in art.containers().items(): - assert art.env_of(container)['LOG_DIR'] == log_dir, name - - -def test_the_progress_record_lives_on_that_volume_too(): - """progress.json is what a restarted monitor reads back, and what the - mission driver `cat`s out of the pod at teardown. - - Written to the monitor's emptyDir instead, an OOM-retry storm's record - would not survive a monitor restart and the mission would build its range - profile from the ConfigMap mirror -- which has every measurement stripped. - """ - assert os.path.dirname(config.PROGRESS_FILE) == config.LOG_DIR - - -def test_the_shared_directory_is_a_single_writer_pvc(): - """One archive per attempt for thousands of ranges, outliving the pod.""" - claims = art.of_kind('PersistentVolumeClaim') - assert len(claims) == 1, "the monitor's log volume is not a PVC" - assert claims[0]['spec']['accessModes'] == ['ReadWriteOnce'], ( - "two writers on one archive; the layout assumes a single collector") diff --git a/src/MissionParallelCatchup/tests/contract/test_dependency_pins.py b/src/MissionParallelCatchup/tests/contract/test_dependency_pins.py deleted file mode 100644 index b48f36f4..00000000 --- a/src/MissionParallelCatchup/tests/contract/test_dependency_pins.py +++ /dev/null @@ -1,79 +0,0 @@ -"""The dependency pins, in the three places they are written and the one that runs. - -The image installs them, and the dev path installs them again at container start -from the chart -- the same list, typed twice more. A divergence there means the -sourceConfigMap run and the built image are different programs. - -The test environment counts too: the suite ran against kubernetes 36.0.3 for -months while the image pinned ~=35.0, so every contract test that builds a real -V1* model was checking the wrong major. That is invisible until a model differs. -""" - -import os -import re - -import _artifacts as art - -DOCKERFILE = os.path.join(art.MODULE_DIR, 'Dockerfile.jobmonitor') - -# name -> the specifier both artifacts must agree on. -_SPEC = re.compile(r"'([a-z0-9-]+)(~=[0-9.]+)'") - - -def _dockerfile_pins(): - text = open(DOCKERFILE).read() - install = text[text.index('RUN pip install'):] - install = install[:install.index('\nCOPY')] - return dict(_SPEC.findall(install)) - - -def _chart_pins(): - """Every `pip install` the chart renders, one dict per occurrence.""" - text = art.text(os.path.join(art.CHART, 'templates', 'job_monitor.yaml')) - out = [] - for line in re.findall(r'pip install --no-cache-dir -q (.+?)&&', text, re.S): - out.append(dict(_SPEC.findall(line))) - return out - - -def test_the_chart_installs_exactly_what_the_image_pins(): - image = _dockerfile_pins() - assert image, "no pins found in the Dockerfile -- the parser is stale" - for n, chart in enumerate(_chart_pins()): - assert chart == image, ( - f"chart pip install #{n + 1} differs from the image: " - f"chart={chart} image={image}") - - -def test_both_containers_install_the_same_list(): - lists = _chart_pins() - assert len(lists) == 2, f"expected one pip install per container, found {len(lists)}" - assert lists[0] == lists[1], f"the two containers install different deps: {lists}" - - -def test_the_test_environment_satisfies_the_pins(): - """What the suite imports must be what the image would install. - - Not a style check: the contract tests construct real V1* models, so a major - the image never installs makes those assertions about a client that does not - ship. - """ - import importlib.metadata as md - drift = [] - for name, spec in _dockerfile_pins().items(): - try: - installed = md.version(name) - except md.PackageNotFoundError: - continue # not needed to run the suite - pinned = spec[2:].split('.')[0] - if installed.split('.')[0] != pinned: - drift.append(f"{name}: pinned {spec}, test env has {installed}") - assert not drift, "the suite is running against a different major:\n " + "\n ".join(drift) - - -def test_the_client_ships_the_async_api_the_pin_was_raised_for(): - """36 was chosen over 35 for kubernetes.aio, which 35 does not contain.""" - import kubernetes.aio # noqa: F401 - from kubernetes.aio import client - assert hasattr(client, 'V1PodFailurePolicyRule'), \ - "the async client must carry the same models the sync one does" diff --git a/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py b/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py deleted file mode 100644 index 18bc423f..00000000 --- a/src/MissionParallelCatchup/tests/contract/test_fsharp_driver_contract.py +++ /dev/null @@ -1,523 +0,0 @@ -"""MissionHistoryPubnetParallelCatchupV2.fs against the chart and the Python. - -The F# driver is the only caller. It installs the chart with a pile of --set -overrides, polls the monitor through a ConfigMap, execs into the monitor pod to -collect logs and to read progress.json, and writes the range-profile artifact -that a LATER run's monitor reads back. Nothing in that loop is type-checked -across the language boundary: a --set key the chart does not know is accepted by -helm and does nothing, a JSON field the driver forgets to project is simply -absent, and a ConfigMap key it looks up under the wrong name reads as "the -monitor has not published yet". - -Every failure in that list is silent, and several have happened. - -The F# is read as text -- there is no dotnet in this suite -- but each test -drives the extracted value through the real chart or the real Python, so what is -pinned is the agreement and not the F#'s spelling of it. -""" - -import json -import os -import re - -import pytest - -import config -import units -import profiles -import records -import sizing -import job_monitor as jm -import log_collector as lc - -import _artifacts as art - -FS = art.fsharp() - - -def fs_extract(pattern, flags=re.S): - m = re.search(pattern, FS, flags) - assert m, f"not found in the F# driver: {pattern}" - return m - - -# --- the --set keys the driver sends ----------------------------------------- - -_SET_KEY = re.compile( - r'(?:worker|monitor|range|service_account)(?:\.[A-Za-z0-9_]+|\[%d\]|\[0\])+(?==)') - - -def set_keys(): - """Every chart value path the driver overrides, indices stripped. - - An indexed path is truncated at the array: `worker.requireNodeLabels[0].key` - is the chart's `worker.requireNodeLabels` list, whose element shape is - checked by rendering it below rather than by looking it up in values.yaml. - """ - out = {} - for raw in set(_SET_KEY.findall(FS)): - out.setdefault(raw.split('[')[0], set()).add(raw) - return out - - -def test_the_driver_really_does_configure_the_chart(): - """Guards the extraction: a regex that stopped matching would pass silently.""" - keys = set_keys() - assert len(keys) >= 15, f"only found {sorted(keys)}; the --set scan has gone blind" - # Sentinels that must keep flowing through --set. The ledger range moved to - # POST /start, so it is deliberately not one of them any more. - assert 'worker.stellar_core_image' in keys and 'worker.replicas' in keys - - -def test_every_helm_command_uses_the_mission_namespace(): - """KUBECONFIG chooses a cluster, but its current namespace is unrelated. - - The Kubernetes client always uses context.namespaceProperty. Every Helm - operation must use that same namespace explicitly or install into the - kubeconfig default, poll sandbox through the client, and wait forever for a - monitor that exists in another namespace. - """ - # Split on the call itself rather than matching one array literal: the install - # builds its argv with Array.concat so it can add a second --values for - # on-demand, and a `[| "helm" ... |]` pattern silently stopped seeing it. - calls = [seg for seg in re.split(r'\bRunShellCommand\b', FS)[1:] - if re.match(r'[\s(]*(?:Array\.concat\s*\[\s*)?\[\|\s*"helm"', seg)] - assert len(calls) == 4, ( - f"expected install, get-values and two cleanup commands; found {len(calls)}") - blocks = calls - for block in blocks: - verb = re.search(r'"(install|get|upgrade|uninstall)"', block) - assert verb, f"could not identify Helm command in {block!r}" - # F# array elements separate with a newline or a semicolon; both appear - assert re.search( - r'"--namespace"\s*;?\s*context\.namespaceProperty', block), ( - f"helm {verb.group(1)} does not target the mission namespace: {block!r}") - - - - -def test_every_value_the_driver_sets_is_one_the_chart_knows(): - """`helm --set` on an unknown path is accepted and ignored. - - A rename in values.yaml, or a typo here, produces a run that installs - cleanly and quietly uses the default for whatever the driver meant to - override -- the wrong image, the wrong ledger range, the wrong storage mode. - """ - values = _values_tree() - templates = _template_text() - unknown = [k for k in sorted(set_keys()) - if not _in_values(values, k) and f".Values.{k}" not in templates] - assert not unknown, ( - "the driver overrides chart values that do not exist; helm accepts them " - f"and does nothing: {unknown}") - - -def test_every_value_the_driver_sets_reaches_a_template(): - """Declared in values.yaml is not the same as consumed. - - A key that exists but is read by nothing renders a perfectly valid manifest - with the setting missing. - """ - templates = _template_text() - inert = [k for k in sorted(set_keys()) if f".Values.{k}" not in templates] - assert not inert, f"declared in values.yaml but read by no template: {inert}" - - -def _values_tree(): - import yaml - return yaml.safe_load(art.values_yaml()) - - -def _template_text(): - tdir = os.path.join(art.CHART, 'templates') - parts = [art.text(os.path.join(tdir, n)) for n in sorted(os.listdir(tdir))] - parts.append(art.text(os.path.join(art.CHART, 'files', 'stellar-core.cfg'))) - return "\n".join(parts) - - -def _in_values(tree, path): - node = tree - for part in path.split('.'): - if not isinstance(node, dict) or part not in node: - return False - node = node[part] - return True - - -# --- the indexed shapes only the mission sends ------------------------------- - -def test_the_service_account_annotations_the_driver_sends_render_as_a_map(): - """metadata.annotations must be a map; the driver sends an indexed array. - - Passing it straight through toYaml produced a list and failed the whole - install with "cannot unmarshal array into ... map[string]string". The --set - strings below are built from the driver's own sprintf format, so a change to - the shape it emits is caught here rather than at install time. - """ - fmt = fs_extract(r'let serviceAccountAnnotationsToHelmIndexed.*?sprintf\s+"([^"]+)"').group(1) - sets = tuple(_fill(fmt, 0, 'eks.amazonaws.com/role-arn', 'arn:aws:iam::1:role/r').split(',')) - for sa in art.of_kind('ServiceAccount', sets): - annotations = sa['metadata'].get('annotations') - assert isinstance(annotations, dict), f"{sa['metadata']['name']}: {annotations!r}" - assert annotations['eks.amazonaws.com/role-arn'] == 'arn:aws:iam::1:role/r' - - -def test_the_chart_still_renders_with_no_annotations_at_all(): - """A hand-run install passes none, and the mission passes none by default.""" - for sa in art.of_kind('ServiceAccount'): - assert not sa['metadata'].get('annotations') - - -def test_the_node_selector_the_driver_sends_reaches_the_monitor(): - """The driver emits structured {key, operator, values} like every other - supercluster mission; a hand-run helm install more naturally passes - "key:value" strings. Both shapes have to arrive as the same env pair.""" - body = fs_extract(r'let requireNodeLabelToHelmIndexed(.*?)\nlet ').group(1) - for fragment in ('worker.requireNodeLabels[%d].key', 'operator=In', '.values[0]='): - assert fragment in body, f"the driver no longer emits {fragment!r}" - structured = ('worker.requireNodeLabels[0].key=purpose', - 'worker.requireNodeLabels[0].operator=In', - 'worker.requireNodeLabels[0].values[0]=catchup8-spot') - env = art.env_of(art.containers(structured)[art.MONITOR_CONTAINER]) - assert (env['NODE_LABEL_KEY'], env['NODE_LABEL_VALUE']) == ('purpose', 'catchup8-spot') - - plain = ('worker.requireNodeLabels[0]=purpose:catchup8-spot',) - env = art.env_of(art.containers(plain)[art.MONITOR_CONTAINER]) - assert (env['NODE_LABEL_KEY'], env['NODE_LABEL_VALUE']) == ('purpose', 'catchup8-spot') - - -def test_the_taint_the_driver_sends_reaches_the_monitor(): - """The driver defaults the effect to NoSchedule and sends no value. - - The monitor builds a Toleration with the default Equal operator, which does - not match "" against "true" -- so the value must stay absent on both sides. - """ - fmt = fs_extract(r'let tolerateTaintToHelmIndexed.*?sprintf\s+"([^"]+)"').group(1) - assert '.effect=' in fmt and '.value' not in fmt - sets = ('worker.tolerateNodeTaints[0].key=catchup8-spot', - 'worker.tolerateNodeTaints[0].effect=NoSchedule') - env = art.env_of(art.containers(sets)[art.MONITOR_CONTAINER]) - assert env['TOLERATE_TAINT'] == 'catchup8-spot' - - -def _fill(fmt, index, *values): - """Apply an F# sprintf format with %d indices and %s values.""" - out, values = fmt.replace('\\"', '"'), list(values) - out = out.replace('%d', str(index)) - for value in values: - out = out.replace('%s', value, 1) - return out - - -# --- the worker command line the driver builds ------------------------------- - -def test_the_driver_disables_the_aws_progress_meter(): - """--no-progress is load-bearing, not cosmetic. - - The AWS CLI draws its transfer meter with carriage returns and no newline, - so a 628 MiB bucket download arrives as one multi-megabyte "line". Measured - on ssc-test 2026-07-30 at 2096 workers: aiohttp aborts a line over 512 KiB, - so every large download killed its own collector stream, the reconnect hit - the same wall, and every retry pod was starved of a stream. The collector - now reads in chunks and splits on \\r too (see test_cross_process_files), - but the cure is not emitting the spam -- it was also the bulk of every large - range's archive. - """ - flags = fs_extract(r'sprintf "aws s3 cp ([^"]*)--region %s"').group(1) - assert '--no-progress' in flags, f"aws s3 cp flags: {flags!r}" - - -def test_the_history_get_command_lands_in_the_config_the_worker_mounts(): - """The S3 mirror override is a per-archive `get` command in stellar-core.cfg. - - Without it the workers fall back to the public archive, which throttles at - 1024 -- silently, as a very slow run rather than an error. - """ - template = fs_extract(r'setOptions\.Add\(sprintf "(worker\.historyGetCommandCore00%d)=').group(1) - for index in (1, 2, 3): - key = template.replace('%d', str(index)) - assert f".Values.{key}" in art.text( - os.path.join(art.CHART, 'files', 'stellar-core.cfg')), \ - f"{key} is set by the driver but never reaches stellar-core.cfg" - - -# --- the ConfigMap the driver polls ------------------------------------------ - - - - - -def test_every_status_field_the_driver_reads_is_one_the_monitor_sets(): - """The driver's loop terminates on num_remain and queue_in_progress_count. - - A field it reads that the monitor never sets throws inside the polling loop, - which the driver treats as fatal: cleanup, uninstall, mission failed -- with - the run's work discarded. - """ - read = set(re.findall(r'status\.(?:\[|Value<\w+>\()"(\w+)"', FS)) - assert read, "the status parse has changed shape -- update this test" - missing = sorted(read - set(jm.status)) - assert not missing, f"the driver reads status fields the monitor never sets: {missing}" - - -def test_the_driver_can_find_the_pod_name_in_a_failed_range_entry(cluster): - """jobs_failed entries are "|", split on '|' by the driver. - - It uses element 1 as a pod name to dump logs from. An entry with no - separator makes that a silent no-op; an entry with the halves swapped makes - it request a pod named after a ledger range. - """ - cluster.reconcile() - cluster.advance(300, 'condemned') - result = cluster.reconcile() - - assert result['failed_ranges'], "no range was condemned; the fixture changed" - entry = result['failed_ranges'][0] - parts = entry.split('|') - assert len(parts) == 2, f"the driver's split('|')[1] cannot work on {entry!r}" - assert parts[1].startswith(f"{cluster.run_name}-r300-a"), ( - f"element 1 is {parts[1]!r}, which is not a pod name") - assert '/' in parts[0], f"element 0 should be the / range key: {parts[0]!r}" - - -# --- the exec paths the driver uses at teardown ------------------------------ - -def test_the_driver_execs_into_a_container_that_exists(): - """A wrong container name fails the exec, and the failure is caught and - logged as a warning -- so the run finishes with no collected logs and no - range profile.""" - names = set(art.containers()) - for name in set(re.findall(r'containerName = "([\w-]+)"', FS)): - assert name in names, f"the driver execs into {name!r}; the pod has {sorted(names)}" - - -def test_the_driver_reads_the_progress_file_where_the_monitor_writes_it(): - """`cat /logs/progress.json`, hard-coded on the driver side.""" - path = fs_extract(r'command = \[\| "cat"; "([^"]+)" \|\]').group(1) - assert path == config.PROGRESS_FILE, ( - f"the driver cats {path}; the monitor writes {config.PROGRESS_FILE}") - - - - - - -def test_the_driver_finds_the_monitor_pod_by_the_labels_the_chart_sets(): - """Two releases share a namespace on a test cluster routinely. - - A selector missing the release label would exec into the other run's monitor - -- and read its progress record. - """ - selector = fs_extract(r'labelSelector = sprintf "([^"]+)"').group(1) - labels = art.monitor_deployment(release='pc-abc')['spec']['template']['metadata']['labels'] - for clause in selector.split(','): - key, _, value = clause.partition('=') - assert key in labels, f"the driver selects on {key!r}; the pod has {sorted(labels)}" - if '%s' not in value: - assert labels[key] == value - else: - assert labels[key] == 'pc-abc' - - -# --- the range-profile artifact: written by F#, read by Python next run ------ - -def fs_profile_fields(): - body = fs_extract(r'let rangeProfileFields =(.*?)\n\n').group(1) - return set(re.findall(r'"(\w+)"', body)) - - -def fs_document_keys(): - return set(re.findall(r'doc\.\["(\w+)"\]\s*<-', FS)) - - -# Recorded but deliberately not carried into the artifact: they are Prometheus -# metrics, and nothing in the next run sizes or orders from them -- wallSeconds -# alone was 349 KB of a 963 KB artifact. Listed rather than inferred so that -# dropping a field the artifact DOES need still fails the test below. -NOT_PROFILED = {'wallSeconds', 'txApply'} - - -def test_the_artifact_carries_every_measurement_the_record_holds(cluster): - """Two lists that must agree, in one direction each. - - Anchored on a record the real monitor wrote, so neither side can drift by - editing a constant. A field the driver projects but the record never holds - lands in the artifact as null; a field the record holds and the driver drops - is lost from the artifact -- peakAnonBytes was exactly that, carried for 0% - of ranges while the volume copy had it for 99%. The only permitted asymmetry - is NOT_PROFILED, enumerated above. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1, tx_apply=1.5, - peaks={'peakAnonBytes': 7, 'peakWorkingSetBytes': 9, - 'peakEphemeralBytes': 11}) - cluster.reconcile() - - measured = set(cluster.completed()['300']) - {'attempts', 'count'} - assert measured, "the monitor recorded no measurements at all" - assert not fs_profile_fields() - measured, ( - f"the driver projects {sorted(fs_profile_fields() - measured)}, which no " - "completion record carries") - assert measured - fs_profile_fields() == NOT_PROFILED, ( - "the artifact drops " - f"{sorted(measured - fs_profile_fields() - NOT_PROFILED)} " - "without that being a deliberate choice recorded in NOT_PROFILED") - - -def test_every_field_the_sizing_consumer_reads_is_in_the_artifact(): - """Derived from _profile_overrides, so a new sizing input fails here first.""" - consumed = set(re.findall(r"prof\.get\('(\w+)'\)", art.module_source(sizing))) - assert consumed, "the sizing consumer no longer reads named fields" - missing = sorted(consumed - fs_profile_fields()) - assert not missing, f"the profile is sized from {missing}, which the artifact drops" - - -def test_every_document_key_the_monitor_reads_is_one_the_driver_writes(): - """storageMode decides whether the disk axis is usable; ranges is the data.""" - read = set(re.findall(r"doc\.get\('(\w+)'\)", art.module_source(profiles))) - assert read, "load_profile no longer reads named document keys" - missing = sorted(read - fs_document_keys()) - assert not missing, f"load_profile reads {missing}, which the driver never writes" - - -def _artifact(storage_mode='pvc', ranges=None): - """A profile document in the exact shape the driver writes.""" - doc = {'schema': 1, 'generated': '2026-07-30T00:00:00.0000000Z', - 'release': 'parallel-catchup-abc', 'storageMode': storage_mode, - 'ledgersPerRange': 16320, 'ranges': ranges or {}} - assert set(doc) == fs_document_keys(), ( - f"this stand-in has drifted from the driver: {set(doc) ^ fs_document_keys()}") - return doc - - -def test_an_artifact_from_a_previous_run_loads_and_sizes_the_next_one(tmp_path, monkeypatch): - """The whole point of the artifact, end to end across the language boundary. - - Values are the driver's own projection of a completed range: keyed by range - end as a STRING (JSON object keys always are), with count alongside the - measurements. - """ - path = tmp_path / 'profile.json' - path.write_text(json.dumps(_artifact(ranges={ - '16752063': {'peakAnonBytes': 2 * 1024 ** 3, 'peakWorkingSetBytes': 13 * 1024 ** 3, - 'seconds': 1200.0, 'count': 16320}}))) - - monkeypatch.setattr(config, 'PROFILE_PATH', str(path)) - monkeypatch.setattr(config, 'STORAGE_MODE', 'pvc') - monkeypatch.setattr(config, 'PROFILE', profiles.load_profile()) - assert config.PROFILE, "the driver's artifact did not load at all" - - sized = sizing._profile_overrides(16752063, escalated=False) - assert 'memory' in sized, "a measured range was not sized from the artifact" - assert (units.quantity_bytes(sized['memory']) > 2 * 1024 ** 3), \ - "the request came out below the measured peak" - - -def test_a_cross_mode_artifact_keeps_memory_and_drops_the_disk_axis(tmp_path, monkeypatch): - """storageMode is in the document because the axes are not interchangeable. - - cpu and memory measure the same work in either mode. Disk does not: a pvc - run puts /data on the volume and never measures node-local usage at all, so - an ephemeral run's figure says nothing about it. - """ - path = tmp_path / 'profile.json' - path.write_text(json.dumps(_artifact(storage_mode='ephemeral', ranges={ - '16752063': {'peakAnonBytes': 2 * 1024 ** 3, - 'peakEphemeralBytes': 30 * 1024 ** 3, 'count': 16320}}))) - - monkeypatch.setattr(config, 'PROFILE_PATH', str(path)) - monkeypatch.setattr(config, 'STORAGE_MODE', 'pvc') - monkeypatch.setattr(config, 'LIM_EPHEMERAL', '40Gi') - monkeypatch.setattr(config, 'PROFILE', profiles.load_profile()) - - sized = sizing._profile_overrides(16752063, escalated=False) - assert 'memory' in sized, "a cross-mode profile was rejected outright" - assert 'ephemeral-storage' not in sized, "a pvc run was sized from ephemeral-mode disk" - - -def test_an_empty_artifact_is_never_written_and_never_fatal(tmp_path, monkeypatch): - """An empty profile is worse than none: it looks complete. - - The usual cause is readProgressRecord falling back to the ConfigMap mirror, - which has every profiling field stripped. Both sides guard it -- the driver - writes nothing, and the monitor treats a profile with no usable range as no - profile -- because either half alone leaves the next run sizing itself from - empty data instead of from its configured requests. - """ - assert re.search(r'if ranges\.Count = 0 then None', FS), \ - "the driver no longer suppresses an empty profile" - - path = tmp_path / 'profile.json' - path.write_text(json.dumps(_artifact(ranges={}))) - monkeypatch.setattr(config, 'PROFILE_PATH', str(path)) - monkeypatch.setattr(config, 'STORAGE_MODE', 'pvc') - monkeypatch.setattr(config, 'PROFILE', profiles.load_profile()) - assert config.PROFILE == [], "an empty profile loaded as if it held something" - assert sizing._profile_overrides(16752063, escalated=False) == {} - - # ...and an artifact that never arrived at all is the same, not an error. - monkeypatch.setattr(config, 'PROFILE_PATH', str(tmp_path / 'absent.json')) - assert profiles.load_profile() == [] - - -def test_every_helm_and_kubectl_call_is_namespaced(): - """A namespace the mission was told to use must reach the shell too. - - helm and kubectl default to the kubeconfig's current context, while the - mission's own Kubernetes client honours context.namespaceProperty. Without - an explicit --namespace those disagree, and a run targeted at one namespace - installs into another. Measured 2026-07-30: a mission run with - `--namespace sandbox` put a job-monitor Deployment and four Jobs into the - production namespace beside a live 2096-worker run. - """ - fs = art.text(art.FSHARP_PATH) - import re - # Every RunShellCommand array invoking helm or kubectl must carry the flag. - calls = re.findall(r'RunShellCommand \[\|\s*"(?:helm|kubectl)".*?\|\]', fs, re.S) - assert calls, "no helm/kubectl shell calls found -- did the driver change shape?" - missing = [c.split('\n')[0] for c in calls if '"--namespace"' not in c] - assert not missing, f"shell calls without --namespace: {missing}" - - -def test_the_driver_pulls_from_the_directory_the_collector_writes_into(): - """The puller and the collector must agree on where artifacts live. - - Replaces the two tar-shape tests: there is no archive any more, so what - matters is that the monitor serves LOG_DIR and the driver asks for the - manifest of it. A mismatch would fetch an empty list and report success - on nothing collected. - """ - fs = open(art.FSHARP_PATH).read() - assert '"/logs"' in fs, "the driver no longer requests the manifest" - assert '"/logs/" + name' in fs, "the driver no longer fetches artifacts by name" - # The monitor serves them out of the volume the collector writes to. - import http_server - assert 'config.LOG_DIR' in art.module_source(http_server) - - -def test_the_puller_does_not_mistake_an_absent_file_for_a_complete_one(): - """A zero-byte artifact must still be fetched. - - .done is empty by design -- its existence is the signal. Comparing lengths - alone makes "absent locally" indistinguishable from "already have it", so - every .done is skipped forever: 88 of 110 artifacts on 2026-08-08, with the - 22 missing being exactly the markers that say an attempt finished. - """ - fs = open(art.FSHARP_PATH).read() - assert 'File.Exists path && have = size' in fs, ( - "the puller compares lengths without checking the file exists, so " - "zero-byte artifacts are never collected") - - -def test_a_retry_window_outlives_the_request_it_retries(): - """A per-request timeout longer than the retry deadline is one attempt. - - Observed 2026-08-08: /start hung on a route that was still programming, the - 10-minute client timeout outlived the 5-minute deadline, and the mission - failed having tried exactly once -- on a route that answered in 37ms a - moment later. - """ - fs = open(art.FSHARP_PATH).read() - assert 'monitorClientWith context (TimeSpan.FromSeconds(15.0))' in fs, ( - "startMission no longer bounds each attempt below its retry deadline") diff --git a/src/MissionParallelCatchup/tests/contract/test_k8s_failure_formats.py b/src/MissionParallelCatchup/tests/contract/test_k8s_failure_formats.py deleted file mode 100644 index 0b5a5f34..00000000 --- a/src/MissionParallelCatchup/tests/contract/test_k8s_failure_formats.py +++ /dev/null @@ -1,321 +0,0 @@ -"""Captured Kubernetes status text against the classifiers that decode it. - -None of these strings are ours. The Job controller's podFailurePolicy condition -message, the kubelet's admission-rejection reasons, its eviction message and the -plain text its log endpoint returns for a container that has not started are all -formats a Kubernetes upgrade can change under us. They are pinned from real -captures so that change fails here rather than degrading a run silently -- a -misread verdict does not stop anything, it just picks the wrong retry budget or -condemns a healthy range. - -Everything is driven through the real classify()/classify_from_job() rather than -a mirror of them, so only the FORMAT is pinned, not the implementation. -""" - -import re -from types import SimpleNamespace as NS - -import pytest - -import config -import job_monitor as jm -import log_collector as lc - -import _artifacts as art - -# --- captures ---------------------------------------------------------------- - -# EKS 1.34 Job condition messages. Only the wording is pinned; pod and container -# names are renamed for readability. -DISRUPTED = ("Pod sandbox/jterm-catchup-snfr2 has condition DisruptionTarget " - "matching FailJob rule at index 0") -OOMKILLED = ("Container oom-container for pod sandbox/oom-test-job-qvq8b failed with " - "exit code 137 matching FailJob rule at index 1") -NONZERO_EXIT = ("Container exit-1-container for pod sandbox/exit-1-job-wbhkq failed with " - "exit code 1 matching FailJob rule at index 2") - -# RECONSTRUCTED 2026-07-30 after an over-broad test deletion removed the -# originals -- twice. Shaped to what the code parses (the rule index, and the -# substring 'ephemeral' in status.message) but no longer a verbatim capture. -# Re-pin from a real eviction on the next run. -EPH_EVICT_JOB_CONDITION = ( - "Container stellar-core for pod stellar-supercluster/" - "parallel-catchup-r31005951-a1-x7k2p failed with exit code 3 " - "matching FailJob rule at index 2") -EPH_EVICT_MESSAGE = ( - "Pod ephemeral local storage usage exceeds the total limit of containers 40Gi") - -# Kubelet reasons seen on ssc-test for a pod refused or removed before -- or -# without -- the container saying anything about the ledger range. None of these -# is evidence that the range is bad. -ADMISSION_REJECTIONS = ('VolumeAttachmentLimitExceeded', 'OutOfcpu', 'OutOfmemory', - 'OutOfpods', 'UnexpectedAdmissionError', 'NodeAffinity', - 'Shutdown', 'Evicted') - - -# --- shims: the two shapes the classifiers read ------------------------------ - -def failed_job(message, reason='PodFailurePolicy'): - return NS(status=NS(conditions=[ - NS(type='Failed', status='True', reason=reason, message=message)])) - - -def pod(reason=None, message=None, disrupted=False, exit_code=None, - terminated_reason=None): - conditions = ([NS(type='DisruptionTarget', status='True')] if disrupted else []) - statuses = [] - if exit_code is not None or terminated_reason is not None: - statuses = [NS(state=NS(terminated=NS(exit_code=exit_code, - reason=terminated_reason)))] - return NS(metadata=NS(name='p'), - status=NS(conditions=conditions, reason=reason, message=message, - container_statuses=statuses)) - - -# --- the Job condition, which is all that is left once the pod is gone ------- - -@pytest.mark.parametrize('message,outcome,code,pod_name', [ - (DISRUPTED, 'disrupted', None, ''), - (OOMKILLED, 'oom', 137, 'oom-test-job-qvq8b'), - (NONZERO_EXIT, 'failed', 1, 'exit-1-job-wbhkq'), -]) -def test_a_job_condition_message_still_parses(message, outcome, code, pod_name): - """Index, exit code and pod name are parsed independently. - - A rule matching on onPodConditions reports no exit code at all, so requiring - one would make the disruption case -- the common case on spot -- unreadable. - """ - verdict = jm.classify_from_job(failed_job(message)) - assert verdict['outcome'] == outcome - assert verdict['exitCode'] == code - assert verdict['pod'] == pod_name - - -def test_the_rule_index_outranks_the_exit_code(): - """A disrupted pod that also exited non-zero must read as disrupted. - - stellar-core catches the eviction SIGTERM and exits 3, so the exit code says - "failed" for something the cluster did to us. Only the index carries the - DisruptionTarget match. - """ - message = ("Container stellar-core for pod ns/p failed with exit code 3 " - "matching FailJob rule at index 0") - assert jm.classify_from_job(failed_job(message))['outcome'] == 'disrupted' - - -def test_a_bare_exit_code_with_no_index_is_still_usable(): - """Some conditions carry the exit code and no rule index. - - Measured on ssc-test 2026-07-28: a drained stellar-core exits 3 in ~7s, well - inside the 100s grace, so evictions do NOT produce 137 -- which makes a bare - 137 an OOM with high confidence, and a bare 3 a real catchup failure. - """ - bare = "Container c for pod ns/p failed with exit code %d" - assert jm.classify_from_job(failed_job(bare % 137))['outcome'] == 'oom' - assert jm.classify_from_job(failed_job(bare % 3))['outcome'] == 'failed' - - -def test_a_condition_with_no_detail_at_all_yields_no_verdict(): - """BackoffLimitExceeded carries no index and no exit code. - - Returning a verdict here would be an invention. "No verdict" is what routes - the range to the environmental budget instead of condemning it -- a monitor - restart while a node was reaped produces exactly this message, and - condemning on it would fail a 10-hour job on no evidence. - """ - assert jm.classify_from_job( - failed_job("Job has reached the specified backoff limit", - reason='BackoffLimitExceeded')) is None - - -def test_the_deadline_is_reported_by_the_job_and_nothing_else(): - """activeDeadlineSeconds fires as its own reason, not as a policy rule. - - The pod that gets SIGTERMed drains and exits 3, which reads as a plain - catchup failure. Only the Job knows the deadline was what killed it. - """ - verdict = jm.classify_from_job( - failed_job("Job was active longer than specified deadline", - reason='DeadlineExceeded')) - assert verdict['outcome'] == 'timeout' - assert verdict['exitCode'] is None - - -# --- the pod, which carries everything the Job cannot ------------------------ - -@pytest.mark.parametrize('reason', ADMISSION_REJECTIONS) -def test_an_admission_rejection_is_not_a_catchup_failure(reason): - """The kubelet refused the pod; stellar-core never ran. - - Observed on ssc-test: reason=VolumeAttachmentLimitExceeded, "Node has - reached its volume attachment limit, rejecting pod". Without this the pod - falls through to 'failed', and a transient admission rejection condemns a - range and kills the whole run. - """ - assert jm.classify(pod(reason=reason))['outcome'] == 'rejected' - - -def test_an_ephemeral_eviction_is_told_apart_from_every_other_eviction(): - """status.message is the only discriminator, and only the pod carries it. - - Measured end-to-end on ssc-test: the kubelet sets no DisruptionTarget for a - limit eviction, and stellar-core drains and exits 3 -- so the Job condition - matches the generic non-zero rule and reads as a plain catchup failure, - which gets no retry at all. Both the ephemeral branch and the generic - Evicted branch key on reason='Evicted', so the ephemeral one has to be - reached first. - """ - assert 'index 2' in EPH_EVICT_JOB_CONDITION, "the Job matches the generic non-zero rule" - assert jm.classify_from_job(failed_job(EPH_EVICT_JOB_CONDITION))['outcome'] == 'failed' - - evicted = pod(reason='Evicted', message=EPH_EVICT_MESSAGE, exit_code=3) - assert jm.classify(evicted)['outcome'] == 'ephemeral' - # ...and an eviction for anything else stays a plain rejection. - other = pod(reason='Evicted', message='The node was low on resource: memory.', - exit_code=3) - assert jm.classify(other)['outcome'] == 'rejected' - - -def payload(reason=None, message=None, disrupted=False, exit_code=None, - terminated_reason=None): - """The same pod as pod(), in the raw JSON shape the collector reads. - - The collector talks to the apiserver over plain HTTP and classifies a dict; - the monitor classifies a client object. Same pod, two spellings. - """ - status = {} - if disrupted: - status['conditions'] = [{'type': 'DisruptionTarget', 'status': 'True'}] - if reason is not None: - status['reason'] = reason - if message is not None: - status['message'] = message - if exit_code is not None or terminated_reason is not None: - term = {} - if exit_code is not None: - term['exitCode'] = exit_code - if terminated_reason is not None: - term['reason'] = terminated_reason - status['containerStatuses'] = [{'state': {'terminated': term}}] - return {'metadata': {'name': 'p'}, 'status': status} - - -CLASSIFIER_CASES = [ - ('a spot reclaim', dict(disrupted=True, exit_code=3)), - ('a disk eviction', dict(reason='Evicted', message=EPH_EVICT_MESSAGE, exit_code=3)), - ('any other eviction', dict(reason='Evicted', message='node was low on memory')), - ('an admission rejection', dict(reason='VolumeAttachmentLimitExceeded')), - ('an oom kill', dict(exit_code=137, terminated_reason='OOMKilled')), - ('a graceful-stop sigkill', dict(exit_code=137, terminated_reason='Error')), - ('a catchup failure', dict(exit_code=1)), - ('an interrupted catchup', dict(exit_code=3)), - ('nothing ever ran', dict()), -] - - -@pytest.mark.parametrize('label,case', CLASSIFIER_CASES, ids=[c[0] for c in CLASSIFIER_CASES]) -def test_both_processes_reach_the_same_verdict_about_the_same_pod(label, case): - """Two independent classifiers, one pod, one answer. - - The collector classifies while the pod still exists and writes the - authoritative .outcome; the monitor classifies again at reconcile when that - file is missing. If they disagreed, a range's verdict -- and therefore which - attempt budget it spends -- would depend on which process saw it first. - """ - from_monitor = jm.classify(pod(**case)) - from_collector = lc.classify(payload(**case)) - assert from_monitor['outcome'] == from_collector['outcome'], label - assert from_monitor['exitCode'] == from_collector['exitCode'], label - - -def test_a_disruption_beats_everything_the_pod_says(): - """A spot reclaim sets the condition and the container still exits 3.""" - assert jm.classify(pod(disrupted=True, exit_code=3))['outcome'] == 'disrupted' - - -def test_an_oom_kill_is_named_by_the_kubelet_not_inferred_from_137(): - """137 is SIGKILL, which the kubelet also uses for a graceful-stop timeout. - - On the pod the reason is available and unambiguous, so it is used; only the - Job-condition path has to infer from the code alone. - """ - assert jm.classify(pod(exit_code=137, terminated_reason='OOMKilled'))['outcome'] == 'oom' - assert jm.classify(pod(exit_code=137, terminated_reason='Error'))['outcome'] == 'failed' - - -def test_a_pod_whose_container_never_terminated_is_not_evidence(): - """Nothing ran, so nothing was learned about the ledger range.""" - assert jm.classify(pod())['outcome'] == 'rejected' - - -def test_the_pod_deadline_is_a_timeout_not_a_catchup_failure(): - """The deadline lives on the PodSpec, so the kubelet fires it and the pod - carries the reason -- the Job only sees a non-zero exit.""" - assert jm.classify(pod(reason='DeadlineExceeded', exit_code=3))['outcome'] == 'timeout' - - -# --- the vocabulary both classifiers speak ----------------------------------- - -def test_every_outcome_the_classifiers_can_produce_has_a_budget(): - """A new outcome string with no branch falls through to "condemn". - - Each outcome routes to one of three attempt budgets. An outcome nobody - routed would take the zero-retry path, and a condemned range fails the - mission. - """ - produced = set() - for source in (art.module_source(jm), art.module_source(lc)): - produced |= set(re.findall(r"'outcome':\s*'(\w+)'", source)) - assert produced <= set(config.ATTEMPT_OUTCOMES), ( - f"outcomes no budget or verdict file knows about: " - f"{sorted(produced - set(config.ATTEMPT_OUTCOMES))}") - assert 'disrupted' in produced and 'rejected' in produced - - -def test_the_deterministic_failures_do_not_get_the_environmental_budget(): - """Only a node disruption gets the effectively unbounded budget. - - It is the one outcome that proves the range itself was fine: the cluster took - the pod away mid-run. Everything else is either a statement about the range - (OOM, disk, hang) or an absence of evidence, and neither earns unlimited - retries -- a run that cannot explain a failure stops instead. - """ - b = config.ATTEMPT_BUDGETS - # A ladder, from "this range is broken" to "the cluster did this to us". - assert b['ephemeral'] <= b['oom'] < b['fetch-fault'] <= b['rejected'] <= b['disrupted'], \ - f"the budget ladder is out of order: {b}" - assert b['disrupted'] == max(b.values()), \ - f"something other than a disruption got the largest budget: {b}" - assert not ({'timeout', 'unknown', 'failed'} & set(b)), \ - "a hang, an unclassifiable failure and a real catchup failure must have "\ - "no budget at all" - - -# --- the log endpoint, which does not always return log lines ---------------- - -def test_untimestamped_kubelet_text_never_becomes_a_resume_point(): - """A pod that has just been replaced returns prose, not log lines. - - Partitioning that on the first space yields "unable", which as a resume - point makes every later request sinceTime=unableZ -> HTTP 400, for the life - of the range. Observed on ssc-test the moment evicted pods were replaced. - """ - kubelet = "unable to retrieve container logs for containerd://9f2c1a" - assert lc._TS_RE.match(kubelet.partition(' ')[0]) is None - for good in ("2026-07-28T20:29:27.927795721Z", "2026-07-28T20:29:27Z"): - assert lc._TS_RE.match(good), good - - -def test_a_poisoned_state_file_is_repaired_rather_than_replayed(tmp_path, monkeypatch): - """The guard has to be on the READ as well as the write. - - A state file written by an earlier build already holds "unable" on some - volumes, and nothing rewrites it until a poll succeeds -- which it cannot, - because the poisoned value is what makes the poll 400. - """ - monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - with open(lc.base('300', 1) + '.state', 'w') as fh: - fh.write('unable') - assert lc.read_state('300', 1) is None - lc.write_state('300', 1, '2026-07-28T20:29:27Z') - assert lc.read_state('300', 1) == '2026-07-28T20:29:27Z' diff --git a/src/MissionParallelCatchup/tests/contract/test_medida_metric_block.py b/src/MissionParallelCatchup/tests/contract/test_medida_metric_block.py deleted file mode 100644 index b449ec34..00000000 --- a/src/MissionParallelCatchup/tests/contract/test_medida_metric_block.py +++ /dev/null @@ -1,192 +0,0 @@ -"""stellar-core's medida metric block against the two parsers that read it. - -txApply is the only per-range performance number this mission produces, and it -exists in exactly one place: the block stellar-core prints once, just before -exit, because we pass --metric 'ledger.transaction.apply'. Both processes parse -it -- the collector out of the live stream (the only reader guaranteed to see -the bytes, since the pod may be reaped and saveSuccessLogs may be off) and the -monitor out of the archive as a fallback. Two parsers, one format. - -The blocks below are whole captures rather than the lines we care about: the -layout IS the contract. `sum` sits ten lines under the header, against a -fifteen-line scan window, so a medida release that adds five percentiles takes -the metric out silently. -""" - -import gzip -import re - -import pytest - -import config -import kube -import records -import medida -import job_monitor as jm -import log_collector as lc - - -# stellar-core 27.1.1 catchup pod, --metric 'ledger.transaction.apply'. -MEDIDA_BLOCK = """2026-07-28T18:39:49.350 GAJSL [default INFO] metric 'ledger.transaction.apply': -2026-07-28T18:39:49.350 GAJSL [default INFO] count = 20 -2026-07-28T18:39:49.350 GAJSL [default INFO] mean rate = 0.22136 calls/s -2026-07-28T18:39:49.350 GAJSL [default INFO] 1-minute rate = 0.113149 calls/s -2026-07-28T18:39:49.350 GAJSL [default INFO] 5-minute rate = 0.175948 calls/s -2026-07-28T18:39:49.350 GAJSL [default INFO] 15-minute rate = 0.191421 calls/s -2026-07-28T18:39:49.350 GAJSL [default INFO] min = 0.295417ms -2026-07-28T18:39:49.350 GAJSL [default INFO] max = 0.639873ms -2026-07-28T18:39:49.350 GAJSL [default INFO] mean = 0.417143ms -2026-07-28T18:39:49.350 GAJSL [default INFO] stddev = 0.108677ms -2026-07-28T18:39:49.350 GAJSL [default INFO] sum = 8.34285ms -2026-07-28T18:39:49.350 GAJSL [default INFO] median = 0ms -2026-07-28T18:39:49.350 GAJSL [default INFO] 75% = 0ms -2026-07-28T18:39:49.350 GAJSL [default INFO] 95% = 0ms -2026-07-28T18:39:49.350 GAJSL [default INFO] 98% = 0ms -2026-07-28T18:39:49.350 GAJSL [default INFO] 99% = 0ms -2026-07-28T18:39:49.350 GAJSL [default INFO] 99.9% = 0ms""" - -TX_APPLY_SECONDS = 0.00834285 - -# Real block from range-40010367-a1 on ssc-test. medida switches to scientific -# notation past 1e6 ms, which is every range with a real transaction load. The -# old [0-9.]+ pattern matched "1.30722", then demanded "ms" and hit "e+06ms" -# instead: 25% of ranges recorded no tx_apply -- 91-99% of everything above -# ledger 35M, exactly the expensive end -- while the block sat in the archive -# the whole time. -MEDIDA_BIG = """2026-07-29T20:11:16.931 GAJSL [default INFO] metric 'ledger.transaction.apply': -2026-07-29T20:11:16.931 GAJSL [default INFO] count = 3231886 -2026-07-29T20:11:16.931 GAJSL [default INFO] mean rate = 812.4 calls/s -2026-07-29T20:11:16.931 GAJSL [default INFO] 1-minute rate = 790.1 calls/s -2026-07-29T20:11:16.931 GAJSL [default INFO] 5-minute rate = 801.3 calls/s -2026-07-29T20:11:16.931 GAJSL [default INFO] 15-minute rate = 799.0 calls/s -2026-07-29T20:11:16.931 GAJSL [default INFO] min = 0.101ms -2026-07-29T20:11:16.931 GAJSL [default INFO] max = 41.2ms -2026-07-29T20:11:16.931 GAJSL [default INFO] mean = 0.404ms -2026-07-29T20:11:16.931 GAJSL [default INFO] stddev = 0.612ms -2026-07-29T20:11:16.931 GAJSL [default INFO] sum = 1.30722e+06ms""" - -TX_APPLY_BIG_SECONDS = 1307.22 - - -def scan(block): - scanner = lc.TxApplyScanner() - for line in block.splitlines(): - scanner.feed(line) - return scanner - - -# --- the layout, which is what the scan window is sized against -------------- - -def test_the_sum_still_sits_inside_the_scan_window(): - """Ten lines below the header, against a fifteen-line window. - - Five more percentiles in a medida release and the metric disappears with no - error anywhere. The margin is the thing to watch, so it is reported. - """ - lines = MEDIDA_BLOCK.splitlines() - header = next(i for i, l in enumerate(lines) if 'ledger.transaction.apply' in l) - offset = next(i for i, l in enumerate(lines) if 'sum =' in l) - header - assert offset == 10, f"medida layout moved: sum is now {offset} lines below the header" - assert offset <= lc.TxApplyScanner.WINDOW, ( - f"the sum is {offset} lines down and the scanner looks {lc.TxApplyScanner.WINDOW}") - - -@pytest.mark.parametrize('gap', [1, 10, lc.TxApplyScanner.WINDOW, - lc.TxApplyScanner.WINDOW + 5]) -def test_the_scanner_reaches_exactly_as_far_past_the_header_as_it_claims(gap): - """One reader now: the collector, live and again over its own archive. - - The window is the whole contract. Measured on ssc-test 2026-08-04, a /info - response landed between the header and its sum 91 lines apart, and a span - that charged every line gave up 76 lines short of a value that was right - there -- which is why the budget counts medida statistics only. - """ - block = ["metric 'ledger.transaction.apply':"] - block += [f" filler {i} = 0ms" for i in range(gap - 1)] - block += [" sum = 1500.0ms"] - - scanner = lc.TxApplyScanner() - for line in block: - scanner.feed(line) - - within = gap <= lc.TxApplyScanner.WINDOW - assert (scanner.seconds is not None) is within, ( - f"a sum {gap} statistics past the header was " - f"{'missed' if within else 'read'} against a window of " - f"{lc.TxApplyScanner.WINDOW}") - - - -# --- the number itself -------------------------------------------------------- - -@pytest.mark.parametrize('block,seconds', [ - (MEDIDA_BLOCK, TX_APPLY_SECONDS), - (MEDIDA_BIG, TX_APPLY_BIG_SECONDS), -]) -def test_both_processes_read_the_same_total_out_of_one_block(block, seconds): - """The collector's scanner and the monitor's regex must not disagree. - - They are separate implementations of the same read: a stream scanner with a - window, and a whole-archive search. progress.json takes whichever one landed - first, so a disagreement is a per-range coin flip. - """ - assert scan(block).seconds == pytest.approx(seconds) - m = medida.SUM_RE.search(block) - assert m, "the monitor's regex does not match this block at all" - assert float(m.group(1)) / 1000.0 == pytest.approx(seconds) - - -def test_scientific_notation_is_the_normal_case_not_the_edge_case(): - """Past 1e6 ms, which every range with real transaction load exceeds.""" - assert 'e+06' in MEDIDA_BIG - assert scan(MEDIDA_BIG).seconds > scan(MEDIDA_BLOCK).seconds - - -def test_no_other_line_in_the_block_looks_like_the_sum(): - """min, max, mean, stddev and the percentiles are all " = ms". - - A pattern loose enough to take one of them would report a per-transaction - latency as a whole-range total -- plausible, wrong, and unnoticeable. - """ - for block in (MEDIDA_BLOCK, MEDIDA_BIG): - matched = [l for l in block.splitlines() if medida.SUM_RE.search(l)] - assert len(matched) == 1, f"matched {len(matched)} lines: {matched}" - assert 'sum =' in matched[0] - - -def test_a_sum_from_another_metric_is_not_this_metric(): - """stellar-core prints many medida blocks; only one is ours.""" - scanner = lc.TxApplyScanner() - for line in ["metric 'ledger.ledger.close':", " sum = 999999.0ms"]: - scanner.feed(line) - assert scanner.seconds is None - - -def test_a_block_split_across_two_polls_still_resolves(): - """One scanner spans the whole poll loop for a pod. - - A poll boundary -- or a reconnect -- landing inside the block must not lose - the header already seen, or the last four lines of a range's life are read - with no idea what metric they belong to. - """ - lines = MEDIDA_BLOCK.splitlines() - scanner = lc.TxApplyScanner() - for line in lines[:4]: - scanner.feed(line) - assert scanner.seconds is None - for line in lines[4:]: - scanner.feed(line) - assert scanner.seconds == pytest.approx(TX_APPLY_SECONDS) - - -def test_the_metric_is_the_one_the_worker_is_told_to_print(): - """--metric on the worker command line and the string the scanner greps. - - stellar-core prints nothing at all without the flag, so a rename on either - side is a run's worth of missing metrics with no error. - """ - script = jm.RESUME_SCRIPT - m = re.search(r"--metric '([^']+)'", script) - assert m, "the worker no longer asks stellar-core for a metric" - assert m.group(1) in lc._TX_METRIC, ( - f"the worker prints {m.group(1)!r}, the collector greps {lc._TX_METRIC!r}") diff --git a/src/MissionParallelCatchup/tests/contract/test_module_packaging.py b/src/MissionParallelCatchup/tests/contract/test_module_packaging.py deleted file mode 100644 index 06c0336b..00000000 --- a/src/MissionParallelCatchup/tests/contract/test_module_packaging.py +++ /dev/null @@ -1,157 +0,0 @@ -"""The repo layout must flatten into the one directory the container runs from. - -apps/ and lib/ are for reading the repo. At runtime there is a single flat /app: -the image COPYs both directories into it, and the dev path mounts a ConfigMap -built with --from-file, whose keys are basenames and cannot contain '/'. The -modules therefore import each other by bare name, and every failure guarded here -is silent in the suite and fatal in the cluster -- an import nothing ships -crash-loops the container, and two same-named files collide into one ConfigMap -key with no warning at all. -""" - -import ast -import os -import re -import sys - -import _artifacts as art - -DOCKERFILE = os.path.join(art.MODULE_DIR, 'Dockerfile.jobmonitor') - -# The two entrypoints of the image. Everything they import from this repo has to -# reach /app with them. -ENTRYPOINTS = ('job_monitor.py', 'log_collector.py') - -# Modules that must be read through rather than copied out of, and the names it -# is never safe to bind: reassigned at startup or replaced by the tests. -READ_THROUGH = ('config', 'kube') - -SOURCE_DIRS = (art.APPS_DIR, art.LIB_DIR) - - -def _py_files(directory): - return [f for f in os.listdir(directory) if f.endswith('.py')] - - -def _local_modules(): - """Everything importable from the source directories, by the name used. - - Packages count: a subdirectory is exactly what the flatness check exists to - catch, so it cannot be invisible here. - """ - names = set() - for d in SOURCE_DIRS: - names |= {f[:-3] for f in _py_files(d) if not f.startswith('_')} - names |= {e for e in os.listdir(d) - if os.path.isfile(os.path.join(d, e, '__init__.py'))} - return names - - -def _path(name): - for d in SOURCE_DIRS: - candidate = os.path.join(d, name) - if os.path.isfile(candidate): - return candidate - raise AssertionError(f"{name} is in neither apps/ nor lib/") - - -def _imports(path): - """(module, names) for every import in `path`; names is empty for plain imports.""" - with open(path) as fh: - tree = ast.parse(fh.read()) - out = [] - for node in ast.walk(tree): - if isinstance(node, ast.Import): - out.extend((a.name, ()) for a in node.names) - elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: - out.append((node.module, tuple(a.name for a in node.names))) - return out - - -def _first_party(name): - local = _local_modules() - return {m for m, _ in _imports(_path(name)) if m in local} - - -def test_the_image_ships_every_module_the_entrypoints_import(): - text = open(DOCKERFILE).read() - # A directory COPY ships everything in it; a file COPY ships just that file. - copied = set() - for target in re.findall(r'^COPY\s+\./(\S+)\s', text, re.M): - if target.endswith('/'): - copied |= set(_py_files(os.path.join(art.MODULE_DIR, target.rstrip('/')))) - else: - copied.add(os.path.basename(target)) - - needed = set(ENTRYPOINTS) - for entry in ENTRYPOINTS: - needed |= {f"{m}.py" for m in _first_party(entry)} - missing = sorted(needed - copied) - assert not missing, f"imported but never COPYd into the image: {missing}" - - -def test_the_source_directories_flatten_without_a_collision(): - """Two same-named files would become one /app file and one ConfigMap key. - - Whichever COPY ran last wins in the image, and `--from-file` silently keeps - one of the two -- so a duplicated basename is a module quietly replaced by - another, not an error anyone sees. - """ - seen = {} - for d in SOURCE_DIRS: - for f in _py_files(d): - seen.setdefault(f, []).append(os.path.relpath(d, art.MODULE_DIR)) - clashes = {f: dirs for f, dirs in seen.items() if len(dirs) > 1} - assert not clashes, f"same basename in more than one source directory: {clashes}" - - -def test_the_modules_import_each_other_by_bare_name(): - """No package-qualified import can survive the flattening. - - `from lib import config` resolves in the repo and fails in /app, where there - is no lib/ -- and it fails at container startup, long after every test here - has passed. - """ - packages = {e for d in SOURCE_DIRS for e in os.listdir(d) - if os.path.isfile(os.path.join(d, e, '__init__.py'))} - packages |= {os.path.basename(d) for d in SOURCE_DIRS} - offenders = [] - for d in SOURCE_DIRS: - for f in _py_files(d): - for module, _ in _imports(os.path.join(d, f)): - if module.split('.')[0] in packages: - offenders.append(f"{f}: import {module}") - assert not offenders, ( - "these do not resolve once apps/ and lib/ flatten into /app:\n " - + "\n ".join(offenders)) - - -def test_no_module_shadows_a_standard_library_name(): - """/app is sys.path[0], so a local name wins over the stdlib module. - - lib/profile.py was written and renamed to profiles.py for exactly this: it - would have shadowed the stdlib profiler for every module in the process, - including anything the kubernetes client imports. The failure is remote from - the cause and appears only in the container. - """ - stdlib = sys.stdlib_module_names - clashes = sorted({f[:-3] for d in SOURCE_DIRS for f in _py_files(d)} & set(stdlib)) - assert not clashes, f"these shadow a stdlib module on a flat sys.path: {clashes}" - - -def test_nothing_copies_names_out_of_the_read_through_modules(): - """`from config import REQ_CPU` binds a copy, and the copy is silently stale. - - config.PROFILE is assigned at startup and the tests monkeypatch the rest; - kube.core_v1/batch_v1 are replaced with a fake cluster. A name bound at - import time sees none of it -- the default is used, and the test passes. - """ - offenders = [] - for d in SOURCE_DIRS: - for f in _py_files(d): - for module, names in _imports(os.path.join(d, f)): - if module in READ_THROUGH and names: - offenders.append(f"{f}: from {module} import {', '.join(names)}") - assert not offenders, ( - "read these through the module (import config; config.X) instead:\n " - + "\n ".join(offenders)) diff --git a/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py b/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py deleted file mode 100644 index f5e95670..00000000 --- a/src/MissionParallelCatchup/tests/contract/test_rendered_job_spec.py +++ /dev/null @@ -1,293 +0,0 @@ -"""The worker Job the monitor renders, against the controllers that read it. - -Three readers on the other side of this boundary, none of them ours: - - the Job controller evaluates podFailurePolicy rules first-match-wins and - reports the winner as "matching FailJob rule at index N". - That INDEX is the whole verdict -- see - test_k8s_failure_formats.py -- so the order the rules are - rendered in is a contract with the message we later decode. - the kubelet honours restartPolicy and terminationGracePeriodSeconds. - the log collector reads the attempt off the POD's labels, not the Job's. - -Everything here is asserted against a real build_job() object rather than the -source text, so a rewrite that keeps the rendered Job identical is free to -happen. -""" - -from types import SimpleNamespace as NS - -import pytest - -import config -import job_monitor as jm -import log_collector as lc - -import _artifacts as art - - -@pytest.fixture -def job(monkeypatch): - """One rendered worker Job, in the mode that needs no cluster.""" - monkeypatch.setattr(config, 'STORAGE_MODE', 'ephemeral') - monkeypatch.setattr(config, 'RUN_NAME', 'pc') - monkeypatch.setattr(config, 'CORE_IMAGE', 'stellar/stellar-core:test') - monkeypatch.setattr(config, 'PROFILE', None) - return jm.build_job(31005951, 16320, 2, None) - - -# --- the Job controller must not own retries --------------------------------- - -def test_the_job_controller_never_replaces_a_failed_pod(job): - """backoffLimit 0 is load-bearing. - - Above 0 the controller replaces the pod on its own schedule: we could not - tell a disruption from a catchup failure, could not count evictions against - their own budget, and could not guarantee the log was archived before the - next attempt started. Escalating a memory limit also needs a NEW Job -- - spec.template is immutable -- so a controller-driven retry would silently - re-run at the limit that just killed the range. - """ - assert job.spec.backoff_limit == 0 - - -def test_a_finished_job_still_has_a_ttl_backstop(job): - """reconcile() reaps finished Jobs, but only while it is running. - - A monitor that is down, wedged, or has lost its RBAC leaves every finished - Job listed on every later pass. The TTL is what bounds that, and it must be - the value the chart configured -- not a second, independent default. - """ - assert job.spec.ttl_seconds_after_finished == config.JOB_TTL_SECONDS - assert config.JOB_TTL_SECONDS > 0, "a TTL of 0 deletes a Job before it can be classified" - - -def test_a_worker_pod_is_never_restarted_in_place(job): - """restartPolicy OnFailure restarts the container inside the same pod. - - Same pod name, same resource limits -- so an OOM would loop forever at the - limit that killed it, the attempt counter would never advance, and the - terminated container state the classifier reads would be overwritten. - """ - assert job.spec.template.spec.restart_policy == 'Never' - - -def test_the_deadline_is_on_the_job_so_it_can_be_patched_live(job, monkeypatch): - """A pod-level deadline is immutable once the pod exists. - - Measured 2026-07-30: 1007 Jobs were repointed in place from 3h to 12h while - their pods kept running, and later 850 pod-level ones could not be corrected - at all -- the only way out would have been deleting every pod. The JobSpec - field is mutable, which is worth more than the Pending time it also counts. - - The cost is that Pending time is charged against the budget. Accepted: at a - flat 12h the worst stall observed (~15 min waiting on Karpenter) spends 2% - of the allowance, against a pod-level field that could not be corrected at - all on 850 already-running pods. - """ - monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', 43200) - j = jm.build_job(300, 420, 1, None) - assert j.spec.active_deadline_seconds == 43200, \ - "the deadline must be patchable, so it belongs on the JobSpec" - assert j.spec.template.spec.active_deadline_seconds is None - - -def test_no_deadline_means_no_field_at_all(job, monkeypatch): - """0 is "off". Rendering it literally would kill every pod instantly.""" - monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', 0) - j = jm.build_job(300, 420, 1, None) - assert j.spec.template.spec.active_deadline_seconds is None - - -def test_the_grace_period_outlasts_a_stellar_core_drain(job): - """stellar-core catches SIGTERM, drains, and exits 3 in ~7s (ssc-test). - - A grace period shorter than the drain turns every eviction into a SIGKILL - and exit 137 -- which the podFailurePolicy classifies as an OOM, spends the - OOM budget instead of the disruption budget, and escalates memory for a - range that never needed any. - """ - grace = job.spec.template.spec.termination_grace_period_seconds - assert grace == config.WORKER_GRACE_SECONDS - assert grace > 7, f"{grace}s does not cover the measured ~7s drain" - - -# --- podFailurePolicy: order IS the protocol --------------------------------- - -def test_every_rule_index_decodes_back_to_the_rule_that_matched(job): - """The round trip: rules[i] -> "rule at index i" -> classify_from_job. - - The controller reports only the index, so the rendered ORDER and the table - the decoder uses are one contract. Rather than assert they are the same - list, this drives a real condition message through the real classifier for - every index that exists -- which is what actually has to hold. - """ - rules = job.spec.pod_failure_policy.rules - assert len(rules) == len(jm.RULE_ORDER), ( - f"{len(rules)} rules rendered but {len(jm.RULE_ORDER)} decodable indices") - for index, expected in enumerate(jm.RULE_ORDER): - msg = (f"Container stellar-core for pod ns/p failed with exit code 1 " - f"matching FailJob rule at index {index}") - verdict = jm.classify_from_job(_failed_job(msg)) - assert verdict['outcome'] == expected, ( - f"rule {index} renders as {expected!r} but decodes as {verdict['outcome']!r}") - - -def test_disruption_is_evaluated_before_any_exit_code(job): - """First match wins, and exit 3 is ambiguous on its own. - - stellar-core exits 3 both for a SIGTERM drain and for a corrupt bucket, so - the DisruptionTarget condition is the only thing that separates a spot - eviction from a broken range. If an exit-code rule were evaluated first, an - eviction would match it, be condemned as a catchup failure, and abort a - whole run -- on spot, routinely. - """ - rules = job.spec.pod_failure_policy.rules - assert rules[0].on_pod_conditions, "index 0 is not the pod-condition rule" - assert [(c.type, c.status) for c in rules[0].on_pod_conditions] \ - == [('DisruptionTarget', 'True')] - assert jm.RULE_ORDER[0] == 'disrupted' - for rule in rules[1:]: - assert rule.on_exit_codes is not None - - -def test_the_oom_rule_is_narrower_than_the_catch_all_and_precedes_it(job): - """137 has to be matched before "any non-zero", or it never matches at all. - - Reaching the 137 rule also proves DisruptionTarget did not match, which is - the only way to tell an OOM kill from a grace-period SIGKILL once the pod - is gone. - """ - rules = job.spec.pod_failure_policy.rules - oom = rules[jm.RULE_ORDER.index('oom')].on_exit_codes - catch_all = rules[jm.RULE_ORDER.index('failed')].on_exit_codes - assert (oom.operator, oom.values) == ('In', [137]) - assert (catch_all.operator, catch_all.values) == ('NotIn', [0]) - assert jm.RULE_ORDER.index('oom') < jm.RULE_ORDER.index('failed') - - -def test_every_rule_fails_the_job_rather_than_counting_it(job): - """A Count action surfaces as BackoffLimitExceeded and loses the index. - - classify_from_job only reads a condition whose reason is PodFailurePolicy; - anything else carries no per-rule detail and returns no verdict at all. - """ - for rule in job.spec.pod_failure_policy.rules: - assert rule.action == 'FailJob' - - -def test_the_exit_code_rules_name_the_container_that_actually_runs(job): - """A containerName that matches nothing makes the rule silently inert. - - The Job would then fall through to the catch-all -- or to no rule -- and an - OOM would arrive with no index at all. - """ - names = {c.name for c in job.spec.template.spec.containers} - for rule in job.spec.pod_failure_policy.rules: - if rule.on_exit_codes is not None: - assert rule.on_exit_codes.container_name in names, ( - f"rule targets container {rule.on_exit_codes.container_name!r}, " - f"pod has {sorted(names)}") - - -def test_the_collector_watches_the_container_the_job_creates(job): - """The collector streams one container by name and samples its memory. - - A rename here leaves it streaming nothing -- and the peak sampler skipping - every container, since it filters on the same name. - """ - names = {c.name for c in job.spec.template.spec.containers} - default = _clean_default('log_collector', 'CONTAINER') - assert default in names, ( - f"the collector follows {default!r}; the Job creates {sorted(names)}") - - -# --- labels: the pod is the collector's only source of the attempt ----------- - -def test_the_pod_carries_its_own_attempt_number(job): - """The collector reads the attempt off the POD, and defaults it to "1". - - With the label only on the Job, every attempt claimed the same - range--a1.* files: measured on ssc-test 2026-07-30, 2246 metrics files - all a1 while 475 a2 pods were running -- so each retry OVERWROTE the first - attempt's peak instead of being maxed against it, destroying exactly the - OOM evidence the resumed chain exists to keep. - """ - labels = job.spec.template.metadata.labels - assert labels[config.LABEL_ATTEMPT] == '2' - assert labels[config.LABEL_RANGE] == '31005951' - assert labels[config.LABEL_RUN] == 'pc' - - -def test_the_job_is_findable_by_the_same_labels_as_its_pod(job): - """reconcile lists Jobs by run label and reads the range and attempt off it. - - The pod list and the Job list have to describe the same universe, or a Job - is reaped while its pod is still streaming. - """ - for key in (config.LABEL_RUN, config.LABEL_RANGE, config.LABEL_ATTEMPT): - assert job.metadata.labels[key] == job.spec.template.metadata.labels[key] - - -def test_the_job_name_encodes_the_range_and_the_attempt(job): - """Name uniqueness IS the dispatch mutex. - - reconcile treats a 409 AlreadyExists as "someone else already dispatched - this attempt" and spends a slot rather than raising. A name that did not - vary with the attempt would make a retry collide with its predecessor - forever; one that did not vary with the range would let two ranges share it. - """ - assert job.metadata.name == jm.job_name(31005951, 2) - assert jm.job_name(1, 1) != jm.job_name(1, 2) != jm.job_name(2, 2) - - -# --- the worker's own inputs ------------------------------------------------- - -def test_the_worker_runs_the_resume_script_for_its_own_range(job): - """The key the script marks /data with is the range identity. - - RESUME only skips new-db when the DB on /data belongs to THIS range; the - mark file is how it knows. A key that did not match the catchup argument - would resume one range's replay into another's database. - """ - command = job.spec.template.spec.containers[0].command - assert command[:2] == ['/bin/sh', '-c'] - script = command[2] - key = jm.job_key(31005951, 16320) - assert f'KEY="{key}"' in script - assert f'catchup "$KEY"' in script - - - - -def test_the_worker_mounts_the_config_the_chart_renders(job): - """The stellar-core.cfg ConfigMap is the chart's, named off the release. - - It is also the object every Job, PVC and the progress ConfigMap are - owner-referenced to, so the name has to be the one owner_ref() reads. - """ - volumes = {v.name: v for v in job.spec.template.spec.volumes} - assert volumes['config'].config_map.name == f"{config.RUN_NAME}-stellar-core-config" - mounts = {m.name: m.mount_path for m in job.spec.template.spec.containers[0].volume_mounts} - assert mounts['config'] == '/config' - assert '/config/stellar-core.cfg' in job.spec.template.spec.containers[0].command[2] - - -def test_data_is_the_path_the_resume_script_probes(job): - """RESUME reads /data/.job-key and the previous incarnation's core log.""" - mounts = {m.name: m.mount_path for m in job.spec.template.spec.containers[0].volume_mounts} - assert mounts['data'] == '/data' - assert 'MARK=/data/.job-key' in job.spec.template.spec.containers[0].command[2] - - -# --- helpers ----------------------------------------------------------------- - -def _failed_job(message, reason='PodFailurePolicy'): - """The shape classify_from_job reads: a Job with one Failed condition.""" - return NS(status=NS(conditions=[ - NS(type='Failed', status='True', reason=reason, message=message)])) - - -def _clean_default(module_name, constant): - """The module's own fallback, read with no ambient env set.""" - return art.defaults(module_name)[constant] diff --git a/src/MissionParallelCatchup/tests/contract/test_worker_log_markers.py b/src/MissionParallelCatchup/tests/contract/test_worker_log_markers.py deleted file mode 100644 index 142b81b4..00000000 --- a/src/MissionParallelCatchup/tests/contract/test_worker_log_markers.py +++ /dev/null @@ -1,133 +0,0 @@ -"""What the worker prints, against the collector that reads it off the stream. - -RESUME_SCRIPT is the worker's entrypoint and it announces its own decision on -stdout. The collector -- a different process, in a different container, that -never sees the Job spec -- recovers that decision by scanning the log stream for -a marker. That marker is the only way anything downstream knows whether an -attempt did the whole range or only its tail, and the difference matters: a -resumed attempt skips the archive download and the bucket apply, which is where -peak memory happens, so profiling it alone under-reports the range by the whole -download-vs-replay gap. On spot, where eviction is routine and resume is the -entire point of durable /data, that would make a run unprofileable. - -The script is executed here rather than quoted: what the collector has to cope -with is the bytes a real /bin/sh emits, not the string literal in job_monitor. - -(The resume DECISION -- when to skip new-db, when a range is already complete -- -is exercised in tests/unit/test_resume_script.py. This file only pins the -handshake between the two processes.) -""" - -import os -import re -import subprocess -import tempfile - -import job_monitor as jm -import log_collector as lc - -TARGET = 16752063 -COUNT = 16320 - - -def _offline_info(lcl): - """`stellar-core offline-info --console`, as 27.1.1 prints it. - - Whole document on purpose: the probe has to reach "num" past the ~40 lines - of bucketlist hashes that sit between it and the "ledger" key. - """ - if lcl is None: - return '{}' - hashes = "\n".join(f' "{i:064x}",' for i in range(40)) - return ('{\n "info" : {\n "ledger" : {\n' - ' "age" : 3,\n' - f' "bucketList" : [\n{hashes}\n ],\n' - f' "num" : {lcl},\n "version" : 23\n' - ' }\n }\n}') - - -def worker_stdout(lcl): - """Run RESUME_SCRIPT with a stubbed stellar-core; return what it printed.""" - script = jm.RESUME_SCRIPT % {'key': f"{TARGET}/{COUNT}", 'target': TARGET, - 'count': COUNT} - d = tempfile.mkdtemp() - data = os.path.join(d, 'data') - os.makedirs(data) - stub = os.path.join(d, 'stellar-core') - with open(stub, 'w') as fh: - fh.write('#!/bin/sh\n' - 'for a in "$@"; do case "$a" in\n' - ' offline-info) cat "$INFO"; exit 0;;\n' - ' new-db) exit 0;;\n' - ' catchup) exit 0;;\n' - 'esac; done\nexit 0\n') - os.chmod(stub, 0o755) - info = os.path.join(d, 'info.json') - with open(info, 'w') as fh: - fh.write(_offline_info(lcl)) - with open(os.path.join(data, '.job-key'), 'w') as fh: - fh.write(f"{TARGET}/{COUNT}") - - script = script.replace('/usr/bin/stellar-core', stub).replace('/data/', data + '/') - r = subprocess.run(['/bin/sh', '-c', script], capture_output=True, text=True, - env=dict(os.environ, INFO=info), timeout=30) - return r.stdout - - -def scan(output): - scanner = lc.TxApplyScanner() - for line in output.splitlines(): - scanner.feed(line) - return scanner - - -def test_the_collector_sees_a_resume_the_worker_announced(): - out = worker_stdout(lcl=TARGET - 100) - assert 'RESUME:' in out, out - assert scan(out).resumed is True, f"the collector missed the marker in:\n{out}" - - -def test_a_declined_resume_is_not_read_as_a_resume(): - """"RESUME DECLINED" means the opposite and shares a prefix with "RESUME:". - - Reading it as a resume chains a fresh attempt onto the attempts before it - and maxes their peaks together, inflating every range that ever restarted. - The colon is what separates them, so it is load-bearing on both sides. - """ - out = worker_stdout(lcl=None) - assert 'RESUME DECLINED' in out, out - assert scan(out).resumed is False, "a declined resume was read as a resume" - - -def test_the_probe_line_is_not_mistaken_for_the_decision(): - """The script also prints "RESUME PROBE: ..." before it has decided anything. - - It reports the LCL it read, on every attempt including a fresh one, so a - marker loose enough to match it would mark every attempt resumed. - """ - out = worker_stdout(lcl=None) - assert 'RESUME PROBE:' in out - assert scan("RESUME PROBE: offline-info reports lcl 42").resumed is False - - -def test_a_range_that_was_already_complete_announces_no_resume(): - """It ran no catchup at all, so there is no measurement to chain.""" - out = worker_stdout(lcl=TARGET) - assert 'ALREADY COMPLETE' in out, out - assert scan(out).resumed is False - - -def test_the_marker_the_collector_greps_is_the_one_the_script_prints(): - """Stated directly, so a rename on either side fails here and not in a run. - - Everything above would still pass if BOTH sides were renamed together -- - which is fine -- but this catches the case where the script's wording drifts - while the constant does not. - """ - assert lc.TxApplyScanner.RESUME_MARK in jm.RESUME_SCRIPT, ( - f"the collector greps {lc.TxApplyScanner.RESUME_MARK!r}, which the script " - "never prints") - # ...and the decline must not contain it, or the two are indistinguishable. - decline = re.search(r'echo "(RESUME DECLINED[^"]*)"', jm.RESUME_SCRIPT) - assert decline, "the script no longer announces a declined resume" - assert lc.TxApplyScanner.RESUME_MARK not in decline.group(1) diff --git a/src/MissionParallelCatchup/tests/data/real-sts-fault-exit3.log.gz b/src/MissionParallelCatchup/tests/data/real-sts-fault-exit3.log.gz deleted file mode 100644 index e1a9d49b49b8f4767f6e315478053b383204229f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8441 zcmc(kWmjBXn{JU58Z@{&1g#>ty9Rd+PO!ouKyVN41S+^&a0wFJHAvwa+}*>8oSg1{ z&ikQ1^mu#hKd{EV=Dx0Zk3}AV1jBsbXbtyt_;KGkM-H$3>)4x#BsO;^CZcvkxoKM# zQ995DQ^1CR^`LdO3OJT)DK@gX?&&@Qfx}TVe)X-n@ZuS2{SANUWZ_r@$ReA3l%soH zSwAZ_gjNbJfLHW7$T2e{OqvD>$i$n7l~-im#GHE_&0JjPnXh9vlTkUc# z>ogFLg(%%E$>S0yj4B$WClxq_6Un?IC^|s`R^X7sy|cT8cb+|JK(FvSD+NjG+a8S+ z36r?eCtKOv9=Z=l_5W;mUnDlk~nWu5m&|YJ+9<^7I zsY;EMZS}X7BsWLop_uIuUTtU{)d?aH_qu~3yN2)CJ(U|+b_#YKv;!wSD;P}yzGzi! zp`aAT;SMN;%>^Gwr2CQt*exdj*ZI6^B{{Bgn`Rd@hOS!z1w=v{Uv*+{p+>8|6uUf~ z3eANVn(c{ub?imfeIOoNPD>ETE_9vp`Q+$sO8^}DNmym#{*YJnh);7EB&{YUru;ku z>-$qgW2lNqk`tU7bUQlaVv}7ve0A!J$l}KYfYhl|ZzU>LD)=l1(xvG%t9if^NDv@~WXfLBM6^B^Sj>f;%mDYpGX5ZL6 z?lX0@apAOjntGQLLpSzIb@jg1vkBLEec)B+$8R0WybA#x6?;WUKrpq$XC%wL=8k=C zIa;g<>)NI@_PDn$dm0?Q7Kwdp!6DptmHanuI*(@)L}*~}{Kf?Sy|eH(3@3MVLbBP% z#lE}ei*b43I^yyI)OdNq=k(vrsUj(~IwTFwCc~)v?}$|Mj(P}bFzDn`FX%ftajH^V zXr6rUD(dtw9Sj-++5!_~aQ5|?daEzj@~y;EOwqHa_by*|fv>&r-o9Z&*xP$U=UF<9 z=L_ZM_nVe-Tx~z7%i>-@n5HS_DHg6~%Sy04lp2&P=c!)u3ei?U#b$HanA`-yUJ0jn zMgMSRC%8hJtoD1%ZcKQ7k-fZ|`boCW)}YgHCs1^Ip9~_5x|}f;N_mSsf!UY+N*col zlR|o`?UY@>^r|~AH0G9A~B7(9jKr4GT~GnZ$mN?XGDm4 zb_s)q>z)D@7qWtEhiIU<$-060WNc&7NI zMR>VieX?{R_jmr*psNh{X%jI+eq|OJ7N9bV6f1|acfc|_|EF$+x$~@IABtQQ-+#9b+ z>qWZ!VomtSCa$eXrbERy|5H;CI|}a=(iCO{0Ed|9!7X3Pdrim&x-b5+1^nOrDIV9MI*X zlQ{7b&p^dQVU4=eGDxS!Q3h7pL&`cn#vS!C!7L4|2J7G?w;Gg(StYw_d%}bJGWb?hfP3cU^Tj*NFaYfF8PvD-dRAz*aK4vFhtGNw~1S&WKk!dWRLj#PpXTZu0kRV<%nH1#~blwCbDt5;Y;ZHc&)my`Kt{EZ!URkX2M_QkcuUot!ihEx}V-!gFR4- z>b~)|Xe`vj6*D8z(xRYh`Y%yrm3dCfpuYLAPcq6ib46D;rM+Q?CJJUg6nqkSe&DYSnXeFD4PammZl}PB?9m`q6Mij^ zH4~m0To>ai6WLS0d8abd)o!|`e^jNkCLpMg?^dnR67a=p0#M71>^mC}y^?}tVEURG ziXp=@vRhlp@s9PtTV)Z4>4t}0e>6pz35`Y5W(w1S`P|YtIYul}j~$ZCC(4sf)uIbo zDO+87>$6I7N#4tcy$^S5ndzHtV6s2I%H=dD2CV&l!Z>MrX*0uo{td`)SaUONy`^Tq z%_nk{Cm*hOxkd{xbyW}N*z^gIRjuBc3TV=%y7EQYQo=n*?Ng(Tnoh%X;pk~KDO9)j z;xB&)W*luI));SSaz0SKlo-tKXy4m1K@|UoxrmvrBt_x86O+5T7sAine3rb37u#l*{3_ z%>Sm#%Nbd2%S#HgsjYU9@8gdCWXh$&cwQSv_vV3&xutWLog9c+$$ z37t%Ehn%W)jv_I&ZRW5JelgM+<%O^bfBw9>RqBo+?ngdk;o8Kv(hf98#9hr~%?W7< znL-bM;e?@_iOZY2RgnnmGuUP(xs~P8Mfq^hjQQy+{kKx@YvSEiPK{A&(1ukRxNk2R zq)5-AV6HO`5PE~zJ7nS?!F5lF9lXau|3$Fq4FP=Aqr=o`&LU&5K0~kEQZfgAL^<}M zm0>tnpJpuP1i@~mogmi()PXglmaWS}C(6xzhNn3D@Oc5OU~rIDRaVNiUQioqc1^Q1 zZ41k;UX-_O+P($+8@kROC$#c&Bjw}qz#{FW%GhUN=#Oti&O>xQoM5?sit zIP$&Ew623t!0BjqIHq7XEgav%eXnF=v7-E}LGP^E0I#H+X1B7M8T-U1pKHah$|mpo z+}%~<6jNtSshX2ZhCit0%d+B52&BZ4|xeEeyxW@YLjhaqjb@bpkbT_wEs0O`HMhkDv&gLO&Z?%0vP+KLvgY#|wlPc`U*+C8LT$8UjIaD0nFHo*j!mj$6SqIok)K4y)&to(k2H3`i29#b8 z49jcv`wi0I(OQ}%=QHA@&?M*Sb_89bMP4W`wu4DDJviHh`E$C>f60Vy{J)P0 z_9R&q90wl$Ccfb2p;a{Oj>9rOVIkp`P@kn#hIS}j4nU_9h%V#5>krzVYiI660^vDK zyhBWXgy)i@>PkkFBwtQNx0n;4J`UXMMKn9SAu0{6^28K7;~bJ4Jdad5ZSIgrvs8o3 z3{a!#HIh25pC@^A$_ThUhRD~v2R|1HBP9SNkN7`oYjsCy} z$z{8uhh$D_KdtM2W|>lm5Epqvx0b@PbFn~@Z4*Y+a6@CvU$}t6UsMvYM>aY`9BrYc zd=JAi4?s{3X84}LI9P{CBt-aw#dRB=q0&|HFB8ZAiixUPc6Mk@ZLRwM1{3A)6tDg= zp?35u&;OAV;6FK$6bcsxC}D(V)cSWv3$QA;WjsScLPmDmn3x!Db=Ti5B5ucg&#*df zPlP%u4P55sz{&}2loGMzf76WAuvzB!=o(3i3sK866MxPrN%vxDPwV*N6>V?BdtWx( z-PCY+Iw@{d+>gw8a!zrbrxypF2-jL2#54iTRvwO?jsl@zD1oIPo2vwBwVw}3EZiil z>1a)cPB5$q^J>WzdfV0YEJgTpJt&U%EIaA&7ay@@z)%!Mf7%+SdPKG^66J05Ce`L# z*!Inw5!Hwk@0P}y$Ri&39&+PkN(RD?vAgcc&$YGs&BPqzhU%t#Zb=oSNX{hxClf@Z zLguX)4@{5*j4OV~@lPfWh(5EZy9RoTkkhS&vb?=?3m>16sMBjrI1w&@t#3I1im;P=%3krSLhnV5LR4OIR2 zFafjo@;_pt*dt`%FA~H5M54&6PZ7dl=Y}gL&@B(!4M+KXK7lKCHm(ma2u-+v6%z|s zdgsF7d`zmvScEX{wT;T-Qrhe13R_!)=jyjg!HhMYHagAKZs1|q{+3Wo8nBJSpo=9g z>E9D)Jsn!J7e#fbE5s)w!ERyFDd9;xM3oTSpBw&q?VX#OGdc!z*@`CMUdO}3)8heY zsRA?#yRz?xunu?UD1z;R7^&0eX|aYF28xN8*SVQrF%6{oW~-y-;4_j49lFEy*hYhn zQrm<;l?BnJ6EKlis#@4;61N-efpKP5(H^*hYF~rXh3JLe?BYM~7GLmolua9{>y@yk zWLGm6NWRq{vL;b7$2FxT1lSU7URQI$KcQ@!4f%wgdlPLe+ioQKmw+oV-r8XM%m!$d zZzI_$s4w({1U>OSey+@|Zeb)}Gig{ebAwMhAJ5XsLgdArd`DH-qlk2w`P?_f^8{Ij z{E+Q0605(bZa3PB(h0-?tS%KcvW;(X3D>(Yh53e|yS{iy**Y&HLxJ|e{X`Z3rC#(UN`#nrl8Q8mBf?*r-Wg7C_ zzgQ?9|49l_r41lv@bM!q^-=U8k5$?tV~`P6B?G|mk5Z)n7bc=-_QbNyug~gX$ zPK_9~ONOBDh{+L&+k|W)N*QoIZiSmU5 z*_({XV-)`4{me^veKIZxi`MuJ4Cy=?f^t5?rdZOnC{k4B^CvdfEBvTP)UCf{*!~{; zKf?k*CHO)O72q!Y*@!&&ZOrg`{6mf$KuMVUNgI&zaMl8O=uRANaS482^x8zlNy#Q{}k+8ZS325%NylrU+*|5 zqQO0jSPS~)nWaKof5IXy=n-7R>W;S!7dy<1OBfsA4w_*^q1%uGk}e z*VlZq_po)>`s{y^&^Y<^Fa86Gx>|P7#4Apa+P|ki{;OvazkS#Bmx-?5r~Bm;-N5HR zcaPv6A%$53p_833aQ;Ty+R|68&|=pq79%8~#L7dh za=w*zcdpu#o~66;B^ufRfFqjKI7I;&E(an>(&K*n62k)8S*5((c-El#UnbxBR*jUvGT3kXHwJRzpenmzwJj`BEV1FywVzG@ zBzg#6`xc)-M4?9=Tx`{ZC)P#Zm6;E2fl0m7wfKug+v)#@MHM@+jve&vCl*@&16YWd zs*OxuRDU=>Z;X_w;^N_?=WL?k1c0@id{nI8qsQXf8yS@;$A6B6b4`VFTY&l2IXU!~ z3g)x_Vw#U1uzqPpEL}4SCE0pD&VSO1&C5%gaaBs!4zK`UTGMs34X!%sPU|@>Nt3Hh z`xlsbod`Y=2TV<*w+hEZ#H$rw7}kt%#gu4&X~m1*TCq1(_**OZe1NhNPW|19;TK$r z@^LAu5^5^#0|)PYU@^G=Xoah*o9h|UV?msy-?4%SyyGt{kiQ`ImYKrWTu^tX_L*-m zKOFtS!m8eyf#<0s5n8&1>v8;BD}G`j81+*t;_bCru`}`o`s@P3Y2=-U_jf*quq;03 zJ?+u`YD7WyFg#q@{{x38{<9=R|CP$O^dEwTlBi~@f@8T zM4d3IjKQ2mr0lhXHJ{=Hw#H|)jen_tyZ9GWaMZE0PuBoBl>Y~)_-`$xn*V7j<g<@6#P>Er2(;Fukv}FyTJfMfB$sHAe_NTR1{nps7^D*`We5IN_8sh3m!*ng>2! ztJadPRL}Crsbq1kAzJYWL0*xlIC@t+G&s+P%te9b%&YelJ;a&&bI%%?0~*;-N5qzf*9`j+a}bpEMCjOpEMyANrMQXl?K zLl265G(E!$G)BKSVu3Bs=)niHxsnvB_=Ex^dp|yAeHfW1GFPuzU%BHCjjRtAvsW_w zWSB*M?Xse4doxyXHMM%i_#(Gs35L?lbqtS)@_dHy8yDL<%c^A8*?jW2w>#(t7lO`? zI6;Q%^pvuWB(%mv@O`jAt@KCCK4dp|;fD@BU8>RRciwTMa%|bJ)^a)ZU7&P4Bfwb{rc{XFO`^K_DF~2~{bvqfq3_Y8YwREZz_-%q$-q z4?pBW(eKViSg`RPG^i%ng(`5V7DDyWZk6VeHi1!I=r-_oLldUFmxkBvyZ`a|_F0Xw z+A;yW9`73~A|NW$FJGXma5ndkLFpNE}J=xPekhB>=p%jg%MqqeS!kODUG-Y925L;S$QXRq(e5H-jMgFZv4 zedzsouWP)fto^7xT^8N#b|}$I2}_jgk{cjuir5a0#Fmt>6wZ|;ErxEqhHF~G-w>Z_ z+Qf_fxNBly5&kiVPML{8t+iQ#d!baZ8KJdPF?Y#Y(y+K<&`;^(Tc)~%99>j5gI6{+ z{HIp5<K4xX0&VmL&12ZxrN+SD>G%?71SrFmD^D8e6TyN$VeR&}#_2Z?mud!H zNTFn)4tR6r)n#Mx=T+k)wC+Y2bCu(Vs6)Cm>{+W>?u`@Mo#%xMRZa}-Bfs+k*Dc#+ k?o4O~D!HYgJi^EuOv^v0gOeKpLN~ V1Job - self.pods = {} # (ns, name) -> V1Pod - self.pvcs = {} # (ns, name) -> V1PersistentVolumeClaim - self.config_maps = {} # (ns, name) -> V1ConfigMap - self.pod_logs = {} # (ns, name) -> str - self.calls = CallLog() - # Deleted names, in order, so a test can assert a reap happened even - # after the object is gone from the dicts. - self.deleted = CallLog() - self._pod_seq = 0 - # Set to an ApiException factory to make the next matching call fail; - # keyed by "verb kind", e.g. {'create job': api_exception(500, 'boom')}. - self.fail_next = {} - self.core_v1 = FakeCoreV1Api(self) - self.batch_v1 = FakeBatchV1Api(self) - - # -- internals ----------------------------------------------------------- - - def _key(self, namespace, name): - return (namespace, name) - - def _maybe_fail(self, verb, kind): - exc = self.fail_next.pop(f"{verb} {kind}", None) - if exc is not None: - raise exc - - def _record(self, verb, kind, name, namespace): - self._maybe_fail(verb, kind) - self.calls.record(verb, kind, name, namespace) - - # -- seeding ------------------------------------------------------------- - - def add_config_map(self, name, data=None, namespace=None, uid=None): - ns = namespace or self.namespace - cm = client.V1ConfigMap( - metadata=client.V1ObjectMeta(name=name, namespace=ns, - uid=uid or f"uid-{name}"), - data=dict(data or {})) - self.config_maps[self._key(ns, name)] = cm - return cm - - # -- inspection ---------------------------------------------------------- - - def job(self, name, namespace=None): - return self.jobs[self._key(namespace or self.namespace, name)] - - def pod(self, name, namespace=None): - return self.pods[self._key(namespace or self.namespace, name)] - - def pod_for_job(self, job_name, namespace=None): - """The Pod the fake created for this Job, or None once it is reaped.""" - ns = namespace or self.namespace - for (pod_ns, _), pod in self.pods.items(): - if pod_ns != ns: - continue - if (pod.metadata.labels or {}).get(JOB_NAME_LABEL) == job_name: - return pod - return None - - def job_names(self, namespace=None): - ns = namespace or self.namespace - return sorted(name for (pod_ns, name) in self.jobs if pod_ns == ns) - - def pvc_names(self, namespace=None): - ns = namespace or self.namespace - return sorted(name for (pvc_ns, name) in self.pvcs if pvc_ns == ns) - - def config_map_data(self, name, namespace=None): - cm = self.config_maps.get(self._key(namespace or self.namespace, name)) - return dict(cm.data or {}) if cm is not None else None - - # -- Job controller emulation ------------------------------------------- - - def _spawn_pod(self, namespace, job): - self._pod_seq += 1 - name = f"{job.metadata.name}-{self._pod_seq:05d}" - labels = dict((job.spec.template.metadata.labels or {}) - if job.spec and job.spec.template and job.spec.template.metadata - else {}) - labels[JOB_NAME_LABEL] = job.metadata.name - labels['job-name'] = job.metadata.name - pod = client.V1Pod( - metadata=client.V1ObjectMeta( - name=name, namespace=namespace, labels=labels, - owner_references=[client.V1OwnerReference( - api_version='batch/v1', kind='Job', name=job.metadata.name, - uid=job.metadata.uid or f"uid-{job.metadata.name}", - controller=True)]), - spec=job.spec.template.spec if job.spec and job.spec.template else None, - status=client.V1PodStatus(phase='Pending', container_statuses=[], - conditions=[])) - self.pods[self._key(namespace, name)] = pod - return pod - - # -- state the monitor branches on -------------------------------------- - - def set_job_running(self, job_name, namespace=None): - job = self.job(job_name, namespace) - job.status = client.V1JobStatus(active=1, start_time=job.status.start_time or _now()) - pod = self.pod_for_job(job_name, namespace) - if pod is not None: - self.set_pod_running(pod.metadata.name, namespace=namespace) - return job - - def set_job_succeeded(self, job_name, namespace=None, seconds=60, - start_time=None, completion_time=None): - job = self.job(job_name, namespace) - start = start_time or job.status.start_time or (_now() - timedelta(seconds=seconds)) - job.status = client.V1JobStatus( - succeeded=1, active=0, start_time=start, - completion_time=completion_time or (start + timedelta(seconds=seconds))) - return job - - def set_job_failed(self, job_name, namespace=None, reason='PodFailurePolicy', - message='', seconds=60, start_time=None): - """Failed with a Job condition -- the message is what classify_from_job parses.""" - job = self.job(job_name, namespace) - start = start_time or job.status.start_time or (_now() - timedelta(seconds=seconds)) - conditions = [] - if reason is not None: - conditions.append(client.V1JobCondition( - type='Failed', status='True', reason=reason, message=message, - last_transition_time=_now())) - job.status = client.V1JobStatus(failed=1, active=0, start_time=start, - conditions=conditions) - return job - - def set_pod_phase(self, pod_name, phase, namespace=None, reason=None, message=None): - pod = self.pod(pod_name, namespace) - pod.status.phase = phase - if reason is not None: - pod.status.reason = reason - if message is not None: - pod.status.message = message - return pod - - def set_pod_running(self, pod_name, namespace=None, ip='10.0.0.1', start_time=None): - pod = self.pod(pod_name, namespace) - pod.status.phase = 'Running' - pod.status.pod_ip = ip - pod.status.start_time = start_time or pod.status.start_time or _now() - pod.status.container_statuses = [client.V1ContainerStatus( - name='stellar-core', image='core', image_id='', ready=True, - restart_count=0, state=client.V1ContainerState( - running=client.V1ContainerStateRunning(started_at=pod.status.start_time)))] - return pod - - def set_pod_terminated(self, pod_name, exit_code=0, reason=None, namespace=None, - seconds=60, start_time=None, finished_at=None, - container='stellar-core', phase=None): - """Terminal container state: exit code plus OOMKilled/Error reason.""" - pod = self.pod(pod_name, namespace) - start = start_time or pod.status.start_time or (_now() - timedelta(seconds=seconds)) - pod.status.start_time = start - pod.status.phase = phase or ('Succeeded' if exit_code == 0 else 'Failed') - pod.status.container_statuses = [client.V1ContainerStatus( - name=container, image='core', image_id='', ready=False, restart_count=0, - state=client.V1ContainerState(terminated=client.V1ContainerStateTerminated( - exit_code=exit_code, - reason=reason or ('Completed' if exit_code == 0 else 'Error'), - started_at=start, - finished_at=finished_at or (start + timedelta(seconds=seconds)))))] - return pod - - def set_pod_condition(self, pod_name, cond_type, status='True', namespace=None, - reason=None): - pod = self.pod(pod_name, namespace) - pod.status.conditions = [c for c in (pod.status.conditions or []) - if c.type != cond_type] - pod.status.conditions.append(client.V1PodCondition( - type=cond_type, status=status, reason=reason, - last_transition_time=_now())) - return pod - - def set_pod_log(self, pod_name, text, namespace=None): - self.pod_logs[self._key(namespace or self.namespace, pod_name)] = text - - def delete_pod(self, pod_name, namespace=None): - """Reap the pod out from under the monitor, the way Karpenter does.""" - self.pods.pop(self._key(namespace or self.namespace, pod_name), None) - - -class _Api: - def __init__(self, cluster): - self._c = cluster - - -class FakeCoreV1Api(_Api): - - # -- ConfigMaps ---------------------------------------------------------- - - def read_namespaced_config_map(self, name, namespace, **_): - self._c._record('read', 'configmap', name, namespace) - cm = self._c.config_maps.get((namespace, name)) - if cm is None: - raise _not_found('configmaps', name) - return copy.deepcopy(cm) - - def create_namespaced_config_map(self, namespace, body, **_): - name = body.metadata.name - self._c._record('create', 'configmap', name, namespace) - if (namespace, name) in self._c.config_maps: - raise _already_exists('configmaps', name) - cm = copy.deepcopy(body) - cm.metadata.namespace = namespace - cm.metadata.uid = cm.metadata.uid or f"uid-{name}" - cm.data = dict(cm.data or {}) - self._c.config_maps[(namespace, name)] = cm - return copy.deepcopy(cm) - - def patch_namespaced_config_map(self, name, namespace, body, **_): - self._c._record('patch', 'configmap', name, namespace) - cm = self._c.config_maps.get((namespace, name)) - if cm is None: - raise _not_found('configmaps', name) - data = body.get('data') if isinstance(body, dict) else (body.data or {}) - cm.data = dict(cm.data or {}) - cm.data.update(data or {}) - return copy.deepcopy(cm) - - def replace_namespaced_config_map(self, name, namespace, body, **_): - self._c._record('replace', 'configmap', name, namespace) - if (namespace, name) not in self._c.config_maps: - raise _not_found('configmaps', name) - cm = copy.deepcopy(body) - cm.metadata.namespace = namespace - cm.data = dict(cm.data or {}) - self._c.config_maps[(namespace, name)] = cm - return copy.deepcopy(cm) - - def delete_namespaced_config_map(self, name, namespace, **_): - self._c._record('delete', 'configmap', name, namespace) - if self._c.config_maps.pop((namespace, name), None) is None: - raise _not_found('configmaps', name) - self._c.deleted.record('delete', 'configmap', name, namespace) - - def list_namespaced_config_map(self, namespace, label_selector=None, **_): - self._c._record('list', 'configmap', '', namespace) - items = [copy.deepcopy(cm) for (ns, _), cm in sorted(self._c.config_maps.items()) - if ns == namespace and _match_selector(cm.metadata.labels, label_selector)] - return client.V1ConfigMapList(items=items) - - # -- Pods ---------------------------------------------------------------- - - def list_namespaced_pod(self, namespace, label_selector=None, field_selector=None, - resource_version=None, **_): - self._c._record('list', 'pod', '', namespace) - items = [copy.deepcopy(p) for (ns, _), p in sorted(self._c.pods.items()) - if ns == namespace - and _match_selector(p.metadata.labels, label_selector) - and _match_fields(p, field_selector)] - return client.V1PodList(items=items) - - def read_namespaced_pod(self, name, namespace, **_): - self._c._record('read', 'pod', name, namespace) - pod = self._c.pods.get((namespace, name)) - if pod is None: - raise _not_found('pods', name) - return copy.deepcopy(pod) - - def read_namespaced_pod_log(self, name, namespace, container=None, tail_lines=None, **_): - self._c._record('read', 'podlog', name, namespace) - if (namespace, name) not in self._c.pods: - raise _not_found('pods', name) - text = self._c.pod_logs.get((namespace, name), '') - if tail_lines: - text = "\n".join(text.splitlines()[-tail_lines:]) - return text - - def delete_namespaced_pod(self, name, namespace, **_): - self._c._record('delete', 'pod', name, namespace) - if self._c.pods.pop((namespace, name), None) is None: - raise _not_found('pods', name) - self._c.deleted.record('delete', 'pod', name, namespace) - - # -- PersistentVolumeClaims --------------------------------------------- - - def read_namespaced_persistent_volume_claim(self, name, namespace, **_): - self._c._record('read', 'pvc', name, namespace) - pvc = self._c.pvcs.get((namespace, name)) - if pvc is None: - raise _not_found('persistentvolumeclaims', name) - return copy.deepcopy(pvc) - - def create_namespaced_persistent_volume_claim(self, namespace, body, **_): - name = body.metadata.name - self._c._record('create', 'pvc', name, namespace) - if (namespace, name) in self._c.pvcs: - raise _already_exists('persistentvolumeclaims', name) - pvc = copy.deepcopy(body) - pvc.metadata.namespace = namespace - pvc.metadata.uid = pvc.metadata.uid or f"uid-{name}" - pvc.status = client.V1PersistentVolumeClaimStatus(phase='Bound') - self._c.pvcs[(namespace, name)] = pvc - return copy.deepcopy(pvc) - - def delete_namespaced_persistent_volume_claim(self, name, namespace, **_): - self._c._record('delete', 'pvc', name, namespace) - if self._c.pvcs.pop((namespace, name), None) is None: - raise _not_found('persistentvolumeclaims', name) - self._c.deleted.record('delete', 'pvc', name, namespace) - - def list_namespaced_persistent_volume_claim(self, namespace, label_selector=None, **_): - self._c._record('list', 'pvc', '', namespace) - items = [copy.deepcopy(p) for (ns, _), p in sorted(self._c.pvcs.items()) - if ns == namespace and _match_selector(p.metadata.labels, label_selector)] - return client.V1PersistentVolumeClaimList(items=items) - - -class FakeBatchV1Api(_Api): - - def create_namespaced_job(self, namespace, body, **_): - name = body.metadata.name - self._c._record('create', 'job', name, namespace) - if (namespace, name) in self._c.jobs: - # Name uniqueness is the monitor's dispatch mutex; it swallows this. - raise _already_exists('jobs.batch', name) - job = copy.deepcopy(body) - job.metadata.namespace = namespace - job.metadata.uid = job.metadata.uid or f"uid-{name}" - # The apiserver assigns this synchronously on every create, so a Job - # without one cannot exist. status.start_time is set by the Job - # controller instead, i.e. after the create response -- but the harness - # has no controller loop, so it stands in for one here. - job.metadata.creation_timestamp = job.metadata.creation_timestamp or _now() - job.status = client.V1JobStatus(active=0, start_time=_now()) - self._c.jobs[(namespace, name)] = job - self._c._spawn_pod(namespace, job) - return copy.deepcopy(job) - - def read_namespaced_job(self, name, namespace, **_): - self._c._record('read', 'job', name, namespace) - job = self._c.jobs.get((namespace, name)) - if job is None: - raise _not_found('jobs.batch', name) - return copy.deepcopy(job) - - def list_namespaced_job(self, namespace, label_selector=None, field_selector=None, **_): - self._c._record('list', 'job', '', namespace) - items = [copy.deepcopy(j) for (ns, _), j in sorted(self._c.jobs.items()) - if ns == namespace and _match_selector(j.metadata.labels, label_selector)] - return client.V1JobList(items=items) - - def delete_namespaced_job(self, name, namespace, propagation_policy=None, body=None, **_): - self._c._record('delete', 'job', name, namespace) - if self._c.jobs.pop((namespace, name), None) is None: - raise _not_found('jobs.batch', name) - self._c.deleted.record('delete', 'job', name, namespace) - # Background/Foreground both reap the pods; Orphan is the only one that - # does not, and the monitor never asks for it. - if propagation_policy != 'Orphan': - for key in [k for k, p in self._c.pods.items() - if k[0] == namespace - and (p.metadata.labels or {}).get(JOB_NAME_LABEL) == name]: - self._c.pods.pop(key, None) - - def patch_namespaced_job(self, name, namespace, body, **_): - self._c._record('patch', 'job', name, namespace) - job = self._c.jobs.get((namespace, name)) - if job is None: - raise _not_found('jobs.batch', name) - return copy.deepcopy(job) - - def replace_namespaced_job(self, name, namespace, body, **_): - self._c._record('replace', 'job', name, namespace) - if (namespace, name) not in self._c.jobs: - raise _not_found('jobs.batch', name) - job = copy.deepcopy(body) - job.metadata.namespace = namespace - self._c.jobs[(namespace, name)] = job - return copy.deepcopy(job) diff --git a/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py b/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py deleted file mode 100644 index 85e53435..00000000 --- a/src/MissionParallelCatchup/tests/reconcile/test_attempt_deadline.py +++ /dev/null @@ -1,318 +0,0 @@ -"""RACE #6 -- the attempt deadline is on the wrong object, and it outranks the pod. - -Two independent defects, both run-ending, both driven here through the real -reconcile() against the fake cluster: - -A. `activeDeadlineSeconds` sits on the JobSpec, so the clock starts when the Job - is created rather than when the container starts. Every second a pod spends - Pending -- waiting for Karpenter, waiting for an image pull -- is charged - against a budget that is meant to bound how long the RANGE runs. During the - node-class outage this run really did sit ~15 minutes Pending, and ranges - then died as "timeouts" having barely executed. - -B. When the Job reports DeadlineExceeded the monitor takes that verdict - unconditionally, over the pod's own terminated reason. A pod the kubelet - OOM-killed inside a Job that also tripped its deadline is filed as a timeout: - no memory escalation, and no budget at all, because a timeout is terminal. - One such event condemns the range and fails the mission. - -Nothing here asserts on source text. Facet B is fully drivable with the shipped -harness. Facet A needs the one thing the fake cluster does not have -- the piece -of Kubernetes that actually enforces a deadline -- so `_DeadlineController` -below supplies it. It is a model of *Kubernetes*, not of job_monitor: it reads -whichever field the monitor set and applies the clock that Kubernetes documents -for that field. A monitor that puts the deadline in the right place survives it; -one that puts it in the wrong place does not. -""" - -import pytest - -import config -import records -import job_monitor as jm - - -DEADLINE = 600 # ATTEMPT_DEADLINE_SECONDS for the facet-A tests - - -# --- the bit of Kubernetes that enforces activeDeadlineSeconds --------------- - -class _DeadlineController: - """Two fields, two clocks. That difference is the entire bug. - - * `JobSpec.activeDeadlineSeconds` is measured from `job.status.startTime`, - which the Job controller stamps when the Job is admitted -- before any pod - is scheduled. Pending time counts against it. On expiry the Job is - terminated with a Failed condition, reason=DeadlineExceeded. - - * `PodSpec.activeDeadlineSeconds` is measured from the pod's own start time, - set by the kubelet when the pod starts running. Pending time does not - count. On expiry the pod is killed (SIGTERM; stellar-core drains and exits - 3) and marked Failed with reason=DeadlineExceeded, and the Job then fails - through its podFailurePolicy like any other non-zero exit. - - This class knows nothing about which one job_monitor chose -- it reads the - Job it was handed. - """ - - @staticmethod - def deadlines(job): - pod_spec = job.spec.template.spec - return (job.spec.active_deadline_seconds, - getattr(pod_spec, 'active_deadline_seconds', None)) - - @classmethod - def run_attempt(cls, cluster, end, pending_seconds, running_seconds, - finishes='succeeded'): - """Play one attempt's timeline out against whatever deadline is set. - - Returns the state the cluster ended up in: 'timeout' if a deadline - fired, otherwise `finishes`. - """ - name = cluster.job_name(end) - job_deadline, pod_deadline = cls.deadlines(cluster.k8s.job(name)) - - if job_deadline is not None and pending_seconds + running_seconds > job_deadline: - # Job-level clock: the Job controller kills it and stamps its own - # condition. The pod is SIGTERMed and drains to exit 3. - cluster.advance(end, 'timeout') - return 'timeout' - - if pod_deadline is not None and running_seconds > pod_deadline: - # Pod-level clock: the kubelet kills the pod and marks it - # DeadlineExceeded. The Job fails through the ordinary exit-code - # rule -- it has no idea a deadline was involved. - pod = cluster.k8s.pod_for_job(name) - cluster.k8s.set_pod_terminated(pod.metadata.name, exit_code=3, - seconds=running_seconds) - cluster.k8s.set_pod_phase(pod.metadata.name, 'Failed', - reason='DeadlineExceeded', - message='Pod was active on the node longer ' - 'than the specified deadline') - cluster.k8s.set_job_failed( - name, message=(f"Container stellar-core for pod {cluster.namespace}/" - f"{pod.metadata.name} failed with exit code 3 " - f"matching FailJob rule at index 2")) - return 'timeout' - - cluster.advance(end, finishes) - return finishes - - -def _job_hit_its_deadline(cluster, end, attempt=None): - """Stamp the Job-level DeadlineExceeded condition, leaving the pod as-is. - - This is the interleaving in facet B: the pod has already recorded a specific - terminated reason (OOMKilled, DisruptionTarget, ...) and the Job *also* - tripped its deadline, so both signals are on the table at once. - """ - name = cluster.job_name(end, attempt) - cluster.k8s.set_job_failed(name, reason='DeadlineExceeded', - message='Job was active longer than specified deadline') - return name - - -def _memory(cluster, job_name): - return cluster.k8s.job(job_name).spec.template.spec.containers[0].resources - - -# --- A: Pending time must not be charged against the runtime budget ---------- - -def test_a_range_that_never_ran_still_fails_when_it_hits_the_deadline(cluster, monkeypatch): - """15 minutes Pending, 100 seconds of work, a 600s budget -- this must pass. - - The range ran for a sixth of its allowance. It is only killed because the - clock was started by the Job's creation instead of by the container's start. - """ - monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) - cluster.reconcile() - - outcome = _DeadlineController.run_attempt( - cluster, 300, pending_seconds=900, running_seconds=100, finishes='succeeded') - cluster.finalize(300, 1, tx_apply=0.5) - cluster.reconcile() - - assert outcome == 'timeout', ( - "the attempt was killed after 100s of running against a 600s budget: " - "the deadline is counting the 900s it spent Pending") - assert '300' in cluster.failed() - assert cluster.completed() == {} - - -def test_a_stall_long_enough_to_hit_the_deadline_condemns_the_range(cluster, monkeypatch): - """The run-ending shape: every attempt stalls, so every attempt "times out". - - A timeout is terminal, so the first stall condemns the range outright -- and - a condemned range fails the mission. - """ - monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) - cluster.reconcile() - - # Keep the stall going until the range settles one way or the other. Four - # passes is more than the 2-attempt timeout budget, so if the deadline is - # counting Pending time this reaches the condemned state. - for attempt in (1, 2, 3, 4): - if '300' in cluster.completed() or '300' in cluster.failed(): - break - _DeadlineController.run_attempt(cluster, 300, pending_seconds=900, - running_seconds=100, finishes='succeeded') - cluster.finalize(300, attempt) - cluster.reconcile() - - # A deadline that is reached is reported, whatever consumed it. At a 12h - # ceiling, a pod that spent the whole budget Pending is a cluster that - # cannot run this mission -- worth failing on, not worth retrying into. - assert '300' in cluster.failed(), ( - "a range that burned its entire deadline must be reported, not retried") - assert '300' not in cluster.completed() - - -def test_a_fleet_wide_stall_that_reaches_the_deadline_is_reported_not_retried(cluster, monkeypatch): - """The outage hits every range at once, not one of them.""" - monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) - monkeypatch.setattr(config, 'PARALLELISM', 3) - cluster.reconcile() - assert sorted(cluster.jobs()) == ['pc-r100-a1', 'pc-r200-a1', 'pc-r300-a1'] - - for end in (300, 200, 100): - _DeadlineController.run_attempt(cluster, end, pending_seconds=1200, - running_seconds=60, finishes='succeeded') - cluster.finalize(end, 1) - cluster.reconcile() - - # Every range burned its whole deadline, so every one is reported. A fleet - # that cannot schedule for the length of the budget is a cluster problem the - # mission must surface, not retry into. - assert sorted(cluster.failed()) == ['100', '200', '300'] - assert cluster.completed() == {} - - -def test_an_attempt_that_really_hangs_is_still_killed_by_the_deadline(cluster, monkeypatch): - """The deadline must keep biting -- a fix that just removes it is not a fix. - - Green before and after: a range that genuinely runs past its budget is - killed, retried once, and then condemned as a timeout with evidence. - """ - monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) - cluster.reconcile() - - for attempt in (1, 2): - outcome = _DeadlineController.run_attempt( - cluster, 300, pending_seconds=10, running_seconds=900, finishes='succeeded') - assert outcome == 'timeout', "a 900s attempt escaped its 600s deadline" - cluster.finalize(300, attempt) - cluster.reconcile() - - assert cluster.failed()['300']['outcome'] == 'timeout' - # Terminal on the FIRST deadline: retrying a wedged range just spends the - # deadline again. Was 2 when a timeout was retryable. - assert cluster.failed()['300']['attempts'] == 1 - assert cluster.completed() == {} - - -# --- B: a Job deadline must not overwrite the pod's own verdict -------------- - -def test_an_oom_inside_a_deadline_exceeded_job_escalates_memory(cluster, monkeypatch): - """The kubelet said OOMKilled. The Job said "ran too long". Both are true. - - Only one of them tells you what to do about it. Filing this as a timeout - means the retry goes out at the same memory limit that just killed it. - """ - monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) - cluster.reconcile() - cluster.advance(300, 'oom') - _job_hit_its_deadline(cluster, 300) - - cluster.reconcile() - - # The pod's own record is unambiguous and durable -- reconcile simply - # ignored it. - assert records.read_outcome('300', 1)['outcome'] == 'oom' - assert 'pc-r300-a2' in cluster.jobs() - resources = _memory(cluster, 'pc-r300-a2') - assert resources.requests['memory'] == '13824Mi', ( - "the retry went out at the same limit that OOM-killed it: the Job's " - "DeadlineExceeded overwrote the kubelet's OOMKilled") - assert resources.requests['memory'] == '13824Mi' - - -def test_two_ooms_inside_deadline_exceeded_jobs_do_not_condemn_the_range(cluster, monkeypatch): - """An OOM gets the range budget (5). A timeout gets 2. Misfiling ends the run.""" - monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) - cluster.reconcile() - - for attempt in (1, 2): - cluster.advance(300, 'oom') - _job_hit_its_deadline(cluster, 300, attempt) - cluster.finalize(300, attempt) - cluster.reconcile() - - assert cluster.failed() == {}, ( - "two OOMs condemned the range at the 2-attempt timeout budget instead " - "of retrying on the 5-attempt range budget") - assert 'pc-r300-a3' in cluster.jobs() - # Two rungs climbed, capped at MAX_MEM (48Gi). - assert _memory(cluster, 'pc-r300-a3').requests['memory'] == '20736Mi' - - -def test_a_disruption_inside_a_deadline_exceeded_job_keeps_its_own_budget(cluster, monkeypatch): - """Spot reclaim is the cluster's fault, and gets MAX_DISRUPTION_ATTEMPTS (20). - - A node drained near the end of a long attempt trips the Job deadline on the - way out, so the two signals arrive together constantly on spot. - """ - monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) - cluster.reconcile() - - for attempt in (1, 2): - cluster.advance(300, 'disrupted') - _job_hit_its_deadline(cluster, 300, attempt) - cluster.finalize(300, attempt) - cluster.reconcile() - - assert records.read_outcome('300', 1)['outcome'] == 'disrupted' - assert cluster.failed() == {}, ( - "two spot evictions condemned the range: the Job's DeadlineExceeded " - "downgraded them to the 2-attempt timeout budget") - assert 'pc-r300-a3' in cluster.jobs() - # An eviction says nothing about how much memory the range wants. - assert _memory(cluster, 'pc-r300-a3').requests['memory'] == config.REQ_MEM - - -def test_an_ephemeral_eviction_inside_a_deadline_exceeded_job_still_grows_the_disk(cluster, monkeypatch): - monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) - monkeypatch.setattr(config, 'LIM_EPHEMERAL', '40Gi') - monkeypatch.setattr(config, 'REQ_EPHEMERAL', '40Gi') - cluster.reconcile() - cluster.advance(300, 'ephemeral') - _job_hit_its_deadline(cluster, 300) - - cluster.reconcile() - - assert records.read_outcome('300', 1)['outcome'] == 'ephemeral' - assert 'pc-r300-a2' in cluster.jobs() - grown = _memory(cluster, 'pc-r300-a2').limits['ephemeral-storage'] - assert grown != '40Gi', ( - "the retry went out at the same ephemeral-storage limit that evicted " - "it: the Job's DeadlineExceeded overwrote the kubelet's eviction") - - -def test_a_deadline_kill_that_drained_to_exit_three_is_still_a_timeout(cluster, monkeypatch): - """The intended exception, which the ranking must not undo. - - A deadline kill SIGTERMs stellar-core, which drains and exits 3 -- the pod - verdict reads a plain `failed`, and nothing on the pod says a deadline was - involved. Here the Job genuinely is the better source, so it must still win. - Green before and after. - """ - monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) - cluster.reconcile() - - for attempt in (1, 2): - cluster.advance(300, 'timeout') - cluster.finalize(300, attempt) - cluster.reconcile() - - assert cluster.failed()['300']['outcome'] == 'timeout', ( - "an exit-3 deadline kill is no longer recognised as a timeout") - assert cluster.failed()['300']['attempts'] == 1 diff --git a/src/MissionParallelCatchup/tests/reconcile/test_completed_range_not_redispatched.py b/src/MissionParallelCatchup/tests/reconcile/test_completed_range_not_redispatched.py deleted file mode 100644 index c93edbd2..00000000 --- a/src/MissionParallelCatchup/tests/reconcile/test_completed_range_not_redispatched.py +++ /dev/null @@ -1,221 +0,0 @@ -"""RACE #1 -- a completed range gets re-dispatched and re-run from scratch. - -The interleaving these tests drive is the real one: - - A. range 300 attempt 1 is lost to node disruption. The verdict is - `disrupted`, 1 < MAX_DISRUPTION_ATTEMPTS, so attempt 2 is created. The - collector died with the node, so no `.done` marker exists for attempt 1 - and the monitor deliberately does NOT delete its Job -- it sits Failed, - waiting on JOB_TTL_SECONDS. - B. attempt 2 reuses the surviving PVC, finds the range already complete and - exits 0. The next pass keys `live` on the highest attempt, records the - range, releases the PVC and reaps -- but the reap is attempt-scoped, so - only attempt 2's Job dies. The Failed attempt-1 Job outlives the winner. - C. The pass after that lists only attempt 1, so `live[300]` is the Failed - Job. Nothing in the `st.failed` branch asks whether the range is already - in `completed`, so the disruption verdict is reached all over again and - attempt 2 is created A SECOND TIME -- against a freshly recreated, empty - PVC, so it replays the whole range from genesis. - -Everything asserted below is observed state: which Jobs and PVCs exist, what -the API was asked to create, what landed in progress.json, and what reconcile() -itself reported. No source text is inspected. -""" - -import config -import job_monitor as jm - - -# -- helpers ----------------------------------------------------------------- - - -def jobs_for(cluster, end): - """Live Job names belonging to one range, oldest attempt first.""" - prefix = f"{cluster.run_name}-r{int(end)}-a" - return sorted((n for n in cluster.jobs() if n.startswith(prefix)), - key=lambda n: int(n.rsplit('-a', 1)[1])) - - -def created(cluster, kind): - return cluster.calls.names(verb='create', kind=kind) - - -def stale_predecessor(cluster): - """Passes 1-2: dispatch, then lose 300/a1 to disruption. - - Leaves the range with two Jobs: Failed a1 (never finalized, so never - reaped) and freshly created a2. - """ - cluster.reconcile() - cluster.advance(300, 'disrupted') # collector dies with the node: - cluster.reconcile() # no finalize() -> no .done - - -def win_on_attempt_two(cluster): - """Pass 3: a2 succeeds and is recorded. Returns reconcile()'s summary.""" - cluster.advance(300, 'succeeded') # newest attempt == a2 - cluster.finalize(300, 2, tx_apply=0.25, peaks={'peakAnonBytes': 4096}) - return cluster.reconcile() - - -# -- the precondition, so a green suite cannot be green by accident ---------- - - -def test_a_disrupted_attempt_that_never_finalized_outlives_its_successor(cluster): - """Setup check: the losing Job really is still there when a2 starts. - - This is intended behaviour -- the monitor refuses to reap an attempt whose - collector never wrote `.done`, because the Job's pod is the last place its - measurements could still be read from. It is the *input* to the race, not - the bug, and it must hold both before and after the fix. - """ - stale_predecessor(cluster) - - assert jobs_for(cluster, 300) == ['pc-r300-a1', 'pc-r300-a2'] - assert cluster.k8s.job('pc-r300-a1').status.failed - # The volume survives on purpose: that is what lets a2 resume instead of - # replaying from genesis. - assert 'pc-data-r300' in cluster.pvcs() - assert cluster.completed() == {} - - -# -- the race ---------------------------------------------------------------- - - -def test_recording_a_range_reaps_every_attempt_not_just_the_winner(cluster): - """Completion is terminal for the RANGE, so no attempt of it may survive. - - RED (attempt-scoped reap): only pc-r300-a2 is deleted and the Failed - pc-r300-a1 is still standing -- which is the entire fuel for the re-run. - """ - stale_predecessor(cluster) - win_on_attempt_two(cluster) - - assert cluster.completed()['300']['attempts'] == 2 # it really recorded - assert jobs_for(cluster, 300) == [] - - -def test_a_recorded_range_is_never_dispatched_again(cluster): - """The core consequence: an already-paid-for range is re-run end to end. - - RED (no `completed` guard in the failed branch): the pass after the record - sees the leftover Failed a1, re-reaches the `disrupted` verdict and creates - pc-r300-a2 for the second time. - """ - stale_predecessor(cluster) - win_on_attempt_two(cluster) - recorded = dict(cluster.completed()['300']) - - for _ in range(3): # the monitor loops forever - cluster.reconcile() - - assert jobs_for(cluster, 300) == [] - # Exactly two Jobs were ever created for this range: a1 and its one retry. - assert created(cluster, 'job').count('pc-r300-a2') == 1 - assert [n for n in created(cluster, 'job') if n.startswith('pc-r300-')] == \ - ['pc-r300-a1', 'pc-r300-a2'] - # And the durable record was not disturbed by the extra passes. - assert cluster.completed()['300'] == recorded - assert cluster.failed() == {} - - -def test_a_released_volume_is_not_resurrected_for_a_completed_range(cluster): - """Why the re-run is worst case: the PVC is gone, so there is nothing to - resume from. build_job() calls ensure_pvc(), which recreates it empty -- - no /data/.job-key, RESUME declined, new-db, full replay from genesis. - - RED: pc-data-r300 is released by the recording pass and then created a - second time by the spurious re-dispatch. - """ - stale_predecessor(cluster) - win_on_attempt_two(cluster) - - assert 'pc-data-r300' not in cluster.pvcs() # released on record - cluster.reconcile() - cluster.reconcile() - - assert 'pc-data-r300' not in cluster.pvcs() - assert created(cluster, 'pvc').count('pc-data-r300') == 1 - - -def test_a_phantom_rerun_does_not_breach_parallelism(cluster): - """The slot freed by 300 goes to 100 -- and then 300 must not take one back. - - Recording 300 frees a slot, so pass 3 dispatches range 100 and the run is - at its cap of 2. The re-dispatch happens *outside* the capacity check (the - failed branch creates the Job and appends to in_progress unconditionally), - so it does not wait for a slot -- it takes a third one. - - RED: in_progress is ['100/420', '200/420', '300/420'] -- three concurrent - ranges under PARALLELISM 2, one of them already finished. - """ - stale_predecessor(cluster) - win_on_attempt_two(cluster) - assert 'pc-r100-a1' in cluster.jobs() # the slot did free up - - result = cluster.reconcile() - - assert '300/420' not in result['in_progress'] - assert sorted(result['in_progress']) == ['100/420', '200/420'] - assert len(result['in_progress']) <= config.PARALLELISM - - -def test_the_range_scoped_reap_still_waits_for_the_done_marker(cluster): - """Widening the reap from one attempt to the whole range must not widen - *when* it fires. Deleting a Job reaps its pod, and .metrics is the only - place peaks live, so nothing may be reaped before the collector has - written `.done` -- JOB_TTL_SECONDS is the backstop for a collector that - never gets there. - - This is the range-scoped half of the guarantee; the attempt-scoped half is - unit/test_reaping.py::test_the_reap_waits_for_the_collectors_done_marker. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - - waiting = cluster.reconcile() # recorded; collector not done - assert '300' in cluster.completed() - assert jobs_for(cluster, 300) == ['pc-r300-a1'] - assert cluster.deleted.names(verb='delete', kind='job') == [] - assert waiting['finalizing'] == ['300/420'], \ - "the mission could publish its final profile before metrics landed" - - cluster.finalize(300, 1, tx_apply=0.1, peaks={'peakAnonBytes': 1}) - finished = cluster.reconcile() - assert jobs_for(cluster, 300) == [] - assert cluster.deleted.names(verb='delete', kind='job') == ['pc-r300-a1'] - assert finished['finalizing'] == [] - - -def test_remaining_never_goes_negative_and_the_run_reports_done(cluster, - monkeypatch): - """The mission driver waits for `remaining == 0 and in_progress == []`. - - With every range finished, that condition must hold and keep holding. - - RED: the leftover Failed a1 puts range 300 back into in_progress while it - is also in completed, so it is subtracted twice -- remaining reads -1, and - in_progress is never empty, so the driver's completion test never fires. - """ - monkeypatch.setattr(config, 'PARALLELISM', 3) # all three ranges at once - - cluster.reconcile() - cluster.advance(300, 'disrupted') - cluster.reconcile() # 300/a1 Failed and unfinalized - - for end, attempt in ((300, 2), (200, 1), (100, 1)): - cluster.advance(end, 'succeeded', attempt=attempt) - cluster.finalize(end, attempt, tx_apply=0.1, peaks={'peakAnonBytes': 1}) - done = cluster.reconcile() - - assert done['completed'] == 3 - assert done['in_progress'] == [] - assert done['remaining'] == 0 - - # ...and it stays done. This is the pass that re-dispatches under the bug. - again = cluster.reconcile() - assert again['completed'] == 3 - assert again['remaining'] == 0 # reads -1 while the bug is present - assert again['in_progress'] == [] - assert again['created'] == 0 - assert cluster.jobs() == [] diff --git a/src/MissionParallelCatchup/tests/reconcile/test_dispatch_not_frozen.py b/src/MissionParallelCatchup/tests/reconcile/test_dispatch_not_frozen.py deleted file mode 100644 index fe7e1552..00000000 --- a/src/MissionParallelCatchup/tests/reconcile/test_dispatch_not_frozen.py +++ /dev/null @@ -1,229 +0,0 @@ -"""RACE #7: a condemned range must not freeze dispatch, or the mission hangs. - -The mission driver (MissionHistoryPubnetParallelCatchupV2.fs) no longer aborts -on the first failure -- it drains first, and only reports once - - num_remain == 0 && queue_in_progress_count == 0 - -which are `reconcile()`'s own `remaining` and `in_progress` verbatim -(job_monitor.reconcile_loop maps them straight into the status JSON). - -If dispatch is gated on `not failed`, the first condemned range stops the -monitor sending any further work. `in_progress` still drains to empty as the -in-flight ranges land, but `remaining` stays pinned at however many ranges were -never dispatched. The driver's condition is then unsatisfiable and the mission -waits forever with an idle, fully-billed node pool -- strictly worse than the -immediate abort it replaced. - -These tests drive the real reconcile() through the fake cluster and assert only -on what it returns and on what ends up in progress.json. No source text. -""" - -import pytest - -import config -import job_monitor as jm - - -# -- helpers (local to this file; the shared harness is not touched) ---------- - -def _end_of(key): - """'300/420' -> 300. `in_progress` entries are job_key(end, count).""" - return int(key.split('/')[0]) - - -def _drained(poll): - """The mission driver's completion test, applied to a reconcile summary.""" - return poll['remaining'] == 0 and not poll['in_progress'] - - -def drive_like_the_mission(cluster, condemn=(), max_passes=40): - """Poll reconcile() the way the driver polls the monitor, until it drains. - - Between polls the cluster does its job: every range currently in flight - finishes -- successfully, unless it is in `condemn`, in which case it exits - 1 (a genuine catchup failure, which the monitor never retries). - - Returns the list of poll results, or None if the run never drained inside - `max_passes` -- which is what a hang looks like when you cannot wait forever. - """ - condemn = {int(e) for e in condemn} - polls = [] - for _ in range(max_passes): - poll = cluster.reconcile() - polls.append(poll) - if _drained(poll): - return polls - for key in list(poll['in_progress']): - end = _end_of(key) - attempt = cluster.attempt_of(end) - if end in condemn: - cluster.advance(end, 'condemned', attempt=attempt) - else: - cluster.advance(end, 'succeeded', attempt=attempt) - cluster.finalize(end, attempt, tx_apply=1.0, - peaks={'peakAnonBytes': 1}) - return None - - -def _why_stuck(polls): - last = polls[-1] - return (f"run never drained; last poll remaining={last['remaining']} " - f"in_progress={last['in_progress']} created={last['created']} " - f"completed={last['completed']} failed={last['failed_ranges']}") - - -# -- the race ---------------------------------------------------------------- - -def test_a_condemned_range_does_not_pin_remaining_above_zero(cluster): - """The exact interleaving: one range condemned while another succeeds. - - Three ranges, PARALLELISM 2, so range 100 is still undispatched when 300 is - condemned. If the condemn freezes dispatch, 100 is never sent, `in_progress` - empties anyway, and `remaining` sticks at 1 -- the deadlock. - """ - first = cluster.reconcile() - assert sorted(first['in_progress']) == ['200/420', '300/420'] - assert first['remaining'] == 1, "range 100 has not been dispatched yet" - - cluster.advance(300, 'condemned') # exit 1: never retried - cluster.advance(200, 'succeeded') - cluster.finalize(200, 1, tx_apply=1.0, peaks={'peakAnonBytes': 1}) - - second = cluster.reconcile() - - # The condemn is recorded and the good range is banked... - assert '300' in cluster.failed() - assert '200' in cluster.completed() - # ...and the range behind them goes out into the freed capacity. Frozen - # dispatch gives in_progress == [] with remaining == 1, and from there the - # driver's `remaining == 0 && in_progress == []` can never come true. - assert second['in_progress'] == ['100/420'], ( - "the condemned range froze dispatch: range 100 was never sent, so the " - "mission's drain condition is now unsatisfiable") - assert second['remaining'] == 0 - - # And it really does finish. - cluster.advance(100, 'succeeded') - cluster.finalize(100, 1, tx_apply=1.0, peaks={'peakAnonBytes': 1}) - third = cluster.reconcile() - assert _drained(third) - assert sorted(cluster.completed()) == ['100', '200'] - assert list(cluster.failed()) == ['300'] - - -def test_the_mission_drains_and_then_fails_instead_of_hanging(cluster): - """End to end through the driver's own loop: it must terminate. - - A run with a condemned range has to reach `remaining == 0 and - in_progress == []` -- the mission then fails on the recorded failure. With - dispatch frozen the loop below simply never exits. - """ - polls = drive_like_the_mission(cluster, condemn=[300]) - - assert polls is not None, ( - "the mission never drained: reconcile() never reported " - "remaining == 0 with in_progress empty, so the driver would poll forever") - assert _drained(polls[-1]), _why_stuck(polls) - - # It drains, but it does not pass: the failure is still reported, which is - # what makes the mission fail after the drain. - assert polls[-1]['failed_ranges'], "the condemned range must still be reported" - assert polls[-1]['failed_ranges'][0].startswith('300/420|') - assert polls[-1]['completed'] == 2, "the other two ranges must still be run" - - -def test_a_condemned_tip_does_not_discard_every_range_behind_it(cluster, - monkeypatch): - """The production shape: one early condemn, nine ranges still to dispatch. - - This is the 2026-07-30 incident at small scale -- a condemned range at the - tip stranded everything queued behind it. Freezing dispatch loses all nine. - """ - monkeypatch.setattr(config, 'LATEST_LEDGER_NUM', 1000) # ends 100..1000 - - polls = drive_like_the_mission(cluster, condemn=[1000]) - - assert polls is not None, "the mission hung with nine ranges never dispatched" - assert _drained(polls[-1]), _why_stuck(polls) - - assert sorted(int(e) for e in cluster.completed()) == [ - 100, 200, 300, 400, 500, 600, 700, 800, 900] - assert list(cluster.failed()) == ['1000'] - assert polls[-1]['total'] == 10 - - -def test_the_stuck_state_never_settles_into_a_reportable_one(cluster): - """A hang is a state that repeats, so poll it the way the driver does. - - Once everything that can finish has finished, every subsequent poll must - report the drained state. Frozen dispatch instead reports the same - `remaining > 0, in_progress == []` forever -- work outstanding, nobody - doing it, no new Jobs. That pair is the deadlock signature. - """ - drive_like_the_mission(cluster, condemn=[300]) - - for _ in range(5): - poll = cluster.reconcile() - assert not (poll['remaining'] > 0 and not poll['in_progress']), ( - f"deadlock signature: remaining={poll['remaining']} with nothing in " - f"flight and created={poll['created']}") - assert _drained(poll) - - # Nothing new was invented to get there, either: three ranges, three Jobs. - assert cluster.calls.names(verb='create', kind='job') == [ - 'pc-r300-a1', 'pc-r200-a1', 'pc-r100-a1'] - - -def test_two_condemned_ranges_still_leave_the_run_drainable(cluster, - monkeypatch): - """More than one failure must not make it worse, and must not double-count. - - `remaining` subtracts completed, failed and in-flight; a second condemn has - to land in `failed` exactly once or the arithmetic stops reaching zero. - """ - monkeypatch.setattr(config, 'LATEST_LEDGER_NUM', 500) # ends 100..500 - - polls = drive_like_the_mission(cluster, condemn=[500, 400]) - - assert polls is not None, "the mission hung after two condemned ranges" - assert _drained(polls[-1]), _why_stuck(polls) - assert sorted(cluster.failed()) == ['400', '500'] - assert sorted(int(e) for e in cluster.completed()) == [100, 200, 300] - assert len(polls[-1]['failed_ranges']) == 2 - - -# -- the safety valve the fix must not take with it --------------------------- - -def test_nothing_gates_dispatch_at_all(cluster): - """The gate is gone on purpose, and no new one may appear. - - `failed` stopped gating dispatch because it deadlocked the driver. `halted` - stopped gating it because its high-water mark lived in memory: a restart - reset it to zero, so the guard was disarmed by the very event it was there - to survive. A reconciler must not gate a decision on state a restart erases. - - The cost is re-running a range, which is idempotent -- the PVC still holds - /data so the attempt resumes at its last closed ledger and the measurements - are re-recorded rather than lost. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1, tx_apply=1.0, peaks={'peakAnonBytes': 1}) - cluster.reconcile() - - cluster.write(config.PROGRESS_FILE, '{}') # the record is wiped underneath us - - poll = cluster.reconcile() - - # The range returns to the pool rather than the run stopping. Nothing is - # created on this pass only because PARALLELISM is already spent on the - # other two ranges -- capacity, not a gate. - assert '300' not in cluster.completed() - assert poll['completed'] == 0 - assert poll['remaining'] + len(poll['in_progress']) == 3 - - # Free a slot and it really is dispatched again: the run is not wedged. - cluster.advance(200, 'succeeded') - cluster.finalize(200, 1, tx_apply=1.0, peaks={'peakAnonBytes': 1}) - assert cluster.reconcile()['created'] >= 1 diff --git a/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py b/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py deleted file mode 100644 index c1bece37..00000000 --- a/src/MissionParallelCatchup/tests/reconcile/test_retry_budgets.py +++ /dev/null @@ -1,547 +0,0 @@ -"""RACE #5 -- attempt budgets are spent from one shared counter. - -Every retry bumps the same attempt index. The cap is then picked from whatever -the LATEST verdict happened to be, and the global index is compared against it. -So cluster churn (spot evictions, admission rejections, monitor restarts) -- -which has its own, deliberately large budget -- silently drains the small -budgets belonging to the causes that actually say something about the range. - -A range evicted five times arrives at attempt 6. Its FIRST genuine OOM is then -compared 6 >= MAX_ATTEMPTS(5) and condemned, having never once been retried for -an OOM and never once had its memory escalated. A condemned range fails the -whole mission. - -Everything below is observed state: which Jobs exist, what resources they were -created with, and what landed in progress.json's failed{}. No source text. -""" - -import config -import units -import sizing -import job_monitor as jm -import records - - -# --- helpers (local to this file on purpose) -------------------------------- - -def dispatch(cluster, end=300): - """Get `end` to attempt 1, running, with nothing else in the way.""" - cluster.reconcile() - assert cluster.attempt_of(end) == 1 - return end - - -def hit(cluster, end, state, times=1): - """Fail the range's newest attempt `times` times in a row with `state`. - - One reconcile per failure, which is what the real loop does: the monitor - sees the failed Job, decides retry-or-condemn, and (if retrying) creates - the successor before the next pass. - """ - for _ in range(times): - n = cluster.attempt_of(end) - cluster.advance(end, state) - if state in ('incomplete', 'unexplained'): - # exit 3 is decided from the archive, and not until .done exists. - cluster.finalize(end, n, archive=( - 'fetch_fault' if state == 'incomplete' - else 'bare')) - cluster.reconcile() - - -def job_exists(cluster, end, attempt): - return jm.job_name(int(end), attempt) in cluster.jobs() - - -def mem_of(cluster, end, attempt): - job = cluster.k8s.job(jm.job_name(int(end), attempt)) - return job.spec.template.spec.containers[0].resources - - -def condemned(cluster, end): - return str(end) in cluster.failed() - - -# --- the race --------------------------------------------------------------- - -def test_five_evictions_do_not_burn_the_whole_oom_budget(cluster): - """5 spot evictions (budget 20) must leave the OOM budget (5) untouched. - - Under the bug the range is on attempt 6 when its first OOM lands, 6 >= 5, - and it is condemned without ever being retried for the OOM -- so its memory - is never escalated and the mission fails on a range that is merely unlucky. - """ - end = dispatch(cluster) - hit(cluster, end, 'disrupted', times=5) - # Five evictions are legal on the disruption budget: the range is alive. - assert not condemned(cluster, end) - assert cluster.attempt_of(end) == 6 - - hit(cluster, end, 'oom') - - assert not condemned(cluster, end), ( - "first OOM after eviction churn condemned the range: the eviction " - f"retries spent the OOM budget. failed={cluster.failed()}") - assert job_exists(cluster, end, 7), ( - f"no attempt 7 was dispatched; live jobs are {cluster.jobs()}") - - # And the whole point of an OOM retry: more memory. One OOM = one rung. - res = mem_of(cluster, end, 7) - assert res.requests['memory'] == '13824Mi' - - -def test_evictions_do_not_burn_the_disk_budget(cluster, monkeypatch): - """Same shape, ephemeral-storage budget (4). Disk evictions repeat until - the range gets more disk, so losing that budget to churn is terminal.""" - monkeypatch.setattr(config, 'LIM_EPHEMERAL', '40Gi') - monkeypatch.setattr(config, 'REQ_EPHEMERAL', '40Gi') - - end = dispatch(cluster) - hit(cluster, end, 'disrupted', times=5) - assert cluster.attempt_of(end) == 6 - - hit(cluster, end, 'ephemeral') - - assert not condemned(cluster, end), ( - "first disk eviction after eviction churn condemned the range; " - f"failed={cluster.failed()}") - assert job_exists(cluster, end, 7) - # One eviction = one rung: 40Gi * EPH_BUMP_FACTOR. Pinned to the exact rung - # rather than "more than 40Gi", so indexing the ladder on `attempt` instead - # of on evictions fails here -- after this much churn that would ask for the - # 6th rung (capped at 200Gi), five times the disk for one eviction. - grown = mem_of(cluster, end, 7).limits['ephemeral-storage'] - assert grown == sizing.eph_for_attempt(2) == '61440Mi', grown - - -def test_evictions_do_not_burn_the_range_budget_for_exit_3(cluster): - """exit 3 ("did not complete") rides the ordinary range budget of 5. - - A range evicted five times gets zero exit-3 retries -- and exit 3 is the - outcome an interrupted-then-resumable range produces, so the retry that was - denied is the one that would have succeeded. - """ - end = dispatch(cluster) - hit(cluster, end, 'disrupted', times=5) - assert cluster.attempt_of(end) == 6 - - hit(cluster, end, 'incomplete') - - assert not condemned(cluster, end), ( - f"first exit-3 after eviction churn condemned the range; " - f"failed={cluster.failed()}") - assert job_exists(cluster, end, 7) - - -def test_memory_ladder_follows_ooms_not_evictions(cluster, monkeypatch): - """Interleaved churn: each OOM must climb exactly one rung, and the second - OOM must still be inside the budget even though the attempt index is 8.""" - # The shipped 48Gi ceiling clamps rung 2 to the same figure rung 8 would - # give, which would make the ladder assertion below prove nothing. Raise it - # so the rung is observable; the budget behaviour under test is unaffected. - monkeypatch.setattr(config, 'MEM_ESCALATION_CAP', '128Gi') - end = dispatch(cluster) - hit(cluster, end, 'disrupted', times=3) # attempts 1-3, now on 4 - hit(cluster, end, 'oom') # OOM #1 on attempt 4 -> a5 - assert job_exists(cluster, end, 5) - assert mem_of(cluster, end, 5).requests['memory'] == '13824Mi' - - hit(cluster, end, 'disrupted', times=2) # attempts 5-6, now on 7 - hit(cluster, end, 'oom') # OOM #2 on attempt 7 -> a8 - - assert not condemned(cluster, end), f"failed={cluster.failed()}" - assert job_exists(cluster, end, 8) - # 24000Mi * 1.5^2 -- two OOMs, six evictions, two rungs. - assert mem_of(cluster, end, 8).requests['memory'] == '20736Mi' - - -# --- the caps must still bind (a fix that just removes them is not a fix) ---- - -def test_the_oom_budget_still_binds(cluster): - """Five real OOMs in a row exhaust the OOM budget and condemn the range.""" - end = dispatch(cluster) - hit(cluster, end, 'oom', times=5) - - assert condemned(cluster, end), ( - f"five consecutive OOMs were not condemned; jobs={cluster.jobs()}") - assert cluster.failed()[str(end)]['outcome'] == 'oom' - assert not job_exists(cluster, end, 6), ( - f"a 6th OOM attempt was dispatched past the budget: {cluster.jobs()}") - - -def test_one_timeout_condemns_the_range(cluster): - """A timeout is terminal -- it has no budget to bind. - - The deadline exists only for a range wedged on an unreachable archive, and - retrying that just spends another 12h to learn the same thing. This used to - allow 2 attempts; the assertion is that a SECOND one is never dispatched. - """ - end = dispatch(cluster) - hit(cluster, end, 'timeout') - - assert condemned(cluster, end), ( - f"the first timeout did not condemn the range; jobs={cluster.jobs()}") - assert cluster.failed()[str(end)]['outcome'] == 'timeout' - assert not job_exists(cluster, end, 2), ( - f"a second attempt was dispatched after a terminal timeout: {cluster.jobs()}") - - -def test_an_archive_without_the_done_marker_does_not_promote_a_fetch_fault(cluster): - """The collector appends as the pod runs, so a present archive is not a - finished one. - - Promoting on a partial archive would read a fetch-fault anchor that a later - line still explains away, and hand the range the fetch-fault budget on - incomplete evidence. The .done marker is what says the collector is finished - with this attempt, so the decision waits for it. - """ - end = dispatch(cluster) - attempt = cluster.attempt_of(end) - cluster.advance(end, 'incomplete') - cluster.archive(end, attempt, 'fetch_fault') # archive, but no .done - cluster.reconcile() - - assert not condemned(cluster, end), f"failed={cluster.failed()}" - assert not job_exists(cluster, end, attempt + 1), ( - "a successor was dispatched off a half-written archive: " - f"{cluster.jobs()}") - assert records._verdict_of(str(end), attempt) != 'fetch-fault', \ - "the verdict was promoted before the collector finished" - - -def test_a_cause_with_no_budget_entry_gets_no_retries(cluster): - """The table is the whole policy: absent means condemned on sight. - - Not reachable through reconcile today -- every outcome that gets as far as - the retry gate is in the table, and the rest return CONDEMN before the cap - is read. This pins the default so a new outcome added to classify() cannot - quietly inherit retries nobody chose for it. - """ - assert jm.budget_for({'outcome': 'a-brand-new-thing'}, 300, 1) == (0, 0) - for terminal in ('timeout', 'unknown'): - assert terminal not in config.ATTEMPT_BUDGETS - assert jm.budget_for({'outcome': terminal}, 300, 1)[1] == 0 - - -def test_a_fetch_fault_does_not_spend_the_oom_budget(cluster): - """An unreachable archive is the cluster's problem, not the range's. - - Before this, an exit-3 fetch fault was recorded as `failed` and the range - budget counted ('oom', 'failed') -- so one unreachable S3 mirror permanently - cost the range one of its five memory escalations. - """ - end = dispatch(cluster) - hit(cluster, end, 'incomplete') # exit 3, fetch fault in the archive - assert job_exists(cluster, end, 2), f"the fetch fault was not retried: {cluster.jobs()}" - - # The OOM ladder still has its whole budget. - hit(cluster, end, 'oom', times=config.ATTEMPT_BUDGETS['oom'] - 1) - assert not condemned(cluster, end), ( - "the fetch fault spent an OOM attempt; " - f"failed={cluster.failed()} jobs={cluster.jobs()}") - hit(cluster, end, 'oom') - assert condemned(cluster, end), ( - f"the OOM budget did not bind after {config.ATTEMPT_BUDGETS['oom']} OOMs") - assert cluster.failed()[str(end)]['outcome'] == 'oom' - - -def test_the_disk_budget_still_binds(cluster, monkeypatch): - """Disk evictions are counted against their OWN budget, and it binds. - - The gap this closes: with the ephemeral arm of budget_for removed, an - eviction falls through to the range budget, whose counter looks only at - ('oom', 'failed'). For a purely disk-evicted range that count is always 0, so - it would be retried forever -- and every other budget test still passed. - """ - monkeypatch.setattr(config, 'LIM_EPHEMERAL', '40Gi') - monkeypatch.setattr(config, 'REQ_EPHEMERAL', '40Gi') - end = dispatch(cluster) - hit(cluster, end, 'ephemeral', times=config.ATTEMPT_BUDGETS['ephemeral']) - - assert condemned(cluster, end), ( - f"{config.ATTEMPT_BUDGETS['ephemeral']} disk evictions were not condemned; " - f"jobs={cluster.jobs()}") - assert cluster.failed()[str(end)]['outcome'] == 'ephemeral' - assert not job_exists(cluster, end, config.ATTEMPT_BUDGETS['ephemeral'] + 1), ( - f"an attempt was dispatched past the disk budget: {cluster.jobs()}") - - -def test_an_eviction_with_no_configured_disk_limit_does_not_wedge_reconcile(cluster): - """LIM_EPHEMERAL is empty by default and the chart ships it empty. - - A pod with no ephemeral-storage limit can still be evicted under node disk - pressure. eph_for_attempt used to parse the empty string and raise, and - reconcile's caller swallows exceptions -- so one such eviction killed every - later pass at the same range: no dispatch, no completions, for the rest of - the run. - """ - assert config.LIM_EPHEMERAL == '', "this test is about the unset default" - end = dispatch(cluster) - hit(cluster, end, 'ephemeral') - - assert not condemned(cluster, end), f"failed={cluster.failed()}" - assert job_exists(cluster, end, 2), ( - f"the eviction was not retried; jobs={cluster.jobs()}") - # And the pass still completes for everything else. - assert cluster.reconcile()['completed'] == 0 - - -def test_the_disk_budget_is_smaller_than_the_range_budget(cluster): - """Pins that the two caps are actually different. - - Both tests above pass if disk silently borrows the range budget, as long as - the caps happen to match. They must not: escalating disk 5 times is a 7.6x - request. - """ - # The MAX_* constants, not ATTEMPT_BUDGETS: the cluster fixture patches the - # map, so asserting on it here would pin the fixture and let production ship - # any ordering it liked. - assert config.MAX_EPHEMERAL_ATTEMPTS < config.MAX_OOM_ATTEMPTS - assert config.MAX_EPHEMERAL_ATTEMPTS < config.MAX_DISRUPTION_ATTEMPTS, ( - "an eviction is the range's own problem, not the cluster's") - - -def test_the_disruption_budget_still_binds(cluster, monkeypatch): - """The environmental budget is effectively unlimited, but it is still a gate. - - Driven at a small cap rather than the configured one: what matters is that - the gate fires at N, and looping to the real value would make this test do a - thousand reconcile passes. test_attempt_budgets_are_ordered_by_whose_fault - pins the production number. - """ - monkeypatch.setitem(config.ATTEMPT_BUDGETS, 'disrupted', 6) - end = dispatch(cluster) - hit(cluster, end, 'disrupted', times=6) - - assert condemned(cluster, end), ( - f"6 evictions were not condemned; jobs={cluster.jobs()}") - assert not job_exists(cluster, end, 7) - - -def test_a_condemned_range_is_decided_once_and_then_cleaned_up(cluster, caplog): - """A condemned Job is not deleted by anything else until JOB_TTL_SECONDS. - - It therefore stays the newest Job for its range, so every later pass - re-derives the same verdict and re-logs the same condemnation -- measured on - the 2026-07-30 run as 15 identical lines over 9 minutes, ending only when the - TTL removed the Job. The reap waits for the collector's marker because - deleting the Job reaps the pod. - """ - import logging - end = dispatch(cluster) - hit(cluster, end, 'condemned') - assert condemned(cluster, end), f"failed={cluster.failed()}" - - # Before the collector finishes: the Job stays, and nothing is re-decided. - caplog.clear() - with caplog.at_level(logging.ERROR): - cluster.reconcile() - assert 'RANGE CONDEMNED' not in caplog.text, \ - "the condemnation was logged again on a later pass" - assert cluster.jobs(), "the Job was reaped before the collector finalized it" - - attempt = cluster.attempt_of(end) - job = jm.job_name(end, attempt) - cluster.finalize(end, attempt) - cluster.reconcile() - assert job not in cluster.jobs(), \ - f"the condemned Job was not reaped: {cluster.jobs()}" - assert not [v for v in cluster.pvcs() if str(end) in v], \ - f"the condemned range kept its volume: {cluster.pvcs()}" - # The other ranges are untouched. - assert len(cluster.jobs()) == 2, cluster.jobs() - # Still condemned, and still the reason the mission fails. - assert condemned(cluster, end) - - -def test_a_genuine_catchup_failure_is_still_never_retried(cluster): - """exit 1 is condemned on attempt 1 regardless of any tally.""" - end = dispatch(cluster) - hit(cluster, end, 'condemned') - - assert condemned(cluster, end) - assert cluster.failed()[str(end)]['attempts'] == 1 - assert not job_exists(cluster, end, 2) - - -def test_a_terminated_pod_with_no_exit_code_is_condemned(cluster): - """No exit code is no evidence, and the run stops rather than guess. - - This reverses an earlier choice, so the cost stays on the record: on the r5 - run 2026-07-30 range 59018943 was condemned exactly this way and failed a - mission that was otherwise 554 for 554. The policy now is that only a node - disruption -- which proves the cluster took the pod away mid-run -- earns a - retry without evidence. Anything the monitor cannot explain fails the run, - because a run that reports success on a range nobody verified is worse. - """ - end = dispatch(cluster) - hit(cluster, end, 'no_exit_code') - - assert condemned(cluster, end), ( - f"a pod reaped before classification was retried on no evidence; " - f"jobs={cluster.jobs()}") - assert not job_exists(cluster, end, 2), ( - f"attempt 2 was dispatched with nothing explaining attempt 1: {cluster.jobs()}") - - -def test_an_unclassified_failure_is_condemned(cluster): - """Same rule for a pod that vanished before anything classified it.""" - end = dispatch(cluster) - hit(cluster, end, 'unknown') - - assert condemned(cluster, end), f"jobs={cluster.jobs()}" - assert cluster.failed()[str(end)]['outcome'] == 'unknown' - assert not job_exists(cluster, end, 2) - - -def test_a_disruption_is_the_only_thing_retried_without_evidence(cluster): - """The counterpart: a disruption proves the range itself was fine.""" - end = dispatch(cluster) - hit(cluster, end, 'disrupted', times=8) - - assert not condemned(cluster, end), ( - f"eight spot evictions condemned a healthy range; failed={cluster.failed()}") - assert job_exists(cluster, end, 9) - - -def test_a_real_catchup_failure_is_still_condemned(cluster): - """The guard above must not swallow the case it is next to: an exit code of - 1 IS evidence, and a range that produces one still fails the mission.""" - end = dispatch(cluster) - hit(cluster, end, 'condemned') - - assert condemned(cluster, end), ( - "exit 1 is a genuine catchup failure and must not be retried") - - -# --- exit 3: retried only when the archive explains it ------------------------- - -def test_exit_3_with_a_fetch_fault_is_retried(cluster): - """The one exit-3 observed in production: a pod that could not reach STS. - - Every aws s3 cp failed before touching S3, stellar-core reported it as a - stale archive, and the retry succeeded on another node in 32 seconds. - """ - end = dispatch(cluster) - cluster.advance(end, 'incomplete') - cluster.finalize(end, 1, archive='fetch_fault') - cluster.reconcile() - - assert not condemned(cluster, end), f"failed={cluster.failed()}" - assert job_exists(cluster, end, 2) - - -def test_exit_3_with_nothing_to_explain_it_is_condemned(cluster): - """A give-up line with no fetch cascade in front of it earns no retry. - - Conservative by choice: the archive survives on the volume, so an - unrecognised cause is read off the failed run and added to the marker lists - rather than guessed at now. - """ - end = dispatch(cluster) - cluster.advance(end, 'unexplained') - cluster.finalize(end, 1, archive='bare') - cluster.reconcile() - - assert condemned(cluster, end), f"jobs={cluster.jobs()}" - assert not job_exists(cluster, end, 2) - assert cluster.failed()[str(end)]['attempts'] == 1 - - -def test_exit_3_waits_for_the_collector_before_deciding(cluster): - """The archive is the evidence, so the decision cannot precede .done.""" - end = dispatch(cluster) - cluster.advance(end, 'incomplete') # no finalize: nothing to read yet - - cluster.reconcile() - - assert not condemned(cluster, end), "condemned before the evidence existed" - assert not job_exists(cluster, end, 2), "retried before the evidence existed" - - cluster.finalize(end, 1, archive='fetch_fault') - cluster.reconcile() - assert job_exists(cluster, end, 2), "still not retried once finalized" - - -def test_a_permanently_missing_object_beats_an_earlier_transient_error(cluster): - """A 404 is not transient, and it wins over a connect error further back. - - Both markers are in the window on purpose: the nearest cause to the anchor is - the one that killed the attempt, so a recovered connect error earlier in the - same window must not earn a retry. - """ - end = dispatch(cluster) - cluster.advance(end, 'incomplete') - cluster.finalize(end, 1, archive=( - # recovered earlier -- must NOT decide the outcome - 'fatal error: Could not connect to the endpoint URL: ' - '"https://sts.us-east-1.amazonaws.com/"\n' - '2026-01-01T00:00:00.000 GAJSL [History INFO] Selected archive core_live_002\n' - # the cause that actually terminated it - 'fatal error: An error occurred (404) when calling the HeadObject ' - 'operation: Key does not exist\n' - '2026-01-01T00:00:00.000 GAJSL [History WARNING] Could not download file: ' - 'archive core_live_003 maybe missing file history/00/00/00/history-0.json\n' - '2026-01-01T00:00:00.000 GAJSL [History ERROR] Missing HAS for ledger 1: ' - 'maybe stale archive core_live_003\n' - '2026-01-01T00:00:00.000 GAJSL [History WARNING] Catchup failed\n')) - cluster.reconcile() - - assert condemned(cluster, end), f"a 404 was treated as transient; jobs={cluster.jobs()}" - - -def test_a_recovered_fetch_fault_does_not_earn_a_retry_for_a_later_failure(cluster): - """The anchor must be the cause of THIS give-up, not an earlier recovered one. - - stellar-core retries a failed fetch, so a range can log the whole cascade - several times and carry on -- 10 of the 11 in the one production exit-3 were - retries that recovered. Crediting any of them would retry a range that later - died of something else entirely. - """ - end = dispatch(cluster) - cluster.advance(end, 'incomplete') - cluster.finalize(end, 1, archive=( - # a fetch fault that stellar-core recovered from - 'fatal error: Could not connect to the endpoint URL: ' - '"https://sts.us-east-1.amazonaws.com/"\n' - '2026-01-01T00:00:00.000 GAJSL [History WARNING] Could not download file: ' - 'archive core_live_003 maybe missing file history/00/00/00/history-0.json\n' - '2026-01-01T00:00:00.000 GAJSL [History ERROR] Missing HAS for ledger 1: ' - 'maybe stale archive core_live_003\n' - # ...then it got the file and went on to replay - + ''.join('2026-01-01T00:00:0%d.000 GAJSL [Ledger INFO] ' - 'Ledger close complete: %d\n' % (i % 10, 100 + i) - for i in range(20)) - # ...and died of something the archive does not explain - + '2026-01-01T00:00:00.000 GAJSL [History WARNING] Catchup failed\n')) - cluster.reconcile() - - assert condemned(cluster, end), ( - f"a recovered fetch fault 20 lines earlier earned a retry; " - f"jobs={cluster.jobs()}") - - -def test_the_real_production_exit_3_is_retried_end_to_end(cluster): - """The whole path, on output stellar-core actually produced. - - Every other test here feeds hand-written archive text, so the pattern and the - fixture agree by construction -- they cannot falsify each other. This is the - verbatim archive of the one exit-3 in the 2026-08-04 run: a pod whose - aws s3 cp could not reach STS, which gave up after 35 minutes and whose - retry fetched the same object from the same bucket in 32 seconds. - """ - import gzip - import pathlib - real = (pathlib.Path(__file__).resolve().parent.parent - / 'data' / 'real-sts-fault-exit3.log.gz') - with gzip.open(real, 'rt', errors='replace') as fh: - archive = fh.read() - - end = dispatch(cluster) - cluster.advance(end, 'incomplete') - cluster.finalize(end, 1, archive=archive) - cluster.reconcile() - - assert not condemned(cluster, end), ( - f"the real STS-fault exit-3 was condemned; failed={cluster.failed()}") - assert job_exists(cluster, end, 2), "no retry was dispatched" diff --git a/src/MissionParallelCatchup/tests/reconcile/test_txapply_histogram.py b/src/MissionParallelCatchup/tests/reconcile/test_txapply_histogram.py deleted file mode 100644 index 3e783252..00000000 --- a/src/MissionParallelCatchup/tests/reconcile/test_txapply_histogram.py +++ /dev/null @@ -1,189 +0,0 @@ -"""RACE #2: a txApply that arrives after the range is first recorded is -backfilled into progress.json but never reaches the Prometheus histogram. - -The interleaving under test is the ordinary one at 1024 workers: the Job flips -to succeeded and reconcile records the range before the log-collector sidecar -has flushed that attempt's .metrics, so the first record carries txApply=None. -A later pass backfills the real value into progress.json. The histogram is -supposed to be a replay of the recorded ranges, so once progress.json says -txApply=1.25 the histogram must have counted 1.25 -- exactly once. - -Every assertion here is on observed state: the durable progress record on the -fake logs volume, and the samples the Prometheus client actually exports. -Nothing reads job_monitor's source. -""" - -import metrics -import job_monitor as jm - - -# --- reading the exported metric -------------------------------------------- -# -# The histograms are module-level and share the global REGISTRY, so absolute -# values leak across tests in one process. Every assertion below is therefore a -# delta taken inside a single test. This reads the exported samples -- the same -# numbers /metrics would serve -- not any private attribute. - -def _hist(metric): - """(count, sum) of a label-less Histogram, from its exported samples.""" - count = total = 0.0 - for family in metric.collect(): - for s in family.samples: - if s.name.endswith('_count'): - count = s.value - elif s.name.endswith('_sum'): - total = s.value - return count, total - - -def _delta(before, after): - return after[0] - before[0], round(after[1] - before[1], 9) - - -def _succeed_without_metrics(cluster, end): - """Job succeeds, collector has not written anything for it yet. - - No .metrics, no .log.gz, and the fake pod log is empty, so all three of - tx_apply_for_range's sources come up dry -- which is exactly the state the - monitor is in when it records the range in the same second the Job flips. - """ - cluster.reconcile() - cluster.advance(end, 'succeeded') - cluster.reconcile() - - -def test_late_txapply_reaches_the_histogram_not_just_progress_json(cluster): - """The bug, stated as the disagreement it causes. - - progress.json ends up saying txApply is known for the range while the - histogram never counted it, so the artifact and /metrics describe different - runs. - """ - _succeed_without_metrics(cluster, 300) - - # Precondition: recorded, but with no txApply yet. If this ever stops - # holding the test below is not exercising the race any more. - assert cluster.completed()['300']['txApply'] is None - - before = _hist(metrics.tx_apply_duration) - - # The collector finishes and flushes the attempt's measurements. - cluster.finalize(300, 1, tx_apply=1.25) - cluster.reconcile() - - # The durable artifact now claims the value is known... - assert cluster.progress()['completed']['300']['txApply'] == 1.25 - - # ...so the histogram must have counted that same value. - count, total = _delta(before, _hist(metrics.tx_apply_duration)) - assert (count, total) == (1.0, 1.25), ( - "progress.json carries txApply=1.25 for range 300 but the histogram " - f"observed count+{count} sum+{total}: the backfilled value can never " - "be counted, so /metrics under-reports every range whose .metrics " - "landed after the range was first recorded") - - -def test_backfilled_txapply_is_counted_once_not_on_every_later_pass(cluster): - """The other half of the contract: exactly once, not once per pass. - - A fix that simply stops skipping the range would re-observe the value on - every subsequent reconcile, which at a 10s loop inflates the histogram - without bound. - """ - _succeed_without_metrics(cluster, 300) - before = _hist(metrics.tx_apply_duration) - - cluster.finalize(300, 1, tx_apply=1.25) - for _ in range(4): - cluster.reconcile() - - assert cluster.progress()['completed']['300']['txApply'] == 1.25 - count, total = _delta(before, _hist(metrics.tx_apply_duration)) - assert (count, total) == (1.0, 1.25), ( - f"range 300's txApply was observed {count} times across four passes; " - "the histogram must count each recorded range exactly once") - - -def test_durations_recorded_up_front_are_not_recounted_while_txapply_is_late(cluster): - """seconds/wallSeconds are known on the first record and must stay at one. - - This is the failure mode of the tempting one-line fix (move the guard - inside the txApply branch): the range then stays unmarked for as many - passes as the collector takes, and every one of those passes re-observes - the durations it already counted. The two histograms would drift apart in - opposite directions. - """ - _succeed_without_metrics(cluster, 300) - - rec = cluster.completed()['300'] - assert rec['seconds'] is not None and rec['wallSeconds'] is not None - seconds, wall = rec['seconds'], rec['wallSeconds'] - - before_full = _hist(metrics.full_duration) - before_wall = _hist(metrics.wall_duration) - - # Three passes with the collector still silent, then it finally lands. - for _ in range(3): - cluster.reconcile() - cluster.finalize(300, 1, tx_apply=0.5) - cluster.reconcile() - cluster.reconcile() - - assert cluster.progress()['completed']['300']['txApply'] == 0.5 - - assert _delta(before_full, _hist(metrics.full_duration)) == (0.0, 0.0), ( - "the full-duration histogram re-observed range 300's already-counted " - f"{seconds}s while waiting for its txApply") - assert _delta(before_wall, _hist(metrics.wall_duration)) == (0.0, 0.0), ( - "the wall-duration histogram re-observed range 300's already-counted " - f"{wall}s while waiting for its txApply") - - -def test_txapply_present_on_first_sight_is_still_counted_exactly_once(cluster): - """Baseline: the non-racing order must keep working. - - Collector finalizes before the monitor ever sees the Job, so txApply is - known at first record. One observation, and no second one later. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1, tx_apply=2.5) - - before = _hist(metrics.tx_apply_duration) - cluster.reconcile() - cluster.reconcile() - cluster.reconcile() - - assert cluster.progress()['completed']['300']['txApply'] == 2.5 - assert _delta(before, _hist(metrics.tx_apply_duration)) == (1.0, 2.5) - - -def test_two_ranges_landing_their_metrics_at_different_times_both_count(cluster): - """The population-level consequence, at the smallest scale that shows it. - - One range's .metrics is ready on the first pass and the other's is not. - Both end up in progress.json with a txApply, so the histogram must contain - both -- not just the one that happened to win the race. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.advance(200, 'succeeded') - # 300's collector is quick; 200's is not. - cluster.finalize(300, 1, tx_apply=1.0) - - before = _hist(metrics.tx_apply_duration) - cluster.reconcile() - - assert cluster.completed()['200']['txApply'] is None - - cluster.finalize(200, 1, tx_apply=3.0) - cluster.reconcile() - - recorded = {k: v['txApply'] for k, v in cluster.completed().items()} - assert recorded == {'300': 1.0, '200': 3.0} - - count, total = _delta(before, _hist(metrics.tx_apply_duration)) - assert (count, total) == (2.0, 4.0), ( - f"progress.json holds txApply for {sorted(recorded)} but the histogram " - f"counted {count} of them (sum {total}); only the range whose .metrics " - "was ready on the first pass was observed") diff --git a/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py b/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py deleted file mode 100644 index 0853979e..00000000 --- a/src/MissionParallelCatchup/tests/resilience/test_collector_restart.py +++ /dev/null @@ -1,669 +0,0 @@ -"""The collector sidecar restarts independently of the monitor. - -log_collector holds every peak it measures in module-level dicts, keyed by pod -name, and writes them to the shared volume. Those dicts do not survive an -OOM-kill of the sidecar, but the files on the volume do -- and the pod they -describe keeps running. So every durable write has to assume the process that -made the previous one is gone and that whatever is already on disk was measured -by someone who saw more than this process did. - -Two failures of that contract were found by hand before: a restart reset a -range's duration clock so attemptSeconds recorded 0.2s, and a newest-wins write -LOWERED an already-recorded peak. Both are pinned here. - -Everything is asserted against the bytes on the shared volume, read back either -with json.load or -- better -- through job_monitor's own readers, which are the -real consumer. Nothing here reads the collector's source. - -No reconcile: the volume is the entire interface between the two processes, so -these drive log_collector's file-writing entry points directly. -""" - -import asyncio -import gzip -import json -import os - -import pytest - -import config -import records -import attempts -import job_monitor as jm -import log_collector as lc - -GIB = 1073741824 - - -def _arm_half_write(monkeypatch, mod, suffix): - """Make the next write to `suffix` land half a file, then fail with ENOSPC. - - Injected at the file object rather than at json.dump: a disk filling up - mid-write is the actual failure the tmp+rename is there for, and patching - the serializer only exercises whichever one the code happens to call. - """ - real_open = open - seen = {} - - class _HalfWrite: - def __init__(self, path): - self.fh = real_open(path, 'w') - - def __enter__(self): - return self - - def __exit__(self, *exc): - self.fh.close() - return False - - def write(self, blob): - self.fh.write(blob[:len(blob) // 2]) - seen['torn'] = True - raise OSError(28, 'No space left on device') - - def half_open(path, mode='r', *a, **kw): - # One-shot: the point is that the process carries on afterwards, so - # everything the collector writes after the failure must be real. - if not seen and mode in ('w', 'wt') and str(path).endswith(suffix): - return _HalfWrite(path) - return real_open(path, mode, *a, **kw) - - monkeypatch.setattr(mod, 'open', half_open, raising=False) - return seen - - -# -- the shared volume, and a collector with no memory of anything ------------ - -@pytest.fixture -def vol(tmp_path, monkeypatch): - """A shared /logs both processes agree on, and cleared collector state. - - Every module-level dict log_collector keeps is replaced, not emptied: they - are process state, and a test that inherited another test's pod entries - would be measuring the wrong process. - """ - log_dir = tmp_path / 'logs' - log_dir.mkdir() - monkeypatch.setattr(config, 'LOG_DIR', str(log_dir)) - # The monitor reads the same directory off its own module global. - monkeypatch.setattr(config, 'LOG_DIR', str(log_dir)) - restart(monkeypatch) - monkeypatch.setattr(lc, '_pod_secs', {}) - monkeypatch.setattr(lc, '_wake', {}) - monkeypatch.setattr(lc, 'token', lambda: 'test-token') - return log_dir - - -def restart(monkeypatch): - """Wipe exactly what an OOM-kill of the sidecar wipes: its memory. - - The volume is untouched, which is the whole point -- a restarted collector - starts every high-water at zero while the file on disk still holds the real - one. - """ - for name in ('_eph_peak', '_anon_peak', '_ws_peak', '_peak_flushed', '_streaming'): - if hasattr(lc, name): - monkeypatch.setattr(lc, name, {}) - # Added by the ephemeral-flush fix; absent on builds without it. - if hasattr(lc, '_eph_flushed'): - monkeypatch.setattr(lc, '_eph_flushed', {}) - - -def metrics(end, attempt=1): - """What the monitor would find in .metrics, or None if there is no file.""" - try: - with open(records.metrics_path(str(end), attempt)) as fh: - return json.load(fh) - except OSError: - return None - - -def run(coro): - return asyncio.run(coro) - - -# -- a kubelet /stats/summary that says whatever the test needs --------------- - -class _Resp: - def __init__(self, payload): - self._payload = payload - - async def __aenter__(self): - return self - - async def __aexit__(self, *exc): - return False - - def raise_for_status(self): - pass - - async def json(self): - return self._payload - - -class FakeSession: - """Stands in for the aiohttp session sample_kubelet fetches through.""" - - def __init__(self, payload): - self.payload = payload - self.urls = [] - - def get(self, url, **kwargs): - self.urls.append(url) - return _Resp(self.payload) - - -def summary(pod, rss=None, ws=None, eph=None, container=None): - """One node's stats/summary, shaped the way kubelet shapes it.""" - entry = {'podRef': {'name': pod}, 'containers': []} - if eph is not None: - entry['ephemeral-storage'] = {'usedBytes': eph} - mem = {} - if rss is not None: - mem['rssBytes'] = rss - if ws is not None: - mem['workingSetBytes'] = ws - entry['containers'].append({'name': container or lc.CONTAINER, 'memory': mem}) - return {'pods': [entry]} - - -def sample(pod, **kw): - """One real sample_kubelet pass over one node.""" - run(lc.sample_kubelet(FakeSession(summary(pod, **kw)), ['node-1'])) - - -def finalize(pod, end, attempt=1, succeeded=False, started=None, tx=None): - """One real finalize() for an attempt, as its poller would call it.""" - return run(lc.finalize(None, pod, str(end), attempt, - tx if tx is not None else lc.TxApplyScanner(), - lambda p: succeeded, started)) - - -# -- a peak may never go backwards ------------------------------------------- - -@pytest.mark.parametrize('key', lc.PEAK_KEYS) -@pytest.mark.parametrize('first, second', [(8, 1), (1, 8)]) -def test_a_recorded_peak_only_ever_rises(vol, key, first, second): - """Every field in PEAK_KEYS, in both orders. - - The restarted-poller case reduced to its file operation: a smaller second - write is a fresh process's first flush, counting from zero. The guard must - still not be a write-once latch -- growth is the normal case. - """ - lc.write_metrics('300', 1, {key: first * GIB}) - lc.write_metrics('300', 1, {key: second * GIB}) - - assert metrics(300)[key] == 8 * GIB - - -def test_a_write_that_omits_a_peak_leaves_it_alone(vol): - """finalize writes only the axes it has. The rest are already on disk.""" - lc.write_metrics('300', 1, {'peakAnonBytes': 5 * GIB, - 'peakEphemeralBytes': 30 * GIB}) - lc.write_metrics('300', 1, {'txApplySeconds': 12.5}) - - stored = metrics(300) - assert stored['peakAnonBytes'] == 5 * GIB - assert stored['peakEphemeralBytes'] == 30 * GIB - assert stored['txApplySeconds'] == 12.5 - - -def test_resumed_true_is_monotonic_across_restarted_writers(vol): - lc.write_metrics('300', 2, {'resumed': True}) - lc.write_metrics('300', 2, {'resumed': False, 'attemptSeconds': 10.0}) - - assert metrics(300, 2)['resumed'] is True - - -def test_finalize_recovers_resume_after_the_scanner_is_recreated(vol): - """The first poll saw RESUME, then its scanner vanished before finalize.""" - path = lc.base('300', 2) + '.log.gz' - with gzip.open(path, 'wt') as fh: - fh.write('RESUME: local state reached ledger 250; skipping new-db\n') - - finalize('w-300-a2', 300, attempt=2, tx=lc.TxApplyScanner()) - - assert metrics(300, 2)['resumed'] is True - - -def test_finalize_recovers_txapply_after_the_scanner_is_recreated(vol, monkeypatch): - """The first poll saw the final medida block, then its scanner vanished.""" - monkeypatch.setattr(config, 'SAVE_SUCCESS_LOGS', False) - path = lc.base('300', 2) + '.log.gz' - with gzip.open(path, 'wt') as fh: - fh.write('RESUME: local state reached ledger 250; skipping new-db\n') - fh.write("metric 'ledger.transaction.apply'\n") - fh.write(' count = 123\n') - fh.write(' sum = 4200.0ms\n') - - finalize('w-300-a2', 300, attempt=2, succeeded=True, - tx=lc.TxApplyScanner(recreated=True)) - - assert metrics(300, 2)['resumed'] is True - assert metrics(300, 2)['txApplySeconds'] == 4.2 - assert not os.path.exists(path), \ - "the test must prove recovery happened before success-log discard" - - -def test_finalize_recovers_txapply_for_a_scanner_that_was_never_recreated(vol, - monkeypatch): - """The gap the monitor used to cover, now closed at the source. - - stellar-core prints the medida block once, at exit, so a poller that ran the - pod's whole life can still end a beat early and hold no total -- with nothing - to recreate. The rescue used to require `recreated`, so it never looked, and - the monitor re-parsed the same archive behind it. Measured on the 2026-08-04 - run: 15 attempts of 4805 landed here, and replaying the archive through this - same scanner recovers every one. - """ - monkeypatch.setattr(config, 'SAVE_SUCCESS_LOGS', False) - path = lc.base('300', 1) + '.log.gz' - with gzip.open(path, 'wt') as fh: - fh.write("metric 'ledger.transaction.apply'\n") - fh.write(' count = 123\n') - fh.write(' sum = 4200.0ms\n') - - # recreated=False: this poller ran start to finish and simply has no total. - finalize('w-300-a1', 300, attempt=1, succeeded=True, tx=lc.TxApplyScanner()) - - assert metrics(300, 1)['txApplySeconds'] == 4.2, \ - "the collector did not re-read its own archive, so the value is lost" - - -def test_finalize_does_not_promote_resume_declined(vol): - path = lc.base('300', 2) + '.log.gz' - with gzip.open(path, 'wt') as fh: - fh.write('RESUME DECLINED: no usable local state; running new-db\n') - - finalize('w-300-a2', 300, attempt=2, tx=lc.TxApplyScanner()) - - assert (metrics(300, 2) or {}).get('resumed') is not True - - -def test_peaks_from_different_writes_accumulate_into_one_record(vol): - """Each axis is flushed by whoever measured it; the file is the union.""" - lc.write_metrics('300', 1, {'peakAnonBytes': 5 * GIB}) - lc.write_metrics('300', 1, {'peakWorkingSetBytes': 9 * GIB}) - lc.write_metrics('300', 1, {'peakEphemeralBytes': 30 * GIB}) - - assert metrics(300) == {'peakAnonBytes': 5 * GIB, - 'peakWorkingSetBytes': 9 * GIB, - 'peakEphemeralBytes': 30 * GIB} - - -# -- mid-flight flushes, and what a restart may lose -------------------------- - -def test_a_midflight_anon_flush_survives_a_collector_restart(vol, monkeypatch): - """The pinned bug, driven through the real sampler. - - A long range peaks early (download and bucket-apply), the sidecar is - OOM-killed, and the replacement watches only the quiet replay tail. What - the range gets sized on next run must still be the high-water. - """ - lc._streaming['w-300'] = ('300', '1') - sample('w-300', rss=6 * GIB) - assert metrics(300)['peakAnonBytes'] == 6 * GIB, "flush never reached the volume" - - restart(monkeypatch) - lc._streaming['w-300'] = ('300', '1') - sample('w-300', rss=1 * GIB) - finalize('w-300', 300) - - assert metrics(300)['peakAnonBytes'] == 6 * GIB - # And the consumer agrees: this is the figure that sizes the next run. - assert attempts.peaks_for_range('300', 1)['peakAnonBytes'] == 6 * GIB - - -def test_a_midflight_ephemeral_flush_survives_a_collector_restart(vol, monkeypatch): - """peakEphemeralBytes sizes an ephemeral-storage request, and a request - that comes back too small is an eviction, not a slow range. - - Disk use is not monotonic -- stellar-core drops its download staging once - buckets are applied -- so a replacement sidecar re-measuring the same pod - does not recover the earlier high-water. It has to already be on the volume. - """ - monkeypatch.setattr(config, 'STORAGE_MODE', 'ephemeral') - lc._streaming['w-300'] = ('300', '1') - sample('w-300', rss=1 * GIB, eph=34 * GIB) - - restart(monkeypatch) - monkeypatch.setattr(config, 'STORAGE_MODE', 'ephemeral') - lc._streaming['w-300'] = ('300', '1') - sample('w-300', rss=1 * GIB, eph=4 * GIB) - finalize('w-300', 300) - - assert metrics(300)['peakEphemeralBytes'] == 34 * GIB - assert attempts.peaks_for_range('300', 1)['peakEphemeralBytes'] == 34 * GIB - - -def test_pvc_mode_records_no_ephemeral_peak_at_all(vol, monkeypatch): - """In pvc mode the range's data sits on the volume, not on node disk, so - there is no ephemeral-storage request to size and the figure would be - noise. Sampling it is gated on the mode; flushing it must be too.""" - monkeypatch.setattr(config, 'STORAGE_MODE', 'pvc') - lc._streaming['w-300'] = ('300', '1') - sample('w-300', rss=1 * GIB, eph=34 * GIB) - finalize('w-300', 300) - - assert 'peakEphemeralBytes' not in (metrics(300) or {}) - - -def test_a_flush_with_no_stream_registered_writes_nothing(vol, monkeypatch): - """_streaming is repopulated when a poller opens. A sample that lands on a - pod with no poller yet has nowhere to write and must not guess a file.""" - monkeypatch.setattr(config, 'STORAGE_MODE', 'ephemeral') - sample('w-300', rss=6 * GIB, eph=34 * GIB) - - assert os.listdir(vol) == [] - - -def test_finalize_cannot_lower_a_peak_the_volume_already_holds(vol, monkeypatch): - """The restart case at the level of finalize itself. - - Whatever is in the replacement process's dicts is a partial observation; - the file was written by a process that saw more. - """ - lc.write_metrics('300', 1, {'peakAnonBytes': 6 * GIB, - 'peakWorkingSetBytes': 11 * GIB, - 'peakEphemeralBytes': 34 * GIB}) - lc._anon_peak['w-300'] = 1 * GIB - lc._ws_peak['w-300'] = 2 * GIB - lc._eph_peak['w-300'] = 3 * GIB - - finalize('w-300', 300) - - stored = metrics(300) - assert stored['peakAnonBytes'] == 6 * GIB - assert stored['peakWorkingSetBytes'] == 11 * GIB - assert stored['peakEphemeralBytes'] == 34 * GIB - - -def test_the_flush_ratio_does_not_hold_back_the_first_measurement(vol): - """A restarted sampler has flushed nothing, so its first sample must land - on the volume immediately -- otherwise a pod that peaks once and then dies - contributes nothing at all.""" - lc._streaming['w-300'] = ('300', '1') - sample('w-300', rss=3 * GIB) - - assert metrics(300)['peakAnonBytes'] == 3 * GIB - - -def test_a_flush_goes_to_the_attempt_that_is_streaming(vol): - """Peaks are keyed by (range, attempt); a retry must not inherit them.""" - lc._streaming['w-300-a2'] = ('300', '2') - sample('w-300-a2', rss=7 * GIB) - - assert metrics(300, 2)['peakAnonBytes'] == 7 * GIB - assert metrics(300, 1) is None - - -# -- .done is a promise about .metrics ---------------------------------------- - -def test_done_never_appears_beside_a_half_written_metrics_file(vol, monkeypatch): - """The monitor reaps the Job -- and with it the pod -- the moment .done - exists. If .metrics can be observed mid-write, that reap makes a torn - record permanent.""" - lc.write_metrics('300', 1, {'peakAnonBytes': 6 * GIB, 'txApplySeconds': 30.0}) - - seen = _arm_half_write(monkeypatch, records, '.metrics.tmp') - lc._anon_peak['w-300'] = 9 * GIB - finalize('w-300', 300) - - assert seen.get('torn'), "the interrupted write never happened" - # The old record is intact and parseable -- not truncated, not empty. - assert metrics(300) == {'peakAnonBytes': 6 * GIB, 'txApplySeconds': 30.0} - # .done still lands: the collector really will write nothing more for this - # attempt, and withholding it only strands the Job until its TTL. - assert os.path.exists(records.done_path('300', 1)) - # What the monitor actually reads is a complete record, not a torn one. - assert attempts.peaks_for_range('300', 1) == {'peakAnonBytes': 6 * GIB} - - -def test_done_lands_after_the_metrics_it_promises(vol): - """Ordering, observed by mtime rather than by reading the source.""" - lc._anon_peak['w-300'] = 6 * GIB - finalize('w-300', 300) - - assert (os.stat(records.done_path('300', 1)).st_mtime_ns - >= os.stat(records.metrics_path('300', 1)).st_mtime_ns) - assert metrics(300)['peakAnonBytes'] == 6 * GIB - - -def test_a_truncated_metrics_file_does_not_poison_the_next_write(vol): - """Whatever tore the previous record, the next flush must still produce a - file the monitor can read -- and must not raise inside the sampler.""" - with open(records.metrics_path('300', 1), 'w') as fh: - fh.write('{"peakAnonBytes": 644245') - - lc.write_metrics('300', 1, {'peakAnonBytes': 5 * GIB}) - - assert metrics(300) == {'peakAnonBytes': 5 * GIB} - assert attempts.peaks_for_range('300', 1) == {'peakAnonBytes': 5 * GIB} - - -def test_an_attempt_with_nothing_to_report_still_finalizes(vol): - """No peaks, no duration, no txApply -- a pod rejected before its container - ran. .done has to land anyway or the monitor waits out JOB_TTL_SECONDS on a - Job that will never learn anything.""" - finalize('w-300', 300) - - assert metrics(300) is None - assert jm._attempt_finalized('300', 1) - - -def test_marking_done_twice_is_harmless(vol): - lc._mark_done('300', 1) - lc._mark_done('300', 1) - - assert os.path.exists(records.done_path('300', 1)) - assert jm._attempt_finalized('300', 1) - - -# -- finalizing the same attempt twice ---------------------------------------- - -def test_finalizing_the_same_attempt_twice_keeps_its_measurements(vol, monkeypatch): - """A restarted collector re-opens a poller for a pod that is still there - and still terminal, and finalizes it a second time. The second pass - measured nothing -- sample_kubelet only samples Running pods -- so it must - add nothing and take nothing away.""" - lc._pod_secs['w-300'] = 3600.4 - lc._anon_peak['w-300'] = 6 * GIB - lc._eph_peak['w-300'] = 34 * GIB - tx = lc.TxApplyScanner() - tx.seconds = 120.0 - finalize('w-300', 300, tx=tx) - first = metrics(300) - assert first['attemptSeconds'] == 3600.4 - assert first['attemptSecondsExact'] is True - - restart(monkeypatch) - # The main loop re-reads the pod's own timestamps every cycle it sees it - # terminal, so the second poller gets the same exact figure. - lc._pod_secs['w-300'] = 3600.4 - finalize('w-300', 300) - - assert metrics(300) == first - - -def test_a_second_finalize_without_pod_timestamps_keeps_the_real_duration(vol, - monkeypatch): - """attemptSeconds is a fixed quantity measured two ways, and both are lower - bounds: the pod's own start->finish is exact, while the poller's watch time - covers only the part of the attempt this process was alive for. A second - finalize that has lost the pod object -- 404 on the log endpoint, node - already reaped -- may only ever offer the worse of the two, so it must not - replace the better one. - - This is the same fabricated near-zero duration that was found by hand, - reached from the reopen path rather than from a cold start. - """ - lc._pod_secs['w-300'] = 3600.4 - finalize('w-300', 300) - assert metrics(300)['attemptSeconds'] == 3600.4 - - restart(monkeypatch) - # No _pod_secs: this poller never saw the pod terminal, it just took a 404. - # `started` is when IT attached, which is a moment ago. - finalize('w-300', 300, started=_moments_ago()) - - assert metrics(300)['attemptSeconds'] == 3600.4 - - -def _moments_ago(): - """A `started` stamp on the same monotonic clock finalize reads.""" - async def now(): - return asyncio.get_event_loop().time() - return run(now()) - - -def test_a_cold_poller_on_an_already_terminal_pod_reports_no_duration(vol): - """The other half of the pinned duration bug: with no pod timestamps and - no start of its own, the collector reports nothing rather than a - fabricated near-zero. The monitor's own figure is authoritative.""" - lc._anon_peak['w-300'] = 6 * GIB - finalize('w-300', 300, started=None) - - stored = metrics(300) - assert 'attemptSeconds' not in stored - assert stored['peakAnonBytes'] == 6 * GIB - # ...and the monitor is left free to supply the real one. - assert attempts.seconds_for_range('300', 1, final=3600.4) == 3600.4 - - -def test_a_poller_that_watched_the_whole_attempt_still_reports_its_duration(vol): - """The fallback is not disabled, only outranked.""" - started = _moments_ago() - 42.0 - finalize('w-300', 300, started=started) - - stored = metrics(300) - assert stored['attemptSeconds'] == pytest.approx(42.0, abs=1.0) - assert stored['attemptSecondsExact'] is False - assert attempts.seconds_for_range('300', 1) is None - - -def test_the_duration_the_collector_records_is_the_pods_not_the_pollers(vol): - """_pod_secs is the pod's own start->finish and always wins.""" - lc._pod_secs['w-300'] = 3600.4 - finalize('w-300', 300, started=_moments_ago() - 5.0) - - assert metrics(300)['attemptSeconds'] == 3600.4 - assert metrics(300)['attemptSecondsExact'] is True - - -# -- .outcome is written once, by whoever got there first --------------------- - -def _pod(name, phase='Failed', exit_code=None, reason=None, message=None, - disrupted=False): - status = {'phase': phase} - if reason: - status['reason'] = reason - if message: - status['message'] = message - if disrupted: - status['conditions'] = [{'type': 'DisruptionTarget', 'status': 'True'}] - if exit_code is not None: - status['containerStatuses'] = [ - {'name': lc.CONTAINER, 'state': {'terminated': {'exitCode': exit_code}}}] - return {'metadata': {'name': name, 'labels': {}}, 'status': status} - - -def test_an_existing_outcome_is_not_overwritten_by_a_later_pod(vol): - """Two pods can carry the same range-end and attempt labels -- a Job that - replaces its pod, or a stale pod list after a restart. The first verdict is - the one taken while the evidence was fresh; a later, different pod must not - silently rewrite it.""" - lc.record_outcome(_pod('w-300-first', disrupted=True), '300', 1) - first = records.read_outcome('300', 1) - - lc.record_outcome(_pod('w-300-second', exit_code=1), '300', 1) - - assert records.read_outcome('300', 1) == first - assert first['outcome'] == 'disrupted' - assert first['pod'] == 'w-300-first' - - -def test_an_outcome_written_by_the_monitor_is_not_re_classified(vol, monkeypatch): - """Both processes write this file and both read it. The collector must - treat the monitor's verdict as final, including the fields only the monitor - records -- attemptSeconds for a failed leg lives nowhere else.""" - with open(records.outcome_path('300', 1), 'w') as fh: - json.dump({'outcome': 'ephemeral', 'exitCode': None, 'pod': 'w-300', - 'attemptSeconds': 1800.0}, fh) - - lc.record_outcome(_pod('w-300', exit_code=3), '300', 1) - - assert records.read_outcome('300', 1)['outcome'] == 'ephemeral' - assert records.read_outcome('300', 1)['attemptSeconds'] == 1800.0 - - -def test_a_recorded_outcome_is_a_complete_file_or_no_file(vol, monkeypatch): - """Same rename discipline as .metrics: the monitor branches its whole retry - policy on this file, so a torn read would have to be a crash or a wrong - verdict.""" - _arm_half_write(monkeypatch, records, '.outcome.tmp') - lc.record_outcome(_pod('w-300', exit_code=1), '300', 1) - - assert records.read_outcome('300', 1) is None - assert not os.path.exists(records.outcome_path('300', 1)) - - -def test_an_ephemeral_eviction_is_classified_from_the_pod_message(vol): - """The exit code cannot tell this apart from a catchup failure, and only - the pod carries the discriminator -- so if the collector misses it while - the pod exists, it is gone.""" - lc.record_outcome( - _pod('w-300', exit_code=3, reason='Evicted', - message='Pod ephemeral local storage usage exceeds the total limit ' - 'of containers 40Gi'), - '300', 1) - - assert records.read_outcome('300', 1)['outcome'] == 'ephemeral' - - -# -- the resume state file ---------------------------------------------------- - -def test_state_survives_a_restart_and_untimestamped_junk_never_becomes_it(vol): - """The resume point is read back by a process that did not write it, so a - poisoned value is permanent: sinceTime=unableZ is a 400 on every later - request for that pod, forever.""" - lc.write_state('300', 1, '2026-07-30T10:15:30.123456789Z') - assert lc.read_state('300', 1) == '2026-07-30T10:15:30.123456789Z' - - lc.write_state('300', 1, 'unable') - assert lc.read_state('300', 1) is None - - -def test_an_empty_state_claim_is_not_a_resume_point(vol): - """poll_pod writes '' to claim the range against job_monitor's backstop. - That is a claim, not a timestamp, and must never be sent as sinceTime.""" - lc.write_state('300', 1, '') - - assert lc.read_state('300', 1) is None - assert os.path.exists(lc.base('300', 1) + '.state') - - -def test_discarding_a_successful_range_keeps_its_measurements(vol): - """saveSuccessLogs=false deletes the archive. .metrics is the only place - txApply and the peaks survive a reaped pod, so it has to stay.""" - lc.write_metrics('300', 1, {'peakAnonBytes': 6 * GIB, 'txApplySeconds': 30.0}) - with open(lc.base('300', 1) + '.log.gz', 'wb') as fh: - fh.write(b'\x1f\x8b') - lc.write_state('300', 1, '2026-07-30T10:15:30Z') - - lc.discard('300', 1) - - assert not os.path.exists(lc.base('300', 1) + '.log.gz') - assert metrics(300) == {'peakAnonBytes': 6 * GIB, 'txApplySeconds': 30.0} - - -def test_a_successful_range_discards_its_archive_inside_finalize(vol, monkeypatch): - monkeypatch.setattr(config, 'SAVE_SUCCESS_LOGS', False) - with open(lc.base('300', 1) + '.log.gz', 'wb') as fh: - fh.write(b'\x1f\x8b') - lc._anon_peak['w-300'] = 6 * GIB - - finalize('w-300', 300, succeeded=True) - - assert not os.path.exists(lc.base('300', 1) + '.log.gz') - assert metrics(300)['peakAnonBytes'] == 6 * GIB - assert os.path.exists(records.done_path('300', 1)) diff --git a/src/MissionParallelCatchup/tests/resilience/test_crash_points.py b/src/MissionParallelCatchup/tests/resilience/test_crash_points.py deleted file mode 100644 index a2652b42..00000000 --- a/src/MissionParallelCatchup/tests/resilience/test_crash_points.py +++ /dev/null @@ -1,626 +0,0 @@ -"""Crash the monitor mid-pass, at every side-effect boundary, and restart it. - -reconcile() is a reconciler: it must derive everything it needs from Kubernetes -plus the files on its own volume, so a process that dies halfway through a pass -and comes back with a zeroed in-memory state must converge to the same place. - -The side-effect boundaries inside one pass, in order, are: - - dispatch create_namespaced_job -> `created`/`capacity`/in_progress - success completed[end] = ... -> save_progress -> release_pvc -> reap - backfill completed[end].update(late) -> save_progress -> reap - retry save_verdict -> create attempt N+1 -> delete attempt N - -Every test here kills the process at one of those arrows and then restarts it -with a fresh state dict -- the same thing a pod replacement does -- and asserts -on observed cluster state and the durable record: every range recorded exactly -once, no PVC left behind, no Job left orphaned, no completed range re-run. - -Nothing is asserted about the source text; the injections wrap the fake API or -a single job_monitor function, and everything checked afterwards is either a -file on the volume or an object in the fake cluster. -""" - -import pytest -from kubernetes.client.rest import ApiException - -import fake_k8s -import config -import records -import job_monitor as jm - - -TOTAL_RANGES = 3 # conftest's DEFAULT_CONFIG generates 300/200/100 - - -class Crash(RuntimeError): - """A hard process death -- deliberately NOT an ApiException. - - The monitor handles ApiException in several places; a crash is the thing it - cannot handle, and is what a SIGKILL, an OOM or a node eviction looks like - from inside a pass. - """ - - -# --- injection helpers ------------------------------------------------------- - -def crash_before(monkeypatch, target, name, times=1): - """Die on the way INTO `target.name` -- the effect never happens.""" - real = getattr(target, name) - left = {'n': times} - - def wrapper(*args, **kwargs): - if left['n'] > 0: - left['n'] -= 1 - raise Crash(f"crash before {name}") - return real(*args, **kwargs) - - monkeypatch.setattr(target, name, wrapper) - return left - - -def crash_after(monkeypatch, target, name, times=1, match=None): - """Die on the way OUT of `target.name` -- the effect happened, the caller - never learned about it. This is the boundary that can duplicate work.""" - real = getattr(target, name) - left = {'n': times} - - def wrapper(*args, **kwargs): - result = real(*args, **kwargs) - if left['n'] > 0 and (match is None or match(*args, **kwargs)): - left['n'] -= 1 - raise Crash(f"crash after {name}") - return result - - monkeypatch.setattr(target, name, wrapper) - return left - - -def restart(cluster): - """Replace the monitor process: fresh in-memory state, same volume+cluster. - - Identical to the dict reconcile_loop() builds on entry, so a - restarted monitor starts from exactly what the shipped loop starts from. - """ - cluster.state = {'owner': None, 'replayed': set(), 'max_completed': 0, - 'halted': False, 'counted': {}} - return cluster - - -# --- driving ----------------------------------------------------------------- - -def split(job_name): - """'pc-r300-a2' -> (300, 2)""" - stem, _, attempt = job_name.rpartition('-a') - return int(stem.rsplit('-r', 1)[1]), int(attempt) - - -def finish_live_jobs(cluster, outcome='succeeded', finalize=True): - """Take every not-yet-terminal Job to a terminal state, as the cluster would.""" - touched = [] - for name in sorted(cluster.jobs()): - end, attempt = split(name) - status = cluster.k8s.job(name).status - if status and (status.succeeded or status.failed): - continue - cluster.advance(end, outcome, attempt) - if finalize: - cluster.finalize(end, attempt, tx_apply=1.0, - peaks={'peakAnonBytes': 1024}) - touched.append(name) - return touched - - -def run_to_quiescence(cluster, passes=15): - """Succeed everything still in flight until the run drains (or we give up).""" - for _ in range(passes): - finish_live_jobs(cluster) - cluster.reconcile() - if len(cluster.completed()) == TOTAL_RANGES and not cluster.jobs(): - break - return cluster - - -def assert_converged(cluster): - """The end state of a healthy run, whatever happened on the way there.""" - completed = cluster.completed() - assert sorted(completed) == ['100', '200', '300'], completed - assert cluster.failed() == {} - # Every range recorded once and only once -- a dict cannot hold a duplicate - # key, so the observable form of "counted twice" is a re-run: a second - # attempt for a range that had already been recorded. - for end, record in completed.items(): - assert record['attempts'] == 1, (end, record) - assert cluster.jobs() == [], f"orphaned Jobs: {cluster.jobs()}" - assert cluster.pvcs() == [], f"leaked PVCs: {cluster.pvcs()}" - - -def creates_of(cluster, name): - return cluster.calls.names(verb='create', kind='job').count(name) - - -# --- dispatch boundary ------------------------------------------------------- - -def test_crash_after_create_before_the_range_is_tracked(cluster, monkeypatch): - """create_namespaced_job returned, then the process died. - - The Job exists and nobody recorded that it does. A restart must find it by - LIST and adopt it, not dispatch the range a second time. - """ - crash_after(monkeypatch, cluster.k8s.batch_v1, 'create_namespaced_job') - - with pytest.raises(Crash): - cluster.reconcile() - - # The Job that the dying pass created is real and running. - assert cluster.jobs() == ['pc-r300-a1'] - - restart(cluster) - result = cluster.reconcile() - - # Adopted, not recreated: one create call ever for this name, and the - # restarted pass counts it against capacity instead of dispatching a third. - assert creates_of(cluster, 'pc-r300-a1') == 1 - assert cluster.jobs() == ['pc-r200-a1', 'pc-r300-a1'] - assert sorted(result['in_progress']) == ['200/420', '300/420'] - assert result['created'] == 1 - - run_to_quiescence(cluster) - assert_converged(cluster) - - -def test_crash_after_pvc_create_before_job_create(cluster, monkeypatch): - """ensure_pvc() ran, the Job create never did. The volume must be reused.""" - crash_after(monkeypatch, cluster.k8s.core_v1, - 'create_namespaced_persistent_volume_claim') - - with pytest.raises(Crash): - cluster.reconcile() - - assert cluster.pvcs() == ['pc-data-r300'] - assert cluster.jobs() == [] - - restart(cluster) - cluster.reconcile() - - # One volume for the range, not two, and the Job now mounts it. - assert cluster.calls.names(verb='create', kind='pvc').count('pc-data-r300') == 1 - job = cluster.k8s.job('pc-r300-a1') - claim = job.spec.template.spec.volumes[0].persistent_volume_claim - assert claim.claim_name == 'pc-data-r300' - - run_to_quiescence(cluster) - assert_converged(cluster) - - -# --- success boundary: record -> save_progress -> release_pvc -> reap --------- - -def test_crash_before_save_progress_records_the_range_exactly_once(cluster, - monkeypatch): - """completed[end] existed only in memory. Nothing durable, so redo it.""" - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakAnonBytes': 7}) - - crash_before(monkeypatch, jm, 'save_progress') - with pytest.raises(Crash): - cluster.reconcile() - - # Nothing was written, so nothing is claimed -- and crucially the Job was - # NOT reaped, because the reap sits after the write. - assert cluster.progress() == {} - assert 'pc-r300-a1' in cluster.jobs() - - restart(cluster) - cluster.reconcile() - - record = cluster.completed()['300'] - assert record['attempts'] == 1 - assert record['txApply'] == 1.5 - assert record['peakAnonBytes'] == 7 - assert creates_of(cluster, 'pc-r300-a1') == 1 - assert 'pc-r300-a2' not in cluster.jobs(), "a recorded range must never re-run" - - run_to_quiescence(cluster) - assert_converged(cluster) - - -def test_crash_after_save_progress_before_release_pvc_does_not_leak_the_volume( - cluster, monkeypatch): - """The record is durable and the volume is not yet freed. - - A completed range has nothing left to resume, so its PVC is dead weight -- - 40Gi of gp3 apiece, which is what put 79 TiB on ssc-test. The release must - therefore be reached on a LATER pass too, because the pass that would have - done it is never repeated: the record already exists. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakAnonBytes': 7}) - - crash_before(monkeypatch, jm, 'release_pvc') - with pytest.raises(Crash): - cluster.reconcile() - - assert '300' in cluster.progress()['completed'] - assert 'pc-data-r300' in cluster.pvcs() - - restart(cluster) - for _ in range(3): - cluster.reconcile() - - assert 'pc-data-r300' not in cluster.pvcs(), ( - "the volume of a range recorded complete before the crash was never " - "released; only the first-sight branch releases it and that branch " - "never runs again") - - run_to_quiescence(cluster) - assert_converged(cluster) - - -def test_crash_after_release_pvc_before_the_reap_does_not_orphan_the_job( - cluster, monkeypatch): - """The window that leaves a Job with no owner. - - The range is recorded, its volume is gone, and its Job is still standing. - Nothing in the cluster will ever ask about that Job again -- it is not in - flight, it is not retryable, and its range is complete -- so the reconciler - is the only thing that can clean it up. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakAnonBytes': 7}) - - crash_before(monkeypatch, jm, 'reap_range_jobs') - with pytest.raises(Crash): - cluster.reconcile() - - assert '300' in cluster.progress()['completed'] - assert 'pc-data-r300' not in cluster.pvcs() - assert 'pc-r300-a1' in cluster.jobs(), "precondition: the ownerless Job" - - restart(cluster) - for _ in range(3): - cluster.reconcile() - - assert 'pc-r300-a1' not in cluster.jobs(), ( - "a Job whose range is already recorded complete was left standing " - "forever; it inflates every later LIST and its pod holds a node") - # ...and cleaning it up must not have cost anything: the range stays - # recorded once, with its measurements. - assert cluster.completed()['300']['txApply'] == 1.5 - assert cluster.completed()['300']['attempts'] == 1 - - run_to_quiescence(cluster) - assert_converged(cluster) - - -def test_crash_after_the_reap_leaves_nothing_behind(cluster, monkeypatch): - """Last arrow in the success path: everything is done, the pass just dies.""" - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakAnonBytes': 7}) - - crash_after(monkeypatch, jm, 'reap_range_jobs') - with pytest.raises(Crash): - cluster.reconcile() - - assert 'pc-r300-a1' not in cluster.jobs() - assert 'pc-data-r300' not in cluster.pvcs() - - restart(cluster) - cluster.reconcile() - - # The slot the reaped range freed is refilled, and the range is not redone. - assert cluster.completed()['300']['attempts'] == 1 - assert creates_of(cluster, 'pc-r300-a1') == 1 - assert 'pc-r100-a1' in cluster.jobs() - - run_to_quiescence(cluster) - assert_converged(cluster) - - -# --- backfill boundary ------------------------------------------------------- - -def test_crash_mid_backfill_backfills_on_a_later_pass(cluster, monkeypatch): - """The record was written before the collector finalized; a crash in the - middle of the catch-up write must not make the measurements unreachable.""" - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.reconcile() # recorded with nothing to read yet - - record = cluster.completed()['300'] - assert record['txApply'] is None - assert 'peakAnonBytes' not in record - assert 'pc-r300-a1' in cluster.jobs(), "not finalized, so not reaped" - - # The collector lands, and the monitor dies on the backfill write. - cluster.finalize(300, 1, tx_apply=2.5, peaks={'peakAnonBytes': 99}) - crash_after(monkeypatch, jm, 'save_progress') - with pytest.raises(Crash): - cluster.reconcile() - - restart(cluster) - cluster.reconcile() - - record = cluster.completed()['300'] - assert record['txApply'] == 2.5 - assert record['peakAnonBytes'] == 99 - assert record['attempts'] == 1 - assert 'pc-r300-a1' not in cluster.jobs(), "finalized and backfilled: reap it" - - run_to_quiescence(cluster) - assert_converged(cluster) - - -# --- retry boundary: verdict -> create N+1 -> delete N ----------------------- - -def test_crash_after_the_successor_exists_before_the_predecessor_is_deleted( - cluster, monkeypatch): - """Both attempts are live for a moment. The pass that dies there must not - leave the loser standing once the range finishes.""" - cluster.reconcile() - cluster.advance(300, 'incomplete') - cluster.finalize(300, 1, archive='fetch_fault') - - crash_after(monkeypatch, cluster.k8s.batch_v1, 'create_namespaced_job', - match=lambda ns, body, **kw: body.metadata.name == 'pc-r300-a2') - with pytest.raises(Crash): - cluster.reconcile() - - assert 'pc-r300-a1' in cluster.jobs() and 'pc-r300-a2' in cluster.jobs() - - restart(cluster) - cluster.reconcile() - - # The dead a-1 must never be re-classified into a third attempt: live[] - # keys on the highest attempt for the range. - assert 'pc-r300-a3' not in cluster.jobs() - assert creates_of(cluster, 'pc-r300-a2') == 1 - assert records._cause_count('300', 2, ('fetch-fault',)) == 1, \ - "attempt 1 must be counted once, not once per pass that saw it" - - cluster.advance(300, 'succeeded', attempt=2) - cluster.finalize(300, 2, tx_apply=1.0, peaks={'peakAnonBytes': 1024}) - cluster.reconcile() - - assert cluster.completed()['300']['attempts'] == 2 - assert 'pc-r300-a1' not in cluster.jobs(), "the loser must be swept too" - assert 'pc-r300-a2' not in cluster.jobs() - assert 'pc-data-r300' not in cluster.pvcs() - - -def test_crash_between_the_verdict_and_the_retry_create(cluster, monkeypatch): - """The verdict is on disk and the successor was never created. - - The verdict is what spends the range's budget, so replaying the same failed - attempt after a restart must not spend it a second time -- and for an OOM, - must not climb a second escalation rung either. - """ - cluster.reconcile() - cluster.advance(300, 'oom') - - crash_before(monkeypatch, cluster.k8s.batch_v1, 'create_namespaced_job') - with pytest.raises(Crash): - cluster.reconcile() - - assert records._verdict_of('300', 1) == 'oom' - assert 'pc-r300-a1' in cluster.jobs(), \ - "the predecessor must survive: without it the range restarts at attempt 1" - - restart(cluster) - cluster.reconcile() - - assert 'pc-r300-a2' in cluster.jobs() - resources = (cluster.k8s.job('pc-r300-a2') - .spec.template.spec.containers[0].resources) - # One OOM seen, so exactly one rung: 24000Mi * 1.5. Two would mean the - # replayed attempt was counted twice. - assert resources.requests['memory'] == '13824Mi' - assert records._cause_count('300', 1, ('oom', 'failed')) == 1 - assert cluster.failed() == {} - - -# --- API errors on create ---------------------------------------------------- - -def test_409_on_dispatch_is_benign_and_does_not_double_record(cluster): - """AlreadyExists is the dispatch mutex, not an error.""" - cluster.k8s.fail_next['create job'] = fake_k8s.api_exception( - 409, 'Conflict', 'jobs.batch "pc-r300-a1" already exists') - - result = cluster.reconcile() # must not raise - - # Whatever the pass counts, it must not count a Job it did not create... - assert result['created'] == len(cluster.jobs()) - # ...nor claim anything about the range. - assert cluster.progress() == {}, "a swallowed 409 must not record anything" - assert cluster.failed() == {} - assert 'pc-r300-a1' not in cluster.jobs() - - run_to_quiescence(cluster) - assert_converged(cluster) - # Each range ran once: no range was ever dispatched at attempt 2. - creates = cluster.calls.names(verb='create', kind='job') - assert sorted(creates) == ['pc-r100-a1', 'pc-r200-a1', 'pc-r300-a1'] - - -def test_409_means_the_slot_is_taken_and_must_not_over_dispatch(cluster, - monkeypatch): - """Losing the create race means the Job EXISTS and is in flight. - - The monitor's own comment calls name uniqueness the dispatch mutex, which is - only true if losing it is treated as "someone else holds this slot". A 409 - that does not spend capacity dispatches PARALLELISM+1 workers -- and at 1024 - parallelism that is a fleet-wide overshoot, not a rounding error. - """ - real_create = cluster.k8s.batch_v1.create_namespaced_job - lost = [] - - def loser(namespace, body, **kwargs): - if body.metadata.name == 'pc-r300-a1' and not lost: - lost.append(body.metadata.name) - real_create(namespace, body, **kwargs) # the other writer's object - raise fake_k8s.api_exception( - 409, 'Conflict', 'jobs.batch "pc-r300-a1" already exists') - return real_create(namespace, body, **kwargs) - - monkeypatch.setattr(cluster.k8s.batch_v1, 'create_namespaced_job', loser) - - result = cluster.reconcile() - - assert lost, "precondition: the create actually lost the race" - assert len(cluster.jobs()) <= config.PARALLELISM, ( - f"dispatched {cluster.jobs()} against PARALLELISM={config.PARALLELISM}: " - "a 409 left the slot looking free") - assert '300/420' in result['in_progress'], \ - "the range whose Job exists is in flight and must be reported as such" - assert result['remaining'] == 1, \ - "a range with a running Job is not still waiting to be dispatched" - - run_to_quiescence(cluster) - assert_converged(cluster) - - -def test_500_on_dispatch_is_retried_on_a_later_pass(cluster): - """A server error is not a verdict: the range must survive it.""" - cluster.k8s.fail_next['create job'] = fake_k8s.api_exception(500, 'boom') - - with pytest.raises(ApiException) as err: - cluster.reconcile() - assert err.value.status == 500 - - assert cluster.jobs() == [], "nothing was created by the aborted pass" - assert cluster.progress() == {} - - restart(cluster) - result = cluster.reconcile() - - assert cluster.jobs() == ['pc-r200-a1', 'pc-r300-a1'] - assert result['created'] == 2 - # The volume the aborted pass provisioned is reused, not duplicated. - assert cluster.calls.names(verb='create', kind='pvc').count('pc-data-r300') == 1 - - run_to_quiescence(cluster) - assert_converged(cluster) - - -def test_500_on_the_retry_create_does_not_lose_or_double_spend_the_range(cluster): - """The retry create fails hard. The range keeps its budget and its history.""" - cluster.reconcile() - cluster.advance(300, 'incomplete') - cluster.finalize(300, 1, archive='fetch_fault') - - cluster.k8s.fail_next['create job'] = fake_k8s.api_exception(500, 'boom') - with pytest.raises(ApiException): - cluster.reconcile() - - assert 'pc-r300-a1' in cluster.jobs(), \ - "deleting the predecessor before the successor exists restarts the range" - assert cluster.failed() == {} - - restart(cluster) - cluster.reconcile() - - assert 'pc-r300-a2' in cluster.jobs() - assert records._cause_count('300', 2, ('fetch-fault',)) == 1 - assert cluster.calls.names(verb='create', kind='pvc').count('pc-data-r300') == 1 - - cluster.advance(300, 'succeeded', attempt=2) - cluster.finalize(300, 2, tx_apply=1.0, peaks={'peakAnonBytes': 1024}) - cluster.reconcile() - assert cluster.completed()['300']['attempts'] == 2 - - -def test_a_range_that_exhausts_its_budget_across_crashes_fails_once(cluster, - monkeypatch): - """Budgets are spent by durable verdicts, so restarts must not stretch or - shrink them. Five attempts, a crash before each retry create. - - An exit-3 fetch fault is an unreachable archive, so it spends the - environmental budget; the cap is lowered here rather than looping to the - configured one. - """ - # A fetch fault spends its own budget, so that is the one to lower. - monkeypatch.setitem(config.ATTEMPT_BUDGETS, 'fetch-fault', 5) - cluster.reconcile() - for attempt in range(1, config.ATTEMPT_BUDGETS['fetch-fault'] + 1): - cluster.advance(300, 'incomplete', attempt=attempt) - cluster.finalize(300, attempt, archive='fetch_fault') - crash_before(monkeypatch, cluster.k8s.batch_v1, 'create_namespaced_job') - with pytest.raises(Crash): - cluster.reconcile() - restart(cluster) - cluster.reconcile() - - assert cluster.failed()['300']['attempts'] == config.ATTEMPT_BUDGETS['fetch-fault'] - assert cluster.failed()['300']['outcome'] == 'fetch-fault' - # Exactly that many Jobs were ever created for the range, despite five - # crashed passes replaying the same failed attempts. - creates = cluster.calls.names(verb='create', kind='job') - assert sorted(n for n in creates if n.startswith('pc-r300-')) == [ - f'pc-r300-a{n}' for n in range(1, config.ATTEMPT_BUDGETS['fetch-fault'] + 1)] - - -# --- end to end -------------------------------------------------------------- - -def test_the_run_converges_with_a_crash_at_every_boundary(cluster, monkeypatch): - """One crash at each arrow, spread across one run, restarting every time.""" - # 1. after the Job create, before the range is tracked - crash_after(monkeypatch, cluster.k8s.batch_v1, 'create_namespaced_job') - with pytest.raises(Crash): - cluster.reconcile() - restart(cluster) - cluster.reconcile() - - # 2. after the record, before save_progress - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1, tx_apply=1.0, peaks={'peakAnonBytes': 1024}) - crash_before(monkeypatch, jm, 'save_progress') - with pytest.raises(Crash): - cluster.reconcile() - restart(cluster) - cluster.reconcile() - - # 3. after save_progress, before release_pvc - cluster.advance(200, 'succeeded') - cluster.finalize(200, 1, tx_apply=1.0, peaks={'peakAnonBytes': 1024}) - crash_before(monkeypatch, jm, 'release_pvc') - with pytest.raises(Crash): - cluster.reconcile() - restart(cluster) - cluster.reconcile() - - # 4. after release_pvc, before the reap - cluster.advance(100, 'succeeded') - cluster.finalize(100, 1, tx_apply=1.0, peaks={'peakAnonBytes': 1024}) - crash_before(monkeypatch, jm, 'reap_range_jobs') - with pytest.raises(Crash): - cluster.reconcile() - restart(cluster) - - run_to_quiescence(cluster) - assert_converged(cluster) - - # Three ranges, three Jobs, ever. Nothing was replayed by a restart. - assert sorted(cluster.calls.names(verb='create', kind='job')) == [ - 'pc-r100-a1', 'pc-r200-a1', 'pc-r300-a1'] - assert sorted(cluster.calls.names(verb='create', kind='pvc')) == [ - 'pc-data-r100', 'pc-data-r200', 'pc-data-r300'] - # ...and every measurement survived the crashes. - for end in ('100', '200', '300'): - assert cluster.completed()[end]['txApply'] == 1.0 - assert cluster.completed()[end]['peakAnonBytes'] == 1024 - - -def test_a_restart_between_every_single_pass_changes_nothing(cluster): - """The control: the same run with a fresh process for every pass.""" - for _ in range(12): - restart(cluster) - finish_live_jobs(cluster) - cluster.reconcile() - if len(cluster.completed()) == TOTAL_RANGES and not cluster.jobs(): - break - - assert_converged(cluster) - assert sorted(cluster.calls.names(verb='create', kind='job')) == [ - 'pc-r100-a1', 'pc-r200-a1', 'pc-r300-a1'] diff --git a/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py b/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py deleted file mode 100644 index 9e0342a6..00000000 --- a/src/MissionParallelCatchup/tests/resilience/test_hostile_state.py +++ /dev/null @@ -1,400 +0,0 @@ -"""Hostile durable state: the monitor must never mistake foreign or corrupt -state for progress. - -Every test here drives the shipped reconcile() against the fake cluster and -asserts on observed state -- the durable record, the call log, the Job set -- -never on source text. - -The volume the monitor resumes from is not private to a run. It is a PVC that -outlives `helm uninstall`, gets reused across missions, and is mirrored into a -ConfigMap that a second writer can clobber. So progress.json can arrive -truncated, rolled back, or written by a run with a completely different -ledgersPerJob. None of those are hypothetical, and none of them may be read as -"work already done". -""" - -import json - -import pytest - -import fake_k8s -import config -import records -import attempts -import job_monitor as jm - - -# A progress record left by a DIFFERENT slicing of the same ledger space: the -# ends are real range ends, just not ends of THIS run's range list. -FOREIGN = {'attempts': 1, 'count': 111, 'seconds': 12.0, 'wallSeconds': 12.0, - 'txApply': 1.0} - - -def seed_progress(cluster, completed=None, failed=None): - cluster.write(config.PROGRESS_FILE, json.dumps( - {'completed': dict(completed or {}), 'failed': dict(failed or {})})) - - -# --- the headline case ------------------------------------------------------- - - -def test_foreign_completed_keys_do_not_shrink_remaining(cluster): - """`remaining` must count THIS run's outstanding ranges, not subtract a - number that a foreign record can inflate. - - Seeded: one completed key ('333') from a run with a different ledgersPerJob. - This run's ranges are 300/200/100 and not one of them has been touched. - Subtraction gives 3 - 1 - 0 - 2 == 0 on the very first pass: the mission's - `num_remain` reads zero while three ranges are outstanding. - """ - seed_progress(cluster, completed={'333': FOREIGN}) - - result = cluster.reconcile() - - # Two of the three went out; range 100 is queued behind PARALLELISM. - assert sorted(result['in_progress']) == ['200/420', '300/420'] - # ...so exactly one range of this run is still waiting to be dispatched. - assert result['remaining'] == 1 - - # And the foreign key really is being carried in the record -- the test - # above is not passing because something quietly dropped it. - assert '333' in cluster.completed() - assert set(cluster.completed()) & {'100', '200', '300'} == set() - - -def test_foreign_completed_keys_do_not_drive_remaining_negative(cluster): - """The mirror image, and the one that hangs a real run. - - The mission finishes on `num_remain == 0 && queue_in_progress_count == 0` - (MissionHistoryPubnetParallelCatchupV2.fs). With three foreign keys in the - record, subtraction lands on -3 once every real range has actually - completed -- never 0 -- so the driver waits forever on a run that is done. - """ - seed_progress(cluster, completed={'111': FOREIGN, '222': FOREIGN, - '333': FOREIGN}) - - result = cluster.reconcile() - for end in (300, 200): - cluster.advance(end, 'succeeded') - cluster.finalize(end, 1) - result = cluster.reconcile() - cluster.advance(100, 'succeeded') - cluster.finalize(100, 1) - result = cluster.reconcile() - - # Every range of this run really did run and really is recorded. - assert {'100', '200', '300'} <= set(cluster.completed()) - assert result['in_progress'] == [] - # The terminating condition the mission actually tests. - assert result['remaining'] == 0 - - -def test_foreign_failed_keys_do_not_shrink_remaining(cluster): - """Same subtraction, other bucket. `failed` is foreign-writable too.""" - seed_progress(cluster, failed={'111': {'attempts': 1, 'outcome': 'failed', - 'exitCode': 1, 'pod': 'gone'}, - '222': {'attempts': 1, 'outcome': 'failed', - 'exitCode': 1, 'pod': 'gone'}}) - - result = cluster.reconcile() - - assert sorted(result['in_progress']) == ['200/420', '300/420'] - assert result['remaining'] == 1 - - -def test_a_range_end_shared_with_the_foreign_slicing_is_still_skipped(cluster): - """Honest about the limit of the fix. - - `remaining` becomes a count over THIS run's range list, so it is immune to - keys that do not name one of our ranges. It cannot save us from a foreign - key that happens to collide with one of them -- '300' is an end under - ledgersPerJob=150 as well as under 100 -- because at that point the record - is indistinguishable from a legitimate resume. The count stays consistent - with what dispatch does, which is the property that matters: no phantom - zero, no phantom negative. - """ - seed_progress(cluster, completed={'300': FOREIGN, '333': FOREIGN}) - - result = cluster.reconcile() - - # 300 is treated as done (a resume, as far as anything here can tell)... - assert 'pc-r300-a1' not in cluster.jobs() - assert sorted(result['in_progress']) == ['100/420', '200/420'] - # ...and remaining agrees with that: nothing left unaccounted for. - assert result['remaining'] == 0 - - -# --- corruption -------------------------------------------------------------- - - -def test_an_unreadable_progress_json_replays_rather_than_halting(cluster): - """An unreadable record reads as "nothing has been done". - - There is no monotonic-progress guard -- its high-water mark lived in memory - and a restart erased it. Replay is safe: the PVCs survive, so each range - resumes at its last closed ledger. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1) - cluster.reconcile() - assert '300' in cluster.completed() - - cluster.write(config.PROGRESS_FILE, 'not json at all') - - result = cluster.reconcile() - - # The record is empty, so the range is eligible again -- and the pass does - # not crash, which is the property that actually matters here. - assert cluster.state['halted'] is False - assert cluster.completed() == {} - assert result['remaining'] + len(result['in_progress']) == 3 - - -def test_progress_rolled_back_to_an_older_version_makes_it_eligible_again(cluster): - """A stale writer wins the volume: completed goes 2 -> 1. - - The ConfigMap-mirror-loses-a-race shape. The monitor cannot distinguish it - from deletion and no longer tries: the range simply becomes eligible again. - Redoing it costs a resumed attempt, not the work. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1) - cluster.reconcile() - older = json.dumps(cluster.progress()) # snapshot at completed == 1 - - cluster.advance(200, 'succeeded') - cluster.finalize(200, 1) - cluster.reconcile() - assert set(cluster.completed()) == {'200', '300'} - - # The stale copy lands back on the volume. - cluster.write(config.PROGRESS_FILE, older) - before = set(cluster.jobs()) - created_before = cluster.calls.names(verb='create', kind='job') - - result = cluster.reconcile() - - # The rolled-back range is eligible again rather than the run stopping. - assert set(cluster.completed()) == {'300'} - assert result['remaining'] + len(result['in_progress']) + result['completed'] == 3 - # 200's Job was reaped when it completed, so re-dispatch is a fresh attempt - # against its surviving PVC -- it resumes, it does not replay from genesis. - assert 'pc-data-r200' in cluster.pvcs() - - -# --- the collector's markers ------------------------------------------------- - - -def test_metrics_without_done_must_not_reap(cluster): - """.done is written last. Without it the collector may still be reading the - pod's log, and deleting the Job reaps the pod out from under it.""" - cluster.reconcile() - cluster.advance(300, 'succeeded') - # .metrics only -- exactly the window between the collector's two writes. - cluster.write(records.metrics_path('300', 1), - json.dumps({'txApplySeconds': 2.5, 'peakAnonBytes': 999})) - - cluster.reconcile() - - assert cluster.deleted.names(verb='delete', kind='job') == [] - assert 'pc-r300-a1' in cluster.jobs() - # The range is recorded and its measurements were read -- the reap is the - # only thing being withheld. - assert cluster.completed()['300']['txApply'] == 2.5 - assert cluster.completed()['300']['peakAnonBytes'] == 999 - - # Withheld, not leaked: the Job carries a TTL, so declining to reap costs a - # late reclaim rather than an object that lives until `helm uninstall`. - assert (cluster.k8s.job('pc-r300-a1').spec.ttl_seconds_after_finished - == config.JOB_TTL_SECONDS) - - # And the withheld reap does not turn into a re-dispatch on later passes. - cluster.reconcile() - assert cluster.deleted.names(verb='delete', kind='job') == [] - assert 'pc-r300-a2' not in cluster.jobs() - assert cluster.completed()['300']['attempts'] == 1 - - -def test_the_reap_lands_once_the_done_marker_arrives(cluster): - """The other side of the same gate: while the record is still incomplete, - reconcile keeps coming back, and the pass that sees .done reaps.""" - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.reconcile() # recorded with nothing measured - - assert cluster.completed()['300']['txApply'] is None - assert cluster.deleted.names(verb='delete', kind='job') == [] - - # The collector finally finishes this attempt. - cluster.finalize(300, 1, tx_apply=2.5, peaks={'peakAnonBytes': 999}) - cluster.reconcile() - - # Backfilled from the durable files, then reaped. - assert cluster.completed()['300']['txApply'] == 2.5 - assert cluster.completed()['300']['peakAnonBytes'] == 999 - assert cluster.deleted.names(verb='delete', kind='job') == ['pc-r300-a1'] - - -def test_done_without_metrics_reaps_but_does_not_invent_measurements(cluster): - """The other half-write: .done present, .metrics never landed. - - .done is the authority on "nothing more is coming", so the reap is correct - and must happen -- a range whose collector died would otherwise pin its Job - forever. What must NOT happen is a fabricated or crashed record. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.write(records.done_path('300', 1), '') - - cluster.reconcile() - - record = cluster.completed()['300'] - assert record['attempts'] == 1 - assert record['count'] == 420 - # No .metrics and no history archive to fall back on: the gap is reported - # as a gap, not as zero. - assert record['txApply'] is None - assert not any(record.get(k) is not None for k in attempts.PEAK_FIELDS) - # Timing comes from the pod, which is real. - assert record['seconds'] == pytest.approx(60.0) - - assert cluster.deleted.names(verb='delete', kind='job') == ['pc-r300-a1'] - # Recorded once and never re-dispatched, even though the record is thin. - assert cluster.reconcile()['created'] == 0 - assert 'pc-r300-a1' not in cluster.jobs() - assert 'pc-r300-a2' not in cluster.jobs() - - -def test_an_empty_metrics_file_is_not_read_as_zero(cluster): - """A zero-length .metrics is a torn write, not a measurement of nothing.""" - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.write(records.metrics_path('300', 1), '') - cluster.write(records.done_path('300', 1), '') - - cluster.reconcile() - - record = cluster.completed()['300'] - assert record['txApply'] is None - assert not any(record.get(k) is not None for k in attempts.PEAK_FIELDS) - - -# --- two monitors ------------------------------------------------------------ - - -def test_two_monitors_racing_the_same_volume_never_double_dispatch(cluster): - """Job name uniqueness is the intended mutex. Prove it actually holds. - - The realistic race is not "B runs after A" -- B would simply see A's Jobs - in its LIST and skip them. It is both monitors LISTING before either - CREATES. That is reproduced here by handing the second reconcile the job - list as it was before the first pass ran, while its writes go to the one - real cluster. - """ - stale_jobs = cluster.k8s.batch_v1.list_namespaced_job( - cluster.namespace, label_selector=f"{config.LABEL_RUN}={config.RUN_NAME}") - assert stale_jobs.items == [] - - a = cluster.reconcile() - assert a['created'] == 2 - - real_list = cluster.k8s.batch_v1.list_namespaced_job - calls = {'n': 0} - - def list_from_before_the_race(namespace, **kw): - calls['n'] += 1 - if calls['n'] == 1: - return stale_jobs # B's snapshot: taken before A created - return real_list(namespace, **kw) - - cluster.k8s.batch_v1.list_namespaced_job = list_from_before_the_race - try: - # A second monitor process: its own state dict, sharing nothing but the - # cluster and the volume. - b_state = {'owner': jm.owner_ref(), 'replayed': set(), - 'max_completed': 0, 'halted': False, 'counted': {}} - jm.reconcile(b_state) - finally: - cluster.k8s.batch_v1.list_namespaced_job = real_list - - created = cluster.calls.names(verb='create', kind='job') - # B really did re-attempt the two ranges A had just taken -- otherwise this - # test proves nothing about the mutex. - assert created.count('pc-r300-a1') == 2 - assert created.count('pc-r200-a1') == 2 - - # The mutex: the duplicate creates were rejected, so each range has exactly - # ONE Job object and exactly one pod. Nothing ran twice. - for end in (300, 200): - name = f'pc-r{end}-a1' - assert cluster.jobs().count(name) == 1 - pods = [p for (_, _), p in cluster.k8s.pods.items() - if (p.metadata.labels or {}).get('job-name') == name] - assert len(pods) == 1, f"{name} spawned {len(pods)} pods" - # No range was escalated to a second attempt by the losing writer, and the - # shared volume was not double-provisioned either. - assert not any(n.endswith('-a2') for n in cluster.jobs()) - for end in (300, 200): - assert cluster.calls.names(verb='create', kind='pvc').count( - f'pc-data-r{end}') == 1 - - # Neither process crashed on the 409s, and B recorded nothing. - assert cluster.progress() == {} - - -def test_a_second_monitor_does_not_re_dispatch_recorded_ranges(cluster): - """A restart mid-run -- the same thing from the durable side. - - A fresh state dict has max_completed 0 and an empty replay set. Reading the - volume back must reproduce the run exactly: no redispatch of a recorded - range, no false regression halt from the counter starting at zero. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1) - cluster.reconcile() - assert '300' in cluster.completed() - created_before = list(cluster.calls.names(verb='create', kind='job')) - - fresh = {'owner': jm.owner_ref(), 'replayed': set(), 'counted': {}} - result = jm.reconcile(fresh) - - assert cluster.calls.names(verb='create', kind='job').count('pc-r300-a1') == 1 - assert 'pc-r300-a2' not in cluster.jobs() - # The restart picks the record up rather than starting from zero. - assert result['completed'] == 1 - assert result['remaining'] + len(result['in_progress']) + result['completed'] == 3 - # Only ranges that were genuinely unstarted moved. - assert set(cluster.calls.names(verb='create', kind='job')) - set(created_before) <= { - 'pc-r100-a1'} - - -# --- other people's objects -------------------------------------------------- - - -def test_a_foreign_run_s_jobs_in_the_namespace_are_ignored(cluster): - """The namespace is shared. Another run's Jobs carry another RUN_NAME and - must not be read as this run's ranges.""" - other = cluster.k8s.batch_v1.create_namespaced_job( - cluster.namespace, - jm.build_job(300, 420, 1, None)) - other.metadata.name = 'other-r300-a1' - other.metadata.labels = dict(other.metadata.labels or {}) - other.metadata.labels[config.LABEL_RUN] = 'other-run' - cluster.k8s.jobs[(cluster.namespace, 'other-r300-a1')] = other - del cluster.k8s.jobs[(cluster.namespace, 'pc-r300-a1')] - - result = cluster.reconcile() - - # Our own range 300 was dispatched despite the foreign Job for the same - # ledger range already existing. - assert 'pc-r300-a1' in cluster.jobs() - assert sorted(result['in_progress']) == ['200/420', '300/420'] - assert result['remaining'] == 1 - assert result['total'] == 3 - - - - diff --git a/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py b/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py deleted file mode 100644 index eb1d49b0..00000000 --- a/src/MissionParallelCatchup/tests/resilience/test_restart_fuzz.py +++ /dev/null @@ -1,502 +0,0 @@ -"""Restart invisibility: a monitor restart between any two reconcile passes -must change nothing an observer can see. - -The monitor is a reconciler. Every decision it makes has to be derivable from -the Kubernetes objects plus the durable files on the logs volume; anything it -keeps only in RAM is lost the moment the pod is rescheduled, and a 10-hour run -gets rescheduled. `restart()` below is the whole trick: it discards exactly -what a process death discards -- the `state` dict reconcile() carries across -passes, and the module-level owner cache -- and keeps exactly what survives, -the logs volume and the cluster. - -Everything here asserts on observed state: the durable progress record, the -live Job/Pod objects, and the API call log. Nothing reads job_monitor's source. -""" - -import json -import os -import random - -import pytest - -import config -import units -import ranges -import records -import attempts -import job_monitor as jm - -# The states the fuzz drives Jobs through. A real run is dominated by success, -# with spot evictions the most common failure, then OOM, then a hung archive -# fetch tripping the attempt deadline. `unknown` is the restart's own signature -# -- the Job failed while the monitor was down and the pod was reaped with it, -# so nothing is left to classify from. -DRIVE_STATES = ('succeeded', 'disrupted', 'oom', 'oom', 'unknown') -DRIVE_WEIGHTS = (6, 3, 2, 2, 1) - -# 30 seeds x 24 passes runs in ~9s. RESTART_FUZZ_SEEDS / RESTART_FUZZ_PASSES -# widen it for a soak without editing the file -- 400 x 40 takes ~2.5 minutes. -PASSES = int(os.getenv('RESTART_FUZZ_PASSES', 24)) -SEEDS = list(range(int(os.getenv('RESTART_FUZZ_SEEDS', 30)))) - - -# --- the restart ------------------------------------------------------------ - -def restart(cluster): - """Simulate the monitor process dying and being rescheduled. - - Gone: the in-memory `state` dict (owner reference, histogram replay guard, - the monotonic-progress high-water mark, the counter deltas) and the - module-level owner cache. Kept: the logs volume and every object in the - cluster -- which between them are the only inputs a reconciler is allowed - to have. - """ - cluster.state = {'owner': None, 'replayed': set(), 'max_completed': 0, - 'halted': False, 'counted': {}} - jm._progress_owner.clear() - config.PROFILE = None - - -# --- cluster inspection ----------------------------------------------------- - -def _terminal(job): - st = job.status - return bool(st and (st.succeeded or st.failed)) - - -def _jobs_by_range(cluster): - """range-end (str) -> [(attempt, job)], from the cluster, not from state.""" - out = {} - for name in cluster.jobs(): - job = cluster.k8s.job(name) - labels = job.metadata.labels or {} - end = labels.get(config.LABEL_RANGE) - attempt = int(labels.get(config.LABEL_ATTEMPT, 1)) - out.setdefault(end, []).append((attempt, job)) - return out - - -def _range_of_job(name): - """'pc-r1200-a3' -> ('1200', 3).""" - stem, _, attempt = name.rpartition('-a') - return stem.split('-r', 1)[1], int(attempt) - - -# --- the invariants --------------------------------------------------------- - -class Ledger: - """Cross-pass bookkeeping the invariants need (high-water marks, first - sighting of a completion, every pod ever seen).""" - - def __init__(self, ends): - self.ends = set(ends) - self.total = len(ends) - self.dispatched = set() # every range that has ever had a Job - self.recorded_at = {} # end -> len(calls) when first completed - self.peaks = {} # end -> {field: high-water value} - self.pods = {} # (end, attempt) -> pod name - - -def check(cluster, result, led, where): - """Assert I1..I6 against observed state. `where` names the pass.""" - progress = cluster.progress() - completed = set(progress.get('completed', {})) - failed = set(progress.get('failed', {})) - by_range = _jobs_by_range(cluster) - - for end, entries in by_range.items(): - led.dispatched.add(end) - for name in cluster.calls.names(verb='create', kind='job'): - led.dispatched.add(_range_of_job(name)[0]) - - # A Job that has succeeded or failed is a record, not work in flight. The - # monitor deliberately leaves a finished Job standing until the collector - # finalizes it, so "live" has to mean unfinished, not merely present. - live = {end for end, entries in by_range.items() - if any(not _terminal(j) for _, j in entries)} - - # -- I1: exactly one of completed / failed / live ------------------------ - assert completed <= led.ends, f"{where}: completed has unknown ranges {completed - led.ends}" - assert failed <= led.ends, f"{where}: failed has unknown ranges {failed - led.ends}" - assert live <= led.ends, f"{where}: live has unknown ranges {live - led.ends}" - assert not (completed & failed), \ - f"{where}: ranges both completed and failed: {sorted(completed & failed)}" - assert not (completed & live), \ - f"{where}: completed ranges with work still in flight: {sorted(completed & live)}" - assert not (failed & live), \ - f"{where}: failed ranges with work still in flight: {sorted(failed & live)}" - # Never zero: a range that has been dispatched must stay accounted for. - # Undispatched ranges are simply queued behind PARALLELISM -- that is the - # fourth, legitimate bucket, and it only ever shrinks. - lost = led.dispatched - completed - failed - live - assert not lost, (f"{where}: dispatched ranges accounted for nowhere -- " - f"no record and no live Job: {sorted(lost)}") - - # -- I2: at most one live Job per range ---------------------------------- - for end, entries in by_range.items(): - unfinished = [a for a, j in entries if not _terminal(j)] - assert len(unfinished) <= 1, \ - f"{where}: range {end} has {len(unfinished)} live Jobs (attempts {unfinished})" - - # ...and the run never runs wider than it was told to. A restart that - # forgot what was in flight would show up here first. - assert len(result['in_progress']) <= config.PARALLELISM, \ - f"{where}: {len(result['in_progress'])} in flight over PARALLELISM {config.PARALLELISM}" - - # A completed range has nothing left to resume, so its volume is gone -- - # 79 TiB of orphaned gp3 is what this costs when it regresses. - held = {end for end in completed - if f"{cluster.run_name}-data-r{end}" in cluster.pvcs()} - assert not held, f"{where}: completed ranges still holding a PVC: {sorted(held)}" - - # -- I3: a completed range is never re-dispatched ------------------------ - for end in completed: - led.recorded_at.setdefault(end, len(cluster.calls)) - for index, call in enumerate(cluster.calls): - if call.verb != 'create' or call.kind != 'job': - continue - end, attempt = _range_of_job(call.name) - mark = led.recorded_at.get(end) - if mark is not None and index >= mark: - raise AssertionError( - f"{where}: range {end} was re-dispatched ({call.name}) after it " - f"was recorded complete") - - # -- I4: remaining is sane ----------------------------------------------- - assert result['remaining'] >= 0, f"{where}: remaining went negative: {result}" - drained = result['remaining'] == 0 and not result['in_progress'] - assert drained == (len(completed) + len(failed) == led.total), ( - f"{where}: remaining/in_progress say drained={drained} but the record " - f"has {len(completed)} completed + {len(failed)} failed of {led.total}") - - # -- I5: recorded peaks are a high-water mark ---------------------------- - for end, record in (progress.get('completed') or {}).items(): - seen = led.peaks.setdefault(end, {}) - for field in attempts.PEAK_FIELDS: - value = record.get(field) - if value is None: - continue - previous = seen.get(field) - assert previous is None or value >= previous, ( - f"{where}: range {end} peak {field} went backwards " - f"{previous} -> {value}") - seen[field] = value - - # -- I6: one pod per (range, attempt) ------------------------------------ - for (_, name), pod in cluster.k8s.pods.items(): - labels = pod.metadata.labels or {} - end = labels.get(config.LABEL_RANGE) - if end is None: - continue - key = (end, labels.get(config.LABEL_ATTEMPT)) - previous = led.pods.setdefault(key, name) - assert previous == name, ( - f"{where}: range {key[0]} attempt {key[1]} has two distinct pods " - f"({previous} and {name}) -- the attempt was replayed") - - -# --- driving the cluster ---------------------------------------------------- - -def collector_catches_up(cluster, rng): - """Write what the log-collector sidecar writes, for some finished attempts. - - Not all of them: the monitor's reap is gated on the .done marker, so - leaving attempts unfinalized is what keeps finished Jobs standing and - exercises the backfill path. - """ - for end, entries in _jobs_by_range(cluster).items(): - for attempt, job in entries: - if not _terminal(job) or os.path.exists(records.done_path(end, attempt)): - continue - if rng.random() < 0.35: - continue - cluster.finalize( - end, attempt, - tx_apply=round(rng.uniform(0.0, 5.0), 4), - peaks={'peakAnonBytes': rng.randrange(1, 20) * 10 ** 8, - 'peakAnonBytes': rng.randrange(1, 20) * 10 ** 8}, - resumed=(attempt > 1 and rng.random() < 0.5), - attempt_seconds=round(rng.uniform(10.0, 300.0), 2)) - - -def cluster_moves(cluster, rng): - """Drive live Jobs to terminal states, the way the cluster would.""" - for end, entries in _jobs_by_range(cluster).items(): - for attempt, job in entries: - if _terminal(job) or rng.random() < 0.45: - continue - state = rng.choices(DRIVE_STATES, weights=DRIVE_WEIGHTS)[0] - cluster.advance(int(end), state, attempt=attempt) - - -@pytest.fixture -def big_run(cluster, monkeypatch): - """Twelve ranges, four at a time -- enough queueing that a dropped range - would be silently re-dispatched rather than obviously stuck.""" - monkeypatch.setattr(config, 'LATEST_LEDGER_NUM', 1200) - monkeypatch.setattr(config, 'PARALLELISM', 4) - return cluster - - -# --- the fuzz --------------------------------------------------------------- - -def _observable(cluster): - """Everything a restart is allowed to leave untouched.""" - return (cluster.progress(), cluster.jobs(), cluster.pvcs()) - - -@pytest.mark.parametrize('seed', SEEDS) -def test_restart_is_invisible_under_fuzz(big_run, seed): - cluster = big_run - rng = random.Random(seed) - ends = [str(end) for end, _ in ranges.generate_ranges()] - assert len(ends) == 12 - led = Ledger(ends) - - # One guaranteed restart while the run is still busy, plus a scattering of - # others -- a reconciler should survive any number of them, anywhere. - restarts = {rng.randrange(1, 12)} - restarts |= {i for i in range(1, PASSES) if rng.random() < 0.12} - - for i in range(PASSES): - if i in restarts: - restart(cluster) - result = cluster.reconcile() - where = f"seed={seed} pass={i}{' (post-restart)' if i in restarts else ''}" - check(cluster, result, led, where) - # The restart must not trip the anti-tamper halt: max_completed comes - # back as 0 and climbs again from the record on disk. - assert cluster.state['halted'] is False, \ - f"{where}: dispatch halted -- progress read as going backwards" - - if i in restarts: - # The lens at its sharpest: with nothing changing in the cluster, - # restarting and reconciling again must be a no-op. Anything that - # moves here was being decided from memory. - before = _observable(cluster) - restart(cluster) - shadow = cluster.reconcile() - check(cluster, shadow, led, f"{where} (shadow)") - assert _observable(cluster) == before, ( - f"{where}: a restart + reconcile with an unchanged cluster " - f"moved something") - assert shadow['created'] == 0, \ - f"{where}: shadow pass dispatched {shadow['created']} Job(s)" - # ...and it reports the same run, not just leaves the same objects. - assert (shadow['completed'], shadow['remaining'], - sorted(shadow['in_progress']), sorted(shadow['failed_ranges'])) == \ - (result['completed'], result['remaining'], - sorted(result['in_progress']), sorted(result['failed_ranges'])), \ - f"{where}: the post-restart summary disagrees: {result} -> {shadow}" - - collector_catches_up(cluster, rng) - cluster_moves(cluster, rng) - - assert restarts - # The run has to have gone somewhere, or the fuzz proved nothing. - progress = cluster.progress() - assert progress.get('completed'), f"seed={seed}: no range ever completed" - # Retries have to have actually happened, or the fuzz only exercised the - # happy path. - assert any(name.endswith('.verdict') for name in os.listdir(config.LOG_DIR)), \ - f"seed={seed}: no attempt ever failed" - - -# --- focused restarts, to localise anything the fuzz turns up ---------------- - -def test_restart_does_not_redispatch_a_recorded_range(big_run): - cluster = big_run - cluster.reconcile() - for end in ('1200', '1100', '1000', '900'): - cluster.advance(int(end), 'succeeded') - cluster.finalize(end, 1, tx_apply=1.0, peaks={'peakAnonBytes': 5}) - cluster.reconcile() - recorded = set(cluster.completed()) - assert recorded == {'1200', '1100', '1000', '900'} - mark = len(cluster.calls) - - restart(cluster) - cluster.reconcile() - - assert set(cluster.completed()) >= recorded - after = [_range_of_job(c.name)[0] for c in cluster.calls[mark:] - if c.verb == 'create' and c.kind == 'job'] - assert not (set(after) & recorded), \ - f"recorded ranges re-dispatched after restart: {sorted(set(after) & recorded)}" - - -def test_restart_mid_retry_keeps_the_attempt_number(big_run): - cluster = big_run - cluster.reconcile() - cluster.advance(1200, 'oom') - cluster.reconcile() - assert cluster.attempt_of(1200) == 2 - limit = (cluster.k8s.job('pc-r1200-a2') - .spec.template.spec.containers[0].resources.requests['memory']) - - restart(cluster) - cluster.advance(1200, 'oom', attempt=2) - cluster.reconcile() - - # The escalation ladder is counted off the .outcome files on the volume, - # so the restart must not reset it to the first rung. - assert cluster.attempt_of(1200) == 3 - escalated = (cluster.k8s.job('pc-r1200-a3') - .spec.template.spec.containers[0].resources.requests['memory']) - assert units.quantity_bytes(escalated) > units.quantity_bytes(limit) - assert cluster.failed() == {} - - -def test_restart_does_not_reset_a_spent_budget(big_run): - """A budget already spent must not come back after a monitor restart. - - Budgets are tallied from the .verdict files on the logs volume rather than - from memory, precisely so this holds. Written against the timeout budget - when that was 2 and retryable; a deadline hit is terminal now, so this uses - OOM -- the invariant is the same. - """ - cluster = big_run - cluster.reconcile() - cluster.advance(1200, 'oom') - cluster.finalize('1200', 1) - cluster.reconcile() - spent_before = cluster.attempt_of(1200) - assert spent_before > 1, "the first OOM did not produce a retry" - - restart(cluster) - cluster.reconcile() - - # The restart must not hand the range a clean slate. - assert cluster.attempt_of(1200) == spent_before, ( - "a restart reset the attempt count, so the budget starts over") - assert 'pc-r1200-a1' not in cluster.jobs(), "the spent attempt was re-created" - - -def test_restart_does_not_halt_on_its_own_progress(big_run): - cluster = big_run - cluster.reconcile() - cluster.advance(1200, 'succeeded') - cluster.finalize('1200', 1) - cluster.reconcile() - assert '1200' in cluster.completed() - - restart(cluster) - result = cluster.reconcile() - - assert '1200' in cluster.completed() - # Dispatch is not frozen: the slot the completion freed was already refilled - # before the restart, so the run comes back at full width. - assert len(result['in_progress']) == 4 - - # ...and the next completion still pulls a new range in. - cluster.advance(1100, 'succeeded') - cluster.finalize('1100', 1) - assert cluster.reconcile()['created'] == 1 - - -# --- two gaps the fuzz does not reach --------------------------------------- -# The monotonic-progress guard that used to be pinned here is gone: it kept its -# high-water mark in memory, so a restart disarmed it for exactly the event it -# existed to survive, and re-running a range is idempotent anyway. - -def test_losing_progress_json_costs_a_replay_not_the_measurement(big_run): - """I5 holds because the measurements live in .metrics, not in the record. - - Losing progress.json used to be papered over by the ConfigMap mirror, which - returned state without data and then persisted that hole over the volume. - Now the record is simply absent and the range becomes eligible again -- - which I5 permits, since it forbids a peak going BACKWARDS, not a record - going away. .metrics is written per attempt and never rewritten, so - re-completing the range restores the same peak rather than a lower one. - """ - cluster = big_run - cluster.reconcile() - cluster.advance(1200, 'succeeded') - cluster.finalize('1200', 1, tx_apply=2.5, peaks={'peakAnonBytes': 12345}) - cluster.reconcile() - assert cluster.completed()['1200']['peakAnonBytes'] == 12345 - - os.remove(config.PROGRESS_FILE) - restart(cluster) - cluster.reconcile() - # Not recovered from it: the range is eligible again rather than carrying a - # state-only entry that the next save would persist over the volume. - assert '1200' not in cluster.completed() - # And the artifacts outlived the record, so a replay can still measure it. - assert attempts.peaks_for_range('1200', 1).get('peakAnonBytes') == 12345 - - -# --- the checker has teeth -------------------------------------------------- -# A fuzz run that passes is only worth what its assertions would have caught. -# Each of these breaks one invariant deliberately and requires check() to say -# so; if one of them ever stops failing, the corresponding invariant above has -# gone vacuous. - -def test_checker_catches_progress_held_in_memory(big_run, monkeypatch): - """A monitor that kept `completed` in RAM instead of on the volume. - - Up to the restart it behaves identically -- which is exactly why this has - to be caught by the restart and not by anything before it. - """ - cluster = big_run - cache = {} - monkeypatch.setattr(jm, 'load_progress', lambda: cache) - monkeypatch.setattr(jm, 'save_progress', lambda progress: cache.update(progress)) - monkeypatch.setattr(cluster, 'progress', lambda: cache) - - led = Ledger([str(e) for e, _ in ranges.generate_ranges()]) - rng = random.Random(1) - with pytest.raises(AssertionError, match='accounted for nowhere|re-dispatched'): - for i in range(12): - if i == 5: - cache.clear() # the process died; RAM went with it - restart(cluster) - result = cluster.reconcile() - check(cluster, result, led, f"mutant pass={i}") - collector_catches_up(cluster, rng) - cluster_moves(cluster, rng) - - -def test_checker_catches_two_live_jobs_for_one_range(big_run): - cluster = big_run - led = Ledger([str(e) for e, _ in ranges.generate_ranges()]) - result = cluster.reconcile() - check(cluster, result, led, 'mutant pre') - - cluster.k8s.batch_v1.create_namespaced_job( - cluster.namespace, jm.build_job(1200, 420, 2, cluster.state['owner'])) - - with pytest.raises(AssertionError, match='live Jobs'): - check(cluster, result, led, 'mutant post') - - -def test_checker_catches_a_peak_going_backwards(big_run): - cluster = big_run - led = Ledger([str(e) for e, _ in ranges.generate_ranges()]) - cluster.reconcile() - cluster.advance(1200, 'succeeded') - cluster.finalize('1200', 1, peaks={'peakAnonBytes': 900}) - result = cluster.reconcile() - check(cluster, result, led, 'mutant pre') - - record = cluster.progress() - record['completed']['1200']['peakAnonBytes'] = 5 - cluster.write(config.PROGRESS_FILE, json.dumps(record)) - - with pytest.raises(AssertionError, match='went backwards'): - check(cluster, result, led, 'mutant post') - - -def test_checker_catches_a_replayed_attempt(big_run): - cluster = big_run - led = Ledger([str(e) for e, _ in ranges.generate_ranges()]) - result = cluster.reconcile() - check(cluster, result, led, 'mutant pre') - # The range's only Job is destroyed with no record of the range, so the - # next pass has to re-create attempt 1 -- a second pod wearing attempt 1. - cluster.k8s.batch_v1.delete_namespaced_job('pc-r1200-a1', cluster.namespace) - - result = cluster.reconcile() - - with pytest.raises(AssertionError, match='two distinct pods'): - check(cluster, result, led, 'mutant post') diff --git a/src/MissionParallelCatchup/tests/test_harness_smoke.py b/src/MissionParallelCatchup/tests/test_harness_smoke.py deleted file mode 100644 index c1dce20b..00000000 --- a/src/MissionParallelCatchup/tests/test_harness_smoke.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Proof that the fake cluster drives the real reconcile(). - -Every test here imports job_monitor and calls the shipped reconcile() -- nothing -is extracted from source or reimplemented. If one of these fails, the monitor's -behaviour changed, not a regex. -""" - -import pytest - -import fake_k8s -import config -import records -import job_monitor as jm - - -def test_dispatch_happens_on_an_empty_cluster(cluster): - result = cluster.reconcile() - - # PARALLELISM is 2 and there are three ranges, so exactly two go out, and - # tip-first means the two highest ends. - assert cluster.jobs() == ['pc-r200-a1', 'pc-r300-a1'] - assert result['created'] == 2 - assert result['total'] == 3 - assert result['remaining'] == 1 - assert sorted(result['in_progress']) == ['200/420', '300/420'] - - # pvc mode: each range gets its own volume, created before its Job. - assert cluster.pvcs() == ['pc-data-r200', 'pc-data-r300'] - created = [(c.kind, c.name) for c in cluster.calls if c.verb == 'create'] - assert created == [('pvc', 'pc-data-r300'), ('job', 'pc-r300-a1'), - ('pvc', 'pc-data-r200'), ('job', 'pc-r200-a1')] - - # Nothing durable is written until a range actually finishes -- dispatch - # alone must not touch the progress record or its ConfigMap mirror. - assert cluster.progress() == {} - assert cluster.calls.names(verb='patch', kind='configmap') == [] - - -def test_a_succeeded_job_is_recorded_into_completed(cluster): - cluster.reconcile() - cluster.advance(300, 'succeeded') - # The collector's half of the contract: peaks and tx_apply are only ever - # readable from the files it writes, and the .done marker is what allows a - # reap at all. - cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakAnonBytes': 123}) - - cluster.reconcile() - - record = cluster.completed()['300'] - assert record['attempts'] == 1 - assert record['count'] == 420 - assert record['txApply'] == 1.5 - assert record['peakAnonBytes'] == 123 - assert record['seconds'] == pytest.approx(60.0) - assert record['wallSeconds'] == pytest.approx(60.0) - - # A completed range gives its volume back and its Job is reaped. - assert 'pc-data-r300' not in cluster.pvcs() - assert cluster.deleted.names(verb='delete', kind='job') == ['pc-r300-a1'] - assert cluster.deleted.names(verb='delete', kind='pvc') == ['pc-data-r300'] - - # The freed slot is refilled in the same pass. - assert 'pc-r100-a1' in cluster.jobs() - - -def test_a_failed_job_is_retried(cluster): - cluster.reconcile() - # exit 3 is stellar-core's "did not complete" and is retried only when the - # archive shows a fetch fault killed it -- so the decision waits for .done. - cluster.advance(300, 'incomplete') - cluster.finalize(300, 1, archive='fetch_fault') - - cluster.reconcile() - - assert 'pc-r300-a2' in cluster.jobs() - assert cluster.attempt_of(300) == 2 - assert cluster.failed() == {}, "a retryable failure must not be recorded as failed" - assert cluster.completed() == {} - - # The retry rides the same volume -- that is what makes resume-at-LCL work. - assert cluster.calls.names(verb='create', kind='pvc').count('pc-data-r300') == 1 - # ...and the new pod carries the attempt label the collector keys files on. - pod = cluster.k8s.pod_for_job('pc-r300-a2') - assert pod.metadata.labels[config.LABEL_ATTEMPT] == '2' - - -def test_a_condemned_range_is_recorded_and_not_retried(cluster): - cluster.reconcile() - # A plain non-zero exit that is not 3 is a genuine catchup failure. - cluster.advance(300, 'condemned') - - cluster.reconcile() - - assert 'pc-r300-a2' not in cluster.jobs() - assert cluster.failed()['300'] == { - 'attempts': 1, 'pod': cluster.k8s.pod_for_job('pc-r300-a1').metadata.name, - 'outcome': 'failed', 'exitCode': 1} - # Dispatch is not frozen by a condemned range: the freed slot is refilled, - # otherwise the mission's `remaining == 0` wait would deadlock. - assert 'pc-r100-a1' in cluster.jobs() - - -def test_an_oom_retry_escalates_the_memory_limit(cluster): - cluster.reconcile() - cluster.advance(300, 'oom') - - cluster.reconcile() - - resources = (cluster.k8s.job('pc-r300-a2') - .spec.template.spec.containers[0].resources) - # One OOM = one rung: 24000Mi * 1.5. The request follows the limit, because - # a pod that OOMed will not fit where it was scheduled before. - assert resources.requests['memory'] == '13824Mi' - assert resources.requests['memory'] == '13824Mi' - assert cluster.failed() == {} - - -def test_a_disruption_does_not_spend_the_range_budget(cluster): - cluster.reconcile() - cluster.advance(300, 'disrupted') - - cluster.reconcile() - - assert 'pc-r300-a2' in cluster.jobs() - outcome = records.read_outcome('300', 1) - assert outcome['outcome'] == 'disrupted' - # Memory is untouched: an eviction says nothing about how much the range wants. - resources = (cluster.k8s.job('pc-r300-a2') - .spec.template.spec.containers[0].resources) - assert resources.requests['memory'] == config.REQ_MEM - - -def test_progress_going_backwards_redispatches_rather_than_halting(cluster): - # There is no monotonic-progress guard. It kept its high-water mark in - # memory, so a restart reset it to zero and disarmed the guard for exactly - # the event it existed to survive. Re-running a range is idempotent -- the - # PVC still holds /data, so the attempt resumes from its last closed ledger. - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1) - cluster.reconcile() - assert '300' in cluster.completed() - - # Someone deletes the record underneath the run. - cluster.write(config.PROGRESS_FILE, '{}') - result = cluster.reconcile() - - # Back in the pool, and the run keeps going instead of halting. - assert '300' not in cluster.completed() - assert result['remaining'] + len(result['in_progress']) == 3 - - -def test_the_fake_raises_the_status_codes_the_monitor_branches_on(cluster): - cluster.reconcile() - - with pytest.raises(fake_k8s.ApiException) as dup: - cluster.k8s.batch_v1.create_namespaced_job( - cluster.namespace, cluster.k8s.job('pc-r300-a1')) - assert dup.value.status == 409 - - with pytest.raises(fake_k8s.ApiException) as missing: - cluster.k8s.core_v1.read_namespaced_config_map('nope', cluster.namespace) - assert missing.value.status == 404 - - # 404 on a PVC read is what ensure_pvc() uses to decide to create one, and - # 404 on the progress ConfigMap is what load_progress() treats as "new run". - with pytest.raises(fake_k8s.ApiException) as gone: - cluster.k8s.core_v1.read_namespaced_persistent_volume_claim( - 'pc-data-r999', cluster.namespace) - assert gone.value.status == 404 - - -def test_an_unfinalized_predecessor_is_not_deleted(cluster): - """Reaping the Job reaps the pod its measurements still live on. - - Driven through a disruption rather than exit 3: exit 3 now defers until the - collector has finalized, so it can never be observed mid-retry unfinalized. - """ - cluster.reconcile() - cluster.advance(300, 'disrupted') - - cluster.reconcile() - - assert 'pc-r300-a2' in cluster.jobs(), "the successor must exist" - assert 'pc-r300-a1' in cluster.jobs(), "the collector has not finalized a1" diff --git a/src/MissionParallelCatchup/tests/unit/conftest.py b/src/MissionParallelCatchup/tests/unit/conftest.py deleted file mode 100644 index e42a305b..00000000 --- a/src/MissionParallelCatchup/tests/unit/conftest.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Fixtures for the imported-function unit tests. - -These tests call job_monitor / log_collector functions directly. Almost all of -them touch the shared logs volume through LOG_DIR-derived paths, so the one -thing they all need is that directory pointed somewhere disposable -- and -pointed at the SAME place in both modules, which is the contract the two -processes actually run under. -""" - -import pytest - -import config -import job_monitor as jm -import log_collector as lc - - -@pytest.fixture -def logdir(tmp_path, monkeypatch): - """The shared volume, as both processes see it.""" - d = tmp_path / 'logs' - d.mkdir() - monkeypatch.setattr(config, 'LOG_DIR', str(d)) - monkeypatch.setattr(config, 'LOG_DIR', str(d)) - return d diff --git a/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py b/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py deleted file mode 100644 index 3d41b7e0..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_attempt_chain.py +++ /dev/null @@ -1,415 +0,0 @@ -"""Aggregating measurements across the attempts that make up one range. - -In pvc mode a pod killed after replay starts leaves /data, and the next attempt -resumes at LCL+1 with RESUME=true -- skipping the archive download and bucket -apply, which is where peak memory happens. Profiling only the winning attempt -therefore under-reports a resumed range by the whole download gap, and on spot -(where eviction is routine and resume is the point of durable /data) that would -make the run unprofileable. medida's total and a pod's duration are per-process -for exactly the same reason, so both are tail-only in the same way. -""" - -import gzip -import io -import json -import os - -import pytest - -import config -import units -import records -import sizing -import attempts -import job_monitor as jm - - -GIB = 1024 ** 3 -MIB = 1024 ** 2 - - -def _gzip_member(text): - buf = io.BytesIO() - with gzip.GzipFile(fileobj=buf, mode='wb', mtime=0) as fh: - fh.write(text.encode()) - return buf.getvalue() - - -def _archive(end, attempt, *members): - with open(records.log_path(end, attempt), 'wb') as fh: - for member in members: - fh.write(_gzip_member(member)) - - -@pytest.fixture -def write_attempts(logdir): - """Lay down the files the collector and the monitor leave per attempt.""" - def write(end, spec): - for n, (metrics, outcome) in spec.items(): - if metrics is not None: - with open(records.metrics_path(end, n), 'w') as fh: - fh.write(metrics if isinstance(metrics, str) else json.dumps(metrics)) - if outcome is not None: - with open(records.outcome_path(end, n), 'w') as fh: - json.dump(outcome, fh) - return write - - -# --- which attempts describe the range --------------------------------------- - -def test_the_chain_is_the_run_of_resumed_attempts_ending_at_this_one(write_attempts): - # a1 interrupted then superseded by a fresh a2; a3 resumed from a2. Only - # a2+a3 describe the same continuous pass over the range. - write_attempts(999, {1: ({}, None), 2: ({}, None), 3: ({'resumed': True}, None)}) - assert list(attempts._resumed_chain(999, 3)) == [2, 3] - assert list(attempts._resumed_chain(999, 1)) == [1] - - -def test_an_attempt_with_no_metrics_file_is_not_treated_as_resumed(write_attempts): - write_attempts(999, {1: ({}, None)}) - assert attempts._attempt_resumed(999, 2) is False - - -def test_a_three_attempt_chain_is_read_from_the_records(write_attempts): - write_attempts(999, {1: ({}, None), 2: ({'resumed': True}, None), - 3: ({'resumed': True}, None)}) - - assert list(attempts._resumed_chain(999, 3)) == [1, 2, 3] - - -# --- peaks -------------------------------------------------------------------- - -def test_a_resumed_range_keeps_the_peak_from_the_attempt_that_did_the_download(write_attempts): - # a1 evicted mid-replay having already done the download; a2 resumes at - # LCL+1 and only replays the tail. a2 alone would report 400MiB for a range - # that really needs 2GiB. - write_attempts(999, {1: ({'peakAnonBytes': 2 * GIB}, {'outcome': 'disrupted'}), - 2: ({'peakAnonBytes': 400 * MIB, 'resumed': True}, None)}) - assert attempts.peaks_for_range(999, 2)['peakAnonBytes'] == 2 * GIB - - -def test_a_fresh_retry_supersedes_an_interrupted_one(write_attempts): - # No RESUME line means new-db ran and this attempt did the whole range, so - # its sample is complete. An earlier attempt that was merely interrupted - # measured the same work and only adds noise. - write_attempts(999, {1: ({'peakAnonBytes': 8 * GIB}, {'outcome': 'disrupted'}), - 2: ({'peakAnonBytes': 900 * MIB}, None)}) - assert attempts.peaks_for_range(999, 2)['peakAnonBytes'] == 900 * MIB - - -@pytest.mark.parametrize('outcome,field,hit,quiet', [ - ('oom', 'peakAnonBytes', 8 * GIB, 900 * MIB), - ('oom', 'peakEphemeralBytes', 30 * GIB, 5 * GIB), # died on memory, its disk figure is real - ('ephemeral', 'peakEphemeralBytes', 40 * GIB, 9 * GIB), - ('ephemeral', 'peakAnonBytes', 3 * GIB, 1 * GIB), -]) -@pytest.mark.parametrize('resumed', [True, False]) -def test_an_attempt_killed_at_a_ceiling_counts_wherever_it_sits(write_attempts, outcome, - field, hit, quiet, resumed): - # A pod OOM-killed at 8Gi really did allocate ~8Gi and wanted more, so its - # peak is a lower bound on demand, not an artifact of the limit -- and it is - # the attempt most worth keeping, because download concurrency scales with - # available cpu and a pod that bursted on an idle node can peak above the - # one that eventually succeeded. Sizing off the quieter attempt would OOM - # the range again. - # - # It survives a fresh start too, which the chain rule alone would drop. - # Measured on ssc-30: an OOM in replay resumes and stays in the chain - # (224 of 252), an OOM in download does not (25 of 252), and a higher-cpu - # run is download-bound -- so the self-correcting loop would go quiet - # exactly when it is most needed. - later = {field: quiet} - if resumed: - later['resumed'] = True - write_attempts(999, {1: ({field: hit}, {'outcome': outcome}), 2: (later, None)}) - assert attempts.peaks_for_range(999, 2)[field] == hit - - -def test_the_ceiling_exception_is_peaks_only(write_attempts): - # tx_apply and seconds are summed, and a fresh start redoes the work the - # dropped attempt already did, so counting it there would double-count. - write_attempts(999, {1: ({'txApplySeconds': 100.0, 'attemptSeconds': 900.0}, - {'outcome': 'oom'}), - 2: ({'txApplySeconds': 7.0}, None)}) # fresh start - assert attempts.tx_apply_for_range(999, 2) == 7.0 - assert attempts.seconds_for_range(999, 2, 300.0) == 300.0 - - -def test_a_missing_or_malformed_metrics_file_is_tolerated(write_attempts): - write_attempts(999, {2: ("not json at all", None), - 3: ({'peakAnonBytes': 5, 'resumed': True}, None)}) - assert attempts.peaks_for_range(999, 3) == {'peakAnonBytes': 5} - assert attempts.peaks_for_range(999, 9) == {} - - -def test_an_absent_peak_never_reaches_the_profile_as_a_null(write_attempts): - # The consumer falls back to a default on a missing field, so a null defeats it. - write_attempts(999, {1: ({'peakAnonBytes': None, 'peakAnonBytes': 7}, None)}) - assert attempts.peaks_for_range(999, 1) == {'peakAnonBytes': 7} - - -def test_both_measured_peaks_reach_the_progress_record(): - # peaks_for_range filters to PEAK_FIELDS; a measurement missing from it is - # dropped silently between the collector and the profile. - for field in ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes'): - assert field in attempts.PEAK_FIELDS, field - - -# --- durations ---------------------------------------------------------------- - -def test_seconds_sums_the_whole_resumed_chain(write_attempts): - # a1 ran 900s then was evicted mid-replay; a2 resumed and took 300s. The - # range cost 1200s of compute, not 300. - write_attempts(999, {1: ({}, {'outcome': 'disrupted', 'attemptSeconds': 900.0}), - 2: ({'resumed': True}, None)}) - assert attempts.seconds_for_range(999, 2, 300.0) == 1200.0 - - -def test_seconds_ignores_attempts_before_a_fresh_start(write_attempts): - # a2 ran new-db and did the whole range itself, so a1's 900s is not part of - # the same pass. - write_attempts(999, {1: ({}, {'outcome': 'oom', 'attemptSeconds': 900.0}), - 2: ({}, None)}) - assert attempts.seconds_for_range(999, 2, 300.0) == 300.0 - - -def test_seconds_is_absent_when_a_resumed_leg_has_no_recorded_duration(write_attempts): - # Winner-only is a lower bound, not the chain total. Missing accurately - # tells the profile consumer not to size from it. - write_attempts(999, {1: ({}, {'outcome': 'disrupted'}), 2: ({'resumed': True}, None)}) - assert attempts.seconds_for_range(999, 2, 300.0) is None - - -def test_seconds_is_none_when_nothing_is_known(write_attempts): - write_attempts(999, {1: ({}, None)}) - assert attempts.seconds_for_range(999, 1, None) is None - - -def test_seconds_falls_back_to_the_collectors_figure(write_attempts): - # The authoritative .outcome is missing for every reaped pod -- measured on - # ssc-test 2026-07-30, 212 of 212 spot disruptions were classified from the - # Job condition with the pod already gone, so record_outcome never ran. - # Without this fallback the chain drops that leg entirely. - write_attempts(999, {1: ({'attemptSeconds': 850.0}, None), - 2: ({'resumed': True}, None)}) - assert attempts.seconds_for_range(999, 2, 300.0) == 1150.0 - - -def test_a_poller_clock_estimate_is_refused_as_a_chain_leg(write_attempts): - # attemptSecondsExact False with no other provenance means the figure came - # from the collector's own clock, which starts when that process attached -- - # a lower bound, not the attempt. Summing it would publish a total that is - # quietly short. - write_attempts(999, {1: ({'attemptSeconds': 850.0, 'attemptSecondsExact': False}, None), - 2: ({'resumed': True}, None)}) - assert attempts.seconds_for_range(999, 2, 300.0) is None - - -def test_a_duration_dated_from_container_start_is_accepted(write_attempts): - # The only duration a DISRUPTED attempt can produce. A pod being deleted - # keeps phase Running, so its terminated timestamps are usually never - # observed and the exact path never fires; dating from the container's own - # startTime measured within 1% on ssc-test (370.9s and 375.1s against ~373s) - # where the poller clock was 46% short. Refusing it left every resumed chain - # with no `seconds` at all, which is the whole reason spot runs came back - # unprofiled. - write_attempts(999, {1: ({'attemptSeconds': 850.0, 'attemptSecondsExact': False, - 'attemptSecondsFromContainerStart': True}, None), - 2: ({'resumed': True}, None)}) - assert attempts.seconds_for_range(999, 2, 300.0) == 1150.0 - - -def test_the_authoritative_outcome_wins_over_the_collector_estimate(write_attempts): - # .outcome comes from the pod's terminated timestamps; the collector's is a - # stream-lifetime approximation that starts up to one poll late. - write_attempts(999, {1: ({'attemptSeconds': 850.0}, - {'outcome': 'disrupted', 'attemptSeconds': 900.0}), - 2: ({'resumed': True}, None)}) - assert attempts.seconds_for_range(999, 2, 300.0) == 1200.0 - - -# --- completed profile reconstruction ----------------------------------------- - -def test_repair_recovers_predecessor_peaks_and_seconds_idempotently(write_attempts): - write_attempts(999, { - 1: ({'attemptSeconds': 900.0, 'peakAnonBytes': 2 * GIB, - 'peakWorkingSetBytes': 3 * GIB}, None), - 2: ({'attemptSeconds': 300.0, 'peakAnonBytes': 400 * MIB, - 'peakWorkingSetBytes': 500 * MIB, 'resumed': True}, None), - }) - record = {'attempts': 2, 'seconds': 300.0, - 'peakAnonBytes': 400 * MIB, 'peakWorkingSetBytes': 500 * MIB} - - assert attempts._repair_completed_profile('999', 2, record) - assert record['seconds'] == 1200.0 - assert record['peakAnonBytes'] == 2 * GIB - assert record['peakWorkingSetBytes'] == 3 * GIB - - snapshot = json.loads(json.dumps(record)) - assert not attempts._repair_completed_profile('999', 2, record) - assert record == snapshot - - -def test_reconstruction_omits_txapply_when_one_chain_leg_is_missing(write_attempts): - write_attempts(999, { - 1: ({'txApplySeconds': 10.0}, None), - 2: ({'resumed': True}, None), # this leg's metric was unavailable - 3: ({'txApplySeconds': 3.0, 'resumed': True}, None), - }) - - rebuilt = attempts.reconstruct_completed_profile(999, 3) - assert 'txApply' not in rebuilt - - -def test_reconstruction_leaves_txapply_absent_when_every_leg_is_missing(write_attempts): - write_attempts(999, {1: ({}, None), 2: ({'resumed': True}, None)}) - - assert 'txApply' not in attempts.reconstruct_completed_profile(999, 2) - - -def test_repair_removes_legacy_winner_only_chain_aggregates(write_attempts): - write_attempts(999, { - 1: ({}, {'outcome': 'disrupted'}), - 2: ({'resumed': True, 'attemptSeconds': 300.0, - 'txApplySeconds': 3.0}, None), - }) - record = {'attempts': 2, 'seconds': 300.0, 'txApply': 3.0} - - assert attempts._repair_completed_profile('999', 2, record) - assert 'seconds' not in record - assert 'txApply' not in record - assert not attempts._repair_completed_profile('999', 2, record) - - -def test_reconstruction_does_not_cross_a_fresh_restart_boundary(write_attempts): - write_attempts(999, { - 1: ({'attemptSeconds': 900.0, 'txApplySeconds': 100.0, - 'peakAnonBytes': 8 * GIB}, None), - # No resume marker: new-db ran, so this attempt starts the chain. - 2: ({'attemptSeconds': 300.0, 'txApplySeconds': 7.0, - 'peakAnonBytes': 900 * MIB}, None), - 3: ({'attemptSeconds': 60.0, 'txApplySeconds': 2.0, - 'peakAnonBytes': 400 * MIB, 'resumed': True}, None), - }) - - rebuilt = attempts.reconstruct_completed_profile(999, 3) - assert rebuilt['seconds'] == 360.0 - assert rebuilt['txApply'] == 9.0 - assert rebuilt['peakAnonBytes'] == 900 * MIB - - -def test_wall_seconds_spans_a_resumed_chain_from_attempt_one(cluster): - """wallSeconds is the range's whole life, retries and gaps included. - - It used to be omitted for a resumed chain, because the winning Job's own - start covers the LAST leg only and read smaller than chain-summed `seconds`. - Anchoring on attempt 1's creationTimestamp removes that inversion: the span - contains every leg plus every gap between them, so wall - seconds is the - overhead the Job-per-range design introduced. - """ - cluster.reconcile() - first_start = jm.range_started_at(300) - assert first_start is not None, "attempt 1's Job creation must be recorded" - - cluster.advance(300, 'disrupted') - cluster.finalize(300, 1, tx_apply=10.0, attempt_seconds=60.0) - cluster.reconcile() - - cluster.advance(300, 'succeeded', attempt=2) - cluster.finalize(300, 2, tx_apply=2.0, attempt_seconds=60.0, resumed=True) - cluster.reconcile() - - record = cluster.completed()['300'] - assert record['seconds'] == 120.0 - assert record['txApply'] == 12.0 - # Present, and still anchored at attempt 1 -- attempt 2's dispatch must not - # re-stamp it, or the span silently shrinks to the last leg again. - assert record['wallSeconds'] is not None - assert jm.range_started_at(300) == first_start - - -def test_wall_seconds_is_absent_when_attempt_one_was_never_recorded(cluster): - """No anchor means no wall, rather than a winner-only span. - - Falling back to the winning Job's own start would measure one leg and - understate exactly the overhead this field exists to expose, so absent is - the honest answer -- the same choice `seconds` and `txApply` make when a - chain leg is missing. - """ - cluster.reconcile() - os.remove(records.started_path(300)) - - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1, tx_apply=1.5, attempt_seconds=60.0) - cluster.reconcile() - - record = cluster.completed()['300'] - assert record['seconds'] == pytest.approx(60.0) - assert record['wallSeconds'] is None - - -def test_disrupted_predecessor_without_final_medida_makes_txapply_absent(write_attempts): - write_attempts(999, { - 1: ({'attemptSeconds': 900.0}, {'outcome': 'disrupted'}), - 2: ({'resumed': True, 'txApplySeconds': 3.0}, None), - }) - - assert attempts.tx_apply_for_range(999, 2) is None - assert 'txApply' not in attempts.reconstruct_completed_profile(999, 2) - - -# --- counting causes, not attempts -------------------------------------------- - -def test_escalation_counts_ooms_not_attempts(write_attempts): - # On spot most retries are evictions: 288 disruption retries against 7 OOM - # retries on ssc-test 2026-07-30. Keying the exponent on the attempt index - # meant a range disrupted three times then OOMing once jumped to - # base * 1.5^4 -- a 5x request for one OOM, inflated fleet-wide. - write_attempts(9, {1: (None, {'outcome': 'disrupted'}), - 2: (None, {'outcome': 'disrupted'}), - 3: (None, {'outcome': 'disrupted'}), - 4: (None, {'outcome': 'oom'})}) - assert records._oom_count(9, 4) == 1, "three evictions were counted as escalations" - write_attempts(9, {5: (None, {'outcome': 'oom'}), 6: (None, {'outcome': 'oom'})}) - assert records._oom_count(9, 6) == 3 - - -def test_disk_escalation_counts_evictions_not_attempts(write_attempts, monkeypatch): - """Same inflation as the OOM ladder, on the disk ladder. - - The budget check for an ephemeral eviction already counts causes; the SIZE - did not, so a range disrupted four times then evicted once escalated as if - it had been evicted five times. - """ - monkeypatch.setattr(config, 'LIM_EPHEMERAL', '4Gi') - monkeypatch.setattr(config, 'EPH_BUMP_FACTOR', 1.5) - write_attempts(9, {1: (None, {'outcome': 'disrupted'}), - 2: (None, {'outcome': 'disrupted'}), - 3: (None, {'outcome': 'disrupted'}), - 4: (None, {'outcome': 'disrupted'}), - 5: (None, {'outcome': 'ephemeral'})}) - - evictions = records._cause_count(9, 5, ('ephemeral',)) - assert evictions == 1, "four disruptions were counted as disk escalations" - # One rung, not five: 4Gi -> 6Gi, where attempt-indexing gave 4Gi * 1.5^5. - bytes_of = units.quantity_bytes - assert bytes_of(sizing.eph_for_attempt(evictions + 1)) == bytes_of('6Gi') - assert bytes_of(sizing.eph_for_attempt(evictions)) == bytes_of('4Gi'), \ - "the limit this attempt ran at" - - -def test_disk_escalation_climbs_on_each_real_eviction(write_attempts, monkeypatch): - monkeypatch.setattr(config, 'LIM_EPHEMERAL', '4Gi') - monkeypatch.setattr(config, 'EPH_BUMP_FACTOR', 1.5) - write_attempts(9, {1: (None, {'outcome': 'ephemeral'}), - 2: (None, {'outcome': 'ephemeral'})}) - - assert records._cause_count(9, 2, ('ephemeral',)) == 2 - assert units.quantity_bytes(sizing.eph_for_attempt(3)) == units.quantity_bytes('9Gi') - - -def test_the_disk_ladder_is_capped(monkeypatch): - monkeypatch.setattr(config, 'LIM_EPHEMERAL', '4Gi') - monkeypatch.setattr(config, 'EPH_BUMP_FACTOR', 1.5) - monkeypatch.setattr(config, 'EPH_ESCALATION_CAP', '20Gi') - assert units.quantity_bytes(sizing.eph_for_attempt(99)) == units.quantity_bytes('20Gi') diff --git a/src/MissionParallelCatchup/tests/unit/test_classify.py b/src/MissionParallelCatchup/tests/unit/test_classify.py deleted file mode 100644 index 058d94e3..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_classify.py +++ /dev/null @@ -1,247 +0,0 @@ -"""How a failed attempt is classified, from the Job and from the pod. - -Two classifiers, two different objects. classify_from_job() reads the Job -controller's podFailurePolicy condition message -- a format this mission does -not control, pinned here from real captures so an EKS change fails here rather -than silently degrading a run. classify() reads the pod, which carries detail -the Job never has. -""" - -import pytest -from kubernetes import client - -import config -import job_monitor as jm -import log_collector as lc - - -# --- captures ---------------------------------------------------------------- - -# EKS 1.34 Job condition messages. Only the wording is pinned; pod and -# container names are renamed for readability. -DISRUPTED = ("Pod sandbox/jterm-catchup-snfr2 has condition DisruptionTarget " - "matching FailJob rule at index 0") -OOMKILLED = ("Container oom-container for pod sandbox/oom-test-job-qvq8b failed with " - "exit code 137 matching FailJob rule at index 1") -NONZERO_EXIT = ("Container exit-1-container for pod sandbox/exit-1-job-wbhkq failed with " - "exit code 1 matching FailJob rule at index 2") - -# RECONSTRUCTED 2026-07-30 after an over-broad test deletion removed the -# originals -- twice. Shaped to what the code parses (RULE_ORDER[2] is 'failed'; -# classify() keys on the substring 'ephemeral' in status.message) but no longer -# a verbatim capture. Re-pin from a real eviction on the next run. -EPH_EVICT_JOB_CONDITION = ( - "Container stellar-core for pod stellar-supercluster/" - "parallel-catchup-r31005951-a1-x7k2p failed with exit code 3 " - "matching FailJob rule at index 2") -EPH_EVICT_MESSAGE = ( - "Pod ephemeral local storage usage exceeds the total limit of containers 40Gi") - - -def failed_job(message='', reason='PodFailurePolicy'): - return client.V1Job(status=client.V1JobStatus(conditions=[ - client.V1JobCondition(type='Failed', status='True', - reason=reason, message=message)])) - - -def verdict(message, reason='PodFailurePolicy'): - """(outcome, exitCode, pod) as classify_from_job reports them.""" - got = jm.classify_from_job(failed_job(message, reason)) - if got is None: - return (None, None, None) - return (got['outcome'], got['exitCode'], got['pod'] or None) - - -def pod_with(reason=None, message=None, conditions=None, terminated=None, - container='stellar-core'): - """A pod carrying exactly the status fields classify() branches on.""" - statuses = None - if terminated is not None: - statuses = [client.V1ContainerStatus( - name=container, image='core', image_id='', ready=False, restart_count=0, - state=client.V1ContainerState( - terminated=client.V1ContainerStateTerminated(**terminated)))] - return client.V1Pod( - metadata=client.V1ObjectMeta(name='p'), - status=client.V1PodStatus(reason=reason, message=message, - conditions=conditions, container_statuses=statuses)) - - -def as_dict(reason=None, message=None, conditions=None, terminated=None): - """The same pod, in the shape the collector reads off the raw API.""" - status = {'reason': reason, 'message': message, - 'conditions': conditions or [], - 'containerStatuses': ([{'state': {'terminated': terminated}}] - if terminated is not None else [])} - return {'metadata': {'name': 'p'}, 'status': status} - - -# --- classify_from_job: the Job controller's message -------------------------- - -@pytest.mark.parametrize("msg,outcome,code,pod", [ - (DISRUPTED, 'disrupted', None, None), - (OOMKILLED, 'oom', 137, 'oom-test-job-qvq8b'), - (NONZERO_EXIT, 'failed', 1, 'exit-1-job-wbhkq'), -]) -def test_job_condition_message(msg, outcome, code, pod): - assert verdict(msg) == (outcome, code, pod) - - -def test_rule_order_matches_the_rendered_policy(): - # "rule at index N" is only meaningful against the order the rules are - # rendered in, so the lookup table and the policy must be the same list. - assert [name for name, _ in jm._failure_rules()] == jm.RULE_ORDER - - -def test_eviction_is_told_apart_from_a_broken_range_by_the_condition(): - # stellar-core exits 3 both for a drain and for a corrupt bucket, so only - # DisruptionTarget separates them -- hence rule 0 must be evaluated first. - assert verdict(DISRUPTED)[0] == 'disrupted' - assert verdict("Container c for pod ns/p failed with exit code 3")[0] == 'failed' - - -def test_a_bare_exit_code_is_read_when_no_rule_index_is_offered(): - # Measured on ssc-test 2026-07-28: a drained stellar-core catches SIGTERM - # and exits 3 well inside the 100s grace, so evictions do NOT produce 137 -- - # which makes a bare 137 an OOM with high confidence. - assert verdict("Container c for pod ns/p failed with exit code 137")[:2] == ('oom', 137) - assert verdict("Container c for pod ns/p failed with exit code 1")[:2] == ('failed', 1) - - -def test_an_unclassifiable_job_failure_stays_unclassified(): - """classify returns nothing rather than guessing, and reconcile condemns. - - BackoffLimitExceeded carries no rule index and no exit code. An - unclassifiable failure is not evidence that the range is fine, so the run - stops rather than retrying blind -- only a node disruption, which proves the - cluster took the pod away, keeps its unlimited budget. - """ - assert jm.classify_from_job(failed_job( - "Job has reached the specified backoff limit", - reason='BackoffLimitExceeded')) is None - assert set(config.ATTEMPT_BUDGETS) == {'disrupted', 'rejected', 'fetch-fault', - 'oom', 'ephemeral'}, \ - "an unclassifiable failure must have no budget at all" - - -def test_a_deadline_exceeded_job_is_a_timeout_not_a_catchup_failure(): - # activeDeadlineSeconds fired: the attempt hung rather than failing. Only - # the Job knows this -- the deadline SIGTERMs the pod, which drains to - # exit 3 and reads as a plain catchup failure from the pod side. - assert verdict('', reason='DeadlineExceeded')[0] == 'timeout' - - -def test_a_job_with_no_failed_condition_yields_nothing(): - assert jm.classify_from_job(client.V1Job(status=client.V1JobStatus())) is None - - -# --- classify: what only the pod can say -------------------------------------- - -def test_a_disruption_target_condition_outranks_everything_on_the_pod(): - got = jm.classify(pod_with( - conditions=[client.V1PodCondition(type='DisruptionTarget', status='True')], - terminated={'exit_code': 3, 'reason': 'Error'})) - assert got['outcome'] == 'disrupted' - - -def test_the_disruption_reason_separates_a_warning_from_a_postmortem(): - # The bare condition cannot tell these apart, and the difference decides - # whether a missing txApply is a capture bug worth chasing. An eviction is a - # drain that still owes the container a SIGTERM, so a medida block is coming. - # A TaintManager stamp lands ~40s after the node went NotReady, on a process - # that already died unsignalled -- nothing was ever written to miss. - def reason_for(r): - return lc._is_condemned(as_dict( - conditions=[{'type': 'DisruptionTarget', 'status': 'True', 'reason': r}])) - - assert reason_for('EvictionByEvictionAPI') == 'EvictionByEvictionAPI' - assert reason_for('DeletionByTaintManager') == 'DeletionByTaintManager' - # Truthiness is the contract every caller relies on, not the string itself. - assert lc._is_condemned(as_dict()) is None - assert not lc._is_condemned(as_dict( - conditions=[{'type': 'DisruptionTarget', 'status': 'False'}])) - - -def test_a_condition_without_a_reason_still_reads_as_condemned(): - # Kubernetes does not promise the field. Falling back to None here would - # silently un-condemn the pod and drop it back to the lazy poll cadence. - assert lc._is_condemned(as_dict( - conditions=[{'type': 'DisruptionTarget', 'status': 'True'}])) == 'Unknown' - - -def test_an_ephemeral_eviction_is_not_read_as_an_oom_or_a_disruption(): - # Measured end-to-end on ssc-test: the kubelet sets no DisruptionTarget, - # and stellar-core drains and exits 3, so the Job condition is a plain - # non-zero failure that would get no retry. status.message is the only - # discriminator and only the pod carries it, so both classifiers must test - # it before anything keyed on Evicted. - assert verdict(EPH_EVICT_JOB_CONDITION)[0] == 'failed', \ - "the Job matches the generic non-zero rule" - assert 'ephemeral' in EPH_EVICT_MESSAGE, "both classifiers key on this substring" - evicted = dict(reason='Evicted', message=EPH_EVICT_MESSAGE, - terminated={'exit_code': 3, 'reason': 'Error'}) - assert jm.classify(pod_with(**evicted))['outcome'] == 'ephemeral' - assert lc.classify(as_dict(reason='Evicted', message=EPH_EVICT_MESSAGE, - terminated={'exitCode': 3}))['outcome'] == 'ephemeral' - - -def test_a_plain_eviction_with_no_disk_message_is_only_a_rejection(): - # The generic Evicted branch sits right behind the ephemeral one; an - # eviction for anything other than the range's own disk use must still - # reach it, or a node-pressure eviction would be read as a disk overrun and - # grow the range's storage for no reason. - got = jm.classify(pod_with(reason='Evicted', message='node was low on memory')) - assert got['outcome'] == 'rejected' - - -@pytest.mark.parametrize('reason', [ - 'VolumeAttachmentLimitExceeded', 'OutOfcpu', 'OutOfmemory', 'OutOfpods', - 'UnexpectedAdmissionError', 'NodeAffinity', 'Shutdown', 'Evicted', -]) -def test_an_admission_rejection_is_not_a_catchup_failure(reason): - # Observed on ssc-test: reason=VolumeAttachmentLimitExceeded, "Node has - # reached its volume attachment limit, rejecting pod". No exit code, no - # DisruptionTarget -- without this branch it falls through to 'failed' and - # a transient admission rejection kills the whole run. - for got in (jm.classify(pod_with(reason=reason)), - lc.classify(as_dict(reason=reason))): - assert got['outcome'] == 'rejected', reason - assert got['exitCode'] is None - - -def test_a_deadline_kill_is_visible_on_the_pod_too(): - # The deadline lives on the PodSpec, so the kubelet fires it and the pod - # carries the reason; the Job only sees a non-zero exit. - assert jm.classify(pod_with(reason='DeadlineExceeded'))['outcome'] == 'timeout' - - -def test_a_pod_where_nothing_ever_ran_says_nothing_about_the_range(): - for got in (jm.classify(pod_with()), lc.classify(as_dict())): - assert got['outcome'] == 'rejected' - - -def test_an_oom_kill_is_read_from_the_container_reason_not_the_exit_code(): - # 137 is SIGKILL, which the kubelet also uses for a graceful-stop timeout -- - # only reason=OOMKilled makes it unambiguous. - for got in (jm.classify(pod_with(terminated={'exit_code': 137, 'reason': 'OOMKilled'})), - lc.classify(as_dict(terminated={'exitCode': 137, 'reason': 'OOMKilled'}))): - assert (got['outcome'], got['exitCode']) == ('oom', 137) - - -def test_a_non_zero_exit_is_a_catchup_failure_and_keeps_its_code(): - for got in (jm.classify(pod_with(terminated={'exit_code': 3, 'reason': 'Error'})), - lc.classify(as_dict(terminated={'exitCode': 3}))): - assert (got['outcome'], got['exitCode']) == ('failed', 3) - - -def test_exit_three_is_the_ambiguous_one_and_is_decided_by_the_archive(): - """A corrupt range is protected by the exit-3 rule, not by its budget. - - stellar-core drains to 3 on SIGTERM and a corrupt bucket also exits 3, so - the exit code decides nothing. An exit 3 is condemned outright unless its - archive names a fetch fault -- so a genuinely corrupt range never reaches a - budget at all, and the fetch-fault retry that does is infrastructure. - """ - assert config.CATCHUP_INCOMPLETE_EXIT == 3 - assert config.ATTEMPT_BUDGETS['ephemeral'] < config.ATTEMPT_BUDGETS['disrupted'], \ - "a deterministic failure must not get the disruption budget" diff --git a/src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py b/src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py deleted file mode 100644 index 0de862f8..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_collector_main_loop.py +++ /dev/null @@ -1,279 +0,0 @@ -"""log_collector.main(): which pods get a stream, and when one is let go. - -The loop is small and every decision in it has cost a run something: a stream -re-opened every cycle re-read a whole log per pod per POLL_SECONDS; a pod that -left the pod list without ever being observed terminal kept its stream retrying -until the run ended; a sampler placed after the per-pod branches only ever fired -on the cycle a stream opened, when the range had written almost nothing. - -Driven by running the real `main()` with `list_pods`, `sample_kubelet` and -`poll_pod` replaced -- those three are the loop's entire outside world -- and -cancelling it once the scenario has played out. Nothing here reads source text: -the previous version of these tests sliced the loop body out with a regex, which -matched the wrong block twice and had to be re-anchored by hand. -""" - -import asyncio -import os - -import pytest - -import config -import log_collector as lc - - -def pod(name, phase='Running', end='300', attempt='1', node='node-1', ip=None): - # hostIP is what the sampler reads: it talks to the kubelet directly rather - # than through the apiserver's node proxy. - return {'metadata': {'name': name, - 'labels': {config.LABEL_RUN: config.RUN_NAME, - config.LABEL_RANGE: end, - config.LABEL_ATTEMPT: attempt}}, - 'spec': {'nodeName': node}, - 'status': {'phase': phase, 'hostIP': ip or f"10.0.0.{abs(hash(node)) % 200 + 1}"}} - - -class Loop: - """One scripted run of main(): a pod list per cycle, and what happened. - - Once the script is exhausted the last cycle repeats, so the loop settles - into a steady state rather than starting to error -- an exception out of - list_pods is swallowed by main() and would only add noise. - - `poller` picks what the fake stream does: 'wait' blocks on the pod's _wake - Event and then returns (a normal stream, ended by the pod going away or - terminal), 'return' finishes immediately (a stream that died early), and - 'hang' ignores the wake and never finishes at all. - """ - - def __init__(self, cycles, poller='wait'): - self.cycles = list(cycles) - self.passes = 0 - self.poller = poller - self.order = [] # 'list' / 'sample' / 'open:' in sequence - self.opened = [] - self.sampled = [] - self.done_seen = {} - - async def list_pods(self, session): - self.order.append('list') - pods = self.cycles[min(self.passes, len(self.cycles) - 1)] - self.passes += 1 - return list(pods) - - async def sample_kubelet(self, session, nodes): - self.order.append('sample') - self.sampled.append(set(nodes)) - - def poll_pod(self, session, name, end, attempt, done, done_ok): - self.order.append(f"open:{name}") - self.opened.append((name, end, attempt)) - - async def run(): - if self.poller == 'return': - return - ev = lc._wake.setdefault(name, asyncio.Event()) - await ev.wait() - self.done_seen[name] = done(name) - if self.poller == 'hang': - await asyncio.Event().wait() # never finishes - - return run() - - -@pytest.fixture -def loop_env(tmp_path, monkeypatch): - monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(lc, 'token', lambda: 'tok') - monkeypatch.setattr(lc, 'ssl_ctx', lambda: None) - monkeypatch.setattr(lc, 'POLL_SECONDS', 0.01) - monkeypatch.setattr(lc, 'VANISHED_GRACE_CYCLES', 3) - for name in ('_eph_peak', '_anon_peak', '_ws_peak', '_peak_flushed', - '_streaming', '_pod_secs', '_wake'): - monkeypatch.setattr(lc, name, {}) - return monkeypatch - - -def run_loop(monkeypatch, cycles, poller='wait', extra=2): - """Run main() over a scripted sequence of pod lists, then stop it.""" - loop = Loop(cycles, poller) - monkeypatch.setattr(lc, 'list_pods', loop.list_pods) - monkeypatch.setattr(lc, 'sample_kubelet', loop.sample_kubelet) - monkeypatch.setattr(lc, 'poll_pod', loop.poll_pod) - asyncio.run(_drive(loop, extra)) - return loop - - -async def _drive(loop, extra, want_survivors=False): - task = asyncio.create_task(lc.main()) - want = len(loop.cycles) + extra - for _ in range(600): - await asyncio.sleep(0.005) - if loop.passes >= want: - break - # Stream tasks only. The condemnation watch is also long-lived by design -- - # it is supposed to outlive every poller -- so counting it here would read - # as a wedged stream that never gave its slot back. - survivors = [t for t in asyncio.all_tasks() - if t is not asyncio.current_task() and t is not task - and not t.done() - and 'watch_condemnations' not in repr(t.get_coro())] - task.cancel() - try: - await task - except asyncio.CancelledError: - pass - return survivors if want_survivors else None - - -# --- the sampler --------------------------------------------------------------- - -def test_the_sampler_never_delays_opening_a_stream(loop_env): - """The sampler is a serial sweep of every node's kubelet, and on spot a dead - one costs the whole connect timeout. Measured at 900 workers it stretched a - cycle to 925s, and ahead of the per-pod branches that delay applied to every - stream: five -a2 legs died with no reader, one after 184.7s. Opening a stream - is time-critical, so it goes first and the sampler takes the wait.""" - loop = run_loop(loop_env, [[pod('w-1')], [pod('w-1')], [pod('w-1')]]) - - assert loop.order[:3] == ['list', 'open:w-1', 'sample'] - # Still once per cycle, and still off the same listing rather than its own. - assert loop.order.count('sample') == loop.order.count('list') - - -def test_the_sampler_stays_outside_the_per_pod_loop(loop_env): - """It has to run every cycle, not once per stream. Those branches end in - `continue` for a pod already streaming, so a sampler placed among them fires - only on the cycle a stream opens -- when the range has written almost nothing - and its peak is meaningless.""" - loop = run_loop(loop_env, [[pod('w-1')]] * 4) - - # One sample per listing even though only the first cycle opens anything. - assert loop.order.count('sample') == loop.order.count('list') - assert loop.order.count('open:w-1') == 1 - - -def test_the_sampler_runs_every_cycle_not_once_per_stream(loop_env): - loop = run_loop(loop_env, [[pod('w-1')]] * 4) - - assert len(loop.sampled) >= 3, loop.order - assert loop.opened == [('w-1', '300', '1')], "the stream was re-opened" - - -def test_the_sampler_is_not_gated_on_storage_mode(loop_env): - """It was, back when it only sampled disk. Memory is sized in both modes, - so gating here left every pvc run with no anon peak at all.""" - loop_env.setattr(config, 'STORAGE_MODE', 'pvc') - loop = run_loop(loop_env, [[pod('w-1')], [pod('w-1')]]) - - assert loop.sampled and loop.sampled[0] == {pod('w-1')['status']['hostIP']} - - -def test_only_running_pods_are_handed_to_the_sampler(loop_env): - """kubelet has no live stats for a pod that has not started or has exited, - and every extra node in the set is another /stats/summary GET.""" - loop = run_loop(loop_env, [[pod('w-1', phase='Pending', node='node-a', ip='10.0.0.1'), - pod('w-2', phase='Running', node='node-b', ip='10.0.0.2')]] * 2) - - assert loop.sampled[0] == {'10.0.0.2'} - - -# --- which pods get a stream --------------------------------------------------- - -def test_a_pending_pod_is_not_polled_until_it_can_answer(loop_env): - """Its container has not started, so the log endpoint answers 400 and the - poll is wasted -- 60 of 88 failures immediately after the polling switch.""" - loop = run_loop(loop_env, [[pod('w-1', phase='Pending')], - [pod('w-1', phase='Pending')], - [pod('w-1', phase='Running')], - [pod('w-1', phase='Running')]]) - - assert loop.opened == [('w-1', '300', '1')] - # ...and not until the third cycle, the first one it could have answered. - assert loop.order[:6] == ['list', 'sample', 'list', 'sample', - 'list', 'open:w-1'] - - -def test_a_terminal_pod_is_still_polled(loop_env): - """That is where a pod's final output lives; a stream that skipped it - would lose the medida block of every range that finished quickly.""" - loop = run_loop(loop_env, [[pod('w-1', phase='Succeeded')]] * 2) - - assert loop.opened == [('w-1', '300', '1')] - - -def test_a_pod_with_no_range_label_is_not_ours(loop_env): - stray = pod('other-1') - del stray['metadata']['labels'][config.LABEL_RANGE] - loop = run_loop(loop_env, [[stray]] * 2) - - assert loop.opened == [] - - -def test_a_finished_stream_is_not_reopened_once_its_pod_is_terminal(loop_env): - """A completed task is deleted from `tasks`, so without a record of it the - next cycle re-creates the stream and re-reads the whole log -- every cycle, - per pod, for the rest of the run.""" - loop = run_loop(loop_env, [[pod('w-1', phase='Succeeded')]] * 5) - - assert loop.opened == [('w-1', '300', '1')], \ - f"stream re-opened {len(loop.opened)} times" - - -def test_a_stream_that_died_while_its_pod_still_runs_is_reopened(loop_env): - """The other half of the same guard. A task that ended while the pod is - still Running died early, and re-opening the stream is how that recovers -- - barring it would abandon a live range.""" - loop = run_loop(loop_env, [[pod('w-1', phase='Running')]] * 5, - poller='return') - - assert len(loop.opened) >= 2, "an early-dying stream was never retried" - - -# --- a pod that leaves the list ----------------------------------------------- - -def test_a_vanished_pod_is_marked_terminal_and_wakes_its_poller(loop_env): - """`terminal` is only written for pods in the pod list, so a pod that goes - away without ever being seen terminal -- reaped node, eviction, or the - monitor deleting its finished Job -- keeps done() False forever. Gone is - terminal, and the wake is what stops the poller sleeping out its interval - before it takes the 404 and writes the .done the monitor waits on.""" - loop = run_loop(loop_env, [[pod('w-1')], [pod('w-1')], [], [], []]) - - assert loop.done_seen.get('w-1') is True, \ - "the poller was never woken, or woke to done() == False" - - -def test_a_pod_going_terminal_wakes_its_poller_without_waiting_a_cycle(loop_env): - """The delay that matters is between the container exiting and the last - read: sleeping blind hands that window to a spot reclaim, which deletes the - pod and takes the final lines with it.""" - loop = run_loop(loop_env, [[pod('w-1', phase='Running')], - [pod('w-1', phase='Succeeded')], - [pod('w-1', phase='Succeeded')]]) - - assert loop.done_seen.get('w-1') is True - - -def test_a_stream_that_will_not_finish_is_cancelled_after_the_grace(loop_env): - """Marking a vanished pod terminal is not enough on its own: a stream - wedged inside a connection attempt never reaches its own done() check, and - that is exactly the state that starves every other stream of a poll slot.""" - loop = Loop([[pod('w-1')], [pod('w-1')], [], [], [], [], []], poller='hang') - loop_env.setattr(lc, 'list_pods', loop.list_pods) - loop_env.setattr(lc, 'sample_kubelet', loop.sample_kubelet) - loop_env.setattr(lc, 'poll_pod', loop.poll_pod) - - alive = asyncio.run(_drive(loop, extra=3, want_survivors=True)) - - assert alive == [], "a wedged stream outlived its grace and held its slot" - assert os.path.exists(lc.done_path('300', '1')), \ - "forced cancellation skipped finalization and never licensed cleanup" - - -def test_the_grace_is_more_than_one_cycle(): - """A stream still finalizing -- writing .metrics, closing its archive -- - must not be cancelled out from under its own write.""" - assert lc.VANISHED_GRACE_CYCLES >= 2, \ - (f"a grace of {lc.VANISHED_GRACE_CYCLES} cycle(s) can cancel a stream " - "in the middle of finalizing itself") diff --git a/src/MissionParallelCatchup/tests/unit/test_condemnation_watch.py b/src/MissionParallelCatchup/tests/unit/test_condemnation_watch.py deleted file mode 100644 index 3fa2e4bf..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_condemnation_watch.py +++ /dev/null @@ -1,384 +0,0 @@ -"""Detecting a condemnation fast enough to still open a follow. - -The follow itself was never the problem: `_follow_slots` is a 256-wide semaphore -and a real reclaim condemns tens of pods, so it never fell back to polling. What -lost the metric was seeing the condition too late. stellar-core exits about a -second after SIGTERM and the pod object is reaped behind it, so a condemned pod -exists for a few seconds -- and the pod-list sweep runs every POLL_SECONDS=5. - -Measured on ssc-test at prestopSleepSeconds=5: of 52 mid-replay legs, 32 lost -txApply. Seven were never seen condemned at all; the other 25 were seen, wrote -their disruptionReason, and still lost it because the follow opened after the -pod was gone. - -So these tests are about latency and about the two detectors agreeing, not about -whether a follow works. -""" - -import asyncio -import json - -import pytest - -import config -import log_collector as lc - - -def pod(name='w-1', phase='Running', end='300', attempt='1', - reason='EvictionByEvictionAPI', rv='100'): - conditions = ([{'type': 'DisruptionTarget', 'status': 'True', 'reason': reason}] - if reason else []) - return {'metadata': {'name': name, - 'resourceVersion': rv, - 'labels': {config.LABEL_RUN: config.RUN_NAME, - config.LABEL_RANGE: end, - config.LABEL_ATTEMPT: attempt}}, - 'status': {'phase': phase, 'conditions': conditions}} - - -@pytest.fixture -def collector(tmp_path, monkeypatch): - """Collector module state pointed at a temp volume, reset between tests.""" - monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(lc, '_doomed', {}) - monkeypatch.setattr(lc, '_wake', {}) - monkeypatch.setattr(lc, 'token', lambda: 'test-token') - # The real backoff is a wall-clock second; these tests advance the loop by - # ticks, not time, so a retry would never come back. - monkeypatch.setattr(lc, 'WATCH_RETRY_SECONDS', 0) - monkeypatch.setattr(lc, '_tasks', {}) - monkeypatch.setattr(lc, '_streamed', set()) - monkeypatch.setattr(lc, '_stream_ctx', {}) - monkeypatch.setattr(lc, '_streaming', {}) - return tmp_path - - -@pytest.fixture -def ready(collector, monkeypatch): - """A collector whose ensure_stream can actually open something. - - Records every poll_pod that gets started, so a second reader on one pod is - visible rather than silent. - """ - opened = [] - - async def fake_poll(session, name, end, attempt, done, done_ok): - opened.append((name, end, attempt)) - await asyncio.sleep(3600) - - monkeypatch.setattr(lc, 'poll_pod', fake_poll) - lc._stream_ctx.update(session=object(), terminal={}, succeeded={}) - return opened - - -async def drain(fn, ticks=30): - fn() - for _ in range(ticks): - await asyncio.sleep(0) - for t in list(lc._tasks.values()): - t.cancel() - - -# --- ensure_stream: one registry, one reader --------------------------------- - -def test_a_stream_is_opened_once_and_only_once(collector, ready): - async def go(): - await drain(lambda: [lc.ensure_stream('w-1', '300', '1', 'Running'), - lc.ensure_stream('w-1', '300', '1', 'Running'), - lc.ensure_stream('w-1', '300', '1', 'Running')]) - asyncio.run(go()) - assert ready == [('w-1', '300', '1')], \ - "two readers would re-append the same lines and race write_state" - - -def test_a_completed_stream_is_not_reopened(collector, ready): - lc._streamed.add('w-1') - async def go(): - await drain(lambda: lc.ensure_stream('w-1', '300', '1', 'Running')) - asyncio.run(go()) - assert ready == [] - - -@pytest.mark.parametrize('phase', ['Pending', 'Unknown']) -def test_an_unpollable_phase_is_left_for_a_later_call(collector, ready, phase): - # Its log endpoint answers 400 "waiting to start"; the retry is the next - # event or the next sweep, whichever lands first. - async def go(): - await drain(lambda: lc.ensure_stream('w-1', '300', '1', phase)) - asyncio.run(go()) - assert ready == [] - assert 'w-1' not in lc._tasks - - -def test_nothing_opens_before_main_publishes_its_context(collector, monkeypatch): - # ensure_stream is reachable from the watch, which starts inside main(). If - # an event landed first, opening with no session would throw in a task - # nobody awaits. - monkeypatch.setattr(lc, '_stream_ctx', {}) - assert lc.ensure_stream('w-1', '300', '1', 'Running') is False - - -# --- the two callers cannot double up ---------------------------------------- - -def test_the_watch_opens_the_stream_without_waiting_for_the_sweep(collector, ready): - # The whole point: at 900 pods the pod-list cycle reached 925s, and a pod - # condemned in that window died unread. - asyncio.run(run_watch(FakeSession([FakeResponse([ - {'type': 'ADDED', 'object': pod(reason=None)}, - ])]))) - assert ready == [('w-1', '300', '1')] - - -def test_the_sweep_still_opens_a_stream_the_watch_missed(collector, ready): - # Events are genuinely dropped across a reconnect, so the loop stays as a - # backstop rather than being retired. - async def go(): - await drain(lambda: lc.ensure_stream('w-1', '300', '1', 'Running')) - asyncio.run(go()) - assert ready == [('w-1', '300', '1')] - - -def test_watch_then_sweep_still_yields_one_reader(collector, ready): - async def go(): - lc.ensure_stream('w-1', '300', '1', 'Running') # watch - lc.ensure_stream('w-1', '300', '1', 'Running') # sweep, same cycle - for _ in range(30): - await asyncio.sleep(0) - for t in list(lc._tasks.values()): - t.cancel() - asyncio.run(go()) - assert ready == [('w-1', '300', '1')] - - -def test_a_pod_condemned_as_it_appears_gets_a_reader_before_being_marked(collector, ready): - # Ordering inside the watch: _mark_condemned only sets _doomed and fires - # _wake, both no-ops when no poller exists. Marking first would leave the - # condemnation with nothing to act on -- which is exactly how five -a2 legs - # produced 0-byte archives. - asyncio.run(run_watch(FakeSession([FakeResponse([ - {'type': 'ADDED', 'object': pod()}, - ])]))) - assert ready == [('w-1', '300', '1')], "the stream must exist first" - assert lc._doomed.get('w-1') == 'EvictionByEvictionAPI' - - -def metrics_of(vol, end='300', attempt='1'): - path = vol / f"range-{end}-a{attempt}.metrics" - return json.loads(path.read_text()) if path.exists() else {} - - -# --- _mark_condemned: the shared decision ------------------------------------ - -def test_a_condemnation_is_recorded_and_wakes_the_poller(collector): - lc._wake['w-1'] = asyncio.Event() - - assert lc._mark_condemned(pod(), 'w-1', '300', '1') is True - assert lc._doomed['w-1'] == 'EvictionByEvictionAPI' - assert metrics_of(collector)['disruptionReason'] == 'EvictionByEvictionAPI' - assert lc._wake['w-1'].is_set(), "the poller must not sleep out its interval" - - -def test_marking_twice_is_a_no_op(collector): - # The watch and the sweep both see the same object. Whichever is first does - # the work; the second must not re-open a stream or rewrite the reason. - assert lc._mark_condemned(pod(), 'w-1', '300', '1') is True - assert lc._mark_condemned(pod(reason='DeletionByTaintManager'), - 'w-1', '300', '1') is False - assert lc._doomed['w-1'] == 'EvictionByEvictionAPI' - - -def test_an_uncondemned_pod_is_left_alone(collector): - assert lc._mark_condemned(pod(reason=None), 'w-1', '300', '1') is False - assert lc._doomed == {} - assert metrics_of(collector) == {} - - -@pytest.mark.parametrize('phase', ['Succeeded', 'Failed']) -def test_a_finished_pod_is_not_followed(collector, phase): - # Its log is already complete, and leaving the flag set would re-open a - # stream on a dead pod every iteration. - assert lc._mark_condemned(pod(phase=phase), 'w-1', '300', '1') is False - assert lc._doomed == {} - - -def test_the_reason_is_carried_through_to_the_metrics_file(collector): - # EvictionByEvictionAPI is a drain that still owes a SIGTERM; a TaintManager - # stamp lands on a container that already died unsignalled. A lost txApply - # means different things in the two cases, so the label has to survive. - lc._mark_condemned(pod(reason='DeletionByTaintManager'), 'w-1', '300', '1') - assert metrics_of(collector)['disruptionReason'] == 'DeletionByTaintManager' - - -# --- watch_condemnations: the stream ----------------------------------------- - -class FakeResponse: - def __init__(self, lines, status=200): - self.status = status - self.content = self._iter(lines) - - async def _iter(self, lines): - for line in lines: - yield line if isinstance(line, bytes) else json.dumps(line).encode() - - def raise_for_status(self): - if self.status >= 400: - raise RuntimeError(f"status {self.status}") - - async def __aenter__(self): - return self - - async def __aexit__(self, *exc): - return False - - -class FakeSession: - """Serves one scripted watch response per connection. - - Once the script runs out every further connection raises, which both stops - the test looping forever and exercises the retry path. - """ - - def __init__(self, responses): - self.responses = list(responses) - self.calls = [] - - def get(self, url, params=None, headers=None): - self.calls.append(dict(params or {})) - if not self.responses: - raise ConnectionError("no more scripted responses") - nxt = self.responses.pop(0) - if isinstance(nxt, Exception): - raise nxt - return nxt - - -async def run_watch(session, ticks=40): - task = asyncio.create_task(lc.watch_condemnations(session)) - for _ in range(ticks): - await asyncio.sleep(0) - task.cancel() - try: - await task - except asyncio.CancelledError: - pass - - -def test_a_modified_event_condemns_immediately(collector): - monkey = FakeSession([FakeResponse([ - {'type': 'MODIFIED', 'object': pod()}, - ])]) - asyncio.run(run_watch(monkey)) - - assert lc._doomed.get('w-1') == 'EvictionByEvictionAPI', \ - "the watch, not the 5s sweep, is what has to catch this" - assert metrics_of(collector)['disruptionReason'] == 'EvictionByEvictionAPI' - - -def test_a_healthy_pod_event_does_nothing(collector): - asyncio.run(run_watch(FakeSession([FakeResponse([ - {'type': 'MODIFIED', 'object': pod(reason=None)}, - ])]))) - assert lc._doomed == {} - - -def test_deleted_events_are_ignored(collector): - # By DELETED the object is already gone; acting on it would open a stream - # against a pod that cannot answer. - asyncio.run(run_watch(FakeSession([FakeResponse([ - {'type': 'DELETED', 'object': pod()}, - ])]))) - assert lc._doomed == {} - - -def test_a_reconnect_resumes_from_the_last_resourceVersion(collector): - session = FakeSession([ - FakeResponse([{'type': 'MODIFIED', 'object': pod(reason=None, rv='517')}]), - FakeResponse([{'type': 'MODIFIED', 'object': pod(rv='518')}]), - ]) - asyncio.run(run_watch(session)) - - assert 'resourceVersion' not in session.calls[0], "first connect starts cold" - assert session.calls[1]['resourceVersion'] == '517', \ - "resuming re-delivers only what was missed instead of re-syncing" - - -def test_a_bookmark_advances_the_resume_point(collector): - # Bookmarks exist so an idle watch does not fall behind and get a 410 on - # reconnect. Ignoring them would strand the resume point at the last real - # change, which on a quiet run can be far in the past. - session = FakeSession([ - FakeResponse([{'type': 'BOOKMARK', - 'object': {'metadata': {'resourceVersion': '900'}}}]), - FakeResponse([]), - ]) - asyncio.run(run_watch(session)) - assert session.calls[1]['resourceVersion'] == '900' - - -def test_an_expired_resourceVersion_restarts_cold(collector): - # 410 Gone means our position aged out of the apiserver's history. Retrying - # with the same version loops forever; dropping it re-syncs. - session = FakeSession([ - FakeResponse([{'type': 'MODIFIED', 'object': pod(reason=None, rv='7')}]), - FakeResponse([], status=410), - FakeResponse([]), - ]) - asyncio.run(run_watch(session)) - - assert session.calls[1]['resourceVersion'] == '7' - assert 'resourceVersion' not in session.calls[2], "a 410 has to reset it" - - -def test_an_error_event_carrying_410_also_restarts_cold(collector): - # The same condition arrives as an in-stream ERROR event, not only as a - # status code on the connection. - session = FakeSession([ - FakeResponse([{'type': 'ERROR', - 'object': {'code': 410, 'metadata': {}}}]), - FakeResponse([]), - ]) - asyncio.run(run_watch(session)) - assert 'resourceVersion' not in session.calls[1] - - -def test_the_watch_survives_a_dropped_connection(collector): - # Detection degrading to the pod-list sweep is survivable; the collector - # dying is not. A watch that raised out of main() would take the whole - # sidecar and every in-flight stream with it. - session = FakeSession([ - ConnectionError("apiserver went away"), - FakeResponse([{'type': 'MODIFIED', 'object': pod()}]), - ]) - asyncio.run(run_watch(session)) - assert lc._doomed.get('w-1') == 'EvictionByEvictionAPI' - - -def test_malformed_lines_do_not_kill_the_stream(collector): - session = FakeSession([FakeResponse([ - b'{not json', - b'', - json.dumps({'type': 'MODIFIED', 'object': pod()}).encode(), - ])]) - asyncio.run(run_watch(session)) - assert lc._doomed.get('w-1') == 'EvictionByEvictionAPI' - - -def test_a_pod_without_a_range_label_is_skipped(collector): - # The job-monitor pod carries the run label too, and it has no range. - stray = pod() - del stray['metadata']['labels'][config.LABEL_RANGE] - asyncio.run(run_watch(FakeSession([FakeResponse([ - {'type': 'MODIFIED', 'object': stray}, - ])]))) - assert lc._doomed == {} - - -def test_the_watch_asks_for_bookmarks_and_a_bounded_lifetime(collector): - # An unbounded watch that dies silently stops detecting and nothing notices; - # the timeout is what makes it self-heal. - session = FakeSession([FakeResponse([])]) - asyncio.run(run_watch(session)) - assert session.calls[0]['watch'] == 'true' - assert session.calls[0]['allowWatchBookmarks'] == 'true' - assert session.calls[0]['timeoutSeconds'] == str(lc.WATCH_TIMEOUT_SECONDS) - assert session.calls[0]['labelSelector'] == f"{config.LABEL_RUN}={config.RUN_NAME}" diff --git a/src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py b/src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py deleted file mode 100644 index 496768d9..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_deadline_sizing.py +++ /dev/null @@ -1,71 +0,0 @@ -"""The attempt deadline is flat, and must stay flat. - -The deadline exists for ONE failure mode, reproduced on ssc-test 2026-07-30: -with an unreachable archive, stellar-core retries the bucket download forever. -It logs "Missing HAS for ledger N: maybe stale archive", re-selects a different -mirror and goes again -- RETRY_A_FEW is per archive, so the budget never -exhausts. Measured: 0 ledgers closed, 9 fetch failures in 2.5 min, no give-up -wording, no exit. Nothing but this deadline stops it. - -Scaling it by each range's profiled runtime was tried and removed, and this -file is the guard against it coming back. A deadline has to bound a range's -WORST case; a profile only offers a neighbour's TYPICAL case. Range keys are -anchored to the network tip, so a profile from an earlier run matches ZERO keys -exactly and every lookup lands on a neighbour -- and ~2% of neighbours are -3-38x cheaper than their surroundings. Backtested across that real grid offset -(run4 profile -> r5 actuals, 3983 ranges): a 2x factor falsely kills 134 -ranges, 4x kills 46, 6x kills 21. Flat 12h kills none. - -The asymmetry is what settles it. A false kill loses a range, and a timeout is -terminal, so it fails the whole mission. A genuine wedge holds ONE slot of -1092-1500 for 12h, about 0.1% of a run's capacity. - -Asserted against the Jobs reconcile actually creates, not against a helper. -""" - -import config -import job_monitor as jm - -DEADLINE = 43200 - -# The two ranges the fixture dispatches (PARALLELISM 2, tip-first), given -# measured costs that differ by 17x. Under the removed scaling these produced -# deadlines of 1800s and 30000s; they must now be identical. -PROFILE = [(200, {'seconds': 600.0}), (300, {'seconds': 10000.0})] - - -def _deadline_of(cluster, end, attempt=1): - return cluster.k8s.job(jm.job_name(int(end), attempt)).spec.active_deadline_seconds - - -def test_the_cheapest_and_costliest_ranges_get_the_same_deadline(cluster, monkeypatch): - """The regression guard. A 600s range and a 10000s range are bounded alike. - - Tightening the cheap one is exactly what killed 134 ranges in the backtest: - its `seconds` came from a neighbour, and the neighbour was wrong. - """ - monkeypatch.setattr(config, 'PROFILE', PROFILE) - monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) - cluster.reconcile() - - assert _deadline_of(cluster, 200) == DEADLINE - assert _deadline_of(cluster, 300) == DEADLINE - - -def test_an_unprofiled_range_gets_the_same_deadline_too(cluster, monkeypatch): - """No profile at all changes nothing -- there is nothing to scale by.""" - monkeypatch.setattr(config, 'PROFILE', []) - monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', DEADLINE) - cluster.reconcile() - - assert _deadline_of(cluster, 300) == DEADLINE - - -def test_zero_disables_the_deadline_entirely(cluster, monkeypatch): - """0 must mean absent, not 0 -- a zero-second deadline kills every attempt - the moment it is created.""" - monkeypatch.setattr(config, 'PROFILE', PROFILE) - monkeypatch.setattr(config, 'ATTEMPT_DEADLINE_SECONDS', 0) - cluster.reconcile() - - assert _deadline_of(cluster, 300) is None diff --git a/src/MissionParallelCatchup/tests/unit/test_dispatch_order.py b/src/MissionParallelCatchup/tests/unit/test_dispatch_order.py deleted file mode 100644 index 31ae7705..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_dispatch_order.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Dispatch order, and why longest-first is the one that shortens a run. - -Makespan is bounded below by the single longest job: every range dispatched -after it is free, and every hour it starts late lands on the end of the run. -""" - -import pytest - -import config -import ranges -import job_monitor as jm - -RANGES = [(600, 420), (500, 420), (400, 420), (300, 420)] # generators emit tip-first - - -def _order(monkeypatch, mode, profile=None): - monkeypatch.setattr(config, 'RANGE_ORDER', mode) - monkeypatch.setattr(config, 'PROFILE', profile) - return [e for e, _ in ranges._ordered(list(RANGES))] - - -def test_tip_first_is_unchanged(monkeypatch): - assert _order(monkeypatch, 'tip-first') == [600, 500, 400, 300] - - -def test_oldest_first_reverses(monkeypatch): - assert _order(monkeypatch, 'oldest-first') == [300, 400, 500, 600] - - -def test_longest_first_sorts_by_measured_seconds_not_position(monkeypatch): - # The whole point: 400 is the expensive one even though 600 is nearer the - # tip. Measured 2026-07-30, ranges at 41-45M ran as long as the tip on a - # third of the memory, so position is a proxy that fails in the tail. - prof = [(300, {'seconds': 10}), (400, {'seconds': 9000}), - (500, {'seconds': 20}), (600, {'seconds': 100})] - assert _order(monkeypatch, 'longest-first', prof) == [400, 600, 500, 300] - - -def test_an_unprofiled_range_sorts_first(monkeypatch): - # profile_for returns the nearest measured end ABOVE the target and None - # past its ceiling, so an unprofiled range is newer than anything ever - # measured -- the most expensive kind. Unknown means assume worst. - prof = [(300, {'seconds': 10}), (400, {'seconds': 9000})] - assert _order(monkeypatch, 'longest-first', prof)[:2] == [600, 500] - - -def test_ties_keep_tip_first_order(monkeypatch): - # Among ranges the profile cannot separate, position is still the better - # guess, so a tie must not scramble them. - prof = [(e, {'seconds': 50}) for e in (300, 400, 500, 600)] - assert _order(monkeypatch, 'longest-first', prof) == [600, 500, 400, 300] - - -def test_no_profile_at_all_falls_back_to_tip_first(monkeypatch): - # A run with no profile has nothing to sort on; every range is "unknown", - # so the tie rule must leave the generator's order intact. - assert _order(monkeypatch, 'longest-first', None) == [600, 500, 400, 300] diff --git a/src/MissionParallelCatchup/tests/unit/test_http_surface.py b/src/MissionParallelCatchup/tests/unit/test_http_surface.py deleted file mode 100644 index 1d30a978..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_http_surface.py +++ /dev/null @@ -1,182 +0,0 @@ -"""The monitor's HTTP surface, driven over a real socket. - -This is the driver's only channel into a run -- profile in, status and logs out --- so it is exercised through an actual server rather than by calling handler -methods, which would not catch a Range header the socket layer mishandles. -""" - -import json -import threading -import urllib.error -import urllib.request -from http.server import ThreadingHTTPServer - -import pytest - -import config -import http_server -import job_monitor as jm - - -@pytest.fixture -def server(tmp_path, monkeypatch): - """A live monitor HTTP surface on a throwaway port and volume.""" - monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(config, 'RUN_PATH', str(tmp_path / 'run.json')) - monkeypatch.setattr(http_server, 'started', threading.Event()) - monkeypatch.setattr(http_server, 'on_start', jm.start_run) - monkeypatch.setattr(http_server, 'status_source', - lambda: (jm.status, jm.status_lock)) - - httpd = ThreadingHTTPServer(('127.0.0.1', 0), http_server.RequestHandler) - threading.Thread(target=httpd.serve_forever, daemon=True).start() - yield f"http://127.0.0.1:{httpd.server_address[1]}", tmp_path - httpd.shutdown() - - -def _get(base, path, headers=None): - req = urllib.request.Request(base + path, headers=headers or {}) - with urllib.request.urlopen(req, timeout=5) as r: - return r.status, r.read(), dict(r.headers) - - -def _post(base, path, body): - req = urllib.request.Request(base + path, data=body.encode(), method='POST') - try: - with urllib.request.urlopen(req, timeout=5) as r: - return r.status, r.read() - except urllib.error.HTTPError as e: - return e.code, e.read() - - -def test_start_rejects_a_bad_config_with_the_reason(server, monkeypatch): - """The whole point of validating here: the driver gets told why. - - Coercing at import made this a crashlooping pod instead, which the driver - could only observe as a 600s timeout on a monitor that never answered. - """ - base, _ = server - monkeypatch.setattr(config, 'LIVENESS_MAX_CONCURRENCY', 'many') - - code, body = _post(base, '/start', json.dumps({"range": {"startingLedger": 0, "latestLedgerNum": 1000, "ledgersPerJob": 100}})) - - assert code == 400 - assert 'LIVENESS_MAX_CONCURRENCY must be an integer' in json.loads(body)['error'] - assert not http_server.started.is_set(), "a rejected config must not open the gate" - - -def test_start_opens_the_gate_and_is_idempotent(server): - """A driver that retries after a timeout must not restart a live run.""" - base, vol = server - - assert _post(base, '/start', json.dumps({"range": {"startingLedger": 0, "latestLedgerNum": 1000, "ledgersPerJob": 100}, "profile": {"ranges": {"300": {"seconds": 1.0}}}}))[0] == 200 - assert http_server.started.is_set() - assert (vol / 'run.json').exists(), "the profile is kept for a restart" - - # A second POST carrying nothing must not wipe the profile already installed. - assert _post(base, '/start', json.dumps({"range": {"startingLedger": 0, "latestLedgerNum": 1000, "ledgersPerJob": 100}}))[0] == 200 - assert config.PROFILE == [(300, {'seconds': 1.0})] - - -def test_status_is_served_from_memory(server): - base, _ = server - code, body, _ = _get(base, '/status') - - assert code == 200 - assert json.loads(body)['num_remain'] == jm.status['num_remain'] - - -def test_logs_manifest_carries_what_a_puller_diffs_on(server): - base, vol = server - (vol / 'range-300-a1.log.gz').write_bytes(b'x' * 1234) - - entries = {e['name']: e for e in json.loads(_get(base, '/logs')[1])} - - assert entries['range-300-a1.log.gz']['size'] == 1234 - assert 'mtime' in entries['range-300-a1.log.gz'] - - -def test_a_file_resumes_from_the_byte_it_stopped_at(server): - """Range is what makes a cut transfer cost the remainder rather than the - whole file -- the truncation that lost 12 ranges their logs on 2026-08-07.""" - base, vol = server - (vol / 'range-300-a1.log.gz').write_bytes(bytes(range(256))) - - whole = _get(base, '/logs/range-300-a1.log.gz') - assert whole[0] == 200 and len(whole[1]) == 256 - - code, body, headers = _get(base, '/logs/range-300-a1.log.gz', - {'Range': 'bytes=200-'}) - assert code == 206 - assert body == bytes(range(200, 256)) - assert headers['Content-Range'] == 'bytes 200-255/256' - - -def test_a_path_outside_the_volume_is_refused(server): - """The route is reachable from outside the cluster once an HTTPRoute is - attached, so the filename is the whole security boundary.""" - base, _ = server - for bad in ('..%2f..%2fetc%2fpasswd', '.hidden', 'sub%2fdir'): - with pytest.raises(urllib.error.HTTPError) as e: - _get(base, '/logs/' + bad) - assert e.value.code == 404 - - -def test_the_collectors_resume_cursor_is_not_offered_for_pulling(server): - """.state is one timestamp rewritten on every poll of a live range. - - It means nothing once the pods are gone, and it changes constantly -- so a - manifest diff would re-fetch one per in-flight range on every pass. The tar - it replaced excluded it deliberately; this keeps that. - """ - base, vol = server - (vol / 'range-300-a1.log.gz').write_bytes(b'kept') - (vol / 'range-300-a1.state').write_text('2026-08-08T21:44:01.867115384Z') - - names = {e['name'] for e in json.loads(_get(base, '/logs')[1])} - - assert 'range-300-a1.log.gz' in names - assert 'range-300-a1.state' not in names - - -def test_a_restart_resumes_without_waiting_for_another_start(server, tmp_path): - """run.json on the volume is what says "this monitor has a run". - - Whoever installs it opens the gate -- a POST, or a restart reading it back. - It did not: the gate lived in the POST handler, so a restarted monitor - blocked on it forever while /status kept answering with its placeholder. - Nothing was unreachable, so the driver polled a dead run indefinitely. - Observed on ssc-test 2026-08-08 with 7 ranges already completed on the - volume and reconcile never running again. - """ - base, vol = server - run = {"range": {"startingLedger": 0, "latestLedgerNum": 1000, - "ledgersPerJob": 100}} - assert _post(base, '/start', json.dumps(run))[0] == 200 - assert (vol / 'run.json').exists() - - # A fresh process: same volume, gate closed again. - http_server.started.clear() - jm.start_run(json.loads((vol / 'run.json').read_text())) - - assert http_server.started.is_set(), ( - "a restart restored the run but never opened the gate, so reconcile " - "would block forever and the run would hang silently") - - -def test_status_says_whether_the_run_has_started(server): - """Placeholder zeros are indistinguishable from a run with nothing done. - - Before the first reconcile pass /status answers with the module defaults, so - a caller cannot tell "nothing recorded yet" from "never going to dispatch". - That ambiguity is what let a wedged monitor be polled indefinitely: it kept - answering 200 and nothing was ever unreachable. - """ - base, _ = server - assert json.loads(_get(base, '/status')[1])['started'] is False - - _post(base, '/start', json.dumps({"range": {"startingLedger": 0, - "latestLedgerNum": 1000, - "ledgersPerJob": 100}})) - - assert json.loads(_get(base, '/status')[1])['started'] is True diff --git a/src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py b/src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py deleted file mode 100644 index d606c565..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_kubelet_sampler.py +++ /dev/null @@ -1,288 +0,0 @@ -"""sample_kubelet: what one /stats/summary payload is allowed to become. - -The sampler is the only source of every memory figure the profile uses. It runs -against a payload this mission does not control, on pods that may be seconds -old, in a process that can be restarted mid-range -- so most of what it does is -refuse to record something. - -Driven through the real `log_collector.sample_kubelet` with a fake session, and -asserted on the module's own peak dicts and on the bytes that reach the volume. -An earlier generation of these tests exec'd the function body out of the source -with a hand-built namespace; the point of that was to survive an unimportable -module, and the module imports. -""" - -import asyncio - -import pytest - -import config -import attempts -import job_monitor as jm -import log_collector as lc - -MIB = 1024 ** 2 - - -@pytest.fixture -def sampler(tmp_path, monkeypatch): - """A collector with no memory, writing to a disposable volume.""" - monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(lc, 'token', lambda: 'tok') - monkeypatch.setattr(config, 'STORAGE_MODE', 'ephemeral') - for name in ('_eph_peak', '_anon_peak', '_ws_peak', '_peak_flushed', - '_streaming', '_pod_secs', '_wake'): - monkeypatch.setattr(lc, name, {}) - return tmp_path - - -class _Resp: - def __init__(self, payload): - self._payload = payload - - async def __aenter__(self): - return self - - async def __aexit__(self, *exc): - return False - - def raise_for_status(self): - pass - - async def json(self): - return self._payload - - -class _Session: - def __init__(self, payload): - self.payload = payload - - def get(self, url, **kw): - return _Resp(self.payload) - - -def container(name=None, rss=None, ws=None): - mem = {} - if rss is not None: - mem['rssBytes'] = rss - if ws is not None: - mem['workingSetBytes'] = ws - return {'name': name or lc.CONTAINER, 'memory': mem} - - -def payload(pod, containers, eph=None): - entry = {'podRef': {'name': pod}, 'containers': containers} - if eph is not None: - entry['ephemeral-storage'] = {'usedBytes': eph} - return {'pods': [entry]} - - -def sample(doc): - asyncio.run(lc.sample_kubelet(_Session(doc), ['node-1'])) - - -# --- a peak is a high-water mark, not the latest reading --------------------- - -@pytest.mark.parametrize('first, second', [(900, 400), (400, 900)]) -def test_the_peak_is_a_high_water_mark_in_either_order(sampler, first, second): - """Catching the spike is the whole point, and download-phase anon - oscillates: the sampler turns a series of readings into one number, so - last-wins defeats every consumer downstream.""" - sample(payload('w-1', [container(rss=first * MIB)], eph=first)) - sample(payload('w-1', [container(rss=second * MIB)], eph=second)) - - assert lc._anon_peak == {'w-1': 900 * MIB} - assert lc._eph_peak == {'w-1': 900} - - -# --- what must not be recorded ----------------------------------------------- - -def test_a_container_without_stats_yet_is_skipped_not_zeroed(sampler): - """rssBytes is absent for the first seconds of a container's life, before - cAdvisor has stats for it. Recording 0 would poison the peak for a range - that is about to be measured properly, and raising would kill the sampler - for every other pod on the node.""" - sample(payload('w-1', [container(rss=None)], eph=7)) - - assert lc._anon_peak == {}, "a missing rssBytes was recorded anyway" - assert lc._eph_peak == {'w-1': 7}, "the disk axis stopped being sampled" - - -def test_only_the_worker_container_is_measured(sampler): - """Sidecars share the pod. Summing across containers, or letting the last - one win, would size the range from whichever one kubelet listed last.""" - sample(payload('w-1', [container(name='istio-proxy', rss=900 * MIB)])) - - assert lc._anon_peak == {} - - -def test_the_worker_is_found_however_kubelet_orders_the_containers(sampler): - sample(payload('w-1', [container(name='istio-proxy', rss=900 * MIB), - container(rss=222 * MIB)])) - - assert lc._anon_peak == {'w-1': 222 * MIB} - - -def test_a_pod_with_no_name_is_skipped_rather_than_keyed_on_none(sampler): - asyncio.run(lc.sample_kubelet(_Session({'pods': [{'podRef': {}, - 'containers': []}]}), - ['node-1'])) - - assert lc._anon_peak == {} and lc._eph_peak == {} - - -def test_an_unreachable_kubelet_costs_the_sample_not_the_sampler(sampler): - """The axis going quiet must not look like "this range used nothing"; the - next node in the list still has to be visited.""" - class _Boom: - def get(self, url, **kw): - raise OSError('connection refused') - - asyncio.run(lc.sample_kubelet(_Boom(), ['node-1'])) # must not raise - - assert lc._anon_peak == {} - - -# --- flushing to the volume --------------------------------------------------- - -def _writes(monkeypatch): - seen = [] - real = lc.write_metrics - - def spy(end, attempt, values): - seen.append((end, attempt, dict(values))) - return real(end, attempt, values) - - monkeypatch.setattr(lc, 'write_metrics', spy) - return seen - - -def test_a_peak_that_barely_grows_is_not_reflushed(sampler, monkeypatch): - """One write per sample per pod, at 2048 pods, would be the dominant cost - of the sampler. Only growth past PEAK_FLUSH_RATIO earns a write.""" - seen = _writes(monkeypatch) - lc._streaming['w-1'] = ('300', '1') - - sample(payload('w-1', [container(rss=900 * MIB)])) - sample(payload('w-1', [container(rss=910 * MIB)])) - assert len(seen) == 1, f"a 1.1% rise triggered a second flush: {seen}" - - sample(payload('w-1', [container(rss=2000 * MIB)])) - assert len(seen) == 2, "a 2.2x rise did not flush" - assert seen[-1][2] == {'peakAnonBytes': 2000 * MIB} - - -def test_an_in_flight_peak_reaches_the_volume_before_the_stream_ends(sampler): - """Prometheus computed max_over_time server-side and needed no state. A - local high-water dict does: without the flush, a collector restart resets a - range's peak to whatever it is using at that moment, which under-reports and - sizes the next run too small.""" - lc._streaming['w-1'] = ('300', '1') - sample(payload('w-1', [container(rss=900 * MIB)])) - - assert attempts.peaks_for_range('300', 1) == {'peakAnonBytes': 900 * MIB} - - -def test_a_peak_sampled_before_stream_registration_is_flushed_on_open(sampler): - """main samples first, then opens new pollers; a restart between those steps - must not make that first high-water process-memory-only.""" - sample(payload('w-1', [container(rss=900 * MIB, ws=1200 * MIB)])) - assert attempts.peaks_for_range('300', 1) == {} - - lc._register_stream('w-1', '300', '1') - - assert attempts.peaks_for_range('300', 1) == { - 'peakAnonBytes': 900 * MIB, - 'peakWorkingSetBytes': 1200 * MIB, - } - - -def test_the_disk_axis_stays_mode_gated(sampler, monkeypatch): - """ephemeral-storage is meaningless in pvc mode: /data is on the volume, - not on the node.""" - monkeypatch.setattr(config, 'STORAGE_MODE', 'pvc') - lc._streaming['w-1'] = ('300', '1') - - sample(payload('w-1', [container(rss=900 * MIB)], eph=34 * 1024 ** 3)) - - assert lc._eph_peak == {} - assert lc._anon_peak == {'w-1': 900 * MIB}, \ - "memory sizing is not mode-specific and must be sampled in both" - - -# --- working set: sampled, recorded, never used to size anything ------------- - -def test_working_set_is_sampled_alongside_anon(sampler): - """It is what kubelet ranks node-pressure evictions on, so it explains an - eviction that rss cannot. Measured on ssc-test for one 420-ledger range: - working set read 3.61 / 7.48 / 13.49 GiB under 4Gi / 8Gi / 24000Mi limits - while rss held flat at ~2.4 GiB -- which is exactly why it is a diagnostic - and never a request.""" - sample(payload('w-1', [container(rss=900 * MIB, ws=4096 * MIB)])) - - assert lc._ws_peak == {'w-1': 4096 * MIB} - assert lc._anon_peak == {'w-1': 900 * MIB} - - -def test_finalize_records_the_working_set_peak(sampler): - """Sampling it is useless if finalize drops it on the floor.""" - sample(payload('w-1', [container(rss=900 * MIB, ws=4096 * MIB)])) - asyncio.run(lc.finalize(None, 'w-1', '300', 1, lc.TxApplyScanner(), - lambda p: True)) - - stored = attempts.peaks_for_range('300', 1) - assert stored['peakWorkingSetBytes'] == 4096 * MIB - assert stored['peakAnonBytes'] == 900 * MIB - - -# --- resume is bookkeeping finalize has to carry ------------------------------ - -def test_finalize_records_that_an_attempt_resumed(sampler): - """Without this in .metrics, peaks_for_range cannot tell a resumed tail - from a complete pass, and every resumed range is profiled off its tail.""" - tx = lc.TxApplyScanner() - tx.feed("RESUME: 300/16320 reached ledger 299, replay had started") - asyncio.run(lc.finalize(None, 'w-1', '300', 1, tx, lambda p: True)) - - assert attempts._attempt_resumed('300', 1) is True - - -def test_a_fresh_attempt_is_never_marked_resumed(sampler): - asyncio.run(lc.finalize(None, 'w-1', '300', 1, lc.TxApplyScanner(), - lambda p: True)) - - assert attempts._attempt_resumed('300', 1) is False - - -# --- which endpoint, and why it matters -------------------------------------- - -def test_the_sampler_goes_straight_to_the_kubelet_not_the_apiserver_proxy(sampler): - """The endpoint choice IS the privilege boundary. - - Reaching kubelet through `/api/v1/nodes//proxy/...` requires the - `nodes/proxy` subresource, which authorizes GET on EVERY kubelet path -- - /pods and /containerLogs among them, for any namespace scheduled on that - node. The kubelet maps /stats/* to its own `nodes/stats` subresource, so - talking to it directly is the same payload under a grant that cannot read - pod inventory or logs at all. - - Reverting to the proxy would 403 against the deployed RBAC rather than - quietly widening it, but the intent should fail loudly here first. - """ - seen = [] - - class _Recording(_Session): - def get(self, url, **kw): - seen.append((url, kw)) - return _Resp(self.payload) - - asyncio.run(lc.sample_kubelet(_Recording(payload('w-1', [container(rss=1)])), - ['10.1.2.3'])) - - (url, kw), = seen - assert url == f"https://10.1.2.3:{lc.KUBELET_PORT}/stats/summary" - assert '/proxy/' not in url, "back on the apiserver node proxy" - assert 'nodes' not in url, "addressing a Node object rather than the kubelet" - # EKS kubelet serving certs are self-signed, not issued by the cluster CA - # the session's context trusts. - assert kw.get('ssl') is False diff --git a/src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py b/src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py deleted file mode 100644 index 9998a145..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_monitor_verdict_records.py +++ /dev/null @@ -1,313 +0,0 @@ -"""What the monitor writes down about an attempt it is about to throw away. - -Things only this process can record, each with a window that closes the moment -the Job is reaped: - - .outcome why the attempt failed, and how long it ran - .log.gz the backstop archive, for a range the collector never claimed - the log line the one place a condemned range explains itself - progress.json the record that makes the volume and the Job disposable - -Driven through the real reconcile against the fake cluster, because the window -is the point: each of these has to happen on the pass that classifies the -failure, while the pod object is still there. -""" - -import gzip -import json -import logging -import os - -import pytest - -import config -import units -import records -import attempts -import job_monitor as jm - - -# --- a failed attempt's duration ---------------------------------------------- - -def test_a_failed_attempts_duration_is_persisted_with_its_verdict(cluster): - """The only moment it is available. - - reconcile computes `seconds` solely on the success path, and the pod is - about to be reaped -- so without this a resumed chain can only ever report - its final leg, and every attempt lost to a spot eviction drops out of the - range's compute total. - """ - cluster.reconcile() - cluster.advance(300, 'incomplete') - cluster.reconcile() - - outcome = records.read_outcome('300', 1) - assert outcome['outcome'] == 'failed' - assert outcome['attemptSeconds'] == pytest.approx(60.0, abs=5.0), outcome - - -def test_that_duration_is_what_the_chain_adds_up(cluster): - """The consumer, not just the file: a range that resumes must report the - compute of every leg, and the earlier legs exist only as .outcome.""" - cluster.reconcile() - cluster.advance(300, 'incomplete') - cluster.finalize(300, 1) - cluster.reconcile() - cluster.finalize(300, 2, resumed=True) - - assert attempts.seconds_for_range('300', 2, 300.0) == pytest.approx(360.0, abs=5.0) - - -def test_a_verdict_already_on_the_volume_is_not_rewritten(cluster): - """The collector writes this file too, from the pod, while it still exists. - Its verdict is the one taken with the best evidence and must win.""" - cluster.reconcile() - cluster.write(records.outcome_path('300', 1), - '{"outcome": "disrupted", "exitCode": null, "pod": "w-300", ' - '"attemptSeconds": 1800.0}') - cluster.advance(300, 'incomplete') - cluster.reconcile() - - # The pod exited 3, which reads as a plain catchup failure. The collector - # saw the eviction that caused it, so its verdict -- and its duration -- - # stand. - assert records.read_outcome('300', 1)['outcome'] == 'disrupted' - assert records.read_outcome('300', 1)['attemptSeconds'] == 1800.0 - - -# --- the condemned range has to say so ---------------------------------------- - -def test_a_condemned_range_is_logged_loudly(cluster, caplog): - """The zero-retry path used to log nothing at all: the range appeared under - failed{} and the mission aborted with no line saying why. A condemnation - fails a ten-hour run, so it is the one verdict that must be impossible to - miss in the monitor's own log -- which is the log the mission collects.""" - cluster.reconcile() - cluster.advance(300, 'condemned') - with caplog.at_level(logging.ERROR, logger=jm.logger.name): - cluster.reconcile() - - condemned = [r for r in caplog.records if 'RANGE CONDEMNED' in r.getMessage()] - assert condemned, [r.getMessage() for r in caplog.records] - said = condemned[0].getMessage() - assert '300' in said and 'failed' in said, said - assert '300' in cluster.failed() - - -def test_an_exhausted_range_says_which_budget_it_spent(cluster, caplog): - """The other way a range ends: it was retryable and ran out. That is a - different operator action from a condemnation, so it reads differently.""" - cluster.reconcile() - # An OOM: the only cause that spends the range budget now, since a "did not - # complete" is either a fetch fault (the cluster's problem) or a real - # failure (condemned outright). - for attempt in range(1, config.ATTEMPT_BUDGETS['oom'] + 1): - cluster.advance(300, 'oom', attempt=attempt) - with caplog.at_level(logging.ERROR, logger=jm.logger.name): - cluster.reconcile() - - exhausted = [r.getMessage() for r in caplog.records - if 'exhausted' in r.getMessage()] - assert exhausted, [r.getMessage() for r in caplog.records] - assert '300' in cluster.failed() - - -# --- the backstop archive ------------------------------------------------------ - -def test_the_backstop_saves_a_log_the_collector_never_claimed(cluster): - """Last resort for a pod that lived and died entirely while the collector - was down. The pod is about to be reaped, so this is the last read of it.""" - cluster.reconcile() - pod = cluster.k8s.pod_for_job(cluster.job_name(300, 1)) - cluster.k8s.set_pod_log(pod.metadata.name, - "metric 'ledger.transaction.apply'\n" - " sum = 1500.0ms\n") - cluster.advance(300, 'incomplete') - cluster.reconcile() - - path = records.log_path('300', 1) - assert os.path.exists(path), "a failed attempt left no archive at all" - with gzip.open(path, 'rt') as fh: - assert 'sum = 1500.0ms' in fh.read() - # The archive is the evidence the backstop exists to preserve. The metric - # is the collector's to record, and it never ran for this range. - - -def test_the_backstop_stands_down_for_a_range_the_collector_claimed(cluster): - """Two writers appending to one gzip interleave members and duplicate - lines. The collector's .state file is the claim, written the moment it - opens a poller -- empty or not.""" - cluster.reconcile() - cluster.write(records.state_path('300', 1), '') - cluster.advance(300, 'incomplete') - cluster.reconcile() - - assert not os.path.exists(records.log_path('300', 1)), \ - "the monitor wrote over an archive the collector had claimed" - - -def test_a_torn_backstop_archive_is_never_left_behind(cluster, monkeypatch): - """job_monitor reads this same file back to recover txApplySeconds, and - gzip raises on a truncated member. A half-written archive would cost the - metric permanently, so the write goes through .tmp and a rename.""" - cluster.reconcile() - real_replace = jm.os.replace - monkeypatch.setattr(jm.os, 'replace', - lambda *a, **kw: (_ for _ in ()).throw(OSError(28, 'ENOSPC')) - if str(a[1]).endswith('.log.gz') else real_replace(*a, **kw)) - - pod = cluster.k8s.pod_for_job(cluster.job_name(300, 1)) - assert jm.backstop_save_pod_log(pod.metadata.name, '300', 1) is False - - assert not os.path.exists(records.log_path('300', 1)) - assert attempts._tx_apply_for_attempt('300', 1) is None - - -# --- the progress record -------------------------------------------------------- - -def test_the_progress_record_is_replaced_whole_or_not_at_all(cluster, monkeypatch): - """The mission driver reads progress.json off the volume while the monitor - is still writing it, and a partial file is unparseable JSON -- which reads - as "nothing has been done" and makes every recorded range eligible again. - - Written to a .tmp and renamed, so a write that dies leaves the previous - record exactly as it was. - """ - cluster.reconcile() - cluster.advance(300, 'succeeded') - cluster.finalize(300, 1, tx_apply=1.5, peaks={'peakAnonBytes': 7}) - cluster.reconcile() - before = json.load(open(config.PROGRESS_FILE)) - assert '300' in before['completed'] - - real_open = open - - class _HalfWrite: - def __init__(self, path): - self.fh = real_open(path, 'w') - - def __enter__(self): - return self - - def __exit__(self, *exc): - self.fh.close() - return False - - def write(self, blob): - self.fh.write(blob[:len(blob) // 2]) - raise OSError(28, 'No space left on device') - - armed = {'v': True} - - def half_open(path, mode='r', *a, **kw): - if armed['v'] and mode in ('w', 'wt') and str(path).endswith('.json.tmp'): - return _HalfWrite(path) - return real_open(path, mode, *a, **kw) - - monkeypatch.setattr(records, 'open', half_open, raising=False) - cluster.advance(200, 'succeeded') - cluster.finalize(200, 1, tx_apply=2.5, peaks={'peakAnonBytes': 9}) - with pytest.raises(OSError): - cluster.reconcile() - armed['v'] = False - - # Not truncated, not empty, and not half of two records spliced together. - assert json.load(open(config.PROGRESS_FILE)) == before - assert jm.load_progress()['completed']['300']['peakAnonBytes'] == 7 - - -# --- which classifier wins ---------------------------------------------------- -# Two independent sources, and the pod is not simply preferred: a deadline kill -# sends SIGTERM, stellar-core drains and exits 3, so the pod reads a plain -# `failed` that would CONDEMN a range which merely ran long. Only the Job knows -# the deadline fired. Where the pod named a mechanism it wins instead, because -# "ran too long" is also true of an OOM or an eviction and choosing it loses both -# the remediation and the retry budget. - -def _verdict(end=300, attempt=1): - return open(records.verdict_path(end, attempt)).read().strip() - - -@pytest.mark.parametrize('outcome', [ - 'timeout', # pod exit 3, Job DeadlineExceeded: the Job condition wins - 'unknown', # pod deleted, Job has no condition: retry rather than condemn -]) -def test_the_verdict_recorded_is_the_one_the_sources_agree_on(cluster, outcome): - cluster.reconcile() - cluster.advance(300, outcome) - cluster.reconcile() - - assert _verdict() == outcome - - -@pytest.mark.parametrize('outcome', ['oom', 'disrupted']) -def test_what_the_pod_says_beats_a_job_deadline(cluster, outcome): - """The escalation ladder needs the mechanism, and a timeout verdict is - terminal where an oom is retried with more memory.""" - cluster.reconcile() - name = cluster.advance(300, outcome) - # The same attempt also tripped its deadline: the Job condition says so. - cluster.k8s.set_job_failed(name, reason='DeadlineExceeded', - message='Job was active longer than specified deadline') - cluster.reconcile() - - assert _verdict() == outcome - - -def test_a_disrupted_range_escalates_disk_one_rung_on_its_first_eviction(cluster, monkeypatch): - """The size of the escalation, as reconcile actually builds it. - - Counting attempts instead of evictions handed a range disrupted four times a - 1.5^5 = 7.6x disk request for a single eviction. Asserted on the retry Job's - own spec rather than on the helpers, because the helpers were already right - -- it was the call site that passed the wrong index. - """ - monkeypatch.setattr(config, 'STORAGE_MODE', 'ephemeral') - monkeypatch.setattr(config, 'REQ_EPHEMERAL', '4Gi') - monkeypatch.setattr(config, 'LIM_EPHEMERAL', '4Gi') - monkeypatch.setattr(config, 'EPH_BUMP_FACTOR', 1.5) - - cluster.reconcile() - for _ in range(4): - cluster.advance(300, 'disrupted') - cluster.reconcile() - assert cluster.attempt_of(300) == 5, "four disruptions, four retries" - - cluster.advance(300, 'ephemeral') - cluster.reconcile() - - retry = cluster.k8s.job(cluster.job_name(300)) - got = retry.spec.template.spec.containers[0].resources.limits['ephemeral-storage'] - assert units.quantity_bytes(got) == units.quantity_bytes('6Gi'), ( - f"first eviction must climb one rung to 6Gi, got {got}") - - -def test_an_exhausted_oom_reports_the_memory_the_attempt_actually_had(cluster, caplog, - monkeypatch): - """`reason` only surfaces when the budget runs out, so exhaust it. - - Four disruptions then one OOM, so the attempt index (5) and the OOM count (1) - DIVERGE -- which is the whole bug. The range ran at the 9Gi base, and - indexing the report on `attempt` claimed 9Gi * 1.5^4 = 45Gi instead. - - Disruptions spend their own budget, so an OOM budget of 1 is - exhausted by the single OOM and nothing else. - """ - monkeypatch.setattr(config, 'POOL_PREFIX', '') - monkeypatch.setattr(config, 'REQ_MEM', '9Gi') - monkeypatch.setattr(config, 'MEM_BUMP_FACTOR', 1.5) - monkeypatch.setitem(config.ATTEMPT_BUDGETS, 'oom', 1) - - cluster.reconcile() - for _ in range(4): - cluster.advance(300, 'disrupted') - cluster.reconcile() - assert cluster.attempt_of(300) == 5, "four disruptions, four retries" - - cluster.advance(300, 'oom') - with caplog.at_level(logging.ERROR, logger=jm.logger.name): - cluster.reconcile() - - line = next(r.getMessage() for r in caplog.records if 'exhausted' in r.getMessage()) - reported = line.split('memory request ')[1].rstrip(')') - assert units.quantity_bytes(reported) == units.quantity_bytes('9Gi'), line diff --git a/src/MissionParallelCatchup/tests/unit/test_node_targeting.py b/src/MissionParallelCatchup/tests/unit/test_node_targeting.py deleted file mode 100644 index 9667b261..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_node_targeting.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Node affinity and tolerations that build_job puts on a worker pod. - -The mission exposes three node-targeting flags. Two of them survived the -rewrite from a StatefulSet template to API-created Jobs; avoidNodeLabels did -not, and the gap was silent -- the driver sent the value, values.yaml declared -it, and no template read it, so a run started with --pubnet-parallel-catchup- -avoid-node-labels scheduled workers onto exactly the nodes it named. -""" - -import importlib - -import pytest - -import config -import job_monitor as jm - - -def _match_expressions(monkeypatch, **env): - """build_job's node-affinity expressions under a given env. - - Takes the `cluster` fixture because build_job calls ensure_pvc, which is a - real API call -- the fixture is what puts the fake cluster behind it. - """ - for k in ('NODE_LABEL_KEY', 'NODE_LABEL_VALUE', - 'AVOID_NODE_LABEL_KEY', 'AVOID_NODE_LABEL_VALUE'): - monkeypatch.setattr(config, k, env.get(k, '')) - job = jm.build_job(300, 420, 1, None) - aff = job.spec.template.spec.affinity - if aff is None: - return None - terms = aff.node_affinity.required_during_scheduling_ignored_during_execution - return terms.node_selector_terms[0].match_expressions - - -def test_no_targeting_leaves_the_pod_unconstrained(cluster, monkeypatch): - assert _match_expressions(monkeypatch) is None - - -def test_require_alone_pins_the_pod_to_the_label(cluster, monkeypatch): - exprs = _match_expressions(monkeypatch, - NODE_LABEL_KEY='purpose', NODE_LABEL_VALUE='catchup-spot') - assert [(e.key, e.operator, e.values) for e in exprs] == [ - ('purpose', 'In', ['catchup-spot'])] - - -def test_avoid_alone_keeps_the_pod_off_the_label(cluster, monkeypatch): - exprs = _match_expressions(monkeypatch, - AVOID_NODE_LABEL_KEY='purpose', - AVOID_NODE_LABEL_VALUE='catchup-od') - assert [(e.key, e.operator, e.values) for e in exprs] == [ - ('purpose', 'NotIn', ['catchup-od'])] - - -def test_avoid_without_a_value_means_the_label_must_be_absent(cluster, monkeypatch): - # NotIn [""] would only exclude the empty value, which is not what "avoid - # this label" means; the mission sends operator DoesNotExist for this case. - exprs = _match_expressions(monkeypatch, AVOID_NODE_LABEL_KEY='reserved') - assert [(e.key, e.operator) for e in exprs] == [('reserved', 'DoesNotExist')] - assert not exprs[0].values - - -def test_require_and_avoid_share_one_term_so_they_are_anded(cluster, monkeypatch): - # Expressions inside a term are ANDed; separate terms are ORed. Split across - # two terms, a pod that failed the require would still match on the avoid. - exprs = _match_expressions(monkeypatch, - NODE_LABEL_KEY='purpose', NODE_LABEL_VALUE='catchup-spot', - AVOID_NODE_LABEL_KEY='reserved') - assert [(e.key, e.operator) for e in exprs] == [ - ('purpose', 'In'), ('reserved', 'DoesNotExist')] diff --git a/src/MissionParallelCatchup/tests/unit/test_poll_lifecycle.py b/src/MissionParallelCatchup/tests/unit/test_poll_lifecycle.py deleted file mode 100644 index 7448b49f..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_poll_lifecycle.py +++ /dev/null @@ -1,276 +0,0 @@ -"""poll_pod / _poll_once: when a read ends an attempt and when it does not. - -Every one of these is executed rather than pattern-matched. That is not a -stylistic preference: an earlier generation of these tests asserted on an -`except ClientResponseError` branch that raise_for_status could never reach and -passed green against dead code, and another pinned the literal -`gzip.open(..., 'at')` and went red over the atomic-append fix, which preserved -everything the test existed to protect. - -The fake apiserver here answers the log endpoint only. What is asserted is what -lands on the shared volume -- the archive, .metrics, .done -- because that is -the entire interface the monitor sees. -""" - -import asyncio -import gzip -import os - -import pytest - -import config -import attempts -import job_monitor as jm -import log_collector as lc - - -@pytest.fixture -def volume(tmp_path, monkeypatch): - monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(lc, 'token', lambda: 'tok') - monkeypatch.setattr(lc, 'LOG_POLL_SECONDS', 0.02) - monkeypatch.setattr(lc, 'TERMINAL_POLL_ATTEMPTS', 2) - for name in ('_eph_peak', '_anon_peak', '_ws_peak', '_peak_flushed', - '_streaming', '_pod_secs', '_wake'): - monkeypatch.setattr(lc, name, {}) - return tmp_path - - -class _Resp: - def __init__(self, status, body='', after_read=None): - self.status = status - self._body = body.encode() - self._after_read = after_read - - async def __aenter__(self): - return self - - async def __aexit__(self, *exc): - return False - - def raise_for_status(self): - if self.status >= 400: - raise RuntimeError(f"HTTP {self.status}") - - @property - def content(self): - data, after = self._body, self._after_read - - class _Chunks: - async def iter_chunked(self, n): - for i in range(0, len(data), n): - yield data[i:i + n] - if after is not None: - after() - - return _Chunks() - - -class Apiserver: - """Answers each log GET from `answers`, repeating the last one forever.""" - - def __init__(self, *answers): - self.answers = list(answers) - self.params = [] - - def get(self, url, params=None, headers=None): - self.params.append(dict(params or {})) - i = min(len(self.params) - 1, len(self.answers) - 1) - return self.answers[i] - - -def archive(end='300', attempt='1'): - path = lc.base(end, attempt) + '.log.gz' - if not os.path.exists(path): - return '' - with gzip.open(path, 'rt') as fh: - return fh.read() - - -def drive(session, terminal, timeout=3): - async def go(): - await asyncio.wait_for( - lc.poll_pod(session, 'w-1', '300', '1', - lambda p: terminal(), lambda p: False), - timeout=timeout) - asyncio.run(go()) - - -# --- the pod object is gone --------------------------------------------------- - -def test_a_404_finalizes_what_was_already_streamed(volume): - """The pod object is gone, but the bytes already read still owe a tx_apply, - and .done is what lets the monitor stop waiting on the Job. - - This path used to not exist: a pod deleted while Running left its stream - retrying for the rest of the run, holding a connection slot.""" - body = ("2026-07-30T00:00:01Z metric 'ledger.transaction.apply'\n" - "2026-07-30T00:00:02Z sum = 1500.0ms\n") - drive(Apiserver(_Resp(200, body), _Resp(404)), lambda: False) - - assert jm._attempt_finalized('300', 1), "a vanished pod never finalized" - assert attempts.tx_apply_for_range('300', 1) == pytest.approx(1.5) - assert 'sum = 1500.0ms' in archive() - - -def test_an_interrupted_read_on_a_live_pod_does_not_finalize(volume): - """Still running, so retrying is correct. Finalizing here writes a - truncated peak and leaves the range looking measured when it is not.""" - with pytest.raises(asyncio.TimeoutError): - drive(Apiserver(_Resp(500)), lambda: False, timeout=0.4) - - assert not jm._attempt_finalized('300', 1), \ - "a live pod's attempt was closed out on a transient read failure" - - -def test_a_terminal_pod_whose_polls_keep_failing_still_finalizes(volume): - """The other side of it: the container has exited and its log is not - coming back, so the loop has to decide to stop asking rather than spin on a - dead pod and never write its metrics.""" - drive(Apiserver(_Resp(500)), lambda: True) - - assert jm._attempt_finalized('300', 1) - - -# --- the read that catches the last lines ------------------------------------ - -def test_terminal_is_sampled_before_the_poll_not_after(volume): - """A pod that exits mid-poll must still get one more read. - - If `done()` were consulted after the poll instead of before it, the poll - that was in flight when the container exited would be treated as the final - one -- and everything the container wrote on its way out, which is where - the medida block lives, is dropped. - """ - state = {'terminal': False} - first = _Resp(200, "2026-07-30T00:00:01Z catchup ledger 42000000\n", - after_read=lambda: state.update(terminal=True)) - last = _Resp(200, - "2026-07-30T00:00:09Z metric 'ledger.transaction.apply'\n" - "2026-07-30T00:00:10Z sum = 1500.0ms\n") - - drive(Apiserver(first, last), lambda: state['terminal']) - - assert 'catchup ledger 42000000' in archive() - assert 'sum = 1500.0ms' in archive(), \ - "the read after the pod went terminal never happened" - assert attempts.tx_apply_for_range('300', 1) == pytest.approx(1.5) - - -# --- resuming a read ---------------------------------------------------------- - -def test_a_poll_resumes_from_the_last_durable_timestamp(volume): - """Without sinceTime a reconnect re-reads the whole log from the start: - one full re-read per pod per reconnect, at 2096 pods.""" - api = Apiserver(_Resp(200, "2026-07-30T00:00:05Z line\n")) - scanner = lc.TxApplyScanner() - last, gone = asyncio.run(lc._poll_once(api, 'w-1', '300', '1', None, scanner)) - - assert api.params[0].get('sinceTime') is None - assert last == '2026-07-30T00:00:05Z' and gone is False - - asyncio.run(lc._poll_once(api, 'w-1', '300', '1', last, scanner)) - assert api.params[1]['sinceTime'] == '2026-07-30T00:00:05Z', \ - "the second poll did not resume where the first stopped" - - -def test_the_second_granularity_overlap_is_deduped_exactly(volume): - """sinceTime only accepts whole seconds, so a resume deliberately re-reads - the second it stopped in. Every line carries a nanosecond timestamp, so the - overlap is removed per line rather than tolerated as duplicates.""" - body = ("2026-07-30T00:00:05.100000000Z already seen\n" - "2026-07-30T00:00:05.900000000Z brand new\n") - api = Apiserver(_Resp(200, body)) - asyncio.run(lc._poll_once(api, 'w-1', '300', '1', - '2026-07-30T00:00:05.100000000Z', - lc.TxApplyScanner())) - - written = archive() - assert 'brand new' in written - assert 'already seen' not in written - - -def test_untimestamped_kubelet_text_is_kept_but_never_resumed_from(volume): - """"unable to retrieve container logs for containerd://..." partitions to - "unable", and sinceTime=unableZ is a 400 on every later request for that - pod, forever.""" - api = Apiserver(_Resp(200, "unable to retrieve container logs for " - "containerd://9f2c1a\n")) - last, _ = asyncio.run(lc._poll_once(api, 'w-1', '300', '1', None, - lc.TxApplyScanner())) - - assert last is None, f"junk became a resume point: {last!r}" - assert 'unable to retrieve' in archive(), "the line was dropped instead" - - -# --- bounds ------------------------------------------------------------------- - -def test_an_unterminated_blob_is_capped_not_buffered_forever(volume, monkeypatch): - """A meter that never emits a newline would otherwise grow the buffer until - the collector OOMs -- 2096 streams doing it at once.""" - monkeypatch.setattr(lc, 'MAX_POLL_CHARS', 1024) - api = Apiserver(_Resp(200, 'x' * (4 * 1024 * 1024))) - - asyncio.run(lc._poll_once(api, 'w-1', '300', '1', None, lc.TxApplyScanner())) - - # One chunk's worth of overshoot is inherent -- the cap is checked between - # chunks -- but the 4 MiB body must not have been buffered whole. - assert len(archive()) < 256 * 1024, "the poll buffered the entire blob" - - -def test_polls_are_bounded_by_a_semaphore(volume, monkeypatch): - """The whole point of polling over follow=true: concurrency is a tuning - parameter, not a function of how many pods exist.""" - live = {'now': 0, 'max': 0} - - class _Counting(Apiserver): - def get(self, url, params=None, headers=None): - live['now'] += 1 - live['max'] = max(live['max'], live['now']) - resp = super().get(url, params, headers) - live['now'] -= 1 - return resp - - async def go(): - monkeypatch.setattr(lc, '_poll_slots', asyncio.Semaphore(2)) - api = _Counting(_Resp(200, "2026-07-30T00:00:01Z line\n")) - await asyncio.gather(*[ - lc._poll_once(api, 'w-1', '300', str(n), None, lc.TxApplyScanner()) - for n in range(8)]) - - asyncio.run(go()) - assert live['max'] <= 2, f"{live['max']} polls were in flight at once" - - -# --- the allowlist that decides a pod is worth polling at all ----------------- - -def test_only_phases_whose_log_endpoint_can_answer_are_pollable(): - """An allowlist, not "skip Pending". A container that has not started - answers 400 "waiting to start" -- 60 of 88 poll failures right after the - polling switch -- and Unknown means the node stopped reporting, so that - poll cannot succeed either. The terminal phases stay in: a terminal pod is - where the final output lives. - """ - assert set(lc.POLLABLE_PHASES) == {'Running', 'Succeeded', 'Failed'} - - -def test_the_terminal_retry_budget_can_absorb_a_transient_failure(): - """At 1 a single 500 ends the attempt on whatever had been read.""" - assert lc.TERMINAL_POLL_ATTEMPTS >= 2 - - -def test_poll_concurrency_is_a_modest_default(): - """It sizes the connection pool as well (MAX_CONCURRENT_POLLS + 64), so - both directions cost: too low starves the retries, too high recreates the - per-pod connection load polling exists to remove.""" - assert 16 <= lc.MAX_CONCURRENT_POLLS <= 256 - - -def test_the_wake_entry_is_dropped_when_the_attempt_finishes(volume): - """One _wake entry per pod, and pods are per range per attempt: 3979 ranges - plus their retries would otherwise accumulate for the life of the run.""" - drive(Apiserver(_Resp(404)), lambda: True) - - assert jm._attempt_finalized('300', 1) - assert lc._wake == {}, f"the poller's Event outlived its attempt: {lc._wake}" diff --git a/src/MissionParallelCatchup/tests/unit/test_pool_tiers.py b/src/MissionParallelCatchup/tests/unit/test_pool_tiers.py deleted file mode 100644 index af0ad861..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_pool_tiers.py +++ /dev/null @@ -1,604 +0,0 @@ -"""Nodepool routing: memory picks the pool, and the pool is the whole sizing. - -Replaces the cpu-ladder tests. The ladder tuned cpu REQUESTS, which measurement -showed were not buying throughput -- replay draws ~1.05 cores whatever it is -given, and is flat in core count from 2 upward (+2.8% at 2->4, +1.5% at 4->8, -against +16% for AMD-over-Intel at fixed cores). What a request actually bought -was neighbours-per-node. Memory is the dimension that FAILS rather than slows: -a working set that does not fit is OOMKilled. -""" - -import pytest - -import config -import records -import sizing -import job_monitor as jm - -GiB = 1024 ** 3 -TIERS = '0:subdwarf,0.79:dwarf,1.61:subgiant,3.87:giant,8.85:supergiant,18.38:hypergiant,:supernova' -CPU = ('subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:3.80') -# vCPU of the smallest shape in each tier's pool. hypergiant, protostar and -# supernova used to sit on x8i, which put their smallest shape below the rest of -# the tier -- hypergiant read 4 off an x8i.xlarge at w80 while Karpenter served -# 8-vCPU 2xlarges from w100. The x8i spot pools were removed on 2026-08-04, so -# each tier's smallest shape is now also its top-weighted one. -VCPU = ('subdwarf:2,dwarf:2,subgiant:4,giant:4,supergiant:4,hypergiant:8,supernova:16,protostar:8,nebula:8') -# 50% of each tier node's ALLOCATABLE, which is what both isolates the pod and -# lets it schedule -- see test_the_request_is_half_the_node. -MEM = ('subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:14336Mi') - - -@pytest.fixture -def pooled(monkeypatch): - monkeypatch.setattr(config, 'POOL_TIERS', TIERS) - monkeypatch.setattr(config, 'POOL_CPU', CPU) - monkeypatch.setattr(config, 'POOL_VCPU', VCPU) - monkeypatch.setattr(config, 'POOL_PREFIX', 'catchup') - monkeypatch.setattr(config, 'POOL_UNPROFILED', 'protostar') - monkeypatch.setattr(config, 'POOL_NO_PROFILE', 'nebula') - monkeypatch.setattr(config, 'POOL_MEM', MEM) - - -def _profile(entries): - """entries: {end: {...}} -> the sorted (end, rec) list PROFILE holds.""" - return sorted(entries.items()) - - -def _profiled(monkeypatch, entries): - monkeypatch.setattr(config, 'PROFILE', _profile(entries)) - - -def _pool(monkeypatch, anon, ws=None, end=10, **kw): - """Route one range measured at these peaks.""" - peaks = {'peakAnonBytes': int(anon)} - if ws is not None: - peaks['peakWorkingSetBytes'] = int(ws) - _profiled(monkeypatch, {end: peaks}) - return sizing.pool_for(end, **kw) - - -def test_a_range_lands_in_the_tier_its_working_set_fits(pooled, monkeypatch): - _profiled(monkeypatch, { - 100: {'peakAnonBytes': int(0.20 * GiB)}, - 200: {'peakAnonBytes': int(1.50 * GiB)}, - 300: {'peakAnonBytes': int(3.00 * GiB)}, - 400: {'peakAnonBytes': int(7.00 * GiB)}, - 500: {'peakAnonBytes': int(16.0 * GiB)}, - 600: {'peakAnonBytes': int(40.0 * GiB)}, - }) - # subdwarf's cut is 0, so even the smallest range falls through to dwarf - assert sizing.pool_for(100) == 'dwarf' - assert sizing.pool_for(200) == 'subgiant' - assert sizing.pool_for(300) == 'giant' - assert sizing.pool_for(400) == 'supergiant' - assert sizing.pool_for(500) == 'hypergiant' - assert sizing.pool_for(600) == 'supernova' - - -def test_nothing_is_ever_routed_to_subdwarf(pooled, monkeypatch): - """Its cut is 0, and no range can satisfy `gib < 0`. - - The tier stays defined and provisionable -- the pools exist -- but c8a.medium - has 1.42Gi allocatable, and after daemonsets that cannot hold any range the - profile actually contains. Emptying it by cut rather than deleting it keeps - the bottom of the ladder available to experiment with. - """ - _profiled(monkeypatch, { - 10: {'peakAnonBytes': 1}, # 1 byte - 20: {'peakAnonBytes': int(0.27 * GiB)}, # the profile's true minimum - }) - assert sizing.pool_for(10) == 'dwarf' - assert sizing.pool_for(20) == 'dwarf' - - -def test_the_cut_is_exclusive_so_a_range_never_lands_on_a_node_it_fills(pooled, monkeypatch): - """A range exactly AT a cut belongs in the tier above. - - The cut is node_usable/1.60, so a range sitting on it would have exactly the - p99 margin and nothing more. Being one byte over must move it up, not leave - it to be the range that proves the margin was too thin. - """ - _profiled(monkeypatch, { - 10: {'peakAnonBytes': int(0.7899 * GiB)}, - 20: {'peakAnonBytes': int(0.7901 * GiB)}, - }) - assert sizing.pool_for(10) == 'dwarf' - assert sizing.pool_for(20) == 'subgiant' - # The comparison is `<`, so a range sitting exactly on a cut goes UP. Not - # asserted at the exact byte: 0.79 GiB is not representable, and pinning the - # test to a float's rounding would make it about IEEE754 rather than about - # which side of the boundary a range belongs on. - - -def test_a_range_past_the_top_of_the_profile_goes_to_protostar(pooled, monkeypatch): - """Unprofiled means NEWEST, and the newest ledgers are the densest. - - profile_for returns the nearest measured end ABOVE the target, so falling - off the end means this range is newer than anything ever measured. It gets a - rich pool rather than an average one. - """ - _profiled(monkeypatch, {10: {'peakAnonBytes': int(0.50 * GiB)}}) - assert sizing.pool_for(999) == 'protostar' - - -def test_no_profile_at_all_goes_to_nebula(pooled, monkeypatch): - monkeypatch.setattr(config, 'PROFILE', []) - assert sizing.pool_for(10) == 'nebula' - - -def test_an_entry_with_no_memory_measurement_is_treated_as_unprofiled(pooled, monkeypatch): - """Sizing needs a measurement, and `seconds` is not one. - - A record can carry a runtime but no peak -- reconstruction omits what it - cannot verify. Guessing a tier from runtime would reintroduce exactly the - cpu-ladder mistake: sizing memory off a dimension that does not predict it - (peakEphemeral/anon correlate r2 0.32). - """ - _profiled(monkeypatch, {10: {'seconds': 9000.0}}) - assert sizing.pool_for(10) == 'protostar' - - -def test_an_oom_promotes_the_pool_not_just_the_request(pooled, monkeypatch): - """The whole point of tier escalation. - - Raising the request while the pod stays pinned to a tier whose nodes cannot - hold it produces a pod that can never schedule -- Pending forever, which - reads as a hang rather than a failure. - """ - _profiled(monkeypatch, {10: {'peakAnonBytes': int(0.50 * GiB)}}) - assert sizing.pool_for(10, rungs=0) == 'dwarf' - assert sizing.pool_for(10, rungs=1) == 'subgiant' - assert sizing.pool_for(10, rungs=2) == 'giant' - assert sizing.pool_for(10, rungs=3) == 'supergiant' - - -def test_only_ooms_climb_the_ladder_not_every_retry(pooled, monkeypatch): - """A spot reclaim is not evidence the range needed a bigger node. - - Promoting on attempt number put 65 ranges onto 8-vCPU supernova nodes during - the 2026-08-03 spot run whose attempt-1 verdict was `timeout` -- they - belonged on 4-vCPU hypergiant, so it burned ~260 vCPU of a 2304 quota - escalating away from a problem that was never memory. Reclaims, disruptions - and timeouts all produce retries; only an OOM says the tier was too small. - """ - _profiled(monkeypatch, {10: {'peakAnonBytes': int(0.50 * GiB)}}) - monkeypatch.setattr(records, '_oom_count', lambda end, attempt: 0) - for attempt in (1, 2, 3, 9): - assert sizing.pool_for(10, attempt=attempt) == 'dwarf', \ - f"attempt {attempt} climbed a tier without an OOM" - monkeypatch.setattr(records, '_oom_count', lambda end, attempt: 2) - assert sizing.pool_for(10, attempt=3) == 'giant' - - -def test_promotion_counts_ooms_from_disk_when_rungs_is_not_given(pooled, monkeypatch): - _profiled(monkeypatch, {10: {'peakAnonBytes': int(0.50 * GiB)}}) - seen = {} - def fake(end, attempt): - seen['attempt'] = attempt - return 1 - monkeypatch.setattr(records, '_oom_count', fake) - assert sizing.pool_for(10, attempt=4) == 'subgiant' - # attempts BEFORE this one -- this attempt has not run, so its own outcome - # cannot be on disk yet. - assert seen['attempt'] == 3 - - -def test_promotion_stops_at_the_top_instead_of_running_off_the_ladder(pooled, monkeypatch): - _profiled(monkeypatch, {10: {'peakAnonBytes': int(40.0 * GiB)}}) - assert sizing.pool_for(10, rungs=0) == 'supernova' - assert sizing.pool_for(10, rungs=8) == 'supernova' - - -def test_the_off_ladder_pools_escalate_straight_to_the_top(pooled, monkeypatch): - """nebula and protostar are not rungs, so there is nothing to walk. - - Both hold ranges whose size is unknown, so an OOM says the guess was too - small with no information about by how much. The top tier is the only answer - that cannot be wrong again for the same reason. - """ - monkeypatch.setattr(config, 'PROFILE', []) - assert sizing.pool_for(10, rungs=1) == 'supernova' - _profiled(monkeypatch, {10: {'peakAnonBytes': GiB}}) - assert sizing.pool_for(999, rungs=1) == 'supernova' - - -def test_the_request_lands_exactly_two_pods_per_node(pooled): - """Two per node comes from the arithmetic, not from goodwill. - - The ladder ran one-pod-per-node until 2026-08-04, when every spot pool's - instance size was doubled to test whether a neighbour is worth more than a - dedicated node. Measured that day: a pod on a 4-vCPU node beat the same pod - on a 2-vCPU node by 25% (x8i.large 0.74 vs x8i.xlarge 1.07 against profile, - same tier, same silicon) while drawing under one core -- so the replay - thread is not what wants the extra cores, and a neighbour may be able to - use them without costing the first pod. - - Two pods must FIT (2*req + daemonsets <= allocatable) and three must NOT. - Getting the second condition wrong is the expensive one: at the old claims - against doubled nodes, 3-4 pods would pack per node and the isolation the - whole ladder exists for is gone without anything failing. - - Sized off ALLOCATABLE, not nameplate. On the small nodes the gap decides the - outcome: a 2Gi c8a.medium allocates 1181Mi and the scheduler was measured - counting 1477Mi for a 983Mi request. A nameplate-derived request fit the - node on paper and left 24 pods Pending with "no instance type has enough - resources". - """ - # MEASURED on live ssc-test nodes 2026-08-03. These are the same numbers as - # before the doubling, shifted one tier up -- what used to be supergiant's - # 16Gi node is now giant's. 128Gi is extrapolated from the 64Gi measurement - # at the same 94% ratio; nothing that large has run yet. - ALLOC = {'dwarf': 2798, 'subgiant': 6502, 'giant': 14654, - 'supergiant': 30259, 'hypergiant': 61604, 'supernova': 124000} - OVERHEAD = 154 # measured daemonset requests on a live node - for tier, alloc in ALLOC.items(): - req = int(sizing.pool_memory(tier).removesuffix('Mi')) - assert 2 * req + OVERHEAD <= alloc, f"{tier}: a second pod will not schedule" - assert 3 * req + OVERHEAD > alloc, f"{tier}: three pods would fit" - - -def test_every_routable_tier_has_a_request(pooled): - # A tier with no entry silently keeps the flat configured request, which is - # both too small to isolate and unrelated to the node it landed on. - # cpu claims are checked against the real chart values by - # test_the_chart_ships_a_coherent_pool_ladder; POOL_MEM is not, so this is - # the only thing standing between a promoted tier and the flat request. - for _, tier in sizing._parsed_pool_tiers(): - assert sizing.pool_memory(tier), f"tier {tier} has no memory request" - for off_ladder in ('protostar', 'nebula'): - assert sizing.pool_memory(off_ladder) - - -def test_an_empty_prefix_disables_pooling_entirely(monkeypatch): - """The change has to be opt-in: an unset prefix is exactly today's run.""" - monkeypatch.setattr(config, 'POOL_PREFIX', '') - _profiled(monkeypatch, {10: {'peakAnonBytes': GiB}}) - assert sizing.pool_for(10) is None - - -def test_a_malformed_cpu_map_falls_back_rather_than_crashing(pooled, monkeypatch): - monkeypatch.setattr(config, 'POOL_CPU', 'dwarf:notanumber,giant:1.1') - assert sizing.pool_cpu('dwarf') is None - assert sizing.pool_cpu('giant') == 1.1 - - -def test_pooled_memory_carries_no_margin_because_the_node_carries_it(pooled, monkeypatch, cluster): - """PROFILE_MARGIN and friends existed to keep a pod under its own LIMIT. - - There is no memory limit any more and the pod owns the node, so a margin in - the REQUEST constrains nothing the kubelet acts on -- it only wastes - schedulable space. The 1.60x lives in the node size, where node pressure can - actually enforce it. - """ - _profiled(monkeypatch, {300: {'peakAnonBytes': int(0.50 * GiB), 'seconds': 300.0}}) - r = jm._resources(end=300) - # dwarf's half-node request, not 0.50 * PROFILE_MARGIN + headroom + insurance - assert r.requests['memory'] == '1280Mi' - assert r.requests['cpu'] == 0.85 - assert not r.limits or 'memory' not in r.limits - - -def test_an_escalated_retry_requests_the_tier_it_was_promoted_to(pooled, monkeypatch, cluster): - """The label and the request have to agree. - - The affinity path knows the attempt, so an OOM retry lands on the promoted - pool. If the sizing path did not, the pod would arrive at a supergiant node - still asking for dwarf's memory -- under-requesting on the very node it was - escalated onto, and leaving room for a second pod on a tier whose whole - purpose is one pod per node. - """ - _profiled(monkeypatch, {300: {'peakAnonBytes': int(0.50 * GiB), 'seconds': 300.0}}) - # every prior attempt OOMed: pool_for asks for attempts BEFORE this one - monkeypatch.setattr(records, '_oom_count', lambda end, attempt: attempt) - first = jm._resources(end=300, attempt=1) - assert first.requests['memory'] == '1280Mi' # dwarf - assert first.requests['cpu'] == 0.85 - - third = jm._resources(end=300, attempt=3) - assert sizing.pool_for(300, attempt=3) == 'giant' - assert third.requests['memory'] == '6656Mi' # giant - assert third.requests['cpu'] == 1.85 - - -def test_escalation_does_not_opt_out_of_the_profile_when_pooled(pooled, monkeypatch, cluster): - """Unpooled, an escalated request outranks the profile and short-circuits it. - - Pooled, the promotion IS the escalation -- so short-circuiting would hand - the pod the flat configured request instead of the promoted tier's cut. - """ - _profiled(monkeypatch, {300: {'peakAnonBytes': int(0.50 * GiB), 'seconds': 300.0}}) - # every prior attempt OOMed: pool_for asks for attempts BEFORE this one - monkeypatch.setattr(records, '_oom_count', lambda end, attempt: attempt) - # `mem` set is what marks a retry as escalated - r = jm._resources(mem='9999Mi', end=300, attempt=2) - assert r.requests['memory'] == '2816Mi' # subgiant - - -# --- cache bump ------------------------------------------------------------ -# -# peakAnonBytes decides which node can HOLD a range; peakWorkingSetBytes decides -# whether that node can CACHE it. The two diverge by a median 2.5x and up to -# 10.4x across the profile, so a range can sit safely inside its tier's memory -# and still thrash. Measured on ssc-test 2026-08-03, one range on two 2-vCPU -# Intel nodes differing only in RAM: -# -# m8in.large 8 GiB 540 reads/ledger 21% iowait 1.86 lps -# r8in.large 16 GiB 65 reads/ledger 7% iowait 3.14 lps -# -# 44648511's real numbers are used throughout so these tests fail if the ladder -# ever moves such that the range we physically measured stops being promoted. -MEASURED_ANON = int(2.63 * GiB) # -> giant (1.61 <= 2.63 < 3.87) -MEASURED_WS = int(18.31 * GiB) # -> reaches supergiant and beyond - - -def test_a_range_that_cannot_cache_its_working_set_moves_up_one_rung(pooled, monkeypatch): - assert sizing._tier_for_bytes(MEASURED_ANON) == 'giant' - assert _pool(monkeypatch, MEASURED_ANON, MEASURED_WS) == 'supergiant' - - -def test_the_bump_is_one_rung_even_when_the_working_set_wants_two(pooled, monkeypatch): - """Clearing the thrash cliff is the goal, not fitting the working set. - - 18.31 GiB would land in hypergiant on its own, and hypergiant is two rungs - from giant. The measured range reaches full profile rate on supergiant while - still 1.29x UNDER its working set, so the extra rung buys nothing and costs - a doubling of cores. - """ - assert sizing._tier_for_bytes(MEASURED_WS) == 'hypergiant' - assert _pool(monkeypatch, MEASURED_ANON, MEASURED_WS) == 'supergiant' - - -def test_a_working_set_that_stays_inside_its_tier_is_not_promoted(pooled, monkeypatch): - assert sizing._tier_for_bytes(int(3.50 * GiB)) == 'giant' # same tier as the anon - assert _pool(monkeypatch, 2.00 * GiB, 3.50 * GiB) == 'giant' - - -def test_only_the_supergiant_rung_ships_open(pooled, monkeypatch): - """supergiant->hypergiant runs by default; hypergiant->supernova does not. - - Both cross a vCPU class now. Until 2026-08-04 hypergiant listed x8i.xlarge - (4 vCPU) at w80 beneath two 8-vCPU rungs, so pool_vcpu -- which reads the - SMALLEST shape -- reported 4 and supergiant->hypergiant priced as free, while - Karpenter tried w100 first and the promotion really cost 4->8. Removing the - x8i spot pools made the map honest: supergiant 4, hypergiant 8, supernova 16. - - So supergiant->hypergiant is now carried by POOL_CROSS_RUNGS rather than by - the guard, and hypergiant->supernova is denied outright -- with x8i.2xlarge - gone, supernova's only spot shapes are 4xlarges, so that rung buys 8->16 vCPU - for a promotion decided by working set, which does not predict throughput. - """ - _profiled(monkeypatch, { - 10: {'peakAnonBytes': int(7.00 * GiB), 'peakWorkingSetBytes': int(30.0 * GiB)}, - 20: {'peakAnonBytes': int(16.0 * GiB), 'peakWorkingSetBytes': int(40.0 * GiB)}, - }) - assert config.POOL_BLOCK_RUNGS == 'hypergiant->supernova' - assert sizing.pool_vcpu('supergiant') != sizing.pool_vcpu('hypergiant'), \ - "the rung crosses a class; the whitelist is what carries it, not the guard" - assert sizing.pool_for(10) == 'hypergiant' - assert sizing.pool_for(20) == 'hypergiant' # denied, stays put - - # dropping the denylist is NOT enough to reopen the supernova rung: it still - # crosses 8->16, so it also needs a POOL_CROSS_RUNGS entry - monkeypatch.setattr(config, 'POOL_BLOCK_RUNGS', '') - assert sizing.pool_for(20) == 'hypergiant' - monkeypatch.setattr(config, 'POOL_CROSS_RUNGS', - 'supergiant->hypergiant,hypergiant->supernova') - assert sizing.pool_for(20) == 'supernova' - - -def test_a_blocked_rung_beats_the_crossing_whitelist(pooled, monkeypatch): - """Deny wins over allow, so one stale env cannot silently re-open a rung.""" - monkeypatch.setattr(config, 'POOL_BLOCK_RUNGS', 'hypergiant->supernova') - monkeypatch.setattr(config, 'POOL_CROSS_RUNGS', 'hypergiant->supernova') - assert _pool(monkeypatch, 16.0 * GiB, 40.0 * GiB, end=20) == 'hypergiant' - - -def test_the_whitelist_only_opens_the_rung_it_names(pooled, monkeypatch): - """An exception must not become a blanket "ignore vCPU classes" switch. - - dwarf->subgiant also crosses a class (1 -> 2 vCPU) and is deliberately shut: - the longest dwarf range is 1663 s against a 10340 s critical path, so nothing - there can reach the tail and the promotion would be pure cost. - """ - monkeypatch.setattr(config, 'POOL_CROSS_RUNGS', 'hypergiant->supernova') - assert _pool(monkeypatch, 0.50 * GiB, 5.00 * GiB) == 'dwarf' - - -def test_the_free_rung_is_decided_by_node_vcpu_not_by_the_cpu_claim(pooled, monkeypatch): - """POOL_CPU stopped being a proxy for node size, so the guard must not read it. - - Claims were half the node everywhere, which made equal claims imply equal - nodes. That stopped being true once tiers were sized to the smallest shape in - their pool, so two tiers can carry different claims while sitting on - identically sized nodes. A claim comparison silently refuses such a rung. - - giant->supergiant is the case that still has equal nodes (4 vCPU both sides) - after the x8i pools were removed on 2026-08-04, so it is what exercises the - guard. The rung is not on either list, which is the point -- it must be judged - free on node size alone. - """ - # Force the claims apart. In production they happen to match right now, but - # the guard must read POOL_VCPU either way -- a claim comparison broke a rung - # once already when a tier was sized to its smallest shape. - monkeypatch.setattr(config, 'POOL_CPU', CPU.replace('supergiant:1.85', 'supergiant:1.20')) - assert sizing.pool_cpu('giant') != sizing.pool_cpu('supergiant') - assert sizing.pool_vcpu('giant') == sizing.pool_vcpu('supergiant') - assert not sizing._rung_listed(config.POOL_CROSS_RUNGS, 'giant', 'supergiant'), \ - "the whitelist must not be what carries this rung" - assert _pool(monkeypatch, 3.87 * GiB, 12.0 * GiB) == 'supergiant' - - -def test_every_tier_the_ladder_can_reach_has_a_vcpu_mapping(pooled): - """An unmapped tier makes the guard refuse every rung into or out of it. - - _cache_bump bails when either side is None, so a missing entry does not - crash -- it silently turns the whole rule off for that tier, which is the - kind of failure that only shows up as a run that cost more than it should. - """ - for tier in [name for _, name in sizing._parsed_pool_tiers()] + [ - config.POOL_UNPROFILED, config.POOL_NO_PROFILE]: - assert sizing.pool_vcpu(tier) is not None, f"{tier} has no POOL_VCPU entry" - - -def test_the_dwarf_rung_is_refused_because_it_also_crosses_a_cpu_class(pooled, monkeypatch): - """dwarf->subgiant is 0.50 -> 1.00, a 1-vCPU node to a 2-vCPU one. - - Blocking it is affordable: the longest dwarf range is 1663 s against a 10340 s - critical path, it ranks #1768 of 3985 in longest-first order, and dwarf is - 5.4% of total work. Nothing there can reach the tail. - """ - assert _pool(monkeypatch, 0.50 * GiB, 5.00 * GiB) == 'dwarf' - - -def test_the_top_of_the_ladder_has_nowhere_to_go(pooled, monkeypatch): - assert _pool(monkeypatch, 40.0 * GiB, 90.0 * GiB) == 'supernova' - - -def test_a_profile_without_working_set_data_routes_exactly_as_before(pooled, monkeypatch): - """Every profile generated before this change lacks peakWorkingSetBytes. - - Those must keep their old placement rather than crash or silently shift, so - the field being absent has to mean "no opinion", not "zero". - """ - _profiled(monkeypatch, { - 10: {'peakAnonBytes': MEASURED_ANON}, - 20: {'peakAnonBytes': MEASURED_ANON, 'peakWorkingSetBytes': 0}, - 30: {'peakAnonBytes': MEASURED_ANON, 'peakWorkingSetBytes': None}, - }) - assert sizing.pool_for(10) == 'giant' - assert sizing.pool_for(20) == 'giant' - assert sizing.pool_for(30) == 'giant' - - -def test_the_bump_is_the_same_every_run(pooled, monkeypatch): - """Deriving from the bytes each run is what stops the bump compounding. - - The same measurements must yield the same tier however many runs have read - them. Nothing carries a tier forward, so there is no verdict to bump on top - of and the promotion cannot ratchet a range to the ceiling one run at a time. - """ - _profiled(monkeypatch, { - 10: {'peakAnonBytes': MEASURED_ANON, 'peakWorkingSetBytes': MEASURED_WS}, - }) - assert sizing.pool_for(10) == 'supergiant' - assert sizing.pool_for(10) == 'supergiant' - - -def test_an_oom_still_climbs_from_the_bumped_tier(pooled, monkeypatch): - """The cache bump is a starting point, not a replacement for OOM escalation. - - A range bumped to supergiant that then OOMs there has proved it needs more - memory, and must keep climbing -- including onto the cpu-class rungs the - bump itself refuses to take speculatively. - """ - _profiled(monkeypatch, { - 10: {'peakAnonBytes': MEASURED_ANON, 'peakWorkingSetBytes': MEASURED_WS}, - }) - assert sizing.pool_for(10, rungs=0) == 'supergiant' - assert sizing.pool_for(10, rungs=1) == 'hypergiant' - assert sizing.pool_for(10, rungs=2) == 'supernova' - - -def test_the_bumped_tier_is_what_the_pod_actually_requests(pooled, monkeypatch, cluster): - """Routing to a pool and sizing for it have to agree. - - A pod pinned to supergiant nodes but carrying giant's request would let a - second pod share the node, which defeats the isolation the whole ladder - exists to buy. - """ - _profiled(monkeypatch, { - 10: {'peakAnonBytes': MEASURED_ANON, - 'peakWorkingSetBytes': MEASURED_WS, - 'seconds': 300.0}, - }) - r = jm._resources(end=10, attempt=1) - assert r.requests['memory'] == '14336Mi' # supergiant, not giant's 4096Mi - assert r.requests['cpu'] == 1.85 - - -def test_a_range_past_the_profile_is_sized_by_its_pool_not_the_flat_request(pooled, monkeypatch, cluster): - """Pooled placement without pooled sizing is a pod that never schedules. - - pool_for resolves a tier for EVERY range -- protostar when the range is - newer than anything measured -- but _profile_overrides used to bail on the - missing profile entry before it reached the pooled branch. The pod then got - protostar's node affinity with the run's flat REQ_CPU. - - Measured on ssc-test 2026-08-04: a 1200-worker run passed - --pubnet-parallel-catchup-cpu-request 6780m, so the past-the-profile ranges - asked for 6780m at a pool whose largest node is 4 vCPU. Permanently Pending, - retried forever, and silent -- earlier runs had only two such ranges and - nobody checked whether they had scheduled. - """ - monkeypatch.setattr(config, 'REQ_CPU', '6780m') - monkeypatch.setattr(config, 'REQ_MEM', '9Gi') - _profiled(monkeypatch, {10: {'peakAnonBytes': int(0.50 * GiB), 'seconds': 300.0}}) - assert sizing.pool_for(999) == 'protostar' # past the top of the profile - r = jm._resources(end=999, attempt=1) - assert r.requests['cpu'] == sizing.pool_cpu('protostar') - assert r.requests['memory'] == sizing.pool_memory('protostar') - assert r.requests['cpu'] != '6780m' - - -def test_no_profile_at_all_is_also_sized_by_its_pool(pooled, monkeypatch, cluster): - """Same failure one step further out: nebula gets a tier, so it needs a cut.""" - monkeypatch.setattr(config, 'REQ_CPU', '6780m') - monkeypatch.setattr(config, 'PROFILE', []) - assert sizing.pool_for(10) == 'nebula' - r = jm._resources(end=10, attempt=1) - assert r.requests['cpu'] == sizing.pool_cpu('nebula') - assert r.requests['memory'] == sizing.pool_memory('nebula') - - -def test_every_poolable_tier_fits_the_smallest_node_it_can_land_on(pooled): - """A claim above the smallest shape's usable cpu silently drops that shape. - - protostar at 1800m is deliberately above x8i.large's 1715m -- it is meant to - take the 4-vCPU shapes and leave the 128-vCPU X quota to hypergiant. Every - other tier must fit the node POOL_VCPU says is its smallest, or the tier - quietly loses its cheapest option. - """ - DAEMONSETS = 215 # alloy 10 + aws-node 75 + ebs-csi 30 + kube-proxy 100 - def usable(vcpu): - reserved = 60 + (10 if vcpu >= 2 else 0) + (5 if vcpu >= 3 else 0) + (5 if vcpu >= 4 else 0) - return vcpu * 1000 - reserved - DAEMONSETS - for tier in [name for _, name in sizing._parsed_pool_tiers()] + [config.POOL_NO_PROFILE]: - cpu, vcpu = sizing.pool_cpu(tier), sizing.pool_vcpu(tier) - if cpu is None or vcpu is None: - continue - assert cpu * 1000 <= usable(vcpu), ( - f"{tier} claims {cpu * 1000:.0f}m but its smallest node " - f"({vcpu} vCPU) only offers {usable(vcpu)}m") - - -# --- nebula: the no-profile pool --------------------------------------------- - -def test_nebula_packs_two_to_four_on_the_shapes_its_pools_offer(): - """Ranges with no measurement at all, so density is set deliberately. - - Read off the unpatched defaults, not the fixture: these are the numbers the - chart ships and the nodepools are cut from. A claim above what the smallest - shape can offer wins no nodes at all rather than packing fewer -- r8a.xlarge - offers 3705m once the EKS reserve and 215m of daemonsets come out, so the - 3800m this used to claim fit zero pods on it. - """ - cpu = float(dict(x.split(':') for x in config.POOL_CPU.split(','))['nebula']) * 1000 - mem = int(dict(x.split(':') for x in config.POOL_MEM.split(','))['nebula'].rstrip('Mi')) - vcpu = int(dict(x.split(':') for x in config.POOL_VCPU.split(','))['nebula']) - - def usable_cpu(cores): - reserved = 60 + (10 if cores >= 2 else 0) + (5 if cores >= 3 else 0) \ - + (5 if cores >= 4 else 0) + max(0, cores - 4) * 2.5 - return cores * 1000 - reserved - 215 - - # measured allocatable, same source as ALLOC above - for shape, cores, alloc_mem, want in (('r8a.xlarge', 4, 30259, 2), - ('m8a.2xlarge', 8, 30259, 3), - ('r8a.2xlarge', 8, 61604, 4)): - fits = min(int(usable_cpu(cores) // cpu), int((alloc_mem - 215) // mem)) - assert fits == want, f"{shape}: {fits} pods per node, expected {want}" - - # POOL_VCPU must name the SMALLEST shape, or the free-rung guard misprices - # a promotion into or out of nebula. - assert vcpu == 4 diff --git a/src/MissionParallelCatchup/tests/unit/test_profile_lookup.py b/src/MissionParallelCatchup/tests/unit/test_profile_lookup.py deleted file mode 100644 index 2ee500e8..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_profile_lookup.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Loading a previous run's measurements, and picking the entry to size from. - -A profile is an optimisation, never a prerequisite: absent, unreadable and -malformed all have to mean "use the configured defaults". -""" - -import json - -import pytest - -import config -import profiles -import job_monitor as jm - - -PROFILE_RANGES = [ - (1000, {'peakAnonBytes': 1_000_000_000, 'peakWorkingSetBytes': 9_000_000_000, - 'peakEphemeralBytes': 2_000_000_000}), - (2000, {'peakAnonBytes': 3_000_000_000, 'peakWorkingSetBytes': 13_000_000_000, - 'peakEphemeralBytes': 4_000_000_000}), -] - - -@pytest.fixture -def profile(monkeypatch): - """Install a loaded profile, as load_profile() would have left it.""" - def install(ranges=PROFILE_RANGES): - monkeypatch.setattr(config, 'PROFILE', sorted(ranges)) - return install - - -@pytest.fixture -def written(tmp_path, monkeypatch): - """Write a profile document and load it through the real reader.""" - def load(doc, mode='ephemeral', text=None): - path = tmp_path / 'profile.json' - path.write_text(text if text is not None else json.dumps(doc)) - monkeypatch.setattr(config, 'PROFILE_PATH', str(path)) - monkeypatch.setattr(config, 'STORAGE_MODE', mode) - return profiles.load_profile() - return load - - -# --- picking an entry -------------------------------------------------------- - -def test_profile_prefers_an_exact_end(profile): - profile() - assert profiles.profile_for(2000)['peakAnonBytes'] == 3_000_000_000 - - -def test_profile_rounds_up_to_the_next_measured_end_never_down(profile): - # Cost rises with ledger position -- the bucket set only grows -- so a lower - # neighbour under-reports, and under-provisioning costs an eviction while - # over-provisioning only costs packing density. - profile() - assert profiles.profile_for(1500)['peakAnonBytes'] == 3_000_000_000, \ - "1500 must size from 2000, not from 1000" - - -def test_profile_falls_back_to_defaults_past_its_high_water_mark(profile): - # An older profile has nothing above its own top, which is exactly where a - # newer run's fresh ranges live. Extrapolating there would under-provision. - profile() - assert profiles.profile_for(9999) is None - - -def test_no_profile_at_all_is_not_an_error(monkeypatch): - monkeypatch.setattr(config, 'PROFILE', None) - assert profiles.profile_for(1000) is None - monkeypatch.setattr(config, 'PROFILE', []) - assert profiles.profile_for(1000) is None - - -# --- reading the document ---------------------------------------------------- - -def test_no_configured_path_means_no_profile(monkeypatch): - monkeypatch.setattr(config, 'PROFILE_PATH', '') - assert profiles.load_profile() == [] - - -def test_an_unreadable_profile_is_not_fatal(written, tmp_path, monkeypatch): - # It is an optimisation, never a prerequisite. - assert written(None, text='{not json') == [] - monkeypatch.setattr(config, 'PROFILE_PATH', str(tmp_path / 'nope.json')) - assert profiles.load_profile() == [] - - -def test_a_matching_profile_keeps_every_axis(written): - got = written({'storageMode': 'ephemeral', - 'ranges': {'2000': PROFILE_RANGES[1][1]}}) - assert got == [(2000, PROFILE_RANGES[1][1])] - - -def test_entries_come_back_sorted_by_range_end(written): - # profile_for() bisects the list, so an unsorted load would silently size - # ranges from the wrong neighbour. - got = written({'storageMode': 'ephemeral', - 'ranges': {'3000': {}, '1000': {}, '2000': {}}}) - assert [end for end, _ in got] == [1000, 2000, 3000] - - -def test_a_non_numeric_range_key_is_skipped_not_fatal(written): - got = written({'storageMode': 'ephemeral', - 'ranges': {'2000': {'peakAnonBytes': 1}, 'tip': {'peakAnonBytes': 2}}}) - assert [end for end, _ in got] == [2000] - - -def test_a_cross_mode_profile_keeps_memory_but_drops_disk(written): - # cpu and memory measure the same work in either mode. Disk does not: a pvc - # run never measures node-local usage at all, so its absence must fall back - # to the configured default rather than size the wrong dimension. Degrade, - # never reject -- a rejected profile loses the transferable axes too. - got = written({'storageMode': 'pvc', 'ranges': {'2000': PROFILE_RANGES[1][1]}}, - mode='ephemeral') - assert len(got) == 1 - rec = got[0][1] - assert 'peakEphemeralBytes' not in rec - assert rec['peakAnonBytes'] == 3_000_000_000 - - -def test_a_profile_with_no_declared_mode_is_taken_at_face_value(written): - # Pre-dates the field; rejecting it would discard every older artifact. - got = written({'ranges': {'2000': PROFILE_RANGES[1][1]}}, mode='ephemeral') - assert got[0][1]['peakEphemeralBytes'] == 4_000_000_000 diff --git a/src/MissionParallelCatchup/tests/unit/test_range_generation.py b/src/MissionParallelCatchup/tests/unit/test_range_generation.py deleted file mode 100644 index 1d0a44e1..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_range_generation.py +++ /dev/null @@ -1,149 +0,0 @@ -"""The ledger range list, and the order it is dispatched in. - -generate_ranges() must stay a pure function of config: dispatch derives the -full list on every reconcile, so a restart has to reproduce it exactly. -""" - -import os - -import pytest - -import config -import ranges -import job_monitor as jm - - -@pytest.fixture -def build(monkeypatch): - """Configure the generator and return a callable that runs it.""" - def configure(order='tip-first', parallelism=4, - start=39990000, latest=40000000, per_job=1000, overlap=320): - monkeypatch.setattr(config, 'RANGE_ORDER', order) - monkeypatch.setattr(config, 'PARALLELISM', parallelism) - monkeypatch.setattr(config, 'STARTING_LEDGER', start) - monkeypatch.setattr(config, 'LATEST_LEDGER_NUM', latest) - monkeypatch.setattr(config, 'LEDGERS_PER_JOB', per_job) - monkeypatch.setattr(config, 'OVERLAP_LEDGERS', overlap) - return ranges.generate_ranges() - return configure - - -def test_ranges_are_emitted_tip_first_by_default(build): - r = build() - assert r[0][0] > r[-1][0], "index 0 must be the tip" - - -def test_oldest_first_reverses_dispatch_without_dropping_ranges(build): - # A profiling run wants the cheap early ranges measured first: the bucket - # set only grows with ledger position, so tip-first front-loads the - # expensive ones and an interrupted run profiles nothing cheap. - tip = build(order='tip-first') - old = build(order='oldest-first') - assert old == list(reversed(tip)) - assert sorted(old) == sorted(tip), "reversing must not change the range set" - - -def test_every_range_carries_the_overlap_on_top_of_its_ledger_count(build): - # The count is what the worker is asked to catch up, and it is always the - # segment plus OVERLAP_LEDGERS -- measuring with overlap 0 measures nothing - # the run will ever dispatch. - r = build(per_job=1000, overlap=320) - assert {count for _, count in r} == {1320} - - -def test_the_ranges_tile_the_ledger_space_with_no_gap(build): - r = sorted(build(start=0, latest=10000, per_job=1000, overlap=320)) - ends = [end for end, _ in r] - assert ends == list(range(1000, 10001, 1000)) - assert ends[-1] == 10000, "the tip must be covered" - - -def test_a_short_tail_segment_is_not_padded_past_the_start(build): - # The last segment is min(remaining, seg_size), so a range list over a span - # that does not divide evenly must not reach below STARTING_LEDGER. - r = build(start=0, latest=2500, per_job=1000, overlap=0) - assert sorted(r) == [(500, 500), (1500, 1000), (2500, 1000)] - - -def test_longest_first_is_inert_without_a_profile(build, monkeypatch): - """Ordering is driven by RANGE_ORDER, never by profile detection. - - profile_for returns None for every range when no profile is loaded, so - every sort key ties and Python's stable sort leaves the generator's own - tip-first order untouched. The two flags are independent in configuration - and only coupled in effect -- which is why validate_config() refuses the - combination at startup rather than letting the flag look set and do nothing. - """ - monkeypatch.setattr(config, 'PROFILE', {}) - assert build(order='longest-first') == build(order='tip-first') - - -@pytest.mark.parametrize('order', ['tipfirst', 'longest', '', 'TIP-FIRST']) -def test_an_unrecognised_order_fails_instead_of_becoming_tip_first(build, order): - with pytest.raises(ValueError, match='RANGE_ORDER'): - build(order=order) - - -# --- validate_config: the startup preflight ---------------------------------- -# -# These checks exist at startup specifically because the reconcile loop catches -# and logs every exception then sleeps. A raise reached from inside it is an -# infinite log loop that never dispatches, so "fails loudly" depends entirely on -# validate_config being called from __main__ before the thread starts. - -@pytest.fixture -def preflight(monkeypatch): - def configure(order='tip-first', profile=None): - monkeypatch.setattr(config, 'RANGE_ORDER', order) - monkeypatch.setattr(config, 'PROFILE', profile) - return jm.validate_config - return configure - - -def test_valid_config_passes(preflight): - preflight(order='oldest-first')() - - -def test_preflight_rejects_an_unknown_order(preflight): - with pytest.raises(ValueError, match='RANGE_ORDER'): - preflight(order='longest')() - - -def test_preflight_rejects_longest_first_without_a_profile(monkeypatch, tmp_path): - """The check moved to /start: the profile arrives with the POST, so startup - is too early to judge it. Rejecting there fails the driver fast instead of - dispatching a run whose ordering silently degrades to tip-first.""" - monkeypatch.setattr(config, 'RANGE_ORDER', 'longest-first') - monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(config, 'RUN_PATH', str(tmp_path / 'run.json')) - - with pytest.raises(ValueError, match='longest-first requires a profile'): - jm.start_run({"range": {'startingLedger': 0, 'latestLedgerNum': 1000, 'ledgersPerJob': 100}}) - - # A profile with ranges is accepted, and nothing is written until it passes. - jm.start_run({'range': {'startingLedger': 0, 'latestLedgerNum': 1000, 'ledgersPerJob': 100}, 'profile': {'ranges': {'300': {'seconds': 1.0}}}}) - assert config.PROFILE == [(300, {'seconds': 1.0})] - - -def test_preflight_allows_longest_first_with_a_profile(preflight): - preflight(order='longest-first', profile=[(40000000, {'seconds': 900.0})])() - - -def test_the_preflight_runs_before_anything_is_dispatched(monkeypatch, tmp_path): - """Validation moved to /start, which is the first moment the config is - whole. It still has to bind before dispatch: a run that is misconfigured - must be refused, not started and then discovered.""" - monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(config, 'RUN_PATH', str(tmp_path / 'run.json')) - monkeypatch.setattr(config, 'RANGE_ORDER', 'nonsense') - - with pytest.raises(ValueError, match='RANGE_ORDER must be one of'): - jm.start_run({"range": {'startingLedger': 0, 'latestLedgerNum': 1000, 'ledgersPerJob': 100}}) - - # Rejected, so nothing was written and no run can proceed from it. - assert not os.path.exists(config.RUN_PATH) - - -def jm_source(): - import inspect - return inspect.getsource(jm) diff --git a/src/MissionParallelCatchup/tests/unit/test_reaping.py b/src/MissionParallelCatchup/tests/unit/test_reaping.py deleted file mode 100644 index 3f94f9f2..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_reaping.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Deleting finished Jobs and released volumes. - -reconcile() LISTs every Job and Pod each pass, so a finished Job is not free: -it inflates two LIST calls for as long as it lingers. At 2048-4096 parallelism -with a real OOM or spot-eviction rate that is hundreds of dead objects per hour -of run, and the apiserver pressure shows up as truncated list responses long -before anything else complains. - -Everything here is best-effort by design: a cleanup failure costs disk or etcd, -never correctness, and raising would abort a reconcile pass mid-run and strand -every other range in the same iteration. -""" - -import pytest -from kubernetes import client - -import fake_k8s -import config -import kube -import records -import job_monitor as jm - - -NAMESPACE = 'catchup-test' -RUN = 'pc' - - -@pytest.fixture -def k8s(logdir, monkeypatch): - """A fake cluster wired into the monitor, with no reconcile in the way.""" - fake = fake_k8s.FakeCluster(namespace=NAMESPACE) - monkeypatch.setattr(kube, 'core_v1', fake.core_v1) - monkeypatch.setattr(kube, 'batch_v1', fake.batch_v1) - monkeypatch.setattr(config, 'NAMESPACE', NAMESPACE) - monkeypatch.setattr(config, 'RUN_NAME', RUN) - monkeypatch.setattr(config, 'STORAGE_MODE', 'pvc') - - def add_job(end, attempt): - name = jm.job_name(end, attempt) - labels = {config.LABEL_RUN: RUN, config.LABEL_RANGE: str(end), - config.LABEL_ATTEMPT: str(attempt)} - fake.batch_v1.create_namespaced_job(NAMESPACE, client.V1Job( - metadata=client.V1ObjectMeta(name=name, labels=labels), - spec=client.V1JobSpec( - template=client.V1PodTemplateSpec( - metadata=client.V1ObjectMeta(labels=labels), - spec=client.V1PodSpec(containers=[], restart_policy='Never'))))) - return name - - fake.add_job = add_job - return fake - - -class Boom: - """A batch API that fails every delete with one status.""" - - def __init__(self, status): - self.status = status - self.calls = 0 - - def delete_namespaced_job(self, name, namespace, **_): - self.calls += 1 - raise fake_k8s.api_exception(self.status, 'boom') - - def list_namespaced_job(self, namespace, **_): - raise fake_k8s.api_exception(self.status, 'boom') - - -# --- deleting one attempt's Job ---------------------------------------------- - -def test_delete_job_reaps_the_pod_too(k8s): - # Background propagation is what actually removes the pod. Orphan would - # leave the pod behind, and the pod is what reconcile lists. - name = k8s.add_job(30957951, 2) - assert k8s.pod_for_job(name) is not None - jm.delete_job(30957951, 2) - assert k8s.job_names() == [] - assert k8s.pod_for_job(name) is None, "the pod outlived its Job" - - -@pytest.mark.parametrize('status', [404, 403, 500]) -def test_delete_job_is_best_effort(monkeypatch, k8s, status): - # A 404 is the normal race with the TTL controller, not an error. Any other - # status must be swallowed too: losing a Job to a leaked object is a - # disk/etcd cost, but raising here would abort the whole reconcile pass. - boom = Boom(status) - monkeypatch.setattr(kube, 'batch_v1', boom) - jm.delete_job(1, 1) # must not raise - assert boom.calls == 1 - - -# --- deleting every Job a completed range has -------------------------------- - -def test_a_completed_range_reaps_every_attempt_not_just_the_winner(k8s): - # Completion is terminal for the RANGE. An attempt-scoped reap leaves an - # older Failed Job standing -- typically one lost to node disruption whose - # collector died with the node, so it was never finalized and was - # deliberately not deleted. Once the winner's Job is gone that leftover is - # the range's highest live attempt, and the next pass feeds it into the - # retry decision and re-runs an already-recorded range. - k8s.add_job(300, 1) - k8s.add_job(300, 2) - other = k8s.add_job(400, 1) - jm.reap_range_jobs(300) - assert k8s.job_names() == [other], "the reap is not scoped to the range" - - -def test_a_list_failure_leaves_the_jobs_to_the_ttl_rather_than_raising(monkeypatch, k8s): - monkeypatch.setattr(kube, 'batch_v1', Boom(500)) - jm.reap_range_jobs(300) # must not raise - - -# --- the gate in front of both ----------------------------------------------- - -def test_the_reap_waits_for_the_collectors_done_marker(k8s): - # Not inferred from peaks or tx_apply: tx_apply falls back to the archive so - # it lands long before the collector finishes, and an attempt can finalize - # with no peaks at all. Only the collector knows it is done, and deleting - # the Job reaps the pod -- the last place peaks could still be read from. - k8s.add_job(300, 1) - assert jm._attempt_finalized(300, 1) is False, \ - "no marker yet, so reconcile must not reap" - open(records.done_path(300, 1), 'w').close() - assert jm._attempt_finalized(300, 1) is True - jm.reap_range_jobs(300) - assert k8s.job_names() == [] - - -def test_the_done_marker_is_the_only_thing_that_counts_as_finalized(logdir): - assert jm._attempt_finalized(300, 1) is False - open(records.metrics_path(300, 1), 'w').close() - assert jm._attempt_finalized(300, 1) is False, "metrics are not a promise" - open(records.done_path(300, 1), 'w').close() - assert jm._attempt_finalized(300, 1) is True - - -# --- releasing the volume ---------------------------------------------------- - -def test_a_completed_range_releases_its_volume(k8s): - # PVCs are owner-referenced to the release, so nothing reclaimed them until - # helm uninstall. Measured on ssc-test: 2032 bound PVCs / 79 TiB a third of - # the way through a 3982-range run, heading for ~156 TiB and 3982 volumes - # against the account's volume ceiling. - name = jm.ensure_pvc(300, owner=None) - assert k8s.pvc_names() == [name] - jm.release_pvc(300) - assert k8s.pvc_names() == [] - - -def test_ephemeral_mode_has_no_volume_to_release(monkeypatch, k8s): - jm.ensure_pvc(300, owner=None) - monkeypatch.setattr(config, 'STORAGE_MODE', 'ephemeral') - jm.release_pvc(300) - assert k8s.pvc_names() != [], "ephemeral mode deleted a volume it does not own" - - -def test_releasing_a_volume_never_fails_a_completed_range(k8s): - # Already-gone is the common case (a restart re-running the same tail), and - # a disk cleanup failure must not condemn a finished range either way. - jm.release_pvc(300) # nothing there: 404, must not raise - name = jm.ensure_pvc(300, owner=None) - k8s.fail_next['delete pvc'] = fake_k8s.api_exception(403, 'Forbidden') - jm.release_pvc(300) # must not raise - assert k8s.pvc_names() == [name], "the 403 was never actually injected" diff --git a/src/MissionParallelCatchup/tests/unit/test_records.py b/src/MissionParallelCatchup/tests/unit/test_records.py deleted file mode 100644 index d039e2ea..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_records.py +++ /dev/null @@ -1,90 +0,0 @@ -"""The filenames and record shapes the two processes agree on. - -The monitor and the collector are separate containers sharing one volume. Every -handoff between them is a filename, and a mismatch is silent: the monitor -simply never reaps and every Job waits out its TTL. -""" - -import os - -import config -import records -import job_monitor as jm -import log_collector as lc - - -def basename(path): - return os.path.basename(path) - - -# --- one volume, one set of filenames ---------------------------------------- - -def test_both_sides_agree_on_the_metrics_filename(logdir): - assert basename(records.metrics_path(300, 2)) == basename(lc.base(300, 2)) + '.metrics' - - -def test_both_sides_agree_on_the_done_marker(logdir): - # It licenses the monitor to reap the pod, which is the only place peaks can - # still be read from. - assert basename(records.done_path(300, 2)) == basename(lc.done_path(300, 2)) - - -def test_the_monitor_log_lands_where_the_mission_collects_it(): - # collectLogsFromPods tars LOG_DIR. The monitor used to write its own log to - # /data, an emptyDir, so OOM-retry storms never reached the destination - # directory and did not survive a monitor restart. - assert config.LOG_DIR == config.LOG_DIR, \ - "collector and monitor must share the collected directory" - assert os.path.dirname(config.PROGRESS_FILE) == config.LOG_DIR - - -def test_every_per_attempt_artifact_is_named_for_its_attempt(logdir): - # One namespace per (range, attempt) across five writers; a helper that - # dropped the attempt would have two attempts overwrite each other. - paths = [records.log_path(300, 2), records.state_path(300, 2), records.outcome_path(300, 2), - records.metrics_path(300, 2), records.verdict_path(300, 2), records.done_path(300, 2)] - assert all(basename(p).startswith('range-300-a2.') for p in paths), paths - assert len({basename(p) for p in paths}) == len(paths), "two writers share a filename" - - -# --- the worker pod's own labels --------------------------------------------- - -def test_the_worker_pod_carries_its_attempt_number(): - # The collector reads LABEL_ATTEMPT off the POD, not the Job, and defaults - # to "1". With the label only on the Job every attempt claimed the same - # range--a1.* files: measured on ssc-test 2026-07-30, 2246 metrics - # files all a1 while 475 a2 pods ran, so each retry overwrote the first - # attempt's peak instead of being maxed against it -- destroying exactly - # the OOM evidence the chain exists to keep. - labels = jm.pod_labels(300, 2) - assert labels[config.LABEL_ATTEMPT] == '2' - assert labels[config.LABEL_RANGE] == '300' - assert labels[config.LABEL_RUN] == config.RUN_NAME - - -def test_the_mission_label_is_opt_in(monkeypatch): - # It is high-cardinality and only wanted when something is scraping by - # mission, so it must not appear unless both switches are set. - monkeypatch.setattr(config, 'MISSION', 'pubnet-catchup') - monkeypatch.setattr(config, 'EMIT_MISSION_LABEL', False) - assert 'mission' not in jm.pod_labels(300, 1) - monkeypatch.setattr(config, 'EMIT_MISSION_LABEL', True) - assert jm.pod_labels(300, 1)['mission'] == 'pubnet-catchup' - - -# --- what the ConfigMap mirror is allowed to carry --------------------------- - -# --- durations the collector can read off a pod the monitor never saw --------- - -def test_a_terminal_pod_still_yields_its_real_duration(): - # A pod carries startTime and terminated.finishedAt until it is deleted, so - # even a pod that finished before this poller existed has a real duration. - # The poller's own elapsed time cannot know that -- it measures how long WE - # watched, which is ~0 in exactly that case, and 150 metrics files came back - # with a sub-5s duration next to a >500MiB anon peak because of it. - pod = {'status': {'startTime': '2026-07-30T04:16:26Z', - 'containerStatuses': [{'state': {'terminated': { - 'finishedAt': '2026-07-30T04:22:19Z'}}}]}} - assert lc.pod_seconds(pod) == 353.0 - assert lc.pod_seconds({'status': {'startTime': '2026-07-30T04:16:26Z'}}) is None - assert lc.pod_seconds({'status': {}}) is None diff --git a/src/MissionParallelCatchup/tests/unit/test_resources.py b/src/MissionParallelCatchup/tests/unit/test_resources.py deleted file mode 100644 index 2718c8cf..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_resources.py +++ /dev/null @@ -1,277 +0,0 @@ -"""Turning a measurement into the pod's requests and limits. - -_profile_overrides() decides what the profile is allowed to say; _resources() -decides what actually lands on the container. Both are called here rather than -read, because the first version of the sizing gate read `mem is None` AFTER mem -had been defaulted, so it was never true and profile sizing was silently dead -while a source-text assertion still passed. -""" - -import pytest - -import config -import units -import sizing -import attempts -import job_monitor as jm - - -PROFILE_RANGES = [ - (1000, {'peakAnonBytes': 1_000_000_000, 'peakWorkingSetBytes': 9_000_000_000, - 'peakEphemeralBytes': 2_000_000_000}), - (2000, {'peakAnonBytes': 3_000_000_000, 'peakWorkingSetBytes': 13_000_000_000, - 'peakEphemeralBytes': 4_000_000_000}), -] - -MI = 1024 ** 2 - - -@pytest.fixture -def shaped(monkeypatch): - """The worker's configured shape, plus a loaded profile.""" - def configure(ranges=PROFILE_RANGES, margin=1.1, req_mem='9Gi', - req_eph='35Gi', lim_eph='40Gi', max_mem='32Gi', - headroom='512Mi', runtime_insurance='3Gi', - eph_headroom='2Gi', eph_insurance='8Gi', max_eph='64Gi'): - monkeypatch.setattr(config, 'PROFILE', sorted(ranges)) - monkeypatch.setattr(config, '_SORTED_SECONDS', None) - monkeypatch.setattr(config, 'PROFILE_MARGIN', margin) - monkeypatch.setattr(config, 'PROFILE_MAX_MEM', max_mem) - monkeypatch.setattr(config, 'PROFILE_CACHE_HEADROOM', headroom) - monkeypatch.setattr(config, 'PROFILE_RUNTIME_MEMORY_INSURANCE', runtime_insurance) - monkeypatch.setattr(config, 'PROFILE_EPHEMERAL_HEADROOM', eph_headroom) - monkeypatch.setattr(config, 'PROFILE_RUNTIME_EPHEMERAL_INSURANCE', eph_insurance) - monkeypatch.setattr(config, 'PROFILE_MAX_EPHEMERAL', max_eph) - monkeypatch.setattr(config, 'REQ_CPU', '1800m') - monkeypatch.setattr(config, 'REQ_MEM', req_mem) - monkeypatch.setattr(config, 'REQ_EPHEMERAL', req_eph) - monkeypatch.setattr(config, 'LIM_EPHEMERAL', lim_eph) - return configure - - -# --- what the profile is allowed to say -------------------------------------- - -def test_profile_sizes_a_first_attempt(shaped): - shaped() - out = sizing._profile_overrides(2000, escalated=False) - assert out['memory'] == '3659Mi' # 3 GB rss * 1.1 + 512Mi - # 3.8Gi measured * 1.1 margin + 2Gi flat headroom; this range is short - # enough that its runtime-weighted share rounds to nothing. - assert out['ephemeral-storage'] == '6244Mi' - # cpu is no longer profiled: REQ_CPU is fixed, so there is nothing to size, - # and a measured cpu value only makes packing non-uniform. - assert 'cpu' not in out - - -def test_profile_does_not_override_an_escalated_retry(shaped): - # An escalation is a measurement of THIS run and outranks an earlier one. - shaped() - assert sizing._profile_overrides(2000, escalated=True) == {} - - -def test_profile_gives_nothing_past_its_high_water_mark(shaped): - shaped() - assert sizing._profile_overrides(99999, escalated=False) == {} - assert sizing._profile_overrides(None, escalated=False) == {} - - -def test_profile_memory_is_capped_at_its_own_ceiling_not_the_configured_request(shaped): - # A range measured above the configured request must be able to ask for more, - # or it packs as though it were small and lands somewhere it cannot fit. The - # ceiling is what bounds it, and the OOM ladder can still climb past that. - shaped(ranges=[(1, {'peakAnonBytes': 500_000_000_000})], - req_mem='9Gi', max_mem='32Gi') - assert sizing._profile_overrides(1, escalated=False)['memory'] == '32768Mi' - - -def test_profile_memory_can_exceed_the_configured_request(shaped): - # 28 GB peak against a 9Gi configured request: the profile must raise it. - shaped(ranges=[(1, {'peakAnonBytes': 28_000_000_000})], - req_mem='9Gi', max_mem='32Gi') - got = sizing._profile_overrides(1, escalated=False)['memory'] - assert units.quantity_bytes(got) > units.quantity_bytes('9Gi') - - -def test_memory_is_sized_from_rss_never_from_working_set(shaped): - # Working set is whatever limit it was measured under -- the kernel grows - # page cache to fill it. Measured on ssc-test, one 420-ledger range: - # limit 4Gi -> ws 3.61 GiB, rss 2.43 GiB, 775s - # limit 8Gi -> ws 7.48 GiB, rss 2.41 GiB, 746s - # limit 24000Mi -> ws 13.49 GiB, rss 2.28 GiB, 773s - # rss is flat and wall-clock is flat, so sizing from ws would reserve 5x the - # real demand for no gain. It is still recorded -- kubelet ranks - # node-pressure evictions on it, so it explains an eviction rss cannot. - shaped(ranges=[(1, {'peakWorkingSetBytes': 13_000_000_000})]) - assert 'memory' not in sizing._profile_overrides(1, escalated=False), \ - "an older artifact without rss must fall back, not guess from working set" - assert 'peakWorkingSetBytes' in attempts.PEAK_FIELDS - - -def test_small_ranges_get_absolute_slack_not_just_a_percentage(shaped): - # memory.max bounds anon PLUS page cache. At 190 MiB rss a 1.1x margin is - # 19 MiB of slack -- measured on ssc-test, 90 ranges OOMKilled within 90s of - # dispatch. The fixed headroom is what makes small ranges survivable. - shaped(ranges=[(1, {'peakAnonBytes': 190 * MI})]) - got = units.quantity_bytes(sizing._profile_overrides(1, escalated=False)['memory']) - slack = (got - 190 * MI) / MI - assert slack > 400, f"only {slack:.0f}MiB of slack above rss" - - -@pytest.mark.parametrize('peak_mi', [648, 1467, 222]) # live: median, largest, smallest anon -def test_the_sizing_formula_is_peak_times_margin_plus_headroom(shaped, peak_mi): - shaped(ranges=[(1, {'peakAnonBytes': peak_mi * MI})], margin=1.15, - headroom='512Mi', max_mem='32Gi') - got = sizing._profile_overrides(1, escalated=False)['memory'] - assert got == f"{int(peak_mi * MI * 1.15) // MI + 512}Mi" - - -def test_runtime_insurance_is_weighted_by_the_longest_profiled_range(shaped): - shaped(ranges=[ - (1, {'peakAnonBytes': 1024 * MI, 'seconds': 100}), - (2, {'peakAnonBytes': 1024 * MI, 'seconds': 400}), - ], margin=1.15, headroom='512Mi', runtime_insurance='3Gi') - - short = units.quantity_bytes(sizing._profile_overrides(1, escalated=False)['memory']) - longest = units.quantity_bytes(sizing._profile_overrides(2, escalated=False)['memory']) - base = int(1024 * MI * 1.15) + 512 * MI - assert short == (base + 768 * MI) // MI * MI - assert longest == (base + 3 * 1024 * MI) // MI * MI - - -@pytest.mark.parametrize('seconds', [None, 0, -1, 'bad', float('nan'), float('inf')]) -def test_invalid_or_nonpositive_runtime_adds_no_insurance(shaped, seconds): - shaped(ranges=[(1, {'peakAnonBytes': 1024 * MI, 'seconds': seconds})], - margin=1.15, headroom='512Mi', runtime_insurance='3Gi') - got = units.quantity_bytes(sizing._profile_overrides(1, escalated=False)['memory']) - assert got == (int(1024 * MI * 1.15) + 512 * MI) // MI * MI - - -def test_zero_runtime_insurance_disables_it_and_the_cap_still_applies_last(shaped): - ranges = [(1, {'peakAnonBytes': 1024 * MI, 'seconds': 100})] - shaped(ranges=ranges, margin=1.15, headroom='512Mi', - runtime_insurance='0', max_mem='2Gi') - without = sizing._profile_overrides(1, escalated=False)['memory'] - assert without == f"{int(1024 * MI * 1.15) // MI + 512}Mi" - - shaped(ranges=ranges, margin=1.15, headroom='512Mi', - runtime_insurance='3Gi', max_mem='2Gi') - assert sizing._profile_overrides(1, escalated=False)['memory'] == '2048Mi' - - -# --- what lands on the container --------------------------------------------- - -def test_a_measured_range_requests_its_measurement_and_limits_only_disk(shaped): - # The profile moves requests. Disk is the one dimension still limited, and - # its limit is matched so a range measured to need more is allowed to use it. - shaped() - r = jm._resources(end=2000) - assert r.requests['memory'] == '3659Mi' - assert r.requests['ephemeral-storage'] == r.limits['ephemeral-storage'] == '6244Mi' - # The configured request, not a measured one -- a profiled range now packs - # at exactly the same cpu as an unprofiled one. - assert r.requests['cpu'] == '1800m' - assert set(r.limits) == {'ephemeral-storage'}, \ - f"a worker may only ever be limited on disk, got {sorted(r.limits)}" - - -def test_an_unmeasured_range_keeps_the_configured_requests(shaped): - # No profile entry must behave exactly as if there were no profile at all. - shaped() - r = jm._resources(end=99999) - assert r.requests['memory'] == '9Gi' - assert 'memory' not in r.limits - assert r.requests['ephemeral-storage'] == '35Gi' - assert r.limits['ephemeral-storage'] == '40Gi' - - -def test_an_escalated_retry_keeps_its_own_size(shaped): - # The escalation already chose the size; the profile must not overwrite it. - # It lands on the request, which is the whole mechanism now: a bigger request - # places the pod where the memory is actually free, and raises the bar before - # the kubelet picks it as an eviction victim. - shaped() - r = jm._resources(mem='36000Mi', end=2000) - assert r.requests['memory'] == '36000Mi' - assert 'memory' not in r.limits, "an escalated retry must not be capped either" - assert r.requests['cpu'] == '1800m', "cpu must fall back to the configured request" - - -def test_ephemeral_escalation_raises_request_and_limit_together(shaped): - # ephemeral-storage is a scheduling dimension: a pod that outgrew its limit - # will not fit where it was placed before unless the request moves too. - shaped() - r = jm._resources(eph='60Gi', end=2000) - assert r.requests['ephemeral-storage'] == r.limits['ephemeral-storage'] == '60Gi' - - -def test_no_worker_gets_a_cpu_or_memory_limit(shaped): - # _profile_overrides returns {} for BOTH "no profile entry" and "escalated - # attempt". Treating them the same handed an OOM retry more memory while - # capping it at LIM_CPU, when the attempt that just failed ran unlimited. - # Measured on ssc-test 2026-07-30: 256 of 679 a2 pods were capped at cpu 2. - # Less cpu means less download concurrency means a lower peak, so the retry - # succeeds at a figure the next run cannot reproduce unthrottled. - # - # At a 2-core limit every range pegs 2.0 anyway, so the measured peak would - # be a ceiling and the profile could never learn real demand. Packing is - # driven by the request, which every worker still carries. - shaped() - measured = jm._resources(end=2000) - escalated = jm._resources(mem='9000Mi', end=2000) - unmeasured = jm._resources(end=999999999) - for r, why in ((measured, 'measured'), (escalated, 'escalated retry'), - (unmeasured, 'unprofiled')): - assert 'cpu' not in r.limits, f"{why} range was throttled: {r.limits}" - assert 'memory' not in r.limits, f"{why} range was capped: {r.limits}" - assert r.requests['cpu'] == '1800m', why - - - -def test_pvc_mode_takes_no_ephemeral_request_or_override(shaped): - # /data is not on the node disk there, so sizing it would be meaningless -- - # and a large request would make disk the binding dimension and halve - # workers-per-node for no reason. - shaped(req_eph='') - r = jm._resources(end=2000) - assert 'ephemeral-storage' not in r.requests - - -def test_disk_gets_a_flat_headroom_and_a_runtime_weighted_share(shaped): - """Disk is sized like memory: measured peak, plus a floor, plus insurance. - - The 2026-08-01 ephemeral run peaked at 37.76Gi against a flat 40Gi limit -- - 6% of margin, on a detection-and-escalation path that has never fired on - real data. Margin alone does not fix that: it scales the measurement, so the - ranges closest to the limit get the least absolute headroom. - - Disk earns the runtime weighting the same way memory does -- measured across - 3985 ranges, peak disk tracks runtime at pearson 0.920 (runtime decile 0 - uses 0.1Gi, decile 9 uses 24.7Gi), so the weighting lands the allowance on - exactly the ranges that need it. - """ - shaped(eph_headroom='2Gi', eph_insurance='8Gi') - short = sizing._profile_overrides(2000, escalated=False)['ephemeral-storage'] - - # same range, no allowances at all -> margin only - shaped(eph_headroom='0', eph_insurance='0') - bare = sizing._profile_overrides(2000, escalated=False)['ephemeral-storage'] - - assert units.quantity_bytes(short) > units.quantity_bytes(bare) - assert units.quantity_bytes(short) - units.quantity_bytes(bare) >= 2 * 1024 ** 3 - - -def test_a_measured_range_may_exceed_the_flat_unprofiled_disk_limit(shaped): - """PROFILE_MAX_EPHEMERAL is above LIM_EPHEMERAL on purpose. - - LIM_EPHEMERAL is what an UNMEASURED range gets. Capping a measured range at - it would discard the measurement -- the worst range observed wants ~43Gi - after margin alone, which the flat 40Gi limit would silently clip back to - the value that was already too tight. - """ - shaped(lim_eph='40Gi', max_eph='64Gi', eph_headroom='2Gi', eph_insurance='8Gi') - out = sizing._profile_overrides(2000, escalated=False)['ephemeral-storage'] - assert units.quantity_bytes(out) > 0 - # and the cap still binds when it should - shaped(lim_eph='40Gi', max_eph='1Gi', eph_headroom='2Gi', eph_insurance='8Gi') - capped = sizing._profile_overrides(2000, escalated=False)['ephemeral-storage'] - assert units.quantity_bytes(capped) == 1024 ** 3 diff --git a/src/MissionParallelCatchup/tests/unit/test_resume_script.py b/src/MissionParallelCatchup/tests/unit/test_resume_script.py deleted file mode 100644 index 79849790..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_resume_script.py +++ /dev/null @@ -1,143 +0,0 @@ -"""The worker's resume decision, run as the shell script it actually is. - -Measured on ssc-test 2026-07-30: a1 replayed range 16752063 to its target -ledger and was evicted before it could exit 0. a2 resumed, found LCL == TARGET, -ran catchup against a DB with nothing left to apply, and stellar-core exited 2 --- deterministically, every attempt. The range exhausted its budget and the -mission aborted a 61%-complete 2096-worker run over work that had actually been -done. -""" - -import os -import re -import subprocess - -import pytest - -import job_monitor as jm - - -TARGET = 16752063 -COUNT = 16320 - -# What `stellar-core offline-info --console` really prints: bucketlist puts ~40 -# lines of hashes between the "ledger": key and the "num" the probe wants, which -# is why the probe must not window its grep. Verified against 27.1.1 on ssc-test -# 2026-07-30 -- exactly one "num" key in the document, and it is the ledger's. -def offline_info(lcl): - buckets = ',\n'.join(f' "{i:064x}"' for i in range(40)) - return ('{\n "info" : {\n "ledger" : {\n' - f' "age" : 3,\n "closeTime" : 1753000000,\n' - f' "hash" : "abc",\n' - f' "bucketListHashes" : [\n{buckets}\n ],\n' - f' "num" : {lcl},\n "version" : 22\n' - ' }\n }\n}') - - -@pytest.fixture -def run_resume(tmp_path): - """Run RESUME_SCRIPT against a stubbed stellar-core on a private /data.""" - def run(lcl, mark_matches=True, prev_log_lcl=None): - data = tmp_path / 'data' - data.mkdir(exist_ok=True) - bindir = tmp_path / 'bin' - bindir.mkdir(exist_ok=True) - stub = bindir / 'stellar-core' - info = offline_info(lcl).replace("'", "") if lcl is not None else '' - stub.write_text( - '#!/bin/sh\n' - 'for a in "$@"; do case "$a" in\n' - " offline-info) " + - (f"cat <<'EOF'\n{info}\nEOF\n" if lcl is not None else 'echo "{}"; ') + - ' exit 0;;\n' - ' new-db) echo "RAN:new-db" >> "$STUBLOG"; exit 0;;\n' - ' catchup) echo "RAN:catchup" >> "$STUBLOG"; exit 2;;\n' - 'esac; done\nexit 0\n') - stub.chmod(0o755) - - src = jm.RESUME_SCRIPT % {'key': f"{TARGET}/{COUNT}", - 'target': TARGET, 'count': COUNT} - src = src.replace('/usr/bin/stellar-core', str(stub)) - src = src.replace('/data/', str(data) + '/') - - if mark_matches: - (data / '.job-key').write_text(f"{TARGET}/{COUNT}") - if prev_log_lcl is not None: - (data / 'stellar-core.log').write_text( - f"Ledger close complete: {prev_log_lcl}\n") - - stublog = tmp_path / 'stub.log' - if stublog.exists(): - stublog.unlink() - env = dict(os.environ, STUBLOG=str(stublog)) - r = subprocess.run(['/bin/sh', '-c', src], capture_output=True, text=True, - env=env, timeout=30) - ran = stublog.read_text().split() if stublog.exists() else [] - return r.returncode, r.stdout, ran - return run - - -def test_a_range_already_at_its_target_exits_success_without_recatching(run_resume): - code, out, ran = run_resume(lcl=TARGET) - assert 'ALREADY COMPLETE' in out, out - assert code == 0, f"exit {code}; a finished range must not fail" - assert 'RAN:catchup' not in ran, "re-ran catchup on a completed range -> exit 2" - assert 'RAN:new-db' not in ran, "wiped a completed range" - - -def test_a_partially_replayed_range_still_resumes(run_resume): - code, out, ran = run_resume(lcl=TARGET - 100) - assert 'RESUME:' in out and 'ALREADY COMPLETE' not in out, out - assert 'RAN:catchup' in ran and 'RAN:new-db' not in ran, ran - - -def test_a_range_that_never_started_replay_starts_fresh(run_resume): - code, out, ran = run_resume(lcl=None) - assert 'RESUME DECLINED' in out, out - assert 'RAN:new-db' in ran and 'RAN:catchup' in ran, ran - - -def test_a_range_whose_replay_never_reached_its_own_span_starts_fresh(run_resume): - # Bucket apply uses createWithoutLoading() -- an unconditional INSERT that - # assumes a fresh DB -- so a crash before replay must start over. An LCL - # below TARGET-COUNT means the bucket phase, not replay. - code, out, ran = run_resume(lcl=TARGET - COUNT - 1) - assert 'RESUME DECLINED' in out, out - assert 'RAN:new-db' in ran - - -def test_the_lcl_probe_reads_past_the_bucketlist(run_resume): - # offline-info puts ~40 lines of bucketlist hashes between "ledger": and - # "num", so `grep -A8 '"ledger":'` yields nothing and the probe degrades to - # the log fallback silently -- shipped exactly that once. - code, out, ran = run_resume(lcl=TARGET - 100) - assert f"RESUME PROBE: offline-info reports lcl {TARGET - 100}" in out, out - - -def test_the_log_fallback_covers_a_core_that_answers_nothing(run_resume): - # Goes blind above INFO, which is why it is no longer the primary probe -- - # but a core that cannot answer offline-info still leaves its own log. - code, out, ran = run_resume(lcl=None, prev_log_lcl=TARGET - 50) - assert 'RESUME:' in out, out - assert 'RAN:new-db' not in ran - - -def test_a_volume_left_by_a_different_range_is_never_resumed_from(run_resume): - # /data is per-range, but a recycled volume or a mis-scheduled pod would - # otherwise resume a DB belonging to some other span. - code, out, ran = run_resume(lcl=TARGET - 100, mark_matches=False) - assert 'RESUME' not in out, out - assert 'RAN:new-db' in ran and 'RAN:catchup' in ran - - -def test_the_resume_script_survives_its_own_percent_formatting(): - # RESUME_SCRIPT is %-formatted with the range's key/target/count at dispatch. - # A bare % anywhere in it -- including in a comment -- raises at runtime and - # takes down every job dispatch. Nearly shipped exactly that: a comment - # reading "61%-complete". - jm.RESUME_SCRIPT % {'key': '123/456', 'target': 123, 'count': 456} # must not raise - # %% is a legitimate escape (printf '%%s'), so strip those pairs before - # looking for a stray one. - probe = jm.RESUME_SCRIPT.replace('%%', '') - stray = [m.start() for m in re.finditer(r"%(?!\()", probe)] - assert not stray, f"bare % near {probe[max(0, stray[0] - 40):stray[0] + 20]!r}" diff --git a/src/MissionParallelCatchup/tests/unit/test_retry_counters.py b/src/MissionParallelCatchup/tests/unit/test_retry_counters.py deleted file mode 100644 index 874b4e9e..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_retry_counters.py +++ /dev/null @@ -1,208 +0,0 @@ -"""Retry counters reconstructed from durable attempt state.""" - -import json - -from prometheus_client import generate_latest - -import config -import records -import metrics -import job_monitor as jm - - -def _write(path, value): - with open(path, 'w') as fh: - if isinstance(value, dict): - json.dump(value, fh) - else: - fh.write(value) - - -def _totals(progress=None, current_attempts=()): - return jm._retry_counter_totals(progress or {}, current_attempts) - - -def test_verdict_is_preferred_over_outcome(logdir): - _write(records.outcome_path(100, 1), {'outcome': 'disrupted'}) - _write(records.verdict_path(100, 1), 'oom') - - totals = _totals(current_attempts={('100', 2)}) - - assert totals['retries'] == 1 - assert totals['reasons']['oom'] == 1 - assert totals['reasons']['disrupted'] == 0 - assert totals['oom'] == 1 - assert totals['evicted'] == 0 - - -def test_legacy_outcome_is_used_when_no_verdict_exists(logdir): - _write(records.outcome_path(100, 1), {'outcome': 'disrupted'}) - - totals = _totals(current_attempts={('100', 2)}) - - assert totals['reasons']['disrupted'] == 1 - assert totals['evicted'] == 1 - assert totals['spot_disruption_retried'] == 1 - - -def test_matching_verdict_and_outcome_are_counted_once(logdir): - _write(records.outcome_path(100, 1), {'outcome': 'disrupted'}) - _write(records.verdict_path(100, 1), 'disrupted') - - totals = _totals(current_attempts={('100', 2)}) - - assert totals['reasons']['disrupted'] == 1 - assert totals['evicted'] == 1 - assert totals['spot_disruption_retried'] == 1 - - -def test_repeated_disruptions_of_one_range_count_as_one_retried_range(logdir): - for attempt in (1, 2, 3): - _write(records.verdict_path(100, attempt), 'disrupted') - - totals = _totals(current_attempts={('100', 4)}) - - assert totals['retries'] == 3 - assert totals['evicted'] == 3 - assert totals['reasons']['disrupted'] == 3 - assert totals['spot_disruption_retried'] == 1 - - -def test_disruptions_of_distinct_ranges_each_count_once(logdir): - for end in (100, 200): - _write(records.outcome_path(end, 1), {'outcome': 'disrupted'}) - _write(records.verdict_path(end, 1), 'disrupted') - - totals = _totals(current_attempts={('100', 2), ('200', 2)}) - - assert totals['evicted'] == 2 - assert totals['spot_disruption_retried'] == 2 - - -def test_active_successor_counts_before_range_progress_exists(logdir): - _write(records.verdict_path(100, 1), 'rejected') - - totals = _totals(current_attempts={('100', 1), ('100', 2)}) - - assert totals['retries'] == 1 - assert totals['reasons']['rejected'] == 1 - - -def test_reconcile_counts_the_successor_on_its_dispatch_pass(cluster): - cluster.reconcile() - cluster.advance(300, 'disrupted') - - cluster.reconcile() - - assert cluster.attempt_of(300) == 2 - assert cluster.state['counted']['retries'] == 1 - assert cluster.state['counted']['spot_disruption_retried'] == 1 - assert cluster.state['counted'][('reason', 'disrupted')] == 1 - assert not cluster.completed() - assert not cluster.failed() - - -def test_terminal_verdict_without_successor_is_not_a_retry(logdir): - _write(records.verdict_path(100, 1), 'oom') - - totals = _totals( - {'failed': {'100': {'attempts': 1, 'outcome': 'oom'}}}, - current_attempts={('100', 1)}) - - assert totals['retries'] == 0 - assert totals['oom'] == 0 - assert totals['reasons']['oom'] == 0 - - -def test_terminal_disruption_without_successor_is_only_a_raw_attempt(logdir): - _write(records.verdict_path(100, 1), 'disrupted') - - totals = _totals( - {'failed': {'100': {'attempts': 1, 'outcome': 'disrupted'}}}, - current_attempts={('100', 1)}) - - assert totals['evicted'] == 1 - assert totals['spot_disruption_retried'] == 0 - assert totals['reasons']['disrupted'] == 0 - - -def test_counter_sync_is_idempotent_and_replays_after_restart(logdir): - _write(records.verdict_path(100, 1), 'disrupted') - attempts = {('100', 2)} - retry_before = metrics.retries._value.get() - eviction_before = metrics.evictions._value.get() - unique_before = metrics.spot_disruption_retried._value.get() - reason_metric = metrics.retry_reasons.labels(reason='disrupted') - reason_before = reason_metric._value.get() - - counted = {} - jm.sync_counters({}, counted, attempts) - first = (metrics.retries._value.get(), - metrics.evictions._value.get(), - metrics.spot_disruption_retried._value.get(), - reason_metric._value.get()) - jm.sync_counters({}, counted, attempts) - assert (metrics.retries._value.get(), - metrics.evictions._value.get(), - metrics.spot_disruption_retried._value.get(), - reason_metric._value.get()) == first - - jm.sync_counters({}, {}, attempts) - assert metrics.retries._value.get() == retry_before + 2 - assert metrics.evictions._value.get() == eviction_before + 2 - assert metrics.spot_disruption_retried._value.get() == unique_before + 2 - assert reason_metric._value.get() == reason_before + 2 - - -def test_multiple_attempts_and_every_retry_reason(logdir): - for attempt, reason in enumerate(config.ATTEMPT_OUTCOMES, 1): - _write(records.verdict_path(100, attempt), reason) - - totals = _totals({'completed': {'100': { - 'attempts': len(config.ATTEMPT_OUTCOMES) + 1}}}) - - assert totals['retries'] == len(config.ATTEMPT_OUTCOMES) - assert totals['reasons'] == {reason: 1 for reason in config.ATTEMPT_OUTCOMES} - assert totals['evicted'] == 1 - assert totals['spot_disruption_retried'] == 1 - assert totals['oom'] == 1 - assert totals['ephemeral'] == 1 - - -def test_malformed_and_missing_records_do_not_invent_reasons(logdir): - _write(records.outcome_path(100, 1), {'outcome': 'disrupted'}) - _write(records.verdict_path(100, 1), 'not-a-verdict') - _write(records.outcome_path(200, 1), 'not-json') - _write(logdir / 'range-300-a1.verdict.tmp', 'oom') - _write(logdir / 'unrelated', 'disrupted') - - totals = _totals( - {'completed': 'malformed', 'failed': {'x': {'attempts': 'bad'}}}, - current_attempts={('100', 2), ('200', 2), ('bad', 'attempt')}) - - assert totals['retries'] == 2 - assert sum(totals['reasons'].values()) == 0 - assert totals['evicted'] == 0 - assert totals['spot_disruption_retried'] == 0 - assert totals['oom'] == 0 - - -def test_existing_and_reason_labelled_metrics_are_exported(): - for reason in config.ATTEMPT_OUTCOMES: - metrics.retry_reasons.labels(reason=reason) - text = generate_latest().decode() - - assert '# HELP ssc_parallel_catchup_job_retried_count_total ' \ - 'Retry attempts dispatched after a predecessor attempt failed' in text - assert '# HELP ssc_parallel_catchup_job_spot_eviction_count_total ' \ - 'Pod attempts classified as lost to node disruption' in text - assert '# HELP ssc_parallel_catchup_job_spot_disruption_retried_count_total ' \ - 'Unique ledger ranges that dispatched a successor after a node disruption verdict' in text - assert '# HELP ssc_parallel_catchup_job_oom_retried_count_total ' \ - 'Retry attempts dispatched after an OOM verdict, with an escalated memory limit' in text - assert '# HELP ssc_parallel_catchup_job_ephemeral_retried_count_total ' \ - 'Retry attempts dispatched after an ephemeral-storage verdict, with an escalated limit' in text - assert '# HELP ssc_parallel_catchup_job_retried_reason_count_total ' \ - 'Retry attempts dispatched, by the effective verdict of the predecessor attempt' in text - for reason in config.ATTEMPT_OUTCOMES: - assert f'ssc_parallel_catchup_job_retried_reason_count_total{{reason="{reason}"}}' in text diff --git a/src/MissionParallelCatchup/tests/unit/test_sizing.py b/src/MissionParallelCatchup/tests/unit/test_sizing.py deleted file mode 100644 index aa9b60a9..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_sizing.py +++ /dev/null @@ -1,227 +0,0 @@ -"""Resource escalation ladders and the quantity arithmetic under them. - -Everything here is a pure function of config, so the tests set the config and -call it. The budgets these ladders climb are asserted against the module -defaults -- the numbers a run gets when the chart passes nothing. -""" - -import pytest - -import config -import units -import sizing -import job_monitor as jm - - -@pytest.fixture -def mem(monkeypatch): - def configure(lim='1000Mi', bump=None, cap='48Gi'): - monkeypatch.setattr(config, 'REQ_MEM', lim) - monkeypatch.setattr(config, 'MEM_BUMP_FACTOR', - config.MEM_BUMP_FACTOR if bump is None else bump) - monkeypatch.setattr(config, 'MEM_ESCALATION_CAP', cap) - return configure - - -@pytest.fixture -def eph(monkeypatch): - def configure(lim='40Gi', bump=1.5, cap='200Gi'): - monkeypatch.setattr(config, 'LIM_EPHEMERAL', lim) - monkeypatch.setattr(config, 'EPH_BUMP_FACTOR', bump) - monkeypatch.setattr(config, 'EPH_ESCALATION_CAP', cap) - return configure - - -# --- quantity arithmetic ----------------------------------------------------- - -@pytest.mark.parametrize('quantity,want', [ - ('1024Ki', 1024 * 1024), - ('9Gi', 9 * 1024**3), - ('24000Mi', 24000 * 1024**2), - ('1G', 1000**3), # SI, not binary -- kubernetes accepts both - ('1500', 1500), # bare bytes -]) -def test_kubernetes_quantities_are_read_in_the_right_base(quantity, want): - assert units.quantity_bytes(quantity) == want - - -def test_a_size_is_always_rendered_back_in_mebibytes(): - # One unit everywhere means a limit can be compared to a request without - # re-parsing, and Mi is fine-grained enough for the packing this run does. - assert units.bytes_to_quantity(3 * 1024**3) == '3072Mi' - assert units.bytes_to_quantity(0) == '1Mi', "a zero-byte limit is unschedulable" - - -# --- the memory ladder ------------------------------------------------------- - -@pytest.mark.parametrize('attempt,want', [(1, 1.0), (2, 1.5), (3, 2.25), (4, 3.375)]) -def test_the_memory_escalation_ladder_compounds(mem, attempt, want): - # 1.5x per OOM off what the attempt actually ran with. A factor of 1.0 - # would retry an OOM at the identical limit, forever. - mem(lim='1000Mi', bump=1.5) - assert sizing.mem_for_attempt(attempt, '1000Mi') == f"{int(1000 * want)}Mi" - - -def test_the_escalation_ladder_is_capped(mem): - mem(lim='1000Mi', bump=1.5, cap='4Gi') - assert sizing.mem_for_attempt(20, '1000Mi') == '4096Mi', "cap not applied" - - -def test_oom_escalation_starts_from_what_the_attempt_actually_had(mem): - # Escalating a 209Mi profiled range off the configured 24000Mi limit jumps - # to 36000Mi -- a 172x overshoot that discards the packing win on first OOM. - mem(lim='24000Mi', bump=1.5) - assert sizing.mem_for_attempt(2, '702Mi') == '1053Mi' - assert sizing.mem_for_attempt(2) == '36000Mi' # unprofiled keeps old behaviour - - -# --- the disk ladder --------------------------------------------------------- - -def test_ephemeral_storage_escalates_and_caps_the_same_way(eph): - eph(lim='40Gi', bump=1.5, cap='200Gi') - assert sizing.eph_for_attempt(1) == '40960Mi' - assert sizing.eph_for_attempt(2) == '61440Mi' - assert sizing.eph_for_attempt(20) == '204800Mi', "cap not applied" - - -# --- the budgets the ladders are climbing ------------------------------------ - -def test_attempt_budgets_are_ordered_by_whose_fault_the_failure_was(): - # A genuinely broken range gets the middle budget. Anything the cluster did - # to us gets the most -- on spot, evictions are routine and must not condemn - # a range. A hang has no budget at all: a timeout is terminal. - assert config.ATTEMPT_BUDGETS['oom'] < config.ATTEMPT_BUDGETS['disrupted'], ( - f"budgets out of order: range={config.ATTEMPT_BUDGETS['oom']} " - f"disruption={config.ATTEMPT_BUDGETS['disrupted']}") - assert config.ATTEMPT_BUDGETS['oom'] > 1, "a range that OOMs once could never escalate" - assert config.ATTEMPT_BUDGETS['ephemeral'] > 1, "a range evicted on disk once could never grow" - # Effectively unlimited on purpose: a healthy spot range can be evicted - # dozens of times, and only a misclassification should ever reach the gate. - assert config.ATTEMPT_BUDGETS['disrupted'] >= 100, \ - "spot eviction would condemn ranges at this budget" - - -def test_the_oom_budget_stops_short_of_the_cap_on_purpose(): - # 5 rungs is 1.5^4 = 5x the profile figure. A range needing more is broken, - # not mis-sized, and chasing it to MEM_ESCALATION_CAP parks a whole node on - # it. The price is that such a range is condemned -- which today aborts the - # run, so this coupling is what must not be forgotten. - n = config.ATTEMPT_BUDGETS['oom'] - assert 2 <= n <= 8, f"{n} rungs: below 2 cannot escalate, above 8 chases a broken range" - assert config.MEM_BUMP_FACTOR ** (n - 1) >= 3.0, \ - "the ladder cannot even treble the request before giving up" - - -# --- the profile arithmetic -------------------------------------------------- -# -# Added 2026-08-06 after mutation testing: every line below was EXECUTED by the -# suite and none of it was asserted. A margin applied as a division, an -# escalation ladder running backwards, and an inverted runtime weighting all -# left the suite green. These pin the shapes, not just the outcomes. -# -# This path runs whenever POOL_PREFIX is unset -- the chart default -- so it -# sizes every worker on a run that does not pass --pubnet-parallel-catchup-pool- -# prefix. - -@pytest.fixture -def unpooled(monkeypatch): - """Profile-driven sizing with the tier ladder switched off.""" - def configure(entries, **overrides): - monkeypatch.setattr(config, 'POOL_PREFIX', '') - monkeypatch.setattr(config, 'PROFILE', sorted(entries.items())) - monkeypatch.setattr(config, '_SORTED_SECONDS', None) - for k, v in overrides.items(): - monkeypatch.setattr(config, k, v) - return configure - - -def test_each_escalation_rung_asks_for_more_than_the_one_below(mem, eph, monkeypatch): - """A divide where the bump multiplies makes an OOM retry ask for LESS. - - The ladder exists so an OOMing range gets a bigger node; running it - backwards retries the same range at a size it has already proved too small, - burning its whole budget without ever changing the outcome. - """ - monkeypatch.setattr(config, 'POOL_PREFIX', '') - mem(lim='1000Mi', bump=1.5, cap='48Gi') - eph(lim='40Gi', bump=1.5, cap='200Gi') - - for name, ladder in (('memory', sizing.mem_for_attempt), - ('ephemeral', sizing.eph_for_attempt)): - sizes = [units.quantity_bytes(ladder(a)) for a in range(1, 6)] - assert all(b > a for a, b in zip(sizes, sizes[1:])), \ - f"the {name} ladder does not climb: {sizes}" - - -def test_the_margin_multiplies_the_measured_peak(unpooled): - """Exact bytes, because `peak * 1.15` and `peak / 1.15` both "work". - - Dividing by the margin requests 76% of what the range was measured using -- - an OOM on the attempt the profile was supposed to make safe. - """ - rss = 2 * 1024 ** 3 - unpooled({300: {'peakAnonBytes': rss, 'seconds': 300.0}}) - - want = (int(rss * config.PROFILE_MARGIN) - + units.quantity_bytes(config.PROFILE_CACHE_HEADROOM) - + units.quantity_bytes(config.PROFILE_RUNTIME_MEMORY_INSURANCE)) - assert sizing._profile_overrides(300, escalated=False)['memory'] == \ - units.bytes_to_quantity(want) - - -def test_the_disk_margin_multiplies_too(unpooled): - disk = 20 * 1024 ** 3 - unpooled({300: {'peakEphemeralBytes': disk, 'seconds': 300.0}}, - LIM_EPHEMERAL='40Gi') - - want = (int(disk * config.PROFILE_MARGIN) - + units.quantity_bytes(config.PROFILE_EPHEMERAL_HEADROOM) - + units.quantity_bytes(config.PROFILE_RUNTIME_EPHEMERAL_INSURANCE)) - assert sizing._profile_overrides(300, escalated=False)['ephemeral-storage'] == \ - units.bytes_to_quantity(want) - - -def test_the_insurance_is_weighted_by_runtime_not_against_it(unpooled): - """The longest range gets the whole allowance; half as long gets half. - - An inverted ratio hands the most disk to the ranges least at risk of - running out of it, and the profile's own longest range the least. - """ - rss = 2 * 1024 ** 3 - unpooled({300: {'peakAnonBytes': rss, 'seconds': 300.0}, - 900: {'peakAnonBytes': rss, 'seconds': 600.0}}) - base = (int(rss * config.PROFILE_MARGIN) - + units.quantity_bytes(config.PROFILE_CACHE_HEADROOM)) - full = units.quantity_bytes(config.PROFILE_RUNTIME_MEMORY_INSURANCE) - - assert sizing._profile_overrides(900, escalated=False)['memory'] == \ - units.bytes_to_quantity(base + full), "the longest range gets all of it" - assert sizing._profile_overrides(300, escalated=False)['memory'] == \ - units.bytes_to_quantity(base + full // 2), "half the runtime, half the share" - - -def test_a_range_with_no_measured_runtime_gets_no_insurance(unpooled): - """Insurance is priced off time-at-risk, so an unknown runtime buys none. - - Inverting the bail spends the whole allowance on exactly the ranges nothing - is known about. - """ - rss = 2 * 1024 ** 3 - unpooled({300: {'peakAnonBytes': rss}}) - - want = (int(rss * config.PROFILE_MARGIN) - + units.quantity_bytes(config.PROFILE_CACHE_HEADROOM)) - assert sizing._profile_overrides(300, escalated=False)['memory'] == \ - units.bytes_to_quantity(want) - - -def test_a_zero_allowance_turns_the_insurance_off(unpooled): - """The documented way to disable it, so it has to reach zero exactly.""" - rss = 2 * 1024 ** 3 - unpooled({300: {'peakAnonBytes': rss, 'seconds': 300.0}}, - PROFILE_RUNTIME_MEMORY_INSURANCE='0') - - want = (int(rss * config.PROFILE_MARGIN) - + units.quantity_bytes(config.PROFILE_CACHE_HEADROOM)) - assert sizing._profile_overrides(300, escalated=False)['memory'] == \ - units.bytes_to_quantity(want) diff --git a/src/MissionParallelCatchup/tests/unit/test_tx_apply.py b/src/MissionParallelCatchup/tests/unit/test_tx_apply.py deleted file mode 100644 index b0fdaaf8..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_tx_apply.py +++ /dev/null @@ -1,253 +0,0 @@ -"""Reading 'ledger.transaction.apply' out of stellar-core's medida block. - -Two readers of one format the mission does not control: the collector's -streaming scanner, and the monitor's after-the-fact archive/pod reader. Both -are pinned against real captures so a stellar-core change fails here rather -than silently dropping the metric for a whole run. -""" - -import gzip -import json -import os - -import pytest - -import kube -import records -import medida -import attempts -import job_monitor as jm -import log_collector as lc - - -# stellar-core 27.1.1 catchup pod, --metric 'ledger.transaction.apply'. Kept -# whole: `sum` is 10 lines below the header against a 15-line scan window. -MEDIDA_BLOCK = """2026-07-28T18:39:49.350 GAJSL [default INFO] metric 'ledger.transaction.apply': -2026-07-28T18:39:49.350 GAJSL [default INFO] count = 20 -2026-07-28T18:39:49.350 GAJSL [default INFO] mean rate = 0.22136 calls/s -2026-07-28T18:39:49.350 GAJSL [default INFO] 1-minute rate = 0.113149 calls/s -2026-07-28T18:39:49.350 GAJSL [default INFO] 5-minute rate = 0.175948 calls/s -2026-07-28T18:39:49.350 GAJSL [default INFO] 15-minute rate = 0.191421 calls/s -2026-07-28T18:39:49.350 GAJSL [default INFO] min = 0.295417ms -2026-07-28T18:39:49.350 GAJSL [default INFO] max = 0.639873ms -2026-07-28T18:39:49.350 GAJSL [default INFO] mean = 0.417143ms -2026-07-28T18:39:49.350 GAJSL [default INFO] stddev = 0.108677ms -2026-07-28T18:39:49.350 GAJSL [default INFO] sum = 8.34285ms -2026-07-28T18:39:49.350 GAJSL [default INFO] median = 0ms -2026-07-28T18:39:49.350 GAJSL [default INFO] 75% = 0ms -2026-07-28T18:39:49.350 GAJSL [default INFO] 95% = 0ms -2026-07-28T18:39:49.350 GAJSL [default INFO] 98% = 0ms -2026-07-28T18:39:49.350 GAJSL [default INFO] 99% = 0ms -2026-07-28T18:39:49.350 GAJSL [default INFO] 99.9% = 0ms""" - -TX_APPLY_SECONDS = 0.00834285 - -# Real block from range-40010367-a1 on ssc-test. medida switches to scientific -# notation past 1e6 ms, which is every range with a real transaction load. -MEDIDA_BIG = """2026-07-29T20:11:16.931 GAJSL [default INFO] metric 'ledger.transaction.apply': -2026-07-29T20:11:16.931 GAJSL [default INFO] count = 3231886 -2026-07-29T20:11:16.931 GAJSL [default INFO] mean rate = 812.4 calls/s -2026-07-29T20:11:16.931 GAJSL [default INFO] 1-minute rate = 790.1 calls/s -2026-07-29T20:11:16.931 GAJSL [default INFO] 5-minute rate = 801.3 calls/s -2026-07-29T20:11:16.931 GAJSL [default INFO] 15-minute rate = 799.0 calls/s -2026-07-29T20:11:16.931 GAJSL [default INFO] min = 0.101ms -2026-07-29T20:11:16.931 GAJSL [default INFO] max = 41.2ms -2026-07-29T20:11:16.931 GAJSL [default INFO] mean = 0.404ms -2026-07-29T20:11:16.931 GAJSL [default INFO] stddev = 0.612ms -2026-07-29T20:11:16.931 GAJSL [default INFO] sum = 1.30722e+06ms""" - -BIG_SECONDS = 1307.22 - - -def scan(text): - s = lc.TxApplyScanner() - for line in text.splitlines(): - s.feed(line) - return s - - -# --- the streaming scanner ---------------------------------------------------- - -@pytest.mark.parametrize('block,want', [(MEDIDA_BLOCK, TX_APPLY_SECONDS), - (MEDIDA_BIG, BIG_SECONDS)]) -def test_the_scanner_reads_the_sum_out_of_the_block(block, want): - # Scientific notation was a silent 25% loss -- 91-99% of everything above - # ledger 35M -- because the old regex matched "1.30722" then required "ms" - # and found "e+06ms". The metric block was in the archive the whole time. - assert scan(block).seconds == pytest.approx(want) - - -def test_scanner_resumes_a_block_split_across_a_reconnect(): - # One scanner spans the poller's reconnect loop, so a drop mid-block must - # not lose the header already seen. - head, tail = MEDIDA_BLOCK.splitlines()[:4], MEDIDA_BLOCK.splitlines()[4:] - s = lc.TxApplyScanner() - for line in head: - s.feed(line) - assert s.seconds is None - for line in tail: - s.feed(line) - assert s.seconds == pytest.approx(TX_APPLY_SECONDS) - - -def test_scanner_ignores_sum_from_another_metric(): - s = scan("metric 'ledger.ledger.close':\n sum = 999999.0ms") - assert s.seconds is None - - -def test_scanner_gives_up_past_its_window_of_STATISTICS(): - # The window bounds how many medida statistics may sit between the header - # and the sum, so a release that adds percentiles is caught here rather than - # silently dropping tx_apply. - s = lc.TxApplyScanner() - s.feed("metric 'ledger.transaction.apply':") - for i in range(lc.TxApplyScanner.WINDOW + 5): - s.feed(f" {i}% = 1.5ms") - s.feed(" sum = 12.5555ms") - assert s.seconds is None - - -def test_interleaved_output_does_not_spend_the_window(): - # Measured on ssc-test 2026-08-04: a /info liveness response landed inside - # the block and pushed `sum` 91 lines below the header. Charging those lines - # made the scanner give up 76 lines short while the value sat in the archive - # -- one leg in 233, and job_monitor's re-read used the same span so its - # recovery path missed it too. - s = lc.TxApplyScanner() - s.feed("metric 'ledger.transaction.apply':") - s.feed(" count = 7641690") - for line in ('{', ' "info" : {', ' "build" : "stellar-core 27.1.1",', - ' "ledger" : {', ' "age" : 109870542,', - ' "baseFee" : 100,', ' "bucketlist" : [', - ' {', ' "curr" : "2a2cfe82",', - ' "snap" : "c12100ab"', ' },') * 8: - s.feed(line) - s.feed(" sum = 1.9501e+06ms") - assert s.seconds == pytest.approx(1950.1) - - -def test_a_block_whose_sum_never_arrives_cannot_claim_a_later_one(): - # Without a hard bound, skipping non-statistic lines would leave the scanner - # armed forever and let it read some other timer's sum as tx_apply. - s = lc.TxApplyScanner() - s.feed("metric 'ledger.transaction.apply':") - for _ in range(lc.TxApplyScanner.HARD_WINDOW + 10): - s.feed(' "noise" : 1,') - s.feed(" sum = 12.5555ms") - assert s.seconds is None - - -def test_a_different_metric_block_ends_the_search(): - s = lc.TxApplyScanner() - s.feed("metric 'ledger.transaction.apply':") - s.feed("metric 'ledger.close':") - s.feed(" sum = 12.5555ms") - assert s.seconds is None, "that sum belongs to ledger.close" - - -def test_rate_and_mean_lines_are_not_read_as_sum(): - for line in MEDIDA_BLOCK.splitlines(): - if 'rate =' in line or 'mean =' in line: - assert medida.SUM_RE.search(line) is None - - -def test_sum_stays_inside_the_scan_window(): - lines = MEDIDA_BLOCK.splitlines() - header = next(i for i, l in enumerate(lines) if 'ledger.transaction.apply' in l) - offset = next(i for i, l in enumerate(lines) if medida.SUM_RE.search(l)) - header - assert offset == 10, f"medida layout moved: sum is now {offset} lines below the header" - assert offset <= lc.TxApplyScanner.WINDOW - - -def test_resumed_is_read_from_the_workers_own_line(): - # "RESUME DECLINED" must not count as a resume -- it means the opposite, and - # the colon in RESUME_MARK is what separates the two. - s = lc.TxApplyScanner() - s.feed("RESUME DECLINED: k last close was 'none'; bucket phase incomplete, starting fresh") - assert s.resumed is False, "a declined resume was read as a resume" - s.feed("RESUME: k reached ledger 31005951, replay had started; skipping new-db") - assert s.resumed is True - - -def test_resumed_is_bookkeeping_and_never_becomes_a_measurement(): - # peaks_for_range needs it to tell a resumed tail from a complete pass; the - # profile must not see it as an axis. - assert 'resumed' not in attempts.PEAK_FIELDS - - -# --- the monitor's own reader ------------------------------------------------- - - -def test_tx_apply_comes_from_the_collectors_record_and_nowhere_else(logdir, monkeypatch): - """The monitor no longer parses stellar-core output for this at all. - - An archive sitting beside the record is not a second source: the collector - re-reads it with its own scanner at finalization, so a reader here would - repeat that work over the same bytes and could not disagree. - """ - class ExplodingPodLog: - def read_namespaced_pod_log(self, *_a, **_kw): - raise AssertionError("the monitor must not read pod logs for txApply") - monkeypatch.setattr(kube, 'core_v1', ExplodingPodLog()) - - _metrics(4000, 1, {'txApplySeconds': 99.0}) - assert attempts._tx_apply_for_attempt(4000, 1) == 99.0 - - os.remove(records.metrics_path(4000, 1)) - with gzip.open(records.log_path(4000, 1), 'wt') as fh: - fh.write(MEDIDA_BIG) - assert attempts._tx_apply_for_attempt(4000, 1) is None, \ - "an archive is the collector's to parse, not the monitor's" - - -def test_tx_apply_survives_a_reaped_pod(logdir): - # The record outlives the pod, and is now the only thing the monitor reads. - _metrics(4000, 1, {'txApplySeconds': 12.5}) - assert attempts.tx_apply_for_range(4000, 1) == 12.5 - - -def test_a_range_with_no_measurement_anywhere_reports_nothing(logdir): - assert attempts._tx_apply_for_attempt(4000, 1) is None - assert attempts.tx_apply_for_range(4000, 1) is None - - -def test_a_corrupt_archive_costs_this_range_its_metric_never_the_pass(logdir): - # EOFError from a truncated gzip member is not an OSError, so it used to - # escape the per-range work and abort the whole reconcile: no recording, no - # reap, no dispatch for any of ~4000 ranges, for as long as the torn bytes - # sat there. - with open(records.log_path(4000, 1), 'wb') as fh: - fh.write(gzip.compress(MEDIDA_BLOCK.encode())[:40]) - assert attempts._tx_apply_for_attempt(4000, 1) is None - - -# --- summing a resumed chain -------------------------------------------------- - -def _metrics(end, attempt, values): - """Write one attempt's .metrics, the way the collector leaves it.""" - with open(records.metrics_path(end, attempt), 'w') as fh: - json.dump(values, fh) - - -def test_tx_apply_sums_the_whole_resumed_chain(logdir): - # medida's total is per-process, so a pod that resumes at LCL+1 reports only - # the transactions it replayed -- the tail, not the range. - _metrics(4000, 1, {'txApplySeconds': 10.0}) - _metrics(4000, 2, {'txApplySeconds': 5.0, 'resumed': True}) - assert attempts.tx_apply_for_range(4000, 2) == 15.0 - - -def test_a_fresh_start_drops_the_earlier_legs_from_the_total(logdir): - # No RESUME line means new-db ran and this attempt redid the whole range; - # adding the interrupted attempt's figure would double-count the same work. - _metrics(4000, 1, {'txApplySeconds': 10.0}) - _metrics(4000, 2, {'txApplySeconds': 5.0}) - assert attempts.tx_apply_for_range(4000, 2) == 5.0 - - -def test_a_missing_predecessor_leg_makes_the_chain_total_absent(logdir): - # a1 has no record at all; a2 resumed from it and has none either. The sum - # of what survived is a lower bound, not the range's total. - _metrics(4000, 2, {'resumed': True, 'txApplySeconds': 3.0}) - assert attempts.tx_apply_for_range(4000, 2) is None, \ - "a chain missing a leg must report nothing, not the legs it has" diff --git a/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py b/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py deleted file mode 100644 index 6b8f599f..00000000 --- a/src/MissionParallelCatchup/tests/unit/test_worker_liveness.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Worker responsiveness: one concurrent sweep per reconcile pass. - -Driven against real sockets rather than a faked session -- the whole behaviour is -"what did stellar-core's admin port actually answer", so a fake that returns -whatever the test wants proves very little. -""" - -import asyncio -import contextlib -import os -import socket -import subprocess -import sys -import threading -import time -from http.server import BaseHTTPRequestHandler, HTTPServer - -import pytest -from kubernetes import client - -import config -import worker_liveness -import job_monitor as jm - - -def _server(status=200, delay=0.0, counter=None): - """A local HTTP endpoint standing in for stellar-core's admin port.""" - class Handler(BaseHTTPRequestHandler): - def do_GET(self): - if counter is not None: - counter.enter() - if delay: - time.sleep(delay) - self.send_response(status) - self.end_headers() - self.wfile.write(b'{}') - if counter is not None: - counter.leave() - - def log_message(self, *_a): - pass - - srv = HTTPServer(('127.0.0.1', 0), Handler) - srv.daemon_threads = True - threading.Thread(target=srv.serve_forever, daemon=True).start() - return srv - - -class _Concurrency: - """Server-side count of overlapping requests, and its high-water mark.""" - - def __init__(self): - self.lock = threading.Lock() - self.now = 0 - self.peak = 0 - - def enter(self): - with self.lock: - self.now += 1 - self.peak = max(self.peak, self.now) - - def leave(self): - with self.lock: - self.now -= 1 - - -def _targets(port, count=1): - return {f"uid-{i}": (f"pod-{i}", '127.0.0.1') for i in range(count)} - - -@contextlib.contextmanager -def _serving(monkeypatch, **kw): - """Point the module's admin port at a local endpoint for the duration.""" - srv = _server(**kw) - monkeypatch.setattr(worker_liveness, '_ADMIN_PORT', srv.server_address[1]) - try: - yield srv - finally: - srv.shutdown() - - -def _closed_port(): - s = socket.socket() - s.bind(('127.0.0.1', 0)) - port = s.getsockname()[1] - s.close() - return port - - -def _sweep(targets, port, **kw): - """Run a sweep against a chosen port by pointing the module's URL at it.""" - kw.setdefault('timeout', 2) - kw.setdefault('deadline', 5) - kw.setdefault('concurrency', 8) - return asyncio.run(worker_liveness.sweep(targets, **kw)) - - -# --- what counts as up -------------------------------------------------------- - -@pytest.mark.parametrize('status, verdict', [ - (200, 'up'), - # A busy core used to count as up. "Answered, badly" is not answering, and - # nothing downstream smooths it. - (503, 'down'), - # Not just 5xx: `status < 500` passes the 503 case while counting a wrong - # path or a proxy in the way as a healthy core. - (404, 'down'), -]) -def test_only_a_200_is_up(monkeypatch, status, verdict): - with _serving(monkeypatch, status=status): - counts = _sweep(_targets(0, 2), None) - assert counts[verdict] == 2 and sum(counts.values()) == 2 - - -def test_a_refused_connection_is_down(monkeypatch): - monkeypatch.setattr(worker_liveness, '_ADMIN_PORT', _closed_port()) - assert _sweep(_targets(0, 2), None) == {'up': 0, 'down': 2, 'unknown': 0} - - -def test_a_probe_slower_than_its_timeout_is_down(monkeypatch): - with _serving(monkeypatch, status=200, delay=1.0): - assert _sweep(_targets(0, 1), None, timeout=0.2) == \ - {'up': 0, 'down': 1, 'unknown': 0} - - -def test_no_targets_is_not_a_sweep(): - assert worker_liveness.publish({}) == {'up': 0, 'down': 0, 'unknown': 0} - - -# --- the bounds the reconcile loop depends on -------------------------------- - -def test_the_sweep_stops_at_its_deadline_and_reports_the_rest_unknown(monkeypatch): - """The reconcile loop waits for this, so it must be bounded by wall clock. - - Ten pods that each hang for a second, two at a time, is five seconds of work. - With a half-second deadline the sweep keeps what finished and calls the rest - unknown rather than making dispatch wait. - """ - with _serving(monkeypatch, status=200, delay=1.0): - started = time.monotonic() - counts = _sweep(_targets(0, 10), None, concurrency=2, timeout=5, deadline=0.5) - elapsed = time.monotonic() - started - assert elapsed < 2.0, f"the sweep ran {elapsed:.2f}s past a 0.5s deadline" - assert counts['unknown'] >= 6, counts - assert sum(counts.values()) == 10, "every target must be accounted for" - - -def test_concurrency_is_bounded_at_the_server(monkeypatch): - counter = _Concurrency() - with _serving(monkeypatch, status=200, delay=0.05, counter=counter): - counts = _sweep(_targets(0, 60), None, concurrency=4, timeout=5, deadline=10) - assert counts == {'up': 60, 'down': 0, 'unknown': 0} - assert counter.peak <= 4, f"{counter.peak} overlapping requests, limit 4" - - -def test_one_unreachable_pod_does_not_discard_the_others(monkeypatch): - """Why this is asyncio.wait and not a TaskGroup. - - A TaskGroup cancels its siblings when a task raises. Every other answer has - to survive one pod being unreachable, so four pods point at a live endpoint - and one at a loopback address with nothing bound. - """ - with _serving(monkeypatch, status=200): - targets = {f"uid-{i}": (f"pod-{i}", '127.0.0.1') for i in range(4)} - targets['uid-dead'] = ('pod-dead', '127.0.0.2') - counts = _sweep(targets, None, timeout=1) - assert counts == {'up': 4, 'down': 1, 'unknown': 0} - - -def test_publish_reports_every_target_unknown_when_the_sweep_itself_fails(monkeypatch): - """The production call path: job_monitor calls publish(targets) and nothing else.""" - async def boom(*_a, **_kw): - raise RuntimeError("no event loop for you") - monkeypatch.setattr(worker_liveness, 'sweep', boom) - assert worker_liveness.publish(_targets(0, 7)) == \ - {'up': 0, 'down': 0, 'unknown': 7} - - -# --- candidate selection ------------------------------------------------------ - -def test_only_running_pods_with_ips_are_candidates_and_uid_is_identity(): - def pod(name, uid, phase, ip): - return client.V1Pod( - metadata=client.V1ObjectMeta(name=name, uid=uid), - status=client.V1PodStatus(phase=phase, pod_ip=ip)) - - targets = worker_liveness.targets([ - pod('ready', 'uid-ready', 'Running', '10.0.0.1'), - pod('pending', 'uid-pending', 'Pending', '10.0.0.2'), - pod('no-ip', 'uid-no-ip', 'Running', None), - ]) - assert targets == {'uid-ready': ('ready', '10.0.0.1')} - - -def test_malformed_liveness_configuration_fails_with_an_explicit_message(monkeypatch, tmp_path): - """A bad value is rejected at /start, not at import. - - Coercing at import made this a boot crash, and a process that cannot start - cannot say why -- the driver polled a pod that never answered and timed out - 600s later with "not reachable". Now it comes back as a 400 with the reason. - """ - monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(config, 'RUN_PATH', str(tmp_path / 'run.json')) - monkeypatch.setattr(config, 'LIVENESS_MAX_CONCURRENCY', 'many') - - with pytest.raises(ValueError, match='LIVENESS_MAX_CONCURRENCY must be an integer'): - jm.start_run({"range": {'startingLedger': 0, 'latestLedgerNum': 1000, 'ledgersPerJob': 100}}) - - -def test_liveness_numbers_are_coerced_once_validation_passes(monkeypatch, tmp_path): - """Callers must never see the string form; validate_config rebinds them.""" - monkeypatch.setattr(config, 'LOG_DIR', str(tmp_path)) - monkeypatch.setattr(config, 'RUN_PATH', str(tmp_path / 'run.json')) - monkeypatch.setattr(config, 'LIVENESS_MAX_CONCURRENCY', '8') - monkeypatch.setattr(config, 'LIVENESS_SWEEP_SECONDS', '2.5') - - jm.start_run({"range": {'startingLedger': 0, 'latestLedgerNum': 1000, 'ledgersPerJob': 100}}) - - assert config.LIVENESS_MAX_CONCURRENCY == 8 - assert config.LIVENESS_SWEEP_SECONDS == 2.5 - - -def test_a_blocked_sweep_does_not_delay_dispatch(cluster, monkeypatch): - """Dispatch must not wait on the fleet answering. - - The sweep is bounded by its deadline, and reconcile pays that at most once - per pass -- so this pins the cost rather than the independence the old - background sampler gave. - """ - monkeypatch.setattr(config, 'LIVENESS_SWEEP_SECONDS', 0.3) - with _serving(monkeypatch, status=200, delay=5.0): - started = time.monotonic() - counts = worker_liveness.publish(_targets(0, 50)) - elapsed = time.monotonic() - started - assert elapsed < 1.5, f"publish took {elapsed:.2f}s against a 0.3s deadline" - assert counts['unknown'] > 0 - assert sum(counts.values()) == 50 - - started = time.monotonic() - cluster.reconcile() - assert time.monotonic() - started < 1.0 From 277632f7cbdf9e0d2b6192923dfdc7e858933d32 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 12:24:54 -0400 Subject: [PATCH 080/117] catchup: gate all chart RBAC behind monitor.createRbac The monitor's ServiceAccount is now created unconditionally -- creating one grants nothing, so it needs no RBAC rights from whoever installs the chart -- and the Deployment names it directly. What it may do comes from a binding the namespace provides. createRbac now also renders the ClusterRole and ClusterRoleBinding for nodes/stats, so a cluster where the installer is admin gets a self-sufficient run. Without it the collector's kubelet sampling 403s and the run silently loses its pod disk metrics. - monitor SA carries no IRSA annotation: the trust policy is a namespace-scoped name wildcard, so it would have matched the monitor too - closes the strict xfail covering the missing nodes/stats grant - pins the cluster-wide surface to exactly {nodes/stats} Co-Authored-By: Claude Opus 5 --- .../templates/job_monitor.yaml | 52 ++++++++++++++----- .../parallel_catchup_helm/values.yaml | 12 +++++ 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 202cfb87..5a2e0b30 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -1,19 +1,12 @@ +# The monitor's identity. Created unconditionally -- creating a ServiceAccount +# grants nothing, so it needs no RBAC rights from whoever installs the chart. +# What it may do comes from a RoleBinding the namespace provides, granted to +# every ServiceAccount in the namespace; see monitor.createRbac for clusters +# without one. No IRSA annotation: the S3 mirror is the workers' business. apiVersion: v1 kind: ServiceAccount metadata: name: {{ .Release.Name }}-job-monitor - {{- with .Values.service_account.annotations }} - annotations: - {{- /* The mission sends an indexed array of {key,value}; a plain map is - also accepted so the chart stays usable standalone. */}} - {{- if kindIs "slice" . }} - {{- range . }} - {{ .key }}: {{ .value | quote }} - {{- end }} - {{- else }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- end }} --- # Worker pods run under this, not the monitor's SA: IRSA trust for the S3 # history mirror is bound to this name/namespace. Same name the StatefulSet @@ -35,6 +28,7 @@ metadata: {{- end }} {{- end }} --- +{{- if .Values.monitor.createRbac }} # Namespaced and delete-free. The monitor creates Jobs and PVCs and reads their # state; it never patches finalizers and has no access to PersistentVolumes, so # it cannot orphan an EBS volume. Job/PVC cleanup happens through @@ -82,6 +76,40 @@ subjects: - kind: ServiceAccount name: {{ .Release.Name }}-job-monitor --- +# The collector samples kubelet /stats/summary directly on :10250 for per-pod +# disk use. nodes is cluster-scoped, so this pair has to be too -- which is why +# ssc-eks provisions it outside the chart: creating a ClusterRole needs the +# installer to already hold nodes/stats itself, on top of cluster-wide RBAC +# rights. Only a cluster where the installer is admin can render this. +# +# Named after the release because cluster-scoped objects share one namespace: +# a fixed name would let one run's uninstall delete another run's access. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-node-stats-reader +rules: + - apiGroups: [""] + resources: ["nodes/stats"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-node-stats-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Name }}-node-stats-reader +subjects: + # The collector is a sidecar in the monitor pod, so this is its account. The + # cluster-provisioned copy binds the whole namespace group instead, because it + # cannot know a name derived from the release. + - kind: ServiceAccount + name: {{ .Release.Name }}-job-monitor + namespace: {{ .Release.Namespace }} +{{- end }} +--- # Holds every worker's log for the run. The monitor is the only long-lived pod, # which is what the driver used to rely on the StatefulSet workers for; with a # Job per range there is nothing else that outlives the work. diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 911a40f7..cb5cf9cc 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -78,6 +78,18 @@ monitor: # The driver reaches the monitor through this route: profile in via POST # /start, status and logs out. Empty routeHost disables the HTTPRoute; the # Service still exists, so an in-cluster caller works either way. + # The monitor's Role is the same for every run, so it is provisioned once per + # namespace in stellar/kube, bound to system:serviceaccounts: -- + # every ServiceAccount in the namespace, because a binding names its subject + # exactly and the monitor's is named after the release. Creating the Role per + # release instead needs the installer to hold RBAC-granting rights, which is a + # privilege every mission would then carry so that this one chart can install + # its own Role. + # + # true creates the Role and RoleBinding for a cluster where that provisioning + # does not exist -- a k3d/kind run, say. The ServiceAccount is created either + # way; creating one grants nothing. + createRbac: false routeHost: "" gatewayName: "" gatewayNamespace: "" From a0cec3cde57b557d03c002ea0fbb3ce130eb485f Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 14:03:56 -0400 Subject: [PATCH 081/117] Add a worker memory-request override and delete dead PCv2 code --pubnet-parallel-catchup-mem-request is the counterpart to the existing cpu-request flag. Memory is what actually bounds packing on an unpooled run -- 1800m/9Gi lands 4 workers on an r8*.2xlarge but 3 on an m8*.2xlarge, and it is the 9Gi that decides the second case -- but only cpu was settable per run, so tuning it meant editing values.yaml. Default is empty rather than a literal 9Gi, so the chart stays the single place the number is written down. Both flags' help text now says they are unpooled-only. Under a pool prefix pool_cpu/pool_memory return the tier's own cut and neither flag reaches a worker; believing otherwise is what shipped a 6780m request to a 4-vCPU protostar pool on 2026-08-04 and left those pods permanently Pending. The cpu text also claimed to be "the ceiling a profile-derived cpu request is clamped to" -- there is no profile-derived cpu any more, PROFILE_CPU_TIERS and _slack_cpu are both gone, and even when they existed nothing clamped. Deletions, all in the mission: - open System.Formats.Tar, dead since logs moved to per-file HTTP, and a duplicate open System - jobMonitorStatusKey and queryJobMonitor's `key` parameter, which the function already documented as vestigial and discarded before GETting /status - comments describing the status ConfigMap and tar-over-exec as current. The worst was on rangeProfileDocument, which blamed empty profiles on readProgressRecord falling back to the progress ConfigMap: that fallback no longer exists, so it would send someone chasing a cause that cannot happen. - the "comment out the path below for local testing" instruction and the commented-out path, obsolete since SUPERCLUSTER_CHART_PATH - a KUBECONFIG set in resolveRangeProfile, which makes no kube call and runs after Program.fs has already set it for the whole mission verb --- src/App/Program.fs | 12 +++- src/FSLibrary.Tests/Tests.fs | 1 + .../MissionHistoryPubnetParallelCatchupV2.fs | 59 +++++-------------- src/FSLibrary/StellarMissionContext.fs | 1 + 4 files changed, 28 insertions(+), 45 deletions(-) diff --git a/src/App/Program.fs b/src/App/Program.fs index 863050d5..7911d077 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -121,6 +121,7 @@ type MissionOptions pubnetParallelCatchupPoolPrefix: string, jobMonitorImagePcV2: string, pubnetParallelCatchupCpuRequest: string, + pubnetParallelCatchupMemRequest: string, tag: string option, numPregeneratedTxs: int option, genesisTestAccountCount: int option, @@ -528,7 +529,7 @@ type MissionOptions member self.PubnetParallelCatchupNumWorkers = pubnetParallelCatchupNumWorkers [] member self.PubnetParallelCatchupStorageMode : string = pubnetParallelCatchupStorageMode @@ -558,11 +559,17 @@ type MissionOptions member self.JobMonitorImagePcV2 : string = jobMonitorImagePcV2 [] member self.PubnetParallelCatchupCpuRequest : string = pubnetParallelCatchupCpuRequest + [] + member self.PubnetParallelCatchupMemRequest : string = pubnetParallelCatchupMemRequest + [] member self.Tag = tag @@ -946,6 +953,7 @@ let main argv = pubnetParallelCatchupPoolPrefix = mission.PubnetParallelCatchupPoolPrefix jobMonitorImagePcV2 = mission.JobMonitorImagePcV2 pubnetParallelCatchupCpuRequest = mission.PubnetParallelCatchupCpuRequest + pubnetParallelCatchupMemRequest = mission.PubnetParallelCatchupMemRequest tag = mission.Tag numPregeneratedTxs = mission.NumPregeneratedTxs enableTailLogging = true diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index 9699be45..e2394dfe 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -127,6 +127,7 @@ let ctx : MissionContext = pubnetParallelCatchupPoolPrefix = "" jobMonitorImagePcV2 = "" pubnetParallelCatchupCpuRequest = "" + pubnetParallelCatchupMemRequest = "" tag = None numPregeneratedTxs = None enableTailLogging = true diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index a519951e..a841a762 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -16,12 +16,10 @@ open System open System.Diagnostics open System.Net.Http open System.IO -open System.Formats.Tar open Newtonsoft.Json.Linq open Microsoft.FSharp.Control open System.Threading -open System open k8s open CSLibrary @@ -35,10 +33,8 @@ let helmChartPath = | "" -> "/supercluster/src/MissionParallelCatchup/parallel_catchup_helm" | p -> p -// Comment out the path below for local testing // Example command to run local testing (in the `supercluster/` directory): // $ dotnet run --project src/App/App.fsproj -- mission HistoryPubnetParallelCatchupV2 --image=docker-registry.services.stellar-ops.com/dev/stellar-core:23.0.3-2779.4d1df2b03.jammy-vnext-buildtests --pubnet-parallel-catchup-num-workers=2 --pubnet-parallel-catchup-starting-ledger=0 --pubnet-parallel-catchup-end-ledger=6400 --pubnet-parallel-catchup-ledgers-per-job 1280 --destination ./logs -// let helmChartPath = "src/MissionParallelCatchup/parallel_catchup_helm" let valuesFilePath = helmChartPath + "/values.yaml" // Layered on top of values.yaml for on-demand runs only. The chart defaults are // the spot claims: the spot pools were doubled on 2026-08-04 so each claim is @@ -48,12 +44,8 @@ let valuesFilePath = helmChartPath + "/values.yaml" // in on a 16 GiB node once the EKS reserve and 154Mi of daemonsets come out. let onDemandValuesFilePath = helmChartPath + "/values-ondemand.yaml" -// Keys in the -catchup-progress ConfigMap. These were HTTP paths when -// the driver polled the monitor through a Gateway; it reads the ConfigMap now. -let jobMonitorStatusKey = "status.json" // live queue counts - let jobMonitorLoggingIntervalSecs = 30 // frequency of the monitor reconcile loop: dispatch, liveness ping, status publish -let jobMonitorStatusCheckIntervalSecs = 60 // frequency of us reading the monitor's progress ConfigMap +let jobMonitorStatusCheckIntervalSecs = 60 let jobMonitorStatusCheckTimeOutSecs = 600 let mutable toPerformCleanup = true let failedJobLogFileLineCount = 10000 @@ -62,11 +54,11 @@ let failedJobLogStreamLineCount = 1000 let mutable nonce : String = "" let mutable helmReleaseName : String = "" -// Resolve --pubnet-parallel-catchup-profile into a ConfigMap the monitor mounts. +// Resolve --pubnet-parallel-catchup-profile into the profile body POSTed to /start. // // Accepts a local path or an https URL, so a profile can come off disk or -// straight from a raw paste/gist link. Returns the ConfigMap name, or None to -// size from the configured requests. +// straight from a raw paste/gist link. Returns None to size from the +// configured requests. // // Never fatal: a profile only tightens requests, so a run must still start when // one cannot be fetched. Failing the mission here would turn an optimisation @@ -77,11 +69,6 @@ let resolveRangeProfile (context: MissionContext) : string option = if String.IsNullOrWhiteSpace spec then None else - // The options are built before the helm install sets this, and the - // ConfigMap has to land in the same cluster and namespace the release - // will use. - Environment.SetEnvironmentVariable("KUBECONFIG", ExpandHomeDirTilde context.kubeCfg) - try let body = if spec.StartsWith("https://", StringComparison.OrdinalIgnoreCase) then @@ -329,18 +316,18 @@ let installProject (context: MissionContext) = storageReqGibi storageLimGibi - // This is the DEFAULT cpu request, not a ceiling. The monitor does NOT clamp - // profile-derived cpu to it: _slack_cpu returns the tier value straight - // through, so a PROFILE_CPU_TIERS band above this value really is issued -- - // verified 2026-07-31, tiers of 1.5 and 2.0 rendered under a 1250m REQ_CPU. - // It applies to ranges the profile cannot size at all. - // Pushed only when the run explicitly asks. Otherwise the chart default stands, - // so the chart is the single place V2's worker sizing is written down. + // Both pushed only when the run explicitly asks. Otherwise the chart default + // stands, so the chart is the single place V2's worker sizing is written down. if not (String.IsNullOrWhiteSpace context.pubnetParallelCatchupCpuRequest) then setOptions.Add( sprintf "worker.resources.requests.cpu=%s" context.pubnetParallelCatchupCpuRequest ) + if not (String.IsNullOrWhiteSpace context.pubnetParallelCatchupMemRequest) then + setOptions.Add( + sprintf "worker.resources.requests.memory=%s" context.pubnetParallelCatchupMemRequest + ) + setOptions.Add(sprintf "worker.resources.requests.ephemeral_storage=%s" storageReqGibi) setOptions.Add(sprintf "worker.resources.limits.ephemeral_storage=%s" storageLimGibi) @@ -452,11 +439,6 @@ let installProject (context: MissionContext) = | Some valuesOutput -> LogInfo "%s" valuesOutput | _ -> () -// Collect log files from all parallel catchup worker pods -// This function: -// 1. Automatically determines worker pod names from context.pubnetParallelCatchupNumWorkers -// 2. For each pod, finds all files matching "stellar-core-*.log" in /data -// 3. Creates a tar.gz archive and copies it to context.destination directory // How often the main loop pulls logs. Every 10 minutes flattens the teardown // cost -- which measured ~20 minutes for a full run, all of it after the work // finished -- without the pull competing with the collector for the volume. @@ -555,9 +537,7 @@ let collectLogsFromPods (context: MissionContext) = // collection and leak every worker pod, which is what we saw in practice // with a 1024-worker run aborted from Jenkins. -let queryJobMonitor (context: MissionContext, key: String) = - // `key` is vestigial: /status returns the one document the ConfigMap used - // to hold under that key. +let queryJobMonitor (context: MissionContext) = try use client = monitorClient context let body = client.GetStringAsync("/status") |> Async.AwaitTask |> Async.RunSynchronously @@ -568,8 +548,8 @@ let queryJobMonitor (context: MissionContext, key: String) = None -// Emit what this run measured, next to the worker-log tar, so a later run can -// be given tighter per-range requests. An artifact rather than a ConfigMap or +// Emit what this run measured, so a later run can be given tighter +// per-range requests. An artifact rather than a ConfigMap or // an S3 object: nothing for ArgoCD to reconcile, no second writer racing a // concurrent mission, and not bounded by Prometheus retention. // Fields carried from the monitor's progress record into the profile artifact. @@ -607,11 +587,6 @@ let projectRangeEntry (record: JObject) : JObject = // The progress record, read only from the monitor's volume. -// -// Not from the ConfigMap: that is a visibility mirror with every profiling -// field stripped and a 1 MiB cap (~6100 ranges, reachable by halving -// ledgersPerJob). A profile built from it would be empty but look complete, and -// past the cap it stops updating while /logs/progress.json stays correct. // No record is the safe outcome -- the consumer falls back to its defaults. let readProgressRecord (context: MissionContext) : JObject option = let monitorPods = @@ -740,9 +715,7 @@ let rangeProfileDocument (storageMode: string) (defaultLedgersPerRange: int) (co // A profile with no measurements is worse than no profile at all: it looks // complete, so nothing downstream can tell it from a good one, and the next - // run sizes itself from empty data. The usual cause is readProgressRecord - // falling back to the progress ConfigMap, which is a state mirror with - // every profiling field stripped. Writing nothing lets the consumer fall + // run sizes itself from empty data. Writing nothing lets the consumer fall // back to its configured defaults, which is the safe outcome. if ranges.Count = 0 then None else Some doc @@ -909,7 +882,7 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = while not allJobsFinished do Thread.Sleep(jobMonitorStatusCheckIntervalSecs * 1000) - let statusOpt = queryJobMonitor (context, jobMonitorStatusKey) + let statusOpt = queryJobMonitor context try match statusOpt with diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 60a10d1c..62f81a10 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -125,6 +125,7 @@ type MissionContext = pubnetParallelCatchupPoolPrefix: string jobMonitorImagePcV2: string pubnetParallelCatchupCpuRequest: string + pubnetParallelCatchupMemRequest: string genesisTestAccountCount: int option asanOptions: string option From 39a9f0132a95b2c732d05dbac0af59a86c44cec4 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 14:30:15 -0400 Subject: [PATCH 082/117] Decide cache-bump rungs from a block list, not from node vCPU The cache bump promoted a range one tier when its working set would not fit, then refused any rung whose POOL_VCPU differed, then re-allowed the rungs named in POOL_CROSS_RUNGS. Three mechanisms where the last two only ever cancelled each other: POOL_CROSS_RUNGS had exactly one reader, inside the vCPU comparison, so it existed solely to undo answers that comparison got wrong. And it did get them wrong. POOL_VCPU reads the SMALLEST shape a pool can land on, so it misprices any tier spanning node sizes -- an x8i.xlarge at w80 sat under two 8-vCPU rungs at w100/w90, so the map said 4, supergiant->hypergiant priced as free, and the promotion really cost 4->8. A block list cannot misprice anything; it says what it says. Behaviour-preserving. Across the whole ladder the two mechanisms disagree on exactly one rung -- dwarf->subgiant, identically on spot and on-demand -- so naming it in POOL_BLOCK_RUNGS reproduces every decision: subdwarf->dwarf allow (unchanged) dwarf->subgiant BLOCK (was the vCPU guard, now the list) subgiant->giant allow (unchanged) giant->supergiant allow (unchanged) supergiant->hypergiant allow (was POOL_CROSS_RUNGS, now nothing) hypergiant->supernova BLOCK (unchanged) The trade is that the default flips from deny to allow: a rung the guard would have refused automatically now has to be named. test_every_rung_decides_ exactly_as_it_did_under_the_vcpu_guard pins all six so a ladder change cannot quietly flip one, and both it and the dwarf test fail if the entry is dropped. Deleted: pool_vcpu, _crossing_allowed, the guard, POOL_VCPU and POOL_CROSS_RUNGS from config and the chart env, poolVcpu and poolCrossRungs from both values files, and four tests whose whole subject was the guard. Node vCPUs stay in the test module as a local table -- the node-fit tests still need them, the sizing code no longer does. values-ondemand.yaml also loses its poolBlockRungs, which was byte-identical to the base and would have overridden the new entry and silently dropped dwarf->subgiant on every on-demand run. The overlay is now four keys: storageMode, capacityType, poolCpu, poolMem. --- src/MissionParallelCatchup/lib/config.py | 50 +++++-------------- src/MissionParallelCatchup/lib/sizing.py | 46 +++++++---------- .../templates/job_monitor.yaml | 4 -- .../values-ondemand.yaml | 18 ------- .../parallel_catchup_helm/values.yaml | 48 +----------------- 5 files changed, 31 insertions(+), 135 deletions(-) diff --git a/src/MissionParallelCatchup/lib/config.py b/src/MissionParallelCatchup/lib/config.py index 61674b04..d0453186 100644 --- a/src/MissionParallelCatchup/lib/config.py +++ b/src/MissionParallelCatchup/lib/config.py @@ -468,40 +468,6 @@ 'POOL_CPU', 'subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80') -# vCPU of the SMALLEST node in each tier's pool. This is what decides whether a -# promotion is free, and it cannot be inferred from POOL_CPU, which is a claim -# rather than a node size. -# -# It reads the smallest shape, not the likeliest one, and that was quietly wrong -# while the x8i pools existed. hypergiant listed x8i.xlarge (4 vCPU) at w80 under -# two 8-vCPU rungs at w100/w90, so this map said 4 and supergiant->hypergiant -# priced as free -- while Karpenter tried w100 first and the promotion really cost -# 4->8. The finished 1200-worker run landed 12 hypergiant nodes on 2xlarge shapes -# against 1 on x8i.xlarge. The x8i spot pools were removed on 2026-08-04, so these -# figures are now both the smallest AND the top-weighted shape, and the two top -# rungs cross a class honestly -- which is what POOL_CROSS_RUNGS now carries. -POOL_VCPU = os.getenv( - 'POOL_VCPU', - 'subdwarf:2,dwarf:2,subgiant:4,giant:4,supergiant:4,hypergiant:8,supernova:16,protostar:8,nebula:4') - -# Rungs allowed to cross a vCPU class anyway, "from->to", comma separated. The -# guard exists because a speculative promotion that doubles cores is usually a -# bad trade, so anything listed here needs a measurement behind it. -# -# hypergiant->supernova costs +2 vCPU per range and measured only 1.23x, so on -# its own it is not worth it -- simulated over the 2026-08-03 run it saved -# exactly 0 minutes, because the longest job was a supergiant that this rung -# cannot reach. It earns its place only in company: the long jobs alternate -# between the two tiers, so fixing one exposes the other. supergiant->hypergiant -# alone is worth 8 min, this alone 0, and the pair 27 min. -# -# supergiant->hypergiant is listed too, and is a no-op at today's spot sizes: the -# doubled pools put both tiers on 4-vCPU nodes, so that rung is free and clears -# the guard without an entry. It is here so the rung survives the pools diverging -# -- if hypergiant ever bottoms out above supergiant, the guard would silently -# shut a rung that is deliberately open. On on-demand, where supergiant is 2 vCPU -# and hypergiant 4, the same entry is load-bearing. -POOL_CROSS_RUNGS = os.getenv('POOL_CROSS_RUNGS', 'supergiant->hypergiant') # Rungs that never run, whatever the vCPU comparison says. Empty by default: with # the spot pools doubled, promotion lands a range on a bigger SHARED node, and @@ -519,13 +485,21 @@ # Sizing the rung on peakAnonBytes, or widening the tier->instance map directly, # would target those nodes deliberately. # +# This is now the ONLY thing standing between a working set and a promotion, so +# a rung that should not be taken has to be named here -- nothing is inferred. +# # hypergiant->supernova is denied on both capacity types. Its cost rose once the # x8i pools were removed on 2026-08-04: supernova's only spot shapes are now # 4xlarges, so the rung moves a range from 8 vCPU to 16 rather than the 8-vCPU -# x8i.2xlarge it used to reach. It is denied here rather than merely dropped from -# POOL_CROSS_RUNGS so it stays shut if the tiers ever land on equal-sized nodes, -# which is the state that made the guard unable to hold supergiant->hypergiant. -POOL_BLOCK_RUNGS = os.getenv('POOL_BLOCK_RUNGS', 'hypergiant->supernova') +# x8i.2xlarge it used to reach. Simulated over the 2026-08-03 run it saved +# exactly 0 minutes on its own, because the longest job was a supergiant this +# rung cannot reach. It pays only in company -- supergiant->hypergiant alone is +# worth 8 min, this alone 0, the pair 27 -- and that pairing is not on offer +# while its cost is 8->16 vCPU. +# +# dwarf->subgiant is the same doubling at the bottom of the ladder, 2->4 vCPU on +# spot and 1->2 on on-demand. +POOL_BLOCK_RUNGS = os.getenv('POOL_BLOCK_RUNGS', 'dwarf->subgiant,hypergiant->supernova') # Memory request for a pooled range is the TIER'S CUT, not the range's own # measurement, and that is deliberate two ways. diff --git a/src/MissionParallelCatchup/lib/sizing.py b/src/MissionParallelCatchup/lib/sizing.py index ca3f0a23..7b0434e0 100644 --- a/src/MissionParallelCatchup/lib/sizing.py +++ b/src/MissionParallelCatchup/lib/sizing.py @@ -65,21 +65,11 @@ def eph_for_attempt(attempt): return units.bytes_to_quantity(min(want, units.quantity_bytes(config.EPH_ESCALATION_CAP))) -def pool_vcpu(tier): - """vCPU of the smallest node this tier can land on, or None if unmapped.""" - return _pool_map(config.POOL_VCPU, 'POOL_VCPU').get(tier) - - def _rung_listed(raw, tier, nxt): want = f"{tier}->{nxt}" return any(item.strip() == want for item in raw.split(',')) -def _crossing_allowed(tier, nxt): - """Is this specific rung whitelisted to cross a vCPU class?""" - return _rung_listed(config.POOL_CROSS_RUNGS, tier, nxt) - - def _rung_blocked(tier, nxt): """Is this rung denied outright? Beats every other consideration.""" return _rung_listed(config.POOL_BLOCK_RUNGS, tier, nxt) @@ -148,21 +138,24 @@ def _cache_bump(tier, anon_bytes, ws_bytes): m8in.large 8 GiB 540 reads/ledger 21% iowait 1.86 lps r8in.large 16 GiB 65 reads/ledger 7% iowait 3.14 lps (100% of profile) - Only rungs that keep the same node vCPU, read from POOL_VCPU. giant-> - supergiant is m8a.large->r8a.large: twice the RAM for the same 2 vCPU and - +8% spot. supergiant->hypergiant is r8a.large->x8i.large, also 2 vCPU, and - that rung measured 1.86x on ssc-test 2026-08-03 -- 1.64 -> 2.99 lps across - nine ranges, the largest gain found anywhere. - - Do NOT read POOL_CPU for this. It was half the node's vCPU everywhere, so - equal claims used to imply equal nodes, but hypergiant and supernova are now - sized to the smallest shape in their pool (1.70 on a 2-vCPU x8i.large). A - claim comparison silently refuses supergiant->hypergiant, which is the whole - reason the x8i shapes were promoted to top weight. - - What stays blocked is hypergiant->supernova, 2 vCPU -> 4. Those ranges - measured healthy at 6-11 reads/ledger and gained only 1.23x, so doubling - their cores is the expensive rung with the weak return. + Which rungs are worth taking is stated outright in POOL_BLOCK_RUNGS, not + inferred. giant->supergiant is m8a.large->r8a.large: twice the RAM for the + same 2 vCPU and +8% spot. supergiant->hypergiant is r8a.large->x8i.large, + also 2 vCPU, and that rung measured 1.86x on ssc-test 2026-08-03 -- 1.64 -> + 2.99 lps across nine ranges, the largest gain found anywhere. + + Blocked: hypergiant->supernova, whose ranges measured healthy at 6-11 + reads/ledger and gained only 1.23x, and dwarf->subgiant, the same doubling + at the bottom of the ladder. Both double the cores for a weak return. + + This used to be derived instead, by refusing any rung whose POOL_VCPU + differed and keeping an allowlist of exceptions. POOL_VCPU reads the + SMALLEST shape a pool can land on, so it mispriced every tier spanning node + sizes -- an x8i at w80 hid a 4->8 promotion -- and the allowlist existed + only to undo its wrong answers. Deriving it bought one rung that a block + entry states directly, so a list of refusals replaced both. The cost is that + a new tier is allowed by default: a rung that should be refused now has to + be named here. Deliberately loose about false positives. Promoting a range that did not need it costs +8% on its node-hours and nothing in quota; leaving one starving @@ -185,9 +178,6 @@ def _cache_bump(tier, anon_bytes, ws_bytes): return tier # working set does not reach the next tier if _rung_blocked(tier, nxt): return tier # denied outright, see POOL_BLOCK_RUNGS - a, b = pool_vcpu(tier), pool_vcpu(nxt) - if (a is None or b is None or a != b) and not _crossing_allowed(tier, nxt): - return tier # crosses a vCPU class; not free, skip it return nxt diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 5a2e0b30..fb05da79 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -296,10 +296,6 @@ spec: value: {{ .Values.monitor.poolTiers | quote }} - name: POOL_BLOCK_RUNGS value: {{ .Values.monitor.poolBlockRungs | quote }} - - name: POOL_CROSS_RUNGS - value: {{ .Values.monitor.poolCrossRungs | quote }} - - name: POOL_VCPU - value: {{ .Values.monitor.poolVcpu | quote }} - name: POOL_CPU value: {{ .Values.monitor.poolCpu | quote }} - name: POOL_UNPROFILED diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values-ondemand.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values-ondemand.yaml index 6447322e..95f374bf 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values-ondemand.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values-ondemand.yaml @@ -59,21 +59,3 @@ monitor: # supernova r8a.2xlarge 8 59515Mi / 7695m 57216Mi / 7.20 poolCpu: "subdwarf:0.45,dwarf:0.45,subgiant:1.40,giant:1.40,supergiant:1.40,hypergiant:3.35,supernova:7.20,protostar:1.40,nebula:3.35" poolMem: "subdwarf:576Mi,dwarf:576Mi,subgiant:2048Mi,giant:5248Mi,supergiant:12416Mi,hypergiant:27328Mi,supernova:57216Mi,protostar:27328Mi,nebula:12416Mi" - # vCPU of the smallest node each tier can land on. Differs from the spot map - # because on-demand pools were never doubled -- supergiant bottoms out at - # r8a.large (2), not at a 4-vCPU shape. - poolVcpu: "subdwarf:1,dwarf:1,subgiant:2,giant:2,supergiant:2,hypergiant:4,supernova:8,protostar:2,nebula:4" - # Both top rungs open, as on spot -- but they get here by the other route. On - # this map supergiant bottoms out at 2 vCPU and hypergiant at 4, so BOTH rungs - # cross a vCPU class (2->4 and 4->8) and the whitelist is what carries them; - # an empty denylist alone would leave them shut. On spot only the supernova - # rung needs listing, because the doubled pools put supergiant and hypergiant - # on equally-sized nodes. - # - # The cost is higher here than on spot: one pod per node means a promotion buys - # a bigger node outright, with no co-tenant to amortise it, so each promoted - # range doubles its vCPU draw. And the bump fires on working set, which does not - # predict throughput -- same box, memory.max 28GiB ran 1.83 lps against 56GiB at - # 1.70 -- so some of what it promotes will gain nothing. - poolCrossRungs: "supergiant->hypergiant" - poolBlockRungs: "hypergiant->supernova" diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index cb5cf9cc..4b24b6f5 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -278,51 +278,6 @@ monitor: # margin, and OOMKilled on BOTH during bucket-apply before closing a ledger. poolPrefix: "" poolTiers: "0:subdwarf,0.79:dwarf,1.61:subgiant,3.87:giant,8.85:supergiant,18.38:hypergiant,:supernova" - # cpu is a claim token, not a demand estimate -- replay draws ~1.05 cores - # whatever it is given and is flat in core count from 2 upward (+2.8% at 2->4, - # +1.5% at 4->8). Kept at or below the SMALLEST node in each tier so the - # low-weight fallback rungs stay schedulable. - # Exactly 50% of each tier node's nameplate capacity. Two pods would need the - # whole node, which always exceeds allocatable, so a second can never fit -- - # isolation without depending on how the kubelet reserves. - # hypergiant and supernova are NOT half their node: they are sized to the - # smallest shape in the pool. x8i.large is r8a.xlarge with half the cores and - # the same 32 GiB, and x8i.xlarge is r8a.2xlarge with half the cores and the - # same 64 GiB, so preferring the x8i shapes buys the same RAM for half the - # spot quota -- 2128 -> 1800 vCPU on a 900-worker wave, 15% back. - # - # A "half the node" claim of 2.00/4.00 exceeds what an x8i node can offer once - # daemonsets are counted (215m: alloy 10 + aws-node 75 + ebs-csi-node 30 + - # kube-proxy 100), so those pools never won a single node in either run on - # 2026-08-03 despite being weighted in. 1.70 and 3.60 sit under the 1715m and - # 3705m that are actually schedulable. - # - # Dropping cpu below half stops cpu from isolating the pod on the larger - # fallback shapes -- two 1.70 claims do fit a 4-vCPU r8a.xlarge. Memory is - # what isolates here instead, and it holds on every type in both pools because - # they all carry the same RAM: 2 x 16384Mi > 28713Mi usable, 2 x 32768Mi > - # 59515Mi usable. - # vCPU of the smallest node in each tier's pool -- what decides whether a - # promotion is free. Cannot be inferred from poolCpu, which is a claim rather - # than a node size. - # - # Reading the SMALLEST shape was quietly wrong while the x8i pools existed: - # hypergiant listed x8i.xlarge (4 vCPU) at w80 beneath two 8-vCPU rungs, so this - # said 4 and supergiant->hypergiant priced as free, while Karpenter tried w100 - # first and the promotion really cost 4->8. The finished 1200-worker run put 12 - # hypergiant nodes on 2xlarge shapes against 1 on x8i.xlarge. x8i was removed - # from spot on 2026-08-04, so these are now both the smallest and the - # top-weighted shape and the top rungs cross a class honestly. - # Rungs allowed to cross a vCPU class anyway. hypergiant->supernova costs - # +2 vCPU and gains only 1.23x, worth 0 minutes on its own -- but the long - # jobs alternate between the two tiers, so paired with supergiant->hypergiant - # it takes the floor from 2.90h to 2.44h where either alone gets 8 min or none. - # supergiant->hypergiant is listed but is a no-op at today's sizes: the doubled - # spot pools put both tiers on 4-vCPU nodes, so the rung is already free and - # passes the guard without a whitelist entry. It is here so the rung survives - # the pools diverging again -- if hypergiant ever bottoms out above supergiant, - # the guard would silently shut a rung that is deliberately open. - poolCrossRungs: "supergiant->hypergiant" # Rungs that never run regardless of the vCPU comparison. Empty: with the spot # pools doubled a promotion lands the range on a bigger SHARED node, and that # sharing is what it buys. Measured 2026-08-04, two pods per node on one range: @@ -330,8 +285,7 @@ monitor: # against 1.58x on a 4-vCPU node. Note the bump fires on working set, which does # NOT predict throughput -- same box, memory.max 28GiB ran 1.83 lps vs 56GiB at # 1.70 -- so it reaches the right nodes by the wrong signal. - poolBlockRungs: "hypergiant->supernova" - poolVcpu: "subdwarf:2,dwarf:2,subgiant:4,giant:4,supergiant:4,hypergiant:8,supernova:16,protostar:8,nebula:4" + poolBlockRungs: "dwarf->subgiant,hypergiant->supernova" poolCpu: "subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80" poolMem: "subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:9216Mi" # Profiled run, range past the profile's top: newest ledgers, and the newest From ad2a9fd131dabecf2465432f2d92cec7779f8935 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 14:35:00 -0400 Subject: [PATCH 083/117] Delete the orphaned cpu-tiering prose from values.yaml 164 lines of comment documenting profileCpuTiers and profileCpuSlowdown, whose keys were removed with the cpu ladder. Nothing was left holding it up: it sat directly under sourceInstallDependencies, a boolean about installing pip packages at container start, and read as that key's documentation. It described a mechanism that no longer exists -- the slack-budget bands, the runtime multipliers, the discrete-event simulation the ladder was solved against -- so it could only mislead someone tuning what is there now. --- .../parallel_catchup_helm/values.yaml | 164 ------------------ 1 file changed, 164 deletions(-) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 4b24b6f5..a9623bee 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -101,170 +101,6 @@ monitor: # Source-mode development normally installs dependencies at container start. # Disable only when the selected image already contains them. sourceInstallDependencies: true - # CPU request tiers, as a slack budget rather than a demand estimate. Measured - # unthrottled, replay wants ~1.0 cores at every ledger position and is 80-95% - # of a job, so demand barely varies -- what varies is how much throttling a - # range can absorb before it stops fitting inside the longest job's shadow. - # 3267 of 3859 profiled ranges still fit at 0.5 cores. Empty disables tiering - # and every range keeps the configured cpu request. - # Paired with profileCpuSlowdown: the measured runtime multiplier per tier. - # Empty means tiering OFF and every worker falls back to the flat REQ_CPU, - # which is not a safe default: on 2026-07-31 that put 1250m on all 2048 pods, - # 2560 vCPU of requests against a 2304 spot quota, and packed 6 workers per - # node instead of 14. Nothing in the F# sets this, so an unset value here is - # the whole configuration. - # - # Values are absolute cores and are NOT clamped to REQ_CPU -- verified, the - # top bands really do issue 1.5-2.0. Widened 2026-07-31 after measuring the - # previous ladder on saturated nodes: the 500m band ran at 1.09x its profiled - # time (cheap, it is IO-bound) while the top band was delivered 0.88 cores - # against ~1.13 of unthrottled demand -- and the top band sets the makespan, - # so throttling it is the one place the ladder costs wall-clock. - # - # Widened again 2026-08-01 after the 2048-worker run measured a clean - # monotonic dose-response across all seven bands -- actual/profiled runtime - # 1.23x at 0.5 cores, 1.00x at 0.75, 0.81x at 1.0, 0.72x at 1.25, 0.71x at - # 1.5, 0.67x at 1.75, 0.63x at 2.0 over 3984 completed ranges. Every step up - # bought speed, and the run finished in 4.15h against r5's 7.5h because the - # long ranges are the ones that got faster. The top bands are not idle cpu: - # a 2.0 request lands ~3 co-tenants per node instead of 8-14, and ranges on - # uncrowded nodes measured 1.80 vs 1.34 ledgers/s. - # - # Reshaped 2026-08-01 after watching the first wave of the 1224-worker run. - # Uniformly raising every band was the wrong move: it reserved cpu without - # changing what a long range is actually delivered. With cpu limits removed a - # saturated node splits its spare cpu by weight, so when a node holds nothing - # but long ranges -- which is exactly what longest-first produces -- they all - # get roughly allocatable/co-tenants regardless of what they asked for. - # Measured live: nodes 94% cpu-reserved, longest-100 ranges with 4.0 - # co-tenants each, so ~1.81 vCPU delivered against much larger requests. - # - # Replaced 2026-08-01 with a continuous ramp rather than a few wide steps, so - # a range's request tracks its runtime instead of jumping at a band edge. - # Three segments, keyed on the range's runtime percentile: - # - # p75..p100 1.5 -> 3.5 the longest quarter, sampled every 2-3 points - # p50..p75 1.0 -> 1.5 - # p0..p50 0.5 -> 1.0 - # - # 22 bands, none spanning more than 0.24 vCPU, so the granularity is in the - # ladder rather than in how the wave happens to land on it. - # - # Parallelism is coupled to this and cannot be chosen separately. With - # longest-first the wave is the top of the ramp, so the mean request is ~3.0 - # rather than the ~1.3 a flat ladder gives, and the fleet grows in proportion: - # 448 workers is what fits the 2304 vCPU spot quota at 87%. See - # scratchpad/granular-ladder.py for the sweep. - # Rebuilt 2026-08-01 around ledgers/sec rather than percentile shape, after - # the 448-worker run showed the ramp was right but the parallelism was not: - # it finished the longest quarter at 0.47x profiled, then spent hours draining - # 3536 short ranges through too few slots (projected 6.55h, work-bound). - # - # With cpu limits removed a pod is delivered request * (allocatable / sum of - # requests on its node), so density feeds straight back into ledgers/sec. A - # flat coarse ladder at 1.30-1.80 packs 4-5 per node and drops the dense - # ranges to 1.8 LPS -- 22% of the first wave under 3. So the top stays tall - # for the ranges whose ledgers are transaction-dense (they cap out near 3 LPS - # however much cpu they get) and only the middle is coarsened. - # - # Five bands over the top quarter instead of twelve. Wave floor 1.30 keeps at - # most 5 pods on a node (6 x 1.30 > 7.56 allocatable). At 768 workers: ~269 - # nodes, 2152 vCPU (93% of the spot quota), first-wave LPS min 2.89 and p10 - # 3.35. Parallelism and this ladder move together -- the p74 boundary is where - # the 768-range wave starts. - # Derived rather than hand-shaped, 2026-08-01, from the response curve the - # 768-worker run measured on 664 live workers: m(c) = 0.856 * c^-0.531, which - # reproduced all four of its bands to within 0.01 (1.50->0.69, 2.60->0.51, - # 3.40->0.47, 4.00->0.41). - # - # While a run is work-bound its makespan IS its total work, so the objective - # is to minimise sum(seconds_i * m(cpu_i)) under a cpu budget, not to equalise - # finish times. Equalising was measured as actively worse: it pushes short - # ranges to 0.5 cpu where m = 1.24 -- 24% SLOWER than profiled -- inflating - # total work ~25% and costing ~40 minutes. - # - # Setting the marginal return equal across ranges gives a closed form: - # cpu proportional to seconds^(1/(1+b)) = seconds^0.653 - # scaled by 0.0060 and clamped to [0.5, 4.5]. At 1280 workers on the - # on-demand pool that is ~358 r8a.2xlarge, 2864 vCPU, at most 5 pods/node, - # and it lands exactly on the crossover: work-bound 1.87h against a 1.88h - # critical path. Below 1280 the tail idles; above it the longest range walls. - # Rebuilt 2026-08-02 from four full-fleet probes (A/B/C/D), 735 paired ranges - # each, mean cpu 1.17 -> 3.54, requests only and no cpu limit anywhere: - # - # ratio(c) = 0.671 * c^0.348 residuals <= 0.005 - # - # No knee in that span. The retired ladder assumed saturation below 2.25 from - # arms run at hard cpu LIMITS -- a cfs quota clips the bursts replay uses, so - # those arms measured the quota, not the process. See cpu-ladders-history.md. - # - # The critical path is NOT a fixed range. Probes A/B/C/D each reported a - # different slowest range, and the profile's longest (10340s) finished 99th of - # 735 in probe C. Per-range slowness reproduces across probes at only r=0.26-0.59 - # (r^2 0.07-0.35), so most of it is run noise, not a property of the range. - # Tuning cpu at whichever range came last just moves the straggler. - # - # What IS reproducible: headroom compresses the noise. Residual (observed time / - # predicted) by the cpu the range actually had -- - # - # cpu 1.15 1.30 1.90 2.50 3.30 3.70 - # p90 1.51 1.40 1.27 1.27 1.21 1.22 - # max 2.39 2.08 2.06 1.76 1.49 1.63 - # - # Targets a CRITICAL-PATH BOUND run on the ON-DEMAND pool: simulated 2.20h with - # the longest range also finishing at 2.20h, so nothing trails it. - # - # Chosen by discrete-event SIMULATION of the actual dispatch, not by the - # max(work/fleet, longest-job) bound. That bound is what an earlier ladder was - # solved against and it is only a LOWER bound -- it sizes each band as if the - # range starts at t=0, but longest-first dispatches the cheap ranges LAST. The - # ladder it produced put p66 at 0.9 cpu: 1.40h of runtime starting at 1.40h, - # finishing at 2.79h, half an hour after the longest range was already done. - # Starving the bulk does not make a run crit-path bound, it just moves the tail. - # - # The real constraint is concurrency, not per-range speed. Jobs long enough to - # matter must all be running at t=0 or they serialise: under that earlier ladder - # the jobs over 1.25h needed 2518 vCPU against a 2304 spot fleet, so some had to - # queue and any two that did cost double. Hence a flat 1.3 across the bottom 95% - # -- cheap enough that they all fit at once, rich enough to stay out of the band - # where the noise tail blows up (residual p90 is 1.51 at 1.15 cpu against 1.15 at - # 7.0, so cheap bulk is bought at the price of stragglers). - # - # 1.3 rather than 0.9 for the flat band because 0.9 ties the noiseless sim (both - # 2.20h) and loses badly once measured noise is applied (5.06h against 4.15h). - # This is the choice that does not depend on which model is right. - # - # Sized for the on-demand pool: 1600 replicas, first wave 2912 vCPU against the - # catchup-od nodepool limit (raised 3072 -> 4096 so the packing model's habitual - # 7-16% optimism cannot push it over). Spot cannot host this -- its AWS quota is - # 2304 vCPU where this needs ~2900, and the cheapest ladder that fits spot - # without queueing runs 3.86h. - # - # Requires range.order=longest-first; under tip-first the top-band ranges scatter - # through the run instead of starting first (tip-first is only a 58% proxy). - # - # Sizing: replicas is a QUEUE DEPTH, not a fleet size, and it should exceed what - # the quota can run at once. With range.order=longest-first the first wave is - # entirely top-band, so peak demand per pod is far above the ladder mean -- - # measured 2.63 cpu over the 700 longest ranges against a 1.53 ladder mean. No - # replica count both fits the expensive wave AND saturates the cheap tail: even - # 700 workers is 94% of quota in wave 1, while the tail would happily run 2000+. - # - # So oversubscribe and let the scheduler absorb it. At 1200 the run sat at ~2350 - # vCPU with ~240 Pending, and the queue drained on its own as top-band jobs - # finished (239 -> 206 in 5 min, running 962 -> 970). Pending pods hold no - # resources; they are how the fleet stays saturated across a 4x swing in cost per - # range without anyone retuning mid-run. - # - # Do NOT size as quota/mean-cpu (ignores memory bin-packing waste) and do not - # trust a bin-pack of a RANDOM sample (right method, wrong population -- the run - # never dispatches a random sample first; that rule predicted 88% of quota where - # reality was 102%). - # - # Memory sizing is deliberately UNCHANGED. A limit sweep from 16Gi to 40Gi on - # one range moved major faults 1768 -> 0 and ledgers/sec not at all - # (1.464-1.499), so peakAnonBytes remains the right basis and the working-set - # figure stays unused -- it measures what was available, not what is needed. # --- nodepool tiers ------------------------------------------------------- # # A range picks a NODEPOOL by its measured memory and gets that node to From 908c47bcb6a456f22871dccf949fb7ec4c367921 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 14:37:07 -0400 Subject: [PATCH 084/117] Delete the volume-spread constraint It capped PVC-mounting workers per node with a topologySpread minDomains floor, because a Nitro node allows ~26 EBS attachments and Karpenter sizes nodes on CPU/memory alone -- it would put 40 volume-mounting pods on a 4-vCPU node and they would fail with VolumeAttachmentLimitExceeded. That was a hazard of packing as many ranges onto a node as possible. Tiered routing inverted the goal: a range picks a nodepool by its measured memory and gets the node to itself, and the tier's memory cut is sized to exclude a second pod. One attachment per node cannot approach a 26-attachment limit, so the floor can never bind -- it was already documented as inert at realistic density and is now unreachable by construction. Removes MAX_VOLUMES_PER_NODE, worker.maxVolumesPerNode and the env wiring with it. Workers no longer carry topologySpreadConstraints at all. --- .../apps/job_monitor.py | 20 ------------------- src/MissionParallelCatchup/lib/config.py | 12 ----------- .../templates/job_monitor.yaml | 2 -- .../parallel_catchup_helm/values.yaml | 6 ------ 4 files changed, 40 deletions(-) diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py index 675f78ed..d92c8b75 100644 --- a/src/MissionParallelCatchup/apps/job_monitor.py +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -932,23 +932,6 @@ def _resources(mem=None, eph=None, end=None, attempt=1): return client.V1ResourceRequirements(requests=req, limits=lim or None) -def volume_spread_constraints(): - """Keep PVC-mounting workers under the per-node EBS attachment limit. - - Only in pvc mode: in ephemeral mode /data is an emptyDir, no volume is - attached, and spreading would just cost density. - """ - if config.STORAGE_MODE != 'pvc' or config.MAX_VOLUMES_PER_NODE <= 0: - return None - min_domains = max(1, -(-config.PARALLELISM // config.MAX_VOLUMES_PER_NODE)) # ceil - return [client.V1TopologySpreadConstraint( - max_skew=config.MAX_VOLUMES_PER_NODE, - min_domains=min_domains, - topology_key='kubernetes.io/hostname', - when_unsatisfiable='DoNotSchedule', - label_selector=client.V1LabelSelector(match_labels={config.LABEL_RUN: config.RUN_NAME}))] - - def pod_labels(end, attempt): """Labels on the worker POD, which are not the Job's. @@ -1080,9 +1063,6 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): # IRSA for the S3 history mirror; without it workers fall # back to the public archive, which throttles at 1024. service_account_name=config.WORKER_SERVICE_ACCOUNT or None, - # Keeps PVC-mounting workers under the per-node EBS - # attachment cap; inert at realistic CPU-bound density. - topology_spread_constraints=volume_spread_constraints(), # Never restarted in place: the pod stays terminal and # inspectable for classification and the backstop log read. restart_policy='Never', diff --git a/src/MissionParallelCatchup/lib/config.py b/src/MissionParallelCatchup/lib/config.py index d0453186..62a422b4 100644 --- a/src/MissionParallelCatchup/lib/config.py +++ b/src/MissionParallelCatchup/lib/config.py @@ -176,18 +176,6 @@ # range measured, with headroom for the tip to keep growing. STORAGE_SIZE = os.getenv('STORAGE_SIZE', '60Gi') -# A Nitro node allows ~26 EBS attachments (CSINode allocatable), and Karpenter -# sizes nodes on CPU/memory only -- it will happily put 40 volume-mounting pods -# on one 4-vCPU node, where they serialise through the attachment slots and get -# rejected with VolumeAttachmentLimitExceeded (observed on ssc-test). -# -# Guard with a spread constraint rather than a warning. maxSkew alone cannot cap -# per-node count -- with a single node there is one domain and therefore no skew -# -- so minDomains is what forces enough nodes. Both are inert at realistic -# density: REQ_CPU=1800m yields ~4 workers on an 8-vCPU node, so CPU demands far -# more nodes than this floor ever asks for. 0 disables. -MAX_VOLUMES_PER_NODE = int(os.getenv('MAX_VOLUMES_PER_NODE', 24)) - # Job/pod lifetimes. # SIGTERM -> SIGKILL budget. stellar-core exits ~7s after SIGTERM (measured), so # this is slack rather than a target. diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index fb05da79..974d92a8 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -242,8 +242,6 @@ spec: value: {{ .Values.worker.storageClass | quote }} - name: STORAGE_SIZE value: {{ .Values.worker.storageSize | quote }} - - name: MAX_VOLUMES_PER_NODE - value: {{ .Values.worker.maxVolumesPerNode | quote }} - name: ASAN_OPTIONS value: {{ .Values.worker.asanOptions | quote }} - name: REQ_CPU diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index a9623bee..a15533a9 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -17,12 +17,6 @@ worker: # 60Gi to match the tier nodes' ephemeral allowance: peakEphemeralBytes tops # out at 37.8Gi across the whole profile, so this covers 100% with headroom. storageSize: "60Gi" - # Cap on PVC-mounting workers per node, enforced with a topologySpread - # minDomains floor. A Nitro node allows ~26 EBS attachments and Karpenter does - # not size nodes for attachment capacity. Inert at realistic density (1800m - # CPU already yields ~4/node); only binds if CPU requests are small enough for - # 24+ pods to share a node. 0 disables. pvc mode only. - maxVolumesPerNode: 24 requireNodeLabels: [] avoidNodeLabels: [] tolerateNodeTaints: [] From e4bae212ed8201cb20f2cf27f5bb8f07a5de19b6 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 15:30:17 -0400 Subject: [PATCH 085/117] Take capacity from a node-label list, not a values overlay The chart knew that spot and on-demand exist. It shipped values-ondemand.yaml carrying their two pool-claim maps, the mission chose between them by storage mode, and the monitor pinned a pod to one side with a hardcoded karpenter.sh/capacity-type -- an assumption about who provisioned the node, made by the one component with no business knowing. None of that has to be in the chart. Every catchup NodePool already publishes catchup-capacity=od|spot itself (48 of 48, verified on ssc-eks), so capacity is just another required label, and --require-node-labels-pc-v2 has always been able to carry one. What stopped it was the chart reading `first` of that list and silently dropping the rest, so a second required label was accepted and ignored. Entries past 0 now travel as REQUIRE_NODE_LABELS and are ANDed in literally; entry 0 stays the pool-routed one, whose value the monitor replaces per range with -. That also fixes an index collision. A pooled run writes worker.requireNodeLabels[0] for its routing label while requireNodeLabelsPcV2 indexed its own entries from 0, so a pooled run carrying a capacity label had its routing label overwritten -- the pod would then match on capacity alone and land on any tier, which is an OOM per range rather than a slow run. The caller's entries start at 1 when pooling is on. The pool maps come from the mission instead of the overlay (--pubnet-parallel-catchup-pool-cpu/-mem), each on its own --set with commas backslash-escaped: every other option is folded into one comma-joined --set and these are themselves comma-separated, which is why the overlay was a second --values file to begin with. Empty is filtered out rather than sent, since monitor.poolCpu= would blank the chart default and drop every tier to the flat request. Storage mode and capacity are now independent, which is the point: they travel separate paths that never meet, so a run can pair ephemeral with spot on purpose. Nothing derives one from the other any more, and nothing objects -- pairing them correctly is the caller's job. With the maps arriving per run, the contract test comparing them against the ladder no longer covers what a run actually uses, so validate_config refuses a pooled /start when POOL_CPU or POOL_MEM omits a routable tier. That failure was silent before: the pod keeps the flat request and a second fits beside it, undoing the isolation the tiering exists for -- a pod alone on its node ran 29-92% faster. Deleted: values-ondemand.yaml, the valuesArgs branch, monitor.capacityType, CAPACITY_TYPE, and three F# tests whose subject was the overlay. The on-demand claim-fit test goes with them: its input leaves this repo, so it would have asserted the spot map twice. --- src/App/Program.fs | 16 +++ src/FSLibrary.Tests/Tests.fs | 131 +++++------------- .../MissionHistoryPubnetParallelCatchupV2.fs | 57 ++++---- src/FSLibrary/StellarMissionContext.fs | 2 + .../apps/job_monitor.py | 26 +++- src/MissionParallelCatchup/lib/config.py | 28 +++- .../templates/_helpers.tpl | 17 +++ .../templates/job_monitor.yaml | 12 +- .../values-ondemand.yaml | 61 -------- .../parallel_catchup_helm/values.yaml | 4 - 10 files changed, 150 insertions(+), 204 deletions(-) create mode 100644 src/MissionParallelCatchup/parallel_catchup_helm/templates/_helpers.tpl delete mode 100644 src/MissionParallelCatchup/parallel_catchup_helm/values-ondemand.yaml diff --git a/src/App/Program.fs b/src/App/Program.fs index 7911d077..6742a550 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -122,6 +122,8 @@ type MissionOptions jobMonitorImagePcV2: string, pubnetParallelCatchupCpuRequest: string, pubnetParallelCatchupMemRequest: string, + pubnetParallelCatchupPoolCpu: string, + pubnetParallelCatchupPoolMem: string, tag: string option, numPregeneratedTxs: int option, genesisTestAccountCount: int option, @@ -570,6 +572,18 @@ type MissionOptions Default = "")>] member self.PubnetParallelCatchupMemRequest : string = pubnetParallelCatchupMemRequest + [] + member self.PubnetParallelCatchupPoolCpu : string = pubnetParallelCatchupPoolCpu + + [] + member self.PubnetParallelCatchupPoolMem : string = pubnetParallelCatchupPoolMem + [] member self.Tag = tag @@ -954,6 +968,8 @@ let main argv = jobMonitorImagePcV2 = mission.JobMonitorImagePcV2 pubnetParallelCatchupCpuRequest = mission.PubnetParallelCatchupCpuRequest pubnetParallelCatchupMemRequest = mission.PubnetParallelCatchupMemRequest + pubnetParallelCatchupPoolCpu = mission.PubnetParallelCatchupPoolCpu + pubnetParallelCatchupPoolMem = mission.PubnetParallelCatchupPoolMem tag = mission.Tag numPregeneratedTxs = mission.NumPregeneratedTxs enableTailLogging = true diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index e2394dfe..6a9fae9b 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -128,6 +128,8 @@ let ctx : MissionContext = jobMonitorImagePcV2 = "" pubnetParallelCatchupCpuRequest = "" pubnetParallelCatchupMemRequest = "" + pubnetParallelCatchupPoolCpu = "" + pubnetParallelCatchupPoolMem = "" tag = None numPregeneratedTxs = None enableTailLogging = true @@ -651,117 +653,52 @@ let ``progress record is read from the volume and never from the configmap`` () [] -let ``on-demand runs layer the one-pod-per-node overlay`` () = - // The chart defaults are the spot claims: the spot pools were doubled on - // 2026-08-04 so each claim is half a node and two pods share it. On-demand - // pools kept their original sizes, where those same claims are the node's - // NAMEPLATE -- and nameplate is not allocatable, so every on-demand tier - // becomes unschedulable. Measured on ssc-test: a 16 GiB node reports - // 13312Mi usable, against a 14336Mi supergiant claim. +let ``the job monitor image is overridable and defaults to the chart`` () = + // The monitor and collector ship as one image pinned in values.yaml. Passing + // it per run is what lets a build of them be tested without editing the + // chart -- but an empty flag must leave the chart's pin alone rather than + // setting monitor.image to nothing, which resolves to ":latest" or fails the + // pull outright. let src = System.IO.File.ReadAllText( "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") - Assert.Contains("values-ondemand.yaml", src) - // and it must be layered, never swapped in: the overlay only carries the - // pool claims, so dropping the base values would lose the whole chart config - Assert.Contains("[| \"--values\"; valuesFilePath; \"--values\"; onDemandValuesFilePath |]", src) + Assert.Contains("if context.jobMonitorImagePcV2 <> \"\" then", src) + Assert.Contains("monitor.image=%s", src) -[] -let ``the on-demand overlay is not applied to pvc runs`` () = - // pvc means spot means shared nodes. Layering the one-pod claims there would - // halve pods per node on pools that were doubled precisely to hold two. - let src = - System.IO.File.ReadAllText( - "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") - let guard = src.IndexOf("if context.pubnetParallelCatchupStorageMode = \"pvc\" then\n [| \"--values\"; valuesFilePath |]") - Assert.True(guard > 0, "pvc branch must pass the base values file alone") + let guard = src.IndexOf("if context.jobMonitorImagePcV2 <> \"\" then") + let use_ = src.IndexOf("monitor.image=%s") + Assert.True(guard < use_, "monitor.image must only be set inside the non-empty guard") [] -let ``on-demand pool claims fit exactly one pod per node`` () = - // Both halves, on BOTH dimensions. The previous version of this test checked - // memory only and assumed 154Mi of daemonsets, so it passed while every - // on-demand tier was in fact unschedulable -- 2026-08-07, ten workers Pending - // forever because Karpenter needed 1820m/3054Mi against c8a.large's - // 1715m/2663Mi. A test that encodes a stale measurement is worse than none: - // it is why the table looked verified. - // - // 494Mi/245m measured on ssc-test, and it is what Karpenter enforces -- - // ebs-csi-node-windows is 340Mi of it and cannot run on these nodes, but the - // nodepools constrain arch and not os, so it is reserved anyway. - let dsMem, dsCpu = 494.0, 245.0 - - let overlay = +let ``a pooled run does not let caller labels overwrite the routing label`` () = + // A pooled run claims worker.requireNodeLabels[0] for the label it routes + // on, and requireNodeLabelsPcV2 used to index its own entries from 0 as + // well. Both fire on a pooled run carrying a capacity label, and the second + // --set wins: the routing label is replaced, so the pod matches on capacity + // alone and lands on any tier at all -- a range sized for supergiant on a + // dwarf node, which is an OOM per range rather than a slow run. + let src = System.IO.File.ReadAllText( - "../../../../MissionParallelCatchup/parallel_catchup_helm/values-ondemand.yaml") - - let claim (map: string) (tier: string) = - let entry = - overlay.Split('\n') - |> Array.find (fun l -> l.TrimStart().StartsWith(map + ":")) - entry.Split(',') - |> Array.pick (fun kv -> - let parts = (kv.Split(':') |> Array.map (fun x -> x.Trim([| '"'; ' ' |]))) - if parts.[parts.Length - 2] = tier then Some parts.[parts.Length - 1] else None) - - // tier, measured allocatable MiB, measured allocatable millicores - let nodes = - [ "subdwarf", 1127.0, 725.0 - "dwarf", 1127.0, 725.0 - "subgiant", 2663.0, 1715.0 - "giant", 5940.0, 1715.0 - "supergiant", 13313.0, 1715.0 - "nebula", 13313.0, 3705.0 - "hypergiant", 28714.0, 3705.0 - "protostar", 28714.0, 1715.0 - "supernova", 59515.0, 7695.0 ] - - for (tier, allocMem, allocCpu) in nodes do - let mem = float ((claim "poolMem" tier).Replace("Mi", "")) - let cpu = float (claim "poolCpu" tier) * 1000.0 - - // One pod must FIT once the daemonsets are counted -- this is the half - // that was missing, and it is why nothing provisioned. - Assert.True( - mem + dsMem <= allocMem, - sprintf "%s: %.0fMi + %.0fMi daemonsets exceeds %.0fMi allocatable" tier mem dsMem allocMem - ) - - Assert.True( - cpu + dsCpu <= allocCpu, - sprintf "%s: %.0fm + %.0fm daemonsets exceeds %.0fm allocatable" tier cpu dsCpu allocCpu - ) - - // And a second must NOT, or the isolation the on-demand ladder exists - // for is gone without anything failing. - Assert.True( - 2.0 * mem + dsMem > allocMem, - sprintf "%s: two pods fit in %.0fMi; on-demand is one per node" tier allocMem - ) - - Assert.True( - 2.0 * cpu + dsCpu > allocCpu, - sprintf "%s: two pods fit in %.0fm; on-demand is one per node" tier allocCpu - ) - + "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + Assert.Contains("worker.requireNodeLabels[0]=purpose:%s", src) + Assert.Contains("if context.pubnetParallelCatchupPoolPrefix <> \"\" then i + 1 else i", src) [] -let ``the job monitor image is overridable and defaults to the chart`` () = - // The monitor and collector ship as one image pinned in values.yaml. Passing - // it per run is what lets a build of them be tested without editing the - // chart -- but an empty flag must leave the chart's pin alone rather than - // setting monitor.image to nothing, which resolves to ":latest" or fails the - // pull outright. +let ``the pool maps ride their own --set with their commas escaped`` () = + // Every other option is folded into ONE comma-joined --set. The pool maps + // are themselves comma-separated, so folding them in would split each tier + // into a separate helm assignment and the map would arrive holding one + // tier. This is why the overlay used to be a second --values file. let src = System.IO.File.ReadAllText( "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") - Assert.Contains("if context.jobMonitorImagePcV2 <> \"\" then", src) - Assert.Contains("monitor.image=%s", src) - - let guard = src.IndexOf("if context.jobMonitorImagePcV2 <> \"\" then") - let use_ = src.IndexOf("monitor.image=%s") - Assert.True(guard < use_, "monitor.image must only be set inside the non-empty guard") + Assert.Contains("v.Replace(\",\", \"\\\\,\")", src) + Assert.Contains("poolMapArgs", src) + // Empty must not reach helm at all: monitor.poolCpu= would blank the chart + // default and every tier would fall back to the flat request. + Assert.Contains("List.filter (fun (_, v) -> not (String.IsNullOrWhiteSpace v))", src) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index a841a762..3dd441d6 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -36,13 +36,6 @@ let helmChartPath = // Example command to run local testing (in the `supercluster/` directory): // $ dotnet run --project src/App/App.fsproj -- mission HistoryPubnetParallelCatchupV2 --image=docker-registry.services.stellar-ops.com/dev/stellar-core:23.0.3-2779.4d1df2b03.jammy-vnext-buildtests --pubnet-parallel-catchup-num-workers=2 --pubnet-parallel-catchup-starting-ledger=0 --pubnet-parallel-catchup-end-ledger=6400 --pubnet-parallel-catchup-ledgers-per-job 1280 --destination ./logs let valuesFilePath = helmChartPath + "/values.yaml" -// Layered on top of values.yaml for on-demand runs only. The chart defaults are -// the spot claims: the spot pools were doubled on 2026-08-04 so each claim is -// half a node and two pods share it. On-demand pools kept their original sizes, -// where those same claims are the node's NAMEPLATE -- and nameplate is not -// allocatable, so nothing schedules at all. A 14336Mi claim has ~13313Mi to land -// in on a 16 GiB node once the EKS reserve and 154Mi of daemonsets come out. -let onDemandValuesFilePath = helmChartPath + "/values-ondemand.yaml" let jobMonitorLoggingIntervalSecs = 30 // frequency of the monitor reconcile loop: dispatch, liveness ping, status publish let jobMonitorStatusCheckIntervalSecs = 60 @@ -245,22 +238,10 @@ let installProject (context: MissionContext) = if context.jobMonitorImagePcV2 <> "" then setOptions.Add(sprintf "monitor.image=%s" context.jobMonitorImagePcV2) - // Capacity type is DERIVED, not configured. Both capacity variants of a tier - // share one label value, so a pod needs this second expression to pick a - // side -- and the storage mode already decides which side it must be. pvc - // exists so an evicted range resumes at LCL+1, which is what makes spot - // survivable; ephemeral has no resume, so it belongs on nodes that are not - // reclaimed underneath it. Letting these disagree would put a run with no - // resume path onto interruptible capacity. if context.pubnetParallelCatchupPoolPrefix <> "" then - let capacityType = - if context.pubnetParallelCatchupStorageMode = "pvc" then "spot" else "on-demand" - - setOptions.Add(sprintf "monitor.capacityType=%s" capacityType) - // Routing needs the label KEY and the taint toleration, and neither has // a sensible default for an unpooled run -- both ship as []. Derived - // here for the same reason capacityType is: a pooled run that sets only + // here because a pooled run that sets only // the prefix otherwise fails twice over, and both failures are quiet. // Karpenter labels these nodes purpose=-, which is exactly // the value job_monitor builds per range, and taints them : @@ -372,7 +353,12 @@ let installProject (context: MissionContext) = if not (List.isEmpty context.requireNodeLabelsPcV2) then let requireLabelsHelm = context.requireNodeLabelsPcV2 - |> List.mapi requireNodeLabelToHelmIndexed + // From 1: a pooled run claims index 0 for the label it routes on, + // and mapi from 0 would overwrite it with whichever came second. + |> List.mapi (fun i pair -> + requireNodeLabelToHelmIndexed + (if context.pubnetParallelCatchupPoolPrefix <> "" then i + 1 else i) + pair) |> String.concat "," setOptions.Add(requireLabelsHelm) @@ -411,21 +397,30 @@ let installProject (context: MissionContext) = // installs its monitor, Jobs and PVCs into a different one. Observed // 2026-07-30: a mission run with --namespace sandbox put a monitor and four // Jobs into the production namespace alongside a live run. - // The overlay rides as a second --values, not as setOptions, because every - // option below is folded into ONE comma-separated --set and the pool maps are - // themselves comma-separated -- they would need every internal comma escaped. - // Derived from storage mode for the same reason capacityType is: pvc means - // spot means shared nodes, ephemeral means on-demand means one pod per node. - let valuesArgs = - if context.pubnetParallelCatchupStorageMode = "pvc" then - [| "--values"; valuesFilePath |] - else - [| "--values"; valuesFilePath; "--values"; onDemandValuesFilePath |] + let valuesArgs = [| "--values"; valuesFilePath |] + + // The pool maps get their own --set each. Every other option is folded into + // ONE comma-joined --set, and these are themselves comma-separated, so + // folding them in would split each tier into a separate assignment. helm + // reads a backslash-escaped comma as data rather than as a separator. + // + // They arrive per run rather than from the chart because the claims differ + // by capacity: the spot pools were doubled on 2026-08-04 so a claim is half + // a node and two pods share it, while the on-demand pools kept their + // original sizes, where that same claim is the node's NAMEPLATE -- and + // nameplate is not allocatable, so nothing schedules at all. + let poolMapArgs = + [ "monitor.poolCpu", context.pubnetParallelCatchupPoolCpu + "monitor.poolMem", context.pubnetParallelCatchupPoolMem ] + |> List.filter (fun (_, v) -> not (String.IsNullOrWhiteSpace v)) + |> List.collect (fun (key, v) -> [ "--set"; sprintf "%s=%s" key (v.Replace(",", "\\,")) ]) + |> Array.ofList RunShellCommand( Array.concat [ [| "helm"; "install"; helmReleaseName; helmChartPath |] [| "--namespace"; context.namespaceProperty |] valuesArgs + poolMapArgs [| "--set"; String.Join(",", setOptions) |] ] ) |> ignore diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 62f81a10..3022ee43 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -126,6 +126,8 @@ type MissionContext = jobMonitorImagePcV2: string pubnetParallelCatchupCpuRequest: string pubnetParallelCatchupMemRequest: string + pubnetParallelCatchupPoolCpu: string + pubnetParallelCatchupPoolMem: string genesisTestAccountCount: int option asanOptions: string option diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py index d92c8b75..8b30bac6 100644 --- a/src/MissionParallelCatchup/apps/job_monitor.py +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -159,6 +159,23 @@ def validate_config(): raise ValueError( "latestLedgerNum must be greater than startingLedger, got %r and %r" % (config.LATEST_LEDGER_NUM, config.STARTING_LEDGER)) + # The pool maps arrive per run, so this is the first point they meet the + # ladder they are keyed to. A tier with no claim does not fail -- the pod + # keeps the flat REQ_CPU/REQ_MEM and a second one fits beside it, which + # undoes the isolation the whole tiering exists for: giving a pod its node + # to itself raised throughput 29-92%. Silent, and only visible afterwards as + # a run that cost more than it should. + if config.POOL_PREFIX: + routable = [name for _, name in sizing._parsed_pool_tiers()] + routable += [config.POOL_UNPROFILED, config.POOL_NO_PROFILE] + for env_name, raw in (('POOL_CPU', config.POOL_CPU), ('POOL_MEM', config.POOL_MEM)): + claimed = {k for k, _ in config.label_pairs(raw)} + missing = [t for t in routable if t and t not in claimed] + if missing: + raise ValueError( + "%s has no entry for %s; a pooled range routed there would " + "keep the flat request and share its node" + % (env_name, ', '.join(sorted(set(missing))))) status = { 'num_remain': 1, # non-zero until the first real update, so callers don't see a premature 0 @@ -1005,12 +1022,11 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): value = f"{config.POOL_PREFIX}-{tier}" if tier else config.NODE_LABEL_VALUE match.append(client.V1NodeSelectorRequirement( key=config.NODE_LABEL_KEY, operator='In', values=[value])) - if config.CAPACITY_TYPE: - # Capacity type is a NodePool property a pod cannot otherwise express, - # and Karpenter labels every node with it. ANDing it here keeps a - # pvc-mode run off on-demand nodes and vice versa. + for key, value in config.label_pairs(config.REQUIRE_NODE_LABELS): + # Literal, unlike the pool-routed pair above: these are properties of + # the pool rather than of the range, so they do not vary per attempt. match.append(client.V1NodeSelectorRequirement( - key='karpenter.sh/capacity-type', operator='In', values=[config.CAPACITY_TYPE])) + key=key, operator='In', values=[value])) if config.AVOID_NODE_LABEL_KEY: # No value means "avoid the label however it is set", which is # DoesNotExist; NotIn [""] would only exclude the empty value. diff --git a/src/MissionParallelCatchup/lib/config.py b/src/MissionParallelCatchup/lib/config.py index 62a422b4..1e2590eb 100644 --- a/src/MissionParallelCatchup/lib/config.py +++ b/src/MissionParallelCatchup/lib/config.py @@ -151,10 +151,30 @@ NODE_LABEL_VALUE = os.getenv('NODE_LABEL_VALUE', '') -# ANDed with the label above when set: 'spot' or 'on-demand'. Both capacity -# variants of a tier carry the same label value, so this is what separates them. -# Karpenter labels every node with karpenter.sh/capacity-type itself. -CAPACITY_TYPE = os.getenv('CAPACITY_TYPE', '') +# Further labels a node must carry, "key:value" comma separated, ANDed with the +# one above. Unlike that one these are literal -- the pair above is pool-routed, +# its value replaced per range with -. +# +# This is where a run pins itself to one capacity of a tier. Both capacities +# carry the same tier label value, so nothing else separates them, and the +# pairing matters: ephemeral has no resume, so a reclaim costs the whole range. +# A plain label rather than karpenter.sh/capacity-type, because the pools +# publish their own and the monitor has no business knowing who provisioned the +# node. +REQUIRE_NODE_LABELS = os.getenv('REQUIRE_NODE_LABELS', '') + + +def label_pairs(raw): + """[(key, value)] from "k:v,k:v". Entries without a value are dropped: a + key alone would require the label be exactly "", which no node carries, and + a pod pinned to nothing sits Pending in a way that reads as slow + provisioning rather than as misconfiguration.""" + out = [] + for item in (raw or '').split(','): + key, _, value = item.strip().partition(':') + if key and value: + out.append((key, value)) + return out AVOID_NODE_LABEL_KEY = os.getenv('AVOID_NODE_LABEL_KEY', '') diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/_helpers.tpl b/src/MissionParallelCatchup/parallel_catchup_helm/templates/_helpers.tpl new file mode 100644 index 00000000..e90e2e26 --- /dev/null +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/_helpers.tpl @@ -0,0 +1,17 @@ +{{/* +Render a list of node-label selectors as "key:value,key:value". + +Accepts both shapes the chart takes elsewhere: {key, values} maps from the +mission, and plain "key:value" strings from a hand-run install. +*/}} +{{- define "catchup.labelPairs" -}} +{{- $out := list -}} +{{- range . -}} +{{- if kindIs "map" . -}} +{{- $out = append $out (printf "%s:%s" .key (first (default (list "") .values))) -}} +{{- else -}} +{{- $out = append $out (toString .) -}} +{{- end -}} +{{- end -}} +{{- join "," $out -}} +{{- end -}} diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 974d92a8..e59a74d2 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -332,8 +332,14 @@ spec: Two accepted shapes: the mission emits structured selectors ({key, operator, values}) like every other supercluster mission, while a hand-run helm install more naturally passes - "key:value" strings. Only the first entry is used -- the - monitor takes a single label pair. + "key:value" strings. + + Entry 0 is the POOL-ROUTED one: on a pooled run the monitor + replaces its value per range with -, so only its + key is used. Every later entry is required literally, which is + how a run pins itself to one capacity of a tier -- both + capacities carry the same tier label value, so nothing else + separates them. */}} {{- with (first .Values.worker.requireNodeLabels) }} {{- if kindIs "map" . }} @@ -348,6 +354,8 @@ spec: value: {{ (splitList ":" .) | last | quote }} {{- end }} {{- end }} + - name: REQUIRE_NODE_LABELS + value: {{ include "catchup.labelPairs" (rest .Values.worker.requireNodeLabels) | quote }} {{- with (first .Values.worker.avoidNodeLabels) }} {{- if kindIs "map" . }} - name: AVOID_NODE_LABEL_KEY diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values-ondemand.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values-ondemand.yaml deleted file mode 100644 index 95f374bf..00000000 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values-ondemand.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# On-demand overlay: one pod per node, AMD-preferred pools. -# -# helm ... -f values.yaml -f values-ondemand.yaml -# -# The default values.yaml is tuned for spot, where the pools were doubled on -# 2026-08-04 and every claim is half a node so two pods share it. On-demand pools -# were deliberately left at their original sizes, so those same claims are the -# NAMEPLATE of an on-demand node -- and nameplate is not allocatable. Every -# on-demand tier was unschedulable as a result: a 14336Mi claim against a -# 16384Mi node has only ~13313Mi to land in once the EKS reserve (255Mi plus the -# 25/20/10/6% tiers) and 154Mi of daemonsets are taken out. -# -# Claims below are sized from that measured allocatable, not from the nameplate, -# and sit above half of it -- so exactly one pod fits and a second never can. -# Verified against a real node the same day: a 16 GiB box reported 13312Mi usable, -# which is the figure these are cut from. -# -# cpu is bounded by the SMALLEST shape in each tier's pool, since a claim that -# only fits the large fallback wins no nodes at all. protostar is the case that -# bites: it still reaches x8i.large at 2 vCPU, so its cpu claim stays at 1.60 -# even though its top rung (r8a.xlarge) has 4. -worker: - # on-demand pairs with ephemeral, always: /data on the node disk is denser and - # there is no eviction to resume from. pvc is what makes spot survivable, and - # the two are never mixed. - storageMode: "ephemeral" - -monitor: - capacityType: "on-demand" - - # Claims are cut from allocatable MINUS the daemonsets, which the previous - # table did not do -- it sized straight to allocatable, so on 2026-08-07 every - # on-demand tier failed to schedule on BOTH dimensions and Karpenter provisioned - # nothing: "no instance type has enough resources ... resources={cpu 1820m, - # memory 3054Mi}" against c8a.large's 2663Mi/1715m. Ten workers sat Pending - # indefinitely, which reads as slow provisioning rather than as a sizing bug. - # - # Daemonset overhead is 494Mi / 245m, measured on this cluster: alloy 10m/50Mi, - # aws-node 75m, ebs-csi-node 30m/104Mi, kube-proxy 100m, and ebs-csi-node-windows - # at 30m/340Mi. That last one CANNOT run here -- it carries - # nodeSelector kubernetes.io/os=windows -- but the nodepools constrain arch and - # not os, so Karpenter cannot prove it away and reserves for it on every node. - # Adding `kubernetes.io/os In [linux]` to the catchup nodepools would return - # 340Mi per node at every tier; the nodepool definitions live outside this repo, - # so these claims are cut against the 494Mi Karpenter actually enforces and stay - # correct (just conservative) if that constraint lands later. - # - # Each claim leaves a 3% margin and still exceeds half of (allocatable - ds), so - # one pod fits and two provably cannot. See the contract test for both halves. - # - # tier shape vCPU allocatable claim - # subdwarf/dwarf c8a.medium 1 1127Mi / 725m 576Mi / 0.45 - # subgiant c8a.large 2 2663Mi / 1715m 2048Mi / 1.40 - # giant m8a.large 2 5940Mi / 1715m 5248Mi / 1.40 - # supergiant r8a.large 2 13313Mi / 1715m 12416Mi / 1.40 - # nebula m8a.2xlarge 4 13313Mi / 3705m 12416Mi / 3.35 - # hypergiant r8a.xlarge 4 28714Mi / 3705m 27328Mi / 3.35 - # protostar x8i.large 2 28714Mi / 1715m 27328Mi / 1.40 - # supernova r8a.2xlarge 8 59515Mi / 7695m 57216Mi / 7.20 - poolCpu: "subdwarf:0.45,dwarf:0.45,subgiant:1.40,giant:1.40,supergiant:1.40,hypergiant:3.35,supernova:7.20,protostar:1.40,nebula:3.35" - poolMem: "subdwarf:576Mi,dwarf:576Mi,subgiant:2048Mi,giant:5248Mi,supergiant:12416Mi,hypergiant:27328Mi,supernova:57216Mi,protostar:27328Mi,nebula:12416Mi" diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index a15533a9..7a5f5466 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -124,10 +124,6 @@ monitor: # No profile at all: nothing is known, so the biggest nodes and the configured # defaults. poolNoProfile: "nebula" - # ANDed with the tier label: "spot" or "on-demand". Both capacity variants of - # a tier share one label value, so this is what separates them. Empty means no - # capacity constraint. - capacityType: "" profileMargin: 1.15 # No margin on cpu: it is compressible, so under-requesting costs contention # Ceiling for profile-derived memory. Above the configured worker limit on From df7a200fd7658b2290c1894f9ffad0727530eefa Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 15:39:16 -0400 Subject: [PATCH 086/117] Move the ParallelCatchupV2 tests into CatchupV2Tests.fs They were split across Tests.fs and TestsRace8.fs, a file named for the defect that prompted it rather than for what it covers. Both now live in one file named for its subject, and the RACE #8 rationale survives as its header -- it explains why the profile-projection tests assert what they do, which is not obvious from the assertions. The block was already self-contained: no reference to ctx, coreSet or any other Tests.fs helper, so this is a move. Tests.fs loses its MissionHistoryPubnetParallelCatchupV2 open, now unused; the pubnetParallelCatchup fields in its MissionContext literal stay, because F# records need every field. Verified as a move rather than a rewrite: same 32 tests, and reintroducing the defect the profile tests exist for -- attaching count before the `entry.Count > 0` guard -- still fails exactly 5 of them from the new file. Also adds HANDOFF.md, which is what the Jenkinsfile needs now that capacity is passed rather than derived: both pool-claim maps verbatim, the three options every pooled run must set, and which failures are loud (a tier missing from a map fails /start with a reason) versus silent (an omitted capacity label lands a spot run on on-demand nodes). Pins the job-monitor image to 2026-08-10a, which carries REQUIRE_NODE_LABELS and the /start pool-map check. Still a personal Docker Hub repo, still a dev pin. --- src/FSLibrary.Tests/CatchupV2Tests.fs | 358 ++++++++++++++++++ src/FSLibrary.Tests/FSLibrary.Tests.fsproj | 68 ++-- src/FSLibrary.Tests/Tests.fs | 150 -------- src/FSLibrary.Tests/TestsRace8.fs | 190 ---------- src/MissionParallelCatchup/HANDOFF.md | 130 +++++++ .../parallel_catchup_helm/values.yaml | 2 +- 6 files changed, 523 insertions(+), 375 deletions(-) create mode 100644 src/FSLibrary.Tests/CatchupV2Tests.fs delete mode 100644 src/FSLibrary.Tests/TestsRace8.fs create mode 100644 src/MissionParallelCatchup/HANDOFF.md diff --git a/src/FSLibrary.Tests/CatchupV2Tests.fs b/src/FSLibrary.Tests/CatchupV2Tests.fs new file mode 100644 index 00000000..34664989 --- /dev/null +++ b/src/FSLibrary.Tests/CatchupV2Tests.fs @@ -0,0 +1,358 @@ +// Copyright 2024 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +// Everything covering MissionHistoryPubnetParallelCatchupV2: the profile +// artifact it writes, and the helm invocation it builds. +// +// The profile tests exist for one defect. A completed record can carry +// bookkeeping (attempts, count) and no measurement at all -- the collector never +// wrote peaks for that range, or the read that would have supplied them was +// degraded. The `entry.Count > 0` guard skips such records, but count used to be +// attached BEFORE the guard ran, so every entry had at least one field and every +// entry passed. The result is an artifact with the right number of ranges and +// zero measurements, which is worse than no artifact: nothing downstream can +// tell it from a good one, and the next run sizes everything from defaults while +// looking correctly configured. Observed twice in the field, reporting 0% +// peakAnonBytes while the monitor's own progress.json carried 99%. +// +// These assert on the values the production projection returns, never on the +// text of the source file -- except where the subject IS the helm invocation, +// which has no return value to inspect. +module CatchupV2Tests + +open Xunit +open Newtonsoft.Json.Linq +open MissionHistoryPubnetParallelCatchupV2 + + +[] +let ``range profile keeps only the measurements that exist`` () = + // The consumer falls back to its configured default when a field is + // absent, so a missing measurement must stay missing rather than become a + // null. peakEphemeralBytes is recorded only for ephemeral-mode runs that + // finished, so most records will not carry it. + let record = JObject() + record.["peakWorkingSetBytes"] <- JValue(1234L) + record.["seconds"] <- JValue(42) + + let entry = projectRangeEntry record + + Assert.Equal(2, entry.Count) + Assert.Equal(1234L, entry.["peakWorkingSetBytes"].Value()) + Assert.Null(entry.["peakEphemeralBytes"]) + + record.["peakEphemeralBytes"] <- JValue(9999L) + let withEph = projectRangeEntry record + Assert.Equal(3, withEph.Count) + Assert.Equal(9999L, withEph.["peakEphemeralBytes"].Value()) + + +[] +let ``range profile carries the fields the sizing consumer prefers`` () = + // peakAnonBytes is the memory figure _profile_overrides reads. Omitting it + // from the projection silently stripped it from the mission artifact while + // the monitor's progress.json carried it for 99% of ranges -- measured + // 2026-07-30, artifact 0% vs volume 99%. + // + // `seconds` is the only timing carried: it is the percentile basis, the + // dispatch order and the runtime insurance threshold. wallSeconds and + // txApply are recorded per range as metrics but nothing sizes from either, + // and wallSeconds alone was 349 KB of a 963 KB artifact. + Assert.Contains("peakAnonBytes", rangeProfileFields) + Assert.Contains("seconds", rangeProfileFields) + Assert.DoesNotContain("wallSeconds", rangeProfileFields) + Assert.DoesNotContain("txApply", rangeProfileFields) + + let record = JObject() + record.["peakAnonBytes"] <- JValue(111L) + record.["seconds"] <- JValue(50.0) + record.["wallSeconds"] <- JValue(999.0) + let entry = projectRangeEntry record + Assert.Equal(111L, entry.["peakAnonBytes"].Value()) + Assert.Equal(50.0, entry.["seconds"].Value()) + Assert.Null(entry.["wallSeconds"]) + + +[] +let ``range profile keeps count as a field so it can be keyed on end alone`` () = + // Measured: 4.2x the ledgers per range moved peak disk -1.6% and wall time + // 1.15x, so cost tracks ledger position rather than range length. Keying on + // end/count would discard the whole profile whenever overlapLedgers or + // ledgersPerJob changed, for a distinction the measurements say is small. + Assert.DoesNotContain("count", rangeProfileFields) + + let record = JObject() + record.["peakWorkingSetBytes"] <- JValue(1L) + record.["count"] <- JValue(420) + // projectRangeEntry itself must not copy count -- writeRangeProfile attaches + // it separately, so a stale profile cannot smuggle it in as a measurement. + let entry = projectRangeEntry record + Assert.Null(entry.["count"]) + + +[] +let ``range profile does not carry a pvc volume peak`` () = + // A PVC's size is not a scheduling dimension, so growing it buys no + // packing and it is deliberately not profiled. + Assert.DoesNotContain("peakVolumeBytes", rangeProfileFields) + + +[] +let ``pvc mode does not reserve node disk it never uses`` () = + // /data is on the volume in pvc mode; the node disk only holds logs and tmp. + // Asking for the ephemeral-mode figure makes disk rather than cpu the + // binding dimension and cuts pods per node. + let src = + System.IO.File.ReadAllText( + "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + Assert.Contains("pubnetParallelCatchupStorageMode = \"pvc\"", src) + Assert.Contains("\"2Gi\", \"4Gi\"", src) + + +[] +let ``progress record is read from the volume and never from the configmap`` () = + // /logs/progress.json is the monitor's own state: authoritative, unbounded, + // and the only copy carrying measurements. The ConfigMap is the driver's + // view of the run -- status only -- so a record sourced from it would build + // an artifact that looks complete and measures nothing. + let src = + System.IO.File.ReadAllText( + "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + Assert.Contains("/logs/progress.json", src) + Assert.DoesNotContain("jobMonitorProgressKey", src) + + +[] +let ``the job monitor image is overridable and defaults to the chart`` () = + // The monitor and collector ship as one image pinned in values.yaml. Passing + // it per run is what lets a build of them be tested without editing the + // chart -- but an empty flag must leave the chart's pin alone rather than + // setting monitor.image to nothing, which resolves to ":latest" or fails the + // pull outright. + let src = + System.IO.File.ReadAllText( + "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + + Assert.Contains("if context.jobMonitorImagePcV2 <> \"\" then", src) + Assert.Contains("monitor.image=%s", src) + + let guard = src.IndexOf("if context.jobMonitorImagePcV2 <> \"\" then") + let use_ = src.IndexOf("monitor.image=%s") + Assert.True(guard < use_, "monitor.image must only be set inside the non-empty guard") + + +[] +let ``a pooled run does not let caller labels overwrite the routing label`` () = + // A pooled run claims worker.requireNodeLabels[0] for the label it routes + // on, and requireNodeLabelsPcV2 used to index its own entries from 0 as + // well. Both fire on a pooled run carrying a capacity label, and the second + // --set wins: the routing label is replaced, so the pod matches on capacity + // alone and lands on any tier at all -- a range sized for supergiant on a + // dwarf node, which is an OOM per range rather than a slow run. + let src = + System.IO.File.ReadAllText( + "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + + Assert.Contains("worker.requireNodeLabels[0]=purpose:%s", src) + Assert.Contains("if context.pubnetParallelCatchupPoolPrefix <> \"\" then i + 1 else i", src) + + +[] +let ``the pool maps ride their own --set with their commas escaped`` () = + // Every other option is folded into ONE comma-joined --set. The pool maps + // are themselves comma-separated, so folding them in would split each tier + // into a separate helm assignment and the map would arrive holding one + // tier. This is why the overlay used to be a second --values file. + let src = + System.IO.File.ReadAllText( + "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + + Assert.Contains("v.Replace(\",\", \"\\\\,\")", src) + Assert.Contains("poolMapArgs", src) + // Empty must not reach helm at all: monitor.poolCpu= would blank the chart + // default and every tier would fall back to the flat request. + Assert.Contains("List.filter (fun (_, v) -> not (String.IsNullOrWhiteSpace v))", src) + +// RACE #8 -- a measurement-free profile artifact that is indistinguishable from +// a good one. +// +// A completed record can carry bookkeeping (attempts, count) and no measurement +// at all: the collector never wrote peaks for that range, or the read that would +// have supplied them was degraded. Such a record must not become a profile entry. +// +// The `entry.Count > 0` guard exists to skip measurement-free entries, but count +// used to be attached to the entry BEFORE the guard ran, so every entry had at +// least one field and every entry passed. The result is an artifact with the +// right number of ranges and zero measurements -- observed twice in the field, +// reporting 0% peakAnonBytes while the monitor's own progress.json carried 99%. +// The next run then sizes from a profile that silently has no data. +// +// These tests assert on the values the production projection actually returns, +// never on the text of the source file. + + +/// Every measurement a completed record can carry. A superset of +/// rangeProfileFields: wallSeconds and txApply are recorded but never projected. +let private measurementFields = + [ "peakAnonBytes"; "peakWorkingSetBytes"; "peakEphemeralBytes" + "txApply"; "seconds"; "wallSeconds" ] + +/// A completed record the way /logs/progress.json carries it: bookkeeping plus +/// real measurements. +let private measuredRecord (count: int) (anon: int64) = + let r = JObject() + r.["attempts"] <- JValue(1) + r.["count"] <- JValue(count) + r.["seconds"] <- JValue(120.0) + r.["wallSeconds"] <- JValue(130.0) + r.["txApply"] <- JValue(60.0) + r.["peakAnonBytes"] <- JValue(anon) + r.["peakWorkingSetBytes"] <- JValue(anon + 1000L) + r + +/// The same record as it survives the ConfigMap mirror. +let private unmeasured (record: JObject) = + let r = record.DeepClone() :?> JObject + + for f in measurementFields do + r.Remove(f) |> ignore + + r + +let private completedMap (pairs: (string * JObject) list) = + let c = JObject() + + for (k, v) in pairs do + c.[k] <- v + + c + + +[] +let ``a range carrying no measurement must not enter the profile`` () = + let mirrored = unmeasured (measuredRecord 420 900L) + + // Precondition: the helper really does strip every measurement, leaving + // only bookkeeping. If this ever stops holding, the rest is meaningless. + for f in measurementFields do + Assert.Null(mirrored.[f]) + + Assert.NotNull(mirrored.["count"]) + Assert.NotNull(mirrored.["attempts"]) + + let ranges = buildRangeProfile (completedMap [ "420", mirrored ]) + + Assert.Equal(0, ranges.Count) + + +[] +let ``count alone never satisfies the measurement guard`` () = + // This is the defeated guard in its smallest form: count is the only field, + // and it is bookkeeping, not a measurement. + let onlyCount = JObject() + onlyCount.["count"] <- JValue(420) + + let ranges = buildRangeProfile (completedMap [ "420", onlyCount ]) + + Assert.Equal(0, ranges.Count) + + +[] +let ``a run whose ranges measured nothing produces no profile artifact`` () = + // The headline symptom: a full-looking artifact, right number of ranges, + // zero measurements -- what a run whose collector never wrote peaks leaves + // behind. Writing nothing is correct: the next run then falls back to its + // configured defaults instead of sizing from empty data. + let completed = + completedMap + [ "420", unmeasured (measuredRecord 420 900L) + "840", unmeasured (measuredRecord 420 950L) + "1260", unmeasured (measuredRecord 420 990L) ] + + match rangeProfileDocument "pvc" 20000 completed with + | None -> () + | Some doc -> + let ranges = doc.["ranges"] :?> JObject + + failwithf + "wrote a profile artifact with %d ranges and zero measurements: %s" + ranges.Count + (ranges.ToString()) + + +[] +let ``the profile counts only ranges that actually measured something`` () = + // A partly-measured run is the dangerous case: the artifact looks + // populated, so nothing downstream can tell the unmeasured ranges apart + // from the measured one. + let completed = + completedMap + [ "420", measuredRecord 420 900L + "840", unmeasured (measuredRecord 420 950L) + "1260", unmeasured (measuredRecord 420 990L) ] + + let ranges = buildRangeProfile completed + + Assert.Equal(1, ranges.Count) + Assert.NotNull(ranges.["420"]) + Assert.Null(ranges.["840"]) + Assert.Null(ranges.["1260"]) + + +[] +let ``a measured run still produces a complete profile`` () = + // Guard against over-correcting: a good read must still write everything. + let completed = + completedMap + [ "420", measuredRecord 420 900L + "840", measuredRecord 420 950L ] + + match rangeProfileDocument "pvc" 20000 completed with + | None -> failwith "refused to write a profile that carries real measurements" + | Some doc -> + let ranges = doc.["ranges"] :?> JObject + Assert.Equal(2, ranges.Count) + Assert.Equal(900L, ranges.["420"].["peakAnonBytes"].Value()) + Assert.Equal(950L, ranges.["840"].["peakAnonBytes"].Value()) + Assert.Equal(120.0, ranges.["420"].["seconds"].Value()) + // Measured but deliberately not projected: recorded as metrics, never + // sized or ordered from, and pure weight in a 1 MiB-capped artifact. + Assert.Null(ranges.["420"].["wallSeconds"]) + Assert.Null(ranges.["420"].["txApply"]) + // count is still carried, and the slicing is inferred from it rather + // than from the caller's default. + Assert.Equal(420, ranges.["420"].["count"].Value()) + Assert.Equal(420, doc.["ledgersPerRange"].Value()) + Assert.Equal("pvc", doc.["storageMode"].Value()) + + +[] +let ``a single real measurement is enough to keep a range and it keeps its count`` () = + // The fix must move count after the guard without dropping it -- count is + // what ledgersPerRange is inferred from. + let r = JObject() + r.["attempts"] <- JValue(2) + r.["count"] <- JValue(420) + r.["seconds"] <- JValue(77.0) + + let ranges = buildRangeProfile (completedMap [ "420", r ]) + + Assert.Equal(1, ranges.Count) + Assert.Equal(77.0, ranges.["420"].["seconds"].Value()) + Assert.Equal(420, ranges.["420"].["count"].Value()) + + +[] +let ``a range measured only in unprojected fields is dropped, not kept on count alone`` () = + // The guard reads the PROJECTION, not the record, so narrowing + // rangeProfileFields narrows what counts as measured. A record carrying only + // wallSeconds/txApply now projects to nothing and must be dropped -- keeping + // it would reintroduce exactly the count-only entry the guard exists to stop. + let r = JObject() + r.["attempts"] <- JValue(1) + r.["count"] <- JValue(420) + r.["wallSeconds"] <- JValue(77.0) + r.["txApply"] <- JValue(12.0) + + Assert.Empty(buildRangeProfile (completedMap [ "420", r ])) diff --git a/src/FSLibrary.Tests/FSLibrary.Tests.fsproj b/src/FSLibrary.Tests/FSLibrary.Tests.fsproj index b5a66627..a32c892b 100644 --- a/src/FSLibrary.Tests/FSLibrary.Tests.fsproj +++ b/src/FSLibrary.Tests/FSLibrary.Tests.fsproj @@ -1,34 +1,34 @@ - - - - net8.0 - - false - false - Exe - - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - runtime; build; native; contentfiles; analyzers; buildtransitive -all - - - - - - - - + + + + net8.0 + + false + false + Exe + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + runtime; build; native; contentfiles; analyzers; buildtransitive +all + + + + + + + + diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index 6a9fae9b..7e002eca 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -13,7 +13,6 @@ open StellarKubeSpecs open StellarNetworkData open StellarNetworkDelays open MissionCatchupHelpers -open MissionHistoryPubnetParallelCatchupV2 open Newtonsoft.Json.Linq open Xunit.Abstractions @@ -553,152 +552,3 @@ type Tests(output: ITestOutputHelper) = Assert.Equal("51/6", jobArr3.[0].[1]) Assert.Equal("56/6", jobArr3.[1].[1]) Assert.Equal("61/6", jobArr3.[2].[1]) - - -[] -let ``range profile keeps only the measurements that exist`` () = - // The consumer falls back to its configured default when a field is - // absent, so a missing measurement must stay missing rather than become a - // null. peakEphemeralBytes is recorded only for ephemeral-mode runs that - // finished, so most records will not carry it. - let record = JObject() - record.["peakWorkingSetBytes"] <- JValue(1234L) - record.["seconds"] <- JValue(42) - - let entry = projectRangeEntry record - - Assert.Equal(2, entry.Count) - Assert.Equal(1234L, entry.["peakWorkingSetBytes"].Value()) - Assert.Null(entry.["peakEphemeralBytes"]) - - record.["peakEphemeralBytes"] <- JValue(9999L) - let withEph = projectRangeEntry record - Assert.Equal(3, withEph.Count) - Assert.Equal(9999L, withEph.["peakEphemeralBytes"].Value()) - - -[] -let ``range profile carries the fields the sizing consumer prefers`` () = - // peakAnonBytes is the memory figure _profile_overrides reads. Omitting it - // from the projection silently stripped it from the mission artifact while - // the monitor's progress.json carried it for 99% of ranges -- measured - // 2026-07-30, artifact 0% vs volume 99%. - // - // `seconds` is the only timing carried: it is the percentile basis, the - // dispatch order and the runtime insurance threshold. wallSeconds and - // txApply are recorded per range as metrics but nothing sizes from either, - // and wallSeconds alone was 349 KB of a 963 KB artifact. - Assert.Contains("peakAnonBytes", rangeProfileFields) - Assert.Contains("seconds", rangeProfileFields) - Assert.DoesNotContain("wallSeconds", rangeProfileFields) - Assert.DoesNotContain("txApply", rangeProfileFields) - - let record = JObject() - record.["peakAnonBytes"] <- JValue(111L) - record.["seconds"] <- JValue(50.0) - record.["wallSeconds"] <- JValue(999.0) - let entry = projectRangeEntry record - Assert.Equal(111L, entry.["peakAnonBytes"].Value()) - Assert.Equal(50.0, entry.["seconds"].Value()) - Assert.Null(entry.["wallSeconds"]) - - -[] -let ``range profile keeps count as a field so it can be keyed on end alone`` () = - // Measured: 4.2x the ledgers per range moved peak disk -1.6% and wall time - // 1.15x, so cost tracks ledger position rather than range length. Keying on - // end/count would discard the whole profile whenever overlapLedgers or - // ledgersPerJob changed, for a distinction the measurements say is small. - Assert.DoesNotContain("count", rangeProfileFields) - - let record = JObject() - record.["peakWorkingSetBytes"] <- JValue(1L) - record.["count"] <- JValue(420) - // projectRangeEntry itself must not copy count -- writeRangeProfile attaches - // it separately, so a stale profile cannot smuggle it in as a measurement. - let entry = projectRangeEntry record - Assert.Null(entry.["count"]) - - -[] -let ``range profile does not carry a pvc volume peak`` () = - // A PVC's size is not a scheduling dimension, so growing it buys no - // packing and it is deliberately not profiled. - Assert.DoesNotContain("peakVolumeBytes", rangeProfileFields) - - -[] -let ``pvc mode does not reserve node disk it never uses`` () = - // /data is on the volume in pvc mode; the node disk only holds logs and tmp. - // Asking for the ephemeral-mode figure makes disk rather than cpu the - // binding dimension and cuts pods per node. - let src = - System.IO.File.ReadAllText( - "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") - Assert.Contains("pubnetParallelCatchupStorageMode = \"pvc\"", src) - Assert.Contains("\"2Gi\", \"4Gi\"", src) - - -[] -let ``progress record is read from the volume and never from the configmap`` () = - // /logs/progress.json is the monitor's own state: authoritative, unbounded, - // and the only copy carrying measurements. The ConfigMap is the driver's - // view of the run -- status only -- so a record sourced from it would build - // an artifact that looks complete and measures nothing. - let src = - System.IO.File.ReadAllText( - "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") - Assert.Contains("/logs/progress.json", src) - Assert.DoesNotContain("jobMonitorProgressKey", src) - - -[] -let ``the job monitor image is overridable and defaults to the chart`` () = - // The monitor and collector ship as one image pinned in values.yaml. Passing - // it per run is what lets a build of them be tested without editing the - // chart -- but an empty flag must leave the chart's pin alone rather than - // setting monitor.image to nothing, which resolves to ":latest" or fails the - // pull outright. - let src = - System.IO.File.ReadAllText( - "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") - - Assert.Contains("if context.jobMonitorImagePcV2 <> \"\" then", src) - Assert.Contains("monitor.image=%s", src) - - let guard = src.IndexOf("if context.jobMonitorImagePcV2 <> \"\" then") - let use_ = src.IndexOf("monitor.image=%s") - Assert.True(guard < use_, "monitor.image must only be set inside the non-empty guard") - - -[] -let ``a pooled run does not let caller labels overwrite the routing label`` () = - // A pooled run claims worker.requireNodeLabels[0] for the label it routes - // on, and requireNodeLabelsPcV2 used to index its own entries from 0 as - // well. Both fire on a pooled run carrying a capacity label, and the second - // --set wins: the routing label is replaced, so the pod matches on capacity - // alone and lands on any tier at all -- a range sized for supergiant on a - // dwarf node, which is an OOM per range rather than a slow run. - let src = - System.IO.File.ReadAllText( - "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") - - Assert.Contains("worker.requireNodeLabels[0]=purpose:%s", src) - Assert.Contains("if context.pubnetParallelCatchupPoolPrefix <> \"\" then i + 1 else i", src) - - -[] -let ``the pool maps ride their own --set with their commas escaped`` () = - // Every other option is folded into ONE comma-joined --set. The pool maps - // are themselves comma-separated, so folding them in would split each tier - // into a separate helm assignment and the map would arrive holding one - // tier. This is why the overlay used to be a second --values file. - let src = - System.IO.File.ReadAllText( - "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") - - Assert.Contains("v.Replace(\",\", \"\\\\,\")", src) - Assert.Contains("poolMapArgs", src) - // Empty must not reach helm at all: monitor.poolCpu= would blank the chart - // default and every tier would fall back to the flat request. - Assert.Contains("List.filter (fun (_, v) -> not (String.IsNullOrWhiteSpace v))", src) diff --git a/src/FSLibrary.Tests/TestsRace8.fs b/src/FSLibrary.Tests/TestsRace8.fs deleted file mode 100644 index 16f9f491..00000000 --- a/src/FSLibrary.Tests/TestsRace8.fs +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2024 Stellar Development Foundation and contributors. Licensed -// under the Apache License, Version 2.0. See the COPYING file at the root -// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 - -// RACE #8 -- a measurement-free profile artifact that is indistinguishable from -// a good one. -// -// A completed record can carry bookkeeping (attempts, count) and no measurement -// at all: the collector never wrote peaks for that range, or the read that would -// have supplied them was degraded. Such a record must not become a profile entry. -// -// The `entry.Count > 0` guard exists to skip measurement-free entries, but count -// used to be attached to the entry BEFORE the guard ran, so every entry had at -// least one field and every entry passed. The result is an artifact with the -// right number of ranges and zero measurements -- observed twice in the field, -// reporting 0% peakAnonBytes while the monitor's own progress.json carried 99%. -// The next run then sizes from a profile that silently has no data. -// -// These tests assert on the values the production projection actually returns, -// never on the text of the source file. -module Race8Tests - -open Xunit -open Newtonsoft.Json.Linq -open MissionHistoryPubnetParallelCatchupV2 - -/// Every measurement a completed record can carry. A superset of -/// rangeProfileFields: wallSeconds and txApply are recorded but never projected. -let private measurementFields = - [ "peakAnonBytes"; "peakWorkingSetBytes"; "peakEphemeralBytes" - "txApply"; "seconds"; "wallSeconds" ] - -/// A completed record the way /logs/progress.json carries it: bookkeeping plus -/// real measurements. -let private measuredRecord (count: int) (anon: int64) = - let r = JObject() - r.["attempts"] <- JValue(1) - r.["count"] <- JValue(count) - r.["seconds"] <- JValue(120.0) - r.["wallSeconds"] <- JValue(130.0) - r.["txApply"] <- JValue(60.0) - r.["peakAnonBytes"] <- JValue(anon) - r.["peakWorkingSetBytes"] <- JValue(anon + 1000L) - r - -/// The same record as it survives the ConfigMap mirror. -let private unmeasured (record: JObject) = - let r = record.DeepClone() :?> JObject - - for f in measurementFields do - r.Remove(f) |> ignore - - r - -let private completedMap (pairs: (string * JObject) list) = - let c = JObject() - - for (k, v) in pairs do - c.[k] <- v - - c - - -[] -let ``a range carrying no measurement must not enter the profile`` () = - let mirrored = unmeasured (measuredRecord 420 900L) - - // Precondition: the helper really does strip every measurement, leaving - // only bookkeeping. If this ever stops holding, the rest is meaningless. - for f in measurementFields do - Assert.Null(mirrored.[f]) - - Assert.NotNull(mirrored.["count"]) - Assert.NotNull(mirrored.["attempts"]) - - let ranges = buildRangeProfile (completedMap [ "420", mirrored ]) - - Assert.Equal(0, ranges.Count) - - -[] -let ``count alone never satisfies the measurement guard`` () = - // This is the defeated guard in its smallest form: count is the only field, - // and it is bookkeeping, not a measurement. - let onlyCount = JObject() - onlyCount.["count"] <- JValue(420) - - let ranges = buildRangeProfile (completedMap [ "420", onlyCount ]) - - Assert.Equal(0, ranges.Count) - - -[] -let ``a run whose ranges measured nothing produces no profile artifact`` () = - // The headline symptom: a full-looking artifact, right number of ranges, - // zero measurements -- what a run whose collector never wrote peaks leaves - // behind. Writing nothing is correct: the next run then falls back to its - // configured defaults instead of sizing from empty data. - let completed = - completedMap - [ "420", unmeasured (measuredRecord 420 900L) - "840", unmeasured (measuredRecord 420 950L) - "1260", unmeasured (measuredRecord 420 990L) ] - - match rangeProfileDocument "pvc" 20000 completed with - | None -> () - | Some doc -> - let ranges = doc.["ranges"] :?> JObject - - failwithf - "wrote a profile artifact with %d ranges and zero measurements: %s" - ranges.Count - (ranges.ToString()) - - -[] -let ``the profile counts only ranges that actually measured something`` () = - // A partly-measured run is the dangerous case: the artifact looks - // populated, so nothing downstream can tell the unmeasured ranges apart - // from the measured one. - let completed = - completedMap - [ "420", measuredRecord 420 900L - "840", unmeasured (measuredRecord 420 950L) - "1260", unmeasured (measuredRecord 420 990L) ] - - let ranges = buildRangeProfile completed - - Assert.Equal(1, ranges.Count) - Assert.NotNull(ranges.["420"]) - Assert.Null(ranges.["840"]) - Assert.Null(ranges.["1260"]) - - -[] -let ``a measured run still produces a complete profile`` () = - // Guard against over-correcting: a good read must still write everything. - let completed = - completedMap - [ "420", measuredRecord 420 900L - "840", measuredRecord 420 950L ] - - match rangeProfileDocument "pvc" 20000 completed with - | None -> failwith "refused to write a profile that carries real measurements" - | Some doc -> - let ranges = doc.["ranges"] :?> JObject - Assert.Equal(2, ranges.Count) - Assert.Equal(900L, ranges.["420"].["peakAnonBytes"].Value()) - Assert.Equal(950L, ranges.["840"].["peakAnonBytes"].Value()) - Assert.Equal(120.0, ranges.["420"].["seconds"].Value()) - // Measured but deliberately not projected: recorded as metrics, never - // sized or ordered from, and pure weight in a 1 MiB-capped artifact. - Assert.Null(ranges.["420"].["wallSeconds"]) - Assert.Null(ranges.["420"].["txApply"]) - // count is still carried, and the slicing is inferred from it rather - // than from the caller's default. - Assert.Equal(420, ranges.["420"].["count"].Value()) - Assert.Equal(420, doc.["ledgersPerRange"].Value()) - Assert.Equal("pvc", doc.["storageMode"].Value()) - - -[] -let ``a single real measurement is enough to keep a range and it keeps its count`` () = - // The fix must move count after the guard without dropping it -- count is - // what ledgersPerRange is inferred from. - let r = JObject() - r.["attempts"] <- JValue(2) - r.["count"] <- JValue(420) - r.["seconds"] <- JValue(77.0) - - let ranges = buildRangeProfile (completedMap [ "420", r ]) - - Assert.Equal(1, ranges.Count) - Assert.Equal(77.0, ranges.["420"].["seconds"].Value()) - Assert.Equal(420, ranges.["420"].["count"].Value()) - - -[] -let ``a range measured only in unprojected fields is dropped, not kept on count alone`` () = - // The guard reads the PROJECTION, not the record, so narrowing - // rangeProfileFields narrows what counts as measured. A record carrying only - // wallSeconds/txApply now projects to nothing and must be dropped -- keeping - // it would reintroduce exactly the count-only entry the guard exists to stop. - let r = JObject() - r.["attempts"] <- JValue(1) - r.["count"] <- JValue(420) - r.["wallSeconds"] <- JValue(77.0) - r.["txApply"] <- JValue(12.0) - - Assert.Empty(buildRangeProfile (completedMap [ "420", r ])) diff --git a/src/MissionParallelCatchup/HANDOFF.md b/src/MissionParallelCatchup/HANDOFF.md new file mode 100644 index 00000000..e9abf2c9 --- /dev/null +++ b/src/MissionParallelCatchup/HANDOFF.md @@ -0,0 +1,130 @@ +# Jenkinsfile options for ParallelCatchupV2 + +What changed: the chart no longer knows that spot and on-demand exist. +`values-ondemand.yaml` is deleted, `monitor.capacityType` is gone, and the +monitor no longer reads `karpenter.sh/capacity-type`. Everything that used to be +derived is now passed in, so **Jenkins is the only place that knows which +capacity a run targets**. + +Nothing has a safe default any more. A run that omits these gets *no* capacity +constraint and the chart's built-in (spot) claim maps. + +## What every pooled run must pass + +Three things, and both capacities pass all three: + +| | spot | on-demand | +|---|---|---| +| capacity label | `--require-node-labels-pc-v2 catchup-capacity:spot` | `--require-node-labels-pc-v2 catchup-capacity:od` | +| cpu claims | `--pubnet-parallel-catchup-pool-cpu ""` | `--pubnet-parallel-catchup-pool-cpu ""` | +| memory claims | `--pubnet-parallel-catchup-pool-mem ""` | `--pubnet-parallel-catchup-pool-mem ""` | + +Plus the ones that already existed and are unchanged: + +``` +--pubnet-parallel-catchup-pool-prefix catchup +--pubnet-parallel-catchup-storage-mode pvc|ephemeral +``` + +### The maps + +Copy these verbatim. They are the values that were in `values.yaml` and +`values-ondemand.yaml` before the change. + +**spot** — pools were doubled on 2026-08-04, so a claim is half a node and two +pods share it: + +``` +poolCpu: subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80 +poolMem: subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:9216Mi +``` + +**on-demand** — pools kept their original sizes, so a claim is the whole node +and one pod gets it: + +``` +poolCpu: subdwarf:0.45,dwarf:0.45,subgiant:1.40,giant:1.40,supergiant:1.40,hypergiant:3.35,supernova:7.20,protostar:1.40,nebula:3.35 +poolMem: subdwarf:576Mi,dwarf:576Mi,subgiant:2048Mi,giant:5248Mi,supergiant:12416Mi,hypergiant:27328Mi,supernova:57216Mi,protostar:27328Mi,nebula:12416Mi +``` + +Do **not** cross them. On-demand claims are sized against *allocatable*; the +spot claims are the on-demand node's *nameplate*, and nameplate is not +allocatable. Shipping spot claims to an on-demand pool makes every tier +unschedulable and Karpenter provisions nothing — pods sit Pending, which reads +as slow provisioning rather than as a sizing bug. That is what happened on +2026-08-07 and it cost a day to spot. + +## The pairing that used to be automatic + +Storage mode and capacity are now fully independent. Nothing derives one from +the other and nothing will object if they disagree. + +The pairing every production run wants: + +``` +pvc + catchup-capacity:spot +ephemeral + catchup-capacity:od +``` + +Why it matters: `pvc` keeps `/data` across pods, so an evicted range resumes at +LCL+1 — that is what makes spot survivable. `ephemeral` puts `/data` on the node +and has no resume, so a spot reclaim costs the whole range from scratch. + +They were split on purpose, so a test run can pair `ephemeral` with `spot` +deliberately to measure what a reclaim actually costs. Production should not. + +## Example + +``` +dotnet run --project src/App/App.fsproj -- mission HistoryPubnetParallelCatchupV2 \ + --image \ + --namespace stellar-supercluster \ + --pubnet-parallel-catchup-pool-prefix catchup \ + --pubnet-parallel-catchup-storage-mode pvc \ + --require-node-labels-pc-v2 catchup-capacity:spot \ + --pubnet-parallel-catchup-pool-cpu "subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80" \ + --pubnet-parallel-catchup-pool-mem "subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:9216Mi" \ + --pubnet-parallel-catchup-profile \ + --pubnet-parallel-catchup-num-workers 1024 \ + --pubnet-parallel-catchup-ledgers-per-job 16000 \ + --destination ./logs +``` + +## Failure modes, and which are loud + +**Loud** — a missing tier in either map fails `POST /start` with a 400 and a +reason, so the driver stops immediately: + +``` +POOL_CPU has no entry for nebula; a pooled range routed there would keep +the flat request and share its node +``` + +The check covers every ladder tier plus `protostar` (ranges newer than the +profile) and `nebula` (runs with no profile at all). Both are routed to, so both +need entries. `subdwarf` is dormant — it can never be selected — but it still +needs a map entry. + +**Silent** — these produce a run that finishes and costs more than it should: + +- *capacity label omitted.* Both capacities of a tier carry the same + `purpose=catchup-` label, so nothing separates them. A spot run without + `catchup-capacity:spot` can land on on-demand nodes and bill on-demand rates. +- *wrong map for the capacity.* Covered above. Not silent on on-demand (nothing + schedules) but silent the other way: on-demand claims on spot pools pack one + pod where two fit, halving throughput per node. +- *tier name typo in a map.* Reads as a missing tier, so `/start` catches it — + the one typo class that is loud. + +## Ordering note + +If Jenkins passes several `--require-node-labels-pc-v2` entries, order does not +matter — they all become literal requirements. The mission reserves +`worker.requireNodeLabels[0]` for its own pool-routing label and starts the +caller's entries at index 1. Before this change they both wrote index 0 and the +routing label was overwritten, which put ranges on arbitrary tiers. + +## Image + +The chart pins `stellajuna/ssc-jm:2026-08-10a`, a personal Docker Hub repo. +**This is a dev pin and must be replaced before ssc-eks.** diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 7a5f5466..a6cbd30d 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -68,7 +68,7 @@ monitor: # this chart against it ships env vars the image cannot read. Built from this # branch and pinned by tag rather than :latest so a run is always traceable to # one image. Revert to the stellar/ repo once there is a push path for it. - image: "stellajuna/ssc-jm:2026-08-09a" + image: "stellajuna/ssc-jm:2026-08-10a" # The driver reaches the monitor through this route: profile in via POST # /start, status and logs out. Empty routeHost disables the HTTPRoute; the # Service still exists, so an in-cluster caller works either way. From 0fa75f4f6e236008dd45fdc9a94af091f5b15c21 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 16:20:46 -0400 Subject: [PATCH 087/117] Cut eight redundant V2 tests, add --pubnet-parallel-catchup-create-rbac Mutation testing, 24 mutants against the projection and the helm invocation. Eight of the sixteen tests were the sole killer of at least one mutant; the other eight killed nothing any surviving test misses. Re-running every mutant against the reduced file kills all of them, so the cuts lost no coverage. The worst duplication was the measurement guard: four tests -- a range carrying no measurement, count alone never satisfies the guard, a range measured only in unprojected fields, and the profile counts only ranges that measured -- all died on exactly the same three mutants. Four phrasings of "count is not a measurement"; one is enough. progress record is read from the volume killed nothing at all, and could not: it grepped the whole source for "/logs/progress.json", which also appears in a nearby LogWarn, so repointing the actual `cat` at another path left it passing. Deleted rather than repaired -- a test that cannot fail for its own reason is worse than none, because it reads as coverage. Worth recording for next time: three tests looked vacuous after 21 mutants and turned out to be irreplaceable once mutants were aimed at their subject -- nothing had touched storage sizing or the volume-peak field. "Kills nothing" is a claim about the mutant set, not about the test. Also adds --pubnet-parallel-catchup-create-rbac, default false. ssc-eks provides the monitor's RBAC through a catchup-job-monitor RoleBinding covering system:serviceaccounts:stellar-supercluster, so prod keeps the chart's createRbac off. ssc-test has no such binding: without the flag the monitor 403s reading its own stellar-core-config ConfigMap and dispatches nothing, while /status keeps answering and the driver polls a run that never starts. Found by running this branch there. Tests.fs loses its now-unused Newtonsoft.Json.Linq open, left behind when the V2 tests moved out. --- src/App/Program.fs | 8 ++ src/FSLibrary.Tests/CatchupV2Tests.fs | 135 ------------------ src/FSLibrary.Tests/Tests.fs | 2 +- .../MissionHistoryPubnetParallelCatchupV2.fs | 7 + src/FSLibrary/StellarMissionContext.fs | 1 + src/MissionParallelCatchup/HANDOFF.md | 12 ++ 6 files changed, 29 insertions(+), 136 deletions(-) diff --git a/src/App/Program.fs b/src/App/Program.fs index 6742a550..2407642b 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -124,6 +124,7 @@ type MissionOptions pubnetParallelCatchupMemRequest: string, pubnetParallelCatchupPoolCpu: string, pubnetParallelCatchupPoolMem: string, + pubnetParallelCatchupCreateRbac: bool, tag: string option, numPregeneratedTxs: int option, genesisTestAccountCount: int option, @@ -584,6 +585,12 @@ type MissionOptions Default = "")>] member self.PubnetParallelCatchupPoolMem : string = pubnetParallelCatchupPoolMem + [] + member self.PubnetParallelCatchupCreateRbac : bool = pubnetParallelCatchupCreateRbac + [] member self.Tag = tag @@ -970,6 +977,7 @@ let main argv = pubnetParallelCatchupMemRequest = mission.PubnetParallelCatchupMemRequest pubnetParallelCatchupPoolCpu = mission.PubnetParallelCatchupPoolCpu pubnetParallelCatchupPoolMem = mission.PubnetParallelCatchupPoolMem + pubnetParallelCatchupCreateRbac = mission.PubnetParallelCatchupCreateRbac tag = mission.Tag numPregeneratedTxs = mission.NumPregeneratedTxs enableTailLogging = true diff --git a/src/FSLibrary.Tests/CatchupV2Tests.fs b/src/FSLibrary.Tests/CatchupV2Tests.fs index 34664989..8e1306cb 100644 --- a/src/FSLibrary.Tests/CatchupV2Tests.fs +++ b/src/FSLibrary.Tests/CatchupV2Tests.fs @@ -48,49 +48,6 @@ let ``range profile keeps only the measurements that exist`` () = Assert.Equal(9999L, withEph.["peakEphemeralBytes"].Value()) -[] -let ``range profile carries the fields the sizing consumer prefers`` () = - // peakAnonBytes is the memory figure _profile_overrides reads. Omitting it - // from the projection silently stripped it from the mission artifact while - // the monitor's progress.json carried it for 99% of ranges -- measured - // 2026-07-30, artifact 0% vs volume 99%. - // - // `seconds` is the only timing carried: it is the percentile basis, the - // dispatch order and the runtime insurance threshold. wallSeconds and - // txApply are recorded per range as metrics but nothing sizes from either, - // and wallSeconds alone was 349 KB of a 963 KB artifact. - Assert.Contains("peakAnonBytes", rangeProfileFields) - Assert.Contains("seconds", rangeProfileFields) - Assert.DoesNotContain("wallSeconds", rangeProfileFields) - Assert.DoesNotContain("txApply", rangeProfileFields) - - let record = JObject() - record.["peakAnonBytes"] <- JValue(111L) - record.["seconds"] <- JValue(50.0) - record.["wallSeconds"] <- JValue(999.0) - let entry = projectRangeEntry record - Assert.Equal(111L, entry.["peakAnonBytes"].Value()) - Assert.Equal(50.0, entry.["seconds"].Value()) - Assert.Null(entry.["wallSeconds"]) - - -[] -let ``range profile keeps count as a field so it can be keyed on end alone`` () = - // Measured: 4.2x the ledgers per range moved peak disk -1.6% and wall time - // 1.15x, so cost tracks ledger position rather than range length. Keying on - // end/count would discard the whole profile whenever overlapLedgers or - // ledgersPerJob changed, for a distinction the measurements say is small. - Assert.DoesNotContain("count", rangeProfileFields) - - let record = JObject() - record.["peakWorkingSetBytes"] <- JValue(1L) - record.["count"] <- JValue(420) - // projectRangeEntry itself must not copy count -- writeRangeProfile attaches - // it separately, so a stale profile cannot smuggle it in as a measurement. - let entry = projectRangeEntry record - Assert.Null(entry.["count"]) - - [] let ``range profile does not carry a pvc volume peak`` () = // A PVC's size is not a scheduling dimension, so growing it buys no @@ -110,19 +67,6 @@ let ``pvc mode does not reserve node disk it never uses`` () = Assert.Contains("\"2Gi\", \"4Gi\"", src) -[] -let ``progress record is read from the volume and never from the configmap`` () = - // /logs/progress.json is the monitor's own state: authoritative, unbounded, - // and the only copy carrying measurements. The ConfigMap is the driver's - // view of the run -- status only -- so a record sourced from it would build - // an artifact that looks complete and measures nothing. - let src = - System.IO.File.ReadAllText( - "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") - Assert.Contains("/logs/progress.json", src) - Assert.DoesNotContain("jobMonitorProgressKey", src) - - [] let ``the job monitor image is overridable and defaults to the chart`` () = // The monitor and collector ship as one image pinned in values.yaml. Passing @@ -229,35 +173,6 @@ let private completedMap (pairs: (string * JObject) list) = c -[] -let ``a range carrying no measurement must not enter the profile`` () = - let mirrored = unmeasured (measuredRecord 420 900L) - - // Precondition: the helper really does strip every measurement, leaving - // only bookkeeping. If this ever stops holding, the rest is meaningless. - for f in measurementFields do - Assert.Null(mirrored.[f]) - - Assert.NotNull(mirrored.["count"]) - Assert.NotNull(mirrored.["attempts"]) - - let ranges = buildRangeProfile (completedMap [ "420", mirrored ]) - - Assert.Equal(0, ranges.Count) - - -[] -let ``count alone never satisfies the measurement guard`` () = - // This is the defeated guard in its smallest form: count is the only field, - // and it is bookkeeping, not a measurement. - let onlyCount = JObject() - onlyCount.["count"] <- JValue(420) - - let ranges = buildRangeProfile (completedMap [ "420", onlyCount ]) - - Assert.Equal(0, ranges.Count) - - [] let ``a run whose ranges measured nothing produces no profile artifact`` () = // The headline symptom: a full-looking artifact, right number of ranges, @@ -281,25 +196,6 @@ let ``a run whose ranges measured nothing produces no profile artifact`` () = (ranges.ToString()) -[] -let ``the profile counts only ranges that actually measured something`` () = - // A partly-measured run is the dangerous case: the artifact looks - // populated, so nothing downstream can tell the unmeasured ranges apart - // from the measured one. - let completed = - completedMap - [ "420", measuredRecord 420 900L - "840", unmeasured (measuredRecord 420 950L) - "1260", unmeasured (measuredRecord 420 990L) ] - - let ranges = buildRangeProfile completed - - Assert.Equal(1, ranges.Count) - Assert.NotNull(ranges.["420"]) - Assert.Null(ranges.["840"]) - Assert.Null(ranges.["1260"]) - - [] let ``a measured run still produces a complete profile`` () = // Guard against over-correcting: a good read must still write everything. @@ -325,34 +221,3 @@ let ``a measured run still produces a complete profile`` () = Assert.Equal(420, ranges.["420"].["count"].Value()) Assert.Equal(420, doc.["ledgersPerRange"].Value()) Assert.Equal("pvc", doc.["storageMode"].Value()) - - -[] -let ``a single real measurement is enough to keep a range and it keeps its count`` () = - // The fix must move count after the guard without dropping it -- count is - // what ledgersPerRange is inferred from. - let r = JObject() - r.["attempts"] <- JValue(2) - r.["count"] <- JValue(420) - r.["seconds"] <- JValue(77.0) - - let ranges = buildRangeProfile (completedMap [ "420", r ]) - - Assert.Equal(1, ranges.Count) - Assert.Equal(77.0, ranges.["420"].["seconds"].Value()) - Assert.Equal(420, ranges.["420"].["count"].Value()) - - -[] -let ``a range measured only in unprojected fields is dropped, not kept on count alone`` () = - // The guard reads the PROJECTION, not the record, so narrowing - // rangeProfileFields narrows what counts as measured. A record carrying only - // wallSeconds/txApply now projects to nothing and must be dropped -- keeping - // it would reintroduce exactly the count-only entry the guard exists to stop. - let r = JObject() - r.["attempts"] <- JValue(1) - r.["count"] <- JValue(420) - r.["wallSeconds"] <- JValue(77.0) - r.["txApply"] <- JValue(12.0) - - Assert.Empty(buildRangeProfile (completedMap [ "420", r ])) diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index 7e002eca..6efd7d97 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -13,7 +13,6 @@ open StellarKubeSpecs open StellarNetworkData open StellarNetworkDelays open MissionCatchupHelpers -open Newtonsoft.Json.Linq open Xunit.Abstractions @@ -129,6 +128,7 @@ let ctx : MissionContext = pubnetParallelCatchupMemRequest = "" pubnetParallelCatchupPoolCpu = "" pubnetParallelCatchupPoolMem = "" + pubnetParallelCatchupCreateRbac = false tag = None numPregeneratedTxs = None enableTailLogging = true diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 3dd441d6..a2648300 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -238,6 +238,13 @@ let installProject (context: MissionContext) = if context.jobMonitorImagePcV2 <> "" then setOptions.Add(sprintf "monitor.image=%s" context.jobMonitorImagePcV2) + // Off by default: ssc-eks provides the binding itself, and creating these + // needs the installer to hold the rights being granted. A cluster without + // that binding gets a 403 on the monitor's first ConfigMap read and + // dispatches nothing -- ssc-test is such a cluster. + if context.pubnetParallelCatchupCreateRbac then + setOptions.Add("monitor.createRbac=true") + if context.pubnetParallelCatchupPoolPrefix <> "" then // Routing needs the label KEY and the taint toleration, and neither has // a sensible default for an unpooled run -- both ship as []. Derived diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 3022ee43..bee51c2e 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -128,6 +128,7 @@ type MissionContext = pubnetParallelCatchupMemRequest: string pubnetParallelCatchupPoolCpu: string pubnetParallelCatchupPoolMem: string + pubnetParallelCatchupCreateRbac: bool genesisTestAccountCount: int option asanOptions: string option diff --git a/src/MissionParallelCatchup/HANDOFF.md b/src/MissionParallelCatchup/HANDOFF.md index e9abf2c9..e8db6df1 100644 --- a/src/MissionParallelCatchup/HANDOFF.md +++ b/src/MissionParallelCatchup/HANDOFF.md @@ -26,6 +26,18 @@ Plus the ones that already existed and are unchanged: --pubnet-parallel-catchup-storage-mode pvc|ephemeral ``` +### Not on ssc-eks + +`--pubnet-parallel-catchup-create-rbac` has the chart create the monitor's Role, +RoleBinding, ClusterRole and ClusterRoleBinding. Leave it off for prod: ssc-eks +provides them already, through a `catchup-job-monitor` RoleBinding covering +`system:serviceaccounts:stellar-supercluster`. + +ssc-test has no such binding, so a run there needs the flag. Without it the +monitor 403s reading its own `-stellar-core-config` ConfigMap, never +resolves the ownerReference, and dispatches nothing -- `/status` keeps answering +with `num_remain` unchanged and the run simply never starts. + ### The maps Copy these verbatim. They are the values that were in `values.yaml` and From d3e191e5a407caba53b3e997d3a9c231e5ea4356 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 16:22:03 -0400 Subject: [PATCH 088/117] Remove HANDOFF.md The Jenkinsfile options it describes belong with whoever owns the Jenkinsfile, not checked into this repo where they would drift from it silently. --- src/MissionParallelCatchup/HANDOFF.md | 142 -------------------------- 1 file changed, 142 deletions(-) delete mode 100644 src/MissionParallelCatchup/HANDOFF.md diff --git a/src/MissionParallelCatchup/HANDOFF.md b/src/MissionParallelCatchup/HANDOFF.md deleted file mode 100644 index e8db6df1..00000000 --- a/src/MissionParallelCatchup/HANDOFF.md +++ /dev/null @@ -1,142 +0,0 @@ -# Jenkinsfile options for ParallelCatchupV2 - -What changed: the chart no longer knows that spot and on-demand exist. -`values-ondemand.yaml` is deleted, `monitor.capacityType` is gone, and the -monitor no longer reads `karpenter.sh/capacity-type`. Everything that used to be -derived is now passed in, so **Jenkins is the only place that knows which -capacity a run targets**. - -Nothing has a safe default any more. A run that omits these gets *no* capacity -constraint and the chart's built-in (spot) claim maps. - -## What every pooled run must pass - -Three things, and both capacities pass all three: - -| | spot | on-demand | -|---|---|---| -| capacity label | `--require-node-labels-pc-v2 catchup-capacity:spot` | `--require-node-labels-pc-v2 catchup-capacity:od` | -| cpu claims | `--pubnet-parallel-catchup-pool-cpu ""` | `--pubnet-parallel-catchup-pool-cpu ""` | -| memory claims | `--pubnet-parallel-catchup-pool-mem ""` | `--pubnet-parallel-catchup-pool-mem ""` | - -Plus the ones that already existed and are unchanged: - -``` ---pubnet-parallel-catchup-pool-prefix catchup ---pubnet-parallel-catchup-storage-mode pvc|ephemeral -``` - -### Not on ssc-eks - -`--pubnet-parallel-catchup-create-rbac` has the chart create the monitor's Role, -RoleBinding, ClusterRole and ClusterRoleBinding. Leave it off for prod: ssc-eks -provides them already, through a `catchup-job-monitor` RoleBinding covering -`system:serviceaccounts:stellar-supercluster`. - -ssc-test has no such binding, so a run there needs the flag. Without it the -monitor 403s reading its own `-stellar-core-config` ConfigMap, never -resolves the ownerReference, and dispatches nothing -- `/status` keeps answering -with `num_remain` unchanged and the run simply never starts. - -### The maps - -Copy these verbatim. They are the values that were in `values.yaml` and -`values-ondemand.yaml` before the change. - -**spot** — pools were doubled on 2026-08-04, so a claim is half a node and two -pods share it: - -``` -poolCpu: subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80 -poolMem: subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:9216Mi -``` - -**on-demand** — pools kept their original sizes, so a claim is the whole node -and one pod gets it: - -``` -poolCpu: subdwarf:0.45,dwarf:0.45,subgiant:1.40,giant:1.40,supergiant:1.40,hypergiant:3.35,supernova:7.20,protostar:1.40,nebula:3.35 -poolMem: subdwarf:576Mi,dwarf:576Mi,subgiant:2048Mi,giant:5248Mi,supergiant:12416Mi,hypergiant:27328Mi,supernova:57216Mi,protostar:27328Mi,nebula:12416Mi -``` - -Do **not** cross them. On-demand claims are sized against *allocatable*; the -spot claims are the on-demand node's *nameplate*, and nameplate is not -allocatable. Shipping spot claims to an on-demand pool makes every tier -unschedulable and Karpenter provisions nothing — pods sit Pending, which reads -as slow provisioning rather than as a sizing bug. That is what happened on -2026-08-07 and it cost a day to spot. - -## The pairing that used to be automatic - -Storage mode and capacity are now fully independent. Nothing derives one from -the other and nothing will object if they disagree. - -The pairing every production run wants: - -``` -pvc + catchup-capacity:spot -ephemeral + catchup-capacity:od -``` - -Why it matters: `pvc` keeps `/data` across pods, so an evicted range resumes at -LCL+1 — that is what makes spot survivable. `ephemeral` puts `/data` on the node -and has no resume, so a spot reclaim costs the whole range from scratch. - -They were split on purpose, so a test run can pair `ephemeral` with `spot` -deliberately to measure what a reclaim actually costs. Production should not. - -## Example - -``` -dotnet run --project src/App/App.fsproj -- mission HistoryPubnetParallelCatchupV2 \ - --image \ - --namespace stellar-supercluster \ - --pubnet-parallel-catchup-pool-prefix catchup \ - --pubnet-parallel-catchup-storage-mode pvc \ - --require-node-labels-pc-v2 catchup-capacity:spot \ - --pubnet-parallel-catchup-pool-cpu "subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80" \ - --pubnet-parallel-catchup-pool-mem "subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:9216Mi" \ - --pubnet-parallel-catchup-profile \ - --pubnet-parallel-catchup-num-workers 1024 \ - --pubnet-parallel-catchup-ledgers-per-job 16000 \ - --destination ./logs -``` - -## Failure modes, and which are loud - -**Loud** — a missing tier in either map fails `POST /start` with a 400 and a -reason, so the driver stops immediately: - -``` -POOL_CPU has no entry for nebula; a pooled range routed there would keep -the flat request and share its node -``` - -The check covers every ladder tier plus `protostar` (ranges newer than the -profile) and `nebula` (runs with no profile at all). Both are routed to, so both -need entries. `subdwarf` is dormant — it can never be selected — but it still -needs a map entry. - -**Silent** — these produce a run that finishes and costs more than it should: - -- *capacity label omitted.* Both capacities of a tier carry the same - `purpose=catchup-` label, so nothing separates them. A spot run without - `catchup-capacity:spot` can land on on-demand nodes and bill on-demand rates. -- *wrong map for the capacity.* Covered above. Not silent on on-demand (nothing - schedules) but silent the other way: on-demand claims on spot pools pack one - pod where two fit, halving throughput per node. -- *tier name typo in a map.* Reads as a missing tier, so `/start` catches it — - the one typo class that is loud. - -## Ordering note - -If Jenkins passes several `--require-node-labels-pc-v2` entries, order does not -matter — they all become literal requirements. The mission reserves -`worker.requireNodeLabels[0]` for its own pool-routing label and starts the -caller's entries at index 1. Before this change they both wrote index 0 and the -routing label was overwritten, which put ranges on arbitrary tiers. - -## Image - -The chart pins `stellajuna/ssc-jm:2026-08-10a`, a personal Docker Hub repo. -**This is a dev pin and must be replaced before ssc-eks.** From 7e12140cf921d26ee0bea2208ad77eb1670bda46 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 16:23:39 -0400 Subject: [PATCH 089/117] Cut the field-projection test Unlike the eight cut before it, this one was not redundant: it was the sole killer of two mutants, and both now survive. Dropping peakEphemeralBytes or peakWorkingSetBytes from rangeProfileFields no longer fails anything. What that costs, so it is not rediscovered the hard way: the artifact would still be written, still carry the right number of ranges, and still look complete, while the next run sizes every range from defaults on the missing axis. That is the RACE #8 failure shape -- nothing downstream can tell a measurement-free profile from a good one -- narrowed to one field. drop-seconds and volume-peak-profiled are still covered, by "a measured run still produces a complete profile" and "range profile does not carry a pvc volume peak" respectively. --- src/FSLibrary.Tests/CatchupV2Tests.fs | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/FSLibrary.Tests/CatchupV2Tests.fs b/src/FSLibrary.Tests/CatchupV2Tests.fs index 8e1306cb..e4de3d78 100644 --- a/src/FSLibrary.Tests/CatchupV2Tests.fs +++ b/src/FSLibrary.Tests/CatchupV2Tests.fs @@ -26,28 +26,6 @@ open Newtonsoft.Json.Linq open MissionHistoryPubnetParallelCatchupV2 -[] -let ``range profile keeps only the measurements that exist`` () = - // The consumer falls back to its configured default when a field is - // absent, so a missing measurement must stay missing rather than become a - // null. peakEphemeralBytes is recorded only for ephemeral-mode runs that - // finished, so most records will not carry it. - let record = JObject() - record.["peakWorkingSetBytes"] <- JValue(1234L) - record.["seconds"] <- JValue(42) - - let entry = projectRangeEntry record - - Assert.Equal(2, entry.Count) - Assert.Equal(1234L, entry.["peakWorkingSetBytes"].Value()) - Assert.Null(entry.["peakEphemeralBytes"]) - - record.["peakEphemeralBytes"] <- JValue(9999L) - let withEph = projectRangeEntry record - Assert.Equal(3, withEph.Count) - Assert.Equal(9999L, withEph.["peakEphemeralBytes"].Value()) - - [] let ``range profile does not carry a pvc volume peak`` () = // A PVC's size is not a scheduling dimension, so growing it buys no From f496adceb77f8efdc1c6ea311e218fb6bca742c0 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 16:30:45 -0400 Subject: [PATCH 090/117] Cut the volume-peak test It asserted "peakVolumeBytes" is absent from rangeProfileFields. Nothing in the repo produces that field: no collector writes it, no sizer reads it, and the name appears nowhere outside the test. It guarded against a change nobody was making, and anyone deliberately profiling volume size would have deleted the assertion in the same edit. It scored as irreplaceable in the mutation run only because the mutant that killed it -- adding peakVolumeBytes to the field list -- was one I wrote to give it something to kill. A test that dies only to a mutant designed around it measures the mutant, not the test. --- src/FSLibrary.Tests/CatchupV2Tests.fs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/FSLibrary.Tests/CatchupV2Tests.fs b/src/FSLibrary.Tests/CatchupV2Tests.fs index e4de3d78..2b8c1681 100644 --- a/src/FSLibrary.Tests/CatchupV2Tests.fs +++ b/src/FSLibrary.Tests/CatchupV2Tests.fs @@ -26,13 +26,6 @@ open Newtonsoft.Json.Linq open MissionHistoryPubnetParallelCatchupV2 -[] -let ``range profile does not carry a pvc volume peak`` () = - // A PVC's size is not a scheduling dimension, so growing it buys no - // packing and it is deliberately not profiled. - Assert.DoesNotContain("peakVolumeBytes", rangeProfileFields) - - [] let ``pvc mode does not reserve node disk it never uses`` () = // /data is on the volume in pvc mode; the node disk only holds logs and tmp. From daacac9974a1798d36534bb9b92750c0210b0b52 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 16:39:49 -0400 Subject: [PATCH 091/117] Stop sizing ephemeral storage at all in pvc mode The mission sent 2Gi/4Gi for pvc runs, a figure invented to be small rather than measured. In pvc mode /data is on the volume and the node disk holds only logs and tmp, so any request there reserves disk nothing uses and makes disk rather than cpu the binding dimension for packing. The monitor was already written for this. _resources reads an empty REQ_EPHEMERAL as "leave both axes off the pod" and drops any profile-derived disk override with it, and eph_for_attempt returns None when no limit is configured, so escalation is a no-op rather than a wrong number. Those branches were unreachable while the mission always sent a value. Verified: a pvc pod now comes out with requests {cpu, memory} and no limits at all, an ephemeral one is unchanged at 35Gi/40Gi. Consequence worth naming: a pvc worker now has no ephemeral-storage limit, so it cannot be evicted for exceeding its own. Node disk pressure can still evict it. That is the accepted trade -- the limit only ever guarded a dimension this mode does not write to. The test that covered this goes too. It asserted on the text of the mission source, matching "2Gi", "4Gi" -- a string that no longer exists. --- src/FSLibrary.Tests/CatchupV2Tests.fs | 12 ----- .../MissionHistoryPubnetParallelCatchupV2.fs | 44 ++++++++++--------- 2 files changed, 23 insertions(+), 33 deletions(-) diff --git a/src/FSLibrary.Tests/CatchupV2Tests.fs b/src/FSLibrary.Tests/CatchupV2Tests.fs index 2b8c1681..845d24e5 100644 --- a/src/FSLibrary.Tests/CatchupV2Tests.fs +++ b/src/FSLibrary.Tests/CatchupV2Tests.fs @@ -26,18 +26,6 @@ open Newtonsoft.Json.Linq open MissionHistoryPubnetParallelCatchupV2 -[] -let ``pvc mode does not reserve node disk it never uses`` () = - // /data is on the volume in pvc mode; the node disk only holds logs and tmp. - // Asking for the ephemeral-mode figure makes disk rather than cpu the - // binding dimension and cuts pods per node. - let src = - System.IO.File.ReadAllText( - "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") - Assert.Contains("pubnetParallelCatchupStorageMode = \"pvc\"", src) - Assert.Contains("\"2Gi\", \"4Gi\"", src) - - [] let ``the job monitor image is overridable and defaults to the chart`` () = // The monitor and collector ship as one image pinned in values.yaml. Passing diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index a2648300..38f3dec2 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -285,25 +285,6 @@ let installProject (context: MissionContext) = // in parallel_catchup_helm/values.yaml, overridable per run with // --pubnet-parallel-catchup-cpu-request. let resourceRequirements = ParallelCatchupCoreResourceRequirements - // StellarKubeSpecs sizes ephemeral-storage for ephemeral mode, where /data is - // an emptyDir on the node. In pvc mode /data is on the volume and the node - // disk only holds logs and tmp, so asking for the full amount reserves disk - // nothing uses -- and makes disk, not cpu, the binding dimension for packing. - let storageReqGibi, storageLimGibi = - if context.pubnetParallelCatchupStorageMode = "pvc" then - "2Gi", "4Gi" - else - resourceRequirements.Requests.["ephemeral-storage"].ToString(), - resourceRequirements.Limits.["ephemeral-storage"].ToString() - - LogInfo - "Worker storage from StellarKubeCfg:\n\ - Storage request: %s\n\ - Storage limit: %s\n\ - (cpu and memory come from the chart; workers run with no cpu or memory limit)" - storageReqGibi - storageLimGibi - // Both pushed only when the run explicitly asks. Otherwise the chart default // stands, so the chart is the single place V2's worker sizing is written down. if not (String.IsNullOrWhiteSpace context.pubnetParallelCatchupCpuRequest) then @@ -316,8 +297,29 @@ let installProject (context: MissionContext) = sprintf "worker.resources.requests.memory=%s" context.pubnetParallelCatchupMemRequest ) - setOptions.Add(sprintf "worker.resources.requests.ephemeral_storage=%s" storageReqGibi) - setOptions.Add(sprintf "worker.resources.limits.ephemeral_storage=%s" storageLimGibi) + // Ephemeral-storage is an ephemeral-mode concept: /data is an emptyDir on the + // node there, and StellarKubeSpecs sizes it. In pvc mode /data is on the + // volume and the node disk holds only logs and tmp, so the run reserves + // nothing -- _resources already reads an empty REQ_EPHEMERAL as "leave both + // axes off the pod", and a request sized for the other mode would make disk + // rather than cpu the binding dimension for packing. + if context.pubnetParallelCatchupStorageMode <> "pvc" then + LogInfo + "Worker ephemeral storage from StellarKubeCfg: request %s, limit %s" + (resourceRequirements.Requests.["ephemeral-storage"].ToString()) + (resourceRequirements.Limits.["ephemeral-storage"].ToString()) + + setOptions.Add( + sprintf + "worker.resources.requests.ephemeral_storage=%s" + (resourceRequirements.Requests.["ephemeral-storage"].ToString()) + ) + + setOptions.Add( + sprintf + "worker.resources.limits.ephemeral_storage=%s" + (resourceRequirements.Limits.["ephemeral-storage"].ToString()) + ) // Construct command for fetching history files from S3 for core node // `index` and set the corresponding Helm option From f59bd495e0ba7169255277c4c17b6c32d493879d Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 16:48:52 -0400 Subject: [PATCH 092/117] Drop the two size-only assertions from the profile test They asserted wallSeconds and txApply do not reach the artifact. Both are still recorded per range -- verified against a real ssc-test run, whose progress.json carried txApply=0.00025 and wallSeconds=42.0 while the artifact correctly held only peakAnonBytes, peakWorkingSetBytes, seconds and count. But leaking them would break nothing. load_profile_doc copies each record wholesale and sizing reads named keys with .get(), so an unknown field is ignored. The assertions guarded artifact weight, not behaviour, and weight is already the stated reason those fields are excluded. The three mutants this test uniquely kills -- storageMode and ledgersPerRange missing from the doc, and the count-before-guard defect -- still die. Also fixes unmeasured's doc comment, which described the record "as it survives the ConfigMap mirror". That mirror was deleted when status moved to /status; the helper now says what it actually builds. --- src/FSLibrary.Tests/CatchupV2Tests.fs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/FSLibrary.Tests/CatchupV2Tests.fs b/src/FSLibrary.Tests/CatchupV2Tests.fs index 845d24e5..4ffdbcb7 100644 --- a/src/FSLibrary.Tests/CatchupV2Tests.fs +++ b/src/FSLibrary.Tests/CatchupV2Tests.fs @@ -114,7 +114,8 @@ let private measuredRecord (count: int) (anon: int64) = r.["peakWorkingSetBytes"] <- JValue(anon + 1000L) r -/// The same record as it survives the ConfigMap mirror. +/// The same record with every measurement stripped: what a range leaves behind +/// when the collector never wrote peaks for it, or the read was degraded. let private unmeasured (record: JObject) = let r = record.DeepClone() :?> JObject @@ -171,10 +172,6 @@ let ``a measured run still produces a complete profile`` () = Assert.Equal(900L, ranges.["420"].["peakAnonBytes"].Value()) Assert.Equal(950L, ranges.["840"].["peakAnonBytes"].Value()) Assert.Equal(120.0, ranges.["420"].["seconds"].Value()) - // Measured but deliberately not projected: recorded as metrics, never - // sized or ordered from, and pure weight in a 1 MiB-capped artifact. - Assert.Null(ranges.["420"].["wallSeconds"]) - Assert.Null(ranges.["420"].["txApply"]) // count is still carried, and the slicing is inferred from it rather // than from the caller's default. Assert.Equal(420, ranges.["420"].["count"].Value()) From 15f9c939d33803ffa177a8f579b60c9e7e453fe7 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 17:05:51 -0400 Subject: [PATCH 093/117] Name the profile guard test after what it actually prevents It was "a run whose ranges measured nothing produces no profile artifact", which describes the least interesting case it covers. The empty-artifact scenario is caught by a human looking at the profile they pass in. What is not caught by looking is a MIX. profile_for resolves a range to the nearest measured end ABOVE it, so one measurement-free entry captures every range beneath it and hides the real measurement further up. Demonstrated against the live sizing code: with a junk entry at 1200 alongside a real one at 1600, range 1100 resolves to {'count': 1600} and routes to protostar; with the guard doing its job it resolves to the 5 GiB record and routes to supergiant. It is silent the whole way. The junk entry is a truthy record, so _profile_overrides proceeds, finds no peakAnonBytes, _tier_for_bytes returns None, and pool_for treats the range as past the profile. Nothing logs a difference, and the run just costs more. Test body unchanged -- it already asserted the right thing. Only the name and the comment were wrong about why. --- src/FSLibrary.Tests/CatchupV2Tests.fs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/FSLibrary.Tests/CatchupV2Tests.fs b/src/FSLibrary.Tests/CatchupV2Tests.fs index 4ffdbcb7..95e3618c 100644 --- a/src/FSLibrary.Tests/CatchupV2Tests.fs +++ b/src/FSLibrary.Tests/CatchupV2Tests.fs @@ -134,11 +134,20 @@ let private completedMap (pairs: (string * JObject) list) = [] -let ``a run whose ranges measured nothing produces no profile artifact`` () = - // The headline symptom: a full-looking artifact, right number of ranges, - // zero measurements -- what a run whose collector never wrote peaks leaves - // behind. Writing nothing is correct: the next run then falls back to its - // configured defaults instead of sizing from empty data. +let ``a measurement-free record cannot become a profile entry`` () = + // A range the collector never sampled carries bookkeeping and nothing else. + // Letting it into the artifact does not merely add a useless entry -- it + // SHADOWS the real ones. profile_for resolves a range to the nearest + // measured end ABOVE it, so a junk entry at 1200 captures every range below + // it and hides the good measurement at 1600. The junk resolves as a truthy + // record with no peakAnonBytes, so _tier_for_bytes returns None and the + // range routes to protostar instead of the supergiant its neighbour implies. + // Verified: with the entry present, range 1100 sizes to protostar; without + // it, supergiant. Nothing logs the difference. + // + // When NO range measured, the whole document is refused -- an artifact with + // the right range count and zero measurements is indistinguishable from a + // good one. let completed = completedMap [ "420", unmeasured (measuredRecord 420 900L) From 8ef98140786d580257198b7611e24acb95c7af10 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 17:08:51 -0400 Subject: [PATCH 094/117] Cover the mixed case: some ranges measured, some not The two existing tests feed all-measured or all-unmeasured records, and a guard that decides per RUN rather than per RECORD passes both. Demonstrated with a sticky guard -- one that latches on the first measurement and lets every later record through: all-unmeasured never latches so the document is still refused, all-measured latches immediately so everything is still written, and both tests go green while every junk record following a real one now reaches the artifact. That is the case that costs something, and it is also the realistic one: a run misses a few ranges, not zero and not all. profile_for resolves a range to the nearest measured end ABOVE it, so one junk entry captures every range beneath it and hides the real measurement further up -- range 1100 routes to protostar with a junk entry at 1200 present, and to supergiant without it. The new test is the sole killer of the sticky-guard mutant; the other two do not notice it. --- src/FSLibrary.Tests/CatchupV2Tests.fs | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/FSLibrary.Tests/CatchupV2Tests.fs b/src/FSLibrary.Tests/CatchupV2Tests.fs index 95e3618c..df9d5070 100644 --- a/src/FSLibrary.Tests/CatchupV2Tests.fs +++ b/src/FSLibrary.Tests/CatchupV2Tests.fs @@ -186,3 +186,32 @@ let ``a measured run still produces a complete profile`` () = Assert.Equal(420, ranges.["420"].["count"].Value()) Assert.Equal(420, doc.["ledgersPerRange"].Value()) Assert.Equal("pvc", doc.["storageMode"].Value()) + + +[] +let ``a measured range does not drag its unmeasured neighbours in`` () = + // The realistic shape: a run where the collector missed a few ranges, not + // zero and not all. Neither all-measured nor all-unmeasured catches a guard + // that decides per RUN rather than per RECORD -- one that latches on the + // first real measurement passes both, and then every later junk record + // rides in behind it. + // + // Which is the case that costs something. profile_for resolves a range to + // the nearest measured end ABOVE it, so a junk entry at 1200 captures every + // range beneath it and hides the real 1600. Measured against the sizing + // code: range 1100 routes to protostar with the junk entry present and to + // supergiant without it. + let completed = + completedMap + [ "1200", unmeasured (measuredRecord 400 900L) + "1600", measuredRecord 400 950L + "2000", unmeasured (measuredRecord 400 990L) ] + + match rangeProfileDocument "pvc" 20000 completed with + | None -> failwith "refused a profile that carries one real measurement" + | Some doc -> + let ranges = doc.["ranges"] :?> JObject + Assert.Equal(1, ranges.Count) + Assert.NotNull(ranges.["1600"]) + Assert.Null(ranges.["1200"]) + Assert.Null(ranges.["2000"]) From 54ac4125b45239651e2dae0f8159469b8a174219 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 17:11:46 -0400 Subject: [PATCH 095/117] Tighten the comments in CatchupV2Tests Same facts, a third of the lines. The shadowing explanation appeared in two test comments and the module header; it now lives once, in the header, since it is the reason the whole group exists rather than a property of any one test. --- src/FSLibrary.Tests/CatchupV2Tests.fs | 93 +++++++++------------------ 1 file changed, 29 insertions(+), 64 deletions(-) diff --git a/src/FSLibrary.Tests/CatchupV2Tests.fs b/src/FSLibrary.Tests/CatchupV2Tests.fs index df9d5070..18edc3f6 100644 --- a/src/FSLibrary.Tests/CatchupV2Tests.fs +++ b/src/FSLibrary.Tests/CatchupV2Tests.fs @@ -28,11 +28,8 @@ open MissionHistoryPubnetParallelCatchupV2 [] let ``the job monitor image is overridable and defaults to the chart`` () = - // The monitor and collector ship as one image pinned in values.yaml. Passing - // it per run is what lets a build of them be tested without editing the - // chart -- but an empty flag must leave the chart's pin alone rather than - // setting monitor.image to nothing, which resolves to ":latest" or fails the - // pull outright. + // An empty flag must leave the chart's pin alone. monitor.image= resolves to + // ":latest" or fails the pull. let src = System.IO.File.ReadAllText( "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") @@ -47,12 +44,9 @@ let ``the job monitor image is overridable and defaults to the chart`` () = [] let ``a pooled run does not let caller labels overwrite the routing label`` () = - // A pooled run claims worker.requireNodeLabels[0] for the label it routes - // on, and requireNodeLabelsPcV2 used to index its own entries from 0 as - // well. Both fire on a pooled run carrying a capacity label, and the second - // --set wins: the routing label is replaced, so the pod matches on capacity - // alone and lands on any tier at all -- a range sized for supergiant on a - // dwarf node, which is an OOM per range rather than a slow run. + // A pooled run claims index 0; the caller's entries must start at 1. Both at + // 0 and the second --set wins, so the pod matches on capacity alone and lands + // on any tier -- a supergiant range on a dwarf node, an OOM per range. let src = System.IO.File.ReadAllText( "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") @@ -63,46 +57,38 @@ let ``a pooled run does not let caller labels overwrite the routing label`` () = [] let ``the pool maps ride their own --set with their commas escaped`` () = - // Every other option is folded into ONE comma-joined --set. The pool maps - // are themselves comma-separated, so folding them in would split each tier - // into a separate helm assignment and the map would arrive holding one - // tier. This is why the overlay used to be a second --values file. + // Every other option is folded into ONE comma-joined --set; these maps are + // themselves comma-separated, so folding them in delivers a map of one tier. let src = System.IO.File.ReadAllText( "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") Assert.Contains("v.Replace(\",\", \"\\\\,\")", src) Assert.Contains("poolMapArgs", src) - // Empty must not reach helm at all: monitor.poolCpu= would blank the chart - // default and every tier would fall back to the flat request. + // Empty must not reach helm: monitor.poolCpu= blanks the chart default. Assert.Contains("List.filter (fun (_, v) -> not (String.IsNullOrWhiteSpace v))", src) -// RACE #8 -- a measurement-free profile artifact that is indistinguishable from -// a good one. +// The profile artifact. A completed record always carries bookkeeping (attempts, +// count) and may carry no measurement at all, when the collector never sampled +// that range. Such a record must not become an entry. // -// A completed record can carry bookkeeping (attempts, count) and no measurement -// at all: the collector never wrote peaks for that range, or the read that would -// have supplied them was degraded. Such a record must not become a profile entry. +// RACE #8: count was attached BEFORE the `entry.Count > 0` guard, so every entry +// was non-empty and none were skipped. Seen twice in the field -- 0% +// peakAnonBytes in the artifact against 99% in progress.json. // -// The `entry.Count > 0` guard exists to skip measurement-free entries, but count -// used to be attached to the entry BEFORE the guard ran, so every entry had at -// least one field and every entry passed. The result is an artifact with the -// right number of ranges and zero measurements -- observed twice in the field, -// reporting 0% peakAnonBytes while the monitor's own progress.json carried 99%. -// The next run then sizes from a profile that silently has no data. -// -// These tests assert on the values the production projection actually returns, -// never on the text of the source file. +// What that costs: profile_for resolves a range to the nearest measured end +// ABOVE it, so one junk entry at 1200 captures every range beneath it and hides +// the real 1600. Range 1100 then routes to protostar instead of supergiant, and +// nothing logs it. -/// Every measurement a completed record can carry. A superset of -/// rangeProfileFields: wallSeconds and txApply are recorded but never projected. +/// Superset of rangeProfileFields: wallSeconds and txApply are recorded, never +/// projected. let private measurementFields = [ "peakAnonBytes"; "peakWorkingSetBytes"; "peakEphemeralBytes" "txApply"; "seconds"; "wallSeconds" ] -/// A completed record the way /logs/progress.json carries it: bookkeeping plus -/// real measurements. +/// A record as /logs/progress.json carries it. let private measuredRecord (count: int) (anon: int64) = let r = JObject() r.["attempts"] <- JValue(1) @@ -114,8 +100,7 @@ let private measuredRecord (count: int) (anon: int64) = r.["peakWorkingSetBytes"] <- JValue(anon + 1000L) r -/// The same record with every measurement stripped: what a range leaves behind -/// when the collector never wrote peaks for it, or the read was degraded. +/// The same record with every measurement stripped. let private unmeasured (record: JObject) = let r = record.DeepClone() :?> JObject @@ -135,19 +120,8 @@ let private completedMap (pairs: (string * JObject) list) = [] let ``a measurement-free record cannot become a profile entry`` () = - // A range the collector never sampled carries bookkeeping and nothing else. - // Letting it into the artifact does not merely add a useless entry -- it - // SHADOWS the real ones. profile_for resolves a range to the nearest - // measured end ABOVE it, so a junk entry at 1200 captures every range below - // it and hides the good measurement at 1600. The junk resolves as a truthy - // record with no peakAnonBytes, so _tier_for_bytes returns None and the - // range routes to protostar instead of the supergiant its neighbour implies. - // Verified: with the entry present, range 1100 sizes to protostar; without - // it, supergiant. Nothing logs the difference. - // - // When NO range measured, the whole document is refused -- an artifact with - // the right range count and zero measurements is indistinguishable from a - // good one. + // Nothing measured -- the whole document is refused, rather than written + // with the right range count and no data. let completed = completedMap [ "420", unmeasured (measuredRecord 420 900L) @@ -167,7 +141,7 @@ let ``a measurement-free record cannot become a profile entry`` () = [] let ``a measured run still produces a complete profile`` () = - // Guard against over-correcting: a good read must still write everything. + // The over-correction guard: a good read must still write everything. let completed = completedMap [ "420", measuredRecord 420 900L @@ -181,8 +155,7 @@ let ``a measured run still produces a complete profile`` () = Assert.Equal(900L, ranges.["420"].["peakAnonBytes"].Value()) Assert.Equal(950L, ranges.["840"].["peakAnonBytes"].Value()) Assert.Equal(120.0, ranges.["420"].["seconds"].Value()) - // count is still carried, and the slicing is inferred from it rather - // than from the caller's default. + // Slicing is inferred from the records, not from the caller's 20000. Assert.Equal(420, ranges.["420"].["count"].Value()) Assert.Equal(420, doc.["ledgersPerRange"].Value()) Assert.Equal("pvc", doc.["storageMode"].Value()) @@ -190,17 +163,9 @@ let ``a measured run still produces a complete profile`` () = [] let ``a measured range does not drag its unmeasured neighbours in`` () = - // The realistic shape: a run where the collector missed a few ranges, not - // zero and not all. Neither all-measured nor all-unmeasured catches a guard - // that decides per RUN rather than per RECORD -- one that latches on the - // first real measurement passes both, and then every later junk record - // rides in behind it. - // - // Which is the case that costs something. profile_for resolves a range to - // the nearest measured end ABOVE it, so a junk entry at 1200 captures every - // range beneath it and hides the real 1600. Measured against the sizing - // code: range 1100 routes to protostar with the junk entry present and to - // supergiant without it. + // The realistic shape, and the only one that catches a guard deciding per RUN + // rather than per RECORD: one that latches on the first measurement passes + // both tests above while every later junk record rides in behind it. let completed = completedMap [ "1200", unmeasured (measuredRecord 400 900L) From 7a633af9d2cbc3b076fcc73e0b1ea676cb149bee Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 17:19:29 -0400 Subject: [PATCH 096/117] Fold the V2 tests back into Tests.fs Six tests do not need a file of their own. They go under a section banner, with Tests.fs regaining the two opens it lost when they moved out. Verified as a move: same 22 tests, and the count-before-guard defect still fails the same two. --- src/FSLibrary.Tests/CatchupV2Tests.fs | 182 --------------------- src/FSLibrary.Tests/FSLibrary.Tests.fsproj | 1 - src/FSLibrary.Tests/Tests.fs | 160 ++++++++++++++++++ 3 files changed, 160 insertions(+), 183 deletions(-) delete mode 100644 src/FSLibrary.Tests/CatchupV2Tests.fs diff --git a/src/FSLibrary.Tests/CatchupV2Tests.fs b/src/FSLibrary.Tests/CatchupV2Tests.fs deleted file mode 100644 index 18edc3f6..00000000 --- a/src/FSLibrary.Tests/CatchupV2Tests.fs +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright 2024 Stellar Development Foundation and contributors. Licensed -// under the Apache License, Version 2.0. See the COPYING file at the root -// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 - -// Everything covering MissionHistoryPubnetParallelCatchupV2: the profile -// artifact it writes, and the helm invocation it builds. -// -// The profile tests exist for one defect. A completed record can carry -// bookkeeping (attempts, count) and no measurement at all -- the collector never -// wrote peaks for that range, or the read that would have supplied them was -// degraded. The `entry.Count > 0` guard skips such records, but count used to be -// attached BEFORE the guard ran, so every entry had at least one field and every -// entry passed. The result is an artifact with the right number of ranges and -// zero measurements, which is worse than no artifact: nothing downstream can -// tell it from a good one, and the next run sizes everything from defaults while -// looking correctly configured. Observed twice in the field, reporting 0% -// peakAnonBytes while the monitor's own progress.json carried 99%. -// -// These assert on the values the production projection returns, never on the -// text of the source file -- except where the subject IS the helm invocation, -// which has no return value to inspect. -module CatchupV2Tests - -open Xunit -open Newtonsoft.Json.Linq -open MissionHistoryPubnetParallelCatchupV2 - - -[] -let ``the job monitor image is overridable and defaults to the chart`` () = - // An empty flag must leave the chart's pin alone. monitor.image= resolves to - // ":latest" or fails the pull. - let src = - System.IO.File.ReadAllText( - "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") - - Assert.Contains("if context.jobMonitorImagePcV2 <> \"\" then", src) - Assert.Contains("monitor.image=%s", src) - - let guard = src.IndexOf("if context.jobMonitorImagePcV2 <> \"\" then") - let use_ = src.IndexOf("monitor.image=%s") - Assert.True(guard < use_, "monitor.image must only be set inside the non-empty guard") - - -[] -let ``a pooled run does not let caller labels overwrite the routing label`` () = - // A pooled run claims index 0; the caller's entries must start at 1. Both at - // 0 and the second --set wins, so the pod matches on capacity alone and lands - // on any tier -- a supergiant range on a dwarf node, an OOM per range. - let src = - System.IO.File.ReadAllText( - "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") - - Assert.Contains("worker.requireNodeLabels[0]=purpose:%s", src) - Assert.Contains("if context.pubnetParallelCatchupPoolPrefix <> \"\" then i + 1 else i", src) - - -[] -let ``the pool maps ride their own --set with their commas escaped`` () = - // Every other option is folded into ONE comma-joined --set; these maps are - // themselves comma-separated, so folding them in delivers a map of one tier. - let src = - System.IO.File.ReadAllText( - "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") - - Assert.Contains("v.Replace(\",\", \"\\\\,\")", src) - Assert.Contains("poolMapArgs", src) - // Empty must not reach helm: monitor.poolCpu= blanks the chart default. - Assert.Contains("List.filter (fun (_, v) -> not (String.IsNullOrWhiteSpace v))", src) - -// The profile artifact. A completed record always carries bookkeeping (attempts, -// count) and may carry no measurement at all, when the collector never sampled -// that range. Such a record must not become an entry. -// -// RACE #8: count was attached BEFORE the `entry.Count > 0` guard, so every entry -// was non-empty and none were skipped. Seen twice in the field -- 0% -// peakAnonBytes in the artifact against 99% in progress.json. -// -// What that costs: profile_for resolves a range to the nearest measured end -// ABOVE it, so one junk entry at 1200 captures every range beneath it and hides -// the real 1600. Range 1100 then routes to protostar instead of supergiant, and -// nothing logs it. - - -/// Superset of rangeProfileFields: wallSeconds and txApply are recorded, never -/// projected. -let private measurementFields = - [ "peakAnonBytes"; "peakWorkingSetBytes"; "peakEphemeralBytes" - "txApply"; "seconds"; "wallSeconds" ] - -/// A record as /logs/progress.json carries it. -let private measuredRecord (count: int) (anon: int64) = - let r = JObject() - r.["attempts"] <- JValue(1) - r.["count"] <- JValue(count) - r.["seconds"] <- JValue(120.0) - r.["wallSeconds"] <- JValue(130.0) - r.["txApply"] <- JValue(60.0) - r.["peakAnonBytes"] <- JValue(anon) - r.["peakWorkingSetBytes"] <- JValue(anon + 1000L) - r - -/// The same record with every measurement stripped. -let private unmeasured (record: JObject) = - let r = record.DeepClone() :?> JObject - - for f in measurementFields do - r.Remove(f) |> ignore - - r - -let private completedMap (pairs: (string * JObject) list) = - let c = JObject() - - for (k, v) in pairs do - c.[k] <- v - - c - - -[] -let ``a measurement-free record cannot become a profile entry`` () = - // Nothing measured -- the whole document is refused, rather than written - // with the right range count and no data. - let completed = - completedMap - [ "420", unmeasured (measuredRecord 420 900L) - "840", unmeasured (measuredRecord 420 950L) - "1260", unmeasured (measuredRecord 420 990L) ] - - match rangeProfileDocument "pvc" 20000 completed with - | None -> () - | Some doc -> - let ranges = doc.["ranges"] :?> JObject - - failwithf - "wrote a profile artifact with %d ranges and zero measurements: %s" - ranges.Count - (ranges.ToString()) - - -[] -let ``a measured run still produces a complete profile`` () = - // The over-correction guard: a good read must still write everything. - let completed = - completedMap - [ "420", measuredRecord 420 900L - "840", measuredRecord 420 950L ] - - match rangeProfileDocument "pvc" 20000 completed with - | None -> failwith "refused to write a profile that carries real measurements" - | Some doc -> - let ranges = doc.["ranges"] :?> JObject - Assert.Equal(2, ranges.Count) - Assert.Equal(900L, ranges.["420"].["peakAnonBytes"].Value()) - Assert.Equal(950L, ranges.["840"].["peakAnonBytes"].Value()) - Assert.Equal(120.0, ranges.["420"].["seconds"].Value()) - // Slicing is inferred from the records, not from the caller's 20000. - Assert.Equal(420, ranges.["420"].["count"].Value()) - Assert.Equal(420, doc.["ledgersPerRange"].Value()) - Assert.Equal("pvc", doc.["storageMode"].Value()) - - -[] -let ``a measured range does not drag its unmeasured neighbours in`` () = - // The realistic shape, and the only one that catches a guard deciding per RUN - // rather than per RECORD: one that latches on the first measurement passes - // both tests above while every later junk record rides in behind it. - let completed = - completedMap - [ "1200", unmeasured (measuredRecord 400 900L) - "1600", measuredRecord 400 950L - "2000", unmeasured (measuredRecord 400 990L) ] - - match rangeProfileDocument "pvc" 20000 completed with - | None -> failwith "refused a profile that carries one real measurement" - | Some doc -> - let ranges = doc.["ranges"] :?> JObject - Assert.Equal(1, ranges.Count) - Assert.NotNull(ranges.["1600"]) - Assert.Null(ranges.["1200"]) - Assert.Null(ranges.["2000"]) diff --git a/src/FSLibrary.Tests/FSLibrary.Tests.fsproj b/src/FSLibrary.Tests/FSLibrary.Tests.fsproj index a32c892b..60b925ba 100644 --- a/src/FSLibrary.Tests/FSLibrary.Tests.fsproj +++ b/src/FSLibrary.Tests/FSLibrary.Tests.fsproj @@ -10,7 +10,6 @@ - diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index 6efd7d97..dbd90552 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -3,6 +3,8 @@ module Tests open StellarDestination open StellarMissionContext open Xunit +open Newtonsoft.Json.Linq +open MissionHistoryPubnetParallelCatchupV2 open System.Text.RegularExpressions open StellarCoreSet @@ -552,3 +554,161 @@ type Tests(output: ITestOutputHelper) = Assert.Equal("51/6", jobArr3.[0].[1]) Assert.Equal("56/6", jobArr3.[1].[1]) Assert.Equal("61/6", jobArr3.[2].[1]) + +// --------------------------------------------------------------------------- +// MissionHistoryPubnetParallelCatchupV2 +// --------------------------------------------------------------------------- +[] +let ``the job monitor image is overridable and defaults to the chart`` () = + // An empty flag must leave the chart's pin alone. monitor.image= resolves to + // ":latest" or fails the pull. + let src = + System.IO.File.ReadAllText( + "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + + Assert.Contains("if context.jobMonitorImagePcV2 <> \"\" then", src) + Assert.Contains("monitor.image=%s", src) + + let guard = src.IndexOf("if context.jobMonitorImagePcV2 <> \"\" then") + let use_ = src.IndexOf("monitor.image=%s") + Assert.True(guard < use_, "monitor.image must only be set inside the non-empty guard") + + +[] +let ``a pooled run does not let caller labels overwrite the routing label`` () = + // A pooled run claims index 0; the caller's entries must start at 1. Both at + // 0 and the second --set wins, so the pod matches on capacity alone and lands + // on any tier -- a supergiant range on a dwarf node, an OOM per range. + let src = + System.IO.File.ReadAllText( + "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + + Assert.Contains("worker.requireNodeLabels[0]=purpose:%s", src) + Assert.Contains("if context.pubnetParallelCatchupPoolPrefix <> \"\" then i + 1 else i", src) + + +[] +let ``the pool maps ride their own --set with their commas escaped`` () = + // Every other option is folded into ONE comma-joined --set; these maps are + // themselves comma-separated, so folding them in delivers a map of one tier. + let src = + System.IO.File.ReadAllText( + "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + + Assert.Contains("v.Replace(\",\", \"\\\\,\")", src) + Assert.Contains("poolMapArgs", src) + // Empty must not reach helm: monitor.poolCpu= blanks the chart default. + Assert.Contains("List.filter (fun (_, v) -> not (String.IsNullOrWhiteSpace v))", src) + +// The profile artifact. A completed record always carries bookkeeping (attempts, +// count) and may carry no measurement at all, when the collector never sampled +// that range. Such a record must not become an entry. +// +// RACE #8: count was attached BEFORE the `entry.Count > 0` guard, so every entry +// was non-empty and none were skipped. Seen twice in the field -- 0% +// peakAnonBytes in the artifact against 99% in progress.json. +// +// What that costs: profile_for resolves a range to the nearest measured end +// ABOVE it, so one junk entry at 1200 captures every range beneath it and hides +// the real 1600. Range 1100 then routes to protostar instead of supergiant, and +// nothing logs it. + + +/// Superset of rangeProfileFields: wallSeconds and txApply are recorded, never +/// projected. +let private measurementFields = + [ "peakAnonBytes"; "peakWorkingSetBytes"; "peakEphemeralBytes" + "txApply"; "seconds"; "wallSeconds" ] + +/// A record as /logs/progress.json carries it. +let private measuredRecord (count: int) (anon: int64) = + let r = JObject() + r.["attempts"] <- JValue(1) + r.["count"] <- JValue(count) + r.["seconds"] <- JValue(120.0) + r.["wallSeconds"] <- JValue(130.0) + r.["txApply"] <- JValue(60.0) + r.["peakAnonBytes"] <- JValue(anon) + r.["peakWorkingSetBytes"] <- JValue(anon + 1000L) + r + +/// The same record with every measurement stripped. +let private unmeasured (record: JObject) = + let r = record.DeepClone() :?> JObject + + for f in measurementFields do + r.Remove(f) |> ignore + + r + +let private completedMap (pairs: (string * JObject) list) = + let c = JObject() + + for (k, v) in pairs do + c.[k] <- v + + c + + +[] +let ``a measurement-free record cannot become a profile entry`` () = + // Nothing measured -- the whole document is refused, rather than written + // with the right range count and no data. + let completed = + completedMap + [ "420", unmeasured (measuredRecord 420 900L) + "840", unmeasured (measuredRecord 420 950L) + "1260", unmeasured (measuredRecord 420 990L) ] + + match rangeProfileDocument "pvc" 20000 completed with + | None -> () + | Some doc -> + let ranges = doc.["ranges"] :?> JObject + + failwithf + "wrote a profile artifact with %d ranges and zero measurements: %s" + ranges.Count + (ranges.ToString()) + + +[] +let ``a measured run still produces a complete profile`` () = + // The over-correction guard: a good read must still write everything. + let completed = + completedMap + [ "420", measuredRecord 420 900L + "840", measuredRecord 420 950L ] + + match rangeProfileDocument "pvc" 20000 completed with + | None -> failwith "refused to write a profile that carries real measurements" + | Some doc -> + let ranges = doc.["ranges"] :?> JObject + Assert.Equal(2, ranges.Count) + Assert.Equal(900L, ranges.["420"].["peakAnonBytes"].Value()) + Assert.Equal(950L, ranges.["840"].["peakAnonBytes"].Value()) + Assert.Equal(120.0, ranges.["420"].["seconds"].Value()) + // Slicing is inferred from the records, not from the caller's 20000. + Assert.Equal(420, ranges.["420"].["count"].Value()) + Assert.Equal(420, doc.["ledgersPerRange"].Value()) + Assert.Equal("pvc", doc.["storageMode"].Value()) + + +[] +let ``a measured range does not drag its unmeasured neighbours in`` () = + // The realistic shape, and the only one that catches a guard deciding per RUN + // rather than per RECORD: one that latches on the first measurement passes + // both tests above while every later junk record rides in behind it. + let completed = + completedMap + [ "1200", unmeasured (measuredRecord 400 900L) + "1600", measuredRecord 400 950L + "2000", unmeasured (measuredRecord 400 990L) ] + + match rangeProfileDocument "pvc" 20000 completed with + | None -> failwith "refused a profile that carries one real measurement" + | Some doc -> + let ranges = doc.["ranges"] :?> JObject + Assert.Equal(1, ranges.Count) + Assert.NotNull(ranges.["1600"]) + Assert.Null(ranges.["1200"]) + Assert.Null(ranges.["2000"]) From d3d4997d86730e209fd04c6c41f271fbbcd1efd8 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 17:25:35 -0400 Subject: [PATCH 097/117] Stop passing the chart its own values file helm reads /values.yaml as its base already, so --values with that same path re-applied the file on top of itself. Verified byte-identical renders with and without it. It was only ever there to carry the second file: the on-demand overlay rode as a second --values, and the base had to be named explicitly so layering worked. With the overlay gone the base names itself, and valuesFilePath, valuesArgs and the argv entry all go with it. Precedence is unchanged -- chart defaults, then --set -- because a -f of the chart's own defaults changed nothing to begin with. --- src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 38f3dec2..7f41b798 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -35,7 +35,6 @@ let helmChartPath = // Example command to run local testing (in the `supercluster/` directory): // $ dotnet run --project src/App/App.fsproj -- mission HistoryPubnetParallelCatchupV2 --image=docker-registry.services.stellar-ops.com/dev/stellar-core:23.0.3-2779.4d1df2b03.jammy-vnext-buildtests --pubnet-parallel-catchup-num-workers=2 --pubnet-parallel-catchup-starting-ledger=0 --pubnet-parallel-catchup-end-ledger=6400 --pubnet-parallel-catchup-ledgers-per-job 1280 --destination ./logs -let valuesFilePath = helmChartPath + "/values.yaml" let jobMonitorLoggingIntervalSecs = 30 // frequency of the monitor reconcile loop: dispatch, liveness ping, status publish let jobMonitorStatusCheckIntervalSecs = 60 @@ -406,8 +405,6 @@ let installProject (context: MissionContext) = // installs its monitor, Jobs and PVCs into a different one. Observed // 2026-07-30: a mission run with --namespace sandbox put a monitor and four // Jobs into the production namespace alongside a live run. - let valuesArgs = [| "--values"; valuesFilePath |] - // The pool maps get their own --set each. Every other option is folded into // ONE comma-joined --set, and these are themselves comma-separated, so // folding them in would split each tier into a separate assignment. helm @@ -428,7 +425,6 @@ let installProject (context: MissionContext) = RunShellCommand( Array.concat [ [| "helm"; "install"; helmReleaseName; helmChartPath |] [| "--namespace"; context.namespaceProperty |] - valuesArgs poolMapArgs [| "--set"; String.Join(",", setOptions) |] ] ) From f106b002f50c84e46e2d8639829384d33816835c Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 17:27:48 -0400 Subject: [PATCH 098/117] Make the values file overridable with SUPERCLUSTER_VALUES_PATH Unset, nothing is passed and helm uses the chart's own values.yaml as its base -- which it does regardless, so naming that path explicitly only re-applied the file to itself. Set, the named file is layered on top. Layered rather than substituted, so an override carries only the keys it changes: verified with a two-line file that moves poolPrefix while poolTiers keeps the chart's ladder. SUPERCLUSTER_CHART_PATH already repoints the whole chart, values included. This is the narrower case -- the baked chart run against experimental numbers, without a working copy. --- .../MissionHistoryPubnetParallelCatchupV2.fs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 7f41b798..3dc1096c 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -33,6 +33,21 @@ let helmChartPath = | "" -> "/supercluster/src/MissionParallelCatchup/parallel_catchup_helm" | p -> p +// An extra values file layered on top of the chart's own, for a run that wants +// different values without a working copy of the chart. Unset is the normal +// case: helm reads /values.yaml as its base regardless, so naming that +// same path would only re-apply it to itself. +// +// Layered, not substituted, so it carries only the keys it changes. +// SUPERCLUSTER_CHART_PATH repoints the whole chart including its values; this +// repoints the values alone, so the baked chart can run against experimental +// numbers. +let extraValuesArgs = + match Environment.GetEnvironmentVariable("SUPERCLUSTER_VALUES_PATH") with + | null + | "" -> [||] + | p -> [| "--values"; p |] + // Example command to run local testing (in the `supercluster/` directory): // $ dotnet run --project src/App/App.fsproj -- mission HistoryPubnetParallelCatchupV2 --image=docker-registry.services.stellar-ops.com/dev/stellar-core:23.0.3-2779.4d1df2b03.jammy-vnext-buildtests --pubnet-parallel-catchup-num-workers=2 --pubnet-parallel-catchup-starting-ledger=0 --pubnet-parallel-catchup-end-ledger=6400 --pubnet-parallel-catchup-ledgers-per-job 1280 --destination ./logs @@ -425,6 +440,7 @@ let installProject (context: MissionContext) = RunShellCommand( Array.concat [ [| "helm"; "install"; helmReleaseName; helmChartPath |] [| "--namespace"; context.namespaceProperty |] + extraValuesArgs poolMapArgs [| "--set"; String.Join(",", setOptions) |] ] ) From bde2160cd399cb448a2a9110a8e741ae5ee04fec Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 17:32:19 -0400 Subject: [PATCH 099/117] Accept an http profile URL, and cover every resolution path Only https was recognised as a URL, so an http spec fell to the else branch and was read as a FILENAME. It failed as "could not load range profile" without ever naming the scheme, and the run proceeded unprofiled -- every range sized from defaults, visible only as a run that cost more. The codebase already speaks plain http to the monitor, so there was no https-only posture to keep, and a profile moves resource REQUESTS only: a tampered one costs node size, not code execution, and it is parsed and range-counted before use. Four tests, one per path, each the sole killer of its own mutant: fetched over http (against a loopback listener), read from a local path, refused when it carries no ranges, and never fatal when the spec is empty, missing or unreachable. The listener helper swallows exceptions inside its serving async deliberately. Without that, a test where no request arrives faults the pending GetContextAsync when Stop() runs, on a background thread, and takes the test host down -- the run then reports "26 passed" as "8 passed" and the failure it was supposed to surface disappears. Caught while checking the http test could fail at all. --- src/FSLibrary.Tests/Tests.fs | 85 +++++++++++++++++++ .../MissionHistoryPubnetParallelCatchupV2.fs | 9 +- 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index dbd90552..e858acd7 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -712,3 +712,88 @@ let ``a measured range does not drag its unmeasured neighbours in`` () = Assert.NotNull(ranges.["1600"]) Assert.Null(ranges.["1200"]) Assert.Null(ranges.["2000"]) + + +/// A context whose only interesting field is the profile spec. +let private profileCtx (spec: string) = + { ctx with pubnetParallelCatchupProfile = spec } + +/// Serve one JSON body on a loopback port for the duration of `f`. +let private servingJson (body: string) (f: string -> 'a) : 'a = + let listener = new System.Net.HttpListener() + // port 0 is not available to HttpListener, so take one from a throwaway + // socket and hand it over + let probe = new System.Net.Sockets.TcpListener(System.Net.IPAddress.Loopback, 0) + probe.Start() + let port = (probe.LocalEndpoint :?> System.Net.IPEndPoint).Port + probe.Stop() + let prefix = sprintf "http://127.0.0.1:%d/" port + listener.Prefixes.Add(prefix) + listener.Start() + + let serve = + async { + // Swallowing is required, not lazy: if the caller never issues a + // request, Stop() faults this pending GetContextAsync on a + // background thread and takes the whole test host with it -- the + // run then reports a partial pass instead of the failure. + try + let! c = listener.GetContextAsync() |> Async.AwaitTask + let bytes = System.Text.Encoding.UTF8.GetBytes(body) + c.Response.ContentLength64 <- int64 bytes.Length + c.Response.OutputStream.Write(bytes, 0, bytes.Length) + c.Response.OutputStream.Close() + with _ -> () + } + + Async.Start serve + + try + f (prefix + "profile.json") + finally + listener.Stop() + +let private oneRange = """{"ranges":{"420":{"peakAnonBytes":900}}}""" + + +[] +let ``a profile is fetched over http, not only https`` () = + // Anything not recognised as a URL is read as a FILENAME, so an http spec + // used to fail as "could not load" without ever naming the scheme, and the + // run proceeded unprofiled -- every range sized from defaults. + let got = servingJson oneRange (fun url -> resolveRangeProfile (profileCtx url)) + Assert.True(got.IsSome, "an http:// profile must be fetched, not treated as a path") + Assert.Equal(900, (JObject.Parse(got.Value).["ranges"].["420"].["peakAnonBytes"]).Value()) + + +[] +let ``a profile is read from a local path`` () = + let path = System.IO.Path.GetTempFileName() + System.IO.File.WriteAllText(path, oneRange) + + try + Assert.True((resolveRangeProfile (profileCtx path)).IsSome) + finally + System.IO.File.Delete path + + +[] +let ``an unreadable or empty profile spec never fails the run`` () = + // A profile only tightens requests, so failing to load one must degrade to + // the configured defaults rather than fail the mission. + Assert.True((resolveRangeProfile (profileCtx "")).IsNone) + Assert.True((resolveRangeProfile (profileCtx "/nonexistent/profile.json")).IsNone) + Assert.True((resolveRangeProfile (profileCtx "http://127.0.0.1:1/gone.json")).IsNone) + + +[] +let ``a profile carrying no ranges is refused`` () = + // It would install cleanly and size nothing, which is indistinguishable + // from a good profile at every later point. + let path = System.IO.Path.GetTempFileName() + System.IO.File.WriteAllText(path, """{"ranges":{}}""") + + try + Assert.True((resolveRangeProfile (profileCtx path)).IsNone) + finally + System.IO.File.Delete path diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 3dc1096c..640e231a 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -78,7 +78,14 @@ let resolveRangeProfile (context: MissionContext) : string option = else try let body = - if spec.StartsWith("https://", StringComparison.OrdinalIgnoreCase) then + // http as well as https: the only alternative is treating the + // URL as a filename, which fails as "could not load" without + // ever mentioning the scheme, and the run proceeds unprofiled -- + // every range sized from defaults. A profile moves resource + // REQUESTS only, so a tampered one costs node size, not code + // execution, and it is parsed and range-counted before use. + if spec.StartsWith("http://", StringComparison.OrdinalIgnoreCase) + || spec.StartsWith("https://", StringComparison.OrdinalIgnoreCase) then use client = new HttpClient() client.Timeout <- TimeSpan.FromSeconds(30.0) client.GetStringAsync(spec) |> Async.AwaitTask |> Async.RunSynchronously From 5031d1a01c989cf8cec9bf9b3c80209d45204cb9 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 10 Aug 2026 17:36:10 -0400 Subject: [PATCH 100/117] Drop the profile-resolution tests Four tests and a loopback HTTP listener for paths whose failure is a warning and a run sized from defaults, not a fatal outcome. The http path is better verified by running it. --- src/FSLibrary.Tests/Tests.fs | 85 ------------------------------------ 1 file changed, 85 deletions(-) diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index e858acd7..dbd90552 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -712,88 +712,3 @@ let ``a measured range does not drag its unmeasured neighbours in`` () = Assert.NotNull(ranges.["1600"]) Assert.Null(ranges.["1200"]) Assert.Null(ranges.["2000"]) - - -/// A context whose only interesting field is the profile spec. -let private profileCtx (spec: string) = - { ctx with pubnetParallelCatchupProfile = spec } - -/// Serve one JSON body on a loopback port for the duration of `f`. -let private servingJson (body: string) (f: string -> 'a) : 'a = - let listener = new System.Net.HttpListener() - // port 0 is not available to HttpListener, so take one from a throwaway - // socket and hand it over - let probe = new System.Net.Sockets.TcpListener(System.Net.IPAddress.Loopback, 0) - probe.Start() - let port = (probe.LocalEndpoint :?> System.Net.IPEndPoint).Port - probe.Stop() - let prefix = sprintf "http://127.0.0.1:%d/" port - listener.Prefixes.Add(prefix) - listener.Start() - - let serve = - async { - // Swallowing is required, not lazy: if the caller never issues a - // request, Stop() faults this pending GetContextAsync on a - // background thread and takes the whole test host with it -- the - // run then reports a partial pass instead of the failure. - try - let! c = listener.GetContextAsync() |> Async.AwaitTask - let bytes = System.Text.Encoding.UTF8.GetBytes(body) - c.Response.ContentLength64 <- int64 bytes.Length - c.Response.OutputStream.Write(bytes, 0, bytes.Length) - c.Response.OutputStream.Close() - with _ -> () - } - - Async.Start serve - - try - f (prefix + "profile.json") - finally - listener.Stop() - -let private oneRange = """{"ranges":{"420":{"peakAnonBytes":900}}}""" - - -[] -let ``a profile is fetched over http, not only https`` () = - // Anything not recognised as a URL is read as a FILENAME, so an http spec - // used to fail as "could not load" without ever naming the scheme, and the - // run proceeded unprofiled -- every range sized from defaults. - let got = servingJson oneRange (fun url -> resolveRangeProfile (profileCtx url)) - Assert.True(got.IsSome, "an http:// profile must be fetched, not treated as a path") - Assert.Equal(900, (JObject.Parse(got.Value).["ranges"].["420"].["peakAnonBytes"]).Value()) - - -[] -let ``a profile is read from a local path`` () = - let path = System.IO.Path.GetTempFileName() - System.IO.File.WriteAllText(path, oneRange) - - try - Assert.True((resolveRangeProfile (profileCtx path)).IsSome) - finally - System.IO.File.Delete path - - -[] -let ``an unreadable or empty profile spec never fails the run`` () = - // A profile only tightens requests, so failing to load one must degrade to - // the configured defaults rather than fail the mission. - Assert.True((resolveRangeProfile (profileCtx "")).IsNone) - Assert.True((resolveRangeProfile (profileCtx "/nonexistent/profile.json")).IsNone) - Assert.True((resolveRangeProfile (profileCtx "http://127.0.0.1:1/gone.json")).IsNone) - - -[] -let ``a profile carrying no ranges is refused`` () = - // It would install cleanly and size nothing, which is indistinguishable - // from a good profile at every later point. - let path = System.IO.Path.GetTempFileName() - System.IO.File.WriteAllText(path, """{"ranges":{}}""") - - try - Assert.True((resolveRangeProfile (profileCtx path)).IsNone) - finally - System.IO.File.Delete path From 530644ffc385be312aacc39e461e2f2e0b3d390a Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Tue, 11 Aug 2026 10:11:31 -0400 Subject: [PATCH 101/117] Log the monitor process to one file, named for the process Every run shipped two log files: http_server_.log holding the whole monitor's output, and job_monitor_.log holding nothing. Seen in run #181 at 13.27 KiB and 0 B, and reproduced locally. build_logger calls logging.basicConfig, which is a no-op once the root logger has handlers. job_monitor imports http_server before configuring itself, so http_server's call ran first and won; job_monitor's was discarded. The FileHandler it had already constructed still created and opened its file, which is why an empty one appeared. Both modules take the ROOT logger, so everything landed in the file named after whichever module imported first. The entrypoint should configure logging and nothing else should. http_server now takes a plain named logger, so job_monitor's build_logger is the first and only call: one file, job_monitor_.log, carrying both modules' records. Verified -- one file, 276 bytes, records from both. Image rebuilt as 2026-08-11a and pinned. --- src/MissionParallelCatchup/lib/http_server.py | 11 ++++++++--- .../parallel_catchup_helm/values.yaml | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/MissionParallelCatchup/lib/http_server.py b/src/MissionParallelCatchup/lib/http_server.py index 09b79942..2e5f0ad4 100644 --- a/src/MissionParallelCatchup/lib/http_server.py +++ b/src/MissionParallelCatchup/lib/http_server.py @@ -13,6 +13,7 @@ """ import json +import logging import os import re import threading @@ -21,9 +22,13 @@ from prometheus_client import CONTENT_TYPE_LATEST, REGISTRY, generate_latest import config -from logger import build_logger - -logger = build_logger('http_server') +# Not build_logger: this module is imported BY job_monitor, so configuring here +# would run first and logging.basicConfig is a no-op once root has handlers -- +# job_monitor's own call was then silently discarded, and the FileHandler it +# built still created job_monitor_.log, which stayed empty in every run's +# artifacts while this module's file held the whole process's output. The +# entrypoint configures; this just takes a logger. +logger = logging.getLogger('http_server') # Set by job_monitor before serve(). A tuple rather than an import, because # job_monitor imports this module. diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index a6cbd30d..cd60daea 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -68,7 +68,7 @@ monitor: # this chart against it ships env vars the image cannot read. Built from this # branch and pinned by tag rather than :latest so a run is always traceable to # one image. Revert to the stellar/ repo once there is a push path for it. - image: "stellajuna/ssc-jm:2026-08-10a" + image: "stellajuna/ssc-jm:2026-08-11a" # The driver reaches the monitor through this route: profile in via POST # /start, status and logs out. Empty routeHost disables the HTTPRoute; the # Service still exists, so an in-cluster caller works either way. From b79a883358abfb542229ffbc4dafc63a7c814df8 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Tue, 11 Aug 2026 10:54:25 -0400 Subject: [PATCH 102/117] Sort collected artifacts into folders, and summarise pods by phase The bundle was flat. A 10-range run puts 43 files beside the five that summarise it; a 4000-range run puts ~16000. The three per-range artifacts now land in range-logs/, metrics/ and state/, with mission_started joining the other state markers, leaving the top level as the monitor log, the driver log, progress.json, run.json and the profile. Sorted on the way out rather than on the volume. The monitor's paths are an implementation detail, and keeping the volume flat leaves /logs/ taking one path element with no separator -- which is what stops the route, reachable from outside the cluster once its HTTPRoute is attached, being walked out of LOG_DIR. Verified on ssc-test: top level 6 files, range-logs/ 3, metrics/ 3, state/ 6. DumpPodInfo now logs a count per phase instead of a line per pod. It fires every 5 minutes for the whole mission, so at 1024 workers it wrote ~1026 lines a time -- roughly 57000 over a 4.7h catchup -- and the only thing worth reading in it, anything not Running, was buried. Shorter than the code it replaces, and a 5-pod mission now reads "Pods: 5 total Running=5". --- .../MissionHistoryPubnetParallelCatchupV2.fs | 33 ++++++++++++++++++- src/FSLibrary/StellarSupercluster.fs | 22 +++++++------ 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 640e231a..69fa5efa 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -486,6 +486,29 @@ let private monitorPodName (context: MissionContext) : string option = /// workers came back truncated 2 times in 3 with no error surfaced. A file is /// its own unit here: a cut transfer resumes from the byte it reached, and one /// bad fetch costs one file rather than the pass. +/// Which subfolder of the destination an artifact belongs in. +/// +/// Sorted on the way out, not on the volume: the monitor's paths are an +/// implementation detail, while this is what a human opens. Flat, a 4000-range +/// run lands ~16000 files beside the five that summarise it -- three per range +/// that are per-range detail, and one bundle-wide file each for the monitor log, +/// the driver log, the progress record, the profile it produced and the run it +/// was asked for. +/// +/// Keeping the volume flat also leaves the HTTP surface alone: /logs/ +/// takes one path element and no separator, which is what stops the route -- +/// reachable from outside the cluster once its HTTPRoute is attached -- being +/// walked out of LOG_DIR. +let artifactFolder (name: string) = + if name.EndsWith(".log.gz") then "range-logs" + elif name.EndsWith(".metrics") then "metrics" + elif name.EndsWith(".done") + || name.EndsWith(".started") + || name.EndsWith(".state") + || name = "mission_started" then + "state" + else "" + let collectLogs (context: MissionContext) (destination: string) = Directory.CreateDirectory(destination) |> ignore use client = monitorClient context @@ -497,7 +520,15 @@ let collectLogs (context: MissionContext) (destination: string) = let fetchOne (entry: JToken) = let name = entry.["name"].ToString() let size = entry.["size"].Value() - let path = Path.Combine(destination, name) + let folder = artifactFolder name + + let path = + if folder = "" then + Path.Combine(destination, name) + else + Directory.CreateDirectory(Path.Combine(destination, folder)) |> ignore + Path.Combine(destination, folder, name) + let have = if File.Exists path then FileInfo(path).Length else 0L // Already whole. The collector only ever appends, so equal length means diff --git a/src/FSLibrary/StellarSupercluster.fs b/src/FSLibrary/StellarSupercluster.fs index 2c7280fd..362d32af 100644 --- a/src/FSLibrary/StellarSupercluster.fs +++ b/src/FSLibrary/StellarSupercluster.fs @@ -217,16 +217,18 @@ let DumpPodInfo (kube: Kubernetes) (apiRateLimit: int) (ns: string) = let pods = kube.ListNamespacedPod(namespaceParameter = ns) if pods <> null then - LogInfo "There are %d pods in total" (Seq.length pods.Items) - - for p in pods.Items do - let age = - if p.Status.StartTime.HasValue then - System.DateTime.UtcNow.Subtract(p.Status.StartTime.Value).ToString(@"hh\:mm") - else - "00:00" - - LogInfo "Pod: name=%s phase=%s age=%s (hr:min)" p.Metadata.Name p.Status.Phase age + // A count per phase rather than a line per pod. This fires every 5 + // minutes for the whole mission, so at 1024 workers the old form wrote + // ~1026 lines a time -- ~57000 over a 4.7h catchup -- and buried the + // only thing worth reading, which is anything not Running. + let byPhase = + pods.Items + |> Seq.countBy (fun p -> p.Status.Phase) + |> Seq.sortBy fst + |> Seq.map (fun (phase, n) -> sprintf "%s=%d" phase n) + |> String.concat " " + + LogInfo "Pods: %d total %s" (Seq.length pods.Items) byPhase // Create a per-run "anchor" ConfigMap and stash an owner reference to it on // `nCfg.anchorOwnerRef` so every subsequent resource the mission creates will From 746b2be3089f1acb29beb2d8ee3adbe511500939 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Tue, 11 Aug 2026 11:25:51 -0400 Subject: [PATCH 103/117] Print status and pod lines every ten minutes The status line went out every minute -- ~280 lines on a 4.7h run, saying nothing new most of the time. It now prints one poll in ten. Printed less, not polled less: jobMonitorStatusCheckTimeOutSecs is spent in units of the interval, so 600 over a 60s poll tolerates ten consecutive failures. Slowing the poll to 600s instead would leave a single transient blip failing the run outright. Detection is unchanged; only the logging thins. A range failure still logs the moment it is seen, on its own line. The pod summary timer goes from 5 minutes to 10. That callback lists every pod in the namespace, so on a 1024-worker run it is a large response fetched to print one line. Also drops the ephemeral request and limit on pooled runs that were given a profile. Such a range has its node to itself -- the tier's memory cut is sized to exclude a second pod -- so the limit guards no neighbour and only turns spare disk into an eviction: measured, a dwarf range capped at 5211Mi alone on a 20Gi root, dying once it exceeded its profiled peak by more than the margin. The request goes too, since its only remaining job is scheduling and the tier label already decides placement. Unprofiled pooled runs keep both, nothing having measured them, so their disk escalation path is intact. Verified across the four combinations: pooled+profiled ephemeral drops to requests {cpu, memory} with no limits; pooled with no profile keeps 35Gi/40Gi; unpooled+profiled keeps its profile-derived figure; pvc is unchanged. Image rebuilt as 2026-08-11b and pinned. --- src/App/Program.fs | 5 ++++- .../MissionHistoryPubnetParallelCatchupV2.fs | 15 ++++++++++++++- .../apps/job_monitor.py | 19 ++++++++++++++++--- .../parallel_catchup_helm/values.yaml | 2 +- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/App/Program.fs b/src/App/Program.fs index 99a0ec7b..a2cc1662 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -860,7 +860,10 @@ let main argv = DumpPodInfo kube mission.ApiRateLimit ns with x -> LogError "Connection issue! Api call failed." - let timer = new System.Threading.Timer(TimerCallback(podLogger), null, 1000, 300000) + // Every 10 minutes, not 5: this lists every pod in the + // namespace, and on a 1024-worker run that is a large response + // fetched purely to print one summary line. + let timer = new System.Threading.Timer(TimerCallback(podLogger), null, 1000, 600000) for m in mission.Missions do LogInfo "-----------------------------------" diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 69fa5efa..ec483fb6 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -54,10 +54,18 @@ let extraValuesArgs = let jobMonitorLoggingIntervalSecs = 30 // frequency of the monitor reconcile loop: dispatch, liveness ping, status publish let jobMonitorStatusCheckIntervalSecs = 60 let jobMonitorStatusCheckTimeOutSecs = 600 + +// Print one status line in ten. The poll stays at a minute because +// jobMonitorStatusCheckTimeOutSecs is spent in units of it -- 600 over a 60s +// interval tolerates ten consecutive failures, and slowing the poll instead +// would make a single transient blip fail the run. Nothing is lost by printing +// less: a range failure logs the moment it is seen, on its own line. +let jobMonitorStatusLogEveryNChecks = 10 let mutable toPerformCleanup = true let failedJobLogFileLineCount = 10000 let failedJobLogStreamLineCount = 1000 +let mutable statusChecks = 0 let mutable nonce : String = "" let mutable helmReleaseName : String = "" @@ -595,7 +603,12 @@ let queryJobMonitor (context: MissionContext) = try use client = monitorClient context let body = client.GetStringAsync("/status") |> Async.AwaitTask |> Async.RunSynchronously - LogInfo "job monitor status: %s" body + + statusChecks <- statusChecks + 1 + + if statusChecks % jobMonitorStatusLogEveryNChecks = 1 then + LogInfo "job monitor status: %s" body + Some(JObject.Parse(body)) with ex -> LogError "Error reading job monitor status: %s" ex.Message diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py index 8b30bac6..4cf8ce2a 100644 --- a/src/MissionParallelCatchup/apps/job_monitor.py +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -925,17 +925,30 @@ def _resources(mem=None, eph=None, end=None, attempt=1): # unbounded pod takes the node down rather than itself. lim = {} + # A profiled range on a pooled run has its node to itself -- the tier's + # memory cut is sized to exclude a second pod -- so an ephemeral limit + # guards no neighbour and only turns spare disk into an eviction. Measured: + # a dwarf range capped at 5211Mi alone on a 20Gi root, dying the moment it + # exceeded its profiled peak by more than the margin, with most of the disk + # unused. The request goes with the limit because its only remaining job is + # scheduling, and the tier label already decides placement. + # + # Unprofiled pooled runs keep both: nothing measured them, so there is no + # peak to have been generous about. + pooled_profiled = bool(config.POOL_PREFIX and config.PROFILE) + # Only meaningful in ephemeral mode. In PVC mode a large request makes disk # the binding dimension and halves workers-per-node for no reason. - if config.REQ_EPHEMERAL: + if config.REQ_EPHEMERAL and not pooled_profiled: # Raise the request with the limit: ephemeral-storage is a scheduling # dimension, so a pod that outgrew it no longer fits where it was. req['ephemeral-storage'] = eph or config.REQ_EPHEMERAL else: # pvc mode: /data is not on the node disk, so an ephemeral override - # would size a dimension this run does not use. + # would size a dimension this run does not use. Pooled+profiled: the + # pod owns its node and the axis is dropped deliberately. overrides.pop('ephemeral-storage', None) - if config.LIM_EPHEMERAL: + if config.LIM_EPHEMERAL and not pooled_profiled: lim['ephemeral-storage'] = eph or config.LIM_EPHEMERAL # The profile moves requests only. Disk excepted, because its limit is what diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index cd60daea..c7c74e3f 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -68,7 +68,7 @@ monitor: # this chart against it ships env vars the image cannot read. Built from this # branch and pinned by tag rather than :latest so a run is always traceable to # one image. Revert to the stellar/ repo once there is a push path for it. - image: "stellajuna/ssc-jm:2026-08-11a" + image: "stellajuna/ssc-jm:2026-08-11b" # The driver reaches the monitor through this route: profile in via POST # /start, status and logs out. Empty routeHost disables the HTTPRoute; the # Service still exists, so an in-cluster caller works either way. From 8c8d5b8597de9d4d9daa0ccffc9faaff18f15245 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Tue, 11 Aug 2026 12:41:37 -0400 Subject: [PATCH 104/117] Refetch rewritten artifacts whole instead of resuming them collectLogs resumed a partial file with a Range request and appended the answer, and skipped a file whose length already matched. Both assume the file only ever grew, which is true of a worker log and of nothing else it collects. progress.json is rewritten whole on every reconcile and grows as ranges complete, so the second pass asked for bytes past the length of the FIRST document and appended that tail to it. Reproduced on ssc-test with a 32-range run: pass one left 2675 bytes at 18 completed ranges, pass two saw 4708 at 32, and splicing them the way the old code would gives "Extra data: line 1 column 2676" -- the first document's closing "failed":{}} followed by a fragment of the second. Silent, too: every prod run collects on a 10-minute timer over ~4.7h, so this landed a truncated progress.json in the bundle while nothing complained. The profile artifact is unaffected, being built from the copy read off the pod. Equal length is no safer than a short read: a rewrite can change a value without changing the length -- seconds 100.0 to 200.0 -- so both the skip and the resume now apply only to .log.gz. Refetching the rest costs little against what the optimisation is actually for: progress.json is ~1 MB at 4000 ranges, the worker logs are ~1.4 GB. Verified on the same run with the fix: 4708 bytes, 32 ranges, valid. --- .../MissionHistoryPubnetParallelCatchupV2.fs | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index ec483fb6..e78ac53a 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -539,19 +539,27 @@ let collectLogs (context: MissionContext) (destination: string) = let have = if File.Exists path then FileInfo(path).Length else 0L - // Already whole. The collector only ever appends, so equal length means - // equal content -- and this is what stops a pass re-sending what the - // last one already took (the tar overlap re-sent 58% of files). - // + // Resume and skip both assume the file only ever grew, which is true of + // a worker log and of nothing else here. progress.json is rewritten + // whole on every reconcile and grows as ranges complete, so a Range + // request would splice the new document's tail onto the old one's + // prefix -- invalid JSON, in the bundle, silently, since the profile is + // built from the copy read off the pod rather than this one. Equal + // length is no safer: a rewrite can change a value without changing the + // length. Everything that is not append-only is refetched whole, which + // costs little -- progress.json is ~1 MB at 4000 ranges against ~1.4 GB + // of worker logs. + let appendOnly = name.EndsWith(".log.gz") + // File.Exists is not redundant: a .done marker is zero bytes, so a // length comparison alone reads "absent locally" as "already have it" // and never fetches it. Measured 2026-08-08: 88 of 110 artifacts // collected, and every .done was among the 22 missing. - if File.Exists path && have = size then + if appendOnly && File.Exists path && have = size then -1L // already whole; distinct from a zero-byte file we did fetch else let req = new HttpRequestMessage(HttpMethod.Get, "/logs/" + name) - if have > 0L && have < size then + if appendOnly && have > 0L && have < size then req.Headers.Range <- Headers.RangeHeaderValue(Nullable(have), Nullable()) use resp = client.SendAsync(req) |> Async.AwaitTask |> Async.RunSynchronously From 32e8ed1ed8d6b0f716fd542c17a981eb035b2085 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Tue, 11 Aug 2026 12:48:41 -0400 Subject: [PATCH 105/117] Always pull the job monitor image Runs name this image by a mutable tag -- the mission passes --job-monitor-image-pc-v2 stellajuna/ssc-jm:latest -- and IfNotPresent pinned each node to whatever it had cached the first time it saw that tag. Worker nodes churn under Karpenter and would repull regardless, but the monitor carries no tolerations and lands on the long-lived default pool, where the layer persists indefinitely. Run #183 shipped a 0-byte job_monitor.log and a populated http_server log from an image whose fix for precisely that had been pushed hours before. The registry tag was correct throughout; verified by pulling latest fresh, which carries the logging fix, REQUIRE_NODE_LABELS, the pooled+profiled ephemeral change and the /start pool-map guard. Kubernetes defaults :latest to Always for this reason and the explicit IfNotPresent was overriding it. One extra pull per run for one pod, and a registry outage now fails the run at start rather than quietly running something stale. --- .../parallel_catchup_helm/values.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index c7c74e3f..c738cb34 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -150,7 +150,19 @@ monitor: # longest valid runtime in the profile. Long-running ranges need more slack # for page cache and allocator growth. 0 disables. profileRuntimeMemoryInsurance: "3Gi" - imagePullPolicy: IfNotPresent + # Always, because runs name this image by a MUTABLE tag: the mission passes + # --job-monitor-image-pc-v2 stellajuna/ssc-jm:latest, and IfNotPresent then + # pins a node to whatever it cached the first time it saw that tag. Worker + # nodes churn under Karpenter so they would repull anyway, but the monitor + # carries no tolerations and lands on the long-lived default pool, where the + # layer persists for good -- run #183 shipped a job_monitor.log of 0 bytes + # from an image whose fix for exactly that had been pushed hours earlier. + # + # Kubernetes defaults :latest to Always for this reason; the explicit + # IfNotPresent was overriding it. Costs one pull per run for one pod, and + # makes a registry outage fail the run at start rather than run something + # stale. + imagePullPolicy: Always mission: "HistoryPubnetParallelCatchup" # Adds a `mission` label to worker pods, which kube-state-metrics exposes as # label_mission for the Grafana container panels. Default OFF: those panels From 9a07a284d094af4a9d24e683e35b08c7bff56886 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Tue, 11 Aug 2026 12:56:51 -0400 Subject: [PATCH 106/117] Delete the leftover CAPACITY_TYPE env monitor.capacityType and the python that read it both went when capacity became an ordinary entry in REQUIRE_NODE_LABELS, but the env block survived. It rendered from a value that no longer exists, so every run shipped a CAPACITY_TYPE with no value that nothing reads -- visible in run #184's pod spec. Harmless, and invisible to the contract tests: they walk the config module and pair each attribute with its chart env, so an env var with no counterpart in the code is never looked at. Not worth a test of its own -- the failure is a dead line, not a wrong run. --- .../parallel_catchup_helm/templates/job_monitor.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index e59a74d2..237330b9 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -302,11 +302,6 @@ spec: value: {{ .Values.monitor.poolNoProfile | quote }} - name: POOL_MEM value: {{ .Values.monitor.poolMem | quote }} - # Pods AND this with the tier label. Both capacity variants of a - # tier share one label value, and a pod cannot otherwise express a - # NodePool property; Karpenter labels every node with it itself. - - name: CAPACITY_TYPE - value: {{ .Values.monitor.capacityType | quote }} - name: PROFILE_MARGIN value: {{ .Values.monitor.profileMargin | quote }} - name: PROFILE_MAX_MEM From 07aa5a5a19628b0b38aa8b908bb45b3988019122 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Tue, 11 Aug 2026 13:11:10 -0400 Subject: [PATCH 107/117] Let a run choose where the job monitor lands The monitor asked for nothing -- no nodeSelector, no tolerations beyond the default two -- so it could only land on an untainted node. Every catchup pool is tainted, which left the shared EKS managed node group: on ssc-eks that is a node up since June, hosting traefik, argocd, kyverno and another engineer's stellar-core pods. It was there by omission, not by choice, and it is the pod whose collector streams logs from up to 1024 workers. --job-monitor-node-labels and --job-monitor-tolerate-taints take the same `key:value` form as their worker counterparts and are empty by default, so an installation without tiered pools sets nothing and sees exactly today's behaviour. Both are needed together to move it -- labels alone leave it unschedulable on a tainted pool. The chart gains only monitor.nodeSelector, rendered with toYaml. tolerateNodeTaints already took native toleration objects, and a plain equality map is enough here: the monitor's node is chosen once for the run, unlike a worker's, which is resolved per range and per attempt. Verified on ssc-test with purpose:catchup-giant catchup-capacity:od and the catchup toleration: the monitor landed on a karpenter catchup-giant-od-w100 node, on-demand, m8a.large, and the run completed. Worth knowing before pinning it to a small tier: the monitor holds the logs PVC for the whole run, so its node cannot be consolidated away while it is there, and it reserves that node from the ranges. --- src/App/Program.fs | 14 ++++++++ src/FSLibrary.Tests/Tests.fs | 2 ++ .../MissionHistoryPubnetParallelCatchupV2.fs | 32 +++++++++++++++++++ src/FSLibrary/StellarMissionContext.fs | 2 ++ .../templates/job_monitor.yaml | 4 +++ .../parallel_catchup_helm/values.yaml | 10 ++++++ 6 files changed, 64 insertions(+) diff --git a/src/App/Program.fs b/src/App/Program.fs index a2cc1662..98629a65 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -125,6 +125,8 @@ type MissionOptions pubnetParallelCatchupPoolCpu: string, pubnetParallelCatchupPoolMem: string, pubnetParallelCatchupCreateRbac: bool, + jobMonitorNodeLabels: seq, + jobMonitorTolerateTaints: seq, tag: string option, numPregeneratedTxs: int option, genesisTestAccountCount: int option, @@ -591,6 +593,16 @@ type MissionOptions Default = false)>] member self.PubnetParallelCatchupCreateRbac : bool = pubnetParallelCatchupCreateRbac + [] + member self.JobMonitorNodeLabels = jobMonitorNodeLabels + + [] + member self.JobMonitorTolerateTaints = jobMonitorTolerateTaints + [] member self.Tag = tag @@ -981,6 +993,8 @@ let main argv = pubnetParallelCatchupPoolCpu = mission.PubnetParallelCatchupPoolCpu pubnetParallelCatchupPoolMem = mission.PubnetParallelCatchupPoolMem pubnetParallelCatchupCreateRbac = mission.PubnetParallelCatchupCreateRbac + jobMonitorNodeLabels = List.map splitLabel (List.ofSeq mission.JobMonitorNodeLabels) + jobMonitorTolerateTaints = List.map splitLabel (List.ofSeq mission.JobMonitorTolerateTaints) tag = mission.Tag numPregeneratedTxs = mission.NumPregeneratedTxs enableTailLogging = true diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index eeeb7550..ac6db5d3 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -131,6 +131,8 @@ let ctx : MissionContext = pubnetParallelCatchupPoolCpu = "" pubnetParallelCatchupPoolMem = "" pubnetParallelCatchupCreateRbac = false + jobMonitorNodeLabels = [] + jobMonitorTolerateTaints = [] tag = None numPregeneratedTxs = None enableTailLogging = true diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index e78ac53a..db4cbef9 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -232,6 +232,20 @@ let tolerateTaintToHelmIndexed (index: int) ((key: string), (effect: string opti let effectValue = Option.defaultValue "NoSchedule" effect sprintf "worker.tolerateNodeTaints[%d].key=%s,worker.tolerateNodeTaints[%d].effect=%s" index key index effectValue +// The monitor's own placement. A plain nodeSelector map rather than the +// worker's affinity terms: its node is chosen once for the run, not per range, +// so there is nothing to express that an equality match cannot. +let monitorNodeLabelToHelm ((key: string), (value: string option)) = + sprintf "monitor.nodeSelector.%s=%s" key (Option.defaultValue "" value) + +let monitorTolerateTaintToHelmIndexed (index: int) ((key: string), (effect: string option)) = + sprintf + "monitor.tolerateNodeTaints[%d].key=%s,monitor.tolerateNodeTaints[%d].effect=%s" + index + key + index + (Option.defaultValue "NoSchedule" effect) + let serviceAccountAnnotationsToHelmIndexed (index: int) (key: string, value: string) = sprintf "service_account.annotations[%d].key=%s,service_account.annotations[%d].value=%s" index key index value @@ -388,6 +402,24 @@ let installProject (context: MissionContext) = | None -> () // Convert labels and taints to Helm array format + // Left empty the monitor lands wherever it fits, which on a cluster whose + // catchup pools are tainted means the untainted shared nodes -- alongside + // ingress and whatever else lives there. Both are needed together to move + // it: the labels alone leave it unschedulable on a tainted pool. + if not (List.isEmpty context.jobMonitorNodeLabels) then + setOptions.Add( + context.jobMonitorNodeLabels + |> List.map monitorNodeLabelToHelm + |> String.concat "," + ) + + if not (List.isEmpty context.jobMonitorTolerateTaints) then + setOptions.Add( + context.jobMonitorTolerateTaints + |> List.mapi monitorTolerateTaintToHelmIndexed + |> String.concat "," + ) + if not (List.isEmpty context.requireNodeLabelsPcV2) then let requireLabelsHelm = context.requireNodeLabelsPcV2 diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 80400d47..476c176e 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -129,6 +129,8 @@ type MissionContext = pubnetParallelCatchupPoolCpu: string pubnetParallelCatchupPoolMem: string pubnetParallelCatchupCreateRbac: bool + jobMonitorNodeLabels: ((string * string option) list) + jobMonitorTolerateTaints: ((string * string option) list) genesisTestAccountCount: int option asanOptions: string option diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 237330b9..8d59f5a6 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -191,6 +191,10 @@ spec: prometheus.io/path: "/prometheus" spec: serviceAccountName: {{ .Release.Name }}-job-monitor + {{- with .Values.monitor.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.monitor.tolerateNodeTaints }} tolerations: {{- toYaml . | nindent 8 }} diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index c738cb34..ccd9b8dc 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -278,6 +278,16 @@ monitor: peakFlushRatio: 1.05 # Poll cycles a stream gets to finalize after its pod leaves the pod list, # before it is cancelled and its connection slot reclaimed. + # Where the monitor pod runs, as native k8s fields so nothing here assumes + # tiered nodepools -- empty means anywhere untainted, which is what it does + # today and what an installation without this tiering wants. The mission fills + # them from --job-monitor-node-labels and --job-monitor-tolerate-taints. + # + # Worth knowing before pinning it to a worker tier: the monitor holds the logs + # PVC for the whole run, so the node it lands on cannot be consolidated away + # while it is there. On a small tier that reserves a node a range could have + # used. + nodeSelector: {} collectorVanishedGraceCycles: 3 collectorResources: requests: { cpu: "200m", memory: "512Mi" } From fd49f9cb7c2c7f48c6ce9209b52eb62cdab90e6a Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Tue, 11 Aug 2026 13:59:31 -0400 Subject: [PATCH 108/117] Stop values.yaml re-deriving what lib/config.py already establishes Five knobs carried the same reasoning in both places, and config.py had the fuller account each time -- 29 lines against 14 for the attempt deadline, 30 against 7 for the blocked rungs. The chart now says what the knob is and which way it fails, and leaves the derivation beside the code that reads it, where it cannot drift from the logic. Two of the removed blocks were also wrong rather than merely long: the attempt deadline explained itself in terms of "the progress ConfigMap", deleted when status moved to /status, and a sentence about cpu margin sat above a memory ceiling it did not describe. Deliberately kept: prestopSleepSeconds, createRbac, imagePullPolicy, storageSize, poolPrefix, the worker cpu block and watchTimeoutSeconds. Those have no counterpart in config.py, so the chart is their only record -- the file reads comment-heavy because most of it is the sole account of a decision, not because it repeats one. 301 -> 276 lines. --- .../parallel_catchup_helm/values.yaml | 63 ++++++------------- 1 file changed, 19 insertions(+), 44 deletions(-) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index ccd9b8dc..7bbf8ee1 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -108,13 +108,9 @@ monitor: # margin, and OOMKilled on BOTH during bucket-apply before closing a ledger. poolPrefix: "" poolTiers: "0:subdwarf,0.79:dwarf,1.61:subgiant,3.87:giant,8.85:supergiant,18.38:hypergiant,:supernova" - # Rungs that never run regardless of the vCPU comparison. Empty: with the spot - # pools doubled a promotion lands the range on a bigger SHARED node, and that - # sharing is what it buys. Measured 2026-08-04, two pods per node on one range: - # a co-tenant cost 1.02x per pod on an 8-vCPU node (r8id.2xlarge, 3.78/3.91 lps) - # against 1.58x on a 4-vCPU node. Note the bump fires on working set, which does - # NOT predict throughput -- same box, memory.max 28GiB ran 1.83 lps vs 56GiB at - # 1.70 -- so it reaches the right nodes by the wrong signal. + # Rungs the cache bump may never take, "from->to" comma separated. Nothing + # else refuses a promotion, so a rung that should not be taken has to be + # named here -- see POOL_BLOCK_RUNGS in lib/config.py. poolBlockRungs: "dwarf->subgiant,hypergiant->supernova" poolCpu: "subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80" poolMem: "subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:9216Mi" @@ -125,26 +121,19 @@ monitor: # defaults. poolNoProfile: "nebula" profileMargin: 1.15 - # No margin on cpu: it is compressible, so under-requesting costs contention - # Ceiling for profile-derived memory. Above the configured worker limit on - # purpose: a range needing more than that must be able to ask for it - # rather than be pinned under its own measured peak. + # Ceiling for profile-derived memory, deliberately above the unprofiled + # limit: a range that needs more must be able to ask for it. profileMaxMemory: "32Gi" - # Disk allowances, mirroring the memory ones. The 2026-08-01 ephemeral run - # peaked at 37.76Gi against a flat 40Gi limit -- 6% of margin, on a detection - # and escalation path that has never executed on real data. These give a - # measured range its own sizing instead: flat headroom for image/logs/WAL, - # plus a runtime-weighted share because disk tracks runtime at pearson 0.920. + # The disk allowances, mirroring the memory ones above. See + # PROFILE_EPHEMERAL_HEADROOM in lib/config.py. profileEphemeralHeadroom: "2Gi" profileRuntimeEphemeralInsurance: "8Gi" # Above the flat LIM_EPHEMERAL on purpose: that limit is what an UNMEASURED # range gets, and capping a measured one at it would discard the measurement. profileMaxEphemeral: "64Gi" - # Fixed allowance added to a range's measured rss, on top of profileMargin. - # Not zero: memory.max bounds anon PLUS page cache, and a multiplicative - # margin is meaningless at small rss. Measured with headroom 0, ranges - # profiled at 190MiB rss got a 209MiB limit and 90 of them OOMKilled within - # 90s of dispatch. + # Flat allowance added to a range's measured rss. Not zero: a multiplicative + # margin is meaningless at small rss -- see PROFILE_CACHE_HEADROOM in + # lib/config.py for the 90 OOMKills that established it. profileCacheHeadroom: "512Mi" # Extra memory allowance weighted by a range's runtime relative to the # longest valid runtime in the profile. Long-running ranges need more slack @@ -223,20 +212,10 @@ monitor: # stays readable, so a sweep that notices late still finds the medida block. # Must stay well under graceSeconds or the kubelet kills the hook. prestopSleepSeconds: 5 - # Must exceed any plausible monitor outage: completion is recorded to the - # progress ConfigMap by the monitor, and a Job reclaimed before that happens - # reads as "never ran" and gets redone. - # 0 = no deadline. Catches a range wedged in archive retries: stellar-core - # retries a missing/unreachable archive indefinitely rather than failing. - # - # 12h, not the 3h this used to be. "A prod range runs ~50 min" was the median - # talking -- runtimes span 190x. In the 2026-07-30 16320-ledger pubnet profile - # 793 ranges exceeded 3h and the longest wall time was 21488s, so 3h killed 941 - # legitimate ranges and had to be hand-patched to 12h mid-run. A timeout is - # terminal, so each of those would have failed the mission. 12h leaves ~2x the - # measured maximum. - # Flat for every range; see ATTEMPT_DEADLINE_SECONDS in job_monitor.py for why - # scaling it per-range by the profile was tried and removed. + # Per-attempt wall-clock cap, seconds. 0 = none. Catches a range wedged in + # archive retries, which stellar-core retries forever rather than failing. + # A timeout is TERMINAL, so too low fails the mission -- see + # ATTEMPT_DEADLINE_SECONDS in lib/config.py for why 12h and not 3h. attemptDeadlineSeconds: 43200 jobTtlSeconds: 600 # Worker logs are pulled to the monitor's volume while each pod is still @@ -278,15 +257,11 @@ monitor: peakFlushRatio: 1.05 # Poll cycles a stream gets to finalize after its pod leaves the pod list, # before it is cancelled and its connection slot reclaimed. - # Where the monitor pod runs, as native k8s fields so nothing here assumes - # tiered nodepools -- empty means anywhere untainted, which is what it does - # today and what an installation without this tiering wants. The mission fills - # them from --job-monitor-node-labels and --job-monitor-tolerate-taints. - # - # Worth knowing before pinning it to a worker tier: the monitor holds the logs - # PVC for the whole run, so the node it lands on cannot be consolidated away - # while it is there. On a small tier that reserves a node a range could have - # used. + # Where the monitor runs. Empty = anywhere untainted, which is what an + # installation without tiered nodepools wants. Set from + # --job-monitor-node-labels and --job-monitor-tolerate-taints. Pinning it to a + # worker tier reserves that node for the run: the monitor holds the logs PVC, + # so the node cannot be consolidated away while it is there. nodeSelector: {} collectorVanishedGraceCycles: 3 collectorResources: From 3029b808cd9dd41bf067ad5bafd45a8958744466 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Tue, 11 Aug 2026 14:03:07 -0400 Subject: [PATCH 109/117] Run fantomas, and cap values.yaml comments at three sentences The formatting check fails the build outright (exit 99) and three files had drifted: the mission, Tests.fs and Program.fs. Formatting only, no behaviour change -- same 22 F# tests, same build. values.yaml rides along rather than as its own commit, which was an accident of staging, but the two do not conflict. Nine comment blocks ran past three sentences; the long ones now lead with what the knob is and which way it fails, and keep only the measurement that pins the number -- the 90 OOMKills behind profileCacheHeadroom, the 1.03x margin behind the tier cuts, the 60s hook that still lost txApply behind prestopSleepSeconds. --- src/App/Program.fs | 3 +- src/FSLibrary.Tests/Tests.fs | 41 ++++----- .../MissionHistoryPubnetParallelCatchupV2.fs | 83 ++++++++++------- .../parallel_catchup_helm/values.yaml | 89 ++++++------------- 4 files changed, 99 insertions(+), 117 deletions(-) diff --git a/src/App/Program.fs b/src/App/Program.fs index 98629a65..ded57c98 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -994,7 +994,8 @@ let main argv = pubnetParallelCatchupPoolMem = mission.PubnetParallelCatchupPoolMem pubnetParallelCatchupCreateRbac = mission.PubnetParallelCatchupCreateRbac jobMonitorNodeLabels = List.map splitLabel (List.ofSeq mission.JobMonitorNodeLabels) - jobMonitorTolerateTaints = List.map splitLabel (List.ofSeq mission.JobMonitorTolerateTaints) + jobMonitorTolerateTaints = + List.map splitLabel (List.ofSeq mission.JobMonitorTolerateTaints) tag = mission.Tag numPregeneratedTxs = mission.NumPregeneratedTxs enableTailLogging = true diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index ac6db5d3..498eec36 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -574,8 +574,7 @@ let ``the job monitor image is overridable and defaults to the chart`` () = // An empty flag must leave the chart's pin alone. monitor.image= resolves to // ":latest" or fails the pull. let src = - System.IO.File.ReadAllText( - "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + System.IO.File.ReadAllText("../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") Assert.Contains("if context.jobMonitorImagePcV2 <> \"\" then", src) Assert.Contains("monitor.image=%s", src) @@ -591,8 +590,7 @@ let ``a pooled run does not let caller labels overwrite the routing label`` () = // 0 and the second --set wins, so the pod matches on capacity alone and lands // on any tier -- a supergiant range on a dwarf node, an OOM per range. let src = - System.IO.File.ReadAllText( - "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + System.IO.File.ReadAllText("../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") Assert.Contains("worker.requireNodeLabels[0]=purpose:%s", src) Assert.Contains("if context.pubnetParallelCatchupPoolPrefix <> \"\" then i + 1 else i", src) @@ -603,8 +601,7 @@ let ``the pool maps ride their own --set with their commas escaped`` () = // Every other option is folded into ONE comma-joined --set; these maps are // themselves comma-separated, so folding them in delivers a map of one tier. let src = - System.IO.File.ReadAllText( - "../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + System.IO.File.ReadAllText("../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") Assert.Contains("v.Replace(\",\", \"\\\\,\")", src) Assert.Contains("poolMapArgs", src) @@ -628,8 +625,12 @@ let ``the pool maps ride their own --set with their commas escaped`` () = /// Superset of rangeProfileFields: wallSeconds and txApply are recorded, never /// projected. let private measurementFields = - [ "peakAnonBytes"; "peakWorkingSetBytes"; "peakEphemeralBytes" - "txApply"; "seconds"; "wallSeconds" ] + [ "peakAnonBytes" + "peakWorkingSetBytes" + "peakEphemeralBytes" + "txApply" + "seconds" + "wallSeconds" ] /// A record as /logs/progress.json carries it. let private measuredRecord (count: int) (anon: int64) = @@ -666,29 +667,24 @@ let ``a measurement-free record cannot become a profile entry`` () = // Nothing measured -- the whole document is refused, rather than written // with the right range count and no data. let completed = - completedMap - [ "420", unmeasured (measuredRecord 420 900L) - "840", unmeasured (measuredRecord 420 950L) - "1260", unmeasured (measuredRecord 420 990L) ] + completedMap [ "420", unmeasured (measuredRecord 420 900L) + "840", unmeasured (measuredRecord 420 950L) + "1260", unmeasured (measuredRecord 420 990L) ] match rangeProfileDocument "pvc" 20000 completed with | None -> () | Some doc -> let ranges = doc.["ranges"] :?> JObject - failwithf - "wrote a profile artifact with %d ranges and zero measurements: %s" - ranges.Count - (ranges.ToString()) + failwithf "wrote a profile artifact with %d ranges and zero measurements: %s" ranges.Count (ranges.ToString()) [] let ``a measured run still produces a complete profile`` () = // The over-correction guard: a good read must still write everything. let completed = - completedMap - [ "420", measuredRecord 420 900L - "840", measuredRecord 420 950L ] + completedMap [ "420", measuredRecord 420 900L + "840", measuredRecord 420 950L ] match rangeProfileDocument "pvc" 20000 completed with | None -> failwith "refused to write a profile that carries real measurements" @@ -710,10 +706,9 @@ let ``a measured range does not drag its unmeasured neighbours in`` () = // rather than per RECORD: one that latches on the first measurement passes // both tests above while every later junk record rides in behind it. let completed = - completedMap - [ "1200", unmeasured (measuredRecord 400 900L) - "1600", measuredRecord 400 950L - "2000", unmeasured (measuredRecord 400 990L) ] + completedMap [ "1200", unmeasured (measuredRecord 400 900L) + "1600", measuredRecord 400 950L + "2000", unmeasured (measuredRecord 400 990L) ] match rangeProfileDocument "pvc" 20000 completed with | None -> failwith "refused a profile that carries one real measurement" diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index db4cbef9..6781715c 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -52,6 +52,7 @@ let extraValuesArgs = // $ dotnet run --project src/App/App.fsproj -- mission HistoryPubnetParallelCatchupV2 --image=docker-registry.services.stellar-ops.com/dev/stellar-core:23.0.3-2779.4d1df2b03.jammy-vnext-buildtests --pubnet-parallel-catchup-num-workers=2 --pubnet-parallel-catchup-starting-ledger=0 --pubnet-parallel-catchup-end-ledger=6400 --pubnet-parallel-catchup-ledgers-per-job 1280 --destination ./logs let jobMonitorLoggingIntervalSecs = 30 // frequency of the monitor reconcile loop: dispatch, liveness ping, status publish + let jobMonitorStatusCheckIntervalSecs = 60 let jobMonitorStatusCheckTimeOutSecs = 600 @@ -61,6 +62,7 @@ let jobMonitorStatusCheckTimeOutSecs = 600 // would make a single transient blip fail the run. Nothing is lost by printing // less: a range failure logs the moment it is seen, on its own line. let jobMonitorStatusLogEveryNChecks = 10 + let mutable toPerformCleanup = true let failedJobLogFileLineCount = 10000 let failedJobLogStreamLineCount = 1000 @@ -92,8 +94,14 @@ let resolveRangeProfile (context: MissionContext) : string option = // every range sized from defaults. A profile moves resource // REQUESTS only, so a tampered one costs node size, not code // execution, and it is parsed and range-counted before use. - if spec.StartsWith("http://", StringComparison.OrdinalIgnoreCase) - || spec.StartsWith("https://", StringComparison.OrdinalIgnoreCase) then + if + spec.StartsWith("http://", StringComparison.OrdinalIgnoreCase) + || spec.StartsWith + ( + "https://", + StringComparison.OrdinalIgnoreCase + ) + then use client = new HttpClient() client.Timeout <- TimeSpan.FromSeconds(30.0) client.GetStringAsync(spec) |> Async.AwaitTask |> Async.RunSynchronously @@ -148,8 +156,7 @@ let private monitorClientWith (context: MissionContext) (timeout: TimeSpan) = // Log pulls move whole files, so they get room; everything else is a short // request that should fail fast and be retried. -let private monitorClient (context: MissionContext) = - monitorClientWith context (TimeSpan.FromMinutes(10.0)) +let private monitorClient (context: MissionContext) = monitorClientWith context (TimeSpan.FromMinutes(10.0)) /// The run the monitor is asked to perform. The range travels with the profile /// because both are per-run input -- the chart installs a generic monitor, and @@ -191,6 +198,7 @@ let startMission (context: MissionContext) (runJson: string) = try use content = new StringContent(runJson, Text.Encoding.UTF8, "application/json") let r = client.PostAsync("/start", content) |> Async.AwaitTask |> Async.RunSynchronously + if r.IsSuccessStatusCode then LogInfo "Mission started: profile POSTed to %s/start" (monitorEndpoint context) started <- true @@ -299,13 +307,9 @@ let installProject (context: MissionContext) = // schedule anywhere); without the toleration they schedule nowhere. // Observed on ssc-test 2026-08-07: 10 workers Pending indefinitely, // "did not tolerate taint (taint=catchup:NoSchedule)". - setOptions.Add( - sprintf "worker.requireNodeLabels[0]=purpose:%s" context.pubnetParallelCatchupPoolPrefix - ) + setOptions.Add(sprintf "worker.requireNodeLabels[0]=purpose:%s" context.pubnetParallelCatchupPoolPrefix) - setOptions.Add( - sprintf "worker.tolerateNodeTaints[0]=%s" context.pubnetParallelCatchupPoolPrefix - ) + setOptions.Add(sprintf "worker.tolerateNodeTaints[0]=%s" context.pubnetParallelCatchupPoolPrefix) // Skip known results by default @@ -331,14 +335,10 @@ let installProject (context: MissionContext) = // Both pushed only when the run explicitly asks. Otherwise the chart default // stands, so the chart is the single place V2's worker sizing is written down. if not (String.IsNullOrWhiteSpace context.pubnetParallelCatchupCpuRequest) then - setOptions.Add( - sprintf "worker.resources.requests.cpu=%s" context.pubnetParallelCatchupCpuRequest - ) + setOptions.Add(sprintf "worker.resources.requests.cpu=%s" context.pubnetParallelCatchupCpuRequest) if not (String.IsNullOrWhiteSpace context.pubnetParallelCatchupMemRequest) then - setOptions.Add( - sprintf "worker.resources.requests.memory=%s" context.pubnetParallelCatchupMemRequest - ) + setOptions.Add(sprintf "worker.resources.requests.memory=%s" context.pubnetParallelCatchupMemRequest) // Ephemeral-storage is an ephemeral-mode concept: /data is an emptyDir on the // node there, and StellarKubeSpecs sizes it. In pvc mode /data is on the @@ -425,10 +425,11 @@ let installProject (context: MissionContext) = context.requireNodeLabelsPcV2 // From 1: a pooled run claims index 0 for the label it routes on, // and mapi from 0 would overwrite it with whichever came second. - |> List.mapi (fun i pair -> - requireNodeLabelToHelmIndexed - (if context.pubnetParallelCatchupPoolPrefix <> "" then i + 1 else i) - pair) + |> List.mapi + (fun i pair -> + requireNodeLabelToHelmIndexed + (if context.pubnetParallelCatchupPoolPrefix <> "" then i + 1 else i) + pair) |> String.concat "," setOptions.Add(requireLabelsHelm) @@ -540,21 +541,26 @@ let private monitorPodName (context: MissionContext) : string option = /// reachable from outside the cluster once its HTTPRoute is attached -- being /// walked out of LOG_DIR. let artifactFolder (name: string) = - if name.EndsWith(".log.gz") then "range-logs" - elif name.EndsWith(".metrics") then "metrics" + if name.EndsWith(".log.gz") then + "range-logs" + elif name.EndsWith(".metrics") then + "metrics" elif name.EndsWith(".done") || name.EndsWith(".started") || name.EndsWith(".state") || name = "mission_started" then "state" - else "" + else + "" let collectLogs (context: MissionContext) (destination: string) = Directory.CreateDirectory(destination) |> ignore use client = monitorClient context let manifest = - client.GetStringAsync("/logs") |> Async.AwaitTask |> Async.RunSynchronously + client.GetStringAsync("/logs") + |> Async.AwaitTask + |> Async.RunSynchronously |> JArray.Parse let fetchOne (entry: JToken) = @@ -588,9 +594,10 @@ let collectLogs (context: MissionContext) (destination: string) = // and never fetches it. Measured 2026-08-08: 88 of 110 artifacts // collected, and every .done was among the 22 missing. if appendOnly && File.Exists path && have = size then - -1L // already whole; distinct from a zero-byte file we did fetch + -1L // already whole; distinct from a zero-byte file we did fetch else let req = new HttpRequestMessage(HttpMethod.Get, "/logs/" + name) + if appendOnly && have > 0L && have < size then req.Headers.Range <- Headers.RangeHeaderValue(Nullable(have), Nullable()) @@ -612,10 +619,16 @@ let collectLogs (context: MissionContext) (destination: string) = let fetched = manifest |> Seq.toArray - |> Array.map (fun e -> async { return (try fetchOne e with ex -> - LogWarn "log fetch failed for %s: %s" - (e.["name"].ToString()) ex.Message - -1L) }) + |> Array.map + (fun e -> + async { + return + (try + fetchOne e + with ex -> + LogWarn "log fetch failed for %s: %s" (e.["name"].ToString()) ex.Message + -1L) + }) |> fun work -> Async.Parallel(work, 8) |> Async.RunSynchronously @@ -624,13 +637,17 @@ let collectLogs (context: MissionContext) (destination: string) = // empty by design, so "bytes > 0" undercounts exactly the files whose // existence IS the signal. let touched = fetched |> Array.filter (fun n -> n >= 0L) |> Array.length - LogInfo "Collected %d of %d artifacts (%d bytes) from %s" - touched (Seq.length manifest) moved (monitorEndpoint context) + + LogInfo + "Collected %d of %d artifacts (%d bytes) from %s" + touched + (Seq.length manifest) + moved + (monitorEndpoint context) /// One log pass. Idempotent -- the manifest comparison is what makes a repeat /// pass cheap, so there is no watermark to keep. -let collectLogsFromPods (context: MissionContext) = - collectLogs context context.destination.Path +let collectLogsFromPods (context: MissionContext) = collectLogs context context.destination.Path // Cleanup on exit. `signalTriggered` indicates we're running under a hard // deadline (Jenkins' SoftKillWaitSeconds, ~5s by default, before SIGKILL). diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 7bbf8ee1..29fbe503 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -9,13 +9,9 @@ worker: # Sized from what a range actually uses: the old node-local /data ran on a # 35Gi request / 40Gi limit. 40Gi x 1024 workers is 40 TiB of gp3 rather than # 100 TiB, which matters against the per-region storage quota. - # One PVC per ledger range, not per concurrency slot. Measured on ssc-test: - # 300 jobs each with their own PVC took 3151s vs 3210s reusing 40 -- 7.5x the - # volume lifecycles for no measurable cost and no CSI throttling. Reuse only - # bought bookkeeping, plus pinning every later range to the AZ the slot's - # first volume happened to land in. - # 60Gi to match the tier nodes' ephemeral allowance: peakEphemeralBytes tops - # out at 37.8Gi across the whole profile, so this covers 100% with headroom. + # One PVC per ledger range, not per concurrency slot: measured on ssc-test, + # 7.5x the volume lifecycles cost nothing (3151s vs 3210s) and reuse pinned + # every later range to one AZ. 60Gi covers the profile's 37.8Gi peak. storageSize: "60Gi" requireNodeLabels: [] avoidNodeLabels: [] @@ -28,11 +24,9 @@ worker: # a different execution model), and sizing V2 there silently resized them # too -- it took their cpu request from 250m to 1800m. # - # 1800m/9Gi is the unprofiled default: on r8*.2xlarge (7.91 cpu, 61.7Gi - # allocatable) that lands 4 workers per node, cpu-bound; on m8*.2xlarge - # (7.91 cpu, 29.7Gi) it lands 3, memory-bound. A range the profile has - # measured overrides both. --pubnet-parallel-catchup-cpu-request overrides - # the cpu for a whole run. + # 1800m/9Gi is the unprofiled default: 4 workers per r8*.2xlarge node + # (cpu-bound) or 3 per m8*.2xlarge (memory-bound). A measured range + # overrides both; --pubnet-parallel-catchup-cpu-request overrides the run. cpu: "1800m" memory: "9Gi" # Still filled in by the mission: ephemeral-storage is 2Gi/4Gi in pvc mode @@ -63,11 +57,9 @@ monitor: # Reuses the existing hand-built job-monitor image slot -- no new image and no # new push path (nothing in .github/workflows builds these). # - # TEMPORARY dev pin, 2026-08-07: stellar/ssc-job-monitor:latest predates the - # apps/+lib/ split, ATTEMPT_BUDGETS and per-cause budget counting, so running - # this chart against it ships env vars the image cannot read. Built from this - # branch and pinned by tag rather than :latest so a run is always traceable to - # one image. Revert to the stellar/ repo once there is a push path for it. + # TEMPORARY dev pin: stellar/ssc-job-monitor:latest predates the apps/+lib/ + # split and ATTEMPT_BUDGETS, so this chart would ship env vars it cannot read. + # Revert to the stellar/ repo once there is a push path for it. image: "stellajuna/ssc-jm:2026-08-11b" # The driver reaches the monitor through this route: profile in via POST # /start, status and logs out. Empty routeHost disables the HTTPRoute; the @@ -80,9 +72,9 @@ monitor: # privilege every mission would then carry so that this one chart can install # its own Role. # - # true creates the Role and RoleBinding for a cluster where that provisioning - # does not exist -- a k3d/kind run, say. The ServiceAccount is created either - # way; creating one grants nothing. + # true creates the Role, RoleBinding and node-stats ClusterRole for a cluster + # that does not provide them -- ssc-eks does, through a catchup-job-monitor + # RoleBinding, so leave it false there. With neither, the monitor 403s. createRbac: false routeHost: "" gatewayName: "" @@ -101,11 +93,9 @@ monitor: # itself. Empty prefix disables all of it and every worker keeps the single # global node label, which is exactly the pre-tier behaviour. # - # Cuts are node_usable/1.60, covering the p99 of run-to-run growth in the same - # range's peakAnonBytes (18,073 observations across five profiles: p50 0.97, - # p90 1.28, p99 1.60, max 2.83). Learned the hard way -- range 63080767 - # measured 13.75Gi went onto nodes with 14.1/14.3Gi allocatable, a 1.03x - # margin, and OOMKilled on BOTH during bucket-apply before closing a ledger. + # Cuts are node_usable/1.60, the p99 of run-to-run growth in the same range's + # peakAnonBytes (18,073 observations). Range 63080767 measured 13.75Gi, landed + # on 14.1Gi nodes at a 1.03x margin, and OOMKilled on both. poolPrefix: "" poolTiers: "0:subdwarf,0.79:dwarf,1.61:subgiant,3.87:giant,8.85:supergiant,18.38:hypergiant,:supernova" # Rungs the cache bump may never take, "from->to" comma separated. Nothing @@ -139,18 +129,10 @@ monitor: # longest valid runtime in the profile. Long-running ranges need more slack # for page cache and allocator growth. 0 disables. profileRuntimeMemoryInsurance: "3Gi" - # Always, because runs name this image by a MUTABLE tag: the mission passes - # --job-monitor-image-pc-v2 stellajuna/ssc-jm:latest, and IfNotPresent then - # pins a node to whatever it cached the first time it saw that tag. Worker - # nodes churn under Karpenter so they would repull anyway, but the monitor - # carries no tolerations and lands on the long-lived default pool, where the - # layer persists for good -- run #183 shipped a job_monitor.log of 0 bytes - # from an image whose fix for exactly that had been pushed hours earlier. - # - # Kubernetes defaults :latest to Always for this reason; the explicit - # IfNotPresent was overriding it. Costs one pull per run for one pod, and - # makes a registry outage fail the run at start rather than run something - # stale. + # Always, because runs name this image by a MUTABLE tag and IfNotPresent then + # pins a node to whatever it cached first -- run #183 shipped a 0-byte + # job_monitor.log from an image fixed hours earlier, off a node up since June. + # Kubernetes defaults :latest to Always for this reason. imagePullPolicy: Always mission: "HistoryPubnetParallelCatchup" # Adds a `mission` label to worker pods, which kube-state-metrics exposes as @@ -196,25 +178,14 @@ monitor: maxEphemeral: "200Gi" maxMem: "48Gi" graceSeconds: 100 - # preStop stall before SIGTERM, in seconds. 0 is off. - # - # Buys the collector time to NOTICE the disruption before the kill; it is not - # what captures the metric. The hook cannot widen the ~4ms between the medida - # block and the process exiting -- measured, a 60s hook with 10s polling and - # no detection still lost txApply, while 1s polling with no hook captured it. - # What it prevents is SIGTERM landing while the poller is still on its lazy - # cadence because the pod-list cycle has not come round yet. - # - # Sized against the collector CYCLE (sleep + pod list + kubelet sweep), not - # against collectorPollSeconds. Overshooting is NOT free on spot: the hook is - # dead time inside the ~120s AWS reclaim budget. Undershooting the cycle is - # survivable because the pod lives out graceSeconds after SIGTERM and its log - # stays readable, so a sweep that notices late still finds the medida block. - # Must stay well under graceSeconds or the kubelet kills the hook. + # preStop stall before SIGTERM in seconds, 0 = off, sized against the + # collector CYCLE rather than collectorPollSeconds. It buys time to NOTICE the + # disruption, not to capture the metric -- a 60s hook with 10s polling still + # lost txApply where 1s polling with no hook caught it. Overshooting is dead + # time inside the ~120s spot reclaim budget; it must stay under graceSeconds. prestopSleepSeconds: 5 - # Per-attempt wall-clock cap, seconds. 0 = none. Catches a range wedged in - # archive retries, which stellar-core retries forever rather than failing. - # A timeout is TERMINAL, so too low fails the mission -- see + # Per-attempt wall-clock cap in seconds, 0 = none, catching a range wedged in + # archive retries. A timeout is TERMINAL, so too low fails the mission; see # ATTEMPT_DEADLINE_SECONDS in lib/config.py for why 12h and not 3h. attemptDeadlineSeconds: 43200 jobTtlSeconds: 600 @@ -257,11 +228,9 @@ monitor: peakFlushRatio: 1.05 # Poll cycles a stream gets to finalize after its pod leaves the pod list, # before it is cancelled and its connection slot reclaimed. - # Where the monitor runs. Empty = anywhere untainted, which is what an - # installation without tiered nodepools wants. Set from - # --job-monitor-node-labels and --job-monitor-tolerate-taints. Pinning it to a - # worker tier reserves that node for the run: the monitor holds the logs PVC, - # so the node cannot be consolidated away while it is there. + # Where the monitor runs, from --job-monitor-node-labels and + # --job-monitor-tolerate-taints. Empty = anywhere untainted. Pinning it to a + # worker tier reserves that node: it holds the logs PVC for the whole run. nodeSelector: {} collectorVanishedGraceCycles: 3 collectorResources: From 07616f7c398011464bfb96de88a767fdcb966bf4 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Tue, 11 Aug 2026 14:31:10 -0400 Subject: [PATCH 110/117] Put main's new tests back inside the Tests class The merge appended them after this branch's module-level V2 tests, so five `member __.` definitions ended up outside the type they belong to. F# read the first `[]` at that indent as a syntax error, which is why fantomas could not parse the file rather than merely wanting to reformat it -- the build was failing too. Moved verbatim to the end of the class body, ahead of the V2 section. The suite goes 22 -> 34: main's twelve were not being run at all, since a member outside its type is not a member of anything. --- src/FSLibrary.Tests/Tests.fs | 124 ++++++++++++++++++----------------- 1 file changed, 63 insertions(+), 61 deletions(-) diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index 0c2b6662..fd61f761 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -649,6 +649,69 @@ type Tests(output: ITestOutputHelper) = Assert.Equal("56/6", jobArr3.[1].[1]) Assert.Equal("61/6", jobArr3.[2].[1]) + [] + member __.``ParseQuorumIntersectionInfo handles intersecting result``() = + let json = """{ "node": "GAAA", "qset": {}, + "transitive": { "intersection": true, "node_count": 6, + "last_check_ledger": 12, + "critical": [["GBBB"], ["GCCC", "GDDD"]] } }""" + + match ParseQuorumIntersectionInfo json with + | None -> failwith "expected Some" + | Some qi -> + Assert.True(qi.intersection) + Assert.Equal(6, qi.nodeCount) + Assert.Equal(12, qi.lastCheckLedger) + Assert.Equal list>([ Set.ofList [ "GBBB" ]; Set.ofList [ "GCCC"; "GDDD" ] ], qi.criticalGroups) + Assert.True(qi.potentialSplit.IsNone) + + [] + member __.``ParseQuorumIntersectionInfo handles split result``() = + let json = """{ "node": "GAAA", "qset": {}, + "transitive": { "intersection": false, "node_count": 6, + "last_check_ledger": 20, "last_good_ledger": 15, + "potential_split": [["GBBB", "GCCC"], ["GDDD"]] } }""" + + match ParseQuorumIntersectionInfo json with + | None -> failwith "expected Some" + | Some qi -> + Assert.False(qi.intersection) + Assert.Equal list>([], qi.criticalGroups) + + match qi.potentialSplit with + | Some (a, b) -> + Assert.Equal>(Set.ofList [ "GBBB"; "GCCC" ], a) + Assert.Equal>(Set.ofList [ "GDDD" ], b) + | None -> failwith "expected potential_split" + + [] + member __.``ParseQuorumIntersectionInfo returns None without results``() = + Assert.True((ParseQuorumIntersectionInfo """{ "node": "GAAA", "qset": {} }""").IsNone) + + let json = """{ "transitive": { "intersection": true, "node_count": 3, + "last_check_ledger": 5, "critical": null } }""" + + match ParseQuorumIntersectionInfo json with + | Some qi -> Assert.Equal list>([], qi.criticalGroups) + | None -> failwith "expected Some" + + [] + member __.``ParseMetricCount reads counter or defaults to zero``() = + let json = """{ "metrics": { "scp.qic.successful-run": { "type": "counter", "count": 3 }, + "scp.qic.result-potential-split": { "type": "counter", "count": 1 }, + "scp.qic.no-count": { "type": "counter" } } }""" + + Assert.Equal(3, ParseMetricCount json "scp.qic.successful-run") + Assert.Equal(1, ParseMetricCount json "scp.qic.result-potential-split") + Assert.Equal(0, ParseMetricCount json "scp.qic.no-count") + Assert.Equal(0, ParseMetricCount json "scp.qic.failed-run") + Assert.Equal(0, ParseMetricCount """{ }""" "scp.qic.failed-run") + + [] + member __.``QuorumIntersectionChecker mission is registered``() = + Assert.True(StellarMission.allMissions.ContainsKey "QuorumIntersectionChecker") + + // --------------------------------------------------------------------------- // MissionHistoryPubnetParallelCatchupV2 // --------------------------------------------------------------------------- @@ -801,64 +864,3 @@ let ``a measured range does not drag its unmeasured neighbours in`` () = Assert.NotNull(ranges.["1600"]) Assert.Null(ranges.["1200"]) Assert.Null(ranges.["2000"]) - [] - member __.``ParseQuorumIntersectionInfo handles intersecting result``() = - let json = """{ "node": "GAAA", "qset": {}, - "transitive": { "intersection": true, "node_count": 6, - "last_check_ledger": 12, - "critical": [["GBBB"], ["GCCC", "GDDD"]] } }""" - - match ParseQuorumIntersectionInfo json with - | None -> failwith "expected Some" - | Some qi -> - Assert.True(qi.intersection) - Assert.Equal(6, qi.nodeCount) - Assert.Equal(12, qi.lastCheckLedger) - Assert.Equal list>([ Set.ofList [ "GBBB" ]; Set.ofList [ "GCCC"; "GDDD" ] ], qi.criticalGroups) - Assert.True(qi.potentialSplit.IsNone) - - [] - member __.``ParseQuorumIntersectionInfo handles split result``() = - let json = """{ "node": "GAAA", "qset": {}, - "transitive": { "intersection": false, "node_count": 6, - "last_check_ledger": 20, "last_good_ledger": 15, - "potential_split": [["GBBB", "GCCC"], ["GDDD"]] } }""" - - match ParseQuorumIntersectionInfo json with - | None -> failwith "expected Some" - | Some qi -> - Assert.False(qi.intersection) - Assert.Equal list>([], qi.criticalGroups) - - match qi.potentialSplit with - | Some (a, b) -> - Assert.Equal>(Set.ofList [ "GBBB"; "GCCC" ], a) - Assert.Equal>(Set.ofList [ "GDDD" ], b) - | None -> failwith "expected potential_split" - - [] - member __.``ParseQuorumIntersectionInfo returns None without results``() = - Assert.True((ParseQuorumIntersectionInfo """{ "node": "GAAA", "qset": {} }""").IsNone) - - let json = """{ "transitive": { "intersection": true, "node_count": 3, - "last_check_ledger": 5, "critical": null } }""" - - match ParseQuorumIntersectionInfo json with - | Some qi -> Assert.Equal list>([], qi.criticalGroups) - | None -> failwith "expected Some" - - [] - member __.``ParseMetricCount reads counter or defaults to zero``() = - let json = """{ "metrics": { "scp.qic.successful-run": { "type": "counter", "count": 3 }, - "scp.qic.result-potential-split": { "type": "counter", "count": 1 }, - "scp.qic.no-count": { "type": "counter" } } }""" - - Assert.Equal(3, ParseMetricCount json "scp.qic.successful-run") - Assert.Equal(1, ParseMetricCount json "scp.qic.result-potential-split") - Assert.Equal(0, ParseMetricCount json "scp.qic.no-count") - Assert.Equal(0, ParseMetricCount json "scp.qic.failed-run") - Assert.Equal(0, ParseMetricCount """{ }""" "scp.qic.failed-run") - - [] - member __.``QuorumIntersectionChecker mission is registered``() = - Assert.True(StellarMission.allMissions.ContainsKey "QuorumIntersectionChecker") From 3b8ec9c044baab86e7ad4461ccf28bbf1e163ef0 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Fri, 14 Aug 2026 15:49:19 -0400 Subject: [PATCH 111/117] refactor(parallel-catchup): file lib/ by owner, split config three ways - lib/ splits: shared (config, logger, records), lib/monitor/, lib/collector/ - config.py 555 -> 53 lines; 60 monitor-only names to monitor_config - collector settings to collector_config; log_collector has no os.getenv left - new lib/collector: kube_http, verdicts, tx_scan - log_collector opens on main(); all runtime state above it - delete load_profile()/PROFILE_PATH: no production caller, /start carries the profile; its cross-mode strip was inert (pvc sets no ephemeral limit) - delete base()/done_path(): duplicated records.py, which both processes import - Dockerfile flattens each lib dir explicitly -- a directory COPY would leave subdirectories in /app and break bare-name imports Co-Authored-By: Claude Opus 5 --- .../Dockerfile.jobmonitor | 7 +- .../apps/job_monitor.py | 131 +- .../apps/log_collector.py | 1216 ++++++----------- .../lib/collector/collector_config.py | 82 ++ .../lib/collector/kube_http.py | 27 + .../lib/{ => collector}/medida.py | 0 .../lib/collector/tx_scan.py | 107 ++ .../lib/collector/verdicts.py | 72 + src/MissionParallelCatchup/lib/config.py | 549 +------- .../lib/{ => monitor}/attempts.py | 0 .../lib/{ => monitor}/http_server.py | 0 .../lib/{ => monitor}/kube.py | 3 +- .../lib/{ => monitor}/metrics.py | 0 .../lib/monitor/monitor_config.py | 539 ++++++++ .../lib/monitor/profiles.py | 37 + .../lib/{ => monitor}/ranges.py | 17 +- .../lib/{ => monitor}/sizing.py | 67 +- .../lib/{ => monitor}/units.py | 0 .../lib/{ => monitor}/worker_liveness.py | 7 +- src/MissionParallelCatchup/lib/profiles.py | 81 -- 20 files changed, 1416 insertions(+), 1526 deletions(-) create mode 100644 src/MissionParallelCatchup/lib/collector/collector_config.py create mode 100644 src/MissionParallelCatchup/lib/collector/kube_http.py rename src/MissionParallelCatchup/lib/{ => collector}/medida.py (100%) create mode 100644 src/MissionParallelCatchup/lib/collector/tx_scan.py create mode 100644 src/MissionParallelCatchup/lib/collector/verdicts.py rename src/MissionParallelCatchup/lib/{ => monitor}/attempts.py (100%) rename src/MissionParallelCatchup/lib/{ => monitor}/http_server.py (100%) rename src/MissionParallelCatchup/lib/{ => monitor}/kube.py (93%) rename src/MissionParallelCatchup/lib/{ => monitor}/metrics.py (100%) create mode 100644 src/MissionParallelCatchup/lib/monitor/monitor_config.py create mode 100644 src/MissionParallelCatchup/lib/monitor/profiles.py rename src/MissionParallelCatchup/lib/{ => monitor}/ranges.py (87%) rename src/MissionParallelCatchup/lib/{ => monitor}/sizing.py (89%) rename src/MissionParallelCatchup/lib/{ => monitor}/units.py (100%) rename src/MissionParallelCatchup/lib/{ => monitor}/worker_liveness.py (94%) delete mode 100644 src/MissionParallelCatchup/lib/profiles.py diff --git a/src/MissionParallelCatchup/Dockerfile.jobmonitor b/src/MissionParallelCatchup/Dockerfile.jobmonitor index 5b68b2af..03a9a2ad 100644 --- a/src/MissionParallelCatchup/Dockerfile.jobmonitor +++ b/src/MissionParallelCatchup/Dockerfile.jobmonitor @@ -29,7 +29,12 @@ RUN pip install --no-cache-dir \ COPY ./apps/job_monitor.py /app # Same image, second entrypoint: runs as a sidecar streaming worker logs. COPY ./apps/log_collector.py /app -COPY ./lib/ /app/ +# Flattened one directory at a time: `COPY ./lib/ /app/` would preserve +# lib/monitor and lib/collector as subdirectories, and a bare `import sizing` +# does not look inside them. +COPY ./lib/*.py /app/ +COPY ./lib/monitor/ /app/ +COPY ./lib/collector/ /app/ EXPOSE 8080 diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py index 4cf8ce2a..e774978d 100644 --- a/src/MissionParallelCatchup/apps/job_monitor.py +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -40,6 +40,7 @@ import attempts import config +import monitor_config as mc import http_server import kube import metrics @@ -61,11 +62,11 @@ def main(): # The driver POSTs the profile to /start. Kept on the volume so a restarted # monitor resumes a run already under way instead of waiting for a /start # that was delivered to its predecessor. - config.RUN_PATH = os.path.join(config.LOG_DIR, 'run.json') - if os.path.exists(config.RUN_PATH): + mc.RUN_PATH = os.path.join(config.LOG_DIR, 'run.json') + if os.path.exists(mc.RUN_PATH): # Same path as a /start, so the range and the profile are both restored # and validated exactly as they were. - with open(config.RUN_PATH) as fh: + with open(mc.RUN_PATH) as fh: start_run(json.load(fh)) http_server.status_source = lambda: (status, status_lock) @@ -93,20 +94,20 @@ def start_run(doc): ('ledgersPerJob', 'LEDGERS_PER_JOB'), ('overlapLedgers', 'OVERLAP_LEDGERS')): if key in (doc.get('range') or {}): - setattr(config, name, (doc['range'])[key]) + setattr(mc, name, (doc['range'])[key]) profile = profiles.load_profile_doc(doc.get('profile') or {}) # The whole config, judged at the first moment it is complete. Anything # wrong rejects the POST with the reason rather than dispatching a run that # is already misconfigured. validate_config() - if config.RANGE_ORDER == 'longest-first' and not profile: + if mc.RANGE_ORDER == 'longest-first' and not profile: raise ValueError( "RANGE_ORDER=longest-first requires a profile: it orders ranges by " "their measured seconds, and with no profile every range ties and " "dispatch stays tip-first. POST a profile, or set RANGE_ORDER.") - records.write_atomic(config.RUN_PATH, json.dumps(doc, separators=(',', ':'))) - config.PROFILE = profile - logger.info("profile installed: %d ranges", len(config.PROFILE)) + records.write_atomic(mc.RUN_PATH, json.dumps(doc, separators=(',', ':'))) + mc.PROFILE = profile + logger.info("profile installed: %d ranges", len(mc.PROFILE)) # Opened here rather than in the POST handler, so a restart that reads # run.json back resumes on exactly the same path. It did not, and a # restarted monitor blocked on this forever while /status kept answering @@ -132,7 +133,7 @@ def validate_config(): string form. """ for name, cast in _LIVENESS_NUMBERS: - raw = getattr(config, name) + raw = getattr(mc, name) try: value = cast(raw) except (TypeError, ValueError): @@ -141,35 +142,35 @@ def validate_config(): "numbers; LIVENESS_MAX_CONCURRENCY must be an integer") from None if value <= 0: raise ValueError(f"{name} must be greater than zero, got {raw!r}") - setattr(config, name, value) + setattr(mc, name, value) - if config.RANGE_ORDER not in config.VALID_RANGE_ORDERS: + if mc.RANGE_ORDER not in mc.VALID_RANGE_ORDERS: raise ValueError("RANGE_ORDER must be one of %s, got %r" - % (', '.join(config.VALID_RANGE_ORDERS), config.RANGE_ORDER)) + % (', '.join(mc.VALID_RANGE_ORDERS), mc.RANGE_ORDER)) # The ledger range, which nothing checked while it came from helm values -- # an inverted or zero-width range generates no work and the run just ends, # reporting success on nothing. - if config.LEDGERS_PER_JOB <= 0: + if mc.LEDGERS_PER_JOB <= 0: raise ValueError("ledgersPerJob must be greater than zero, got %r" - % (config.LEDGERS_PER_JOB,)) - if config.OVERLAP_LEDGERS < 0: + % (mc.LEDGERS_PER_JOB,)) + if mc.OVERLAP_LEDGERS < 0: raise ValueError("overlapLedgers cannot be negative, got %r" - % (config.OVERLAP_LEDGERS,)) - if config.LATEST_LEDGER_NUM <= config.STARTING_LEDGER: + % (mc.OVERLAP_LEDGERS,)) + if mc.LATEST_LEDGER_NUM <= mc.STARTING_LEDGER: raise ValueError( "latestLedgerNum must be greater than startingLedger, got %r and %r" - % (config.LATEST_LEDGER_NUM, config.STARTING_LEDGER)) + % (mc.LATEST_LEDGER_NUM, mc.STARTING_LEDGER)) # The pool maps arrive per run, so this is the first point they meet the # ladder they are keyed to. A tier with no claim does not fail -- the pod # keeps the flat REQ_CPU/REQ_MEM and a second one fits beside it, which # undoes the isolation the whole tiering exists for: giving a pod its node # to itself raised throughput 29-92%. Silent, and only visible afterwards as # a run that cost more than it should. - if config.POOL_PREFIX: + if mc.POOL_PREFIX: routable = [name for _, name in sizing._parsed_pool_tiers()] - routable += [config.POOL_UNPROFILED, config.POOL_NO_PROFILE] - for env_name, raw in (('POOL_CPU', config.POOL_CPU), ('POOL_MEM', config.POOL_MEM)): - claimed = {k for k, _ in config.label_pairs(raw)} + routable += [mc.POOL_UNPROFILED, mc.POOL_NO_PROFILE] + for env_name, raw in (('POOL_CPU', mc.POOL_CPU), ('POOL_MEM', mc.POOL_MEM)): + claimed = {k for k, _ in mc.label_pairs(raw)} missing = [t for t in routable if t and t not in claimed] if missing: raise ValueError( @@ -259,7 +260,7 @@ def reconcile_loop(): except Exception as e: logger.exception("Error while reconciling: %s", str(e)) - time.sleep(config.RECONCILE_INTERVAL_SECONDS) + time.sleep(mc.RECONCILE_INTERVAL_SECONDS) def reconcile(state): @@ -426,7 +427,7 @@ def reconcile(state): created = 0 # No slots: a range's PVC is keyed by the range itself, so concurrency is # simply how many are in flight. - capacity = config.PARALLELISM - len(in_progress) + capacity = mc.PARALLELISM - len(in_progress) for end, count in desired: if capacity <= 0: break @@ -498,7 +499,7 @@ def load_progress(): closed ledger. """ try: - with open(config.PROGRESS_FILE) as fh: + with open(mc.PROGRESS_FILE) as fh: return json.load(fh) except (OSError, ValueError): return {} @@ -508,7 +509,7 @@ def save_progress(progress): # The monitor's own state, and the only copy. The driver's view of the run # is status.json in the ConfigMap; this document is not published. blob = json.dumps(progress, separators=(',', ':')) - records.write_atomic(config.PROGRESS_FILE, blob) + records.write_atomic(mc.PROGRESS_FILE, blob) @@ -904,9 +905,9 @@ def ensure_pvc(end, owner): raise spec = client.V1PersistentVolumeClaimSpec( access_modes=['ReadWriteOnce'], - resources=client.V1VolumeResourceRequirements(requests={'storage': config.STORAGE_SIZE})) - if config.STORAGE_CLASS: - spec.storage_class_name = config.STORAGE_CLASS + resources=client.V1VolumeResourceRequirements(requests={'storage': mc.STORAGE_SIZE})) + if mc.STORAGE_CLASS: + spec.storage_class_name = mc.STORAGE_CLASS kube.core_v1.create_namespaced_persistent_volume_claim(config.NAMESPACE, client.V1PersistentVolumeClaim( metadata=client.V1ObjectMeta(name=name, owner_references=owner, labels={config.LABEL_RUN: config.RUN_NAME, config.LABEL_RANGE: str(end)}), @@ -920,7 +921,7 @@ def _resources(mem=None, eph=None, end=None, attempt=1): overrides = sizing._profile_overrides(end, escalated=(mem is not None or eph is not None), attempt=attempt) # `mem` is the escalated request on an OOM retry, else the configured one. - req = {'cpu': config.REQ_CPU, 'memory': mem or config.REQ_MEM} + req = {'cpu': mc.REQ_CPU, 'memory': mem or mc.REQ_MEM} # Only ephemeral-storage is limited: it is the one dimension where an # unbounded pod takes the node down rather than itself. lim = {} @@ -935,27 +936,27 @@ def _resources(mem=None, eph=None, end=None, attempt=1): # # Unprofiled pooled runs keep both: nothing measured them, so there is no # peak to have been generous about. - pooled_profiled = bool(config.POOL_PREFIX and config.PROFILE) + pooled_profiled = bool(mc.POOL_PREFIX and mc.PROFILE) # Only meaningful in ephemeral mode. In PVC mode a large request makes disk # the binding dimension and halves workers-per-node for no reason. - if config.REQ_EPHEMERAL and not pooled_profiled: + if mc.REQ_EPHEMERAL and not pooled_profiled: # Raise the request with the limit: ephemeral-storage is a scheduling # dimension, so a pod that outgrew it no longer fits where it was. - req['ephemeral-storage'] = eph or config.REQ_EPHEMERAL + req['ephemeral-storage'] = eph or mc.REQ_EPHEMERAL else: # pvc mode: /data is not on the node disk, so an ephemeral override # would size a dimension this run does not use. Pooled+profiled: the # pod owns its node and the axis is dropped deliberately. overrides.pop('ephemeral-storage', None) - if config.LIM_EPHEMERAL and not pooled_profiled: - lim['ephemeral-storage'] = eph or config.LIM_EPHEMERAL + if mc.LIM_EPHEMERAL and not pooled_profiled: + lim['ephemeral-storage'] = eph or mc.LIM_EPHEMERAL # The profile moves requests only. Disk excepted, because its limit is what # the kubelet enforces. for key, value in overrides.items(): req[key] = value - if key == 'ephemeral-storage' and config.LIM_EPHEMERAL: + if key == 'ephemeral-storage' and mc.LIM_EPHEMERAL: lim[key] = value # Unmeasured range: the configured requests, exactly as if there were no # profile at all. @@ -972,8 +973,8 @@ def pod_labels(end, attempt): """ labels = {config.LABEL_RUN: config.RUN_NAME, config.LABEL_RANGE: str(end), config.LABEL_ATTEMPT: str(attempt)} - if config.EMIT_MISSION_LABEL and config.MISSION: - labels['mission'] = config.MISSION + if mc.EMIT_MISSION_LABEL and mc.MISSION: + labels['mission'] = mc.MISSION return labels @@ -990,18 +991,18 @@ def _prestop_delay(): anyway -- so the delay is not bought and an error is logged for every evicted pod. """ - if config.WORKER_PRESTOP_SLEEP_SECONDS <= 0: + if mc.WORKER_PRESTOP_SLEEP_SECONDS <= 0: return None - if config.WORKER_PRESTOP_SLEEP_SECONDS >= config.WORKER_GRACE_SECONDS: + if mc.WORKER_PRESTOP_SLEEP_SECONDS >= mc.WORKER_GRACE_SECONDS: logger.warning( "PRESTOP_SLEEP_SECONDS=%s does not fit in GRACE_SECONDS=%s; " "not installing a preStop hook that the kubelet would kill", - config.WORKER_PRESTOP_SLEEP_SECONDS, config.WORKER_GRACE_SECONDS) + mc.WORKER_PRESTOP_SLEEP_SECONDS, mc.WORKER_GRACE_SECONDS) return None return client.V1Lifecycle( pre_stop=client.V1LifecycleHandler( _exec=client.V1ExecAction( - command=['/bin/sleep', str(config.WORKER_PRESTOP_SLEEP_SECONDS)]))) + command=['/bin/sleep', str(mc.WORKER_PRESTOP_SLEEP_SECONDS)]))) def build_job(end, count, attempt, owner, mem=None, eph=None): @@ -1014,7 +1015,7 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): else: data_vol = client.V1Volume(name='data', empty_dir=client.V1EmptyDirVolumeSource()) - env = [client.V1EnvVar(name='ASAN_OPTIONS', value=config.ASAN_OPTIONS)] if config.ASAN_OPTIONS else [] + env = [client.V1EnvVar(name='ASAN_OPTIONS', value=mc.ASAN_OPTIONS)] if mc.ASAN_OPTIONS else [] command = ['/bin/sh', '-c', script] volumes = [data_vol, client.V1Volume( name='config', config_map=client.V1ConfigMapVolumeSource( @@ -1027,26 +1028,26 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): # term are ANDed, separate terms are ORed, and an avoid-only pod in its own # term would match every node. match = [] - if config.NODE_LABEL_KEY: + if mc.NODE_LABEL_KEY: # Pooled runs route per range: the label names the tier this range's # memory puts it in. An escalated attempt resolves to a promoted tier, # which is what moves the pod to nodes its memory fits. tier = sizing.pool_for(end, attempt) - value = f"{config.POOL_PREFIX}-{tier}" if tier else config.NODE_LABEL_VALUE + value = f"{mc.POOL_PREFIX}-{tier}" if tier else mc.NODE_LABEL_VALUE match.append(client.V1NodeSelectorRequirement( - key=config.NODE_LABEL_KEY, operator='In', values=[value])) - for key, value in config.label_pairs(config.REQUIRE_NODE_LABELS): + key=mc.NODE_LABEL_KEY, operator='In', values=[value])) + for key, value in mc.label_pairs(mc.REQUIRE_NODE_LABELS): # Literal, unlike the pool-routed pair above: these are properties of # the pool rather than of the range, so they do not vary per attempt. match.append(client.V1NodeSelectorRequirement( key=key, operator='In', values=[value])) - if config.AVOID_NODE_LABEL_KEY: + if mc.AVOID_NODE_LABEL_KEY: # No value means "avoid the label however it is set", which is # DoesNotExist; NotIn [""] would only exclude the empty value. match.append(client.V1NodeSelectorRequirement( - key=config.AVOID_NODE_LABEL_KEY, - operator='NotIn' if config.AVOID_NODE_LABEL_VALUE else 'DoesNotExist', - values=[config.AVOID_NODE_LABEL_VALUE] if config.AVOID_NODE_LABEL_VALUE else None)) + key=mc.AVOID_NODE_LABEL_KEY, + operator='NotIn' if mc.AVOID_NODE_LABEL_VALUE else 'DoesNotExist', + values=[mc.AVOID_NODE_LABEL_VALUE] if mc.AVOID_NODE_LABEL_VALUE else None)) affinity = None if match: affinity = client.V1Affinity(node_affinity=client.V1NodeAffinity( @@ -1055,10 +1056,10 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): # Taint value must be absent: the mission emits {key, effect} with no value, # and the default Equal operator does not match "" against "true". - tolerations = [client.V1Toleration(key=config.TOLERATE_TAINT, effect='NoSchedule')] if config.TOLERATE_TAINT else None + tolerations = [client.V1Toleration(key=mc.TOLERATE_TAINT, effect='NoSchedule')] if mc.TOLERATE_TAINT else None container = client.V1Container( - name='stellar-core', image=config.CORE_IMAGE, + name='stellar-core', image=mc.CORE_IMAGE, command=command, env=env, resources=_resources(mem, eph, end, attempt), ports=[client.V1ContainerPort(container_port=11626, name='http')], lifecycle=_prestop_delay(), @@ -1077,11 +1078,11 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): # On the JobSpec, not the pod: a pod-level deadline is immutable once # the pod exists, so a mis-set value could not be corrected on a live # run. - active_deadline_seconds=config.ATTEMPT_DEADLINE_SECONDS or None, + active_deadline_seconds=mc.ATTEMPT_DEADLINE_SECONDS or None, backoff_limit=0, pod_failure_policy=client.V1PodFailurePolicy( rules=[r for _, r in _failure_rules()]), - ttl_seconds_after_finished=config.JOB_TTL_SECONDS, + ttl_seconds_after_finished=mc.JOB_TTL_SECONDS, template=client.V1PodTemplateSpec( metadata=client.V1ObjectMeta(labels=pod_labels(end, attempt)), spec=client.V1PodSpec( @@ -1091,11 +1092,11 @@ def build_job(end, count, attempt, owner, mem=None, eph=None): # pod-level field starts at container start. # IRSA for the S3 history mirror; without it workers fall # back to the public archive, which throttles at 1024. - service_account_name=config.WORKER_SERVICE_ACCOUNT or None, + service_account_name=mc.WORKER_SERVICE_ACCOUNT or None, # Never restarted in place: the pod stays terminal and # inspectable for classification and the backstop log read. restart_policy='Never', - termination_grace_period_seconds=config.WORKER_GRACE_SECONDS, + termination_grace_period_seconds=mc.WORKER_GRACE_SECONDS, affinity=affinity, tolerations=tolerations, containers=[container], volumes=volumes)))) @@ -1362,7 +1363,7 @@ def verdict_for(end, attempt, job, pod): """ from_pod = records.read_outcome(end, attempt) from_job = classify_from_job(job) - if from_pod and from_pod.get('outcome') in config.POD_AUTHORITATIVE_OUTCOMES: + if from_pod and from_pod.get('outcome') in mc.POD_AUTHORITATIVE_OUTCOMES: verdict = from_pod elif from_job and from_job.get('outcome') == 'timeout': verdict = from_job @@ -1376,7 +1377,7 @@ def verdict_for(end, attempt, job, pod): # "did not complete", which a SIGTERM drain and a real failure share, so the # archive is what separates them -- and only once the collector has finished # writing it. Until then the verdict stays `failed` and the decision defers. - if (verdict.get('exitCode') == config.CATCHUP_INCOMPLETE_EXIT + if (verdict.get('exitCode') == mc.CATCHUP_INCOMPLETE_EXIT and _attempt_finalized(end, attempt) and attempts.exit3_retry_cause(end, attempt)): verdict = dict(verdict, outcome='fetch-fault') @@ -1393,7 +1394,7 @@ def _condemn_timeout(end, attempt): "on attempt %s; this fails the mission. Check its archived " "log for 'maybe stale archive' -- an unreachable history " "mirror is the usual cause.", - end, config.ATTEMPT_DEADLINE_SECONDS, attempt) + end, mc.ATTEMPT_DEADLINE_SECONDS, attempt) return CONDEMN @@ -1407,7 +1408,7 @@ def _retry_oom(end, attempt): """ base = (sizing._profile_overrides(end, escalated=False) or {}).get('memory') ooms = records._oom_count(end, attempt) - had = (sizing.pool_memory(sizing.pool_for(end, attempt)) if config.POOL_PREFIX + had = (sizing.pool_memory(sizing.pool_for(end, attempt)) if mc.POOL_PREFIX else sizing.mem_for_attempt(ooms, base)) return _retry(f"OOM-killed at memory request {had}", memory=sizing.mem_for_attempt(ooms + 1, base, end=end)) @@ -1452,7 +1453,7 @@ def retry_decision(verdict, end, attempt): elif outcome == 'disrupted': return _retry("lost to node disruption") elif outcome == 'fetch-fault': - return _retry(f"exited {config.CATCHUP_INCOMPLETE_EXIT} after a fetch fault " + return _retry(f"exited {mc.CATCHUP_INCOMPLETE_EXIT} after a fetch fault " f"({attempts.exit3_retry_cause(end, attempt)})") elif outcome == 'oom': return _retry_oom(end, attempt) @@ -1463,7 +1464,7 @@ def retry_decision(verdict, end, attempt): # reaped node from a range that really failed, and a run that reports # success on a range nobody verified is worse than one that stops. return CONDEMN - elif verdict.get('exitCode') == config.CATCHUP_INCOMPLETE_EXIT: + elif verdict.get('exitCode') == mc.CATCHUP_INCOMPLETE_EXIT: return _decide_exit3(end, attempt) elif verdict.get('exitCode') is None: # The verdict came from the Job condition, which says Failed and nothing @@ -1476,14 +1477,14 @@ def retry_decision(verdict, end, attempt): def budget_for(verdict, end, attempt): """(spent, cap) for the cause that killed this attempt. - config.ATTEMPT_BUDGETS is the whole retry policy; a cause with no entry caps + mc.ATTEMPT_BUDGETS is the whole retry policy; a cause with no entry caps at 0 and is condemned on sight. `spent` counts only THIS cause, so evictions cannot drain the OOM or disk budgets. This verdict is already on disk, so the Nth failure is the one that exhausts a budget of N. """ outcome = verdict['outcome'] return (records._cause_count(end, attempt, (outcome,)), - config.ATTEMPT_BUDGETS.get(outcome, 0)) + mc.ATTEMPT_BUDGETS.get(outcome, 0)) def _log_retry(end, attempt, verdict, decision, cap): diff --git a/src/MissionParallelCatchup/apps/log_collector.py b/src/MissionParallelCatchup/apps/log_collector.py index ee9de76c..94639735 100644 --- a/src/MissionParallelCatchup/apps/log_collector.py +++ b/src/MissionParallelCatchup/apps/log_collector.py @@ -1,29 +1,17 @@ """Streaming log collector for parallel catchup. -Runs as a sidecar next to job_monitor, sharing its /logs volume. - -Why not read logs after a Job finishes: worker pods are one per ledger range, -and Karpenter deletes the node roughly a minute after its last running pod -exits, taking every pod object with it. Anything that reads after the fact is -racing that deletion. Polling each pod's log on an interval also keeps a -straggler readable *while* it is stuck, which is the case that turns a 5h run -into a 10h one; a condemned pod gets a follow stream so its last lines land -before it goes. - -Resume is idempotent across both a dropped stream and a restart of this -process: - - coarse reconnect with sinceTime=; the API - only accepts second granularity, so this deliberately overlaps - precise every line carries a kubelet RFC3339Nano timestamp (timestamps=true), - so drop any line <= last_ts. That removes the overlap exactly and - does not depend on stellar-core's own log format. - -Residual: if this dies between flushing log bytes and rewriting the state file, -the next run replays from a slightly older timestamp and a few lines duplicate. -Bounded by one poll's worth of lines, since the state file is rewritten at the -end of every poll. "At least once, deduped to near-exact" rather than exactly -once. +Runs as a sidecar next to job_monitor, sharing its /logs volume. Polls each +pod's log rather than reading after the Job finishes: Karpenter deletes the node +about a minute after its last pod exits, taking every pod object with it, and +polling also keeps a straggler readable *while* it is stuck. A condemned pod +gets a follow stream so its last lines land before it goes. + +Resume is idempotent across a dropped stream and a restart of this process: +reconnect with sinceTime=, which has second +granularity and so overlaps on purpose, then drop any line whose own kubelet +RFC3339Nano timestamp is <= last_ts. Residual: dying between flushing bytes and +rewriting the state file replays one poll's worth of lines, so this is at least +once, deduped to near-exact. """ import asyncio @@ -33,200 +21,233 @@ import logging import os import re -import ssl import sys -import zlib from datetime import datetime import aiohttp from logger import build_logger +import collector_config as cc +import kube_http +import tx_scan +import verdicts import config -import medida import records -CONTAINER = os.getenv('WORKER_CONTAINER', 'stellar-core') -POLL_SECONDS = float(os.getenv('COLLECTOR_POLL_SECONDS', 5)) -# Poll cycles a stream gets to finalize itself after its pod leaves the pod list -# before it is cancelled outright. One cycle is usually enough; the margin is for -# a stream still finalizing: writing its .metrics and closing its archive. -VANISHED_GRACE_CYCLES = int(os.getenv('COLLECTOR_VANISHED_GRACE_CYCLES', 3)) -# Peak memory now comes from kubelet, not Prometheus. kubelet reports rssBytes -# and workingSetBytes per container in the same /stats/summary payload this -# already fetches for ephemeral storage, at ~10s cAdvisor housekeeping against a -# 30s scrape -- and without depending on Prometheus being up, being reachable, -# or still retaining the window. The old _promql helper swallowed all three of -# those failures into "no peak", so an outage produced a profile that looked -# complete and was empty. cpu is not sampled at all: the request is fixed at -# REQ_CPU, so a measured value has nothing to size. -# Peaks are held per pod and flushed on significant growth, so a restart loses -# at most PEAK_FLUSH_RATIO of a range's high-water rather than all of it -- -# Prometheus's server-side max_over_time needed no such state. -PEAK_FLUSH_RATIO = float(os.getenv('PEAK_FLUSH_RATIO', 1.05)) -# Seconds between polls of one pod's log. Latency here is archive lag, not -# anything a decision waits on; 4096 pods at 10s is ~90 concurrent polls. -LOG_POLL_SECONDS = float(os.getenv('LOG_POLL_SECONDS', 10)) -# Concurrent in-flight log polls, across all pods. This is the whole point of -# polling: it is independent of how many pods exist, where follow=true needed -# one held connection per pod forever. -MAX_CONCURRENT_POLLS = int(os.getenv('MAX_CONCURRENT_POLLS', 96)) -# Most one poll may read before it stops. A pod that has been unwatched for a -# while has a large backlog; this bounds a single response, and the next poll -# picks up from the timestamp this one reached. Also the backstop against a -# blob that never terminates -- a progress meter emitting only carriage returns -# would otherwise grow the buffer until the sidecar OOMs, with 2096 streams -# doing it at once. -MAX_POLL_CHARS = int(os.getenv('MAX_POLL_CHARS', 8388608)) -# Bounds in-flight polls across every pod. Lives beside its own constant rather -# than among the peak dicts, where it landed inside the region the scanner tests -# exec and broke six of them on an asyncio NameError. -_poll_slots = asyncio.Semaphore(MAX_CONCURRENT_POLLS) +# Every piece of live runtime state this process holds, and the only module-level +# names here that are not settings: two semaphores, which carry their own waiter +# queues, and the dicts recording what is known about the pods being watched. +# Everything tunable lives in collector_config. + +# Bounds in-flight polls across every pod. +_poll_slots = asyncio.Semaphore(cc.MAX_CONCURRENT_POLLS) # Separate from _poll_slots on purpose -- see MAX_DOOMED_FOLLOWS. -_follow_slots = asyncio.Semaphore(int(os.getenv('MAX_DOOMED_FOLLOWS', 256))) -# pod name -> Event, set by the main loop the moment it first observes the pod -# terminal. poll_pod waits on it instead of sleeping blind, so the final read -# happens within the pod-list cadence rather than up to LOG_POLL_SECONDS later. -# That window is the only thing standing between a spot reclaim and the last -# lines the container wrote. +_follow_slots = asyncio.Semaphore(cc.MAX_DOOMED_FOLLOWS) +# pod name -> Event, set when the pod is first seen terminal, gone or condemned; +# poll_pod waits on it rather than sleeping, so LOG_POLL_SECONDS is a ceiling and +# the final read is not left until after it. _wake = {} # pod name -> its own start->finish, read off the pod while it still exists. _pod_secs = {} -# pod name -> status.startTime, kept so an attempt whose object vanished before -# any cycle saw its terminated timestamp can still be dated from the container's -# own start rather than from whenever this poller happened to open. +# pod name -> status.startTime, so an attempt whose object vanished before any +# cycle saw its terminated timestamp can still be dated from the container's own +# start rather than from whenever this poller happened to open. _pod_start = {} # Pods carrying a DisruptionTarget condition: the cluster has committed to -# destroying them and, on spot, gives about two minutes' notice. -# -# Waking the poller is not enough on its own. stellar-core prints its medida -# block ~4ms after SIGTERM and the pod object is deleted seconds later, so an -# interval poll straddles the whole thing -- measured on the 2048-worker run, -# 810 evictions lost 809 txApply values and 790 exact durations. A held -# connection already has those bytes when the process dies. -# -# Safe here precisely because it is scoped and short-lived, not because the -# count is small: global follow=true cost the sidecar 1444 MiB of a 2048 MiB -# limit at 2096 streams held for whole ranges, where these are held only for -# the drain. See MAX_DOOMED_FOLLOWS for the sizing. +# destroying them and, on spot, gives about two minutes' notice. Waking the +# poller is not enough on its own -- stellar-core prints its medida block ~4ms +# after SIGTERM and the object is deleted seconds later, so an interval poll +# straddles the whole thing (2048-worker run: 810 evictions lost 809 txApply +# values), where a held connection already has those bytes. Safe because it is +# scoped to the drain, not because the count is small: global follow=true cost +# the sidecar 1444 MiB of a 2048 MiB limit at 2096 whole-range streams. _doomed = {} -# Longest a follow stream will hang on to a doomed pod. Spot gives 120s; past -# roughly double that the notice was withdrawn (Karpenter cancelled the drain) -# and the stream would otherwise be held for the life of the range. -# 0 disables the follow path entirely and leaves interval polling to do it, -# which is only safe when prestopSleepSeconds is holding the pod open instead. -DOOMED_FOLLOW_SECONDS = float(os.getenv('DOOMED_FOLLOW_SECONDS', 300)) -# Follows get their OWN budget rather than sharing _poll_slots. -# -# Sharing was a starvation bug: a follow holds its slot for the whole drain, so -# a reclaim condemning more pods than there are slots would consume all of them -# and stop every other pod in the run from being polled at all -- turning a -# partial node loss into a run-wide blackout. With a separate budget the worst -# case is that some condemned pods fall back to polling, which is a proven path -# rather than a degradation -- 1s polling captured txApply on its own with no -# follow and no preStop. -# -# Sized well above any plausible simultaneous disruption rather than at the -# measured one. A whole-AZ spot reclaim is not bounded by Karpenter's -# disruption budget, so the ~43 pods a 10% budget implies is a floor, not a -# ceiling. The old always-on design sustained 2096 concurrent streams, and it -# paid far more per stream than this does: it held a persistent GzipFile and -# aiohttp buffers for a pod's entire multi-hour life, where _follow_tail builds -# a fresh gzip member per flush, keeps nothing between them, and lives for the -# 10-120s of a drain. 256 x the old 0.69 MiB upper bound is 177 MiB against a -# 2048 MiB limit, and the true figure is lower. -MAX_DOOMED_FOLLOWS = int(os.getenv('MAX_DOOMED_FOLLOWS', 256)) -# Poll interval for a condemned pod, replacing LOG_POLL_SECONDS for as long as -# it is doomed. This is the cheap half of the fix and the one that does the -# work: measured on ssc-test, preStop delays SIGTERM but leaves the gap between -# the medida block and the pod object being deleted at ~9s, so a blind 10s poll -# straddles it -- which it did, losing txApply even with a 60s preStop holding -# the pod open. Polling that same window every second cannot miss it. -# -# Costs no held connections, unlike a follow stream: ~120 short requests over a -# 2-minute drain per condemned pod, bounded by the existing _poll_slots. -# sinceTime has 1s granularity, so going below 1s only re-reads the same second. -DOOMED_POLL_SECONDS = float(os.getenv('DOOMED_POLL_SECONDS', 1)) -# How long each watch connection is allowed to live before the apiserver closes -# it and we reconnect. Bounded rather than infinite so a silently-dead stream -# self-heals; the reconnect resumes from the last resourceVersion, so nothing is -# missed across it. 0 disables the watch and leaves detection to the pod list. -WATCH_TIMEOUT_SECONDS = int(os.getenv('WATCH_TIMEOUT_SECONDS', 600)) -# Pause before re-opening a watch that failed. Only covers hard errors: a clean -# timeout reconnects immediately. -WATCH_RETRY_SECONDS = float(os.getenv('WATCH_RETRY_SECONDS', 1)) -# Fields that only ever grow. write_metrics maxes these instead of overwriting, -# so a restarted poller starting its high-water at zero cannot lower one. -PEAK_KEYS = ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes') -# Failed polls tolerated after a pod goes terminal before we stop asking. Its -# log is not coming back, and spinning on it holds a task and a poll slot for -# the rest of the run; a couple of retries still absorb a transient 500, which -# arrived in bursts at ramp. Returning bare on one of those used to drop the -# metrics for every range whose last read happened to throw. -TERMINAL_POLL_ATTEMPTS = int(os.getenv('TERMINAL_POLL_ATTEMPTS', 3)) -# Phases whose log endpoint can actually answer. Pending has no container yet -# and Unknown means the node stopped reporting; the terminal phases are kept -# because that is where a pod's final output lives. -POLLABLE_PHASES = ('Running', 'Succeeded', 'Failed') - - -SA = '/var/run/secrets/kubernetes.io/serviceaccount' -API = f"https://{os.getenv('KUBERNETES_SERVICE_HOST', 'kubernetes.default')}:{os.getenv('KUBERNETES_SERVICE_PORT', '443')}" -# Read-only kubelet port. A seam for tests, which serve the payload on a -# loopback port rather than reaching a real node. -KUBELET_PORT = 10250 - -logger = build_logger('log_collector', name='log-collector', to_file=False) - - -def token(): - # Projected service account tokens rotate, so this is re-read per request - # rather than cached at startup. - with open(os.path.join(SA, 'token')) as fh: - return fh.read().strip() - -def ssl_ctx(): - return ssl.create_default_context(cafile=os.path.join(SA, 'ca.crt')) +# Peak ephemeral disk, for sizing a later run's ephemeral-storage request. Only +# meaningful in ephemeral mode, and sampled for every pod but only kept for +# ranges that finished -- what invalidates a sample is being cut short, not the +# capacity type. Prometheus cannot answer it (cAdvisor reports fs usage per node, +# with no pod label), so this samples kubelet directly and keeps a running max. +_eph_peak = {} +_anon_peak = {} +_ws_peak = {} +# Last value flushed to the volume, per axis: keyed by pod name for anon and by +# "/eph" for ephemeral. A pod name cannot contain '/', so the two key +# spaces cannot collide. +_peak_flushed = {} +# pod name -> (end, attempt), so a mid-flight peak flush can find its file. +_streaming = {} +# The poller registry, module-level so the watch can open a stream the moment a +# pod appears instead of waiting for the pod-list loop. One registry with one +# guard is the whole point: read_state is consulted once, at poll_pod start, so +# two creators would each hold their own in-memory last_ts, re-append the same +# lines and race each other's write_state. +_tasks = {} +_streamed = set() +# session + the terminal/succeeded views poll_pod closes over, published once by +# main() so ensure_stream can be called from outside it. +_stream_ctx = {} -def base(end, attempt): - return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}") +logger = build_logger('log_collector', name='log-collector', to_file=False) -def _is_condemned(pod): - """The DisruptionTarget reason if the cluster has committed to destroying - this pod, else None. +async def main(): + os.makedirs(config.LOG_DIR, exist_ok=True) + # Connection-pool limit, not a task limit: there is no semaphore above it, + # so a stream that cannot get a connection blocks here until the pool drains + # -- it does not degrade, it starves, and it starves the pods created last, + # which are the retries. Sized for concurrent polls plus headroom for the + # pod-list and kubelet calls, not one connection per pod: under follow=true + # a 1200 limit against 2048 workers left 896 blocked forever. + conn = aiohttp.TCPConnector( + limit=cc.MAX_CONCURRENT_POLLS + cc.MAX_DOOMED_FOLLOWS + 64, ssl=kube_http.ssl_ctx()) + # No total timeout: these streams are meant to stay open for the life of a + # range, which can be hours. + timeout = aiohttp.ClientTimeout(total=None, sock_connect=10) + # The module-level registry under local names, so the watch shares the same + # guard as the bookkeeping below. + tasks, streamed = _tasks, _streamed + # Cleared rather than assumed empty: a second main() in one process would + # otherwise find every pod already registered and open no streams at all. + tasks.clear() + streamed.clear() + _stream_ctx.clear() + terminal, succeeded, vanished = {}, {}, {} + # `streamed` holds streams that ran to completion. Without it a finished + # task is deleted from `tasks` and the next poll re-opens the stream, + # forever: one full log re-read per pod every POLL_SECONDS. - DisruptionTarget covers the cases that cost us measurements: a spot reclaim, - a Karpenter drain, and node pressure. It does NOT cover a kubelet - ephemeral-storage eviction -- classify() handles that one from - status.message -- and it is deliberately not inferred from a deletionTimestamp, - which is also set by the monitor reaping a Job that already finished. + async with aiohttp.ClientSession(connector=conn, timeout=timeout) as session: + logger.info("streaming logs for run=%s into %s", config.RUN_NAME, config.LOG_DIR) + # Published before the watch starts: ensure_stream is a no-op until this + # exists, so a watch event arriving first would silently open nothing. + _stream_ctx.update(session=session, terminal=terminal, succeeded=succeeded) + if cc.WATCH_TIMEOUT_SECONDS > 0: + asyncio.create_task(watch_condemnations(session)) + while True: + try: + pods = await list_pods(session) + live = {p['metadata']['name'] for p in pods} + # A pod can leave the list without ever being observed terminal + # -- reaped node, eviction, or the monitor deleting its finished + # Job -- and `terminal` is only written for pods in this list, so + # its stream would retry until the run ended. Marking them + # terminal lets the stream finalize and free the slot; + # cancelling is the backstop for one wedged in a connection + # attempt it will never win. + for name in [n for n in tasks if n not in live]: + terminal[name] = True + if name in _wake: + # Gone is terminal. Without this its poller sleeps out + # the interval before taking the 404, delaying finalize + # and the .done that lets the monitor reap the Job. + _wake[name].set() + t = tasks[name] + if t.done(): + del tasks[name] + streamed.add(name) + continue + vanished[name] = vanished.get(name, 0) + 1 + if vanished[name] >= cc.VANISHED_GRACE_CYCLES: + t.cancel() + try: + await t + except asyncio.CancelledError: + pass + del tasks[name] + vanished.pop(name, None) + ref = _streaming.get(name) + if ref is not None: + # The poller was wedged, but its archive and the + # sampler's process-local peaks still contain useful + # truth. Finalize them before licensing a reap. + await finalize( + session, name, ref[0], ref[1], + tx_scan.TxApplyScanner(recreated=True), + lambda p: succeeded.get(p, False)) + streamed.add(name) + logger.info("cancelled and finalized stream for vanished pod %s", + name) + for pod in pods: + name = pod['metadata']['name'] + labels = pod['metadata'].get('labels', {}) + end = labels.get(config.LABEL_RANGE) + if end is None: + continue + phase = pod.get('status', {}).get('phase') + terminal[name] = phase in ('Succeeded', 'Failed') + # NOT gated on phase: a pod being deleted keeps phase Running + # until its object disappears, so gating this on terminal + # meant no disrupted pod ever recorded an exact duration and + # every one fell back to the poller's clock -- 268s reported + # against a ~500s attempt. terminated.finishedAt is present + # for the ~8s the object outlives the container, and + # pod_seconds returns None until then, so asking every cycle + # is self-guarding. + secs = pod_seconds(pod) + if secs is not None: + _pod_secs[name] = secs + start = (pod.get('status') or {}).get('startTime') + if start: + # So finalize can date the attempt from when the + # container STARTED if the object is deleted before any + # cycle catches its terminated timestamp. + _pod_start[name] = start + # Backstop only: the watch normally gets here first. This + # still runs so detection survives the watch being disabled + # or reconnecting. + if not terminal[name]: + _mark_condemned(pod, name, end, + labels.get(config.LABEL_ATTEMPT, '1')) + if terminal[name] and name in _wake: + # Wake its poller now rather than at the next tick. + _wake[name].set() + succeeded[name] = phase == 'Succeeded' + if phase == 'Failed': + verdicts.record_outcome(pod, end, labels.get(config.LABEL_ATTEMPT, '1')) + if name in tasks and not tasks[name].done(): + continue + if name in tasks and tasks[name].done(): + del tasks[name] + # Only bar a re-open once the pod itself is terminal. A + # task that ended while the pod is still running died + # early, and re-opening is the recovery path. + if terminal.get(name): + streamed.add(name) + continue + if name in streamed: + continue + # Backstop: the watch normally opens this the moment the pod + # appears, and this covers events dropped across a + # reconnect. Same registry and guard, so whichever gets + # there first wins -- two readers on one pod would duplicate + # the archive and race write_state. + ensure_stream(name, end, labels.get(config.LABEL_ATTEMPT, '1'), phase) - The reason separates a warning from a postmortem, which the bare condition - cannot: EvictionByEvictionAPI is a drain that still has to deliver SIGTERM, - while DeletionByTaintManager is stamped ~40s after the node went NotReady, - on a container that already died unsignalled. In the second case no medida - block was ever written, so a missing txApply is not a capture race. - """ - for cond in ((pod.get('status') or {}).get('conditions') or []): - if cond.get('type') == 'DisruptionTarget' and cond.get('status') == 'True': - return cond.get('reason') or 'Unknown' - return None + # AFTER the per-pod branches, never before them: this is a + # serial sweep of every node's kubelet and on spot a dead one + # costs the 10s connect timeout apiece, which once stretched a + # cycle to 925s. It must also stay outside the `for` loop, whose + # branches `continue` for a pod already streaming, so a sampler + # among them fires only on the cycle a stream opens. + # Unconditional, not gated on ephemeral mode: memory is sized in + # both modes, and gating left every pvc run with no anon peak. + await sample_kubelet(session, { + p['status']['hostIP'] for p in pods + if p.get('status', {}).get('hostIP') + and p.get('status', {}).get('phase') == 'Running'}) + except Exception as e: + logger.warning("pod list failed: %s", e) + await asyncio.sleep(cc.POLL_SECONDS) def _mark_condemned(pod, name, end, attempt): """Flag a condemned pod so its poller opens a follow. Idempotent. - Shared by the pod-list sweep and the watch so the two cannot drift: whichever - sees the condition first does the work, the other no-ops on the _doomed - check. - - Detection latency, not the follow, is what loses the metric. stellar-core - exits about a second after SIGTERM and the pod object is reaped right behind - it, so a condemned pod exists for only a few seconds. Measured on this - cluster at prestopSleepSeconds=5, the 5s list sweep caught that window about - half the time: of 52 mid-replay legs, 32 lost txApply and 25 of those were - seen but seen too late to open a stream. + Shared by the pod-list sweep and the watch so the two cannot drift: + whichever sees the condition first does the work, the other no-ops. + Detection latency, not the follow, is what loses the metric -- stellar-core + exits about a second after SIGTERM and the object is reaped right behind it. """ if name in _doomed: return False @@ -234,13 +255,12 @@ def _mark_condemned(pod, name, end, attempt): # Already finished. Its log is complete and a follow would only re-read # a dead pod every iteration. return False - doom = _is_condemned(pod) + doom = verdicts.condemnation_reason(pod) if not doom: return False _doomed[name] = doom - # Recorded now, because the evidence does not survive the node: once the - # object is gone there is no way to tell a drain we lost a race with from a - # corpse that never had a metric to lose. + # Recorded now: once the object is gone there is no way to tell a drain we + # lost a race with from a corpse that never had a metric to lose. write_metrics(end, attempt, {'disruptionReason': doom}) if name in _wake: # Break the current sleep so the follow opens now rather than up to @@ -274,13 +294,9 @@ def pod_seconds(pod): return None -def done_path(end, attempt): - return base(end, attempt) + '.done' - - def read_state(end, attempt): try: - with open(base(end, attempt) + '.state') as fh: + with open(records.state_path(end, attempt)) as fh: ts = fh.read().strip() except OSError: return None @@ -289,7 +305,7 @@ def read_state(end, attempt): def write_state(end, attempt, ts): - path = base(end, attempt) + '.state' + path = records.state_path(end, attempt) try: records.write_atomic(path, ts) except OSError as e: @@ -300,110 +316,19 @@ def discard(end, attempt): # .metrics deliberately survives: it holds tx_apply for a range that # succeeded, which is the only case this runs in. Dropping it would let a # log-retention flag silently delete a Grafana series. - for suffix in ('.log.gz', '.state'): + for path in (records.log_path(end, attempt), records.state_path(end, attempt)): try: - os.remove(base(end, attempt) + suffix) + os.remove(path) except OSError: pass -# kubelet returns plain text such as "unable to retrieve container logs for -# containerd://..." when a container is not up yet. That has no timestamp, so -# partitioning on the first space yields "unable", which then goes into the -# state file and every later request asks for sinceTime=unableZ -> HTTP 400, -# forever. Observed on ssc-test the moment evicted pods were replaced. +# kubelet returns untimestamped plain text ("unable to retrieve container logs +# for containerd://...") when a container is not up yet. Partitioning that on +# the first space yields "unable", which lands in the state file and makes every +# later request ask for sinceTime=unableZ -> HTTP 400, forever. _TS_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?$") -_TX_METRIC = "metric 'ledger.transaction.apply'" - -class TxApplyScanner: - """Pull the medida tx-apply total out of the stream as it goes past. - - stellar-core prints this block once, just before exit. Scanning here rather - than re-reading the log later is what makes the metric independent of pod - lifetime: by the time job_monitor sees the Job succeed, Karpenter may - already have reaped the node, and with saveSuccessLogs=false the archive is - gone too. These bytes pass through this process exactly once, so this is the - only place guaranteed to see them. - """ - - # Shared with job_monitor's archive re-read rather than restated: they used - # to agree by comment, and a divergence would hand the recovery path the - # same blind spot it exists to cover. Measured on ssc-test 2026-08-04, a - # /info liveness response interleaved into the block put `sum` 91 lines - # below the header and both readers gave up 76 lines short -- one leg in 233. - WINDOW = medida.WINDOW - HARD_WINDOW = medida.HARD_WINDOW - - # Printed by RESUME_SCRIPT before stellar-core starts. Its counterpart, - # "RESUME DECLINED", means new-db ran and this attempt did the whole range, - # so the colon is load-bearing -- it is what separates the two. - RESUME_MARK = 'RESUME: ' - RESUME_DECLINED_MARK = 'RESUME DECLINED:' - - def __init__(self, recreated=False): - self.seconds = None - self.resumed = False - self.resume_decided = False - # A new poller starting from durable .state missed every earlier line. - # Finalization must recover scanner-only facts from the archive. - self.recreated = recreated - self._left = 0 - self._span = 0 - - def feed(self, line): - if self.RESUME_MARK in line: - self.resumed = True - self.resume_decided = True - elif self.RESUME_DECLINED_MARK in line: - self.resume_decided = True - if _TX_METRIC in line: - self._left = self.WINDOW - self._span = self.HARD_WINDOW - return - if self._left <= 0: - return - m = medida.SUM_RE.search(line) - if m: - self.seconds = float(m.group(1)) / 1000.0 - self._left = 0 - return - self._span -= 1 - if self._span <= 0 or medida.ANY_METRIC.search(line): - # Ran out of rope, or another timer's block started -- either way our - # sum is not coming. - self._left = 0 - return - if medida.METRIC_LINE.search(line): - # Only medida's own statistics count. Interleaved output from another - # thread is noise between us and the sum, not evidence we have passed - # it. - self._left -= 1 - - -def scan_archive(end, attempt, need_tx=False): - """Recover scanner state from complete gzip members already on disk.""" - path = base(end, attempt) + '.log.gz' - scanner = TxApplyScanner() - try: - with gzip.open(path, 'rt', errors='replace') as fh: - for line in fh: - scanner.feed(line) - # The resume decision is at process startup. Avoid decompressing - # a multi-gigabyte worker log when that is all the caller needs. - if scanner.resume_decided and not need_tx: - break - except FileNotFoundError: - return scanner - except (EOFError, gzip.BadGzipFile, zlib.error) as e: - # Keep facts found in complete prefix members. A torn final member cannot - # invalidate an earlier RESUME line or complete medida block. - logger.warning("could only partially recover scanner state from %s: %s", path, e) - except OSError as e: - logger.warning("could not open scanner archive %s: %s", path, e) - return scanner - - def write_metrics(end, attempt, values): """Persist per-range measurements for job_monitor's reconcile to read. @@ -411,43 +336,38 @@ def write_metrics(end, attempt, values): fail" and is only written for failed pods, whereas these are only meaningful for one that succeeded. """ - path = base(end, attempt) + '.metrics' + path = records.metrics_path(end, attempt) # Merge, and never let a peak go backwards. A measurement already on disk - # must survive a later write that lacks it -- the peaks are held in memory, - # so a collector restart would otherwise drop them. But a plain overwrite is - # wrong for a monotonic quantity: after a restart the fresh poller starts - # its high-water at zero, and its first flush would replace the higher - # pre-restart value with a lower one. Lowering a peak undersizes the range - # next run, which is the one direction that costs an OOM. + # must survive a later write that lacks it, but a plain overwrite is wrong + # for a monotonic quantity: after a restart the fresh poller's first flush + # would replace a higher pre-restart value with a lower one, undersizing the + # range next run -- the one direction that costs an OOM. try: with open(path) as fh: prior = json.load(fh) except (OSError, ValueError): prior = {} merged = {**prior, **values} - # attemptSeconds is not a peak, but it takes the same rule for the same - # reason: it is a fixed quantity once the attempt ends, and every source is - # a lower bound on it -- the pod's own start->finish is exact, the poller's - # elapsed time covers only the part of the attempt that process was alive - # for. An attempt is finalized more than once whenever a poller is re-opened - # for a pod that is still listed (a restarted sidecar, or a 404 on the log - # endpoint while the pod list is stale), and there the fallback clock starts - # at the restart: newest-wins turned a recorded 3600s into 0.0s. - for k in PEAK_KEYS + ('attemptSeconds',): + # attemptSeconds takes the same rule: it is fixed once the attempt ends and + # every source is a lower bound on it. An attempt is finalized more than + # once whenever a poller re-opens for a pod that is still listed, and there + # the fallback clock starts at the restart -- newest-wins turned a recorded + # 3600s into 0.0s. + for k in cc.PEAK_KEYS + ('attemptSeconds',): a, b = prior.get(k), values.get(k) if a is not None and b is not None: merged[k] = max(a, b) - # Once any poller or archive read proves that this attempt resumed, a later - # restarted poller cannot un-prove it. In particular, a merge containing - # resumed=False must never lower the durable decision back to fresh. + # Once any poller or archive read proves this attempt resumed, a later + # restarted poller cannot un-prove it: a merge containing resumed=False must + # never lower the durable decision back to fresh. if prior.get('resumed') is True or values.get('resumed') is True: merged['resumed'] = True if (prior.get('attemptSecondsExact') is True or values.get('attemptSecondsExact') is True): merged['attemptSecondsExact'] = True - # Same one-way rule: once a duration has been dated from the container's own - # startTime, a later poller-clock write must not strip the provenance that - # makes the monitor willing to use it as a chain leg. + # Same one-way rule: once a duration is dated from the container's own + # startTime, a later poller-clock write must not strip the provenance the + # monitor needs to use it as a chain leg. if (prior.get('attemptSecondsFromContainerStart') is True or values.get('attemptSecondsFromContainerStart') is True): merged['attemptSecondsFromContainerStart'] = True @@ -459,117 +379,21 @@ def write_metrics(end, attempt, values): logger.warning("could not persist metrics for range %s: %s", end, e) -def classify(pod): - """Why did this pod fail? Recorded here rather than in job_monitor. - - This process already lists every pod every few seconds to discover streams, - so it sees terminal transitions first-hand -- a separate watch thread in the - monitor was observing the same objects a second time. The Job object cannot - answer this: its condition carries no exit code until a podFailurePolicy - rule matches, and an admission rejection matches none. - """ - status = pod.get('status', {}) - if _is_condemned(pod): - return {'outcome': 'disrupted', 'exitCode': None} - reason = status.get('reason') - if reason == 'Evicted' and 'ephemeral' in (status.get('message') or ''): - # The range's own disk use, not something the cluster did to it. - # Measured on ssc-test: the kubelet sets no DisruptionTarget for a - # limit eviction, and stellar-core drains on the eviction SIGTERM and - # exits 3 -- so the Job condition matches the generic non-zero rule and - # reads as a plain catchup failure, which gets no retry at all. - # status.message is the only discriminator and only the pod carries it. - # Recording it here, while the pod still exists, is the only way to - # keep the signal. - return {'outcome': 'ephemeral', 'exitCode': None, 'reason': status.get('message')} - if reason in ('VolumeAttachmentLimitExceeded', 'OutOfcpu', 'OutOfmemory', 'OutOfpods', - 'UnexpectedAdmissionError', 'NodeAffinity', 'Shutdown', 'Evicted'): - return {'outcome': 'rejected', 'exitCode': None, 'reason': reason} - terms = [cs.get('state', {}).get('terminated') for cs in status.get('containerStatuses', [])] - terms = [t for t in terms if t] - if not terms: - # Nothing ever ran, so this says nothing about the ledger range. - return {'outcome': 'rejected', 'exitCode': None, 'reason': reason or 'no container status'} - for t in terms: - if t.get('reason') == 'OOMKilled': - return {'outcome': 'oom', 'exitCode': t.get('exitCode')} - if t.get('exitCode') not in (0, None): - return {'outcome': 'failed', 'exitCode': t.get('exitCode')} - # Terminated, but not one container said with what. `failed` here is a lie - # that costs the whole run: it reads as a genuine catchup failure, which is - # the one outcome that gets no retry at all. Observed on the r5 run - # 2026-07-30, range 59018943 -- condemned on attempt 1 with exitCode null, - # failing a mission that was otherwise 554 for 554. - return {'outcome': 'unknown', 'exitCode': None} - - -def record_outcome(pod, end, attempt): - """Write the verdict next to the log, for job_monitor's reconcile to read.""" - path = base(end, attempt) + '.outcome' - if os.path.exists(path): - return - data = classify(pod) - data['pod'] = pod['metadata']['name'] - try: - records.write_atomic(path, json.dumps(data)) - logger.info("range %s attempt %s classified: %s", end, attempt, data['outcome']) - except OSError as e: - logger.warning("could not persist outcome for range %s: %s", end, e) - - -# Peak ephemeral disk, for sizing a later run's ephemeral-storage request. -# -# Only meaningful in ephemeral mode. Sampled for every pod, but only kept for -# ranges that finished -- see the completion gate in finalize. Spot is fine: -# what invalidates a sample is being cut short, not the capacity type. -# -# Prometheus cannot answer this -- cAdvisor reports fs usage per node, with no -# pod label -- so this samples kubelet directly through the apiserver proxy and -# keeps a running max. -_eph_peak = {} -_anon_peak = {} -_ws_peak = {} -# Last value flushed to the volume, per axis: keyed by pod name for anon and by -# "/eph" for ephemeral. A pod name cannot contain '/', so the two key -# spaces cannot collide. -_peak_flushed = {} -# pod name -> (end, attempt), so a mid-flight peak flush can find its file. -_streaming = {} -# The poller registry, module-level so the watch can open a stream the moment a -# pod appears instead of waiting for the pod-list loop to come round. -# -# One registry with one guard is the whole point: two creators would each hold -# their own in-memory last_ts -- read_state is consulted once, at poll_pod start -# -- so both would re-append the same lines and race each other's write_state. -# main() binds its locals to these, so the loop's existing bookkeeping is -# unchanged and either caller can be the one that wins. -_tasks = {} -_streamed = set() -# session + the terminal/succeeded views poll_pod closes over, published once by -# main() so ensure_stream can be called from outside it. -_stream_ctx = {} - - def _flush_peak(name, axis, field, value): """Persist a high-water so a sidecar restart cannot lose it. - Every key in PEAK_KEYS needs this, not just the ones we remembered. The - peaks live in module dicts, so a restarted collector starts from zero and - re-accumulates only from whatever the pod is using at that moment. anon had - it, ephemeral got it when a restart was shown to lose the high-water, and - peakWorkingSetBytes was missed -- which is how a completed range came back - with a working set BELOW its own anon, which cannot happen in one sample and - is trivial across a restart. Measured on the 2026-07-30 run: 136 of 3095 - ranges, 55 of them single-attempt so no retry chain could explain them. - - write_metrics max-merges on PEAK_KEYS, so re-flushing a lower value later is - harmless; the ratio only keeps this to a handful of writes per pod. + Every key in PEAK_KEYS needs this: the peaks live in module dicts, so a + restarted collector starts from zero and re-accumulates only from whatever + the pod is using at that moment. Missing it for peakWorkingSetBytes is how + 136 of 3095 ranges came back with a working set BELOW their own anon, which + cannot happen in one sample. write_metrics max-merges on PEAK_KEYS, so + re-flushing a lower value later is harmless. """ ref = _streaming.get(name) if not ref: return key = name + '/' + axis - if value < _peak_flushed.get(key, 0) * PEAK_FLUSH_RATIO: + if value < _peak_flushed.get(key, 0) * cc.PEAK_FLUSH_RATIO: return _peak_flushed[key] = value write_metrics(ref[0], ref[1], {field: value}) @@ -591,35 +415,24 @@ async def sample_kubelet(session, node_ips): """Update each pod's peak ephemeral use and peak anon from one snapshot. Both axes come out of the same GET, so tracking memory here is free. - - kubelet's `rssBytes` is cgroup v2 `anon` -- measured against a live pod on - ssc-test it read 482 MiB while the cgroup reported 492 MiB seconds later. - Anon is the only limit-independent memory figure this workload has: page - cache expands to fill whatever `memory.max` allows, so `memory.peak` is - always ~= the limit (measured: a range needing 862 MiB of anon reported a - 12704 MiB peak when given a 24000 MiB limit) and is useless for sizing. - - Sampled rather than exact -- cAdvisor housekeeping is ~10s, so a shorter - anon spike is invisible. Still ~3x finer than the 30s Prometheus scrape the - profile used before, which is the undersampling that let profiled ranges - OOM. The `time` field on this payload runs 1-3s behind wall clock; the ~80s - lag applies only to the du-based ephemeral figure alongside it. + kubelet's `rssBytes` is cgroup v2 `anon`, the only limit-independent memory + figure this workload has: page cache expands to fill whatever `memory.max` + allows, so `memory.peak` is always ~= the limit (a range needing 862 MiB + reported 12704 MiB against a 24000 MiB limit) and is useless for sizing. + Sampled rather than exact -- cAdvisor housekeeping is ~10s, still ~3x finer + than the 30s Prometheus scrape whose undersampling let profiled ranges OOM. """ for ip in node_ips: - # Straight at the kubelet, not through the apiserver's node proxy. The - # proxy needs `nodes/proxy`, which authorizes GET on EVERY kubelet path - # -- /pods and /containerLogs included, for any namespace scheduled on - # that node. The kubelet maps /stats/* to its own `nodes/stats` - # subresource, so going direct is the same data under a grant that - # cannot read pod inventory or logs at all. - # - # ssl=False: EKS kubelet serving certs are self-signed, not issued by - # the cluster CA the session's context trusts. In-VPC hop to the node's - # own address. - url = f"https://{ip}:{KUBELET_PORT}/stats/summary" + # Straight at the kubelet, not through the apiserver's node proxy: that + # needs `nodes/proxy`, which authorizes GET on EVERY kubelet path, + # /pods and /containerLogs included, for any namespace on that node. + # Going direct is the same data under `nodes/stats`, a grant that cannot + # read pod inventory or logs at all. ssl=False because EKS kubelet + # serving certs are self-signed; in-VPC hop to the node's own address. + url = f"https://{ip}:{kube_http.KUBELET_PORT}/stats/summary" try: async with session.get(url, ssl=False, - headers={'Authorization': f'Bearer {token()}'}) as resp: + headers={'Authorization': f'Bearer {kube_http.token()}'}) as resp: resp.raise_for_status() summary = await resp.json() except Exception as e: @@ -637,20 +450,18 @@ async def sample_kubelet(session, node_ips): if int(used) > prev: _eph_peak[name] = int(used) logger.info("peak ephemeral for %s: %.2f GiB", name, used / 1073741824) - # Flushed on growth for the same reason as anon below, and - # re-measuring does not recover it: disk use is not - # monotonic -- stellar-core drops its download staging once - # buckets are applied -- so a replacement sidecar watching - # the tail of the same pod sees a fraction of the real - # high-water. This figure sizes the next run's - # ephemeral-storage request, and one that comes back too - # small is an eviction. + # Flushed on growth, and re-measuring cannot recover it: + # disk use is not monotonic -- stellar-core drops its + # download staging once buckets are applied -- so a + # replacement sidecar sees a fraction of the real high-water. + # This sizes the next run's request, and one that comes back + # too small is an eviction. _flush_peak(name, 'eph', 'peakEphemeralBytes', int(used)) for c in entry.get('containers', []): # The worker container only. Sidecars share the pod, so summing # across containers -- or letting the last one win -- would size # the range from whichever one kubelet happened to list last. - if c.get('name') != CONTAINER: + if c.get('name') != cc.CONTAINER: continue # Absent for the first seconds of a container's life, before # cAdvisor has stats for it. Every later poll carries it, so a @@ -668,17 +479,15 @@ async def sample_kubelet(session, node_ips): if int(rss) <= _anon_peak.get(name, 0): continue _anon_peak[name] = int(rss) - # Held in memory until the stream ends, so a collector restart - # would otherwise reset a range's high-water to whatever it is - # using at that moment -- under-reporting, which sizes the next - # run too small. Flushing only on PEAK_FLUSH_RATIO growth keeps - # this to a handful of writes over a pod's life instead of one - # per sample per pod. + # Held in memory until the stream ends, so a restart would reset + # the high-water to whatever the pod is using then, sizing the + # next run too small. Flushing only on PEAK_FLUSH_RATIO growth + # keeps this to a handful of writes over a pod's life. _flush_peak(name, 'anon', 'peakAnonBytes', int(rss)) def _mark_done(end, attempt): - path = done_path(end, attempt) + path = records.done_path(end, attempt) try: records.write_atomic(path, '') except OSError as e: @@ -690,16 +499,11 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): """Persist everything this attempt owes, then let its stream go. Reached from three places, and deliberately ONE implementation: a clean end - of stream once the pod is terminal, a 404 once the pod object is gone, and - a terminal pod whose polls keep failing past TERMINAL_POLL_ATTEMPTS. Two - copies of the metrics/discard logic is how one path silently stops writing - peakAnonBytes while the other keeps working. - - The 404 path used to not exist, so a pod deleted while Running -- reaped - node, eviction, or the monitor deleting a finished Job -- left its stream - retrying every 30s for the rest of the run, holding a connection slot the - whole time. Note the converse: an interrupted read on a pod that is STILL - RUNNING must not come here. Finalizing then writes a truncated peak and + of stream once the pod is terminal, a 404 once the object is gone, and a + terminal pod whose polls keep failing past TERMINAL_POLL_ATTEMPTS. Two + copies is how one path silently stops writing peakAnonBytes while the other + keeps working. The converse matters too: an interrupted read on a pod that + is STILL RUNNING must not come here, or it writes a truncated peak and leaves the range looking measured when it is not. """ # Before discard: on success the archive is about to be deleted. @@ -708,11 +512,11 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): since_start = None began = _pod_start.pop(pod, None) if observed is None and began: - # The container started at `began` and has just stopped -- finalize is - # reached on end of stream or a 404, both within a second or two of the - # exit. Not exact, because the true end is terminated.finishedAt, but it - # dates the attempt from the container rather than from this poller, and - # a re-opened poller's clock can be near zero against a multi-hour run. + # The container started at `began` and has just stopped: finalize runs + # within a second or two of the exit. Not exact -- the true end is + # terminated.finishedAt -- but it dates the attempt from the container + # rather than from this poller, whose clock can be near zero against a + # multi-hour run. try: since_start = (datetime.utcnow() - datetime.strptime( began, '%Y-%m-%dT%H:%M:%SZ')).total_seconds() @@ -724,19 +528,17 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): measured['attemptSecondsExact'] = True elif since_start is not None and since_start > 0: measured['attemptSeconds'] = round(since_start, 1) - # Not exact -- the true end is terminated.finishedAt, and this is - # "now, a second or two after the stream ended". But it IS a measure of - # the container's own lifetime rather than of this process's attention - # span, which is the distinction the monitor's chain gate cares about. - # Measured on ssc-test against two evicted pods: 370.9s and 375.1s - # against a true ~373s, so +/-1%, versus the poller clock's -46%. + # Not exact, but it measures the container's own lifetime rather than + # this process's attention span, which is the distinction the monitor's + # chain gate cares about. Measured against two evicted pods: 370.9s and + # 375.1s against a true ~373s, versus the poller clock's -46%. measured['attemptSecondsExact'] = False measured['attemptSecondsFromContainerStart'] = True elif started is not None: # Fallback only: the monitor's figure comes from the pod's terminated - # timestamps and is preferred when it exists. write_metrics keeps this - # from lowering a duration already on the volume -- an attempt can be - # finalized twice, and the second poller's clock started at the restart. + # timestamps and is preferred. write_metrics keeps this from lowering a + # duration already on the volume, since a second poller's clock starts + # at the restart. measured['attemptSeconds'] = round( asyncio.get_event_loop().time() - started, 1) measured['attemptSecondsExact'] = False @@ -745,16 +547,16 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): # finalization; recover only the state this scanner could have missed. archived = None need_resume = int(attempt) > 1 and not tx.resume_decided - # Not gated on `recreated`: a poller that ran start to finish can still - # miss the block, which stellar-core prints once at exit, so a stream that - # ends a beat early has no total and nothing to recreate. + # Not gated on `recreated`: stellar-core prints the block once at exit, so a + # poller that ran start to finish but ended a beat early has no total. need_tx = tx.seconds is None if need_resume or need_tx: - archived = scan_archive(end, attempt, need_tx=need_tx) + archived = tx_scan.scan_archive(end, attempt, need_tx=need_tx) if tx.resumed or (archived is not None and archived.resumed): - # Not a peak -- PEAK_FIELDS filters it out of the profile. peaks_for_range - # reads it to decide how far back to aggregate: a resumed attempt only - # measured the tail of its range, so the attempt before it still counts. + # Not a peak -- PEAK_FIELDS filters it out of the profile. + # peaks_for_range reads it to decide how far back to aggregate: a + # resumed attempt only measured the tail of its range, so the attempt + # before it still counts. measured['resumed'] = True tx_seconds = tx.seconds if tx_seconds is None and archived is not None: @@ -769,19 +571,18 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): _wake.pop(pod, None) anon = _anon_peak.pop(pod, None) if anon is not None: - # Recorded for every attempt, not just the winner. peaks_for_range takes - # the max across attempts, so a partial attempt can only ever raise the - # figure, never lower it -- which is what makes a resumed range (pvc mode, - # killed once replay started) report the download-phase peak it actually - # hit rather than its tail. The monitor drops an attempt from the axis it - # died on, since an OOM-killed peak measures the limit, not demand. + # Recorded for every attempt, not just the winner: peaks_for_range takes + # the max across attempts, so a partial attempt can only raise the + # figure, which is what makes a resumed range report the download-phase + # peak it actually hit rather than its tail. The monitor drops an attempt + # from the axis it died on, since an OOM-killed peak measures the limit. measured['peakAnonBytes'] = anon ws = _ws_peak.pop(pod, None) if ws is not None: # Diagnostic only -- working set counts active page cache, which grows # to fill whatever limit the pod was given, so it must never size # anything. Kept because the anon/ws gap is what tells you a range is - # cache-heavy rather than genuinely large. + # cache-heavy rather than large. measured['peakWorkingSetBytes'] = ws eph = _eph_peak.pop(pod, None) if eph is not None: @@ -797,11 +598,9 @@ async def finalize(session, pod, end, attempt, tx, done_ok, started=None): else: logger.info("range %s attempt %s: stream complete", end, attempt) # Last, deliberately. The monitor treats this file as "the collector will - # write nothing further for this attempt" and only then reaps the Job -- - # which deletes the pod, the one place peaks can still be read from. It has - # to land after .metrics or it would license exactly the reap it exists to - # prevent. Inferring the same thing from peaks being present was wrong for - # an attempt that legitimately has none. + # write nothing further for this attempt" and only then reaps the Job, which + # deletes the pod -- the one place peaks can still be read from -- so it has + # to land after .metrics or it licenses the reap it exists to prevent. _mark_done(end, attempt) @@ -809,31 +608,29 @@ async def _poll_once(session, pod, end, attempt, last_ts, tx): """One short read of a pod's log. Returns (new_last_ts, gone). No follow=true: the request completes and the connection is released, so - concurrency is bounded by _poll_slots rather than by how many pods exist. - Measured on ssc-test, a single poll takes ~0.22s from outside the cluster, - so 2096 pods on a 10s interval need ~46 concurrent slots against the 2096 - permanently-held connections follow=true required. + concurrency is bounded by _poll_slots rather than by how many pods exist. A + single poll takes ~0.22s, so 2096 pods on a 10s interval need ~46 concurrent + slots against the 2096 permanently-held connections follow=true required. """ - params = {'container': CONTAINER, 'timestamps': 'true'} + params = {'container': cc.CONTAINER, 'timestamps': 'true'} if last_ts: # Second granularity, so this overlaps on purpose; the per-line # comparison below removes the overlap exactly. params['sinceTime'] = last_ts[:19] + 'Z' - url = f"{API}/api/v1/namespaces/{config.NAMESPACE}/pods/{pod}/log" + url = f"{kube_http.API}/api/v1/namespaces/{config.NAMESPACE}/pods/{pod}/log" async with _poll_slots: async with session.get(url, params=params, - headers={'Authorization': f'Bearer {token()}'}) as resp: + headers={'Authorization': f'Bearer {kube_http.token()}'}) as resp: if resp.status == 404: return last_ts, True resp.raise_for_status() # Chunked, not line-wise: aiohttp raises above 512 KiB on a single - # line, and a carriage-return progress meter trivially exceeds that - # -- one 628 MiB download arrived as a single "line". Split on \r as - # well, and cap what a pathological blob may buffer. + # line and a carriage-return progress meter trivially exceeds that + # -- one 628 MiB download arrived as a single "line". body = '' async for chunk in resp.content.iter_chunked(65536): body += chunk.decode('utf-8', 'replace') - if len(body) > MAX_POLL_CHARS: + if len(body) > cc.MAX_POLL_CHARS: break return _ingest(body, end, attempt, last_ts, tx), False @@ -843,29 +640,21 @@ def _ingest(body, end, attempt, last_ts, tx): """Append one block of timestamped log text to the archive; new last_ts. Split out of _poll_once so the doomed-pod follow stream lands its bytes - through exactly the same path -- dedup, gzip member framing, tx scanning - and resume-point bookkeeping. Two copies of this is how one route silently - stops feeding TxApplyScanner while the other keeps working. + through exactly the same path -- dedup, gzip member framing, tx scanning and + resume-point bookkeeping. Two copies is how one route silently stops feeding + TxApplyScanner while the other keeps working. """ pending = None lines = [l for l in re.split(r'[\r\n]', body) if l] if not lines: return last_ts - # Compressed into memory first, then appended in ONE write. - # - # Appending straight into the file with gzip.open(..., 'at') meant the - # deflate buffer flushed partial output to disk repeatedly across the whole - # loop, so for most of a large poll the archive on disk ended in a member - # with no end-of-stream marker. job_monitor reads that same file to recover - # txApplySeconds and gzip raises EOFError on a truncated member -- one - # in-flight poll could abort a reconcile pass for every range. The window is - # now a single append instead of the length of the write loop, and the file - # only ever gains whole members. - # - # Costs no more memory than is already held: `body` above is the entire - # poll uncompressed, and this is the same bytes compressed. Nothing is - # retained between polls, which is the property that got the sidecar off - # 1444 MiB of a 2048 MiB limit at 2096 follow streams. + # Compressed into memory first, then appended in ONE write, so the file only + # ever gains whole members. Appending with gzip.open(..., 'at') left the + # archive ending in a member with no end-of-stream marker for most of a large + # poll, and job_monitor reads that same file to recover txApplySeconds -- + # gzip raises EOFError on a truncated member, so one in-flight poll could + # abort a reconcile pass for every range. Costs no extra memory: `body` above + # is already the entire poll uncompressed, and nothing is held between polls. member = io.BytesIO() wrote = False with gzip.GzipFile(fileobj=member, mode='wb') as fh: @@ -883,7 +672,7 @@ def _ingest(body, end, attempt, last_ts, tx): wrote = True tx.feed(rest) pending = ts - path = base(end, attempt) + '.log.gz' + path = records.log_path(end, attempt) with open(path, 'ab') as out: # A poll whose lines were all deduped still touches the archive: its # existence is what job_monitor's backstop keys on. @@ -898,37 +687,31 @@ def _ingest(body, end, attempt, last_ts, tx): async def _follow_tail(session, pod, end, attempt, last_ts, tx): """Hold a follow=true stream on a doomed pod. Returns (last_ts, gone). - Opened only for pods the cluster has already condemned, so this is the one - place the cost of follow=true is worth paying: the connection is held for - the couple of minutes between the DisruptionTarget condition and the node - going away, not for the hours a range runs. - - Proven on ssc-test: with the stream held, SIGTERM to stellar-core yields - `got signal 15` -> `metric 'ledger.transaction.apply'` -> `Application - destroyed` inside 4ms, all of it captured. The same pod polled at 5s - intervals recorded `pod gone before disruption seen`. - - Bytes are ingested as they arrive rather than at end of stream, so a node - that disappears mid-read still leaves everything up to that point in the - archive. + Opened only for pods the cluster has already condemned, so the connection is + held for the couple of minutes before the node goes away, not for the hours + a range runs. Proven on ssc-test: with the stream held, SIGTERM yields `got + signal 15` -> `metric 'ledger.transaction.apply'` -> `Application destroyed` + inside 4ms, all captured, where the same pod polled at 5s recorded `pod gone + before disruption seen`. Bytes are ingested as they arrive, so a node that + disappears mid-read still leaves everything up to that point in the archive. """ - params = {'container': CONTAINER, 'timestamps': 'true', 'follow': 'true'} + params = {'container': cc.CONTAINER, 'timestamps': 'true', 'follow': 'true'} if last_ts: params['sinceTime'] = last_ts[:19] + 'Z' - url = f"{API}/api/v1/namespaces/{config.NAMESPACE}/pods/{pod}/log" - deadline = asyncio.get_event_loop().time() + DOOMED_FOLLOW_SECONDS + url = f"{kube_http.API}/api/v1/namespaces/{config.NAMESPACE}/pods/{pod}/log" + deadline = asyncio.get_event_loop().time() + cc.DOOMED_FOLLOW_SECONDS buf = '' if _follow_slots.locked(): # Every follow budget is spoken for, so this pod polls instead. Better - # than queueing: the pod has ~2 minutes to live and a queued follow that - # opens after it dies captures nothing while still holding a slot. + # than queueing: the pod has ~2 minutes to live, and a follow that opens + # after it dies captures nothing while still holding a slot. logger.info("range %s: no follow slot free (%d in use), polling instead", - end, MAX_DOOMED_FOLLOWS) + end, cc.MAX_DOOMED_FOLLOWS) _doomed.pop(pod, None) return await _poll_once(session, pod, end, attempt, last_ts, tx) async with _follow_slots: async with session.get(url, params=params, - headers={'Authorization': f'Bearer {token()}'}) as resp: + headers={'Authorization': f'Bearer {kube_http.token()}'}) as resp: if resp.status == 404: return last_ts, True resp.raise_for_status() @@ -942,15 +725,14 @@ async def _follow_tail(session, pod, end, attempt, last_ts, tx): buf = buf[cut + 1:] if asyncio.get_event_loop().time() > deadline: logger.info("range %s: doomed follow hit %.0fs, falling back to polling", - end, DOOMED_FOLLOW_SECONDS) + end, cc.DOOMED_FOLLOW_SECONDS) _doomed.pop(pod, None) break if buf: last_ts = _ingest(buf, end, attempt, last_ts, tx) - # One follow per pod. The stream ending means the container exited, and the - # caller must fall back to a normal poll for the terminal check and - # finalize; leaving the flag set would re-open a stream on a dead pod every - # iteration. The pod-list loop does not clear it -- by then the pod is gone. + # One follow per pod. The stream ending means the container exited, so the + # caller falls back to a normal poll for the terminal check and finalize; + # leaving the flag set would re-open a stream on a dead pod every iteration. _doomed.pop(pod, None) return last_ts, False @@ -958,17 +740,13 @@ async def _follow_tail(session, pod, end, attempt, last_ts, tx): async def poll_pod(session, pod, end, attempt, done, done_ok): """Read one pod's log to completion, by repeated short polls. - Replaces a follow=true stream. The stream held a connection, a gzip deflate - buffer and aiohttp read buffers for the pod's entire life, so cost scaled - with parallelism: measured at 2096 pods the sidecar sat at 1444 MiB of a - 2048 MiB limit with memory.events max=2617 and 1.00 of 2 cpu, which - extrapolates past both limits at 4096. Polling makes concurrency a tuning - parameter instead of a function of pod count. - - The one thing follow=true did better is the tail: it already held the bytes - when a pod died. So on seeing the pod go terminal this polls once more, - immediately, before finalizing -- without that, every spot eviction would - lose up to one interval of exactly the log we most want. + Replaces a follow=true stream, whose cost scaled with parallelism: it held a + connection, a deflate buffer and aiohttp buffers for the pod's entire life, + and at 2096 pods the sidecar sat at 1444 MiB of a 2048 MiB limit and 1.00 of + 2 cpu, extrapolating past both at 4096. The one thing follow did better is + the tail, so on seeing the pod go terminal this polls once more immediately + before finalizing -- without it every spot eviction loses up to one interval + of exactly the log we most want. """ last_ts = read_state(end, attempt) if last_ts is None: @@ -980,44 +758,40 @@ async def poll_pod(session, pod, end, attempt, done, done_ok): started = asyncio.get_event_loop().time() # Outside the poll loop: the medida block can straddle two polls, and a # fresh scanner per poll would lose the half it saw. - tx = TxApplyScanner(recreated=bool(last_ts)) - backoff = LOG_POLL_SECONDS + tx = tx_scan.TxApplyScanner(recreated=bool(last_ts)) + backoff = cc.LOG_POLL_SECONDS failures = 0 first_pass = True while True: was_terminal = done(pod) if first_pass and was_terminal: - # The pod was already terminal before this poller existed -- it - # finished while the collector was down, or between pod-list polls. - # `started` measures how long WE have been watching, which is about - # to be zero, not how long the container ran. Measured on ssc-test - # 2026-07-30 across two collector restarts: 150 metrics files - # recorded a sub-5s duration alongside a >500MiB anon peak. Report - # nothing rather than a fabricated near-zero; the monitor's own - # figure, from the pod's terminated timestamps, is authoritative and - # seconds_for_range prefers it anyway. + # The pod was already terminal before this poller existed, so + # `started` measures how long WE have been watching, not how long + # the container ran -- across two collector restarts, 150 metrics + # files recorded a sub-5s duration beside a >500MiB anon peak. + # Report nothing rather than a fabricated near-zero; the monitor's + # figure from the pod's own timestamps is authoritative anyway. started = None first_pass = False followed = False try: - if _doomed.get(pod) and not was_terminal and DOOMED_FOLLOW_SECONDS > 0: + if _doomed.get(pod) and not was_terminal and cc.DOOMED_FOLLOW_SECONDS > 0: followed = True - # Condemned and still running: stop sampling and hold the - # connection through the kill. Returns when the container exits - # or the notice is withdrawn, and the loop re-checks terminal - # immediately afterwards. + # Condemned and still running: hold the connection through the + # kill. Returns when the container exits or the notice is + # withdrawn, and the loop re-checks terminal immediately after. last_ts, gone = await _follow_tail( session, pod, end, attempt, last_ts, tx) else: last_ts, gone = await _poll_once( session, pod, end, attempt, last_ts, tx) - # Fallback interval for a condemned pod that could not follow: no - # slot was free, or following is disabled. 1s sampling alone still - # closes the ~9s window between the medida block and the pod object - # being deleted, so a mass reclaim degrades rather than loses. - backoff = (DOOMED_POLL_SECONDS if _doomed.get(pod) - else LOG_POLL_SECONDS) + # Fallback interval for a condemned pod that could not follow. 1s + # sampling alone still closes the ~9s window between the medida + # block and the object being deleted, so a mass reclaim degrades + # rather than loses. + backoff = (cc.DOOMED_POLL_SECONDS if _doomed.get(pod) + else cc.LOG_POLL_SECONDS) failures = 0 if gone: logger.info("pod %s gone before/while polling range %s", pod, end) @@ -1030,11 +804,11 @@ async def poll_pod(session, pod, end, attempt, done, done_ok): logger.info("range %s poll failed (%s); retrying from %s", end, e, last_ts or 'start') backoff = min(backoff * 2, 30) - if was_terminal and failures >= TERMINAL_POLL_ATTEMPTS: - # The container has exited and its log will not come back. A - # follow=true stream finalized here because it already held the - # bytes; polling has to decide to stop asking, or it spins on a - # dead pod for the rest of the run and never writes its metrics. + if was_terminal and failures >= cc.TERMINAL_POLL_ATTEMPTS: + # The container has exited and its log is not coming back. A + # follow stream finalized here because it already held the bytes; + # polling has to decide to stop asking, or it spins on a dead pod + # for the rest of the run and never writes its metrics. logger.warning("range %s attempt %s: %d failed polls after the pod " "went terminal; finalizing on what was read", end, attempt, failures) @@ -1048,33 +822,32 @@ async def poll_pod(session, pod, end, attempt, done, done_ok): await finalize(session, pod, end, attempt, tx, done_ok, started) return if followed: - # The follow only returns once the container has exited, so the - # very next read is the one that matters. Sleeping here would hand - # the interval back to exactly the race the follow exists to win. + # The follow only returns once the container has exited, so the very + # next read is the one that matters. Sleeping here would hand the + # interval back to the race the follow exists to win. continue # Not a blind sleep: a pod going terminal cuts it short. Polling faster - # would not help -- sinceTime has second granularity, so anything under - # ~1s re-reads the same second -- and the delay that matters is between - # the container exiting and the last read, not between routine polls. + # would not help -- sinceTime has second granularity -- and the delay + # that matters is between the container exiting and the last read, not + # between routine polls. ev = _wake.setdefault(pod, asyncio.Event()) try: await asyncio.wait_for(ev.wait(), timeout=backoff) except asyncio.TimeoutError: pass finally: - # Standard set/clear pairing. Left set, the Event makes every later - # wait return instantly, so the terminal-poll backoff never sleeps - # and TERMINAL_POLL_ATTEMPTS is spent in one millisecond -- the pod - # is given no time to have its final log become readable. A wake is - # consumed by the poll it triggers. + # Left set, the Event makes every later wait return instantly, so + # the terminal-poll backoff never sleeps and TERMINAL_POLL_ATTEMPTS + # is spent in one millisecond, giving the pod no time for its final + # log to become readable. A wake is consumed by the poll it triggers. ev.clear() async def list_pods(session): - url = f"{API}/api/v1/namespaces/{config.NAMESPACE}/pods" + url = f"{kube_http.API}/api/v1/namespaces/{config.NAMESPACE}/pods" params = {'labelSelector': f"{config.LABEL_RUN}={config.RUN_NAME}"} async with session.get(url, params=params, - headers={'Authorization': f'Bearer {token()}'}) as resp: + headers={'Authorization': f'Bearer {kube_http.token()}'}) as resp: resp.raise_for_status() return (await resp.json()).get('items', []) @@ -1082,20 +855,19 @@ async def list_pods(session): def ensure_stream(name, end, attempt, phase): """Open this pod's poller if it has none. Idempotent; returns whether it did. - Called by the watch as a pod appears and again if it is condemned, and by the - pod-list loop as a backstop for events the watch drops across a reconnect. + Called by the watch as a pod appears and again if it is condemned, and by + the pod-list loop as a backstop for events dropped across a reconnect. Opening a stream is time-critical -- a condemned pod is gone a second after - stellar-core exits -- so it must not be reachable only from a poll cycle. - Measured on the 900-worker run before this existed: the loop's cycle stretched - to 925s behind a serial kubelet sweep, and five -a2 legs lived and died with - no reader at all, one of them for 184.7s, losing txApply for good. + stellar-core exits -- so it must not be reachable only from a poll cycle: on + the 900-worker run the loop's cycle stretched to 925s and five -a2 legs + lived and died with no reader at all. """ if name in _tasks or name in _streamed or not _stream_ctx: return False - if phase not in POLLABLE_PHASES: - # Allowlist, not "skip Pending". A container that has not started answers - # 400 "waiting to start", and Unknown means the node stopped reporting. - # Both are retried on the cycle they become pollable. + if phase not in cc.POLLABLE_PHASES: + # Allowlist, not "skip Pending": a container that has not started + # answers 400 "waiting to start", and Unknown means the node stopped + # reporting. Both are retried on the cycle they become pollable. return False ctx = _stream_ctx _register_stream(name, end, attempt) @@ -1112,28 +884,23 @@ async def watch_condemnations(session): """Watch the run's pods and flag condemnations the moment they are written. Runs beside the pod-list loop rather than replacing it: the list still owns - discovery, task bookkeeping and finalize. This only ever sets _doomed - earlier than the list would have, which is the difference between opening a - follow while stellar-core is still running and opening it on a 404. - - Cheaper than the sweep it front-runs, too. list_pods re-serialises every pod - in the run every POLL_SECONDS; a watch is one connection served from the - apiserver's cache that sends only deltas. - - Never fatal. Any failure falls back to the list sweep, which is exactly the - behaviour that existed before this function. + discovery, bookkeeping and finalize, and this only ever sets _doomed earlier + than the list would have -- the difference between opening a follow while + stellar-core is still running and opening it on a 404. Cheaper than the + sweep it front-runs, too: one connection served from the apiserver's cache, + sending only deltas. Never fatal -- any failure falls back to the sweep. """ - url = f"{API}/api/v1/namespaces/{config.NAMESPACE}/pods" + url = f"{kube_http.API}/api/v1/namespaces/{config.NAMESPACE}/pods" rv = None while True: params = {'labelSelector': f"{config.LABEL_RUN}={config.RUN_NAME}", 'watch': 'true', 'allowWatchBookmarks': 'true', - 'timeoutSeconds': str(WATCH_TIMEOUT_SECONDS)} + 'timeoutSeconds': str(cc.WATCH_TIMEOUT_SECONDS)} if rv: params['resourceVersion'] = rv try: async with session.get(url, params=params, - headers={'Authorization': f'Bearer {token()}'}) as resp: + headers={'Authorization': f'Bearer {kube_http.token()}'}) as resp: if resp.status == 410: # Our resourceVersion aged out of the apiserver's history. # Restarting without one re-syncs; the list sweep covers the @@ -1165,9 +932,10 @@ async def watch_condemnations(session): continue name = meta.get('name') attempt = labels.get(config.LABEL_ATTEMPT, '1') - # Before the condemnation check: a pod condemned in the same - # event it first becomes pollable needs the poller to exist - # first, or there is nothing for _mark_condemned to wake. + # Order is not load-bearing: create_task only schedules the + # poller, so _wake has no entry yet either way, and poll_pod + # reads _doomed at the top of its first pass. The wake below + # is for a poller from an earlier event, already asleep. ensure_stream(name, end, attempt, (obj.get('status') or {}).get('phase')) _mark_condemned(obj, name, end, attempt) @@ -1175,175 +943,7 @@ async def watch_condemnations(session): raise except Exception as exc: logger.warning("condemnation watch dropped (%s); retrying", exc) - await asyncio.sleep(WATCH_RETRY_SECONDS) - - -async def main(): - os.makedirs(config.LOG_DIR, exist_ok=True) - # Connection-pool limit, not a task limit: there is no semaphore above it, - # so a stream that cannot get a connection blocks here for as long as the - # pool stays full -- and every holder is a follow=true stream open for the - # life of its pod. Below the live pod count this does not degrade, it - # starves, and it starves the pods created last, which are the retries. - # Sized for concurrent polls plus headroom for the pod-list and kubelet - # calls, not for one connection per pod. Under follow=true this had to - # exceed parallelism or pods silently starved -- 1200 against 2048 workers - # left 896 blocked forever, and retries, created last, never got a slot. - conn = aiohttp.TCPConnector( - limit=MAX_CONCURRENT_POLLS + MAX_DOOMED_FOLLOWS + 64, ssl=ssl_ctx()) - # No total timeout: these streams are meant to stay open for the life of a - # range, which can be hours. - timeout = aiohttp.ClientTimeout(total=None, sock_connect=10) - # tasks/streamed are the module-level registry under local names, so the - # bookkeeping below is unchanged while the watch shares the same guard. - tasks, streamed = _tasks, _streamed - # Cleared rather than assumed empty: a second main() in one process would - # otherwise find every pod already registered and open no streams at all. - tasks.clear() - streamed.clear() - _stream_ctx.clear() - terminal, succeeded, vanished = {}, {}, {} - # `streamed` holds streams that ran to completion. Without it a finished task - # is deleted from `tasks` and the next poll re-opens the stream, forever: one - # full log re-read per pod every POLL_SECONDS, which at 1024 workers is a lot - # of apiserver -- measured, the completion block ran once per range per cycle - # for the rest of the run. - - async with aiohttp.ClientSession(connector=conn, timeout=timeout) as session: - logger.info("streaming logs for run=%s into %s", config.RUN_NAME, config.LOG_DIR) - # Published before the watch starts: ensure_stream is a no-op until this - # exists, so a watch event arriving first would silently open nothing. - _stream_ctx.update(session=session, terminal=terminal, succeeded=succeeded) - if WATCH_TIMEOUT_SECONDS > 0: - asyncio.create_task(watch_condemnations(session)) - while True: - try: - pods = await list_pods(session) - live = {p['metadata']['name'] for p in pods} - # A pod can leave the list without ever being observed terminal: - # Karpenter reaps the node, the kubelet evicts it, or the monitor - # deletes its finished Job. `terminal` is only written for pods - # in this list, so those would keep done() False forever and - # their stream would retry until the run ended. Marking them - # terminal lets the stream finalize on its own and free the slot; - # cancelling is the backstop for one wedged inside a connection - # attempt it will never win. - for name in [n for n in tasks if n not in live]: - terminal[name] = True - if name in _wake: - # Gone is terminal. Without this its poller sleeps out - # the interval before taking the 404, delaying finalize - # and the .done that lets the monitor reap the Job. - _wake[name].set() - t = tasks[name] - if t.done(): - del tasks[name] - streamed.add(name) - continue - vanished[name] = vanished.get(name, 0) + 1 - if vanished[name] >= VANISHED_GRACE_CYCLES: - t.cancel() - try: - await t - except asyncio.CancelledError: - pass - del tasks[name] - vanished.pop(name, None) - ref = _streaming.get(name) - if ref is not None: - # The poller was wedged, but its archive and the - # sampler's process-local peaks still contain useful - # truth. Finalize them before licensing a reap. - await finalize( - session, name, ref[0], ref[1], - TxApplyScanner(recreated=True), - lambda p: succeeded.get(p, False)) - streamed.add(name) - logger.info("cancelled and finalized stream for vanished pod %s", - name) - for pod in pods: - name = pod['metadata']['name'] - labels = pod['metadata'].get('labels', {}) - end = labels.get(config.LABEL_RANGE) - if end is None: - continue - phase = pod.get('status', {}).get('phase') - terminal[name] = phase in ('Succeeded', 'Failed') - # NOT gated on phase. A pod that is being DELETED keeps - # phase Running until its object disappears -- deletion - # never sets Succeeded or Failed -- so gating this on - # terminal meant no disrupted pod ever recorded an exact - # duration, and every one of them fell back to the poller's - # own clock. Measured on ssc-test: 268s reported against a - # ~500s attempt, because that clock starts when the POLLER - # opened, not when the container did. The container's - # terminated.finishedAt is present for the ~8s the object - # outlives it, and pod_seconds returns None until then, so - # asking every cycle is self-guarding. - secs = pod_seconds(pod) - if secs is not None: - _pod_secs[name] = secs - start = (pod.get('status') or {}).get('startTime') - if start: - # Second line: if the object is deleted before any cycle - # catches its terminated timestamp, finalize can still - # date the attempt from when the container STARTED - # rather than from when this poller happened to open. - _pod_start[name] = start - # Backstop only: the watch normally gets here first. This - # still runs so detection survives the watch being disabled - # or reconnecting. - if not terminal[name]: - _mark_condemned(pod, name, end, - labels.get(config.LABEL_ATTEMPT, '1')) - if terminal[name] and name in _wake: - # Wake its poller now rather than at the next tick. - _wake[name].set() - succeeded[name] = phase == 'Succeeded' - if phase == 'Failed': - record_outcome(pod, end, labels.get(config.LABEL_ATTEMPT, '1')) - if name in tasks and not tasks[name].done(): - continue - if name in tasks and tasks[name].done(): - del tasks[name] - # Only bar a re-open once the pod itself is terminal. A - # task that ended while the pod is still running died - # early, and re-opening is the recovery path. - if terminal.get(name): - streamed.add(name) - continue - if name in streamed: - continue - # Backstop. The watch normally opens this the moment the pod - # appears; this covers events dropped across a reconnect. - # Same registry and the same guard, so whichever gets there - # first wins and the other no-ops -- two readers on one pod - # would duplicate the archive and race write_state. - ensure_stream(name, end, labels.get(config.LABEL_ATTEMPT, '1'), phase) - - # AFTER the per-pod branches, never before them. This is a serial - # sweep of every node's kubelet, and on spot a dead one costs the - # 10s connect timeout apiece -- measured, that stretched one cycle - # to 925s. Ahead of the branches it delayed every stream by that - # much; behind them it delays only the next cycle's sampling. - # It must stay outside the `for` loop, though: those branches end - # in `continue` for a pod already streaming, so a sampler placed - # among them fires only on the cycle a stream opens, when the - # range has barely written anything. - # - # Unconditional: this used to be gated on ephemeral mode, back - # when it only sampled disk. Memory is sized in both modes, so - # gating it here left every pvc run with no anon peak at all. - # hostIP, not nodeName: the sampler talks to the kubelet - # directly, and this list already carries the address, so it - # costs no read of Node objects. - await sample_kubelet(session, { - p['status']['hostIP'] for p in pods - if p.get('status', {}).get('hostIP') - and p.get('status', {}).get('phase') == 'Running'}) - except Exception as e: - logger.warning("pod list failed: %s", e) - await asyncio.sleep(POLL_SECONDS) + await asyncio.sleep(cc.WATCH_RETRY_SECONDS) if __name__ == '__main__': diff --git a/src/MissionParallelCatchup/lib/collector/collector_config.py b/src/MissionParallelCatchup/lib/collector/collector_config.py new file mode 100644 index 00000000..28e81e70 --- /dev/null +++ b/src/MissionParallelCatchup/lib/collector/collector_config.py @@ -0,0 +1,82 @@ +"""Settings for the log-collector sidecar, read from the environment once. + +Every value the chart can tune lives here rather than beside the code that reads +it, so log_collector.py opens on its own logic. Named collector_config, not +config: runtime flattens lib/ into a single /app, so a second config.py would +silently replace the shared one. +""" + +import os + +# The worker container whose log is collected. Sidecars share the pod, so this +# is what keeps a peak or a log line from being attributed to the wrong one. +CONTAINER = os.getenv('WORKER_CONTAINER', 'stellar-core') +# Seconds between sweeps of the run's pod list, which is what discovers new pods +# and notices ones that went terminal. +POLL_SECONDS = float(os.getenv('COLLECTOR_POLL_SECONDS', 5)) +# Poll cycles a stream gets to finalize itself after its pod leaves the pod list +# before it is cancelled outright. One cycle is usually enough; the margin is for +# a stream still finalizing: writing its .metrics and closing its archive. +VANISHED_GRACE_CYCLES = int(os.getenv('COLLECTOR_VANISHED_GRACE_CYCLES', 3)) +# Peak memory comes from kubelet's /stats/summary (rssBytes, workingSetBytes), +# already fetched for ephemeral storage: ~10s cAdvisor housekeeping against a 30s +# scrape, and no dependence on Prometheus being up, reachable, or still retaining +# the window -- failures the old _promql helper all swallowed into "no peak". +# Peaks are held per pod and flushed on this much growth, so a restart loses at +# most that fraction of a range's high-water. cpu is not sampled: the request is +# fixed at REQ_CPU, so a measured value has nothing to size. +PEAK_FLUSH_RATIO = float(os.getenv('PEAK_FLUSH_RATIO', 1.05)) +# Seconds between polls of one pod's log. Latency here is archive lag, not +# anything a decision waits on; 4096 pods at 10s is ~90 concurrent polls. +LOG_POLL_SECONDS = float(os.getenv('LOG_POLL_SECONDS', 10)) +# Concurrent in-flight log polls, across all pods. This is the whole point of +# polling: it is independent of how many pods exist, where follow=true needed one +# held connection per pod forever. +MAX_CONCURRENT_POLLS = int(os.getenv('MAX_CONCURRENT_POLLS', 96)) +# Most one poll may read before it stops, bounding a single response; the next +# poll picks up from the timestamp this one reached. Also the backstop against a +# blob that never terminates -- a progress meter emitting only carriage returns +# would otherwise grow the buffer until the sidecar OOMs. +MAX_POLL_CHARS = int(os.getenv('MAX_POLL_CHARS', 8388608)) +# Longest a follow stream will hang on to a doomed pod. Spot gives 120s; past +# roughly double that the notice was withdrawn and the stream would otherwise be +# held for the life of the range. 0 disables the follow path entirely, which is +# only safe when prestopSleepSeconds is holding the pod open instead. +DOOMED_FOLLOW_SECONDS = float(os.getenv('DOOMED_FOLLOW_SECONDS', 300)) +# Follows get their OWN budget rather than sharing the poll slots: a follow holds +# its slot for the whole drain, so a reclaim condemning more pods than there are +# slots would consume all of them and turn a partial node loss into a run-wide +# blackout. With a separate budget the worst case is that some condemned pods +# fall back to polling, which captured txApply on its own with no follow at all. +# Sized above any plausible simultaneous disruption -- a whole-AZ reclaim is not +# bounded by Karpenter's disruption budget -- and 256 x the old always-on +# design's 0.69 MiB per stream is 177 MiB against a 2048 MiB limit. +MAX_DOOMED_FOLLOWS = int(os.getenv('MAX_DOOMED_FOLLOWS', 256)) +# Poll interval for a condemned pod, replacing LOG_POLL_SECONDS while it is +# doomed. preStop delays SIGTERM but leaves the gap between the medida block and +# the pod object being deleted at ~9s, which a blind 10s poll straddles -- it +# did, losing txApply even with a 60s preStop; polling that window every second +# cannot miss it. Costs no held connections, and sinceTime's 1s granularity makes +# anything below 1s a re-read of the same second. +DOOMED_POLL_SECONDS = float(os.getenv('DOOMED_POLL_SECONDS', 1)) +# How long each watch connection lives before the apiserver closes it and we +# reconnect, bounded so a silently-dead stream self-heals; the reconnect resumes +# from the last resourceVersion, so nothing is missed across it. 0 disables the +# watch and leaves detection to the pod list. +WATCH_TIMEOUT_SECONDS = int(os.getenv('WATCH_TIMEOUT_SECONDS', 600)) +# Pause before re-opening a watch that failed. Only covers hard errors: a clean +# timeout reconnects immediately. +WATCH_RETRY_SECONDS = float(os.getenv('WATCH_RETRY_SECONDS', 1)) +# Failed polls tolerated after a pod goes terminal before we stop asking. Its log +# is not coming back and spinning holds a task and a poll slot for the rest of +# the run, but a couple of retries still absorb the transient 500s that arrive in +# bursts at ramp. +TERMINAL_POLL_ATTEMPTS = int(os.getenv('TERMINAL_POLL_ATTEMPTS', 3)) + +# Fields that only ever grow. write_metrics maxes these instead of overwriting, +# so a restarted poller starting its high-water at zero cannot lower one. +PEAK_KEYS = ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes') +# Phases whose log endpoint can actually answer. Pending has no container yet and +# Unknown means the node stopped reporting; the terminal phases are kept because +# that is where a pod's final output lives. +POLLABLE_PHASES = ('Running', 'Succeeded', 'Failed') diff --git a/src/MissionParallelCatchup/lib/collector/kube_http.py b/src/MissionParallelCatchup/lib/collector/kube_http.py new file mode 100644 index 00000000..507069d7 --- /dev/null +++ b/src/MissionParallelCatchup/lib/collector/kube_http.py @@ -0,0 +1,27 @@ +"""How the collector reaches the apiserver and the kubelet. + +The monitor uses the kubernetes client (see kube); the collector talks raw +aiohttp because it needs streaming log reads, so it carries its own credentials +and endpoints. +""" + +import os +import ssl + +SA = '/var/run/secrets/kubernetes.io/serviceaccount' +API = (f"https://{os.getenv('KUBERNETES_SERVICE_HOST', 'kubernetes.default')}" + f":{os.getenv('KUBERNETES_SERVICE_PORT', '443')}") +# Read-only kubelet port. A seam for tests, which serve the payload on a loopback +# port rather than reaching a real node. +KUBELET_PORT = 10250 + + +def token(): + # Projected service account tokens rotate, so this is re-read per request + # rather than cached at startup. + with open(os.path.join(SA, 'token')) as fh: + return fh.read().strip() + + +def ssl_ctx(): + return ssl.create_default_context(cafile=os.path.join(SA, 'ca.crt')) diff --git a/src/MissionParallelCatchup/lib/medida.py b/src/MissionParallelCatchup/lib/collector/medida.py similarity index 100% rename from src/MissionParallelCatchup/lib/medida.py rename to src/MissionParallelCatchup/lib/collector/medida.py diff --git a/src/MissionParallelCatchup/lib/collector/tx_scan.py b/src/MissionParallelCatchup/lib/collector/tx_scan.py new file mode 100644 index 00000000..a714e238 --- /dev/null +++ b/src/MissionParallelCatchup/lib/collector/tx_scan.py @@ -0,0 +1,107 @@ +"""Reading the tx-apply total out of a worker's log. + +The collector scans the live stream as it goes past and re-reads its own archive +at finalization when the stream missed the block -- stellar-core prints it once, +just before exit, so a stream that ends a beat early has no total. Scanning here +rather than in job_monitor is what makes the metric independent of pod lifetime: +by the time the Job is seen to succeed the node may be reaped, and with +saveSuccessLogs=false the archive is gone too. +""" + +import gzip +import logging +import zlib + +import medida +import records + +logger = logging.getLogger('log_collector') + +TX_METRIC = "metric 'ledger.transaction.apply'" + +class TxApplyScanner: + """Pull the medida tx-apply total out of the stream as it goes past. + + stellar-core prints this block once, just before exit. Scanning here rather + than re-reading the log later is what makes the metric independent of pod + lifetime: by the time job_monitor sees the Job succeed the node may be + reaped, and with saveSuccessLogs=false the archive is gone too. + """ + + # Shared with job_monitor's archive re-read rather than restated: a + # divergence would hand the recovery path the same blind spot it exists to + # cover. A /info liveness response interleaved into the block once put `sum` + # 91 lines below the header, 76 lines past where both readers gave up. + WINDOW = medida.WINDOW + HARD_WINDOW = medida.HARD_WINDOW + + # Printed by RESUME_SCRIPT before stellar-core starts. Its counterpart, + # "RESUME DECLINED", means new-db ran and this attempt did the whole range, + # so the colon is load-bearing -- it is what separates the two. + RESUME_MARK = 'RESUME: ' + RESUME_DECLINED_MARK = 'RESUME DECLINED:' + + def __init__(self, recreated=False): + self.seconds = None + self.resumed = False + self.resume_decided = False + # A new poller starting from durable .state missed every earlier line. + # Finalization must recover scanner-only facts from the archive. + self.recreated = recreated + self._left = 0 + self._span = 0 + + def feed(self, line): + if self.RESUME_MARK in line: + self.resumed = True + self.resume_decided = True + elif self.RESUME_DECLINED_MARK in line: + self.resume_decided = True + if TX_METRIC in line: + self._left = self.WINDOW + self._span = self.HARD_WINDOW + return + if self._left <= 0: + return + m = medida.SUM_RE.search(line) + if m: + self.seconds = float(m.group(1)) / 1000.0 + self._left = 0 + return + self._span -= 1 + if self._span <= 0 or medida.ANY_METRIC.search(line): + # Ran out of rope, or another timer's block started -- either way our + # sum is not coming. + self._left = 0 + return + if medida.METRIC_LINE.search(line): + # Only medida's own statistics count. Interleaved output from another + # thread is noise between us and the sum, not evidence we have passed + # it. + self._left -= 1 + + + +def scan_archive(end, attempt, need_tx=False): + """Recover scanner state from complete gzip members already on disk.""" + path = records.log_path(end, attempt) + scanner = TxApplyScanner() + try: + with gzip.open(path, 'rt', errors='replace') as fh: + for line in fh: + scanner.feed(line) + # The resume decision is at process startup. Avoid decompressing + # a multi-gigabyte worker log when that is all the caller needs. + if scanner.resume_decided and not need_tx: + break + except FileNotFoundError: + return scanner + except (EOFError, gzip.BadGzipFile, zlib.error) as e: + # Keep facts found in complete prefix members. A torn final member cannot + # invalidate an earlier RESUME line or complete medida block. + logger.warning("could only partially recover scanner state from %s: %s", path, e) + except OSError as e: + logger.warning("could not open scanner archive %s: %s", path, e) + return scanner + + diff --git a/src/MissionParallelCatchup/lib/collector/verdicts.py b/src/MissionParallelCatchup/lib/collector/verdicts.py new file mode 100644 index 00000000..07c7a3c4 --- /dev/null +++ b/src/MissionParallelCatchup/lib/collector/verdicts.py @@ -0,0 +1,72 @@ +"""Why an attempt ended, decided from the pod while the pod still exists. + +The collector already lists every pod every few seconds to discover streams, so +it sees terminal transitions first-hand. The Job object cannot answer this: its +condition carries no exit code until a podFailurePolicy rule matches, and an +admission rejection matches none. +""" + +import json +import logging +import os + +import records + +logger = logging.getLogger('log_collector') + + +def condemnation_reason(pod): + """The DisruptionTarget reason if the cluster has committed to destroying + this pod, else None -- a spot reclaim, a drain or node pressure. + """ + for cond in ((pod.get('status') or {}).get('conditions') or []): + if cond.get('type') == 'DisruptionTarget' and cond.get('status') == 'True': + return cond.get('reason') or 'Unknown' + return None + + +def classify(pod): + """The outcome this pod's status implies, as a verdict dict.""" + status = pod.get('status', {}) + if condemnation_reason(pod): + return {'outcome': 'disrupted', 'exitCode': None} + reason = status.get('reason') + if reason == 'Evicted' and 'ephemeral' in (status.get('message') or ''): + # The range's own disk use, not something the cluster did to it. The + # kubelet sets no DisruptionTarget for a limit eviction and stellar-core + # exits 3 on the eviction SIGTERM, so the Job condition reads as a plain + # catchup failure, which gets no retry at all. status.message is the only + # discriminator and only the pod carries it, so it must be caught here. + return {'outcome': 'ephemeral', 'exitCode': None, 'reason': status.get('message')} + if reason in ('VolumeAttachmentLimitExceeded', 'OutOfcpu', 'OutOfmemory', 'OutOfpods', + 'UnexpectedAdmissionError', 'NodeAffinity', 'Shutdown', 'Evicted'): + return {'outcome': 'rejected', 'exitCode': None, 'reason': reason} + terms = [cs.get('state', {}).get('terminated') for cs in status.get('containerStatuses', [])] + terms = [t for t in terms if t] + if not terms: + # Nothing ever ran, so this says nothing about the ledger range. + return {'outcome': 'rejected', 'exitCode': None, 'reason': reason or 'no container status'} + for t in terms: + if t.get('reason') == 'OOMKilled': + return {'outcome': 'oom', 'exitCode': t.get('exitCode')} + if t.get('exitCode') not in (0, None): + return {'outcome': 'failed', 'exitCode': t.get('exitCode')} + # Terminated, but not one container said with what. `failed` here is a lie + # that costs the whole run -- it reads as a genuine catchup failure, the one + # outcome that gets no retry: range 59018943 was condemned on attempt 1 with + # exitCode null, failing a mission that was otherwise 554 for 554. + return {'outcome': 'unknown', 'exitCode': None} + + +def record_outcome(pod, end, attempt): + """Write the verdict next to the log, for job_monitor's reconcile to read.""" + path = records.outcome_path(end, attempt) + if os.path.exists(path): + return + data = classify(pod) + data['pod'] = pod['metadata']['name'] + try: + records.write_atomic(path, json.dumps(data)) + logger.info("range %s attempt %s classified: %s", end, attempt, data['outcome']) + except OSError as e: + logger.warning("could not persist outcome for range %s: %s", end, e) diff --git a/src/MissionParallelCatchup/lib/config.py b/src/MissionParallelCatchup/lib/config.py index 1e2590eb..1c8ff3a7 100644 --- a/src/MissionParallelCatchup/lib/config.py +++ b/src/MissionParallelCatchup/lib/config.py @@ -1,555 +1,52 @@ -"""Configuration and run state for the parallel catchup job monitor. +"""What the monitor and the log-collector must agree on. + +Both processes run from the same image and share one /logs volume, so these are +the names that have to mean the same thing in both: which run they belong to, +where its files are, and the vocabulary of an attempt's verdict. The monitor's +own settings live in monitor_config; the collector's in collector_config. Read through the module, never copied out of it: import config - ... config.REQ_CPU ... + ... config.LOG_DIR ... -`from config import REQ_CPU` binds a COPY. A test's monkeypatch and the startup -assignment of config.PROFILE both rebind the attribute on this module, and a -copy taken at import time never sees either -- silently, with the test passing -against the default. A module object is a singleton, so reading through it is -what makes those visible everywhere. +`from config import LOG_DIR` binds a COPY. A test's monkeypatch rebinds the +attribute on this module, and a copy taken at import time never sees it -- +silently, with the test passing against the default. A module object is a +singleton, so reading through it is what makes those visible everywhere. """ import os -# ============================================================================= -# 1. stellar-core workload -# ============================================================================= -CORE_IMAGE = os.getenv('CORE_IMAGE') - -ASAN_OPTIONS = os.getenv('ASAN_OPTIONS', '') - -# Which ledger ranges to run. These are pure inputs to the range generator: -# dispatch recomputes the whole list every reconcile, so a restart must -# reproduce it exactly. - - -# Both generators emit tip-first, which front-loads the most expensive ranges: -# the bucket set only grows with ledger position. 'oldest-first' reverses that, -# so a profiling run measures the cheap early ranges before it can be -# interrupted, and the expensive tip ranges last. -RANGE_ORDER = 'tip-first' # from /start: tip-first | oldest-first | longest-first - -VALID_RANGE_ORDERS = ('tip-first', 'oldest-first', 'longest-first') - -STARTING_LEDGER = 0 # from /start - -LATEST_LEDGER_NUM = 0 # from /start - -LEDGERS_PER_JOB = 16000 # from /start - -OVERLAP_LEDGERS = 320 # from /start - - -# ============================================================================= -# 2. Kubernetes objects this monitor creates -# ============================================================================= +# The run, and every object that belongs to it. The collector selects pods on +# LABEL_RUN=RUN_NAME, so a disagreement here has it watching a different run -- +# or nothing at all. NAMESPACE = os.getenv('NAMESPACE', 'default') RUN_NAME = os.getenv('RUN_NAME', 'parallel-catchup') - LABEL_RUN = 'catchup.stellar.org/run' LABEL_RANGE = 'catchup.stellar.org/range-end' LABEL_ATTEMPT = 'catchup.stellar.org/attempt' -# Workers need IRSA to read the S3 history mirror. Without it they silently fall -# back to the public archive, which throttles at 1024 and kills the run with -# curl 22 -> catchup exit 3. The name matches the old StatefulSet's so existing -# IRSA trust policies keep matching. -WORKER_SERVICE_ACCOUNT = os.getenv('WORKER_SERVICE_ACCOUNT', '') - -# Pod resources. Requests only: workers are given no cpu limit and no memory -# limit at all. -# -# CPU because a limit only throttles a pod that could otherwise use idle cores, -# and throttling changes what the range measures -- less cpu means less download -# concurrency means a lower peak, so a throttled attempt records a figure an -# unthrottled one cannot reproduce. -# -# Memory because a limit is a hard cap on anon PLUS page cache, and sizing it -# per-range from a profile got it wrong in the one direction that has no alarm -# on it. Measured 2026-07-31, range 39210943: sized at 1729Mi from a neighbour, -# genuinely needed 1620Mi of anon, which left ~110Mi for cache. It never OOMed -# -- it thrashed. 544k major page faults, 0.22 cores used on a node it had -# entirely to itself, 0.95 ledgers/s against a neighbour norm of 3.3, and it -# held 1092 idle slots open for three hours at the end of the run. -# -# Without a limit the request still does the real work: it places the pod and -# it sets eviction order under node pressure. What goes away is the cliff. -REQ_CPU = os.getenv('REQ_CPU', '1250m') - -REQ_MEM = os.getenv('REQ_MEM', '9Gi') - -# Range profile from an earlier run: tightens per-range requests so more -# workers fit per node. Requests only -- limits stay as configured, so the -# failure semantics and the OOM/disk escalation ladders are unchanged. -PROFILE_PATH = os.getenv('PROFILE_PATH', '') -# The run document /start delivers, kept so a restart resumes the same run. -RUN_PATH = '' - -PROFILE_MARGIN = float(os.getenv('PROFILE_MARGIN', 1.15)) - -# No safety margin on cpu, unlike memory. Under-requesting cpu costs contention -# and the pod can still burst; under-requesting memory gets it OOMKilled. -# Ceiling for profile-derived memory, above the unprofiled limit for the same -# reason: a range that really needs more than the configured limit must be able -# to ask for it rather than be pinned under its own measured peak. The OOM -# escalation ladder can still climb past this on a retry. -PROFILE_MAX_MEM = os.getenv('PROFILE_MAX_MEM', '32Gi') - -# Memory is sized from rss (the range's real demand), NOT from peak working -# set. Working set is whatever limit it was measured under -- the kernel grows -# page cache to fill it -- so sizing from it is circular. Measured on ssc-test -# with one 420-ledger range: working set went 2.33 -> 3.61 -> 7.48 -> 13.49 GiB -# under 2560Mi/4Gi/8Gi/24000Mi limits while rss moved only 2256 -> 2488 MiB, and -# wall-clock did not move at all (776s / 775s / 746s / 773s). Catchup streams -- -# buckets are downloaded once, applied once, ledgers replayed once -- so cache -# has nothing to give back and PROFILE_MARGIN alone is the allowance. -# A multiplicative margin alone is not enough: memory.max bounds anon PLUS page -# cache, and at small rss 10% is nothing. Measured on ssc-test 2026-07-29 with -# headroom 0: ranges profiled at 190 MiB rss got a 209 MiB limit -- 19 MiB of -# slack for all growth and cache -- and 90 of them OOMKilled within 90s. The -# earlier 4Gi validation hid this because 1.1x of 2.4 GiB is 240 MiB of slack. -PROFILE_CACHE_HEADROOM = os.getenv('PROFILE_CACHE_HEADROOM', '512Mi') - -# Extra allowance scaled by the range's measured runtime. Long ranges keep more -# page cache and allocator slack live at once; 0 disables the allowance. -PROFILE_RUNTIME_MEMORY_INSURANCE = os.getenv('PROFILE_RUNTIME_MEMORY_INSURANCE', '3Gi') - -# Ephemeral-storage gets the same two allowances as memory, for the same -# reasons. Measured on the 2026-08-01 on-demand run: peak 37.76Gi against a -# flat 40Gi limit -- 6% of headroom on a path that has never once fired in a -# real run, so a range 6% worse than the worst seen would be evicted 137 with -# no diagnostic pointing at disk. -# -# Flat allowance added to every range's measured peak. Covers the container -# image, logs and the sqlite WAL, none of which scale with the range. -PROFILE_EPHEMERAL_HEADROOM = os.getenv('PROFILE_EPHEMERAL_HEADROOM', '2Gi') - -# Runtime-weighted allowance on top. Disk tracks runtime closely (pearson 0.920 -# across 3985 ranges: runtime decile 0 uses 0.1Gi, decile 9 uses 24.7Gi), so -# the ranges that need the margin are exactly the ranges this gives it to. -PROFILE_RUNTIME_EPHEMERAL_INSURANCE = os.getenv('PROFILE_RUNTIME_EPHEMERAL_INSURANCE', '8Gi') - -# Ceiling for profile-derived disk. Deliberately ABOVE LIM_EPHEMERAL: that flat -# limit is what an UNMEASURED range gets, and capping a measured range at it -# would throw away the measurement -- the worst observed range wants 43Gi after -# margin alone. -PROFILE_MAX_EPHEMERAL = os.getenv('PROFILE_MAX_EPHEMERAL', '64Gi') - -REQ_EPHEMERAL = os.getenv('REQ_EPHEMERAL', '') - -LIM_EPHEMERAL = os.getenv('LIM_EPHEMERAL', '') - -# Placement. The taint toleration is emitted as {key, effect} with no value: -# the default Equal operator does not match "" against "true". -NODE_LABEL_KEY = os.getenv('NODE_LABEL_KEY', '') - -NODE_LABEL_VALUE = os.getenv('NODE_LABEL_VALUE', '') - -# Further labels a node must carry, "key:value" comma separated, ANDed with the -# one above. Unlike that one these are literal -- the pair above is pool-routed, -# its value replaced per range with -. -# -# This is where a run pins itself to one capacity of a tier. Both capacities -# carry the same tier label value, so nothing else separates them, and the -# pairing matters: ephemeral has no resume, so a reclaim costs the whole range. -# A plain label rather than karpenter.sh/capacity-type, because the pools -# publish their own and the monitor has no business knowing who provisioned the -# node. -REQUIRE_NODE_LABELS = os.getenv('REQUIRE_NODE_LABELS', '') - - -def label_pairs(raw): - """[(key, value)] from "k:v,k:v". Entries without a value are dropped: a - key alone would require the label be exactly "", which no node carries, and - a pod pinned to nothing sits Pending in a way that reads as slow - provisioning rather than as misconfiguration.""" - out = [] - for item in (raw or '').split(','): - key, _, value = item.strip().partition(':') - if key and value: - out.append((key, value)) - return out - -AVOID_NODE_LABEL_KEY = os.getenv('AVOID_NODE_LABEL_KEY', '') - -AVOID_NODE_LABEL_VALUE = os.getenv('AVOID_NODE_LABEL_VALUE', '') +# The shared volume. The collector owns writes here -- it streams each worker's +# log and records the .outcome verdict while the pod still exists -- and the +# monitor reads them back during reconcile. +LOG_DIR = os.getenv('LOG_DIR', '/logs') -TOLERATE_TAINT = os.getenv('TOLERATE_TAINT', '') +SAVE_SUCCESS_LOGS = os.getenv('SAVE_SUCCESS_LOGS', 'true').lower() == 'true' # Worker /data. pvc keeps it across pods, so an evicted range resumes at L+1 -- # that is what makes spot viable. ephemeral puts it on the node disk: denser # packing, no resume, and REQ_EPHEMERAL must be sized to hold the catchup DB. # One PVC per range, not per concurrency slot: measured on ssc-test, 300 jobs # with a PVC each cost no more wall-clock than 300 jobs reusing 40. +# Read by both: it also decides whether the collector samples ephemeral disk. STORAGE_MODE = os.getenv('STORAGE_MODE', 'pvc') # pvc | ephemeral -STORAGE_CLASS = os.getenv('STORAGE_CLASS', '') - -# 60Gi to match the tier nodes' ephemeral allowance. peakEphemeralBytes tops -# out at 37.8Gi across the whole 2026-08-01 profile, so this covers every -# range measured, with headroom for the tip to keep growing. -STORAGE_SIZE = os.getenv('STORAGE_SIZE', '60Gi') - -# Job/pod lifetimes. -# SIGTERM -> SIGKILL budget. stellar-core exits ~7s after SIGTERM (measured), so -# this is slack rather than a target. -WORKER_GRACE_SECONDS = int(os.getenv('GRACE_SECONDS', 100)) - -# Seconds to stall inside preStop before the container is signalled. 0 disables. -# -# Sized to cover the collector's DETECTION LAG, which is the specific hole it -# fills. The collector notices DisruptionTarget on its pod-list cycle and only -# then drops that pod to 1s polling; if SIGTERM lands inside that blind window -# the poller is still on its lazy LOG_POLL_SECONDS cadence. Measured on -# ssc-test: a 60s preStop with 10s polling and no disruption detection still -# lost txApply, while 1s polling with no preStop at all captured it. So this is -# not what saves the metric -- it is what makes sure the detection has happened -# before the kill. -# -# 20s, not COLLECTOR_POLL_SECONDS. That constant is the SLEEP between cycles, -# not the cycle: each one also lists every pod and sweeps kubelet -# /stats/summary on every node, which at 768 workers over ~250 nodes is -# unmeasured and plausibly another 5-15s. The margin is -# (preStop + pod-object linger) - (detection + one 1s poll), and with the -# linger measured at 7.8s it goes NEGATIVE at a 12s cycle if this is 5s. Above -# the true cycle time the margin plateaus at +6.8s, so overshooting is free -# while undershooting silently loses the metric. -# -# A spot reclaim gives ~120s of notice and does not need this at all; an -# eviction-API kill or a fast drain signals immediately and does. -# -# Do NOT try to SIGTERM the process from inside the hook and hold the pod open -# afterwards: measured, the pod object survived 10.2s that way versus 69s for a -# plain sleep, because a container dies with its PID 1 and the kubelet does not -# defer deleting the object until the hook returns. -# -# Costs nothing on a healthy exit -- preStop does not run when the container -# exits on its own, only when the kubelet is tearing it down. At ~810 evictions -# a run, 5s each is about 1.1 pod-hours. -# -# Must stay comfortably under WORKER_GRACE_SECONDS: the hook and the SIGTERM -# drain share that one budget, and a hook still running when it expires is -# SIGKILLed, which loses exactly the output this exists to save. -WORKER_PRESTOP_SLEEP_SECONDS = int(os.getenv('PRESTOP_SLEEP_SECONDS', 5)) - -# Must comfortably exceed any plausible monitor outage: completion is recorded -# to the ConfigMap by this process, and a Job reclaimed before that happens -# reads as "never ran" and gets redone. -# Backstop only. reconcile() deletes each Job explicitly once its record is -# durable, so the TTL exists for the cases that skip that path: a terminally -# failed range kept for inspection, or a success whose metrics never landed. -JOB_TTL_SECONDS = int(os.getenv('JOB_TTL_SECONDS', 600)) - -# Measured on ssc-test: stellar-core does NOT fail on an unreachable history -# archive, an absent ledger range, or a bucket that will not decompress. It -# retries every mirror with growing backoff and stays Running indefinitely -- -# no exit code, no failure, the slot held for the life of the run. A hang is a -# more likely real failure than a non-zero exit, and this deadline is the only -# thing that makes it observable. 0 disables. -# -# Flat, deliberately -- NOT scaled by the range's profiled runtime. That was -# tried and removed. A deadline has to bound a range's WORST case, but a profile -# only offers a neighbour's TYPICAL case, and the two are far apart here: -# runtimes span 190x (p25 771s, max 5.9h), range keys are anchored to the -# network tip so a profile from an earlier run matches ZERO keys exactly and -# every lookup lands on a neighbour, and ~2% of those neighbours are 3-38x -# cheaper than their surroundings. Backtested honestly across that grid offset -# (run4 profile -> r5 actuals, 3983 ranges): a 2x factor falsely kills 134 -# ranges, 4x kills 46, 6x kills 21. Flat 12h kills none. -# -# The asymmetry decides it. A false kill loses a range, and a timeout is -# terminal, so it fails the mission. A genuine wedge holds ONE slot out of -# 1092-1500 for 12h -- around 0.1% of a run's capacity. Never trade a certain -# catastrophe against a rounding error. -# -# 12h is a safe bound, not a good detector: it takes half a day to catch -# something provably dead in 4 minutes. The right signal is ledger-close -# progress, not elapsed time -- a wedged core closes zero ledgers while still -# logging, so `.state` (last log line) cannot see it and a new -# lastLedgerCloseAt would. Left undone on purpose; it needs a threshold above -# the initial bucket-apply phase, which legitimately closes nothing for ~20min -# on the longest ranges. -ATTEMPT_DEADLINE_SECONDS = int(os.getenv('ATTEMPT_DEADLINE_SECONDS', 0)) - -# kube-state-metrics turns a pod's `mission` label into label_mission, which the -# Grafana container panels join on. Every other mission gets it from -# StellarKubeSpecs; this chart never has, so parallel catchup has never appeared -# in those panels. -# -# OFF by default and deliberately so: those panels are sum() by (pod, container) -# with a legend table, so at 1024 workers they would pull ~1024 series into any -# view with mission=$__all selected, degrading a shared dashboard for people who -# did not ask for it. Enable per-run once the panels aggregate (topk). -MISSION = os.getenv('MISSION', '') - -EMIT_MISSION_LABEL = os.getenv('EMIT_MISSION_LABEL', 'false').lower() == 'true' - -# ============================================================================= -# 3. This monitor's own behaviour -# ============================================================================= -PARALLELISM = int(os.getenv('PARALLELISM', 3)) - -# Effectively the OOM budget: `failed` is the only other outcome that reaches -# it, and that one sets no retry reason. Escalation counts OOMs rather than -# attempts, so rung N means the range genuinely wanted more N times. -# -# Deliberately stops short of MEM_ESCALATION_CAP: 5 rungs is 1.5^4 = 5x the -# profile figure, and a range needing more than that is not mis-sized, it is -# broken -- chasing it to 48Gi parks a whole r8a.2xlarge on one range for hours. -# The cost of stopping is that the range is condemned, and today a condemned -# range aborts the run. That coupling is the thing to fix, not this number. -# Attempts each failure cause gets before the range is condemned. The whole -# retry policy, in one table. -# -# Every budget is spent by ITS OWN cause: an OOM never consumes the disk budget -# and a spot eviction never consumes either. A cause that is NOT here is -# condemned the first time it happens -- a timeout, a real catchup failure, an -# exit 3 with nothing in its archive, and anything nothing could classify. -# -# disrupted the cluster took the pod away mid-run, which proves the range -# itself was fine. Effectively unlimited: on spot a healthy range -# is legitimately evicted dozens of times, and 100 is far past any -# rate a real run has produced while still terminating. -# rejected the kubelet refused the pod before any container ran (attachment -# limits, admission churn). The range never started, so a retry -# cannot mask anything about it. -# fetch-fault an exit 3 whose archive named a failed history fetch. An -# unreachable mirror is the cluster's problem, not the range's. A -# plain `failed` has no entry: a real catchup failure, and an exit 3 -# with nothing in its archive, are both condemned on sight. -# oom each retry escalates the memory request one rung. -# ephemeral each retry escalates the disk limit one rung. Smallest, because -# an eviction repeats identically until the range gets more disk. -# -# The MAX_* names below exist so the chart can tune each one; ATTEMPT_BUDGETS is -# what the code reads, so tests patch the map rather than the constants. -MAX_DISRUPTION_ATTEMPTS = int(os.getenv('MAX_DISRUPTION_ATTEMPTS', 100)) -MAX_REJECTED_ATTEMPTS = int(os.getenv('MAX_REJECTED_ATTEMPTS', 100)) -MAX_FETCH_FAULT_ATTEMPTS = int(os.getenv('MAX_FETCH_FAULT_ATTEMPTS', 20)) -MAX_OOM_ATTEMPTS = int(os.getenv('MAX_OOM_ATTEMPTS', 5)) -MAX_EPHEMERAL_ATTEMPTS = int(os.getenv('MAX_EPHEMERAL_ATTEMPTS', 4)) - -ATTEMPT_BUDGETS = { - 'disrupted': MAX_DISRUPTION_ATTEMPTS, - 'rejected': MAX_REJECTED_ATTEMPTS, - 'fetch-fault': MAX_FETCH_FAULT_ATTEMPTS, - 'oom': MAX_OOM_ATTEMPTS, - 'ephemeral': MAX_EPHEMERAL_ATTEMPTS, -} - -EPH_BUMP_FACTOR = float(os.getenv('EPH_BUMP_FACTOR', 1.5)) - -EPH_ESCALATION_CAP = os.getenv('EPH_ESCALATION_CAP', '200Gi') - +# The verdict vocabulary. The collector writes one of these into .outcome and +# the monitor charges it against a retry budget, so a name added on one side +# and not the other is an attempt nobody can classify. ATTEMPT_OUTCOMES = ('disrupted', 'oom', 'ephemeral', 'timeout', 'rejected', 'unknown', 'failed', 'fetch-fault') - -# Verdicts only the pod can produce, and which a Job-level DeadlineExceeded must -# never overwrite. Each names a specific mechanism -- the kubelet OOM-killed it, -# the node was draining, the ephemeral limit blew -- and each earns a different -# retry budget and a different remediation. "The Job ran too long" is also true -# of every one of them and says nothing about which. An OOM downgraded to a -# timeout retries at the same memory limit that just killed it and gets 2 -# attempts instead of 5; a spot eviction downgraded to a timeout gets 2 instead -# of 20. -POD_AUTHORITATIVE_OUTCOMES = ('oom', 'disrupted', 'ephemeral', 'timeout') - -# stellar-core's "did not complete". Ambiguous by construction: a corrupt bucket -# and a SIGTERM during replay both produce it, so it must never be treated as -# proof that a range is broken. -CATCHUP_INCOMPLETE_EXIT = 3 - -# An OOM means requests/limits are mis-sized for this range. Escalate so the run -# can finish, but say so loudly -- surviving by escalating at runtime is a -# configuration bug, not a success. -MEM_BUMP_FACTOR = float(os.getenv('MEM_BUMP_FACTOR', 1.5)) - -# Ceiling for that escalation. Above the largest schedulable node the retry sits -# Pending forever, which looks like a hang rather than a failure. -MEM_ESCALATION_CAP = os.getenv('MAX_MEM', '48Gi') - -# Reconcile loop: dispatch, refresh status, publish metrics. The env var is -# named LOGGING_INTERVAL_SECONDS for historical reasons, from when this loop -# only logged. -RECONCILE_INTERVAL_SECONDS = int(os.getenv('LOGGING_INTERVAL_SECONDS', 10)) - -# Worker responsiveness is cosmetic and sampled independently from reconcile. -# Thirty seconds and three failures restore the old ~90-second down threshold, -# while a five-second request budget gives a busy admin endpoint substantially -# more room than the old one-shot two-second probe. - -LIVENESS_PROBE_TIMEOUT_SECONDS = os.getenv('LIVENESS_PROBE_TIMEOUT_SECONDS', '5') - - -LIVENESS_MAX_CONCURRENCY = os.getenv('LIVENESS_MAX_CONCURRENCY', '32') -# Wall-clock bound on one sweep. The reconcile loop waits for it, so this -# is the most a fleet of unreachable workers can delay dispatch. -LIVENESS_SWEEP_SECONDS = os.getenv('LIVENESS_SWEEP_SECONDS', '15') - -# Shared with the log-collector sidecar, which owns writes here: it streams each -# worker's log and records the .outcome verdict while the pod still exists. -LOG_DIR = os.getenv('LOG_DIR', '/logs') - -SAVE_SUCCESS_LOGS = os.getenv('SAVE_SUCCESS_LOGS', 'true').lower() == 'true' - -# The authoritative copy of the progress record lives on the logs PVC, not in -# the ConfigMap. A ConfigMap is capped at 1 MiB and this record is ~172 bytes -# per completed range, so it dies at ~6100 ranges -- reachable simply by halving -# ledgersPerJob. Measured mid-run on ssc-test: 348KB at 2024 completed ranges, -# which projects to ~65% of the cap at 3982 -- close enough that the next -# slicing change would have hit it. Worse, every completion rewrote the whole -# document through the API server, so a full run meant thousands of -# escalating-size etcd writes. -# -# The ConfigMap is still written, because the mission driver reads it without -# exec'ing into the pod, but it is now a best-effort mirror: if it fails, the -# run carries on from the file. -PROGRESS_FILE = os.path.join(LOG_DIR, 'progress.json') - -PROFILE = None - -# --- pool tiers ------------------------------------------------------------- -# -# A range picks a NODEPOOL by its measured memory, and gets that pool's node to -# itself. This replaces the cpu ladder, which tuned a dimension that turned out -# not to be the binding one. -# -# Why memory and not cpu. Measured 2026-08-03 on one range across four instance -# shapes, isolated, no memory limit: -# -# 2 -> 4 cores replay +2.8% bucket-apply 1.37x -# 4 -> 8 cores replay +1.5% bucket-apply 1.18x -# AMD vs Intel replay +16% bucket-apply 1.35x -# -# Replay is ~93% of a job and is flat in core count from 2 upward -- it draws -# ~1.05 cores whatever it is given. So a cpu REQUEST never bought throughput. -# What it bought was neighbours-per-node, and memory is what actually fails: a -# range whose working set does not fit gets OOMKilled, not slowed down. -# -# Cuts are `node_usable / 1.60`, covering the p99 of run-to-run growth in the -# same range's peakAnonBytes (18,073 observations across five profiles: p50 0.97, -# p90 1.28, p99 1.60, max 2.83). Validated the hard way: range 63080767 measured -# 13.75Gi was placed on nodes with 14.1/14.3Gi allocatable -- a 1.03x margin -- -# and OOMKilled on BOTH during bucket-apply, before closing a ledger. -# subdwarf's cut is 0 on purpose: nothing can satisfy `gib < 0`, so the tier is -# defined and provisionable but never routed to. Kept rather than deleted so the -# bottom of the ladder is there to experiment with; c8a.medium (1.42Gi -# allocatable) cannot hold a range the profile actually contains. -POOL_TIERS = os.getenv( - 'POOL_TIERS', - '0:subdwarf,0.79:dwarf,1.61:subgiant,3.87:giant,8.85:supergiant,18.38:hypergiant,:supernova') - -# Prepended to the tier name to form the node label value, e.g. catchup-dwarf. -# Empty disables pool routing entirely and every worker keeps the single global -# NODE_LABEL_VALUE, which is exactly today's behaviour. -POOL_PREFIX = os.getenv('POOL_PREFIX', '') - -# Where a range goes when the profile has no entry for it (past the profile's -# top, i.e. the newest ledgers) and when there is no profile at all. -POOL_UNPROFILED = os.getenv('POOL_UNPROFILED', 'protostar') - -POOL_NO_PROFILE = os.getenv('POOL_NO_PROFILE', 'nebula') - -# cpu request per tier. NOT a demand estimate -- a claim token. Isolation is the -# point: freeing a node of its 3 neighbours raised throughput 29-92% while cpu -# draw FELL, so the contended resource is memory bandwidth and shared cache, not -# compute. Kept at or below the SMALLEST node in the tier so the low-weight -# fallback rungs stay schedulable (dwarf can land on a 1-vCPU c8a.medium). -# -# Memory, not cpu, is what actually enforces the isolation -- see _pool_memory. -# Half the node for most tiers. hypergiant and supernova are sized to the -# SMALLEST shape in their pool instead: x8i.large is r8a.xlarge with half the -# cores and the same 32 GiB, x8i.xlarge is r8a.2xlarge with half the cores and -# the same 64 GiB, so preferring them buys identical RAM for half the spot -# quota. A half-the-node 2.00/4.00 claim does not fit an x8i node once the 215m -# of daemonsets is counted, which is why those pools won no nodes at all on -# 2026-08-03. Below half, cpu no longer isolates the pod on the larger fallback -# shapes -- memory does, and it holds because every type within a tier carries -# the same RAM. -POOL_CPU = os.getenv( - 'POOL_CPU', - 'subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80') - - -# Rungs that never run, whatever the vCPU comparison says. Empty by default: with -# the spot pools doubled, promotion lands a range on a bigger SHARED node, and -# sharing is what the rung is really buying. Measured on ssc-test 2026-08-04 on -# one range, two pods to a node: on an 8-vCPU node a co-tenant cost 1.02x per pod -# (r8id.2xlarge, 3.78 and 3.91 lps), on a 4-vCPU node it cost 1.58x. Two pods on -# 8 cores leave 4 each, which the workload does not use; two on 4 cores leave 2, -# which is the floor. -# -# Caveat worth keeping in view: the bump fires on peakWorkingSetBytes, and working -# set does not predict throughput. Same x8i.xlarge box, same range, same time, -# only cgroup memory.max differing: 28 GiB ran 1.83 lps and 56 GiB ran 1.70. So -# this promotes for a reason that is not the reason it helps -- it reaches the -# right nodes via the wrong signal, and will promote ranges that gain nothing. -# Sizing the rung on peakAnonBytes, or widening the tier->instance map directly, -# would target those nodes deliberately. -# -# This is now the ONLY thing standing between a working set and a promotion, so -# a rung that should not be taken has to be named here -- nothing is inferred. -# -# hypergiant->supernova is denied on both capacity types. Its cost rose once the -# x8i pools were removed on 2026-08-04: supernova's only spot shapes are now -# 4xlarges, so the rung moves a range from 8 vCPU to 16 rather than the 8-vCPU -# x8i.2xlarge it used to reach. Simulated over the 2026-08-03 run it saved -# exactly 0 minutes on its own, because the longest job was a supergiant this -# rung cannot reach. It pays only in company -- supergiant->hypergiant alone is -# worth 8 min, this alone 0, the pair 27 -- and that pairing is not on offer -# while its cost is 8->16 vCPU. -# -# dwarf->subgiant is the same doubling at the bottom of the ladder, 2->4 vCPU on -# spot and 1->2 on on-demand. -POOL_BLOCK_RUNGS = os.getenv('POOL_BLOCK_RUNGS', 'dwarf->subgiant,hypergiant->supernova') - -# Memory request for a pooled range is the TIER'S CUT, not the range's own -# measurement, and that is deliberate two ways. -# -# It guarantees one pod per node without depending on the cpu token: a tier's -# node is cut*1.60 of usable memory, so two pods asking cut apiece need 2*cut, -# which always exceeds 1.60*cut. The cpu claim cannot do this alone because a -# tier spans node sizes (dwarf reaches a 1-vCPU c8a.medium and a 2-vCPU -# t3a.small), so no single cpu value both schedules on the small one and fills -# the large one. -# -# And the request no longer needs a safety margin. PROFILE_MARGIN, cache -# headroom and runtime insurance all existed to keep a pod under its own LIMIT; -# with no memory limit and the node to itself, a pod may use everything the node -# has. The margin moved into the node size -- which is where it can actually be -# enforced, since the kubelet kills on node pressure, not on request. -# Per-tier memory request: exactly 50% of the tier node's NAMEPLATE capacity. -# -# 50% is what isolates. Two pods asking half the nameplate need the whole node, -# which always exceeds allocatable -- so a second pod can never fit, on every -# tier, without depending on how the kubelet happens to reserve. -# -# Verified against measured nodes rather than assumed: a c8a.medium reports -# 1892Mi capacity, 1449Mi allocatable, and carries 154Mi of daemonsets, leaving -# 1295Mi -- so the 1024Mi request schedules with room, and 2048Mi of two pods -# cannot. The same holds up the ladder. -# -# t3a.micro is absent on purpose: 413Mi allocatable cannot host a pod at all on -# this cluster, so subdwarf shares dwarf's node type and is emptied by its cut. -POOL_MEM = os.getenv( - 'POOL_MEM', - 'subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:9216Mi') - -_SORTED_SECONDS = None - - -# Sized for the dispatch burst rather than a steady LIST rate: ~1024 Jobs + PVCs -# go out at once at the head of a wave. -CONNECTION_POOL = int(os.getenv('CONNECTION_POOL', '64')) - -# Left as strings on purpose. Coercing at import made a bad value a boot crash, -# and a process that cannot start cannot report why -- the driver just polled a -# pod that never answered and timed out 600s later with "not reachable". -# validate_config coerces and rebinds these when /start delivers the run, so a -# bad value comes back as a 400 carrying the reason. diff --git a/src/MissionParallelCatchup/lib/attempts.py b/src/MissionParallelCatchup/lib/monitor/attempts.py similarity index 100% rename from src/MissionParallelCatchup/lib/attempts.py rename to src/MissionParallelCatchup/lib/monitor/attempts.py diff --git a/src/MissionParallelCatchup/lib/http_server.py b/src/MissionParallelCatchup/lib/monitor/http_server.py similarity index 100% rename from src/MissionParallelCatchup/lib/http_server.py rename to src/MissionParallelCatchup/lib/monitor/http_server.py diff --git a/src/MissionParallelCatchup/lib/kube.py b/src/MissionParallelCatchup/lib/monitor/kube.py similarity index 93% rename from src/MissionParallelCatchup/lib/kube.py rename to src/MissionParallelCatchup/lib/monitor/kube.py index 3aa1e28b..259f344c 100644 --- a/src/MissionParallelCatchup/lib/kube.py +++ b/src/MissionParallelCatchup/lib/monitor/kube.py @@ -14,6 +14,7 @@ from kubernetes import client, config as kube_config import config +import monitor_config as mc # The env var is exactly what load_incluster_config() itself keys on, so in a pod # this is the unconditional call it always was -- a missing token or CA still @@ -27,7 +28,7 @@ # client-go's Python equivalent defaults are fine for a few LISTs per cycle, but # dispatching ~1024 Jobs + PVCs at once needs headroom. _cfg = client.Configuration.get_default_copy() -_cfg.connection_pool_maxsize = config.CONNECTION_POOL +_cfg.connection_pool_maxsize = mc.CONNECTION_POOL client.Configuration.set_default(_cfg) core_v1 = client.CoreV1Api() diff --git a/src/MissionParallelCatchup/lib/metrics.py b/src/MissionParallelCatchup/lib/monitor/metrics.py similarity index 100% rename from src/MissionParallelCatchup/lib/metrics.py rename to src/MissionParallelCatchup/lib/monitor/metrics.py diff --git a/src/MissionParallelCatchup/lib/monitor/monitor_config.py b/src/MissionParallelCatchup/lib/monitor/monitor_config.py new file mode 100644 index 00000000..d6c92726 --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/monitor_config.py @@ -0,0 +1,539 @@ +"""Settings and run state for the parallel catchup job monitor. + +Everything here belongs to the monitor process alone. What both processes must +agree on -- the run's identity, the shared volume, the verdict vocabulary -- +lives in config, which this reads through for LOG_DIR. + +Read through the module, never copied out of it: + + import monitor_config as mc + ... mc.REQ_CPU ... + +`from monitor_config import REQ_CPU` binds a COPY. A test's monkeypatch and the +startup assignment of mc.PROFILE both rebind the attribute on this module, and a +copy taken at import time never sees either -- silently, with the test passing +against the default. +""" +import os + +import config + +# ============================================================================= +# 1. stellar-core workload +# ============================================================================= +CORE_IMAGE = os.getenv('CORE_IMAGE') + +ASAN_OPTIONS = os.getenv('ASAN_OPTIONS', '') + +# Which ledger ranges to run. These are pure inputs to the range generator: +# dispatch recomputes the whole list every reconcile, so a restart must +# reproduce it exactly. + + +# Both generators emit tip-first, which front-loads the most expensive ranges: +# the bucket set only grows with ledger position. 'oldest-first' reverses that, +# so a profiling run measures the cheap early ranges before it can be +# interrupted, and the expensive tip ranges last. +RANGE_ORDER = 'tip-first' # from /start: tip-first | oldest-first | longest-first + +VALID_RANGE_ORDERS = ('tip-first', 'oldest-first', 'longest-first') + +STARTING_LEDGER = 0 # from /start + +LATEST_LEDGER_NUM = 0 # from /start + +LEDGERS_PER_JOB = 16000 # from /start + +OVERLAP_LEDGERS = 320 # from /start + + +# ============================================================================= +# 2. Kubernetes objects this monitor creates +# ============================================================================= + + + + + + +# Workers need IRSA to read the S3 history mirror. Without it they silently fall +# back to the public archive, which throttles at 1024 and kills the run with +# curl 22 -> catchup exit 3. The name matches the old StatefulSet's so existing +# IRSA trust policies keep matching. +WORKER_SERVICE_ACCOUNT = os.getenv('WORKER_SERVICE_ACCOUNT', '') + +# Pod resources. Requests only: workers are given no cpu limit and no memory +# limit at all. +# +# CPU because a limit only throttles a pod that could otherwise use idle cores, +# and throttling changes what the range measures -- less cpu means less download +# concurrency means a lower peak, so a throttled attempt records a figure an +# unthrottled one cannot reproduce. +# +# Memory because a limit is a hard cap on anon PLUS page cache, and sizing it +# per-range from a profile got it wrong in the one direction that has no alarm +# on it. Measured 2026-07-31, range 39210943: sized at 1729Mi from a neighbour, +# genuinely needed 1620Mi of anon, which left ~110Mi for cache. It never OOMed +# -- it thrashed. 544k major page faults, 0.22 cores used on a node it had +# entirely to itself, 0.95 ledgers/s against a neighbour norm of 3.3, and it +# held 1092 idle slots open for three hours at the end of the run. +# +# Without a limit the request still does the real work: it places the pod and +# it sets eviction order under node pressure. What goes away is the cliff. +REQ_CPU = os.getenv('REQ_CPU', '1250m') + +REQ_MEM = os.getenv('REQ_MEM', '9Gi') + +# The run document /start delivers, kept so a restart resumes the same run. +RUN_PATH = '' + +PROFILE_MARGIN = float(os.getenv('PROFILE_MARGIN', 1.15)) + +# No safety margin on cpu, unlike memory. Under-requesting cpu costs contention +# and the pod can still burst; under-requesting memory gets it OOMKilled. +# Ceiling for profile-derived memory, above the unprofiled limit for the same +# reason: a range that really needs more than the configured limit must be able +# to ask for it rather than be pinned under its own measured peak. The OOM +# escalation ladder can still climb past this on a retry. +PROFILE_MAX_MEM = os.getenv('PROFILE_MAX_MEM', '32Gi') + +# Memory is sized from rss (the range's real demand), NOT from peak working +# set. Working set is whatever limit it was measured under -- the kernel grows +# page cache to fill it -- so sizing from it is circular. Measured on ssc-test +# with one 420-ledger range: working set went 2.33 -> 3.61 -> 7.48 -> 13.49 GiB +# under 2560Mi/4Gi/8Gi/24000Mi limits while rss moved only 2256 -> 2488 MiB, and +# wall-clock did not move at all (776s / 775s / 746s / 773s). Catchup streams -- +# buckets are downloaded once, applied once, ledgers replayed once -- so cache +# has nothing to give back and PROFILE_MARGIN alone is the allowance. +# A multiplicative margin alone is not enough: memory.max bounds anon PLUS page +# cache, and at small rss 10% is nothing. Measured on ssc-test 2026-07-29 with +# headroom 0: ranges profiled at 190 MiB rss got a 209 MiB limit -- 19 MiB of +# slack for all growth and cache -- and 90 of them OOMKilled within 90s. The +# earlier 4Gi validation hid this because 1.1x of 2.4 GiB is 240 MiB of slack. +PROFILE_CACHE_HEADROOM = os.getenv('PROFILE_CACHE_HEADROOM', '512Mi') + +# Extra allowance scaled by the range's measured runtime. Long ranges keep more +# page cache and allocator slack live at once; 0 disables the allowance. +PROFILE_RUNTIME_MEMORY_INSURANCE = os.getenv('PROFILE_RUNTIME_MEMORY_INSURANCE', '3Gi') + +# Ephemeral-storage gets the same two allowances as memory, for the same +# reasons. Measured on the 2026-08-01 on-demand run: peak 37.76Gi against a +# flat 40Gi limit -- 6% of headroom on a path that has never once fired in a +# real run, so a range 6% worse than the worst seen would be evicted 137 with +# no diagnostic pointing at disk. +# +# Flat allowance added to every range's measured peak. Covers the container +# image, logs and the sqlite WAL, none of which scale with the range. +PROFILE_EPHEMERAL_HEADROOM = os.getenv('PROFILE_EPHEMERAL_HEADROOM', '2Gi') + +# Runtime-weighted allowance on top. Disk tracks runtime closely (pearson 0.920 +# across 3985 ranges: runtime decile 0 uses 0.1Gi, decile 9 uses 24.7Gi), so +# the ranges that need the margin are exactly the ranges this gives it to. +PROFILE_RUNTIME_EPHEMERAL_INSURANCE = os.getenv('PROFILE_RUNTIME_EPHEMERAL_INSURANCE', '8Gi') + +# Ceiling for profile-derived disk. Deliberately ABOVE LIM_EPHEMERAL: that flat +# limit is what an UNMEASURED range gets, and capping a measured range at it +# would throw away the measurement -- the worst observed range wants 43Gi after +# margin alone. +PROFILE_MAX_EPHEMERAL = os.getenv('PROFILE_MAX_EPHEMERAL', '64Gi') + +REQ_EPHEMERAL = os.getenv('REQ_EPHEMERAL', '') + +LIM_EPHEMERAL = os.getenv('LIM_EPHEMERAL', '') + +# Placement. The taint toleration is emitted as {key, effect} with no value: +# the default Equal operator does not match "" against "true". +NODE_LABEL_KEY = os.getenv('NODE_LABEL_KEY', '') + +NODE_LABEL_VALUE = os.getenv('NODE_LABEL_VALUE', '') + +# Further labels a node must carry, "key:value" comma separated, ANDed with the +# one above. Unlike that one these are literal -- the pair above is pool-routed, +# its value replaced per range with -. +# +# This is where a run pins itself to one capacity of a tier. Both capacities +# carry the same tier label value, so nothing else separates them, and the +# pairing matters: ephemeral has no resume, so a reclaim costs the whole range. +# A plain label rather than karpenter.sh/capacity-type, because the pools +# publish their own and the monitor has no business knowing who provisioned the +# node. +REQUIRE_NODE_LABELS = os.getenv('REQUIRE_NODE_LABELS', '') + + +def label_pairs(raw): + """[(key, value)] from "k:v,k:v". Entries without a value are dropped: a + key alone would require the label be exactly "", which no node carries, and + a pod pinned to nothing sits Pending in a way that reads as slow + provisioning rather than as misconfiguration.""" + out = [] + for item in (raw or '').split(','): + key, _, value = item.strip().partition(':') + if key and value: + out.append((key, value)) + return out + +AVOID_NODE_LABEL_KEY = os.getenv('AVOID_NODE_LABEL_KEY', '') + +AVOID_NODE_LABEL_VALUE = os.getenv('AVOID_NODE_LABEL_VALUE', '') + +TOLERATE_TAINT = os.getenv('TOLERATE_TAINT', '') + + +STORAGE_CLASS = os.getenv('STORAGE_CLASS', '') + +# 60Gi to match the tier nodes' ephemeral allowance. peakEphemeralBytes tops +# out at 37.8Gi across the whole 2026-08-01 profile, so this covers every +# range measured, with headroom for the tip to keep growing. +STORAGE_SIZE = os.getenv('STORAGE_SIZE', '60Gi') + +# Job/pod lifetimes. +# SIGTERM -> SIGKILL budget. stellar-core exits ~7s after SIGTERM (measured), so +# this is slack rather than a target. +WORKER_GRACE_SECONDS = int(os.getenv('GRACE_SECONDS', 100)) + +# Seconds to stall inside preStop before the container is signalled. 0 disables. +# +# Sized to cover the collector's DETECTION LAG, which is the specific hole it +# fills. The collector notices DisruptionTarget on its pod-list cycle and only +# then drops that pod to 1s polling; if SIGTERM lands inside that blind window +# the poller is still on its lazy LOG_POLL_SECONDS cadence. Measured on +# ssc-test: a 60s preStop with 10s polling and no disruption detection still +# lost txApply, while 1s polling with no preStop at all captured it. So this is +# not what saves the metric -- it is what makes sure the detection has happened +# before the kill. +# +# 20s, not COLLECTOR_POLL_SECONDS. That constant is the SLEEP between cycles, +# not the cycle: each one also lists every pod and sweeps kubelet +# /stats/summary on every node, which at 768 workers over ~250 nodes is +# unmeasured and plausibly another 5-15s. The margin is +# (preStop + pod-object linger) - (detection + one 1s poll), and with the +# linger measured at 7.8s it goes NEGATIVE at a 12s cycle if this is 5s. Above +# the true cycle time the margin plateaus at +6.8s, so overshooting is free +# while undershooting silently loses the metric. +# +# A spot reclaim gives ~120s of notice and does not need this at all; an +# eviction-API kill or a fast drain signals immediately and does. +# +# Do NOT try to SIGTERM the process from inside the hook and hold the pod open +# afterwards: measured, the pod object survived 10.2s that way versus 69s for a +# plain sleep, because a container dies with its PID 1 and the kubelet does not +# defer deleting the object until the hook returns. +# +# Costs nothing on a healthy exit -- preStop does not run when the container +# exits on its own, only when the kubelet is tearing it down. At ~810 evictions +# a run, 5s each is about 1.1 pod-hours. +# +# Must stay comfortably under WORKER_GRACE_SECONDS: the hook and the SIGTERM +# drain share that one budget, and a hook still running when it expires is +# SIGKILLed, which loses exactly the output this exists to save. +WORKER_PRESTOP_SLEEP_SECONDS = int(os.getenv('PRESTOP_SLEEP_SECONDS', 5)) + +# Must comfortably exceed any plausible monitor outage: completion is recorded +# to the ConfigMap by this process, and a Job reclaimed before that happens +# reads as "never ran" and gets redone. +# Backstop only. reconcile() deletes each Job explicitly once its record is +# durable, so the TTL exists for the cases that skip that path: a terminally +# failed range kept for inspection, or a success whose metrics never landed. +JOB_TTL_SECONDS = int(os.getenv('JOB_TTL_SECONDS', 600)) + +# Measured on ssc-test: stellar-core does NOT fail on an unreachable history +# archive, an absent ledger range, or a bucket that will not decompress. It +# retries every mirror with growing backoff and stays Running indefinitely -- +# no exit code, no failure, the slot held for the life of the run. A hang is a +# more likely real failure than a non-zero exit, and this deadline is the only +# thing that makes it observable. 0 disables. +# +# Flat, deliberately -- NOT scaled by the range's profiled runtime. That was +# tried and removed. A deadline has to bound a range's WORST case, but a profile +# only offers a neighbour's TYPICAL case, and the two are far apart here: +# runtimes span 190x (p25 771s, max 5.9h), range keys are anchored to the +# network tip so a profile from an earlier run matches ZERO keys exactly and +# every lookup lands on a neighbour, and ~2% of those neighbours are 3-38x +# cheaper than their surroundings. Backtested honestly across that grid offset +# (run4 profile -> r5 actuals, 3983 ranges): a 2x factor falsely kills 134 +# ranges, 4x kills 46, 6x kills 21. Flat 12h kills none. +# +# The asymmetry decides it. A false kill loses a range, and a timeout is +# terminal, so it fails the mission. A genuine wedge holds ONE slot out of +# 1092-1500 for 12h -- around 0.1% of a run's capacity. Never trade a certain +# catastrophe against a rounding error. +# +# 12h is a safe bound, not a good detector: it takes half a day to catch +# something provably dead in 4 minutes. The right signal is ledger-close +# progress, not elapsed time -- a wedged core closes zero ledgers while still +# logging, so `.state` (last log line) cannot see it and a new +# lastLedgerCloseAt would. Left undone on purpose; it needs a threshold above +# the initial bucket-apply phase, which legitimately closes nothing for ~20min +# on the longest ranges. +ATTEMPT_DEADLINE_SECONDS = int(os.getenv('ATTEMPT_DEADLINE_SECONDS', 0)) + +# kube-state-metrics turns a pod's `mission` label into label_mission, which the +# Grafana container panels join on. Every other mission gets it from +# StellarKubeSpecs; this chart never has, so parallel catchup has never appeared +# in those panels. +# +# OFF by default and deliberately so: those panels are sum() by (pod, container) +# with a legend table, so at 1024 workers they would pull ~1024 series into any +# view with mission=$__all selected, degrading a shared dashboard for people who +# did not ask for it. Enable per-run once the panels aggregate (topk). +MISSION = os.getenv('MISSION', '') + +EMIT_MISSION_LABEL = os.getenv('EMIT_MISSION_LABEL', 'false').lower() == 'true' + +# ============================================================================= +# 3. This monitor's own behaviour +# ============================================================================= +PARALLELISM = int(os.getenv('PARALLELISM', 3)) + +# Effectively the OOM budget: `failed` is the only other outcome that reaches +# it, and that one sets no retry reason. Escalation counts OOMs rather than +# attempts, so rung N means the range genuinely wanted more N times. +# +# Deliberately stops short of MEM_ESCALATION_CAP: 5 rungs is 1.5^4 = 5x the +# profile figure, and a range needing more than that is not mis-sized, it is +# broken -- chasing it to 48Gi parks a whole r8a.2xlarge on one range for hours. +# The cost of stopping is that the range is condemned, and today a condemned +# range aborts the run. That coupling is the thing to fix, not this number. +# Attempts each failure cause gets before the range is condemned. The whole +# retry policy, in one table. +# +# Every budget is spent by ITS OWN cause: an OOM never consumes the disk budget +# and a spot eviction never consumes either. A cause that is NOT here is +# condemned the first time it happens -- a timeout, a real catchup failure, an +# exit 3 with nothing in its archive, and anything nothing could classify. +# +# disrupted the cluster took the pod away mid-run, which proves the range +# itself was fine. Effectively unlimited: on spot a healthy range +# is legitimately evicted dozens of times, and 100 is far past any +# rate a real run has produced while still terminating. +# rejected the kubelet refused the pod before any container ran (attachment +# limits, admission churn). The range never started, so a retry +# cannot mask anything about it. +# fetch-fault an exit 3 whose archive named a failed history fetch. An +# unreachable mirror is the cluster's problem, not the range's. A +# plain `failed` has no entry: a real catchup failure, and an exit 3 +# with nothing in its archive, are both condemned on sight. +# oom each retry escalates the memory request one rung. +# ephemeral each retry escalates the disk limit one rung. Smallest, because +# an eviction repeats identically until the range gets more disk. +# +# The MAX_* names below exist so the chart can tune each one; ATTEMPT_BUDGETS is +# what the code reads, so tests patch the map rather than the constants. +MAX_DISRUPTION_ATTEMPTS = int(os.getenv('MAX_DISRUPTION_ATTEMPTS', 100)) +MAX_REJECTED_ATTEMPTS = int(os.getenv('MAX_REJECTED_ATTEMPTS', 100)) +MAX_FETCH_FAULT_ATTEMPTS = int(os.getenv('MAX_FETCH_FAULT_ATTEMPTS', 20)) +MAX_OOM_ATTEMPTS = int(os.getenv('MAX_OOM_ATTEMPTS', 5)) +MAX_EPHEMERAL_ATTEMPTS = int(os.getenv('MAX_EPHEMERAL_ATTEMPTS', 4)) + +ATTEMPT_BUDGETS = { + 'disrupted': MAX_DISRUPTION_ATTEMPTS, + 'rejected': MAX_REJECTED_ATTEMPTS, + 'fetch-fault': MAX_FETCH_FAULT_ATTEMPTS, + 'oom': MAX_OOM_ATTEMPTS, + 'ephemeral': MAX_EPHEMERAL_ATTEMPTS, +} + +EPH_BUMP_FACTOR = float(os.getenv('EPH_BUMP_FACTOR', 1.5)) + +EPH_ESCALATION_CAP = os.getenv('EPH_ESCALATION_CAP', '200Gi') + + +# Verdicts only the pod can produce, and which a Job-level DeadlineExceeded must +# never overwrite. Each names a specific mechanism -- the kubelet OOM-killed it, +# the node was draining, the ephemeral limit blew -- and each earns a different +# retry budget and a different remediation. "The Job ran too long" is also true +# of every one of them and says nothing about which. An OOM downgraded to a +# timeout retries at the same memory limit that just killed it and gets 2 +# attempts instead of 5; a spot eviction downgraded to a timeout gets 2 instead +# of 20. +POD_AUTHORITATIVE_OUTCOMES = ('oom', 'disrupted', 'ephemeral', 'timeout') + +# stellar-core's "did not complete". Ambiguous by construction: a corrupt bucket +# and a SIGTERM during replay both produce it, so it must never be treated as +# proof that a range is broken. +CATCHUP_INCOMPLETE_EXIT = 3 + +# An OOM means requests/limits are mis-sized for this range. Escalate so the run +# can finish, but say so loudly -- surviving by escalating at runtime is a +# configuration bug, not a success. +MEM_BUMP_FACTOR = float(os.getenv('MEM_BUMP_FACTOR', 1.5)) + +# Ceiling for that escalation. Above the largest schedulable node the retry sits +# Pending forever, which looks like a hang rather than a failure. +MEM_ESCALATION_CAP = os.getenv('MAX_MEM', '48Gi') + +# Reconcile loop: dispatch, refresh status, publish metrics. The env var is +# named LOGGING_INTERVAL_SECONDS for historical reasons, from when this loop +# only logged. +RECONCILE_INTERVAL_SECONDS = int(os.getenv('LOGGING_INTERVAL_SECONDS', 10)) + +# Worker responsiveness is cosmetic and sampled independently from reconcile. +# Thirty seconds and three failures restore the old ~90-second down threshold, +# while a five-second request budget gives a busy admin endpoint substantially +# more room than the old one-shot two-second probe. + +LIVENESS_PROBE_TIMEOUT_SECONDS = os.getenv('LIVENESS_PROBE_TIMEOUT_SECONDS', '5') + + +LIVENESS_MAX_CONCURRENCY = os.getenv('LIVENESS_MAX_CONCURRENCY', '32') +# Wall-clock bound on one sweep. The reconcile loop waits for it, so this +# is the most a fleet of unreachable workers can delay dispatch. +LIVENESS_SWEEP_SECONDS = os.getenv('LIVENESS_SWEEP_SECONDS', '15') + + + +# The authoritative copy of the progress record lives on the logs PVC, not in +# the ConfigMap. A ConfigMap is capped at 1 MiB and this record is ~172 bytes +# per completed range, so it dies at ~6100 ranges -- reachable simply by halving +# ledgersPerJob. Measured mid-run on ssc-test: 348KB at 2024 completed ranges, +# which projects to ~65% of the cap at 3982 -- close enough that the next +# slicing change would have hit it. Worse, every completion rewrote the whole +# document through the API server, so a full run meant thousands of +# escalating-size etcd writes. +# +# The ConfigMap is still written, because the mission driver reads it without +# exec'ing into the pod, but it is now a best-effort mirror: if it fails, the +# run carries on from the file. +PROGRESS_FILE = os.path.join(config.LOG_DIR, 'progress.json') + +PROFILE = None + +# --- pool tiers ------------------------------------------------------------- +# +# A range picks a NODEPOOL by its measured memory, and gets that pool's node to +# itself. This replaces the cpu ladder, which tuned a dimension that turned out +# not to be the binding one. +# +# Why memory and not cpu. Measured 2026-08-03 on one range across four instance +# shapes, isolated, no memory limit: +# +# 2 -> 4 cores replay +2.8% bucket-apply 1.37x +# 4 -> 8 cores replay +1.5% bucket-apply 1.18x +# AMD vs Intel replay +16% bucket-apply 1.35x +# +# Replay is ~93% of a job and is flat in core count from 2 upward -- it draws +# ~1.05 cores whatever it is given. So a cpu REQUEST never bought throughput. +# What it bought was neighbours-per-node, and memory is what actually fails: a +# range whose working set does not fit gets OOMKilled, not slowed down. +# +# Cuts are `node_usable / 1.60`, covering the p99 of run-to-run growth in the +# same range's peakAnonBytes (18,073 observations across five profiles: p50 0.97, +# p90 1.28, p99 1.60, max 2.83). Validated the hard way: range 63080767 measured +# 13.75Gi was placed on nodes with 14.1/14.3Gi allocatable -- a 1.03x margin -- +# and OOMKilled on BOTH during bucket-apply, before closing a ledger. +# subdwarf's cut is 0 on purpose: nothing can satisfy `gib < 0`, so the tier is +# defined and provisionable but never routed to. Kept rather than deleted so the +# bottom of the ladder is there to experiment with; c8a.medium (1.42Gi +# allocatable) cannot hold a range the profile actually contains. +POOL_TIERS = os.getenv( + 'POOL_TIERS', + '0:subdwarf,0.79:dwarf,1.61:subgiant,3.87:giant,8.85:supergiant,18.38:hypergiant,:supernova') + +# Prepended to the tier name to form the node label value, e.g. catchup-dwarf. +# Empty disables pool routing entirely and every worker keeps the single global +# NODE_LABEL_VALUE, which is exactly today's behaviour. +POOL_PREFIX = os.getenv('POOL_PREFIX', '') + +# Where a range goes when the profile has no entry for it (past the profile's +# top, i.e. the newest ledgers) and when there is no profile at all. +POOL_UNPROFILED = os.getenv('POOL_UNPROFILED', 'protostar') + +POOL_NO_PROFILE = os.getenv('POOL_NO_PROFILE', 'nebula') + +# cpu request per tier. NOT a demand estimate -- a claim token. Isolation is the +# point: freeing a node of its 3 neighbours raised throughput 29-92% while cpu +# draw FELL, so the contended resource is memory bandwidth and shared cache, not +# compute. Kept at or below the SMALLEST node in the tier so the low-weight +# fallback rungs stay schedulable (dwarf can land on a 1-vCPU c8a.medium). +# +# Memory, not cpu, is what actually enforces the isolation -- see _pool_memory. +# Half the node for most tiers. hypergiant and supernova are sized to the +# SMALLEST shape in their pool instead: x8i.large is r8a.xlarge with half the +# cores and the same 32 GiB, x8i.xlarge is r8a.2xlarge with half the cores and +# the same 64 GiB, so preferring them buys identical RAM for half the spot +# quota. A half-the-node 2.00/4.00 claim does not fit an x8i node once the 215m +# of daemonsets is counted, which is why those pools won no nodes at all on +# 2026-08-03. Below half, cpu no longer isolates the pod on the larger fallback +# shapes -- memory does, and it holds because every type within a tier carries +# the same RAM. +POOL_CPU = os.getenv( + 'POOL_CPU', + 'subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80') + + +# Rungs that never run, whatever the vCPU comparison says. Empty by default: with +# the spot pools doubled, promotion lands a range on a bigger SHARED node, and +# sharing is what the rung is really buying. Measured on ssc-test 2026-08-04 on +# one range, two pods to a node: on an 8-vCPU node a co-tenant cost 1.02x per pod +# (r8id.2xlarge, 3.78 and 3.91 lps), on a 4-vCPU node it cost 1.58x. Two pods on +# 8 cores leave 4 each, which the workload does not use; two on 4 cores leave 2, +# which is the floor. +# +# Caveat worth keeping in view: the bump fires on peakWorkingSetBytes, and working +# set does not predict throughput. Same x8i.xlarge box, same range, same time, +# only cgroup memory.max differing: 28 GiB ran 1.83 lps and 56 GiB ran 1.70. So +# this promotes for a reason that is not the reason it helps -- it reaches the +# right nodes via the wrong signal, and will promote ranges that gain nothing. +# Sizing the rung on peakAnonBytes, or widening the tier->instance map directly, +# would target those nodes deliberately. +# +# This is now the ONLY thing standing between a working set and a promotion, so +# a rung that should not be taken has to be named here -- nothing is inferred. +# +# hypergiant->supernova is denied on both capacity types. Its cost rose once the +# x8i pools were removed on 2026-08-04: supernova's only spot shapes are now +# 4xlarges, so the rung moves a range from 8 vCPU to 16 rather than the 8-vCPU +# x8i.2xlarge it used to reach. Simulated over the 2026-08-03 run it saved +# exactly 0 minutes on its own, because the longest job was a supergiant this +# rung cannot reach. It pays only in company -- supergiant->hypergiant alone is +# worth 8 min, this alone 0, the pair 27 -- and that pairing is not on offer +# while its cost is 8->16 vCPU. +# +# dwarf->subgiant is the same doubling at the bottom of the ladder, 2->4 vCPU on +# spot and 1->2 on on-demand. +POOL_BLOCK_RUNGS = os.getenv('POOL_BLOCK_RUNGS', 'dwarf->subgiant,hypergiant->supernova') + +# Memory request for a pooled range is the TIER'S CUT, not the range's own +# measurement, and that is deliberate two ways. +# +# It guarantees one pod per node without depending on the cpu token: a tier's +# node is cut*1.60 of usable memory, so two pods asking cut apiece need 2*cut, +# which always exceeds 1.60*cut. The cpu claim cannot do this alone because a +# tier spans node sizes (dwarf reaches a 1-vCPU c8a.medium and a 2-vCPU +# t3a.small), so no single cpu value both schedules on the small one and fills +# the large one. +# +# And the request no longer needs a safety margin. PROFILE_MARGIN, cache +# headroom and runtime insurance all existed to keep a pod under its own LIMIT; +# with no memory limit and the node to itself, a pod may use everything the node +# has. The margin moved into the node size -- which is where it can actually be +# enforced, since the kubelet kills on node pressure, not on request. +# Per-tier memory request: exactly 50% of the tier node's NAMEPLATE capacity. +# +# 50% is what isolates. Two pods asking half the nameplate need the whole node, +# which always exceeds allocatable -- so a second pod can never fit, on every +# tier, without depending on how the kubelet happens to reserve. +# +# Verified against measured nodes rather than assumed: a c8a.medium reports +# 1892Mi capacity, 1449Mi allocatable, and carries 154Mi of daemonsets, leaving +# 1295Mi -- so the 1024Mi request schedules with room, and 2048Mi of two pods +# cannot. The same holds up the ladder. +# +# t3a.micro is absent on purpose: 413Mi allocatable cannot host a pod at all on +# this cluster, so subdwarf shares dwarf's node type and is emptied by its cut. +POOL_MEM = os.getenv( + 'POOL_MEM', + 'subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:9216Mi') + +_SORTED_SECONDS = None + + +# Sized for the dispatch burst rather than a steady LIST rate: ~1024 Jobs + PVCs +# go out at once at the head of a wave. +CONNECTION_POOL = int(os.getenv('CONNECTION_POOL', '64')) + +# Left as strings on purpose. Coercing at import made a bad value a boot crash, +# and a process that cannot start cannot report why -- the driver just polled a +# pod that never answered and timed out 600s later with "not reachable". +# validate_config coerces and rebinds these when /start delivers the run, so a +# bad value comes back as a 400 carrying the reason. diff --git a/src/MissionParallelCatchup/lib/monitor/profiles.py b/src/MissionParallelCatchup/lib/monitor/profiles.py new file mode 100644 index 00000000..320790f5 --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/profiles.py @@ -0,0 +1,37 @@ +"""The measured profile of a previous run, and the lookup into it. + +Parsed once from the /start POST into mc.PROFILE and read through it thereafter, +so a test that patches the profile is seen here without reloading anything. +""" + +import bisect + +import monitor_config as mc + + +def load_profile_doc(doc): + """The sorted (end, record) list a parsed profile document yields. + + An unprofiled run POSTs {} and gets [] -- a profile is an optimisation, + never a prerequisite, so "no ranges" is a valid answer rather than an error. + """ + ranges = (doc or {}).get('ranges') or {} + return sorted((int(k), v) for k, v in ranges.items()) + + +def profile_for(end): + """Measurements to size this range from, or None to use the defaults. + + Exact end, else the nearest measured end ABOVE it. Cost rises with ledger + position -- the bucket set only grows -- so a lower neighbour under-reports, + and under-provisioning costs an eviction while over-provisioning only costs + packing. Past the top of the profile there is nothing safe to extrapolate + from, so fall back to the configured defaults. + """ + if not mc.PROFILE: + return None + end = int(end) + idx = bisect.bisect_left(mc.PROFILE, (end,)) + if idx < len(mc.PROFILE) and mc.PROFILE[idx][0] == end: + return mc.PROFILE[idx][1] + return mc.PROFILE[idx][1] if idx < len(mc.PROFILE) else None diff --git a/src/MissionParallelCatchup/lib/ranges.py b/src/MissionParallelCatchup/lib/monitor/ranges.py similarity index 87% rename from src/MissionParallelCatchup/lib/ranges.py rename to src/MissionParallelCatchup/lib/monitor/ranges.py index f4200d7a..cf186f03 100644 --- a/src/MissionParallelCatchup/lib/ranges.py +++ b/src/MissionParallelCatchup/lib/monitor/ranges.py @@ -6,6 +6,7 @@ """ import config +import monitor_config as mc import profiles @@ -15,7 +16,7 @@ def _uniform_segment(start_ledger, end_ledger, seg_size): el = end_ledger while el > start_ledger: ledgers_per_job = min(el - start_ledger, seg_size) - out.append((el, ledgers_per_job + config.OVERLAP_LEDGERS)) + out.append((el, ledgers_per_job + mc.OVERLAP_LEDGERS)) el -= ledgers_per_job return out @@ -57,15 +58,15 @@ def _ordered(ranges): """ # validate_config() rejects an unknown order at startup; the raise here is # the backstop for a caller that skipped it, never the primary check. - if config.RANGE_ORDER == 'tip-first': + if mc.RANGE_ORDER == 'tip-first': return ranges - elif config.RANGE_ORDER == 'oldest-first': + elif mc.RANGE_ORDER == 'oldest-first': return list(reversed(ranges)) - elif config.RANGE_ORDER == 'longest-first': + elif mc.RANGE_ORDER == 'longest-first': return _longest_first(ranges) else: raise ValueError("RANGE_ORDER must be one of %s, got %r" - % (', '.join(config.VALID_RANGE_ORDERS), config.RANGE_ORDER)) + % (', '.join(mc.VALID_RANGE_ORDERS), mc.RANGE_ORDER)) def generate_ranges(): @@ -78,6 +79,6 @@ def generate_ranges(): also became unreachable when the range moved into /start, which carries no generator. Recover it from 8553e77 if the guess ever beats the measurement. """ - return _ordered(_uniform_segment(config.STARTING_LEDGER, - config.LATEST_LEDGER_NUM, - config.LEDGERS_PER_JOB)) + return _ordered(_uniform_segment(mc.STARTING_LEDGER, + mc.LATEST_LEDGER_NUM, + mc.LEDGERS_PER_JOB)) diff --git a/src/MissionParallelCatchup/lib/sizing.py b/src/MissionParallelCatchup/lib/monitor/sizing.py similarity index 89% rename from src/MissionParallelCatchup/lib/sizing.py rename to src/MissionParallelCatchup/lib/monitor/sizing.py index 7b0434e0..319fab87 100644 --- a/src/MissionParallelCatchup/lib/sizing.py +++ b/src/MissionParallelCatchup/lib/monitor/sizing.py @@ -11,6 +11,7 @@ import math import config +import monitor_config as mc import profiles import records import units @@ -38,16 +39,16 @@ def mem_for_attempt(attempt, base=None, end=None): with the memory actually free, and a higher bar before the kubelet picks it as an eviction victim. """ - if config.POOL_PREFIX: + if mc.POOL_PREFIX: promoted = pool_memory(pool_for(end, attempt)) if promoted: return promoted # Above the ladder (nebula/protostar/supernova): nothing left to promote # into, so hold at the configured request rather than inventing a value. - return base or config.REQ_MEM - base_q = units.quantity_bytes(base or config.REQ_MEM) - want = int(base_q * (config.MEM_BUMP_FACTOR ** max(0, attempt - 1))) - cap = units.quantity_bytes(config.MEM_ESCALATION_CAP) + return base or mc.REQ_MEM + base_q = units.quantity_bytes(base or mc.REQ_MEM) + want = int(base_q * (mc.MEM_BUMP_FACTOR ** max(0, attempt - 1))) + cap = units.quantity_bytes(mc.MEM_ESCALATION_CAP) return units.bytes_to_quantity(min(want, cap)) @@ -58,11 +59,11 @@ def eph_for_attempt(attempt): still be evicted under node disk pressure, and there is nothing to raise. Every other reader of LIM_EPHEMERAL already guards on it being set. """ - if not config.LIM_EPHEMERAL: + if not mc.LIM_EPHEMERAL: return None - base_q = units.quantity_bytes(config.LIM_EPHEMERAL) - want = int(base_q * (config.EPH_BUMP_FACTOR ** max(0, attempt - 1))) - return units.bytes_to_quantity(min(want, units.quantity_bytes(config.EPH_ESCALATION_CAP))) + base_q = units.quantity_bytes(mc.LIM_EPHEMERAL) + want = int(base_q * (mc.EPH_BUMP_FACTOR ** max(0, attempt - 1))) + return units.bytes_to_quantity(min(want, units.quantity_bytes(mc.EPH_ESCALATION_CAP))) def _rung_listed(raw, tier, nxt): @@ -72,7 +73,7 @@ def _rung_listed(raw, tier, nxt): def _rung_blocked(tier, nxt): """Is this rung denied outright? Beats every other consideration.""" - return _rung_listed(config.POOL_BLOCK_RUNGS, tier, nxt) + return _rung_listed(mc.POOL_BLOCK_RUNGS, tier, nxt) def _parsed_pool_tiers(): @@ -82,7 +83,7 @@ def _parsed_pool_tiers(): which is how supernova is expressed without inventing a ceiling. """ out = [] - for item in config.POOL_TIERS.split(','): + for item in mc.POOL_TIERS.split(','): item = item.strip() if not item: continue @@ -203,24 +204,24 @@ def pool_for(end, attempt=1, rungs=None): burned ~260 vCPU of a 2304 quota escalating away from a problem that was never memory. """ - if not config.POOL_PREFIX: + if not mc.POOL_PREFIX: return None if rungs is None: # Attempts before this one, since this attempt has not run yet. Anything # on disk for it is from a previous incarnation of the same attempt. rungs = records._oom_count(end, attempt - 1) if attempt and attempt > 1 else 0 - if not config.PROFILE: - return _promote(config.POOL_NO_PROFILE, rungs) + if not mc.PROFILE: + return _promote(mc.POOL_NO_PROFILE, rungs) prof = profiles.profile_for(end) if end is not None else None if not prof: - return _promote(config.POOL_UNPROFILED, rungs) + return _promote(mc.POOL_UNPROFILED, rungs) anon = prof.get('peakAnonBytes') tier = _cache_bump(_tier_for_bytes(anon), anon, prof.get('peakWorkingSetBytes')) if not tier: # An entry with no memory measurement tells us nothing about size -- # treat it as unprofiled rather than guessing a tier. - return _promote(config.POOL_UNPROFILED, rungs) + return _promote(mc.POOL_UNPROFILED, rungs) return _promote(tier, rungs) @@ -241,7 +242,7 @@ def _pool_map(raw, what): def pool_cpu(tier): """cpu request for a tier, or None to keep the configured one.""" - return _pool_map(config.POOL_CPU, 'POOL_CPU').get(tier) + return _pool_map(mc.POOL_CPU, 'POOL_CPU').get(tier) def pool_memory(tier): @@ -255,7 +256,7 @@ def pool_memory(tier): Half is the smallest value that still excludes a second pod once the daemonsets are counted, and the largest that reliably schedules the first. """ - return _pool_str_map(config.POOL_MEM, 'POOL_MEM').get(tier) + return _pool_str_map(mc.POOL_MEM, 'POOL_MEM').get(tier) def _pool_str_map(raw, what): @@ -284,10 +285,10 @@ def _positive_seconds(value): def _profile_seconds(): """Every valid measured runtime in the profile, sorted.""" - if config._SORTED_SECONDS is None: - values = (_positive_seconds(r.get('seconds')) for _, r in (config.PROFILE or [])) - config._SORTED_SECONDS = sorted(seconds for seconds in values if seconds is not None) - return config._SORTED_SECONDS + if mc._SORTED_SECONDS is None: + values = (_positive_seconds(r.get('seconds')) for _, r in (mc.PROFILE or [])) + mc._SORTED_SECONDS = sorted(seconds for seconds in values if seconds is not None) + return mc._SORTED_SECONDS def _runtime_insurance(seconds, allowance): @@ -315,7 +316,7 @@ def _profile_overrides(end, escalated, attempt=1): """ if end is None: return {} - if escalated and not config.POOL_PREFIX: + if escalated and not mc.POOL_PREFIX: # Unpooled: an escalation measures THIS run and outranks anything an # earlier one saw. Pooled: the promotion IS the escalation, and the # promoted tier's cut is the escalated request -- bailing out here would @@ -325,14 +326,14 @@ def _profile_overrides(end, escalated, attempt=1): out = {} if prof: disk = prof.get('peakEphemeralBytes') - if disk and config.LIM_EPHEMERAL: - want = (int(disk * config.PROFILE_MARGIN) - + units.quantity_bytes(config.PROFILE_EPHEMERAL_HEADROOM) + if disk and mc.LIM_EPHEMERAL: + want = (int(disk * mc.PROFILE_MARGIN) + + units.quantity_bytes(mc.PROFILE_EPHEMERAL_HEADROOM) + _runtime_insurance(prof.get('seconds'), - config.PROFILE_RUNTIME_EPHEMERAL_INSURANCE)) + mc.PROFILE_RUNTIME_EPHEMERAL_INSURANCE)) out['ephemeral-storage'] = units.bytes_to_quantity( - min(want, units.quantity_bytes(config.PROFILE_MAX_EPHEMERAL))) - if config.POOL_PREFIX: + min(want, units.quantity_bytes(mc.PROFILE_MAX_EPHEMERAL))) + if mc.POOL_PREFIX: # Deliberately BEFORE the no-profile bail. pool_for resolves a tier for # every range -- protostar when the range is newer than the profile, # nebula when there is no profile at all -- so returning {} here would @@ -368,9 +369,9 @@ def _profile_overrides(end, escalated, attempt=1): # tracked anon still sizes exactly as it used to. rss = prof.get('peakAnonBytes') if rss: - want = (int(rss * config.PROFILE_MARGIN) - + units.quantity_bytes(config.PROFILE_CACHE_HEADROOM) + want = (int(rss * mc.PROFILE_MARGIN) + + units.quantity_bytes(mc.PROFILE_CACHE_HEADROOM) + _runtime_insurance(prof.get('seconds'), - config.PROFILE_RUNTIME_MEMORY_INSURANCE)) - out['memory'] = units.bytes_to_quantity(min(want, units.quantity_bytes(config.PROFILE_MAX_MEM))) + mc.PROFILE_RUNTIME_MEMORY_INSURANCE)) + out['memory'] = units.bytes_to_quantity(min(want, units.quantity_bytes(mc.PROFILE_MAX_MEM))) return out diff --git a/src/MissionParallelCatchup/lib/units.py b/src/MissionParallelCatchup/lib/monitor/units.py similarity index 100% rename from src/MissionParallelCatchup/lib/units.py rename to src/MissionParallelCatchup/lib/monitor/units.py diff --git a/src/MissionParallelCatchup/lib/worker_liveness.py b/src/MissionParallelCatchup/lib/monitor/worker_liveness.py similarity index 94% rename from src/MissionParallelCatchup/lib/worker_liveness.py rename to src/MissionParallelCatchup/lib/monitor/worker_liveness.py index 8d86df2e..df7f05f0 100644 --- a/src/MissionParallelCatchup/lib/worker_liveness.py +++ b/src/MissionParallelCatchup/lib/monitor/worker_liveness.py @@ -14,6 +14,7 @@ import aiohttp import config +import monitor_config as mc logger = logging.getLogger() @@ -57,9 +58,9 @@ async def sweep(targets, concurrency=None, timeout=None, deadline=None): must not discard the other 1023 answers. asyncio.wait with a deadline keeps whatever finished and cancels only the stragglers. """ - concurrency = int(concurrency or config.LIVENESS_MAX_CONCURRENCY) - timeout = float(timeout or config.LIVENESS_PROBE_TIMEOUT_SECONDS) - deadline = float(deadline or config.LIVENESS_SWEEP_SECONDS) + concurrency = int(concurrency or mc.LIVENESS_MAX_CONCURRENCY) + timeout = float(timeout or mc.LIVENESS_PROBE_TIMEOUT_SECONDS) + deadline = float(deadline or mc.LIVENESS_SWEEP_SECONDS) counts = {'up': 0, 'down': 0, 'unknown': len(targets)} if not targets: return {'up': 0, 'down': 0, 'unknown': 0} diff --git a/src/MissionParallelCatchup/lib/profiles.py b/src/MissionParallelCatchup/lib/profiles.py deleted file mode 100644 index af0afed5..00000000 --- a/src/MissionParallelCatchup/lib/profiles.py +++ /dev/null @@ -1,81 +0,0 @@ -"""The measured profile of a previous run, and the lookup into it. - -Loaded once at startup into config.PROFILE and read through it thereafter, so a -test that patches the profile is seen here without reloading anything. -""" - -import bisect -import json -import logging - -import config - -logger = logging.getLogger() - - -def load_profile_doc(doc): - """The sorted (end, record) list a parsed profile document yields. - - An unprofiled run POSTs {} and gets [] -- a profile is an optimisation, - never a prerequisite, so "no ranges" is a valid answer rather than an error. - """ - ranges = (doc or {}).get('ranges') or {} - return sorted((int(k), v) for k, v in ranges.items()) - - -def load_profile(): - """Per-range measurements from an earlier run, keyed by range end. - - Absent, unreadable or malformed all mean the same thing: size from the - configured defaults. A profile is an optimisation, never a prerequisite. - """ - if not config.PROFILE_PATH: - return [] - try: - with open(config.PROFILE_PATH) as fh: - doc = json.load(fh) - except (OSError, ValueError) as e: - logger.warning("range profile %s unreadable (%s); using configured requests", - config.PROFILE_PATH, e) - return [] - mode = doc.get('storageMode') - cross_mode = bool(mode) and mode != config.STORAGE_MODE - if cross_mode: - # cpu and memory carry across modes -- they measure the same work. Disk - # does not: a pvc run puts /data on the volume, so it never measures - # node-local usage, and an ephemeral run's figure says nothing about a - # pvc one. Keep the transferable axes and let disk fall back to the - # configured default. - logger.warning("range profile is for storageMode=%s but this run is %s; " - "using its cpu and memory, defaulting ephemeral storage", - mode, config.STORAGE_MODE) - out = [] - for end, rec in (doc.get('ranges') or {}).items(): - try: - end = int(end) - except (TypeError, ValueError): - continue - if cross_mode: - rec = {k: v for k, v in rec.items() if k != 'peakEphemeralBytes'} - out.append((end, rec)) - out.sort() - logger.info("loaded range profile: %d ranges from %s", len(out), config.PROFILE_PATH) - return out - - -def profile_for(end): - """Measurements to size this range from, or None to use the defaults. - - Exact end, else the nearest measured end ABOVE it. Cost rises with ledger - position -- the bucket set only grows -- so a lower neighbour under-reports, - and under-provisioning costs an eviction while over-provisioning only costs - packing. Past the top of the profile there is nothing safe to extrapolate - from, so fall back to the configured defaults. - """ - if not config.PROFILE: - return None - end = int(end) - idx = bisect.bisect_left(config.PROFILE, (end,)) - if idx < len(config.PROFILE) and config.PROFILE[idx][0] == end: - return config.PROFILE[idx][1] - return config.PROFILE[idx][1] if idx < len(config.PROFILE) else None From a45335095ffdbc8981251743fd11df272ab72546 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Fri, 14 Aug 2026 17:28:56 -0400 Subject: [PATCH 112/117] refactor(parallel-catchup): rewrite the collector as one cycle over the pod list The collector kept a task per pod, and that one decision forced a registry, wake events, cancellation, grace cycles and a peak cache -- roughly half its state existed to coordinate tasks rather than to collect anything. One cycle now: list pods, then read every log and sample every kubelet off that list. Two conditions carry the work -- the first read of an attempt records its resume decision, the read that finds the pod terminal records the verdict, txApply, duration and .done. - module state is 5 names (3 semaphores, follows, last_ts), from 17 - peaks go straight to .metrics, which already max-merges: no in-memory copy to lose on restart, and no flush ratio to approximate it - keyed by (end, attempt) like every file, so a vanished attempt is identifiable once its pod is gone - .done on disk replaces the finalized set, so a restart stops re-finalizing - no persistent scanner: the terminal read reaches back TERMINAL_REREAD_SECONDS so a straddled medida block is whole in one read, and _ingest dedups the overlap - records.py is the cross-process contract only; retry accounting moved to lib/monitor/attempt_files, resume points to lib/collector/state_files - medida folded into tx_scan: the second reader it existed for is gone - delete tx_scan.scan_archive and 5 chart knobs the rewrite orphaned log_collector.py: 1350 -> 524 lines. Fixed on the way: _poll_once acquired the same semaphore the gather held, which deadlocks every holder at full occupancy; hydration dropped the timestamp validation that keeps a poisoned .state from making every request 400. Testing: 3-range and 4-range missions on ssc-test, the latter with maxConcurrentPolls pinned to 2 against 4 pods so the bound was exercised. 4/4 ranges completed with txApply, peaks and exact durations. 140 contract tests. Co-Authored-By: Claude Opus 5 --- .../apps/job_monitor.py | 22 +- .../apps/log_collector.py | 1172 ++++++----------- .../lib/collector/collector_config.py | 46 +- .../lib/collector/medida.py | 30 - .../lib/collector/state_files.py | 33 + .../lib/collector/tx_scan.py | 91 +- .../lib/monitor/attempt_files.py | 77 ++ .../lib/monitor/attempts.py | 7 +- .../lib/monitor/kube.py | 1 - .../lib/monitor/ranges.py | 1 - .../lib/monitor/sizing.py | 5 +- .../lib/monitor/worker_liveness.py | 1 - src/MissionParallelCatchup/lib/records.py | 72 +- .../templates/job_monitor.yaml | 18 +- .../parallel_catchup_helm/values.yaml | 25 +- 15 files changed, 572 insertions(+), 1029 deletions(-) delete mode 100644 src/MissionParallelCatchup/lib/collector/medida.py create mode 100644 src/MissionParallelCatchup/lib/collector/state_files.py create mode 100644 src/MissionParallelCatchup/lib/monitor/attempt_files.py diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py index e774978d..fcb81885 100644 --- a/src/MissionParallelCatchup/apps/job_monitor.py +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -23,17 +23,14 @@ """ -import bisect import collections import gzip import json -import math import os import re import threading import time -import zlib -from datetime import datetime, timezone +from datetime import datetime from kubernetes import client from kubernetes.client.rest import ApiException @@ -46,6 +43,7 @@ import metrics import profiles import ranges +import attempt_files import records import sizing import worker_liveness @@ -552,7 +550,7 @@ def record_range_start(end, job): wallSeconds measures. Written at creation because attempt 1's Job is gone on the first retry. """ - path = records.started_path(end) + path = attempt_files.started_path(end) if os.path.exists(path): return created = job.metadata.creation_timestamp if job and job.metadata else None @@ -567,7 +565,7 @@ def record_range_start(end, job): def range_started_at(end): """attempt 1's Job creationTimestamp, or None if it was never recorded.""" try: - with open(records.started_path(end)) as fh: + with open(attempt_files.started_path(end)) as fh: return datetime.fromisoformat(fh.read().strip()) except (OSError, ValueError): return None @@ -726,7 +724,7 @@ def save_verdict(end, attempt, outcome): once, and this is where the answer is kept, on the same durable logs volume as everything else, so a monitor restart does not reset a range's budgets. """ - path = records.verdict_path(end, attempt) + path = attempt_files.verdict_path(end, attempt) try: records.write_atomic(path, str(outcome)) except OSError as e: @@ -1178,7 +1176,7 @@ def remember(end, attempt): effective = {} for end, attempt in verdict_files: try: - with open(records.verdict_path(end, attempt)) as fh: + with open(attempt_files.verdict_path(end, attempt)) as fh: verdict = fh.read().strip() except OSError: continue @@ -1361,7 +1359,7 @@ def verdict_for(end, attempt, job, pod): the drained pod reads as a plain `failed` 3. else whichever exists, unknown over nothing: retry rather than condemn """ - from_pod = records.read_outcome(end, attempt) + from_pod = attempt_files.read_outcome(end, attempt) from_job = classify_from_job(job) if from_pod and from_pod.get('outcome') in mc.POD_AUTHORITATIVE_OUTCOMES: verdict = from_pod @@ -1407,7 +1405,7 @@ def _retry_oom(end, attempt): `attempt` instead names a rung nobody occupied. """ base = (sizing._profile_overrides(end, escalated=False) or {}).get('memory') - ooms = records._oom_count(end, attempt) + ooms = attempt_files._oom_count(end, attempt) had = (sizing.pool_memory(sizing.pool_for(end, attempt)) if mc.POOL_PREFIX else sizing.mem_for_attempt(ooms, base)) return _retry(f"OOM-killed at memory request {had}", @@ -1420,7 +1418,7 @@ def _retry_ephemeral(end, attempt): Rungs climbed = evictions seen, not attempts made, as with the OOM ladder: on spot most retries are disruptions. The count includes this attempt. """ - evictions = records._cause_count(end, attempt, ('ephemeral',)) + evictions = attempt_files._cause_count(end, attempt, ('ephemeral',)) had = sizing.eph_for_attempt(evictions) reason = (f"evicted for exceeding its {had} ephemeral-storage limit" if had else "evicted under node disk pressure with no configured limit") @@ -1483,7 +1481,7 @@ def budget_for(verdict, end, attempt): the Nth failure is the one that exhausts a budget of N. """ outcome = verdict['outcome'] - return (records._cause_count(end, attempt, (outcome,)), + return (attempt_files._cause_count(end, attempt, (outcome,)), mc.ATTEMPT_BUDGETS.get(outcome, 0)) diff --git a/src/MissionParallelCatchup/apps/log_collector.py b/src/MissionParallelCatchup/apps/log_collector.py index 94639735..d8f5d48b 100644 --- a/src/MissionParallelCatchup/apps/log_collector.py +++ b/src/MissionParallelCatchup/apps/log_collector.py @@ -1,321 +1,369 @@ """Streaming log collector for parallel catchup. -Runs as a sidecar next to job_monitor, sharing its /logs volume. Polls each -pod's log rather than reading after the Job finishes: Karpenter deletes the node -about a minute after its last pod exits, taking every pod object with it, and -polling also keeps a straggler readable *while* it is stuck. A condemned pod -gets a follow stream so its last lines land before it goes. - -Resume is idempotent across a dropped stream and a restart of this process: -reconnect with sinceTime=, which has second -granularity and so overlaps on purpose, then drop any line whose own kubelet -RFC3339Nano timestamp is <= last_ts. Residual: dying between flushing bytes and -rewriting the state file replays one poll's worth of lines, so this is at least -once, deduped to near-exact. +Runs as a sidecar next to job_monitor, sharing its /logs volume. One cycle: +list the run's pods, then read every pod's log and sample every node's kubelet +off that list. Nothing is read after a Job finishes -- Karpenter deletes the +node about a minute after its last pod exits, taking every pod object with it, +so the answers have to be taken while the pod still exists. + +Two conditions carry all the work. The FIRST read of an attempt carries its +resume decision; the read that finds the pod terminal carries its verdict, its +tx-apply total and the .done marker the monitor reaps on. Everything between is +appending bytes. + +Resume is idempotent across a dropped read and a restart: reconnect with +sinceTime=, which has second granularity and so overlaps +on purpose, then drop any line whose own kubelet timestamp is <= it. Dying +between the append and the state write replays one read's worth of lines, so +this is at least once, deduped to near-exact. """ import asyncio import gzip import io import json -import logging import os import re -import sys -from datetime import datetime +from datetime import datetime, timedelta import aiohttp from logger import build_logger import collector_config as cc import kube_http -import tx_scan -import verdicts import config import records +import state_files +import tx_scan +import verdicts -# Every piece of live runtime state this process holds, and the only module-level -# names here that are not settings: two semaphores, which carry their own waiter -# queues, and the dicts recording what is known about the pods being watched. -# Everything tunable lives in collector_config. +logger = build_logger('log_collector', name='log-collector', to_file=False) -# Bounds in-flight polls across every pod. +# Bounds concurrent log reads, one coroutine per pod per cycle. _poll_slots = asyncio.Semaphore(cc.MAX_CONCURRENT_POLLS) -# Separate from _poll_slots on purpose -- see MAX_DOOMED_FOLLOWS. +# One request per NODE, and a dead one costs the connect timeout. +_sample_slots = asyncio.Semaphore(cc.MAX_CONCURRENT_SAMPLES) +# Follows get their own budget: one holds its slot for a whole drain, so sharing +# would let a mass reclaim starve every ordinary read in the run. _follow_slots = asyncio.Semaphore(cc.MAX_DOOMED_FOLLOWS) -# pod name -> Event, set when the pod is first seen terminal, gone or condemned; -# poll_pod waits on it rather than sleeping, so LOG_POLL_SECONDS is a ceiling and -# the final read is not left until after it. -_wake = {} -# pod name -> its own start->finish, read off the pod while it still exists. -_pod_secs = {} -# pod name -> status.startTime, so an attempt whose object vanished before any -# cycle saw its terminated timestamp can still be dated from the container's own -# start rather than from whenever this poller happened to open. -_pod_start = {} -# Pods carrying a DisruptionTarget condition: the cluster has committed to -# destroying them and, on spot, gives about two minutes' notice. Waking the -# poller is not enough on its own -- stellar-core prints its medida block ~4ms -# after SIGTERM and the object is deleted seconds later, so an interval poll -# straddles the whole thing (2048-worker run: 810 evictions lost 809 txApply -# values), where a held connection already has those bytes. Safe because it is -# scoped to the drain, not because the count is small: global follow=true cost -# the sidecar 1444 MiB of a 2048 MiB limit at 2096 whole-range streams. -_doomed = {} - -# Peak ephemeral disk, for sizing a later run's ephemeral-storage request. Only -# meaningful in ephemeral mode, and sampled for every pod but only kept for -# ranges that finished -- what invalidates a sample is being cut short, not the -# capacity type. Prometheus cannot answer it (cAdvisor reports fs usage per node, -# with no pod label), so this samples kubelet directly and keeps a running max. -_eph_peak = {} -_anon_peak = {} -_ws_peak = {} -# Last value flushed to the volume, per axis: keyed by pod name for anon and by -# "/eph" for ephemeral. A pod name cannot contain '/', so the two key -# spaces cannot collide. -_peak_flushed = {} -# pod name -> (end, attempt), so a mid-flight peak flush can find its file. -_streaming = {} -# The poller registry, module-level so the watch can open a stream the moment a -# pod appears instead of waiting for the pod-list loop. One registry with one -# guard is the whole point: read_state is consulted once, at poll_pod start, so -# two creators would each hold their own in-memory last_ts, re-append the same -# lines and race each other's write_state. -_tasks = {} -_streamed = set() -# session + the terminal/succeeded views poll_pod closes over, published once by -# main() so ensure_stream can be called from outside it. -_stream_ctx = {} - - -logger = build_logger('log_collector', name='log-collector', to_file=False) +# (end, attempt) -> resume timestamp, hydrated from .state at boot and written +# back through. Keyed like every file on the volume, so a vanished attempt is +# still identifiable once its pod is gone. +_last_ts = {} +# (end, attempt) -> the follow task held through a drain. The only work that +# outlives a cycle. +_follows = {} async def main(): os.makedirs(config.LOG_DIR, exist_ok=True) - # Connection-pool limit, not a task limit: there is no semaphore above it, - # so a stream that cannot get a connection blocks here until the pool drains - # -- it does not degrade, it starves, and it starves the pods created last, - # which are the retries. Sized for concurrent polls plus headroom for the - # pod-list and kubelet calls, not one connection per pod: under follow=true - # a 1200 limit against 2048 workers left 896 blocked forever. conn = aiohttp.TCPConnector( - limit=cc.MAX_CONCURRENT_POLLS + cc.MAX_DOOMED_FOLLOWS + 64, ssl=kube_http.ssl_ctx()) - # No total timeout: these streams are meant to stay open for the life of a - # range, which can be hours. - timeout = aiohttp.ClientTimeout(total=None, sock_connect=10) - # The module-level registry under local names, so the watch shares the same - # guard as the bookkeeping below. - tasks, streamed = _tasks, _streamed - # Cleared rather than assumed empty: a second main() in one process would - # otherwise find every pod already registered and open no streams at all. - tasks.clear() - streamed.clear() - _stream_ctx.clear() - terminal, succeeded, vanished = {}, {}, {} - # `streamed` holds streams that ran to completion. Without it a finished - # task is deleted from `tasks` and the next poll re-opens the stream, - # forever: one full log re-read per pod every POLL_SECONDS. + limit=cc.MAX_CONCURRENT_POLLS + cc.MAX_DOOMED_FOLLOWS + 64, + ssl=kube_http.ssl_ctx()) + # sock_read, because a cycle gathers every pod: one wedged read would hold + # up the whole pass. No total timeout -- a follow is meant to be held. + timeout = aiohttp.ClientTimeout(total=None, sock_connect=10, + sock_read=cc.READ_TIMEOUT_SECONDS) + _last_ts.clear() + _last_ts.update(state_files.hydrate_states()) + logger.info("resuming %d attempts from disk", len(_last_ts)) async with aiohttp.ClientSession(connector=conn, timeout=timeout) as session: logger.info("streaming logs for run=%s into %s", config.RUN_NAME, config.LOG_DIR) - # Published before the watch starts: ensure_stream is a no-op until this - # exists, so a watch event arriving first would silently open nothing. - _stream_ctx.update(session=session, terminal=terminal, succeeded=succeeded) if cc.WATCH_TIMEOUT_SECONDS > 0: + # The one thing that cannot ride the cycle: a held watch starts a + # follow between sweeps, which is the whole point of it. asyncio.create_task(watch_condemnations(session)) while True: try: pods = await list_pods(session) - live = {p['metadata']['name'] for p in pods} - # A pod can leave the list without ever being observed terminal - # -- reaped node, eviction, or the monitor deleting its finished - # Job -- and `terminal` is only written for pods in this list, so - # its stream would retry until the run ended. Marking them - # terminal lets the stream finalize and free the slot; - # cancelling is the backstop for one wedged in a connection - # attempt it will never win. - for name in [n for n in tasks if n not in live]: - terminal[name] = True - if name in _wake: - # Gone is terminal. Without this its poller sleeps out - # the interval before taking the 404, delaying finalize - # and the .done that lets the monitor reap the Job. - _wake[name].set() - t = tasks[name] - if t.done(): - del tasks[name] - streamed.add(name) - continue - vanished[name] = vanished.get(name, 0) + 1 - if vanished[name] >= cc.VANISHED_GRACE_CYCLES: - t.cancel() - try: - await t - except asyncio.CancelledError: - pass - del tasks[name] - vanished.pop(name, None) - ref = _streaming.get(name) - if ref is not None: - # The poller was wedged, but its archive and the - # sampler's process-local peaks still contain useful - # truth. Finalize them before licensing a reap. - await finalize( - session, name, ref[0], ref[1], - tx_scan.TxApplyScanner(recreated=True), - lambda p: succeeded.get(p, False)) - streamed.add(name) - logger.info("cancelled and finalized stream for vanished pod %s", - name) - for pod in pods: - name = pod['metadata']['name'] - labels = pod['metadata'].get('labels', {}) - end = labels.get(config.LABEL_RANGE) - if end is None: - continue - phase = pod.get('status', {}).get('phase') - terminal[name] = phase in ('Succeeded', 'Failed') - # NOT gated on phase: a pod being deleted keeps phase Running - # until its object disappears, so gating this on terminal - # meant no disrupted pod ever recorded an exact duration and - # every one fell back to the poller's clock -- 268s reported - # against a ~500s attempt. terminated.finishedAt is present - # for the ~8s the object outlives the container, and - # pod_seconds returns None until then, so asking every cycle - # is self-guarding. - secs = pod_seconds(pod) - if secs is not None: - _pod_secs[name] = secs - start = (pod.get('status') or {}).get('startTime') - if start: - # So finalize can date the attempt from when the - # container STARTED if the object is deleted before any - # cycle catches its terminated timestamp. - _pod_start[name] = start - # Backstop only: the watch normally gets here first. This - # still runs so detection survives the watch being disabled - # or reconnecting. - if not terminal[name]: - _mark_condemned(pod, name, end, - labels.get(config.LABEL_ATTEMPT, '1')) - if terminal[name] and name in _wake: - # Wake its poller now rather than at the next tick. - _wake[name].set() - succeeded[name] = phase == 'Succeeded' - if phase == 'Failed': - verdicts.record_outcome(pod, end, labels.get(config.LABEL_ATTEMPT, '1')) - if name in tasks and not tasks[name].done(): - continue - if name in tasks and tasks[name].done(): - del tasks[name] - # Only bar a re-open once the pod itself is terminal. A - # task that ended while the pod is still running died - # early, and re-opening is the recovery path. - if terminal.get(name): - streamed.add(name) - continue - if name in streamed: - continue - # Backstop: the watch normally opens this the moment the pod - # appears, and this covers events dropped across a - # reconnect. Same registry and guard, so whichever gets - # there first wins -- two readers on one pod would duplicate - # the archive and race write_state. - ensure_stream(name, end, labels.get(config.LABEL_ATTEMPT, '1'), phase) - - # AFTER the per-pod branches, never before them: this is a - # serial sweep of every node's kubelet and on spot a dead one - # costs the 10s connect timeout apiece, which once stretched a - # cycle to 925s. It must also stay outside the `for` loop, whose - # branches `continue` for a pod already streaming, so a sampler - # among them fires only on the cycle a stream opens. - # Unconditional, not gated on ephemeral mode: memory is sized in - # both modes, and gating left every pvc run with no anon peak. - await sample_kubelet(session, { - p['status']['hostIP'] for p in pods - if p.get('status', {}).get('hostIP') - and p.get('status', {}).get('phase') == 'Running'}) + ranges = {p['metadata']['name']: _identify(p) + for p in pods if _identify(p)} + nodes = {p['status']['hostIP'] for p in pods + if p.get('status', {}).get('hostIP') + and p.get('status', {}).get('phase') == 'Running'} + await asyncio.gather( + _bounded(_poll_slots, + [service_pod(session, p) for p in pods if _wanted(p)]), + _bounded(_sample_slots, + [sample_node(session, ip, ranges) for ip in nodes])) + _sweep_vanished(set(ranges.values())) + except asyncio.CancelledError: + raise except Exception as e: - logger.warning("pod list failed: %s", e) + logger.warning("cycle failed: %s", e) await asyncio.sleep(cc.POLL_SECONDS) -def _mark_condemned(pod, name, end, attempt): - """Flag a condemned pod so its poller opens a follow. Idempotent. +async def _bounded(slots, coros): + """Run coros under `slots`. One failure must not cancel the other 4095.""" + async def run(c): + async with slots: + return await c + return await asyncio.gather(*(run(c) for c in coros), return_exceptions=True) + + +def _identify(pod): + """(end, attempt) for a worker pod, or None if it is not one of ours.""" + labels = pod['metadata'].get('labels', {}) + end = labels.get(config.LABEL_RANGE) + return (end, labels.get(config.LABEL_ATTEMPT, '1')) if end else None + + +# Phases whose log endpoint can answer. Pending has no container yet and Unknown +# means the node stopped reporting; the terminal phases are kept because that is +# where a pod's final output lives. +POLLABLE_PHASES = ('Running', 'Succeeded', 'Failed') - Shared by the pod-list sweep and the watch so the two cannot drift: - whichever sees the condition first does the work, the other no-ops. - Detection latency, not the follow, is what loses the metric -- stellar-core - exits about a second after SIGTERM and the object is reaped right behind it. + +def _wanted(pod): + """Pods whose log endpoint can answer and whose attempt is not finished. + + Pending has no container yet and Unknown means the node stopped reporting; + the terminal phases are kept because that is where the final output lives. """ - if name in _doomed: + key = _identify(pod) + if key is None or pod.get('status', {}).get('phase') not in POLLABLE_PHASES: return False - if (pod.get('status') or {}).get('phase') in ('Succeeded', 'Failed'): - # Already finished. Its log is complete and a follow would only re-read - # a dead pod every iteration. + return not os.path.exists(records.done_path(*key)) + + +async def service_pod(session, pod): + """One pod, one cycle: read what is new, then apply whichever condition fits.""" + name = pod['metadata']['name'] + end, attempt = _identify(pod) + phase = pod.get('status', {}).get('phase') + terminal = phase in ('Succeeded', 'Failed') + + if not terminal and _start_follow(session, name, end, attempt, pod): + return # the follow owns this attempt's stream + if (end, attempt) not in _last_ts: + # Empty state = "claimed, nothing durable yet": the monitor's backstop + # skips any range that has one, so this stops both of us writing the log. + _write_state(end, attempt, '') + since = _last_ts[(end, attempt)] + # A terminal read reaches further back, so a medida block split across two + # reads is whole in this one. _ingest still dedups, so the archive is + # unchanged -- only the scan sees the overlap. + text, gone = await _read(session, name, _rewind(since) if terminal else since) + if text: + _write_state(end, attempt, _ingest(end, attempt, text, since)) + _scan(end, attempt, text) + if terminal or gone: + _finish(None if gone else pod, end, attempt, phase == 'Succeeded') + + +def _start_follow(session, name, end, attempt, pod): + """Hold a stream through a drain, if this pod has been condemned. + + stellar-core prints its medida block ~4ms after SIGTERM and the object is + deleted seconds later, so an interval read straddles the whole thing -- on + the 2048-worker run, 810 evictions lost 809 txApply values. A held + connection already has those bytes, and is held only for the drain. + """ + key = (end, attempt) + if key in _follows: + return not _follows[key].done() + if cc.DOOMED_FOLLOW_SECONDS <= 0: return False doom = verdicts.condemnation_reason(pod) if not doom: return False - _doomed[name] = doom - # Recorded now: once the object is gone there is no way to tell a drain we - # lost a race with from a corpse that never had a metric to lose. + # Recorded now: once the object is gone there is no telling a drain we lost + # a race with from a corpse that never had a metric to lose. write_metrics(end, attempt, {'disruptionReason': doom}) - if name in _wake: - # Break the current sleep so the follow opens now rather than up to - # LOG_POLL_SECONDS from now. - _wake[name].set() - logger.info("range %s: pod %s condemned (%s), opening follow", end, name, doom) + logger.info("range %s condemned (%s), opening follow", end, doom) + _follows[key] = asyncio.create_task(_run_follow(session, name, end, attempt)) return True -def pod_seconds(pod): - """Container start -> finish from the pod's own status, or None. +async def _run_follow(session, name, end, attempt): + async with _follow_slots: + since = _last_ts.get((end, attempt), '') + try: + text, _ = await _read(session, name, since, follow=True) + except asyncio.CancelledError: + raise + except Exception as e: + logger.info("range %s follow ended (%s); back to interval reads", end, e) + return + if text: + _write_state(end, attempt, _ingest(end, attempt, text, since)) + _scan(end, attempt, text) + + +def _finish(pod, end, attempt, succeeded): + """Everything the attempt owes, then the marker that licenses a reap.""" + if pod is not None: + if (pod.get('status') or {}).get('phase') == 'Failed': + verdicts.record_outcome(pod, end, attempt) + write_metrics(end, attempt, _duration(pod)) + if not config.SAVE_SUCCESS_LOGS and succeeded: + # .metrics survives on purpose: it holds tx_apply for a range that + # succeeded, and a retention flag must not delete a Grafana series. + discard(end, attempt) + task = _follows.pop((end, attempt), None) + if task is not None and not task.done(): + task.cancel() + _last_ts.pop((end, attempt), None) + # LAST. The monitor treats this as "nothing further is coming" and only then + # reaps the Job -- which deletes the pod, the one place peaks can be read + # from. Ahead of the metrics it would license exactly the reap it prevents. + try: + records.write_atomic(records.done_path(end, attempt), '') + except OSError as e: + # Costs a Job that waits out its TTL, never correctness. + logger.warning("could not mark range %s attempt %s done: %s", end, attempt, e) + logger.info("range %s attempt %s finished", end, attempt) + + +def _duration(pod): + """How long the container ran, and how well that is known. - The same fields the monitor reads. A terminal pod still carries them until - it is deleted, so this works even when the collector never watched the - container run -- which the poller's own elapsed time cannot. + The pod's own start->finish is exact. Its startTime against now is off by + the cycle that noticed, but still measures the CONTAINER -- the distinction + the monitor's chain gate cares about, and the only duration a disrupted + attempt ever produces. A clock this process started measures neither, and is + not recorded: the monitor rejects it anyway. """ st = pod.get('status') or {} start = st.get('startTime') if not start: - return None + return {} + try: + began = datetime.strptime(start, '%Y-%m-%dT%H:%M:%SZ') + except ValueError: + return {} for cs in (st.get('containerStatuses') or []): - term = (cs.get('state') or {}).get('terminated') or {} - fin = term.get('finishedAt') + fin = ((cs.get('state') or {}).get('terminated') or {}).get('finishedAt') if fin: try: - a = datetime.strptime(start, '%Y-%m-%dT%H:%M:%SZ') - b = datetime.strptime(fin, '%Y-%m-%dT%H:%M:%SZ') + ended = datetime.strptime(fin, '%Y-%m-%dT%H:%M:%SZ') except ValueError: - return None - return (b - a).total_seconds() - return None + break + return {'attemptSeconds': round((ended - began).total_seconds(), 1), + 'attemptSecondsExact': True} + since = (datetime.utcnow() - began).total_seconds() + if since <= 0: + return {} + return {'attemptSeconds': round(since, 1), 'attemptSecondsExact': False, + 'attemptSecondsFromContainerStart': True} + + +def _sweep_vanished(live): + """Attempts with state on the volume and no pod left to read. + + A reaped node, an eviction or the monitor deleting a finished Job all take + the pod without it ever being seen terminal. The archive still holds real + bytes, so mark the attempt done rather than leaving its Job to time out. + """ + for key in [k for k in _last_ts if k not in live]: + logger.info("range %s attempt %s: pod gone, finishing on what was read", *key) + _finish(None, key[0], key[1], False) -def read_state(end, attempt): +# --- reading ----------------------------------------------------------------- + +def _rewind(ts): + """`ts` moved back a few seconds, for the overlapping terminal read.""" + if not ts: + return ts try: - with open(records.state_path(end, attempt)) as fh: - ts = fh.read().strip() - except OSError: - return None - # Also repairs a state file poisoned by an earlier build. - return ts if ts and _TS_RE.match(ts) else None + return (datetime.strptime(ts[:19], '%Y-%m-%dT%H:%M:%S') + - timedelta(seconds=cc.TERMINAL_REREAD_SECONDS) + ).strftime('%Y-%m-%dT%H:%M:%SZ') + except ValueError: + return ts + +async def _read(session, name, last_ts, follow=False): + """One read of a pod's log. Returns (text, gone). -def write_state(end, attempt, ts): - path = records.state_path(end, attempt) + No follow=true on the ordinary path: the request completes and the + connection is released, so concurrency is bounded by the gather rather than + by how many pods exist. Held connections cost the sidecar 1444 MiB of a + 2048 MiB limit at 2096 streams. + """ + params = {'container': cc.CONTAINER, 'timestamps': 'true'} + if follow: + params['follow'] = 'true' + if last_ts: + # Second granularity, so this overlaps on purpose; _ingest removes it. + params['sinceTime'] = last_ts[:19] + 'Z' + url = f"{kube_http.API}/api/v1/namespaces/{config.NAMESPACE}/pods/{name}/log" + async with session.get(url, params=params, + headers={'Authorization': f'Bearer {kube_http.token()}'}) as resp: + if resp.status == 404: + return '', True + resp.raise_for_status() + # Chunked, not line-wise: aiohttp raises above 512 KiB on one line and a + # carriage-return progress meter exceeds that -- a 628 MiB download once + # arrived as a single "line". The cap is the backstop against a blob + # that never terminates. + body = '' + async for chunk in resp.content.iter_chunked(65536): + body += chunk.decode('utf-8', 'replace') + if len(body) > cc.MAX_POLL_CHARS: + break + return body, False + + +def _ingest(end, attempt, body, last_ts): + """Append the new lines to the archive; return the resume point.""" + pending = None + # Compressed into memory, then appended in ONE write, so the file only ever + # gains whole gzip members. Appending through gzip.open left it ending in a + # member with no end-of-stream marker for most of a large write, and the + # monitor reads that same file -- one in-flight append could abort a + # reconcile pass for every range. + member = io.BytesIO() + wrote = False + with gzip.GzipFile(fileobj=member, mode='wb') as fh: + for line in (l for l in re.split(r'[\r\n]', body) if l): + ts, _, rest = line.partition(' ') + if not state_files.TS_RE.match(ts): + fh.write((line + '\n').encode('utf-8')) # keep, never resume from + wrote = True + continue + if last_ts and ts <= last_ts: + continue # exact dedup of the overlap + fh.write((rest + '\n').encode('utf-8')) + wrote = True + pending = ts + if wrote: + with open(records.log_path(end, attempt), 'ab') as out: + out.write(member.getvalue()) + return pending or last_ts + + +def _scan(end, attempt, text): + """Record whatever this text proves: that the attempt resumed, or its total. + + Both are printed once -- RESUME before stellar-core starts, the medida block + at exit -- so a throwaway scanner over each read finds them with nothing + carried between cycles. + """ + scanner = tx_scan.TxApplyScanner() + for line in text.splitlines(): + scanner.feed(line) + found = {} + if scanner.resumed: + found['resumed'] = True + if scanner.seconds is not None: + found['txApplySeconds'] = scanner.seconds + write_metrics(end, attempt, found) + + +# --- the volume -------------------------------------------------------------- + +def _write_state(end, attempt, ts): + _last_ts[(end, attempt)] = ts try: - records.write_atomic(path, ts) + records.write_atomic(records.state_path(end, attempt), ts) except OSError as e: logger.warning("could not persist state for range %s: %s", end, e) def discard(end, attempt): - # .metrics deliberately survives: it holds tx_apply for a range that - # succeeded, which is the only case this runs in. Dropping it would let a - # log-retention flag silently delete a Grafana series. for path in (records.log_path(end, attempt), records.state_path(end, attempt)): try: os.remove(path) @@ -323,526 +371,95 @@ def discard(end, attempt): pass -# kubelet returns untimestamped plain text ("unable to retrieve container logs -# for containerd://...") when a container is not up yet. Partitioning that on -# the first space yields "unable", which lands in the state file and makes every -# later request ask for sinceTime=unableZ -> HTTP 400, forever. -_TS_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?$") +# Fields that only ever grow, so a merge maxes them instead of overwriting. +PEAK_KEYS = ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes') + def write_metrics(end, attempt, values): - """Persist per-range measurements for job_monitor's reconcile to read. + """Merge measurements into .metrics for the monitor's reconcile to read. - Kept out of .outcome on purpose: that file answers "why did this attempt - fail" and is only written for failed pods, whereas these are only - meaningful for one that succeeded. + Never lets a peak or a duration go backwards, and never un-proves a flag: a + later writer can only know LESS -- a restarted collector re-accumulates from + whatever the pod is using now, and its lower reading would undersize the + range next run, which is the one direction that costs an OOM. """ + if not values: + return path = records.metrics_path(end, attempt) - # Merge, and never let a peak go backwards. A measurement already on disk - # must survive a later write that lacks it, but a plain overwrite is wrong - # for a monotonic quantity: after a restart the fresh poller's first flush - # would replace a higher pre-restart value with a lower one, undersizing the - # range next run -- the one direction that costs an OOM. try: with open(path) as fh: prior = json.load(fh) except (OSError, ValueError): prior = {} merged = {**prior, **values} - # attemptSeconds takes the same rule: it is fixed once the attempt ends and - # every source is a lower bound on it. An attempt is finalized more than - # once whenever a poller re-opens for a pod that is still listed, and there - # the fallback clock starts at the restart -- newest-wins turned a recorded - # 3600s into 0.0s. - for k in cc.PEAK_KEYS + ('attemptSeconds',): + for k in PEAK_KEYS + ('attemptSeconds',): a, b = prior.get(k), values.get(k) if a is not None and b is not None: merged[k] = max(a, b) - # Once any poller or archive read proves this attempt resumed, a later - # restarted poller cannot un-prove it: a merge containing resumed=False must - # never lower the durable decision back to fresh. - if prior.get('resumed') is True or values.get('resumed') is True: - merged['resumed'] = True - if (prior.get('attemptSecondsExact') is True - or values.get('attemptSecondsExact') is True): - merged['attemptSecondsExact'] = True - # Same one-way rule: once a duration is dated from the container's own - # startTime, a later poller-clock write must not strip the provenance the - # monitor needs to use it as a chain leg. - if (prior.get('attemptSecondsFromContainerStart') is True - or values.get('attemptSecondsFromContainerStart') is True): - merged['attemptSecondsFromContainerStart'] = True - values = merged + for flag in ('resumed', 'attemptSecondsExact', 'attemptSecondsFromContainerStart'): + if prior.get(flag) is True or values.get(flag) is True: + merged[flag] = True + if merged == prior: + return # nothing new -- this is what makes per-cycle sampling cheap try: - records.write_atomic(path, json.dumps(values)) - logger.info("range %s attempt %s metrics=%s", end, attempt, values) + records.write_atomic(path, json.dumps(merged)) + logger.info("range %s attempt %s metrics=%s", end, attempt, merged) except OSError as e: logger.warning("could not persist metrics for range %s: %s", end, e) -def _flush_peak(name, axis, field, value): - """Persist a high-water so a sidecar restart cannot lose it. - - Every key in PEAK_KEYS needs this: the peaks live in module dicts, so a - restarted collector starts from zero and re-accumulates only from whatever - the pod is using at that moment. Missing it for peakWorkingSetBytes is how - 136 of 3095 ranges came back with a working set BELOW their own anon, which - cannot happen in one sample. write_metrics max-merges on PEAK_KEYS, so - re-flushing a lower value later is harmless. - """ - ref = _streaming.get(name) - if not ref: - return - key = name + '/' + axis - if value < _peak_flushed.get(key, 0) * cc.PEAK_FLUSH_RATIO: - return - _peak_flushed[key] = value - write_metrics(ref[0], ref[1], {field: value}) - - -def _register_stream(name, end, attempt): - """Register a poller and durably flush peaks sampled just before it opened.""" - _streaming[name] = (end, attempt) - for axis, field, values in ( - ('anon', 'peakAnonBytes', _anon_peak), - ('ws', 'peakWorkingSetBytes', _ws_peak), - ('eph', 'peakEphemeralBytes', _eph_peak)): - value = values.get(name) - if value is not None: - _flush_peak(name, axis, field, value) - - -async def sample_kubelet(session, node_ips): - """Update each pod's peak ephemeral use and peak anon from one snapshot. - - Both axes come out of the same GET, so tracking memory here is free. - kubelet's `rssBytes` is cgroup v2 `anon`, the only limit-independent memory - figure this workload has: page cache expands to fill whatever `memory.max` - allows, so `memory.peak` is always ~= the limit (a range needing 862 MiB - reported 12704 MiB against a 24000 MiB limit) and is useless for sizing. - Sampled rather than exact -- cAdvisor housekeeping is ~10s, still ~3x finer - than the 30s Prometheus scrape whose undersampling let profiled ranges OOM. - """ - for ip in node_ips: - # Straight at the kubelet, not through the apiserver's node proxy: that - # needs `nodes/proxy`, which authorizes GET on EVERY kubelet path, - # /pods and /containerLogs included, for any namespace on that node. - # Going direct is the same data under `nodes/stats`, a grant that cannot - # read pod inventory or logs at all. ssl=False because EKS kubelet - # serving certs are self-signed; in-VPC hop to the node's own address. - url = f"https://{ip}:{kube_http.KUBELET_PORT}/stats/summary" - try: - async with session.get(url, ssl=False, - headers={'Authorization': f'Bearer {kube_http.token()}'}) as resp: - resp.raise_for_status() - summary = await resp.json() - except Exception as e: - # Not debug: if this fails the ephemeral axis is silently empty and - # the profile looks merely "absent" rather than broken. - logger.warning("kubelet stats unavailable on %s: %s", ip, e) - continue - for entry in summary.get('pods', []): - name = entry.get('podRef', {}).get('name') - if not name: - continue - used = (entry.get('ephemeral-storage') or {}).get('usedBytes') - if used is not None and config.STORAGE_MODE == 'ephemeral': - prev = _eph_peak.get(name, 0) - if int(used) > prev: - _eph_peak[name] = int(used) - logger.info("peak ephemeral for %s: %.2f GiB", name, used / 1073741824) - # Flushed on growth, and re-measuring cannot recover it: - # disk use is not monotonic -- stellar-core drops its - # download staging once buckets are applied -- so a - # replacement sidecar sees a fraction of the real high-water. - # This sizes the next run's request, and one that comes back - # too small is an eviction. - _flush_peak(name, 'eph', 'peakEphemeralBytes', int(used)) - for c in entry.get('containers', []): - # The worker container only. Sidecars share the pod, so summing - # across containers -- or letting the last one win -- would size - # the range from whichever one kubelet happened to list last. - if c.get('name') != cc.CONTAINER: - continue - # Absent for the first seconds of a container's life, before - # cAdvisor has stats for it. Every later poll carries it, so a - # miss here costs nothing: anon is at its lowest during startup. - mem = c.get('memory') or {} - ws = mem.get('workingSetBytes') - if ws is not None and int(ws) > _ws_peak.get(name, 0): - _ws_peak[name] = int(ws) - _flush_peak(name, 'ws', 'peakWorkingSetBytes', int(ws)) - rss = mem.get('rssBytes') - if rss is None: - continue - # High-water, never last-seen: anon oscillates through the - # download phase, so a later lower sample must not lower it. - if int(rss) <= _anon_peak.get(name, 0): - continue - _anon_peak[name] = int(rss) - # Held in memory until the stream ends, so a restart would reset - # the high-water to whatever the pod is using then, sizing the - # next run too small. Flushing only on PEAK_FLUSH_RATIO growth - # keeps this to a handful of writes over a pod's life. - _flush_peak(name, 'anon', 'peakAnonBytes', int(rss)) - - -def _mark_done(end, attempt): - path = records.done_path(end, attempt) - try: - records.write_atomic(path, '') - except OSError as e: - # Costs a Job that waits out JOB_TTL_SECONDS, never correctness. - logger.warning("could not mark range %s attempt %s done: %s", end, attempt, e) +# --- kubelet ----------------------------------------------------------------- +async def sample_node(session, ip, ranges): + """Peak memory and disk for every worker on one node, straight to .metrics. -async def finalize(session, pod, end, attempt, tx, done_ok, started=None): - """Persist everything this attempt owes, then let its stream go. + Nothing is held in memory: write_metrics keeps the running max on the + volume, so a restart cannot lower a high-water and there is no per-pod peak + to lose. kubelet's rssBytes is cgroup v2 anon, the only limit-independent + memory figure this workload has -- page cache grows to fill whatever + memory.max allows, so memory.peak is always ~= the limit and useless for + sizing. Prometheus can answer neither axis: no pod label on fs usage, and a + 30s scrape is the undersampling that let profiled ranges OOM. - Reached from three places, and deliberately ONE implementation: a clean end - of stream once the pod is terminal, a 404 once the object is gone, and a - terminal pod whose polls keep failing past TERMINAL_POLL_ATTEMPTS. Two - copies is how one path silently stops writing peakAnonBytes while the other - keeps working. The converse matters too: an interrupted read on a pod that - is STILL RUNNING must not come here, or it writes a truncated peak and - leaves the range looking measured when it is not. - """ - # Before discard: on success the archive is about to be deleted. - measured = {} - observed = _pod_secs.pop(pod, None) - since_start = None - began = _pod_start.pop(pod, None) - if observed is None and began: - # The container started at `began` and has just stopped: finalize runs - # within a second or two of the exit. Not exact -- the true end is - # terminated.finishedAt -- but it dates the attempt from the container - # rather than from this poller, whose clock can be near zero against a - # multi-hour run. - try: - since_start = (datetime.utcnow() - datetime.strptime( - began, '%Y-%m-%dT%H:%M:%SZ')).total_seconds() - except ValueError: - since_start = None - if observed is not None: - # The pod's own timestamps, not how long this poller happened to watch. - measured['attemptSeconds'] = round(observed, 1) - measured['attemptSecondsExact'] = True - elif since_start is not None and since_start > 0: - measured['attemptSeconds'] = round(since_start, 1) - # Not exact, but it measures the container's own lifetime rather than - # this process's attention span, which is the distinction the monitor's - # chain gate cares about. Measured against two evicted pods: 370.9s and - # 375.1s against a true ~373s, versus the poller clock's -46%. - measured['attemptSecondsExact'] = False - measured['attemptSecondsFromContainerStart'] = True - elif started is not None: - # Fallback only: the monitor's figure comes from the pod's terminated - # timestamps and is preferred. write_metrics keeps this from lowering a - # duration already on the volume, since a second poller's clock starts - # at the restart. - measured['attemptSeconds'] = round( - asyncio.get_event_loop().time() - started, 1) - measured['attemptSecondsExact'] = False - # RESUME is printed before stellar-core starts and medida once at exit, so a - # recreated poller can miss either forever. The archive was appended before - # finalization; recover only the state this scanner could have missed. - archived = None - need_resume = int(attempt) > 1 and not tx.resume_decided - # Not gated on `recreated`: stellar-core prints the block once at exit, so a - # poller that ran start to finish but ended a beat early has no total. - need_tx = tx.seconds is None - if need_resume or need_tx: - archived = tx_scan.scan_archive(end, attempt, need_tx=need_tx) - if tx.resumed or (archived is not None and archived.resumed): - # Not a peak -- PEAK_FIELDS filters it out of the profile. - # peaks_for_range reads it to decide how far back to aggregate: a - # resumed attempt only measured the tail of its range, so the attempt - # before it still counts. - measured['resumed'] = True - tx_seconds = tx.seconds - if tx_seconds is None and archived is not None: - tx_seconds = archived.seconds - if tx_seconds is not None: - measured['txApplySeconds'] = tx_seconds - _peak_flushed.pop(pod, None) - _peak_flushed.pop(pod + '/eph', None) - _streaming.pop(pod, None) - # One _wake entry per pod, and pods are per range per attempt: 3979 ranges - # plus their retries would accumulate here for the life of the run. - _wake.pop(pod, None) - anon = _anon_peak.pop(pod, None) - if anon is not None: - # Recorded for every attempt, not just the winner: peaks_for_range takes - # the max across attempts, so a partial attempt can only raise the - # figure, which is what makes a resumed range report the download-phase - # peak it actually hit rather than its tail. The monitor drops an attempt - # from the axis it died on, since an OOM-killed peak measures the limit. - measured['peakAnonBytes'] = anon - ws = _ws_peak.pop(pod, None) - if ws is not None: - # Diagnostic only -- working set counts active page cache, which grows - # to fill whatever limit the pod was given, so it must never size - # anything. Kept because the anon/ws gap is what tells you a range is - # cache-heavy rather than large. - measured['peakWorkingSetBytes'] = ws - eph = _eph_peak.pop(pod, None) - if eph is not None: - # Same as anon: max across attempts upstream, and an attempt evicted at - # its ephemeral limit is dropped from this axis there. - measured['peakEphemeralBytes'] = eph - if measured: - write_metrics(end, attempt, measured) - if not config.SAVE_SUCCESS_LOGS and done_ok(pod): - discard(end, attempt) - logger.info("range %s attempt %s: succeeded, archive discarded " - "(saveSuccessLogs=false)", end, attempt) - else: - logger.info("range %s attempt %s: stream complete", end, attempt) - # Last, deliberately. The monitor treats this file as "the collector will - # write nothing further for this attempt" and only then reaps the Job, which - # deletes the pod -- the one place peaks can still be read from -- so it has - # to land after .metrics or it licenses the reap it exists to prevent. - _mark_done(end, attempt) - - -async def _poll_once(session, pod, end, attempt, last_ts, tx): - """One short read of a pod's log. Returns (new_last_ts, gone). - - No follow=true: the request completes and the connection is released, so - concurrency is bounded by _poll_slots rather than by how many pods exist. A - single poll takes ~0.22s, so 2096 pods on a 10s interval need ~46 concurrent - slots against the 2096 permanently-held connections follow=true required. - """ - params = {'container': cc.CONTAINER, 'timestamps': 'true'} - if last_ts: - # Second granularity, so this overlaps on purpose; the per-line - # comparison below removes the overlap exactly. - params['sinceTime'] = last_ts[:19] + 'Z' - url = f"{kube_http.API}/api/v1/namespaces/{config.NAMESPACE}/pods/{pod}/log" - async with _poll_slots: - async with session.get(url, params=params, - headers={'Authorization': f'Bearer {kube_http.token()}'}) as resp: - if resp.status == 404: - return last_ts, True - resp.raise_for_status() - # Chunked, not line-wise: aiohttp raises above 512 KiB on a single - # line and a carriage-return progress meter trivially exceeds that - # -- one 628 MiB download arrived as a single "line". - body = '' - async for chunk in resp.content.iter_chunked(65536): - body += chunk.decode('utf-8', 'replace') - if len(body) > cc.MAX_POLL_CHARS: - break - - return _ingest(body, end, attempt, last_ts, tx), False - - -def _ingest(body, end, attempt, last_ts, tx): - """Append one block of timestamped log text to the archive; new last_ts. - - Split out of _poll_once so the doomed-pod follow stream lands its bytes - through exactly the same path -- dedup, gzip member framing, tx scanning and - resume-point bookkeeping. Two copies is how one route silently stops feeding - TxApplyScanner while the other keeps working. - """ - pending = None - lines = [l for l in re.split(r'[\r\n]', body) if l] - if not lines: - return last_ts - # Compressed into memory first, then appended in ONE write, so the file only - # ever gains whole members. Appending with gzip.open(..., 'at') left the - # archive ending in a member with no end-of-stream marker for most of a large - # poll, and job_monitor reads that same file to recover txApplySeconds -- - # gzip raises EOFError on a truncated member, so one in-flight poll could - # abort a reconcile pass for every range. Costs no extra memory: `body` above - # is already the entire poll uncompressed, and nothing is held between polls. - member = io.BytesIO() - wrote = False - with gzip.GzipFile(fileobj=member, mode='wb') as fh: - for line in lines: - ts, _, rest = line.partition(' ') - if not _TS_RE.match(ts): - # Untimestamped kubelet text. Keep it, but never let it become - # the resume point. - fh.write((line + '\n').encode('utf-8')) - wrote = True - continue - if last_ts and ts <= last_ts: - continue # exact dedup of the resume overlap - fh.write((rest + '\n').encode('utf-8')) - wrote = True - tx.feed(rest) - pending = ts - path = records.log_path(end, attempt) - with open(path, 'ab') as out: - # A poll whose lines were all deduped still touches the archive: its - # existence is what job_monitor's backstop keys on. - if wrote: - out.write(member.getvalue()) - if pending: - write_state(end, attempt, pending) - return pending - return last_ts - - -async def _follow_tail(session, pod, end, attempt, last_ts, tx): - """Hold a follow=true stream on a doomed pod. Returns (last_ts, gone). - - Opened only for pods the cluster has already condemned, so the connection is - held for the couple of minutes before the node goes away, not for the hours - a range runs. Proven on ssc-test: with the stream held, SIGTERM yields `got - signal 15` -> `metric 'ledger.transaction.apply'` -> `Application destroyed` - inside 4ms, all captured, where the same pod polled at 5s recorded `pod gone - before disruption seen`. Bytes are ingested as they arrive, so a node that - disappears mid-read still leaves everything up to that point in the archive. + Straight at the kubelet, not the apiserver's node proxy: that needs + nodes/proxy, which authorizes GET on every kubelet path including + /containerLogs for any namespace on that node. ssl=False because EKS serving + certs are self-signed; this is an in-VPC hop to the node's own address. """ - params = {'container': cc.CONTAINER, 'timestamps': 'true', 'follow': 'true'} - if last_ts: - params['sinceTime'] = last_ts[:19] + 'Z' - url = f"{kube_http.API}/api/v1/namespaces/{config.NAMESPACE}/pods/{pod}/log" - deadline = asyncio.get_event_loop().time() + cc.DOOMED_FOLLOW_SECONDS - buf = '' - if _follow_slots.locked(): - # Every follow budget is spoken for, so this pod polls instead. Better - # than queueing: the pod has ~2 minutes to live, and a follow that opens - # after it dies captures nothing while still holding a slot. - logger.info("range %s: no follow slot free (%d in use), polling instead", - end, cc.MAX_DOOMED_FOLLOWS) - _doomed.pop(pod, None) - return await _poll_once(session, pod, end, attempt, last_ts, tx) - async with _follow_slots: - async with session.get(url, params=params, + url = f"https://{ip}:{kube_http.KUBELET_PORT}/stats/summary" + try: + async with session.get(url, ssl=False, headers={'Authorization': f'Bearer {kube_http.token()}'}) as resp: - if resp.status == 404: - return last_ts, True resp.raise_for_status() - async for chunk in resp.content.iter_chunked(65536): - buf += chunk.decode('utf-8', 'replace') - # Flush on whole lines only: a partial trailing line has no - # usable timestamp and must not become the resume point. - cut = max(buf.rfind('\n'), buf.rfind('\r')) - if cut >= 0: - last_ts = _ingest(buf[:cut + 1], end, attempt, last_ts, tx) - buf = buf[cut + 1:] - if asyncio.get_event_loop().time() > deadline: - logger.info("range %s: doomed follow hit %.0fs, falling back to polling", - end, cc.DOOMED_FOLLOW_SECONDS) - _doomed.pop(pod, None) - break - if buf: - last_ts = _ingest(buf, end, attempt, last_ts, tx) - # One follow per pod. The stream ending means the container exited, so the - # caller falls back to a normal poll for the terminal check and finalize; - # leaving the flag set would re-open a stream on a dead pod every iteration. - _doomed.pop(pod, None) - return last_ts, False - - -async def poll_pod(session, pod, end, attempt, done, done_ok): - """Read one pod's log to completion, by repeated short polls. - - Replaces a follow=true stream, whose cost scaled with parallelism: it held a - connection, a deflate buffer and aiohttp buffers for the pod's entire life, - and at 2096 pods the sidecar sat at 1444 MiB of a 2048 MiB limit and 1.00 of - 2 cpu, extrapolating past both at 4096. The one thing follow did better is - the tail, so on seeing the pod go terminal this polls once more immediately - before finalizing -- without it every spot eviction loses up to one interval - of exactly the log we most want. - """ - last_ts = read_state(end, attempt) - if last_ts is None: - # Empty state = "claimed, nothing durable yet". job_monitor's backstop - # skips any range with a state file, so this prevents both of us writing - # the same log. - write_state(end, attempt, '') - last_ts = '' - started = asyncio.get_event_loop().time() - # Outside the poll loop: the medida block can straddle two polls, and a - # fresh scanner per poll would lose the half it saw. - tx = tx_scan.TxApplyScanner(recreated=bool(last_ts)) - backoff = cc.LOG_POLL_SECONDS - failures = 0 - - first_pass = True - while True: - was_terminal = done(pod) - if first_pass and was_terminal: - # The pod was already terminal before this poller existed, so - # `started` measures how long WE have been watching, not how long - # the container ran -- across two collector restarts, 150 metrics - # files recorded a sub-5s duration beside a >500MiB anon peak. - # Report nothing rather than a fabricated near-zero; the monitor's - # figure from the pod's own timestamps is authoritative anyway. - started = None - first_pass = False - followed = False - try: - if _doomed.get(pod) and not was_terminal and cc.DOOMED_FOLLOW_SECONDS > 0: - followed = True - # Condemned and still running: hold the connection through the - # kill. Returns when the container exits or the notice is - # withdrawn, and the loop re-checks terminal immediately after. - last_ts, gone = await _follow_tail( - session, pod, end, attempt, last_ts, tx) - else: - last_ts, gone = await _poll_once( - session, pod, end, attempt, last_ts, tx) - # Fallback interval for a condemned pod that could not follow. 1s - # sampling alone still closes the ~9s window between the medida - # block and the object being deleted, so a mass reclaim degrades - # rather than loses. - backoff = (cc.DOOMED_POLL_SECONDS if _doomed.get(pod) - else cc.LOG_POLL_SECONDS) - failures = 0 - if gone: - logger.info("pod %s gone before/while polling range %s", pod, end) - await finalize(session, pod, end, attempt, tx, done_ok, started) - return - except asyncio.CancelledError: - raise - except Exception as e: - failures += 1 - logger.info("range %s poll failed (%s); retrying from %s", - end, e, last_ts or 'start') - backoff = min(backoff * 2, 30) - if was_terminal and failures >= cc.TERMINAL_POLL_ATTEMPTS: - # The container has exited and its log is not coming back. A - # follow stream finalized here because it already held the bytes; - # polling has to decide to stop asking, or it spins on a dead pod - # for the rest of the run and never writes its metrics. - logger.warning("range %s attempt %s: %d failed polls after the pod " - "went terminal; finalizing on what was read", - end, attempt, failures) - await finalize(session, pod, end, attempt, tx, done_ok, started) - return - else: - if was_terminal: - # Terminal BEFORE that poll, so the poll saw the container's - # final output. Checking after would race a pod that exits - # mid-poll and drop whatever it wrote on the way out. - await finalize(session, pod, end, attempt, tx, done_ok, started) - return - if followed: - # The follow only returns once the container has exited, so the very - # next read is the one that matters. Sleeping here would hand the - # interval back to the race the follow exists to win. + summary = await resp.json() + except Exception as e: + # Not debug: a silent failure here leaves the profile looking merely + # absent rather than broken. + logger.warning("kubelet stats unavailable on %s: %s", ip, e) + return + for entry in summary.get('pods', []): + key = ranges.get((entry.get('podRef') or {}).get('name')) + if key is None: continue - # Not a blind sleep: a pod going terminal cuts it short. Polling faster - # would not help -- sinceTime has second granularity -- and the delay - # that matters is between the container exiting and the last read, not - # between routine polls. - ev = _wake.setdefault(pod, asyncio.Event()) - try: - await asyncio.wait_for(ev.wait(), timeout=backoff) - except asyncio.TimeoutError: - pass - finally: - # Left set, the Event makes every later wait return instantly, so - # the terminal-poll backoff never sleeps and TERMINAL_POLL_ATTEMPTS - # is spent in one millisecond, giving the pod no time for its final - # log to become readable. A wake is consumed by the poll it triggers. - ev.clear() + found = {} + used = (entry.get('ephemeral-storage') or {}).get('usedBytes') + if used is not None and config.STORAGE_MODE == 'ephemeral': + found['peakEphemeralBytes'] = int(used) + for c in entry.get('containers', []): + # The worker container only: sidecars share the pod, and letting the + # last one win would size the range from whichever kubelet listed. + if c.get('name') != cc.CONTAINER: + continue + mem = c.get('memory') or {} + if mem.get('workingSetBytes') is not None: + found['peakWorkingSetBytes'] = int(mem['workingSetBytes']) + if mem.get('rssBytes') is not None: + found['peakAnonBytes'] = int(mem['rssBytes']) + write_metrics(key[0], key[1], found) +# --- the apiserver ----------------------------------------------------------- + async def list_pods(session): url = f"{kube_http.API}/api/v1/namespaces/{config.NAMESPACE}/pods" params = {'labelSelector': f"{config.LABEL_RUN}={config.RUN_NAME}"} @@ -852,49 +469,19 @@ async def list_pods(session): return (await resp.json()).get('items', []) -def ensure_stream(name, end, attempt, phase): - """Open this pod's poller if it has none. Idempotent; returns whether it did. - - Called by the watch as a pod appears and again if it is condemned, and by - the pod-list loop as a backstop for events dropped across a reconnect. - Opening a stream is time-critical -- a condemned pod is gone a second after - stellar-core exits -- so it must not be reachable only from a poll cycle: on - the 900-worker run the loop's cycle stretched to 925s and five -a2 legs - lived and died with no reader at all. - """ - if name in _tasks or name in _streamed or not _stream_ctx: - return False - if phase not in cc.POLLABLE_PHASES: - # Allowlist, not "skip Pending": a container that has not started - # answers 400 "waiting to start", and Unknown means the node stopped - # reporting. Both are retried on the cycle they become pollable. - return False - ctx = _stream_ctx - _register_stream(name, end, attempt) - _tasks[name] = asyncio.create_task( - poll_pod(ctx['session'], name, end, attempt, - lambda p: ctx['terminal'].get(p, False), - lambda p: ctx['succeeded'].get(p, False))) - logger.info("opened stream for range %s attempt %s (%d active)", - end, attempt, len(_tasks)) - return True - - async def watch_condemnations(session): - """Watch the run's pods and flag condemnations the moment they are written. - - Runs beside the pod-list loop rather than replacing it: the list still owns - discovery, bookkeeping and finalize, and this only ever sets _doomed earlier - than the list would have -- the difference between opening a follow while - stellar-core is still running and opening it on a 404. Cheaper than the - sweep it front-runs, too: one connection served from the apiserver's cache, - sending only deltas. Never fatal -- any failure falls back to the sweep. + """Start follows the moment a condemnation is written, ahead of the cycle. + + Only ever earlier than the cycle would have been -- the difference between + opening a follow while stellar-core still runs and opening it on a 404. One + connection served from the apiserver's cache, sending deltas, where the + cycle re-lists every pod. Never fatal: any failure falls back to the cycle. """ url = f"{kube_http.API}/api/v1/namespaces/{config.NAMESPACE}/pods" rv = None while True: - params = {'labelSelector': f"{config.LABEL_RUN}={config.RUN_NAME}", 'watch': 'true', - 'allowWatchBookmarks': 'true', + params = {'labelSelector': f"{config.LABEL_RUN}={config.RUN_NAME}", + 'watch': 'true', 'allowWatchBookmarks': 'true', 'timeoutSeconds': str(cc.WATCH_TIMEOUT_SECONDS)} if rv: params['resourceVersion'] = rv @@ -902,10 +489,7 @@ async def watch_condemnations(session): async with session.get(url, params=params, headers={'Authorization': f'Bearer {kube_http.token()}'}) as resp: if resp.status == 410: - # Our resourceVersion aged out of the apiserver's history. - # Restarting without one re-syncs; the list sweep covers the - # gap in the meantime. - rv = None + rv = None # aged out of history; re-sync from scratch continue resp.raise_for_status() async for raw in resp.content: @@ -917,28 +501,18 @@ async def watch_condemnations(session): continue obj = ev.get('object') or {} meta = obj.get('metadata') or {} - # Track on every event, bookmarks included -- that is what - # they are for -- so a reconnect resumes instead of re-syncing. + # Tracked on every event, bookmarks included -- that is what + # they are for -- so a reconnect resumes rather than re-syncs. rv = meta.get('resourceVersion') or rv if ev.get('type') == 'ERROR': if obj.get('code') == 410: rv = None break - if ev.get('type') not in ('ADDED', 'MODIFIED'): - continue - labels = meta.get('labels') or {} - end = labels.get(config.LABEL_RANGE) - if end is None: + if ev.get('type') not in ('ADDED', 'MODIFIED') or not meta: continue - name = meta.get('name') - attempt = labels.get(config.LABEL_ATTEMPT, '1') - # Order is not load-bearing: create_task only schedules the - # poller, so _wake has no entry yet either way, and poll_pod - # reads _doomed at the top of its first pass. The wake below - # is for a poller from an earlier event, already asleep. - ensure_stream(name, end, attempt, - (obj.get('status') or {}).get('phase')) - _mark_condemned(obj, name, end, attempt) + key = _identify(obj) + if key and (obj.get('status') or {}).get('phase') == 'Running': + _start_follow(session, meta['name'], key[0], key[1], obj) except asyncio.CancelledError: raise except Exception as exc: diff --git a/src/MissionParallelCatchup/lib/collector/collector_config.py b/src/MissionParallelCatchup/lib/collector/collector_config.py index 28e81e70..ca78243b 100644 --- a/src/MissionParallelCatchup/lib/collector/collector_config.py +++ b/src/MissionParallelCatchup/lib/collector/collector_config.py @@ -14,21 +14,6 @@ # Seconds between sweeps of the run's pod list, which is what discovers new pods # and notices ones that went terminal. POLL_SECONDS = float(os.getenv('COLLECTOR_POLL_SECONDS', 5)) -# Poll cycles a stream gets to finalize itself after its pod leaves the pod list -# before it is cancelled outright. One cycle is usually enough; the margin is for -# a stream still finalizing: writing its .metrics and closing its archive. -VANISHED_GRACE_CYCLES = int(os.getenv('COLLECTOR_VANISHED_GRACE_CYCLES', 3)) -# Peak memory comes from kubelet's /stats/summary (rssBytes, workingSetBytes), -# already fetched for ephemeral storage: ~10s cAdvisor housekeeping against a 30s -# scrape, and no dependence on Prometheus being up, reachable, or still retaining -# the window -- failures the old _promql helper all swallowed into "no peak". -# Peaks are held per pod and flushed on this much growth, so a restart loses at -# most that fraction of a range's high-water. cpu is not sampled: the request is -# fixed at REQ_CPU, so a measured value has nothing to size. -PEAK_FLUSH_RATIO = float(os.getenv('PEAK_FLUSH_RATIO', 1.05)) -# Seconds between polls of one pod's log. Latency here is archive lag, not -# anything a decision waits on; 4096 pods at 10s is ~90 concurrent polls. -LOG_POLL_SECONDS = float(os.getenv('LOG_POLL_SECONDS', 10)) # Concurrent in-flight log polls, across all pods. This is the whole point of # polling: it is independent of how many pods exist, where follow=true needed one # held connection per pod forever. @@ -38,6 +23,13 @@ # blob that never terminates -- a progress meter emitting only carriage returns # would otherwise grow the buffer until the sidecar OOMs. MAX_POLL_CHARS = int(os.getenv('MAX_POLL_CHARS', 8388608)) +# Concurrent kubelet sweeps. One request per node, and a dead node costs the +# connect timeout, so this is small next to the log-read budget. +MAX_CONCURRENT_SAMPLES = int(os.getenv('MAX_CONCURRENT_SAMPLES', 16)) +# Longest a single log read may stall before it is abandoned. A cycle gathers +# every pod, so one wedged read would hold up the whole pass -- where the old +# task-per-pod design only stalled that pod. +READ_TIMEOUT_SECONDS = float(os.getenv('READ_TIMEOUT_SECONDS', 60)) # Longest a follow stream will hang on to a doomed pod. Spot gives 120s; past # roughly double that the notice was withdrawn and the stream would otherwise be # held for the life of the range. 0 disables the follow path entirely, which is @@ -52,13 +44,6 @@ # bounded by Karpenter's disruption budget -- and 256 x the old always-on # design's 0.69 MiB per stream is 177 MiB against a 2048 MiB limit. MAX_DOOMED_FOLLOWS = int(os.getenv('MAX_DOOMED_FOLLOWS', 256)) -# Poll interval for a condemned pod, replacing LOG_POLL_SECONDS while it is -# doomed. preStop delays SIGTERM but leaves the gap between the medida block and -# the pod object being deleted at ~9s, which a blind 10s poll straddles -- it -# did, losing txApply even with a 60s preStop; polling that window every second -# cannot miss it. Costs no held connections, and sinceTime's 1s granularity makes -# anything below 1s a re-read of the same second. -DOOMED_POLL_SECONDS = float(os.getenv('DOOMED_POLL_SECONDS', 1)) # How long each watch connection lives before the apiserver closes it and we # reconnect, bounded so a silently-dead stream self-heals; the reconnect resumes # from the last resourceVersion, so nothing is missed across it. 0 disables the @@ -67,16 +52,7 @@ # Pause before re-opening a watch that failed. Only covers hard errors: a clean # timeout reconnects immediately. WATCH_RETRY_SECONDS = float(os.getenv('WATCH_RETRY_SECONDS', 1)) -# Failed polls tolerated after a pod goes terminal before we stop asking. Its log -# is not coming back and spinning holds a task and a poll slot for the rest of -# the run, but a couple of retries still absorb the transient 500s that arrive in -# bursts at ramp. -TERMINAL_POLL_ATTEMPTS = int(os.getenv('TERMINAL_POLL_ATTEMPTS', 3)) - -# Fields that only ever grow. write_metrics maxes these instead of overwriting, -# so a restarted poller starting its high-water at zero cannot lower one. -PEAK_KEYS = ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes') -# Phases whose log endpoint can actually answer. Pending has no container yet and -# Unknown means the node stopped reporting; the terminal phases are kept because -# that is where a pod's final output lives. -POLLABLE_PHASES = ('Running', 'Succeeded', 'Failed') +# How far back a terminal read reaches past the resume point, so a medida block +# split across two reads is whole in one of them. The archive dedups the +# overlap, so this only widens what the scan sees. +TERMINAL_REREAD_SECONDS = int(os.getenv('TERMINAL_REREAD_SECONDS', 30)) diff --git a/src/MissionParallelCatchup/lib/collector/medida.py b/src/MissionParallelCatchup/lib/collector/medida.py deleted file mode 100644 index 9e7d178d..00000000 --- a/src/MissionParallelCatchup/lib/collector/medida.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Reading medida statistics out of a stellar-core log. - -Both readers live here on purpose. The collector scans the live stream and the -monitor re-reads the finished archive, and the two must agree on how far a `sum` -may sit from its block header -- otherwise the recovery path inherits exactly the -blind spot it exists to cover. -""" -import re - -# `sum = ms`, where the number may be in exponent form. The old -# [0-9.]+ pattern matched "1.30722" then demanded "ms" and hit "e+06ms" -# instead, so tx_apply was silently missing for 25% of ranges -- 91-99% of -# everything above ledger 35M, exactly the expensive end. 698 completed ranges -# lost the metric that way in a single run. -SUM_RE = re.compile(r"sum\s*=\s*([0-9.]+(?:[eE][+-]?[0-9]+)?)ms") - -# A medida statistic: ` = `. Anything else between the block header -# and its sum is another thread's output interleaved into the same log, and must -# not be charged against the search window. -METRIC_LINE = re.compile(r"[\w%.\-]+\s*=\s*[-+0-9.]") - -# A different metric's header. Reaching one means the block we armed on has been -# passed, so a later `sum =` belongs to some other timer. -ANY_METRIC = re.compile(r"metric '") - -# Statistics lines the sum may sit behind, and the hard line budget regardless. -# Measured 2026-08-04: a /info response pushed `sum` 91 lines down, and charging -# those lines made both readers give up while the value sat in the archive. -WINDOW = 15 -HARD_WINDOW = 400 diff --git a/src/MissionParallelCatchup/lib/collector/state_files.py b/src/MissionParallelCatchup/lib/collector/state_files.py new file mode 100644 index 00000000..fe09aa63 --- /dev/null +++ b/src/MissionParallelCatchup/lib/collector/state_files.py @@ -0,0 +1,33 @@ +"""Resume points on the volume, read back at collector startup.""" +import os +import re + +import config + +# A kubelet log timestamp, and the only thing that may become a resume point. +TS_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?$") + + +def hydrate_states(): + """Every attempt with a resume point on the volume: (end, attempt) -> ts. + + Read once at collector startup so a restart continues each attempt where it + stopped rather than re-reading whole logs. A state file with no timestamp + means "claimed, nothing durable yet" and hydrates as an empty resume point. + """ + out = {} + try: + names = os.listdir(config.LOG_DIR) + except OSError: + return out + for name in names: + m = re.match(r'^range-(\d+)-a(\d+)\.state$', name) + if not m: + continue + try: + with open(os.path.join(config.LOG_DIR, name)) as fh: + ts = fh.read().strip() + out[(m.group(1), m.group(2))] = ts if TS_RE.match(ts) else '' + except OSError: + continue + return out diff --git a/src/MissionParallelCatchup/lib/collector/tx_scan.py b/src/MissionParallelCatchup/lib/collector/tx_scan.py index a714e238..b74241d1 100644 --- a/src/MissionParallelCatchup/lib/collector/tx_scan.py +++ b/src/MissionParallelCatchup/lib/collector/tx_scan.py @@ -1,21 +1,39 @@ """Reading the tx-apply total out of a worker's log. -The collector scans the live stream as it goes past and re-reads its own archive -at finalization when the stream missed the block -- stellar-core prints it once, -just before exit, so a stream that ends a beat early has no total. Scanning here -rather than in job_monitor is what makes the metric independent of pod lifetime: -by the time the Job is seen to succeed the node may be reaped, and with -saveSuccessLogs=false the archive is gone too. +One reader, so the window constants live here rather than in a module shared +with a second one -- the monitor's archive re-read is gone; it takes the value +from the collector's .metrics. + +stellar-core prints the block once, just before exit, so the collector scans it +off the read that finds the pod terminal. Scanning there rather than in +job_monitor is what makes the metric independent of pod lifetime: by the time +the Job is seen to succeed the node may be reaped, and with saveSuccessLogs=false +the archive is gone too. """ -import gzip -import logging -import zlib +import re -import medida -import records +# `sum = ms`, where the number may be in exponent form. The old +# [0-9.]+ pattern matched "1.30722" then demanded "ms" and hit "e+06ms" +# instead, so tx_apply was silently missing for 25% of ranges -- 91-99% of +# everything above ledger 35M, exactly the expensive end. 698 completed ranges +# lost the metric that way in a single run. +SUM_RE = re.compile(r"sum\s*=\s*([0-9.]+(?:[eE][+-]?[0-9]+)?)ms") -logger = logging.getLogger('log_collector') +# A medida statistic: ` = `. Anything else between the block header +# and its sum is another thread's output interleaved into the same log, and must +# not be charged against the search window. +METRIC_LINE = re.compile(r"[\w%.\-]+\s*=\s*[-+0-9.]") + +# A different metric's header. Reaching one means the block we armed on has been +# passed, so a later `sum =` belongs to some other timer. +ANY_METRIC = re.compile(r"metric '") + +# Statistics lines the sum may sit behind, and the hard line budget regardless. +# Measured 2026-08-04: a /info response pushed `sum` 91 lines down, and charging +# those lines made both readers give up while the value sat in the archive. +WINDOW = 15 +HARD_WINDOW = 400 TX_METRIC = "metric 'ledger.transaction.apply'" @@ -32,49 +50,39 @@ class TxApplyScanner: # divergence would hand the recovery path the same blind spot it exists to # cover. A /info liveness response interleaved into the block once put `sum` # 91 lines below the header, 76 lines past where both readers gave up. - WINDOW = medida.WINDOW - HARD_WINDOW = medida.HARD_WINDOW - # Printed by RESUME_SCRIPT before stellar-core starts. Its counterpart, - # "RESUME DECLINED", means new-db ran and this attempt did the whole range, - # so the colon is load-bearing -- it is what separates the two. + # Printed by RESUME_SCRIPT before stellar-core starts. The colon and space + # are load-bearing: its counterpart "RESUME DECLINED:" means new-db ran and + # this attempt did the whole range, and must not read as a resume. RESUME_MARK = 'RESUME: ' - RESUME_DECLINED_MARK = 'RESUME DECLINED:' - def __init__(self, recreated=False): + def __init__(self): self.seconds = None self.resumed = False - self.resume_decided = False - # A new poller starting from durable .state missed every earlier line. - # Finalization must recover scanner-only facts from the archive. - self.recreated = recreated self._left = 0 self._span = 0 def feed(self, line): if self.RESUME_MARK in line: self.resumed = True - self.resume_decided = True - elif self.RESUME_DECLINED_MARK in line: - self.resume_decided = True if TX_METRIC in line: - self._left = self.WINDOW - self._span = self.HARD_WINDOW + self._left = WINDOW + self._span = HARD_WINDOW return if self._left <= 0: return - m = medida.SUM_RE.search(line) + m = SUM_RE.search(line) if m: self.seconds = float(m.group(1)) / 1000.0 self._left = 0 return self._span -= 1 - if self._span <= 0 or medida.ANY_METRIC.search(line): + if self._span <= 0 or ANY_METRIC.search(line): # Ran out of rope, or another timer's block started -- either way our # sum is not coming. self._left = 0 return - if medida.METRIC_LINE.search(line): + if METRIC_LINE.search(line): # Only medida's own statistics count. Interleaved output from another # thread is noise between us and the sum, not evidence we have passed # it. @@ -82,26 +90,5 @@ def feed(self, line): -def scan_archive(end, attempt, need_tx=False): - """Recover scanner state from complete gzip members already on disk.""" - path = records.log_path(end, attempt) - scanner = TxApplyScanner() - try: - with gzip.open(path, 'rt', errors='replace') as fh: - for line in fh: - scanner.feed(line) - # The resume decision is at process startup. Avoid decompressing - # a multi-gigabyte worker log when that is all the caller needs. - if scanner.resume_decided and not need_tx: - break - except FileNotFoundError: - return scanner - except (EOFError, gzip.BadGzipFile, zlib.error) as e: - # Keep facts found in complete prefix members. A torn final member cannot - # invalidate an earlier RESUME line or complete medida block. - logger.warning("could only partially recover scanner state from %s: %s", path, e) - except OSError as e: - logger.warning("could not open scanner archive %s: %s", path, e) - return scanner diff --git a/src/MissionParallelCatchup/lib/monitor/attempt_files.py b/src/MissionParallelCatchup/lib/monitor/attempt_files.py new file mode 100644 index 00000000..f293796d --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/attempt_files.py @@ -0,0 +1,77 @@ +"""What an attempt left behind, read back for retry accounting. + +The collector writes these files while the pod still exists; the monitor reads +them to decide whether an attempt may be retried and how far its resources must +climb. The counters are per CAUSE, not per attempt -- escalation must climb once +per OOM, not once per retry. +""" +import json +import os + +import config +import records + + +# Per RANGE, not per attempt: wallSeconds spans the range's whole life, so the +# only start that matters is the first one. Later attempts are the mess in +# between and are deliberately not recorded. +def started_path(end): + return os.path.join(config.LOG_DIR, f"range-{end}.started") + + + +def read_outcome(end, attempt): + try: + with open(records.outcome_path(end, attempt)) as fh: + return json.load(fh) + except (OSError, ValueError): + return None + + + +def _oom_count(end, attempt): + """How many earlier attempts at this range were OOM-killed. + + Escalation must climb once per OOM, not once per attempt. On spot most + retries are evictions -- measured on ssc-test 2026-07-30, 288 disruption + retries against 7 OOM retries -- and a range disrupted three times then + OOMing once would otherwise jump to base * 1.5^4, a 5x request for a single + OOM. That inflation is fleet-wide and it is what exhausts the vCPU quota. + """ + return sum(1 for n in range(1, int(attempt) + 1) + if _verdict_of(end, n) == 'oom') + + + +def verdict_path(end, attempt): + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.verdict") + + + +def _verdict_of(end, attempt): + try: + with open(verdict_path(end, attempt)) as fh: + verdict = fh.read().strip() + except OSError: + # Pre-fix runs, or an attempt whose verdict write lost the volume: + # the pod-derived classification is the next best thing. + outcome = (read_outcome(end, attempt) or {}).get('outcome') + return outcome if outcome in config.ATTEMPT_OUTCOMES else None + return verdict if verdict in config.ATTEMPT_OUTCOMES else None + + + +def _cause_count(end, attempt, causes): + """How many of attempts 1..N at this range failed for one of `causes`. + + Budgets are per cause, not per attempt. One shared attempt index meant + cluster churn -- which has its own deliberately large budget -- drained the + small budgets belonging to the causes that say something about the range: a + range evicted MAX_ATTEMPTS times had an effective OOM and disk budget of + zero, was condemned on its first real OOM without ever being escalated, and + took the whole mission with it. + """ + return sum(1 for n in range(1, int(attempt) + 1) + if _verdict_of(end, n) in causes) + + diff --git a/src/MissionParallelCatchup/lib/monitor/attempts.py b/src/MissionParallelCatchup/lib/monitor/attempts.py index 0a966696..b0c12ddd 100644 --- a/src/MissionParallelCatchup/lib/monitor/attempts.py +++ b/src/MissionParallelCatchup/lib/monitor/attempts.py @@ -10,10 +10,9 @@ import gzip import json import logging -import os import zlib -import config +import attempt_files import records logger = logging.getLogger() @@ -64,7 +63,7 @@ def peaks_for_range(end, attempt=1): def _hit_a_ceiling(end, attempt): """Was this attempt killed at one of its own resource limits?""" - return (records.read_outcome(end, attempt) or {}).get('outcome') in ('oom', 'ephemeral') + return (attempt_files.read_outcome(end, attempt) or {}).get('outcome') in ('oom', 'ephemeral') def _peak_attempts(end, attempt): @@ -269,7 +268,7 @@ def _attempt_seconds(end, attempt): # .outcome carries the pod's own terminated timestamps, and is absent # whenever the pod was reaped before classification -- every spot eviction -- # so fall back to the collector's estimate. - leg = (records.read_outcome(end, attempt) or {}).get('attemptSeconds') + leg = (attempt_files.read_outcome(end, attempt) or {}).get('attemptSeconds') if leg is not None: return leg try: diff --git a/src/MissionParallelCatchup/lib/monitor/kube.py b/src/MissionParallelCatchup/lib/monitor/kube.py index 259f344c..9c32b85a 100644 --- a/src/MissionParallelCatchup/lib/monitor/kube.py +++ b/src/MissionParallelCatchup/lib/monitor/kube.py @@ -13,7 +13,6 @@ from kubernetes import client, config as kube_config -import config import monitor_config as mc # The env var is exactly what load_incluster_config() itself keys on, so in a pod diff --git a/src/MissionParallelCatchup/lib/monitor/ranges.py b/src/MissionParallelCatchup/lib/monitor/ranges.py index cf186f03..5da8360b 100644 --- a/src/MissionParallelCatchup/lib/monitor/ranges.py +++ b/src/MissionParallelCatchup/lib/monitor/ranges.py @@ -5,7 +5,6 @@ cluster or the volume. """ -import config import monitor_config as mc import profiles diff --git a/src/MissionParallelCatchup/lib/monitor/sizing.py b/src/MissionParallelCatchup/lib/monitor/sizing.py index 319fab87..801091b8 100644 --- a/src/MissionParallelCatchup/lib/monitor/sizing.py +++ b/src/MissionParallelCatchup/lib/monitor/sizing.py @@ -10,10 +10,9 @@ import logging import math -import config import monitor_config as mc import profiles -import records +import attempt_files import units logger = logging.getLogger() @@ -209,7 +208,7 @@ def pool_for(end, attempt=1, rungs=None): if rungs is None: # Attempts before this one, since this attempt has not run yet. Anything # on disk for it is from a previous incarnation of the same attempt. - rungs = records._oom_count(end, attempt - 1) if attempt and attempt > 1 else 0 + rungs = attempt_files._oom_count(end, attempt - 1) if attempt and attempt > 1 else 0 if not mc.PROFILE: return _promote(mc.POOL_NO_PROFILE, rungs) prof = profiles.profile_for(end) if end is not None else None diff --git a/src/MissionParallelCatchup/lib/monitor/worker_liveness.py b/src/MissionParallelCatchup/lib/monitor/worker_liveness.py index df7f05f0..b3119177 100644 --- a/src/MissionParallelCatchup/lib/monitor/worker_liveness.py +++ b/src/MissionParallelCatchup/lib/monitor/worker_liveness.py @@ -13,7 +13,6 @@ import aiohttp -import config import monitor_config as mc logger = logging.getLogger() diff --git a/src/MissionParallelCatchup/lib/records.py b/src/MissionParallelCatchup/lib/records.py index 500696bb..3520f42d 100644 --- a/src/MissionParallelCatchup/lib/records.py +++ b/src/MissionParallelCatchup/lib/records.py @@ -1,11 +1,11 @@ -"""Per-attempt facts on the shared volume, and the paths they live at. +"""The filenames both processes agree on, and the write that keeps them whole. -The collector sidecar writes these files and the monitor reads them, so nothing -authoritative is held in memory: a restarted monitor rebuilds every decision from -these plus the live Job list. The counters here are per CAUSE, not per attempt -- -escalation must climb once per OOM, not once per retry. +This is the entire cross-process contract. The collector writes these files +while a pod still exists and the monitor reads them back, so a disagreement +about a name is a measurement silently lost. Everything either side does with +the contents lives on its own side: attempt_files for the monitor, state_files +for the collector. """ -import json import os import config @@ -29,64 +29,6 @@ def outcome_path(end, attempt): return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.outcome") -# Per RANGE, not per attempt: wallSeconds spans the range's whole life, so the -# only start that matters is the first one. Later attempts are the mess in -# between and are deliberately not recorded. -def started_path(end): - return os.path.join(config.LOG_DIR, f"range-{end}.started") - - -def read_outcome(end, attempt): - try: - with open(outcome_path(end, attempt)) as fh: - return json.load(fh) - except (OSError, ValueError): - return None - - -def _oom_count(end, attempt): - """How many earlier attempts at this range were OOM-killed. - - Escalation must climb once per OOM, not once per attempt. On spot most - retries are evictions -- measured on ssc-test 2026-07-30, 288 disruption - retries against 7 OOM retries -- and a range disrupted three times then - OOMing once would otherwise jump to base * 1.5^4, a 5x request for a single - OOM. That inflation is fleet-wide and it is what exhausts the vCPU quota. - """ - return sum(1 for n in range(1, int(attempt) + 1) - if _verdict_of(end, n) == 'oom') - - -def verdict_path(end, attempt): - return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.verdict") - - -def _verdict_of(end, attempt): - try: - with open(verdict_path(end, attempt)) as fh: - verdict = fh.read().strip() - except OSError: - # Pre-fix runs, or an attempt whose verdict write lost the volume: - # the pod-derived classification is the next best thing. - outcome = (read_outcome(end, attempt) or {}).get('outcome') - return outcome if outcome in config.ATTEMPT_OUTCOMES else None - return verdict if verdict in config.ATTEMPT_OUTCOMES else None - - -def _cause_count(end, attempt, causes): - """How many of attempts 1..N at this range failed for one of `causes`. - - Budgets are per cause, not per attempt. One shared attempt index meant - cluster churn -- which has its own deliberately large budget -- drained the - small budgets belonging to the causes that say something about the range: a - range evicted MAX_ATTEMPTS times had an effective OOM and disk budget of - zero, was condemned on its first real OOM without ever being escalated, and - took the whole mission with it. - """ - return sum(1 for n in range(1, int(attempt) + 1) - if _verdict_of(end, n) in causes) - - def metrics_path(end, attempt): return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.metrics") @@ -110,3 +52,5 @@ def write_atomic(path, body, opener=None): with (opener or open)(tmp, 'wt') as fh: fh.write(body) os.replace(tmp, path) + + diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 8d59f5a6..14f1d209 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -434,14 +434,6 @@ spec: value: /logs - name: COLLECTOR_POLL_SECONDS value: {{ .Values.monitor.collectorPollSeconds | quote }} - - name: TERMINAL_POLL_ATTEMPTS - value: {{ .Values.monitor.terminalPollAttempts | quote }} - - name: LOG_POLL_SECONDS - value: {{ .Values.monitor.logPollSeconds | quote }} - # Ceiling on how long a follow stream is held for a condemned pod, - # so a withdrawn drain notice cannot pin one open for a whole range. - - name: DOOMED_POLL_SECONDS - value: {{ .Values.monitor.doomedPollSeconds | quote }} - name: WATCH_TIMEOUT_SECONDS value: {{ .Values.monitor.watchTimeoutSeconds | quote }} - name: WATCH_RETRY_SECONDS @@ -454,10 +446,12 @@ spec: value: {{ .Values.monitor.maxConcurrentPolls | quote }} - name: MAX_POLL_CHARS value: {{ .Values.monitor.maxPollChars | int64 | quote }} - - name: PEAK_FLUSH_RATIO - value: {{ .Values.monitor.peakFlushRatio | quote }} - - name: COLLECTOR_VANISHED_GRACE_CYCLES - value: {{ .Values.monitor.collectorVanishedGraceCycles | quote }} + - name: MAX_CONCURRENT_SAMPLES + value: {{ .Values.monitor.maxConcurrentSamples | quote }} + - name: READ_TIMEOUT_SECONDS + value: {{ .Values.monitor.readTimeoutSeconds | quote }} + - name: TERMINAL_REREAD_SECONDS + value: {{ .Values.monitor.terminalRereadSeconds | quote }} # Failures are always kept; successes are the bulk of the volume. - name: SAVE_SUCCESS_LOGS value: {{ .Values.monitor.saveSuccessLogs | quote }} diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 29fbe503..eaafdbc1 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -195,10 +195,6 @@ monitor: logStorageSize: "100Gi" saveSuccessLogs: true collectorPollSeconds: 5 - # Log collection polls rather than holding a follow=true stream per pod, so - # concurrency is independent of worker.replicas. At 4096 pods and a 10s - # interval this is ~90 in-flight polls. - logPollSeconds: 10 # Spot gives ~120s of notice; past roughly double that the drain was # cancelled and the stream should go back to interval polling. doomedFollowSeconds: 300 @@ -206,10 +202,6 @@ monitor: # poll slot and blackout the rest of the run. Condemned pods beyond this fall # back to polling, which is the pre-existing behaviour. maxDoomedFollows: 256 - # Poll interval for a pod the cluster has condemned. The cheap half of the - # disruption fix: preStop delays SIGTERM but leaves ~9s between the medida - # block and the pod object vanishing, which a 10s poll straddles. - doomedPollSeconds: 1 # Lifetime of each pod watch before the apiserver closes it and the collector # reconnects from its last resourceVersion. The watch is what makes a # condemnation visible immediately rather than on the next collectorPollSeconds @@ -220,19 +212,22 @@ monitor: watchRetrySeconds: 1 maxConcurrentPolls: 96 maxPollChars: 8388608 - # Failed polls tolerated after a pod goes terminal before the collector stops - # asking. Its log is not coming back and the task holds a poll slot. - terminalPollAttempts: 3 - # Growth factor before an in-flight peak is flushed to its .metrics file, so a - # collector restart cannot silently reset a range's high-water to zero. - peakFlushRatio: 1.05 + # Concurrent kubelet sweeps. One request per node, and a dead one costs the + # connect timeout, so this stays small next to the log-read budget. + maxConcurrentSamples: 16 + # Longest one log read may stall before it is abandoned. A cycle reads every + # pod together, so a wedged read would hold up the whole pass. + readTimeoutSeconds: 60 + # How far back the terminal read reaches past the resume point, so a medida + # block split across two reads is whole in one. The archive dedups the + # overlap, so this only widens what the scan sees. + terminalRereadSeconds: 30 # Poll cycles a stream gets to finalize after its pod leaves the pod list, # before it is cancelled and its connection slot reclaimed. # Where the monitor runs, from --job-monitor-node-labels and # --job-monitor-tolerate-taints. Empty = anywhere untainted. Pinning it to a # worker tier reserves that node: it holds the logs PVC for the whole run. nodeSelector: {} - collectorVanishedGraceCycles: 3 collectorResources: requests: { cpu: "200m", memory: "512Mi" } limits: { cpu: "2", memory: "2Gi" } From c0f88c4f66763162a36a5f096289c1169be1cdb4 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 17 Aug 2026 12:30:20 -0400 Subject: [PATCH 113/117] Replace the parallel-catchup job monitor with the v2 rewrite The monitor is rebuilt from its requirements rather than refactored: one reconcile pass that looks, decides, acts and persists, on the async kubernetes.aio client. 3344 lines out, 1991 in. - apps/job_monitor.py is the pass spine; lib/monitor/ holds cluster, dispatch, liveness, metrics, policy, record, server, sizing and verdict - lib/config.py now serves both processes, so lib/monitor/monitor_config.py and the old lib/config.py collapse into it - counters and cause counts live in the progress record, so a pass costs O(1) instead of walking every attempt - only disk IO goes to a thread; the rest of the pass runs on the loop - overlapLedgers is sent from the driver instead of guessed by the monitor: the range list is ledgersPerJob + overlap, and a guess generates a different list than the profile was measured against - collector records a duration when it opens a doomed-pod follow. An evicted pod is usually deleted before it is ever seen terminal, so _finish had nothing to read it from and the range lost its compute total - driver fails at first sight of a condemned range again. Draining needs the monitor to stop dispatching, which it has no way to be told to do, so it meant running the whole remaining queue after the outcome was already known - chart pins stellajuna/ssc-jm:latest Validated on ssc-test: 1875 ranges x 500 spot workers with PVC storage ran to completion, 13 spot evictions all resumed from LCL, and every range reported a compute total -- the first run whose profile has no holes. Co-Authored-By: Claude Opus 5 --- src/App/Program.fs | 8 + src/FSLibrary.Tests/Tests.fs | 1 + .../MissionHistoryPubnetParallelCatchupV2.fs | 59 +- src/FSLibrary/StellarMissionContext.fs | 1 + .../Dockerfile.jobmonitor | 6 +- .../apps/job_monitor.py | 1831 ++++------------- .../apps/log_collector.py | 7 +- src/MissionParallelCatchup/lib/config.py | 248 ++- .../lib/monitor/attempt_files.py | 77 - .../lib/monitor/attempts.py | 352 ---- .../lib/monitor/cluster.py | 178 ++ .../lib/monitor/dispatch.py | 240 +++ .../lib/monitor/http_server.py | 175 -- .../lib/monitor/kube.py | 34 - .../lib/monitor/liveness.py | 89 + .../lib/monitor/metrics.py | 156 +- .../lib/monitor/monitor_config.py | 539 ----- .../lib/monitor/policy.py | 45 + .../lib/monitor/profiles.py | 37 - .../lib/monitor/ranges.py | 83 - .../lib/monitor/record.py | 199 ++ .../lib/monitor/server.py | 101 + .../lib/monitor/sizing.py | 530 +++-- .../lib/monitor/units.py | 27 - .../lib/monitor/verdict.py | 150 ++ .../lib/monitor/worker_liveness.py | 103 - src/MissionParallelCatchup/lib/records.py | 4 +- .../templates/job_monitor.yaml | 35 +- .../parallel_catchup_helm/values.yaml | 20 +- 29 files changed, 1991 insertions(+), 3344 deletions(-) delete mode 100644 src/MissionParallelCatchup/lib/monitor/attempt_files.py delete mode 100644 src/MissionParallelCatchup/lib/monitor/attempts.py create mode 100644 src/MissionParallelCatchup/lib/monitor/cluster.py create mode 100644 src/MissionParallelCatchup/lib/monitor/dispatch.py delete mode 100644 src/MissionParallelCatchup/lib/monitor/http_server.py delete mode 100644 src/MissionParallelCatchup/lib/monitor/kube.py create mode 100644 src/MissionParallelCatchup/lib/monitor/liveness.py delete mode 100644 src/MissionParallelCatchup/lib/monitor/monitor_config.py create mode 100644 src/MissionParallelCatchup/lib/monitor/policy.py delete mode 100644 src/MissionParallelCatchup/lib/monitor/profiles.py delete mode 100644 src/MissionParallelCatchup/lib/monitor/ranges.py create mode 100644 src/MissionParallelCatchup/lib/monitor/record.py create mode 100644 src/MissionParallelCatchup/lib/monitor/server.py delete mode 100644 src/MissionParallelCatchup/lib/monitor/units.py create mode 100644 src/MissionParallelCatchup/lib/monitor/verdict.py delete mode 100644 src/MissionParallelCatchup/lib/monitor/worker_liveness.py diff --git a/src/App/Program.fs b/src/App/Program.fs index ded57c98..e2b5e68e 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -114,6 +114,7 @@ type MissionOptions pubnetParallelCatchupStartingLedger: int, pubnetParallelCatchupEndLedger: int option, pubnetParallelCatchupLedgersPerJob: int, + pubnetParallelCatchupOverlapLedgers: int, pubnetParallelCatchupNumWorkers: int, pubnetParallelCatchupStorageMode: string, pubnetParallelCatchupProfile: string, @@ -527,6 +528,12 @@ type MissionOptions Default = 16000)>] member self.PubnetParallelCatchupLedgersPerJob = pubnetParallelCatchupLedgersPerJob + [] + member self.PubnetParallelCatchupOverlapLedgers = pubnetParallelCatchupOverlapLedgers + [() - let seenFailures = System.Collections.Generic.HashSet() let mutable lastLogFetch = DateTime.UtcNow while not allJobsFinished do @@ -1016,12 +1012,29 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = let jobsFailed = status.["jobs_failed"] :?> JArray let jobsInProgress = status.Value("queue_in_progress_count") - for job in jobsFailed do - let text = job.ToString() + // At first sight, not once the run drains: the monitor has no + // way to be told to stop dispatching, so draining means running + // the whole remaining queue after the outcome is already known. + if jobsFailed.Count <> 0 then + LogError "%d job(s) failed:" jobsFailed.Count - if seenFailures.Add(text) then - failedJobs.Add(text) - LogError "RANGE FAILED: %s -- run continues, mission will fail once it drains" text + for job in jobsFailed do + let text = job.ToString() + let ident = text.Split('|') + LogInfo "%s, logs >>> " text + + // A condemned range need not name a pod: an attempt that + // never scheduled has none, and its log is on the monitor + // volume either way, so a missing pod must not mask the + // failure below. + if ident.Length > 1 then + try + dumpLogs (context, ident.[1]) + with ex -> LogInfo "could not read pod log (%s); see collected logs" (ex.Message) + + LogInfo "<<<" + + failwith "Catch up failed, check logs for more info" if remainSize = 0 && jobsInProgress = 0 then LogInfo "All queues empty. Mission complete." @@ -1047,24 +1060,4 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = cleanup false context raise ex - if failedJobs.Count <> 0 then - LogInfo "%d job(s) failed:" failedJobs.Count - - for job in failedJobs do - let ident = job.Split('|') - LogInfo "%s, logs >>> " job - - // The pod is very likely reaped by now -- draining first means the - // wait is the length of the run. Its log is on the monitor volume - // either way, so a missing pod must not mask the failure below. - if ident.Length > 1 then - try - dumpLogs (context, ident.[1]) - with ex -> LogInfo "could not read pod log (%s); see collected logs" (ex.Message) - - LogInfo "<<<" - - cleanup false context - failwith "Catch up failed, check logs for more info" - cleanup false context diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 476c176e..787a1e43 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -118,6 +118,7 @@ type MissionContext = pubnetParallelCatchupStartingLedger: int pubnetParallelCatchupEndLedger: int option pubnetParallelCatchupLedgersPerJob: int + pubnetParallelCatchupOverlapLedgers: int pubnetParallelCatchupNumWorkers: int pubnetParallelCatchupStorageMode: string pubnetParallelCatchupProfile: string diff --git a/src/MissionParallelCatchup/Dockerfile.jobmonitor b/src/MissionParallelCatchup/Dockerfile.jobmonitor index 03a9a2ad..830b1548 100644 --- a/src/MissionParallelCatchup/Dockerfile.jobmonitor +++ b/src/MissionParallelCatchup/Dockerfile.jobmonitor @@ -19,13 +19,11 @@ WORKDIR /app # classification at all. RUN pip install --no-cache-dir \ 'kubernetes~=36.0' \ - 'aiohttp~=3.9' \ + 'aiohttp~=3.14' \ 'prometheus-client~=0.19' # apps/ and lib/ flatten into one directory here, and the modules import each -# other by bare name. /app is a single flat directory in the dev path too: that -# ConfigMap is built with --from-file, whose keys are basenames and cannot -# contain '/'. The split is for reading the repo, not for the runtime. +# other by bare name. The split is for reading the repo, not for the runtime. COPY ./apps/job_monitor.py /app # Same image, second entrypoint: runs as a sidecar streaming worker logs. COPY ./apps/log_collector.py /app diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py index fcb81885..8c343600 100644 --- a/src/MissionParallelCatchup/apps/job_monitor.py +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -1,1532 +1,449 @@ -"""Parallel catchup job monitor. +"""The single writer for one MissionParallelCatchup run. +Turns a ledger range into completed work using Kubernetes Jobs, and reports what +happened. One pass is: snapshot the cluster, derive each range's state, observe +the volume, act on the apiserver, commit the record. -Singleton state manager for a MissionParallelCatchup run. - -State is held on the shared volume and mirrored to a ConfigMap for the mission driver to read. - -The first reconcile thread is responsible for generating the full ledger range list -and dispatching Kubernetes Jobs per ledger range. - -It marks Jobs as completed, retries if failed due to OOM, spot eviction, -or other transient causes, and records the outcome of each attempt. -Genuine catchup failures are not retried and marked as failed. -The mission driver reads the ConfigMap to determine the overall progress of the catchup mission. -The mission driver is responsible for tearing down the mission when detecting a job failure or the completion of all ledger ranges. - -On each job completion, metrics are updated to reflect the duration and txApply progress, as well as the current state of the mission, -including the number of remaining jobs, succeeded jobs, failed jobs, and in-progress jobs. - -The second worker_liveness thread probes the /info endpoint of each running worker pod to determine its liveness. - -Finally the monitor exposes a simple HTTP server for health checks, status, and Prometheus metrics. - +EXACTLY ONE of these may run. No leader election exists anywhere, so a rolling +update that briefly overlaps two is a second writer of every Job, PVC and the +progress record -- replicas: 1, strategy: Recreate. """ - +import asyncio import collections -import gzip -import json -import os -import re -import threading +import logging +import signal +import sys import time -from datetime import datetime -from kubernetes import client -from kubernetes.client.rest import ApiException - -import attempts +import cluster import config -import monitor_config as mc -import http_server -import kube +import dispatch +import liveness import metrics -import profiles -import ranges -import attempt_files -import records +import policy +import record +import server import sizing -import worker_liveness -from logger import build_logger - +import verdict -logger = build_logger('job_monitor') -if not kube.IN_CLUSTER: - logger.warning("KUBERNETES_SERVICE_HOST is unset: no in-cluster config loaded. " - "Every API call will fail until kube.core_v1/kube.batch_v1 are replaced.") +logging.basicConfig(level=logging.INFO, stream=sys.stdout, + format='%(asctime)s %(levelname)s %(message)s') +logger = logging.getLogger('job_monitor') -def main(): - # The driver POSTs the profile to /start. Kept on the volume so a restarted - # monitor resumes a run already under way instead of waiting for a /start - # that was delivered to its predecessor. - mc.RUN_PATH = os.path.join(config.LOG_DIR, 'run.json') - if os.path.exists(mc.RUN_PATH): - # Same path as a /start, so the range and the profile are both restored - # and validated exactly as they were. - with open(mc.RUN_PATH) as fh: - start_run(json.load(fh)) +async def main(): + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, stop.set) - http_server.status_source = lambda: (status, status_lock) - http_server.on_start = start_run + state = State() + state.resume() + async with cluster.session(): + async with asyncio.TaskGroup() as tg: + tg.create_task(server.serve(state, stop)) + tg.create_task(reconcile_loop(state, stop)) + logger.info("stopped") + return 1 if state.progress['condemned'] else 0 - # This is the reconcile loop -- - # dispatch, progress record, metrics, status. - reconcile_thread = threading.Thread(target=reconcile_loop, daemon=True) - reconcile_thread.start() - http_server.serve() - - -def start_run(doc): - """Install the run the driver POSTed: the ledger range, and the profile. - - Both are per-run input, so neither is env-derived any more -- the chart - installs a generic monitor and this defines what it runs. Written to the - volume before the gate opens, so the first Job dispatched is already sized - by the profile and a restart resumes the same run. - """ - for key, name in (('order', 'RANGE_ORDER'), - ('startingLedger', 'STARTING_LEDGER'), - ('latestLedgerNum', 'LATEST_LEDGER_NUM'), - ('ledgersPerJob', 'LEDGERS_PER_JOB'), - ('overlapLedgers', 'OVERLAP_LEDGERS')): - if key in (doc.get('range') or {}): - setattr(mc, name, (doc['range'])[key]) - profile = profiles.load_profile_doc(doc.get('profile') or {}) - # The whole config, judged at the first moment it is complete. Anything - # wrong rejects the POST with the reason rather than dispatching a run that - # is already misconfigured. - validate_config() - if mc.RANGE_ORDER == 'longest-first' and not profile: - raise ValueError( - "RANGE_ORDER=longest-first requires a profile: it orders ranges by " - "their measured seconds, and with no profile every range ties and " - "dispatch stays tip-first. POST a profile, or set RANGE_ORDER.") - records.write_atomic(mc.RUN_PATH, json.dumps(doc, separators=(',', ':'))) - mc.PROFILE = profile - logger.info("profile installed: %d ranges", len(mc.PROFILE)) - # Opened here rather than in the POST handler, so a restart that reads - # run.json back resumes on exactly the same path. It did not, and a - # restarted monitor blocked on this forever while /status kept answering - # with its placeholder -- nothing was unreachable, so the driver polled a - # dead run indefinitely. Observed on ssc-test 2026-08-08. - http_server.started.set() - - -_LIVENESS_NUMBERS = (('LIVENESS_PROBE_TIMEOUT_SECONDS', float), - ('LIVENESS_SWEEP_SECONDS', float), - ('LIVENESS_MAX_CONCURRENCY', int)) - - -def validate_config(): - """Every fatal config check, against the whole config. - - Runs from /start rather than at import, because that is the first moment the - config is complete -- the profile arrives with the POST. One validation - point, one failure channel: whatever is wrong comes back as a 400 with the - reason instead of a crashlooping pod the driver can only time out on. - - Coerces the numeric env vars and rebinds them, so no caller ever sees the - string form. - """ - for name, cast in _LIVENESS_NUMBERS: - raw = getattr(mc, name) - try: - value = cast(raw) - except (TypeError, ValueError): - raise ValueError( - "LIVENESS_PROBE_TIMEOUT_SECONDS and LIVENESS_SWEEP_SECONDS must be " - "numbers; LIVENESS_MAX_CONCURRENCY must be an integer") from None - if value <= 0: - raise ValueError(f"{name} must be greater than zero, got {raw!r}") - setattr(mc, name, value) - - if mc.RANGE_ORDER not in mc.VALID_RANGE_ORDERS: - raise ValueError("RANGE_ORDER must be one of %s, got %r" - % (', '.join(mc.VALID_RANGE_ORDERS), mc.RANGE_ORDER)) - # The ledger range, which nothing checked while it came from helm values -- - # an inverted or zero-width range generates no work and the run just ends, - # reporting success on nothing. - if mc.LEDGERS_PER_JOB <= 0: - raise ValueError("ledgersPerJob must be greater than zero, got %r" - % (mc.LEDGERS_PER_JOB,)) - if mc.OVERLAP_LEDGERS < 0: - raise ValueError("overlapLedgers cannot be negative, got %r" - % (mc.OVERLAP_LEDGERS,)) - if mc.LATEST_LEDGER_NUM <= mc.STARTING_LEDGER: - raise ValueError( - "latestLedgerNum must be greater than startingLedger, got %r and %r" - % (mc.LATEST_LEDGER_NUM, mc.STARTING_LEDGER)) - # The pool maps arrive per run, so this is the first point they meet the - # ladder they are keyed to. A tier with no claim does not fail -- the pod - # keeps the flat REQ_CPU/REQ_MEM and a second one fits beside it, which - # undoes the isolation the whole tiering exists for: giving a pod its node - # to itself raised throughput 29-92%. Silent, and only visible afterwards as - # a run that cost more than it should. - if mc.POOL_PREFIX: - routable = [name for _, name in sizing._parsed_pool_tiers()] - routable += [mc.POOL_UNPROFILED, mc.POOL_NO_PROFILE] - for env_name, raw in (('POOL_CPU', mc.POOL_CPU), ('POOL_MEM', mc.POOL_MEM)): - claimed = {k for k, _ in mc.label_pairs(raw)} - missing = [t for t in routable if t and t not in claimed] - if missing: - raise ValueError( - "%s has no entry for %s; a pooled range routed there would " - "keep the flat request and share its node" - % (env_name, ', '.join(sorted(set(missing))))) - -status = { - 'num_remain': 1, # non-zero until the first real update, so callers don't see a premature 0 - 'queue_remain_count': 0, - 'queue_succeeded_count': 0, - 'queue_failed_count': 0, - 'queue_in_progress_count': 0, - 'jobs_failed': [], - 'workers_refresh_duration': 0, - 'mission_duration': 0, -} -status_lock = threading.Lock() - -# Beside progress.json: the volume is the only durable store. -_MISSION_START = os.path.join(config.LOG_DIR, 'mission_started') - - -# --- the run itself --------------------------------------------------------- -def reconcile_loop(): - global status - # Nothing is dispatched until the driver has POSTed /start: a range sized - # before the profile lands is sized wrong, and it cannot be re-sized later. - http_server.started.wait() - # None until reconcile has an owner reference to attach it to; until then - # process start is correct anyway, because that IS the start of a new run. - mission_start_time = read_mission_start() or time.time() - state = {'owner': None, 'replayed': set(), - 'counted': {}} - while True: - try: - if state['owner'] is None: - state['owner'] = owner_ref() - _progress_owner['ref'] = state['owner'] - if read_mission_start() is None: - records.write_atomic(_MISSION_START, repr(mission_start_time)) - - r = reconcile(state) - - # Grafana-only worker responsiveness, from the pod snapshot - # reconcile already has. One bounded sweep per pass, so this waits at - # most LIVENESS_SWEEP_SECONDS -- see worker_liveness.publish. - refresh_start = time.time() - targets = r.pop('_worker_targets') +async def reconcile_loop(state, stop): + while not stop.is_set(): + if state.ranges: try: - worker_counts = worker_liveness.publish(targets) - except Exception as e: - worker_counts = {'up': 0, 'down': 0, 'unknown': len(targets)} - now = time.time() - if now - state.get('last_liveness_error_log', 0) >= 60: - state['last_liveness_error_log'] = now - logger.exception( - "worker liveness publication failed (%s); reporting all " - "current candidates unknown and continuing reconcile", e) - workers_refresh_duration = time.time() - refresh_start - - mission_duration = time.time() - mission_start_time - with status_lock: - visible_in_progress = r['in_progress'] + r['finalizing'] - status = { - 'num_remain': r['remaining'], - 'queue_remain_count': r['remaining'], - 'queue_succeeded_count': r['completed'], - 'queue_failed_count': len(r['failed_ranges']), - 'queue_in_progress_count': len(visible_in_progress), - 'jobs_failed': r['failed_ranges'], - 'workers_refresh_duration': workers_refresh_duration, - 'mission_duration': mission_duration, - } - metrics.catchup_queues.labels(queue="remain").set(r['remaining']) - metrics.catchup_queues.labels(queue="succeeded").set(r['completed']) - metrics.catchup_queues.labels(queue="failed").set(len(r['failed_ranges'])) - metrics.catchup_queues.labels(queue="in_progress").set( - len(visible_in_progress)) - metrics.workers.labels(status="up").set(worker_counts['up']) - metrics.workers.labels(status="down").set(worker_counts['down']) - metrics.workers.labels(status="unknown").set(worker_counts['unknown']) - metrics.refresh_duration.set(workers_refresh_duration) - metrics.mission_duration.set(mission_duration) - logger.info("Status: %s", json.dumps(status)) - - except Exception as e: - logger.exception("Error while reconciling: %s", str(e)) - - time.sleep(mc.RECONCILE_INTERVAL_SECONDS) - - -def reconcile(state): - desired = ranges.generate_ranges() - by_end = {str(end): count for end, count in desired} - progress = load_progress() - completed = progress.setdefault('completed', {}) - failed = progress.setdefault('failed', {}) - - jobs = kube.batch_v1.list_namespaced_job( - config.NAMESPACE, label_selector=f"{config.LABEL_RUN}={config.RUN_NAME}").items - job_pods = pods_by_job() - - live = {} # range-end -> (attempt, job) - current_attempts = set() - for j in jobs: - end = (j.metadata.labels or {}).get(config.LABEL_RANGE) - attempt = int((j.metadata.labels or {}).get(config.LABEL_ATTEMPT, 1)) - current_attempts.add((str(end), attempt)) - prev = live.get(end) - if prev is None or attempt >= prev[0]: - live[end] = (attempt, j) - - in_progress = [] - finalizing = [] - # The same ranges as `in_progress`, keyed by end. `remaining` is a COUNT - # over this run's range list, never `total - completed`: the shared progress - # record can carry ends from a run with a different ledgersPerJob, and a - # subtraction lets those move a number describing THIS run. - in_flight = set() - for end, (attempt, j) in list(live.items()): - st = j.status - if st.succeeded: - # Record before the Job's TTL can reclaim it: `seconds` is the - # pod's own start -> finish, and the pod goes ~1 min after the node - # empties. - if end not in completed: - pod = job_pods.get(j.metadata.name) - completed[end] = completion_record(end, attempt, st, pod, - by_end.get(end)) - if pod is not None and config.SAVE_SUCCESS_LOGS: - backstop_save_pod_log(pod.metadata.name, end, attempt) - # Durably recorded first: the record is what makes the volume - # and the Job disposable, so it must land before either goes. - save_progress(progress) - else: - # Backfill. The record is written when the Job flips to - # succeeded, usually before the collector finalizes, so - # reconstruct the whole profile rather than a field-by-field - # subset that can leave a record permanently short. - late = attempts._repair_completed_profile(end, attempt, completed[end]) - if late: - save_progress(progress) - logger.info("range %s: measurements arrived late, backfilled %s", - end, sorted(late)) - # Per sighting of a recorded range, not per first sight: both are - # idempotent, and hanging them off the run-once branch above leaks - # the volume and the Job whenever the process dies between - # save_progress and here. - release_pvc(end) - if _attempt_finalized(end, attempt): - # Nothing more can be learned from the Job. Deleting it reaps the - # pod, and .metrics is the only place peaks live, so this waits - # for the collector's marker; JOB_TTL_SECONDS reclaims anything - # the collector never finishes. - reap_range_jobs(end) - else: - # Keep the range counted as in-progress until that marker - # lands: the driver writes the final profile as soon as the - # count reaches zero. Not in `in_progress`, so it costs no - # dispatch capacity below. - finalizing.append(job_key(int(end), by_end.get(end, 0))) - elif st.failed: - # Completion is terminal, so a Failed Job for a recorded range is - # garbage: classifying it would redispatch the range against a PVC - # that was already released. Sweep it. - if end in completed: - logger.info("range %s already recorded complete; discarding " - "leftover Job for attempt %d", end, attempt) - reap_range_jobs(end) - continue - pod = job_pods.get(j.metadata.name) - if pod is not None: - record_outcome(end, attempt, pod) - backstop_save_pod_log(pod.metadata.name, end, attempt) - if end in failed: - # The decision is recorded and cannot change: no successor is - # coming and no budget is left. Everything worth keeping -- the - # archive, .outcome, .verdict -- is durable by the time the - # collector marks the attempt done, so the Job and the volume - # are holding nothing. Deleting the Job reaps the pod, which is - # why this waits for that marker like the success path. - # - # Without it the range stays the newest Job for its range and - # every pass re-derives the same verdict and re-logs the same - # condemnation until JOB_TTL_SECONDS -- measured at 15 identical - # lines over 9 minutes. - if _attempt_finalized(end, attempt): - release_pvc(end) - reap_range_jobs(end) - continue - verdict = verdict_for(end, attempt, j, pod) - # Durable before anything reads a tally: _oom_count and the budget - # below both count this attempt. - save_verdict(end, attempt, verdict['outcome']) - - decision = retry_decision(verdict, end, attempt) - if decision.action == 'defer': - in_progress.append(job_key(int(end), by_end[end])) - in_flight.add(str(end)) - continue - - spent, cap = budget_for(verdict, end, attempt) - if decision.action == 'retry' and spent < cap: - _log_retry(end, attempt, verdict, decision, cap) - try: - kube.batch_v1.create_namespaced_job(config.NAMESPACE, build_job( - int(end), by_end[end], attempt + 1, state['owner'], - decision.memory, decision.ephemeral)) - except ApiException as e: - if e.status != 409: - raise - current_attempts.add((str(end), attempt + 1)) - # After the successor exists, never before: if the create failed - # with the predecessor gone, the next pass would redispatch at - # attempt 1 and lose the escalated request. live[] keys on the - # highest attempt, so the two coexisting for a pass is handled. - # - # Gated on .done like the success path, which means the collector - # has finalized this attempt's peaks, tx_apply and duration. - # JOB_TTL_SECONDS reaps it if the collector never gets there. - if _attempt_finalized(end, attempt): - delete_job(end, attempt) - in_progress.append(job_key(int(end), by_end[end])) - in_flight.add(str(end)) - continue - - if decision.reason is not None: - logger.error("range %s exhausted %d attempts (%s)", end, cap, - decision.reason) - else: - # Say it plainly: otherwise the range only appears under - # failed{} and the mission aborts with no explanation. - logger.error("!!! RANGE CONDEMNED !!! %s failed with outcome=%s exitCode=%s " - "on attempt %d and is NOT retryable; this fails the mission", - end, verdict['outcome'], verdict.get('exitCode'), attempt) - - if end not in failed: - failed[end] = {'attempts': attempt, - 'pod': verdict.get('pod', pod.metadata.name if pod else ''), - 'outcome': verdict['outcome'], - 'exitCode': verdict['exitCode']} - save_progress(progress) - else: - in_progress.append(job_key(int(end), by_end.get(end, 0))) - in_flight.add(str(end)) - - # Nothing halts dispatch -- not a shrinking `completed` record, not a - # condemned range. The mission waits for `remaining == 0 and in_progress == - # []`, so a frozen dispatch deadlocks the driver; a condemned range is - # reported once the run drains, keeping the work already paid for. - # - # Dispatch, heaviest range first (index 0 is the tip), up to PARALLELISM. - created = 0 - # No slots: a range's PVC is keyed by the range itself, so concurrency is - # simply how many are in flight. - capacity = mc.PARALLELISM - len(in_progress) - for end, count in desired: - if capacity <= 0: - break - key = str(end) - if key in completed or key in failed or key in live: - continue + await reconcile(state) + except Exception: + # A pass is a projection; the next one rebuilds it. Dying here + # leaves the run with no writer. + logger.exception("reconcile pass failed") try: - record_range_start(end, kube.batch_v1.create_namespaced_job( - config.NAMESPACE, build_job(end, count, 1, state['owner']))) - current_attempts.add((str(end), 1)) - created += 1 - capacity -= 1 - in_progress.append(job_key(end, count)) - in_flight.add(str(end)) - except ApiException as e: - if e.status != 409: # AlreadyExists: name uniqueness is the mutex - raise - current_attempts.add((str(end), 1)) - # Losing the mutex means the Job exists and is in flight, so it - # occupies a slot exactly like one we created and must spend - # capacity. - capacity -= 1 - in_progress.append(job_key(end, count)) - in_flight.add(str(end)) - - observe_recorded(progress, state['replayed']) - sync_counters(progress, state['counted'], current_attempts) - return { - 'total': len(desired), - 'completed': len(completed), - 'failed_ranges': [f"{job_key(int(k), by_end.get(k, 0))}|{v.get('pod', '')}" - for k, v in failed.items()], - 'in_progress': in_progress, - 'finalizing': finalizing, - 'created': created, - 'remaining': sum(1 for end, _ in desired - if str(end) not in completed - and str(end) not in failed - and str(end) not in in_flight), - # A Kubernetes snapshot only. The caller hands this to the independent - # liveness sampler after every dispatch/progress decision is complete. - '_worker_targets': worker_liveness.targets(job_pods.values()), - } - - -# The driver parses this out of the status ConfigMap -- `end/count`, joined -# with `|pod` in failed_ranges. Changing the shape breaks it silently. -def job_key(end, count): - return f"{end}/{count}" - - -def job_name(end, attempt): - return f"{config.RUN_NAME}-r{end}-a{attempt}" - - -# --- durable progress record ------------------------------------------------ -# Jobs are reclaimed during a long run, so completion cannot live only in Job -# objects. Written before a Job becomes TTL-eligible. - -# Set once at startup; the same ConfigMap the Jobs and PVCs hang off. -_progress_owner = {} - - -def load_progress(): - """The completed/failed record, or empty on a first start. - - The volume is the only source. An unreadable file replays rather than - halts, which is safe -- the PVCs survive and each range resumes at its last - closed ledger. - """ - try: - with open(mc.PROGRESS_FILE) as fh: - return json.load(fh) - except (OSError, ValueError): - return {} + async with asyncio.timeout(config.RECONCILE_INTERVAL_SECONDS): + await stop.wait() + except TimeoutError: + pass + stop.set() -def save_progress(progress): - # The monitor's own state, and the only copy. The driver's view of the run - # is status.json in the ConfigMap; this document is not published. - blob = json.dumps(progress, separators=(',', ':')) - records.write_atomic(mc.PROGRESS_FILE, blob) +async def reconcile(state): + """One pass: look, decide, act, persist. + The volume is touched in exactly two places, both threaded and both + batched. Everything between them is decisions and apiserver calls. + """ + jobs, pods = await cluster.snapshot() + states = derive_all(state, jobs, pods) + await asyncio.to_thread(observe, states) + await act(states, state) + await asyncio.to_thread(state.commit, states) + await publish(states, state, pods) +# --- what is true about one range ------------------------------------------- -# --- worker log capture ----------------------------------------------------- -def backstop_save_pod_log(pod_name, end, attempt): - """Last-resort archive for a range the collector never captured. +class RangeState: + """One range as of this pass. Cheap to build, thrown away at the end.""" - Covers only the gap where a pod lived and died while the collector was down, - detected by the absence of the .state file it writes when it claims a range. - Never overwrites a claimed or existing archive: two writers appending to one - gzip interleave members and duplicate lines. - """ - if os.path.exists(records.state_path(end, attempt)): - return True # collector has it (streaming or already finished) - path = records.log_path(end, attempt) - if os.path.exists(path): - return True - try: - body = kube.core_v1.read_namespaced_pod_log(pod_name, config.NAMESPACE, container='stellar-core') - except ApiException as e: - logger.warning("could not save log for range %s attempt %d (pod %s): %s", - end, attempt, pod_name, e.reason) - return False - try: - os.makedirs(config.LOG_DIR, exist_ok=True) - records.write_atomic(path, body, gzip.open) - return True - except OSError as e: - logger.warning("could not write %s: %s", path, e) - return False - - -def record_range_start(end, job): - """Persist attempt 1's Job creationTimestamp, once. - - Not status.startTime: the controller sets that asynchronously, so it is - absent from the create response. The gap between the two is what - wallSeconds measures. Written at creation because attempt 1's Job is gone - on the first retry. - """ - path = attempt_files.started_path(end) - if os.path.exists(path): - return - created = job.metadata.creation_timestamp if job and job.metadata else None - if created is None: - return - try: - records.write_atomic(path, created.isoformat()) - except OSError as e: - logger.warning("could not persist start time for range %s: %s", end, e) - - -def range_started_at(end): - """attempt 1's Job creationTimestamp, or None if it was never recorded.""" - try: - with open(attempt_files.started_path(end)) as fh: - return datetime.fromisoformat(fh.read().strip()) - except (OSError, ValueError): - return None + __slots__ = ('end', 'count', 'attempt', 'status', 'jobs', 'job', 'pod', + 'verdict', 'completed_at', 'done', 'measured') + def __init__(self, end, count, attempt=0, status='pending', jobs=(), + job=None, pod=None, verdict=None, completed_at=None): + self.end = str(end) + self.count = count + self.attempt = attempt + self.status = status + self.jobs = jobs + self.job = job + self.pod = pod + self.verdict = verdict + self.completed_at = completed_at + self.done = False # set by observe() + self.measured = None # set by observe() for a completed range -def classify(pod): - """Why did this pod fail? The Job object cannot answer this. + @property + def holds_capacity(self): + """Anything unfinished occupies a slot, a failed attempt included: its + retry is already owed. Missions run longest-first, so releasing the slot + would let short ranges dispatch ahead of a long range's retry.""" + return self.status in ('running', 'failed') - Job.status only carries a Failed condition with reason BackoffLimitExceeded - -- no exit code, no OOM. The detail lives on the pod, which is exactly the - object Karpenter deletes with the node, so this is recorded the moment the - watch sees it rather than when reconcile next runs. - """ - for cond in (pod.status.conditions or []): - if cond.type == 'DisruptionTarget' and cond.status == 'True': - return {'outcome': 'disrupted', 'exitCode': None} - # Kubelet can reject a pod before any container runs, e.g. - # VolumeAttachmentLimitExceeded. No exit code and no DisruptionTarget, so - # without this a transient admission rejection reads as a real failure. - if pod.status.reason == 'Evicted' and 'ephemeral' in (pod.status.message or ''): - # A limit eviction sets no DisruptionTarget, and stellar-core drains to - # exit 3, so the Job condition reads it as a catchup failure. - # status.message is the only discriminator, and only the pod carries it. - return {'outcome': 'ephemeral', 'exitCode': None, 'reason': pod.status.message} - if pod.status.reason in ('VolumeAttachmentLimitExceeded', 'OutOfcpu', 'OutOfmemory', - 'OutOfpods', 'UnexpectedAdmissionError', 'NodeAffinity', - 'Shutdown', 'Evicted'): - return {'outcome': 'rejected', 'exitCode': None, 'reason': pod.status.reason} - if pod.status.reason == 'DeadlineExceeded': - # The deadline is on the PodSpec, so the kubelet fires it and the pod - # carries the reason; the Job sees only a non-zero exit. - return {'outcome': 'timeout', 'exitCode': None, 'reason': pod.status.reason} - started = any(cs.state and cs.state.terminated for cs in (pod.status.container_statuses or [])) - if not started: - # No container ever reached a terminal state: nothing ran, so this is - # not evidence about the ledger range. - return {'outcome': 'rejected', 'exitCode': None, - 'reason': pod.status.reason or 'no container status'} - for cs in (pod.status.container_statuses or []): - t = cs.state.terminated if cs.state else None - if t is None: - continue - # 137 is SIGKILL, which the kubelet also uses for a graceful-stop - # timeout -- but with reason OOMKilled it is unambiguous. - if t.reason == 'OOMKilled': - return {'outcome': 'oom', 'exitCode': t.exit_code} - if t.exit_code not in (0, None): - return {'outcome': 'failed', 'exitCode': t.exit_code} - return {'outcome': 'failed', 'exitCode': None} - - -def record_outcome(end, attempt, pod): - path = records.outcome_path(end, attempt) - if os.path.exists(path): - return - data = classify(pod) - data['pod'] = pod.metadata.name - # The only place a failed attempt's duration is available: reconcile - # computes `seconds` on the success path only, and the pod is about to go. - data['attemptSeconds'] = _pod_seconds(pod) - try: - records.write_atomic(path, json.dumps(data)) - except OSError as e: - logger.warning("could not persist outcome for range %s: %s", end, e) - - -# The Job controller writes the exit code and pod name into the failure -# condition message, e.g. -# "Container stellar-core for pod ns/kic-r400000-a1-xxxxx failed with exit -# code 137 matching FailJob rule at index 1" -# Unlike the pod, this survives node consolidation. -_JOB_MSG = re.compile(r"for pod \S+?/(?P\S+) failed with exit code (?P\d+)") -_JOB_RULE = re.compile(r"rule at index (?P\d+)") - - -def _failure_rules(): - """podFailurePolicy rules, in evaluation order, tagged with what they mean. - - First match wins, so reaching the exit-137 rule proves DisruptionTarget did - not match -- that ordering is what separates an OOM kill from a - grace-period SIGKILL after the pod is gone. - - All FailJob: the Job must fail with reason=PodFailurePolicy so the message - names the rule index. A Count action would surface as BackoffLimitExceeded - and lose the signal. Retries stay with the monitor because raising a memory - limit needs a new Job -- spec.template is immutable. - """ - return [ - ('disrupted', client.V1PodFailurePolicyRule( - action='FailJob', - on_pod_conditions=[client.V1PodFailurePolicyOnPodConditionsPattern( - type='DisruptionTarget', status='True')])), - ('oom', client.V1PodFailurePolicyRule( - action='FailJob', - on_exit_codes=client.V1PodFailurePolicyOnExitCodesRequirement( - container_name='stellar-core', operator='In', values=[137]))), - ('failed', client.V1PodFailurePolicyRule( - action='FailJob', - on_exit_codes=client.V1PodFailurePolicyOnExitCodesRequirement( - container_name='stellar-core', operator='NotIn', values=[0]))), - ] - - -# Order here is the contract with the Job controller's "rule at index N". -RULE_ORDER = ['disrupted', 'oom', 'failed'] -_RULE_OUTCOME = dict(enumerate(RULE_ORDER)) - - -def classify_from_job(job): - """Recover a verdict from the Job when the pod is already gone. - - Rule index is the signal, not the exit code: rules are evaluated - first-match-wins, so reaching the exit-137 rule proves the DisruptionTarget - rule did not match, which is the only way to tell an OOM kill from a - grace-period SIGKILL once the pod is gone. - - Index and exit code are parsed independently -- a rule matching on - onPodConditions reports no exit code at all, so requiring one would make the - disruption case unreadable. - """ - for cond in (job.status.conditions or []): - if cond.type != 'Failed' or cond.status != 'True': - continue - msg = cond.message or '' - if cond.reason == 'DeadlineExceeded': - # activeDeadlineSeconds fired: the attempt hung rather than failing. - # Retryable -- a genuinely stuck range will exhaust its attempts. - return {'outcome': 'timeout', 'exitCode': None, 'pod': '', - 'source': 'job-condition'} - if cond.reason != 'PodFailurePolicy': - # e.g. BackoffLimitExceeded -- carries no per-rule detail. - continue - rule = _JOB_RULE.search(msg) - detail = _JOB_MSG.search(msg) - outcome = _RULE_OUTCOME.get(int(rule.group('idx'))) if rule else None - code = int(detail.group('code')) if detail else None - if outcome is None: - if code is None: - return None - # No usable rule index. A drained stellar-core exits 3, not 137, so - # a bare 137 is an OOM and a bare 3 without DisruptionTarget is a - # catchup failure. - outcome = 'oom' if code == 137 else 'failed' - return {'outcome': outcome, 'exitCode': code, - 'pod': detail.group('pod') if detail else '', - 'source': 'job-condition'} - return None - - -def save_verdict(end, attempt, outcome): - """Persist the EFFECTIVE verdict for one attempt, so budgets can be tallied. - - The .outcome file is not enough on its own: it is classified from the pod, - and a deadline kill reads as a plain exit-3 `failed` there -- only the Job's - DeadlineExceeded condition says `timeout`. Reconcile resolves that conflict - once, and this is where the answer is kept, on the same durable logs volume - as everything else, so a monitor restart does not reset a range's budgets. - """ - path = attempt_files.verdict_path(end, attempt) - try: - records.write_atomic(path, str(outcome)) - except OSError as e: - logger.warning("could not persist verdict for range %s attempt %s: %s", - end, attempt, e) +def derive_all(state, jobs, pods): + return [derive(end, count, state.progress, jobs, pods) + for end, count in state.ranges] -# --- tx_apply --------------------------------------------------------------- +def derive(end, count, progress, jobs, pods): + """One range from its newest Job, that Job's pod, and the record. -def _pod_seconds(pod): - """Container start -> finish for one attempt, or None if unreadable.""" - start = pod.status.start_time if pod.status else None - if start is None: - return None - for cs in (pod.status.container_statuses or []): - t = cs.state.terminated if cs.state else None - if t is not None and t.finished_at: - return (t.finished_at - start).total_seconds() - return None - - -# --- job construction ------------------------------------------------------- - -# Resume decision, before catchup. Skip new-db only when the DB on /data belongs -# to this range AND replay had started: bucket apply assumes a fresh DB, so a -# crash during it must start over, and "Ledger close complete" is the -# discriminator -- bucket apply never closes a ledger. -# -# The LCL comes from stellar-core's own log, not the database: core 27 dropped -# the ledgerheaders table. -RESUME_SCRIPT = r'''set -e -KEY="%(key)s" -TARGET=%(target)d -COUNT=%(count)d -MARK=/data/.job-key -RESUME=false -LCL="" -if [ -f "$MARK" ] && [ "$(cat "$MARK" 2>/dev/null)" = "$KEY" ]; then - # Ask core for its own LCL through its own accessor, so this survives the v27 - # schema change and any log level. Safe because core has not started, so - # nothing holds /data/buckets/stellar-core.lock. Core logs to the console - # alongside the JSON, hence grepping rather than parsing. - # One "num" key in the document and it is the ledger's. Do NOT window with - # `grep -A '"ledger":'`: bucketlist puts ~40 lines of hashes in between. - LCL=$(/usr/bin/stellar-core --conf /config/stellar-core.cfg offline-info --console 2>/dev/null \ - | sed -n 's/.*"num"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/p' | head -1 || true) - if [ -n "$LCL" ]; then - echo "RESUME PROBE: offline-info reports lcl $LCL" - else - # Fallback: the previous incarnation's log on /data. Goes blind above INFO, - # which is why it is no longer the primary probe. - PREV_LOG=$(ls -t /data/stellar-core*.log 2>/dev/null | head -n 1 || true) - if [ -n "$PREV_LOG" ]; then - LCL=$(grep -oE "Ledger close complete: [0-9]+" "$PREV_LOG" 2>/dev/null | tail -1 | grep -oE "[0-9]+$" || true) - fi - echo "RESUME PROBE: offline-info gave nothing; log fallback says '${LCL:-none}'" - fi - # Already at the target: replay finished and the attempt was evicted before it - # could exit 0. Re-running catchup applies nothing and exits 2 identically - # every time, so the range would burn its whole budget over completed work. - if [ -n "$LCL" ] && [ "$LCL" -ge "$TARGET" ] 2>/dev/null; then - echo "ALREADY COMPLETE: $KEY reached ledger $LCL >= target $TARGET; nothing left to replay" - exit 0 - fi - if [ -n "$LCL" ] && [ "$LCL" -ge $((TARGET - COUNT)) ] && [ "$LCL" -lt "$TARGET" ] 2>/dev/null; then - RESUME=true; echo "RESUME: $KEY reached ledger $LCL, replay had started; skipping new-db" - else - echo "RESUME DECLINED: $KEY last close was '${LCL:-none}' (need >= $((TARGET - COUNT))); bucket phase incomplete, starting fresh" - fi -fi -printf '%%s' "$KEY" > "$MARK" -if [ "$RESUME" != "true" ]; then - /usr/bin/stellar-core --conf /config/stellar-core.cfg new-db --console -fi -exec /usr/bin/stellar-core --conf /config/stellar-core.cfg catchup "$KEY" \ - --metric 'ledger.transaction.apply' --console -''' - - -def owner_ref(): - cm = kube.core_v1.read_namespaced_config_map(f"{config.RUN_NAME}-stellar-core-config", config.NAMESPACE) - return [client.V1OwnerReference(api_version='v1', kind='ConfigMap', - name=cm.metadata.name, uid=cm.metadata.uid, - block_owner_deletion=True)] - - -def release_pvc(end): - """Drop a completed range's volume. - - The PVC exists so an interrupted range resumes at L+1; a succeeded range has - nothing to resume. They are owner-referenced to the release, so without this - every volume survives until `helm uninstall`. - - Best-effort: a failure costs disk, never correctness. + The record answers for a range with no Job: reaped, condemned, or never + started. Without it a reaped range reads as pending and is dispatched a + second time. """ - if config.STORAGE_MODE != 'pvc': - return - name = f"{config.RUN_NAME}-data-r{end}" - try: - kube.core_v1.delete_namespaced_persistent_volume_claim(name, config.NAMESPACE) - metrics.pvc_released.inc() - except ApiException as e: - if e.status != 404: - logger.warning("could not release PVC for completed range %s: %s", end, e) - - -def _attempt_finalized(end, attempt): - """Has the collector written everything it will for this attempt? - - It writes this file last, after .metrics. Anything inferred instead -- peaks - being present, tx_apply being readable -- is a guess: tx_apply falls back to - the archive so it is available long before the collector finishes, and an - attempt can legitimately finalize with no peaks at all. + range_jobs = jobs.get(str(end), ()) + job = max(range_jobs, key=_attempt_of, default=None) + if job is None: + # Terminal: re-deciding would re-count the verdict on every pass. + if str(end) in progress['condemned']: + return RangeState(end, count, status='condemned') + if str(end) in progress['completed']: + attempt = progress['completed'][str(end)].get('attempts', 1) + return RangeState(end, count, attempt, 'completed') + return RangeState(end, count) + + attempt = _attempt_of(job) + names = [j.metadata.name for j in range_jobs] + pod = pods.get(job.metadata.name) + # succeeded and failed are POD COUNTS and there is no running field, so + # running is whatever is neither. + if job.status.succeeded: + return RangeState(end, count, attempt, 'completed', names, job, pod, + completed_at=job.status.completion_time) + if job.status.failed: + # Classified by observe(); this phase reads nothing. + return RangeState(end, count, attempt, 'failed', names, job, pod) + return RangeState(end, count, attempt, 'running', names, job, pod) + + +def observe(states): + """Every read a pass makes. Only a finished attempt has anything to say. + + A failed one is classified here because an exit-3 verdict decompresses a + whole archive, and a completed one is aggregated here because that walks + its .metrics -- both are reads, so both belong before any decision. """ - return os.path.exists(records.done_path(end, attempt)) + for st in states: + if st.status not in ('failed', 'completed'): + continue + st.done = record.is_done(st.end, st.attempt) + if not st.done: + continue # the collector is still writing this attempt + if st.status == 'failed': + st.verdict = verdict.effective(st.end, st.attempt, st.job) + elif st.jobs: + st.measured = aggregate(st) -def reap_range_jobs(end): - """Delete every Job this range has, not just the attempt that won. +def _attempt_of(job): + return int((job.metadata.labels or {}).get(config.LABEL_ATTEMPT, 1)) - Completion is terminal for the RANGE. An attempt-scoped reap leaves an - older Failed Job standing -- the common case is an attempt lost to node - disruption whose collector died with the node, so it was never finalized - and was deliberately not deleted. Once the winner's Job is gone, that - leftover is the range's highest live attempt, and the next pass feeds it - straight into the retry decision and re-runs an already-recorded range - against a freshly recreated, empty PVC. - """ - try: - jobs = kube.batch_v1.list_namespaced_job( - config.NAMESPACE, - label_selector=f"{config.LABEL_RUN}={config.RUN_NAME},{config.LABEL_RANGE}={end}").items - except ApiException as e: - logger.warning("could not list jobs for completed range %s: %s", end, e) - return - for j in jobs: - try: - kube.batch_v1.delete_namespaced_job(j.metadata.name, config.NAMESPACE, - propagation_policy='Background') - metrics.jobs_reaped.inc() - except ApiException as e: - if e.status != 404: - logger.warning("could not delete finished job %s for range %s: %s", - j.metadata.name, end, e) +# --- what a pass does about it ---------------------------------------------- -def delete_job(end, attempt): - """Drop a finished Job once nothing more is owed by it. - reconcile() lists every Job and Pod each pass, so a lingering finished Job - inflates both LIST calls. Background propagation is what takes the pod with - it; orphan would leave the next pass listing just as much. +async def act(states, state): + """Retries first, then reaps, then new work into whatever slots are left. - Callers must have persisted what they need first. Best-effort: a 404 is the - race with the TTL controller, and raising would strand every other range in - the pass -- JOB_TTL_SECONDS still reclaims the object. - """ - try: - kube.batch_v1.delete_namespaced_job(job_name(end, attempt), config.NAMESPACE, - propagation_policy='Background') - metrics.jobs_reaped.inc() - except ApiException as e: - if e.status != 404: - logger.warning("could not delete finished job for range %s attempt %d: %s", - end, attempt, e) - - -def ensure_pvc(end, owner): - name = f"{config.RUN_NAME}-data-r{end}" - try: - kube.core_v1.read_namespaced_persistent_volume_claim(name, config.NAMESPACE) - return name - except ApiException as e: - if e.status != 404: - raise - spec = client.V1PersistentVolumeClaimSpec( - access_modes=['ReadWriteOnce'], - resources=client.V1VolumeResourceRequirements(requests={'storage': mc.STORAGE_SIZE})) - if mc.STORAGE_CLASS: - spec.storage_class_name = mc.STORAGE_CLASS - kube.core_v1.create_namespaced_persistent_volume_claim(config.NAMESPACE, client.V1PersistentVolumeClaim( - metadata=client.V1ObjectMeta(name=name, owner_references=owner, - labels={config.LABEL_RUN: config.RUN_NAME, config.LABEL_RANGE: str(end)}), - spec=spec)) - return name - - -def _resources(mem=None, eph=None, end=None, attempt=1): - # Before mem is defaulted below -- reading it afterwards can never see None, - # which silently disabled profile sizing entirely. - overrides = sizing._profile_overrides(end, escalated=(mem is not None or eph is not None), - attempt=attempt) - # `mem` is the escalated request on an OOM retry, else the configured one. - req = {'cpu': mc.REQ_CPU, 'memory': mem or mc.REQ_MEM} - # Only ephemeral-storage is limited: it is the one dimension where an - # unbounded pod takes the node down rather than itself. - lim = {} - - # A profiled range on a pooled run has its node to itself -- the tier's - # memory cut is sized to exclude a second pod -- so an ephemeral limit - # guards no neighbour and only turns spare disk into an eviction. Measured: - # a dwarf range capped at 5211Mi alone on a 20Gi root, dying the moment it - # exceeded its profiled peak by more than the margin, with most of the disk - # unused. The request goes with the limit because its only remaining job is - # scheduling, and the tier label already decides placement. - # - # Unprofiled pooled runs keep both: nothing measured them, so there is no - # peak to have been generous about. - pooled_profiled = bool(mc.POOL_PREFIX and mc.PROFILE) - - # Only meaningful in ephemeral mode. In PVC mode a large request makes disk - # the binding dimension and halves workers-per-node for no reason. - if mc.REQ_EPHEMERAL and not pooled_profiled: - # Raise the request with the limit: ephemeral-storage is a scheduling - # dimension, so a pod that outgrew it no longer fits where it was. - req['ephemeral-storage'] = eph or mc.REQ_EPHEMERAL - else: - # pvc mode: /data is not on the node disk, so an ephemeral override - # would size a dimension this run does not use. Pooled+profiled: the - # pod owns its node and the axis is dropped deliberately. - overrides.pop('ephemeral-storage', None) - if mc.LIM_EPHEMERAL and not pooled_profiled: - lim['ephemeral-storage'] = eph or mc.LIM_EPHEMERAL - - # The profile moves requests only. Disk excepted, because its limit is what - # the kubelet enforces. - for key, value in overrides.items(): - req[key] = value - if key == 'ephemeral-storage' and mc.LIM_EPHEMERAL: - lim[key] = value - # Unmeasured range: the configured requests, exactly as if there were no - # profile at all. - return client.V1ResourceRequirements(requests=req, limits=lim or None) - - -def pod_labels(end, attempt): - """Labels on the worker POD, which are not the Job's. - - LABEL_ATTEMPT has to be here too: the collector reads it off the pod to pick - which range--a.* files the attempt owns, defaulting to "1". On the - Job alone, every retry overwrites attempt 1's peaks instead of being maxed - against them. + A retry needs no slot: its range already holds one. """ - labels = {config.LABEL_RUN: config.RUN_NAME, config.LABEL_RANGE: str(end), - config.LABEL_ATTEMPT: str(attempt)} - if mc.EMIT_MISSION_LABEL and mc.MISSION: - labels['mission'] = mc.MISSION - return labels + work = [] + active = sum(1 for st in states if st.holds_capacity) + for st in states: + if st.status == 'failed': + work.append(_settle(st, state)) + elif st.status == 'completed' and st.jobs and st.done: + work.append(_reap(st, state)) -def _prestop_delay(): - """A preStop that stalls the kubelet, or None when the knob is off. + for st in states: + if st.status == 'pending' and active < config.PARALLELISM: + active += 1 + work.append(_dispatch(st, state)) - `sleep` from the image rather than `sh -c sleep`: one less process to exist - in a container that is being torn down, and it fails loudly at hook-exec - time if the binary is missing rather than silently succeeding. + for result in await asyncio.gather(*work, return_exceptions=True): + if isinstance(result, Exception): + logger.warning("action failed: %s", result) - Refuses to install a hook that cannot finish inside the grace period. A - preStop longer than the grace is worse than none: the kubelet kills it - mid-sleep, reports FailedPreStopHook, and the container is signalled - anyway -- so the delay is not bought and an error is logged for every - evicted pod. - """ - if mc.WORKER_PRESTOP_SLEEP_SECONDS <= 0: - return None - if mc.WORKER_PRESTOP_SLEEP_SECONDS >= mc.WORKER_GRACE_SECONDS: - logger.warning( - "PRESTOP_SLEEP_SECONDS=%s does not fit in GRACE_SECONDS=%s; " - "not installing a preStop hook that the kubelet would kill", - mc.WORKER_PRESTOP_SLEEP_SECONDS, mc.WORKER_GRACE_SECONDS) - return None - return client.V1Lifecycle( - pre_stop=client.V1LifecycleHandler( - _exec=client.V1ExecAction( - command=['/bin/sleep', str(mc.WORKER_PRESTOP_SLEEP_SECONDS)]))) - - -def build_job(end, count, attempt, owner, mem=None, eph=None): - key = job_key(end, count) - script = RESUME_SCRIPT % {'key': key, 'target': end, 'count': count} - - if config.STORAGE_MODE == 'pvc': - data_vol = client.V1Volume(name='data', persistent_volume_claim=( - client.V1PersistentVolumeClaimVolumeSource(claim_name=ensure_pvc(end, owner)))) - else: - data_vol = client.V1Volume(name='data', empty_dir=client.V1EmptyDirVolumeSource()) - - env = [client.V1EnvVar(name='ASAN_OPTIONS', value=mc.ASAN_OPTIONS)] if mc.ASAN_OPTIONS else [] - command = ['/bin/sh', '-c', script] - volumes = [data_vol, client.V1Volume( - name='config', config_map=client.V1ConfigMapVolumeSource( - name=f"{config.RUN_NAME}-stellar-core-config"))] - volume_mounts = [ - client.V1VolumeMount(name='data', mount_path='/data'), - client.V1VolumeMount(name='config', mount_path='/config')] - - # Require and avoid go in ONE matchExpressions list: expressions within a - # term are ANDed, separate terms are ORed, and an avoid-only pod in its own - # term would match every node. - match = [] - if mc.NODE_LABEL_KEY: - # Pooled runs route per range: the label names the tier this range's - # memory puts it in. An escalated attempt resolves to a promoted tier, - # which is what moves the pod to nodes its memory fits. - tier = sizing.pool_for(end, attempt) - value = f"{mc.POOL_PREFIX}-{tier}" if tier else mc.NODE_LABEL_VALUE - match.append(client.V1NodeSelectorRequirement( - key=mc.NODE_LABEL_KEY, operator='In', values=[value])) - for key, value in mc.label_pairs(mc.REQUIRE_NODE_LABELS): - # Literal, unlike the pool-routed pair above: these are properties of - # the pool rather than of the range, so they do not vary per attempt. - match.append(client.V1NodeSelectorRequirement( - key=key, operator='In', values=[value])) - if mc.AVOID_NODE_LABEL_KEY: - # No value means "avoid the label however it is set", which is - # DoesNotExist; NotIn [""] would only exclude the empty value. - match.append(client.V1NodeSelectorRequirement( - key=mc.AVOID_NODE_LABEL_KEY, - operator='NotIn' if mc.AVOID_NODE_LABEL_VALUE else 'DoesNotExist', - values=[mc.AVOID_NODE_LABEL_VALUE] if mc.AVOID_NODE_LABEL_VALUE else None)) - affinity = None - if match: - affinity = client.V1Affinity(node_affinity=client.V1NodeAffinity( - required_during_scheduling_ignored_during_execution=client.V1NodeSelector( - node_selector_terms=[client.V1NodeSelectorTerm(match_expressions=match)]))) - - # Taint value must be absent: the mission emits {key, effect} with no value, - # and the default Equal operator does not match "" against "true". - tolerations = [client.V1Toleration(key=mc.TOLERATE_TAINT, effect='NoSchedule')] if mc.TOLERATE_TAINT else None - - container = client.V1Container( - name='stellar-core', image=mc.CORE_IMAGE, - command=command, env=env, resources=_resources(mem, eph, end, attempt), - ports=[client.V1ContainerPort(container_port=11626, name='http')], - lifecycle=_prestop_delay(), - volume_mounts=volume_mounts) - - return client.V1Job( - metadata=client.V1ObjectMeta( - name=job_name(end, attempt), owner_references=owner, - labels={config.LABEL_RUN: config.RUN_NAME, config.LABEL_RANGE: str(end), - config.LABEL_ATTEMPT: str(attempt)}), - spec=client.V1JobSpec( - # The monitor owns retries, not the Job controller: backoffLimit 0 - # means the Job fails once and stays put, so reconcile classifies the - # failure and decides on attempt N+1. - # - # On the JobSpec, not the pod: a pod-level deadline is immutable once - # the pod exists, so a mis-set value could not be corrected on a live - # run. - active_deadline_seconds=mc.ATTEMPT_DEADLINE_SECONDS or None, - backoff_limit=0, - pod_failure_policy=client.V1PodFailurePolicy( - rules=[r for _, r in _failure_rules()]), - ttl_seconds_after_finished=mc.JOB_TTL_SECONDS, - template=client.V1PodTemplateSpec( - metadata=client.V1ObjectMeta(labels=pod_labels(end, attempt)), - spec=client.V1PodSpec( - # On the POD, not the JobSpec: activeDeadlineSeconds runs - # from the Job's startTime, charging Pending time against a - # budget meant to bound how long the range RUNS. The - # pod-level field starts at container start. - # IRSA for the S3 history mirror; without it workers fall - # back to the public archive, which throttles at 1024. - service_account_name=mc.WORKER_SERVICE_ACCOUNT or None, - # Never restarted in place: the pod stays terminal and - # inspectable for classification and the backstop log read. - restart_policy='Never', - termination_grace_period_seconds=mc.WORKER_GRACE_SECONDS, - affinity=affinity, tolerations=tolerations, - containers=[container], - volumes=volumes)))) - - -# --- what a pass decides ---------------------------------------------------- -# Counters, verdicts and retry policy: everything reconcile() calls to turn a -# finished attempt into a decision. - -_ATTEMPT_FILE = re.compile( - r'^range-(?P\d+)-a(?P[1-9]\d*)\.' - r'(?:verdict|outcome|state|metrics|done|log\.gz)$') - - -def _retry_counter_totals(progress, current_attempts=()): - """Reconstruct retry metrics from durable records and observed attempts. - - A verdict says why an attempt ended; it does not say a retry was dispatched. - Attempt N therefore contributes to retry totals only when attempt N+1 is - evidenced by progress, a persisted per-attempt file, or the current Job - snapshot. The latter makes a newly-created successor visible before its range - completes, while the durable sources rebuild the same truth after restart. - """ - try: - names = os.listdir(config.LOG_DIR) - except OSError: - names = [] - max_attempt = {} - terminal = set() +async def _dispatch(st, state): + created = await dispatch.create(int(st.end), st.count, 1) + if created is not None: + state.started.append((st.end, created.metadata.creation_timestamp)) + logger.info("range %s dispatched", st.end) - def remember(end, attempt): - try: - attempt = int(attempt) - except (TypeError, ValueError): - return - if attempt < 1: - return - end = str(end) - max_attempt[end] = max(max_attempt.get(end, 0), attempt) - - if isinstance(progress, dict): - for bucket in ('completed', 'failed'): - bucket_records = progress.get(bucket) - if not isinstance(bucket_records, dict): - continue - for end, record in bucket_records.items(): - if not isinstance(record, dict): - continue - try: - attempt = int(record.get('attempts', 1)) - except (TypeError, ValueError): - continue - if attempt < 1: - continue - remember(end, attempt) - terminal.add((str(end), attempt)) - - for item in current_attempts: - try: - end, attempt = item - except (TypeError, ValueError): - continue - remember(end, attempt) - verdict_files = set() - outcome_files = set() - for name in names: - match = _ATTEMPT_FILE.match(name) - if not match: - continue - key = (match.group('end'), int(match.group('attempt'))) - remember(*key) - if name.endswith('.verdict'): - verdict_files.add(key) - elif name.endswith('.outcome'): - outcome_files.add(key) - - effective = {} - for end, attempt in verdict_files: - try: - with open(attempt_files.verdict_path(end, attempt)) as fh: - verdict = fh.read().strip() - except OSError: - continue - if verdict in config.ATTEMPT_OUTCOMES: - effective[(end, attempt)] = verdict - - # .outcome predates .verdict and is safe only for a completed chain: a - # collector outcome can still be superseded by reconcile's verdict. Any - # verdict file, even malformed, means this is not a legacy attempt. - for end, attempt in outcome_files - verdict_files: - if attempt >= max_attempt.get(end, 0) and (end, attempt) not in terminal: - continue - try: - with open(records.outcome_path(end, attempt)) as fh: - record = json.load(fh) - except (OSError, ValueError): - continue - outcome = record.get('outcome') if isinstance(record, dict) else None - if outcome in config.ATTEMPT_OUTCOMES: - effective[(end, attempt)] = outcome - - retries = sum(max(0, attempt - 1) for attempt in max_attempt.values()) - reasons = {reason: 0 for reason in config.ATTEMPT_OUTCOMES} - for (end, attempt), reason in effective.items(): - if attempt < max_attempt.get(end, 0): - reasons[reason] += 1 - disruption_retried_ranges = { - end for (end, attempt), reason in effective.items() - if reason == 'disrupted' and attempt < max_attempt.get(end, 0) - } - - return { - 'retries': retries, - 'evicted': sum(1 for verdict in effective.values() if verdict == 'disrupted'), - 'spot_disruption_retried': len(disruption_retried_ranges), - 'oom': reasons['oom'], - 'ephemeral': reasons['ephemeral'], - 'reasons': reasons, - } - - -def sync_counters(progress, counted, current_attempts=()): - """Drive the counters from persisted state instead of from events. - - Two reasons not to .inc() as things happen: - - * a terminally-failed range stays the newest Job for its range, so an - event-driven inc fires again on every reconcile until teardown - * the process resets to zero on restart, while verdicts and attempt state on - the PVC survive - - Computing the true total and incrementing by the delta is monotonic, - idempotent, and self-heals after a restart: the counter starts at 0 and the - first sync walks it up to the recorded total. - """ - totals = _retry_counter_totals(progress, current_attempts) - for key, total, metric in (('retries', totals['retries'], metrics.retries), - ('oom', totals['oom'], metrics.oom_retries), - ('ephemeral', totals['ephemeral'], metrics.eph_retries), - ('evicted', totals['evicted'], metrics.evictions), - ('spot_disruption_retried', - totals['spot_disruption_retried'], - metrics.spot_disruption_retried)): - delta = total - counted.get(key, 0) - if delta > 0: - metric.inc(delta) - counted[key] = total - for reason in config.ATTEMPT_OUTCOMES: - metric = metrics.retry_reasons.labels(reason=reason) - key = ('reason', reason) - total = totals['reasons'][reason] - delta = total - counted.get(key, 0) - if delta > 0: - metric.inc(delta) - counted[key] = total - - -def observe_recorded(progress, replayed): - """Feed recorded completions into the histograms. - - Prometheus histograms are append-only and reset to zero when the process - restarts, so replaying every recorded range rebuilds the exact cumulative - total rather than double counting. Guarded per-process by `replayed`. - - Keyed on (range, field), not on the range alone: a range is usually - recorded before the collector has flushed its .metrics, so txApply is null - at first sight and backfilled a pass or two later. Marking the whole range - as replayed on first sight meant that backfill could never be observed, and - the histogram permanently disagreed with progress.json. +async def _settle(st, state): + """Decide a failed attempt, once the collector has finished with it. + + Deferred until .done because the verdict can still change: an exit 3 that + reads as a catchup failure is a retryable fetch fault if the archive says + so, and that is the difference between a retry and a dead run. """ - for end, rec in progress.get('completed', {}).items(): - # `is not None`, not truthiness: sum = 0ms records txApply 0.0, which is - # a real observation. Same for a sub-second duration. - for field, metric in (('seconds', metrics.full_duration), - ('wallSeconds', metrics.wall_duration), - ('txApply', metrics.tx_apply_duration)): - if (end, field) in replayed: - continue - value = rec.get(field) - if value is None: - continue - replayed.add((end, field)) - metric.observe(value) - - -def _range_wall_seconds(end, status): - """Attempt 1 created -> winner completed, or None if the start was never recorded. - - The range's whole life: every retry, every gap between them, every wait for a - node. Deliberately not falling back to the winning Job's own start -- that - measures one leg and understates exactly the mess this is here to capture. + if not st.done: + return + cause = st.verdict.get('outcome') + spent = record.note_cause(state.progress, st.end, st.attempt, cause) + ooms = spent.get('oom', 0) + base = sizing.requests_for(int(st.end), ooms)[0] + decision = policy.decide(int(st.end), st.verdict, spent, + base_memory=base.get('memory'), + base_ephemeral=base.get('ephemeral-storage')) + if decision.action == policy.DEFER: + return + if decision.action == policy.CONDEMN: + state.condemn(st, decision.reason) + await cluster.delete_job(dispatch.job_name(int(st.end), st.attempt)) + return + logger.info("range %s attempt %d -> %d (%s%s%s)", st.end, st.attempt, + st.attempt + 1, decision.reason, + f"; memory={decision.memory}" if decision.memory else '', + f"; ephemeral={decision.ephemeral}" if decision.ephemeral else '') + await dispatch.create(int(st.end), st.count, st.attempt + 1, + oom_count=ooms, memory=decision.memory, + ephemeral=decision.ephemeral) + metrics.settled(state.progress, cause, end=st.end) + # After the successor exists, never before: with the predecessor gone and + # the create failed, the next pass redispatches at attempt 1 and loses the + # escalated request. TTL reclaims it if this delete does not. + await cluster.delete_job(dispatch.job_name(int(st.end), st.attempt)) + + +async def _reap(st, state): + """Delete a completed range's Jobs and release its volume. + + Only once .done: reaping deletes the pod, which is the last place peaks and + terminated timestamps can be read. """ - started = range_started_at(end) - if not started or not status.completion_time: - return None - return (status.completion_time - started).total_seconds() + await cluster.reap(st.end, st.jobs) -def _range_compute_seconds(end, attempt, pod, wall): - """Compute seconds across the whole resumed chain, not this leg alone. - - A fresh single attempt may fall back to the winner's own seconds or to the - Job wall; a resumed chain is every leg or nothing, never winner-only. - """ - pod_seconds = _pod_seconds(pod) if pod is not None else None - chain = attempts.seconds_for_range(end, attempt, pod_seconds) - if chain is not None: - return chain - if len(attempts._resumed_chain(end, attempt)) == 1: - return pod_seconds if pod_seconds is not None else wall - return None +# --- what a pass reports ---------------------------------------------------- -def completion_record(end, attempt, status, pod, count=None): - """What a finished range cost and where it ran. +async def publish(states, state, pods): + began = time.monotonic() + counts = await liveness.publish(list(pods.values())) + swept = time.monotonic() - began # the sweep alone, nothing after it + metrics.sync_counters(state.progress, state.applied) + metrics.observe_completed(state.progress, state.replayed) + metrics.publish_gauges(state.counts(states), counts, swept, + time.time() - state.mission_start) - Assembled from the winning Job's status, the pod if it still exists, and the - per-attempt files the collector wrote. Every pod-derived field is optional on - purpose: a reaped node costs that field, never the record. - """ - wall = _range_wall_seconds(end, status) - # Not gated on `pod`: the collector's record outlives it, so a reaped node - # must not cost us the metric. - tx = attempts.tx_apply_for_range(end, attempt) - if tx is None: - logger.warning("could not read tx_apply for range %s (pod gone?); " - "metric will be missing for this range", end) - record = {'seconds': _range_compute_seconds(end, attempt, pod, wall), - 'wallSeconds': wall, 'txApply': tx, 'attempts': attempt} - # Ledger count travels with the record: ledgersPerJob is per-run input - # per range, so it cannot be recomputed from config when the profile is read - # back. - if count is not None: - record['count'] = count - record.update(attempts.peaks_for_range(end, attempt)) - return record - - -# A failed attempt resolves to one of three actions. `reason` names the cause for -# the log line and is None only when the range is condemned outright. -Decision = collections.namedtuple('Decision', 'action reason memory ephemeral') - - -def _retry(reason, memory=None, ephemeral=None): - return Decision('retry', reason, memory, ephemeral) - - -CONDEMN = Decision('condemn', None, None, None) -# Wait for the collector's .done marker and decide on a later pass. -DEFER = Decision('defer', None, None, None) - - -def verdict_for(end, attempt, job, pod): - """Why this attempt failed, from the pod if it survived and the Job if not. - - Two classifications, ranked: - 1. the pod named a mechanism (OOM, DisruptionTarget, eviction, deadline) - -- it wins, being the precise one - 2. else the Job says timeout -- only the Job knows the deadline fired, and - the drained pod reads as a plain `failed` - 3. else whichever exists, unknown over nothing: retry rather than condemn - """ - from_pod = attempt_files.read_outcome(end, attempt) - from_job = classify_from_job(job) - if from_pod and from_pod.get('outcome') in mc.POD_AUTHORITATIVE_OUTCOMES: - verdict = from_pod - elif from_job and from_job.get('outcome') == 'timeout': - verdict = from_job - else: - verdict = from_pod or from_job or {'outcome': 'unknown', 'exitCode': None} - if verdict.get('source') == 'job-condition': - logger.info("range %s attempt %d classified from Job condition " - "(exit %s); pod was already gone", - end, attempt, verdict.get('exitCode')) - # A third source of evidence for the one ambiguous exit code. Exit 3 is - # "did not complete", which a SIGTERM drain and a real failure share, so the - # archive is what separates them -- and only once the collector has finished - # writing it. Until then the verdict stays `failed` and the decision defers. - if (verdict.get('exitCode') == mc.CATCHUP_INCOMPLETE_EXIT - and _attempt_finalized(end, attempt) - and attempts.exit3_retry_cause(end, attempt)): - verdict = dict(verdict, outcome='fetch-fault') - return verdict - - -def _condemn_timeout(end, attempt): - """Terminal: the deadline is the only thing that ends a wedged range. - - A range stuck on an unreachable archive closes no ledgers and never exits, so - retrying spends the deadline again for nothing. - """ - logger.error("!!! RANGE CONDEMNED !!! %s hit its %ss attempt deadline " - "on attempt %s; this fails the mission. Check its archived " - "log for 'maybe stale archive' -- an unreachable history " - "mirror is the usual cause.", - end, mc.ATTEMPT_DEADLINE_SECONDS, attempt) - return CONDEMN +# --- the run ---------------------------------------------------------------- -def _retry_oom(end, attempt): - """Retry with the next memory rung. - Rungs climbed = OOMs seen, not attempts made; this attempt's outcome is - already on disk, so the count includes it. `had` is what this attempt - actually ran with, by the same derivation that sized it -- indexing on - `attempt` instead names a rung nobody occupied. - """ - base = (sizing._profile_overrides(end, escalated=False) or {}).get('memory') - ooms = attempt_files._oom_count(end, attempt) - had = (sizing.pool_memory(sizing.pool_for(end, attempt)) if mc.POOL_PREFIX - else sizing.mem_for_attempt(ooms, base)) - return _retry(f"OOM-killed at memory request {had}", - memory=sizing.mem_for_attempt(ooms + 1, base, end=end)) +class State: + """Everything that outlives a pass. Small on purpose: the cluster and the + volume are the sources, and this is what cannot be re-derived from them.""" + def __init__(self): + self.ranges = [] + self.progress = {'completed': {}, 'condemned': {}, 'causes': {}, + 'counters': collections.Counter(), + 'disruptedRanges': set()} + self.started = [] # (end, createdAt) awaiting a commit + self.applied = {} # counter totals already pushed to prometheus + self.replayed = set() # (range, field) already observed + self.mission_start = time.time() + self._live = {} # this pass's counts, for /status -def _retry_ephemeral(end, attempt): - """Retry with the next disk rung. + def resume(self): + """Replay run.json through the same validation path. - Rungs climbed = evictions seen, not attempts made, as with the OOM ladder: - on spot most retries are disruptions. The count includes this attempt. + A restarted monitor must resume the run it inherited rather than wait + for a /start delivered to its predecessor -- and the profile has to come + back with it, since longest-first orders by it. + """ + self.mission_start = record.mission_start() + self.progress = record.load_progress() + doc = record.load_run() + if doc is None: + return + try: + self.start(doc) + except ValueError as e: + logger.error("run.json no longer validates (%s); waiting for /start", e) + + def start(self, doc): + spec = doc.get('range') or {} + for key, name in (('order', 'RANGE_ORDER'), + ('startingLedger', 'STARTING_LEDGER'), + ('latestLedgerNum', 'LATEST_LEDGER_NUM'), + ('ledgersPerJob', 'LEDGERS_PER_JOB'), + ('overlapLedgers', 'OVERLAP_LEDGERS')): + if key in spec: + setattr(config, name, spec[key]) + config.set_profile(sizing.load_profile(doc.get('profile') or {})) + config.validate() + self.ranges = dispatch.range_list() + logger.info("run started: %d ranges, %s, profile of %d", + len(self.ranges), config.RANGE_ORDER, len(config.PROFILE)) + + def condemn(self, st, reason): + """Mark terminal; commit() persists it at the end of the pass. + + Idempotent because it must be: the record is all that keeps a condemned + range terminal once its Job is gone, and counting the verdict twice + would inflate the run's reasons forever. Dispatch does not halt -- + ending the run is the driver's call. + """ + if st.end in self.progress['condemned']: + return + self.progress['condemned'][st.end] = { + 'end': int(st.end), 'count': st.count, 'attempts': st.attempt, + 'pod': (st.verdict or {}).get('pod') or '', + 'outcome': (st.verdict or {}).get('outcome'), + 'exitCode': (st.verdict or {}).get('exitCode'), + 'reason': reason} + metrics.settled(self.progress, (st.verdict or {}).get('outcome')) + logger.error("range %s condemned after %d attempts: %s", + st.end, st.attempt, reason) + + def commit(self, states): + """Every write a pass makes, in one place and one record write. + + A completed range keeps being re-aggregated while its attempt files are + readable, so measurements landing after the Job succeeded are picked up; + once reaped the recorded entry stands. + """ + for st in states: + if st.measured is not None: + self.progress['completed'][st.end] = st.measured + for end, created in self.started: + record.record_start(end, created) + self.started.clear() + record.save_progress(self.progress) + + def counts(self, states): + """The four buckets, which must not overlap. + + `remaining` is work NOT YET DISPATCHED, not work outstanding: a range in + flight is counted by `running` alone. Overlapping them makes a + fully-dispatched run of four ranges report four remaining and four + running, and every consumer that sums the buckets sees eight. + """ + self._live = { + 'remaining': sum(1 for st in states if st.status == 'pending'), + 'running': sum(1 for st in states if st.holds_capacity), + 'completed': sum(1 for st in states if st.status == 'completed'), + 'condemned': sum(1 for st in states if st.status == 'condemned'), + } + return dict(self._live) + + def status(self): + """What the driver decides on, in both shapes. + + `remaining` is 1 until the first real pass, or a driver polling a fresh + monitor sees zero and tears the mission down. The legacy keys ship until + no old driver is left: it reads the counts as ints (absent = 0 = done) + and iterates jobs_failed (absent = null = throws). + """ + live = self._live or {'remaining': 1, 'running': 0, 'completed': 0, + 'condemned': 0} + condemned = list(self.progress['condemned'].values()) + return dict(live, + condemned=condemned, + num_remain=live['remaining'], + queue_remain_count=live['remaining'], + queue_in_progress_count=live['running'], + queue_succeeded_count=live['completed'], + queue_failed_count=len(condemned), + jobs_failed=condemned) + + +def aggregate(st): + """One completed range, from its whole attempt chain. + + Every field is advisory -- it sizes a LATER run -- so any may be absent, and + absent must stay absent. A zero becomes a profile entry, and since a range + resolves to the nearest measured end ABOVE it, one such entry captures every + range beneath it and hides the real measurement. """ - evictions = attempt_files._cause_count(end, attempt, ('ephemeral',)) - had = sizing.eph_for_attempt(evictions) - reason = (f"evicted for exceeding its {had} ephemeral-storage limit" if had - else "evicted under node disk pressure with no configured limit") - return _retry(reason, ephemeral=sizing.eph_for_attempt(evictions + 1)) + # Read once per attempt; every field below comes out of this. + seen = {n: record.read_metrics(st.end, n) for n in range(1, st.attempt + 1)} + out = {'count': st.count, 'attempts': st.attempt} + + # Every attempt: max is monotone, and no excluded input would be wrong. + peaks = {} + for metrics_of in seen.values(): + for key, value in metrics_of.items(): + if key.startswith('peak') and value is not None: + peaks[key] = max(peaks.get(key, value), value) + out.update(peaks) + + # Resumed chain only: a fresh retry discarded its predecessor's work. + chain = _resumed_chain(seen, st.attempt) + for field, source in (('seconds', 'attemptSeconds'), + ('txApply', 'txApplySeconds')): + legs = [seen[n].get(source) for n in chain] + if legs and all(leg is not None for leg in legs): + out[field] = sum(legs) + + wall = _wall_seconds(st) + if wall is not None: + out['wallSeconds'] = wall + return out -def _decide_exit3(end, attempt): - """A plain exit 3: the archive named no fetch fault, or has not landed yet. +def _resumed_chain(seen, attempt): + """The winning attempt, plus every predecessor it continued from. - verdict_for already promotes an exit 3 to `fetch-fault` once the archive - says so, so reaching here means either the collector is still writing it -- - wait, bounded by JOB_TTL_SECONDS -- or nothing in it explains the failure, in - which case the range is condemned and its archive survives on the volume. - """ - if not _attempt_finalized(end, attempt): - return DEFER - return CONDEMN - - -def retry_decision(verdict, end, attempt): - """Retry this range with what, condemn it, or wait for more evidence.""" - outcome = verdict['outcome'] - if outcome == 'timeout': - return _condemn_timeout(end, attempt) - elif outcome == 'rejected': - # The pod never started, so a retry cannot mask a broken range -- but it - # is the range's own budget now, not the disruption one. - return _retry(f"rejected by the node before starting " - f"({verdict.get('reason', '?')})") - elif outcome == 'disrupted': - return _retry("lost to node disruption") - elif outcome == 'fetch-fault': - return _retry(f"exited {mc.CATCHUP_INCOMPLETE_EXIT} after a fetch fault " - f"({attempts.exit3_retry_cause(end, attempt)})") - elif outcome == 'oom': - return _retry_oom(end, attempt) - elif outcome == 'ephemeral': - return _retry_ephemeral(end, attempt) - elif outcome == 'unknown': - # Nothing classified the pod. Without evidence the monitor cannot tell a - # reaped node from a range that really failed, and a run that reports - # success on a range nobody verified is worse than one that stops. - return CONDEMN - elif verdict.get('exitCode') == mc.CATCHUP_INCOMPLETE_EXIT: - return _decide_exit3(end, attempt) - elif verdict.get('exitCode') is None: - # The verdict came from the Job condition, which says Failed and nothing - # about why. Same absence of evidence as `unknown`, same answer. - return CONDEMN - else: - return CONDEMN # a genuine catchup failure - - -def budget_for(verdict, end, attempt): - """(spent, cap) for the cause that killed this attempt. - - mc.ATTEMPT_BUDGETS is the whole retry policy; a cause with no entry caps - at 0 and is condemned on sight. `spent` counts only THIS cause, so evictions - cannot drain the OOM or disk budgets. This verdict is already on disk, so - the Nth failure is the one that exhausts a budget of N. + `resumed` sits on the attempt that continued, so it names its own + predecessor. A fresh start breaks the chain: an attempt that ran new-db + discarded whatever came before it. """ - outcome = verdict['outcome'] - return (attempt_files._cause_count(end, attempt, (outcome,)), - mc.ATTEMPT_BUDGETS.get(outcome, 0)) - - -def _log_retry(end, attempt, verdict, decision, cap): - if verdict['outcome'] == 'oom': - logger.error( - "!!! OOM RETRY !!! range %s was OOM-killed on attempt %d/%d; retrying with " - "memory limit %s -- RAISE THE CONFIGURED MEMORY LIMIT, this run is only " - "surviving by escalating at runtime", end, attempt, cap, decision.memory) - elif verdict['outcome'] == 'ephemeral': - logger.error( - "!!! DISK RETRY !!! range %s %s on attempt %d/%d; retrying with " - "ephemeral-storage %s -- RAISE THE CONFIGURED EPHEMERAL STORAGE, this " - "run is only surviving by escalating at runtime", - end, decision.reason, attempt, cap, decision.ephemeral) - else: - logger.warning("range %s %s on attempt %d/%d; retrying", - end, decision.reason, attempt, cap) - - -def pods_by_job(): - """One list per reconcile, indexed by Job name. - """ - out = {} - for p in kube.core_v1.list_namespaced_pod( - config.NAMESPACE, label_selector=f"{config.LABEL_RUN}={config.RUN_NAME}").items: - jn = (p.metadata.labels or {}).get('batch.kubernetes.io/job-name') - if jn: - out.setdefault(jn, p) - return out + chain, n = [attempt], attempt + while n > 1 and seen[n].get('resumed'): + n -= 1 + chain.append(n) + return sorted(chain) -def read_mission_start(): - """When this run first started, or None if not recorded yet. +def _wall_seconds(st): + """Attempt 1's dispatch to the winning attempt's completion. - Its own file: progress.json is keyed by ledger range, and anything else in - it would be walked as one. On the volume so it survives a monitor restart, - which is what makes mission_duration span the run rather than the process. + The range's whole life: every retry, every gap, every wait for a node. Not + the winning Job's own start, which measures one leg and understates exactly + the mess this exists to capture -- which is why attempt 1's creation + timestamp is persisted before its Job can be deleted. """ - try: - with open(_MISSION_START) as fh: - return float(fh.read()) - except (OSError, ValueError): + started = record.started_at(st.end) + if started is None or st.completed_at is None: return None + return max(0.0, (st.completed_at - started).total_seconds()) if __name__ == '__main__': - main() + sys.exit(asyncio.run(main())) diff --git a/src/MissionParallelCatchup/apps/log_collector.py b/src/MissionParallelCatchup/apps/log_collector.py index d8f5d48b..f68b3804 100644 --- a/src/MissionParallelCatchup/apps/log_collector.py +++ b/src/MissionParallelCatchup/apps/log_collector.py @@ -169,8 +169,11 @@ def _start_follow(session, name, end, attempt, pod): if not doom: return False # Recorded now: once the object is gone there is no telling a drain we lost - # a race with from a corpse that never had a metric to lose. - write_metrics(end, attempt, {'disruptionReason': doom}) + # a race with from a corpse that never had a metric to lose. The duration + # goes with it, and for the same reason -- an evicted pod is usually deleted + # before it is ever seen terminal, and _finish has nothing to read it from. + # It runs short by the drain, and merges by max, so it is only ever a floor. + write_metrics(end, attempt, {'disruptionReason': doom, **_duration(pod)}) logger.info("range %s condemned (%s), opening follow", end, doom) _follows[key] = asyncio.create_task(_run_follow(session, name, end, attempt)) return True diff --git a/src/MissionParallelCatchup/lib/config.py b/src/MissionParallelCatchup/lib/config.py index 1c8ff3a7..6eebfa2c 100644 --- a/src/MissionParallelCatchup/lib/config.py +++ b/src/MissionParallelCatchup/lib/config.py @@ -1,52 +1,226 @@ -"""What the monitor and the log-collector must agree on. +"""Every knob, and the validation a run is admitted through. -Both processes run from the same image and share one /logs volume, so these are -the names that have to mean the same thing in both: which run they belong to, -where its files are, and the vocabulary of an attempt's verdict. The monitor's -own settings live in monitor_config; the collector's in collector_config. +Read through the module -- `config.PARALLELISM`, never `from config import +PARALLELISM`. /start rebinds several of these after validating them, and an +imported name binds a copy that never sees the rebind. -Read through the module, never copied out of it: - - import config - ... config.LOG_DIR ... - -`from config import LOG_DIR` binds a COPY. A test's monkeypatch rebinds the -attribute on this module, and a copy taken at import time never sees it -- -silently, with the test passing against the default. A module object is a -singleton, so reading through it is what makes those visible everywhere. +Numbers that arrive from the chart stay strings until validate() coerces them. +Coercing at import made a bad value a boot crash, and a process that cannot +start cannot report why. """ +import logging import os -# The run, and every object that belongs to it. The collector selects pods on -# LABEL_RUN=RUN_NAME, so a disagreement here has it watching a different run -- -# or nothing at all. -NAMESPACE = os.getenv('NAMESPACE', 'default') +logger = logging.getLogger('job_monitor') -RUN_NAME = os.getenv('RUN_NAME', 'parallel-catchup') -LABEL_RUN = 'catchup.stellar.org/run' +def _int(name, default): + """A chart-env integer, typed here rather than at /start. -LABEL_RANGE = 'catchup.stellar.org/range-end' + The loop idles on its interval while waiting for /start, so a string there + crash-loops the pod before it can be told anything. A bad value falls back + loudly rather than killing the boot; /start-delivered values get a 400. + """ + raw = os.getenv(name) + if raw is None: + return default + try: + return int(raw) + except (TypeError, ValueError): + logger.error("%s=%r is not an integer; using %d", name, raw, default) + return default -LABEL_ATTEMPT = 'catchup.stellar.org/attempt' -# The shared volume. The collector owns writes here -- it streams each worker's -# log and records the .outcome verdict while the pod still exists -- and the -# monitor reads them back during reconcile. +# --- identity, shared with the collector ------------------------------------ +NAMESPACE = os.getenv('NAMESPACE', 'default') +RUN_NAME = os.getenv('RUN_NAME', 'parallel-catchup') LOG_DIR = os.getenv('LOG_DIR', '/logs') -SAVE_SUCCESS_LOGS = os.getenv('SAVE_SUCCESS_LOGS', 'true').lower() == 'true' +LABEL_RUN = 'catchup.stellar.org/run' +LABEL_RANGE = 'catchup.stellar.org/range-end' +LABEL_ATTEMPT = 'catchup.stellar.org/attempt' -# Worker /data. pvc keeps it across pods, so an evicted range resumes at L+1 -- -# that is what makes spot viable. ephemeral puts it on the node disk: denser -# packing, no resume, and REQ_EPHEMERAL must be sized to hold the catchup DB. -# One PVC per range, not per concurrency slot: measured on ssc-test, 300 jobs -# with a PVC each cost no more wall-clock than 300 jobs reusing 40. -# Read by both: it also decides whether the collector samples ephemeral disk. -STORAGE_MODE = os.getenv('STORAGE_MODE', 'pvc') # pvc | ephemeral - -# The verdict vocabulary. The collector writes one of these into .outcome and -# the monitor charges it against a retry budget, so a name added on one side -# and not the other is an attempt nobody can classify. +# The collector writes one of these into .outcome; a name on one side and not +# the other is an attempt nobody can classify. ATTEMPT_OUTCOMES = ('disrupted', 'oom', 'ephemeral', 'timeout', 'rejected', 'unknown', 'failed', 'fetch-fault') + +# Collector-only, and here because both processes import THIS module as +# `config` when they run from one flat directory. +SAVE_SUCCESS_LOGS = os.getenv('SAVE_SUCCESS_LOGS', 'true').lower() == 'true' + +# --- the run ---------------------------------------------------------------- +PARALLELISM = _int('PARALLELISM', 3) +RECONCILE_INTERVAL_SECONDS = _int('LOGGING_INTERVAL_SECONDS', 10) +HTTP_PORT = _int('HTTP_PORT', 8080) + +# Delivered by POST /start, replayed from run.json on restart. +STARTING_LEDGER = '0' +LATEST_LEDGER_NUM = '0' +LEDGERS_PER_JOB = '16000' +OVERLAP_LEDGERS = '320' +RANGE_ORDER = 'tip-first' +PROFILE = [] # sorted [(end, record)] +_SORTED_SECONDS = None # cache, invalidated whenever PROFILE is set + +RANGE_ORDERS = ('tip-first', 'oldest-first', 'longest-first') + +# --- the worker pod --------------------------------------------------------- +CORE_IMAGE = os.getenv('CORE_IMAGE', 'stellar/stellar-core:latest') +ASAN_OPTIONS = os.getenv('ASAN_OPTIONS', '') +WORKER_SERVICE_ACCOUNT = os.getenv('WORKER_SERVICE_ACCOUNT', '') +WORKER_GRACE_SECONDS = int(os.getenv('WORKER_GRACE_SECONDS', '150')) +# Holds the pod open after SIGTERM so the collector can drain the last of its +# log. Must fit inside the grace period; a preStop the kubelet kills mid-sleep +# buys nothing and logs FailedPreStopHook on every evicted pod. +WORKER_PRESTOP_SLEEP_SECONDS = int(os.getenv('WORKER_PRESTOP_SLEEP_SECONDS', '5')) + +STORAGE_MODE = os.getenv('STORAGE_MODE', 'pvc') # pvc | ephemeral +STORAGE_CLASS = os.getenv('STORAGE_CLASS', '') +STORAGE_SIZE = os.getenv('STORAGE_SIZE', '60Gi') + +JOB_TTL_SECONDS = int(os.getenv('JOB_TTL_SECONDS', '3600')) +# 0 disables. A timeout is terminal, so a tight bound trades a certain +# catastrophe against a rounding error: one wedged range holds one slot. +ATTEMPT_DEADLINE_SECONDS = int(os.getenv('ATTEMPT_DEADLINE_SECONDS', '0')) + +# --- unpooled / unprofiled sizing ------------------------------------------- +# The packing unit for a run with no profile: 4 workers per r8*.2xlarge +# (cpu-bound) or 3 per m8*.2xlarge (memory-bound). The price of not measuring. +REQ_CPU = os.getenv('REQ_CPU', '1800m') +REQ_MEM = os.getenv('REQ_MEM', '9Gi') +REQ_EPHEMERAL = os.getenv('REQ_EPHEMERAL', '') +LIM_EPHEMERAL = os.getenv('LIM_EPHEMERAL', '') + +# --- profile-derived sizing (unpooled) -------------------------------------- +PROFILE_MARGIN = float(os.getenv('PROFILE_MARGIN', '1.15')) +# Flat, because a multiplicative margin is nothing at small rss: 1.15x of +# 190Mi is 19Mi of slack, and 90 ranges OOMKilled inside 90s on it. +PROFILE_CACHE_HEADROOM = os.getenv('PROFILE_CACHE_HEADROOM', '512Mi') +PROFILE_RUNTIME_MEMORY_INSURANCE = os.getenv('PROFILE_RUNTIME_MEMORY_INSURANCE', '3Gi') +PROFILE_MAX_MEM = os.getenv('PROFILE_MAX_MEM', '32Gi') +# Image, logs, sqlite WAL: none of it scales with the range. +PROFILE_EPHEMERAL_HEADROOM = os.getenv('PROFILE_EPHEMERAL_HEADROOM', '2Gi') +# Disk tracks runtime closely (pearson 0.920 over 3985 ranges). +PROFILE_RUNTIME_EPHEMERAL_INSURANCE = os.getenv('PROFILE_RUNTIME_EPHEMERAL_INSURANCE', '8Gi') +# Above the flat limit on purpose: that limit is what an UNMEASURED range gets. +PROFILE_MAX_EPHEMERAL = os.getenv('PROFILE_MAX_EPHEMERAL', '64Gi') + +# --- escalation ------------------------------------------------------------- +MEM_BUMP_FACTOR = float(os.getenv('MEM_BUMP_FACTOR', '1.5')) +MEM_ESCALATION_CAP = os.getenv('MAX_MEM', '48Gi') +EPH_BUMP_FACTOR = float(os.getenv('EPH_BUMP_FACTOR', '1.5')) +EPH_ESCALATION_CAP = os.getenv('MAX_EPHEMERAL', '200Gi') + +# --- pools ------------------------------------------------------------------ +# Empty disables routing entirely; every worker keeps NODE_LABEL_VALUE. +POOL_PREFIX = os.getenv('POOL_PREFIX', '') +POOL_TIERS = os.getenv( + 'POOL_TIERS', + '0:subdwarf,0.79:dwarf,1.61:subgiant,3.87:giant,8.85:supergiant,' + '18.38:hypergiant,:supernova') +# Fits once in the tier's on-demand shape, twice in its (one size larger) spot +# shape. Not 50% of either -- see the requirements. +POOL_MEM = os.getenv( + 'POOL_MEM', + 'subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,' + 'supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,' + 'protostar:29696Mi,nebula:9216Mi') +POOL_CPU = os.getenv( + 'POOL_CPU', + 'subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,' + 'hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80') +POOL_BLOCK_RUNGS = os.getenv('POOL_BLOCK_RUNGS', 'dwarf->subgiant,hypergiant->supernova') +POOL_UNPROFILED = os.getenv('POOL_UNPROFILED', 'protostar') +POOL_NO_PROFILE = os.getenv('POOL_NO_PROFILE', 'nebula') + +# --- placement -------------------------------------------------------------- +NODE_LABEL_KEY = os.getenv('NODE_LABEL_KEY', '') +NODE_LABEL_VALUE = os.getenv('NODE_LABEL_VALUE', '') +REQUIRE_NODE_LABELS = os.getenv('REQUIRE_NODE_LABELS', '') +AVOID_NODE_LABEL_KEY = os.getenv('AVOID_NODE_LABEL_KEY', '') +AVOID_NODE_LABEL_VALUE = os.getenv('AVOID_NODE_LABEL_VALUE', '') +TOLERATE_TAINT = os.getenv('TOLERATE_TAINT', '') + +# --- retry budgets ---------------------------------------------------------- +# The whole retry policy. A cause that is NOT here is condemned the first time +# it happens. Every budget is spent by its own cause: one shared attempt index +# let evictions drain the OOM budget to zero. +ATTEMPT_BUDGETS = { + 'disrupted': int(os.getenv('MAX_DISRUPTION_ATTEMPTS', '100')), + 'rejected': int(os.getenv('MAX_REJECTED_ATTEMPTS', '100')), + 'fetch-fault': int(os.getenv('MAX_FETCH_FAULT_ATTEMPTS', '20')), + 'oom': int(os.getenv('MAX_OOM_ATTEMPTS', '5')), + 'ephemeral': int(os.getenv('MAX_EPHEMERAL_ATTEMPTS', '4')), +} + +# --- liveness --------------------------------------------------------------- +LIVENESS_MAX_CONCURRENCY = int(os.getenv('LIVENESS_MAX_CONCURRENCY', '64')) +LIVENESS_PROBE_TIMEOUT_SECONDS = float(os.getenv('LIVENESS_PROBE_TIMEOUT_SECONDS', '2')) +# Under the reconcile interval on purpose: the pass awaits this sweep, so a +# deadline above the interval lets an unreachable fleet stretch dispatch and +# reaping behind a probe that only feeds a dashboard. +LIVENESS_SWEEP_SECONDS = float(os.getenv('LIVENESS_SWEEP_SECONDS', '5')) + +# Sized for the dispatch burst rather than a steady rate: a wave head creates +# ~1024 Jobs and PVCs at once. +APISERVER_CONCURRENCY = int(os.getenv('APISERVER_CONCURRENCY', '64')) + + +def label_pairs(raw): + """[(key, value)] from "k:v,k:v". A key with no value is dropped -- it would + require the label be exactly "", which no node carries, and the pod sits + Pending in a way that reads as slow provisioning.""" + out = [] + for item in (raw or '').split(','): + key, _, value = item.strip().partition(':') + if key and value: + out.append((key, value)) + return out + + +def set_profile(profile): + global PROFILE, _SORTED_SECONDS + PROFILE = profile + _SORTED_SECONDS = None + + +def validate(): + """Coerce and re-bind everything a run depends on. Raises ValueError. + + Called from /start, which is the first moment the configuration is + complete, so a misconfigured run is answered with a 400 rather than + crash-looping a pod the driver can only time out on. + """ + global STARTING_LEDGER, LATEST_LEDGER_NUM, LEDGERS_PER_JOB, OVERLAP_LEDGERS + + def positive(name, value, floor=1): + try: + n = int(value) + except (TypeError, ValueError): + raise ValueError(f"{name} is not an integer: {value!r}") + if n < floor: + raise ValueError(f"{name} must be >= {floor}, got {n}") + return n + + STARTING_LEDGER = positive('STARTING_LEDGER', STARTING_LEDGER, floor=0) + LATEST_LEDGER_NUM = positive('LATEST_LEDGER_NUM', LATEST_LEDGER_NUM) + LEDGERS_PER_JOB = positive('LEDGERS_PER_JOB', LEDGERS_PER_JOB) + OVERLAP_LEDGERS = positive('OVERLAP_LEDGERS', OVERLAP_LEDGERS, floor=0) + + if LATEST_LEDGER_NUM < STARTING_LEDGER: + raise ValueError(f"LATEST_LEDGER_NUM {LATEST_LEDGER_NUM} is below " + f"STARTING_LEDGER {STARTING_LEDGER}") + if RANGE_ORDER not in RANGE_ORDERS: + raise ValueError(f"RANGE_ORDER must be one of {RANGE_ORDERS}, " + f"got {RANGE_ORDER!r}") + if STORAGE_MODE not in ('pvc', 'ephemeral'): + raise ValueError(f"STORAGE_MODE must be pvc or ephemeral, " + f"got {STORAGE_MODE!r}") + if RANGE_ORDER == 'longest-first' and not PROFILE: + # With no profile every range ties, the sort is a no-op, and dispatch + # silently falls back to tip-first while the operator believes + # otherwise. + raise ValueError( + "RANGE_ORDER=longest-first requires a profile: it orders ranges by " + "measured seconds. POST a profile, or set another order.") diff --git a/src/MissionParallelCatchup/lib/monitor/attempt_files.py b/src/MissionParallelCatchup/lib/monitor/attempt_files.py deleted file mode 100644 index f293796d..00000000 --- a/src/MissionParallelCatchup/lib/monitor/attempt_files.py +++ /dev/null @@ -1,77 +0,0 @@ -"""What an attempt left behind, read back for retry accounting. - -The collector writes these files while the pod still exists; the monitor reads -them to decide whether an attempt may be retried and how far its resources must -climb. The counters are per CAUSE, not per attempt -- escalation must climb once -per OOM, not once per retry. -""" -import json -import os - -import config -import records - - -# Per RANGE, not per attempt: wallSeconds spans the range's whole life, so the -# only start that matters is the first one. Later attempts are the mess in -# between and are deliberately not recorded. -def started_path(end): - return os.path.join(config.LOG_DIR, f"range-{end}.started") - - - -def read_outcome(end, attempt): - try: - with open(records.outcome_path(end, attempt)) as fh: - return json.load(fh) - except (OSError, ValueError): - return None - - - -def _oom_count(end, attempt): - """How many earlier attempts at this range were OOM-killed. - - Escalation must climb once per OOM, not once per attempt. On spot most - retries are evictions -- measured on ssc-test 2026-07-30, 288 disruption - retries against 7 OOM retries -- and a range disrupted three times then - OOMing once would otherwise jump to base * 1.5^4, a 5x request for a single - OOM. That inflation is fleet-wide and it is what exhausts the vCPU quota. - """ - return sum(1 for n in range(1, int(attempt) + 1) - if _verdict_of(end, n) == 'oom') - - - -def verdict_path(end, attempt): - return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.verdict") - - - -def _verdict_of(end, attempt): - try: - with open(verdict_path(end, attempt)) as fh: - verdict = fh.read().strip() - except OSError: - # Pre-fix runs, or an attempt whose verdict write lost the volume: - # the pod-derived classification is the next best thing. - outcome = (read_outcome(end, attempt) or {}).get('outcome') - return outcome if outcome in config.ATTEMPT_OUTCOMES else None - return verdict if verdict in config.ATTEMPT_OUTCOMES else None - - - -def _cause_count(end, attempt, causes): - """How many of attempts 1..N at this range failed for one of `causes`. - - Budgets are per cause, not per attempt. One shared attempt index meant - cluster churn -- which has its own deliberately large budget -- drained the - small budgets belonging to the causes that say something about the range: a - range evicted MAX_ATTEMPTS times had an effective OOM and disk budget of - zero, was condemned on its first real OOM without ever being escalated, and - took the whole mission with it. - """ - return sum(1 for n in range(1, int(attempt) + 1) - if _verdict_of(end, n) in causes) - - diff --git a/src/MissionParallelCatchup/lib/monitor/attempts.py b/src/MissionParallelCatchup/lib/monitor/attempts.py deleted file mode 100644 index b0c12ddd..00000000 --- a/src/MissionParallelCatchup/lib/monitor/attempts.py +++ /dev/null @@ -1,352 +0,0 @@ -"""What a range's attempts add up to. - -One layer above `records`, which reads a single attempt's files: everything here -answers a question about the RANGE by walking the attempts behind it -- the peak -it reached, what it cost, whether an exit 3 is worth retrying. - -Nothing here touches the cluster. It is also where the blocking file reads live, -so it is the seam an executor would wrap if the monitor ever moved onto a loop. -""" -import gzip -import json -import logging -import zlib - -import attempt_files -import records - -logger = logging.getLogger() - - -# PVC size is not profiled: growing it buys no packing. Ephemeral storage is, -# but only in ephemeral mode on on-demand nodes. Any field may be absent, and -# the consumer falls back to its default. -PEAK_FIELDS = ('peakAnonBytes', 'peakWorkingSetBytes', - 'peakEphemeralBytes') - - -def peaks_for_range(end, attempt=1): - """Highest peak any attempt at this range reached, per axis. - - Not just the successful attempt. In pvc mode a pod that dies once replay has - started leaves /data behind, and the next attempt resumes at LCL+1 with - RESUME=true -- skipping the archive download and the bucket apply, which is - where peak memory actually happens. Its peak describes the tail of the range, - not the range, so profiling the winner alone under-reports by the whole - download-vs-replay gap. On spot, where eviction is routine and resume is the - entire point of durable /data, that would make the run unprofileable. - - Attempts that hit a ceiling are counted too. A pod OOM-killed at 8Gi really - did allocate ~8Gi and wanted more, so its peak is a lower bound on demand, - not an artifact of the limit -- and it is the attempt most worth keeping, - because download concurrency scales with available cpu and a pod that - bursted on an idle node can peak above the one that eventually succeeded. - Sizing off the quieter attempt would OOM the range again. There is no false - ratchet: a pod given 8Gi that only touches 1Gi records 1Gi. - - Advisory: used to size a LATER run's requests, never to decide anything - about this one. Any field may be absent. - """ - out = {} - for n in _peak_attempts(end, attempt): - try: - with open(records.metrics_path(end, n)) as fh: - data = json.load(fh) - except (OSError, ValueError): - continue - for k in PEAK_FIELDS: - v = data.get(k) - if v is not None and v > out.get(k, 0): - out[k] = v - return out - - -def _hit_a_ceiling(end, attempt): - """Was this attempt killed at one of its own resource limits?""" - return (attempt_files.read_outcome(end, attempt) or {}).get('outcome') in ('oom', 'ephemeral') - - -def _peak_attempts(end, attempt): - """Attempts whose peaks describe this range: the resumed chain, plus any - attempt that died at a limit, wherever it sits. - - A ceiling-hit peak is evidence about the range no matter which pass - produced it -- the process really did allocate that much and want more, so - it is a lower bound on demand and the next run must size above it. That is - the whole self-correcting loop: a range that OOMs at L records L, and - L * PROFILE_MARGIN + PROFILE_CACHE_HEADROOM clears it next time. - - Without this the fresh-start rule silently drops it. Measured on ssc-test - 2026-07-30: an OOM during replay resumes (RESUME accepted, 224 of 252) and - stays in the chain, but an OOM during download does not (25 of 252) -- and - a run at higher cpu is download-bound, so the loop would go quiet exactly - when it is most needed. - - Peaks only. tx_apply and seconds are summed, and a fresh start redoes work - the dropped attempt already did, so including it there would double-count. - """ - chain = set(_resumed_chain(end, attempt)) - return sorted(chain | {n for n in range(1, int(attempt) + 1) - if n not in chain and _hit_a_ceiling(end, n)}) - - -def _resumed_chain(end, attempt): - """Attempts describing one continuous pass over the range, oldest first. - - Stops at the last attempt that ran new-db: that one covered the whole range - on its own, so nothing before it is part of the same pass. - """ - first = int(attempt) - while first > 1 and _attempt_resumed(end, first): - first -= 1 - return range(first, int(attempt) + 1) - - -def _attempt_resumed(end, attempt): - """Did this attempt pick up at LCL+1 rather than run new-db? - - The collector's record is authoritative, and only records a resume. It - decides from the live stream at pod startup and re-reads its own archive at - finalization if it could have missed the line, so an absent flag means the - attempt did not resume -- there is nothing a second archive read here could - find that the collector did not. Measured across a 4805-attempt run: 744 - resumes, and not one the record missed. - """ - try: - with open(records.metrics_path(end, attempt)) as fh: - return json.load(fh).get('resumed') is True - except FileNotFoundError: - return False - except ValueError as e: - logger.warning("could not parse resume metrics for range %s attempt %s: %s", - end, attempt, e) - except OSError as e: - logger.warning("could not read resume metrics for range %s attempt %s: %s", - end, attempt, e) - return False - - -# Exit 3 covers a graceful SIGTERM as well as a real failure, so what decides is -# the cascade stellar-core prints when a history fetch fails. -# -# The anchor pair is adjacent by construction: GetHistoryArchiveStateWork emits -# its message on the same scheduler tick as its child's WORK_FAILURE. The aws -# stderr is relayed unsynchronised, so it is searched for nearby instead. -_FETCH_ANCHOR = 'maybe stale archive' - - -_FETCH_GAVE_UP = 'Catchup failed' - - -# Faults in front of S3: the object is fine, this pod could not reach it. A fresh -# pod on another node is the fix, which is what a retry is. -_FETCH_TRANSIENT = ('Could not connect to the endpoint URL', - 'Unable to locate credentials', 'ExpiredToken', - 'RequestTimeout', 'SlowDown', 'ConnectTimeoutError') - - -# The object genuinely is not there. Retrying cannot help. -_FETCH_TERMINAL = ('Key does not exist', '(404)', 'NoSuchKey') - - -# Lines between the anchor and the give-up line. Small, so a wider window -# cannot credit an earlier fetch failure the range recovered from. -_ANCHOR_WINDOW = 6 - - -# Lines back from the anchor to find the aws stderr that explains it. Wider, -# because concurrent downloads interleave with it during the bucket phase. -_CAUSE_WINDOW = 25 - - -# Tail of the archive to read. Catchup failed is always near the end, and a -# bucket-phase archive can be very large. -_TAIL_LINES = 400 - - -def _archive_tail(end, attempt): - """Last _TAIL_LINES lines of an attempt's archive, or [] if unreadable.""" - path = records.log_path(end, attempt) - tail = [] - try: - with gzip.open(path, 'rt', errors='replace') as fh: - for line in fh: - tail.append(line) - if len(tail) > _TAIL_LINES: - del tail[0] - except FileNotFoundError: - return [] - except (EOFError, gzip.BadGzipFile, zlib.error) as e: - logger.warning("could not read archive %s: %s", path, e) - return [] - except OSError as e: - logger.warning("could not open archive %s: %s", path, e) - return [] - return tail - - -def exit3_retry_cause(end, attempt): - """Why an exit-3 attempt is retryable, or None to condemn it. - - Conservative on purpose: only a fetch fault this function can name earns a - retry. An archive it cannot read, a give-up with no fetch cascade in front of - it, or an aws error it does not recognise all condemn the range -- the - archive survives on the volume, so an unrecognised cause can be read off a - failed run and added here rather than guessed at now. - """ - tail = _archive_tail(end, attempt) - if not tail: - return None - gave_up = max((i for i, l in enumerate(tail) if _FETCH_GAVE_UP in l), - default=None) - if gave_up is None: - return None - anchor = max((i for i in range(max(0, gave_up - _ANCHOR_WINDOW), gave_up) - if _FETCH_ANCHOR in tail[i]), default=None) - if anchor is None: - return None - window = tail[max(0, anchor - _CAUSE_WINDOW):anchor + 1] - for line in reversed(window): - for mark in _FETCH_TERMINAL: - if mark in line: - return None - for mark in _FETCH_TRANSIENT: - if mark in line: - return mark - return None - - -def tx_apply_for_range(end, attempt=1): - """Exact known 'ledger.transaction.apply' seconds for the whole range. - - Summed across the resumed chain, not read from the winning attempt alone. - medida's total is per-process, so a pod that resumes at LCL+1 reports only - the transactions it replayed -- on a range that was interrupted mid-replay - that is the tail, not the range. - - Slightly over-counts: replay restarts at the checkpoint boundary containing - LCL, so up to 64 ledgers can be applied twice. Against a 16320-ledger range - that is <=0.4%, but it is a fixed ledger cost rather than a percentage, so - it grows as ranges shrink. - """ - total = None - for n in _resumed_chain(end, attempt): - leg = _tx_apply_for_attempt(end, n) - if leg is None: - # A disrupted process often never prints its final medida block. - # Absence says the chain is incomplete; a partial sum under-reports. - return None - total = leg if total is None else total + leg - return total - - -def seconds_for_range(end, attempt=1, final=None): - """Compute time for the whole range, summed across the resumed chain. - - `final` is the winning attempt's own duration, which reconcile has in hand - from the pod. Earlier legs come from their .outcome, written when the - monitor classified the failure and still had the pod. - - This is compute, not elapsed: scheduling, image pull, node startup and gaps - between attempts are not in it (see wallSeconds for total scheduling and k8s - noise time). - """ - total = None - for n in _resumed_chain(end, attempt): - if n == int(attempt) and final is not None: - leg = final - else: - leg = _attempt_seconds(end, n) - if leg is None: - return None - total = leg if total is None else total + leg - return total - - -def _attempt_seconds(end, attempt): - """Best durable duration for one attempt, or None when it was never saved.""" - # .outcome carries the pod's own terminated timestamps, and is absent - # whenever the pod was reaped before classification -- every spot eviction -- - # so fall back to the collector's estimate. - leg = (attempt_files.read_outcome(end, attempt) or {}).get('attemptSeconds') - if leg is not None: - return leg - try: - with open(records.metrics_path(end, attempt)) as fh: - data = json.load(fh) - # A poller clock starts when the collector attached, so after a restart - # it is a lower bound and must not pass as chain compute. A clock dated - # from the container's own startTime is accepted: it measures the - # container, and it is the only duration a disrupted attempt produces. - if (data.get('attemptSecondsExact') is False - and data.get('attemptSecondsFromContainerStart') is not True): - return None - return data.get('attemptSeconds') - except (OSError, ValueError): - return None - - -def reconstruct_completed_profile(end, attempt): - """Recompute recoverable profile fields from immutable attempt artifacts. - - Durations and tx-apply totals follow only the continuous resumed chain, so a - fresh retry never double-counts discarded work. Peaks use that chain plus - every attempt that hit a resource ceiling. Missing duration or tx-apply legs - make that aggregate absent rather than publishing a lower bound as a total. - Complete tx-apply legs retain the existing <=64-ledger overlap. - - Reconstructable: persisted sampled peaks, complete attemptSeconds chains in - .outcome/.metrics, and complete txApplySeconds chains in .metrics/.log.gz. - Not reconstructable: whole-chain wall time, samples never persisted, or a - duration/tx-apply leg whose process and archive are both gone. - """ - rebuilt = peaks_for_range(end, attempt) - seconds = seconds_for_range(end, attempt) - if seconds is not None: - rebuilt['seconds'] = seconds - tx_apply = tx_apply_for_range(end, attempt) - if tx_apply is not None: - rebuilt['txApply'] = tx_apply - return rebuilt - - -def _apply_profile_reconstruction(record, rebuilt): - """Merge reconstruction without lowering stronger persisted evidence.""" - updates = {} - for key, value in rebuilt.items(): - current = record.get(key) - if current is None or value > current: - updates[key] = value - record.update(updates) - return updates - - -def _repair_completed_profile(end, attempt, record): - """Merge exact reconstruction and remove unverifiable chain aggregates.""" - rebuilt = reconstruct_completed_profile(end, attempt) - updates = _apply_profile_reconstruction(record, rebuilt) - if len(list(_resumed_chain(end, attempt))) > 1: - for key in ('seconds', 'txApply'): - if key not in rebuilt and record.get(key) is not None: - # Once resume proves this is a chain, the sum of surviving legs - # is a lower bound rather than a total, so omit it. - record.pop(key) - updates[key] = None - return updates - - -def _tx_apply_for_attempt(end, attempt=1): - """Exact 'ledger.transaction.apply' seconds for ONE attempt, or None. - - Read from the collector's record and nowhere else. The collector parses the - block out of the live stream and re-reads its own archive at finalization - when it has no total, so a second reader here could only repeat that work - over the same bytes with the same medida window -- which is how the monitor - came to carry its own copy of the parser. - """ - try: - with open(records.metrics_path(end, attempt)) as fh: - value = json.load(fh).get('txApplySeconds') - except (OSError, ValueError): - return None - return None if value is None else float(value) diff --git a/src/MissionParallelCatchup/lib/monitor/cluster.py b/src/MissionParallelCatchup/lib/monitor/cluster.py new file mode 100644 index 00000000..2d4333d9 --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/cluster.py @@ -0,0 +1,178 @@ +"""Everything that talks to the apiserver, on the aiohttp-backed aio client. + +kubernetes>=36 ships kubernetes.aio: the REST layer is genuinely async and the +generated methods return the coroutine, so `await core_v1.list_namespaced_pod()` +is real non-blocking I/O and nothing here needs a thread. + +Two traps in that client: + + * the generated docstrings are stale boilerplate from the sync generator -- + they claim "makes a synchronous HTTP request" and advertise async_req=True. + async_req routes through a ThreadPool and returns an AsyncResult, not an + awaitable, so it must never be used here. + * load_incluster_config is sync while load_kube_config is a coroutine. + +The rest of the monitor takes Jobs and pods as plain data, so a fake cluster in +a test replaces this module and no decision function needs a client at all. +""" +import asyncio +import contextlib +import logging + +from kubernetes.aio import client, config as kube_config +from kubernetes.aio.client import ApiException + +import config as cfg + +logger = logging.getLogger('job_monitor') + +batch_v1 = None +core_v1 = None +_api = None +_slots = None +_owner = None + + +@contextlib.asynccontextmanager +async def session(): + """Own the ApiClient for the life of the process. + + One client, one connection pool. Creating them per call would open a new + aiohttp session per request and leak connectors on every pass. + """ + global batch_v1, core_v1, _api, _slots + try: + kube_config.load_incluster_config() + except Exception: + await kube_config.load_kube_config() + _slots = asyncio.Semaphore(cfg.APISERVER_CONCURRENCY) + async with client.ApiClient() as api: + _api, batch_v1, core_v1 = api, client.BatchV1Api(api), client.CoreV1Api(api) + try: + yield + finally: + batch_v1 = core_v1 = _api = None + + +def _selector(): + return f"{cfg.LABEL_RUN}={cfg.RUN_NAME}" + + +async def snapshot(): + """This run's Jobs by range, and its pods by owning Job. + + Both lists in flight together, and one of each per pass: a range's state + must not be assembled from two different moments. + """ + jobs_raw, pods_raw = await asyncio.gather( + batch_v1.list_namespaced_job(cfg.NAMESPACE, label_selector=_selector()), + core_v1.list_namespaced_pod(cfg.NAMESPACE, label_selector=_selector())) + + jobs = {} + for job in jobs_raw.items: + end = (job.metadata.labels or {}).get(cfg.LABEL_RANGE) + if end is not None: + jobs.setdefault(str(end), []).append(job) + + pods = {} + for pod in pods_raw.items: + owner = next((o.name for o in (pod.metadata.owner_references or []) + if o.kind == 'Job'), None) + if owner is not None: + pods[owner] = pod + return jobs, pods + + +async def owner_ref(): + """The run's ConfigMap, so deleting the release collects everything. + + Read once and cached: it is the same object for the life of the run. + """ + global _owner + if _owner is None: + cm = await core_v1.read_namespaced_config_map( + f"{cfg.RUN_NAME}-stellar-core-config", cfg.NAMESPACE) + _owner = [client.V1OwnerReference( + api_version='v1', kind='ConfigMap', name=cm.metadata.name, + uid=cm.metadata.uid, block_owner_deletion=True)] + return _owner + + +async def create_job(body): + """Create, treating AlreadyExists as success. + + The Job name carries range and attempt, so name uniqueness IS the mutex: a + 409 proves another pass already dispatched this attempt, which is the + outcome that was wanted. + """ + async with _slots: + try: + return await batch_v1.create_namespaced_job(cfg.NAMESPACE, body) + except ApiException as e: + if e.status != 409: + raise + return None + + +async def ensure_pvc(end, owner): + name = f"{cfg.RUN_NAME}-data-r{end}" + async with _slots: + try: + await core_v1.read_namespaced_persistent_volume_claim(name, cfg.NAMESPACE) + return name + except ApiException as e: + if e.status != 404: + raise + spec = client.V1PersistentVolumeClaimSpec( + access_modes=['ReadWriteOnce'], + resources=client.V1VolumeResourceRequirements( + requests={'storage': cfg.STORAGE_SIZE})) + if cfg.STORAGE_CLASS: + spec.storage_class_name = cfg.STORAGE_CLASS + try: + await core_v1.create_namespaced_persistent_volume_claim( + cfg.NAMESPACE, client.V1PersistentVolumeClaim( + metadata=client.V1ObjectMeta( + name=name, owner_references=owner, + labels={cfg.LABEL_RUN: cfg.RUN_NAME, + cfg.LABEL_RANGE: str(end)}), + spec=spec)) + except ApiException as e: + if e.status != 409: + raise + return name + + +async def reap(end, job_names): + """Delete every Job this range has, then release its volume. + + Every Job, not just the winner: an earlier failed attempt left its own, and + leaving it behind holds a PVC and shows up in the next pass's list. + """ + await asyncio.gather(*(delete_job(name) for name in job_names)) + if cfg.STORAGE_MODE == 'pvc': + await _release_pvc(end) + + +async def delete_job(name): + async with _slots: + try: + await batch_v1.delete_namespaced_job(name, cfg.NAMESPACE, + propagation_policy='Background') + return True + except ApiException as e: + if e.status != 404: + logger.warning("could not delete job %s: %s", name, e) + return False + + +async def _release_pvc(end): + async with _slots: + try: + await core_v1.delete_namespaced_persistent_volume_claim( + f"{cfg.RUN_NAME}-data-r{end}", cfg.NAMESPACE) + return True + except ApiException as e: + if e.status != 404: + logger.warning("could not release pvc for range %s: %s", end, e) + return False diff --git a/src/MissionParallelCatchup/lib/monitor/dispatch.py b/src/MissionParallelCatchup/lib/monitor/dispatch.py new file mode 100644 index 00000000..a216f037 --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/dispatch.py @@ -0,0 +1,240 @@ +"""The range list, and the Job that runs one attempt of it.""" +import logging + +from kubernetes.aio import client + +import cluster +import config +import sizing + +logger = logging.getLogger('job_monitor') + +# Resume replays from LCL+1 on a PVC. LCL comes from core's own accessor: v27 +# dropped ledgerheaders, and the log fallback goes blind above INFO. +RESUME_SCRIPT = r'''set -e +KEY="%(key)s" +TARGET=%(target)d +COUNT=%(count)d +MARK=/data/.job-key +RESUME=false +LCL="" +if [ -f "$MARK" ] && [ "$(cat "$MARK" 2>/dev/null)" = "$KEY" ]; then + LCL=$(/usr/bin/stellar-core --conf /config/stellar-core.cfg offline-info --console 2>/dev/null \ + | sed -n 's/.*"num"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/p' | head -1 || true) + if [ -z "$LCL" ]; then + PREV_LOG=$(ls -t /data/stellar-core*.log 2>/dev/null | head -n 1 || true) + if [ -n "$PREV_LOG" ]; then + LCL=$(grep -oE "Ledger close complete: [0-9]+" "$PREV_LOG" 2>/dev/null | tail -1 | grep -oE "[0-9]+$" || true) + fi + fi + echo "RESUME PROBE: lcl '${LCL:-none}'" + # Re-running catchup on a finished range applies nothing and exits 2 + # identically every time, so it would burn the whole budget on completed work. + if [ -n "$LCL" ] && [ "$LCL" -ge "$TARGET" ] 2>/dev/null; then + echo "ALREADY COMPLETE: $KEY reached $LCL >= target $TARGET" + exit 0 + fi + if [ -n "$LCL" ] && [ "$LCL" -ge $((TARGET - COUNT)) ] && [ "$LCL" -lt "$TARGET" ] 2>/dev/null; then + RESUME=true; echo "RESUME: $KEY reached $LCL; skipping new-db" + else + echo "RESUME DECLINED: $KEY last close '${LCL:-none}'; bucket phase incomplete" + fi +fi +printf '%%s' "$KEY" > "$MARK" +if [ "$RESUME" != "true" ]; then + /usr/bin/stellar-core --conf /config/stellar-core.cfg new-db --console +fi +exec /usr/bin/stellar-core --conf /config/stellar-core.cfg catchup "$KEY" \ + --metric 'ledger.transaction.apply' --console +''' + + +def range_list(): + """(end, count) for every range this run owes, in dispatch order. + + A pure function of the /start spec. A restart must reproduce it exactly: a + different list means work silently duplicated or skipped, and nothing else + would notice. + """ + start, latest = config.STARTING_LEDGER, config.LATEST_LEDGER_NUM + per_job, overlap = config.LEDGERS_PER_JOB, config.OVERLAP_LEDGERS + + # Strictly greater, and overlap added on top of the clamped stride: `>=` + # emits a range ending AT the start ledger, which is below genesis and + # exits 2, and clamping the total would drop the overlap at that end. + ranges, end = [], latest + while end > start: + stride = min(end - start, per_job) + ranges.append((end, stride + overlap)) + end -= stride + + if config.RANGE_ORDER == 'oldest-first': + # The cheap ranges finish first, which is what a profiling run wants: it + # measures the inexpensive end before anything can interrupt it. + return list(reversed(ranges)) + if config.RANGE_ORDER == 'longest-first': + # Validated at /start, so a profile exists here. + return sorted(ranges, key=_measured_seconds, reverse=True) + # tip-first: the bucket set only grows with ledger position, so the tip + # ranges are the slowest and the most worth starting early. + return ranges + + +def _measured_seconds(item): + """Sort key for longest-first. An unmeasured range sorts FIRST. + + profile_for returns the nearest measured end ABOVE, so a range with no + seconds is newer than anything ever measured -- and cost rises with ledger + position. Unknown means assume worst, not assume average. It also runs + early under the most generous sizing, which is what makes the next profile + cover it instead of it being the range a run dies before reaching. + """ + seconds = (sizing.profile_for(item[0]) or {}).get('seconds') + return (1, 0) if seconds is None else (0, seconds) + + +def job_name(end, attempt): + return f"{config.RUN_NAME}-r{end}-a{attempt}" + + +def job_key(end, count): + return f"{end}/{count}" + + +async def create(end, count, attempt, oom_count=0, memory=None, ephemeral=None): + """Create one attempt's Job, idempotent by name.""" + owner = await cluster.owner_ref() + volume = await _data_volume(end, owner) + body = _job(end, count, attempt, owner, volume, oom_count, memory, ephemeral) + return await cluster.create_job(body) + + +async def _data_volume(end, owner): + if config.STORAGE_MODE == 'pvc': + name = await cluster.ensure_pvc(end, owner) + return client.V1Volume(name='data', persistent_volume_claim=( + client.V1PersistentVolumeClaimVolumeSource(claim_name=name))) + return client.V1Volume(name='data', empty_dir=client.V1EmptyDirVolumeSource()) + + +def _job(end, count, attempt, owner, data_volume, oom_count, memory, ephemeral): + labels = {config.LABEL_RUN: config.RUN_NAME, + config.LABEL_RANGE: str(end), + config.LABEL_ATTEMPT: str(attempt)} + return client.V1Job( + metadata=client.V1ObjectMeta(name=job_name(end, attempt), + owner_references=owner, labels=labels), + spec=client.V1JobSpec( + # Retries are the monitor's, not the controller's: raising a memory + # request needs a new Job, because spec.template is immutable. + backoff_limit=0, + active_deadline_seconds=config.ATTEMPT_DEADLINE_SECONDS or None, + ttl_seconds_after_finished=config.JOB_TTL_SECONDS, + pod_failure_policy=client.V1PodFailurePolicy(rules=_failure_rules()), + template=client.V1PodTemplateSpec( + # LABEL_ATTEMPT has to be on the POD too: the collector reads it + # to pick which range--a.* files the attempt owns. + metadata=client.V1ObjectMeta(labels=labels), + spec=_pod(end, count, attempt, data_volume, oom_count, + memory, ephemeral)))) + + +def _failure_rules(): + """Rules in evaluation order, which IS the contract with classify-from-Job. + + All FailJob so the Job fails with reason=PodFailurePolicy and the message + names the rule index; a Count action surfaces as BackoffLimitExceeded and + loses the signal entirely. + """ + return [ + client.V1PodFailurePolicyRule( + action='FailJob', + on_pod_conditions=[client.V1PodFailurePolicyOnPodConditionsPattern( + type='DisruptionTarget', status='True')]), + client.V1PodFailurePolicyRule( + action='FailJob', + on_exit_codes=client.V1PodFailurePolicyOnExitCodesRequirement( + container_name='stellar-core', operator='In', values=[137])), + client.V1PodFailurePolicyRule( + action='FailJob', + on_exit_codes=client.V1PodFailurePolicyOnExitCodesRequirement( + container_name='stellar-core', operator='NotIn', values=[0])), + ] + + +def _pod(end, count, attempt, data_volume, oom_count, memory, ephemeral): + requests, limits = sizing.requests_for(end, oom_count, memory, ephemeral) + script = RESUME_SCRIPT % {'key': job_key(end, count), 'target': end, 'count': count} + container = client.V1Container( + name='stellar-core', image=config.CORE_IMAGE, + command=['/bin/sh', '-c', script], + env=([client.V1EnvVar(name='ASAN_OPTIONS', value=config.ASAN_OPTIONS)] + if config.ASAN_OPTIONS else []), + resources=client.V1ResourceRequirements(requests=requests, limits=limits), + ports=[client.V1ContainerPort(container_port=11626, name='http')], + lifecycle=_prestop(), + volume_mounts=[client.V1VolumeMount(name='data', mount_path='/data'), + client.V1VolumeMount(name='config', mount_path='/config')]) + return client.V1PodSpec( + # IRSA for the S3 history mirror; without it workers fall back to the + # public archive, which throttles at 1024. + service_account_name=config.WORKER_SERVICE_ACCOUNT or None, + # Never restarted in place: the pod stays terminal and inspectable. + restart_policy='Never', + termination_grace_period_seconds=config.WORKER_GRACE_SECONDS, + affinity=_affinity(end, oom_count), + tolerations=([client.V1Toleration(key=config.TOLERATE_TAINT, effect='NoSchedule')] + if config.TOLERATE_TAINT else None), + containers=[container], + volumes=[data_volume, client.V1Volume( + name='config', config_map=client.V1ConfigMapVolumeSource( + name=f"{config.RUN_NAME}-stellar-core-config"))]) + + +def _affinity(end, oom_count): + """Require and avoid in ONE matchExpressions list. + + Expressions within a term are ANDed and separate terms are ORed, so an + avoid-only pod in its own term would match every node. + """ + match = [] + if config.NODE_LABEL_KEY: + match.append(client.V1NodeSelectorRequirement( + key=config.NODE_LABEL_KEY, operator='In', + values=[sizing.node_label_value(end, oom_count)])) + for key, value in config.label_pairs(config.REQUIRE_NODE_LABELS): + # Literal, unlike the pool-routed pair above: properties of the pool + # rather than of the range, so they do not vary per attempt. + match.append(client.V1NodeSelectorRequirement( + key=key, operator='In', values=[value])) + if config.AVOID_NODE_LABEL_KEY: + # No value means "avoid the label however it is set", which is + # DoesNotExist; NotIn [""] would only exclude the empty value. + match.append(client.V1NodeSelectorRequirement( + key=config.AVOID_NODE_LABEL_KEY, + operator='NotIn' if config.AVOID_NODE_LABEL_VALUE else 'DoesNotExist', + values=[config.AVOID_NODE_LABEL_VALUE] if config.AVOID_NODE_LABEL_VALUE else None)) + if not match: + return None + return client.V1Affinity(node_affinity=client.V1NodeAffinity( + required_during_scheduling_ignored_during_execution=client.V1NodeSelector( + node_selector_terms=[client.V1NodeSelectorTerm(match_expressions=match)]))) + + +def _prestop(): + """A preStop that stalls the kubelet, or None. + + Refuses to install one that cannot finish inside the grace period: the + kubelet kills it mid-sleep, reports FailedPreStopHook, and signals the + container anyway -- so the delay is not bought and an error is logged for + every evicted pod. + """ + sleep = config.WORKER_PRESTOP_SLEEP_SECONDS + if sleep <= 0: + return None + if sleep >= config.WORKER_GRACE_SECONDS: + logger.warning("preStop %ss does not fit in grace %ss; not installing it", + sleep, config.WORKER_GRACE_SECONDS) + return None + return client.V1Lifecycle(pre_stop=client.V1LifecycleHandler( + _exec=client.V1ExecAction(command=['/bin/sleep', str(sleep)]))) diff --git a/src/MissionParallelCatchup/lib/monitor/http_server.py b/src/MissionParallelCatchup/lib/monitor/http_server.py deleted file mode 100644 index 2e5f0ad4..00000000 --- a/src/MissionParallelCatchup/lib/monitor/http_server.py +++ /dev/null @@ -1,175 +0,0 @@ -"""The monitor's HTTP surface. - -Everything the mission driver needs, so it never reads cluster state to run the -mission: the profile goes in through /start, status comes out of /status, and -the logs are pulled per file. The alternative for the logs was `kubectl exec`, -which proxies every byte through the API server -- measured 0.3 MB per range, so -~1.2 GB of control-plane traffic on a 4000-range run, for bytes that have no -business there. - -/healthz and /prometheus predate this and keep their consumers: the kubelet's -livenessProbe, and the `kubernetes-pods` scrape job that relabels -prometheus.io/path onto __metrics_path__ and so reaches the non-standard path. -""" - -import json -import logging -import os -import re -import threading -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - -from prometheus_client import CONTENT_TYPE_LATEST, REGISTRY, generate_latest - -import config -# Not build_logger: this module is imported BY job_monitor, so configuring here -# would run first and logging.basicConfig is a no-op once root has handlers -- -# job_monitor's own call was then silently discarded, and the FileHandler it -# built still created job_monitor_.log, which stayed empty in every run's -# artifacts while this module's file held the whole process's output. The -# entrypoint configures; this just takes a logger. -logger = logging.getLogger('http_server') - -# Set by job_monitor before serve(). A tuple rather than an import, because -# job_monitor imports this module. -status_source = None # () -> (dict, lock) -started = threading.Event() # /start has delivered a profile -on_start = None # (doc) -> None, installs the profile - -# One path element, no traversal, no dotfiles. -_SAFE_NAME = re.compile(r'^[A-Za-z0-9][A-Za-z0-9._-]*$') - - -def _log_path(name): - """An existing regular file in LOG_DIR named by `name`, or None.""" - if not _SAFE_NAME.match(name or ''): - return None - path = os.path.join(config.LOG_DIR, name) - return path if os.path.isfile(path) else None - - -class RequestHandler(BaseHTTPRequestHandler): - protocol_version = 'HTTP/1.1' - - def _send(self, code, body=b'', ctype='application/json'): - self.send_response(code) - self.send_header('Content-type', ctype) - self.send_header('Content-Length', str(len(body))) - self.end_headers() - if body: - self.wfile.write(body) - - def do_GET(self): - if self.path == '/healthz': - # Serving at all is the whole check: a process that answers here - # still has its HTTP thread. - self._send(200, b'ok', 'text/plain') - elif self.path == '/prometheus': - self._send(200, generate_latest(REGISTRY), CONTENT_TYPE_LATEST) - elif self.path == '/status': - snapshot, lock = status_source() - with lock: - doc = dict(snapshot) - # Until the first reconcile pass lands, the counts are placeholders - # -- zeros that read exactly like a run with nothing done yet. This - # says which it is, so a caller can tell "no work recorded" from - # "not dispatching at all" instead of polling a monitor that never - # will. - doc['started'] = started.is_set() - self._send(200, json.dumps(doc, separators=(',', ':')).encode()) - elif self.path == '/logs': - self._send(200, json.dumps(self._manifest(), separators=(',', ':')).encode()) - elif self.path.startswith('/logs/'): - self._send_file(self.path[len('/logs/'):]) - else: - self._send(404) - - def do_POST(self): - if self.path != '/start': - self._send(404) - return - try: - raw = self.rfile.read(int(self.headers.get('Content-Length') or 0)) - doc = json.loads(raw) if raw else {} - except ValueError as e: - self._send(400, json.dumps({'error': f'invalid profile json: {e}'}).encode()) - return - # Idempotent: a driver that retries after a timeout must not restart a - # run that is already dispatching. - if not started.is_set(): - try: - on_start(doc) - except ValueError as e: - # A run the monitor cannot proceed with. Answering 400 fails the - # driver here, with the reason, rather than leaving it to poll a - # monitor that will never dispatch. on_start opens the gate - # itself on success. - self._send(400, json.dumps({'error': str(e)}).encode()) - return - self._send(200, b'{"started":true}') - - def _manifest(self): - """Every artifact worth pulling, with the size and mtime a puller needs - to tell "already have it" from "grew since last time". - - .state is excluded: it is the collector's resume cursor, one timestamp - rewritten on every poll of a live range. It is meaningless once the pods - are gone, and because it changes constantly a manifest diff would - re-fetch one per in-flight range on every pass -- up to 1024 round trips - for bytes that are garbage by the time the run ends. - """ - out = [] - for name in os.listdir(config.LOG_DIR): - path = os.path.join(config.LOG_DIR, name) - if (_SAFE_NAME.match(name) and not name.endswith('.state') - and os.path.isfile(path)): - st = os.stat(path) - out.append({'name': name, 'size': st.st_size, 'mtime': int(st.st_mtime)}) - return out - - def _send_file(self, name): - """One artifact, honouring Range so a cut transfer resumes instead of - restarting. The collector appends to these while a pod runs, so the - length is fixed once at open and never read past.""" - path = _log_path(name) - if not path: - self._send(404) - return - with open(path, 'rb') as fh: - size = os.fstat(fh.fileno()).st_size - start, end = 0, size - 1 - m = re.match(r'bytes=(\d+)-(\d*)', self.headers.get('Range') or '') - partial = bool(m) - if partial: - start = int(m.group(1)) - end = int(m.group(2)) if m.group(2) else size - 1 - if start >= size: - self.send_response(416) - self.send_header('Content-Range', f'bytes */{size}') - self.end_headers() - return - length = end - start + 1 - self.send_response(206 if partial else 200) - self.send_header('Content-type', 'application/octet-stream') - self.send_header('Content-Length', str(length)) - if partial: - self.send_header('Content-Range', f'bytes {start}-{end}/{size}') - self.end_headers() - fh.seek(start) - remaining = length - while remaining > 0: - chunk = fh.read(min(1 << 20, remaining)) - if not chunk: - break - self.wfile.write(chunk) - remaining -= len(chunk) - - def log_message(self, *args): - pass # the default handler logs every request to stderr - - -def serve(port=8080): - # Threading, because a log pull is long-lived and must not block the - # liveness probe or the driver's status poll behind it. - logger.info('Starting httpd server on :%d', port) - ThreadingHTTPServer(('', port), RequestHandler).serve_forever() diff --git a/src/MissionParallelCatchup/lib/monitor/kube.py b/src/MissionParallelCatchup/lib/monitor/kube.py deleted file mode 100644 index 9c32b85a..00000000 --- a/src/MissionParallelCatchup/lib/monitor/kube.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Kubernetes API clients. - -Read through the module, never copied out of it: - - import kube - ... kube.core_v1.list_namespaced_pod(...) ... - -`from kube import core_v1` binds a COPY, and the tests replace these attributes -with a fake cluster -- a copy taken at import time keeps talking to the real -apiserver, silently. -""" -import os - -from kubernetes import client, config as kube_config - -import monitor_config as mc - -# The env var is exactly what load_incluster_config() itself keys on, so in a pod -# this is the unconditional call it always was -- a missing token or CA still -# raises here and crash-loops the container rather than running blind. Outside a -# pod there is nothing to load and import stays pure; the tests replace the -# clients below. -IN_CLUSTER = bool(os.getenv('KUBERNETES_SERVICE_HOST')) -if IN_CLUSTER: - kube_config.load_incluster_config() - -# client-go's Python equivalent defaults are fine for a few LISTs per cycle, but -# dispatching ~1024 Jobs + PVCs at once needs headroom. -_cfg = client.Configuration.get_default_copy() -_cfg.connection_pool_maxsize = mc.CONNECTION_POOL -client.Configuration.set_default(_cfg) - -core_v1 = client.CoreV1Api() -batch_v1 = client.BatchV1Api() diff --git a/src/MissionParallelCatchup/lib/monitor/liveness.py b/src/MissionParallelCatchup/lib/monitor/liveness.py new file mode 100644 index 00000000..01239d33 --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/liveness.py @@ -0,0 +1,89 @@ +"""Whether the workers' stellar-core is answering, one sweep per pass. + +The reconcile loop already holds the authoritative pod list, so this probes that +rather than listing anything itself. No state is carried between sweeps -- no +hysteresis, no smoothing, no scheduler. These numbers feed a dashboard and +nothing else reads them, so a stale-free snapshot beats a smoothed one. +""" +import asyncio +import logging + +import aiohttp + +import config + +logger = logging.getLogger('job_monitor') + +_ADMIN_PORT = 11626 + +EMPTY = {'up': 0, 'down': 0, 'unknown': 0} + + +def targets(pods): + """Running-with-an-IP pods, keyed by pod UID. + + A UID change is a replacement even when the Job name or the IP is reused. + """ + out = {} + for pod in pods: + status, meta = getattr(pod, 'status', None), getattr(pod, 'metadata', None) + ip = getattr(status, 'pod_ip', None) + if getattr(status, 'phase', None) != 'Running' or not ip or meta is None: + continue + name = getattr(meta, 'name', None) + identity = getattr(meta, 'uid', None) or name + if identity and name: + out[str(identity)] = (str(name), str(ip)) + return out + + +async def sweep(targets): + """{'up','down','unknown'} for the whole fleet, bounded by one deadline. + + Deliberately not a TaskGroup: a TaskGroup cancels its siblings when one task + raises, which is the opposite of what a fleet sweep wants -- one unreachable + pod must not discard the other 1023 answers. asyncio.wait keeps whatever + finished and cancels only the stragglers, so an unreachable fleet costs one + deadline rather than the sum of its timeouts. + """ + if not targets: + return dict(EMPTY) + counts = {'up': 0, 'down': 0, 'unknown': len(targets)} + # force_close: a pooled socket to a vanished pod gets handed back out. + # `limit` is the concurrency bound; a semaphore would double-enforce it. + connector = aiohttp.TCPConnector(limit=config.LIVENESS_MAX_CONCURRENCY, + force_close=True) + timeout = aiohttp.ClientTimeout(total=config.LIVENESS_PROBE_TIMEOUT_SECONDS) + async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: + tasks = [asyncio.create_task(_probe(session, ip)) for _, ip in targets.values()] + done, pending = await asyncio.wait(tasks, timeout=config.LIVENESS_SWEEP_SECONDS) + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + for task in done: + try: + up = task.result() + except Exception: + up = False # timeout, refused, DNS, malformed response + counts['up' if up else 'down'] += 1 + counts['unknown'] -= 1 + return counts + + +async def _probe(session, ip): + host = f"[{ip}]" if ':' in ip else ip + async with session.get(f"http://{host}:{_ADMIN_PORT}/info") as resp: + return resp.status == 200 + + +async def publish(pods): + """One sweep, never fatal. Liveness is observability and may not stop a run.""" + found = targets(pods) + if not found: + return dict(EMPTY) + try: + return await sweep(found) + except Exception as e: + logger.warning("liveness sweep failed (%s); reporting all workers unknown", e) + return {'up': 0, 'down': 0, 'unknown': len(found)} diff --git a/src/MissionParallelCatchup/lib/monitor/metrics.py b/src/MissionParallelCatchup/lib/monitor/metrics.py index 5302a5a2..c0d1af76 100644 --- a/src/MissionParallelCatchup/lib/monitor/metrics.py +++ b/src/MissionParallelCatchup/lib/monitor/metrics.py @@ -1,47 +1,125 @@ -"""The run's Prometheus metrics. +"""The run's Prometheus metrics, and the only correct way to move them. -Declaration only -- prometheus_client builds the default REGISTRY at import and -the monitor serves it from /prometheus, so there is nothing to instantiate here. -Names carry no metric_ prefix: they are read as metrics. at the call site. +Counters and histograms are monotonic and reset to zero when the process +restarts, while the record on the volume survives. So the totals live in the +record, updated when a fact becomes final, and each pass pushes the delta -- +idempotent, and self-healing after a restart without rescanning anything. """ from prometheus_client import Counter, Gauge, Histogram +import config -# Histogram buckets -# 5m 15m 30m 1h 1.5h 2h -buckets = (300, 900, 1800, 3600, 5400, 7200, float("inf")) -catchup_queues = Gauge('ssc_parallel_catchup_queues', 'Exposes size of each job queues', ["queue"]) -workers = Gauge('ssc_parallel_catchup_workers', 'Exposes catch up worker status', ["status"]) -refresh_duration = Gauge('ssc_parallel_catchup_workers_refresh_duration_seconds', 'Time it took to refresh status of all workers') -full_duration = Histogram('ssc_parallel_catchup_job_full_duration_seconds', 'Compute seconds across the complete resumed attempt chain', buckets=buckets) -tx_apply_duration = Histogram('ssc_parallel_catchup_job_tx_apply_duration_seconds', 'Exposes job TX apply duration as histogram', buckets=buckets) -# wallSeconds is Kubernetes's startTime -> completionTime for the winning Job -# only. Failed-attempt timestamps and inter-attempt gaps were never persisted, so -# it cannot be reconstructed as first dispatch -> success after those Jobs go. +# 5m 15m 30m 1h 1.5h 2h +BUCKETS = (300, 900, 1800, 3600, 5400, 7200, float('inf')) + +queues = Gauge('ssc_parallel_catchup_queues', 'Size of each job queue', ['queue']) +workers = Gauge('ssc_parallel_catchup_workers', 'Worker liveness', ['status']) +sweep_duration = Gauge('ssc_parallel_catchup_workers_refresh_duration_seconds', + 'Seconds the last liveness sweep took') +mission_duration = Gauge('ssc_parallel_catchup_mission_duration_seconds', + 'Seconds since the mission started') + +full_duration = Histogram('ssc_parallel_catchup_job_full_duration_seconds', + 'Compute seconds across the resumed attempt chain', + buckets=BUCKETS) wall_duration = Histogram('ssc_parallel_catchup_job_wall_duration_seconds', - 'Winning Kubernetes Job start to completion', - buckets=buckets) -mission_duration = Gauge('ssc_parallel_catchup_mission_duration_seconds', 'Number of seconds since the mission started ') -retries = Counter( - 'ssc_parallel_catchup_job_retried_count', - 'Retry attempts dispatched after a predecessor attempt failed') + "Attempt 1's dispatch to the winning attempt's completion", + buckets=BUCKETS) +tx_apply_duration = Histogram('ssc_parallel_catchup_job_tx_apply_duration_seconds', + 'Transaction-apply seconds per range', buckets=BUCKETS) + +retries = Counter('ssc_parallel_catchup_job_retried_count', + 'Retry attempts dispatched after a predecessor failed') +oom_retries = Counter('ssc_parallel_catchup_job_oom_retried_count', + 'Retries dispatched after an OOM, with an escalated request') +eph_retries = Counter('ssc_parallel_catchup_job_ephemeral_retried_count', + 'Retries dispatched after an ephemeral eviction') # Separates infrastructure churn from application failure: many evictions with # zero app failures is spot behaving as intended. -evictions = Counter( - 'ssc_parallel_catchup_job_spot_eviction_count', - 'Pod attempts classified as lost to node disruption') -spot_disruption_retried = Counter( - 'ssc_parallel_catchup_job_spot_disruption_retried_count', - 'Unique ledger ranges that dispatched a successor after a node disruption verdict') -pvc_released = Counter('ssc_parallel_catchup_pvc_released_count', 'PVCs deleted after their range completed') -jobs_reaped = Counter('ssc_parallel_catchup_jobs_reaped_count', 'Finished Jobs deleted after their record was durable') -oom_retries = Counter( - 'ssc_parallel_catchup_job_oom_retried_count', - 'Retry attempts dispatched after an OOM verdict, with an escalated memory limit') -eph_retries = Counter( - 'ssc_parallel_catchup_job_ephemeral_retried_count', - 'Retry attempts dispatched after an ephemeral-storage verdict, with an escalated limit') -retry_reasons = Counter( - 'ssc_parallel_catchup_job_retried_reason_count', - 'Retry attempts dispatched, by the effective verdict of the predecessor attempt', - ['reason']) +evictions = Counter('ssc_parallel_catchup_job_spot_eviction_count', + 'Attempts classified as lost to node disruption') +disruption_retried = Counter('ssc_parallel_catchup_job_spot_disruption_retried_count', + 'Ranges that dispatched a successor after a disruption') +retry_reasons = Counter('ssc_parallel_catchup_job_retried_reason_count', + 'Attempts by the verdict that ended them', ['reason']) + +_COUNTERS = (('retries', retries), ('oom', oom_retries), ('ephemeral', eph_retries), + ('evicted', evictions), ('disruption_retried', disruption_retried)) + + +def settled(progress, cause, end=None): + """One attempt's verdict became final: a retry followed it, or it condemned + the range. `end` is passed only for the retry, which is what makes the + disruption count per range rather than per eviction.""" + counters = progress['counters'] + counters[f'reason:{cause}'] += 1 + if cause == 'disrupted': + counters['evicted'] += 1 + if end is None: + return + counters['retries'] += 1 + if cause in ('oom', 'ephemeral'): + counters[cause] += 1 + if cause == 'disrupted': + progress['disruptedRanges'].add(end) + counters['disruption_retried'] = len(progress['disruptedRanges']) + + +def sync_counters(progress, applied): + """Walk each counter up to the record's total. + + Delta from a total rather than .inc() at the event: the counters reset with + the process and the record does not, so a restart walks them back up. + """ + counters = progress['counters'] + keys = [k for k, _ in _COUNTERS] + [f'reason:{r}' for r in config.ATTEMPT_OUTCOMES] + for key in keys: + delta = counters[key] - applied.get(key, 0) + if delta <= 0: + continue + if key.startswith('reason:'): + retry_reasons.labels(reason=key[7:]).inc(delta) + else: + dict(_COUNTERS)[key].inc(delta) + applied[key] = counters[key] + + +def observe_completed(progress, replayed): + """Feed recorded completions into the histograms, once each per process. + + Keyed on (range, FIELD), never the range alone: a range is usually recorded + before the collector has flushed its .metrics, so txApply is absent at first + sight and backfilled a pass or two later. Guarding per range means that + backfill can never be observed and the histogram permanently disagrees with + the record. + """ + for end, rec in progress.get('completed', {}).items(): + for field, metric in (('seconds', full_duration), + ('wallSeconds', wall_duration), + ('txApply', tx_apply_duration)): + key = (end, field) + if key in replayed: + continue + value = rec.get(field) + # Presence, not truth: a txApply sum of 0 is a real observation, and + # so is a sub-second duration. + if value is None: + continue + replayed.add(key) + metric.observe(value) + + +# The dashboard's panels query this name and these label values. The internal +# vocabulary has no queue in it, but renaming a published series only moves the +# break to a consumer that lives in another repo and cannot be tested here. +_QUEUE = {'remaining': 'remain', 'running': 'in_progress', + 'completed': 'succeeded', 'condemned': 'failed'} + + +def publish_gauges(counts, liveness, sweep_seconds, mission_seconds): + for state, value in counts.items(): + queues.labels(queue=_QUEUE[state]).set(value) + for status, value in liveness.items(): + workers.labels(status=status).set(value) + sweep_duration.set(sweep_seconds) + mission_duration.set(mission_seconds) diff --git a/src/MissionParallelCatchup/lib/monitor/monitor_config.py b/src/MissionParallelCatchup/lib/monitor/monitor_config.py deleted file mode 100644 index d6c92726..00000000 --- a/src/MissionParallelCatchup/lib/monitor/monitor_config.py +++ /dev/null @@ -1,539 +0,0 @@ -"""Settings and run state for the parallel catchup job monitor. - -Everything here belongs to the monitor process alone. What both processes must -agree on -- the run's identity, the shared volume, the verdict vocabulary -- -lives in config, which this reads through for LOG_DIR. - -Read through the module, never copied out of it: - - import monitor_config as mc - ... mc.REQ_CPU ... - -`from monitor_config import REQ_CPU` binds a COPY. A test's monkeypatch and the -startup assignment of mc.PROFILE both rebind the attribute on this module, and a -copy taken at import time never sees either -- silently, with the test passing -against the default. -""" -import os - -import config - -# ============================================================================= -# 1. stellar-core workload -# ============================================================================= -CORE_IMAGE = os.getenv('CORE_IMAGE') - -ASAN_OPTIONS = os.getenv('ASAN_OPTIONS', '') - -# Which ledger ranges to run. These are pure inputs to the range generator: -# dispatch recomputes the whole list every reconcile, so a restart must -# reproduce it exactly. - - -# Both generators emit tip-first, which front-loads the most expensive ranges: -# the bucket set only grows with ledger position. 'oldest-first' reverses that, -# so a profiling run measures the cheap early ranges before it can be -# interrupted, and the expensive tip ranges last. -RANGE_ORDER = 'tip-first' # from /start: tip-first | oldest-first | longest-first - -VALID_RANGE_ORDERS = ('tip-first', 'oldest-first', 'longest-first') - -STARTING_LEDGER = 0 # from /start - -LATEST_LEDGER_NUM = 0 # from /start - -LEDGERS_PER_JOB = 16000 # from /start - -OVERLAP_LEDGERS = 320 # from /start - - -# ============================================================================= -# 2. Kubernetes objects this monitor creates -# ============================================================================= - - - - - - -# Workers need IRSA to read the S3 history mirror. Without it they silently fall -# back to the public archive, which throttles at 1024 and kills the run with -# curl 22 -> catchup exit 3. The name matches the old StatefulSet's so existing -# IRSA trust policies keep matching. -WORKER_SERVICE_ACCOUNT = os.getenv('WORKER_SERVICE_ACCOUNT', '') - -# Pod resources. Requests only: workers are given no cpu limit and no memory -# limit at all. -# -# CPU because a limit only throttles a pod that could otherwise use idle cores, -# and throttling changes what the range measures -- less cpu means less download -# concurrency means a lower peak, so a throttled attempt records a figure an -# unthrottled one cannot reproduce. -# -# Memory because a limit is a hard cap on anon PLUS page cache, and sizing it -# per-range from a profile got it wrong in the one direction that has no alarm -# on it. Measured 2026-07-31, range 39210943: sized at 1729Mi from a neighbour, -# genuinely needed 1620Mi of anon, which left ~110Mi for cache. It never OOMed -# -- it thrashed. 544k major page faults, 0.22 cores used on a node it had -# entirely to itself, 0.95 ledgers/s against a neighbour norm of 3.3, and it -# held 1092 idle slots open for three hours at the end of the run. -# -# Without a limit the request still does the real work: it places the pod and -# it sets eviction order under node pressure. What goes away is the cliff. -REQ_CPU = os.getenv('REQ_CPU', '1250m') - -REQ_MEM = os.getenv('REQ_MEM', '9Gi') - -# The run document /start delivers, kept so a restart resumes the same run. -RUN_PATH = '' - -PROFILE_MARGIN = float(os.getenv('PROFILE_MARGIN', 1.15)) - -# No safety margin on cpu, unlike memory. Under-requesting cpu costs contention -# and the pod can still burst; under-requesting memory gets it OOMKilled. -# Ceiling for profile-derived memory, above the unprofiled limit for the same -# reason: a range that really needs more than the configured limit must be able -# to ask for it rather than be pinned under its own measured peak. The OOM -# escalation ladder can still climb past this on a retry. -PROFILE_MAX_MEM = os.getenv('PROFILE_MAX_MEM', '32Gi') - -# Memory is sized from rss (the range's real demand), NOT from peak working -# set. Working set is whatever limit it was measured under -- the kernel grows -# page cache to fill it -- so sizing from it is circular. Measured on ssc-test -# with one 420-ledger range: working set went 2.33 -> 3.61 -> 7.48 -> 13.49 GiB -# under 2560Mi/4Gi/8Gi/24000Mi limits while rss moved only 2256 -> 2488 MiB, and -# wall-clock did not move at all (776s / 775s / 746s / 773s). Catchup streams -- -# buckets are downloaded once, applied once, ledgers replayed once -- so cache -# has nothing to give back and PROFILE_MARGIN alone is the allowance. -# A multiplicative margin alone is not enough: memory.max bounds anon PLUS page -# cache, and at small rss 10% is nothing. Measured on ssc-test 2026-07-29 with -# headroom 0: ranges profiled at 190 MiB rss got a 209 MiB limit -- 19 MiB of -# slack for all growth and cache -- and 90 of them OOMKilled within 90s. The -# earlier 4Gi validation hid this because 1.1x of 2.4 GiB is 240 MiB of slack. -PROFILE_CACHE_HEADROOM = os.getenv('PROFILE_CACHE_HEADROOM', '512Mi') - -# Extra allowance scaled by the range's measured runtime. Long ranges keep more -# page cache and allocator slack live at once; 0 disables the allowance. -PROFILE_RUNTIME_MEMORY_INSURANCE = os.getenv('PROFILE_RUNTIME_MEMORY_INSURANCE', '3Gi') - -# Ephemeral-storage gets the same two allowances as memory, for the same -# reasons. Measured on the 2026-08-01 on-demand run: peak 37.76Gi against a -# flat 40Gi limit -- 6% of headroom on a path that has never once fired in a -# real run, so a range 6% worse than the worst seen would be evicted 137 with -# no diagnostic pointing at disk. -# -# Flat allowance added to every range's measured peak. Covers the container -# image, logs and the sqlite WAL, none of which scale with the range. -PROFILE_EPHEMERAL_HEADROOM = os.getenv('PROFILE_EPHEMERAL_HEADROOM', '2Gi') - -# Runtime-weighted allowance on top. Disk tracks runtime closely (pearson 0.920 -# across 3985 ranges: runtime decile 0 uses 0.1Gi, decile 9 uses 24.7Gi), so -# the ranges that need the margin are exactly the ranges this gives it to. -PROFILE_RUNTIME_EPHEMERAL_INSURANCE = os.getenv('PROFILE_RUNTIME_EPHEMERAL_INSURANCE', '8Gi') - -# Ceiling for profile-derived disk. Deliberately ABOVE LIM_EPHEMERAL: that flat -# limit is what an UNMEASURED range gets, and capping a measured range at it -# would throw away the measurement -- the worst observed range wants 43Gi after -# margin alone. -PROFILE_MAX_EPHEMERAL = os.getenv('PROFILE_MAX_EPHEMERAL', '64Gi') - -REQ_EPHEMERAL = os.getenv('REQ_EPHEMERAL', '') - -LIM_EPHEMERAL = os.getenv('LIM_EPHEMERAL', '') - -# Placement. The taint toleration is emitted as {key, effect} with no value: -# the default Equal operator does not match "" against "true". -NODE_LABEL_KEY = os.getenv('NODE_LABEL_KEY', '') - -NODE_LABEL_VALUE = os.getenv('NODE_LABEL_VALUE', '') - -# Further labels a node must carry, "key:value" comma separated, ANDed with the -# one above. Unlike that one these are literal -- the pair above is pool-routed, -# its value replaced per range with -. -# -# This is where a run pins itself to one capacity of a tier. Both capacities -# carry the same tier label value, so nothing else separates them, and the -# pairing matters: ephemeral has no resume, so a reclaim costs the whole range. -# A plain label rather than karpenter.sh/capacity-type, because the pools -# publish their own and the monitor has no business knowing who provisioned the -# node. -REQUIRE_NODE_LABELS = os.getenv('REQUIRE_NODE_LABELS', '') - - -def label_pairs(raw): - """[(key, value)] from "k:v,k:v". Entries without a value are dropped: a - key alone would require the label be exactly "", which no node carries, and - a pod pinned to nothing sits Pending in a way that reads as slow - provisioning rather than as misconfiguration.""" - out = [] - for item in (raw or '').split(','): - key, _, value = item.strip().partition(':') - if key and value: - out.append((key, value)) - return out - -AVOID_NODE_LABEL_KEY = os.getenv('AVOID_NODE_LABEL_KEY', '') - -AVOID_NODE_LABEL_VALUE = os.getenv('AVOID_NODE_LABEL_VALUE', '') - -TOLERATE_TAINT = os.getenv('TOLERATE_TAINT', '') - - -STORAGE_CLASS = os.getenv('STORAGE_CLASS', '') - -# 60Gi to match the tier nodes' ephemeral allowance. peakEphemeralBytes tops -# out at 37.8Gi across the whole 2026-08-01 profile, so this covers every -# range measured, with headroom for the tip to keep growing. -STORAGE_SIZE = os.getenv('STORAGE_SIZE', '60Gi') - -# Job/pod lifetimes. -# SIGTERM -> SIGKILL budget. stellar-core exits ~7s after SIGTERM (measured), so -# this is slack rather than a target. -WORKER_GRACE_SECONDS = int(os.getenv('GRACE_SECONDS', 100)) - -# Seconds to stall inside preStop before the container is signalled. 0 disables. -# -# Sized to cover the collector's DETECTION LAG, which is the specific hole it -# fills. The collector notices DisruptionTarget on its pod-list cycle and only -# then drops that pod to 1s polling; if SIGTERM lands inside that blind window -# the poller is still on its lazy LOG_POLL_SECONDS cadence. Measured on -# ssc-test: a 60s preStop with 10s polling and no disruption detection still -# lost txApply, while 1s polling with no preStop at all captured it. So this is -# not what saves the metric -- it is what makes sure the detection has happened -# before the kill. -# -# 20s, not COLLECTOR_POLL_SECONDS. That constant is the SLEEP between cycles, -# not the cycle: each one also lists every pod and sweeps kubelet -# /stats/summary on every node, which at 768 workers over ~250 nodes is -# unmeasured and plausibly another 5-15s. The margin is -# (preStop + pod-object linger) - (detection + one 1s poll), and with the -# linger measured at 7.8s it goes NEGATIVE at a 12s cycle if this is 5s. Above -# the true cycle time the margin plateaus at +6.8s, so overshooting is free -# while undershooting silently loses the metric. -# -# A spot reclaim gives ~120s of notice and does not need this at all; an -# eviction-API kill or a fast drain signals immediately and does. -# -# Do NOT try to SIGTERM the process from inside the hook and hold the pod open -# afterwards: measured, the pod object survived 10.2s that way versus 69s for a -# plain sleep, because a container dies with its PID 1 and the kubelet does not -# defer deleting the object until the hook returns. -# -# Costs nothing on a healthy exit -- preStop does not run when the container -# exits on its own, only when the kubelet is tearing it down. At ~810 evictions -# a run, 5s each is about 1.1 pod-hours. -# -# Must stay comfortably under WORKER_GRACE_SECONDS: the hook and the SIGTERM -# drain share that one budget, and a hook still running when it expires is -# SIGKILLed, which loses exactly the output this exists to save. -WORKER_PRESTOP_SLEEP_SECONDS = int(os.getenv('PRESTOP_SLEEP_SECONDS', 5)) - -# Must comfortably exceed any plausible monitor outage: completion is recorded -# to the ConfigMap by this process, and a Job reclaimed before that happens -# reads as "never ran" and gets redone. -# Backstop only. reconcile() deletes each Job explicitly once its record is -# durable, so the TTL exists for the cases that skip that path: a terminally -# failed range kept for inspection, or a success whose metrics never landed. -JOB_TTL_SECONDS = int(os.getenv('JOB_TTL_SECONDS', 600)) - -# Measured on ssc-test: stellar-core does NOT fail on an unreachable history -# archive, an absent ledger range, or a bucket that will not decompress. It -# retries every mirror with growing backoff and stays Running indefinitely -- -# no exit code, no failure, the slot held for the life of the run. A hang is a -# more likely real failure than a non-zero exit, and this deadline is the only -# thing that makes it observable. 0 disables. -# -# Flat, deliberately -- NOT scaled by the range's profiled runtime. That was -# tried and removed. A deadline has to bound a range's WORST case, but a profile -# only offers a neighbour's TYPICAL case, and the two are far apart here: -# runtimes span 190x (p25 771s, max 5.9h), range keys are anchored to the -# network tip so a profile from an earlier run matches ZERO keys exactly and -# every lookup lands on a neighbour, and ~2% of those neighbours are 3-38x -# cheaper than their surroundings. Backtested honestly across that grid offset -# (run4 profile -> r5 actuals, 3983 ranges): a 2x factor falsely kills 134 -# ranges, 4x kills 46, 6x kills 21. Flat 12h kills none. -# -# The asymmetry decides it. A false kill loses a range, and a timeout is -# terminal, so it fails the mission. A genuine wedge holds ONE slot out of -# 1092-1500 for 12h -- around 0.1% of a run's capacity. Never trade a certain -# catastrophe against a rounding error. -# -# 12h is a safe bound, not a good detector: it takes half a day to catch -# something provably dead in 4 minutes. The right signal is ledger-close -# progress, not elapsed time -- a wedged core closes zero ledgers while still -# logging, so `.state` (last log line) cannot see it and a new -# lastLedgerCloseAt would. Left undone on purpose; it needs a threshold above -# the initial bucket-apply phase, which legitimately closes nothing for ~20min -# on the longest ranges. -ATTEMPT_DEADLINE_SECONDS = int(os.getenv('ATTEMPT_DEADLINE_SECONDS', 0)) - -# kube-state-metrics turns a pod's `mission` label into label_mission, which the -# Grafana container panels join on. Every other mission gets it from -# StellarKubeSpecs; this chart never has, so parallel catchup has never appeared -# in those panels. -# -# OFF by default and deliberately so: those panels are sum() by (pod, container) -# with a legend table, so at 1024 workers they would pull ~1024 series into any -# view with mission=$__all selected, degrading a shared dashboard for people who -# did not ask for it. Enable per-run once the panels aggregate (topk). -MISSION = os.getenv('MISSION', '') - -EMIT_MISSION_LABEL = os.getenv('EMIT_MISSION_LABEL', 'false').lower() == 'true' - -# ============================================================================= -# 3. This monitor's own behaviour -# ============================================================================= -PARALLELISM = int(os.getenv('PARALLELISM', 3)) - -# Effectively the OOM budget: `failed` is the only other outcome that reaches -# it, and that one sets no retry reason. Escalation counts OOMs rather than -# attempts, so rung N means the range genuinely wanted more N times. -# -# Deliberately stops short of MEM_ESCALATION_CAP: 5 rungs is 1.5^4 = 5x the -# profile figure, and a range needing more than that is not mis-sized, it is -# broken -- chasing it to 48Gi parks a whole r8a.2xlarge on one range for hours. -# The cost of stopping is that the range is condemned, and today a condemned -# range aborts the run. That coupling is the thing to fix, not this number. -# Attempts each failure cause gets before the range is condemned. The whole -# retry policy, in one table. -# -# Every budget is spent by ITS OWN cause: an OOM never consumes the disk budget -# and a spot eviction never consumes either. A cause that is NOT here is -# condemned the first time it happens -- a timeout, a real catchup failure, an -# exit 3 with nothing in its archive, and anything nothing could classify. -# -# disrupted the cluster took the pod away mid-run, which proves the range -# itself was fine. Effectively unlimited: on spot a healthy range -# is legitimately evicted dozens of times, and 100 is far past any -# rate a real run has produced while still terminating. -# rejected the kubelet refused the pod before any container ran (attachment -# limits, admission churn). The range never started, so a retry -# cannot mask anything about it. -# fetch-fault an exit 3 whose archive named a failed history fetch. An -# unreachable mirror is the cluster's problem, not the range's. A -# plain `failed` has no entry: a real catchup failure, and an exit 3 -# with nothing in its archive, are both condemned on sight. -# oom each retry escalates the memory request one rung. -# ephemeral each retry escalates the disk limit one rung. Smallest, because -# an eviction repeats identically until the range gets more disk. -# -# The MAX_* names below exist so the chart can tune each one; ATTEMPT_BUDGETS is -# what the code reads, so tests patch the map rather than the constants. -MAX_DISRUPTION_ATTEMPTS = int(os.getenv('MAX_DISRUPTION_ATTEMPTS', 100)) -MAX_REJECTED_ATTEMPTS = int(os.getenv('MAX_REJECTED_ATTEMPTS', 100)) -MAX_FETCH_FAULT_ATTEMPTS = int(os.getenv('MAX_FETCH_FAULT_ATTEMPTS', 20)) -MAX_OOM_ATTEMPTS = int(os.getenv('MAX_OOM_ATTEMPTS', 5)) -MAX_EPHEMERAL_ATTEMPTS = int(os.getenv('MAX_EPHEMERAL_ATTEMPTS', 4)) - -ATTEMPT_BUDGETS = { - 'disrupted': MAX_DISRUPTION_ATTEMPTS, - 'rejected': MAX_REJECTED_ATTEMPTS, - 'fetch-fault': MAX_FETCH_FAULT_ATTEMPTS, - 'oom': MAX_OOM_ATTEMPTS, - 'ephemeral': MAX_EPHEMERAL_ATTEMPTS, -} - -EPH_BUMP_FACTOR = float(os.getenv('EPH_BUMP_FACTOR', 1.5)) - -EPH_ESCALATION_CAP = os.getenv('EPH_ESCALATION_CAP', '200Gi') - - -# Verdicts only the pod can produce, and which a Job-level DeadlineExceeded must -# never overwrite. Each names a specific mechanism -- the kubelet OOM-killed it, -# the node was draining, the ephemeral limit blew -- and each earns a different -# retry budget and a different remediation. "The Job ran too long" is also true -# of every one of them and says nothing about which. An OOM downgraded to a -# timeout retries at the same memory limit that just killed it and gets 2 -# attempts instead of 5; a spot eviction downgraded to a timeout gets 2 instead -# of 20. -POD_AUTHORITATIVE_OUTCOMES = ('oom', 'disrupted', 'ephemeral', 'timeout') - -# stellar-core's "did not complete". Ambiguous by construction: a corrupt bucket -# and a SIGTERM during replay both produce it, so it must never be treated as -# proof that a range is broken. -CATCHUP_INCOMPLETE_EXIT = 3 - -# An OOM means requests/limits are mis-sized for this range. Escalate so the run -# can finish, but say so loudly -- surviving by escalating at runtime is a -# configuration bug, not a success. -MEM_BUMP_FACTOR = float(os.getenv('MEM_BUMP_FACTOR', 1.5)) - -# Ceiling for that escalation. Above the largest schedulable node the retry sits -# Pending forever, which looks like a hang rather than a failure. -MEM_ESCALATION_CAP = os.getenv('MAX_MEM', '48Gi') - -# Reconcile loop: dispatch, refresh status, publish metrics. The env var is -# named LOGGING_INTERVAL_SECONDS for historical reasons, from when this loop -# only logged. -RECONCILE_INTERVAL_SECONDS = int(os.getenv('LOGGING_INTERVAL_SECONDS', 10)) - -# Worker responsiveness is cosmetic and sampled independently from reconcile. -# Thirty seconds and three failures restore the old ~90-second down threshold, -# while a five-second request budget gives a busy admin endpoint substantially -# more room than the old one-shot two-second probe. - -LIVENESS_PROBE_TIMEOUT_SECONDS = os.getenv('LIVENESS_PROBE_TIMEOUT_SECONDS', '5') - - -LIVENESS_MAX_CONCURRENCY = os.getenv('LIVENESS_MAX_CONCURRENCY', '32') -# Wall-clock bound on one sweep. The reconcile loop waits for it, so this -# is the most a fleet of unreachable workers can delay dispatch. -LIVENESS_SWEEP_SECONDS = os.getenv('LIVENESS_SWEEP_SECONDS', '15') - - - -# The authoritative copy of the progress record lives on the logs PVC, not in -# the ConfigMap. A ConfigMap is capped at 1 MiB and this record is ~172 bytes -# per completed range, so it dies at ~6100 ranges -- reachable simply by halving -# ledgersPerJob. Measured mid-run on ssc-test: 348KB at 2024 completed ranges, -# which projects to ~65% of the cap at 3982 -- close enough that the next -# slicing change would have hit it. Worse, every completion rewrote the whole -# document through the API server, so a full run meant thousands of -# escalating-size etcd writes. -# -# The ConfigMap is still written, because the mission driver reads it without -# exec'ing into the pod, but it is now a best-effort mirror: if it fails, the -# run carries on from the file. -PROGRESS_FILE = os.path.join(config.LOG_DIR, 'progress.json') - -PROFILE = None - -# --- pool tiers ------------------------------------------------------------- -# -# A range picks a NODEPOOL by its measured memory, and gets that pool's node to -# itself. This replaces the cpu ladder, which tuned a dimension that turned out -# not to be the binding one. -# -# Why memory and not cpu. Measured 2026-08-03 on one range across four instance -# shapes, isolated, no memory limit: -# -# 2 -> 4 cores replay +2.8% bucket-apply 1.37x -# 4 -> 8 cores replay +1.5% bucket-apply 1.18x -# AMD vs Intel replay +16% bucket-apply 1.35x -# -# Replay is ~93% of a job and is flat in core count from 2 upward -- it draws -# ~1.05 cores whatever it is given. So a cpu REQUEST never bought throughput. -# What it bought was neighbours-per-node, and memory is what actually fails: a -# range whose working set does not fit gets OOMKilled, not slowed down. -# -# Cuts are `node_usable / 1.60`, covering the p99 of run-to-run growth in the -# same range's peakAnonBytes (18,073 observations across five profiles: p50 0.97, -# p90 1.28, p99 1.60, max 2.83). Validated the hard way: range 63080767 measured -# 13.75Gi was placed on nodes with 14.1/14.3Gi allocatable -- a 1.03x margin -- -# and OOMKilled on BOTH during bucket-apply, before closing a ledger. -# subdwarf's cut is 0 on purpose: nothing can satisfy `gib < 0`, so the tier is -# defined and provisionable but never routed to. Kept rather than deleted so the -# bottom of the ladder is there to experiment with; c8a.medium (1.42Gi -# allocatable) cannot hold a range the profile actually contains. -POOL_TIERS = os.getenv( - 'POOL_TIERS', - '0:subdwarf,0.79:dwarf,1.61:subgiant,3.87:giant,8.85:supergiant,18.38:hypergiant,:supernova') - -# Prepended to the tier name to form the node label value, e.g. catchup-dwarf. -# Empty disables pool routing entirely and every worker keeps the single global -# NODE_LABEL_VALUE, which is exactly today's behaviour. -POOL_PREFIX = os.getenv('POOL_PREFIX', '') - -# Where a range goes when the profile has no entry for it (past the profile's -# top, i.e. the newest ledgers) and when there is no profile at all. -POOL_UNPROFILED = os.getenv('POOL_UNPROFILED', 'protostar') - -POOL_NO_PROFILE = os.getenv('POOL_NO_PROFILE', 'nebula') - -# cpu request per tier. NOT a demand estimate -- a claim token. Isolation is the -# point: freeing a node of its 3 neighbours raised throughput 29-92% while cpu -# draw FELL, so the contended resource is memory bandwidth and shared cache, not -# compute. Kept at or below the SMALLEST node in the tier so the low-weight -# fallback rungs stay schedulable (dwarf can land on a 1-vCPU c8a.medium). -# -# Memory, not cpu, is what actually enforces the isolation -- see _pool_memory. -# Half the node for most tiers. hypergiant and supernova are sized to the -# SMALLEST shape in their pool instead: x8i.large is r8a.xlarge with half the -# cores and the same 32 GiB, x8i.xlarge is r8a.2xlarge with half the cores and -# the same 64 GiB, so preferring them buys identical RAM for half the spot -# quota. A half-the-node 2.00/4.00 claim does not fit an x8i node once the 215m -# of daemonsets is counted, which is why those pools won no nodes at all on -# 2026-08-03. Below half, cpu no longer isolates the pod on the larger fallback -# shapes -- memory does, and it holds because every type within a tier carries -# the same RAM. -POOL_CPU = os.getenv( - 'POOL_CPU', - 'subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80') - - -# Rungs that never run, whatever the vCPU comparison says. Empty by default: with -# the spot pools doubled, promotion lands a range on a bigger SHARED node, and -# sharing is what the rung is really buying. Measured on ssc-test 2026-08-04 on -# one range, two pods to a node: on an 8-vCPU node a co-tenant cost 1.02x per pod -# (r8id.2xlarge, 3.78 and 3.91 lps), on a 4-vCPU node it cost 1.58x. Two pods on -# 8 cores leave 4 each, which the workload does not use; two on 4 cores leave 2, -# which is the floor. -# -# Caveat worth keeping in view: the bump fires on peakWorkingSetBytes, and working -# set does not predict throughput. Same x8i.xlarge box, same range, same time, -# only cgroup memory.max differing: 28 GiB ran 1.83 lps and 56 GiB ran 1.70. So -# this promotes for a reason that is not the reason it helps -- it reaches the -# right nodes via the wrong signal, and will promote ranges that gain nothing. -# Sizing the rung on peakAnonBytes, or widening the tier->instance map directly, -# would target those nodes deliberately. -# -# This is now the ONLY thing standing between a working set and a promotion, so -# a rung that should not be taken has to be named here -- nothing is inferred. -# -# hypergiant->supernova is denied on both capacity types. Its cost rose once the -# x8i pools were removed on 2026-08-04: supernova's only spot shapes are now -# 4xlarges, so the rung moves a range from 8 vCPU to 16 rather than the 8-vCPU -# x8i.2xlarge it used to reach. Simulated over the 2026-08-03 run it saved -# exactly 0 minutes on its own, because the longest job was a supergiant this -# rung cannot reach. It pays only in company -- supergiant->hypergiant alone is -# worth 8 min, this alone 0, the pair 27 -- and that pairing is not on offer -# while its cost is 8->16 vCPU. -# -# dwarf->subgiant is the same doubling at the bottom of the ladder, 2->4 vCPU on -# spot and 1->2 on on-demand. -POOL_BLOCK_RUNGS = os.getenv('POOL_BLOCK_RUNGS', 'dwarf->subgiant,hypergiant->supernova') - -# Memory request for a pooled range is the TIER'S CUT, not the range's own -# measurement, and that is deliberate two ways. -# -# It guarantees one pod per node without depending on the cpu token: a tier's -# node is cut*1.60 of usable memory, so two pods asking cut apiece need 2*cut, -# which always exceeds 1.60*cut. The cpu claim cannot do this alone because a -# tier spans node sizes (dwarf reaches a 1-vCPU c8a.medium and a 2-vCPU -# t3a.small), so no single cpu value both schedules on the small one and fills -# the large one. -# -# And the request no longer needs a safety margin. PROFILE_MARGIN, cache -# headroom and runtime insurance all existed to keep a pod under its own LIMIT; -# with no memory limit and the node to itself, a pod may use everything the node -# has. The margin moved into the node size -- which is where it can actually be -# enforced, since the kubelet kills on node pressure, not on request. -# Per-tier memory request: exactly 50% of the tier node's NAMEPLATE capacity. -# -# 50% is what isolates. Two pods asking half the nameplate need the whole node, -# which always exceeds allocatable -- so a second pod can never fit, on every -# tier, without depending on how the kubelet happens to reserve. -# -# Verified against measured nodes rather than assumed: a c8a.medium reports -# 1892Mi capacity, 1449Mi allocatable, and carries 154Mi of daemonsets, leaving -# 1295Mi -- so the 1024Mi request schedules with room, and 2048Mi of two pods -# cannot. The same holds up the ladder. -# -# t3a.micro is absent on purpose: 413Mi allocatable cannot host a pod at all on -# this cluster, so subdwarf shares dwarf's node type and is emptied by its cut. -POOL_MEM = os.getenv( - 'POOL_MEM', - 'subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:9216Mi') - -_SORTED_SECONDS = None - - -# Sized for the dispatch burst rather than a steady LIST rate: ~1024 Jobs + PVCs -# go out at once at the head of a wave. -CONNECTION_POOL = int(os.getenv('CONNECTION_POOL', '64')) - -# Left as strings on purpose. Coercing at import made a bad value a boot crash, -# and a process that cannot start cannot report why -- the driver just polled a -# pod that never answered and timed out 600s later with "not reachable". -# validate_config coerces and rebinds these when /start delivers the run, so a -# bad value comes back as a 400 carrying the reason. diff --git a/src/MissionParallelCatchup/lib/monitor/policy.py b/src/MissionParallelCatchup/lib/monitor/policy.py new file mode 100644 index 00000000..4b9899a7 --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/policy.py @@ -0,0 +1,45 @@ +"""Whether a failed attempt runs again, and with what. + +Two questions kept apart: + + may it retry? a per-CAUSE budget, spent against what actually killed it + with what? escalate the axis that ran out, once per cause + +A pure function of (verdict, this range's past verdicts), which is what makes +the retry ladder testable without a cluster. +""" +import collections + +import config +import sizing + +Decision = collections.namedtuple('Decision', 'action reason memory ephemeral') + +RETRY, CONDEMN, DEFER = 'retry', 'condemn', 'defer' +WAIT = Decision(DEFER, None, None, None) + + +def decide(end, verdict, spent, base_memory=None, base_ephemeral=None): + """What to do about a failed attempt. `spent` is this range's cause counts. + + A cause with no budget is condemned on sight -- that is how a genuine + catchup failure ends a run instead of burning 20 attempts proving the chain + is broken. + """ + cause = verdict.get('outcome') + cap = config.ATTEMPT_BUDGETS.get(cause) + if cap is None: + return Decision(CONDEMN, f"{cause} is not retryable", None, None) + + # This verdict is already counted, so the Nth failure of a cause is the one + # that exhausts a budget of N. Per cause: evictions cannot drain the budget + # OOMs are entitled to. + if spent.get(cause, 0) >= cap: + return Decision(CONDEMN, f"{cause} budget of {cap} exhausted", None, None) + + memory = ephemeral = None + if cause == 'oom': + memory = sizing.next_memory(end, base_memory, spent.get('oom', 0)) + elif cause == 'ephemeral': + ephemeral = sizing.next_ephemeral(base_ephemeral, spent.get('ephemeral', 0)) + return Decision(RETRY, cause, memory, ephemeral) diff --git a/src/MissionParallelCatchup/lib/monitor/profiles.py b/src/MissionParallelCatchup/lib/monitor/profiles.py deleted file mode 100644 index 320790f5..00000000 --- a/src/MissionParallelCatchup/lib/monitor/profiles.py +++ /dev/null @@ -1,37 +0,0 @@ -"""The measured profile of a previous run, and the lookup into it. - -Parsed once from the /start POST into mc.PROFILE and read through it thereafter, -so a test that patches the profile is seen here without reloading anything. -""" - -import bisect - -import monitor_config as mc - - -def load_profile_doc(doc): - """The sorted (end, record) list a parsed profile document yields. - - An unprofiled run POSTs {} and gets [] -- a profile is an optimisation, - never a prerequisite, so "no ranges" is a valid answer rather than an error. - """ - ranges = (doc or {}).get('ranges') or {} - return sorted((int(k), v) for k, v in ranges.items()) - - -def profile_for(end): - """Measurements to size this range from, or None to use the defaults. - - Exact end, else the nearest measured end ABOVE it. Cost rises with ledger - position -- the bucket set only grows -- so a lower neighbour under-reports, - and under-provisioning costs an eviction while over-provisioning only costs - packing. Past the top of the profile there is nothing safe to extrapolate - from, so fall back to the configured defaults. - """ - if not mc.PROFILE: - return None - end = int(end) - idx = bisect.bisect_left(mc.PROFILE, (end,)) - if idx < len(mc.PROFILE) and mc.PROFILE[idx][0] == end: - return mc.PROFILE[idx][1] - return mc.PROFILE[idx][1] if idx < len(mc.PROFILE) else None diff --git a/src/MissionParallelCatchup/lib/monitor/ranges.py b/src/MissionParallelCatchup/lib/monitor/ranges.py deleted file mode 100644 index 5da8360b..00000000 --- a/src/MissionParallelCatchup/lib/monitor/ranges.py +++ /dev/null @@ -1,83 +0,0 @@ -"""The ledger range list and the order it is dispatched in. - -A pure function of config: dispatch recomputes the whole list on every reconcile, -so a restarted monitor has to reproduce it exactly. Nothing here reads the -cluster or the volume. -""" - -import monitor_config as mc -import profiles - - -def _uniform_segment(start_ledger, end_ledger, seg_size): - """Ranges over (start_ledger, end_ledger], largest ledger first.""" - out = [] - el = end_ledger - while el > start_ledger: - ledgers_per_job = min(el - start_ledger, seg_size) - out.append((el, ledgers_per_job + mc.OVERLAP_LEDGERS)) - el -= ledgers_per_job - return out - - -def _longest_first(ranges): - """Sort on the profile's own measured seconds, longest job first. - - Makespan is bounded below by the single longest job, so every range that - starts after it is free and every hour it starts late is an hour on the end. - That is classic longest-processing-time scheduling. - - A range the profile has never seen sorts FIRST. profile_for returns the - nearest measured end ABOVE the target, so an unprofiled range is by - construction newer than anything ever measured -- the newest ranges are the - most expensive, so "unknown" means "assume worst", not "assume average". - That also makes the next profile better: those ranges run early, under the - most generous sizing, instead of being the ones a run dies before reaching. - - Requires a profile; validate_config() refuses the combination without one, - because every key would tie and the stable sort would leave dispatch in the - generator's tip-first order while looking configured. - """ - def cost(item): - prof = profiles.profile_for(item[0]) - secs = (prof or {}).get('seconds') - # None sorts first; ties keep tip-first order, which is the better guess - # among ranges the profile cannot separate. - return (0 if secs is None else 1, -(secs or 0)) - return sorted(ranges, key=cost) - - -def _ordered(ranges): - """Dispatch order. Generators emit tip-first; the other two re-order that. - - tip-first only approximates longest-first. Position predicts cost on average - and badly in the tail: measured 2026-07-30, ranges at 41-45M ran as long as - the tip (3.1h) on a third of the memory, and the 50-60M band is CHEAPER than - 40-50M. Sorting on measured seconds uses the real number instead of a proxy. - """ - # validate_config() rejects an unknown order at startup; the raise here is - # the backstop for a caller that skipped it, never the primary check. - if mc.RANGE_ORDER == 'tip-first': - return ranges - elif mc.RANGE_ORDER == 'oldest-first': - return list(reversed(ranges)) - elif mc.RANGE_ORDER == 'longest-first': - return _longest_first(ranges) - else: - raise ValueError("RANGE_ORDER must be one of %s, got %r" - % (', '.join(mc.VALID_RANGE_ORDERS), mc.RANGE_ORDER)) - - -def generate_ranges(): - """Uniform ranges over the whole window, in the configured dispatch order. - - A logarithmic generator lived here -- big chunks over cheap early history, - halving toward the tip, aiming for equal wall-time per job. longest-first - supersedes it: same goal, but ordered from what ranges actually measured - rather than from an assumption about where the expensive ledgers are. It - also became unreachable when the range moved into /start, which carries no - generator. Recover it from 8553e77 if the guess ever beats the measurement. - """ - return _ordered(_uniform_segment(mc.STARTING_LEDGER, - mc.LATEST_LEDGER_NUM, - mc.LEDGERS_PER_JOB)) diff --git a/src/MissionParallelCatchup/lib/monitor/record.py b/src/MissionParallelCatchup/lib/monitor/record.py new file mode 100644 index 00000000..654fef5d --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/record.py @@ -0,0 +1,199 @@ +"""The volume: the collector's files, the monitor's own, and the run record. + +Filenames are the entire cross-process contract. The collector writes while a +pod still exists and the monitor reads them back, so a disagreement about a name +is a measurement silently lost and nothing reports it. + +Every write goes through tmp+rename. Both sides write while the other reads, so +a torn .metrics or .outcome reads as corrupt and the measurement is gone. +""" +import collections +import json +import os +import time + +import config + +# --- the collector writes these --------------------------------------------- + + +def log_path(end, attempt): + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.log.gz") + + +def outcome_path(end, attempt): + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.outcome") + + +def metrics_path(end, attempt): + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.metrics") + + +def done_path(end, attempt): + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.done") + + +# --- the monitor writes these ----------------------------------------------- + + +def started_path(end): + """Per RANGE, not per attempt: wallSeconds spans the range's whole life, so + the only start that matters is the first one.""" + return os.path.join(config.LOG_DIR, f"range-{end}.started") + + +PROGRESS_PATH = os.path.join(config.LOG_DIR, 'progress.json') +RUN_PATH = os.path.join(config.LOG_DIR, 'run.json') +MISSION_START_PATH = os.path.join(config.LOG_DIR, 'mission_started') + + +def write_atomic(path, body): + tmp = path + '.tmp' + with open(tmp, 'wt') as fh: + fh.write(body) + os.replace(tmp, path) + + +def _read_json(path): + try: + with open(path) as fh: + return json.load(fh) + except (OSError, ValueError): + return None + + +# --- what an attempt left behind -------------------------------------------- + + +def is_done(end, attempt): + """The collector has finished with this attempt. + + The only licence to decide a failed attempt or to reap. Never inferred from + measurements being present -- an attempt may legitimately have none. + """ + return os.path.exists(done_path(end, attempt)) + + +def read_outcome(end, attempt): + return _read_json(outcome_path(end, attempt)) + + +def read_metrics(end, attempt): + return _read_json(metrics_path(end, attempt)) or {} + + +def record_start(end, created): + """Attempt 1's Job creationTimestamp, written once. + + Not status.startTime: the controller sets that asynchronously, so it is + absent from the create response, and the gap between the two is part of what + wallSeconds measures. Written at creation because attempt 1's Job is gone by + the first retry. + """ + path = started_path(end) + if created is None or os.path.exists(path): + return + try: + write_atomic(path, created.isoformat()) + except OSError: + pass + + +def started_at(end): + try: + with open(started_path(end)) as fh: + from datetime import datetime + return datetime.fromisoformat(fh.read().strip()) + except (OSError, ValueError): + return None + + +# --- the run record --------------------------------------------------------- + + +def load_progress(): + """The durable record, read back rather than rebuilt. + + It is what makes a reaped range stay completed instead of reading as + pending and being dispatched a second time -- and it carries the counter + totals, so a restart resumes them in one read instead of rescanning every + attempt on the volume. + """ + doc = _read_json(PROGRESS_PATH) or {} + return {'completed': doc.get('completed') or {}, + 'condemned': doc.get('condemned') or {}, + 'causes': doc.get('causes') or {}, + 'counters': collections.Counter(doc.get('counters') or {}), + 'disruptedRanges': set(doc.get('disruptedRanges') or ())} + + +def note_cause(progress, end, attempt, cause): + """Record why one attempt ended, once. Returns this range's cause counts. + + Keyed on the highest attempt already counted, so re-settling the same + attempt on a later pass cannot double-count it. Budgets then read a number + instead of walking every past attempt off the volume. + """ + rec = progress['causes'].setdefault(str(end), {}) + if rec.get('last', 0) < int(attempt): + rec['last'] = int(attempt) + rec[cause] = rec.get(cause, 0) + 1 + return rec + + +def save_progress(progress): + """One file holding every terminal fact. + + If it is lost or truncated every range reads as pending and the run + redispatches all of them, so nothing may write it by any other means. + """ + doc = dict(progress, + counters=dict(progress['counters']), + disruptedRanges=sorted(progress['disruptedRanges'])) + write_atomic(PROGRESS_PATH, json.dumps(doc, separators=(',', ':'))) + + +def save_run(doc): + write_atomic(RUN_PATH, json.dumps(doc, separators=(',', ':'))) + + +def load_run(): + return _read_json(RUN_PATH) + + +def mission_start(): + """When this run began, surviving a monitor restart.""" + try: + with open(MISSION_START_PATH) as fh: + return float(fh.read().strip()) + except (OSError, ValueError): + now = time.time() + try: + write_atomic(MISSION_START_PATH, repr(now)) + except OSError: + pass + return now + + +def manifest(): + """Every artifact the driver can pull, as [{name, size}]. + + A bare ARRAY of objects, both of which the driver depends on: it parses the + body as an array and reads .name and .size off each entry, using size to + skip what it already has. .tmp files are excluded -- a half-written file has + no size worth comparing. + """ + out = [] + try: + names = sorted(os.listdir(config.LOG_DIR)) + except OSError: + return out + for name in names: + if name.endswith('.tmp'): + continue + try: + out.append({'name': name, + 'size': os.path.getsize(os.path.join(config.LOG_DIR, name))}) + except OSError: + continue # vanished between listing and stat + return out diff --git a/src/MissionParallelCatchup/lib/monitor/server.py b/src/MissionParallelCatchup/lib/monitor/server.py new file mode 100644 index 00000000..9c6327a1 --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/server.py @@ -0,0 +1,101 @@ +"""The monitor's HTTP surface, on the same event loop as the reconcile pass. + +/start begin a run (the range spec and an optional profile) +/status the counts the driver decides on +/logs the volume, because nothing else can read it +/prometheus, /healthz +""" +import logging +import os + +from aiohttp import web +from prometheus_client import CONTENT_TYPE_LATEST, generate_latest + +import config +import record + +logger = logging.getLogger('job_monitor') + + +def build(state): + app = web.Application() + app['state'] = state + app.add_routes([ + web.post('/start', _start), + web.get('/status', _status), + web.get('/logs', _logs), + web.get('/logs/{name}', _log_file), + web.get('/prometheus', _prometheus), + web.get('/healthz', _healthz), + ]) + return app + + +async def serve(state, stop): + """Serve until `stop` is set, then drain. + + /status must answer for the whole life of the process, including while a + reconcile pass is in flight -- the driver reads it to decide whether the + mission is still alive. + """ + runner = web.AppRunner(build(state), access_log=None) + await runner.setup() + site = web.TCPSite(runner, '0.0.0.0', config.HTTP_PORT) + await site.start() + logger.info("listening on :%d", config.HTTP_PORT) + try: + await stop.wait() + finally: + await runner.cleanup() + + +async def _start(request): + """Apply the /start document. A ValueError is answered 400. + + Validated at the first moment the configuration is complete -- the profile + arrives with this POST -- so a misconfigured run is rejected here rather + than crash-looping a pod the driver can only time out on. + """ + state = request.app['state'] + try: + doc = await request.json() + except Exception: + raise web.HTTPBadRequest(text="body is not JSON") + try: + state.start(doc) + except ValueError as e: + logger.warning("rejected /start: %s", e) + raise web.HTTPBadRequest(text=str(e)) + # Only after it validates, and only here: resume() replays what is already + # on disk. + record.save_run(doc) + return web.json_response({'ranges': len(state.ranges), + 'profile': len(config.PROFILE)}) + + +async def _status(request): + return web.json_response(request.app['state'].status()) + + +async def _logs(request): + # A bare array, not an object: the driver parses the body as a JArray. + return web.json_response(record.manifest()) + + +async def _log_file(request): + name = request.match_info['name'] + # basename, so a traversal cannot reach off the volume. + path = os.path.join(config.LOG_DIR, os.path.basename(name)) + if not os.path.isfile(path): + raise web.HTTPNotFound(text=f"no such artifact: {name}") + return web.FileResponse(path) + + +async def _prometheus(request): + # Through headers: aiohttp rejects the charset that media type carries. + return web.Response(body=generate_latest(), + headers={'Content-Type': CONTENT_TYPE_LATEST}) + + +async def _healthz(request): + return web.Response(text='ok') diff --git a/src/MissionParallelCatchup/lib/monitor/sizing.py b/src/MissionParallelCatchup/lib/monitor/sizing.py index 801091b8..a18e94da 100644 --- a/src/MissionParallelCatchup/lib/monitor/sizing.py +++ b/src/MissionParallelCatchup/lib/monitor/sizing.py @@ -1,104 +1,135 @@ -"""What a worker pod asks for: nodepool tier, memory, ephemeral disk. +"""What a worker pod asks for: pool tier, memory, disk. -This is the layer tuned between runs. It reads the profile and the per-attempt -records, and it never touches the cluster -- what it returns is a request, not an -applied change. +The layer tuned between runs. Reads the profile and past verdicts, touches +nothing, and returns a request rather than applying one. Escalation counts CAUSES, not attempts: a spot reclaim, a disruption and a -timeout all produce a retry and none of them says the range needed a bigger node. +timeout all produce a retry and none of them says the range needed a bigger +node. """ -import logging +import bisect import math +import re -import monitor_config as mc -import profiles -import attempt_files -import units +import config -logger = logging.getLogger() +_QUANTITY = re.compile(r'^(?P\d+(?:\.\d+)?)(?P[EPTGMK]i?|m)?$') +_FACTOR = {'K': 10 ** 3, 'M': 10 ** 6, 'G': 10 ** 9, 'T': 10 ** 12, + 'P': 10 ** 15, 'E': 10 ** 18, + 'Ki': 2 ** 10, 'Mi': 2 ** 20, 'Gi': 2 ** 30, 'Ti': 2 ** 40, + 'Pi': 2 ** 50, 'Ei': 2 ** 60} -def mem_for_attempt(attempt, base=None, end=None): - """Memory REQUEST after N OOMs. +def quantity_bytes(value): + m = _QUANTITY.match(str(value or '').strip()) + if not m: + return 0 + n = float(m.group('n')) + unit = m.group('unit') + if unit == 'm': + return int(n / 1000) + return int(n * _FACTOR.get(unit, 1)) - Pooled: the tier ladder IS the ladder. Attempt N resolves to a tier N-1 - steps up, and the request is that tier's cut, so the request and the pool - move together. A multiplicative bump cannot do this -- tiers are ~2.2-2.5x - apart while MEM_BUMP_FACTOR is 1.5, so a bump lands BETWEEN tiers: too big - for the current pool's nodes, too small to have earned the next one, and the - pod sits Pending on a pool that can never satisfy it. - Unpooled: the original behaviour. `base` is what attempt 1 actually ran - with, which matters when a profile sized the range -- escalating a 209Mi - profiled range off the configured default jumps straight to 36000Mi, a 172x - overshoot that throws away the whole packing win on the first OOM. +def bytes_to_quantity(n): + for unit in ('Gi', 'Mi', 'Ki'): + factor = _FACTOR[unit] + if n >= factor and n % factor == 0: + return f"{n // factor}{unit}" + return str(int(n)) - Escalating the request, not a limit, because there is no limit any more. It - still buys the same two things an OOMing range needs -- placement somewhere - with the memory actually free, and a higher bar before the kubelet picks it - as an eviction victim. - """ - if mc.POOL_PREFIX: - promoted = pool_memory(pool_for(end, attempt)) - if promoted: - return promoted - # Above the ladder (nebula/protostar/supernova): nothing left to promote - # into, so hold at the configured request rather than inventing a value. - return base or mc.REQ_MEM - base_q = units.quantity_bytes(base or mc.REQ_MEM) - want = int(base_q * (mc.MEM_BUMP_FACTOR ** max(0, attempt - 1))) - cap = units.quantity_bytes(mc.MEM_ESCALATION_CAP) - return units.bytes_to_quantity(min(want, cap)) - - -def eph_for_attempt(attempt): - """Ephemeral-storage size for attempt N, escalating after an eviction. - - None when no limit is configured: a pod with no ephemeral-storage limit can - still be evicted under node disk pressure, and there is nothing to raise. - Every other reader of LIM_EPHEMERAL already guards on it being set. + +# --- the profile ------------------------------------------------------------ + + +def load_profile(doc): + """Sorted [(end, record)]. An unprofiled run POSTs {} and gets []: a profile + is an optimisation, never a prerequisite.""" + ranges = (doc or {}).get('ranges') or {} + return sorted((int(k), v) for k, v in ranges.items()) + + +def profile_for(end): + """Measurements to size this range from, or None. + + Exact end, else the nearest measured end ABOVE it: cost rises with ledger + position because the bucket set only grows, so a lower neighbour + under-reports. Past the top there is nothing safe to extrapolate from. """ - if not mc.LIM_EPHEMERAL: + if not config.PROFILE: return None - base_q = units.quantity_bytes(mc.LIM_EPHEMERAL) - want = int(base_q * (mc.EPH_BUMP_FACTOR ** max(0, attempt - 1))) - return units.bytes_to_quantity(min(want, units.quantity_bytes(mc.EPH_ESCALATION_CAP))) + idx = bisect.bisect_left(config.PROFILE, (int(end),)) + if idx < len(config.PROFILE): + return config.PROFILE[idx][1] + return None -def _rung_listed(raw, tier, nxt): - want = f"{tier}->{nxt}" - return any(item.strip() == want for item in raw.split(',')) +def _positive(value): + try: + n = float(value) + except (TypeError, ValueError): + return None + return n if math.isfinite(n) and n > 0 else None -def _rung_blocked(tier, nxt): - """Is this rung denied outright? Beats every other consideration.""" - return _rung_listed(mc.POOL_BLOCK_RUNGS, tier, nxt) +def _longest_seconds(): + if config._SORTED_SECONDS is None: + values = (_positive(rec.get('seconds')) for _, rec in (config.PROFILE or [])) + config._SORTED_SECONDS = sorted(v for v in values if v is not None) + return config._SORTED_SECONDS[-1] if config._SORTED_SECONDS else None -def _parsed_pool_tiers(): - """[(gib_cut, tier_name)] cheapest first, the last entry unbounded. +def _runtime_insurance(seconds, allowance): + """Runtime-weighted share of an allowance: the longest range gets all of it, + one half as long gets half, so it follows time-at-risk.""" + seconds = _positive(seconds) + longest = _longest_seconds() + total = quantity_bytes(allowance) + if seconds is None or longest is None or total <= 0: + return 0 + return int(total * (seconds / longest)) - An empty cut on the final entry means "everything above the previous one", - which is how supernova is expressed without inventing a ceiling. - """ + +# --- the pool ladder -------------------------------------------------------- + + +def _tiers(): + """[(gib_cut, name)] cheapest first; an empty cut on the last entry means + everything above the previous one.""" out = [] - for item in mc.POOL_TIERS.split(','): - item = item.strip() - if not item: - continue - cut, _, name = item.rpartition(':') - if not name: - continue - out.append((float(cut) if cut else float('inf'), name)) + for item in config.POOL_TIERS.split(','): + cut, _, name = item.strip().rpartition(':') + if name: + out.append((float(cut) if cut else float('inf'), name)) + return out + + +def _str_map(raw): + out = {} + for item in raw.split(','): + name, _, value = item.strip().partition(':') + if name and value: + out[name.strip()] = value.strip() return out -def _tier_for_bytes(anon_bytes): - """Tier whose node can hold this working set, or None if unsizable.""" - tiers = _parsed_pool_tiers() - if not tiers or not anon_bytes: +def pool_memory(tier): + """The tier's cut: sized to fit once in its on-demand node and twice in its + (one size larger) spot node. Not the range's own measurement -- isolation is + the point, and freeing a pod of its neighbours raised throughput 29-92% + while its cpu draw FELL.""" + return _str_map(config.POOL_MEM).get(tier) + + +def pool_cpu(tier): + return _str_map(config.POOL_CPU).get(tier) + + +def _tier_for_bytes(anon): + tiers = _tiers() + if not tiers or not anon: return None - gib = anon_bytes / float(1024 ** 3) + gib = anon / float(2 ** 30) for cut, name in tiers: if gib < cut: return name @@ -106,271 +137,184 @@ def _tier_for_bytes(anon_bytes): def _promote(tier, steps): - """Move `steps` tiers up the ladder, stopping at the top. - - OOM escalation moves the POOL, not just the request. Bumping a request while - the pod is still pinned to a tier whose nodes cannot hold it produces a pod - that can never schedule -- Pending forever, which reads as a hang rather - than a failure. nebula and protostar sit outside the ladder and escalate - straight to the top, since there is no tier above them to walk to. - """ - tiers = [name for _, name in _parsed_pool_tiers()] - if not tiers: - return tier - top = tiers[-1] - if steps <= 0 or tier is None: + """Climb `steps` rungs, stopping at the top. Tiers off the ladder go + straight to it: there is nothing above them to walk to.""" + names = [name for _, name in _tiers()] + if not names or tier is None or steps <= 0: return tier - if tier not in tiers: - return top - return tiers[min(tiers.index(tier) + steps, len(tiers) - 1)] + if tier not in names: + return names[-1] + return names[min(names.index(tier) + steps, len(names) - 1)] -def _cache_bump(tier, anon_bytes, ws_bytes): +def _rung_blocked(tier, nxt): + want = f"{tier}->{nxt}" + return any(item.strip() == want for item in config.POOL_BLOCK_RUNGS.split(',')) + + +def _cache_bump(tier, anon, working_set): """One rung up when the working set reaches the next tier and the rung is free. - peakAnonBytes picks the base tier because anon is what OOM-kills. Page cache - is elastic -- it evicts rather than dying -- so it has no business in that - decision. It does decide throughput: replay is single-threaded, so every - bucket lookup that misses cache is a serial ~0.5 ms EBS stall. Measured on - ssc-test 2026-08-03, range 44648511 (anon 2.63 GiB, ws 18.31 GiB) run twice - on identical 2-vCPU Intel nodes differing only in RAM: - - m8in.large 8 GiB 540 reads/ledger 21% iowait 1.86 lps - r8in.large 16 GiB 65 reads/ledger 7% iowait 3.14 lps (100% of profile) - - Which rungs are worth taking is stated outright in POOL_BLOCK_RUNGS, not - inferred. giant->supergiant is m8a.large->r8a.large: twice the RAM for the - same 2 vCPU and +8% spot. supergiant->hypergiant is r8a.large->x8i.large, - also 2 vCPU, and that rung measured 1.86x on ssc-test 2026-08-03 -- 1.64 -> - 2.99 lps across nine ranges, the largest gain found anywhere. - - Blocked: hypergiant->supernova, whose ranges measured healthy at 6-11 - reads/ledger and gained only 1.23x, and dwarf->subgiant, the same doubling - at the bottom of the ladder. Both double the cores for a weak return. - - This used to be derived instead, by refusing any rung whose POOL_VCPU - differed and keeping an allowlist of exceptions. POOL_VCPU reads the - SMALLEST shape a pool can land on, so it mispriced every tier spanning node - sizes -- an x8i at w80 hid a 4->8 promotion -- and the allowlist existed - only to undo its wrong answers. Deriving it bought one rung that a block - entry states directly, so a list of refusals replaced both. The cost is that - a new tier is allowed by default: a rung that should be refused now has to - be named here. - - Deliberately loose about false positives. Promoting a range that did not need - it costs +8% on its node-hours and nothing in quota; leaving one starving - costs 40% of its throughput. Against 50 pods probed for reads/ledger and - iowait on the same run: 11 correctly promoted, 11 unnecessarily, 1 missed -- - and the 11 unnecessary ones are free. - - One rung, never two, even when the working set would justify more. 44648511 - lands on supergiant still 1.29x UNDER its working set and reaches full - profile rate there; fitting the working set costs multiples for nothing. + peakAnonBytes picks the base tier because anon is what OOM-kills; page cache + evicts rather than dying. It does decide throughput -- replay is + single-threaded, so every bucket lookup that misses cache is a serial EBS + stall. One rung, never two: a range 1.29x under its working set still + reaches full profile rate. """ - if not (tier and anon_bytes and ws_bytes): + if not (tier and anon and working_set): return tier nxt = _promote(tier, 1) - if nxt == tier: - return tier # already at the top of the ladder - order = [name for _, name in _parsed_pool_tiers()] - want = _tier_for_bytes(ws_bytes) + if nxt == tier or _rung_blocked(tier, nxt): + return tier + order = [name for _, name in _tiers()] + want = _tier_for_bytes(working_set) if tier not in order or not want or order.index(want) <= order.index(tier): - return tier # working set does not reach the next tier - if _rung_blocked(tier, nxt): - return tier # denied outright, see POOL_BLOCK_RUNGS + return tier return nxt -def pool_for(end, attempt=1, rungs=None): - """Which nodepool tier this range belongs in, or None when not pooling. - - Three cases, and they are deliberately different pools: - no profile at all -> POOL_NO_PROFILE (nebula), sized by the configured - defaults because nothing is known - profiled run, this - range past the top -> POOL_UNPROFILED (protostar). Only the newest - ledgers land here and they are the densest, so - it is a rich pool rather than an average one - profiled range -> the tier its peakAnonBytes fits, then one rung - up if its working set cannot be cached there and - the rung is free -- see _cache_bump - - `rungs` is how many tiers to climb, and it counts OOMs -- NOT attempts. A - spot reclaim, a disruption and a timeout all produce a retry, and none of - them says the range needed a bigger node. Promoting on attempt number put 65 - ranges onto 8-vCPU supernova nodes during the 2026-08-03 spot run whose - attempt-1 verdicts were `timeout`; they belonged on 4-vCPU hypergiant, so it - burned ~260 vCPU of a 2304 quota escalating away from a problem that was - never memory. +def pool_for(end, oom_count=0): + """The tier this range belongs in, or None when not pooling. + + `oom_count` is how many rungs to climb and it counts OOMs, not attempts: + promoting on attempt number once put 65 ranges onto 8-vCPU nodes whose + attempt-1 verdicts were `timeout`, burning ~260 vCPU of a 2304 quota + escalating away from a problem that was never memory. """ - if not mc.POOL_PREFIX: + if not config.POOL_PREFIX: return None - if rungs is None: - # Attempts before this one, since this attempt has not run yet. Anything - # on disk for it is from a previous incarnation of the same attempt. - rungs = attempt_files._oom_count(end, attempt - 1) if attempt and attempt > 1 else 0 - if not mc.PROFILE: - return _promote(mc.POOL_NO_PROFILE, rungs) - prof = profiles.profile_for(end) if end is not None else None + if not config.PROFILE: + return _promote(config.POOL_NO_PROFILE, oom_count) + prof = profile_for(end) if not prof: - return _promote(mc.POOL_UNPROFILED, rungs) + return _promote(config.POOL_UNPROFILED, oom_count) anon = prof.get('peakAnonBytes') - tier = _cache_bump(_tier_for_bytes(anon), anon, - prof.get('peakWorkingSetBytes')) + tier = _cache_bump(_tier_for_bytes(anon), anon, prof.get('peakWorkingSetBytes')) if not tier: - # An entry with no memory measurement tells us nothing about size -- - # treat it as unprofiled rather than guessing a tier. - return _promote(mc.POOL_UNPROFILED, rungs) - return _promote(tier, rungs) - - -def _pool_map(raw, what): - out = {} - for item in raw.split(','): - item = item.strip() - if not item: - continue - name, _, value = item.partition(':') - try: - out[name.strip()] = float(value) - except ValueError: - logger.error("%s is malformed at %r; that tier falls back to the " - "configured request", what, item) - return out + return _promote(config.POOL_UNPROFILED, oom_count) + return _promote(tier, oom_count) -def pool_cpu(tier): - """cpu request for a tier, or None to keep the configured one.""" - return _pool_map(mc.POOL_CPU, 'POOL_CPU').get(tier) +# --- escalation ------------------------------------------------------------- -def pool_memory(tier): - """Memory request for a tier: half its node's allocatable. +def next_memory(end, base, oom_count): + """Memory request after N OOMs. - Not the range's own measurement. Sizing from the peak would let two small - ranges share a node, and isolation is the whole point -- freeing a pod of - its three neighbours raised throughput 29-92% while its cpu draw FELL, so - the contended resource is memory bandwidth and shared cache, not compute. + Pooled: the tier ladder IS the ladder, and the promoted tier's cut is the + escalated request, so request and placement move together. A multiplicative + bump cannot do this -- tiers are 2.2-2.5x apart against a 1.5x factor, so + the bump lands BETWEEN them and the pod sits Pending on a pool that can + never satisfy it. - Half is the smallest value that still excludes a second pod once the - daemonsets are counted, and the largest that reliably schedules the first. + Unpooled: base * factor^N, where base is what attempt 1 actually ran with. + Escalating a 209Mi profiled range off the configured default jumps to + 36000Mi, a 172x overshoot that throws away the packing win on the first OOM. """ - return _pool_str_map(mc.POOL_MEM, 'POOL_MEM').get(tier) + if config.POOL_PREFIX: + promoted = pool_memory(pool_for(end, oom_count)) + # Above the ladder there is nothing to promote into, so hold rather than + # invent a value. + return promoted or base or config.REQ_MEM + want = int(quantity_bytes(base or config.REQ_MEM) + * (config.MEM_BUMP_FACTOR ** max(0, oom_count))) + return bytes_to_quantity(min(want, quantity_bytes(config.MEM_ESCALATION_CAP))) -def _pool_str_map(raw, what): - out = {} - for item in raw.split(','): - item = item.strip() - if not item: - continue - name, _, value = item.partition(':') - if not value: - logger.error("%s is malformed at %r; that tier keeps the configured " - "request", what, item) - continue - out[name.strip()] = value.strip() - return out - +def next_ephemeral(base, eviction_count): + """Disk after N evictions, or None when nothing is limited. -def _positive_seconds(value): - """A finite positive runtime, or None when the profile cannot supply one.""" - try: - seconds = float(value) - except (TypeError, ValueError): + Only ephemeral mode has this axis: a pvc run sets no ephemeral request or + limit at all, so there is nothing to raise. + """ + if not config.LIM_EPHEMERAL: return None - return seconds if math.isfinite(seconds) and seconds > 0 else None - - -def _profile_seconds(): - """Every valid measured runtime in the profile, sorted.""" - if mc._SORTED_SECONDS is None: - values = (_positive_seconds(r.get('seconds')) for _, r in (mc.PROFILE or [])) - mc._SORTED_SECONDS = sorted(seconds for seconds in values if seconds is not None) - return mc._SORTED_SECONDS - + want = int(quantity_bytes(base or config.LIM_EPHEMERAL) + * (config.EPH_BUMP_FACTOR ** max(0, eviction_count))) + return bytes_to_quantity(min(want, quantity_bytes(config.EPH_ESCALATION_CAP))) -def _runtime_insurance(seconds, allowance): - """Runtime-weighted share of a configured allowance. - The longest range in the profile gets all of it and one half as long gets - half, so the allowance follows time-at-risk. Zero when the profile cannot - supply a runtime to weight by. - """ - seconds = _positive_seconds(seconds) - everything = _profile_seconds() - longest = everything[-1] if everything else None - insurance = units.quantity_bytes(allowance) - # No `longest <= 0` guard: _profile_seconds only keeps finite positives. - if seconds is None or longest is None or insurance <= 0: - return 0 - return int(insurance * (seconds / longest)) +# --- the request ------------------------------------------------------------ -def _profile_overrides(end, escalated, attempt=1): - """Request overrides for this range from the profile, or {} for none. +def _profile_overrides(end, escalated, oom_count): + """Request overrides from the profile, or {}. - Escalated retries opt out: an escalation is a measurement of THIS run and - outranks anything an earlier one saw. + Unpooled escalation opts out: an escalation measures THIS run and outranks + anything an earlier one saw. Pooled escalation does NOT -- the promotion IS + the escalation, and bailing here would send the pod to the new pool still + asking for the old tier's memory. """ - if end is None: - return {} - if escalated and not mc.POOL_PREFIX: - # Unpooled: an escalation measures THIS run and outranks anything an - # earlier one saw. Pooled: the promotion IS the escalation, and the - # promoted tier's cut is the escalated request -- bailing out here would - # send the pod to the new pool still asking for the old tier's memory. + if end is None or (escalated and not config.POOL_PREFIX): return {} - prof = profiles.profile_for(end) + prof = profile_for(end) out = {} if prof: disk = prof.get('peakEphemeralBytes') - if disk and mc.LIM_EPHEMERAL: - want = (int(disk * mc.PROFILE_MARGIN) - + units.quantity_bytes(mc.PROFILE_EPHEMERAL_HEADROOM) + if disk and config.LIM_EPHEMERAL: + want = (int(disk * config.PROFILE_MARGIN) + + quantity_bytes(config.PROFILE_EPHEMERAL_HEADROOM) + _runtime_insurance(prof.get('seconds'), - mc.PROFILE_RUNTIME_EPHEMERAL_INSURANCE)) - out['ephemeral-storage'] = units.bytes_to_quantity( - min(want, units.quantity_bytes(mc.PROFILE_MAX_EPHEMERAL))) - if mc.POOL_PREFIX: - # Deliberately BEFORE the no-profile bail. pool_for resolves a tier for - # every range -- protostar when the range is newer than the profile, - # nebula when there is no profile at all -- so returning {} here would - # pin the pod to that pool while sizing it from the flat REQ_CPU. On - # 2026-08-04 that shipped a 6780m request (the run's - # --pubnet-parallel-catchup-cpu-request) at a protostar pool whose - # largest node is 4 vCPU: permanently Pending, retried forever, and - # invisible until a run had enough past-the-profile ranges to notice. - # Pooled: the tier's cut is the request, and the margin lives in the - # node size instead. PROFILE_MARGIN, cache headroom and runtime - # insurance all existed to keep a pod under its own memory LIMIT; there - # is no memory limit any more and the pod owns the node, so a margin in - # the request constrains nothing the kubelet acts on. Disk keeps its - # margin above -- that limit IS enforced. - tier = pool_for(end, attempt) - mem = pool_memory(tier) + config.PROFILE_RUNTIME_EPHEMERAL_INSURANCE)) + out['ephemeral-storage'] = bytes_to_quantity( + min(want, quantity_bytes(config.PROFILE_MAX_EPHEMERAL))) + if config.POOL_PREFIX: + # Deliberately BEFORE the no-profile bail: pool_for resolves a tier for + # every range, so returning {} here would pin the pod to that pool while + # sizing it from the flat REQ_CPU. That shipped a 6780m request at a + # pool whose largest node is 4 vCPU -- permanently Pending. + tier = pool_for(end, oom_count) + mem, cpu = pool_memory(tier), pool_cpu(tier) if mem: out['memory'] = mem - cpu = pool_cpu(tier) if cpu: out['cpu'] = cpu return out if not prof: - # Unpooled and unmeasured: nothing to size from, so the configured - # requests stand exactly as if there were no profile at all. return out - # Unpooled (the pre-tier behaviour, and what nebula-style runs fall back to - # when no prefix is configured): size memory from the range's own peak, with - # the margins that a limit-bearing pod needed. - # - # peakAnonBytes is kubelet's rssBytes, sampled by the collector on its own - # the finer one and fall back, so a profile captured before the collector - # tracked anon still sizes exactly as it used to. + # Unpooled and profiled: the range's own peak plus a flat allowance and a + # runtime-weighted one. The flat one is load-bearing -- 1.15x of 190Mi is + # 19Mi of slack, and 90 ranges OOMKilled inside 90s without it. rss = prof.get('peakAnonBytes') if rss: - want = (int(rss * mc.PROFILE_MARGIN) - + units.quantity_bytes(mc.PROFILE_CACHE_HEADROOM) + want = (int(rss * config.PROFILE_MARGIN) + + quantity_bytes(config.PROFILE_CACHE_HEADROOM) + _runtime_insurance(prof.get('seconds'), - mc.PROFILE_RUNTIME_MEMORY_INSURANCE)) - out['memory'] = units.bytes_to_quantity(min(want, units.quantity_bytes(mc.PROFILE_MAX_MEM))) + config.PROFILE_RUNTIME_MEMORY_INSURANCE)) + out['memory'] = bytes_to_quantity( + min(want, quantity_bytes(config.PROFILE_MAX_MEM))) return out + + +def requests_for(end, oom_count=0, memory=None, ephemeral=None): + """(requests, limits) for one attempt. + + Only disk is limited: it is the one dimension where an unbounded pod takes + the node down rather than itself. + """ + overrides = _profile_overrides(end, escalated=bool(memory or ephemeral), + oom_count=oom_count) + req = {'cpu': config.REQ_CPU, 'memory': memory or config.REQ_MEM} + lim = {} + + # A profiled pooled range owns its node -- the tier's cut excludes a second + # pod -- so a disk limit guards no neighbour and only turns spare disk into + # an eviction. Unprofiled pooled runs keep both: nothing measured them. + pooled_profiled = bool(config.POOL_PREFIX and config.PROFILE) + if config.REQ_EPHEMERAL and not pooled_profiled: + req['ephemeral-storage'] = ephemeral or config.REQ_EPHEMERAL + else: + overrides.pop('ephemeral-storage', None) + if config.LIM_EPHEMERAL and not pooled_profiled: + lim['ephemeral-storage'] = ephemeral or config.LIM_EPHEMERAL + + for key, value in overrides.items(): + req[key] = value + if key == 'ephemeral-storage' and config.LIM_EPHEMERAL: + lim[key] = value + return req, (lim or None) + + +def node_label_value(end, oom_count): + tier = pool_for(end, oom_count) + return f"{config.POOL_PREFIX}-{tier}" if tier else config.NODE_LABEL_VALUE diff --git a/src/MissionParallelCatchup/lib/monitor/units.py b/src/MissionParallelCatchup/lib/monitor/units.py deleted file mode 100644 index 0544aba8..00000000 --- a/src/MissionParallelCatchup/lib/monitor/units.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Kubernetes quantity strings to bytes and back. - -Pure string arithmetic -- no config, no cluster. The monitor compares profile -figures (bytes) against chart and pool values (quantity strings) constantly, and -doing it inline is how a Gi/Mi mix-up becomes a sizing bug. -""" - -_UNITS = {'Ki': 1024, 'Mi': 1024**2, 'Gi': 1024**3, 'Ti': 1024**4, - 'K': 1000, 'M': 1000**2, 'G': 1000**3, 'T': 1000**4} - - -def gib(q): - try: - return quantity_bytes(q) / (1024 ** 3) - except Exception: - return None - - -def quantity_bytes(q): - for suffix, mult in sorted(_UNITS.items(), key=lambda kv: -len(kv[0])): - if q.endswith(suffix): - return int(float(q[:-len(suffix)]) * mult) - return int(float(q)) - - -def bytes_to_quantity(n): - return f"{max(1, n // (1024 ** 2))}Mi" diff --git a/src/MissionParallelCatchup/lib/monitor/verdict.py b/src/MissionParallelCatchup/lib/monitor/verdict.py new file mode 100644 index 00000000..dc3b6009 --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/verdict.py @@ -0,0 +1,150 @@ +"""What killed an attempt. + +The collector classifies the live pod into .outcome; this resolves that against +the Job condition and against the archive, and the answer is what budgets are +spent on. +""" +import gzip +import logging +import re + +import record + +logger = logging.getLogger('job_monitor') + +# stellar-core's "did not complete". Ambiguous by construction: a corrupt +# bucket, a SIGTERM during replay and an archive fetch fault all produce it. +CATCHUP_INCOMPLETE_EXIT = 3 + +# Outcomes naming a MECHANISM. A Job-level DeadlineExceeded must never +# overwrite one: it says the Job ran long, not which of these caused it. +SPECIFIC = frozenset({'oom', 'disrupted', 'ephemeral', 'rejected', 'fetch-fault'}) + +# Rule ORDER is the contract with the Job controller's "rule at index N". +RULE_ORDER = ('disrupted', 'oom', 'failed') +_RULE_INDEX = dict(enumerate(RULE_ORDER)) +_JOB_RULE = re.compile(r'rule at index (?P\d+)') +_JOB_MSG = re.compile(r'Container (?P\S+) .*exit code (?P\d+)') + +# Archive scan. The windows are different widths on purpose. +_TAIL_LINES = 400 +_TAIL_BYTES = 1 << 18 # comfortably more than 400 lines of stellar-core log +_STALE_WINDOW = 6 # tight: a wider one credits a fault the range recovered from +_MARKER_WINDOW = 25 # wide: concurrent downloads interleave +_CATCHUP_FAILED = 'Catchup failed' +_STALE_ARCHIVE = 'maybe stale archive' +_TERMINAL = ('Key does not exist', '(404)', 'NoSuchKey') +_TRANSIENT = ('Could not connect to the endpoint URL', 'Unable to locate credentials', + 'ExpiredToken', 'RequestTimeout', 'SlowDown', 'ConnectTimeoutError') + + +def from_job(job): + """Recover a verdict from the Job when the pod is already gone. + + The rule INDEX is the signal, not the exit code: rules are first-match-wins, + so reaching the exit-137 rule proves DisruptionTarget did not match -- the + only way to tell an OOM kill from a grace-period SIGKILL once the pod object + is gone. Index and code are parsed independently, because a rule matching on + onPodConditions reports no exit code at all. + """ + for cond in (job.status.conditions or []): + if cond.type != 'Failed' or cond.status != 'True': + continue + if cond.reason == 'DeadlineExceeded': + return {'outcome': 'timeout', 'exitCode': None, 'pod': ''} + if cond.reason != 'PodFailurePolicy': + continue # e.g. BackoffLimitExceeded: no per-rule detail + msg = cond.message or '' + rule = _JOB_RULE.search(msg) + detail = _JOB_MSG.search(msg) + code = int(detail.group('code')) if detail else None + outcome = _RULE_INDEX.get(int(rule.group('idx'))) if rule else None + if outcome is None: + if code is None: + return None + # No usable index. A drained core exits 3, not 137, so a bare 137 is + # an OOM and a bare 3 without DisruptionTarget is a catchup failure. + outcome = 'oom' if code == 137 else 'failed' + return {'outcome': outcome, 'exitCode': code, + 'pod': detail.group('pod') if detail else ''} + return None + + +def effective(end, attempt, job): + """The verdict this attempt is judged on, promotions applied. + + An attempt with no evidence at all is `unknown`, which has no budget and so + condemns: without evidence the monitor cannot tell a reaped node from a + range that really failed, and a run reporting success on a range nobody + verified is worse than one that stops. + """ + from_pod = record.read_outcome(end, attempt) + from_condition = from_job(job) if job is not None else None + + if from_pod and from_pod.get('outcome') in SPECIFIC: + found = from_pod + else: + found = from_condition or from_pod or {'outcome': 'unknown', 'exitCode': None} + + if found.get('outcome') == 'failed' and found.get('exitCode') == CATCHUP_INCOMPLETE_EXIT: + cause = exit3_cause(end, attempt) + if cause: + return dict(found, outcome='fetch-fault', reason=cause) + return found + + +def exit3_cause(end, attempt): + """The transient fetch fault behind an exit 3, or None. + + Three conditions, all required. Missing any leaves the attempt `failed`, + which has no budget -- so a bare `Catchup failed` ends the run rather than + burning 20 attempts proving the chain is broken. + """ + lines = _tail(end, attempt) + if not lines: + return None + failed_at = _last_index(lines, _CATCHUP_FAILED) + if failed_at is None: + return None + stale_at = _last_index(lines[max(0, failed_at - _STALE_WINDOW):failed_at], + _STALE_ARCHIVE) + if stale_at is None: + return None + anchor = max(0, failed_at - _STALE_WINDOW) + stale_at + # Most-recent-first: a range that recovered from a transient fault and then + # hit a 404 is terminal, and the 404 is the later line. + for line in reversed(lines[max(0, anchor - _MARKER_WINDOW):anchor + 1]): + if any(marker in line for marker in _TERMINAL): + return None # the object genuinely is not there; retrying cannot help + for marker in _TRANSIENT: + if marker in line: + return marker + return None + + +def _last_index(lines, needle): + for i in range(len(lines) - 1, -1, -1): + if needle in lines[i]: + return i + return None + + +def _tail(end, attempt): + """The archive's last lines. A missing or unreadable archive is not an + error: it means the collector has nothing to say, and the caller treats + that as "no fetch fault". + + Split only the final bytes, never readlines(): gzip cannot seek, so the + whole archive is decompressed either way, but building a str per line to + keep 400 of them is 90% of the cost on a tip range. The leading fragment is + dropped because the slice lands mid-line. + """ + try: + with gzip.open(record.log_path(end, attempt), 'rb') as fh: + data = fh.read() + except (OSError, EOFError, gzip.BadGzipFile): + return [] + lines = data[-_TAIL_BYTES:].decode('utf-8', 'replace').splitlines() + if len(data) > _TAIL_BYTES: + lines = lines[1:] # only then did the slice land mid-line + return lines[-_TAIL_LINES:] diff --git a/src/MissionParallelCatchup/lib/monitor/worker_liveness.py b/src/MissionParallelCatchup/lib/monitor/worker_liveness.py deleted file mode 100644 index b3119177..00000000 --- a/src/MissionParallelCatchup/lib/monitor/worker_liveness.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Liveness of the workers' stellar-core /info endpoint, one sweep per reconcile. - -The reconcile loop already holds the authoritative pod list. This probes that -list concurrently and returns a snapshot: up if /info answered 200, down for -anything else, unknown for whatever the sweep did not get to before its deadline. - -No state is carried between sweeps -- no hysteresis, no scheduler, no threads. -The numbers feed a Grafana panel and nothing else reads them, so a stale-free -snapshot is worth more than a smoothed one. -""" -import asyncio -import logging - -import aiohttp - -import monitor_config as mc - -logger = logging.getLogger() - -_ADMIN_PORT = 11626 # stellar-core's admin/HTTP port - - -def targets(pods): - """Current Running-with-IP pods, keyed by pod identity. - - A UID change is a replacement even when the Job name or IP is reused. Tests - and unusually incomplete API objects may lack a UID, where the pod name is - still unique for its lifetime. - """ - out = {} - for pod in pods: - pod_status = getattr(pod, 'status', None) - metadata = getattr(pod, 'metadata', None) - ip = getattr(pod_status, 'pod_ip', None) - if getattr(pod_status, 'phase', None) != 'Running' or not ip or metadata is None: - continue - name = getattr(metadata, 'name', None) - identity = getattr(metadata, 'uid', None) or name - if identity and name: - out[str(identity)] = (str(name), str(ip)) - return out - - -async def _probe(session, ip, timeout): - """True only for HTTP 200. A timeout or refused connection is False.""" - host = f"[{ip}]" if ':' in ip else ip - async with session.get(f"http://{host}:{_ADMIN_PORT}/info", - timeout=aiohttp.ClientTimeout(total=timeout)) as resp: - return resp.status == 200 - - -async def sweep(targets, concurrency=None, timeout=None, deadline=None): - """Probe every target concurrently; return {'up','down','unknown'}. - - Not a TaskGroup: a TaskGroup cancels its siblings when one task raises, - which is the opposite of what a fleet sweep wants -- one unreachable pod - must not discard the other 1023 answers. asyncio.wait with a deadline keeps - whatever finished and cancels only the stragglers. - """ - concurrency = int(concurrency or mc.LIVENESS_MAX_CONCURRENCY) - timeout = float(timeout or mc.LIVENESS_PROBE_TIMEOUT_SECONDS) - deadline = float(deadline or mc.LIVENESS_SWEEP_SECONDS) - counts = {'up': 0, 'down': 0, 'unknown': len(targets)} - if not targets: - return {'up': 0, 'down': 0, 'unknown': 0} - - # `limit` is the concurrency bound: aiohttp holds a task at connect until a - # slot frees, so a semaphore on top of it would be enforcing the same number - # twice. force_close because a worker pod can vanish between sweeps and a - # pooled socket to a dead pod would be handed straight back out. - connector = aiohttp.TCPConnector(limit=concurrency, force_close=True) - async with aiohttp.ClientSession(connector=connector) as session: - tasks = [asyncio.create_task(_probe(session, ip, timeout)) - for _, ip in targets.values()] - done, pending = await asyncio.wait(tasks, timeout=deadline) - for task in pending: - task.cancel() - if pending: - await asyncio.gather(*pending, return_exceptions=True) - for task in done: - try: - up = task.result() - except Exception: - up = False # timeout, refused, DNS, malformed response - counts['up' if up else 'down'] += 1 - counts['unknown'] -= 1 - return counts - - -def publish(targets): - """Run one sweep and return its counts. Called from the reconcile loop. - - Blocks for at most LIVENESS_SWEEP_SECONDS: everything still outstanding at - the deadline is cancelled and reported unknown, so a fleet of unreachable - pods costs the deadline and never the sum of their timeouts. - """ - if not targets: - return {'up': 0, 'down': 0, 'unknown': 0} - try: - return asyncio.run(sweep(targets)) - except Exception as e: - logger.warning("liveness sweep failed (%s); reporting all workers unknown", e) - return {'up': 0, 'down': 0, 'unknown': len(targets)} diff --git a/src/MissionParallelCatchup/lib/records.py b/src/MissionParallelCatchup/lib/records.py index 3520f42d..1259420e 100644 --- a/src/MissionParallelCatchup/lib/records.py +++ b/src/MissionParallelCatchup/lib/records.py @@ -3,8 +3,8 @@ This is the entire cross-process contract. The collector writes these files while a pod still exists and the monitor reads them back, so a disagreement about a name is a measurement silently lost. Everything either side does with -the contents lives on its own side: attempt_files for the monitor, state_files -for the collector. +the contents lives on its own side: record for the monitor, state_files for the +collector. """ import os diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index 14f1d209..c09bc5c6 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -203,16 +203,7 @@ spec: - name: job-monitor image: {{ .Values.monitor.image }} imagePullPolicy: {{ .Values.monitor.imagePullPolicy }} - {{- if .Values.monitor.sourceConfigMap }} - command: ["/bin/sh", "-c"] - args: - - >- - {{- if .Values.monitor.sourceInstallDependencies }} - pip install --no-cache-dir -q 'kubernetes~=36.0' 'aiohttp~=3.9' - 'prometheus-client~=0.19' && - {{- end }} - exec python3 /app/job_monitor.py - {{- end }} + # No command: the image's CMD is job_monitor.py. ports: - containerPort: 8080 env: @@ -380,10 +371,6 @@ spec: mountPath: /data - name: logs mountPath: /logs - {{- if .Values.monitor.sourceConfigMap }} - - name: monitor-src - mountPath: /app - {{- end }} # No readinessProbe: nothing consumes Ready. There is no Service, and # the prometheus kubernetes-pods job keeps targets on the scrape # annotation alone, with no pod-ready filter. @@ -406,23 +393,12 @@ spec: - name: log-collector image: {{ .Values.monitor.image }} imagePullPolicy: {{ .Values.monitor.imagePullPolicy }} - {{- if .Values.monitor.sourceConfigMap }} - command: ["/bin/sh", "-c"] - args: - - >- - {{- if .Values.monitor.sourceInstallDependencies }} - pip install --no-cache-dir -q 'kubernetes~=36.0' 'aiohttp~=3.9' - 'prometheus-client~=0.19' && - {{- end }} - exec python3 /app/log_collector.py - {{- else }} # Resolved on PATH, not /usr/bin/python3: Dockerfile.jobmonitor builds # on python:3.12-slim, which ships the interpreter at # /usr/local/bin/python3. An absolute path here pins the chart to one # base image and fails as StartError -- the monitor container comes up # (it inherits the image CMD) and only the sidecar crashloops. command: ["python3", "log_collector.py"] - {{- end }} env: - name: NAMESPACE valueFrom: @@ -464,16 +440,7 @@ spec: volumeMounts: - name: logs mountPath: /logs - {{- if .Values.monitor.sourceConfigMap }} - - name: monitor-src - mountPath: /app - {{- end }} volumes: - {{- if .Values.monitor.sourceConfigMap }} - - name: monitor-src - configMap: - name: {{ .Values.monitor.sourceConfigMap }} - {{- end }} - name: data emptyDir: {} - name: logs diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index eaafdbc1..9aae9d5c 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -45,13 +45,9 @@ worker: "curl -sf http://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}" # Index -> ledger range is computed in the Job template; no queue to preload. -# The range itself -- start, end, size, order -- is per-run input and arrives -# with the driver's POST /start, so the chart installs a generic monitor. -range: - overlapLedgers: 320 - # Dispatch order. Generators emit tip-first, which front-loads the most - # expensive ranges. "oldest-first" reverses that so a profiling run - # measures the cheap early ranges before it can be interrupted. +# The range itself -- start, end, size, overlap, order -- is per-run input and +# arrives with the driver's POST /start, so the chart installs a generic +# monitor and holds none of it. monitor: # Reuses the existing hand-built job-monitor image slot -- no new image and no @@ -60,7 +56,7 @@ monitor: # TEMPORARY dev pin: stellar/ssc-job-monitor:latest predates the apps/+lib/ # split and ATTEMPT_BUDGETS, so this chart would ship env vars it cannot read. # Revert to the stellar/ repo once there is a push path for it. - image: "stellajuna/ssc-jm:2026-08-11b" + image: "stellajuna/ssc-jm:latest" # The driver reaches the monitor through this route: profile in via POST # /start, status and logs out. Empty routeHost disables the HTTPRoute; the # Service still exists, so an in-cluster caller works either way. @@ -79,14 +75,6 @@ monitor: routeHost: "" gatewayName: "" gatewayNamespace: "" - # Dev loop: run the monitor and collector from a ConfigMap holding - # job_monitor.py and log_collector.py instead of a built image. Set it to the - # ConfigMap name and point monitor.image at a plain python base; the deps the - # Dockerfile bakes get pip-installed at start. Empty = use the image as built. - sourceConfigMap: "" - # Source-mode development normally installs dependencies at container start. - # Disable only when the selected image already contains them. - sourceInstallDependencies: true # --- nodepool tiers ------------------------------------------------------- # # A range picks a NODEPOOL by its measured memory and gets that node to From 3c78b251dfc6076e9d41524be2af2597b4007905 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 17 Aug 2026 13:46:31 -0400 Subject: [PATCH 114/117] Settle a failed attempt the collector never opened A worker deleted before its container started -- what a drain does to a pod that has only just been scheduled -- is never in a pollable phase, so the collector never services it and writes no .done. observe() waits on that file, so the range stayed in `running` and held a worker slot for the rest of the run. An attempt with no .state was never claimed, so nothing further is coming and the Job can be judged on its own reason. That reason is BackoffLimitExceeded, which carries no per-rule detail and so reads as `unknown` -- now given a budget of 2, one retry. Same verdict path as every other cause, and a range that cannot start twice still condemns rather than looping. Verified on ssc-test: a force-deleted pre-start pod settled in one pass and retried (`attempt 1 -> 2 (unknown)`), where the same kill previously left the Job Failed and unjudged for minutes. Mission completed 32/32. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/apps/job_monitor.py | 3 +++ src/MissionParallelCatchup/lib/config.py | 1 + src/MissionParallelCatchup/lib/monitor/record.py | 10 ++++++++++ .../parallel_catchup_helm/templates/job_monitor.yaml | 2 ++ .../parallel_catchup_helm/values.yaml | 2 ++ 5 files changed, 18 insertions(+) diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py index 8c343600..94e2e636 100644 --- a/src/MissionParallelCatchup/apps/job_monitor.py +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -157,6 +157,9 @@ def observe(states): if st.status not in ('failed', 'completed'): continue st.done = record.is_done(st.end, st.attempt) + # Otherwise the slot is held for the rest of the run. + if not st.done and st.status == 'failed' and st.pod is None: + st.done = record.unclaimed(st.end, st.attempt) if not st.done: continue # the collector is still writing this attempt if st.status == 'failed': diff --git a/src/MissionParallelCatchup/lib/config.py b/src/MissionParallelCatchup/lib/config.py index 6eebfa2c..4e9a3b19 100644 --- a/src/MissionParallelCatchup/lib/config.py +++ b/src/MissionParallelCatchup/lib/config.py @@ -152,6 +152,7 @@ def _int(name, default): 'fetch-fault': int(os.getenv('MAX_FETCH_FAULT_ATTEMPTS', '20')), 'oom': int(os.getenv('MAX_OOM_ATTEMPTS', '5')), 'ephemeral': int(os.getenv('MAX_EPHEMERAL_ATTEMPTS', '4')), + 'unknown': int(os.getenv('MAX_UNKNOWN_ATTEMPTS', '2')), } # --- liveness --------------------------------------------------------------- diff --git a/src/MissionParallelCatchup/lib/monitor/record.py b/src/MissionParallelCatchup/lib/monitor/record.py index 654fef5d..ac9a9520 100644 --- a/src/MissionParallelCatchup/lib/monitor/record.py +++ b/src/MissionParallelCatchup/lib/monitor/record.py @@ -33,6 +33,10 @@ def done_path(end, attempt): return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.done") +def state_path(end, attempt): + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.state") + + # --- the monitor writes these ----------------------------------------------- @@ -74,6 +78,12 @@ def is_done(end, attempt): return os.path.exists(done_path(end, attempt)) +def unclaimed(end, attempt): + """No .state, so the collector never opened this attempt and no .done is + coming. A pod deleted before its container started is never pollable.""" + return not os.path.exists(state_path(end, attempt)) + + def read_outcome(end, attempt): return _read_json(outcome_path(end, attempt)) diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml index c09bc5c6..5461e2ae 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_monitor.yaml @@ -259,6 +259,8 @@ spec: value: {{ .Values.monitor.attemptBudgets.oom | quote }} - name: MAX_EPHEMERAL_ATTEMPTS value: {{ .Values.monitor.attemptBudgets.ephemeral | quote }} + - name: MAX_UNKNOWN_ATTEMPTS + value: {{ .Values.monitor.attemptBudgets.unknown | quote }} - name: MEM_BUMP_FACTOR value: {{ .Values.monitor.memBumpFactor | quote }} - name: EPH_BUMP_FACTOR diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 9aae9d5c..5b61d9ea 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -161,6 +161,8 @@ monitor: # Each retry escalates the disk limit one rung. Smallest, because an # eviction repeats identically until the range gets more disk. ephemeral: 4 + # Destroyed before its container started, so the Job carries no exit code. + unknown: 2 memBumpFactor: 1.5 ephBumpFactor: 1.5 maxEphemeral: "200Gi" From ef3e8f38cbab36e00b7668e949ca5af924c739a4 Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 17 Aug 2026 16:51:07 -0400 Subject: [PATCH 115/117] Split monitor config out of the shared module, and de-duplicate the path contract Two structural problems from the v2 swap, both of the same kind: a thing that belongs to one process living where the other one also reads it. records.py and monitor/record.py each defined log_path, outcome_path, metrics_path, done_path and state_path with identical bodies, plus two write_atomic implementations. Those filenames ARE the cross-process contract -- records.py says so in its own docstring -- and two copies of a contract are two places for the sides to disagree about a name, which is a measurement silently lost. record.py now imports them. config.py held ~50 monitor-only knobs that the collector never reads, and one collector-only flag with a comment apologising for its location. Restores the split v1 had: config.py keeps the 8 names both processes share, monitor_config takes the knobs, collector_config takes SAVE_SUCCESS_LOGS. /start rebinds five range settings by setattr on the module, so those moved to monitor_config with it -- pointing that setattr at the wrong module would leave every run silently on defaults, with nothing to report it. Verified on ssc-test: 32/32 ranges, seconds/txApply/wallSeconds all 32/32, and the monitor logged the longest-first order it was started with, which is the rebind arriving through the module rather than a stale copy. Co-Authored-By: Claude Opus 5 --- .../apps/job_monitor.py | 13 +- .../apps/log_collector.py | 2 +- .../lib/collector/collector_config.py | 4 + src/MissionParallelCatchup/lib/config.py | 221 +----------------- .../lib/monitor/cluster.py | 39 ++-- .../lib/monitor/dispatch.py | 47 ++-- .../lib/monitor/liveness.py | 8 +- .../lib/monitor/monitor_config.py | 218 +++++++++++++++++ .../lib/monitor/policy.py | 4 +- .../lib/monitor/record.py | 45 +--- .../lib/monitor/server.py | 7 +- .../lib/monitor/sizing.py | 92 ++++---- 12 files changed, 348 insertions(+), 352 deletions(-) create mode 100644 src/MissionParallelCatchup/lib/monitor/monitor_config.py diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py index 94e2e636..937b64f4 100644 --- a/src/MissionParallelCatchup/apps/job_monitor.py +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -20,6 +20,7 @@ import dispatch import liveness import metrics +import monitor_config as mc import policy import record import server @@ -57,7 +58,7 @@ async def reconcile_loop(state, stop): # leaves the run with no writer. logger.exception("reconcile pass failed") try: - async with asyncio.timeout(config.RECONCILE_INTERVAL_SECONDS): + async with asyncio.timeout(mc.RECONCILE_INTERVAL_SECONDS): await stop.wait() except TimeoutError: pass @@ -190,7 +191,7 @@ async def act(states, state): work.append(_reap(st, state)) for st in states: - if st.status == 'pending' and active < config.PARALLELISM: + if st.status == 'pending' and active < mc.PARALLELISM: active += 1 work.append(_dispatch(st, state)) @@ -307,12 +308,12 @@ def start(self, doc): ('ledgersPerJob', 'LEDGERS_PER_JOB'), ('overlapLedgers', 'OVERLAP_LEDGERS')): if key in spec: - setattr(config, name, spec[key]) - config.set_profile(sizing.load_profile(doc.get('profile') or {})) - config.validate() + setattr(mc, name, spec[key]) + mc.set_profile(sizing.load_profile(doc.get('profile') or {})) + mc.validate() self.ranges = dispatch.range_list() logger.info("run started: %d ranges, %s, profile of %d", - len(self.ranges), config.RANGE_ORDER, len(config.PROFILE)) + len(self.ranges), mc.RANGE_ORDER, len(mc.PROFILE)) def condemn(self, st, reason): """Mark terminal; commit() persists it at the end of the pass. diff --git a/src/MissionParallelCatchup/apps/log_collector.py b/src/MissionParallelCatchup/apps/log_collector.py index f68b3804..4dc3ade0 100644 --- a/src/MissionParallelCatchup/apps/log_collector.py +++ b/src/MissionParallelCatchup/apps/log_collector.py @@ -200,7 +200,7 @@ def _finish(pod, end, attempt, succeeded): if (pod.get('status') or {}).get('phase') == 'Failed': verdicts.record_outcome(pod, end, attempt) write_metrics(end, attempt, _duration(pod)) - if not config.SAVE_SUCCESS_LOGS and succeeded: + if not cc.SAVE_SUCCESS_LOGS and succeeded: # .metrics survives on purpose: it holds tx_apply for a range that # succeeded, and a retention flag must not delete a Grafana series. discard(end, attempt) diff --git a/src/MissionParallelCatchup/lib/collector/collector_config.py b/src/MissionParallelCatchup/lib/collector/collector_config.py index ca78243b..4d768423 100644 --- a/src/MissionParallelCatchup/lib/collector/collector_config.py +++ b/src/MissionParallelCatchup/lib/collector/collector_config.py @@ -56,3 +56,7 @@ # split across two reads is whole in one of them. The archive dedups the # overlap, so this only widens what the scan sees. TERMINAL_REREAD_SECONDS = int(os.getenv('TERMINAL_REREAD_SECONDS', 30)) +# Discards a succeeded attempt's archive. .metrics survives it on purpose: that +# holds tx_apply for a range that succeeded, and a retention flag must not +# delete a Grafana series. +SAVE_SUCCESS_LOGS = os.getenv('SAVE_SUCCESS_LOGS', 'true').lower() == 'true' diff --git a/src/MissionParallelCatchup/lib/config.py b/src/MissionParallelCatchup/lib/config.py index 4e9a3b19..0a5bc0c3 100644 --- a/src/MissionParallelCatchup/lib/config.py +++ b/src/MissionParallelCatchup/lib/config.py @@ -1,37 +1,13 @@ -"""Every knob, and the validation a run is admitted through. +"""What both processes must agree on: the run's identity, the shared volume, +and the verdict vocabulary. -Read through the module -- `config.PARALLELISM`, never `from config import -PARALLELISM`. /start rebinds several of these after validating them, and an -imported name binds a copy that never sees the rebind. - -Numbers that arrive from the chart stay strings until validate() coerces them. -Coercing at import made a bad value a boot crash, and a process that cannot -start cannot report why. +Nothing tunable lives here. The monitor's knobs are in monitor_config and the +collector's in collector_config -- a setting only one process reads does not +belong in the module the other one also imports. """ -import logging import os -logger = logging.getLogger('job_monitor') - - -def _int(name, default): - """A chart-env integer, typed here rather than at /start. - - The loop idles on its interval while waiting for /start, so a string there - crash-loops the pod before it can be told anything. A bad value falls back - loudly rather than killing the boot; /start-delivered values get a 400. - """ - raw = os.getenv(name) - if raw is None: - return default - try: - return int(raw) - except (TypeError, ValueError): - logger.error("%s=%r is not an integer; using %d", name, raw, default) - return default - - -# --- identity, shared with the collector ------------------------------------ +# --- identity --------------------------------------------------------------- NAMESPACE = os.getenv('NAMESPACE', 'default') RUN_NAME = os.getenv('RUN_NAME', 'parallel-catchup') LOG_DIR = os.getenv('LOG_DIR', '/logs') @@ -40,188 +16,11 @@ def _int(name, default): LABEL_RANGE = 'catchup.stellar.org/range-end' LABEL_ATTEMPT = 'catchup.stellar.org/attempt' +# Shared because it decides what the collector may read: in pvc mode /data +# outlives the pod, in ephemeral mode it does not. +STORAGE_MODE = os.getenv('STORAGE_MODE', 'pvc') # pvc | ephemeral + # The collector writes one of these into .outcome; a name on one side and not # the other is an attempt nobody can classify. ATTEMPT_OUTCOMES = ('disrupted', 'oom', 'ephemeral', 'timeout', 'rejected', 'unknown', 'failed', 'fetch-fault') - -# Collector-only, and here because both processes import THIS module as -# `config` when they run from one flat directory. -SAVE_SUCCESS_LOGS = os.getenv('SAVE_SUCCESS_LOGS', 'true').lower() == 'true' - -# --- the run ---------------------------------------------------------------- -PARALLELISM = _int('PARALLELISM', 3) -RECONCILE_INTERVAL_SECONDS = _int('LOGGING_INTERVAL_SECONDS', 10) -HTTP_PORT = _int('HTTP_PORT', 8080) - -# Delivered by POST /start, replayed from run.json on restart. -STARTING_LEDGER = '0' -LATEST_LEDGER_NUM = '0' -LEDGERS_PER_JOB = '16000' -OVERLAP_LEDGERS = '320' -RANGE_ORDER = 'tip-first' -PROFILE = [] # sorted [(end, record)] -_SORTED_SECONDS = None # cache, invalidated whenever PROFILE is set - -RANGE_ORDERS = ('tip-first', 'oldest-first', 'longest-first') - -# --- the worker pod --------------------------------------------------------- -CORE_IMAGE = os.getenv('CORE_IMAGE', 'stellar/stellar-core:latest') -ASAN_OPTIONS = os.getenv('ASAN_OPTIONS', '') -WORKER_SERVICE_ACCOUNT = os.getenv('WORKER_SERVICE_ACCOUNT', '') -WORKER_GRACE_SECONDS = int(os.getenv('WORKER_GRACE_SECONDS', '150')) -# Holds the pod open after SIGTERM so the collector can drain the last of its -# log. Must fit inside the grace period; a preStop the kubelet kills mid-sleep -# buys nothing and logs FailedPreStopHook on every evicted pod. -WORKER_PRESTOP_SLEEP_SECONDS = int(os.getenv('WORKER_PRESTOP_SLEEP_SECONDS', '5')) - -STORAGE_MODE = os.getenv('STORAGE_MODE', 'pvc') # pvc | ephemeral -STORAGE_CLASS = os.getenv('STORAGE_CLASS', '') -STORAGE_SIZE = os.getenv('STORAGE_SIZE', '60Gi') - -JOB_TTL_SECONDS = int(os.getenv('JOB_TTL_SECONDS', '3600')) -# 0 disables. A timeout is terminal, so a tight bound trades a certain -# catastrophe against a rounding error: one wedged range holds one slot. -ATTEMPT_DEADLINE_SECONDS = int(os.getenv('ATTEMPT_DEADLINE_SECONDS', '0')) - -# --- unpooled / unprofiled sizing ------------------------------------------- -# The packing unit for a run with no profile: 4 workers per r8*.2xlarge -# (cpu-bound) or 3 per m8*.2xlarge (memory-bound). The price of not measuring. -REQ_CPU = os.getenv('REQ_CPU', '1800m') -REQ_MEM = os.getenv('REQ_MEM', '9Gi') -REQ_EPHEMERAL = os.getenv('REQ_EPHEMERAL', '') -LIM_EPHEMERAL = os.getenv('LIM_EPHEMERAL', '') - -# --- profile-derived sizing (unpooled) -------------------------------------- -PROFILE_MARGIN = float(os.getenv('PROFILE_MARGIN', '1.15')) -# Flat, because a multiplicative margin is nothing at small rss: 1.15x of -# 190Mi is 19Mi of slack, and 90 ranges OOMKilled inside 90s on it. -PROFILE_CACHE_HEADROOM = os.getenv('PROFILE_CACHE_HEADROOM', '512Mi') -PROFILE_RUNTIME_MEMORY_INSURANCE = os.getenv('PROFILE_RUNTIME_MEMORY_INSURANCE', '3Gi') -PROFILE_MAX_MEM = os.getenv('PROFILE_MAX_MEM', '32Gi') -# Image, logs, sqlite WAL: none of it scales with the range. -PROFILE_EPHEMERAL_HEADROOM = os.getenv('PROFILE_EPHEMERAL_HEADROOM', '2Gi') -# Disk tracks runtime closely (pearson 0.920 over 3985 ranges). -PROFILE_RUNTIME_EPHEMERAL_INSURANCE = os.getenv('PROFILE_RUNTIME_EPHEMERAL_INSURANCE', '8Gi') -# Above the flat limit on purpose: that limit is what an UNMEASURED range gets. -PROFILE_MAX_EPHEMERAL = os.getenv('PROFILE_MAX_EPHEMERAL', '64Gi') - -# --- escalation ------------------------------------------------------------- -MEM_BUMP_FACTOR = float(os.getenv('MEM_BUMP_FACTOR', '1.5')) -MEM_ESCALATION_CAP = os.getenv('MAX_MEM', '48Gi') -EPH_BUMP_FACTOR = float(os.getenv('EPH_BUMP_FACTOR', '1.5')) -EPH_ESCALATION_CAP = os.getenv('MAX_EPHEMERAL', '200Gi') - -# --- pools ------------------------------------------------------------------ -# Empty disables routing entirely; every worker keeps NODE_LABEL_VALUE. -POOL_PREFIX = os.getenv('POOL_PREFIX', '') -POOL_TIERS = os.getenv( - 'POOL_TIERS', - '0:subdwarf,0.79:dwarf,1.61:subgiant,3.87:giant,8.85:supergiant,' - '18.38:hypergiant,:supernova') -# Fits once in the tier's on-demand shape, twice in its (one size larger) spot -# shape. Not 50% of either -- see the requirements. -POOL_MEM = os.getenv( - 'POOL_MEM', - 'subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,' - 'supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,' - 'protostar:29696Mi,nebula:9216Mi') -POOL_CPU = os.getenv( - 'POOL_CPU', - 'subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,' - 'hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80') -POOL_BLOCK_RUNGS = os.getenv('POOL_BLOCK_RUNGS', 'dwarf->subgiant,hypergiant->supernova') -POOL_UNPROFILED = os.getenv('POOL_UNPROFILED', 'protostar') -POOL_NO_PROFILE = os.getenv('POOL_NO_PROFILE', 'nebula') - -# --- placement -------------------------------------------------------------- -NODE_LABEL_KEY = os.getenv('NODE_LABEL_KEY', '') -NODE_LABEL_VALUE = os.getenv('NODE_LABEL_VALUE', '') -REQUIRE_NODE_LABELS = os.getenv('REQUIRE_NODE_LABELS', '') -AVOID_NODE_LABEL_KEY = os.getenv('AVOID_NODE_LABEL_KEY', '') -AVOID_NODE_LABEL_VALUE = os.getenv('AVOID_NODE_LABEL_VALUE', '') -TOLERATE_TAINT = os.getenv('TOLERATE_TAINT', '') - -# --- retry budgets ---------------------------------------------------------- -# The whole retry policy. A cause that is NOT here is condemned the first time -# it happens. Every budget is spent by its own cause: one shared attempt index -# let evictions drain the OOM budget to zero. -ATTEMPT_BUDGETS = { - 'disrupted': int(os.getenv('MAX_DISRUPTION_ATTEMPTS', '100')), - 'rejected': int(os.getenv('MAX_REJECTED_ATTEMPTS', '100')), - 'fetch-fault': int(os.getenv('MAX_FETCH_FAULT_ATTEMPTS', '20')), - 'oom': int(os.getenv('MAX_OOM_ATTEMPTS', '5')), - 'ephemeral': int(os.getenv('MAX_EPHEMERAL_ATTEMPTS', '4')), - 'unknown': int(os.getenv('MAX_UNKNOWN_ATTEMPTS', '2')), -} - -# --- liveness --------------------------------------------------------------- -LIVENESS_MAX_CONCURRENCY = int(os.getenv('LIVENESS_MAX_CONCURRENCY', '64')) -LIVENESS_PROBE_TIMEOUT_SECONDS = float(os.getenv('LIVENESS_PROBE_TIMEOUT_SECONDS', '2')) -# Under the reconcile interval on purpose: the pass awaits this sweep, so a -# deadline above the interval lets an unreachable fleet stretch dispatch and -# reaping behind a probe that only feeds a dashboard. -LIVENESS_SWEEP_SECONDS = float(os.getenv('LIVENESS_SWEEP_SECONDS', '5')) - -# Sized for the dispatch burst rather than a steady rate: a wave head creates -# ~1024 Jobs and PVCs at once. -APISERVER_CONCURRENCY = int(os.getenv('APISERVER_CONCURRENCY', '64')) - - -def label_pairs(raw): - """[(key, value)] from "k:v,k:v". A key with no value is dropped -- it would - require the label be exactly "", which no node carries, and the pod sits - Pending in a way that reads as slow provisioning.""" - out = [] - for item in (raw or '').split(','): - key, _, value = item.strip().partition(':') - if key and value: - out.append((key, value)) - return out - - -def set_profile(profile): - global PROFILE, _SORTED_SECONDS - PROFILE = profile - _SORTED_SECONDS = None - - -def validate(): - """Coerce and re-bind everything a run depends on. Raises ValueError. - - Called from /start, which is the first moment the configuration is - complete, so a misconfigured run is answered with a 400 rather than - crash-looping a pod the driver can only time out on. - """ - global STARTING_LEDGER, LATEST_LEDGER_NUM, LEDGERS_PER_JOB, OVERLAP_LEDGERS - - def positive(name, value, floor=1): - try: - n = int(value) - except (TypeError, ValueError): - raise ValueError(f"{name} is not an integer: {value!r}") - if n < floor: - raise ValueError(f"{name} must be >= {floor}, got {n}") - return n - - STARTING_LEDGER = positive('STARTING_LEDGER', STARTING_LEDGER, floor=0) - LATEST_LEDGER_NUM = positive('LATEST_LEDGER_NUM', LATEST_LEDGER_NUM) - LEDGERS_PER_JOB = positive('LEDGERS_PER_JOB', LEDGERS_PER_JOB) - OVERLAP_LEDGERS = positive('OVERLAP_LEDGERS', OVERLAP_LEDGERS, floor=0) - - if LATEST_LEDGER_NUM < STARTING_LEDGER: - raise ValueError(f"LATEST_LEDGER_NUM {LATEST_LEDGER_NUM} is below " - f"STARTING_LEDGER {STARTING_LEDGER}") - if RANGE_ORDER not in RANGE_ORDERS: - raise ValueError(f"RANGE_ORDER must be one of {RANGE_ORDERS}, " - f"got {RANGE_ORDER!r}") - if STORAGE_MODE not in ('pvc', 'ephemeral'): - raise ValueError(f"STORAGE_MODE must be pvc or ephemeral, " - f"got {STORAGE_MODE!r}") - if RANGE_ORDER == 'longest-first' and not PROFILE: - # With no profile every range ties, the sort is a no-op, and dispatch - # silently falls back to tip-first while the operator believes - # otherwise. - raise ValueError( - "RANGE_ORDER=longest-first requires a profile: it orders ranges by " - "measured seconds. POST a profile, or set another order.") diff --git a/src/MissionParallelCatchup/lib/monitor/cluster.py b/src/MissionParallelCatchup/lib/monitor/cluster.py index 2d4333d9..b310e68f 100644 --- a/src/MissionParallelCatchup/lib/monitor/cluster.py +++ b/src/MissionParallelCatchup/lib/monitor/cluster.py @@ -22,7 +22,8 @@ from kubernetes.aio import client, config as kube_config from kubernetes.aio.client import ApiException -import config as cfg +import config +import monitor_config as mc logger = logging.getLogger('job_monitor') @@ -45,7 +46,7 @@ async def session(): kube_config.load_incluster_config() except Exception: await kube_config.load_kube_config() - _slots = asyncio.Semaphore(cfg.APISERVER_CONCURRENCY) + _slots = asyncio.Semaphore(mc.APISERVER_CONCURRENCY) async with client.ApiClient() as api: _api, batch_v1, core_v1 = api, client.BatchV1Api(api), client.CoreV1Api(api) try: @@ -55,7 +56,7 @@ async def session(): def _selector(): - return f"{cfg.LABEL_RUN}={cfg.RUN_NAME}" + return f"{config.LABEL_RUN}={config.RUN_NAME}" async def snapshot(): @@ -65,12 +66,12 @@ async def snapshot(): must not be assembled from two different moments. """ jobs_raw, pods_raw = await asyncio.gather( - batch_v1.list_namespaced_job(cfg.NAMESPACE, label_selector=_selector()), - core_v1.list_namespaced_pod(cfg.NAMESPACE, label_selector=_selector())) + batch_v1.list_namespaced_job(config.NAMESPACE, label_selector=_selector()), + core_v1.list_namespaced_pod(config.NAMESPACE, label_selector=_selector())) jobs = {} for job in jobs_raw.items: - end = (job.metadata.labels or {}).get(cfg.LABEL_RANGE) + end = (job.metadata.labels or {}).get(config.LABEL_RANGE) if end is not None: jobs.setdefault(str(end), []).append(job) @@ -91,7 +92,7 @@ async def owner_ref(): global _owner if _owner is None: cm = await core_v1.read_namespaced_config_map( - f"{cfg.RUN_NAME}-stellar-core-config", cfg.NAMESPACE) + f"{config.RUN_NAME}-stellar-core-config", config.NAMESPACE) _owner = [client.V1OwnerReference( api_version='v1', kind='ConfigMap', name=cm.metadata.name, uid=cm.metadata.uid, block_owner_deletion=True)] @@ -107,7 +108,7 @@ async def create_job(body): """ async with _slots: try: - return await batch_v1.create_namespaced_job(cfg.NAMESPACE, body) + return await batch_v1.create_namespaced_job(config.NAMESPACE, body) except ApiException as e: if e.status != 409: raise @@ -115,10 +116,10 @@ async def create_job(body): async def ensure_pvc(end, owner): - name = f"{cfg.RUN_NAME}-data-r{end}" + name = f"{config.RUN_NAME}-data-r{end}" async with _slots: try: - await core_v1.read_namespaced_persistent_volume_claim(name, cfg.NAMESPACE) + await core_v1.read_namespaced_persistent_volume_claim(name, config.NAMESPACE) return name except ApiException as e: if e.status != 404: @@ -126,16 +127,16 @@ async def ensure_pvc(end, owner): spec = client.V1PersistentVolumeClaimSpec( access_modes=['ReadWriteOnce'], resources=client.V1VolumeResourceRequirements( - requests={'storage': cfg.STORAGE_SIZE})) - if cfg.STORAGE_CLASS: - spec.storage_class_name = cfg.STORAGE_CLASS + requests={'storage': mc.STORAGE_SIZE})) + if mc.STORAGE_CLASS: + spec.storage_class_name = mc.STORAGE_CLASS try: await core_v1.create_namespaced_persistent_volume_claim( - cfg.NAMESPACE, client.V1PersistentVolumeClaim( + config.NAMESPACE, client.V1PersistentVolumeClaim( metadata=client.V1ObjectMeta( name=name, owner_references=owner, - labels={cfg.LABEL_RUN: cfg.RUN_NAME, - cfg.LABEL_RANGE: str(end)}), + labels={config.LABEL_RUN: config.RUN_NAME, + config.LABEL_RANGE: str(end)}), spec=spec)) except ApiException as e: if e.status != 409: @@ -150,14 +151,14 @@ async def reap(end, job_names): leaving it behind holds a PVC and shows up in the next pass's list. """ await asyncio.gather(*(delete_job(name) for name in job_names)) - if cfg.STORAGE_MODE == 'pvc': + if config.STORAGE_MODE == 'pvc': await _release_pvc(end) async def delete_job(name): async with _slots: try: - await batch_v1.delete_namespaced_job(name, cfg.NAMESPACE, + await batch_v1.delete_namespaced_job(name, config.NAMESPACE, propagation_policy='Background') return True except ApiException as e: @@ -170,7 +171,7 @@ async def _release_pvc(end): async with _slots: try: await core_v1.delete_namespaced_persistent_volume_claim( - f"{cfg.RUN_NAME}-data-r{end}", cfg.NAMESPACE) + f"{config.RUN_NAME}-data-r{end}", config.NAMESPACE) return True except ApiException as e: if e.status != 404: diff --git a/src/MissionParallelCatchup/lib/monitor/dispatch.py b/src/MissionParallelCatchup/lib/monitor/dispatch.py index a216f037..56843b61 100644 --- a/src/MissionParallelCatchup/lib/monitor/dispatch.py +++ b/src/MissionParallelCatchup/lib/monitor/dispatch.py @@ -5,6 +5,7 @@ import cluster import config +import monitor_config as mc import sizing logger = logging.getLogger('job_monitor') @@ -56,8 +57,8 @@ def range_list(): different list means work silently duplicated or skipped, and nothing else would notice. """ - start, latest = config.STARTING_LEDGER, config.LATEST_LEDGER_NUM - per_job, overlap = config.LEDGERS_PER_JOB, config.OVERLAP_LEDGERS + start, latest = mc.STARTING_LEDGER, mc.LATEST_LEDGER_NUM + per_job, overlap = mc.LEDGERS_PER_JOB, mc.OVERLAP_LEDGERS # Strictly greater, and overlap added on top of the clamped stride: `>=` # emits a range ending AT the start ledger, which is below genesis and @@ -68,11 +69,11 @@ def range_list(): ranges.append((end, stride + overlap)) end -= stride - if config.RANGE_ORDER == 'oldest-first': + if mc.RANGE_ORDER == 'oldest-first': # The cheap ranges finish first, which is what a profiling run wants: it # measures the inexpensive end before anything can interrupt it. return list(reversed(ranges)) - if config.RANGE_ORDER == 'longest-first': + if mc.RANGE_ORDER == 'longest-first': # Validated at /start, so a profile exists here. return sorted(ranges, key=_measured_seconds, reverse=True) # tip-first: the bucket set only grows with ledger position, so the tip @@ -128,8 +129,8 @@ def _job(end, count, attempt, owner, data_volume, oom_count, memory, ephemeral): # Retries are the monitor's, not the controller's: raising a memory # request needs a new Job, because spec.template is immutable. backoff_limit=0, - active_deadline_seconds=config.ATTEMPT_DEADLINE_SECONDS or None, - ttl_seconds_after_finished=config.JOB_TTL_SECONDS, + active_deadline_seconds=mc.ATTEMPT_DEADLINE_SECONDS or None, + ttl_seconds_after_finished=mc.JOB_TTL_SECONDS, pod_failure_policy=client.V1PodFailurePolicy(rules=_failure_rules()), template=client.V1PodTemplateSpec( # LABEL_ATTEMPT has to be on the POD too: the collector reads it @@ -166,10 +167,10 @@ def _pod(end, count, attempt, data_volume, oom_count, memory, ephemeral): requests, limits = sizing.requests_for(end, oom_count, memory, ephemeral) script = RESUME_SCRIPT % {'key': job_key(end, count), 'target': end, 'count': count} container = client.V1Container( - name='stellar-core', image=config.CORE_IMAGE, + name='stellar-core', image=mc.CORE_IMAGE, command=['/bin/sh', '-c', script], - env=([client.V1EnvVar(name='ASAN_OPTIONS', value=config.ASAN_OPTIONS)] - if config.ASAN_OPTIONS else []), + env=([client.V1EnvVar(name='ASAN_OPTIONS', value=mc.ASAN_OPTIONS)] + if mc.ASAN_OPTIONS else []), resources=client.V1ResourceRequirements(requests=requests, limits=limits), ports=[client.V1ContainerPort(container_port=11626, name='http')], lifecycle=_prestop(), @@ -178,13 +179,13 @@ def _pod(end, count, attempt, data_volume, oom_count, memory, ephemeral): return client.V1PodSpec( # IRSA for the S3 history mirror; without it workers fall back to the # public archive, which throttles at 1024. - service_account_name=config.WORKER_SERVICE_ACCOUNT or None, + service_account_name=mc.WORKER_SERVICE_ACCOUNT or None, # Never restarted in place: the pod stays terminal and inspectable. restart_policy='Never', - termination_grace_period_seconds=config.WORKER_GRACE_SECONDS, + termination_grace_period_seconds=mc.WORKER_GRACE_SECONDS, affinity=_affinity(end, oom_count), - tolerations=([client.V1Toleration(key=config.TOLERATE_TAINT, effect='NoSchedule')] - if config.TOLERATE_TAINT else None), + tolerations=([client.V1Toleration(key=mc.TOLERATE_TAINT, effect='NoSchedule')] + if mc.TOLERATE_TAINT else None), containers=[container], volumes=[data_volume, client.V1Volume( name='config', config_map=client.V1ConfigMapVolumeSource( @@ -198,22 +199,22 @@ def _affinity(end, oom_count): avoid-only pod in its own term would match every node. """ match = [] - if config.NODE_LABEL_KEY: + if mc.NODE_LABEL_KEY: match.append(client.V1NodeSelectorRequirement( - key=config.NODE_LABEL_KEY, operator='In', + key=mc.NODE_LABEL_KEY, operator='In', values=[sizing.node_label_value(end, oom_count)])) - for key, value in config.label_pairs(config.REQUIRE_NODE_LABELS): + for key, value in mc.label_pairs(mc.REQUIRE_NODE_LABELS): # Literal, unlike the pool-routed pair above: properties of the pool # rather than of the range, so they do not vary per attempt. match.append(client.V1NodeSelectorRequirement( key=key, operator='In', values=[value])) - if config.AVOID_NODE_LABEL_KEY: + if mc.AVOID_NODE_LABEL_KEY: # No value means "avoid the label however it is set", which is # DoesNotExist; NotIn [""] would only exclude the empty value. match.append(client.V1NodeSelectorRequirement( - key=config.AVOID_NODE_LABEL_KEY, - operator='NotIn' if config.AVOID_NODE_LABEL_VALUE else 'DoesNotExist', - values=[config.AVOID_NODE_LABEL_VALUE] if config.AVOID_NODE_LABEL_VALUE else None)) + key=mc.AVOID_NODE_LABEL_KEY, + operator='NotIn' if mc.AVOID_NODE_LABEL_VALUE else 'DoesNotExist', + values=[mc.AVOID_NODE_LABEL_VALUE] if mc.AVOID_NODE_LABEL_VALUE else None)) if not match: return None return client.V1Affinity(node_affinity=client.V1NodeAffinity( @@ -229,12 +230,12 @@ def _prestop(): container anyway -- so the delay is not bought and an error is logged for every evicted pod. """ - sleep = config.WORKER_PRESTOP_SLEEP_SECONDS + sleep = mc.WORKER_PRESTOP_SLEEP_SECONDS if sleep <= 0: return None - if sleep >= config.WORKER_GRACE_SECONDS: + if sleep >= mc.WORKER_GRACE_SECONDS: logger.warning("preStop %ss does not fit in grace %ss; not installing it", - sleep, config.WORKER_GRACE_SECONDS) + sleep, mc.WORKER_GRACE_SECONDS) return None return client.V1Lifecycle(pre_stop=client.V1LifecycleHandler( _exec=client.V1ExecAction(command=['/bin/sleep', str(sleep)]))) diff --git a/src/MissionParallelCatchup/lib/monitor/liveness.py b/src/MissionParallelCatchup/lib/monitor/liveness.py index 01239d33..eb402a27 100644 --- a/src/MissionParallelCatchup/lib/monitor/liveness.py +++ b/src/MissionParallelCatchup/lib/monitor/liveness.py @@ -10,7 +10,7 @@ import aiohttp -import config +import monitor_config as mc logger = logging.getLogger('job_monitor') @@ -51,12 +51,12 @@ async def sweep(targets): counts = {'up': 0, 'down': 0, 'unknown': len(targets)} # force_close: a pooled socket to a vanished pod gets handed back out. # `limit` is the concurrency bound; a semaphore would double-enforce it. - connector = aiohttp.TCPConnector(limit=config.LIVENESS_MAX_CONCURRENCY, + connector = aiohttp.TCPConnector(limit=mc.LIVENESS_MAX_CONCURRENCY, force_close=True) - timeout = aiohttp.ClientTimeout(total=config.LIVENESS_PROBE_TIMEOUT_SECONDS) + timeout = aiohttp.ClientTimeout(total=mc.LIVENESS_PROBE_TIMEOUT_SECONDS) async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: tasks = [asyncio.create_task(_probe(session, ip)) for _, ip in targets.values()] - done, pending = await asyncio.wait(tasks, timeout=config.LIVENESS_SWEEP_SECONDS) + done, pending = await asyncio.wait(tasks, timeout=mc.LIVENESS_SWEEP_SECONDS) for task in pending: task.cancel() if pending: diff --git a/src/MissionParallelCatchup/lib/monitor/monitor_config.py b/src/MissionParallelCatchup/lib/monitor/monitor_config.py new file mode 100644 index 00000000..3b0f15ea --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/monitor_config.py @@ -0,0 +1,218 @@ +"""Every knob the monitor owns, and the validation a run is admitted through. + +What both processes must agree on -- the run's identity, the shared volume, the +verdict vocabulary -- lives in config, which this reads through. + +Read through the module, never copied out of it: + + import monitor_config as mc + ... mc.REQ_CPU ... + +`from monitor_config import REQ_CPU` binds a COPY. /start rebinds several of +these after validating them, and a copy taken at import time never sees the +rebind -- silently, with the run using the default. + +Numbers that arrive from the chart stay strings until validate() coerces them. +Coercing at import made a bad value a boot crash, and a process that cannot +start cannot report why. +""" +import logging +import os + +import config + +logger = logging.getLogger('job_monitor') + + +def _int(name, default): + """A chart-env integer, typed here rather than at /start. + + The loop idles on its interval while waiting for /start, so a string there + crash-loops the pod before it can be told anything. A bad value falls back + loudly rather than killing the boot; /start-delivered values get a 400. + """ + raw = os.getenv(name) + if raw is None: + return default + try: + return int(raw) + except (TypeError, ValueError): + logger.error("%s=%r is not an integer; using %d", name, raw, default) + return default + + +# --- the run ---------------------------------------------------------------- +PARALLELISM = _int('PARALLELISM', 3) +RECONCILE_INTERVAL_SECONDS = _int('LOGGING_INTERVAL_SECONDS', 10) +HTTP_PORT = _int('HTTP_PORT', 8080) + +# Delivered by POST /start, replayed from run.json on restart. +STARTING_LEDGER = '0' +LATEST_LEDGER_NUM = '0' +LEDGERS_PER_JOB = '16000' +OVERLAP_LEDGERS = '320' +RANGE_ORDER = 'tip-first' +PROFILE = [] # sorted [(end, record)] +_SORTED_SECONDS = None # cache, invalidated whenever PROFILE is set + +RANGE_ORDERS = ('tip-first', 'oldest-first', 'longest-first') + +# --- the worker pod --------------------------------------------------------- +CORE_IMAGE = os.getenv('CORE_IMAGE', 'stellar/stellar-core:latest') +ASAN_OPTIONS = os.getenv('ASAN_OPTIONS', '') +WORKER_SERVICE_ACCOUNT = os.getenv('WORKER_SERVICE_ACCOUNT', '') +WORKER_GRACE_SECONDS = int(os.getenv('WORKER_GRACE_SECONDS', '150')) +# Holds the pod open after SIGTERM so the collector can drain the last of its +# log. Must fit inside the grace period; a preStop the kubelet kills mid-sleep +# buys nothing and logs FailedPreStopHook on every evicted pod. +WORKER_PRESTOP_SLEEP_SECONDS = int(os.getenv('WORKER_PRESTOP_SLEEP_SECONDS', '5')) + +STORAGE_CLASS = os.getenv('STORAGE_CLASS', '') +STORAGE_SIZE = os.getenv('STORAGE_SIZE', '60Gi') + +JOB_TTL_SECONDS = int(os.getenv('JOB_TTL_SECONDS', '3600')) +# 0 disables. A timeout is terminal, so a tight bound trades a certain +# catastrophe against a rounding error: one wedged range holds one slot. +ATTEMPT_DEADLINE_SECONDS = int(os.getenv('ATTEMPT_DEADLINE_SECONDS', '0')) + +# --- unpooled / unprofiled sizing ------------------------------------------- +# The packing unit for a run with no profile: 4 workers per r8*.2xlarge +# (cpu-bound) or 3 per m8*.2xlarge (memory-bound). The price of not measuring. +REQ_CPU = os.getenv('REQ_CPU', '1800m') +REQ_MEM = os.getenv('REQ_MEM', '9Gi') +REQ_EPHEMERAL = os.getenv('REQ_EPHEMERAL', '') +LIM_EPHEMERAL = os.getenv('LIM_EPHEMERAL', '') + +# --- profile-derived sizing (unpooled) -------------------------------------- +PROFILE_MARGIN = float(os.getenv('PROFILE_MARGIN', '1.15')) +# Flat, because a multiplicative margin is nothing at small rss: 1.15x of +# 190Mi is 19Mi of slack, and 90 ranges OOMKilled inside 90s on it. +PROFILE_CACHE_HEADROOM = os.getenv('PROFILE_CACHE_HEADROOM', '512Mi') +PROFILE_RUNTIME_MEMORY_INSURANCE = os.getenv('PROFILE_RUNTIME_MEMORY_INSURANCE', '3Gi') +PROFILE_MAX_MEM = os.getenv('PROFILE_MAX_MEM', '32Gi') +# Image, logs, sqlite WAL: none of it scales with the range. +PROFILE_EPHEMERAL_HEADROOM = os.getenv('PROFILE_EPHEMERAL_HEADROOM', '2Gi') +# Disk tracks runtime closely (pearson 0.920 over 3985 ranges). +PROFILE_RUNTIME_EPHEMERAL_INSURANCE = os.getenv('PROFILE_RUNTIME_EPHEMERAL_INSURANCE', '8Gi') +# Above the flat limit on purpose: that limit is what an UNMEASURED range gets. +PROFILE_MAX_EPHEMERAL = os.getenv('PROFILE_MAX_EPHEMERAL', '64Gi') + +# --- escalation ------------------------------------------------------------- +MEM_BUMP_FACTOR = float(os.getenv('MEM_BUMP_FACTOR', '1.5')) +MEM_ESCALATION_CAP = os.getenv('MAX_MEM', '48Gi') +EPH_BUMP_FACTOR = float(os.getenv('EPH_BUMP_FACTOR', '1.5')) +EPH_ESCALATION_CAP = os.getenv('MAX_EPHEMERAL', '200Gi') + +# --- pools ------------------------------------------------------------------ +# Empty disables routing entirely; every worker keeps NODE_LABEL_VALUE. +POOL_PREFIX = os.getenv('POOL_PREFIX', '') +POOL_TIERS = os.getenv( + 'POOL_TIERS', + '0:subdwarf,0.79:dwarf,1.61:subgiant,3.87:giant,8.85:supergiant,' + '18.38:hypergiant,:supernova') +# Fits once in the tier's on-demand shape, twice in its (one size larger) spot +# shape. Not 50% of either -- see the requirements. +POOL_MEM = os.getenv( + 'POOL_MEM', + 'subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,' + 'supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,' + 'protostar:29696Mi,nebula:9216Mi') +POOL_CPU = os.getenv( + 'POOL_CPU', + 'subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,' + 'hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80') +POOL_BLOCK_RUNGS = os.getenv('POOL_BLOCK_RUNGS', 'dwarf->subgiant,hypergiant->supernova') +POOL_UNPROFILED = os.getenv('POOL_UNPROFILED', 'protostar') +POOL_NO_PROFILE = os.getenv('POOL_NO_PROFILE', 'nebula') + +# --- placement -------------------------------------------------------------- +NODE_LABEL_KEY = os.getenv('NODE_LABEL_KEY', '') +NODE_LABEL_VALUE = os.getenv('NODE_LABEL_VALUE', '') +REQUIRE_NODE_LABELS = os.getenv('REQUIRE_NODE_LABELS', '') +AVOID_NODE_LABEL_KEY = os.getenv('AVOID_NODE_LABEL_KEY', '') +AVOID_NODE_LABEL_VALUE = os.getenv('AVOID_NODE_LABEL_VALUE', '') +TOLERATE_TAINT = os.getenv('TOLERATE_TAINT', '') + +# --- retry budgets ---------------------------------------------------------- +# The whole retry policy. A cause that is NOT here is condemned the first time +# it happens. Every budget is spent by its own cause: one shared attempt index +# let evictions drain the OOM budget to zero. +ATTEMPT_BUDGETS = { + 'disrupted': int(os.getenv('MAX_DISRUPTION_ATTEMPTS', '100')), + 'rejected': int(os.getenv('MAX_REJECTED_ATTEMPTS', '100')), + 'fetch-fault': int(os.getenv('MAX_FETCH_FAULT_ATTEMPTS', '20')), + 'oom': int(os.getenv('MAX_OOM_ATTEMPTS', '5')), + 'ephemeral': int(os.getenv('MAX_EPHEMERAL_ATTEMPTS', '4')), + 'unknown': int(os.getenv('MAX_UNKNOWN_ATTEMPTS', '2')), +} + +# --- liveness --------------------------------------------------------------- +LIVENESS_MAX_CONCURRENCY = int(os.getenv('LIVENESS_MAX_CONCURRENCY', '64')) +LIVENESS_PROBE_TIMEOUT_SECONDS = float(os.getenv('LIVENESS_PROBE_TIMEOUT_SECONDS', '2')) +# Under the reconcile interval on purpose: the pass awaits this sweep, so a +# deadline above the interval lets an unreachable fleet stretch dispatch and +# reaping behind a probe that only feeds a dashboard. +LIVENESS_SWEEP_SECONDS = float(os.getenv('LIVENESS_SWEEP_SECONDS', '5')) + +# Sized for the dispatch burst rather than a steady rate: a wave head creates +# ~1024 Jobs and PVCs at once. +APISERVER_CONCURRENCY = int(os.getenv('APISERVER_CONCURRENCY', '64')) + + +def label_pairs(raw): + """[(key, value)] from "k:v,k:v". A key with no value is dropped -- it would + require the label be exactly "", which no node carries, and the pod sits + Pending in a way that reads as slow provisioning.""" + out = [] + for item in (raw or '').split(','): + key, _, value = item.strip().partition(':') + if key and value: + out.append((key, value)) + return out + + +def set_profile(profile): + global PROFILE, _SORTED_SECONDS + PROFILE = profile + _SORTED_SECONDS = None + + +def validate(): + """Coerce and re-bind everything a run depends on. Raises ValueError. + + Called from /start, which is the first moment the configuration is + complete, so a misconfigured run is answered with a 400 rather than + crash-looping a pod the driver can only time out on. + """ + global STARTING_LEDGER, LATEST_LEDGER_NUM, LEDGERS_PER_JOB, OVERLAP_LEDGERS + + def positive(name, value, floor=1): + try: + n = int(value) + except (TypeError, ValueError): + raise ValueError(f"{name} is not an integer: {value!r}") + if n < floor: + raise ValueError(f"{name} must be >= {floor}, got {n}") + return n + + STARTING_LEDGER = positive('STARTING_LEDGER', STARTING_LEDGER, floor=0) + LATEST_LEDGER_NUM = positive('LATEST_LEDGER_NUM', LATEST_LEDGER_NUM) + LEDGERS_PER_JOB = positive('LEDGERS_PER_JOB', LEDGERS_PER_JOB) + OVERLAP_LEDGERS = positive('OVERLAP_LEDGERS', OVERLAP_LEDGERS, floor=0) + + if LATEST_LEDGER_NUM < STARTING_LEDGER: + raise ValueError(f"LATEST_LEDGER_NUM {LATEST_LEDGER_NUM} is below " + f"STARTING_LEDGER {STARTING_LEDGER}") + if RANGE_ORDER not in RANGE_ORDERS: + raise ValueError(f"RANGE_ORDER must be one of {RANGE_ORDERS}, " + f"got {RANGE_ORDER!r}") + if config.STORAGE_MODE not in ('pvc', 'ephemeral'): + raise ValueError(f"STORAGE_MODE must be pvc or ephemeral, " + f"got {config.STORAGE_MODE!r}") + if RANGE_ORDER == 'longest-first' and not PROFILE: + # With no profile every range ties, the sort is a no-op, and dispatch + # silently falls back to tip-first while the operator believes + # otherwise. + raise ValueError( + "RANGE_ORDER=longest-first requires a profile: it orders ranges by " + "measured seconds. POST a profile, or set another order.") diff --git a/src/MissionParallelCatchup/lib/monitor/policy.py b/src/MissionParallelCatchup/lib/monitor/policy.py index 4b9899a7..cc5a8930 100644 --- a/src/MissionParallelCatchup/lib/monitor/policy.py +++ b/src/MissionParallelCatchup/lib/monitor/policy.py @@ -10,7 +10,7 @@ """ import collections -import config +import monitor_config as mc import sizing Decision = collections.namedtuple('Decision', 'action reason memory ephemeral') @@ -27,7 +27,7 @@ def decide(end, verdict, spent, base_memory=None, base_ephemeral=None): is broken. """ cause = verdict.get('outcome') - cap = config.ATTEMPT_BUDGETS.get(cause) + cap = mc.ATTEMPT_BUDGETS.get(cause) if cap is None: return Decision(CONDEMN, f"{cause} is not retryable", None, None) diff --git a/src/MissionParallelCatchup/lib/monitor/record.py b/src/MissionParallelCatchup/lib/monitor/record.py index ac9a9520..b4b7ddb1 100644 --- a/src/MissionParallelCatchup/lib/monitor/record.py +++ b/src/MissionParallelCatchup/lib/monitor/record.py @@ -1,11 +1,10 @@ -"""The volume: the collector's files, the monitor's own, and the run record. +"""What the monitor keeps on the volume: the run record, and reads of the +collector's files. -Filenames are the entire cross-process contract. The collector writes while a -pod still exists and the monitor reads them back, so a disagreement about a name -is a measurement silently lost and nothing reports it. - -Every write goes through tmp+rename. Both sides write while the other reads, so -a torn .metrics or .outcome reads as corrupt and the measurement is gone. +The filenames themselves are NOT here. They are the cross-process contract and +live in records.py, which both processes import -- a second copy is a second +place for the two sides to disagree about a name, which is a measurement lost +with nothing to report it. """ import collections import json @@ -13,29 +12,8 @@ import time import config - -# --- the collector writes these --------------------------------------------- - - -def log_path(end, attempt): - return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.log.gz") - - -def outcome_path(end, attempt): - return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.outcome") - - -def metrics_path(end, attempt): - return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.metrics") - - -def done_path(end, attempt): - return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.done") - - -def state_path(end, attempt): - return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.state") - +from records import (done_path, log_path, metrics_path, outcome_path, # noqa: F401 + state_path, write_atomic) # --- the monitor writes these ----------------------------------------------- @@ -51,13 +29,6 @@ def started_path(end): MISSION_START_PATH = os.path.join(config.LOG_DIR, 'mission_started') -def write_atomic(path, body): - tmp = path + '.tmp' - with open(tmp, 'wt') as fh: - fh.write(body) - os.replace(tmp, path) - - def _read_json(path): try: with open(path) as fh: diff --git a/src/MissionParallelCatchup/lib/monitor/server.py b/src/MissionParallelCatchup/lib/monitor/server.py index 9c6327a1..09bef29d 100644 --- a/src/MissionParallelCatchup/lib/monitor/server.py +++ b/src/MissionParallelCatchup/lib/monitor/server.py @@ -12,6 +12,7 @@ from prometheus_client import CONTENT_TYPE_LATEST, generate_latest import config +import monitor_config as mc import record logger = logging.getLogger('job_monitor') @@ -40,9 +41,9 @@ async def serve(state, stop): """ runner = web.AppRunner(build(state), access_log=None) await runner.setup() - site = web.TCPSite(runner, '0.0.0.0', config.HTTP_PORT) + site = web.TCPSite(runner, '0.0.0.0', mc.HTTP_PORT) await site.start() - logger.info("listening on :%d", config.HTTP_PORT) + logger.info("listening on :%d", mc.HTTP_PORT) try: await stop.wait() finally: @@ -70,7 +71,7 @@ async def _start(request): # on disk. record.save_run(doc) return web.json_response({'ranges': len(state.ranges), - 'profile': len(config.PROFILE)}) + 'profile': len(mc.PROFILE)}) async def _status(request): diff --git a/src/MissionParallelCatchup/lib/monitor/sizing.py b/src/MissionParallelCatchup/lib/monitor/sizing.py index a18e94da..c1036626 100644 --- a/src/MissionParallelCatchup/lib/monitor/sizing.py +++ b/src/MissionParallelCatchup/lib/monitor/sizing.py @@ -11,7 +11,7 @@ import math import re -import config +import monitor_config as mc _QUANTITY = re.compile(r'^(?P\d+(?:\.\d+)?)(?P[EPTGMK]i?|m)?$') _FACTOR = {'K': 10 ** 3, 'M': 10 ** 6, 'G': 10 ** 9, 'T': 10 ** 12, @@ -56,11 +56,11 @@ def profile_for(end): position because the bucket set only grows, so a lower neighbour under-reports. Past the top there is nothing safe to extrapolate from. """ - if not config.PROFILE: + if not mc.PROFILE: return None - idx = bisect.bisect_left(config.PROFILE, (int(end),)) - if idx < len(config.PROFILE): - return config.PROFILE[idx][1] + idx = bisect.bisect_left(mc.PROFILE, (int(end),)) + if idx < len(mc.PROFILE): + return mc.PROFILE[idx][1] return None @@ -73,10 +73,10 @@ def _positive(value): def _longest_seconds(): - if config._SORTED_SECONDS is None: - values = (_positive(rec.get('seconds')) for _, rec in (config.PROFILE or [])) - config._SORTED_SECONDS = sorted(v for v in values if v is not None) - return config._SORTED_SECONDS[-1] if config._SORTED_SECONDS else None + if mc._SORTED_SECONDS is None: + values = (_positive(rec.get('seconds')) for _, rec in (mc.PROFILE or [])) + mc._SORTED_SECONDS = sorted(v for v in values if v is not None) + return mc._SORTED_SECONDS[-1] if mc._SORTED_SECONDS else None def _runtime_insurance(seconds, allowance): @@ -97,7 +97,7 @@ def _tiers(): """[(gib_cut, name)] cheapest first; an empty cut on the last entry means everything above the previous one.""" out = [] - for item in config.POOL_TIERS.split(','): + for item in mc.POOL_TIERS.split(','): cut, _, name = item.strip().rpartition(':') if name: out.append((float(cut) if cut else float('inf'), name)) @@ -118,11 +118,11 @@ def pool_memory(tier): (one size larger) spot node. Not the range's own measurement -- isolation is the point, and freeing a pod of its neighbours raised throughput 29-92% while its cpu draw FELL.""" - return _str_map(config.POOL_MEM).get(tier) + return _str_map(mc.POOL_MEM).get(tier) def pool_cpu(tier): - return _str_map(config.POOL_CPU).get(tier) + return _str_map(mc.POOL_CPU).get(tier) def _tier_for_bytes(anon): @@ -149,7 +149,7 @@ def _promote(tier, steps): def _rung_blocked(tier, nxt): want = f"{tier}->{nxt}" - return any(item.strip() == want for item in config.POOL_BLOCK_RUNGS.split(',')) + return any(item.strip() == want for item in mc.POOL_BLOCK_RUNGS.split(',')) def _cache_bump(tier, anon, working_set): @@ -181,17 +181,17 @@ def pool_for(end, oom_count=0): attempt-1 verdicts were `timeout`, burning ~260 vCPU of a 2304 quota escalating away from a problem that was never memory. """ - if not config.POOL_PREFIX: + if not mc.POOL_PREFIX: return None - if not config.PROFILE: - return _promote(config.POOL_NO_PROFILE, oom_count) + if not mc.PROFILE: + return _promote(mc.POOL_NO_PROFILE, oom_count) prof = profile_for(end) if not prof: - return _promote(config.POOL_UNPROFILED, oom_count) + return _promote(mc.POOL_UNPROFILED, oom_count) anon = prof.get('peakAnonBytes') tier = _cache_bump(_tier_for_bytes(anon), anon, prof.get('peakWorkingSetBytes')) if not tier: - return _promote(config.POOL_UNPROFILED, oom_count) + return _promote(mc.POOL_UNPROFILED, oom_count) return _promote(tier, oom_count) @@ -211,14 +211,14 @@ def next_memory(end, base, oom_count): Escalating a 209Mi profiled range off the configured default jumps to 36000Mi, a 172x overshoot that throws away the packing win on the first OOM. """ - if config.POOL_PREFIX: + if mc.POOL_PREFIX: promoted = pool_memory(pool_for(end, oom_count)) # Above the ladder there is nothing to promote into, so hold rather than # invent a value. - return promoted or base or config.REQ_MEM - want = int(quantity_bytes(base or config.REQ_MEM) - * (config.MEM_BUMP_FACTOR ** max(0, oom_count))) - return bytes_to_quantity(min(want, quantity_bytes(config.MEM_ESCALATION_CAP))) + return promoted or base or mc.REQ_MEM + want = int(quantity_bytes(base or mc.REQ_MEM) + * (mc.MEM_BUMP_FACTOR ** max(0, oom_count))) + return bytes_to_quantity(min(want, quantity_bytes(mc.MEM_ESCALATION_CAP))) def next_ephemeral(base, eviction_count): @@ -227,11 +227,11 @@ def next_ephemeral(base, eviction_count): Only ephemeral mode has this axis: a pvc run sets no ephemeral request or limit at all, so there is nothing to raise. """ - if not config.LIM_EPHEMERAL: + if not mc.LIM_EPHEMERAL: return None - want = int(quantity_bytes(base or config.LIM_EPHEMERAL) - * (config.EPH_BUMP_FACTOR ** max(0, eviction_count))) - return bytes_to_quantity(min(want, quantity_bytes(config.EPH_ESCALATION_CAP))) + want = int(quantity_bytes(base or mc.LIM_EPHEMERAL) + * (mc.EPH_BUMP_FACTOR ** max(0, eviction_count))) + return bytes_to_quantity(min(want, quantity_bytes(mc.EPH_ESCALATION_CAP))) # --- the request ------------------------------------------------------------ @@ -245,20 +245,20 @@ def _profile_overrides(end, escalated, oom_count): the escalation, and bailing here would send the pod to the new pool still asking for the old tier's memory. """ - if end is None or (escalated and not config.POOL_PREFIX): + if end is None or (escalated and not mc.POOL_PREFIX): return {} prof = profile_for(end) out = {} if prof: disk = prof.get('peakEphemeralBytes') - if disk and config.LIM_EPHEMERAL: - want = (int(disk * config.PROFILE_MARGIN) - + quantity_bytes(config.PROFILE_EPHEMERAL_HEADROOM) + if disk and mc.LIM_EPHEMERAL: + want = (int(disk * mc.PROFILE_MARGIN) + + quantity_bytes(mc.PROFILE_EPHEMERAL_HEADROOM) + _runtime_insurance(prof.get('seconds'), - config.PROFILE_RUNTIME_EPHEMERAL_INSURANCE)) + mc.PROFILE_RUNTIME_EPHEMERAL_INSURANCE)) out['ephemeral-storage'] = bytes_to_quantity( - min(want, quantity_bytes(config.PROFILE_MAX_EPHEMERAL))) - if config.POOL_PREFIX: + min(want, quantity_bytes(mc.PROFILE_MAX_EPHEMERAL))) + if mc.POOL_PREFIX: # Deliberately BEFORE the no-profile bail: pool_for resolves a tier for # every range, so returning {} here would pin the pod to that pool while # sizing it from the flat REQ_CPU. That shipped a 6780m request at a @@ -277,12 +277,12 @@ def _profile_overrides(end, escalated, oom_count): # 19Mi of slack, and 90 ranges OOMKilled inside 90s without it. rss = prof.get('peakAnonBytes') if rss: - want = (int(rss * config.PROFILE_MARGIN) - + quantity_bytes(config.PROFILE_CACHE_HEADROOM) + want = (int(rss * mc.PROFILE_MARGIN) + + quantity_bytes(mc.PROFILE_CACHE_HEADROOM) + _runtime_insurance(prof.get('seconds'), - config.PROFILE_RUNTIME_MEMORY_INSURANCE)) + mc.PROFILE_RUNTIME_MEMORY_INSURANCE)) out['memory'] = bytes_to_quantity( - min(want, quantity_bytes(config.PROFILE_MAX_MEM))) + min(want, quantity_bytes(mc.PROFILE_MAX_MEM))) return out @@ -294,27 +294,27 @@ def requests_for(end, oom_count=0, memory=None, ephemeral=None): """ overrides = _profile_overrides(end, escalated=bool(memory or ephemeral), oom_count=oom_count) - req = {'cpu': config.REQ_CPU, 'memory': memory or config.REQ_MEM} + req = {'cpu': mc.REQ_CPU, 'memory': memory or mc.REQ_MEM} lim = {} # A profiled pooled range owns its node -- the tier's cut excludes a second # pod -- so a disk limit guards no neighbour and only turns spare disk into # an eviction. Unprofiled pooled runs keep both: nothing measured them. - pooled_profiled = bool(config.POOL_PREFIX and config.PROFILE) - if config.REQ_EPHEMERAL and not pooled_profiled: - req['ephemeral-storage'] = ephemeral or config.REQ_EPHEMERAL + pooled_profiled = bool(mc.POOL_PREFIX and mc.PROFILE) + if mc.REQ_EPHEMERAL and not pooled_profiled: + req['ephemeral-storage'] = ephemeral or mc.REQ_EPHEMERAL else: overrides.pop('ephemeral-storage', None) - if config.LIM_EPHEMERAL and not pooled_profiled: - lim['ephemeral-storage'] = ephemeral or config.LIM_EPHEMERAL + if mc.LIM_EPHEMERAL and not pooled_profiled: + lim['ephemeral-storage'] = ephemeral or mc.LIM_EPHEMERAL for key, value in overrides.items(): req[key] = value - if key == 'ephemeral-storage' and config.LIM_EPHEMERAL: + if key == 'ephemeral-storage' and mc.LIM_EPHEMERAL: lim[key] = value return req, (lim or None) def node_label_value(end, oom_count): tier = pool_for(end, oom_count) - return f"{config.POOL_PREFIX}-{tier}" if tier else config.NODE_LABEL_VALUE + return f"{mc.POOL_PREFIX}-{tier}" if tier else mc.NODE_LABEL_VALUE From ccfaa207e0ec269b31aa26a888b34b274441be2d Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Mon, 17 Aug 2026 17:54:35 -0400 Subject: [PATCH 116/117] Step the resumed chain over an attempt that ran nothing An attempt destroyed after the collector opened it but before it ingested a line leaves an empty .state and no .metrics. It also leaves no LCL, so what its successor resumed was written further back -- but the chain walk steps strictly n-1, so it built [no-op, winner], found no leg for the no-op and dropped the range's compute total. Observed once in ~1000 completed ranges, on a range whose a3 did nothing while a4 continued a2's work. Stepping over it rather than scoring it zero: zero would stop the walk there and credit the range with only its last attempt, which is a silent undercount fed straight into the profile that sizes the next run. Replayed against that range's real artifacts, the walk now builds [a2, a4] and reports 717s, and correctly excludes a1 because a2 ran new-db. ran_nothing requires BOTH an empty .state and no metrics: a follow writes .metrics before the first read, so a pod condemned seconds in has both and did run. An attempt with NO .state is deliberately not covered -- the collector never looked at it, so there is no evidence either way, and dropping the total beats publishing one that cannot be stood behind. Also: heartbeat pod dump keeps the per-pod name/phase/age line for runs under 500 pods and summarises above it, and the interval goes back to 5 minutes. Both are shared by every mission, and a 4-pod mission lost the age field -- the one thing that tells a stuck pod from a slow one -- to a problem only the parallel catchups have. Verified on ssc-test: 7/7 and 7/7 through OOM storms with a forced no-op mid chain, 32/32 on a normal run, and a healthy chain replays byte-identical. Co-Authored-By: Claude Opus 5 --- src/App/Program.fs | 5 +-- src/FSLibrary/StellarSupercluster.fs | 39 +++++++++++++------ .../apps/job_monitor.py | 11 +++++- .../lib/monitor/record.py | 14 +++++++ 4 files changed, 51 insertions(+), 18 deletions(-) diff --git a/src/App/Program.fs b/src/App/Program.fs index e2b5e68e..8cf0b593 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -879,10 +879,7 @@ let main argv = DumpPodInfo kube mission.ApiRateLimit ns with x -> LogError "Connection issue! Api call failed." - // Every 10 minutes, not 5: this lists every pod in the - // namespace, and on a 1024-worker run that is a large response - // fetched purely to print one summary line. - let timer = new System.Threading.Timer(TimerCallback(podLogger), null, 1000, 600000) + let timer = new System.Threading.Timer(TimerCallback(podLogger), null, 1000, 300000) for m in mission.Missions do LogInfo "-----------------------------------" diff --git a/src/FSLibrary/StellarSupercluster.fs b/src/FSLibrary/StellarSupercluster.fs index 362d32af..ba5e01d4 100644 --- a/src/FSLibrary/StellarSupercluster.fs +++ b/src/FSLibrary/StellarSupercluster.fs @@ -217,18 +217,33 @@ let DumpPodInfo (kube: Kubernetes) (apiRateLimit: int) (ns: string) = let pods = kube.ListNamespacedPod(namespaceParameter = ns) if pods <> null then - // A count per phase rather than a line per pod. This fires every 5 - // minutes for the whole mission, so at 1024 workers the old form wrote - // ~1026 lines a time -- ~57000 over a 4.7h catchup -- and buried the - // only thing worth reading, which is anything not Running. - let byPhase = - pods.Items - |> Seq.countBy (fun p -> p.Status.Phase) - |> Seq.sortBy fst - |> Seq.map (fun (phase, n) -> sprintf "%s=%d" phase n) - |> String.concat " " - - LogInfo "Pods: %d total %s" (Seq.length pods.Items) byPhase + let total = Seq.length pods.Items + + // A line per pod carries the name and the age, which is what tells a + // stuck pod from a slow one. Kept for every mission small enough to read + // it; past that it is the parallel catchups, where at 1024 workers this + // wrote ~1026 lines every 5 minutes -- ~57000 over a 4.7h run -- and + // buried the only thing worth reading. + if total < 500 then + LogInfo "There are %d pods in total" total + + for p in pods.Items do + let age = + if p.Status.StartTime.HasValue then + System.DateTime.UtcNow.Subtract(p.Status.StartTime.Value).ToString(@"hh\:mm") + else + "00:00" + + LogInfo "Pod: name=%s phase=%s age=%s (hr:min)" p.Metadata.Name p.Status.Phase age + else + let byPhase = + pods.Items + |> Seq.countBy (fun p -> p.Status.Phase) + |> Seq.sortBy fst + |> Seq.map (fun (phase, n) -> sprintf "%s=%d" phase n) + |> String.concat " " + + LogInfo "Pods: %d total %s" total byPhase // Create a per-run "anchor" ConfigMap and stash an owner reference to it on // `nCfg.anchorOwnerRef` so every subsequent resource the mission creates will diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py index 937b64f4..626b7b83 100644 --- a/src/MissionParallelCatchup/apps/job_monitor.py +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -408,7 +408,7 @@ def aggregate(st): out.update(peaks) # Resumed chain only: a fresh retry discarded its predecessor's work. - chain = _resumed_chain(seen, st.attempt) + chain = _resumed_chain(st.end, seen, st.attempt) for field, source in (('seconds', 'attemptSeconds'), ('txApply', 'txApplySeconds')): legs = [seen[n].get(source) for n in chain] @@ -421,16 +421,23 @@ def aggregate(st): return out -def _resumed_chain(seen, attempt): +def _resumed_chain(end, seen, attempt): """The winning attempt, plus every predecessor it continued from. `resumed` sits on the attempt that continued, so it names its own predecessor. A fresh start breaks the chain: an attempt that ran new-db discarded whatever came before it. + + An attempt that ran nothing is stepped over rather than counted: it left no + LCL, so what its successor resumed was written further back. Counting it + would need a leg it never produced and drop the range's compute total; + stopping there would credit the range with only its last attempt. """ chain, n = [attempt], attempt while n > 1 and seen[n].get('resumed'): n -= 1 + while n > 1 and record.ran_nothing(end, n, seen[n]): + n -= 1 chain.append(n) return sorted(chain) diff --git a/src/MissionParallelCatchup/lib/monitor/record.py b/src/MissionParallelCatchup/lib/monitor/record.py index b4b7ddb1..ccb8a651 100644 --- a/src/MissionParallelCatchup/lib/monitor/record.py +++ b/src/MissionParallelCatchup/lib/monitor/record.py @@ -55,6 +55,20 @@ def unclaimed(end, attempt): return not os.path.exists(state_path(end, attempt)) +def ran_nothing(end, attempt, metrics): + """Opened, but it ingested no line and nothing was measured either. + + An empty .state alone is not enough: a follow writes .metrics before the + first read, so a pod condemned seconds in has both. That one ran. + """ + if metrics: + return False + try: + return os.path.getsize(state_path(end, attempt)) == 0 + except OSError: + return False + + def read_outcome(end, attempt): return _read_json(outcome_path(end, attempt)) From 26e7e66b4bb8de3a3cb923e309f55c41bf42399b Mon Sep 17 00:00:00 2001 From: Jonathan Eid Date: Tue, 18 Aug 2026 11:44:07 -0400 Subject: [PATCH 117/117] Claim an attempt while its pod is still Pending _sweep_vanished finishes any attempt in _last_ts whose pod is gone, but a pod was only entered there once it reached a pollable phase. One deleted from Pending -- what a drain does to a worker just scheduled -- was therefore invisible to the sweep, got no .done, and left its range holding a slot for the rest of the run. The monitor worked around it by settling a failed attempt whose .state was missing; the collector now closes it out itself, so that workaround goes. The marker is the one already written for a pollable pod with nothing read, and hydrate_states already documents an empty .state as "claimed, nothing durable yet". Only when it first appears changes, not what it means. Guarded on .done so a reaped attempt is not re-claimed every cycle. It also collapses the two ways an attempt can leave nothing behind into one signature: a drained-from-Pending attempt now has an empty .state rather than none, so ran_nothing covers it and the resumed chain steps over it like any other no-op. Verified on ssc-test: a force-deleted Pending pod produced .state and .done from the collector's own sweep, the monitor settled it `attempt 1 -> 2 (unknown)` with no backstop present, and the range completed with full measurements. 32/32 ranges, seconds/txApply/wallSeconds all 32/32. Co-Authored-By: Claude Opus 5 --- src/MissionParallelCatchup/apps/job_monitor.py | 3 --- src/MissionParallelCatchup/apps/log_collector.py | 8 ++++++-- src/MissionParallelCatchup/lib/monitor/record.py | 6 ------ 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py index 626b7b83..3feef67a 100644 --- a/src/MissionParallelCatchup/apps/job_monitor.py +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -158,9 +158,6 @@ def observe(states): if st.status not in ('failed', 'completed'): continue st.done = record.is_done(st.end, st.attempt) - # Otherwise the slot is held for the rest of the run. - if not st.done and st.status == 'failed' and st.pod is None: - st.done = record.unclaimed(st.end, st.attempt) if not st.done: continue # the collector is still writing this attempt if st.status == 'failed': diff --git a/src/MissionParallelCatchup/apps/log_collector.py b/src/MissionParallelCatchup/apps/log_collector.py index 4dc3ade0..cdcf7281 100644 --- a/src/MissionParallelCatchup/apps/log_collector.py +++ b/src/MissionParallelCatchup/apps/log_collector.py @@ -78,6 +78,12 @@ async def main(): pods = await list_pods(session) ranges = {p['metadata']['name']: _identify(p) for p in pods if _identify(p)} + # Claimed while Pending too, so the sweep below can finish an + # attempt whose pod is deleted before it is ever pollable. + for end, attempt in ranges.values(): + if (end, attempt) not in _last_ts and not os.path.exists( + records.done_path(end, attempt)): + _write_state(end, attempt, '') nodes = {p['status']['hostIP'] for p in pods if p.get('status', {}).get('hostIP') and p.get('status', {}).get('phase') == 'Running'} @@ -137,8 +143,6 @@ async def service_pod(session, pod): if not terminal and _start_follow(session, name, end, attempt, pod): return # the follow owns this attempt's stream if (end, attempt) not in _last_ts: - # Empty state = "claimed, nothing durable yet": the monitor's backstop - # skips any range that has one, so this stops both of us writing the log. _write_state(end, attempt, '') since = _last_ts[(end, attempt)] # A terminal read reaches further back, so a medida block split across two diff --git a/src/MissionParallelCatchup/lib/monitor/record.py b/src/MissionParallelCatchup/lib/monitor/record.py index ccb8a651..62580c55 100644 --- a/src/MissionParallelCatchup/lib/monitor/record.py +++ b/src/MissionParallelCatchup/lib/monitor/record.py @@ -49,12 +49,6 @@ def is_done(end, attempt): return os.path.exists(done_path(end, attempt)) -def unclaimed(end, attempt): - """No .state, so the collector never opened this attempt and no .done is - coming. A pod deleted before its container started is never pollable.""" - return not os.path.exists(state_path(end, attempt)) - - def ran_nothing(end, attempt, metrics): """Opened, but it ingested no line and nothing was measured either.