diff --git a/.gitignore b/.gitignore index 8a57e922..8cfb3cd9 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,11 @@ .fake .ionide .idea + +# Python bytecode from the parallel-catchup monitor/collector tests +__pycache__/ +*.pyc + +# pytest-cov artifacts from the MissionParallelCatchup suite +.coverage +**/__pycache__/ diff --git a/src/App/Program.fs b/src/App/Program.fs index 7f8075a8..8cf0b593 100644 --- a/src/App/Program.fs +++ b/src/App/Program.fs @@ -114,7 +114,20 @@ type MissionOptions pubnetParallelCatchupStartingLedger: int, pubnetParallelCatchupEndLedger: int option, pubnetParallelCatchupLedgersPerJob: int, + pubnetParallelCatchupOverlapLedgers: int, pubnetParallelCatchupNumWorkers: int, + pubnetParallelCatchupStorageMode: string, + pubnetParallelCatchupProfile: string, + pubnetParallelCatchupRangeOrder: string, + pubnetParallelCatchupPoolPrefix: string, + jobMonitorImagePcV2: string, + pubnetParallelCatchupCpuRequest: string, + pubnetParallelCatchupMemRequest: string, + pubnetParallelCatchupPoolCpu: string, + pubnetParallelCatchupPoolMem: string, + pubnetParallelCatchupCreateRbac: bool, + jobMonitorNodeLabels: seq, + jobMonitorTolerateTaints: seq, tag: string option, numPregeneratedTxs: int option, genesisTestAccountCount: int option, @@ -515,12 +528,88 @@ type MissionOptions Default = 16000)>] member self.PubnetParallelCatchupLedgersPerJob = pubnetParallelCatchupLedgersPerJob + [] + member self.PubnetParallelCatchupOverlapLedgers = pubnetParallelCatchupOverlapLedgers + [] member self.PubnetParallelCatchupNumWorkers = pubnetParallelCatchupNumWorkers + [] + member self.PubnetParallelCatchupStorageMode : string = pubnetParallelCatchupStorageMode + + [] + member self.PubnetParallelCatchupProfile : string = pubnetParallelCatchupProfile + + [] + member self.PubnetParallelCatchupRangeOrder : string = pubnetParallelCatchupRangeOrder + + [] + member self.PubnetParallelCatchupPoolPrefix : string = pubnetParallelCatchupPoolPrefix + + [] + member self.JobMonitorImagePcV2 : string = jobMonitorImagePcV2 + + [] + member self.PubnetParallelCatchupCpuRequest : string = pubnetParallelCatchupCpuRequest + + [] + member self.PubnetParallelCatchupMemRequest : string = pubnetParallelCatchupMemRequest + + [] + member self.PubnetParallelCatchupPoolCpu : string = pubnetParallelCatchupPoolCpu + + [] + member self.PubnetParallelCatchupPoolMem : string = pubnetParallelCatchupPoolMem + + [] + member self.PubnetParallelCatchupCreateRbac : bool = pubnetParallelCatchupCreateRbac + + [] + member self.JobMonitorNodeLabels = jobMonitorNodeLabels + + [] + member self.JobMonitorTolerateTaints = jobMonitorTolerateTaints + [] member self.Tag = tag @@ -897,7 +986,21 @@ let main argv = pubnetParallelCatchupStartingLedger = mission.PubnetParallelCatchupStartingLedger pubnetParallelCatchupEndLedger = mission.PubnetParallelCatchupEndLedger pubnetParallelCatchupLedgersPerJob = mission.PubnetParallelCatchupLedgersPerJob + pubnetParallelCatchupOverlapLedgers = mission.PubnetParallelCatchupOverlapLedgers pubnetParallelCatchupNumWorkers = mission.PubnetParallelCatchupNumWorkers + pubnetParallelCatchupStorageMode = mission.PubnetParallelCatchupStorageMode + pubnetParallelCatchupProfile = mission.PubnetParallelCatchupProfile + pubnetParallelCatchupRangeOrder = mission.PubnetParallelCatchupRangeOrder + pubnetParallelCatchupPoolPrefix = mission.PubnetParallelCatchupPoolPrefix + jobMonitorImagePcV2 = mission.JobMonitorImagePcV2 + pubnetParallelCatchupCpuRequest = mission.PubnetParallelCatchupCpuRequest + pubnetParallelCatchupMemRequest = mission.PubnetParallelCatchupMemRequest + pubnetParallelCatchupPoolCpu = mission.PubnetParallelCatchupPoolCpu + pubnetParallelCatchupPoolMem = mission.PubnetParallelCatchupPoolMem + pubnetParallelCatchupCreateRbac = mission.PubnetParallelCatchupCreateRbac + jobMonitorNodeLabels = List.map splitLabel (List.ofSeq mission.JobMonitorNodeLabels) + jobMonitorTolerateTaints = + List.map splitLabel (List.ofSeq mission.JobMonitorTolerateTaints) tag = mission.Tag numPregeneratedTxs = mission.NumPregeneratedTxs enableTailLogging = true diff --git a/src/FSLibrary.Tests/FSLibrary.Tests.fsproj b/src/FSLibrary.Tests/FSLibrary.Tests.fsproj index e9164c38..60b925ba 100644 --- a/src/FSLibrary.Tests/FSLibrary.Tests.fsproj +++ b/src/FSLibrary.Tests/FSLibrary.Tests.fsproj @@ -1,33 +1,33 @@ - - - - net8.0 - - false - false - Exe - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - runtime; build; native; contentfiles; analyzers; buildtransitive -all - - - - - - - - + + + + net8.0 + + false + false + Exe + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + runtime; build; native; contentfiles; analyzers; buildtransitive +all + + + + + + + + diff --git a/src/FSLibrary.Tests/Tests.fs b/src/FSLibrary.Tests/Tests.fs index f5376f77..c5a46ce1 100644 --- a/src/FSLibrary.Tests/Tests.fs +++ b/src/FSLibrary.Tests/Tests.fs @@ -4,6 +4,8 @@ open StellarDestination open StellarDotnetSdk.Accounts open StellarMissionContext open Xunit +open Newtonsoft.Json.Linq +open MissionHistoryPubnetParallelCatchupV2 open System.Text.RegularExpressions open StellarCoreSet @@ -120,7 +122,20 @@ let ctx : MissionContext = pubnetParallelCatchupStartingLedger = 0 pubnetParallelCatchupEndLedger = None pubnetParallelCatchupLedgersPerJob = 16000 + pubnetParallelCatchupOverlapLedgers = 320 pubnetParallelCatchupNumWorkers = 192 + pubnetParallelCatchupStorageMode = "pvc" + pubnetParallelCatchupProfile = "" + pubnetParallelCatchupRangeOrder = "tip-first" + pubnetParallelCatchupPoolPrefix = "" + jobMonitorImagePcV2 = "" + pubnetParallelCatchupCpuRequest = "" + pubnetParallelCatchupMemRequest = "" + pubnetParallelCatchupPoolCpu = "" + pubnetParallelCatchupPoolMem = "" + pubnetParallelCatchupCreateRbac = false + jobMonitorNodeLabels = [] + jobMonitorTolerateTaints = [] tag = None numPregeneratedTxs = None enableTailLogging = true @@ -696,3 +711,157 @@ type Tests(output: ITestOutputHelper) = [] member __.``QuorumIntersectionChecker mission is registered``() = Assert.True(StellarMission.allMissions.ContainsKey "QuorumIntersectionChecker") + + +// --------------------------------------------------------------------------- +// MissionHistoryPubnetParallelCatchupV2 +// --------------------------------------------------------------------------- +[] +let ``the job monitor image is overridable and defaults to the chart`` () = + // An empty flag must leave the chart's pin alone. monitor.image= resolves to + // ":latest" or fails the pull. + let src = + System.IO.File.ReadAllText("../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + + Assert.Contains("if context.jobMonitorImagePcV2 <> \"\" then", src) + Assert.Contains("monitor.image=%s", src) + + let guard = src.IndexOf("if context.jobMonitorImagePcV2 <> \"\" then") + let use_ = src.IndexOf("monitor.image=%s") + Assert.True(guard < use_, "monitor.image must only be set inside the non-empty guard") + + +[] +let ``a pooled run does not let caller labels overwrite the routing label`` () = + // A pooled run claims index 0; the caller's entries must start at 1. Both at + // 0 and the second --set wins, so the pod matches on capacity alone and lands + // on any tier -- a supergiant range on a dwarf node, an OOM per range. + let src = + System.IO.File.ReadAllText("../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + + Assert.Contains("worker.requireNodeLabels[0]=purpose:%s", src) + Assert.Contains("if context.pubnetParallelCatchupPoolPrefix <> \"\" then i + 1 else i", src) + + +[] +let ``the pool maps ride their own --set with their commas escaped`` () = + // Every other option is folded into ONE comma-joined --set; these maps are + // themselves comma-separated, so folding them in delivers a map of one tier. + let src = + System.IO.File.ReadAllText("../../../../FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs") + + Assert.Contains("v.Replace(\",\", \"\\\\,\")", src) + Assert.Contains("poolMapArgs", src) + // Empty must not reach helm: monitor.poolCpu= blanks the chart default. + Assert.Contains("List.filter (fun (_, v) -> not (String.IsNullOrWhiteSpace v))", src) + +// The profile artifact. A completed record always carries bookkeeping (attempts, +// count) and may carry no measurement at all, when the collector never sampled +// that range. Such a record must not become an entry. +// +// RACE #8: count was attached BEFORE the `entry.Count > 0` guard, so every entry +// was non-empty and none were skipped. Seen twice in the field -- 0% +// peakAnonBytes in the artifact against 99% in progress.json. +// +// What that costs: profile_for resolves a range to the nearest measured end +// ABOVE it, so one junk entry at 1200 captures every range beneath it and hides +// the real 1600. Range 1100 then routes to protostar instead of supergiant, and +// nothing logs it. + + +/// Superset of rangeProfileFields: wallSeconds and txApply are recorded, never +/// projected. +let private measurementFields = + [ "peakAnonBytes" + "peakWorkingSetBytes" + "peakEphemeralBytes" + "txApply" + "seconds" + "wallSeconds" ] + +/// A record as /logs/progress.json carries it. +let private measuredRecord (count: int) (anon: int64) = + let r = JObject() + r.["attempts"] <- JValue(1) + r.["count"] <- JValue(count) + r.["seconds"] <- JValue(120.0) + r.["wallSeconds"] <- JValue(130.0) + r.["txApply"] <- JValue(60.0) + r.["peakAnonBytes"] <- JValue(anon) + r.["peakWorkingSetBytes"] <- JValue(anon + 1000L) + r + +/// The same record with every measurement stripped. +let private unmeasured (record: JObject) = + let r = record.DeepClone() :?> JObject + + for f in measurementFields do + r.Remove(f) |> ignore + + r + +let private completedMap (pairs: (string * JObject) list) = + let c = JObject() + + for (k, v) in pairs do + c.[k] <- v + + c + + +[] +let ``a measurement-free record cannot become a profile entry`` () = + // Nothing measured -- the whole document is refused, rather than written + // with the right range count and no data. + let completed = + completedMap [ "420", unmeasured (measuredRecord 420 900L) + "840", unmeasured (measuredRecord 420 950L) + "1260", unmeasured (measuredRecord 420 990L) ] + + match rangeProfileDocument "pvc" 20000 completed with + | None -> () + | Some doc -> + let ranges = doc.["ranges"] :?> JObject + + failwithf "wrote a profile artifact with %d ranges and zero measurements: %s" ranges.Count (ranges.ToString()) + + +[] +let ``a measured run still produces a complete profile`` () = + // The over-correction guard: a good read must still write everything. + let completed = + completedMap [ "420", measuredRecord 420 900L + "840", measuredRecord 420 950L ] + + match rangeProfileDocument "pvc" 20000 completed with + | None -> failwith "refused to write a profile that carries real measurements" + | Some doc -> + let ranges = doc.["ranges"] :?> JObject + Assert.Equal(2, ranges.Count) + Assert.Equal(900L, ranges.["420"].["peakAnonBytes"].Value()) + Assert.Equal(950L, ranges.["840"].["peakAnonBytes"].Value()) + Assert.Equal(120.0, ranges.["420"].["seconds"].Value()) + // Slicing is inferred from the records, not from the caller's 20000. + Assert.Equal(420, ranges.["420"].["count"].Value()) + Assert.Equal(420, doc.["ledgersPerRange"].Value()) + Assert.Equal("pvc", doc.["storageMode"].Value()) + + +[] +let ``a measured range does not drag its unmeasured neighbours in`` () = + // The realistic shape, and the only one that catches a guard deciding per RUN + // rather than per RECORD: one that latches on the first measurement passes + // both tests above while every later junk record rides in behind it. + let completed = + completedMap [ "1200", unmeasured (measuredRecord 400 900L) + "1600", measuredRecord 400 950L + "2000", unmeasured (measuredRecord 400 990L) ] + + match rangeProfileDocument "pvc" 20000 completed with + | None -> failwith "refused a profile that carries one real measurement" + | Some doc -> + let ranges = doc.["ranges"] :?> JObject + Assert.Equal(1, ranges.Count) + Assert.NotNull(ranges.["1600"]) + Assert.Null(ranges.["1200"]) + Assert.Null(ranges.["2000"]) diff --git a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs index 001981ea..fa02c76a 100644 --- a/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs +++ b/src/FSLibrary/MissionHistoryPubnetParallelCatchupV2.fs @@ -20,38 +20,199 @@ open System.IO open Newtonsoft.Json.Linq open Microsoft.FSharp.Control open System.Threading -open System open k8s open CSLibrary // Constants -let helmChartPath = "/supercluster/src/MissionParallelCatchup/parallel_catchup_helm" +// Baked into the supercluster image at this path. Overridable so a local run +// can point at a working copy without editing this file. +let helmChartPath = + match Environment.GetEnvironmentVariable("SUPERCLUSTER_CHART_PATH") with + | null + | "" -> "/supercluster/src/MissionParallelCatchup/parallel_catchup_helm" + | p -> p + +// An extra values file layered on top of the chart's own, for a run that wants +// different values without a working copy of the chart. Unset is the normal +// case: helm reads /values.yaml as its base regardless, so naming that +// same path would only re-apply it to itself. +// +// Layered, not substituted, so it carries only the keys it changes. +// SUPERCLUSTER_CHART_PATH repoints the whole chart including its values; this +// repoints the values alone, so the baked chart can run against experimental +// numbers. +let extraValuesArgs = + match Environment.GetEnvironmentVariable("SUPERCLUSTER_VALUES_PATH") with + | null + | "" -> [||] + | p -> [| "--values"; p |] -// Comment out the path below for local testing // Example command to run local testing (in the `supercluster/` directory): // $ dotnet run --project src/App/App.fsproj -- mission HistoryPubnetParallelCatchupV2 --image=docker-registry.services.stellar-ops.com/dev/stellar-core:23.0.3-2779.4d1df2b03.jammy-vnext-buildtests --pubnet-parallel-catchup-num-workers=2 --pubnet-parallel-catchup-starting-ledger=0 --pubnet-parallel-catchup-end-ledger=6400 --pubnet-parallel-catchup-ledgers-per-job 1280 --destination ./logs -// let helmChartPath = "src/MissionParallelCatchup/parallel_catchup_helm" -let valuesFilePath = helmChartPath + "/values.yaml" - -let defaultJobMonitorHostName = "ssc-job-monitor-eks.services.stellar-ops.com" -let jobMonitorStatusEndPoint = "/status" -let jobMonitorMetricsEndPoint = "/metrics" -let jobMonitorLoggingIntervalSecs = 30 // frequency of job monitor's internal information gathering (querying core endpoint and redis metrics) and logging -let jobMonitorStatusCheckIntervalSecs = 60 // frequency of us querying job monitor's `/status` end point -let jobMonitorMetricsCheckIntervalSecs = 60 // frequency of us querying job monitor's `/metrics` end point + +let jobMonitorLoggingIntervalSecs = 30 // frequency of the monitor reconcile loop: dispatch, liveness ping, status publish + +let jobMonitorStatusCheckIntervalSecs = 60 let jobMonitorStatusCheckTimeOutSecs = 600 + +// Print one status line in ten. The poll stays at a minute because +// jobMonitorStatusCheckTimeOutSecs is spent in units of it -- 600 over a 60s +// interval tolerates ten consecutive failures, and slowing the poll instead +// would make a single transient blip fail the run. Nothing is lost by printing +// less: a range failure logs the moment it is seen, on its own line. +let jobMonitorStatusLogEveryNChecks = 10 + let mutable toPerformCleanup = true let failedJobLogFileLineCount = 10000 let failedJobLogStreamLineCount = 1000 +let mutable statusChecks = 0 let mutable nonce : String = "" let mutable helmReleaseName : String = "" -let jobMonitorHostName (context: MissionContext) = - match context.jobMonitorExternalHost with - | Some host -> host - | None -> defaultJobMonitorHostName // TODO: append it with a nounce to make it session specific +// Resolve --pubnet-parallel-catchup-profile into the profile body POSTed to /start. +// +// Accepts a local path or an https URL, so a profile can come off disk or +// straight from a raw paste/gist link. Returns None to size from the +// configured requests. +// +// Never fatal: a profile only tightens requests, so a run must still start when +// one cannot be fetched. Failing the mission here would turn an optimisation +// into a dependency. +let resolveRangeProfile (context: MissionContext) : string option = + let spec = context.pubnetParallelCatchupProfile + + if String.IsNullOrWhiteSpace spec then + None + else + try + let body = + // http as well as https: the only alternative is treating the + // URL as a filename, which fails as "could not load" without + // ever mentioning the scheme, and the run proceeds unprofiled -- + // every range sized from defaults. A profile moves resource + // REQUESTS only, so a tampered one costs node size, not code + // execution, and it is parsed and range-counted before use. + if + spec.StartsWith("http://", StringComparison.OrdinalIgnoreCase) + || spec.StartsWith + ( + "https://", + StringComparison.OrdinalIgnoreCase + ) + then + use client = new HttpClient() + client.Timeout <- TimeSpan.FromSeconds(30.0) + client.GetStringAsync(spec) |> Async.AwaitTask |> Async.RunSynchronously + else + File.ReadAllText(ExpandHomeDirTilde spec) + + // Parse before shipping it: a 404 page or a truncated download would + // otherwise reach the monitor as an unreadable mount. + let doc = JObject.Parse(body) + + let count = + match doc.["ranges"] with + | :? JObject as r -> r.Count + | _ -> 0 + + if count = 0 then + LogWarn "Range profile %s has no ranges; sizing from configured requests" spec + None + else + LogInfo "Range profile: %d ranges from %s" count spec + Some body + with ex -> + LogWarn "Could not load range profile %s (%s); sizing from configured requests" spec ex.Message + None + + +// The driver talks to the monitor over its HTTPRoute: profile in via POST +// /start, status out of /status, logs pulled per file. The previous channels -- +// a status ConfigMap and `kubectl exec tar` -- both went through the API +// server; the logs alone measured ~0.3 MB per range, so ~1.2 GB of +// control-plane traffic on a 4000-range run, for bytes with no reason to be +// there. +let monitorRouteHost (context: MissionContext) = sprintf "%s.%s" nonce context.routeInternalDomain + +// Where the socket actually goes. An external host (an ELB, say) still needs +// the Host header set to the route hostname, or the gateway cannot match it. +let monitorEndpoint (context: MissionContext) = + match context.routeExternalHost with + | Some h -> h + | None -> monitorRouteHost context + +// The per-request timeout has to be shorter than any retry window built on top +// of it, or one hung request eats the whole window and the retry never happens. +// Observed 2026-08-08: a /start attempt hung on a route that was still +// programming, the 10-minute client timeout outlived the 5-minute deadline, and +// the mission failed after exactly one attempt. +let private monitorClientWith (context: MissionContext) (timeout: TimeSpan) = + let c = new HttpClient(BaseAddress = Uri(sprintf "http://%s" (monitorEndpoint context))) + c.DefaultRequestHeaders.Host <- monitorRouteHost context + c.Timeout <- timeout + c + +// Log pulls move whole files, so they get room; everything else is a short +// request that should fail fast and be retried. +let private monitorClient (context: MissionContext) = monitorClientWith context (TimeSpan.FromMinutes(10.0)) + +/// The run the monitor is asked to perform. The range travels with the profile +/// because both are per-run input -- the chart installs a generic monitor, and +/// this is what makes it a particular run. Validated as one document, so a bad +/// ledger range comes back as a 400 rather than generating no work and +/// reporting success on nothing. +let runDocument (context: MissionContext) (profileJson: string option) : string = + let endLedger = + match context.pubnetParallelCatchupEndLedger with + | Some value -> value + | None -> GetLatestPubnetLedgerNumber() + + // `run`, not `doc`: the profile ARTIFACT this mission writes is also a + // document, and the contract tests scan for its keys by name. + let rangeSpec = JObject() + rangeSpec.["startingLedger"] <- JValue(context.pubnetParallelCatchupStartingLedger) + rangeSpec.["latestLedgerNum"] <- JValue(endLedger) + rangeSpec.["ledgersPerJob"] <- JValue(context.pubnetParallelCatchupLedgersPerJob) + // Sent, not left to the monitor's default: the range list is + // ledgersPerJob + overlap, so a monitor that guesses it generates a + // different list than the one the profile was measured against. + rangeSpec.["overlapLedgers"] <- JValue(context.pubnetParallelCatchupOverlapLedgers) + rangeSpec.["order"] <- JValue(context.pubnetParallelCatchupRangeOrder) + + let run = JObject() + run.["range"] <- rangeSpec + + match profileJson with + | Some body -> run.["profile"] <- JObject.Parse(body) + | None -> () + + run.ToString(Newtonsoft.Json.Formatting.None) + +/// POST the run and let reconcile start. Retried: the route and the pod +/// both need a moment after `helm install`, and until this lands the monitor +/// deliberately dispatches nothing. +let startMission (context: MissionContext) (runJson: string) = + use client = monitorClientWith context (TimeSpan.FromSeconds(15.0)) + let deadline = DateTime.UtcNow.AddMinutes(5.0) + let mutable started = false + + while not started && DateTime.UtcNow < deadline do + try + use content = new StringContent(runJson, Text.Encoding.UTF8, "application/json") + let r = client.PostAsync("/start", content) |> Async.AwaitTask |> Async.RunSynchronously + + if r.IsSuccessStatusCode then + LogInfo "Mission started: profile POSTed to %s/start" (monitorEndpoint context) + started <- true + else + Thread.Sleep(5000) + with _ -> Thread.Sleep(5000) + + if not started then + failwithf "could not reach the job monitor at %s to start the mission" (monitorEndpoint context) + // Helper functions to convert label/taint tuples to Helm-compatible format using indexed notation let requireNodeLabelToHelmIndexed (index: int) ((key: string), (value: string option)) = @@ -83,6 +244,20 @@ let tolerateTaintToHelmIndexed (index: int) ((key: string), (effect: string opti let effectValue = Option.defaultValue "NoSchedule" effect sprintf "worker.tolerateNodeTaints[%d].key=%s,worker.tolerateNodeTaints[%d].effect=%s" index key index effectValue +// The monitor's own placement. A plain nodeSelector map rather than the +// worker's affinity terms: its node is chosen once for the run, not per range, +// so there is nothing to express that an equality match cannot. +let monitorNodeLabelToHelm ((key: string), (value: string option)) = + sprintf "monitor.nodeSelector.%s=%s" key (Option.defaultValue "" value) + +let monitorTolerateTaintToHelmIndexed (index: int) ((key: string), (effect: string option)) = + sprintf + "monitor.tolerateNodeTaints[%d].key=%s,monitor.tolerateNodeTaints[%d].effect=%s" + index + key + index + (Option.defaultValue "NoSchedule" effect) + let serviceAccountAnnotationsToHelmIndexed (index: int) (key: string, value: string) = sprintf "service_account.annotations[%d].key=%s,service_account.annotations[%d].value=%s" index key index value @@ -94,21 +269,52 @@ let installProject (context: MissionContext) = setOptions.Add(sprintf "worker.stellar_core_image=%s" context.image) setOptions.Add(sprintf "worker.replicas=%d" context.pubnetParallelCatchupNumWorkers) - // Set Redis hostname to be unique per release - setOptions.Add(sprintf "redis.hostname=%s-redis" nonce) - - setOptions.Add(sprintf "range_generator.params.starting_ledger=%d" context.pubnetParallelCatchupStartingLedger) - - let endLedger = - match context.pubnetParallelCatchupEndLedger with - | Some value -> value - | None -> GetLatestPubnetLedgerNumber() - - setOptions.Add(sprintf "range_generator.params.latest_ledger_num=%d" endLedger) + // pvc: /data survives the pod so an evicted range resumes at L+1 (what makes + // spot viable). ephemeral: /data is an emptyDir on the node -- denser + // packing, no resume. The ephemeral-storage request below must match: + // ~2Gi for pvc, ~35Gi for ephemeral, or the monitor logs a loud mismatch. + setOptions.Add(sprintf "worker.storageMode=%s" context.pubnetParallelCatchupStorageMode) + + // The route the driver uses for /start, /status and the logs. Templated + // only when routeHost is set, so an in-cluster caller still works without a + // gateway. + setOptions.Add(sprintf "monitor.routeHost=%s" (monitorRouteHost context)) + setOptions.Add(sprintf "monitor.gatewayName=%s" context.gatewayName) + setOptions.Add(sprintf "monitor.gatewayNamespace=%s" context.gatewayNamespace) + + + // Nodepool routing. Empty prefix ships the pre-tier behaviour: one label for + // every worker. Set, each range goes to - where the tier comes + // from its measured peakAnonBytes, and gets that node to itself. + setOptions.Add(sprintf "monitor.poolPrefix=%s" context.pubnetParallelCatchupPoolPrefix) + + // The monitor and collector ship as one image, pinned in the chart. Passing + // it here is what lets a run test a build of them without editing values. + if context.jobMonitorImagePcV2 <> "" then + setOptions.Add(sprintf "monitor.image=%s" context.jobMonitorImagePcV2) + + // Off by default: ssc-eks provides the binding itself, and creating these + // needs the installer to hold the rights being granted. A cluster without + // that binding gets a 403 on the monitor's first ConfigMap read and + // dispatches nothing -- ssc-test is such a cluster. + if context.pubnetParallelCatchupCreateRbac then + setOptions.Add("monitor.createRbac=true") + + if context.pubnetParallelCatchupPoolPrefix <> "" then + // Routing needs the label KEY and the taint toleration, and neither has + // a sensible default for an unpooled run -- both ship as []. Derived + // here because a pooled run that sets only + // the prefix otherwise fails twice over, and both failures are quiet. + // Karpenter labels these nodes purpose=-, which is exactly + // the value job_monitor builds per range, and taints them : + // NoSchedule. Without the key there is no tier affinity at all (the pods + // schedule anywhere); without the toleration they schedule nowhere. + // Observed on ssc-test 2026-08-07: 10 workers Pending indefinitely, + // "did not tolerate taint (taint=catchup:NoSchedule)". + setOptions.Add(sprintf "worker.requireNodeLabels[0]=purpose:%s" context.pubnetParallelCatchupPoolPrefix) + + setOptions.Add(sprintf "worker.tolerateNodeTaints[0]=%s" context.pubnetParallelCatchupPoolPrefix) - setOptions.Add( - sprintf "range_generator.params.uniform_ledgers_per_job=%d" context.pubnetParallelCatchupLedgersPerJob - ) // Skip known results by default setOptions.Add( @@ -123,36 +329,44 @@ let installProject (context: MissionContext) = (Option.defaultValue true context.checkEventsAreConsistentWithEntryDiffs) ) - // read the resource requirements defined in StellarKubeSpecs.fs (where resource for various missions are centralized) + // Only the ephemeral-storage pair is taken from StellarKubeSpecs. Its cpu and + // memory belong to the V1 parallel catchup missions, which share that spec and + // run a different execution model at parallelism 128/256 -- sizing V2 there + // silently resized them too. V2's own worker cpu/memory are the chart defaults + // in parallel_catchup_helm/values.yaml, overridable per run with + // --pubnet-parallel-catchup-cpu-request. let resourceRequirements = ParallelCatchupCoreResourceRequirements - let cpuReqMili = resourceRequirements.Requests.["cpu"].ToString() - let memReqMebi = resourceRequirements.Requests.["memory"].ToString() - let cpuLimMili = resourceRequirements.Limits.["cpu"].ToString() - let memLimMebi = resourceRequirements.Limits.["memory"].ToString() - let storageReqGibi = resourceRequirements.Requests.["ephemeral-storage"].ToString() - let storageLimGibi = resourceRequirements.Limits.["ephemeral-storage"].ToString() + // Both pushed only when the run explicitly asks. Otherwise the chart default + // stands, so the chart is the single place V2's worker sizing is written down. + if not (String.IsNullOrWhiteSpace context.pubnetParallelCatchupCpuRequest) then + setOptions.Add(sprintf "worker.resources.requests.cpu=%s" context.pubnetParallelCatchupCpuRequest) + + if not (String.IsNullOrWhiteSpace context.pubnetParallelCatchupMemRequest) then + setOptions.Add(sprintf "worker.resources.requests.memory=%s" context.pubnetParallelCatchupMemRequest) + + // Ephemeral-storage is an ephemeral-mode concept: /data is an emptyDir on the + // node there, and StellarKubeSpecs sizes it. In pvc mode /data is on the + // volume and the node disk holds only logs and tmp, so the run reserves + // nothing -- _resources already reads an empty REQ_EPHEMERAL as "leave both + // axes off the pod", and a request sized for the other mode would make disk + // rather than cpu the binding dimension for packing. + if context.pubnetParallelCatchupStorageMode <> "pvc" then + LogInfo + "Worker ephemeral storage from StellarKubeCfg: request %s, limit %s" + (resourceRequirements.Requests.["ephemeral-storage"].ToString()) + (resourceRequirements.Limits.["ephemeral-storage"].ToString()) + + setOptions.Add( + sprintf + "worker.resources.requests.ephemeral_storage=%s" + (resourceRequirements.Requests.["ephemeral-storage"].ToString()) + ) - LogInfo - "Resource requirements from StellarKubeCfg:\n\ - CPU request: %s\n\ - CPU limit: %s\n\ - Memory request: %s\n\ - Memory limit: %s\n\ - Storage request: %s\n\ - Storage limit: %s" - cpuReqMili - cpuLimMili - memReqMebi - memLimMebi - storageReqGibi - storageLimGibi - - setOptions.Add(sprintf "worker.resources.requests.cpu=%s" cpuReqMili) - setOptions.Add(sprintf "worker.resources.requests.memory=%s" memReqMebi) - setOptions.Add(sprintf "worker.resources.limits.cpu=%s" cpuLimMili) - setOptions.Add(sprintf "worker.resources.limits.memory=%s" memLimMebi) - setOptions.Add(sprintf "worker.resources.requests.ephemeral_storage=%s" storageReqGibi) - setOptions.Add(sprintf "worker.resources.limits.ephemeral_storage=%s" storageLimGibi) + setOptions.Add( + sprintf + "worker.resources.limits.ephemeral_storage=%s" + (resourceRequirements.Limits.["ephemeral-storage"].ToString()) + ) // Construct command for fetching history files from S3 for core node // `index` and set the corresponding Helm option @@ -160,7 +374,14 @@ let installProject (context: MissionContext) = if index < 1 || index > 3 then failwith "s3HistoryGetCommand: index must be between 1 and 3 inclusive" - let s3GetCommandBase = sprintf "aws s3 cp --region %s" context.s3HistoryMirrorRegionPcV2 + // --no-progress is load-bearing, not cosmetic. The AWS CLI draws its + // transfer meter with carriage returns and no newline, so a 628 MiB + // bucket download arrives as one multi-megabyte "line". The log + // collector reads the pod stream line-wise and aiohttp aborts any line + // over 512 KiB, so every large download killed its own stream, which + // then reconnected and hit the same wall. Measured on ssc-test + // 2026-07-30: it starved every retry pod of a collector stream. + let s3GetCommandBase = sprintf "aws s3 cp --no-progress --region %s" context.s3HistoryMirrorRegionPcV2 let command = sprintf "%s s3://%s/core_live_00%d/{0} {1}" s3GetCommandBase url index setOptions.Add(sprintf "worker.historyGetCommandCore00%d=\"%s\"" index command) @@ -169,13 +390,15 @@ let installProject (context: MissionContext) = | Some mirrorUrl -> [ 1 .. 3 ] |> List.iter (setS3HistoryGetCommand mirrorUrl) | None -> () - setOptions.Add(sprintf "monitor.hostname=%s" (jobMonitorHostName context)) - setOptions.Add(sprintf "monitor.path_prefix=/%s/%s" context.namespaceProperty helmReleaseName) - setOptions.Add(sprintf "monitor.logging_interval_seconds=%d" jobMonitorLoggingIntervalSecs) - // Attach the job-monitor HTTPRoute to the same Gateway as the core route - // (--gateway-name/--gateway-namespace), instead of the values.yaml defaults. - setOptions.Add(sprintf "monitor.gateway_name=%s" context.gatewayName) - setOptions.Add(sprintf "monitor.gateway_namespace=%s" context.gatewayNamespace) + setOptions.Add(sprintf "monitor.loggingIntervalSeconds=%d" jobMonitorLoggingIntervalSecs) + + // Every other mission gets this label from StellarKubeSpecs (Map.add + // "mission" missionName); parallel catchup builds its pods from a helm + // chart instead, so the old worker StatefulSet carried no mission label at + // all. kube-state-metrics turns it into label_mission, which every + // container-level Grafana panel joins on -- which is why this mission has + // never appeared in the dashboard's mission list. + setOptions.Add(sprintf "monitor.mission=%s" context.missionName) // Set ASAN_OPTIONS if provided match context.asanOptions with @@ -183,10 +406,34 @@ let installProject (context: MissionContext) = | None -> () // Convert labels and taints to Helm array format + // Left empty the monitor lands wherever it fits, which on a cluster whose + // catchup pools are tainted means the untainted shared nodes -- alongside + // ingress and whatever else lives there. Both are needed together to move + // it: the labels alone leave it unschedulable on a tainted pool. + if not (List.isEmpty context.jobMonitorNodeLabels) then + setOptions.Add( + context.jobMonitorNodeLabels + |> List.map monitorNodeLabelToHelm + |> String.concat "," + ) + + if not (List.isEmpty context.jobMonitorTolerateTaints) then + setOptions.Add( + context.jobMonitorTolerateTaints + |> List.mapi monitorTolerateTaintToHelmIndexed + |> String.concat "," + ) + if not (List.isEmpty context.requireNodeLabelsPcV2) then let requireLabelsHelm = context.requireNodeLabelsPcV2 - |> List.mapi requireNodeLabelToHelmIndexed + // From 1: a pooled run claims index 0 for the label it routes on, + // and mapi from 0 would overwrite it with whichever came second. + |> List.mapi + (fun i pair -> + requireNodeLabelToHelmIndexed + (if context.pubnetParallelCatchupPoolPrefix <> "" then i + 1 else i) + pair) |> String.concat "," setOptions.Add(requireLabelsHelm) @@ -219,68 +466,192 @@ let installProject (context: MissionContext) = let expandedKubeCfg = ExpandHomeDirTilde context.kubeCfg Environment.SetEnvironmentVariable("KUBECONFIG", expandedKubeCfg) - RunShellCommand [| "helm" - "install" - helmReleaseName - helmChartPath - "--values" - valuesFilePath - "--set" - String.Join(",", setOptions) |] + // --namespace is not optional. Without it helm uses the kubeconfig's current + // context, while every other call in this mission honours + // context.namespaceProperty -- so a run explicitly targeted at one namespace + // installs its monitor, Jobs and PVCs into a different one. Observed + // 2026-07-30: a mission run with --namespace sandbox put a monitor and four + // Jobs into the production namespace alongside a live run. + // The pool maps get their own --set each. Every other option is folded into + // ONE comma-joined --set, and these are themselves comma-separated, so + // folding them in would split each tier into a separate assignment. helm + // reads a backslash-escaped comma as data rather than as a separator. + // + // They arrive per run rather than from the chart because the claims differ + // by capacity: the spot pools were doubled on 2026-08-04 so a claim is half + // a node and two pods share it, while the on-demand pools kept their + // original sizes, where that same claim is the node's NAMEPLATE -- and + // nameplate is not allocatable, so nothing schedules at all. + let poolMapArgs = + [ "monitor.poolCpu", context.pubnetParallelCatchupPoolCpu + "monitor.poolMem", context.pubnetParallelCatchupPoolMem ] + |> List.filter (fun (_, v) -> not (String.IsNullOrWhiteSpace v)) + |> List.collect (fun (key, v) -> [ "--set"; sprintf "%s=%s" key (v.Replace(",", "\\,")) ]) + |> Array.ofList + + RunShellCommand( + Array.concat [ [| "helm"; "install"; helmReleaseName; helmChartPath |] + [| "--namespace"; context.namespaceProperty |] + extraValuesArgs + poolMapArgs + [| "--set"; String.Join(",", setOptions) |] ] + ) |> ignore match RunShellCommand [| "helm" "get" "values" - helmReleaseName |] with + helmReleaseName + "--namespace" + context.namespaceProperty |] with | Some valuesOutput -> LogInfo "%s" valuesOutput | _ -> () -// Collect log files from all parallel catchup worker pods -// This function: -// 1. Automatically determines worker pod names from context.pubnetParallelCatchupNumWorkers -// 2. For each pod, finds all files matching "stellar-core-*.log" in /data -// 3. Creates a tar.gz archive and copies it to context.destination directory -let collectLogsFromPods (context: MissionContext) = - // Generate pod names based on number of workers - // Pod names follow the pattern: -stellar-core-0, -stellar-core-1, etc. - let podNames = - [ 0 .. context.pubnetParallelCatchupNumWorkers - 1 ] - |> List.map (fun i -> sprintf "%s-stellar-core-%d" helmReleaseName i) - - LogInfo "Collecting logs from %d worker pods to directory: %s" (List.length podNames) context.destination.Path - - for podName in podNames do - try - LogInfo "Collecting logs from pod: %s" podName - - // Build the tar command to archive log files - // The command tars all stellar-core-*.log files in /data - // Using `-f -` to write the file contents to stdout - let command = [| "sh"; "-c"; "cd /data && tar -czf - stellar-core-*.log" |] - - // Output file path for this pod's logs - let outputFile = Path.Combine(context.destination.Path, sprintf "%s-logs.tar.gz" podName) - - // Execute the command and capture the tar output to a local file - RemoteCommandRunner.RunRemoteCommandAndCaptureOutput( - kube = context.kube, - ns = context.namespaceProperty, - podName = podName, - containerName = "stellar-core", - command = command, - outputFilePath = outputFile - ) +// How often the main loop pulls logs. Every 10 minutes flattens the teardown +// cost -- which measured ~20 minutes for a full run, all of it after the work +// finished -- without the pull competing with the collector for the volume. +let logFetchIntervalSecs = 600 + +let private monitorPodName (context: MissionContext) : string option = + context + .kube + .ListNamespacedPod( + context.namespaceProperty, + labelSelector = sprintf "app=job-monitor,release=%s" helmReleaseName + ) + .Items + |> Seq.map (fun p -> p.Metadata.Name) + |> Seq.tryHead + +/// Pull every artifact the destination does not already hold. +/// +/// Per file, over the monitor's HTTPRoute. The tar-over-exec this replaces put +/// every byte through the API server -- ~0.3 MB per range, so ~1.2 GB on a +/// 4000-range run -- and streamed the whole volume as one archive, which at 200 +/// workers came back truncated 2 times in 3 with no error surfaced. A file is +/// its own unit here: a cut transfer resumes from the byte it reached, and one +/// bad fetch costs one file rather than the pass. +/// Which subfolder of the destination an artifact belongs in. +/// +/// Sorted on the way out, not on the volume: the monitor's paths are an +/// implementation detail, while this is what a human opens. Flat, a 4000-range +/// run lands ~16000 files beside the five that summarise it -- three per range +/// that are per-range detail, and one bundle-wide file each for the monitor log, +/// the driver log, the progress record, the profile it produced and the run it +/// was asked for. +/// +/// Keeping the volume flat also leaves the HTTP surface alone: /logs/ +/// takes one path element and no separator, which is what stops the route -- +/// reachable from outside the cluster once its HTTPRoute is attached -- being +/// walked out of LOG_DIR. +let artifactFolder (name: string) = + if name.EndsWith(".log.gz") then + "range-logs" + elif name.EndsWith(".metrics") then + "metrics" + elif name.EndsWith(".done") + || name.EndsWith(".started") + || name.EndsWith(".state") + || name = "mission_started" then + "state" + else + "" + +let collectLogs (context: MissionContext) (destination: string) = + Directory.CreateDirectory(destination) |> ignore + use client = monitorClient context + + let manifest = + client.GetStringAsync("/logs") + |> Async.AwaitTask + |> Async.RunSynchronously + |> JArray.Parse + + let fetchOne (entry: JToken) = + let name = entry.["name"].ToString() + let size = entry.["size"].Value() + let folder = artifactFolder name + + let path = + if folder = "" then + Path.Combine(destination, name) + else + Directory.CreateDirectory(Path.Combine(destination, folder)) |> ignore + Path.Combine(destination, folder, name) + + let have = if File.Exists path then FileInfo(path).Length else 0L + + // Resume and skip both assume the file only ever grew, which is true of + // a worker log and of nothing else here. progress.json is rewritten + // whole on every reconcile and grows as ranges complete, so a Range + // request would splice the new document's tail onto the old one's + // prefix -- invalid JSON, in the bundle, silently, since the profile is + // built from the copy read off the pod rather than this one. Equal + // length is no safer: a rewrite can change a value without changing the + // length. Everything that is not append-only is refetched whole, which + // costs little -- progress.json is ~1 MB at 4000 ranges against ~1.4 GB + // of worker logs. + let appendOnly = name.EndsWith(".log.gz") + + // File.Exists is not redundant: a .done marker is zero bytes, so a + // length comparison alone reads "absent locally" as "already have it" + // and never fetches it. Measured 2026-08-08: 88 of 110 artifacts + // collected, and every .done was among the 22 missing. + if appendOnly && File.Exists path && have = size then + -1L // already whole; distinct from a zero-byte file we did fetch + else + let req = new HttpRequestMessage(HttpMethod.Get, "/logs/" + name) - let fileInfo = FileInfo(outputFile) + if appendOnly && have > 0L && have < size then + req.Headers.Range <- Headers.RangeHeaderValue(Nullable(have), Nullable()) - if fileInfo.Exists && fileInfo.Length > 0L then - LogInfo "Successfully collected logs from %s to %s (size: %d bytes)" podName outputFile fileInfo.Length + use resp = client.SendAsync(req) |> Async.AwaitTask |> Async.RunSynchronously + resp.EnsureSuccessStatusCode() |> ignore + let bytes = resp.Content.ReadAsByteArrayAsync() |> Async.AwaitTask |> Async.RunSynchronously + + // Append on a partial answer, replace on a whole one: a server that + // ignored Range would otherwise double the file. + if resp.StatusCode = Net.HttpStatusCode.PartialContent then + use fs = new FileStream(path, FileMode.Append, FileAccess.Write) + fs.Write(bytes, 0, bytes.Length) else - LogWarn "No logs found or empty archive for pod %s" podName + File.WriteAllBytes(path, bytes) + + int64 bytes.Length + + // Bounded: the monitor serves these from the same pod that runs reconcile. + let fetched = + manifest + |> Seq.toArray + |> Array.map + (fun e -> + async { + return + (try + fetchOne e + with ex -> + LogWarn "log fetch failed for %s: %s" (e.["name"].ToString()) ex.Message + -1L) + }) + |> fun work -> Async.Parallel(work, 8) + |> Async.RunSynchronously + + let moved = fetched |> Array.filter (fun n -> n > 0L) |> Array.sum + // >= 0 counts a zero-byte artifact we really did fetch. .done markers are + // empty by design, so "bytes > 0" undercounts exactly the files whose + // existence IS the signal. + let touched = fetched |> Array.filter (fun n -> n >= 0L) |> Array.length - with ex -> - LogWarn "Could not collect logs from pod %s (this is expected if pod doesn't exist): %s" podName ex.Message + LogInfo + "Collected %d of %d artifacts (%d bytes) from %s" + touched + (Seq.length manifest) + moved + (monitorEndpoint context) + +/// One log pass. Idempotent -- the manifest comparison is what makes a repeat +/// pass cheap, so there is no watermark to keep. +let collectLogsFromPods (context: MissionContext) = collectLogs context context.destination.Path // Cleanup on exit. `signalTriggered` indicates we're running under a hard // deadline (Jenkins' SoftKillWaitSeconds, ~5s by default, before SIGKILL). @@ -288,22 +659,259 @@ let collectLogsFromPods (context: MissionContext) = // of the much-slower log collection — otherwise we get SIGKILLed mid- // collection and leak every worker pod, which is what we saw in practice // with a 1024-worker run aborted from Jenkins. + +let queryJobMonitor (context: MissionContext) = + try + use client = monitorClient context + let body = client.GetStringAsync("/status") |> Async.AwaitTask |> Async.RunSynchronously + + statusChecks <- statusChecks + 1 + + if statusChecks % jobMonitorStatusLogEveryNChecks = 1 then + LogInfo "job monitor status: %s" body + + Some(JObject.Parse(body)) + with ex -> + LogError "Error reading job monitor status: %s" ex.Message + None + + +// Emit what this run measured, so a later run can be given tighter +// per-range requests. An artifact rather than a ConfigMap or +// an S3 object: nothing for ArgoCD to reconcile, no second writer racing a +// concurrent mission, and not bounded by Prometheus retention. +// Fields carried from the monitor's progress record into the profile artifact. +// A PVC's size is absent on purpose -- it is not a scheduling dimension, so +// profiling it buys no packing. peakEphemeralBytes appears only for +// ephemeral-mode runs, and only for ranges that finished. +// +// A subset of what the monitor records: it measures more per range than the next +// run can size from, and a measurement nothing reads is pure weight in an +// artifact that was already 963 KB at 4805 ranges. +let rangeProfileFields = + // The memory figure the sizing consumer reads, sampled from kubelet by the + // collector. Omitting it strips it from the artifact while progress.json + // still carries it, so the next run sizes every range from defaults. + [ "peakAnonBytes" + "peakWorkingSetBytes" + "peakEphemeralBytes" + // The only timing the next run sizes from: it sets the percentile basis, + // the dispatch order and the runtime insurance thresholds. wallSeconds and + // txApply are recorded per range as Prometheus metrics but nothing sizes + // or orders from either, so they stay out of the artifact. + "seconds" ] + +// A missing measurement must stay missing rather than become a null: the +// consumer falls back to its configured default when the field is absent. +let projectRangeEntry (record: JObject) : JObject = + let entry = JObject() + + for field in rangeProfileFields do + match record.[field] with + | null -> () + | v -> entry.[field] <- v + + entry + + +// The progress record, read only from the monitor's volume. +// No record is the safe outcome -- the consumer falls back to its defaults. +let readProgressRecord (context: MissionContext) : JObject option = + let monitorPods = + context + .kube + .ListNamespacedPod( + context.namespaceProperty, + labelSelector = sprintf "app=job-monitor,release=%s" helmReleaseName + ) + .Items + |> Seq.map (fun p -> p.Metadata.Name) + |> List.ofSeq + + let fromVolume = + match monitorPods with + | [] -> None + | podName :: _ -> + try + let tmp = Path.Combine(Path.GetTempPath(), sprintf "%s-progress.json" helmReleaseName) + + RemoteCommandRunner.RunRemoteCommandAndCaptureOutput( + kube = context.kube, + ns = context.namespaceProperty, + podName = podName, + containerName = "job-monitor", + command = [| "cat"; "/logs/progress.json" |], + outputFilePath = tmp + ) + + let fi = FileInfo(tmp) + + if fi.Exists && fi.Length > 0L then + LogInfo "Progress record read from the monitor volume (%d bytes)" fi.Length + Some(JObject.Parse(File.ReadAllText(tmp))) + else + None + with ex -> + LogWarn "Could not read /logs/progress.json (%s); no range profile will be written" ex.Message + None + + fromVolume + + +// The `ranges` map of a profile artifact, built from a progress record's +// `completed` map. Pure, so the projection can be exercised without a cluster. +let buildRangeProfile (completed: JObject) : JObject = + let ranges = JObject() + + // Keyed on the range end alone, with count kept as a field. + // + // Measured on ssc-test: 4.2x the ledgers per range (100 -> 420, the + // default 320 overlap) moved peak disk by -1.6% and wall time by + // 1.15x. Cost tracks ledger position -- how big the bucket set is to + // download and apply -- far more than range length. Putting count in + // the key would therefore discard the whole profile whenever + // overlapLedgers or ledgersPerJob changed, to preserve a distinction + // the measurements say is small. + // + // A consumer should resolve a range by exact end, else the nearest + // measured end, else a run-wide fallback -- so a re-sliced run still + // gets useful numbers instead of zero matches. + for prop in completed.Properties() do + let record = prop.Value :?> JObject + let entry = projectRangeEntry record + + // The guard decides on measurements alone. count is bookkeeping, not a + // measurement, so it is attached only after the entry has been found to + // carry something real. Attaching it first made every entry non-empty + // and defeated the guard completely: a record that measured nothing + // still sailed through and produced a range holding only a count. + if entry.Count > 0 then + match record.["count"] with + | null -> () + | v -> entry.["count"] <- v + + // Same end from two differently-sized ranges: keep the + // larger, since sizing from the smaller would under-provision. + let existing = ranges.[prop.Name] + + let keep = + isNull existing + || (let a = entry.["count"] + let b = (existing :?> JObject).["count"] + isNull b || (not (isNull a) && a.Value() >= b.Value())) + + if keep then ranges.[prop.Name] <- entry + + ranges + + +// The profile document to write, or None when there is nothing worth writing. +let rangeProfileDocument (storageMode: string) (defaultLedgersPerRange: int) (completed: JObject) : JObject option = + let ranges = buildRangeProfile completed + + // Slicing and storage mode both go in the name, because a profile is + // only valid for the shape it was measured at. + // + // Slicing: keys are range ends and cost tracks range length, so a + // 39382-ledger profile fed into a 16320-ledger run resolves through + // nearest-end fallback and sizes everything wrong. Measured on + // ssc-test -- it produced 1025 OOM retries in one run. + // + // Mode: an ephemeral profile carries peakEphemeralBytes and a pvc one + // does not, so crossing them silently defaults the disk axis. + let ledgersPerRange = + let counts = + ranges.Properties() + |> Seq.choose + (fun p -> + match (p.Value :?> JObject).["count"] with + | null -> None + | v -> Some(v.Value())) + |> Seq.toList + + match counts with + | [] -> defaultLedgersPerRange + | _ -> counts |> List.countBy id |> List.maxBy snd |> fst + + let doc = JObject() + doc.["schema"] <- JValue(1) + doc.["generated"] <- JValue(DateTime.UtcNow.ToString("o")) + doc.["release"] <- JValue(helmReleaseName) + doc.["storageMode"] <- JValue(storageMode) + doc.["ledgersPerRange"] <- JValue(ledgersPerRange) + doc.["ranges"] <- ranges + + // A profile with no measurements is worse than no profile at all: it looks + // complete, so nothing downstream can tell it from a good one, and the next + // run sizes itself from empty data. Writing nothing lets the consumer fall + // back to its configured defaults, which is the safe outcome. + if ranges.Count = 0 then None else Some doc + + +let writeRangeProfile (context: MissionContext) = + match readProgressRecord context with + | None -> LogInfo "No progress record to build a range profile from" + | Some progress -> + try + let completed = progress.["completed"] :?> JObject + + let docOpt = + rangeProfileDocument + context.pubnetParallelCatchupStorageMode + context.pubnetParallelCatchupLedgersPerJob + completed + + match docOpt with + | None -> LogWarn "Progress record carried no measurements; not writing a range profile" + | Some doc -> + let ranges = doc.["ranges"] :?> JObject + let ledgersPerRange = doc.["ledgersPerRange"].Value() + + let path = + Path.Combine( + context.destination.Path, + sprintf + "%s-profile-%dledgers-%s.json" + helmReleaseName + ledgersPerRange + context.pubnetParallelCatchupStorageMode + ) + + File.WriteAllText(path, doc.ToString()) + LogInfo "Wrote range profile for %d ranges to %s" ranges.Count path + with ex -> LogWarn "Failed to write range profile: %s" ex.Message + + let cleanup (signalTriggered: bool) (context: MissionContext) = if toPerformCleanup then toPerformCleanup <- false + // Before either branch: `helm uninstall` takes the monitor pod, and with + // it the volume the profile is read from, so an aborted run would + // otherwise lose every measurement it had already taken. One pod exec + // and a local file write -- cheap enough for the abort path's few + // seconds, and a run stopped part-way is exactly when the partial + // profile is most wanted. + try + writeRangeProfile context + with ex -> LogWarn "Failed to write range profile: %s" ex.Message + if signalTriggered then // Abort path: resources first, logs are nice-to-have. // Skip log collection entirely — even parallelized it can't beat // Jenkins' ~5s grace before SIGKILL, and it can't beat the per-pod // terminationGracePeriodSeconds (default 30s) when scaled to 1024 // workers. Whatever logs were captured inline by the failure - // handler in the main loop are still on disk. + // handler in the main loop are still on disk -- and since the main + // loop now fetches every logFetchIntervalSecs, an abort keeps every + // part up to the last pass rather than losing the run's logs whole. LogInfo "Signal-triggered cleanup: uninstalling release %s" helmReleaseName RunShellCommand [| "helm" "uninstall" - helmReleaseName |] + helmReleaseName + "--namespace" + context.namespaceProperty |] |> ignore else // Normal / legitimate-failure path: pods are still alive through @@ -320,7 +928,9 @@ let cleanup (signalTriggered: bool) (context: MissionContext) = RunShellCommand [| "helm" "uninstall" - helmReleaseName |] + helmReleaseName + "--namespace" + context.namespaceProperty |] |> ignore let mutable cleanupContext : MissionContext option = None @@ -344,20 +954,6 @@ Console.CancelKeyPress.Add Environment.Exit(0)) -let queryJobMonitor (context: MissionContext, path: String, endPoint: String) = - try - use client = new HttpClient() - let url = "http://" + jobMonitorHostName context + path + endPoint - let response = client.GetStringAsync(url).Result - - LogInfo "job monitor query '%s', got response: %s" url response - let json = JObject.Parse(response) - Some(json) - with ex -> - LogError "Error querying job monitor '%s': %s" endPoint ex.Message - None - - let dumpLogs (context: MissionContext, podName: String) = let stream = context.kube.ReadNamespacedPodLog( @@ -394,14 +990,19 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = installProject context + // The monitor dispatches nothing until this arrives, so a profile that + // cannot be delivered fails the run here rather than silently sizing every + // range as unprofiled. + startMission context (runDocument context (resolveRangeProfile context)) + let mutable allJobsFinished = false let mutable timeoutLeft = jobMonitorStatusCheckTimeOutSecs - let mutable timeBeforeNextMetricsCheck = jobMonitorMetricsCheckIntervalSecs - let jobMonitorPath = "/" + context.namespaceProperty + "/" + helmReleaseName + + let mutable lastLogFetch = DateTime.UtcNow while not allJobsFinished do Thread.Sleep(jobMonitorStatusCheckIntervalSecs * 1000) - let statusOpt = queryJobMonitor (context, jobMonitorPath, jobMonitorStatusEndPoint) + let statusOpt = queryJobMonitor context try match statusOpt with @@ -409,33 +1010,47 @@ let historyPubnetParallelCatchupV2 (context: MissionContext) = timeoutLeft <- jobMonitorStatusCheckTimeOutSecs let remainSize = status.Value("num_remain") let jobsFailed = status.["jobs_failed"] :?> JArray - let JobsInProgress = status.["jobs_in_progress"] :?> JArray + let jobsInProgress = status.Value("queue_in_progress_count") + // At first sight, not once the run drains: the monitor has no + // way to be told to stop dispatching, so draining means running + // the whole remaining queue after the outcome is already known. if jobsFailed.Count <> 0 then - LogInfo "One or more jobs have failed:" + LogError "%d job(s) failed:" jobsFailed.Count for job in jobsFailed do - let ident = job.ToString().Split('|') - let key = ident.[0] - let podName = ident.[1] - LogInfo "%s, logs >>> " (job.ToString()) - dumpLogs (context, podName) + let text = job.ToString() + let ident = text.Split('|') + LogInfo "%s, logs >>> " text + + // A condemned range need not name a pod: an attempt that + // never scheduled has none, and its log is on the monitor + // volume either way, so a missing pod must not mask the + // failure below. + if ident.Length > 1 then + try + dumpLogs (context, ident.[1]) + with ex -> LogInfo "could not read pod log (%s); see collected logs" (ex.Message) + LogInfo "<<<" failwith "Catch up failed, check logs for more info" - if remainSize = 0 && JobsInProgress.Count = 0 then - // All jobs completed — perform a final query on the metrics - queryJobMonitor (context, jobMonitorPath, jobMonitorMetricsEndPoint) |> ignore + if remainSize = 0 && jobsInProgress = 0 then LogInfo "All queues empty. Mission complete." allJobsFinished <- true - // check the metrics - timeBeforeNextMetricsCheck <- timeBeforeNextMetricsCheck - jobMonitorStatusCheckIntervalSecs - - if timeBeforeNextMetricsCheck <= 0 then - queryJobMonitor (context, jobMonitorPath, jobMonitorMetricsEndPoint) |> ignore - timeBeforeNextMetricsCheck <- jobMonitorMetricsCheckIntervalSecs + // Pull the logs written since the last pass, so teardown moves a + // delta instead of the whole volume. Measured at ~20 minutes for + // a full run, all of it after the work had finished. Isolated in + // its own try: a failed pass re-fetches the same window next + // time and must never take the mission down. + if (DateTime.UtcNow - lastLogFetch).TotalSeconds >= float logFetchIntervalSecs then + lastLogFetch <- DateTime.UtcNow + + try + collectLogsFromPods context + with ex -> LogWarn "Incremental log collection failed: %s" ex.Message | None -> LogError "no status" diff --git a/src/FSLibrary/StellarKubeSpecs.fs b/src/FSLibrary/StellarKubeSpecs.fs index 027530f1..256c7796 100644 --- a/src/FSLibrary/StellarKubeSpecs.fs +++ b/src/FSLibrary/StellarKubeSpecs.fs @@ -128,6 +128,13 @@ let SimulatePubnetTier1PerfCoreResourceRequirements : V1ResourceRequirements = let ParallelCatchupCoreResourceRequirements : V1ResourceRequirements = // When doing parallel catchup, we give each container // 0.25 vCPUs, 8Gi RAM and 35 GB of disk bursting to 2vCPU, 28Gi (28672Mi) and 40 GB + // + // Shared with the V1 parallel catchup missions (pubnet and testnet), which run + // coreSets through RunParallelJobsInRandomOrder at parallelism 128 and 256. + // V2 does NOT take its cpu or memory from here -- it is a Job-per-range mission + // with a different packing model and sources those from the helm chart + // (worker.resources.requests in parallel_catchup_helm/values.yaml). It still + // reads the ephemeral-storage pair below for its ephemeral storage mode. makeResourceRequirementsWithStorageLimit 250 8192 35 2000 28672 40 let NonParallelCatchupCoreResourceRequirements : V1ResourceRequirements = diff --git a/src/FSLibrary/StellarMissionContext.fs b/src/FSLibrary/StellarMissionContext.fs index 5641ea59..787a1e43 100644 --- a/src/FSLibrary/StellarMissionContext.fs +++ b/src/FSLibrary/StellarMissionContext.fs @@ -118,7 +118,20 @@ type MissionContext = pubnetParallelCatchupStartingLedger: int pubnetParallelCatchupEndLedger: int option pubnetParallelCatchupLedgersPerJob: int + pubnetParallelCatchupOverlapLedgers: int pubnetParallelCatchupNumWorkers: int + pubnetParallelCatchupStorageMode: string + pubnetParallelCatchupProfile: string + pubnetParallelCatchupRangeOrder: string + pubnetParallelCatchupPoolPrefix: string + jobMonitorImagePcV2: string + pubnetParallelCatchupCpuRequest: string + pubnetParallelCatchupMemRequest: string + pubnetParallelCatchupPoolCpu: string + pubnetParallelCatchupPoolMem: string + pubnetParallelCatchupCreateRbac: bool + jobMonitorNodeLabels: ((string * string option) list) + jobMonitorTolerateTaints: ((string * string option) list) genesisTestAccountCount: int option asanOptions: string option diff --git a/src/FSLibrary/StellarSupercluster.fs b/src/FSLibrary/StellarSupercluster.fs index 2c7280fd..ba5e01d4 100644 --- a/src/FSLibrary/StellarSupercluster.fs +++ b/src/FSLibrary/StellarSupercluster.fs @@ -217,16 +217,33 @@ let DumpPodInfo (kube: Kubernetes) (apiRateLimit: int) (ns: string) = let pods = kube.ListNamespacedPod(namespaceParameter = ns) if pods <> null then - LogInfo "There are %d pods in total" (Seq.length pods.Items) - - for p in pods.Items do - let age = - if p.Status.StartTime.HasValue then - System.DateTime.UtcNow.Subtract(p.Status.StartTime.Value).ToString(@"hh\:mm") - else - "00:00" + let total = Seq.length pods.Items + + // A line per pod carries the name and the age, which is what tells a + // stuck pod from a slow one. Kept for every mission small enough to read + // it; past that it is the parallel catchups, where at 1024 workers this + // wrote ~1026 lines every 5 minutes -- ~57000 over a 4.7h run -- and + // buried the only thing worth reading. + if total < 500 then + LogInfo "There are %d pods in total" total + + for p in pods.Items do + let age = + if p.Status.StartTime.HasValue then + System.DateTime.UtcNow.Subtract(p.Status.StartTime.Value).ToString(@"hh\:mm") + else + "00:00" - LogInfo "Pod: name=%s phase=%s age=%s (hr:min)" p.Metadata.Name p.Status.Phase age + LogInfo "Pod: name=%s phase=%s age=%s (hr:min)" p.Metadata.Name p.Status.Phase age + else + let byPhase = + pods.Items + |> Seq.countBy (fun p -> p.Status.Phase) + |> Seq.sortBy fst + |> Seq.map (fun (phase, n) -> sprintf "%s=%d" phase n) + |> String.concat " " + + LogInfo "Pods: %d total %s" total byPhase // Create a per-run "anchor" ConfigMap and stash an owner reference to it on // `nCfg.anchorOwnerRef` so every subsequent resource the mission creates will diff --git a/src/MissionParallelCatchup/.dockerignore b/src/MissionParallelCatchup/.dockerignore new file mode 100644 index 00000000..7c3a8d38 --- /dev/null +++ b/src/MissionParallelCatchup/.dockerignore @@ -0,0 +1,2 @@ +**/__pycache__/ +**/*.pyc diff --git a/src/MissionParallelCatchup/Dockerfile.jobmonitor b/src/MissionParallelCatchup/Dockerfile.jobmonitor index 9244ba01..830b1548 100644 --- a/src/MissionParallelCatchup/Dockerfile.jobmonitor +++ b/src/MissionParallelCatchup/Dockerfile.jobmonitor @@ -1,20 +1,39 @@ -FROM --platform=linux/amd64 ubuntu:24.04 +FROM --platform=linux/amd64 python:3.12-slim VOLUME /data WORKDIR /app -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - python3 \ - python3-redis \ - python3-requests \ - python3-prometheus-client && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* +# Client major tracks a cluster minor (36.x <-> Kubernetes 1.36). Upstream's +# compatibility table marks an exact match as supported and everything above or +# below as partial, which is where the 1.34 clusters sit today -- the same +# category 1.35 would be. Every API this mission uses (batch/v1 Job, +# podFailurePolicy, ttlSecondsAfterFinished) has been GA since 1.31. +# +# 36 rather than 35 because kubernetes.aio, the aiohttp-backed async client, +# does not exist before it: 35.0.0 ships no kubernetes/aio at all. +# +# The floor matters independently: podFailurePolicy (V1PodFailurePolicy*) needs +# >=26 and V1VolumeResourceRequirements needs >=29. Ubuntu's distro package is +# 22.6.0 and has neither, so it cannot express this mission's failure +# classification at all. +RUN pip install --no-cache-dir \ + 'kubernetes~=36.0' \ + 'aiohttp~=3.14' \ + 'prometheus-client~=0.19' -COPY ./job_monitor.py /app +# apps/ and lib/ flatten into one directory here, and the modules import each +# other by bare name. The split is for reading the repo, not for the runtime. +COPY ./apps/job_monitor.py /app +# Same image, second entrypoint: runs as a sidecar streaming worker logs. +COPY ./apps/log_collector.py /app +# Flattened one directory at a time: `COPY ./lib/ /app/` would preserve +# lib/monitor and lib/collector as subdirectories, and a bare `import sizing` +# does not look inside them. +COPY ./lib/*.py /app/ +COPY ./lib/monitor/ /app/ +COPY ./lib/collector/ /app/ EXPOSE 8080 -CMD ["/usr/bin/python3", "job_monitor.py"] +CMD ["python3", "job_monitor.py"] diff --git a/src/MissionParallelCatchup/apps/job_monitor.py b/src/MissionParallelCatchup/apps/job_monitor.py new file mode 100644 index 00000000..3feef67a --- /dev/null +++ b/src/MissionParallelCatchup/apps/job_monitor.py @@ -0,0 +1,457 @@ +"""The single writer for one MissionParallelCatchup run. + +Turns a ledger range into completed work using Kubernetes Jobs, and reports what +happened. One pass is: snapshot the cluster, derive each range's state, observe +the volume, act on the apiserver, commit the record. + +EXACTLY ONE of these may run. No leader election exists anywhere, so a rolling +update that briefly overlaps two is a second writer of every Job, PVC and the +progress record -- replicas: 1, strategy: Recreate. +""" +import asyncio +import collections +import logging +import signal +import sys +import time + +import cluster +import config +import dispatch +import liveness +import metrics +import monitor_config as mc +import policy +import record +import server +import sizing +import verdict + +logging.basicConfig(level=logging.INFO, stream=sys.stdout, + format='%(asctime)s %(levelname)s %(message)s') +logger = logging.getLogger('job_monitor') + + +async def main(): + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, stop.set) + + state = State() + state.resume() + async with cluster.session(): + async with asyncio.TaskGroup() as tg: + tg.create_task(server.serve(state, stop)) + tg.create_task(reconcile_loop(state, stop)) + logger.info("stopped") + return 1 if state.progress['condemned'] else 0 + + +async def reconcile_loop(state, stop): + while not stop.is_set(): + if state.ranges: + try: + await reconcile(state) + except Exception: + # A pass is a projection; the next one rebuilds it. Dying here + # leaves the run with no writer. + logger.exception("reconcile pass failed") + try: + async with asyncio.timeout(mc.RECONCILE_INTERVAL_SECONDS): + await stop.wait() + except TimeoutError: + pass + stop.set() + + +async def reconcile(state): + """One pass: look, decide, act, persist. + + The volume is touched in exactly two places, both threaded and both + batched. Everything between them is decisions and apiserver calls. + """ + jobs, pods = await cluster.snapshot() + states = derive_all(state, jobs, pods) + await asyncio.to_thread(observe, states) + await act(states, state) + await asyncio.to_thread(state.commit, states) + await publish(states, state, pods) + + +# --- what is true about one range ------------------------------------------- + + +class RangeState: + """One range as of this pass. Cheap to build, thrown away at the end.""" + + __slots__ = ('end', 'count', 'attempt', 'status', 'jobs', 'job', 'pod', + 'verdict', 'completed_at', 'done', 'measured') + + def __init__(self, end, count, attempt=0, status='pending', jobs=(), + job=None, pod=None, verdict=None, completed_at=None): + self.end = str(end) + self.count = count + self.attempt = attempt + self.status = status + self.jobs = jobs + self.job = job + self.pod = pod + self.verdict = verdict + self.completed_at = completed_at + self.done = False # set by observe() + self.measured = None # set by observe() for a completed range + + @property + def holds_capacity(self): + """Anything unfinished occupies a slot, a failed attempt included: its + retry is already owed. Missions run longest-first, so releasing the slot + would let short ranges dispatch ahead of a long range's retry.""" + return self.status in ('running', 'failed') + + +def derive_all(state, jobs, pods): + return [derive(end, count, state.progress, jobs, pods) + for end, count in state.ranges] + + +def derive(end, count, progress, jobs, pods): + """One range from its newest Job, that Job's pod, and the record. + + The record answers for a range with no Job: reaped, condemned, or never + started. Without it a reaped range reads as pending and is dispatched a + second time. + """ + range_jobs = jobs.get(str(end), ()) + job = max(range_jobs, key=_attempt_of, default=None) + if job is None: + # Terminal: re-deciding would re-count the verdict on every pass. + if str(end) in progress['condemned']: + return RangeState(end, count, status='condemned') + if str(end) in progress['completed']: + attempt = progress['completed'][str(end)].get('attempts', 1) + return RangeState(end, count, attempt, 'completed') + return RangeState(end, count) + + attempt = _attempt_of(job) + names = [j.metadata.name for j in range_jobs] + pod = pods.get(job.metadata.name) + # succeeded and failed are POD COUNTS and there is no running field, so + # running is whatever is neither. + if job.status.succeeded: + return RangeState(end, count, attempt, 'completed', names, job, pod, + completed_at=job.status.completion_time) + if job.status.failed: + # Classified by observe(); this phase reads nothing. + return RangeState(end, count, attempt, 'failed', names, job, pod) + return RangeState(end, count, attempt, 'running', names, job, pod) + + +def observe(states): + """Every read a pass makes. Only a finished attempt has anything to say. + + A failed one is classified here because an exit-3 verdict decompresses a + whole archive, and a completed one is aggregated here because that walks + its .metrics -- both are reads, so both belong before any decision. + """ + for st in states: + if st.status not in ('failed', 'completed'): + continue + st.done = record.is_done(st.end, st.attempt) + if not st.done: + continue # the collector is still writing this attempt + if st.status == 'failed': + st.verdict = verdict.effective(st.end, st.attempt, st.job) + elif st.jobs: + st.measured = aggregate(st) + + +def _attempt_of(job): + return int((job.metadata.labels or {}).get(config.LABEL_ATTEMPT, 1)) + + +# --- what a pass does about it ---------------------------------------------- + + +async def act(states, state): + """Retries first, then reaps, then new work into whatever slots are left. + + A retry needs no slot: its range already holds one. + """ + work = [] + active = sum(1 for st in states if st.holds_capacity) + + for st in states: + if st.status == 'failed': + work.append(_settle(st, state)) + elif st.status == 'completed' and st.jobs and st.done: + work.append(_reap(st, state)) + + for st in states: + if st.status == 'pending' and active < mc.PARALLELISM: + active += 1 + work.append(_dispatch(st, state)) + + for result in await asyncio.gather(*work, return_exceptions=True): + if isinstance(result, Exception): + logger.warning("action failed: %s", result) + + +async def _dispatch(st, state): + created = await dispatch.create(int(st.end), st.count, 1) + if created is not None: + state.started.append((st.end, created.metadata.creation_timestamp)) + logger.info("range %s dispatched", st.end) + + +async def _settle(st, state): + """Decide a failed attempt, once the collector has finished with it. + + Deferred until .done because the verdict can still change: an exit 3 that + reads as a catchup failure is a retryable fetch fault if the archive says + so, and that is the difference between a retry and a dead run. + """ + if not st.done: + return + cause = st.verdict.get('outcome') + spent = record.note_cause(state.progress, st.end, st.attempt, cause) + ooms = spent.get('oom', 0) + base = sizing.requests_for(int(st.end), ooms)[0] + decision = policy.decide(int(st.end), st.verdict, spent, + base_memory=base.get('memory'), + base_ephemeral=base.get('ephemeral-storage')) + if decision.action == policy.DEFER: + return + if decision.action == policy.CONDEMN: + state.condemn(st, decision.reason) + await cluster.delete_job(dispatch.job_name(int(st.end), st.attempt)) + return + logger.info("range %s attempt %d -> %d (%s%s%s)", st.end, st.attempt, + st.attempt + 1, decision.reason, + f"; memory={decision.memory}" if decision.memory else '', + f"; ephemeral={decision.ephemeral}" if decision.ephemeral else '') + await dispatch.create(int(st.end), st.count, st.attempt + 1, + oom_count=ooms, memory=decision.memory, + ephemeral=decision.ephemeral) + metrics.settled(state.progress, cause, end=st.end) + # After the successor exists, never before: with the predecessor gone and + # the create failed, the next pass redispatches at attempt 1 and loses the + # escalated request. TTL reclaims it if this delete does not. + await cluster.delete_job(dispatch.job_name(int(st.end), st.attempt)) + + +async def _reap(st, state): + """Delete a completed range's Jobs and release its volume. + + Only once .done: reaping deletes the pod, which is the last place peaks and + terminated timestamps can be read. + """ + await cluster.reap(st.end, st.jobs) + + +# --- what a pass reports ---------------------------------------------------- + + +async def publish(states, state, pods): + began = time.monotonic() + counts = await liveness.publish(list(pods.values())) + swept = time.monotonic() - began # the sweep alone, nothing after it + metrics.sync_counters(state.progress, state.applied) + metrics.observe_completed(state.progress, state.replayed) + metrics.publish_gauges(state.counts(states), counts, swept, + time.time() - state.mission_start) + + +# --- the run ---------------------------------------------------------------- + + +class State: + """Everything that outlives a pass. Small on purpose: the cluster and the + volume are the sources, and this is what cannot be re-derived from them.""" + + def __init__(self): + self.ranges = [] + self.progress = {'completed': {}, 'condemned': {}, 'causes': {}, + 'counters': collections.Counter(), + 'disruptedRanges': set()} + self.started = [] # (end, createdAt) awaiting a commit + self.applied = {} # counter totals already pushed to prometheus + self.replayed = set() # (range, field) already observed + self.mission_start = time.time() + self._live = {} # this pass's counts, for /status + + def resume(self): + """Replay run.json through the same validation path. + + A restarted monitor must resume the run it inherited rather than wait + for a /start delivered to its predecessor -- and the profile has to come + back with it, since longest-first orders by it. + """ + self.mission_start = record.mission_start() + self.progress = record.load_progress() + doc = record.load_run() + if doc is None: + return + try: + self.start(doc) + except ValueError as e: + logger.error("run.json no longer validates (%s); waiting for /start", e) + + def start(self, doc): + spec = doc.get('range') or {} + for key, name in (('order', 'RANGE_ORDER'), + ('startingLedger', 'STARTING_LEDGER'), + ('latestLedgerNum', 'LATEST_LEDGER_NUM'), + ('ledgersPerJob', 'LEDGERS_PER_JOB'), + ('overlapLedgers', 'OVERLAP_LEDGERS')): + if key in spec: + setattr(mc, name, spec[key]) + mc.set_profile(sizing.load_profile(doc.get('profile') or {})) + mc.validate() + self.ranges = dispatch.range_list() + logger.info("run started: %d ranges, %s, profile of %d", + len(self.ranges), mc.RANGE_ORDER, len(mc.PROFILE)) + + def condemn(self, st, reason): + """Mark terminal; commit() persists it at the end of the pass. + + Idempotent because it must be: the record is all that keeps a condemned + range terminal once its Job is gone, and counting the verdict twice + would inflate the run's reasons forever. Dispatch does not halt -- + ending the run is the driver's call. + """ + if st.end in self.progress['condemned']: + return + self.progress['condemned'][st.end] = { + 'end': int(st.end), 'count': st.count, 'attempts': st.attempt, + 'pod': (st.verdict or {}).get('pod') or '', + 'outcome': (st.verdict or {}).get('outcome'), + 'exitCode': (st.verdict or {}).get('exitCode'), + 'reason': reason} + metrics.settled(self.progress, (st.verdict or {}).get('outcome')) + logger.error("range %s condemned after %d attempts: %s", + st.end, st.attempt, reason) + + def commit(self, states): + """Every write a pass makes, in one place and one record write. + + A completed range keeps being re-aggregated while its attempt files are + readable, so measurements landing after the Job succeeded are picked up; + once reaped the recorded entry stands. + """ + for st in states: + if st.measured is not None: + self.progress['completed'][st.end] = st.measured + for end, created in self.started: + record.record_start(end, created) + self.started.clear() + record.save_progress(self.progress) + + def counts(self, states): + """The four buckets, which must not overlap. + + `remaining` is work NOT YET DISPATCHED, not work outstanding: a range in + flight is counted by `running` alone. Overlapping them makes a + fully-dispatched run of four ranges report four remaining and four + running, and every consumer that sums the buckets sees eight. + """ + self._live = { + 'remaining': sum(1 for st in states if st.status == 'pending'), + 'running': sum(1 for st in states if st.holds_capacity), + 'completed': sum(1 for st in states if st.status == 'completed'), + 'condemned': sum(1 for st in states if st.status == 'condemned'), + } + return dict(self._live) + + def status(self): + """What the driver decides on, in both shapes. + + `remaining` is 1 until the first real pass, or a driver polling a fresh + monitor sees zero and tears the mission down. The legacy keys ship until + no old driver is left: it reads the counts as ints (absent = 0 = done) + and iterates jobs_failed (absent = null = throws). + """ + live = self._live or {'remaining': 1, 'running': 0, 'completed': 0, + 'condemned': 0} + condemned = list(self.progress['condemned'].values()) + return dict(live, + condemned=condemned, + num_remain=live['remaining'], + queue_remain_count=live['remaining'], + queue_in_progress_count=live['running'], + queue_succeeded_count=live['completed'], + queue_failed_count=len(condemned), + jobs_failed=condemned) + + +def aggregate(st): + """One completed range, from its whole attempt chain. + + Every field is advisory -- it sizes a LATER run -- so any may be absent, and + absent must stay absent. A zero becomes a profile entry, and since a range + resolves to the nearest measured end ABOVE it, one such entry captures every + range beneath it and hides the real measurement. + """ + # Read once per attempt; every field below comes out of this. + seen = {n: record.read_metrics(st.end, n) for n in range(1, st.attempt + 1)} + out = {'count': st.count, 'attempts': st.attempt} + + # Every attempt: max is monotone, and no excluded input would be wrong. + peaks = {} + for metrics_of in seen.values(): + for key, value in metrics_of.items(): + if key.startswith('peak') and value is not None: + peaks[key] = max(peaks.get(key, value), value) + out.update(peaks) + + # Resumed chain only: a fresh retry discarded its predecessor's work. + chain = _resumed_chain(st.end, seen, st.attempt) + for field, source in (('seconds', 'attemptSeconds'), + ('txApply', 'txApplySeconds')): + legs = [seen[n].get(source) for n in chain] + if legs and all(leg is not None for leg in legs): + out[field] = sum(legs) + + wall = _wall_seconds(st) + if wall is not None: + out['wallSeconds'] = wall + return out + + +def _resumed_chain(end, seen, attempt): + """The winning attempt, plus every predecessor it continued from. + + `resumed` sits on the attempt that continued, so it names its own + predecessor. A fresh start breaks the chain: an attempt that ran new-db + discarded whatever came before it. + + An attempt that ran nothing is stepped over rather than counted: it left no + LCL, so what its successor resumed was written further back. Counting it + would need a leg it never produced and drop the range's compute total; + stopping there would credit the range with only its last attempt. + """ + chain, n = [attempt], attempt + while n > 1 and seen[n].get('resumed'): + n -= 1 + while n > 1 and record.ran_nothing(end, n, seen[n]): + n -= 1 + chain.append(n) + return sorted(chain) + + +def _wall_seconds(st): + """Attempt 1's dispatch to the winning attempt's completion. + + The range's whole life: every retry, every gap, every wait for a node. Not + the winning Job's own start, which measures one leg and understates exactly + the mess this exists to capture -- which is why attempt 1's creation + timestamp is persisted before its Job can be deleted. + """ + started = record.started_at(st.end) + if started is None or st.completed_at is None: + return None + return max(0.0, (st.completed_at - started).total_seconds()) + + +if __name__ == '__main__': + sys.exit(asyncio.run(main())) diff --git a/src/MissionParallelCatchup/apps/log_collector.py b/src/MissionParallelCatchup/apps/log_collector.py new file mode 100644 index 00000000..cdcf7281 --- /dev/null +++ b/src/MissionParallelCatchup/apps/log_collector.py @@ -0,0 +1,531 @@ +"""Streaming log collector for parallel catchup. + +Runs as a sidecar next to job_monitor, sharing its /logs volume. One cycle: +list the run's pods, then read every pod's log and sample every node's kubelet +off that list. Nothing is read after a Job finishes -- Karpenter deletes the +node about a minute after its last pod exits, taking every pod object with it, +so the answers have to be taken while the pod still exists. + +Two conditions carry all the work. The FIRST read of an attempt carries its +resume decision; the read that finds the pod terminal carries its verdict, its +tx-apply total and the .done marker the monitor reaps on. Everything between is +appending bytes. + +Resume is idempotent across a dropped read and a restart: reconnect with +sinceTime=, which has second granularity and so overlaps +on purpose, then drop any line whose own kubelet timestamp is <= it. Dying +between the append and the state write replays one read's worth of lines, so +this is at least once, deduped to near-exact. +""" + +import asyncio +import gzip +import io +import json +import os +import re +from datetime import datetime, timedelta + +import aiohttp +from logger import build_logger +import collector_config as cc +import kube_http +import config +import records +import state_files +import tx_scan +import verdicts + +logger = build_logger('log_collector', name='log-collector', to_file=False) + +# Bounds concurrent log reads, one coroutine per pod per cycle. +_poll_slots = asyncio.Semaphore(cc.MAX_CONCURRENT_POLLS) +# One request per NODE, and a dead one costs the connect timeout. +_sample_slots = asyncio.Semaphore(cc.MAX_CONCURRENT_SAMPLES) +# Follows get their own budget: one holds its slot for a whole drain, so sharing +# would let a mass reclaim starve every ordinary read in the run. +_follow_slots = asyncio.Semaphore(cc.MAX_DOOMED_FOLLOWS) +# (end, attempt) -> resume timestamp, hydrated from .state at boot and written +# back through. Keyed like every file on the volume, so a vanished attempt is +# still identifiable once its pod is gone. +_last_ts = {} +# (end, attempt) -> the follow task held through a drain. The only work that +# outlives a cycle. +_follows = {} + + +async def main(): + os.makedirs(config.LOG_DIR, exist_ok=True) + conn = aiohttp.TCPConnector( + limit=cc.MAX_CONCURRENT_POLLS + cc.MAX_DOOMED_FOLLOWS + 64, + ssl=kube_http.ssl_ctx()) + # sock_read, because a cycle gathers every pod: one wedged read would hold + # up the whole pass. No total timeout -- a follow is meant to be held. + timeout = aiohttp.ClientTimeout(total=None, sock_connect=10, + sock_read=cc.READ_TIMEOUT_SECONDS) + _last_ts.clear() + _last_ts.update(state_files.hydrate_states()) + logger.info("resuming %d attempts from disk", len(_last_ts)) + + async with aiohttp.ClientSession(connector=conn, timeout=timeout) as session: + logger.info("streaming logs for run=%s into %s", config.RUN_NAME, config.LOG_DIR) + if cc.WATCH_TIMEOUT_SECONDS > 0: + # The one thing that cannot ride the cycle: a held watch starts a + # follow between sweeps, which is the whole point of it. + asyncio.create_task(watch_condemnations(session)) + while True: + try: + pods = await list_pods(session) + ranges = {p['metadata']['name']: _identify(p) + for p in pods if _identify(p)} + # Claimed while Pending too, so the sweep below can finish an + # attempt whose pod is deleted before it is ever pollable. + for end, attempt in ranges.values(): + if (end, attempt) not in _last_ts and not os.path.exists( + records.done_path(end, attempt)): + _write_state(end, attempt, '') + nodes = {p['status']['hostIP'] for p in pods + if p.get('status', {}).get('hostIP') + and p.get('status', {}).get('phase') == 'Running'} + await asyncio.gather( + _bounded(_poll_slots, + [service_pod(session, p) for p in pods if _wanted(p)]), + _bounded(_sample_slots, + [sample_node(session, ip, ranges) for ip in nodes])) + _sweep_vanished(set(ranges.values())) + except asyncio.CancelledError: + raise + except Exception as e: + logger.warning("cycle failed: %s", e) + await asyncio.sleep(cc.POLL_SECONDS) + + +async def _bounded(slots, coros): + """Run coros under `slots`. One failure must not cancel the other 4095.""" + async def run(c): + async with slots: + return await c + return await asyncio.gather(*(run(c) for c in coros), return_exceptions=True) + + +def _identify(pod): + """(end, attempt) for a worker pod, or None if it is not one of ours.""" + labels = pod['metadata'].get('labels', {}) + end = labels.get(config.LABEL_RANGE) + return (end, labels.get(config.LABEL_ATTEMPT, '1')) if end else None + + +# Phases whose log endpoint can answer. Pending has no container yet and Unknown +# means the node stopped reporting; the terminal phases are kept because that is +# where a pod's final output lives. +POLLABLE_PHASES = ('Running', 'Succeeded', 'Failed') + + +def _wanted(pod): + """Pods whose log endpoint can answer and whose attempt is not finished. + + Pending has no container yet and Unknown means the node stopped reporting; + the terminal phases are kept because that is where the final output lives. + """ + key = _identify(pod) + if key is None or pod.get('status', {}).get('phase') not in POLLABLE_PHASES: + return False + return not os.path.exists(records.done_path(*key)) + + +async def service_pod(session, pod): + """One pod, one cycle: read what is new, then apply whichever condition fits.""" + name = pod['metadata']['name'] + end, attempt = _identify(pod) + phase = pod.get('status', {}).get('phase') + terminal = phase in ('Succeeded', 'Failed') + + if not terminal and _start_follow(session, name, end, attempt, pod): + return # the follow owns this attempt's stream + if (end, attempt) not in _last_ts: + _write_state(end, attempt, '') + since = _last_ts[(end, attempt)] + # A terminal read reaches further back, so a medida block split across two + # reads is whole in this one. _ingest still dedups, so the archive is + # unchanged -- only the scan sees the overlap. + text, gone = await _read(session, name, _rewind(since) if terminal else since) + if text: + _write_state(end, attempt, _ingest(end, attempt, text, since)) + _scan(end, attempt, text) + if terminal or gone: + _finish(None if gone else pod, end, attempt, phase == 'Succeeded') + + +def _start_follow(session, name, end, attempt, pod): + """Hold a stream through a drain, if this pod has been condemned. + + stellar-core prints its medida block ~4ms after SIGTERM and the object is + deleted seconds later, so an interval read straddles the whole thing -- on + the 2048-worker run, 810 evictions lost 809 txApply values. A held + connection already has those bytes, and is held only for the drain. + """ + key = (end, attempt) + if key in _follows: + return not _follows[key].done() + if cc.DOOMED_FOLLOW_SECONDS <= 0: + return False + doom = verdicts.condemnation_reason(pod) + if not doom: + return False + # Recorded now: once the object is gone there is no telling a drain we lost + # a race with from a corpse that never had a metric to lose. The duration + # goes with it, and for the same reason -- an evicted pod is usually deleted + # before it is ever seen terminal, and _finish has nothing to read it from. + # It runs short by the drain, and merges by max, so it is only ever a floor. + write_metrics(end, attempt, {'disruptionReason': doom, **_duration(pod)}) + logger.info("range %s condemned (%s), opening follow", end, doom) + _follows[key] = asyncio.create_task(_run_follow(session, name, end, attempt)) + return True + + +async def _run_follow(session, name, end, attempt): + async with _follow_slots: + since = _last_ts.get((end, attempt), '') + try: + text, _ = await _read(session, name, since, follow=True) + except asyncio.CancelledError: + raise + except Exception as e: + logger.info("range %s follow ended (%s); back to interval reads", end, e) + return + if text: + _write_state(end, attempt, _ingest(end, attempt, text, since)) + _scan(end, attempt, text) + + +def _finish(pod, end, attempt, succeeded): + """Everything the attempt owes, then the marker that licenses a reap.""" + if pod is not None: + if (pod.get('status') or {}).get('phase') == 'Failed': + verdicts.record_outcome(pod, end, attempt) + write_metrics(end, attempt, _duration(pod)) + if not cc.SAVE_SUCCESS_LOGS and succeeded: + # .metrics survives on purpose: it holds tx_apply for a range that + # succeeded, and a retention flag must not delete a Grafana series. + discard(end, attempt) + task = _follows.pop((end, attempt), None) + if task is not None and not task.done(): + task.cancel() + _last_ts.pop((end, attempt), None) + # LAST. The monitor treats this as "nothing further is coming" and only then + # reaps the Job -- which deletes the pod, the one place peaks can be read + # from. Ahead of the metrics it would license exactly the reap it prevents. + try: + records.write_atomic(records.done_path(end, attempt), '') + except OSError as e: + # Costs a Job that waits out its TTL, never correctness. + logger.warning("could not mark range %s attempt %s done: %s", end, attempt, e) + logger.info("range %s attempt %s finished", end, attempt) + + +def _duration(pod): + """How long the container ran, and how well that is known. + + The pod's own start->finish is exact. Its startTime against now is off by + the cycle that noticed, but still measures the CONTAINER -- the distinction + the monitor's chain gate cares about, and the only duration a disrupted + attempt ever produces. A clock this process started measures neither, and is + not recorded: the monitor rejects it anyway. + """ + st = pod.get('status') or {} + start = st.get('startTime') + if not start: + return {} + try: + began = datetime.strptime(start, '%Y-%m-%dT%H:%M:%SZ') + except ValueError: + return {} + for cs in (st.get('containerStatuses') or []): + fin = ((cs.get('state') or {}).get('terminated') or {}).get('finishedAt') + if fin: + try: + ended = datetime.strptime(fin, '%Y-%m-%dT%H:%M:%SZ') + except ValueError: + break + return {'attemptSeconds': round((ended - began).total_seconds(), 1), + 'attemptSecondsExact': True} + since = (datetime.utcnow() - began).total_seconds() + if since <= 0: + return {} + return {'attemptSeconds': round(since, 1), 'attemptSecondsExact': False, + 'attemptSecondsFromContainerStart': True} + + +def _sweep_vanished(live): + """Attempts with state on the volume and no pod left to read. + + A reaped node, an eviction or the monitor deleting a finished Job all take + the pod without it ever being seen terminal. The archive still holds real + bytes, so mark the attempt done rather than leaving its Job to time out. + """ + for key in [k for k in _last_ts if k not in live]: + logger.info("range %s attempt %s: pod gone, finishing on what was read", *key) + _finish(None, key[0], key[1], False) + + +# --- reading ----------------------------------------------------------------- + +def _rewind(ts): + """`ts` moved back a few seconds, for the overlapping terminal read.""" + if not ts: + return ts + try: + return (datetime.strptime(ts[:19], '%Y-%m-%dT%H:%M:%S') + - timedelta(seconds=cc.TERMINAL_REREAD_SECONDS) + ).strftime('%Y-%m-%dT%H:%M:%SZ') + except ValueError: + return ts + + +async def _read(session, name, last_ts, follow=False): + """One read of a pod's log. Returns (text, gone). + + No follow=true on the ordinary path: the request completes and the + connection is released, so concurrency is bounded by the gather rather than + by how many pods exist. Held connections cost the sidecar 1444 MiB of a + 2048 MiB limit at 2096 streams. + """ + params = {'container': cc.CONTAINER, 'timestamps': 'true'} + if follow: + params['follow'] = 'true' + if last_ts: + # Second granularity, so this overlaps on purpose; _ingest removes it. + params['sinceTime'] = last_ts[:19] + 'Z' + url = f"{kube_http.API}/api/v1/namespaces/{config.NAMESPACE}/pods/{name}/log" + async with session.get(url, params=params, + headers={'Authorization': f'Bearer {kube_http.token()}'}) as resp: + if resp.status == 404: + return '', True + resp.raise_for_status() + # Chunked, not line-wise: aiohttp raises above 512 KiB on one line and a + # carriage-return progress meter exceeds that -- a 628 MiB download once + # arrived as a single "line". The cap is the backstop against a blob + # that never terminates. + body = '' + async for chunk in resp.content.iter_chunked(65536): + body += chunk.decode('utf-8', 'replace') + if len(body) > cc.MAX_POLL_CHARS: + break + return body, False + + +def _ingest(end, attempt, body, last_ts): + """Append the new lines to the archive; return the resume point.""" + pending = None + # Compressed into memory, then appended in ONE write, so the file only ever + # gains whole gzip members. Appending through gzip.open left it ending in a + # member with no end-of-stream marker for most of a large write, and the + # monitor reads that same file -- one in-flight append could abort a + # reconcile pass for every range. + member = io.BytesIO() + wrote = False + with gzip.GzipFile(fileobj=member, mode='wb') as fh: + for line in (l for l in re.split(r'[\r\n]', body) if l): + ts, _, rest = line.partition(' ') + if not state_files.TS_RE.match(ts): + fh.write((line + '\n').encode('utf-8')) # keep, never resume from + wrote = True + continue + if last_ts and ts <= last_ts: + continue # exact dedup of the overlap + fh.write((rest + '\n').encode('utf-8')) + wrote = True + pending = ts + if wrote: + with open(records.log_path(end, attempt), 'ab') as out: + out.write(member.getvalue()) + return pending or last_ts + + +def _scan(end, attempt, text): + """Record whatever this text proves: that the attempt resumed, or its total. + + Both are printed once -- RESUME before stellar-core starts, the medida block + at exit -- so a throwaway scanner over each read finds them with nothing + carried between cycles. + """ + scanner = tx_scan.TxApplyScanner() + for line in text.splitlines(): + scanner.feed(line) + found = {} + if scanner.resumed: + found['resumed'] = True + if scanner.seconds is not None: + found['txApplySeconds'] = scanner.seconds + write_metrics(end, attempt, found) + + +# --- the volume -------------------------------------------------------------- + +def _write_state(end, attempt, ts): + _last_ts[(end, attempt)] = ts + try: + records.write_atomic(records.state_path(end, attempt), ts) + except OSError as e: + logger.warning("could not persist state for range %s: %s", end, e) + + +def discard(end, attempt): + for path in (records.log_path(end, attempt), records.state_path(end, attempt)): + try: + os.remove(path) + except OSError: + pass + + +# Fields that only ever grow, so a merge maxes them instead of overwriting. +PEAK_KEYS = ('peakAnonBytes', 'peakWorkingSetBytes', 'peakEphemeralBytes') + + +def write_metrics(end, attempt, values): + """Merge measurements into .metrics for the monitor's reconcile to read. + + Never lets a peak or a duration go backwards, and never un-proves a flag: a + later writer can only know LESS -- a restarted collector re-accumulates from + whatever the pod is using now, and its lower reading would undersize the + range next run, which is the one direction that costs an OOM. + """ + if not values: + return + path = records.metrics_path(end, attempt) + try: + with open(path) as fh: + prior = json.load(fh) + except (OSError, ValueError): + prior = {} + merged = {**prior, **values} + for k in PEAK_KEYS + ('attemptSeconds',): + a, b = prior.get(k), values.get(k) + if a is not None and b is not None: + merged[k] = max(a, b) + for flag in ('resumed', 'attemptSecondsExact', 'attemptSecondsFromContainerStart'): + if prior.get(flag) is True or values.get(flag) is True: + merged[flag] = True + if merged == prior: + return # nothing new -- this is what makes per-cycle sampling cheap + try: + records.write_atomic(path, json.dumps(merged)) + logger.info("range %s attempt %s metrics=%s", end, attempt, merged) + except OSError as e: + logger.warning("could not persist metrics for range %s: %s", end, e) + + +# --- kubelet ----------------------------------------------------------------- + +async def sample_node(session, ip, ranges): + """Peak memory and disk for every worker on one node, straight to .metrics. + + Nothing is held in memory: write_metrics keeps the running max on the + volume, so a restart cannot lower a high-water and there is no per-pod peak + to lose. kubelet's rssBytes is cgroup v2 anon, the only limit-independent + memory figure this workload has -- page cache grows to fill whatever + memory.max allows, so memory.peak is always ~= the limit and useless for + sizing. Prometheus can answer neither axis: no pod label on fs usage, and a + 30s scrape is the undersampling that let profiled ranges OOM. + + Straight at the kubelet, not the apiserver's node proxy: that needs + nodes/proxy, which authorizes GET on every kubelet path including + /containerLogs for any namespace on that node. ssl=False because EKS serving + certs are self-signed; this is an in-VPC hop to the node's own address. + """ + url = f"https://{ip}:{kube_http.KUBELET_PORT}/stats/summary" + try: + async with session.get(url, ssl=False, + headers={'Authorization': f'Bearer {kube_http.token()}'}) as resp: + resp.raise_for_status() + summary = await resp.json() + except Exception as e: + # Not debug: a silent failure here leaves the profile looking merely + # absent rather than broken. + logger.warning("kubelet stats unavailable on %s: %s", ip, e) + return + for entry in summary.get('pods', []): + key = ranges.get((entry.get('podRef') or {}).get('name')) + if key is None: + continue + found = {} + used = (entry.get('ephemeral-storage') or {}).get('usedBytes') + if used is not None and config.STORAGE_MODE == 'ephemeral': + found['peakEphemeralBytes'] = int(used) + for c in entry.get('containers', []): + # The worker container only: sidecars share the pod, and letting the + # last one win would size the range from whichever kubelet listed. + if c.get('name') != cc.CONTAINER: + continue + mem = c.get('memory') or {} + if mem.get('workingSetBytes') is not None: + found['peakWorkingSetBytes'] = int(mem['workingSetBytes']) + if mem.get('rssBytes') is not None: + found['peakAnonBytes'] = int(mem['rssBytes']) + write_metrics(key[0], key[1], found) + + +# --- the apiserver ----------------------------------------------------------- + +async def list_pods(session): + url = f"{kube_http.API}/api/v1/namespaces/{config.NAMESPACE}/pods" + params = {'labelSelector': f"{config.LABEL_RUN}={config.RUN_NAME}"} + async with session.get(url, params=params, + headers={'Authorization': f'Bearer {kube_http.token()}'}) as resp: + resp.raise_for_status() + return (await resp.json()).get('items', []) + + +async def watch_condemnations(session): + """Start follows the moment a condemnation is written, ahead of the cycle. + + Only ever earlier than the cycle would have been -- the difference between + opening a follow while stellar-core still runs and opening it on a 404. One + connection served from the apiserver's cache, sending deltas, where the + cycle re-lists every pod. Never fatal: any failure falls back to the cycle. + """ + url = f"{kube_http.API}/api/v1/namespaces/{config.NAMESPACE}/pods" + rv = None + while True: + params = {'labelSelector': f"{config.LABEL_RUN}={config.RUN_NAME}", + 'watch': 'true', 'allowWatchBookmarks': 'true', + 'timeoutSeconds': str(cc.WATCH_TIMEOUT_SECONDS)} + if rv: + params['resourceVersion'] = rv + try: + async with session.get(url, params=params, + headers={'Authorization': f'Bearer {kube_http.token()}'}) as resp: + if resp.status == 410: + rv = None # aged out of history; re-sync from scratch + continue + resp.raise_for_status() + async for raw in resp.content: + if not raw.strip(): + continue + try: + ev = json.loads(raw) + except ValueError: + continue + obj = ev.get('object') or {} + meta = obj.get('metadata') or {} + # Tracked on every event, bookmarks included -- that is what + # they are for -- so a reconnect resumes rather than re-syncs. + rv = meta.get('resourceVersion') or rv + if ev.get('type') == 'ERROR': + if obj.get('code') == 410: + rv = None + break + if ev.get('type') not in ('ADDED', 'MODIFIED') or not meta: + continue + key = _identify(obj) + if key and (obj.get('status') or {}).get('phase') == 'Running': + _start_follow(session, meta['name'], key[0], key[1], obj) + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning("condemnation watch dropped (%s); retrying", exc) + await asyncio.sleep(cc.WATCH_RETRY_SECONDS) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/src/MissionParallelCatchup/job_monitor.py b/src/MissionParallelCatchup/job_monitor.py deleted file mode 100644 index 3778eeb2..00000000 --- a/src/MissionParallelCatchup/job_monitor.py +++ /dev/null @@ -1,248 +0,0 @@ -import os -import redis -import requests -import json -import sys -import logging -import threading -import time -from http.server import BaseHTTPRequestHandler, HTTPServer -from prometheus_client import Gauge, Counter, Histogram, generate_latest, REGISTRY, CONTENT_TYPE_LATEST -from datetime import datetime, timezone - -# Configuration - -# Histogram buckets -# 5m 15m 30m 1h 1.5h 2h -metric_buckets = (300, 900, 1800, 3600, 5400, 7200, float("inf")) -REDIS_HOST = os.getenv('REDIS_HOST', 'redis') -REDIS_PORT = int(os.getenv('REDIS_PORT', '6379')) -JOB_QUEUE = os.getenv('JOB_QUEUE', 'ranges') -SUCCESS_QUEUE = os.getenv('SUCCESS_QUEUE', 'succeeded') -FAILED_QUEUE = os.getenv('FAILED_QUEUE', 'failed') -PROGRESS_QUEUE = os.getenv('PROGRESS_QUEUE', 'in_progress') -METRICS = os.getenv('METRICS', 'metrics') -JOB_OWNERS = os.getenv('JOB_OWNERS', 'job_owners') -WORKER_PREFIX = os.getenv('WORKER_PREFIX', 'stellar-core') -NAMESPACE = os.getenv('NAMESPACE', 'default') -WORKER_COUNT = int(os.getenv('WORKER_COUNT', 3)) -LOGGING_INTERVAL_SECONDS = int(os.getenv('LOGGING_INTERVAL_SECONDS', 10)) -STUCK_JOB_PING_RETRIES = 3 -STUCK_JOB_PING_DELAY_SECS = 30 - -def get_logging_level(): - name_to_level = { - 'CRITICAL': logging.CRITICAL, - 'ERROR': logging.ERROR, - 'WARNING': logging.WARNING, - 'INFO': logging.INFO, - 'DEBUG': logging.DEBUG, - } - result = name_to_level.get(os.getenv('LOGGING_LEVEL', 'INFO')) - if result is not None: - return result - else: - return logging.INFO - -# Initialize Redis client -redis_client = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True) - -# Configure logging -log_file_name = f"job_monitor_{datetime.now(timezone.utc).strftime('%Y-%m-%d_%H-%M-%S')}.log" -log_file_path = os.path.join('/data', log_file_name) -logging.basicConfig(level=get_logging_level(), format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler(log_file_path), -]) -logger = logging.getLogger() - -# In-memory status data structure and threading lock -status = { - 'num_remain': 1, # initialize the job remaining to non-zero to indicate something is running, just the status hasn't been updated yet - 'queue_remain_count': 0, - 'queue_succeeded_count': 0, - 'queue_failed_count': 0, - 'queue_in_progress_count': 0, - 'jobs_failed': [], - 'jobs_in_progress': [], - 'workers': [], - 'workers_up': 0, - 'workers_down': 0, - 'workers_refresh_duration': 0, - 'mission_duration': 0, -} -status_lock = threading.Lock() - -metrics = { - 'metrics': [] -} -metrics_lock = threading.Lock() - -# Create metrics -metric_catchup_queues = Gauge('ssc_parallel_catchup_queues', 'Exposes size of each job queues', ["queue"]) -metric_workers = Gauge('ssc_parallel_catchup_workers', 'Exposes catch up worker status', ["status"]) -metric_refresh_duration = Gauge('ssc_parallel_catchup_workers_refresh_duration_seconds', 'Time it took to refresh status of all workers') -metric_full_duration = Histogram('ssc_parallel_catchup_job_full_duration_seconds', 'Exposes full job duration as histogram', buckets=metric_buckets) -metric_tx_apply_duration = Histogram('ssc_parallel_catchup_job_tx_apply_duration_seconds', 'Exposes job TX apply duration as histogram', buckets=metric_buckets) -metric_mission_duration = Gauge('ssc_parallel_catchup_mission_duration_seconds', 'Number of seconds since the mission started ') -metric_retries = Counter('ssc_parallel_catchup_job_retried_count', 'Number of jobs that were retried') - - -class RequestHandler(BaseHTTPRequestHandler): - def do_GET(self): - if self.path == '/status': - self.send_response(200) - self.send_header('Content-type', 'application/json') - self.end_headers() - with status_lock: - self.wfile.write(json.dumps(status).encode()) - elif self.path == '/prometheus': - self.send_response(200) - self.send_header('Content-type', CONTENT_TYPE_LATEST) - self.end_headers() - self.wfile.write(generate_latest(REGISTRY)) - elif self.path == '/metrics': - self.send_response(200) - self.send_header('Content-type', 'application/json') - self.end_headers() - with metrics_lock: - self.wfile.write(json.dumps(metrics).encode()) - else: - self.send_response(404) - self.end_headers() - -def retry_jobs_in_progress(): - while redis_client.llen(PROGRESS_QUEUE) > 0: - job = redis_client.lmove(PROGRESS_QUEUE, JOB_QUEUE, "RIGHT", "LEFT") - metric_retries.inc() - logger.info("moved job %s from %s to %s", job, PROGRESS_QUEUE, JOB_QUEUE) - -def ping_worker(pod_name, retries=1): - """Ping a worker's /info endpoint. Returns True if reachable within the given number of attempts.""" - worker_dns = f"{pod_name}.{WORKER_PREFIX}.{NAMESPACE}.svc.cluster.local" - for attempt in range(1, retries + 1): - try: - requests.get(f"http://{worker_dns}:11626/info", timeout=5) - return True - except requests.exceptions.RequestException: - if attempt < retries: - logger.info("Worker %s unreachable (attempt %d/%d), retrying in %ds", - pod_name, attempt, retries, STUCK_JOB_PING_DELAY_SECS) - time.sleep(STUCK_JOB_PING_DELAY_SECS) - return False - -def update_status_and_metrics(): - global status - mission_start_time = time.time() - while True: - try: - # --- Phase 1: Read queue status from Redis (fast, authoritative) --- - queue_remain_count = redis_client.llen(JOB_QUEUE) - queue_succeeded_count = redis_client.llen(SUCCESS_QUEUE) - jobs_failed = redis_client.lrange(FAILED_QUEUE, 0, -1) - jobs_in_progress = redis_client.lrange(PROGRESS_QUEUE, 0, -1) - queue_failed_count = len(jobs_failed) - queue_in_progress_count = len(jobs_in_progress) - - # --- Phase 2: Quick single-ping check of workers that own in-progress jobs --- - job_owners = redis_client.hgetall(JOB_OWNERS) # {job_key: pod_name} - active_workers = set(job_owners.values()) - worker_statuses = [] - workers_up = 0 - workers_down = 0 - - logger.info("Starting worker liveness check for %d active workers", len(active_workers)) - workers_refresh_start_time = time.time() - for pod_name in active_workers: - if ping_worker(pod_name, retries=1): - worker_statuses.append({'pod': pod_name, 'status': 'running'}) - workers_up += 1 - else: - worker_statuses.append({'pod': pod_name, 'status': 'down'}) - workers_down += 1 - workers_refresh_duration = time.time() - workers_refresh_start_time - logger.info("Finished workers liveness check") - - # --- Phase 3: Retry stuck jobs (only if ALL active workers down AND in_progress non-empty) --- - # Note: queue_in_progress_count is from Phase 1 and may be stale by this point. - # If a job completed during Phase 2 pings, we may enter this block unnecessarily. - # This is harmless: retry_jobs_in_progress() will find an empty in_progress queue - # and exit immediately. We waste at most 90 seconds (30 secs x 3 pings) per stale worker. - if workers_down > 0 and workers_up == 0 and queue_in_progress_count > 0: - logger.warning("All %d active workers appear down with %d jobs in progress. " - "Entering retry confirmation.", workers_down, queue_in_progress_count) - - # Re-ping each active worker with retries to confirm they're truly down - any_recovered = False - for pod_name in active_workers: - if ping_worker(pod_name, retries=STUCK_JOB_PING_RETRIES): - logger.info("Worker %s responded during retry confirmation. Exiting retry block.", pod_name) - any_recovered = True - break - - if not any_recovered: - logger.error("All active workers confirmed down after %d retry attempts. Retrying stuck jobs.", - STUCK_JOB_PING_RETRIES) - retry_jobs_in_progress() - for job_key in list(job_owners.keys()): - redis_client.hdel(JOB_OWNERS, job_key) - else: - logger.info("At least one worker recovered. Skipping job retry.") - - mission_duration = time.time() - mission_start_time - - # Update the status - with status_lock: - status = { - 'num_remain': queue_remain_count, # Needed for backwards compatibility with the rest of the code - 'queue_remain_count': queue_remain_count, - 'queue_succeeded_count': queue_succeeded_count, - 'queue_failed_count': queue_failed_count, - 'queue_in_progress_count': queue_in_progress_count, - 'jobs_failed': jobs_failed, - 'jobs_in_progress': jobs_in_progress, - 'workers': worker_statuses, - 'workers_up': workers_up, - 'workers_down': workers_down, - 'workers_refresh_duration': workers_refresh_duration, - 'mission_duration': mission_duration, - } - metric_catchup_queues.labels(queue="remain").set(queue_remain_count) - metric_catchup_queues.labels(queue="succeeded").set(queue_succeeded_count) - metric_catchup_queues.labels(queue="failed").set(queue_failed_count) - metric_catchup_queues.labels(queue="in_progress").set(queue_in_progress_count) - metric_workers.labels(status="up").set(workers_up) - metric_workers.labels(status="down").set(workers_down) - metric_refresh_duration.set(workers_refresh_duration) - metric_mission_duration.set(mission_duration) - logger.info("Status: %s", json.dumps(status)) - - # update the metrics - new_metrics = redis_client.spop(METRICS, 1000) - if len(new_metrics) > 0: - with metrics_lock: - metrics['metrics'].extend(new_metrics) - for timing in new_metrics: - # Example: 36024000/8320|213|1.47073e+06ms|2965s - _, _, tx_apply, full_duration = timing.split('|') - metric_full_duration.observe(float(full_duration.rstrip('s'))) - metric_tx_apply_duration.observe(float(tx_apply.rstrip('ms'))/1000) - logger.info("Metrics: %s", json.dumps(metrics)) - - except Exception as e: - logger.error("Error while getting status: %s", str(e)) - - time.sleep(LOGGING_INTERVAL_SECONDS) - -def run(server_class=HTTPServer, handler_class=RequestHandler): - server_address = ('', 8080) - httpd = server_class(server_address, handler_class) - logger.info('Starting httpd server...') - httpd.serve_forever() - -if __name__ == '__main__': - log_thread = threading.Thread(target=update_status_and_metrics) - log_thread.daemon = True - log_thread.start() - - run() diff --git a/src/MissionParallelCatchup/lib/collector/collector_config.py b/src/MissionParallelCatchup/lib/collector/collector_config.py new file mode 100644 index 00000000..4d768423 --- /dev/null +++ b/src/MissionParallelCatchup/lib/collector/collector_config.py @@ -0,0 +1,62 @@ +"""Settings for the log-collector sidecar, read from the environment once. + +Every value the chart can tune lives here rather than beside the code that reads +it, so log_collector.py opens on its own logic. Named collector_config, not +config: runtime flattens lib/ into a single /app, so a second config.py would +silently replace the shared one. +""" + +import os + +# The worker container whose log is collected. Sidecars share the pod, so this +# is what keeps a peak or a log line from being attributed to the wrong one. +CONTAINER = os.getenv('WORKER_CONTAINER', 'stellar-core') +# Seconds between sweeps of the run's pod list, which is what discovers new pods +# and notices ones that went terminal. +POLL_SECONDS = float(os.getenv('COLLECTOR_POLL_SECONDS', 5)) +# Concurrent in-flight log polls, across all pods. This is the whole point of +# polling: it is independent of how many pods exist, where follow=true needed one +# held connection per pod forever. +MAX_CONCURRENT_POLLS = int(os.getenv('MAX_CONCURRENT_POLLS', 96)) +# Most one poll may read before it stops, bounding a single response; the next +# poll picks up from the timestamp this one reached. Also the backstop against a +# blob that never terminates -- a progress meter emitting only carriage returns +# would otherwise grow the buffer until the sidecar OOMs. +MAX_POLL_CHARS = int(os.getenv('MAX_POLL_CHARS', 8388608)) +# Concurrent kubelet sweeps. One request per node, and a dead node costs the +# connect timeout, so this is small next to the log-read budget. +MAX_CONCURRENT_SAMPLES = int(os.getenv('MAX_CONCURRENT_SAMPLES', 16)) +# Longest a single log read may stall before it is abandoned. A cycle gathers +# every pod, so one wedged read would hold up the whole pass -- where the old +# task-per-pod design only stalled that pod. +READ_TIMEOUT_SECONDS = float(os.getenv('READ_TIMEOUT_SECONDS', 60)) +# Longest a follow stream will hang on to a doomed pod. Spot gives 120s; past +# roughly double that the notice was withdrawn and the stream would otherwise be +# held for the life of the range. 0 disables the follow path entirely, which is +# only safe when prestopSleepSeconds is holding the pod open instead. +DOOMED_FOLLOW_SECONDS = float(os.getenv('DOOMED_FOLLOW_SECONDS', 300)) +# Follows get their OWN budget rather than sharing the poll slots: a follow holds +# its slot for the whole drain, so a reclaim condemning more pods than there are +# slots would consume all of them and turn a partial node loss into a run-wide +# blackout. With a separate budget the worst case is that some condemned pods +# fall back to polling, which captured txApply on its own with no follow at all. +# Sized above any plausible simultaneous disruption -- a whole-AZ reclaim is not +# bounded by Karpenter's disruption budget -- and 256 x the old always-on +# design's 0.69 MiB per stream is 177 MiB against a 2048 MiB limit. +MAX_DOOMED_FOLLOWS = int(os.getenv('MAX_DOOMED_FOLLOWS', 256)) +# How long each watch connection lives before the apiserver closes it and we +# reconnect, bounded so a silently-dead stream self-heals; the reconnect resumes +# from the last resourceVersion, so nothing is missed across it. 0 disables the +# watch and leaves detection to the pod list. +WATCH_TIMEOUT_SECONDS = int(os.getenv('WATCH_TIMEOUT_SECONDS', 600)) +# Pause before re-opening a watch that failed. Only covers hard errors: a clean +# timeout reconnects immediately. +WATCH_RETRY_SECONDS = float(os.getenv('WATCH_RETRY_SECONDS', 1)) +# How far back a terminal read reaches past the resume point, so a medida block +# split across two reads is whole in one of them. The archive dedups the +# overlap, so this only widens what the scan sees. +TERMINAL_REREAD_SECONDS = int(os.getenv('TERMINAL_REREAD_SECONDS', 30)) +# Discards a succeeded attempt's archive. .metrics survives it on purpose: that +# holds tx_apply for a range that succeeded, and a retention flag must not +# delete a Grafana series. +SAVE_SUCCESS_LOGS = os.getenv('SAVE_SUCCESS_LOGS', 'true').lower() == 'true' diff --git a/src/MissionParallelCatchup/lib/collector/kube_http.py b/src/MissionParallelCatchup/lib/collector/kube_http.py new file mode 100644 index 00000000..507069d7 --- /dev/null +++ b/src/MissionParallelCatchup/lib/collector/kube_http.py @@ -0,0 +1,27 @@ +"""How the collector reaches the apiserver and the kubelet. + +The monitor uses the kubernetes client (see kube); the collector talks raw +aiohttp because it needs streaming log reads, so it carries its own credentials +and endpoints. +""" + +import os +import ssl + +SA = '/var/run/secrets/kubernetes.io/serviceaccount' +API = (f"https://{os.getenv('KUBERNETES_SERVICE_HOST', 'kubernetes.default')}" + f":{os.getenv('KUBERNETES_SERVICE_PORT', '443')}") +# Read-only kubelet port. A seam for tests, which serve the payload on a loopback +# port rather than reaching a real node. +KUBELET_PORT = 10250 + + +def token(): + # Projected service account tokens rotate, so this is re-read per request + # rather than cached at startup. + with open(os.path.join(SA, 'token')) as fh: + return fh.read().strip() + + +def ssl_ctx(): + return ssl.create_default_context(cafile=os.path.join(SA, 'ca.crt')) diff --git a/src/MissionParallelCatchup/lib/collector/state_files.py b/src/MissionParallelCatchup/lib/collector/state_files.py new file mode 100644 index 00000000..fe09aa63 --- /dev/null +++ b/src/MissionParallelCatchup/lib/collector/state_files.py @@ -0,0 +1,33 @@ +"""Resume points on the volume, read back at collector startup.""" +import os +import re + +import config + +# A kubelet log timestamp, and the only thing that may become a resume point. +TS_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z?$") + + +def hydrate_states(): + """Every attempt with a resume point on the volume: (end, attempt) -> ts. + + Read once at collector startup so a restart continues each attempt where it + stopped rather than re-reading whole logs. A state file with no timestamp + means "claimed, nothing durable yet" and hydrates as an empty resume point. + """ + out = {} + try: + names = os.listdir(config.LOG_DIR) + except OSError: + return out + for name in names: + m = re.match(r'^range-(\d+)-a(\d+)\.state$', name) + if not m: + continue + try: + with open(os.path.join(config.LOG_DIR, name)) as fh: + ts = fh.read().strip() + out[(m.group(1), m.group(2))] = ts if TS_RE.match(ts) else '' + except OSError: + continue + return out diff --git a/src/MissionParallelCatchup/lib/collector/tx_scan.py b/src/MissionParallelCatchup/lib/collector/tx_scan.py new file mode 100644 index 00000000..b74241d1 --- /dev/null +++ b/src/MissionParallelCatchup/lib/collector/tx_scan.py @@ -0,0 +1,94 @@ +"""Reading the tx-apply total out of a worker's log. + +One reader, so the window constants live here rather than in a module shared +with a second one -- the monitor's archive re-read is gone; it takes the value +from the collector's .metrics. + +stellar-core prints the block once, just before exit, so the collector scans it +off the read that finds the pod terminal. Scanning there rather than in +job_monitor is what makes the metric independent of pod lifetime: by the time +the Job is seen to succeed the node may be reaped, and with saveSuccessLogs=false +the archive is gone too. +""" + +import re + +# `sum = ms`, where the number may be in exponent form. The old +# [0-9.]+ pattern matched "1.30722" then demanded "ms" and hit "e+06ms" +# instead, so tx_apply was silently missing for 25% of ranges -- 91-99% of +# everything above ledger 35M, exactly the expensive end. 698 completed ranges +# lost the metric that way in a single run. +SUM_RE = re.compile(r"sum\s*=\s*([0-9.]+(?:[eE][+-]?[0-9]+)?)ms") + +# A medida statistic: ` = `. Anything else between the block header +# and its sum is another thread's output interleaved into the same log, and must +# not be charged against the search window. +METRIC_LINE = re.compile(r"[\w%.\-]+\s*=\s*[-+0-9.]") + +# A different metric's header. Reaching one means the block we armed on has been +# passed, so a later `sum =` belongs to some other timer. +ANY_METRIC = re.compile(r"metric '") + +# Statistics lines the sum may sit behind, and the hard line budget regardless. +# Measured 2026-08-04: a /info response pushed `sum` 91 lines down, and charging +# those lines made both readers give up while the value sat in the archive. +WINDOW = 15 +HARD_WINDOW = 400 + +TX_METRIC = "metric 'ledger.transaction.apply'" + +class TxApplyScanner: + """Pull the medida tx-apply total out of the stream as it goes past. + + stellar-core prints this block once, just before exit. Scanning here rather + than re-reading the log later is what makes the metric independent of pod + lifetime: by the time job_monitor sees the Job succeed the node may be + reaped, and with saveSuccessLogs=false the archive is gone too. + """ + + # Shared with job_monitor's archive re-read rather than restated: a + # divergence would hand the recovery path the same blind spot it exists to + # cover. A /info liveness response interleaved into the block once put `sum` + # 91 lines below the header, 76 lines past where both readers gave up. + + # Printed by RESUME_SCRIPT before stellar-core starts. The colon and space + # are load-bearing: its counterpart "RESUME DECLINED:" means new-db ran and + # this attempt did the whole range, and must not read as a resume. + RESUME_MARK = 'RESUME: ' + + def __init__(self): + self.seconds = None + self.resumed = False + self._left = 0 + self._span = 0 + + def feed(self, line): + if self.RESUME_MARK in line: + self.resumed = True + if TX_METRIC in line: + self._left = WINDOW + self._span = HARD_WINDOW + return + if self._left <= 0: + return + m = SUM_RE.search(line) + if m: + self.seconds = float(m.group(1)) / 1000.0 + self._left = 0 + return + self._span -= 1 + if self._span <= 0 or ANY_METRIC.search(line): + # Ran out of rope, or another timer's block started -- either way our + # sum is not coming. + self._left = 0 + return + if METRIC_LINE.search(line): + # Only medida's own statistics count. Interleaved output from another + # thread is noise between us and the sum, not evidence we have passed + # it. + self._left -= 1 + + + + + diff --git a/src/MissionParallelCatchup/lib/collector/verdicts.py b/src/MissionParallelCatchup/lib/collector/verdicts.py new file mode 100644 index 00000000..07c7a3c4 --- /dev/null +++ b/src/MissionParallelCatchup/lib/collector/verdicts.py @@ -0,0 +1,72 @@ +"""Why an attempt ended, decided from the pod while the pod still exists. + +The collector already lists every pod every few seconds to discover streams, so +it sees terminal transitions first-hand. The Job object cannot answer this: its +condition carries no exit code until a podFailurePolicy rule matches, and an +admission rejection matches none. +""" + +import json +import logging +import os + +import records + +logger = logging.getLogger('log_collector') + + +def condemnation_reason(pod): + """The DisruptionTarget reason if the cluster has committed to destroying + this pod, else None -- a spot reclaim, a drain or node pressure. + """ + for cond in ((pod.get('status') or {}).get('conditions') or []): + if cond.get('type') == 'DisruptionTarget' and cond.get('status') == 'True': + return cond.get('reason') or 'Unknown' + return None + + +def classify(pod): + """The outcome this pod's status implies, as a verdict dict.""" + status = pod.get('status', {}) + if condemnation_reason(pod): + return {'outcome': 'disrupted', 'exitCode': None} + reason = status.get('reason') + if reason == 'Evicted' and 'ephemeral' in (status.get('message') or ''): + # The range's own disk use, not something the cluster did to it. The + # kubelet sets no DisruptionTarget for a limit eviction and stellar-core + # exits 3 on the eviction SIGTERM, so the Job condition reads as a plain + # catchup failure, which gets no retry at all. status.message is the only + # discriminator and only the pod carries it, so it must be caught here. + return {'outcome': 'ephemeral', 'exitCode': None, 'reason': status.get('message')} + if reason in ('VolumeAttachmentLimitExceeded', 'OutOfcpu', 'OutOfmemory', 'OutOfpods', + 'UnexpectedAdmissionError', 'NodeAffinity', 'Shutdown', 'Evicted'): + return {'outcome': 'rejected', 'exitCode': None, 'reason': reason} + terms = [cs.get('state', {}).get('terminated') for cs in status.get('containerStatuses', [])] + terms = [t for t in terms if t] + if not terms: + # Nothing ever ran, so this says nothing about the ledger range. + return {'outcome': 'rejected', 'exitCode': None, 'reason': reason or 'no container status'} + for t in terms: + if t.get('reason') == 'OOMKilled': + return {'outcome': 'oom', 'exitCode': t.get('exitCode')} + if t.get('exitCode') not in (0, None): + return {'outcome': 'failed', 'exitCode': t.get('exitCode')} + # Terminated, but not one container said with what. `failed` here is a lie + # that costs the whole run -- it reads as a genuine catchup failure, the one + # outcome that gets no retry: range 59018943 was condemned on attempt 1 with + # exitCode null, failing a mission that was otherwise 554 for 554. + return {'outcome': 'unknown', 'exitCode': None} + + +def record_outcome(pod, end, attempt): + """Write the verdict next to the log, for job_monitor's reconcile to read.""" + path = records.outcome_path(end, attempt) + if os.path.exists(path): + return + data = classify(pod) + data['pod'] = pod['metadata']['name'] + try: + records.write_atomic(path, json.dumps(data)) + logger.info("range %s attempt %s classified: %s", end, attempt, data['outcome']) + except OSError as e: + logger.warning("could not persist outcome for range %s: %s", end, e) diff --git a/src/MissionParallelCatchup/lib/config.py b/src/MissionParallelCatchup/lib/config.py new file mode 100644 index 00000000..0a5bc0c3 --- /dev/null +++ b/src/MissionParallelCatchup/lib/config.py @@ -0,0 +1,26 @@ +"""What both processes must agree on: the run's identity, the shared volume, +and the verdict vocabulary. + +Nothing tunable lives here. The monitor's knobs are in monitor_config and the +collector's in collector_config -- a setting only one process reads does not +belong in the module the other one also imports. +""" +import os + +# --- identity --------------------------------------------------------------- +NAMESPACE = os.getenv('NAMESPACE', 'default') +RUN_NAME = os.getenv('RUN_NAME', 'parallel-catchup') +LOG_DIR = os.getenv('LOG_DIR', '/logs') + +LABEL_RUN = 'catchup.stellar.org/run' +LABEL_RANGE = 'catchup.stellar.org/range-end' +LABEL_ATTEMPT = 'catchup.stellar.org/attempt' + +# Shared because it decides what the collector may read: in pvc mode /data +# outlives the pod, in ephemeral mode it does not. +STORAGE_MODE = os.getenv('STORAGE_MODE', 'pvc') # pvc | ephemeral + +# The collector writes one of these into .outcome; a name on one side and not +# the other is an attempt nobody can classify. +ATTEMPT_OUTCOMES = ('disrupted', 'oom', 'ephemeral', 'timeout', + 'rejected', 'unknown', 'failed', 'fetch-fault') diff --git a/src/MissionParallelCatchup/lib/logger.py b/src/MissionParallelCatchup/lib/logger.py new file mode 100644 index 00000000..7bd47fd9 --- /dev/null +++ b/src/MissionParallelCatchup/lib/logger.py @@ -0,0 +1,60 @@ +"""Logging setup shared by the monitor and the collector sidecar. + +Both processes write to the same logs volume and both had their own copy of this +bootstrap, which is how they came to disagree about the directory. +""" +import logging +import os +import sys +import tempfile +from datetime import datetime, timezone + + +def get_logging_level(): + name_to_level = { + 'CRITICAL': logging.CRITICAL, + 'ERROR': logging.ERROR, + 'WARNING': logging.WARNING, + 'INFO': logging.INFO, + 'DEBUG': logging.DEBUG, + } + result = name_to_level.get(os.getenv('LOGGING_LEVEL', 'INFO')) + return result if result is not None else logging.INFO + + +def log_dir(): + """The directory the log file goes in. + + On the logs PVC, not the monitor's emptyDir: /data dies with the pod, and the + mission tars /logs -- so an OOM-retry storm, the loudest signal this thing + produces, was visible only in `kubectl logs` and never reached the run's + destination directory. Falls back to /data if LOG_DIR is not mounted. + """ + chosen = os.getenv('LOG_DIR', '/logs') + if not os.path.isdir(chosen): + chosen = '/data' + # Last resort, and unreachable in a pod: /data is the monitor's own emptyDir, + # so one of the two above always exists there. Off-cluster neither does, and + # a FileHandler on a missing directory made this module impossible to import + # -- which is why nothing here was ever tested against a real reconcile(). + if not os.path.isdir(chosen): + chosen = tempfile.gettempdir() + return chosen + + +def build_logger(file_prefix, name=None, to_file=True): + """Configure root logging and return the logger to use. + + `file_prefix` names the per-process log file; `name` is the logger name, and + defaults to the root logger. `to_file=False` is stdout only, for a process + that should not add a writer to the shared logs volume. + """ + handlers = [logging.StreamHandler(sys.stdout)] + if to_file: + stamp = datetime.now(timezone.utc).strftime('%Y-%m-%d_%H-%M-%S') + handlers.append(logging.FileHandler( + os.path.join(log_dir(), f"{file_prefix}_{stamp}.log"))) + logging.basicConfig(level=get_logging_level(), + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=handlers) + return logging.getLogger(name) diff --git a/src/MissionParallelCatchup/lib/monitor/cluster.py b/src/MissionParallelCatchup/lib/monitor/cluster.py new file mode 100644 index 00000000..b310e68f --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/cluster.py @@ -0,0 +1,179 @@ +"""Everything that talks to the apiserver, on the aiohttp-backed aio client. + +kubernetes>=36 ships kubernetes.aio: the REST layer is genuinely async and the +generated methods return the coroutine, so `await core_v1.list_namespaced_pod()` +is real non-blocking I/O and nothing here needs a thread. + +Two traps in that client: + + * the generated docstrings are stale boilerplate from the sync generator -- + they claim "makes a synchronous HTTP request" and advertise async_req=True. + async_req routes through a ThreadPool and returns an AsyncResult, not an + awaitable, so it must never be used here. + * load_incluster_config is sync while load_kube_config is a coroutine. + +The rest of the monitor takes Jobs and pods as plain data, so a fake cluster in +a test replaces this module and no decision function needs a client at all. +""" +import asyncio +import contextlib +import logging + +from kubernetes.aio import client, config as kube_config +from kubernetes.aio.client import ApiException + +import config +import monitor_config as mc + +logger = logging.getLogger('job_monitor') + +batch_v1 = None +core_v1 = None +_api = None +_slots = None +_owner = None + + +@contextlib.asynccontextmanager +async def session(): + """Own the ApiClient for the life of the process. + + One client, one connection pool. Creating them per call would open a new + aiohttp session per request and leak connectors on every pass. + """ + global batch_v1, core_v1, _api, _slots + try: + kube_config.load_incluster_config() + except Exception: + await kube_config.load_kube_config() + _slots = asyncio.Semaphore(mc.APISERVER_CONCURRENCY) + async with client.ApiClient() as api: + _api, batch_v1, core_v1 = api, client.BatchV1Api(api), client.CoreV1Api(api) + try: + yield + finally: + batch_v1 = core_v1 = _api = None + + +def _selector(): + return f"{config.LABEL_RUN}={config.RUN_NAME}" + + +async def snapshot(): + """This run's Jobs by range, and its pods by owning Job. + + Both lists in flight together, and one of each per pass: a range's state + must not be assembled from two different moments. + """ + jobs_raw, pods_raw = await asyncio.gather( + batch_v1.list_namespaced_job(config.NAMESPACE, label_selector=_selector()), + core_v1.list_namespaced_pod(config.NAMESPACE, label_selector=_selector())) + + jobs = {} + for job in jobs_raw.items: + end = (job.metadata.labels or {}).get(config.LABEL_RANGE) + if end is not None: + jobs.setdefault(str(end), []).append(job) + + pods = {} + for pod in pods_raw.items: + owner = next((o.name for o in (pod.metadata.owner_references or []) + if o.kind == 'Job'), None) + if owner is not None: + pods[owner] = pod + return jobs, pods + + +async def owner_ref(): + """The run's ConfigMap, so deleting the release collects everything. + + Read once and cached: it is the same object for the life of the run. + """ + global _owner + if _owner is None: + cm = await core_v1.read_namespaced_config_map( + f"{config.RUN_NAME}-stellar-core-config", config.NAMESPACE) + _owner = [client.V1OwnerReference( + api_version='v1', kind='ConfigMap', name=cm.metadata.name, + uid=cm.metadata.uid, block_owner_deletion=True)] + return _owner + + +async def create_job(body): + """Create, treating AlreadyExists as success. + + The Job name carries range and attempt, so name uniqueness IS the mutex: a + 409 proves another pass already dispatched this attempt, which is the + outcome that was wanted. + """ + async with _slots: + try: + return await batch_v1.create_namespaced_job(config.NAMESPACE, body) + except ApiException as e: + if e.status != 409: + raise + return None + + +async def ensure_pvc(end, owner): + name = f"{config.RUN_NAME}-data-r{end}" + async with _slots: + try: + await core_v1.read_namespaced_persistent_volume_claim(name, config.NAMESPACE) + return name + except ApiException as e: + if e.status != 404: + raise + spec = client.V1PersistentVolumeClaimSpec( + access_modes=['ReadWriteOnce'], + resources=client.V1VolumeResourceRequirements( + requests={'storage': mc.STORAGE_SIZE})) + if mc.STORAGE_CLASS: + spec.storage_class_name = mc.STORAGE_CLASS + try: + await core_v1.create_namespaced_persistent_volume_claim( + config.NAMESPACE, client.V1PersistentVolumeClaim( + metadata=client.V1ObjectMeta( + name=name, owner_references=owner, + labels={config.LABEL_RUN: config.RUN_NAME, + config.LABEL_RANGE: str(end)}), + spec=spec)) + except ApiException as e: + if e.status != 409: + raise + return name + + +async def reap(end, job_names): + """Delete every Job this range has, then release its volume. + + Every Job, not just the winner: an earlier failed attempt left its own, and + leaving it behind holds a PVC and shows up in the next pass's list. + """ + await asyncio.gather(*(delete_job(name) for name in job_names)) + if config.STORAGE_MODE == 'pvc': + await _release_pvc(end) + + +async def delete_job(name): + async with _slots: + try: + await batch_v1.delete_namespaced_job(name, config.NAMESPACE, + propagation_policy='Background') + return True + except ApiException as e: + if e.status != 404: + logger.warning("could not delete job %s: %s", name, e) + return False + + +async def _release_pvc(end): + async with _slots: + try: + await core_v1.delete_namespaced_persistent_volume_claim( + f"{config.RUN_NAME}-data-r{end}", config.NAMESPACE) + return True + except ApiException as e: + if e.status != 404: + logger.warning("could not release pvc for range %s: %s", end, e) + return False diff --git a/src/MissionParallelCatchup/lib/monitor/dispatch.py b/src/MissionParallelCatchup/lib/monitor/dispatch.py new file mode 100644 index 00000000..56843b61 --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/dispatch.py @@ -0,0 +1,241 @@ +"""The range list, and the Job that runs one attempt of it.""" +import logging + +from kubernetes.aio import client + +import cluster +import config +import monitor_config as mc +import sizing + +logger = logging.getLogger('job_monitor') + +# Resume replays from LCL+1 on a PVC. LCL comes from core's own accessor: v27 +# dropped ledgerheaders, and the log fallback goes blind above INFO. +RESUME_SCRIPT = r'''set -e +KEY="%(key)s" +TARGET=%(target)d +COUNT=%(count)d +MARK=/data/.job-key +RESUME=false +LCL="" +if [ -f "$MARK" ] && [ "$(cat "$MARK" 2>/dev/null)" = "$KEY" ]; then + LCL=$(/usr/bin/stellar-core --conf /config/stellar-core.cfg offline-info --console 2>/dev/null \ + | sed -n 's/.*"num"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/p' | head -1 || true) + if [ -z "$LCL" ]; then + PREV_LOG=$(ls -t /data/stellar-core*.log 2>/dev/null | head -n 1 || true) + if [ -n "$PREV_LOG" ]; then + LCL=$(grep -oE "Ledger close complete: [0-9]+" "$PREV_LOG" 2>/dev/null | tail -1 | grep -oE "[0-9]+$" || true) + fi + fi + echo "RESUME PROBE: lcl '${LCL:-none}'" + # Re-running catchup on a finished range applies nothing and exits 2 + # identically every time, so it would burn the whole budget on completed work. + if [ -n "$LCL" ] && [ "$LCL" -ge "$TARGET" ] 2>/dev/null; then + echo "ALREADY COMPLETE: $KEY reached $LCL >= target $TARGET" + exit 0 + fi + if [ -n "$LCL" ] && [ "$LCL" -ge $((TARGET - COUNT)) ] && [ "$LCL" -lt "$TARGET" ] 2>/dev/null; then + RESUME=true; echo "RESUME: $KEY reached $LCL; skipping new-db" + else + echo "RESUME DECLINED: $KEY last close '${LCL:-none}'; bucket phase incomplete" + fi +fi +printf '%%s' "$KEY" > "$MARK" +if [ "$RESUME" != "true" ]; then + /usr/bin/stellar-core --conf /config/stellar-core.cfg new-db --console +fi +exec /usr/bin/stellar-core --conf /config/stellar-core.cfg catchup "$KEY" \ + --metric 'ledger.transaction.apply' --console +''' + + +def range_list(): + """(end, count) for every range this run owes, in dispatch order. + + A pure function of the /start spec. A restart must reproduce it exactly: a + different list means work silently duplicated or skipped, and nothing else + would notice. + """ + start, latest = mc.STARTING_LEDGER, mc.LATEST_LEDGER_NUM + per_job, overlap = mc.LEDGERS_PER_JOB, mc.OVERLAP_LEDGERS + + # Strictly greater, and overlap added on top of the clamped stride: `>=` + # emits a range ending AT the start ledger, which is below genesis and + # exits 2, and clamping the total would drop the overlap at that end. + ranges, end = [], latest + while end > start: + stride = min(end - start, per_job) + ranges.append((end, stride + overlap)) + end -= stride + + if mc.RANGE_ORDER == 'oldest-first': + # The cheap ranges finish first, which is what a profiling run wants: it + # measures the inexpensive end before anything can interrupt it. + return list(reversed(ranges)) + if mc.RANGE_ORDER == 'longest-first': + # Validated at /start, so a profile exists here. + return sorted(ranges, key=_measured_seconds, reverse=True) + # tip-first: the bucket set only grows with ledger position, so the tip + # ranges are the slowest and the most worth starting early. + return ranges + + +def _measured_seconds(item): + """Sort key for longest-first. An unmeasured range sorts FIRST. + + profile_for returns the nearest measured end ABOVE, so a range with no + seconds is newer than anything ever measured -- and cost rises with ledger + position. Unknown means assume worst, not assume average. It also runs + early under the most generous sizing, which is what makes the next profile + cover it instead of it being the range a run dies before reaching. + """ + seconds = (sizing.profile_for(item[0]) or {}).get('seconds') + return (1, 0) if seconds is None else (0, seconds) + + +def job_name(end, attempt): + return f"{config.RUN_NAME}-r{end}-a{attempt}" + + +def job_key(end, count): + return f"{end}/{count}" + + +async def create(end, count, attempt, oom_count=0, memory=None, ephemeral=None): + """Create one attempt's Job, idempotent by name.""" + owner = await cluster.owner_ref() + volume = await _data_volume(end, owner) + body = _job(end, count, attempt, owner, volume, oom_count, memory, ephemeral) + return await cluster.create_job(body) + + +async def _data_volume(end, owner): + if config.STORAGE_MODE == 'pvc': + name = await cluster.ensure_pvc(end, owner) + return client.V1Volume(name='data', persistent_volume_claim=( + client.V1PersistentVolumeClaimVolumeSource(claim_name=name))) + return client.V1Volume(name='data', empty_dir=client.V1EmptyDirVolumeSource()) + + +def _job(end, count, attempt, owner, data_volume, oom_count, memory, ephemeral): + labels = {config.LABEL_RUN: config.RUN_NAME, + config.LABEL_RANGE: str(end), + config.LABEL_ATTEMPT: str(attempt)} + return client.V1Job( + metadata=client.V1ObjectMeta(name=job_name(end, attempt), + owner_references=owner, labels=labels), + spec=client.V1JobSpec( + # Retries are the monitor's, not the controller's: raising a memory + # request needs a new Job, because spec.template is immutable. + backoff_limit=0, + active_deadline_seconds=mc.ATTEMPT_DEADLINE_SECONDS or None, + ttl_seconds_after_finished=mc.JOB_TTL_SECONDS, + pod_failure_policy=client.V1PodFailurePolicy(rules=_failure_rules()), + template=client.V1PodTemplateSpec( + # LABEL_ATTEMPT has to be on the POD too: the collector reads it + # to pick which range--a.* files the attempt owns. + metadata=client.V1ObjectMeta(labels=labels), + spec=_pod(end, count, attempt, data_volume, oom_count, + memory, ephemeral)))) + + +def _failure_rules(): + """Rules in evaluation order, which IS the contract with classify-from-Job. + + All FailJob so the Job fails with reason=PodFailurePolicy and the message + names the rule index; a Count action surfaces as BackoffLimitExceeded and + loses the signal entirely. + """ + return [ + client.V1PodFailurePolicyRule( + action='FailJob', + on_pod_conditions=[client.V1PodFailurePolicyOnPodConditionsPattern( + type='DisruptionTarget', status='True')]), + client.V1PodFailurePolicyRule( + action='FailJob', + on_exit_codes=client.V1PodFailurePolicyOnExitCodesRequirement( + container_name='stellar-core', operator='In', values=[137])), + client.V1PodFailurePolicyRule( + action='FailJob', + on_exit_codes=client.V1PodFailurePolicyOnExitCodesRequirement( + container_name='stellar-core', operator='NotIn', values=[0])), + ] + + +def _pod(end, count, attempt, data_volume, oom_count, memory, ephemeral): + requests, limits = sizing.requests_for(end, oom_count, memory, ephemeral) + script = RESUME_SCRIPT % {'key': job_key(end, count), 'target': end, 'count': count} + container = client.V1Container( + name='stellar-core', image=mc.CORE_IMAGE, + command=['/bin/sh', '-c', script], + env=([client.V1EnvVar(name='ASAN_OPTIONS', value=mc.ASAN_OPTIONS)] + if mc.ASAN_OPTIONS else []), + resources=client.V1ResourceRequirements(requests=requests, limits=limits), + ports=[client.V1ContainerPort(container_port=11626, name='http')], + lifecycle=_prestop(), + volume_mounts=[client.V1VolumeMount(name='data', mount_path='/data'), + client.V1VolumeMount(name='config', mount_path='/config')]) + return client.V1PodSpec( + # IRSA for the S3 history mirror; without it workers fall back to the + # public archive, which throttles at 1024. + service_account_name=mc.WORKER_SERVICE_ACCOUNT or None, + # Never restarted in place: the pod stays terminal and inspectable. + restart_policy='Never', + termination_grace_period_seconds=mc.WORKER_GRACE_SECONDS, + affinity=_affinity(end, oom_count), + tolerations=([client.V1Toleration(key=mc.TOLERATE_TAINT, effect='NoSchedule')] + if mc.TOLERATE_TAINT else None), + containers=[container], + volumes=[data_volume, client.V1Volume( + name='config', config_map=client.V1ConfigMapVolumeSource( + name=f"{config.RUN_NAME}-stellar-core-config"))]) + + +def _affinity(end, oom_count): + """Require and avoid in ONE matchExpressions list. + + Expressions within a term are ANDed and separate terms are ORed, so an + avoid-only pod in its own term would match every node. + """ + match = [] + if mc.NODE_LABEL_KEY: + match.append(client.V1NodeSelectorRequirement( + key=mc.NODE_LABEL_KEY, operator='In', + values=[sizing.node_label_value(end, oom_count)])) + for key, value in mc.label_pairs(mc.REQUIRE_NODE_LABELS): + # Literal, unlike the pool-routed pair above: properties of the pool + # rather than of the range, so they do not vary per attempt. + match.append(client.V1NodeSelectorRequirement( + key=key, operator='In', values=[value])) + if mc.AVOID_NODE_LABEL_KEY: + # No value means "avoid the label however it is set", which is + # DoesNotExist; NotIn [""] would only exclude the empty value. + match.append(client.V1NodeSelectorRequirement( + key=mc.AVOID_NODE_LABEL_KEY, + operator='NotIn' if mc.AVOID_NODE_LABEL_VALUE else 'DoesNotExist', + values=[mc.AVOID_NODE_LABEL_VALUE] if mc.AVOID_NODE_LABEL_VALUE else None)) + if not match: + return None + return client.V1Affinity(node_affinity=client.V1NodeAffinity( + required_during_scheduling_ignored_during_execution=client.V1NodeSelector( + node_selector_terms=[client.V1NodeSelectorTerm(match_expressions=match)]))) + + +def _prestop(): + """A preStop that stalls the kubelet, or None. + + Refuses to install one that cannot finish inside the grace period: the + kubelet kills it mid-sleep, reports FailedPreStopHook, and signals the + container anyway -- so the delay is not bought and an error is logged for + every evicted pod. + """ + sleep = mc.WORKER_PRESTOP_SLEEP_SECONDS + if sleep <= 0: + return None + if sleep >= mc.WORKER_GRACE_SECONDS: + logger.warning("preStop %ss does not fit in grace %ss; not installing it", + sleep, mc.WORKER_GRACE_SECONDS) + return None + return client.V1Lifecycle(pre_stop=client.V1LifecycleHandler( + _exec=client.V1ExecAction(command=['/bin/sleep', str(sleep)]))) diff --git a/src/MissionParallelCatchup/lib/monitor/liveness.py b/src/MissionParallelCatchup/lib/monitor/liveness.py new file mode 100644 index 00000000..eb402a27 --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/liveness.py @@ -0,0 +1,89 @@ +"""Whether the workers' stellar-core is answering, one sweep per pass. + +The reconcile loop already holds the authoritative pod list, so this probes that +rather than listing anything itself. No state is carried between sweeps -- no +hysteresis, no smoothing, no scheduler. These numbers feed a dashboard and +nothing else reads them, so a stale-free snapshot beats a smoothed one. +""" +import asyncio +import logging + +import aiohttp + +import monitor_config as mc + +logger = logging.getLogger('job_monitor') + +_ADMIN_PORT = 11626 + +EMPTY = {'up': 0, 'down': 0, 'unknown': 0} + + +def targets(pods): + """Running-with-an-IP pods, keyed by pod UID. + + A UID change is a replacement even when the Job name or the IP is reused. + """ + out = {} + for pod in pods: + status, meta = getattr(pod, 'status', None), getattr(pod, 'metadata', None) + ip = getattr(status, 'pod_ip', None) + if getattr(status, 'phase', None) != 'Running' or not ip or meta is None: + continue + name = getattr(meta, 'name', None) + identity = getattr(meta, 'uid', None) or name + if identity and name: + out[str(identity)] = (str(name), str(ip)) + return out + + +async def sweep(targets): + """{'up','down','unknown'} for the whole fleet, bounded by one deadline. + + Deliberately not a TaskGroup: a TaskGroup cancels its siblings when one task + raises, which is the opposite of what a fleet sweep wants -- one unreachable + pod must not discard the other 1023 answers. asyncio.wait keeps whatever + finished and cancels only the stragglers, so an unreachable fleet costs one + deadline rather than the sum of its timeouts. + """ + if not targets: + return dict(EMPTY) + counts = {'up': 0, 'down': 0, 'unknown': len(targets)} + # force_close: a pooled socket to a vanished pod gets handed back out. + # `limit` is the concurrency bound; a semaphore would double-enforce it. + connector = aiohttp.TCPConnector(limit=mc.LIVENESS_MAX_CONCURRENCY, + force_close=True) + timeout = aiohttp.ClientTimeout(total=mc.LIVENESS_PROBE_TIMEOUT_SECONDS) + async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: + tasks = [asyncio.create_task(_probe(session, ip)) for _, ip in targets.values()] + done, pending = await asyncio.wait(tasks, timeout=mc.LIVENESS_SWEEP_SECONDS) + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + for task in done: + try: + up = task.result() + except Exception: + up = False # timeout, refused, DNS, malformed response + counts['up' if up else 'down'] += 1 + counts['unknown'] -= 1 + return counts + + +async def _probe(session, ip): + host = f"[{ip}]" if ':' in ip else ip + async with session.get(f"http://{host}:{_ADMIN_PORT}/info") as resp: + return resp.status == 200 + + +async def publish(pods): + """One sweep, never fatal. Liveness is observability and may not stop a run.""" + found = targets(pods) + if not found: + return dict(EMPTY) + try: + return await sweep(found) + except Exception as e: + logger.warning("liveness sweep failed (%s); reporting all workers unknown", e) + return {'up': 0, 'down': 0, 'unknown': len(found)} diff --git a/src/MissionParallelCatchup/lib/monitor/metrics.py b/src/MissionParallelCatchup/lib/monitor/metrics.py new file mode 100644 index 00000000..c0d1af76 --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/metrics.py @@ -0,0 +1,125 @@ +"""The run's Prometheus metrics, and the only correct way to move them. + +Counters and histograms are monotonic and reset to zero when the process +restarts, while the record on the volume survives. So the totals live in the +record, updated when a fact becomes final, and each pass pushes the delta -- +idempotent, and self-healing after a restart without rescanning anything. +""" +from prometheus_client import Counter, Gauge, Histogram + +import config + +# 5m 15m 30m 1h 1.5h 2h +BUCKETS = (300, 900, 1800, 3600, 5400, 7200, float('inf')) + +queues = Gauge('ssc_parallel_catchup_queues', 'Size of each job queue', ['queue']) +workers = Gauge('ssc_parallel_catchup_workers', 'Worker liveness', ['status']) +sweep_duration = Gauge('ssc_parallel_catchup_workers_refresh_duration_seconds', + 'Seconds the last liveness sweep took') +mission_duration = Gauge('ssc_parallel_catchup_mission_duration_seconds', + 'Seconds since the mission started') + +full_duration = Histogram('ssc_parallel_catchup_job_full_duration_seconds', + 'Compute seconds across the resumed attempt chain', + buckets=BUCKETS) +wall_duration = Histogram('ssc_parallel_catchup_job_wall_duration_seconds', + "Attempt 1's dispatch to the winning attempt's completion", + buckets=BUCKETS) +tx_apply_duration = Histogram('ssc_parallel_catchup_job_tx_apply_duration_seconds', + 'Transaction-apply seconds per range', buckets=BUCKETS) + +retries = Counter('ssc_parallel_catchup_job_retried_count', + 'Retry attempts dispatched after a predecessor failed') +oom_retries = Counter('ssc_parallel_catchup_job_oom_retried_count', + 'Retries dispatched after an OOM, with an escalated request') +eph_retries = Counter('ssc_parallel_catchup_job_ephemeral_retried_count', + 'Retries dispatched after an ephemeral eviction') +# Separates infrastructure churn from application failure: many evictions with +# zero app failures is spot behaving as intended. +evictions = Counter('ssc_parallel_catchup_job_spot_eviction_count', + 'Attempts classified as lost to node disruption') +disruption_retried = Counter('ssc_parallel_catchup_job_spot_disruption_retried_count', + 'Ranges that dispatched a successor after a disruption') +retry_reasons = Counter('ssc_parallel_catchup_job_retried_reason_count', + 'Attempts by the verdict that ended them', ['reason']) + +_COUNTERS = (('retries', retries), ('oom', oom_retries), ('ephemeral', eph_retries), + ('evicted', evictions), ('disruption_retried', disruption_retried)) + + +def settled(progress, cause, end=None): + """One attempt's verdict became final: a retry followed it, or it condemned + the range. `end` is passed only for the retry, which is what makes the + disruption count per range rather than per eviction.""" + counters = progress['counters'] + counters[f'reason:{cause}'] += 1 + if cause == 'disrupted': + counters['evicted'] += 1 + if end is None: + return + counters['retries'] += 1 + if cause in ('oom', 'ephemeral'): + counters[cause] += 1 + if cause == 'disrupted': + progress['disruptedRanges'].add(end) + counters['disruption_retried'] = len(progress['disruptedRanges']) + + +def sync_counters(progress, applied): + """Walk each counter up to the record's total. + + Delta from a total rather than .inc() at the event: the counters reset with + the process and the record does not, so a restart walks them back up. + """ + counters = progress['counters'] + keys = [k for k, _ in _COUNTERS] + [f'reason:{r}' for r in config.ATTEMPT_OUTCOMES] + for key in keys: + delta = counters[key] - applied.get(key, 0) + if delta <= 0: + continue + if key.startswith('reason:'): + retry_reasons.labels(reason=key[7:]).inc(delta) + else: + dict(_COUNTERS)[key].inc(delta) + applied[key] = counters[key] + + +def observe_completed(progress, replayed): + """Feed recorded completions into the histograms, once each per process. + + Keyed on (range, FIELD), never the range alone: a range is usually recorded + before the collector has flushed its .metrics, so txApply is absent at first + sight and backfilled a pass or two later. Guarding per range means that + backfill can never be observed and the histogram permanently disagrees with + the record. + """ + for end, rec in progress.get('completed', {}).items(): + for field, metric in (('seconds', full_duration), + ('wallSeconds', wall_duration), + ('txApply', tx_apply_duration)): + key = (end, field) + if key in replayed: + continue + value = rec.get(field) + # Presence, not truth: a txApply sum of 0 is a real observation, and + # so is a sub-second duration. + if value is None: + continue + replayed.add(key) + metric.observe(value) + + +# The dashboard's panels query this name and these label values. The internal +# vocabulary has no queue in it, but renaming a published series only moves the +# break to a consumer that lives in another repo and cannot be tested here. +_QUEUE = {'remaining': 'remain', 'running': 'in_progress', + 'completed': 'succeeded', 'condemned': 'failed'} + + +def publish_gauges(counts, liveness, sweep_seconds, mission_seconds): + for state, value in counts.items(): + queues.labels(queue=_QUEUE[state]).set(value) + for status, value in liveness.items(): + workers.labels(status=status).set(value) + sweep_duration.set(sweep_seconds) + mission_duration.set(mission_seconds) diff --git a/src/MissionParallelCatchup/lib/monitor/monitor_config.py b/src/MissionParallelCatchup/lib/monitor/monitor_config.py new file mode 100644 index 00000000..3b0f15ea --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/monitor_config.py @@ -0,0 +1,218 @@ +"""Every knob the monitor owns, and the validation a run is admitted through. + +What both processes must agree on -- the run's identity, the shared volume, the +verdict vocabulary -- lives in config, which this reads through. + +Read through the module, never copied out of it: + + import monitor_config as mc + ... mc.REQ_CPU ... + +`from monitor_config import REQ_CPU` binds a COPY. /start rebinds several of +these after validating them, and a copy taken at import time never sees the +rebind -- silently, with the run using the default. + +Numbers that arrive from the chart stay strings until validate() coerces them. +Coercing at import made a bad value a boot crash, and a process that cannot +start cannot report why. +""" +import logging +import os + +import config + +logger = logging.getLogger('job_monitor') + + +def _int(name, default): + """A chart-env integer, typed here rather than at /start. + + The loop idles on its interval while waiting for /start, so a string there + crash-loops the pod before it can be told anything. A bad value falls back + loudly rather than killing the boot; /start-delivered values get a 400. + """ + raw = os.getenv(name) + if raw is None: + return default + try: + return int(raw) + except (TypeError, ValueError): + logger.error("%s=%r is not an integer; using %d", name, raw, default) + return default + + +# --- the run ---------------------------------------------------------------- +PARALLELISM = _int('PARALLELISM', 3) +RECONCILE_INTERVAL_SECONDS = _int('LOGGING_INTERVAL_SECONDS', 10) +HTTP_PORT = _int('HTTP_PORT', 8080) + +# Delivered by POST /start, replayed from run.json on restart. +STARTING_LEDGER = '0' +LATEST_LEDGER_NUM = '0' +LEDGERS_PER_JOB = '16000' +OVERLAP_LEDGERS = '320' +RANGE_ORDER = 'tip-first' +PROFILE = [] # sorted [(end, record)] +_SORTED_SECONDS = None # cache, invalidated whenever PROFILE is set + +RANGE_ORDERS = ('tip-first', 'oldest-first', 'longest-first') + +# --- the worker pod --------------------------------------------------------- +CORE_IMAGE = os.getenv('CORE_IMAGE', 'stellar/stellar-core:latest') +ASAN_OPTIONS = os.getenv('ASAN_OPTIONS', '') +WORKER_SERVICE_ACCOUNT = os.getenv('WORKER_SERVICE_ACCOUNT', '') +WORKER_GRACE_SECONDS = int(os.getenv('WORKER_GRACE_SECONDS', '150')) +# Holds the pod open after SIGTERM so the collector can drain the last of its +# log. Must fit inside the grace period; a preStop the kubelet kills mid-sleep +# buys nothing and logs FailedPreStopHook on every evicted pod. +WORKER_PRESTOP_SLEEP_SECONDS = int(os.getenv('WORKER_PRESTOP_SLEEP_SECONDS', '5')) + +STORAGE_CLASS = os.getenv('STORAGE_CLASS', '') +STORAGE_SIZE = os.getenv('STORAGE_SIZE', '60Gi') + +JOB_TTL_SECONDS = int(os.getenv('JOB_TTL_SECONDS', '3600')) +# 0 disables. A timeout is terminal, so a tight bound trades a certain +# catastrophe against a rounding error: one wedged range holds one slot. +ATTEMPT_DEADLINE_SECONDS = int(os.getenv('ATTEMPT_DEADLINE_SECONDS', '0')) + +# --- unpooled / unprofiled sizing ------------------------------------------- +# The packing unit for a run with no profile: 4 workers per r8*.2xlarge +# (cpu-bound) or 3 per m8*.2xlarge (memory-bound). The price of not measuring. +REQ_CPU = os.getenv('REQ_CPU', '1800m') +REQ_MEM = os.getenv('REQ_MEM', '9Gi') +REQ_EPHEMERAL = os.getenv('REQ_EPHEMERAL', '') +LIM_EPHEMERAL = os.getenv('LIM_EPHEMERAL', '') + +# --- profile-derived sizing (unpooled) -------------------------------------- +PROFILE_MARGIN = float(os.getenv('PROFILE_MARGIN', '1.15')) +# Flat, because a multiplicative margin is nothing at small rss: 1.15x of +# 190Mi is 19Mi of slack, and 90 ranges OOMKilled inside 90s on it. +PROFILE_CACHE_HEADROOM = os.getenv('PROFILE_CACHE_HEADROOM', '512Mi') +PROFILE_RUNTIME_MEMORY_INSURANCE = os.getenv('PROFILE_RUNTIME_MEMORY_INSURANCE', '3Gi') +PROFILE_MAX_MEM = os.getenv('PROFILE_MAX_MEM', '32Gi') +# Image, logs, sqlite WAL: none of it scales with the range. +PROFILE_EPHEMERAL_HEADROOM = os.getenv('PROFILE_EPHEMERAL_HEADROOM', '2Gi') +# Disk tracks runtime closely (pearson 0.920 over 3985 ranges). +PROFILE_RUNTIME_EPHEMERAL_INSURANCE = os.getenv('PROFILE_RUNTIME_EPHEMERAL_INSURANCE', '8Gi') +# Above the flat limit on purpose: that limit is what an UNMEASURED range gets. +PROFILE_MAX_EPHEMERAL = os.getenv('PROFILE_MAX_EPHEMERAL', '64Gi') + +# --- escalation ------------------------------------------------------------- +MEM_BUMP_FACTOR = float(os.getenv('MEM_BUMP_FACTOR', '1.5')) +MEM_ESCALATION_CAP = os.getenv('MAX_MEM', '48Gi') +EPH_BUMP_FACTOR = float(os.getenv('EPH_BUMP_FACTOR', '1.5')) +EPH_ESCALATION_CAP = os.getenv('MAX_EPHEMERAL', '200Gi') + +# --- pools ------------------------------------------------------------------ +# Empty disables routing entirely; every worker keeps NODE_LABEL_VALUE. +POOL_PREFIX = os.getenv('POOL_PREFIX', '') +POOL_TIERS = os.getenv( + 'POOL_TIERS', + '0:subdwarf,0.79:dwarf,1.61:subgiant,3.87:giant,8.85:supergiant,' + '18.38:hypergiant,:supernova') +# Fits once in the tier's on-demand shape, twice in its (one size larger) spot +# shape. Not 50% of either -- see the requirements. +POOL_MEM = os.getenv( + 'POOL_MEM', + 'subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,' + 'supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,' + 'protostar:29696Mi,nebula:9216Mi') +POOL_CPU = os.getenv( + 'POOL_CPU', + 'subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,' + 'hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80') +POOL_BLOCK_RUNGS = os.getenv('POOL_BLOCK_RUNGS', 'dwarf->subgiant,hypergiant->supernova') +POOL_UNPROFILED = os.getenv('POOL_UNPROFILED', 'protostar') +POOL_NO_PROFILE = os.getenv('POOL_NO_PROFILE', 'nebula') + +# --- placement -------------------------------------------------------------- +NODE_LABEL_KEY = os.getenv('NODE_LABEL_KEY', '') +NODE_LABEL_VALUE = os.getenv('NODE_LABEL_VALUE', '') +REQUIRE_NODE_LABELS = os.getenv('REQUIRE_NODE_LABELS', '') +AVOID_NODE_LABEL_KEY = os.getenv('AVOID_NODE_LABEL_KEY', '') +AVOID_NODE_LABEL_VALUE = os.getenv('AVOID_NODE_LABEL_VALUE', '') +TOLERATE_TAINT = os.getenv('TOLERATE_TAINT', '') + +# --- retry budgets ---------------------------------------------------------- +# The whole retry policy. A cause that is NOT here is condemned the first time +# it happens. Every budget is spent by its own cause: one shared attempt index +# let evictions drain the OOM budget to zero. +ATTEMPT_BUDGETS = { + 'disrupted': int(os.getenv('MAX_DISRUPTION_ATTEMPTS', '100')), + 'rejected': int(os.getenv('MAX_REJECTED_ATTEMPTS', '100')), + 'fetch-fault': int(os.getenv('MAX_FETCH_FAULT_ATTEMPTS', '20')), + 'oom': int(os.getenv('MAX_OOM_ATTEMPTS', '5')), + 'ephemeral': int(os.getenv('MAX_EPHEMERAL_ATTEMPTS', '4')), + 'unknown': int(os.getenv('MAX_UNKNOWN_ATTEMPTS', '2')), +} + +# --- liveness --------------------------------------------------------------- +LIVENESS_MAX_CONCURRENCY = int(os.getenv('LIVENESS_MAX_CONCURRENCY', '64')) +LIVENESS_PROBE_TIMEOUT_SECONDS = float(os.getenv('LIVENESS_PROBE_TIMEOUT_SECONDS', '2')) +# Under the reconcile interval on purpose: the pass awaits this sweep, so a +# deadline above the interval lets an unreachable fleet stretch dispatch and +# reaping behind a probe that only feeds a dashboard. +LIVENESS_SWEEP_SECONDS = float(os.getenv('LIVENESS_SWEEP_SECONDS', '5')) + +# Sized for the dispatch burst rather than a steady rate: a wave head creates +# ~1024 Jobs and PVCs at once. +APISERVER_CONCURRENCY = int(os.getenv('APISERVER_CONCURRENCY', '64')) + + +def label_pairs(raw): + """[(key, value)] from "k:v,k:v". A key with no value is dropped -- it would + require the label be exactly "", which no node carries, and the pod sits + Pending in a way that reads as slow provisioning.""" + out = [] + for item in (raw or '').split(','): + key, _, value = item.strip().partition(':') + if key and value: + out.append((key, value)) + return out + + +def set_profile(profile): + global PROFILE, _SORTED_SECONDS + PROFILE = profile + _SORTED_SECONDS = None + + +def validate(): + """Coerce and re-bind everything a run depends on. Raises ValueError. + + Called from /start, which is the first moment the configuration is + complete, so a misconfigured run is answered with a 400 rather than + crash-looping a pod the driver can only time out on. + """ + global STARTING_LEDGER, LATEST_LEDGER_NUM, LEDGERS_PER_JOB, OVERLAP_LEDGERS + + def positive(name, value, floor=1): + try: + n = int(value) + except (TypeError, ValueError): + raise ValueError(f"{name} is not an integer: {value!r}") + if n < floor: + raise ValueError(f"{name} must be >= {floor}, got {n}") + return n + + STARTING_LEDGER = positive('STARTING_LEDGER', STARTING_LEDGER, floor=0) + LATEST_LEDGER_NUM = positive('LATEST_LEDGER_NUM', LATEST_LEDGER_NUM) + LEDGERS_PER_JOB = positive('LEDGERS_PER_JOB', LEDGERS_PER_JOB) + OVERLAP_LEDGERS = positive('OVERLAP_LEDGERS', OVERLAP_LEDGERS, floor=0) + + if LATEST_LEDGER_NUM < STARTING_LEDGER: + raise ValueError(f"LATEST_LEDGER_NUM {LATEST_LEDGER_NUM} is below " + f"STARTING_LEDGER {STARTING_LEDGER}") + if RANGE_ORDER not in RANGE_ORDERS: + raise ValueError(f"RANGE_ORDER must be one of {RANGE_ORDERS}, " + f"got {RANGE_ORDER!r}") + if config.STORAGE_MODE not in ('pvc', 'ephemeral'): + raise ValueError(f"STORAGE_MODE must be pvc or ephemeral, " + f"got {config.STORAGE_MODE!r}") + if RANGE_ORDER == 'longest-first' and not PROFILE: + # With no profile every range ties, the sort is a no-op, and dispatch + # silently falls back to tip-first while the operator believes + # otherwise. + raise ValueError( + "RANGE_ORDER=longest-first requires a profile: it orders ranges by " + "measured seconds. POST a profile, or set another order.") diff --git a/src/MissionParallelCatchup/lib/monitor/policy.py b/src/MissionParallelCatchup/lib/monitor/policy.py new file mode 100644 index 00000000..cc5a8930 --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/policy.py @@ -0,0 +1,45 @@ +"""Whether a failed attempt runs again, and with what. + +Two questions kept apart: + + may it retry? a per-CAUSE budget, spent against what actually killed it + with what? escalate the axis that ran out, once per cause + +A pure function of (verdict, this range's past verdicts), which is what makes +the retry ladder testable without a cluster. +""" +import collections + +import monitor_config as mc +import sizing + +Decision = collections.namedtuple('Decision', 'action reason memory ephemeral') + +RETRY, CONDEMN, DEFER = 'retry', 'condemn', 'defer' +WAIT = Decision(DEFER, None, None, None) + + +def decide(end, verdict, spent, base_memory=None, base_ephemeral=None): + """What to do about a failed attempt. `spent` is this range's cause counts. + + A cause with no budget is condemned on sight -- that is how a genuine + catchup failure ends a run instead of burning 20 attempts proving the chain + is broken. + """ + cause = verdict.get('outcome') + cap = mc.ATTEMPT_BUDGETS.get(cause) + if cap is None: + return Decision(CONDEMN, f"{cause} is not retryable", None, None) + + # This verdict is already counted, so the Nth failure of a cause is the one + # that exhausts a budget of N. Per cause: evictions cannot drain the budget + # OOMs are entitled to. + if spent.get(cause, 0) >= cap: + return Decision(CONDEMN, f"{cause} budget of {cap} exhausted", None, None) + + memory = ephemeral = None + if cause == 'oom': + memory = sizing.next_memory(end, base_memory, spent.get('oom', 0)) + elif cause == 'ephemeral': + ephemeral = sizing.next_ephemeral(base_ephemeral, spent.get('ephemeral', 0)) + return Decision(RETRY, cause, memory, ephemeral) diff --git a/src/MissionParallelCatchup/lib/monitor/record.py b/src/MissionParallelCatchup/lib/monitor/record.py new file mode 100644 index 00000000..62580c55 --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/record.py @@ -0,0 +1,188 @@ +"""What the monitor keeps on the volume: the run record, and reads of the +collector's files. + +The filenames themselves are NOT here. They are the cross-process contract and +live in records.py, which both processes import -- a second copy is a second +place for the two sides to disagree about a name, which is a measurement lost +with nothing to report it. +""" +import collections +import json +import os +import time + +import config +from records import (done_path, log_path, metrics_path, outcome_path, # noqa: F401 + state_path, write_atomic) + +# --- the monitor writes these ----------------------------------------------- + + +def started_path(end): + """Per RANGE, not per attempt: wallSeconds spans the range's whole life, so + the only start that matters is the first one.""" + return os.path.join(config.LOG_DIR, f"range-{end}.started") + + +PROGRESS_PATH = os.path.join(config.LOG_DIR, 'progress.json') +RUN_PATH = os.path.join(config.LOG_DIR, 'run.json') +MISSION_START_PATH = os.path.join(config.LOG_DIR, 'mission_started') + + +def _read_json(path): + try: + with open(path) as fh: + return json.load(fh) + except (OSError, ValueError): + return None + + +# --- what an attempt left behind -------------------------------------------- + + +def is_done(end, attempt): + """The collector has finished with this attempt. + + The only licence to decide a failed attempt or to reap. Never inferred from + measurements being present -- an attempt may legitimately have none. + """ + return os.path.exists(done_path(end, attempt)) + + +def ran_nothing(end, attempt, metrics): + """Opened, but it ingested no line and nothing was measured either. + + An empty .state alone is not enough: a follow writes .metrics before the + first read, so a pod condemned seconds in has both. That one ran. + """ + if metrics: + return False + try: + return os.path.getsize(state_path(end, attempt)) == 0 + except OSError: + return False + + +def read_outcome(end, attempt): + return _read_json(outcome_path(end, attempt)) + + +def read_metrics(end, attempt): + return _read_json(metrics_path(end, attempt)) or {} + + +def record_start(end, created): + """Attempt 1's Job creationTimestamp, written once. + + Not status.startTime: the controller sets that asynchronously, so it is + absent from the create response, and the gap between the two is part of what + wallSeconds measures. Written at creation because attempt 1's Job is gone by + the first retry. + """ + path = started_path(end) + if created is None or os.path.exists(path): + return + try: + write_atomic(path, created.isoformat()) + except OSError: + pass + + +def started_at(end): + try: + with open(started_path(end)) as fh: + from datetime import datetime + return datetime.fromisoformat(fh.read().strip()) + except (OSError, ValueError): + return None + + +# --- the run record --------------------------------------------------------- + + +def load_progress(): + """The durable record, read back rather than rebuilt. + + It is what makes a reaped range stay completed instead of reading as + pending and being dispatched a second time -- and it carries the counter + totals, so a restart resumes them in one read instead of rescanning every + attempt on the volume. + """ + doc = _read_json(PROGRESS_PATH) or {} + return {'completed': doc.get('completed') or {}, + 'condemned': doc.get('condemned') or {}, + 'causes': doc.get('causes') or {}, + 'counters': collections.Counter(doc.get('counters') or {}), + 'disruptedRanges': set(doc.get('disruptedRanges') or ())} + + +def note_cause(progress, end, attempt, cause): + """Record why one attempt ended, once. Returns this range's cause counts. + + Keyed on the highest attempt already counted, so re-settling the same + attempt on a later pass cannot double-count it. Budgets then read a number + instead of walking every past attempt off the volume. + """ + rec = progress['causes'].setdefault(str(end), {}) + if rec.get('last', 0) < int(attempt): + rec['last'] = int(attempt) + rec[cause] = rec.get(cause, 0) + 1 + return rec + + +def save_progress(progress): + """One file holding every terminal fact. + + If it is lost or truncated every range reads as pending and the run + redispatches all of them, so nothing may write it by any other means. + """ + doc = dict(progress, + counters=dict(progress['counters']), + disruptedRanges=sorted(progress['disruptedRanges'])) + write_atomic(PROGRESS_PATH, json.dumps(doc, separators=(',', ':'))) + + +def save_run(doc): + write_atomic(RUN_PATH, json.dumps(doc, separators=(',', ':'))) + + +def load_run(): + return _read_json(RUN_PATH) + + +def mission_start(): + """When this run began, surviving a monitor restart.""" + try: + with open(MISSION_START_PATH) as fh: + return float(fh.read().strip()) + except (OSError, ValueError): + now = time.time() + try: + write_atomic(MISSION_START_PATH, repr(now)) + except OSError: + pass + return now + + +def manifest(): + """Every artifact the driver can pull, as [{name, size}]. + + A bare ARRAY of objects, both of which the driver depends on: it parses the + body as an array and reads .name and .size off each entry, using size to + skip what it already has. .tmp files are excluded -- a half-written file has + no size worth comparing. + """ + out = [] + try: + names = sorted(os.listdir(config.LOG_DIR)) + except OSError: + return out + for name in names: + if name.endswith('.tmp'): + continue + try: + out.append({'name': name, + 'size': os.path.getsize(os.path.join(config.LOG_DIR, name))}) + except OSError: + continue # vanished between listing and stat + return out diff --git a/src/MissionParallelCatchup/lib/monitor/server.py b/src/MissionParallelCatchup/lib/monitor/server.py new file mode 100644 index 00000000..09bef29d --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/server.py @@ -0,0 +1,102 @@ +"""The monitor's HTTP surface, on the same event loop as the reconcile pass. + +/start begin a run (the range spec and an optional profile) +/status the counts the driver decides on +/logs the volume, because nothing else can read it +/prometheus, /healthz +""" +import logging +import os + +from aiohttp import web +from prometheus_client import CONTENT_TYPE_LATEST, generate_latest + +import config +import monitor_config as mc +import record + +logger = logging.getLogger('job_monitor') + + +def build(state): + app = web.Application() + app['state'] = state + app.add_routes([ + web.post('/start', _start), + web.get('/status', _status), + web.get('/logs', _logs), + web.get('/logs/{name}', _log_file), + web.get('/prometheus', _prometheus), + web.get('/healthz', _healthz), + ]) + return app + + +async def serve(state, stop): + """Serve until `stop` is set, then drain. + + /status must answer for the whole life of the process, including while a + reconcile pass is in flight -- the driver reads it to decide whether the + mission is still alive. + """ + runner = web.AppRunner(build(state), access_log=None) + await runner.setup() + site = web.TCPSite(runner, '0.0.0.0', mc.HTTP_PORT) + await site.start() + logger.info("listening on :%d", mc.HTTP_PORT) + try: + await stop.wait() + finally: + await runner.cleanup() + + +async def _start(request): + """Apply the /start document. A ValueError is answered 400. + + Validated at the first moment the configuration is complete -- the profile + arrives with this POST -- so a misconfigured run is rejected here rather + than crash-looping a pod the driver can only time out on. + """ + state = request.app['state'] + try: + doc = await request.json() + except Exception: + raise web.HTTPBadRequest(text="body is not JSON") + try: + state.start(doc) + except ValueError as e: + logger.warning("rejected /start: %s", e) + raise web.HTTPBadRequest(text=str(e)) + # Only after it validates, and only here: resume() replays what is already + # on disk. + record.save_run(doc) + return web.json_response({'ranges': len(state.ranges), + 'profile': len(mc.PROFILE)}) + + +async def _status(request): + return web.json_response(request.app['state'].status()) + + +async def _logs(request): + # A bare array, not an object: the driver parses the body as a JArray. + return web.json_response(record.manifest()) + + +async def _log_file(request): + name = request.match_info['name'] + # basename, so a traversal cannot reach off the volume. + path = os.path.join(config.LOG_DIR, os.path.basename(name)) + if not os.path.isfile(path): + raise web.HTTPNotFound(text=f"no such artifact: {name}") + return web.FileResponse(path) + + +async def _prometheus(request): + # Through headers: aiohttp rejects the charset that media type carries. + return web.Response(body=generate_latest(), + headers={'Content-Type': CONTENT_TYPE_LATEST}) + + +async def _healthz(request): + return web.Response(text='ok') diff --git a/src/MissionParallelCatchup/lib/monitor/sizing.py b/src/MissionParallelCatchup/lib/monitor/sizing.py new file mode 100644 index 00000000..c1036626 --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/sizing.py @@ -0,0 +1,320 @@ +"""What a worker pod asks for: pool tier, memory, disk. + +The layer tuned between runs. Reads the profile and past verdicts, touches +nothing, and returns a request rather than applying one. + +Escalation counts CAUSES, not attempts: a spot reclaim, a disruption and a +timeout all produce a retry and none of them says the range needed a bigger +node. +""" +import bisect +import math +import re + +import monitor_config as mc + +_QUANTITY = re.compile(r'^(?P\d+(?:\.\d+)?)(?P[EPTGMK]i?|m)?$') +_FACTOR = {'K': 10 ** 3, 'M': 10 ** 6, 'G': 10 ** 9, 'T': 10 ** 12, + 'P': 10 ** 15, 'E': 10 ** 18, + 'Ki': 2 ** 10, 'Mi': 2 ** 20, 'Gi': 2 ** 30, 'Ti': 2 ** 40, + 'Pi': 2 ** 50, 'Ei': 2 ** 60} + + +def quantity_bytes(value): + m = _QUANTITY.match(str(value or '').strip()) + if not m: + return 0 + n = float(m.group('n')) + unit = m.group('unit') + if unit == 'm': + return int(n / 1000) + return int(n * _FACTOR.get(unit, 1)) + + +def bytes_to_quantity(n): + for unit in ('Gi', 'Mi', 'Ki'): + factor = _FACTOR[unit] + if n >= factor and n % factor == 0: + return f"{n // factor}{unit}" + return str(int(n)) + + +# --- the profile ------------------------------------------------------------ + + +def load_profile(doc): + """Sorted [(end, record)]. An unprofiled run POSTs {} and gets []: a profile + is an optimisation, never a prerequisite.""" + ranges = (doc or {}).get('ranges') or {} + return sorted((int(k), v) for k, v in ranges.items()) + + +def profile_for(end): + """Measurements to size this range from, or None. + + Exact end, else the nearest measured end ABOVE it: cost rises with ledger + position because the bucket set only grows, so a lower neighbour + under-reports. Past the top there is nothing safe to extrapolate from. + """ + if not mc.PROFILE: + return None + idx = bisect.bisect_left(mc.PROFILE, (int(end),)) + if idx < len(mc.PROFILE): + return mc.PROFILE[idx][1] + return None + + +def _positive(value): + try: + n = float(value) + except (TypeError, ValueError): + return None + return n if math.isfinite(n) and n > 0 else None + + +def _longest_seconds(): + if mc._SORTED_SECONDS is None: + values = (_positive(rec.get('seconds')) for _, rec in (mc.PROFILE or [])) + mc._SORTED_SECONDS = sorted(v for v in values if v is not None) + return mc._SORTED_SECONDS[-1] if mc._SORTED_SECONDS else None + + +def _runtime_insurance(seconds, allowance): + """Runtime-weighted share of an allowance: the longest range gets all of it, + one half as long gets half, so it follows time-at-risk.""" + seconds = _positive(seconds) + longest = _longest_seconds() + total = quantity_bytes(allowance) + if seconds is None or longest is None or total <= 0: + return 0 + return int(total * (seconds / longest)) + + +# --- the pool ladder -------------------------------------------------------- + + +def _tiers(): + """[(gib_cut, name)] cheapest first; an empty cut on the last entry means + everything above the previous one.""" + out = [] + for item in mc.POOL_TIERS.split(','): + cut, _, name = item.strip().rpartition(':') + if name: + out.append((float(cut) if cut else float('inf'), name)) + return out + + +def _str_map(raw): + out = {} + for item in raw.split(','): + name, _, value = item.strip().partition(':') + if name and value: + out[name.strip()] = value.strip() + return out + + +def pool_memory(tier): + """The tier's cut: sized to fit once in its on-demand node and twice in its + (one size larger) spot node. Not the range's own measurement -- isolation is + the point, and freeing a pod of its neighbours raised throughput 29-92% + while its cpu draw FELL.""" + return _str_map(mc.POOL_MEM).get(tier) + + +def pool_cpu(tier): + return _str_map(mc.POOL_CPU).get(tier) + + +def _tier_for_bytes(anon): + tiers = _tiers() + if not tiers or not anon: + return None + gib = anon / float(2 ** 30) + for cut, name in tiers: + if gib < cut: + return name + return tiers[-1][1] + + +def _promote(tier, steps): + """Climb `steps` rungs, stopping at the top. Tiers off the ladder go + straight to it: there is nothing above them to walk to.""" + names = [name for _, name in _tiers()] + if not names or tier is None or steps <= 0: + return tier + if tier not in names: + return names[-1] + return names[min(names.index(tier) + steps, len(names) - 1)] + + +def _rung_blocked(tier, nxt): + want = f"{tier}->{nxt}" + return any(item.strip() == want for item in mc.POOL_BLOCK_RUNGS.split(',')) + + +def _cache_bump(tier, anon, working_set): + """One rung up when the working set reaches the next tier and the rung is free. + + peakAnonBytes picks the base tier because anon is what OOM-kills; page cache + evicts rather than dying. It does decide throughput -- replay is + single-threaded, so every bucket lookup that misses cache is a serial EBS + stall. One rung, never two: a range 1.29x under its working set still + reaches full profile rate. + """ + if not (tier and anon and working_set): + return tier + nxt = _promote(tier, 1) + if nxt == tier or _rung_blocked(tier, nxt): + return tier + order = [name for _, name in _tiers()] + want = _tier_for_bytes(working_set) + if tier not in order or not want or order.index(want) <= order.index(tier): + return tier + return nxt + + +def pool_for(end, oom_count=0): + """The tier this range belongs in, or None when not pooling. + + `oom_count` is how many rungs to climb and it counts OOMs, not attempts: + promoting on attempt number once put 65 ranges onto 8-vCPU nodes whose + attempt-1 verdicts were `timeout`, burning ~260 vCPU of a 2304 quota + escalating away from a problem that was never memory. + """ + if not mc.POOL_PREFIX: + return None + if not mc.PROFILE: + return _promote(mc.POOL_NO_PROFILE, oom_count) + prof = profile_for(end) + if not prof: + return _promote(mc.POOL_UNPROFILED, oom_count) + anon = prof.get('peakAnonBytes') + tier = _cache_bump(_tier_for_bytes(anon), anon, prof.get('peakWorkingSetBytes')) + if not tier: + return _promote(mc.POOL_UNPROFILED, oom_count) + return _promote(tier, oom_count) + + +# --- escalation ------------------------------------------------------------- + + +def next_memory(end, base, oom_count): + """Memory request after N OOMs. + + Pooled: the tier ladder IS the ladder, and the promoted tier's cut is the + escalated request, so request and placement move together. A multiplicative + bump cannot do this -- tiers are 2.2-2.5x apart against a 1.5x factor, so + the bump lands BETWEEN them and the pod sits Pending on a pool that can + never satisfy it. + + Unpooled: base * factor^N, where base is what attempt 1 actually ran with. + Escalating a 209Mi profiled range off the configured default jumps to + 36000Mi, a 172x overshoot that throws away the packing win on the first OOM. + """ + if mc.POOL_PREFIX: + promoted = pool_memory(pool_for(end, oom_count)) + # Above the ladder there is nothing to promote into, so hold rather than + # invent a value. + return promoted or base or mc.REQ_MEM + want = int(quantity_bytes(base or mc.REQ_MEM) + * (mc.MEM_BUMP_FACTOR ** max(0, oom_count))) + return bytes_to_quantity(min(want, quantity_bytes(mc.MEM_ESCALATION_CAP))) + + +def next_ephemeral(base, eviction_count): + """Disk after N evictions, or None when nothing is limited. + + Only ephemeral mode has this axis: a pvc run sets no ephemeral request or + limit at all, so there is nothing to raise. + """ + if not mc.LIM_EPHEMERAL: + return None + want = int(quantity_bytes(base or mc.LIM_EPHEMERAL) + * (mc.EPH_BUMP_FACTOR ** max(0, eviction_count))) + return bytes_to_quantity(min(want, quantity_bytes(mc.EPH_ESCALATION_CAP))) + + +# --- the request ------------------------------------------------------------ + + +def _profile_overrides(end, escalated, oom_count): + """Request overrides from the profile, or {}. + + Unpooled escalation opts out: an escalation measures THIS run and outranks + anything an earlier one saw. Pooled escalation does NOT -- the promotion IS + the escalation, and bailing here would send the pod to the new pool still + asking for the old tier's memory. + """ + if end is None or (escalated and not mc.POOL_PREFIX): + return {} + prof = profile_for(end) + out = {} + if prof: + disk = prof.get('peakEphemeralBytes') + if disk and mc.LIM_EPHEMERAL: + want = (int(disk * mc.PROFILE_MARGIN) + + quantity_bytes(mc.PROFILE_EPHEMERAL_HEADROOM) + + _runtime_insurance(prof.get('seconds'), + mc.PROFILE_RUNTIME_EPHEMERAL_INSURANCE)) + out['ephemeral-storage'] = bytes_to_quantity( + min(want, quantity_bytes(mc.PROFILE_MAX_EPHEMERAL))) + if mc.POOL_PREFIX: + # Deliberately BEFORE the no-profile bail: pool_for resolves a tier for + # every range, so returning {} here would pin the pod to that pool while + # sizing it from the flat REQ_CPU. That shipped a 6780m request at a + # pool whose largest node is 4 vCPU -- permanently Pending. + tier = pool_for(end, oom_count) + mem, cpu = pool_memory(tier), pool_cpu(tier) + if mem: + out['memory'] = mem + if cpu: + out['cpu'] = cpu + return out + if not prof: + return out + # Unpooled and profiled: the range's own peak plus a flat allowance and a + # runtime-weighted one. The flat one is load-bearing -- 1.15x of 190Mi is + # 19Mi of slack, and 90 ranges OOMKilled inside 90s without it. + rss = prof.get('peakAnonBytes') + if rss: + want = (int(rss * mc.PROFILE_MARGIN) + + quantity_bytes(mc.PROFILE_CACHE_HEADROOM) + + _runtime_insurance(prof.get('seconds'), + mc.PROFILE_RUNTIME_MEMORY_INSURANCE)) + out['memory'] = bytes_to_quantity( + min(want, quantity_bytes(mc.PROFILE_MAX_MEM))) + return out + + +def requests_for(end, oom_count=0, memory=None, ephemeral=None): + """(requests, limits) for one attempt. + + Only disk is limited: it is the one dimension where an unbounded pod takes + the node down rather than itself. + """ + overrides = _profile_overrides(end, escalated=bool(memory or ephemeral), + oom_count=oom_count) + req = {'cpu': mc.REQ_CPU, 'memory': memory or mc.REQ_MEM} + lim = {} + + # A profiled pooled range owns its node -- the tier's cut excludes a second + # pod -- so a disk limit guards no neighbour and only turns spare disk into + # an eviction. Unprofiled pooled runs keep both: nothing measured them. + pooled_profiled = bool(mc.POOL_PREFIX and mc.PROFILE) + if mc.REQ_EPHEMERAL and not pooled_profiled: + req['ephemeral-storage'] = ephemeral or mc.REQ_EPHEMERAL + else: + overrides.pop('ephemeral-storage', None) + if mc.LIM_EPHEMERAL and not pooled_profiled: + lim['ephemeral-storage'] = ephemeral or mc.LIM_EPHEMERAL + + for key, value in overrides.items(): + req[key] = value + if key == 'ephemeral-storage' and mc.LIM_EPHEMERAL: + lim[key] = value + return req, (lim or None) + + +def node_label_value(end, oom_count): + tier = pool_for(end, oom_count) + return f"{mc.POOL_PREFIX}-{tier}" if tier else mc.NODE_LABEL_VALUE diff --git a/src/MissionParallelCatchup/lib/monitor/verdict.py b/src/MissionParallelCatchup/lib/monitor/verdict.py new file mode 100644 index 00000000..dc3b6009 --- /dev/null +++ b/src/MissionParallelCatchup/lib/monitor/verdict.py @@ -0,0 +1,150 @@ +"""What killed an attempt. + +The collector classifies the live pod into .outcome; this resolves that against +the Job condition and against the archive, and the answer is what budgets are +spent on. +""" +import gzip +import logging +import re + +import record + +logger = logging.getLogger('job_monitor') + +# stellar-core's "did not complete". Ambiguous by construction: a corrupt +# bucket, a SIGTERM during replay and an archive fetch fault all produce it. +CATCHUP_INCOMPLETE_EXIT = 3 + +# Outcomes naming a MECHANISM. A Job-level DeadlineExceeded must never +# overwrite one: it says the Job ran long, not which of these caused it. +SPECIFIC = frozenset({'oom', 'disrupted', 'ephemeral', 'rejected', 'fetch-fault'}) + +# Rule ORDER is the contract with the Job controller's "rule at index N". +RULE_ORDER = ('disrupted', 'oom', 'failed') +_RULE_INDEX = dict(enumerate(RULE_ORDER)) +_JOB_RULE = re.compile(r'rule at index (?P\d+)') +_JOB_MSG = re.compile(r'Container (?P\S+) .*exit code (?P\d+)') + +# Archive scan. The windows are different widths on purpose. +_TAIL_LINES = 400 +_TAIL_BYTES = 1 << 18 # comfortably more than 400 lines of stellar-core log +_STALE_WINDOW = 6 # tight: a wider one credits a fault the range recovered from +_MARKER_WINDOW = 25 # wide: concurrent downloads interleave +_CATCHUP_FAILED = 'Catchup failed' +_STALE_ARCHIVE = 'maybe stale archive' +_TERMINAL = ('Key does not exist', '(404)', 'NoSuchKey') +_TRANSIENT = ('Could not connect to the endpoint URL', 'Unable to locate credentials', + 'ExpiredToken', 'RequestTimeout', 'SlowDown', 'ConnectTimeoutError') + + +def from_job(job): + """Recover a verdict from the Job when the pod is already gone. + + The rule INDEX is the signal, not the exit code: rules are first-match-wins, + so reaching the exit-137 rule proves DisruptionTarget did not match -- the + only way to tell an OOM kill from a grace-period SIGKILL once the pod object + is gone. Index and code are parsed independently, because a rule matching on + onPodConditions reports no exit code at all. + """ + for cond in (job.status.conditions or []): + if cond.type != 'Failed' or cond.status != 'True': + continue + if cond.reason == 'DeadlineExceeded': + return {'outcome': 'timeout', 'exitCode': None, 'pod': ''} + if cond.reason != 'PodFailurePolicy': + continue # e.g. BackoffLimitExceeded: no per-rule detail + msg = cond.message or '' + rule = _JOB_RULE.search(msg) + detail = _JOB_MSG.search(msg) + code = int(detail.group('code')) if detail else None + outcome = _RULE_INDEX.get(int(rule.group('idx'))) if rule else None + if outcome is None: + if code is None: + return None + # No usable index. A drained core exits 3, not 137, so a bare 137 is + # an OOM and a bare 3 without DisruptionTarget is a catchup failure. + outcome = 'oom' if code == 137 else 'failed' + return {'outcome': outcome, 'exitCode': code, + 'pod': detail.group('pod') if detail else ''} + return None + + +def effective(end, attempt, job): + """The verdict this attempt is judged on, promotions applied. + + An attempt with no evidence at all is `unknown`, which has no budget and so + condemns: without evidence the monitor cannot tell a reaped node from a + range that really failed, and a run reporting success on a range nobody + verified is worse than one that stops. + """ + from_pod = record.read_outcome(end, attempt) + from_condition = from_job(job) if job is not None else None + + if from_pod and from_pod.get('outcome') in SPECIFIC: + found = from_pod + else: + found = from_condition or from_pod or {'outcome': 'unknown', 'exitCode': None} + + if found.get('outcome') == 'failed' and found.get('exitCode') == CATCHUP_INCOMPLETE_EXIT: + cause = exit3_cause(end, attempt) + if cause: + return dict(found, outcome='fetch-fault', reason=cause) + return found + + +def exit3_cause(end, attempt): + """The transient fetch fault behind an exit 3, or None. + + Three conditions, all required. Missing any leaves the attempt `failed`, + which has no budget -- so a bare `Catchup failed` ends the run rather than + burning 20 attempts proving the chain is broken. + """ + lines = _tail(end, attempt) + if not lines: + return None + failed_at = _last_index(lines, _CATCHUP_FAILED) + if failed_at is None: + return None + stale_at = _last_index(lines[max(0, failed_at - _STALE_WINDOW):failed_at], + _STALE_ARCHIVE) + if stale_at is None: + return None + anchor = max(0, failed_at - _STALE_WINDOW) + stale_at + # Most-recent-first: a range that recovered from a transient fault and then + # hit a 404 is terminal, and the 404 is the later line. + for line in reversed(lines[max(0, anchor - _MARKER_WINDOW):anchor + 1]): + if any(marker in line for marker in _TERMINAL): + return None # the object genuinely is not there; retrying cannot help + for marker in _TRANSIENT: + if marker in line: + return marker + return None + + +def _last_index(lines, needle): + for i in range(len(lines) - 1, -1, -1): + if needle in lines[i]: + return i + return None + + +def _tail(end, attempt): + """The archive's last lines. A missing or unreadable archive is not an + error: it means the collector has nothing to say, and the caller treats + that as "no fetch fault". + + Split only the final bytes, never readlines(): gzip cannot seek, so the + whole archive is decompressed either way, but building a str per line to + keep 400 of them is 90% of the cost on a tip range. The leading fragment is + dropped because the slice lands mid-line. + """ + try: + with gzip.open(record.log_path(end, attempt), 'rb') as fh: + data = fh.read() + except (OSError, EOFError, gzip.BadGzipFile): + return [] + lines = data[-_TAIL_BYTES:].decode('utf-8', 'replace').splitlines() + if len(data) > _TAIL_BYTES: + lines = lines[1:] # only then did the slice land mid-line + return lines[-_TAIL_LINES:] diff --git a/src/MissionParallelCatchup/lib/records.py b/src/MissionParallelCatchup/lib/records.py new file mode 100644 index 00000000..1259420e --- /dev/null +++ b/src/MissionParallelCatchup/lib/records.py @@ -0,0 +1,56 @@ +"""The filenames both processes agree on, and the write that keeps them whole. + +This is the entire cross-process contract. The collector writes these files +while a pod still exists and the monitor reads them back, so a disagreement +about a name is a measurement silently lost. Everything either side does with +the contents lives on its own side: record for the monitor, state_files for the +collector. +""" +import os + +import config + + +def log_path(end, attempt): + """Canonical archive name, written by the log-collector sidecar. + + Deliberately carries no ok/failed suffix: which ranges failed is recorded in + the progress ConfigMap, and encoding it here would mean two components + disagreeing about a filename. + """ + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.log.gz") + + +def state_path(end, attempt): + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.state") + + +def outcome_path(end, attempt): + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.outcome") + + +def metrics_path(end, attempt): + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.metrics") + + +def done_path(end, attempt): + return os.path.join(config.LOG_DIR, f"range-{end}-a{attempt}.done") + + +def write_atomic(path, body, opener=None): + """Write `body` through tmp+rename so a reader never sees a partial file. + + Both processes write these files and both read them back: the collector + writes while the monitor polls, so a torn .metrics or .outcome reads as + corrupt and the measurement is lost, and a restarted monitor decides a + range's remaining budget from .outcome and .verdict. + """ + tmp = path + '.tmp' + # `opener` resolves at call time, not as a default argument, so the write + # seam stays patchable -- the tmp+rename discipline is only worth having if + # a test can crash a write mid-flight and prove the real path is untouched. + with (opener or open)(tmp, 'wt') as fh: + fh.write(body) + os.replace(tmp, path) + + diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/files/logarithmic_range_generator.sh b/src/MissionParallelCatchup/parallel_catchup_helm/files/logarithmic_range_generator.sh deleted file mode 100644 index ac344260..00000000 --- a/src/MissionParallelCatchup/parallel_catchup_helm/files/logarithmic_range_generator.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh - -# Check if required environment variables are set -if [ -z "$LOGARITHMIC_FLOOR_LEDGERS" ]; then echo "LOGARITHMIC_FLOOR_LEDGERS not set"; exit 1; fi -if [ -z "$OVERLAP_LEDGERS" ]; then echo "OVERLAP_LEDGERS not set"; exit 1; fi -if [ -z "$STARTING_LEDGER" ]; then echo "STARTING_LEDGER not set"; exit 1; fi -if [ -z "$LATEST_LEDGER_NUM" ]; then echo "LATEST_LEDGER_NUM not set"; exit 1; fi -if [ -z "$NUM_PARALLELISM" ]; then echo "NUM_PARALLELISM not set"; exit 1; fi -if [ -z "$REDIS_HOST" ]; then echo "REDIS_HOST not set"; exit 1; fi -if [ -z "$REDIS_PORT" ]; then echo "REDIS_PORT not set"; exit 1; fi - -floorSize=$LOGARITHMIC_FLOOR_LEDGERS -overlapLedgers=$OVERLAP_LEDGERS -startLedger=$(echo "$STARTING_LEDGER" | awk '{printf "%d", $1}') -latestLedgerNum=$(echo "$LATEST_LEDGER_NUM" | awk '{printf "%d", $1}') -numParallelism=$NUM_PARALLELISM - -echo "starting logarithmic range generationg with the following inputs: -floorSize=$floorSize -overlapLedgers=$overlapLedgers -startLedger=$startLedger -latestLedgerNum=$latestLedgerNum -numParallelism=$numParallelism" - -generate_uniform () { - sl="$1" - el="$2" - ss="$3" - echo "generating uniform ranges from parameters: startLedger=$sl, endLedger=$el, segSize=$ss" - while [ "$el" -gt "$sl" ]; do - # clamp the segment size to the num of remaining ledgers to avoid doing redundant work - ledgersPerJob=$((el - sl)) - if [ "$ledgersPerJob" -gt "$ss" ]; then - ledgersPerJob=$ss - fi - ledgersToApply=$((ledgersPerJob + overlapLedgers)); - echo "${el}/${ledgersToApply}"; - # our queue assumes push-left-pop-right, but since we are generating the ranges in reverse order, here we push right - redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" RPUSH ranges "${el}/${ledgersToApply}"; - el=$(( el - ledgersPerJob )); - # sleep for a short duration to avoid overloading the redis-cli connection - sleep 1 - done -} - -endLedger=$((latestLedgerNum / 2)) -chunkSize=$(( (endLedger - startLedger + 1) / numParallelism )) -while [ "$chunkSize" -gt "$floorSize" ]; do - generate_uniform "$startLedger" "$endLedger" "$chunkSize" - startLedger=$(( endLedger + 1 )) - chunkSize=$(( chunkSize / 2 )) - endLedger=$((startLedger + (chunkSize * numParallelism) )) -done - -# treat the rest with one uniform-ranged chunk -generate_uniform "$((endLedger+1))" "$latestLedgerNum" "$floorSize" diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/files/uniform_range_generator.sh b/src/MissionParallelCatchup/parallel_catchup_helm/files/uniform_range_generator.sh deleted file mode 100644 index 2f412b63..00000000 --- a/src/MissionParallelCatchup/parallel_catchup_helm/files/uniform_range_generator.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/sh - -# Check if required environment variables are set -if [ -z "$LEDGERS_PER_JOB" ]; then echo "LEDGERS_PER_JOB not set"; exit 1; fi -if [ -z "$OVERLAP_LEDGERS" ]; then echo "OVERLAP_LEDGERS not set"; exit 1; fi -if [ -z "$STARTING_LEDGER" ]; then echo "STARTING_LEDGER not set"; exit 1; fi -if [ -z "$LATEST_LEDGER_NUM" ]; then echo "LATEST_LEDGER_NUM not set"; exit 1; fi -if [ -z "$REDIS_HOST" ]; then echo "REDIS_HOST not set"; exit 1; fi -if [ -z "$REDIS_PORT" ]; then echo "REDIS_PORT not set"; exit 1; fi - -ledgersPerJob=$LEDGERS_PER_JOB -overlapLedgers=$OVERLAP_LEDGERS -startingLedger=$(echo "$STARTING_LEDGER" | awk '{printf "%d", $1}') -endRange=$(echo "$LATEST_LEDGER_NUM" | awk '{printf "%d", $1}') -ledgersToApply=$((ledgersPerJob + overlapLedgers)); - -echo "$(date) Generating uniform ledger ranges from parameters -ledgersPerJob=$ledgersPerJob, -overlapLedgers=$overlapLedgers, -ledgersToApply=$ledgersToApply, -startingLedger=$startingLedger, -endRange=$endRange" - -# Store redis commands in a file for bulk upload in a transaction -CMD_FILE=redis_bulk_load -echo "MULTI">$CMD_FILE # Start redis transaction -while [ "$endRange" -gt "$startingLedger" ]; do - echo "${endRange}/${ledgersToApply}"; - echo "RPUSH ranges \"${endRange}/${ledgersToApply}\"">>$CMD_FILE - endRange=$(( endRange - ledgersPerJob )); -done -echo "EXEC">>$CMD_FILE # Close redis transaction - -echo "$(date) Created file $CMD_FILE with $(wc -l $CMD_FILE) lines. Loading into redis" -for i in $(seq 1 6);do - redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" <$CMD_FILE - if [ $? -eq 0 ]; then - break - else - echo "$(date) Error inserting data. Sleeping and retrying" - sleep 5 - fi -done - -echo "$(date) Finished generating ranges" diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/files/worker.sh b/src/MissionParallelCatchup/parallel_catchup_helm/files/worker.sh deleted file mode 100644 index b8efc18b..00000000 --- a/src/MissionParallelCatchup/parallel_catchup_helm/files/worker.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/sh - -# Check if required environment variables are set -if [ -z "$REDIS_HOST" ]; then echo "REDIS_HOST not set"; exit 1; fi -if [ -z "$REDIS_PORT" ]; then echo "REDIS_PORT not set"; exit 1; fi -if [ -z "$JOB_QUEUE" ]; then echo "JOB_QUEUE not set"; exit 1; fi -if [ -z "$PROGRESS_QUEUE" ]; then echo "PROGRESS_QUEUE not set"; exit 1; fi -if [ -z "$FAILED_QUEUE" ]; then echo "FAILED_QUEUE not set"; exit 1; fi -if [ -z "$SUCCESS_QUEUE" ]; then echo "SUCCESS_QUEUE not set"; exit 1; fi -if [ -z "$METRICS" ]; then echo "METRICS not set"; exit 1; fi -if [ -z "$JOB_OWNERS" ]; then echo "JOB_OWNERS not set"; exit 1; fi -if [ -z "$RELEASE_NAME" ]; then echo "RELEASE_NAME not set"; exit 1; fi -if [ -z "$POD_NAME" ]; then echo "POD_NAME not set"; exit 1; fi - -# ensure redis-cli is available -if [ ! "$(redis-cli --version)" ]; then - echo "redis-cli not found, please ensure running with a supported stellar-core version" - exit 1 -fi - -SLEEP_INTERVAL=10 -LOG_DIR="/data" - -while true; do -# Fetch the next job key from the Redis queue. -# Our ranges are generated in the order we want to run them from left to right, so we always pull from the left -JOB_KEY=$(redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" LMOVE "$JOB_QUEUE" "$PROGRESS_QUEUE" LEFT LEFT) -LMOVE_EXIT_CODE=$? - -# Only process a job if the command succeeded AND we got a non-empty job key -if [ $LMOVE_EXIT_CODE -eq 0 ] && [ -n "$JOB_KEY" ]; then - # Register ownership so the monitor knows which worker owns this job - redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" HSET "$JOB_OWNERS" "$JOB_KEY" "$POD_NAME" - if [ $? -ne 0 ]; then - echo "Error: Failed to register job ownership for $JOB_KEY. Exiting." - exit 1 - fi - - # Start timer - START_TIME=$(date +%s) - echo "Processing job: $JOB_KEY" - - # Run stellar-core: create new-db then catchup - /usr/bin/stellar-core --conf /config/stellar-core.cfg new-db --console && \ - /usr/bin/stellar-core --conf /config/stellar-core.cfg catchup "$JOB_KEY" \ - --metric 'ledger.transaction.apply' --console - STELLAR_CORE_EXIT_CODE=$? - - # End timer and duration - END_TIME=$(date +%s) - DURATION=$((END_TIME - START_TIME))s - echo "Finish processing job: $JOB_KEY, duration: $DURATION" - - # Check if both commands succeeded - if [ $STELLAR_CORE_EXIT_CODE -eq 0 ]; then - echo "Successfully processed job: $JOB_KEY" - QUEUE_COMMAND="LPUSH $SUCCESS_QUEUE \"$JOB_KEY\"" - else - echo "Error processing job: $JOB_KEY (exit code: $STELLAR_CORE_EXIT_CODE)" - QUEUE_COMMAND="LPUSH $FAILED_QUEUE \"$JOB_KEY|$POD_NAME\"" - fi - - # Parse and extract the metrics from the log file - LOG_FILE=$(ls -t "$LOG_DIR"/stellar-core*.log 2>/dev/null | head -n 1) - if [ -z "$LOG_FILE" ]; then - echo "No log file found in $LOG_DIR" - exit 1 - fi - - tx_apply_ms=$(tac "$LOG_FILE" | grep -m 1 -B 11 "metric 'ledger.transaction.apply':" | grep "sum =" | awk '{print $NF}') - echo "Log file: $LOG_FILE" - echo "ledger.transaction.apply sum: $tx_apply_ms" - # Validate metric was extracted successfully - if [ -z "$tx_apply_ms" ]; then - echo "Warning: Failed to extract metric 'ledger.transaction.apply' from log file" - tx_apply_ms="N/A" - fi - - # Push metrics to redis in a transaction to ensure data consistency. Retry for 5min on failures - # Extract the pod ordinal (last hyphen-separated segment) from pod name like "release-name-stellar-core-0" - core_id=$(echo "$POD_NAME" | awk -F'-' '{print $NF}') - # Validate core_id was extracted successfully - if [ -z "$core_id" ]; then - echo "Error: Failed to extract core_id from POD_NAME: $POD_NAME" - core_id="N/A" - fi - - result=1 # Initialize to failure - for i in $(seq 1 30);do - redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" <-, so only its + key is used. Every later entry is required literally, which is + how a run pins itself to one capacity of a tier -- both + capacities carry the same tier label value, so nothing else + separates them. + */}} + {{- with (first .Values.worker.requireNodeLabels) }} + {{- if kindIs "map" . }} + - name: NODE_LABEL_KEY + value: {{ .key | quote }} + - name: NODE_LABEL_VALUE + value: {{ (first (default (list "") .values)) | quote }} + {{- else }} + - name: NODE_LABEL_KEY + value: {{ (splitList ":" .) | first | quote }} + - name: NODE_LABEL_VALUE + value: {{ (splitList ":" .) | last | quote }} + {{- end }} + {{- end }} + - name: REQUIRE_NODE_LABELS + value: {{ include "catchup.labelPairs" (rest .Values.worker.requireNodeLabels) | quote }} + {{- with (first .Values.worker.avoidNodeLabels) }} + {{- if kindIs "map" . }} + - name: AVOID_NODE_LABEL_KEY + value: {{ .key | quote }} + - name: AVOID_NODE_LABEL_VALUE + value: {{ (first (default (list "") .values)) | quote }} + {{- else }} + - name: AVOID_NODE_LABEL_KEY + value: {{ (splitList ":" .) | first | quote }} + - name: AVOID_NODE_LABEL_VALUE + value: {{ (splitList ":" .) | last | quote }} + {{- end }} + {{- end }} + {{- with (first .Values.worker.tolerateNodeTaints) }} + - name: TOLERATE_TAINT + value: {{ if kindIs "map" . }}{{ .key | quote }}{{ else }}{{ (splitList ":" .) | first | quote }}{{ end }} + {{- end }} + resources: + {{- toYaml .Values.monitor.resources | nindent 12 }} + # job_monitor.py mirrors its log to /data so it survives the pod. + volumeMounts: + - name: data + mountPath: /data + - name: logs + mountPath: /logs + # No readinessProbe: nothing consumes Ready. There is no Service, and + # the prometheus kubernetes-pods job keeps targets on the scrape + # annotation alone, with no pod-ready filter. + # + # A dead pod watch stops all failure classification, which degrades + # silently: later failures record outcome=unknown and get retried + # blindly. Restart the container rather than run half-blind. + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 60 + failureThreshold: 3 + # Streams each worker pod's log from the moment it starts, so nothing + # is lost when Karpenter reaps the node ~1 min after the pod exits -- + # and a stuck range is readable while it is still stuck. Separate + # container so its memory limit is its own: holding ~1024 open streams + # must not be affected by, or affect, the reconcile loop. + - name: log-collector + image: {{ .Values.monitor.image }} + imagePullPolicy: {{ .Values.monitor.imagePullPolicy }} + # Resolved on PATH, not /usr/bin/python3: Dockerfile.jobmonitor builds + # on python:3.12-slim, which ships the interpreter at + # /usr/local/bin/python3. An absolute path here pins the chart to one + # base image and fails as StartError -- the monitor container comes up + # (it inherits the image CMD) and only the sidecar crashloops. + command: ["python3", "log_collector.py"] + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: RUN_NAME + value: {{ .Release.Name | quote }} + - name: LOG_DIR + value: /logs + - name: COLLECTOR_POLL_SECONDS + value: {{ .Values.monitor.collectorPollSeconds | quote }} + - name: WATCH_TIMEOUT_SECONDS + value: {{ .Values.monitor.watchTimeoutSeconds | quote }} + - name: WATCH_RETRY_SECONDS + value: {{ .Values.monitor.watchRetrySeconds | quote }} + - name: MAX_DOOMED_FOLLOWS + value: {{ .Values.monitor.maxDoomedFollows | quote }} + - name: DOOMED_FOLLOW_SECONDS + value: {{ .Values.monitor.doomedFollowSeconds | quote }} + - name: MAX_CONCURRENT_POLLS + value: {{ .Values.monitor.maxConcurrentPolls | quote }} + - name: MAX_POLL_CHARS + value: {{ .Values.monitor.maxPollChars | int64 | quote }} + - name: MAX_CONCURRENT_SAMPLES + value: {{ .Values.monitor.maxConcurrentSamples | quote }} + - name: READ_TIMEOUT_SECONDS + value: {{ .Values.monitor.readTimeoutSeconds | quote }} + - name: TERMINAL_REREAD_SECONDS + value: {{ .Values.monitor.terminalRereadSeconds | quote }} + # Failures are always kept; successes are the bulk of the volume. + - name: SAVE_SUCCESS_LOGS + value: {{ .Values.monitor.saveSuccessLogs | quote }} + # The peak-ephemeral sampler runs only in ephemeral mode. Without + # this the collector defaults to pvc and silently records nothing. + - name: STORAGE_MODE + value: {{ .Values.worker.storageMode | quote }} + resources: + {{- toYaml .Values.monitor.collectorResources | nindent 12 }} + volumeMounts: + - name: logs + mountPath: /logs + volumes: + - name: data + emptyDir: {} + - name: logs + persistentVolumeClaim: + claimName: {{ .Release.Name }}-job-monitor-logs diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_preload_redis.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_preload_redis.yaml deleted file mode 100644 index 14de3a7b..00000000 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/job_preload_redis.yaml +++ /dev/null @@ -1,64 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ .Release.Name }}-preload-redis -spec: - template: - spec: - containers: - - name: preload - image: redis:7 - command: ["/bin/sh", "-c"] - args: - - |- - case "$STRATEGY" in - "uniform") - /bin/sh /scripts/uniform_range_generator.sh - ;; - "logarithmic") - /bin/sh /scripts/logarithmic_range_generator.sh - ;; - *) - echo 'Error: Unknown strategy' && exit 1 - ;; - esac - envFrom: - - configMapRef: - name: {{ .Release.Name }}-range-generator-config - volumeMounts: - - name: script - mountPath: /scripts - initContainers: - - name: wait-for-redis - image: redis:7 - command: ['sh', '-c', "until redis-cli -h {{ .Values.redis.hostname }} -p {{ .Values.redis.port }} ping; do echo waiting for redis; sleep 2; done;"] - restartPolicy: OnFailure - volumes: - - name: script - configMap: - name: {{ .Release.Name }}-generator-script ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-generator-script -data: - uniform_range_generator.sh: |- - {{- (.Files.Get "files/uniform_range_generator.sh") | nindent 4 }} - logarithmic_range_generator.sh: |- - {{- (.Files.Get "files/logarithmic_range_generator.sh") | nindent 4 }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-range-generator-config -data: - STRATEGY: "{{ .Values.range_generator.strategy }}" - STARTING_LEDGER: "{{ .Values.range_generator.params.starting_ledger }}" - LATEST_LEDGER_NUM: "{{ .Values.range_generator.params.latest_ledger_num }}" - OVERLAP_LEDGERS: "{{ .Values.range_generator.params.overlap_ledgers }}" - LEDGERS_PER_JOB: "{{ .Values.range_generator.params.uniform_ledgers_per_job }}" - LOGARITHMIC_FLOOR_LEDGERS: "{{ .Values.range_generator.params.logarithmic_floor_ledgers }}" - NUM_PARALLELISM: "192" - REDIS_HOST: "{{ .Values.redis.hostname }}" - REDIS_PORT: "{{ .Values.redis.port }}" diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/templates/redis_queue.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/templates/redis_queue.yaml deleted file mode 100644 index aaba65fb..00000000 --- a/src/MissionParallelCatchup/parallel_catchup_helm/templates/redis_queue.yaml +++ /dev/null @@ -1,37 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: "{{ .Values.redis.hostname }}" -spec: - type: ClusterIP - ports: - - port: {{ .Values.redis.port }} - targetPort: 6379 - selector: - app: {{ .Values.redis.hostname }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.redis.hostname }} -spec: - replicas: 1 - selector: - matchLabels: - app: {{ .Values.redis.hostname }} - template: - metadata: - labels: - app: {{ .Values.redis.hostname }} - spec: - containers: - - name: redis - image: redis:7 - ports: - - containerPort: 6379 - command: ["redis-server"] - resources: - requests: - cpu: "{{ .Values.redis.resources.requests.cpu }}" - memory: "{{ .Values.redis.resources.requests.memory }}" \ No newline at end of file diff --git a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml index 36f3ea3d..5b61d9ea 100644 --- a/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml +++ b/src/MissionParallelCatchup/parallel_catchup_helm/values.yaml @@ -1,32 +1,41 @@ -redis: - hostname: "" # to be set by the mission - port: 6379 - job_queue: "ranges" - success_queue: "succeeded" - failed_queue: "failed" - progress_queue: "in_progress" - metrics: "metrics" - job_owners: "job_owners" - resources: - requests: - cpu: "100m" - memory: "100Mi" - worker: stellar_core_image: "stellar/stellar-core:latest" replicas: 5 + # pvc: /data survives the pod, so an evicted range resumes at L+1 -- this is + # what makes spot viable. ephemeral: /data is an emptyDir on the node, no + # resume, and ephemeral_storage must be sized to hold it. + storageMode: "pvc" + storageClass: "" + # Sized from what a range actually uses: the old node-local /data ran on a + # 35Gi request / 40Gi limit. 40Gi x 1024 workers is 40 TiB of gp3 rather than + # 100 TiB, which matters against the per-region storage quota. + # One PVC per ledger range, not per concurrency slot: measured on ssc-test, + # 7.5x the volume lifecycles cost nothing (3151s vs 3210s) and reuse pinned + # every later range to one AZ. 60Gi covers the profile's 37.8Gi peak. + storageSize: "60Gi" requireNodeLabels: [] avoidNodeLabels: [] tolerateNodeTaints: [] asanOptions: "quarantine_size_mb=1:malloc_context_size=5:alloc_dealloc_mismatch=0" - resources: # resources below are left empty on purpose, they are read and overridden from `StellarKubeCfg.fs` + resources: requests: - cpu: "" - memory: "" + # V2's worker sizing lives HERE, not in StellarKubeSpecs.fs. That spec is + # shared with the V1 parallel catchup missions (parallelism 128 and 256, + # a different execution model), and sizing V2 there silently resized them + # too -- it took their cpu request from 250m to 1800m. + # + # 1800m/9Gi is the unprofiled default: 4 workers per r8*.2xlarge node + # (cpu-bound) or 3 per m8*.2xlarge (memory-bound). A measured range + # overrides both; --pubnet-parallel-catchup-cpu-request overrides the run. + cpu: "1800m" + memory: "9Gi" + # Still filled in by the mission: ephemeral-storage is 2Gi/4Gi in pvc mode + # and the StellarKubeSpecs value in ephemeral mode. ephemeral_storage: "" limits: - cpu: "" - memory: "" + # cpu and memory are deliberately absent -- workers run unlimited on both. + # See REQ_CPU/REQ_MEM in job_monitor.py. Only disk is capped, because that + # is the one dimension where an unbounded pod takes the node with it. ephemeral_storage: "" historyGetCommandCore001: "curl -sf http://history.stellar.org/prd/core-live/core_live_001/{0} -o {1}" @@ -35,26 +44,187 @@ worker: historyGetCommandCore003: "curl -sf http://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}" +# Index -> ledger range is computed in the Job template; no queue to preload. +# The range itself -- start, end, size, overlap, order -- is per-run input and +# arrives with the driver's POST /start, so the chart installs a generic +# monitor and holds none of it. + monitor: - gateway_name: "traefik-gateway-private" - gateway_namespace: "traefik" - hostname: "" # to be set by the mission - path_prefix: "/default" - logging_interval_seconds: 300 - logging_level: "INFO" # 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL' + # Reuses the existing hand-built job-monitor image slot -- no new image and no + # new push path (nothing in .github/workflows builds these). + # + # TEMPORARY dev pin: stellar/ssc-job-monitor:latest predates the apps/+lib/ + # split and ATTEMPT_BUDGETS, so this chart would ship env vars it cannot read. + # Revert to the stellar/ repo once there is a push path for it. + image: "stellajuna/ssc-jm:latest" + # The driver reaches the monitor through this route: profile in via POST + # /start, status and logs out. Empty routeHost disables the HTTPRoute; the + # Service still exists, so an in-cluster caller works either way. + # The monitor's Role is the same for every run, so it is provisioned once per + # namespace in stellar/kube, bound to system:serviceaccounts: -- + # every ServiceAccount in the namespace, because a binding names its subject + # exactly and the monitor's is named after the release. Creating the Role per + # release instead needs the installer to hold RBAC-granting rights, which is a + # privilege every mission would then carry so that this one chart can install + # its own Role. + # + # true creates the Role, RoleBinding and node-stats ClusterRole for a cluster + # that does not provide them -- ssc-eks does, through a catchup-job-monitor + # RoleBinding, so leave it false there. With neither, the monitor 403s. + createRbac: false + routeHost: "" + gatewayName: "" + gatewayNamespace: "" + # --- nodepool tiers ------------------------------------------------------- + # + # A range picks a NODEPOOL by its measured memory and gets that node to + # itself. Empty prefix disables all of it and every worker keeps the single + # global node label, which is exactly the pre-tier behaviour. + # + # Cuts are node_usable/1.60, the p99 of run-to-run growth in the same range's + # peakAnonBytes (18,073 observations). Range 63080767 measured 13.75Gi, landed + # on 14.1Gi nodes at a 1.03x margin, and OOMKilled on both. + poolPrefix: "" + poolTiers: "0:subdwarf,0.79:dwarf,1.61:subgiant,3.87:giant,8.85:supergiant,18.38:hypergiant,:supernova" + # Rungs the cache bump may never take, "from->to" comma separated. Nothing + # else refuses a promotion, so a rung that should not be taken has to be + # named here -- see POOL_BLOCK_RUNGS in lib/config.py. + poolBlockRungs: "dwarf->subgiant,hypergiant->supernova" + poolCpu: "subdwarf:0.85,dwarf:0.85,subgiant:1.85,giant:1.85,supergiant:1.85,hypergiant:1.85,supernova:3.80,protostar:1.85,nebula:1.80" + poolMem: "subdwarf:1280Mi,dwarf:1280Mi,subgiant:2816Mi,giant:6656Mi,supergiant:14336Mi,hypergiant:29696Mi,supernova:60416Mi,protostar:29696Mi,nebula:9216Mi" + # Profiled run, range past the profile's top: newest ledgers, and the newest + # are the densest, so a rich pool rather than an average one. + poolUnprofiled: "protostar" + # No profile at all: nothing is known, so the biggest nodes and the configured + # defaults. + poolNoProfile: "nebula" + profileMargin: 1.15 + # Ceiling for profile-derived memory, deliberately above the unprofiled + # limit: a range that needs more must be able to ask for it. + profileMaxMemory: "32Gi" + # The disk allowances, mirroring the memory ones above. See + # PROFILE_EPHEMERAL_HEADROOM in lib/config.py. + profileEphemeralHeadroom: "2Gi" + profileRuntimeEphemeralInsurance: "8Gi" + # Above the flat LIM_EPHEMERAL on purpose: that limit is what an UNMEASURED + # range gets, and capping a measured one at it would discard the measurement. + profileMaxEphemeral: "64Gi" + # Flat allowance added to a range's measured rss. Not zero: a multiplicative + # margin is meaningless at small rss -- see PROFILE_CACHE_HEADROOM in + # lib/config.py for the 90 OOMKills that established it. + profileCacheHeadroom: "512Mi" + # Extra memory allowance weighted by a range's runtime relative to the + # longest valid runtime in the profile. Long-running ranges need more slack + # for page cache and allocator growth. 0 disables. + profileRuntimeMemoryInsurance: "3Gi" + # Always, because runs name this image by a MUTABLE tag and IfNotPresent then + # pins a node to whatever it cached first -- run #183 shipped a 0-byte + # job_monitor.log from an image fixed hours earlier, off a node up since June. + # Kubernetes defaults :latest to Always for this reason. + imagePullPolicy: Always + mission: "HistoryPubnetParallelCatchup" + # Adds a `mission` label to worker pods, which kube-state-metrics exposes as + # label_mission for the Grafana container panels. Default OFF: those panels + # are per-pod, so 1024 workers would swamp any view with mission=$__all and + # slow the shared dashboard for everyone. Turn on per-run for a small run, or + # once the panels aggregate. + emitMissionLabel: false + loggingIntervalSeconds: 10 + # Cosmetic stellar-core /info sampling. A fixed pool probes each Running pod + # about once per interval; three consecutive failures preserve the old + # approximately 90-second down threshold without coupling network I/O to the + # 10-second reconcile loop. Unprobed/intermediate workers report unknown. + livenessProbeTimeoutSeconds: 5 + # One sweep per reconcile pass, and the reconcile loop waits for it -- so this + # is the most a fleet of unreachable workers can delay dispatch. + livenessSweepSeconds: 15 + # Concurrent probes per sweep. At 2096 workers this is the throughput that has + # to fit inside livenessSweepSeconds: 32 slots x sub-second local responses + # clears the fleet, and a fleet that does not answer is bounded by the sweep. + livenessMaxConcurrency: 32 + # Attempts each failure cause gets before the range is condemned. Every budget + # is spent by its own cause, and a cause that is not here is condemned the + # first time it happens -- a timeout, a real catchup failure, an exit 3 with + # nothing in its archive, and anything the monitor could not classify. + attemptBudgets: + # The cluster took the pod away mid-run, which proves the range was fine. + # Effectively unlimited: on spot a healthy range is evicted dozens of times. + disrupted: 100 + # Refused by the kubelet before any container ran (attachment limits, + # admission churn). The range never started, so a retry masks nothing. + rejected: 100 + # An exit 3 whose archive named a fetch fault: an unreachable history + # mirror is the cluster's problem, not the range's. + fetchFault: 20 + # Each retry escalates the memory request one rung. + oom: 5 + # Each retry escalates the disk limit one rung. Smallest, because an + # eviction repeats identically until the range gets more disk. + ephemeral: 4 + # Destroyed before its container started, so the Job carries no exit code. + unknown: 2 + memBumpFactor: 1.5 + ephBumpFactor: 1.5 + maxEphemeral: "200Gi" + maxMem: "48Gi" + graceSeconds: 100 + # preStop stall before SIGTERM in seconds, 0 = off, sized against the + # collector CYCLE rather than collectorPollSeconds. It buys time to NOTICE the + # disruption, not to capture the metric -- a 60s hook with 10s polling still + # lost txApply where 1s polling with no hook caught it. Overshooting is dead + # time inside the ~120s spot reclaim budget; it must stay under graceSeconds. + prestopSleepSeconds: 5 + # Per-attempt wall-clock cap in seconds, 0 = none, catching a range wedged in + # archive retries. A timeout is TERMINAL, so too low fails the mission; see + # ATTEMPT_DEADLINE_SECONDS in lib/config.py for why 12h and not 3h. + attemptDeadlineSeconds: 43200 + jobTtlSeconds: 600 + # Worker logs are pulled to the monitor's volume while each pod is still + # alive, then collected in one exec at teardown instead of ~1024. + logStorageClass: "" + logStorageSize: "100Gi" + saveSuccessLogs: true + collectorPollSeconds: 5 + # Spot gives ~120s of notice; past roughly double that the drain was + # cancelled and the stream should go back to interval polling. + doomedFollowSeconds: 300 + # Follow streams get their own budget so a mass reclaim cannot consume every + # poll slot and blackout the rest of the run. Condemned pods beyond this fall + # back to polling, which is the pre-existing behaviour. + maxDoomedFollows: 256 + # Lifetime of each pod watch before the apiserver closes it and the collector + # reconnects from its last resourceVersion. The watch is what makes a + # condemnation visible immediately rather than on the next collectorPollSeconds + # sweep: measured at prestopSleepSeconds=5, the sweep alone caught that window + # about half the time and lost 32 of 52 mid-replay legs. 0 disables the watch + # and leaves detection to the sweep. + watchTimeoutSeconds: 600 + watchRetrySeconds: 1 + maxConcurrentPolls: 96 + maxPollChars: 8388608 + # Concurrent kubelet sweeps. One request per node, and a dead one costs the + # connect timeout, so this stays small next to the log-read budget. + maxConcurrentSamples: 16 + # Longest one log read may stall before it is abandoned. A cycle reads every + # pod together, so a wedged read would hold up the whole pass. + readTimeoutSeconds: 60 + # How far back the terminal read reaches past the resume point, so a medida + # block split across two reads is whole in one. The archive dedups the + # overlap, so this only widens what the scan sees. + terminalRereadSeconds: 30 + # Poll cycles a stream gets to finalize after its pod leaves the pod list, + # before it is cancelled and its connection slot reclaimed. + # Where the monitor runs, from --job-monitor-node-labels and + # --job-monitor-tolerate-taints. Empty = anywhere untainted. Pinning it to a + # worker tier reserves that node: it holds the logs PVC for the whole run. + nodeSelector: {} + collectorResources: + requests: { cpu: "200m", memory: "512Mi" } + limits: { cpu: "2", memory: "2Gi" } + tolerateNodeTaints: [] resources: - requests: - cpu: "100m" - memory: "100Mi" - -range_generator: - strategy: "uniform" # "uniform" or "logarithmic" - params: - starting_ledger: 0 - latest_ledger_num: 100000 - overlap_ledgers: 320 - uniform_ledgers_per_job: 16000 # only for strategy="uniform" - logarithmic_floor_ledgers: 16000 # only for strategy="logarithmic" + requests: { cpu: "200m", memory: "512Mi" } + limits: { cpu: "2", memory: "2Gi" } service_account: annotations: []