diff --git a/cmd/relayfile-cli/checkpoint_lifecycle.go b/cmd/relayfile-cli/checkpoint_lifecycle.go new file mode 100644 index 00000000..62f9ea8f --- /dev/null +++ b/cmd/relayfile-cli/checkpoint_lifecycle.go @@ -0,0 +1,1711 @@ +package main + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/exec" + "os/signal" + "path/filepath" + "regexp" + "strconv" + "strings" + "syscall" + "time" + + "github.com/agentworkforce/relayfile/internal/mountlease" + "github.com/agentworkforce/relayfile/internal/mountscope" + "github.com/agentworkforce/relayfile/internal/mountsync" +) + +const ( + checkpointLifecycleVersion = 1 + checkpointStopTimeout = 10 * time.Second + checkpointResumeTimeout = 60 * time.Second + checkpointCLIInputMaxBytes = 16 * 1024 +) + +var ( + checkpointLifecycleIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$`) + checkpointDigestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + checkpointRevisionPattern = regexp.MustCompile(`^(?:0|rev_[0-9]+)$`) + checkpointCursorPattern = regexp.MustCompile(`^(?:0|evt_[0-9]+)$`) +) + +type checkpointCLIError struct { + code string + exitCode int + err error +} + +func (e *checkpointCLIError) Error() string { return e.code + ": " + e.err.Error() } +func (e *checkpointCLIError) Unwrap() error { return e.err } +func (e *checkpointCLIError) ExitCode() int { return e.exitCode } + +func checkpointError(code string, exitCode int, err error) error { + if err == nil { + err = errors.New(code) + } + return &checkpointCLIError{code: code, exitCode: exitCode, err: err} +} + +func decodeStrictCheckpointInput(stdin io.Reader, out any) error { + payload, err := io.ReadAll(io.LimitReader(stdin, checkpointCLIInputMaxBytes+1)) + if err != nil { + return err + } + if len(payload) > checkpointCLIInputMaxBytes { + return fmt.Errorf("checkpoint input exceeds %d bytes", checkpointCLIInputMaxBytes) + } + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(out); err != nil { + return err + } + if decoder.Decode(&struct{}{}) != io.EOF { + return errors.New("checkpoint input must contain exactly one JSON document") + } + return nil +} + +type checkpointMountConfig struct { + Version int `json:"version"` + Server string `json:"server"` + CredentialsFile string `json:"credentialsFile"` + WorkspaceID string `json:"workspaceId"` + LocalRoot string `json:"localRoot"` + RemotePaths []string `json:"remotePaths"` + LocalLayout string `json:"localLayout"` + EventProvider string `json:"eventProvider,omitempty"` + StateFile string `json:"stateFile,omitempty"` + StateDir string `json:"stateDir"` + MountKind string `json:"mountKind"` + Mode string `json:"mode"` + Interval string `json:"interval"` + IntervalJitter float64 `json:"intervalJitter"` + Timeout string `json:"timeout"` + BootstrapTimeout string `json:"bootstrapTimeout"` + BootstrapMaxFilesPerCycle int `json:"bootstrapMaxFilesPerCycle"` + FullPullMinInterval string `json:"fullPullMinInterval"` + CursorTimeout string `json:"cursorTimeout"` + ForceFullReconcile bool `json:"forceFullReconcile"` + WebsocketEnabled bool `json:"websocketEnabled"` + LowMemory bool `json:"lowMemory"` + PprofAddr string `json:"pprofAddr,omitempty"` + MemlogInterval string `json:"memlogInterval"` +} + +type checkpointLifecycleState struct { + Version int `json:"version"` + Kind string `json:"kind"` + ResumeID string `json:"resumeId"` + WorkspaceID string `json:"workspaceId"` + LocalRoot string `json:"localRoot"` + RemoteRoot string `json:"remoteRoot"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + Status string `json:"status"` + Config checkpointMountConfig `json:"config"` + Receipt *mountsync.CheckpointSeal `json:"receipt,omitempty"` + ResumeProof *mountsync.CheckpointSealOwnership `json:"resumeProof,omitempty"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + SealedAt string `json:"sealedAt,omitempty"` + ResumedAt string `json:"resumedAt,omitempty"` + LastError string `json:"lastError,omitempty"` +} + +type checkpointSealEnvelope struct { + Version int `json:"version"` + Kind string `json:"kind"` + Status string `json:"status"` + WorkspaceID string `json:"workspaceId"` + LocalRoot string `json:"localRoot"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + Receipt mountsync.CheckpointSeal `json:"receipt"` + Health mountsync.CheckpointVerificationHealth `json:"health"` + ResumeID string `json:"resumeId"` + SealedAt string `json:"sealedAt"` +} + +type checkpointResumeEnvelope struct { + Version int `json:"version"` + Kind string `json:"kind"` + WorkspaceID string `json:"workspaceId"` + LocalRoot string `json:"localRoot"` + ResumeID string `json:"resumeId"` + Status string `json:"status"` + ResumedAt string `json:"resumedAt"` +} + +type checkpointResumeInput struct { + ResumeID string `json:"resumeId"` +} + +type checkpointVerificationInput struct { + VerificationID string `json:"verificationId"` + Receipt mountsync.CheckpointSeal `json:"receipt"` +} + +type checkpointDestinationVerificationEnvelope struct { + Version int `json:"version"` + Kind string `json:"kind"` + VerificationID string `json:"verificationId"` + WorkspaceID string `json:"workspaceId"` + LocalRoot string `json:"localRoot"` + RemoteRoot string `json:"remoteRoot"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + Status string `json:"status"` + Observed mountsync.CheckpointObservedState `json:"observed"` + Health mountsync.CheckpointVerificationHealth `json:"health"` + VerifiedAt string `json:"verifiedAt"` +} + +type checkpointHandbackInput struct { + HandbackID string `json:"handbackId"` + ConsumerIdempotencyKey string `json:"consumerIdempotencyKey"` + Receipt mountsync.CheckpointSeal `json:"receipt"` +} + +type checkpointHandbackEnvelope struct { + Version int `json:"version"` + Kind string `json:"kind"` + HandbackID string `json:"handbackId"` + WorkspaceID string `json:"workspaceId"` + LocalRoot string `json:"localRoot"` + RemoteRoot string `json:"remoteRoot"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + Status string `json:"status"` + Proof mountsync.CheckpointSealOwnership `json:"proof"` + Health mountsync.CheckpointVerificationHealth `json:"health"` + ReleasedAt string `json:"releasedAt"` +} + +type checkpointHandbackLifecycle struct { + Version int `json:"version"` + Kind string `json:"kind"` + HandbackID string `json:"handbackId"` + ConsumerIdempotencyKey string `json:"consumerIdempotencyKey"` + WorkspaceID string `json:"workspaceId"` + LocalRoot string `json:"localRoot"` + RemoteRoot string `json:"remoteRoot"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + Status string `json:"status"` + Config checkpointMountConfig `json:"config"` + Receipt mountsync.CheckpointSeal `json:"receipt"` + Result *checkpointHandbackEnvelope `json:"result,omitempty"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + LastError string `json:"lastError,omitempty"` +} + +type checkpointVerificationLifecycle struct { + Version int `json:"version"` + Kind string `json:"kind"` + VerificationID string `json:"verificationId"` + WorkspaceID string `json:"workspaceId"` + LocalRoot string `json:"localRoot"` + RemoteRoot string `json:"remoteRoot"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + Status string `json:"status"` + Config checkpointMountConfig `json:"config"` + Receipt mountsync.CheckpointSeal `json:"receipt"` + Result *checkpointDestinationVerificationEnvelope `json:"result,omitempty"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + LastError string `json:"lastError,omitempty"` +} + +type activeCheckpointMount struct { + record workspaceRecord + pid daemonPIDState + config checkpointMountConfig +} + +type checkpointLease interface { + Release() error +} + +var ( + checkpointStartMount = startCheckpointMountProcess + checkpointResolveActive = resolveActiveCheckpointMount + checkpointStopActive = stopCheckpointMount + checkpointIssueStopped = issueCheckpointForStoppedMount + checkpointVerifyStopped = verifyCheckpointForStoppedMount + checkpointHandbackStopped = handbackCheckpointForStoppedMount + checkpointEnsureSource = ensureCheckpointSourceReady + checkpointBurnReceipt = burnCheckpointReceiptForResume + checkpointWaitMountReady = waitCheckpointMountReady + checkpointWaitSourceProof = waitCheckpointSourceProof +) + +func runMountCheckpointSeal(args []string, stdout io.Writer) error { + fs := flag.NewFlagSet("mount checkpoint-seal", flag.ContinueOnError) + fs.SetOutput(io.Discard) + localRoot := fs.String("root", "", "absolute local mount root") + lifecycleID := fs.String("lifecycle-id", "", "stable controller-persisted cutover lifecycle id") + sessionID := fs.String("session", "", "live session identifier") + generation := fs.Uint64("generation", 0, "strictly increasing migration generation") + timeout := fs.Duration("timeout", 30*time.Second, "checkpoint deadline") + ttl := fs.Duration("ttl", mountsync.DefaultCheckpointSealTTL, "server receipt TTL") + jsonOutput := fs.Bool("json", false, "emit the machine contract") + if err := fs.Parse(args); err != nil { + return checkpointError("checkpoint_invalid_input", 2, err) + } + root, err := normalizeAbsoluteLocalRoot(*localRoot) + if err != nil || !checkpointLifecycleIDPattern.MatchString(strings.TrimSpace(*lifecycleID)) || !checkpointLifecycleIDPattern.MatchString(strings.TrimSpace(*sessionID)) || *generation == 0 || *timeout <= 0 || *ttl < time.Second || *ttl > mountsync.MaxCheckpointSealTTL || !*jsonOutput || fs.NArg() != 0 { + return checkpointError("checkpoint_invalid_input", 2, errors.New("--root, --lifecycle-id, --session, --generation, and --json are required; timeout/ttl must be positive and ttl <= 5m")) + } + release, err := acquireCheckpointLifecycleLock(root) + if err != nil { + return checkpointError("checkpoint_lifecycle_conflict", 3, err) + } + defer release() + lifecycleCtx, stopSignals := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stopSignals() + + controllerLifecycleID := strings.TrimSpace(*lifecycleID) + if existing, ok, err := loadCheckpointLifecycleIfExists(controllerLifecycleID); err != nil { + return checkpointError("checkpoint_lifecycle_state_invalid", 3, err) + } else if ok { + if !sameCheckpointLocalRoot(existing.LocalRoot, root) || existing.SessionID != strings.TrimSpace(*sessionID) || existing.Generation != *generation { + return checkpointError("checkpoint_lifecycle_identity_conflict", 3, errors.New("lifecycle-id is already bound to a different root/session/generation")) + } + if existing.Status == "sealed" && existing.Receipt != nil { + return writeJSON(stdout, checkpointEnvelopeFromState(existing)) + } + if existing.Status == "preparing" || existing.Status == "stopped" { + recoveryErr := checkpointEnsureSource(existing.Config, checkpointResumeTimeout) + existing.LastError = "recovered interrupted pre-seal lifecycle" + if recoveryErr != nil { + existing.Status = "recovery-failed" + existing.LastError += ": " + recoveryErr.Error() + _ = saveCheckpointLifecycle(existing) + return checkpointError("checkpoint_source_recovery_failed", 4, errors.New(existing.LastError)) + } + existing.Status = "preseal-failed-source-ready" + existing.ResumedAt = time.Now().UTC().Format(time.RFC3339Nano) + _ = saveCheckpointLifecycle(existing) + return checkpointError("checkpoint_lifecycle_interrupted", 3, errors.New("interrupted pre-seal attempt was recovered; use a newer generation and lifecycle-id")) + } + return checkpointError("checkpoint_lifecycle_terminal", 3, fmt.Errorf("lifecycle-id already has terminal state %q", existing.Status)) + } + if existing, ok, err := findCheckpointLifecycle(root, strings.TrimSpace(*sessionID), *generation); err != nil { + return checkpointError("checkpoint_lifecycle_state_invalid", 3, err) + } else if ok { + return checkpointError("checkpoint_lifecycle_identity_conflict", 3, fmt.Errorf("root/session/generation is already bound to lifecycle-id %q", existing.ResumeID)) + } + if err := rejectStaleCheckpointGeneration(root, strings.TrimSpace(*sessionID), *generation); err != nil { + return checkpointError("checkpoint_generation_stale", 3, err) + } + + active, err := checkpointResolveActive(root) + if err != nil { + return err + } + now := time.Now().UTC() + state := checkpointLifecycleState{ + Version: checkpointLifecycleVersion, Kind: "relayfile-checkpoint-lifecycle", + ResumeID: controllerLifecycleID, WorkspaceID: active.config.WorkspaceID, + LocalRoot: root, RemoteRoot: mountscope.FirstPath(active.config.RemotePaths, "/"), + SessionID: strings.TrimSpace(*sessionID), Generation: *generation, + Status: "preparing", Config: active.config, + CreatedAt: now.Format(time.RFC3339Nano), UpdatedAt: now.Format(time.RFC3339Nano), + } + if err := saveCheckpointLifecycle(state); err != nil { + return checkpointError("checkpoint_lifecycle_state_failed", 4, err) + } + + stopCtx, stopCancel := context.WithTimeout(lifecycleCtx, checkpointStopTimeout) + lease, stopErr := checkpointStopActive(stopCtx, active) + stopCancel() + if stopErr != nil { + state.LastError = stopErr.Error() + if recoveryErr := checkpointEnsureSource(state.Config, checkpointResumeTimeout); recoveryErr != nil { + state.Status = "recovery-failed" + state.LastError += "; source recovery: " + recoveryErr.Error() + _ = saveCheckpointLifecycle(state) + return checkpointError("checkpoint_source_recovery_failed", 4, errors.New(state.LastError)) + } + state.Status = "preseal-failed-source-ready" + state.ResumedAt = time.Now().UTC().Format(time.RFC3339Nano) + _ = saveCheckpointLifecycle(state) + return checkpointError("checkpoint_daemon_stop_failed", 4, stopErr) + } + state.Status = "stopped" + _ = saveCheckpointLifecycle(state) + + checkpointCtx, checkpointCancel := context.WithTimeout(lifecycleCtx, *timeout) + receipt, checkpointErr := checkpointIssueStopped(checkpointCtx, state.Config, state.SessionID, state.Generation, int(ttl.Seconds())) + checkpointCancel() + _ = lease.Release() + if checkpointErr != nil { + recoveryErr := checkpointEnsureSource(state.Config, checkpointResumeTimeout) + state.LastError = checkpointErr.Error() + if recoveryErr != nil { + state.Status = "recovery-failed" + state.LastError += "; source recovery: " + recoveryErr.Error() + _ = saveCheckpointLifecycle(state) + return checkpointError("checkpoint_source_recovery_failed", 4, errors.New(state.LastError)) + } + state.Status = "preseal-failed-source-ready" + state.ResumedAt = time.Now().UTC().Format(time.RFC3339Nano) + _ = saveCheckpointLifecycle(state) + return checkpointError("checkpoint_nonconverged", 5, checkpointErr) + } + state.Status = "sealed" + state.Receipt = &receipt + state.SealedAt = time.Now().UTC().Format(time.RFC3339Nano) + state.UpdatedAt = state.SealedAt + if err := saveCheckpointLifecycle(state); err != nil { + // A server seal exists but was not durably handed off. Restarting the + // source makes that undisclosed receipt stale before returning failure. + recoveryErr := checkpointEnsureSource(state.Config, checkpointResumeTimeout) + if recoveryErr != nil { + return checkpointError("checkpoint_source_recovery_failed", 4, fmt.Errorf("persist sealed lifecycle: %v; source recovery: %w", err, recoveryErr)) + } + return checkpointError("checkpoint_lifecycle_state_failed", 4, err) + } + return writeJSON(stdout, checkpointEnvelopeFromState(state)) +} + +func runMountResumeSeal(args []string, stdin io.Reader, stdout io.Writer) error { + fs := flag.NewFlagSet("mount resume-seal", flag.ContinueOnError) + fs.SetOutput(io.Discard) + localRoot := fs.String("root", "", "absolute local mount root") + timeout := fs.Duration("timeout", checkpointResumeTimeout, "resume readiness deadline") + jsonOutput := fs.Bool("json", false, "emit the machine contract") + if err := fs.Parse(args); err != nil { + return checkpointError("resume_invalid_input", 2, err) + } + root, err := normalizeAbsoluteLocalRoot(*localRoot) + if err != nil || *timeout <= 0 || !*jsonOutput || fs.NArg() != 0 { + return checkpointError("resume_invalid_input", 2, errors.New("--root and --json are required and timeout must be positive")) + } + var input checkpointResumeInput + if err := decodeStrictCheckpointInput(stdin, &input); err != nil || !checkpointLifecycleIDPattern.MatchString(strings.TrimSpace(input.ResumeID)) { + return checkpointError("resume_invalid_input", 2, errors.New("stdin must contain exactly one JSON object with a valid resumeId")) + } + lockCtx, lockCancel := context.WithTimeout(context.Background(), *timeout) + release, err := acquireCheckpointLifecycleLockWait(lockCtx, root) + lockCancel() + if err != nil { + return checkpointError("checkpoint_lifecycle_conflict", 3, err) + } + defer release() + state, err := loadCheckpointLifecycle(strings.TrimSpace(input.ResumeID)) + if err != nil { + return checkpointError("resume_not_found", 3, err) + } + if !sameCheckpointLocalRoot(state.LocalRoot, root) { + return checkpointError("resume_root_mismatch", 2, errors.New("resumeId is not bound to the requested local root")) + } + if state.Status == "ready" { + if err := checkpointWaitMountReady(state.Config, *timeout); err != nil { + return checkpointError("resume_readiness_failed", 4, err) + } + return writeJSON(stdout, resumeEnvelopeFromState(state)) + } + if state.Status == "preparing" || state.Status == "stopped" || state.Status == "preseal-failed-source-ready" { + if err := checkpointEnsureSource(state.Config, *timeout); err != nil { + state.Status = "recovery-failed" + state.LastError = err.Error() + _ = saveCheckpointLifecycle(state) + return checkpointError("resume_readiness_failed", 4, err) + } + state.Status = "ready" + state.ResumedAt = time.Now().UTC().Format(time.RFC3339Nano) + state.LastError = "" + if err := saveCheckpointLifecycle(state); err != nil { + return checkpointError("resume_lifecycle_state_failed", 4, err) + } + return writeJSON(stdout, resumeEnvelopeFromState(state)) + } + if state.Status != "sealed" && state.Status != "resuming" { + return checkpointError("resume_lifecycle_terminal", 3, fmt.Errorf("lifecycle state %q cannot be resumed", state.Status)) + } + if state.Receipt == nil || state.Receipt.SealToken == "" || state.Receipt.Root != state.RemoteRoot { + return checkpointError("resume_lifecycle_state_invalid", 3, errors.New("sealed lifecycle is missing its remote receipt")) + } + state.Status = "resuming" + if err := saveCheckpointLifecycle(state); err != nil { + return checkpointError("resume_lifecycle_state_failed", 4, err) + } + var proof mountsync.CheckpointSealOwnership + if state.ResumeProof != nil { + proof = *state.ResumeProof + } else { + proof, err = checkpointBurnReceipt(state, *timeout) + if err != nil { + var httpErr *mountsync.HTTPError + if errors.As(err, &httpErr) && httpErr.Code == "checkpoint_handback_required" { + state.Status = "sealed" + state.LastError = err.Error() + _ = saveCheckpointLifecycle(state) + return checkpointError("resume_handback_required", 3, err) + } + if errors.As(err, &httpErr) && (httpErr.Code == "checkpoint_resume_conflict" || httpErr.Code == "checkpoint_replayed") { + state.Status = "ownership-conflict" + state.LastError = err.Error() + _ = saveCheckpointLifecycle(state) + return checkpointError("resume_ownership_conflict", 3, err) + } + state.LastError = err.Error() + _ = saveCheckpointLifecycle(state) + return checkpointError("resume_receipt_burn_failed", 4, err) + } + if err := validateSourceResumeProof(state, proof); err != nil { + state.Status = "ownership-conflict" + state.LastError = err.Error() + _ = saveCheckpointLifecycle(state) + return checkpointError("resume_ownership_conflict", 3, err) + } + state.ResumeProof = &proof + state.LastError = "" + if err := saveCheckpointLifecycle(state); err != nil { + return checkpointError("resume_lifecycle_state_failed", 4, fmt.Errorf("persist source-resume proof before restart: %w", err)) + } + } + if err := checkpointEnsureSource(state.Config, *timeout); err != nil { + state.LastError = err.Error() + _ = saveCheckpointLifecycle(state) + return checkpointError("resume_readiness_failed", 4, err) + } + if err := checkpointWaitSourceProof(state.Config, proof, *timeout); err != nil { + state.LastError = err.Error() + _ = saveCheckpointLifecycle(state) + return checkpointError("resume_nonconverged", 5, err) + } + state.Status = "ready" + state.ResumedAt = time.Now().UTC().Format(time.RFC3339Nano) + state.UpdatedAt = state.ResumedAt + state.LastError = "" + state.Receipt.SealToken = "" + if err := saveCheckpointLifecycle(state); err != nil { + return checkpointError("resume_lifecycle_state_failed", 4, err) + } + return writeJSON(stdout, resumeEnvelopeFromState(state)) +} + +func runMountVerifySeal(args []string, stdin io.Reader, stdout io.Writer) error { + fs := flag.NewFlagSet("mount verify-seal", flag.ContinueOnError) + fs.SetOutput(io.Discard) + localRoot := fs.String("root", "", "absolute local destination mount root") + timeout := fs.Duration("timeout", checkpointResumeTimeout, "verification and recovery deadline") + jsonOutput := fs.Bool("json", false, "emit the machine contract") + if err := fs.Parse(args); err != nil { + return checkpointError("verification_invalid_input", 2, err) + } + root, err := normalizeAbsoluteLocalRoot(*localRoot) + if err != nil || *timeout <= 0 || !*jsonOutput || fs.NArg() != 0 { + return checkpointError("verification_invalid_input", 2, errors.New("--root and --json are required and timeout must be positive")) + } + var input checkpointVerificationInput + if err := decodeStrictCheckpointInput(stdin, &input); err != nil || !checkpointLifecycleIDPattern.MatchString(strings.TrimSpace(input.VerificationID)) { + return checkpointError("verification_invalid_input", 2, errors.New("stdin must contain only a valid verificationId and consumed receipt")) + } + input.VerificationID = strings.TrimSpace(input.VerificationID) + if err := validateDestinationReceipt(input.Receipt); err != nil { + return checkpointError("verification_invalid_input", 2, err) + } + + lockCtx, lockCancel := context.WithTimeout(context.Background(), *timeout) + release, err := acquireCheckpointLifecycleLockWait(lockCtx, root) + lockCancel() + if err != nil { + return checkpointError("verification_lifecycle_conflict", 3, err) + } + defer release() + + if existing, ok, err := loadCheckpointVerificationIfExists(input.VerificationID); err != nil { + return checkpointError("verification_lifecycle_state_invalid", 3, err) + } else if ok { + if !sameCheckpointLocalRoot(existing.LocalRoot, root) || !sameDestinationReceipt(existing.Receipt, input.Receipt) { + return checkpointError("verification_identity_conflict", 3, errors.New("verificationId is already bound to another root or receipt")) + } + switch existing.Status { + case "converged": + if existing.Result == nil { + return checkpointError("verification_lifecycle_state_invalid", 3, errors.New("converged verification has no result")) + } + if err := checkpointWaitMountReady(existing.Config, *timeout); err != nil { + return checkpointError("verification_recovery_failed", 4, err) + } + return writeJSON(stdout, *existing.Result) + case "verified": + if existing.Result == nil { + return checkpointError("verification_lifecycle_state_invalid", 3, errors.New("verified lifecycle has no result")) + } + if err := checkpointEnsureSource(existing.Config, *timeout); err != nil { + existing.Status = "recovery-failed" + existing.LastError = err.Error() + _ = saveCheckpointVerification(existing) + return checkpointError("verification_recovery_failed", 4, err) + } + existing.Status = "converged" + existing.LastError = "" + if err := saveCheckpointVerification(existing); err != nil { + return checkpointError("verification_lifecycle_state_failed", 4, err) + } + return writeJSON(stdout, *existing.Result) + case "preparing": + if err := checkpointEnsureSource(existing.Config, *timeout); err != nil { + return checkpointError("verification_recovery_failed", 4, err) + } + case "stopped", "verifying": + return continueStoppedDestinationVerification(existing, *timeout, stdout) + case "diverged-source-ready": + return checkpointError("verification_nonconverged", 5, errors.New(existing.LastError)) + default: + return checkpointError("verification_lifecycle_terminal", 3, fmt.Errorf("verificationId has state %q", existing.Status)) + } + } + + active, err := checkpointResolveActive(root) + if err != nil { + return err + } + if active.config.WorkspaceID != input.Receipt.WorkspaceID || mountscope.FirstPath(active.config.RemotePaths, "/") != input.Receipt.Root { + return checkpointError("verification_identity_conflict", 3, errors.New("active destination mount does not match consumed receipt")) + } + now := time.Now().UTC().Format(time.RFC3339Nano) + state := checkpointVerificationLifecycle{ + Version: checkpointLifecycleVersion, Kind: "relayfile-destination-verification-lifecycle", + VerificationID: input.VerificationID, WorkspaceID: input.Receipt.WorkspaceID, + LocalRoot: root, RemoteRoot: input.Receipt.Root, SessionID: input.Receipt.SessionID, Generation: input.Receipt.Generation, + Status: "preparing", Config: active.config, Receipt: input.Receipt, CreatedAt: now, UpdatedAt: now, + } + if err := saveCheckpointVerification(state); err != nil { + return checkpointError("verification_lifecycle_state_failed", 4, err) + } + stopCtx, stopCancel := context.WithTimeout(context.Background(), checkpointStopTimeout) + lease, stopErr := checkpointStopActive(stopCtx, active) + stopCancel() + if stopErr != nil { + return recoverDestinationAfterVerificationFailure(state, "verification_daemon_stop_failed", 4, stopErr, *timeout) + } + state.Status = "stopped" + if err := saveCheckpointVerification(state); err != nil { + _ = lease.Release() + return recoverDestinationAfterVerificationFailure(state, "verification_lifecycle_state_failed", 4, err, *timeout) + } + return verifyAndRecoverDestination(state, lease, *timeout, stdout) +} + +func runMountHandbackSeal(args []string, stdin io.Reader, stdout io.Writer) error { + fs := flag.NewFlagSet("mount handback-seal", flag.ContinueOnError) + fs.SetOutput(io.Discard) + localRoot := fs.String("root", "", "absolute local destination mount root") + timeout := fs.Duration("timeout", checkpointResumeTimeout, "final drain and handback deadline") + jsonOutput := fs.Bool("json", false, "emit the machine contract") + if err := fs.Parse(args); err != nil { + return checkpointError("handback_invalid_input", 2, err) + } + root, err := normalizeAbsoluteLocalRoot(*localRoot) + if err != nil || *timeout <= 0 || !*jsonOutput || fs.NArg() != 0 { + return checkpointError("handback_invalid_input", 2, errors.New("--root and --json are required and timeout must be positive")) + } + var input checkpointHandbackInput + if err := decodeStrictCheckpointInput(stdin, &input); err != nil || + !checkpointLifecycleIDPattern.MatchString(strings.TrimSpace(input.HandbackID)) || !checkpointLifecycleIDPattern.MatchString(strings.TrimSpace(input.ConsumerIdempotencyKey)) || validateDestinationReceipt(input.Receipt) != nil { + return checkpointError("handback_invalid_input", 2, errors.New("stdin must contain only handbackId, original consumerIdempotencyKey, and a consumed full-root receipt")) + } + input.HandbackID = strings.TrimSpace(input.HandbackID) + input.ConsumerIdempotencyKey = strings.TrimSpace(input.ConsumerIdempotencyKey) + lockCtx, lockCancel := context.WithTimeout(context.Background(), *timeout) + release, err := acquireCheckpointLifecycleLockWait(lockCtx, root) + lockCancel() + if err != nil { + return checkpointError("handback_lifecycle_conflict", 3, err) + } + defer release() + + if existing, ok, err := loadCheckpointHandbackIfExists(input.HandbackID); err != nil { + return checkpointError("handback_lifecycle_state_invalid", 3, err) + } else if ok { + if !sameCheckpointLocalRoot(existing.LocalRoot, root) || existing.ConsumerIdempotencyKey != input.ConsumerIdempotencyKey || !sameDestinationReceipt(existing.Receipt, input.Receipt) { + return checkpointError("handback_identity_conflict", 3, errors.New("handbackId is already bound to another root, consumer, or receipt")) + } + if existing.Status == "released" { + if existing.Result == nil { + return checkpointError("handback_lifecycle_state_invalid", 3, errors.New("released handback has no result")) + } + return writeJSON(stdout, *existing.Result) + } + if existing.Status == "failed-destination-ready" { + return checkpointError("handback_nonconverged", 5, errors.New(existing.LastError)) + } + if existing.Status != "preparing" && existing.Status != "stopped" && existing.Status != "handing-back" && existing.Status != "handback-unknown" { + return checkpointError("handback_lifecycle_terminal", 3, fmt.Errorf("handbackId has state %q", existing.Status)) + } + lease, err := acquireStoppedHandbackLease(existing) + if err != nil { + return checkpointError("handback_destination_stop_failed", 4, err) + } + existing.Status = "stopped" + if err := saveCheckpointHandback(existing); err != nil { + _ = lease.Release() + return checkpointError("handback_lifecycle_state_failed", 4, err) + } + return finishCheckpointHandback(existing, lease, *timeout, stdout) + } + + active, err := checkpointResolveActive(root) + if err != nil { + return err + } + if active.config.WorkspaceID != input.Receipt.WorkspaceID || mountscope.FirstPath(active.config.RemotePaths, "/") != input.Receipt.Root { + return checkpointError("handback_identity_conflict", 3, errors.New("active destination mount does not match consumed receipt")) + } + now := time.Now().UTC().Format(time.RFC3339Nano) + state := checkpointHandbackLifecycle{ + Version: checkpointLifecycleVersion, Kind: "relayfile-checkpoint-handback-lifecycle", + HandbackID: input.HandbackID, ConsumerIdempotencyKey: input.ConsumerIdempotencyKey, + WorkspaceID: input.Receipt.WorkspaceID, LocalRoot: root, RemoteRoot: input.Receipt.Root, + SessionID: input.Receipt.SessionID, Generation: input.Receipt.Generation, + Status: "preparing", Config: active.config, Receipt: input.Receipt, CreatedAt: now, UpdatedAt: now, + } + if err := saveCheckpointHandback(state); err != nil { + return checkpointError("handback_lifecycle_state_failed", 4, err) + } + stopCtx, stopCancel := context.WithTimeout(context.Background(), checkpointStopTimeout) + lease, stopErr := checkpointStopActive(stopCtx, active) + stopCancel() + if stopErr != nil { + state.LastError = stopErr.Error() + if recoveryErr := checkpointEnsureSource(state.Config, *timeout); recoveryErr != nil { + state.Status = "recovery-failed" + state.LastError += "; destination recovery: " + recoveryErr.Error() + _ = saveCheckpointHandback(state) + return checkpointError("handback_destination_recovery_failed", 4, errors.New(state.LastError)) + } + state.Status = "failed-destination-ready" + _ = saveCheckpointHandback(state) + return checkpointError("handback_destination_stop_failed", 4, stopErr) + } + state.Status = "stopped" + if err := saveCheckpointHandback(state); err != nil { + _ = lease.Release() + _ = checkpointEnsureSource(state.Config, *timeout) + return checkpointError("handback_lifecycle_state_failed", 4, err) + } + return finishCheckpointHandback(state, lease, *timeout, stdout) +} + +func acquireStoppedHandbackLease(state checkpointHandbackLifecycle) (checkpointLease, error) { + lease, err := mountlease.Acquire(state.Config.Server, state.Config.WorkspaceID, state.Config.LocalRoot) + if err == nil { + return lease, nil + } + if state.Status != "preparing" { + return nil, fmt.Errorf("destination mount lease is not available while handback is %s: %w", state.Status, err) + } + active, resolveErr := checkpointResolveActive(state.LocalRoot) + if resolveErr != nil { + return nil, fmt.Errorf("resolve interrupted destination: %w", resolveErr) + } + ctx, cancel := context.WithTimeout(context.Background(), checkpointStopTimeout) + defer cancel() + return checkpointStopActive(ctx, active) +} + +func finishCheckpointHandback(state checkpointHandbackLifecycle, lease checkpointLease, timeout time.Duration, stdout io.Writer) error { + state.Status = "handing-back" + if err := saveCheckpointHandback(state); err != nil { + _ = lease.Release() + return checkpointError("handback_lifecycle_state_failed", 4, err) + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + proof, health, handbackErr := checkpointHandbackStopped(ctx, state.Config, state.Receipt, state.ConsumerIdempotencyKey, state.HandbackID) + cancel() + if handbackErr != nil { + _ = lease.Release() + state.LastError = handbackErr.Error() + if definitiveCheckpointHandbackHTTPFailure(handbackErr) { + if recoveryErr := checkpointEnsureSource(state.Config, timeout); recoveryErr != nil { + state.Status = "recovery-failed" + state.LastError += "; destination recovery: " + recoveryErr.Error() + _ = saveCheckpointHandback(state) + return checkpointError("handback_destination_recovery_failed", 4, errors.New(state.LastError)) + } + state.Status = "failed-destination-ready" + _ = saveCheckpointHandback(state) + return checkpointError("handback_nonconverged", 5, handbackErr) + } + // A transport failure or exhausted retryable HTTP response after POST is + // ambiguous: the application may have committed before a proxy/gateway + // emitted 5xx. Leave the destination stopped and require an exact retry + // with the same handbackId; never restart into split-brain. + state.Status = "handback-unknown" + if err := saveCheckpointHandback(state); err != nil { + return checkpointError("handback_lifecycle_state_failed", 4, fmt.Errorf("persist ambiguous handback while destination remains stopped: %w", err)) + } + return checkpointError("handback_result_unknown", 4, handbackErr) + } + result := checkpointHandbackEnvelope{ + Version: checkpointLifecycleVersion, Kind: "relayfile-checkpoint-handback", HandbackID: state.HandbackID, + WorkspaceID: state.WorkspaceID, LocalRoot: state.LocalRoot, RemoteRoot: state.RemoteRoot, + SessionID: state.SessionID, Generation: state.Generation, Status: "released", + Proof: proof, Health: health, ReleasedAt: proof.ReleasedAt, + } + state.Status = "released" + state.Result = &result + state.LastError = "" + if err := saveCheckpointHandback(state); err != nil { + _ = lease.Release() + return checkpointError("handback_lifecycle_state_failed", 4, err) + } + if err := lease.Release(); err != nil { + return checkpointError("handback_lease_release_failed", 4, err) + } + return writeJSON(stdout, result) +} + +func definitiveCheckpointHandbackHTTPFailure(err error) bool { + var httpErr *mountsync.HTTPError + if !errors.As(err, &httpErr) { + return false + } + code := strings.TrimSpace(httpErr.Code) + // checkpoint_diverged is emitted by the authoritative store while it still + // owns the mutation lock and before ownership can be released. Authentication, + // request-shape/size, gateway, and generic HTTP responses can instead be the + // exhausted retry observed after an earlier POST committed but lost its + // response, so none of those responses is safe evidence for restart. + return httpErr.StatusCode == 409 && code == "checkpoint_diverged" +} + +func continueStoppedDestinationVerification(state checkpointVerificationLifecycle, timeout time.Duration, stdout io.Writer) error { + acquiredLease, err := mountlease.Acquire(state.Config.Server, state.Config.WorkspaceID, state.Config.LocalRoot) + var lease checkpointLease + if err == nil { + lease = acquiredLease + } + if err != nil { + if readyErr := checkpointWaitMountReady(state.Config, 500*time.Millisecond); readyErr == nil { + active, resolveErr := checkpointResolveActive(state.LocalRoot) + if resolveErr != nil { + return checkpointError("verification_recovery_failed", 4, resolveErr) + } + ctx, cancel := context.WithTimeout(context.Background(), checkpointStopTimeout) + lease, err = checkpointStopActive(ctx, active) + cancel() + } + } + if err != nil { + return recoverDestinationAfterVerificationFailure(state, "verification_recovery_failed", 4, err, timeout) + } + return verifyAndRecoverDestination(state, lease, timeout, stdout) +} + +func verifyAndRecoverDestination(state checkpointVerificationLifecycle, lease checkpointLease, timeout time.Duration, stdout io.Writer) error { + state.Status = "verifying" + _ = saveCheckpointVerification(state) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + verification, verifyErr := checkpointVerifyStopped(ctx, state.Config, state.Receipt) + cancel() + if verifyErr != nil { + _ = lease.Release() + return recoverDestinationAfterVerificationFailure(state, "verification_nonconverged", 5, verifyErr, timeout) + } + verifiedAt := time.Now().UTC().Format(time.RFC3339Nano) + result := checkpointDestinationVerificationEnvelope{ + Version: checkpointLifecycleVersion, Kind: "relayfile-destination-verification", VerificationID: state.VerificationID, + WorkspaceID: state.WorkspaceID, LocalRoot: state.LocalRoot, RemoteRoot: state.RemoteRoot, + SessionID: state.SessionID, Generation: state.Generation, Status: "converged", + Observed: verification.Observed, Health: verification.Health, VerifiedAt: verifiedAt, + } + state.Status = "verified" + state.Result = &result + state.LastError = "" + if err := saveCheckpointVerification(state); err != nil { + _ = lease.Release() + return recoverDestinationAfterVerificationFailure(state, "verification_lifecycle_state_failed", 4, err, timeout) + } + releaseErr := lease.Release() + if err := checkpointEnsureSource(state.Config, timeout); err != nil { + if releaseErr != nil { + err = fmt.Errorf("release verification lease: %v; destination recovery: %w", releaseErr, err) + } + state.Status = "recovery-failed" + state.LastError = err.Error() + _ = saveCheckpointVerification(state) + return checkpointError("verification_recovery_failed", 4, err) + } + if releaseErr != nil { + return checkpointError("verification_recovery_failed", 4, releaseErr) + } + state.Status = "converged" + if err := saveCheckpointVerification(state); err != nil { + return checkpointError("verification_lifecycle_state_failed", 4, err) + } + return writeJSON(stdout, result) +} + +func recoverDestinationAfterVerificationFailure(state checkpointVerificationLifecycle, code string, exitCode int, cause error, timeout time.Duration) error { + state.LastError = cause.Error() + if recoveryErr := checkpointEnsureSource(state.Config, timeout); recoveryErr != nil { + state.Status = "recovery-failed" + state.LastError += "; destination recovery: " + recoveryErr.Error() + _ = saveCheckpointVerification(state) + return checkpointError("verification_recovery_failed", 4, errors.New(state.LastError)) + } + if exitCode == 5 { + state.Status = "diverged-source-ready" + } else { + state.Status = "failed-source-ready" + } + _ = saveCheckpointVerification(state) + return checkpointError(code, exitCode, cause) +} + +func normalizeAbsoluteLocalRoot(raw string) (string, error) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" || !filepath.IsAbs(trimmed) { + return "", errors.New("local root must be absolute") + } + cleaned := filepath.Clean(trimmed) + info, err := os.Stat(cleaned) + if err != nil { + return "", fmt.Errorf("local root must be an existing directory: %w", err) + } + if !info.IsDir() { + return "", errors.New("local root must be an existing directory") + } + resolved, err := filepath.EvalSymlinks(cleaned) + if err != nil { + return "", fmt.Errorf("resolve local root: %w", err) + } + return filepath.Clean(resolved), nil +} + +func resolveActiveCheckpointMount(localRoot string) (activeCheckpointMount, error) { + catalog, err := loadWorkspaceCatalog() + if err != nil { + return activeCheckpointMount{}, checkpointError("checkpoint_catalog_invalid", 2, err) + } + var matches []workspaceRecord + for _, record := range catalog.Workspaces { + if sameCheckpointLocalRoot(record.LocalDir, localRoot) { + matches = append(matches, record) + } + } + if len(matches) != 1 { + return activeCheckpointMount{}, checkpointError("checkpoint_catalog_mismatch", 2, fmt.Errorf("expected one catalog workspace for local root, found %d", len(matches))) + } + record := matches[0] + pidState, structured := readDaemonPIDState(localRoot) + if !structured || !pidState.Registered || pidState.CheckpointConfig == nil { + return activeCheckpointMount{}, checkpointError("checkpoint_restart_contract_missing", 2, errors.New("active daemon lacks a registered non-secret checkpoint restart contract")) + } + pid, verified := verifyDaemonProcess(localRoot, record.ID) + if !verified || pid != pidState.PID || !processAlive(pid) { + return activeCheckpointMount{}, checkpointError("checkpoint_daemon_identity_invalid", 2, errors.New("registered PID does not identify the active workspace daemon")) + } + running, stalePID, err := runningMountDaemons(localRoot, record.ID, record.Name) + if err != nil || stalePID != 0 || len(running) != 1 || running[0].PID != pid { + return activeCheckpointMount{}, checkpointError("checkpoint_daemon_identity_invalid", 2, fmt.Errorf("active daemon discovery was not unique: running=%d stalePid=%d err=%v", len(running), stalePID, err)) + } + config := *pidState.CheckpointConfig + if err := validateCheckpointMountConfig(config, record, localRoot); err != nil { + return activeCheckpointMount{}, err + } + if err := waitCheckpointMountReady(config, 2*time.Second); err != nil { + return activeCheckpointMount{}, checkpointError("checkpoint_source_not_ready", 4, err) + } + return activeCheckpointMount{record: record, pid: pidState, config: config}, nil +} + +func validateCheckpointMountConfig(config checkpointMountConfig, record workspaceRecord, localRoot string) error { + if config.Version != checkpointLifecycleVersion || config.Mode != defaultMountMode { + if strings.EqualFold(config.Mode, "fuse") { + return checkpointError("checkpoint_fuse_unsupported", 2, errors.New("FUSE checkpoint requires daemon IPC and is not available")) + } + return checkpointError("checkpoint_restart_contract_invalid", 2, errors.New("unsupported restart contract version or mode")) + } + if config.LocalLayout != mountscope.LayoutExact || len(mountscope.NormalizePaths(config.RemotePaths, "/")) != 1 { + return checkpointError("checkpoint_topology_unsupported", 2, errors.New("live checkpoint v1 requires one exact full-root poll mount")) + } + remoteRoot := mountscope.FirstPath(config.RemotePaths, "/") + if remoteRoot != "/" { + return checkpointError("checkpoint_topology_unsupported", 2, errors.New("live checkpoint v1 supports only the full remote root /")) + } + if config.WorkspaceID != record.ID || !sameCheckpointLocalRoot(config.LocalRoot, localRoot) || remoteRoot != mountscope.FirstPath(record.RemotePaths, "/") || config.Server == "" { + return checkpointError("checkpoint_restart_contract_invalid", 2, errors.New("PID restart identity does not match the workspace catalog")) + } + if config.CredentialsFile == "" || !filepath.IsAbs(config.CredentialsFile) { + return checkpointError("checkpoint_credentials_unrecoverable", 2, errors.New("daemon was not started from a delegated credential file; secret argv cannot be reconstructed safely")) + } + info, err := os.Lstat(config.CredentialsFile) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o077 != 0 { + return checkpointError("checkpoint_credentials_unrecoverable", 2, errors.New("delegated credential file is missing, non-regular, or not private")) + } + for name, raw := range map[string]string{ + "interval": config.Interval, "timeout": config.Timeout, "bootstrapTimeout": config.BootstrapTimeout, + "fullPullMinInterval": config.FullPullMinInterval, "cursorTimeout": config.CursorTimeout, "memlogInterval": config.MemlogInterval, + } { + if _, err := parseCheckpointDuration(raw, name == "fullPullMinInterval"); err != nil { + return checkpointError("checkpoint_restart_contract_invalid", 2, fmt.Errorf("%s: %w", name, err)) + } + } + return nil +} + +func parseCheckpointDuration(raw string, allowNegativeOne bool) (time.Duration, error) { + if allowNegativeOne && strings.TrimSpace(raw) == "-1ns" { + return -1, nil + } + value, err := time.ParseDuration(strings.TrimSpace(raw)) + if err != nil { + return 0, err + } + if value < 0 && !(allowNegativeOne && value == -1) { + return 0, errors.New("negative duration") + } + return value, nil +} + +func stopCheckpointMount(ctx context.Context, active activeCheckpointMount) (checkpointLease, error) { + process, err := os.FindProcess(active.pid.PID) + if err != nil { + return nil, err + } + if err := process.Signal(syscall.SIGTERM); err != nil && !isProcessAlreadyGone(err) { + return nil, err + } + for { + lease, leaseErr := mountlease.Acquire(active.config.Server, active.config.WorkspaceID, active.config.LocalRoot) + if leaseErr == nil && !processAlive(active.pid.PID) { + _ = os.Remove(mountPIDFile(active.config.LocalRoot)) + return lease, nil + } + if leaseErr == nil { + _ = lease.Release() + } + select { + case <-ctx.Done(): + return nil, fmt.Errorf("daemon did not exit and release its mount lease: %w", ctx.Err()) + case <-time.After(50 * time.Millisecond): + } + } +} + +func issueCheckpointForStoppedMount(ctx context.Context, config checkpointMountConfig, sessionID string, generation uint64, ttlSeconds int) (mountsync.CheckpointSeal, error) { + bundle, loadedPath, err := loadDelegatedCredentials(config.CredentialsFile) + if err != nil { + return mountsync.CheckpointSeal{}, err + } + bundle, err = refreshDelegatedCredentials(loadedPath, bundle, false) + if err != nil { + return mountsync.CheckpointSeal{}, err + } + if !workspaceRequestMatchesDelegatedCredentials(config.WorkspaceID, bundle.Workspace()) { + return mountsync.CheckpointSeal{}, errors.New("delegated credential workspace does not match restart contract") + } + interval, _ := parseCheckpointDuration(config.Interval, false) + bootstrapTimeout, _ := parseCheckpointDuration(config.BootstrapTimeout, false) + fullPullMinInterval, _ := parseCheckpointDuration(config.FullPullMinInterval, true) + cursorTimeout, _ := parseCheckpointDuration(config.CursorTimeout, false) + client := mountsync.NewHTTPClient(config.Server, bundle.BearerToken(), mountsync.NewSyncHTTPClient()) + syncer, err := mountsync.NewSyncer(client, mountsync.SyncerOptions{ + WorkspaceID: config.WorkspaceID, RemoteRoot: mountscope.FirstPath(config.RemotePaths, "/"), EventProvider: config.EventProvider, + LocalRoot: config.LocalRoot, StateFile: config.StateFile, StateDir: config.StateDir, MountKind: config.MountKind, + ValidateState: true, Scopes: delegatedBundleAvailableScopes(bundle), RootCtx: ctx, Mode: config.Mode, + Interval: interval, LowMemory: boolPtr(config.LowMemory), BootstrapTimeout: bootstrapTimeout, + BootstrapMaxFilesPerCycle: config.BootstrapMaxFilesPerCycle, FullPullMinInterval: fullPullMinInterval, + CursorTimeout: cursorTimeout, ForceFullReconcile: boolPtr(config.ForceFullReconcile), SyncMode: "mirror", + }) + if err != nil { + return mountsync.CheckpointSeal{}, err + } + return syncer.CheckpointAndSeal(ctx, mountsync.CheckpointAndSealOptions{SessionID: sessionID, Generation: generation, TTLSeconds: ttlSeconds}) +} + +func verifyCheckpointForStoppedMount(ctx context.Context, config checkpointMountConfig, receipt mountsync.CheckpointSeal) (mountsync.CheckpointVerification, error) { + bundle, loadedPath, err := loadDelegatedCredentials(config.CredentialsFile) + if err != nil { + return mountsync.CheckpointVerification{}, err + } + bundle, err = refreshDelegatedCredentials(loadedPath, bundle, false) + if err != nil { + return mountsync.CheckpointVerification{}, err + } + if !workspaceRequestMatchesDelegatedCredentials(config.WorkspaceID, bundle.Workspace()) { + return mountsync.CheckpointVerification{}, errors.New("delegated credential workspace does not match destination restart contract") + } + interval, _ := parseCheckpointDuration(config.Interval, false) + bootstrapTimeout, _ := parseCheckpointDuration(config.BootstrapTimeout, false) + fullPullMinInterval, _ := parseCheckpointDuration(config.FullPullMinInterval, true) + cursorTimeout, _ := parseCheckpointDuration(config.CursorTimeout, false) + client := mountsync.NewHTTPClient(config.Server, bundle.BearerToken(), mountsync.NewSyncHTTPClient()) + syncer, err := mountsync.NewSyncer(client, mountsync.SyncerOptions{ + WorkspaceID: config.WorkspaceID, RemoteRoot: mountscope.FirstPath(config.RemotePaths, "/"), EventProvider: config.EventProvider, + LocalRoot: config.LocalRoot, StateFile: config.StateFile, StateDir: config.StateDir, MountKind: config.MountKind, + ValidateState: true, Scopes: delegatedBundleAvailableScopes(bundle), RootCtx: ctx, Mode: config.Mode, + Interval: interval, LowMemory: boolPtr(config.LowMemory), BootstrapTimeout: bootstrapTimeout, + BootstrapMaxFilesPerCycle: config.BootstrapMaxFilesPerCycle, FullPullMinInterval: fullPullMinInterval, + CursorTimeout: cursorTimeout, ForceFullReconcile: boolPtr(config.ForceFullReconcile), SyncMode: "mirror", + }) + if err != nil { + return mountsync.CheckpointVerification{}, err + } + return syncer.VerifyCheckpoint(ctx, receipt) +} + +func handbackCheckpointForStoppedMount(ctx context.Context, config checkpointMountConfig, receipt mountsync.CheckpointSeal, consumerKey, handbackKey string) (mountsync.CheckpointSealOwnership, mountsync.CheckpointVerificationHealth, error) { + bundle, loadedPath, err := loadDelegatedCredentials(config.CredentialsFile) + if err != nil { + return mountsync.CheckpointSealOwnership{}, mountsync.CheckpointVerificationHealth{}, err + } + bundle, err = refreshDelegatedCredentials(loadedPath, bundle, false) + if err != nil { + return mountsync.CheckpointSealOwnership{}, mountsync.CheckpointVerificationHealth{}, err + } + if !workspaceRequestMatchesDelegatedCredentials(config.WorkspaceID, bundle.Workspace()) { + return mountsync.CheckpointSealOwnership{}, mountsync.CheckpointVerificationHealth{}, errors.New("delegated credential workspace does not match destination handback contract") + } + interval, _ := parseCheckpointDuration(config.Interval, false) + bootstrapTimeout, _ := parseCheckpointDuration(config.BootstrapTimeout, false) + fullPullMinInterval, _ := parseCheckpointDuration(config.FullPullMinInterval, true) + cursorTimeout, _ := parseCheckpointDuration(config.CursorTimeout, false) + client := mountsync.NewHTTPClient(config.Server, bundle.BearerToken(), mountsync.NewSyncHTTPClient()) + syncer, err := mountsync.NewSyncer(client, mountsync.SyncerOptions{ + WorkspaceID: config.WorkspaceID, RemoteRoot: mountscope.FirstPath(config.RemotePaths, "/"), EventProvider: config.EventProvider, + LocalRoot: config.LocalRoot, StateFile: config.StateFile, StateDir: config.StateDir, MountKind: config.MountKind, + ValidateState: true, Scopes: delegatedBundleAvailableScopes(bundle), RootCtx: ctx, Mode: config.Mode, + Interval: interval, LowMemory: boolPtr(config.LowMemory), BootstrapTimeout: bootstrapTimeout, + BootstrapMaxFilesPerCycle: config.BootstrapMaxFilesPerCycle, FullPullMinInterval: fullPullMinInterval, + CursorTimeout: cursorTimeout, ForceFullReconcile: boolPtr(config.ForceFullReconcile), SyncMode: "mirror", + }) + if err != nil { + return mountsync.CheckpointSealOwnership{}, mountsync.CheckpointVerificationHealth{}, err + } + return syncer.HandbackCheckpoint(ctx, receipt, consumerKey, handbackKey) +} + +func ensureCheckpointSourceReady(config checkpointMountConfig, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + started := false + for time.Now().Before(deadline) { + if err := waitCheckpointMountReady(config, 100*time.Millisecond); err == nil { + return nil + } + state, ok := readDaemonPIDState(config.LocalRoot) + if (!ok || !state.Registered || !processAlive(state.PID)) && !started { + remaining := time.Until(deadline) + ctx, cancel := context.WithTimeout(context.Background(), remaining) + err := checkpointStartMount(ctx, config) + cancel() + if err != nil { + return err + } + started = true + } + time.Sleep(50 * time.Millisecond) + } + return fmt.Errorf("source mount did not become ready within %s", timeout) +} + +func startCheckpointMountProcess(ctx context.Context, config checkpointMountConfig) error { + executable := resolvedSelfExecutable() + if executable == "" { + return errors.New("resolve relayfile executable") + } + args := checkpointMountArgs(config) + cmd := exec.CommandContext(ctx, executable, append([]string{"mount"}, args...)...) + cmd.Env = checkpointSubprocessEnv(os.Environ()) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("start source mount: %w: %s", err, strings.TrimSpace(string(output))) + } + return nil +} + +func checkpointMountArgs(config checkpointMountConfig) []string { + args := []string{ + config.WorkspaceID, config.LocalRoot, "--background", + "--server", config.Server, "--creds-file", config.CredentialsFile, + "--local-layout", config.LocalLayout, "--mode", config.Mode, + "--interval", config.Interval, "--interval-jitter", strconv.FormatFloat(config.IntervalJitter, 'g', -1, 64), + "--timeout", config.Timeout, "--bootstrap-timeout", config.BootstrapTimeout, + "--bootstrap-max-files-per-cycle", strconv.Itoa(config.BootstrapMaxFilesPerCycle), + "--full-pull-min-interval", config.FullPullMinInterval, "--cursor-timeout", config.CursorTimeout, + "--websocket=" + strconv.FormatBool(config.WebsocketEnabled), "--low-memory=" + strconv.FormatBool(config.LowMemory), + "--memlog-interval", config.MemlogInterval, + } + for _, remotePath := range config.RemotePaths { + args = append(args, "--remote-path", remotePath) + } + for _, pair := range [][2]string{{"--provider", config.EventProvider}, {"--state-file", config.StateFile}, {"--state-dir", config.StateDir}, {"--mount-kind", config.MountKind}, {"--pprof-addr", config.PprofAddr}} { + if strings.TrimSpace(pair[1]) != "" { + args = append(args, pair[0], pair[1]) + } + } + if config.ForceFullReconcile { + args = append(args, "--full-reconcile") + } + return args +} + +func checkpointSubprocessEnv(env []string) []string { + blocked := map[string]struct{}{ + "RELAYFILE_TOKEN": {}, "RELAYFILE_SERVER": {}, "RELAYFILE_BASE_URL": {}, "RELAYFILE_REMOTE_PATH": {}, + "RELAYFILE_MOUNT_CREDS_FILE": {}, "RELAYFILE_DELEGATED_CREDENTIALS_FILE": {}, "RELAYFILE_MOUNT_PATHS_FILE": {}, + } + out := make([]string, 0, len(env)) + for _, item := range env { + key := strings.SplitN(item, "=", 2)[0] + if _, skip := blocked[key]; skip || strings.HasPrefix(key, "RELAYFILE_MOUNT_") { + continue + } + out = append(out, item) + } + return out +} + +func waitCheckpointMountReady(config checkpointMountConfig, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + pidState, ok := readDaemonPIDState(config.LocalRoot) + if ok && pidState.Registered && pidState.WorkspaceID == config.WorkspaceID && processAlive(pidState.PID) && pidState.CheckpointConfig != nil { + payload, err := os.ReadFile(filepath.Join(config.LocalRoot, ".relay", "state.json")) + var state syncStateFile + if err == nil && json.Unmarshal(payload, &state) == nil && state.WorkspaceID == config.WorkspaceID && state.Mode == defaultMountMode && state.RemoteRoot == mountscope.FirstPath(config.RemotePaths, "/") && state.Daemon != nil && state.Daemon.PID == pidState.PID && state.LastSuccessfulReconcileAt != "" && state.LastError == nil && state.PendingConflicts == 0 { + return nil + } + } + if time.Now().After(deadline) { + return fmt.Errorf("mount did not become ready within %s", timeout) + } + time.Sleep(50 * time.Millisecond) + } +} + +func burnCheckpointReceiptForResume(state checkpointLifecycleState, timeout time.Duration) (mountsync.CheckpointSealOwnership, error) { + bundle, loadedPath, err := loadDelegatedCredentials(state.Config.CredentialsFile) + if err != nil { + return mountsync.CheckpointSealOwnership{}, err + } + bundle, err = refreshDelegatedCredentials(loadedPath, bundle, false) + if err != nil { + return mountsync.CheckpointSealOwnership{}, err + } + client := mountsync.NewHTTPClient(state.Config.Server, bundle.BearerToken(), mountsync.NewSyncHTTPClient()) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + proof, err := client.ResumeCheckpointSeal(ctx, state.WorkspaceID, mountsync.CheckpointSealResumeRequest{ + SealToken: state.Receipt.SealToken, Root: state.Receipt.Root, SessionID: state.SessionID, + Generation: state.Generation, ResumeIdempotencyKey: "source-resume:" + checkpointHash(state.ResumeID)[:24], + }) + return proof, err +} + +func waitCheckpointSourceProof(config checkpointMountConfig, proof mountsync.CheckpointSealOwnership, timeout time.Duration) error { + syncer, err := newCheckpointSourceProofVerifier(config) + if err != nil { + return err + } + deadline := time.Now().Add(timeout) + var lastErr error + for { + remaining := time.Until(deadline) + if remaining <= 0 { + if lastErr == nil { + lastErr = context.DeadlineExceeded + } + return fmt.Errorf("source did not converge to resume proof within %s: %w", timeout, lastErr) + } + ctx, cancel := context.WithTimeout(context.Background(), remaining) + _, lastErr = syncer.VerifyCheckpointOwnership(ctx, proof) + cancel() + if lastErr == nil { + return nil + } + pause := 50 * time.Millisecond + if remaining < pause { + pause = remaining + } + time.Sleep(pause) + } +} + +func newCheckpointSourceProofVerifier(config checkpointMountConfig) (*mountsync.Syncer, error) { + bundle, loadedPath, err := loadDelegatedCredentials(config.CredentialsFile) + if err != nil { + return nil, err + } + bundle, err = refreshDelegatedCredentials(loadedPath, bundle, false) + if err != nil { + return nil, err + } + if !workspaceRequestMatchesDelegatedCredentials(config.WorkspaceID, bundle.Workspace()) { + return nil, errors.New("delegated credential workspace does not match source resume contract") + } + interval, _ := parseCheckpointDuration(config.Interval, false) + bootstrapTimeout, _ := parseCheckpointDuration(config.BootstrapTimeout, false) + fullPullMinInterval, _ := parseCheckpointDuration(config.FullPullMinInterval, true) + cursorTimeout, _ := parseCheckpointDuration(config.CursorTimeout, false) + client := mountsync.NewHTTPClient(config.Server, bundle.BearerToken(), mountsync.NewSyncHTTPClient()) + syncer, err := mountsync.NewSyncer(client, mountsync.SyncerOptions{ + WorkspaceID: config.WorkspaceID, RemoteRoot: mountscope.FirstPath(config.RemotePaths, "/"), EventProvider: config.EventProvider, + LocalRoot: config.LocalRoot, StateFile: config.StateFile, StateDir: config.StateDir, MountKind: config.MountKind, + ValidateState: true, Scopes: delegatedBundleAvailableScopes(bundle), RootCtx: context.Background(), Mode: config.Mode, + Interval: interval, LowMemory: boolPtr(config.LowMemory), BootstrapTimeout: bootstrapTimeout, + BootstrapMaxFilesPerCycle: config.BootstrapMaxFilesPerCycle, FullPullMinInterval: fullPullMinInterval, + CursorTimeout: cursorTimeout, ForceFullReconcile: boolPtr(config.ForceFullReconcile), SyncMode: "mirror", + }) + if err != nil { + return nil, err + } + return syncer, nil +} + +func checkpointEnvelopeFromState(state checkpointLifecycleState) checkpointSealEnvelope { + return checkpointSealEnvelope{ + Version: checkpointLifecycleVersion, Kind: "relayfile-checkpoint-seal", Status: "sealed", WorkspaceID: state.WorkspaceID, + LocalRoot: state.LocalRoot, SessionID: state.SessionID, Generation: state.Generation, + Receipt: *state.Receipt, Health: mountsync.CheckpointVerificationHealth{}, ResumeID: state.ResumeID, SealedAt: state.SealedAt, + } +} + +func resumeEnvelopeFromState(state checkpointLifecycleState) checkpointResumeEnvelope { + return checkpointResumeEnvelope{ + Version: checkpointLifecycleVersion, Kind: "relayfile-resume-seal", WorkspaceID: state.WorkspaceID, + LocalRoot: state.LocalRoot, ResumeID: state.ResumeID, Status: "ready", ResumedAt: state.ResumedAt, + } +} + +func newResumeID() (string, error) { + var raw [32]byte + if _, err := rand.Read(raw[:]); err != nil { + return "", err + } + return "rsm_" + base64.RawURLEncoding.EncodeToString(raw[:]), nil +} + +func checkpointLifecycleDir() string { return filepath.Join(configDir(), "checkpoint-resumes") } + +func checkpointVerificationDir() string { + return filepath.Join(configDir(), "checkpoint-verifications") +} + +func checkpointHandbackDir() string { + return filepath.Join(configDir(), "checkpoint-handbacks") +} + +func checkpointLifecyclePath(resumeID string) string { + return filepath.Join(checkpointLifecycleDir(), checkpointHash(resumeID)+".json") +} + +func checkpointVerificationPath(verificationID string) string { + return filepath.Join(checkpointVerificationDir(), checkpointHash(verificationID)+".json") +} + +func checkpointHandbackPath(handbackID string) string { + return filepath.Join(checkpointHandbackDir(), checkpointHash(handbackID)+".json") +} + +func checkpointHash(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} + +func ensureCheckpointLifecycleDir() error { + dir := checkpointLifecycleDir() + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + if err := os.Chmod(dir, 0o700); err != nil { + return err + } + info, err := os.Lstat(dir) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o077 != 0 { + return errors.New("checkpoint lifecycle directory is not a private real directory") + } + return nil +} + +func ensureCheckpointVerificationDir() error { + dir := checkpointVerificationDir() + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + if err := os.Chmod(dir, 0o700); err != nil { + return err + } + info, err := os.Lstat(dir) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o077 != 0 { + return errors.New("checkpoint verification directory is not a private real directory") + } + return nil +} + +func ensureCheckpointHandbackDir() error { + dir := checkpointHandbackDir() + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + if err := os.Chmod(dir, 0o700); err != nil { + return err + } + info, err := os.Lstat(dir) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o077 != 0 { + return errors.New("checkpoint handback directory is not a private real directory") + } + return nil +} + +func saveCheckpointHandback(state checkpointHandbackLifecycle) error { + remotePaths := mountscope.NormalizePaths(state.Config.RemotePaths, "/") + if state.Version != checkpointLifecycleVersion || !checkpointLifecycleIDPattern.MatchString(state.HandbackID) || !checkpointLifecycleIDPattern.MatchString(state.ConsumerIdempotencyKey) || + !sameCheckpointLocalRoot(state.LocalRoot, state.Config.LocalRoot) || state.Config.WorkspaceID != state.WorkspaceID || len(remotePaths) != 1 || remotePaths[0] != state.RemoteRoot || + state.Receipt.WorkspaceID != state.WorkspaceID || state.Receipt.Root != state.RemoteRoot || state.Receipt.SessionID != state.SessionID || state.Receipt.Generation != state.Generation || validateDestinationReceipt(state.Receipt) != nil { + return errors.New("invalid checkpoint handback lifecycle identity") + } + if state.Result != nil { + proof := state.Result.Proof + if state.Result.HandbackID != state.HandbackID || state.Result.WorkspaceID != state.WorkspaceID || !sameCheckpointLocalRoot(state.Result.LocalRoot, state.LocalRoot) || state.Result.RemoteRoot != state.RemoteRoot || + state.Result.SessionID != state.SessionID || state.Result.Generation != state.Generation || state.Result.Status != "released" || proof.Status != "released" || proof.SealID != state.Receipt.SealID || + proof.WorkspaceID != state.WorkspaceID || proof.Root != state.RemoteRoot || proof.SessionID != state.SessionID || proof.Generation != state.Generation || proof.ConsumedAt != state.Receipt.ConsumedAt || + !checkpointDigestPattern.MatchString(proof.Digest) || !checkpointRevisionPattern.MatchString(proof.WorkspaceRevision) || !checkpointCursorPattern.MatchString(proof.EventCursor) || proof.PreparedAt == "" || proof.ReleasedAt == "" || proof.SourceResumedAt != "" { + return errors.New("checkpoint handback result identity mismatch") + } + for _, raw := range []string{proof.PreparedAt, proof.ReleasedAt} { + if _, err := time.Parse(time.RFC3339Nano, raw); err != nil { + return errors.New("checkpoint handback result timestamps must be RFC3339") + } + } + } + if err := ensureCheckpointHandbackDir(); err != nil { + return err + } + state.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano) + payload, err := json.MarshalIndent(state, "", " ") + if err != nil { + return err + } + payload = append(payload, '\n') + return writeFileAtomically(checkpointHandbackPath(state.HandbackID), payload, 0o600) +} + +func loadCheckpointHandbackIfExists(handbackID string) (checkpointHandbackLifecycle, bool, error) { + var state checkpointHandbackLifecycle + if !checkpointLifecycleIDPattern.MatchString(handbackID) { + return state, false, errors.New("invalid handbackId") + } + path := checkpointHandbackPath(handbackID) + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return state, false, nil + } + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o077 != 0 { + return state, false, errors.New("handback lifecycle is missing, non-regular, or not private") + } + payload, err := os.ReadFile(path) + if err != nil { + return state, false, err + } + if err := json.Unmarshal(payload, &state); err != nil { + return state, false, err + } + if state.Version != checkpointLifecycleVersion || subtle.ConstantTimeCompare([]byte(state.HandbackID), []byte(handbackID)) != 1 { + return state, false, errors.New("handback lifecycle identity mismatch") + } + return state, true, nil +} + +func saveCheckpointVerification(state checkpointVerificationLifecycle) error { + remotePaths := mountscope.NormalizePaths(state.Config.RemotePaths, "/") + if state.Version != checkpointLifecycleVersion || !checkpointLifecycleIDPattern.MatchString(state.VerificationID) || !sameCheckpointLocalRoot(state.LocalRoot, state.Config.LocalRoot) || + state.Config.WorkspaceID != state.WorkspaceID || len(remotePaths) != 1 || remotePaths[0] != state.RemoteRoot || + state.Receipt.WorkspaceID != state.WorkspaceID || state.Receipt.Root != state.RemoteRoot || state.Receipt.SessionID != state.SessionID || state.Receipt.Generation != state.Generation || validateDestinationReceipt(state.Receipt) != nil { + return errors.New("invalid checkpoint verification lifecycle identity") + } + if state.Result != nil && (state.Result.VerificationID != state.VerificationID || state.Result.WorkspaceID != state.WorkspaceID || !sameCheckpointLocalRoot(state.Result.LocalRoot, state.LocalRoot) || state.Result.RemoteRoot != state.RemoteRoot || state.Result.SessionID != state.SessionID || state.Result.Generation != state.Generation) { + return errors.New("checkpoint verification result identity mismatch") + } + if err := ensureCheckpointVerificationDir(); err != nil { + return err + } + state.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano) + payload, err := json.MarshalIndent(state, "", " ") + if err != nil { + return err + } + payload = append(payload, '\n') + return writeFileAtomically(checkpointVerificationPath(state.VerificationID), payload, 0o600) +} + +func loadCheckpointVerificationIfExists(verificationID string) (checkpointVerificationLifecycle, bool, error) { + var state checkpointVerificationLifecycle + if !checkpointLifecycleIDPattern.MatchString(verificationID) { + return state, false, errors.New("invalid verificationId") + } + path := checkpointVerificationPath(verificationID) + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return state, false, nil + } + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o077 != 0 { + return state, false, errors.New("verification lifecycle is missing, non-regular, or not private") + } + payload, err := os.ReadFile(path) + if err != nil { + return state, false, err + } + if err := json.Unmarshal(payload, &state); err != nil { + return state, false, err + } + if state.Version != checkpointLifecycleVersion || subtle.ConstantTimeCompare([]byte(state.VerificationID), []byte(verificationID)) != 1 { + return state, false, errors.New("verification lifecycle identity mismatch") + } + return state, true, nil +} + +func validateDestinationReceipt(receipt mountsync.CheckpointSeal) error { + if receipt.SealID == "" || receipt.SealToken != "" || receipt.WorkspaceID == "" || receipt.Root != "/" || !checkpointLifecycleIDPattern.MatchString(receipt.SessionID) || receipt.Generation == 0 || + !checkpointDigestPattern.MatchString(receipt.Digest) || !checkpointRevisionPattern.MatchString(receipt.WorkspaceRevision) || !checkpointCursorPattern.MatchString(receipt.EventCursor) || receipt.ConsumedAt == "" { + return errors.New("receipt must be a consumed full-root checkpoint seal without sealToken") + } + for _, raw := range []string{receipt.IssuedAt, receipt.ExpiresAt, receipt.ConsumedAt} { + if _, err := time.Parse(time.RFC3339Nano, raw); err != nil { + return errors.New("receipt timestamps must be RFC3339") + } + } + return nil +} + +func sameDestinationReceipt(left, right mountsync.CheckpointSeal) bool { + return left.SealToken == "" && right.SealToken == "" && left.SealID == right.SealID && left.WorkspaceID == right.WorkspaceID && left.Root == right.Root && left.SessionID == right.SessionID && left.Generation == right.Generation && + left.Digest == right.Digest && left.WorkspaceRevision == right.WorkspaceRevision && left.EventCursor == right.EventCursor && left.IssuedAt == right.IssuedAt && left.ExpiresAt == right.ExpiresAt && left.ConsumedAt == right.ConsumedAt +} + +func validateSourceResumeProof(state checkpointLifecycleState, proof mountsync.CheckpointSealOwnership) error { + if state.Receipt == nil || proof.Status != "source-resumed" || proof.SealID != state.Receipt.SealID || proof.WorkspaceID != state.WorkspaceID || proof.Root != state.RemoteRoot || + proof.SessionID != state.SessionID || proof.Generation != state.Generation || !checkpointDigestPattern.MatchString(proof.Digest) || + !checkpointRevisionPattern.MatchString(proof.WorkspaceRevision) || !checkpointCursorPattern.MatchString(proof.EventCursor) || proof.ReleasedAt == "" || proof.SourceResumedAt == "" { + return errors.New("source-resume proof does not match the sealed lifecycle") + } + for _, raw := range []string{proof.ReleasedAt, proof.SourceResumedAt} { + if _, err := time.Parse(time.RFC3339Nano, raw); err != nil { + return errors.New("source-resume proof timestamps must be RFC3339") + } + } + if proof.ConsumedAt != "" { + if _, err := time.Parse(time.RFC3339Nano, proof.ConsumedAt); err != nil { + return errors.New("source-resume consumedAt must be RFC3339") + } + } + return nil +} + +func saveCheckpointLifecycle(state checkpointLifecycleState) error { + if state.Version != checkpointLifecycleVersion || !checkpointLifecycleIDPattern.MatchString(state.ResumeID) { + return errors.New("invalid checkpoint lifecycle identity") + } + if state.ResumeProof != nil { + if err := validateSourceResumeProof(state, *state.ResumeProof); err != nil { + return err + } + } + if err := ensureCheckpointLifecycleDir(); err != nil { + return err + } + state.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano) + payload, err := json.MarshalIndent(state, "", " ") + if err != nil { + return err + } + payload = append(payload, '\n') + return writeFileAtomically(checkpointLifecyclePath(state.ResumeID), payload, 0o600) +} + +func loadCheckpointLifecycle(resumeID string) (checkpointLifecycleState, error) { + var state checkpointLifecycleState + if !checkpointLifecycleIDPattern.MatchString(resumeID) { + return state, errors.New("invalid resumeId") + } + path := checkpointLifecyclePath(resumeID) + info, err := os.Lstat(path) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o077 != 0 { + return state, errors.New("resume lifecycle not found or not private") + } + payload, err := os.ReadFile(path) + if err != nil { + return state, err + } + if err := json.Unmarshal(payload, &state); err != nil { + return state, err + } + if state.Version != checkpointLifecycleVersion || subtle.ConstantTimeCompare([]byte(state.ResumeID), []byte(resumeID)) != 1 { + return state, errors.New("resume lifecycle identity mismatch") + } + return state, nil +} + +func loadCheckpointLifecycleIfExists(resumeID string) (checkpointLifecycleState, bool, error) { + var state checkpointLifecycleState + if !checkpointLifecycleIDPattern.MatchString(resumeID) { + return state, false, errors.New("invalid lifecycle-id") + } + path := checkpointLifecyclePath(resumeID) + if _, err := os.Lstat(path); errors.Is(err, os.ErrNotExist) { + return state, false, nil + } else if err != nil { + return state, false, err + } + state, err := loadCheckpointLifecycle(resumeID) + return state, err == nil, err +} + +func findCheckpointLifecycle(localRoot, sessionID string, generation uint64) (checkpointLifecycleState, bool, error) { + states, err := listCheckpointLifecycles() + if err != nil { + return checkpointLifecycleState{}, false, err + } + for _, state := range states { + if sameCheckpointLocalRoot(state.LocalRoot, localRoot) && state.SessionID == sessionID && state.Generation == generation { + return state, true, nil + } + } + return checkpointLifecycleState{}, false, nil +} + +func rejectStaleCheckpointGeneration(localRoot, sessionID string, generation uint64) error { + states, err := listCheckpointLifecycles() + if err != nil { + return err + } + for _, state := range states { + if sameCheckpointLocalRoot(state.LocalRoot, localRoot) && state.SessionID == sessionID && state.Generation >= generation { + return fmt.Errorf("generation %d is not newer than lifecycle generation %d", generation, state.Generation) + } + } + return nil +} + +func listCheckpointLifecycles() ([]checkpointLifecycleState, error) { + if err := ensureCheckpointLifecycleDir(); err != nil { + return nil, err + } + entries, err := os.ReadDir(checkpointLifecycleDir()) + if err != nil { + return nil, err + } + states := make([]checkpointLifecycleState, 0) + for _, entry := range entries { + if entry.IsDir() || strings.HasPrefix(entry.Name(), "lock-") || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + path := filepath.Join(checkpointLifecycleDir(), entry.Name()) + info, err := os.Lstat(path) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o077 != 0 { + return nil, fmt.Errorf("lifecycle %s is not a private regular file", entry.Name()) + } + payload, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var state checkpointLifecycleState + if err := json.Unmarshal(payload, &state); err != nil { + return nil, fmt.Errorf("parse lifecycle %s: %w", entry.Name(), err) + } + states = append(states, state) + } + return states, nil +} + +func acquireCheckpointLifecycleLock(localRoot string) (func(), error) { + return acquireCheckpointLifecycleLockContext(context.Background(), localRoot, false) +} + +func acquireCheckpointLifecycleLockWait(ctx context.Context, localRoot string) (func(), error) { + if ctx == nil { + return nil, errors.New("checkpoint lifecycle wait requires a context") + } + return acquireCheckpointLifecycleLockContext(ctx, localRoot, true) +} + +func acquireCheckpointLifecycleLockContext(waitCtx context.Context, localRoot string, wait bool) (func(), error) { + if err := ensureCheckpointLifecycleDir(); err != nil { + return nil, err + } + lockPath := filepath.Join(checkpointLifecycleDir(), "lock-"+checkpointHash(localRoot)+".json") + nonce, err := newResumeID() + if err != nil { + return nil, err + } + payload, _ := json.Marshal(map[string]any{"pid": os.Getpid(), "nonce": nonce}) + for { + file, err := os.OpenFile(lockPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err == nil { + if _, err := file.Write(payload); err != nil { + _ = file.Close() + _ = os.Remove(lockPath) + return nil, err + } + if err := file.Close(); err != nil { + _ = os.Remove(lockPath) + return nil, err + } + return func() { + current, _ := os.ReadFile(lockPath) + if subtle.ConstantTimeCompare(current, payload) == 1 { + _ = os.Remove(lockPath) + } + }, nil + } + if !errors.Is(err, os.ErrExist) { + return nil, err + } + var owner struct { + PID int `json:"pid"` + } + current, _ := os.ReadFile(lockPath) + if json.Unmarshal(current, &owner) == nil && processAlive(owner.PID) { + if !wait { + return nil, fmt.Errorf("checkpoint lifecycle is active in pid %d", owner.PID) + } + select { + case <-waitCtx.Done(): + return nil, fmt.Errorf("checkpoint lifecycle remained active in pid %d: %w", owner.PID, waitCtx.Err()) + case <-time.After(50 * time.Millisecond): + continue + } + } + _ = os.Remove(lockPath) + } +} + +func sameCheckpointLocalRoot(a, b string) bool { + left, err := filepath.Abs(strings.TrimSpace(a)) + if err != nil { + return false + } + right, err := filepath.Abs(strings.TrimSpace(b)) + if err != nil { + return false + } + if resolved, resolveErr := filepath.EvalSymlinks(left); resolveErr == nil { + left = resolved + } + if resolved, resolveErr := filepath.EvalSymlinks(right); resolveErr == nil { + right = resolved + } + return filepath.Clean(left) == filepath.Clean(right) +} diff --git a/cmd/relayfile-cli/checkpoint_lifecycle_test.go b/cmd/relayfile-cli/checkpoint_lifecycle_test.go new file mode 100644 index 00000000..82e0087b --- /dev/null +++ b/cmd/relayfile-cli/checkpoint_lifecycle_test.go @@ -0,0 +1,967 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/agentworkforce/relayfile/internal/mountlease" + "github.com/agentworkforce/relayfile/internal/mountscope" + "github.com/agentworkforce/relayfile/internal/mountsync" +) + +type fakeCheckpointLease struct{ released bool } + +func (l *fakeCheckpointLease) Release() error { l.released = true; return nil } + +func installCheckpointLifecycleSeams(t *testing.T, active activeCheckpointMount, receipt mountsync.CheckpointSeal) (*fakeCheckpointLease, *int, *int) { + t.Helper() + originalResolve := checkpointResolveActive + originalStop := checkpointStopActive + originalIssue := checkpointIssueStopped + originalVerify := checkpointVerifyStopped + originalHandback := checkpointHandbackStopped + originalEnsure := checkpointEnsureSource + originalBurn := checkpointBurnReceipt + originalWait := checkpointWaitMountReady + originalWaitSourceProof := checkpointWaitSourceProof + lease := &fakeCheckpointLease{} + ensureCalls := 0 + burnCalls := 0 + checkpointResolveActive = func(string) (activeCheckpointMount, error) { return active, nil } + checkpointStopActive = func(context.Context, activeCheckpointMount) (checkpointLease, error) { return lease, nil } + checkpointIssueStopped = func(context.Context, checkpointMountConfig, string, uint64, int) (mountsync.CheckpointSeal, error) { + return receipt, nil + } + checkpointVerifyStopped = func(context.Context, checkpointMountConfig, mountsync.CheckpointSeal) (mountsync.CheckpointVerification, error) { + return mountsync.CheckpointVerification{ + Observed: mountsync.CheckpointObservedState{Digest: receipt.Digest, WorkspaceRevision: receipt.WorkspaceRevision, EventCursor: receipt.EventCursor}, + }, nil + } + checkpointHandbackStopped = func(_ context.Context, _ checkpointMountConfig, consumed mountsync.CheckpointSeal, _, _ string) (mountsync.CheckpointSealOwnership, mountsync.CheckpointVerificationHealth, error) { + return mountsync.CheckpointSealOwnership{ + SealID: consumed.SealID, WorkspaceID: consumed.WorkspaceID, Root: consumed.Root, + SessionID: consumed.SessionID, Generation: consumed.Generation, Status: "released", + Digest: consumed.Digest, WorkspaceRevision: consumed.WorkspaceRevision, EventCursor: consumed.EventCursor, + ConsumedAt: consumed.ConsumedAt, PreparedAt: "2026-08-23T12:00:09Z", ReleasedAt: "2026-08-23T12:00:10Z", + }, mountsync.CheckpointVerificationHealth{}, nil + } + checkpointEnsureSource = func(checkpointMountConfig, time.Duration) error { ensureCalls++; return nil } + checkpointBurnReceipt = func(checkpointLifecycleState, time.Duration) (mountsync.CheckpointSealOwnership, error) { + burnCalls++ + return mountsync.CheckpointSealOwnership{ + SealID: receipt.SealID, WorkspaceID: receipt.WorkspaceID, Root: receipt.Root, + SessionID: receipt.SessionID, Generation: receipt.Generation, Status: "source-resumed", + Digest: receipt.Digest, WorkspaceRevision: receipt.WorkspaceRevision, EventCursor: receipt.EventCursor, + ConsumedAt: receipt.ConsumedAt, ReleasedAt: "2026-08-23T12:00:10Z", SourceResumedAt: "2026-08-23T12:00:11Z", + }, nil + } + checkpointWaitMountReady = func(checkpointMountConfig, time.Duration) error { return nil } + checkpointWaitSourceProof = func(checkpointMountConfig, mountsync.CheckpointSealOwnership, time.Duration) error { return nil } + t.Cleanup(func() { + checkpointResolveActive = originalResolve + checkpointStopActive = originalStop + checkpointIssueStopped = originalIssue + checkpointVerifyStopped = originalVerify + checkpointHandbackStopped = originalHandback + checkpointEnsureSource = originalEnsure + checkpointBurnReceipt = originalBurn + checkpointWaitMountReady = originalWait + checkpointWaitSourceProof = originalWaitSourceProof + }) + return lease, &ensureCalls, &burnCalls +} + +func consumedCheckpointTestReceipt(receipt mountsync.CheckpointSeal) mountsync.CheckpointSeal { + receipt.SealToken = "" + receipt.ConsumedAt = "2026-08-23T12:00:05Z" + return receipt +} + +func checkpointInputAtLimit(t *testing.T, base []byte) []byte { + t.Helper() + if len(base) > checkpointCLIInputMaxBytes { + t.Fatalf("base input is already %d bytes", len(base)) + } + payload := append([]byte(nil), base...) + payload = append(payload, bytes.Repeat([]byte(" "), checkpointCLIInputMaxBytes-len(payload))...) + return payload +} + +func checkpointTestActive(t *testing.T) (activeCheckpointMount, mountsync.CheckpointSeal) { + t.Helper() + root := t.TempDir() + credentials := filepath.Join(t.TempDir(), "delegated.json") + if err := os.WriteFile(credentials, []byte("{}\n"), 0o600); err != nil { + t.Fatal(err) + } + config := checkpointMountConfig{ + Version: checkpointLifecycleVersion, Server: "https://relayfile.test", CredentialsFile: credentials, + WorkspaceID: "ws_checkpoint", LocalRoot: root, RemotePaths: []string{"/"}, + LocalLayout: mountscope.LayoutExact, Mode: defaultMountMode, Interval: "30s", Timeout: "15s", + BootstrapTimeout: "0s", BootstrapMaxFilesPerCycle: 2000, FullPullMinInterval: "24h0m0s", + CursorTimeout: "1m0s", WebsocketEnabled: true, MemlogInterval: "0s", + } + receipt := mountsync.CheckpointSeal{ + SealID: "cps_123", SealToken: "one-use-token", WorkspaceID: config.WorkspaceID, + Root: "/", SessionID: "session-123", Generation: 2, + Digest: "sha256:" + strings.Repeat("a", 64), WorkspaceRevision: "rev_1", EventCursor: "evt_1", + IssuedAt: "2026-08-23T12:00:00Z", ExpiresAt: "2026-08-23T12:01:00Z", + } + return activeCheckpointMount{record: workspaceRecord{ID: config.WorkspaceID, LocalDir: root, RemotePaths: []string{"/"}, LocalLayout: mountscope.LayoutExact}, config: config}, receipt +} + +func TestMountCheckpointSealEmitsDistinctLocalAndRemoteRootsAndIsCrashIdempotent(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, receipt := checkpointTestActive(t) + lease, _, _ := installCheckpointLifecycleSeams(t, active, receipt) + args := []string{"--root", active.config.LocalRoot, "--lifecycle-id", "rsm_controller_123", "--session", "session-123", "--generation", "2", "--json"} + var first bytes.Buffer + if err := runMountCheckpointSeal(args, &first); err != nil { + t.Fatalf("checkpoint: %v", err) + } + if !lease.released { + t.Fatal("checkpoint did not release the acquired mount lease") + } + var envelope checkpointSealEnvelope + if err := json.Unmarshal(first.Bytes(), &envelope); err != nil { + t.Fatalf("decode envelope: %v", err) + } + if envelope.Status != "sealed" || !sameCheckpointLocalRoot(envelope.LocalRoot, active.config.LocalRoot) || envelope.Receipt.Root != "/" || envelope.ResumeID == "" { + t.Fatalf("ambiguous or incomplete roots: %+v", envelope) + } + if envelope.Health != (mountsync.CheckpointVerificationHealth{}) { + t.Fatalf("successful final drain reported non-zero health: %+v", envelope.Health) + } + if !strings.Contains(first.String(), `"outboxNeedsAttention": false`) { + t.Fatalf("checkpoint health wire type is not an explicit boolean: %s", first.String()) + } + var replay bytes.Buffer + if err := runMountCheckpointSeal(args, &replay); err != nil { + t.Fatalf("response-loss retry: %v", err) + } + var replayEnvelope checkpointSealEnvelope + _ = json.Unmarshal(replay.Bytes(), &replayEnvelope) + if replayEnvelope.ResumeID != envelope.ResumeID || replayEnvelope.Receipt.SealToken != envelope.Receipt.SealToken { + t.Fatalf("checkpoint retry changed durable handoff: first=%+v retry=%+v", envelope, replayEnvelope) + } + conflicting := []string{"--root", active.config.LocalRoot, "--lifecycle-id", "rsm_controller_123", "--session", "session-other", "--generation", "2", "--json"} + if err := runMountCheckpointSeal(conflicting, &bytes.Buffer{}); checkpointExitCode(err) != 3 { + t.Fatalf("lifecycle identity conflict = %v", err) + } + staleArgs := []string{"--root", active.config.LocalRoot, "--lifecycle-id", "rsm_controller_stale", "--session", "session-123", "--generation", "1", "--json"} + if err := runMountCheckpointSeal(staleArgs, &bytes.Buffer{}); checkpointExitCode(err) != 3 { + t.Fatalf("stale generation error = %v", err) + } +} + +func TestMountCheckpointRequiresControllerLifecycleIntentBeforeStop(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, receipt := checkpointTestActive(t) + _, _, _ = installCheckpointLifecycleSeams(t, active, receipt) + stopCalls := 0 + originalStop := checkpointStopActive + checkpointStopActive = func(context.Context, activeCheckpointMount) (checkpointLease, error) { + stopCalls++ + return &fakeCheckpointLease{}, nil + } + t.Cleanup(func() { checkpointStopActive = originalStop }) + err := runMountCheckpointSeal([]string{"--root", active.config.LocalRoot, "--session", receipt.SessionID, "--generation", "2", "--json"}, &bytes.Buffer{}) + if checkpointExitCode(err) != 2 || stopCalls != 0 { + t.Fatalf("missing lifecycle intent err=%v stopCalls=%d", err, stopCalls) + } +} + +func TestMountCheckpointPresealFailureRestartsBeforeReturning(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, receipt := checkpointTestActive(t) + _, ensureCalls, _ := installCheckpointLifecycleSeams(t, active, receipt) + checkpointIssueStopped = func(context.Context, checkpointMountConfig, string, uint64, int) (mountsync.CheckpointSeal, error) { + return mountsync.CheckpointSeal{}, errors.New("server divergence") + } + err := runMountCheckpointSeal([]string{"--root", active.config.LocalRoot, "--lifecycle-id", "rsm_controller_fail", "--session", "session-fail", "--generation", "3", "--json"}, &bytes.Buffer{}) + if checkpointExitCode(err) != 5 || *ensureCalls != 1 { + t.Fatalf("preseal failure err=%v ensureCalls=%d", err, *ensureCalls) + } +} + +func TestMountResumeSealIsCrashRecoverableIdempotentAndRootBound(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, receipt := checkpointTestActive(t) + _, ensureCalls, burnCalls := installCheckpointLifecycleSeams(t, active, receipt) + state := checkpointLifecycleState{ + Version: checkpointLifecycleVersion, Kind: "relayfile-checkpoint-lifecycle", ResumeID: "rsm_test_123", + WorkspaceID: active.config.WorkspaceID, LocalRoot: active.config.LocalRoot, RemoteRoot: receipt.Root, + SessionID: receipt.SessionID, Generation: receipt.Generation, Status: "sealed", Config: active.config, + Receipt: &receipt, CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), SealedAt: time.Now().UTC().Format(time.RFC3339Nano), + } + if err := saveCheckpointLifecycle(state); err != nil { + t.Fatal(err) + } + stdin := func() *strings.Reader { return strings.NewReader(`{"resumeId":"rsm_test_123"}`) } + var output bytes.Buffer + if err := runMountResumeSeal([]string{"--root", active.config.LocalRoot, "--json"}, stdin(), &output); err != nil { + t.Fatalf("resume: %v", err) + } + if *burnCalls != 1 || *ensureCalls != 1 { + t.Fatalf("resume calls burn=%d ensure=%d", *burnCalls, *ensureCalls) + } + if err := runMountResumeSeal([]string{"--root", active.config.LocalRoot, "--json"}, stdin(), &bytes.Buffer{}); err != nil { + t.Fatalf("idempotent resume: %v", err) + } + wrongRoot := t.TempDir() + if err := runMountResumeSeal([]string{"--root", wrongRoot, "--json"}, stdin(), &bytes.Buffer{}); checkpointExitCode(err) != 2 { + t.Fatalf("wrong root error = %v", err) + } +} + +func TestMountResumeSealRejectsNonExactJSONDocument(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + root := t.TempDir() + for name, input := range map[string]string{ + "malformed": `{"resumeId":`, + "trailing second object": `{"resumeId":"rsm_exact"}{"resumeId":"rsm_other"}`, + "trailing garbage": `{"resumeId":"rsm_exact"}not-json`, + "unknown field": `{"resumeId":"rsm_exact","extra":true}`, + } { + t.Run(name, func(t *testing.T) { + var output bytes.Buffer + err := runMountResumeSeal([]string{"--root", root, "--json"}, strings.NewReader(input), &output) + if checkpointExitCode(err) != 2 || !strings.Contains(err.Error(), "exactly one JSON object") || output.Len() != 0 { + t.Fatalf("non-exact JSON err=%v output=%q", err, output.String()) + } + }) + } +} + +func TestCheckpointCommandInputCutoffRejectsHiddenTrailingBytes(t *testing.T) { + t.Run("exact boundary remains a complete decodable document", func(t *testing.T) { + payload := checkpointInputAtLimit(t, []byte(`{"resumeId":"rsm_boundary"}`)) + var input checkpointResumeInput + if err := decodeStrictCheckpointInput(bytes.NewReader(payload), &input); err != nil || input.ResumeID != "rsm_boundary" { + t.Fatalf("exact-boundary decode input=%+v err=%v", input, err) + } + }) + + t.Run("resume second object just beyond cutoff has no lifecycle effect", func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + root := t.TempDir() + payload := append(checkpointInputAtLimit(t, []byte(`{"resumeId":"rsm_cutoff"}`)), []byte(`{}`)...) + err := runMountResumeSeal([]string{"--root", root, "--json"}, bytes.NewReader(payload), &bytes.Buffer{}) + if checkpointExitCode(err) != 2 { + t.Fatalf("resume cutoff error=%v", err) + } + if _, statErr := os.Stat(checkpointLifecyclePath("rsm_cutoff")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("resume cutoff created lifecycle: %v", statErr) + } + }) + + t.Run("verify garbage just beyond cutoff has no mount effect", func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, issued := checkpointTestActive(t) + receipt := consumedCheckpointTestReceipt(issued) + installCheckpointLifecycleSeams(t, active, receipt) + resolveCalls := 0 + checkpointResolveActive = func(string) (activeCheckpointMount, error) { + resolveCalls++ + return active, nil + } + input := checkpointVerificationInput{VerificationID: "vrf_cutoff", Receipt: receipt} + base, _ := json.Marshal(input) + payload := append(checkpointInputAtLimit(t, base), 'x') + err := runMountVerifySeal([]string{"--root", active.config.LocalRoot, "--json"}, bytes.NewReader(payload), &bytes.Buffer{}) + if checkpointExitCode(err) != 2 || resolveCalls != 0 { + t.Fatalf("verify cutoff error=%v resolveCalls=%d", err, resolveCalls) + } + if _, statErr := os.Stat(checkpointVerificationPath(input.VerificationID)); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("verify cutoff created lifecycle: %v", statErr) + } + }) + + t.Run("handback second object just beyond cutoff has no mount effect", func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, issued := checkpointTestActive(t) + receipt := consumedCheckpointTestReceipt(issued) + installCheckpointLifecycleSeams(t, active, receipt) + resolveCalls := 0 + checkpointResolveActive = func(string) (activeCheckpointMount, error) { + resolveCalls++ + return active, nil + } + input := checkpointHandbackInput{HandbackID: "handback-cutoff", ConsumerIdempotencyKey: "cutover-cutoff", Receipt: receipt} + base, _ := json.Marshal(input) + payload := append(checkpointInputAtLimit(t, base), []byte(`{}`)...) + err := runMountHandbackSeal([]string{"--root", active.config.LocalRoot, "--json"}, bytes.NewReader(payload), &bytes.Buffer{}) + if checkpointExitCode(err) != 2 || resolveCalls != 0 { + t.Fatalf("handback cutoff error=%v resolveCalls=%d", err, resolveCalls) + } + if _, statErr := os.Stat(checkpointHandbackPath(input.HandbackID)); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("handback cutoff created lifecycle: %v", statErr) + } + }) +} + +func TestMountResumeSealCannotReportReadyBeforeResumeProofMaterializes(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, receipt := checkpointTestActive(t) + _, _, burnCalls := installCheckpointLifecycleSeams(t, active, receipt) + state := checkpointLifecycleState{ + Version: checkpointLifecycleVersion, Kind: "relayfile-checkpoint-lifecycle", ResumeID: "rsm_delayed_pull", + WorkspaceID: active.config.WorkspaceID, LocalRoot: active.config.LocalRoot, RemoteRoot: receipt.Root, + SessionID: receipt.SessionID, Generation: receipt.Generation, Status: "sealed", Config: active.config, + Receipt: &receipt, CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), SealedAt: time.Now().UTC().Format(time.RFC3339Nano), + } + if err := saveCheckpointLifecycle(state); err != nil { + t.Fatal(err) + } + destinationTurn := filepath.Join(active.config.LocalRoot, "destination-turn.txt") + checkpointWaitSourceProof = func(_ checkpointMountConfig, _ mountsync.CheckpointSealOwnership, _ time.Duration) error { + if _, err := os.Stat(destinationTurn); err != nil { + return fmt.Errorf("%w: destination turn not materialized", mountsync.ErrCheckpointNonConverged) + } + return nil + } + input := func() *strings.Reader { return strings.NewReader(`{"resumeId":"rsm_delayed_pull"}`) } + var premature bytes.Buffer + err := runMountResumeSeal([]string{"--root", active.config.LocalRoot, "--json"}, input(), &premature) + if checkpointExitCode(err) != 5 || premature.Len() != 0 { + t.Fatalf("premature source admission err=%v output=%q", err, premature.String()) + } + loaded, loadErr := loadCheckpointLifecycle("rsm_delayed_pull") + if loadErr != nil || loaded.Status != "resuming" || loaded.ResumeProof == nil || loaded.Receipt == nil || loaded.Receipt.SealToken == "" { + t.Fatalf("nonconverged lifecycle=%+v err=%v", loaded, loadErr) + } + if err := os.WriteFile(destinationTurn, []byte("destination turn\n"), 0o644); err != nil { + t.Fatal(err) + } + var ready bytes.Buffer + if err := runMountResumeSeal([]string{"--root", active.config.LocalRoot, "--json"}, input(), &ready); err != nil { + t.Fatalf("resume after destination turn materialized: %v", err) + } + loaded, loadErr = loadCheckpointLifecycle("rsm_delayed_pull") + if loadErr != nil || loaded.Status != "ready" || loaded.ResumeProof == nil || loaded.Receipt == nil || loaded.Receipt.SealToken != "" || *burnCalls != 1 { + t.Fatalf("ready lifecycle=%+v burnCalls=%d err=%v", loaded, *burnCalls, loadErr) + } +} + +func TestMountResumeSealFailsClosedUntilDestinationHandback(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, receipt := checkpointTestActive(t) + _, ensureCalls, _ := installCheckpointLifecycleSeams(t, active, receipt) + state := checkpointLifecycleState{ + Version: checkpointLifecycleVersion, Kind: "relayfile-checkpoint-lifecycle", ResumeID: "rsm_destination_owned", + WorkspaceID: active.config.WorkspaceID, LocalRoot: active.config.LocalRoot, RemoteRoot: receipt.Root, + SessionID: receipt.SessionID, Generation: receipt.Generation, Status: "sealed", Config: active.config, + Receipt: &receipt, CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), SealedAt: time.Now().UTC().Format(time.RFC3339Nano), + } + if err := saveCheckpointLifecycle(state); err != nil { + t.Fatal(err) + } + checkpointBurnReceipt = func(checkpointLifecycleState, time.Duration) (mountsync.CheckpointSealOwnership, error) { + return mountsync.CheckpointSealOwnership{}, &mountsync.HTTPError{StatusCode: 409, Code: "checkpoint_handback_required", Message: "destination still owns seal"} + } + err := runMountResumeSeal([]string{"--root", active.config.LocalRoot, "--json"}, strings.NewReader(`{"resumeId":"rsm_destination_owned"}`), &bytes.Buffer{}) + if checkpointExitCode(err) != 3 || !strings.Contains(err.Error(), "checkpoint_handback_required") || *ensureCalls != 0 { + t.Fatalf("destination-owned resume err=%v ensureCalls=%d", err, *ensureCalls) + } + loaded, loadErr := loadCheckpointLifecycle("rsm_destination_owned") + if loadErr != nil || loaded.Status != "sealed" { + t.Fatalf("premature resume lifecycle=%+v err=%v", loaded, loadErr) + } +} + +func TestMountVerifySealOwnsVerdictRestartsAndIsResponseLossIdempotent(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, issued := checkpointTestActive(t) + receipt := consumedCheckpointTestReceipt(issued) + lease, ensureCalls, _ := installCheckpointLifecycleSeams(t, active, receipt) + input := checkpointVerificationInput{VerificationID: "vrf_controller_123", Receipt: receipt} + payload, _ := json.Marshal(input) + args := []string{"--root", active.config.LocalRoot, "--json"} + var first bytes.Buffer + if err := runMountVerifySeal(args, bytes.NewReader(payload), &first); err != nil { + t.Fatalf("verify: %v", err) + } + if !lease.released || *ensureCalls != 1 { + t.Fatalf("verification did not release/restart: lease=%v ensure=%d", lease.released, *ensureCalls) + } + var envelope checkpointDestinationVerificationEnvelope + if err := json.Unmarshal(first.Bytes(), &envelope); err != nil { + t.Fatal(err) + } + if envelope.Version != 1 || envelope.Kind != "relayfile-destination-verification" || envelope.VerificationID != input.VerificationID || envelope.WorkspaceID != receipt.WorkspaceID || envelope.RemoteRoot != "/" || envelope.SessionID != receipt.SessionID || envelope.Generation != receipt.Generation || envelope.Status != "converged" || envelope.Observed.Digest != receipt.Digest || envelope.Observed.WorkspaceRevision != receipt.WorkspaceRevision || envelope.Observed.EventCursor != receipt.EventCursor || envelope.VerifiedAt == "" { + t.Fatalf("invalid verification envelope: %+v", envelope) + } + if !strings.Contains(first.String(), `"outboxNeedsAttention": false`) { + t.Fatalf("verification health wire type is not an explicit boolean: %s", first.String()) + } + var replay bytes.Buffer + if err := runMountVerifySeal(args, bytes.NewReader(payload), &replay); err != nil { + t.Fatalf("response-loss replay: %v", err) + } + var replayEnvelope checkpointDestinationVerificationEnvelope + _ = json.Unmarshal(replay.Bytes(), &replayEnvelope) + if replayEnvelope.VerifiedAt != envelope.VerifiedAt || replayEnvelope.Observed != envelope.Observed || replayEnvelope.Health != envelope.Health { + t.Fatalf("verification replay changed verdict: first=%+v replay=%+v", envelope, replayEnvelope) + } + changed := input + changed.Receipt.EventCursor = "evt_999" + changedPayload, _ := json.Marshal(changed) + if err := runMountVerifySeal(args, bytes.NewReader(changedPayload), &bytes.Buffer{}); checkpointExitCode(err) != 3 { + t.Fatalf("changed receipt replay = %v", err) + } +} + +func TestMountVerifySealRejectsNonNativeRevisionAndCursorBeforeStop(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, issued := checkpointTestActive(t) + receipt := consumedCheckpointTestReceipt(issued) + stopCalls := 0 + originalStop := checkpointStopActive + checkpointStopActive = func(context.Context, activeCheckpointMount) (checkpointLease, error) { + stopCalls++ + return &fakeCheckpointLease{}, nil + } + t.Cleanup(func() { checkpointStopActive = originalStop }) + for name, mutate := range map[string]func(*mountsync.CheckpointSeal){ + "bare revision": func(value *mountsync.CheckpointSeal) { value.WorkspaceRevision = "12" }, + "bare cursor": func(value *mountsync.CheckpointSeal) { value.EventCursor = "12" }, + } { + t.Run(name, func(t *testing.T) { + candidate := receipt + mutate(&candidate) + payload, _ := json.Marshal(checkpointVerificationInput{VerificationID: "vrf_invalid_" + strings.ReplaceAll(name, " ", "_"), Receipt: candidate}) + err := runMountVerifySeal([]string{"--root", active.config.LocalRoot, "--json"}, bytes.NewReader(payload), &bytes.Buffer{}) + if checkpointExitCode(err) != 2 { + t.Fatalf("invalid wire receipt error = %v", err) + } + }) + } + if stopCalls != 0 { + t.Fatalf("invalid receipt stopped destination mount %d times", stopCalls) + } +} + +func TestMountVerifySealServerFailureFailsClosedAfterDestinationRecovery(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, issued := checkpointTestActive(t) + receipt := consumedCheckpointTestReceipt(issued) + lease, ensureCalls, _ := installCheckpointLifecycleSeams(t, active, receipt) + checkpointVerifyStopped = func(context.Context, checkpointMountConfig, mountsync.CheckpointSeal) (mountsync.CheckpointVerification, error) { + return mountsync.CheckpointVerification{}, errors.New("server unavailable") + } + payload, _ := json.Marshal(checkpointVerificationInput{VerificationID: "vrf_server_down", Receipt: receipt}) + err := runMountVerifySeal([]string{"--root", active.config.LocalRoot, "--json"}, bytes.NewReader(payload), &bytes.Buffer{}) + if checkpointExitCode(err) != 5 || !strings.Contains(err.Error(), "server unavailable") || !lease.released || *ensureCalls != 1 { + t.Fatalf("server failure err=%v lease=%v ensure=%d", err, lease.released, *ensureCalls) + } +} + +func TestMountHandbackSealStopsDrainsReleasesAndIsResponseLossIdempotent(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, issued := checkpointTestActive(t) + receipt := consumedCheckpointTestReceipt(issued) + lease, ensureCalls, _ := installCheckpointLifecycleSeams(t, active, receipt) + handbackCalls := 0 + checkpointHandbackStopped = func(_ context.Context, _ checkpointMountConfig, consumed mountsync.CheckpointSeal, consumerKey, handbackKey string) (mountsync.CheckpointSealOwnership, mountsync.CheckpointVerificationHealth, error) { + handbackCalls++ + if consumerKey != "cutover-job-handback" || handbackKey != "handback-job-one" { + t.Fatalf("handback identities consumer=%q handback=%q", consumerKey, handbackKey) + } + return mountsync.CheckpointSealOwnership{ + SealID: consumed.SealID, WorkspaceID: consumed.WorkspaceID, Root: consumed.Root, + SessionID: consumed.SessionID, Generation: consumed.Generation, Status: "released", + Digest: consumed.Digest, WorkspaceRevision: "rev_3", EventCursor: consumed.EventCursor, + ConsumedAt: consumed.ConsumedAt, PreparedAt: "2026-08-23T12:00:09Z", ReleasedAt: "2026-08-23T12:00:10Z", + }, mountsync.CheckpointVerificationHealth{}, nil + } + input := checkpointHandbackInput{HandbackID: "handback-job-one", ConsumerIdempotencyKey: "cutover-job-handback", Receipt: receipt} + payload, _ := json.Marshal(input) + args := []string{"--root", active.config.LocalRoot, "--json"} + var first bytes.Buffer + if err := runMountHandbackSeal(args, bytes.NewReader(payload), &first); err != nil { + t.Fatalf("handback: %v", err) + } + if !lease.released || *ensureCalls != 0 || handbackCalls != 1 { + t.Fatalf("handback lease=%v restart=%d calls=%d", lease.released, *ensureCalls, handbackCalls) + } + var envelope checkpointHandbackEnvelope + if err := json.Unmarshal(first.Bytes(), &envelope); err != nil { + t.Fatal(err) + } + if envelope.Kind != "relayfile-checkpoint-handback" || envelope.Status != "released" || envelope.HandbackID != input.HandbackID || envelope.Proof.Status != "released" || envelope.Proof.SourceResumedAt != "" { + t.Fatalf("handback envelope=%+v", envelope) + } + if !strings.Contains(first.String(), `"outboxNeedsAttention": false`) || strings.Contains(first.String(), "sealToken") { + t.Fatalf("handback output leaked bearer or changed health wire type: %s", first.String()) + } + var replay bytes.Buffer + if err := runMountHandbackSeal(args, bytes.NewReader(payload), &replay); err != nil { + t.Fatalf("handback response-loss retry: %v", err) + } + if handbackCalls != 1 || replay.String() != first.String() { + t.Fatalf("handback retry calls=%d first=%s replay=%s", handbackCalls, first.String(), replay.String()) + } + saved, ok, err := loadCheckpointHandbackIfExists(input.HandbackID) + if err != nil || !ok || saved.Result == nil { + t.Fatalf("load released handback lifecycle=%+v ok=%v err=%v", saved, ok, err) + } + for name, preparedAt := range map[string]string{ + "missing preparedAt": "", + "malformed preparedAt": "not-a-time", + } { + t.Run(name, func(t *testing.T) { + candidate := saved + result := *saved.Result + result.Proof.PreparedAt = preparedAt + candidate.Result = &result + if err := saveCheckpointHandback(candidate); err == nil { + t.Fatal("released handback lifecycle accepted invalid preparedAt") + } + }) + } +} + +func TestMountHandbackFailureRecoveryDistinguishesDefinitiveFromAmbiguous(t *testing.T) { + t.Run("definitive server rejection restarts destination", func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, issued := checkpointTestActive(t) + receipt := consumedCheckpointTestReceipt(issued) + _, ensureCalls, _ := installCheckpointLifecycleSeams(t, active, receipt) + checkpointHandbackStopped = func(context.Context, checkpointMountConfig, mountsync.CheckpointSeal, string, string) (mountsync.CheckpointSealOwnership, mountsync.CheckpointVerificationHealth, error) { + return mountsync.CheckpointSealOwnership{}, mountsync.CheckpointVerificationHealth{}, &mountsync.HTTPError{StatusCode: 409, Code: "checkpoint_diverged", Message: "destination not drained"} + } + payload, _ := json.Marshal(checkpointHandbackInput{HandbackID: "handback-definitive", ConsumerIdempotencyKey: "cutover-definitive", Receipt: receipt}) + err := runMountHandbackSeal([]string{"--root", active.config.LocalRoot, "--json"}, bytes.NewReader(payload), &bytes.Buffer{}) + if checkpointExitCode(err) != 5 || *ensureCalls != 1 { + t.Fatalf("definitive handback err=%v restart=%d", err, *ensureCalls) + } + }) + + t.Run("transport ambiguity leaves destination stopped for exact retry", func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, issued := checkpointTestActive(t) + receipt := consumedCheckpointTestReceipt(issued) + _, ensureCalls, _ := installCheckpointLifecycleSeams(t, active, receipt) + checkpointHandbackStopped = func(context.Context, checkpointMountConfig, mountsync.CheckpointSeal, string, string) (mountsync.CheckpointSealOwnership, mountsync.CheckpointVerificationHealth, error) { + return mountsync.CheckpointSealOwnership{}, mountsync.CheckpointVerificationHealth{}, errors.New("connection reset after POST") + } + payload, _ := json.Marshal(checkpointHandbackInput{HandbackID: "handback-ambiguous", ConsumerIdempotencyKey: "cutover-ambiguous", Receipt: receipt}) + err := runMountHandbackSeal([]string{"--root", active.config.LocalRoot, "--json"}, bytes.NewReader(payload), &bytes.Buffer{}) + if checkpointExitCode(err) != 4 || !strings.Contains(err.Error(), "handback_result_unknown") || *ensureCalls != 0 { + t.Fatalf("ambiguous handback err=%v restart=%d", err, *ensureCalls) + } + state, ok, loadErr := loadCheckpointHandbackIfExists("handback-ambiguous") + if loadErr != nil || !ok || state.Status != "handback-unknown" { + t.Fatalf("ambiguous lifecycle=%+v ok=%v err=%v", state, ok, loadErr) + } + }) + + for _, tc := range []struct { + name string + status int + code string + }{ + {name: "unknown-409", status: 409, code: "conflict"}, + {name: "released-under-another-handback-key", status: 409, code: "checkpoint_handback_conflict"}, + {name: "not-found-after-release-or-gc", status: 404, code: "checkpoint_seal_not_found"}, + {name: "source-already-resumed", status: 409, code: "checkpoint_replayed"}, + {name: "resume-ownership-conflict", status: 409, code: "checkpoint_resume_conflict"}, + } { + tc := tc + t.Run(tc.name+" stays stopped and ambiguous", func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, issued := checkpointTestActive(t) + receipt := consumedCheckpointTestReceipt(issued) + _, ensureCalls, _ := installCheckpointLifecycleSeams(t, active, receipt) + checkpointHandbackStopped = func(context.Context, checkpointMountConfig, mountsync.CheckpointSeal, string, string) (mountsync.CheckpointSealOwnership, mountsync.CheckpointVerificationHealth, error) { + return mountsync.CheckpointSealOwnership{}, mountsync.CheckpointVerificationHealth{}, &mountsync.HTTPError{StatusCode: tc.status, Code: tc.code, Message: "ownership outcome is not safe to invert"} + } + handbackID := "handback-" + tc.name + payload, _ := json.Marshal(checkpointHandbackInput{HandbackID: handbackID, ConsumerIdempotencyKey: "cutover-" + tc.name, Receipt: receipt}) + err := runMountHandbackSeal([]string{"--root", active.config.LocalRoot, "--json"}, bytes.NewReader(payload), &bytes.Buffer{}) + if checkpointExitCode(err) != 4 || !strings.Contains(err.Error(), "handback_result_unknown") || *ensureCalls != 0 { + t.Fatalf("ambiguous semantic HTTP error=%v restart=%d", err, *ensureCalls) + } + state, ok, loadErr := loadCheckpointHandbackIfExists(handbackID) + if loadErr != nil || !ok || state.Status != "handback-unknown" { + t.Fatalf("ambiguous semantic lifecycle=%+v ok=%v err=%v", state, ok, loadErr) + } + }) + } + + for _, tc := range []struct { + name string + status int + code string + retryCount int + }{ + {name: "bad-request", status: 400, code: "bad_request", retryCount: 1}, + {name: "unauthorized", status: 401, code: "unauthorized", retryCount: 2}, + {name: "forbidden", status: 403, code: "forbidden", retryCount: 2}, + {name: "payload-too-large", status: 413, code: "payload_too_large", retryCount: 1}, + } { + tc := tc + t.Run(tc.name+" after committed response loss stays stopped", func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, issued := checkpointTestActive(t) + receipt := consumedCheckpointTestReceipt(issued) + lease, ensureCalls, _ := installCheckpointLifecycleSeams(t, active, receipt) + committed := false + handbackCalls := 0 + checkpointHandbackStopped = func(context.Context, checkpointMountConfig, mountsync.CheckpointSeal, string, string) (mountsync.CheckpointSealOwnership, mountsync.CheckpointVerificationHealth, error) { + handbackCalls++ + committed = true + return mountsync.CheckpointSealOwnership{}, mountsync.CheckpointVerificationHealth{}, &mountsync.HTTPError{ + StatusCode: tc.status, + Code: tc.code, + Message: "retry response received after the original POST committed", + } + } + handbackID := "handback-postcommit-" + tc.name + payload, _ := json.Marshal(checkpointHandbackInput{HandbackID: handbackID, ConsumerIdempotencyKey: "cutover-postcommit-" + tc.name, Receipt: receipt}) + args := []string{"--root", active.config.LocalRoot, "--json"} + for attempt := 0; attempt < tc.retryCount; attempt++ { + var output bytes.Buffer + err := runMountHandbackSeal(args, bytes.NewReader(payload), &output) + if checkpointExitCode(err) != 4 || !strings.Contains(err.Error(), "handback_result_unknown") || output.Len() != 0 || *ensureCalls != 0 || !committed { + t.Fatalf("attempt %d err=%v output=%q restart=%d committed=%v", attempt+1, err, output.String(), *ensureCalls, committed) + } + state, ok, loadErr := loadCheckpointHandbackIfExists(handbackID) + if loadErr != nil || !ok || state.Status != "handback-unknown" || state.Result != nil { + t.Fatalf("attempt %d lifecycle=%+v ok=%v err=%v", attempt+1, state, ok, loadErr) + } + } + if !lease.released || handbackCalls != tc.retryCount { + t.Fatalf("lease=%v handbackCalls=%d want=%d", lease.released, handbackCalls, tc.retryCount) + } + }) + } + + t.Run("503 after application commit stays stopped and exact retry recovers proof", func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, issued := checkpointTestActive(t) + receipt := consumedCheckpointTestReceipt(issued) + lease, ensureCalls, _ := installCheckpointLifecycleSeams(t, active, receipt) + committed := false + handbackCalls := 0 + checkpointHandbackStopped = func(_ context.Context, _ checkpointMountConfig, consumed mountsync.CheckpointSeal, consumerKey, handbackKey string) (mountsync.CheckpointSealOwnership, mountsync.CheckpointVerificationHealth, error) { + handbackCalls++ + if consumerKey != "cutover-503-after-commit" || handbackKey != "handback-503-after-commit" { + t.Fatalf("changed idempotency identity consumer=%q handback=%q", consumerKey, handbackKey) + } + proof := mountsync.CheckpointSealOwnership{ + SealID: consumed.SealID, WorkspaceID: consumed.WorkspaceID, Root: consumed.Root, + SessionID: consumed.SessionID, Generation: consumed.Generation, Status: "released", + Digest: consumed.Digest, WorkspaceRevision: "rev_3", EventCursor: consumed.EventCursor, + ConsumedAt: consumed.ConsumedAt, PreparedAt: "2026-08-23T12:00:09Z", ReleasedAt: "2026-08-23T12:00:10Z", + } + if !committed { + committed = true + return mountsync.CheckpointSealOwnership{}, mountsync.CheckpointVerificationHealth{}, &mountsync.HTTPError{StatusCode: 503, Code: "upstream_unavailable", Message: "gateway lost committed response"} + } + return proof, mountsync.CheckpointVerificationHealth{}, nil + } + input := checkpointHandbackInput{HandbackID: "handback-503-after-commit", ConsumerIdempotencyKey: "cutover-503-after-commit", Receipt: receipt} + payload, _ := json.Marshal(input) + args := []string{"--root", active.config.LocalRoot, "--json"} + var first bytes.Buffer + err := runMountHandbackSeal(args, bytes.NewReader(payload), &first) + if checkpointExitCode(err) != 4 || !strings.Contains(err.Error(), "handback_result_unknown") || first.Len() != 0 || *ensureCalls != 0 || !lease.released || !committed { + t.Fatalf("503-after-commit err=%v output=%q restart=%d lease=%v committed=%v", err, first.String(), *ensureCalls, lease.released, committed) + } + state, ok, loadErr := loadCheckpointHandbackIfExists(input.HandbackID) + if loadErr != nil || !ok || state.Status != "handback-unknown" || state.Result != nil { + t.Fatalf("ambiguous 503 lifecycle=%+v ok=%v err=%v", state, ok, loadErr) + } + var retry bytes.Buffer + if err := runMountHandbackSeal(args, bytes.NewReader(payload), &retry); err != nil { + t.Fatalf("idempotent handback recovery: %v", err) + } + if *ensureCalls != 0 || handbackCalls != 2 || !strings.Contains(retry.String(), `"status": "released"`) { + t.Fatalf("recovered handback restart=%d calls=%d output=%s", *ensureCalls, handbackCalls, retry.String()) + } + state, ok, loadErr = loadCheckpointHandbackIfExists(input.HandbackID) + if loadErr != nil || !ok || state.Status != "released" || state.Result == nil { + t.Fatalf("recovered lifecycle=%+v ok=%v err=%v", state, ok, loadErr) + } + }) +} + +func TestCheckpointHandbackHTTPFailureClassification(t *testing.T) { + for _, httpErr := range []*mountsync.HTTPError{ + {StatusCode: 409, Code: "checkpoint_diverged"}, + } { + if !definitiveCheckpointHandbackHTTPFailure(httpErr) { + t.Errorf("HTTP %d/%s should be definitive", httpErr.StatusCode, httpErr.Code) + } + } + for _, httpErr := range []*mountsync.HTTPError{ + {StatusCode: 0}, + {StatusCode: 400, Code: "bad_request"}, + {StatusCode: 400, Code: "gateway_bad_request"}, + {StatusCode: 401, Code: "unauthorized"}, + {StatusCode: 403, Code: "forbidden"}, + {StatusCode: 404, Code: "checkpoint_seal_not_found"}, + {StatusCode: 404, Code: "not_found"}, + {StatusCode: 409, Code: "conflict"}, + {StatusCode: 409, Code: "checkpoint_handback_conflict"}, + {StatusCode: 409, Code: "checkpoint_replayed"}, + {StatusCode: 409, Code: "checkpoint_resume_conflict"}, + {StatusCode: 408, Code: "request_timeout"}, + {StatusCode: 425, Code: "too_early"}, + {StatusCode: 429, Code: "rate_limited"}, + {StatusCode: 413, Code: "payload_too_large"}, + {StatusCode: 500, Code: "internal_error"}, + {StatusCode: 502, Code: "bad_gateway"}, + {StatusCode: 503, Code: "upstream_unavailable"}, + {StatusCode: 504, Code: "gateway_timeout"}, + } { + if definitiveCheckpointHandbackHTTPFailure(httpErr) { + t.Errorf("HTTP %d/%s should remain ambiguous", httpErr.StatusCode, httpErr.Code) + } + } + if definitiveCheckpointHandbackHTTPFailure(errors.New("connection reset after POST")) { + t.Error("transport failure should remain ambiguous") + } +} + +func TestCheckpointAbortAndImmediateResumeSerializeAndRecover(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, receipt := checkpointTestActive(t) + _, ensureCalls, burnCalls := installCheckpointLifecycleSeams(t, active, receipt) + issueStarted := make(chan struct{}) + releaseIssue := make(chan struct{}) + checkpointIssueStopped = func(context.Context, checkpointMountConfig, string, uint64, int) (mountsync.CheckpointSeal, error) { + close(issueStarted) + <-releaseIssue + return mountsync.CheckpointSeal{}, context.Canceled + } + checkpointArgs := []string{"--root", active.config.LocalRoot, "--lifecycle-id", "rsm_abort_resume", "--session", "session-abort", "--generation", "4", "--json"} + checkpointDone := make(chan error, 1) + go func() { checkpointDone <- runMountCheckpointSeal(checkpointArgs, &bytes.Buffer{}) }() + <-issueStarted + resumeDone := make(chan error, 1) + go func() { + resumeDone <- runMountResumeSeal([]string{"--root", active.config.LocalRoot, "--timeout", "5s", "--json"}, strings.NewReader(`{"resumeId":"rsm_abort_resume"}`), &bytes.Buffer{}) + }() + select { + case err := <-resumeDone: + t.Fatalf("resume overtook in-flight checkpoint: %v", err) + case <-time.After(100 * time.Millisecond): + } + close(releaseIssue) + if err := <-checkpointDone; checkpointExitCode(err) != 5 { + t.Fatalf("aborted checkpoint error = %v", err) + } + if err := <-resumeDone; err != nil { + t.Fatalf("immediate resume: %v", err) + } + if *ensureCalls < 2 || *burnCalls != 0 { + t.Fatalf("abort recovery calls ensure=%d burn=%d", *ensureCalls, *burnCalls) + } +} + +func TestCheckpointRestartContractRejectsFUSEAndNeverPersistsTokens(t *testing.T) { + active, _ := checkpointTestActive(t) + active.config.Mode = "fuse" + err := validateCheckpointMountConfig(active.config, active.record, active.config.LocalRoot) + if checkpointExitCode(err) != 2 || !strings.Contains(err.Error(), "checkpoint_fuse_unsupported") { + t.Fatalf("fuse validation error = %v", err) + } + active.config.Mode = defaultMountMode + args := checkpointMountArgs(active.config) + joined := strings.Join(args, " ") + if strings.Contains(joined, "--token") || strings.Contains(joined, "one-use-token") { + t.Fatalf("restart argv contains bearer material: %s", joined) + } + env := checkpointSubprocessEnv([]string{"PATH=/bin", "RELAYFILE_TOKEN=secret", "RELAYFILE_MOUNT_MODE=fuse"}) + if strings.Join(env, " ") != "PATH=/bin" { + t.Fatalf("restart environment retained unsafe overrides: %v", env) + } +} + +func TestCheckpointRestartContractRejectsScopedRootBeforeStop(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, receipt := checkpointTestActive(t) + active.config.RemotePaths = []string{"/sessions"} + active.record.RemotePaths = []string{"/sessions"} + receipt.Root = "/sessions" + stopCalls := 0 + originalResolve := checkpointResolveActive + originalStop := checkpointStopActive + checkpointResolveActive = func(string) (activeCheckpointMount, error) { + if err := validateCheckpointMountConfig(active.config, active.record, active.config.LocalRoot); err != nil { + return activeCheckpointMount{}, err + } + return active, nil + } + checkpointStopActive = func(context.Context, activeCheckpointMount) (checkpointLease, error) { + stopCalls++ + return &fakeCheckpointLease{}, nil + } + t.Cleanup(func() { + checkpointResolveActive = originalResolve + checkpointStopActive = originalStop + }) + err := runMountCheckpointSeal([]string{"--root", active.config.LocalRoot, "--lifecycle-id", "rsm_controller_scoped", "--session", receipt.SessionID, "--generation", "2", "--json"}, &bytes.Buffer{}) + if checkpointExitCode(err) != 2 || !strings.Contains(err.Error(), "checkpoint_topology_unsupported") || stopCalls != 0 { + t.Fatalf("scoped root err=%v stopCalls=%d", err, stopCalls) + } +} + +func TestCheckpointRestartContractNormalizesMissingRemotePathsToRoot(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + active, receipt := checkpointTestActive(t) + active.config.RemotePaths = nil + active.record.RemotePaths = nil + if err := validateCheckpointMountConfig(active.config, active.record, active.config.LocalRoot); err != nil { + t.Fatalf("legacy root mount validation: %v", err) + } + installCheckpointLifecycleSeams(t, active, receipt) + + var output bytes.Buffer + err := runMountCheckpointSeal([]string{ + "--root", active.config.LocalRoot, + "--lifecycle-id", "rsm_controller_legacy_root", + "--session", receipt.SessionID, + "--generation", "2", + "--json", + }, &output) + if err != nil { + t.Fatalf("checkpoint with omitted remotePaths: %v", err) + } + var envelope checkpointSealEnvelope + if err := json.Unmarshal(output.Bytes(), &envelope); err != nil { + t.Fatalf("decode checkpoint envelope: %v", err) + } + state, err := loadCheckpointLifecycle("rsm_controller_legacy_root") + if err != nil { + t.Fatalf("load checkpoint lifecycle: %v", err) + } + if state.RemoteRoot != "/" || envelope.Receipt.Root != "/" { + t.Fatalf("omitted remotePaths did not normalize to root: %+v", envelope) + } +} + +func TestCheckpointLifecycleWaitLockRejectsNilContext(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + if _, err := acquireCheckpointLifecycleLockWait(nil, t.TempDir()); err == nil || !strings.Contains(err.Error(), "requires a context") { + t.Fatalf("nil wait context error = %v", err) + } +} + +func TestStopCheckpointMountSignalsRealProcessAndWaitsForLease(t *testing.T) { + if os.Getenv("RELAYFILE_CHECKPOINT_HELPER") == "1" { + t.Skip("helper branch is handled by TestCheckpointLifecycleHelperProcess") + } + root := t.TempDir() + server := "https://relayfile.stop.test" + workspace := "ws_stop_real" + readyReader, readyWriter, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer readyReader.Close() + cmd := exec.Command(os.Args[0], "-test.run=TestCheckpointLifecycleHelperProcess") + cmd.ExtraFiles = []*os.File{readyWriter} + cmd.Env = append(os.Environ(), + "RELAYFILE_CHECKPOINT_HELPER=1", + "RELAYFILE_CHECKPOINT_HELPER_SERVER="+server, + "RELAYFILE_CHECKPOINT_HELPER_WORKSPACE="+workspace, + "RELAYFILE_CHECKPOINT_HELPER_ROOT="+root, + ) + if err := cmd.Start(); err != nil { + _ = readyWriter.Close() + t.Fatal(err) + } + _ = readyWriter.Close() + defer func() { + if cmd.ProcessState == nil { + _ = cmd.Process.Kill() + } + }() + waited := make(chan error, 1) + go func() { waited <- cmd.Wait() }() + if err := readyReader.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + t.Fatal(err) + } + ready := make([]byte, 1) + if n, err := readyReader.Read(ready); err != nil || n != 1 || ready[0] != 1 { + t.Fatalf("helper readiness n=%d byte=%v err=%v", n, ready, err) + } + if lease, err := mountlease.Acquire(server, workspace, root); !errors.Is(err, mountlease.ErrHeld) { + if err == nil { + _ = lease.Release() + } + t.Fatalf("helper signaled ready without holding mount lease: %v", err) + } + active := activeCheckpointMount{pid: daemonPIDState{PID: cmd.Process.Pid}, config: checkpointMountConfig{Server: server, WorkspaceID: workspace, LocalRoot: root}} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + lease, err := stopCheckpointMount(ctx, active) + cancel() + if err != nil { + t.Fatalf("stop real daemon: %v", err) + } + _ = lease.Release() + if err := <-waited; err != nil { + t.Fatalf("helper exit: %v", err) + } +} + +func TestCheckpointLifecycleHelperProcess(t *testing.T) { + if os.Getenv("RELAYFILE_CHECKPOINT_HELPER") != "1" { + return + } + signals := make(chan os.Signal, 1) + signalNotify(signals) + defer signal.Stop(signals) + lease, err := mountlease.Acquire( + os.Getenv("RELAYFILE_CHECKPOINT_HELPER_SERVER"), + os.Getenv("RELAYFILE_CHECKPOINT_HELPER_WORKSPACE"), + os.Getenv("RELAYFILE_CHECKPOINT_HELPER_ROOT"), + ) + if err != nil { + os.Exit(41) + } + readyWriter := os.NewFile(uintptr(3), "checkpoint-helper-ready") + if readyWriter == nil { + _ = lease.Release() + os.Exit(42) + } + if _, err := readyWriter.Write([]byte{1}); err != nil { + _ = readyWriter.Close() + _ = lease.Release() + os.Exit(43) + } + _ = readyWriter.Close() + <-signals + _ = lease.Release() +} + +func signalNotify(ch chan<- os.Signal) { + // Kept behind a helper so the real-process test remains small and the + // production signal path is still the one exercised by stopCheckpointMount. + signal.Notify(ch, syscall.SIGTERM) +} + +func checkpointExitCode(err error) int { + if err == nil { + return 0 + } + var coded interface{ ExitCode() int } + if errors.As(err, &coded) { + return coded.ExitCode() + } + return 1 +} diff --git a/cmd/relayfile-cli/main.go b/cmd/relayfile-cli/main.go index 391847fe..948a43fb 100644 --- a/cmd/relayfile-cli/main.go +++ b/cmd/relayfile-cli/main.go @@ -528,6 +528,10 @@ type daemonPIDState struct { // stop/restart confirm a recorded PID still belongs to a relayfile // daemon before signaling it, guarding against PID reuse. Executable string `json:"executable,omitempty"` + // CheckpointConfig is the complete non-secret restart contract for this + // daemon. It is copied into a private resume record before live migration; + // bearer tokens are never persisted in argv or this catalog. + CheckpointConfig *checkpointMountConfig `json:"checkpointConfig,omitempty"` } type mountDaemonProcess struct { @@ -590,7 +594,12 @@ func main() { log.SetFlags(0) if err := run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr); err != nil { fmt.Fprintln(os.Stderr, "error:", err) - os.Exit(1) + exitCode := 1 + var coded interface{ ExitCode() int } + if errors.As(err, &coded) { + exitCode = coded.ExitCode() + } + os.Exit(exitCode) } } @@ -635,6 +644,18 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) error { // `start` and `on` are friendlier aliases for `mount`. Same flags, // same foreground/background behavior; pass --background to detach. // `on` migrates the agent-relay `relay on` mount UX into relayfile. + if len(args) > 1 && args[1] == "checkpoint-seal" { + return runMountCheckpointSeal(args[2:], stdout) + } + if len(args) > 1 && args[1] == "resume-seal" { + return runMountResumeSeal(args[2:], stdin, stdout) + } + if len(args) > 1 && args[1] == "verify-seal" { + return runMountVerifySeal(args[2:], stdin, stdout) + } + if len(args) > 1 && args[1] == "handback-seal" { + return runMountHandbackSeal(args[2:], stdin, stdout) + } return runMount(args[1:]) case "restart": return runRestart(args[1:], stdout) @@ -738,7 +759,17 @@ func printHelpForArgs(args []string, stdout io.Writer) { case "pull": fmt.Fprintln(stdout, "Usage: relayfile pull [--workspace NAME] [--provider PROVIDER] [--reason TEXT]") case "mount", "start", "on": - printMountHelp(stdout) + if subcommand == "checkpoint-seal" { + fmt.Fprintln(stdout, "Usage: relayfile mount checkpoint-seal --root ABS_LOCAL_ROOT --lifecycle-id STABLE_ID --session ID --generation N [--timeout 30s] [--ttl 60s] --json") + } else if subcommand == "resume-seal" { + fmt.Fprintln(stdout, "Usage: printf '{\"resumeId\":\"...\"}' | relayfile mount resume-seal --root ABS_LOCAL_ROOT [--timeout 60s] --json") + } else if subcommand == "verify-seal" { + fmt.Fprintln(stdout, "Usage: printf '{\"verificationId\":\"...\",\"receipt\":{...}}' | relayfile mount verify-seal --root ABS_LOCAL_ROOT [--timeout 60s] --json") + } else if subcommand == "handback-seal" { + fmt.Fprintln(stdout, "Usage: printf '{\"handbackId\":\"...\",\"consumerIdempotencyKey\":\"...\",\"receipt\":{...}}' | relayfile mount handback-seal --root ABS_LOCAL_ROOT [--timeout 60s] --json") + } else { + printMountHelp(stdout) + } case "restart": fmt.Fprintln(stdout, "Usage: relayfile restart [WORKSPACE] [--foreground]") case "tree", "ls": @@ -7545,6 +7576,32 @@ func runMount(args []string) error { LogFile: logFile, StartedAt: time.Now().UTC().Format(time.RFC3339), Executable: resolvedSelfExecutable(), + CheckpointConfig: &checkpointMountConfig{ + Version: 1, + Server: strings.TrimRight(strings.TrimSpace(*server), "/"), + CredentialsFile: absolutePathIfSet(delegatedCredsPath), + WorkspaceID: workspaceID, + LocalRoot: absLocalDir, + RemotePaths: append([]string(nil), record.RemotePaths...), + LocalLayout: resolvedLocalLayout, + EventProvider: strings.TrimSpace(*eventProvider), + StateFile: absolutePathIfSet(*stateFile), + StateDir: absolutePathIfSet(*stateDir), + MountKind: mountsync.NormalizeMountKind(*mountKind), + Mode: defaultMountMode, + Interval: interval.String(), + IntervalJitter: *intervalJitter, + Timeout: timeout.String(), + BootstrapTimeout: bootstrapTimeout.String(), + BootstrapMaxFilesPerCycle: *bootstrapMaxFiles, + FullPullMinInterval: fullPullMinInterval.String(), + CursorTimeout: cursorTimeout.String(), + ForceFullReconcile: *fullReconcile, + WebsocketEnabled: *websocketEnabled, + LowMemory: *lowMemory, + PprofAddr: strings.TrimSpace(*pprofAddr), + MemlogInterval: memlogInterval.String(), + }, }); err != nil { return err } diff --git a/cmd/relayfile-mount/main.go b/cmd/relayfile-mount/main.go index f299ef35..97f7556e 100644 --- a/cmd/relayfile-mount/main.go +++ b/cmd/relayfile-mount/main.go @@ -75,6 +75,10 @@ type mountConfig struct { once bool flushOutboxOnce bool pushLocalOnce bool + checkpointAndSeal bool + checkpointSession string + checkpointGeneration uint64 + checkpointSealTTL time.Duration mode string fuseContentTTL time.Duration } @@ -125,6 +129,10 @@ func main() { once := flag.Bool("once", false, "run one sync cycle and exit") flushOutboxOnce := flag.Bool("flush-outbox-once", false, "flush durable writeback outbox once and exit without reconciling the local mirror") pushLocalOnce := flag.Bool("push-local-once", false, "ingest pending local writeback drafts (one pushLocal pass) then flush the outbox once and exit; no pullRemote/digest/reconcile — the teardown drain for last-moment drafts") + checkpointAndSeal := flag.Bool("checkpoint-and-seal", false, "drain a managed mount, verify durable convergence, emit a one-use server checkpoint seal as JSON, and exit") + checkpointSession := flag.String("checkpoint-session", "", "session identifier bound into --checkpoint-and-seal") + checkpointGeneration := flag.Uint64("checkpoint-generation", 0, "strictly increasing migration generation bound into --checkpoint-and-seal") + checkpointSealTTL := flag.Duration("checkpoint-seal-ttl", mountsync.DefaultCheckpointSealTTL, "one-use checkpoint seal lifetime (maximum 5m)") flag.Parse() fullPullMinInterval, err := parseDurationWithNegativeOne(*fullPullMinIntervalArg) if err != nil { @@ -220,6 +228,10 @@ func main() { once: *once, flushOutboxOnce: *flushOutboxOnce, pushLocalOnce: *pushLocalOnce, + checkpointAndSeal: *checkpointAndSeal, + checkpointSession: strings.TrimSpace(*checkpointSession), + checkpointGeneration: *checkpointGeneration, + checkpointSealTTL: *checkpointSealTTL, mode: resolvedMode, fuseContentTTL: *fuseContentTTL, } @@ -276,6 +288,18 @@ func resolveSyncMode(mode string) (string, error) { } func executeMount(rootCtx context.Context, cfg mountConfig, runPoll pollRunner, runFuse fuseRunner) error { + if cfg.checkpointAndSeal && cfg.mode != mountModePoll { + return fmt.Errorf("--checkpoint-and-seal requires --mode=%s", mountModePoll) + } + if cfg.checkpointAndSeal && (cfg.once || cfg.flushOutboxOnce || cfg.pushLocalOnce) { + return errors.New("--checkpoint-and-seal cannot be combined with --once, --flush-outbox-once, or --push-local-once") + } + if cfg.checkpointAndSeal && (cfg.checkpointSession == "" || cfg.checkpointGeneration == 0) { + return errors.New("--checkpoint-and-seal requires --checkpoint-session and a positive --checkpoint-generation") + } + if cfg.checkpointAndSeal && (cfg.checkpointSealTTL < time.Second || cfg.checkpointSealTTL > mountsync.MaxCheckpointSealTTL) { + return fmt.Errorf("--checkpoint-seal-ttl must be between 1s and %s", mountsync.MaxCheckpointSealTTL) + } if cfg.mode == mountModeFuse && cfg.syncMode == syncModePullOnly { return fmt.Errorf("--sync-mode=%s is not supported with --mode=%s; use --mode=%s", syncModePullOnly, mountModeFuse, mountModePoll) } @@ -465,6 +489,21 @@ func runSinglePollingMount(rootCtx context.Context, cfg mountConfig) error { if _, err := mountsync.StartDiagnostics(rootCtx, cfg.pprofAddr, cfg.memlogInterval, log.Default()); err != nil { return fmt.Errorf("start diagnostics: %w", err) } + if cfg.checkpointAndSeal { + ctx, cancel := context.WithTimeout(rootCtx, checkpointOperationTimeout(cfg.timeout)) + defer cancel() + seal, err := syncer.CheckpointAndSeal(ctx, mountsync.CheckpointAndSealOptions{ + SessionID: cfg.checkpointSession, Generation: cfg.checkpointGeneration, + TTLSeconds: int(cfg.checkpointSealTTL / time.Second), + }) + if err != nil { + return fmt.Errorf("checkpoint and seal: %w", err) + } + if err := json.NewEncoder(os.Stdout).Encode(seal); err != nil { + return fmt.Errorf("encode checkpoint seal: %w", err) + } + return nil + } if cfg.pushLocalOnce { ctx, cancel := context.WithTimeout(rootCtx, cfg.timeout) defer cancel() @@ -607,6 +646,14 @@ func runSinglePollingMount(rootCtx context.Context, cfg mountConfig) error { } } +func checkpointOperationTimeout(configured time.Duration) time.Duration { + const minimum = 30 * time.Second + if configured < minimum { + return minimum + } + return configured +} + type mountCredsFile struct { Token string `json:"token"` AccessToken string `json:"accessToken,omitempty"` diff --git a/cmd/relayfile-mount/main_test.go b/cmd/relayfile-mount/main_test.go index 07d0c6fc..0c405912 100644 --- a/cmd/relayfile-mount/main_test.go +++ b/cmd/relayfile-mount/main_test.go @@ -188,6 +188,69 @@ func TestResolveMountMode(t *testing.T) { } } +func TestCheckpointAndSealCLIValidationFailsBeforeStartingMount(t *testing.T) { + tests := []struct { + name string + cfg mountConfig + want string + }{ + { + name: "fuse stays mounted", + cfg: mountConfig{checkpointAndSeal: true, mode: mountModeFuse, checkpointSession: "session-1", checkpointGeneration: 1, checkpointSealTTL: time.Minute}, + want: "requires --mode=poll", + }, + { + name: "identity required", + cfg: mountConfig{checkpointAndSeal: true, mode: mountModePoll, checkpointSealTTL: time.Minute}, + want: "requires --checkpoint-session", + }, + { + name: "ttl bounded", + cfg: mountConfig{checkpointAndSeal: true, mode: mountModePoll, checkpointSession: "session-1", checkpointGeneration: 1, checkpointSealTTL: mountsync.MaxCheckpointSealTTL + time.Second}, + want: "--checkpoint-seal-ttl must be between", + }, + { + name: "one-shot modes are exclusive", + cfg: mountConfig{checkpointAndSeal: true, once: true, mode: mountModePoll, checkpointSession: "session-1", checkpointGeneration: 1, checkpointSealTTL: time.Minute}, + want: "cannot be combined", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + pollCalled := false + fuseCalled := false + err := executeMount( + context.Background(), + tc.cfg, + func(context.Context, mountConfig) error { pollCalled = true; return nil }, + func(context.Context, mountConfig) error { fuseCalled = true; return nil }, + ) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("expected error containing %q, got %v", tc.want, err) + } + if pollCalled || fuseCalled { + t.Fatal("invalid checkpoint request must fail before starting a mount") + } + }) + } +} + +func TestCheckpointOperationTimeoutHasThirtySecondFloor(t *testing.T) { + for _, tc := range []struct { + configured time.Duration + want time.Duration + }{ + {configured: 0, want: 30 * time.Second}, + {configured: 15 * time.Second, want: 30 * time.Second}, + {configured: 30 * time.Second, want: 30 * time.Second}, + {configured: 45 * time.Second, want: 45 * time.Second}, + } { + if got := checkpointOperationTimeout(tc.configured); got != tc.want { + t.Fatalf("checkpointOperationTimeout(%s) = %s, want %s", tc.configured, got, tc.want) + } + } +} + func TestResolveLocalLayout(t *testing.T) { tests := []struct { name string diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 20982392..0ac4182f 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -2,6 +2,7 @@ package httpapi import ( "archive/tar" + "bytes" "compress/gzip" "encoding/base64" "encoding/json" @@ -134,12 +135,20 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.handleAdminSync(w, r) return } + if r.URL.Path == "/v1/admin/checkpoint-seals" && r.Method == http.MethodGet { + s.handleAdminCheckpointSeals(w, r) + return + } parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/"), "/") if len(parts) >= 5 && parts[0] == "v1" && parts[1] == "admin" && parts[2] == "replay" { s.handleAdminReplay(w, r, parts) return } + if len(parts) == 5 && parts[0] == "v1" && parts[1] == "admin" && parts[2] == "checkpoint-seals" && parts[4] == "reconcile-source" { + s.handleAdminCheckpointSealReconcile(w, r, parts[3]) + return + } if len(parts) < 4 || parts[0] != "v1" || parts[1] != "workspaces" { writeError(w, http.StatusNotFound, "not_found", "route not found", getCorrelationID(r)) @@ -222,6 +231,24 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { case len(parts) == 5 && parts[3] == "sync" && parts[4] == "refresh" && r.Method == http.MethodPost: requiredScope = "sync:trigger" route = "sync_refresh" + case len(parts) == 5 && parts[3] == "sync" && parts[4] == "checkpoint-seals" && r.Method == http.MethodPost: + requiredScope = "sync:trigger" + route = "checkpoint_seal_issue" + case len(parts) == 6 && parts[3] == "sync" && parts[4] == "checkpoint-seals" && parts[5] == "consume" && r.Method == http.MethodPost: + requiredScope = "sync:trigger" + route = "checkpoint_seal_consume" + case len(parts) == 6 && parts[3] == "sync" && parts[4] == "checkpoint-seals" && parts[5] == "recover-consume" && r.Method == http.MethodPost: + requiredScope = "sync:trigger" + route = "checkpoint_seal_recover_consume" + case len(parts) == 6 && parts[3] == "sync" && parts[4] == "checkpoint-seals" && parts[5] == "verify" && r.Method == http.MethodPost: + requiredScope = "sync:trigger" + route = "checkpoint_seal_verify" + case len(parts) == 6 && parts[3] == "sync" && parts[4] == "checkpoint-seals" && parts[5] == "handback" && r.Method == http.MethodPost: + requiredScope = "sync:trigger" + route = "checkpoint_seal_handback" + case len(parts) == 6 && parts[3] == "sync" && parts[4] == "checkpoint-seals" && parts[5] == "resume" && r.Method == http.MethodPost: + requiredScope = "sync:trigger" + route = "checkpoint_seal_resume" case len(parts) == 4 && parts[3] == "ops" && r.Method == http.MethodGet: requiredScope = "ops:read" route = "ops_list" @@ -354,6 +381,18 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.handleSyncDeadLetterReplay(w, r, workspaceID, parts[5], correlationID) case "sync_refresh": s.handleSyncRefresh(w, r, workspaceID, correlationID) + case "checkpoint_seal_issue": + s.handleCheckpointSealIssue(w, r, workspaceID, correlationID, claims) + case "checkpoint_seal_consume": + s.handleCheckpointSealConsume(w, r, workspaceID, correlationID, claims) + case "checkpoint_seal_recover_consume": + s.handleCheckpointSealRecoverConsume(w, r, workspaceID, correlationID, claims) + case "checkpoint_seal_verify": + s.handleCheckpointSealVerify(w, r, workspaceID, correlationID, claims) + case "checkpoint_seal_handback": + s.handleCheckpointSealHandback(w, r, workspaceID, correlationID, claims) + case "checkpoint_seal_resume": + s.handleCheckpointSealResume(w, r, workspaceID, correlationID, claims) case "ops_list": s.handleOpsList(w, r, workspaceID, correlationID) case "op_replay": @@ -485,6 +524,63 @@ func (s *Server) handleAdminBackends(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, s.store.GetBackendStatus()) } +func (s *Server) handleAdminCheckpointSeals(w http.ResponseWriter, r *http.Request) { + claims, authErr := authorizeBearer(r.Header.Get("Authorization"), s.bearerVerifier, "", "", "", time.Now().UTC()) + if authErr != nil { + writeError(w, authErr.status, authErr.code, authErr.message, getCorrelationID(r)) + return + } + if !hasAnyScope(claims.Scopes, "admin:read", "admin:replay") { + writeError(w, http.StatusForbidden, "forbidden", "missing required scope: admin:read", getCorrelationID(r)) + return + } + correlationID := getCorrelationID(r) + if correlationID == "" { + writeError(w, http.StatusBadRequest, "bad_request", "missing X-Correlation-Id header", "") + return + } + writeJSON(w, http.StatusOK, s.store.GetCheckpointSealRetentionSummary(r.URL.Query().Get("workspaceId"), time.Now().UTC())) +} + +func (s *Server) handleAdminCheckpointSealReconcile(w http.ResponseWriter, r *http.Request, sealID string) { + if r.Method != http.MethodPost { + writeError(w, http.StatusNotFound, "not_found", "route not found", getCorrelationID(r)) + return + } + if _, authErr := authorizeBearer(r.Header.Get("Authorization"), s.bearerVerifier, "", "admin:replay", "", time.Now().UTC()); authErr != nil { + writeError(w, authErr.status, authErr.code, authErr.message, getCorrelationID(r)) + return + } + correlationID := getCorrelationID(r) + if correlationID == "" { + writeError(w, http.StatusBadRequest, "bad_request", "missing X-Correlation-Id header", "") + return + } + var body relayfile.CheckpointSealAdminReconcileRequest + if !s.decodeStrictJSONBody(w, r, correlationID, &body) { + return + } + record, err := s.store.ReconcileCheckpointSealSource(sealID, body, time.Now().UTC()) + if err != nil { + switch { + case errors.Is(err, relayfile.ErrInvalidInput): + writeError(w, http.StatusBadRequest, "bad_request", "invalid checkpoint reconciliation request", correlationID) + case errors.Is(err, relayfile.ErrNotFound): + writeError(w, http.StatusNotFound, "checkpoint_seal_not_found", "checkpoint seal not found", correlationID) + case errors.Is(err, relayfile.ErrCheckpointHandbackRequired): + writeError(w, http.StatusConflict, "checkpoint_handback_required", "consumed destination ownership must be handed back before source reconciliation", correlationID) + case errors.Is(err, relayfile.ErrCheckpointAdminConflict): + writeError(w, http.StatusConflict, "checkpoint_admin_conflict", "checkpoint reconciliation identity conflicts with durable state", correlationID) + default: + log.Printf("checkpoint administrative reconciliation failed (correlationId=%s sealId=%s): %v", correlationID, sealID, err) + writeError(w, http.StatusInternalServerError, "internal_error", "checkpoint reconciliation failed", correlationID) + } + return + } + log.Printf("checkpoint administrative reconciliation completed (correlationId=%s sealId=%s workspaceId=%s status=%s)", correlationID, record.SealID, record.WorkspaceID, record.OwnershipStatus) + writeJSON(w, http.StatusOK, record) +} + func (s *Server) handleAdminIngress(w http.ResponseWriter, r *http.Request) { claims, authErr := authorizeBearer(r.Header.Get("Authorization"), s.bearerVerifier, "", "", "", time.Now().UTC()) if authErr != nil { @@ -2223,6 +2319,194 @@ func (s *Server) handleSyncRefresh(w http.ResponseWriter, r *http.Request, works writeJSON(w, http.StatusAccepted, resp) } +func (s *Server) handleCheckpointSealIssue(w http.ResponseWriter, r *http.Request, workspaceID, correlationID string, claims tokenClaims) { + var body relayfile.CheckpointSealRequest + if !s.decodeStrictJSONBody(w, r, correlationID, &body) { + return + } + if strings.TrimSpace(body.IssuanceIdempotencyKey) == "" { + writeError(w, http.StatusBadRequest, "bad_request", "issuanceIdempotencyKey is required", correlationID) + return + } + root, err := relayfile.NormalizeCheckpointRoot(body.Root) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "invalid checkpoint root", correlationID) + return + } + // A seal certifies the complete root, not a caller-selected projection. + // Require both read and write authority over that root; path-scoped tokens + // remain supported when they cover the exact subtree. If ACLs hide any + // descendant, the server-computed digest cannot match the mount's digest and + // issuance fails closed as checkpoint_diverged. + if !scopeMatchesPath(claims.Scopes, "fs:read", root) || !scopeMatchesPath(claims.Scopes, "fs:write", root) { + writeError(w, http.StatusForbidden, "forbidden", "checkpoint sealing requires fs:read and fs:write authority for the complete root", correlationID) + return + } + body.Root = root + body.Issuer = claims.AgentName + seal, err := s.store.IssueCheckpointSeal(workspaceID, body, time.Now().UTC()) + if err != nil { + writeCheckpointSealError(w, err, correlationID) + return + } + writeJSON(w, http.StatusCreated, seal) +} + +func (s *Server) handleCheckpointSealConsume(w http.ResponseWriter, r *http.Request, workspaceID, correlationID string, claims tokenClaims) { + var body relayfile.CheckpointSealConsumeRequest + if !s.decodeStrictJSONBody(w, r, correlationID, &body) { + return + } + root, err := relayfile.NormalizeCheckpointRoot(body.Root) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "invalid checkpoint root", correlationID) + return + } + if !scopeMatchesPath(claims.Scopes, "fs:read", root) || !scopeMatchesPath(claims.Scopes, "fs:write", root) { + writeError(w, http.StatusForbidden, "forbidden", "checkpoint consume requires fs:read and fs:write authority for the complete root", correlationID) + return + } + body.Root = root + body.ConsumerPrincipal = claims.AgentName + seal, err := s.store.ConsumeCheckpointSeal(workspaceID, body, time.Now().UTC()) + if err != nil { + writeCheckpointSealError(w, err, correlationID) + return + } + writeJSON(w, http.StatusOK, seal) +} + +func (s *Server) handleCheckpointSealRecoverConsume(w http.ResponseWriter, r *http.Request, workspaceID, correlationID string, claims tokenClaims) { + var body relayfile.CheckpointSealConsumeRecoveryRequest + if !s.decodeStrictJSONBody(w, r, correlationID, &body) { + return + } + root, err := relayfile.NormalizeCheckpointRoot(body.Root) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "invalid checkpoint root", correlationID) + return + } + if !scopeMatchesPath(claims.Scopes, "fs:read", root) { + writeError(w, http.StatusForbidden, "forbidden", "consume recovery requires fs:read authority for the complete root", correlationID) + return + } + body.Root = root + body.ConsumerPrincipal = claims.AgentName + seal, err := s.store.RecoverConsumedCheckpointSeal(workspaceID, body, time.Now().UTC()) + if err != nil { + writeCheckpointSealError(w, err, correlationID) + return + } + writeJSON(w, http.StatusOK, seal) +} + +func (s *Server) handleCheckpointSealVerify(w http.ResponseWriter, r *http.Request, workspaceID, correlationID string, claims tokenClaims) { + var body relayfile.CheckpointSealVerifyRequest + if !s.decodeStrictJSONBodyWithMessage(w, r, correlationID, &body, "invalid checkpoint verification body") { + return + } + root, err := relayfile.NormalizeCheckpointRoot(body.Root) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "invalid checkpoint root", correlationID) + return + } + if !scopeMatchesPath(claims.Scopes, "fs:read", root) { + writeError(w, http.StatusForbidden, "forbidden", "checkpoint verification requires fs:read authority for the complete root", correlationID) + return + } + body.Root = root + body.ConsumerPrincipal = claims.AgentName + seal, err := s.store.VerifyConsumedCheckpointSeal(workspaceID, body, time.Now().UTC()) + if err != nil { + writeCheckpointSealError(w, err, correlationID) + return + } + writeJSON(w, http.StatusOK, seal) +} + +func (s *Server) handleCheckpointSealHandback(w http.ResponseWriter, r *http.Request, workspaceID, correlationID string, claims tokenClaims) { + var body relayfile.CheckpointSealHandbackRequest + if !s.decodeStrictJSONBody(w, r, correlationID, &body) { + return + } + root, err := relayfile.NormalizeCheckpointRoot(body.Root) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "invalid checkpoint root", correlationID) + return + } + if !scopeMatchesPath(claims.Scopes, "fs:read", root) || !scopeMatchesPath(claims.Scopes, "fs:write", root) { + writeError(w, http.StatusForbidden, "forbidden", "checkpoint handback requires fs:read and fs:write authority for the complete root", correlationID) + return + } + body.Root = root + body.ConsumerPrincipal = claims.AgentName + proof, err := s.store.HandbackCheckpointSeal(workspaceID, body, time.Now().UTC()) + if err != nil { + writeCheckpointSealError(w, err, correlationID) + return + } + writeJSON(w, http.StatusOK, proof) +} + +func (s *Server) handleCheckpointSealResume(w http.ResponseWriter, r *http.Request, workspaceID, correlationID string, claims tokenClaims) { + var body relayfile.CheckpointSealResumeRequest + if !s.decodeStrictJSONBody(w, r, correlationID, &body) { + return + } + root, err := relayfile.NormalizeCheckpointRoot(body.Root) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "invalid checkpoint root", correlationID) + return + } + if !scopeMatchesPath(claims.Scopes, "fs:read", root) || !scopeMatchesPath(claims.Scopes, "fs:write", root) { + writeError(w, http.StatusForbidden, "forbidden", "checkpoint source resume requires fs:read and fs:write authority for the complete root", correlationID) + return + } + body.Root = root + proof, err := s.store.ResumeCheckpointSeal(workspaceID, body, time.Now().UTC()) + if err != nil { + writeCheckpointSealError(w, err, correlationID) + return + } + writeJSON(w, http.StatusOK, proof) +} + +func writeCheckpointSealError(w http.ResponseWriter, err error, correlationID string) { + switch { + case errors.Is(err, relayfile.ErrInvalidInput): + writeError(w, http.StatusBadRequest, "bad_request", err.Error(), correlationID) + case errors.Is(err, relayfile.ErrNotFound): + writeError(w, http.StatusNotFound, "checkpoint_seal_not_found", err.Error(), correlationID) + case errors.Is(err, relayfile.ErrCheckpointDiverged): + writeError(w, http.StatusConflict, "checkpoint_diverged", err.Error(), correlationID) + case errors.Is(err, relayfile.ErrCheckpointGenerationStale): + writeError(w, http.StatusConflict, "checkpoint_generation_stale", err.Error(), correlationID) + case errors.Is(err, relayfile.ErrCheckpointIssuanceConflict): + writeError(w, http.StatusConflict, "checkpoint_issuance_conflict", err.Error(), correlationID) + case errors.Is(err, relayfile.ErrCheckpointConsumerConflict): + writeError(w, http.StatusConflict, "checkpoint_consumer_conflict", err.Error(), correlationID) + case errors.Is(err, relayfile.ErrCheckpointUnconsumed): + writeError(w, http.StatusConflict, "checkpoint_unconsumed", err.Error(), correlationID) + case errors.Is(err, relayfile.ErrCheckpointHandbackRequired): + writeError(w, http.StatusConflict, "checkpoint_handback_required", err.Error(), correlationID) + case errors.Is(err, relayfile.ErrCheckpointHandbackUnprepared): + writeError(w, http.StatusConflict, "checkpoint_handback_unprepared", err.Error(), correlationID) + case errors.Is(err, relayfile.ErrCheckpointHandbackConflict): + writeError(w, http.StatusConflict, "checkpoint_handback_conflict", err.Error(), correlationID) + case errors.Is(err, relayfile.ErrCheckpointResumeConflict): + writeError(w, http.StatusConflict, "checkpoint_resume_conflict", err.Error(), correlationID) + case errors.Is(err, relayfile.ErrCheckpointExpired): + writeError(w, http.StatusConflict, "checkpoint_expired", err.Error(), correlationID) + case errors.Is(err, relayfile.ErrCheckpointReplay): + writeError(w, http.StatusConflict, "checkpoint_replayed", err.Error(), correlationID) + case errors.Is(err, relayfile.ErrCheckpointStale): + writeError(w, http.StatusConflict, "checkpoint_stale", err.Error(), correlationID) + default: + log.Printf("checkpoint seal operation failed (correlationId=%s): %v", correlationID, err) + writeError(w, http.StatusInternalServerError, "internal_error", "checkpoint seal operation failed", correlationID) + } +} + func (s *Server) handleOpsList(w http.ResponseWriter, r *http.Request, workspaceID, correlationID string) { limit := parseBoundedInt(r.URL.Query().Get("limit"), 100, 1, 1000) feed, err := s.store.ListOperations( @@ -2486,6 +2770,24 @@ func (s *Server) decodeJSONBody(w http.ResponseWriter, r *http.Request, correlat return true } +func (s *Server) decodeStrictJSONBody(w http.ResponseWriter, r *http.Request, correlationID string, dst any) bool { + return s.decodeStrictJSONBodyWithMessage(w, r, correlationID, dst, "invalid json body") +} + +func (s *Server) decodeStrictJSONBodyWithMessage(w http.ResponseWriter, r *http.Request, correlationID string, dst any, invalidMessage string) bool { + body, ok := s.readRequestBody(w, r, correlationID) + if !ok { + return false + } + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(dst); err != nil || decoder.Decode(&struct{}{}) != io.EOF { + writeError(w, http.StatusBadRequest, "bad_request", invalidMessage, correlationID) + return false + } + return true +} + func stringPropertiesFromAny(values map[string]any) map[string]string { if len(values) == 0 { return nil diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index f8f1a927..c39c8163 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -14,6 +14,8 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" + "path/filepath" "strconv" "strings" "sync/atomic" @@ -7010,6 +7012,476 @@ func newForkTestServer(t *testing.T) http.Handler { return NewServer(store) } +func TestCheckpointSealHTTPServerComputesAndConsumesAuthoritativeSeal(t *testing.T) { + store := relayfile.NewStoreWithOptions(relayfile.StoreOptions{DisableWorkers: true}) + t.Cleanup(store.Close) + write, err := store.WriteFile(relayfile.WriteRequest{ + WorkspaceID: "ws_checkpoint_http", + Path: "/sessions/transcript.jsonl", + IfMatch: "0", + Content: "turn one\n", + }) + if err != nil { + t.Fatalf("seed checkpoint file: %v", err) + } + contentHash := sha256.Sum256([]byte("turn one\n")) + digest, err := relayfile.ComputeCheckpointDigest("/sessions", []relayfile.CheckpointDigestEntry{{ + Path: "/sessions/transcript.jsonl", Revision: write.TargetRevision, ContentHash: fmt.Sprintf("%x", contentHash[:]), + }}) + if err != nil { + t.Fatalf("compute checkpoint digest: %v", err) + } + server := NewServer(store) + fullToken := mustTestJWT(t, "dev-secret", "ws_checkpoint_http", "LocalController", []string{"fs:read", "fs:write", "sync:trigger"}, time.Now().Add(time.Hour)) + issue := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals", + headers: map[string]string{ + "Authorization": "Bearer " + fullToken, "X-Correlation-Id": "corr-checkpoint-issue", + }, + body: map[string]any{"root": "/sessions", "sessionId": "thread-http", "generation": 4, "expectedDigest": digest, "issuanceIdempotencyKey": "issue-http-1"}, + }) + if issue.Code != http.StatusCreated { + t.Fatalf("issue status = %d: %s", issue.Code, issue.Body.String()) + } + var seal relayfile.CheckpointSeal + if err := json.NewDecoder(issue.Body).Decode(&seal); err != nil { + t.Fatalf("decode issued seal: %v", err) + } + if seal.Digest != digest || seal.SealToken == "" || seal.EventCursor == "" { + t.Fatalf("issued seal = %+v", seal) + } + firstToken := seal.SealToken + retriedIssue := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals", + headers: map[string]string{ + "Authorization": "Bearer " + fullToken, "X-Correlation-Id": "corr-checkpoint-issue-retry", + }, + body: map[string]any{"root": "/sessions", "sessionId": "thread-http", "generation": 4, "expectedDigest": digest, "issuanceIdempotencyKey": "issue-http-1"}, + }) + var retriedSeal relayfile.CheckpointSeal + if retriedIssue.Code != http.StatusCreated || json.NewDecoder(retriedIssue.Body).Decode(&retriedSeal) != nil { + t.Fatalf("retry issue = %d: %s", retriedIssue.Code, retriedIssue.Body.String()) + } + if retriedSeal.SealID != seal.SealID || retriedSeal.SealToken == "" || retriedSeal.SealToken == firstToken { + t.Fatalf("retry did not preserve seal identity and rotate bearer: first=%+v retry=%+v", seal, retriedSeal) + } + seal = retriedSeal + + syncOnlyToken := mustTestJWT(t, "dev-secret", "ws_checkpoint_http", "CloudAcquire", []string{"sync:trigger"}, time.Now().Add(time.Hour)) + consumeBody := map[string]any{"sealToken": seal.SealToken, "root": "/sessions", "sessionId": "thread-http", "generation": 4, "consumerIdempotencyKey": "cloud-acquire-http-1"} + deniedConsume := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals/consume", + headers: map[string]string{ + "Authorization": "Bearer " + syncOnlyToken, "X-Correlation-Id": "corr-checkpoint-consume-no-files", + }, + body: consumeBody, + }) + if deniedConsume.Code != http.StatusForbidden { + t.Fatalf("consume without complete root authority = %d: %s", deniedConsume.Code, deniedConsume.Body.String()) + } + scopedElsewhereToken := mustTestJWT(t, "dev-secret", "ws_checkpoint_http", "CloudAcquire", []string{"sync:trigger", "relayfile:fs:read:/other/**", "relayfile:fs:write:/other/**"}, time.Now().Add(time.Hour)) + deniedScopedConsume := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals/consume", + headers: map[string]string{ + "Authorization": "Bearer " + scopedElsewhereToken, "X-Correlation-Id": "corr-checkpoint-consume-wrong-scope", + }, + body: consumeBody, + }) + if deniedScopedConsume.Code != http.StatusForbidden { + t.Fatalf("consume with incomplete root authority = %d: %s", deniedScopedConsume.Code, deniedScopedConsume.Body.String()) + } + cloudToken := mustTestJWT(t, "dev-secret", "ws_checkpoint_http", "CloudAcquire", []string{"fs:read", "fs:write", "sync:trigger"}, time.Now().Add(time.Hour)) + lostTokenConsume := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals/consume", + headers: map[string]string{ + "Authorization": "Bearer " + cloudToken, "X-Correlation-Id": "corr-checkpoint-consume-lost-token", + }, + body: map[string]any{"sealToken": firstToken, "root": "/sessions", "sessionId": "thread-http", "generation": 4, "consumerIdempotencyKey": "cloud-acquire-lost-token"}, + }) + if lostTokenConsume.Code != http.StatusNotFound { + t.Fatalf("rotated lost token consume = %d: %s", lostTokenConsume.Code, lostTokenConsume.Body.String()) + } + consume := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals/consume", + headers: map[string]string{ + "Authorization": "Bearer " + cloudToken, "X-Correlation-Id": "corr-checkpoint-consume", + }, + body: consumeBody, + }) + if consume.Code != http.StatusOK { + t.Fatalf("consume status = %d: %s", consume.Code, consume.Body.String()) + } + var consumed relayfile.CheckpointSeal + if err := json.NewDecoder(consume.Body).Decode(&consumed); err != nil { + t.Fatalf("decode consumed seal: %v", err) + } + if consumed.SealToken != "" || consumed.ConsumedAt == "" { + t.Fatalf("consume must return a safe receipt without sealToken: %+v", consumed) + } + recoveryToken := mustTestJWT(t, "dev-secret", "ws_checkpoint_http", "CloudAcquire", []string{"fs:read", "sync:trigger"}, time.Now().Add(2*time.Hour)) + recoverConsume := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals/recover-consume", + headers: map[string]string{ + "Authorization": "Bearer " + recoveryToken, "X-Correlation-Id": "corr-checkpoint-recover-consume", + }, + body: map[string]any{"root": "/sessions", "sessionId": "thread-http", "generation": 4, "consumerIdempotencyKey": "cloud-acquire-http-1"}, + }) + if recoverConsume.Code != http.StatusOK || strings.Contains(recoverConsume.Body.String(), "sealToken") { + t.Fatalf("same-principal consume recovery = %d %s", recoverConsume.Code, recoverConsume.Body.String()) + } + var recovered relayfile.CheckpointSeal + if err := json.NewDecoder(recoverConsume.Body).Decode(&recovered); err != nil || recovered.SealID != consumed.SealID || recovered.ConsumedAt != consumed.ConsumedAt { + t.Fatalf("recovered receipt = %+v err=%v", recovered, err) + } + otherPrincipalToken := mustTestJWT(t, "dev-secret", "ws_checkpoint_http", "OtherAgent", []string{"fs:read", "fs:write", "sync:trigger"}, time.Now().Add(time.Hour)) + wrongPrincipalRecovery := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals/recover-consume", + headers: map[string]string{ + "Authorization": "Bearer " + otherPrincipalToken, "X-Correlation-Id": "corr-checkpoint-recover-wrong-principal", + }, + body: map[string]any{"root": "/sessions", "sessionId": "thread-http", "generation": 4, "consumerIdempotencyKey": "cloud-acquire-http-1"}, + }) + if wrongPrincipalRecovery.Code != http.StatusConflict || !strings.Contains(wrongPrincipalRecovery.Body.String(), "checkpoint_consumer_conflict") { + t.Fatalf("wrong-principal consume recovery = %d %s", wrongPrincipalRecovery.Code, wrongPrincipalRecovery.Body.String()) + } + verifyBody := map[string]any{ + "sealId": consumed.SealID, "root": consumed.Root, "sessionId": consumed.SessionID, + "generation": consumed.Generation, "digest": consumed.Digest, "workspaceRevision": consumed.WorkspaceRevision, + "eventCursor": consumed.EventCursor, "issuedAt": consumed.IssuedAt, "expiresAt": consumed.ExpiresAt, "consumedAt": consumed.ConsumedAt, + } + verify := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals/verify", + headers: map[string]string{ + "Authorization": "Bearer " + recoveryToken, "X-Correlation-Id": "corr-checkpoint-verify", + }, + body: verifyBody, + }) + if verify.Code != http.StatusOK || strings.Contains(verify.Body.String(), "sealToken") { + t.Fatalf("verify status = %d, response must not echo seal token: %s", verify.Code, verify.Body.String()) + } + verifyBody["sealToken"] = seal.SealToken + verifyWithBearer := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals/verify", + headers: map[string]string{ + "Authorization": "Bearer " + recoveryToken, "X-Correlation-Id": "corr-checkpoint-verify-bearer", + }, + body: verifyBody, + }) + if verifyWithBearer.Code != http.StatusBadRequest { + t.Fatalf("verify accepted sealToken in body: %d %s", verifyWithBearer.Code, verifyWithBearer.Body.String()) + } + delete(verifyBody, "sealToken") + verifyWrongPrincipal := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals/verify", + headers: map[string]string{ + "Authorization": "Bearer " + fullToken, "X-Correlation-Id": "corr-checkpoint-verify-wrong-principal", + }, + body: verifyBody, + }) + if verifyWrongPrincipal.Code != http.StatusConflict || !strings.Contains(verifyWrongPrincipal.Body.String(), "checkpoint_consumer_conflict") { + t.Fatalf("verify with wrong principal = %d: %s", verifyWrongPrincipal.Code, verifyWrongPrincipal.Body.String()) + } + verifyWithoutRead := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals/verify", + headers: map[string]string{ + "Authorization": "Bearer " + syncOnlyToken, "X-Correlation-Id": "corr-checkpoint-verify-no-read", + }, + body: verifyBody, + }) + if verifyWithoutRead.Code != http.StatusForbidden { + t.Fatalf("verify without full-root read authority = %d: %s", verifyWithoutRead.Code, verifyWithoutRead.Body.String()) + } + replay := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals/consume", + headers: map[string]string{ + "Authorization": "Bearer " + cloudToken, "X-Correlation-Id": "corr-checkpoint-replay", + }, + body: consumeBody, + }) + if replay.Code != http.StatusOK { + t.Fatalf("idempotent replay status = %d: %s", replay.Code, replay.Body.String()) + } + var replayed relayfile.CheckpointSeal + if err := json.NewDecoder(replay.Body).Decode(&replayed); err != nil { + t.Fatalf("decode replayed consume: %v", err) + } + if replayed.SealToken != "" || replayed.ConsumedAt != consumed.ConsumedAt || replayed.SealID != consumed.SealID { + t.Fatalf("consume replay changed or leaked safe receipt: first=%+v replay=%+v", consumed, replayed) + } + wrongPrincipalConsume := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals/consume", + headers: map[string]string{ + "Authorization": "Bearer " + otherPrincipalToken, "X-Correlation-Id": "corr-checkpoint-consume-wrong-principal", + }, + body: consumeBody, + }) + if wrongPrincipalConsume.Code != http.StatusConflict || !strings.Contains(wrongPrincipalConsume.Body.String(), "checkpoint_consumer_conflict") { + t.Fatalf("wrong-principal consume replay = %d %s", wrongPrincipalConsume.Code, wrongPrincipalConsume.Body.String()) + } + differentConsumer := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals/consume", + headers: map[string]string{ + "Authorization": "Bearer " + cloudToken, "X-Correlation-Id": "corr-checkpoint-other-consumer", + }, + body: map[string]any{"sealToken": seal.SealToken, "root": "/sessions", "sessionId": "thread-http", "generation": 4, "consumerIdempotencyKey": "cloud-acquire-http-2"}, + }) + if differentConsumer.Code != http.StatusConflict || !strings.Contains(differentConsumer.Body.String(), "checkpoint_replayed") { + t.Fatalf("different consumer replay status = %d: %s", differentConsumer.Code, differentConsumer.Body.String()) + } + changedIdentity := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_http/sync/checkpoint-seals/consume", + headers: map[string]string{ + "Authorization": "Bearer " + cloudToken, "X-Correlation-Id": "corr-checkpoint-key-conflict", + }, + body: map[string]any{"sealToken": seal.SealToken, "root": "/other", "sessionId": "thread-http", "generation": 4, "consumerIdempotencyKey": "cloud-acquire-http-1"}, + }) + if changedIdentity.Code != http.StatusConflict || !strings.Contains(changedIdentity.Body.String(), "checkpoint_consumer_conflict") { + t.Fatalf("consumer identity conflict status = %d: %s", changedIdentity.Code, changedIdentity.Body.String()) + } +} + +func TestCheckpointSealInternalPersistenceErrorIsNotLeaked(t *testing.T) { + stateFile := filepath.Join(t.TempDir(), "state.json") + store := relayfile.NewStoreWithOptions(relayfile.StoreOptions{StateFile: stateFile, DisableWorkers: true}) + t.Cleanup(store.Close) + write, err := store.WriteFile(relayfile.WriteRequest{ + WorkspaceID: "ws_checkpoint_error", Path: "/transcript.jsonl", IfMatch: "0", Content: "turn one\n", + }) + if err != nil { + t.Fatal(err) + } + hash := sha256.Sum256([]byte("turn one\n")) + digest, err := relayfile.ComputeCheckpointDigest("/", []relayfile.CheckpointDigestEntry{{ + Path: "/transcript.jsonl", Revision: write.TargetRevision, ContentHash: fmt.Sprintf("%x", hash[:]), + }}) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(stateFile); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(stateFile, 0o700); err != nil { + t.Fatal(err) + } + server := NewServer(store) + token := mustTestJWT(t, "dev-secret", "ws_checkpoint_error", "LocalController", []string{"fs:read", "fs:write", "sync:trigger"}, time.Now().Add(time.Hour)) + response := doRequest(t, server, request{ + method: http.MethodPost, path: "/v1/workspaces/ws_checkpoint_error/sync/checkpoint-seals", + headers: map[string]string{"Authorization": "Bearer " + token, "X-Correlation-Id": "corr-checkpoint-persist-error"}, + body: map[string]any{"root": "/", "sessionId": "thread-error", "generation": 1, "expectedDigest": digest, "issuanceIdempotencyKey": "issue-error-1"}, + }) + if response.Code != http.StatusInternalServerError { + t.Fatalf("status = %d: %s", response.Code, response.Body.String()) + } + if strings.Contains(response.Body.String(), stateFile) || !strings.Contains(response.Body.String(), "checkpoint seal operation failed") { + t.Fatalf("internal persistence detail leaked or generic message missing: %s", response.Body.String()) + } +} + +func TestAdminCheckpointSealRetentionAndReconciliationHTTP(t *testing.T) { + store := relayfile.NewStoreWithOptions(relayfile.StoreOptions{DisableWorkers: true}) + t.Cleanup(store.Close) + now := time.Now().UTC() + write, err := store.WriteFile(relayfile.WriteRequest{WorkspaceID: "ws_admin_checkpoint", Path: "/transcript.jsonl", IfMatch: "0", Content: "turn\n"}) + if err != nil { + t.Fatal(err) + } + hash := sha256.Sum256([]byte("turn\n")) + digest, err := relayfile.ComputeCheckpointDigest("/", []relayfile.CheckpointDigestEntry{{Path: "/transcript.jsonl", Revision: write.TargetRevision, ContentHash: fmt.Sprintf("%x", hash[:])}}) + if err != nil { + t.Fatal(err) + } + issued, err := store.IssueCheckpointSeal("ws_admin_checkpoint", relayfile.CheckpointSealRequest{ + Root: "/", SessionID: "thread-admin-http", Generation: 1, ExpectedDigest: digest, + IssuanceIdempotencyKey: "issue-admin-http-1", Issuer: "LocalController", + }, now) + if err != nil { + t.Fatal(err) + } + server := NewServer(store) + readToken := mustTestJWT(t, "dev-secret", "ws_admin_checkpoint", "Observer", []string{"admin:read"}, now.Add(time.Hour)) + replayToken := mustTestJWT(t, "dev-secret", "ws_admin_checkpoint", "Operator", []string{"admin:replay"}, now.Add(time.Hour)) + status := doRequest(t, server, request{ + method: http.MethodGet, path: "/v1/admin/checkpoint-seals?workspaceId=ws_admin_checkpoint", + headers: map[string]string{"Authorization": "Bearer " + readToken, "X-Correlation-Id": "corr-admin-checkpoint-status"}, + }) + if status.Code != http.StatusOK || !strings.Contains(status.Body.String(), `"unresumedTotal":1`) || strings.Contains(status.Body.String(), issued.SealToken) { + t.Fatalf("admin status = %d: %s", status.Code, status.Body.String()) + } + body := map[string]any{ + "workspaceId": "ws_admin_checkpoint", "root": "/", "sessionId": issued.SessionID, + "generation": issued.Generation, "expectedOwnershipStatus": "unconsumed", + "reconciliationIdempotencyKey": "admin-reconcile-http-1", "confirmSourceReady": true, + } + denied := doRequest(t, server, request{ + method: http.MethodPost, path: "/v1/admin/checkpoint-seals/" + issued.SealID + "/reconcile-source", + headers: map[string]string{"Authorization": "Bearer " + readToken, "X-Correlation-Id": "corr-admin-checkpoint-denied"}, body: body, + }) + if denied.Code != http.StatusForbidden { + t.Fatalf("admin read token reconciled ownership: %d %s", denied.Code, denied.Body.String()) + } + reconciled := doRequest(t, server, request{ + method: http.MethodPost, path: "/v1/admin/checkpoint-seals/" + issued.SealID + "/reconcile-source", + headers: map[string]string{"Authorization": "Bearer " + replayToken, "X-Correlation-Id": "corr-admin-checkpoint-reconcile"}, body: body, + }) + if reconciled.Code != http.StatusOK || !strings.Contains(reconciled.Body.String(), `"ownershipStatus":"source-resumed"`) { + t.Fatalf("admin reconcile = %d: %s", reconciled.Code, reconciled.Body.String()) + } +} + +func TestCheckpointHandbackHTTPIsStrictTokenlessAndSourceGated(t *testing.T) { + store := relayfile.NewStoreWithOptions(relayfile.StoreOptions{DisableWorkers: true}) + t.Cleanup(store.Close) + write, err := store.WriteFile(relayfile.WriteRequest{WorkspaceID: "ws_handback_http", Path: "/transcript.jsonl", IfMatch: "0", Content: "turn\n"}) + if err != nil { + t.Fatal(err) + } + contentHash := sha256.Sum256([]byte("turn\n")) + digest, err := relayfile.ComputeCheckpointDigest("/", []relayfile.CheckpointDigestEntry{{Path: "/transcript.jsonl", Revision: write.TargetRevision, ContentHash: fmt.Sprintf("%x", contentHash[:])}}) + if err != nil { + t.Fatal(err) + } + server := NewServer(store) + fullToken := mustTestJWT(t, "dev-secret", "ws_handback_http", "CutoverController", []string{"fs:read", "fs:write", "sync:trigger"}, time.Now().Add(time.Hour)) + syncOnlyToken := mustTestJWT(t, "dev-secret", "ws_handback_http", "CutoverController", []string{"sync:trigger"}, time.Now().Add(time.Hour)) + issue := doRequest(t, server, request{ + method: http.MethodPost, path: "/v1/workspaces/ws_handback_http/sync/checkpoint-seals", + headers: map[string]string{"Authorization": "Bearer " + fullToken, "X-Correlation-Id": "corr-handback-issue"}, + body: map[string]any{"root": "/", "sessionId": "thread-handback-http", "generation": 1, "expectedDigest": digest, "issuanceIdempotencyKey": "issue-handback-http-1"}, + }) + var issued relayfile.CheckpointSeal + if issue.Code != http.StatusCreated || json.NewDecoder(issue.Body).Decode(&issued) != nil { + t.Fatalf("issue = %d %s", issue.Code, issue.Body.String()) + } + consumerKey := "cutover-http-one" + consume := doRequest(t, server, request{ + method: http.MethodPost, path: "/v1/workspaces/ws_handback_http/sync/checkpoint-seals/consume", + headers: map[string]string{"Authorization": "Bearer " + fullToken, "X-Correlation-Id": "corr-handback-consume"}, + body: map[string]any{"sealToken": issued.SealToken, "root": "/", "sessionId": issued.SessionID, "generation": issued.Generation, "consumerIdempotencyKey": consumerKey}, + }) + var consumed relayfile.CheckpointSeal + if consume.Code != http.StatusOK || json.NewDecoder(consume.Body).Decode(&consumed) != nil { + t.Fatalf("consume = %d %s", consume.Code, consume.Body.String()) + } + resumeBody := map[string]any{ + "sealToken": issued.SealToken, "root": "/", "sessionId": issued.SessionID, + "generation": issued.Generation, "resumeIdempotencyKey": "source-resume-http-one", + } + premature := doRequest(t, server, request{ + method: http.MethodPost, path: "/v1/workspaces/ws_handback_http/sync/checkpoint-seals/resume", + headers: map[string]string{"Authorization": "Bearer " + fullToken, "X-Correlation-Id": "corr-handback-premature-resume"}, body: resumeBody, + }) + if premature.Code != http.StatusConflict || !strings.Contains(premature.Body.String(), "checkpoint_handback_required") { + t.Fatalf("premature resume = %d %s", premature.Code, premature.Body.String()) + } + handbackBody := map[string]any{ + "phase": "prepare", + "sealId": consumed.SealID, "root": "/", "sessionId": consumed.SessionID, + "generation": consumed.Generation, "consumedAt": consumed.ConsumedAt, + "consumerIdempotencyKey": consumerKey, "handbackIdempotencyKey": "handback-http-one", + "expectedDigest": consumed.Digest, + } + forbidden := doRequest(t, server, request{ + method: http.MethodPost, path: "/v1/workspaces/ws_handback_http/sync/checkpoint-seals/handback", + headers: map[string]string{"Authorization": "Bearer " + syncOnlyToken, "X-Correlation-Id": "corr-handback-forbidden"}, body: handbackBody, + }) + if forbidden.Code != http.StatusForbidden { + t.Fatalf("handback without file authority = %d %s", forbidden.Code, forbidden.Body.String()) + } + handbackBody["sealToken"] = issued.SealToken + strict := doRequest(t, server, request{ + method: http.MethodPost, path: "/v1/workspaces/ws_handback_http/sync/checkpoint-seals/handback", + headers: map[string]string{"Authorization": "Bearer " + fullToken, "X-Correlation-Id": "corr-handback-strict"}, body: handbackBody, + }) + if strict.Code != http.StatusBadRequest { + t.Fatalf("handback accepted sealToken = %d %s", strict.Code, strict.Body.String()) + } + delete(handbackBody, "sealToken") + otherFullToken := mustTestJWT(t, "dev-secret", "ws_handback_http", "OtherAgent", []string{"fs:read", "fs:write", "sync:trigger"}, time.Now().Add(time.Hour)) + wrongPrincipal := doRequest(t, server, request{ + method: http.MethodPost, path: "/v1/workspaces/ws_handback_http/sync/checkpoint-seals/handback", + headers: map[string]string{"Authorization": "Bearer " + otherFullToken, "X-Correlation-Id": "corr-handback-wrong-principal"}, body: handbackBody, + }) + if wrongPrincipal.Code != http.StatusConflict || !strings.Contains(wrongPrincipal.Body.String(), "checkpoint_consumer_conflict") { + t.Fatalf("wrong-principal handback = %d %s", wrongPrincipal.Code, wrongPrincipal.Body.String()) + } + prepare := doRequest(t, server, request{ + method: http.MethodPost, path: "/v1/workspaces/ws_handback_http/sync/checkpoint-seals/handback", + headers: map[string]string{"Authorization": "Bearer " + fullToken, "X-Correlation-Id": "corr-handback-prepare"}, body: handbackBody, + }) + if prepare.Code != http.StatusOK || strings.Contains(prepare.Body.String(), "sealToken") || !strings.Contains(prepare.Body.String(), `"status":"prepared"`) || strings.Contains(prepare.Body.String(), `"releasedAt"`) { + t.Fatalf("prepare handback = %d %s", prepare.Code, prepare.Body.String()) + } + prematureAfterPrepare := doRequest(t, server, request{ + method: http.MethodPost, path: "/v1/workspaces/ws_handback_http/sync/checkpoint-seals/resume", + headers: map[string]string{"Authorization": "Bearer " + fullToken, "X-Correlation-Id": "corr-handback-prepared-resume"}, body: resumeBody, + }) + if prematureAfterPrepare.Code != http.StatusConflict || !strings.Contains(prematureAfterPrepare.Body.String(), "checkpoint_handback_required") { + t.Fatalf("resume after prepare = %d %s", prematureAfterPrepare.Code, prematureAfterPrepare.Body.String()) + } + handbackBody["phase"] = "commit" + handback := doRequest(t, server, request{ + method: http.MethodPost, path: "/v1/workspaces/ws_handback_http/sync/checkpoint-seals/handback", + headers: map[string]string{"Authorization": "Bearer " + fullToken, "X-Correlation-Id": "corr-handback-release"}, body: handbackBody, + }) + if handback.Code != http.StatusOK || strings.Contains(handback.Body.String(), "sealToken") || !strings.Contains(handback.Body.String(), `"status":"released"`) { + t.Fatalf("handback = %d %s", handback.Code, handback.Body.String()) + } + resume := doRequest(t, server, request{ + method: http.MethodPost, path: "/v1/workspaces/ws_handback_http/sync/checkpoint-seals/resume", + headers: map[string]string{"Authorization": "Bearer " + fullToken, "X-Correlation-Id": "corr-handback-resume"}, body: resumeBody, + }) + if resume.Code != http.StatusOK || strings.Contains(resume.Body.String(), "sealToken") || !strings.Contains(resume.Body.String(), `"status":"source-resumed"`) { + t.Fatalf("resume = %d %s", resume.Code, resume.Body.String()) + } +} + +func TestCheckpointSealHTTPRequiresCompleteRootAuthorityAndRejectsCallerDigest(t *testing.T) { + store := relayfile.NewStoreWithOptions(relayfile.StoreOptions{DisableWorkers: true}) + t.Cleanup(store.Close) + server := NewServer(store) + readOnlyToken := mustTestJWT(t, "dev-secret", "ws_checkpoint_auth", "LocalController", []string{"fs:read", "sync:trigger"}, time.Now().Add(time.Hour)) + denied := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_auth/sync/checkpoint-seals", + headers: map[string]string{ + "Authorization": "Bearer " + readOnlyToken, "X-Correlation-Id": "corr-checkpoint-denied", + }, + body: map[string]any{"root": "/", "sessionId": "thread-auth", "generation": 1, "expectedDigest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "issuanceIdempotencyKey": "issue-auth-denied-1"}, + }) + if denied.Code != http.StatusForbidden { + t.Fatalf("read-only issue status = %d: %s", denied.Code, denied.Body.String()) + } + fullToken := mustTestJWT(t, "dev-secret", "ws_checkpoint_auth", "LocalController", []string{"fs:read", "fs:write", "sync:trigger"}, time.Now().Add(time.Hour)) + diverged := doRequest(t, server, request{ + method: http.MethodPost, + path: "/v1/workspaces/ws_checkpoint_auth/sync/checkpoint-seals", + headers: map[string]string{ + "Authorization": "Bearer " + fullToken, "X-Correlation-Id": "corr-checkpoint-diverged", + }, + body: map[string]any{"root": "/", "sessionId": "thread-auth", "generation": 1, "expectedDigest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "issuanceIdempotencyKey": "issue-auth-diverged-1"}, + }) + if diverged.Code != http.StatusConflict || !strings.Contains(diverged.Body.String(), "checkpoint_diverged") { + t.Fatalf("caller digest status = %d: %s", diverged.Code, diverged.Body.String()) + } +} + func newMergeTestServer(t *testing.T) http.Handler { t.Helper() store := relayfile.NewStoreWithOptions(relayfile.StoreOptions{DisableWorkers: true}) diff --git a/internal/mountsync/realtime_collaboration_test.go b/internal/mountsync/realtime_collaboration_test.go index 70a24df4..26cb3742 100644 --- a/internal/mountsync/realtime_collaboration_test.go +++ b/internal/mountsync/realtime_collaboration_test.go @@ -840,15 +840,30 @@ func TestLocalChangeBatcherCoalescesBurst(t *testing.T) { func TestLocalChangeBatcherUsesQuietWindowAcrossStaggeredSave(t *testing.T) { batches := make(chan []LocalChange, 2) - batcher := NewLocalChangeBatcher(5*time.Millisecond, func(changes []LocalChange) { + clock := newFakeLocalChangeClock() + batcher := newLocalChangeBatcherWithClock(5*time.Millisecond, func(changes []LocalChange) { batches <- changes - }) + }, clock) defer batcher.Close() for i := 0; i < 5; i++ { batcher.Add(fmt.Sprintf("shared/%02d.txt", i), fsnotify.Write) - time.Sleep(3 * time.Millisecond) + if i < 4 { + clock.Advance(3 * time.Millisecond) + select { + case batch := <-batches: + t.Fatalf("batch flushed before the extended quiet deadline: %+v", batch) + default: + } + } + } + clock.Advance(4 * time.Millisecond) + select { + case batch := <-batches: + t.Fatalf("batch flushed before a full quiet window: %+v", batch) + default: } + clock.Advance(time.Millisecond) select { case batch := <-batches: if len(batch) != 5 { @@ -860,10 +875,141 @@ func TestLocalChangeBatcherUsesQuietWindowAcrossStaggeredSave(t *testing.T) { select { case extra := <-batches: t.Fatalf("staggered save produced an extra batch: %+v", extra) - case <-time.After(20 * time.Millisecond): + default: + } +} + +func TestLocalChangeBatcherReschedulesInactiveTimer(t *testing.T) { + batches := make(chan []LocalChange, 1) + clock := newFakeLocalChangeClock() + batcher := newLocalChangeBatcherWithClock(5*time.Millisecond, func(changes []LocalChange) { + batches <- changes + }, clock) + + batcher.Add("shared/one.txt", fsnotify.Write) + batcher.mu.Lock() + if !batcher.timer.Stop() { + batcher.mu.Unlock() + t.Fatal("timer expired before deterministic race setup") + } + batcher.mu.Unlock() + + // This is the state Add observes when an AfterFunc callback has expired or + // started: timer is non-nil, but Stop returns false. The Add must still + // reschedule it and eventually flush both changes. + batcher.Add("shared/two.txt", fsnotify.Write) + clock.Advance(5 * time.Millisecond) + select { + case batch := <-batches: + if len(batch) != 2 { + t.Fatalf("batch size = %d, want 2", len(batch)) + } + batcher.Close() + default: + t.Fatal("inactive timer was not rescheduled") + } +} + +func TestLocalChangeBatcherCapsSustainedChurn(t *testing.T) { + batches := make(chan []LocalChange, 1) + clock := newFakeLocalChangeClock() + batcher := newLocalChangeBatcherWithClock(5*time.Millisecond, func(changes []LocalChange) { + batches <- changes + }, clock) + defer batcher.Close() + + batcher.Add("shared/00.txt", fsnotify.Write) + for i := 1; i <= 12; i++ { + clock.Advance(4 * time.Millisecond) + batcher.Add(fmt.Sprintf("shared/%02d.txt", i), fsnotify.Write) + } + clock.Advance(time.Millisecond) + select { + case batch := <-batches: + t.Fatalf("sustained churn flushed before maxWait: %+v", batch) + default: + } + clock.Advance(time.Millisecond) + select { + case batch := <-batches: + if len(batch) != 13 { + t.Fatalf("maxWait batch size = %d, want 13", len(batch)) + } + default: + t.Fatal("sustained churn did not flush at maxWait") + } +} + +type fakeLocalChangeClock struct { + mu sync.Mutex + now time.Time + timers []*fakeLocalChangeTimer +} + +type fakeLocalChangeTimer struct { + clock *fakeLocalChangeClock + deadline time.Time + callback func() + active bool +} + +func newFakeLocalChangeClock() *fakeLocalChangeClock { + return &fakeLocalChangeClock{now: time.Unix(1, 0)} +} + +func (c *fakeLocalChangeClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *fakeLocalChangeClock) AfterFunc(delay time.Duration, callback func()) localChangeTimer { + c.mu.Lock() + defer c.mu.Unlock() + timer := &fakeLocalChangeTimer{clock: c, deadline: c.now.Add(delay), callback: callback, active: true} + c.timers = append(c.timers, timer) + return timer +} + +func (c *fakeLocalChangeClock) Advance(delta time.Duration) { + c.mu.Lock() + c.now = c.now.Add(delta) + c.mu.Unlock() + for { + var callback func() + c.mu.Lock() + for _, timer := range c.timers { + if timer.active && !timer.deadline.After(c.now) { + timer.active = false + callback = timer.callback + break + } + } + c.mu.Unlock() + if callback == nil { + return + } + callback() } } +func (t *fakeLocalChangeTimer) Stop() bool { + t.clock.mu.Lock() + defer t.clock.mu.Unlock() + wasActive := t.active + t.active = false + return wasActive +} + +func (t *fakeLocalChangeTimer) Reset(delay time.Duration) bool { + t.clock.mu.Lock() + defer t.clock.mu.Unlock() + wasActive := t.active + t.active = true + t.deadline = t.clock.now.Add(delay) + return wasActive +} + func TestWatcherBatchCheckpointsPrivateStateAfterVisibilityPath(t *testing.T) { client := &fakeClient{files: map[string]RemoteFile{}} client.bulkWriteResponseFunc = func(_ context.Context, _ string, files []BulkWriteFile) (BulkWriteResponse, error) { diff --git a/internal/mountsync/syncer.go b/internal/mountsync/syncer.go index 7cae79ab..ad5553fb 100644 --- a/internal/mountsync/syncer.go +++ b/internal/mountsync/syncer.go @@ -42,6 +42,15 @@ import ( var ErrConflict = errors.New("revision conflict") +var ( + ErrCheckpointConcurrentMutation = errors.New("local workspace changed while checkpoint seal was being created") + ErrCheckpointUnmanagedRoot = errors.New("checkpoint sealing requires an existing managed Relayfile mount") + ErrCheckpointNonConverged = errors.New("local and durable Relayfile state did not converge") + checkpointSessionPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$`) + checkpointRevisionPattern = regexp.MustCompile(`^(?:0|rev_[0-9]+)$`) + checkpointEventCursorPattern = regexp.MustCompile(`^(?:0|evt_[0-9]+)$`) +) + var sensitiveLogQueryValue = regexp.MustCompile(`(?i)([?&](?:token|access_token|api_key)=)[^&#\s"']*`) // ErrSchemaValidation is returned when the cloud rejects a writeback because @@ -446,6 +455,42 @@ type RemoteClient interface { MergeFile(ctx context.Context, workspaceID, path, strategy, baseRevision, baseContent, content, contentType string) (MergeResult, error) } +type checkpointSealClient interface { + IssueCheckpointSeal(ctx context.Context, workspaceID string, request CheckpointSealRequest) (CheckpointSeal, error) +} + +type checkpointSealVerifierClient interface { + VerifyCheckpointSeal(ctx context.Context, workspaceID string, request CheckpointSealVerifyRequest) (CheckpointSeal, error) +} + +type checkpointSealHandbackClient interface { + HandbackCheckpointSeal(ctx context.Context, workspaceID string, request CheckpointSealHandbackRequest) (CheckpointSealOwnership, error) +} + +type CheckpointAndSealOptions struct { + SessionID string + Generation uint64 + TTLSeconds int +} + +type CheckpointObservedState struct { + Digest string `json:"digest"` + WorkspaceRevision string `json:"workspaceRevision"` + EventCursor string `json:"eventCursor"` +} + +type CheckpointVerificationHealth struct { + PendingWriteback int `json:"pendingWriteback"` + Conflicts int `json:"conflicts"` + OutboxPending int `json:"outboxPending"` + OutboxNeedsAttention bool `json:"outboxNeedsAttention"` +} + +type CheckpointVerification struct { + Observed CheckpointObservedState `json:"observed"` + Health CheckpointVerificationHealth `json:"health"` +} + // Merge strategy constants mirror the server's internal/relayfile package. // They are duplicated here rather than imported to keep internal/mountsync // free of a dependency on the store package. @@ -754,6 +799,42 @@ func (c *HTTPClient) DeleteFile(ctx context.Context, workspaceID, path, baseRevi return c.doJSON(ctx, http.MethodDelete, fmt.Sprintf("/v1/workspaces/%s/fs/file?%s", url.PathEscape(workspaceID), q.Encode()), headers, nil, nil) } +func (c *HTTPClient) IssueCheckpointSeal(ctx context.Context, workspaceID string, request CheckpointSealRequest) (CheckpointSeal, error) { + var out CheckpointSeal + err := c.doJSON(ctx, http.MethodPost, fmt.Sprintf("/v1/workspaces/%s/sync/checkpoint-seals", url.PathEscape(workspaceID)), nil, request, &out) + return out, err +} + +func (c *HTTPClient) ConsumeCheckpointSeal(ctx context.Context, workspaceID string, request CheckpointSealConsumeRequest) (CheckpointSeal, error) { + var out CheckpointSeal + err := c.doJSON(ctx, http.MethodPost, fmt.Sprintf("/v1/workspaces/%s/sync/checkpoint-seals/consume", url.PathEscape(workspaceID)), nil, request, &out) + return out, err +} + +func (c *HTTPClient) RecoverConsumedCheckpointSeal(ctx context.Context, workspaceID string, request CheckpointSealConsumeRecoveryRequest) (CheckpointSeal, error) { + var out CheckpointSeal + err := c.doJSON(ctx, http.MethodPost, fmt.Sprintf("/v1/workspaces/%s/sync/checkpoint-seals/recover-consume", url.PathEscape(workspaceID)), nil, request, &out) + return out, err +} + +func (c *HTTPClient) VerifyCheckpointSeal(ctx context.Context, workspaceID string, request CheckpointSealVerifyRequest) (CheckpointSeal, error) { + var out CheckpointSeal + err := c.doJSON(ctx, http.MethodPost, fmt.Sprintf("/v1/workspaces/%s/sync/checkpoint-seals/verify", url.PathEscape(workspaceID)), nil, request, &out) + return out, err +} + +func (c *HTTPClient) HandbackCheckpointSeal(ctx context.Context, workspaceID string, request CheckpointSealHandbackRequest) (CheckpointSealOwnership, error) { + var out CheckpointSealOwnership + err := c.doJSON(ctx, http.MethodPost, fmt.Sprintf("/v1/workspaces/%s/sync/checkpoint-seals/handback", url.PathEscape(workspaceID)), nil, request, &out) + return out, err +} + +func (c *HTTPClient) ResumeCheckpointSeal(ctx context.Context, workspaceID string, request CheckpointSealResumeRequest) (CheckpointSealOwnership, error) { + var out CheckpointSealOwnership + err := c.doJSON(ctx, http.MethodPost, fmt.Sprintf("/v1/workspaces/%s/sync/checkpoint-seals/resume", url.PathEscape(workspaceID)), nil, request, &out) + return out, err +} + func (c *HTTPClient) ExportFiles(ctx context.Context, workspaceID, path string) ([]RemoteFile, error) { q := url.Values{} q.Set("format", "json") @@ -1320,6 +1401,7 @@ type Syncer struct { // Test seam for proving that background local-tree hashing yields the // real-time state lock. Production uses readLocalSnapshot directly. readLocalSnapshotFn func(string, bool) (localSnapshot, error) + checkpointTestHook func(string) // credExpiresAt is the RFC3339 expiry of the delegated access token, // set by the CLI layer via SetCredentialExpiry and included in the // public state as credExpiresInSecs so operators get advance warning. @@ -2318,6 +2400,684 @@ func (s *Syncer) PushLocalAndFlushOnce(ctx context.Context) error { return s.saveState() } +// CheckpointAndSeal is the teardown boundary for a managed mount. It drains +// missed local writes (including deletes), proves the local tree is stable +// across repeated hash scans, and asks Relayfile to issue a one-use seal from +// its own durable digest/cursor under the server mutation lock. +// +// The caller must already have stopped the agent at a turn boundary. No +// filesystem API can prevent an unrelated process from writing after this +// method returns; changes during the operation are detected and fail closed. +func (s *Syncer) CheckpointAndSeal(ctx context.Context, options CheckpointAndSealOptions) (CheckpointSeal, error) { + if ctx == nil { + ctx = context.Background() + } + sessionID := strings.TrimSpace(options.SessionID) + if !checkpointSessionPattern.MatchString(sessionID) || options.Generation == 0 || options.TTLSeconds < 0 || options.TTLSeconds > int(MaxCheckpointSealTTL/time.Second) { + return CheckpointSeal{}, fmt.Errorf("%w: invalid checkpoint session, generation, or TTL", ErrCheckpointNonConverged) + } + if err := ctx.Err(); err != nil { + return CheckpointSeal{}, fmt.Errorf("%w: %v", ErrCheckpointNonConverged, err) + } + if _, ok := ctx.Deadline(); !ok { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 30*time.Second) + defer cancel() + } + if s.pullOnly { + return CheckpointSeal{}, fmt.Errorf("%w: pull-only mounts cannot drain local changes", ErrCheckpointNonConverged) + } + if len(s.scopes) > 0 && !checkpointScopeGrantsOpsRead(s.scopes) { + return CheckpointSeal{}, fmt.Errorf("%w: checkpoint drain requires ops:read to settle asynchronous write receipts", ErrCheckpointNonConverged) + } + if s.remoteRoot != "/" { + return CheckpointSeal{}, fmt.Errorf("%w: live checkpoint v1 supports only the full remote root /", ErrCheckpointUnmanagedRoot) + } + issuer, ok := s.client.(checkpointSealClient) + if !ok { + return CheckpointSeal{}, fmt.Errorf("%w: server/client checkpoint seal contract unavailable", ErrCheckpointNonConverged) + } + if err := s.assertManagedCheckpointRoot(); err != nil { + return CheckpointSeal{}, err + } + if err := s.assertMountRootInvariant(); err != nil { + return CheckpointSeal{}, err + } + + s.localMutationMu.Lock() + defer s.localMutationMu.Unlock() + s.mu.Lock() + defer s.mu.Unlock() + if err := s.loadState(); err != nil { + return CheckpointSeal{}, fmt.Errorf("%w: load private mount state: %v", ErrCheckpointUnmanagedRoot, err) + } + if !s.state.BootstrapComplete { + return CheckpointSeal{}, fmt.Errorf("%w: mount bootstrap is incomplete", ErrCheckpointNonConverged) + } + if s.localWatcherActive { + return CheckpointSeal{}, fmt.Errorf("%w: sealing requires a stopped, watcherless mount", ErrCheckpointNonConverged) + } + if len(s.state.QuarantinedPaths) > 0 || len(s.state.SkippedMaterializations) > 0 || s.state.IncrementalBacklogDraining || s.state.IncrementalCheckpoint != nil || len(s.state.IncrementalReadNotReadySince) > 0 { + return CheckpointSeal{}, fmt.Errorf("%w: mount has quarantined, skipped, or incomplete remote state", ErrCheckpointNonConverged) + } + + // The daemon may have been stopped before its watcher observed the final + // mutation. Force watcherless detection and run two passes so missing-file + // deletion uses its existing two-observation safety gate. + previousPolling := s.pollLocalChanges + previousRecovery := s.recoverStartupDrift + previousRootCtx := s.rootCtx + s.pollLocalChanges = true + s.recoverStartupDrift = true + s.rootCtx = ctx + defer func() { + s.pollLocalChanges = previousPolling + s.recoverStartupDrift = previousRecovery + s.rootCtx = previousRootCtx + }() + for pass := 0; pass < 2; pass++ { + conflicted, err := s.pushLocal(ctx) + if err != nil { + return CheckpointSeal{}, fmt.Errorf("%w: drain local changes: %v", ErrCheckpointNonConverged, err) + } + if len(conflicted) > 0 { + return CheckpointSeal{}, fmt.Errorf("%w: %d conflict(s) materialized during drain", ErrCheckpointNonConverged, len(conflicted)) + } + if err := s.settleCheckpointOutbox(ctx); err != nil { + return CheckpointSeal{}, fmt.Errorf("%w: settle durable outbox: %v", ErrCheckpointNonConverged, err) + } + } + health, err := s.checkpointVerificationHealthLocked() + if err != nil { + return CheckpointSeal{}, fmt.Errorf("%w: inspect final drain health: %v", ErrCheckpointNonConverged, err) + } + if health.PendingWriteback != 0 || health.Conflicts != 0 || health.OutboxPending != 0 || health.OutboxNeedsAttention { + return CheckpointSeal{}, fmt.Errorf("%w: final drain pendingWriteback=%d conflicts=%d outboxPending=%d outboxNeedsAttention=%t", ErrCheckpointNonConverged, health.PendingWriteback, health.Conflicts, health.OutboxPending, health.OutboxNeedsAttention) + } + s.markSyncSuccess() + if err := s.saveStateWithoutLocalScan(); err != nil { + return CheckpointSeal{}, fmt.Errorf("persist drained mount state: %w", err) + } + + firstDigest, err := s.checkpointLocalDigestLocked() + if err != nil { + return CheckpointSeal{}, err + } + s.runCheckpointTestHook("after-first-scan") + secondDigest, err := s.checkpointLocalDigestLocked() + if err != nil { + return CheckpointSeal{}, fmt.Errorf("%w: %v", ErrCheckpointConcurrentMutation, err) + } + if firstDigest != secondDigest { + return CheckpointSeal{}, ErrCheckpointConcurrentMutation + } + + seal, err := issuer.IssueCheckpointSeal(ctx, s.workspace, CheckpointSealRequest{ + Root: s.remoteRoot, + SessionID: sessionID, + Generation: options.Generation, + ExpectedDigest: secondDigest, + TTLSeconds: options.TTLSeconds, + IssuanceIdempotencyKey: checkpointIssuanceIdempotencyKey(s.workspace, s.remoteRoot, sessionID, options.Generation, secondDigest, options.TTLSeconds), + }) + if err != nil { + return CheckpointSeal{}, fmt.Errorf("%w: issue authoritative server seal: %w", ErrCheckpointNonConverged, err) + } + s.runCheckpointTestHook("after-issue") + finalDigest, err := s.checkpointLocalDigestLocked() + if err != nil { + return CheckpointSeal{}, fmt.Errorf("%w: %v", ErrCheckpointConcurrentMutation, err) + } + if finalDigest != secondDigest || seal.Digest != secondDigest || seal.WorkspaceID != s.workspace || seal.Root != s.remoteRoot || seal.SessionID != sessionID || seal.Generation != options.Generation || strings.TrimSpace(seal.SealToken) == "" { + return CheckpointSeal{}, ErrCheckpointConcurrentMutation + } + return seal, nil +} + +func checkpointIssuanceIdempotencyKey(workspaceID, root, sessionID string, generation uint64, digest string, ttlSeconds int) string { + h := sha256.New() + for _, value := range []string{ + strings.TrimSpace(workspaceID), root, strings.TrimSpace(sessionID), + strconv.FormatUint(generation, 10), strings.TrimSpace(digest), strconv.Itoa(ttlSeconds), + } { + _, _ = fmt.Fprintf(h, "%d:%s\x00", len(value), value) + } + return "checkpoint-issue-" + hex.EncodeToString(h.Sum(nil)[:16]) +} + +// HandbackCheckpoint drains a stopped destination and atomically releases its +// server-side ownership back to the original source. It intentionally does not +// issue a second seal (which would be the same-generation replay); the original +// consumer identity and an independent handback key bind this transition. +func (s *Syncer) HandbackCheckpoint(ctx context.Context, consumed CheckpointSeal, consumerKey, handbackKey string) (CheckpointSealOwnership, CheckpointVerificationHealth, error) { + var health CheckpointVerificationHealth + if ctx == nil { + ctx = context.Background() + } + consumerKey = strings.TrimSpace(consumerKey) + handbackKey = strings.TrimSpace(handbackKey) + if err := validateConsumedCheckpointReceipt(consumed); err != nil || !checkpointSessionPattern.MatchString(consumerKey) || !checkpointSessionPattern.MatchString(handbackKey) { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: invalid consumed receipt or handback identity", ErrCheckpointNonConverged) + } + if err := ctx.Err(); err != nil { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: %v", ErrCheckpointNonConverged, err) + } + if _, ok := ctx.Deadline(); !ok { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 30*time.Second) + defer cancel() + } + if s.pullOnly { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: pull-only mounts cannot drain local changes", ErrCheckpointNonConverged) + } + if len(s.scopes) > 0 && !checkpointScopeGrantsOpsRead(s.scopes) { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: handback drain requires ops:read to settle asynchronous write receipts", ErrCheckpointNonConverged) + } + if s.remoteRoot != "/" || consumed.Root != "/" || consumed.WorkspaceID != s.workspace { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: handback v1 requires the exact full workspace root", ErrCheckpointUnmanagedRoot) + } + handbackClient, ok := s.client.(checkpointSealHandbackClient) + if !ok { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: server/client checkpoint handback contract unavailable", ErrCheckpointNonConverged) + } + if err := s.assertManagedCheckpointRoot(); err != nil { + return CheckpointSealOwnership{}, health, err + } + if err := s.assertMountRootInvariant(); err != nil { + return CheckpointSealOwnership{}, health, err + } + + s.localMutationMu.Lock() + defer s.localMutationMu.Unlock() + s.mu.Lock() + defer s.mu.Unlock() + if err := s.loadState(); err != nil { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: load private mount state: %v", ErrCheckpointUnmanagedRoot, err) + } + if !s.state.BootstrapComplete { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: destination mount bootstrap is incomplete", ErrCheckpointNonConverged) + } + if s.localWatcherActive { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: handback requires a stopped, watcherless destination verifier", ErrCheckpointNonConverged) + } + if len(s.state.QuarantinedPaths) > 0 || len(s.state.SkippedMaterializations) > 0 || s.state.IncrementalBacklogDraining || s.state.IncrementalCheckpoint != nil || len(s.state.IncrementalReadNotReadySince) > 0 { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: destination has quarantined, skipped, or incomplete remote state", ErrCheckpointNonConverged) + } + + previousPolling := s.pollLocalChanges + previousRecovery := s.recoverStartupDrift + previousRootCtx := s.rootCtx + s.pollLocalChanges = true + s.recoverStartupDrift = true + s.rootCtx = ctx + defer func() { + s.pollLocalChanges = previousPolling + s.recoverStartupDrift = previousRecovery + s.rootCtx = previousRootCtx + }() + for pass := 0; pass < 2; pass++ { + conflicted, err := s.pushLocal(ctx) + if err != nil { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: drain destination changes: %v", ErrCheckpointNonConverged, err) + } + if len(conflicted) > 0 { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: %d conflict(s) materialized during handback drain", ErrCheckpointNonConverged, len(conflicted)) + } + if err := s.settleCheckpointOutbox(ctx); err != nil { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: settle destination outbox: %v", ErrCheckpointNonConverged, err) + } + } + var err error + health, err = s.checkpointVerificationHealthLocked() + if err != nil { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: inspect handback health: %v", ErrCheckpointNonConverged, err) + } + if health.PendingWriteback != 0 || health.Conflicts != 0 || health.OutboxPending != 0 || health.OutboxNeedsAttention { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: handback pendingWriteback=%d conflicts=%d outboxPending=%d outboxNeedsAttention=%t", ErrCheckpointNonConverged, health.PendingWriteback, health.Conflicts, health.OutboxPending, health.OutboxNeedsAttention) + } + s.markSyncSuccess() + if err := s.saveStateWithoutLocalScan(); err != nil { + return CheckpointSealOwnership{}, health, fmt.Errorf("persist handback mount state: %w", err) + } + firstDigest, err := s.checkpointLocalDigestLocked() + if err != nil { + return CheckpointSealOwnership{}, health, err + } + s.runCheckpointTestHook("handback-after-first-scan") + secondDigest, err := s.checkpointLocalDigestLocked() + if err != nil || firstDigest != secondDigest { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: destination changed during handback", ErrCheckpointConcurrentMutation) + } + request := CheckpointSealHandbackRequest{ + Phase: CheckpointHandbackPhasePrepare, + SealID: consumed.SealID, Root: consumed.Root, SessionID: consumed.SessionID, + Generation: consumed.Generation, ConsumedAt: consumed.ConsumedAt, + ConsumerIdempotencyKey: consumerKey, HandbackIdempotencyKey: handbackKey, + ExpectedDigest: secondDigest, + } + prepared, err := handbackClient.HandbackCheckpointSeal(ctx, s.workspace, request) + if err != nil { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: prepare destination handback: %w", ErrCheckpointNonConverged, err) + } + // A commit can succeed durably while its response is lost. Exact retries + // restart at prepare; the server returns the already-released proof bound to + // the same handback key and digest. Accept that terminal replay rather than + // requiring a second (impossible) transition. + if prepared.Status == "released" { + if prepared.SealID != consumed.SealID || prepared.WorkspaceID != s.workspace || prepared.Root != "/" || prepared.SessionID != consumed.SessionID || prepared.Generation != consumed.Generation || + prepared.Digest != secondDigest || !checkpointEventCursorPattern.MatchString(strings.TrimSpace(prepared.EventCursor)) || !checkpointRevisionPattern.MatchString(strings.TrimSpace(prepared.WorkspaceRevision)) || prepared.ConsumedAt != consumed.ConsumedAt || strings.TrimSpace(prepared.PreparedAt) == "" || strings.TrimSpace(prepared.ReleasedAt) == "" || prepared.SourceResumedAt != "" { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: server returned a changed released handback replay", ErrCheckpointNonConverged) + } + for _, raw := range []string{prepared.PreparedAt, prepared.ReleasedAt} { + if _, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(raw)); err != nil { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: server returned malformed released handback replay time", ErrCheckpointNonConverged) + } + } + // The released proof is durable recovery for a commit whose response was + // lost. It still cannot be reported as a successful local handback until + // a scan performed after this RPC proves no process appended while the + // proof was in flight. + postReplayDigest, err := s.checkpointLocalDigestLocked() + if err != nil || postReplayDigest != secondDigest { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: destination changed while recovering released handback proof", ErrCheckpointConcurrentMutation) + } + return prepared, health, nil + } + if prepared.Status != "prepared" || prepared.SealID != consumed.SealID || prepared.WorkspaceID != s.workspace || prepared.Root != "/" || prepared.SessionID != consumed.SessionID || prepared.Generation != consumed.Generation || + prepared.Digest != secondDigest || !checkpointEventCursorPattern.MatchString(strings.TrimSpace(prepared.EventCursor)) || !checkpointRevisionPattern.MatchString(strings.TrimSpace(prepared.WorkspaceRevision)) || prepared.ConsumedAt != consumed.ConsumedAt || strings.TrimSpace(prepared.PreparedAt) == "" || prepared.ReleasedAt != "" || prepared.SourceResumedAt != "" { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: server returned a changed handback preparation", ErrCheckpointNonConverged) + } + if _, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(prepared.PreparedAt)); err != nil { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: server returned malformed handback preparation time", ErrCheckpointNonConverged) + } + s.runCheckpointTestHook("handback-after-prepare") + closingDigest, err := s.checkpointLocalDigestLocked() + if err != nil || closingDigest != secondDigest { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: destination changed after handback preparation", ErrCheckpointConcurrentMutation) + } + + request.Phase = CheckpointHandbackPhaseCommit + proof, err := handbackClient.HandbackCheckpointSeal(ctx, s.workspace, request) + if err != nil { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: commit destination handback: %w", ErrCheckpointNonConverged, err) + } + // Commit releases remote ownership. A local append can occur inside the + // commit RPC even though the mount daemon and watcher are stopped, so the + // pre-commit closing scan is not sufficient. Fail fenced instead of + // reporting success if the local bytes no longer match the released proof. + postCommitDigest, postCommitErr := s.checkpointLocalDigestLocked() + if postCommitErr != nil || postCommitDigest != secondDigest { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: destination changed while committing handback; remote release outcome must be recovered with the same handback identity", ErrCheckpointConcurrentMutation) + } + if proof.Status != "released" || proof.SealID != consumed.SealID || proof.WorkspaceID != s.workspace || proof.Root != "/" || proof.SessionID != consumed.SessionID || proof.Generation != consumed.Generation || + proof.Digest != secondDigest || proof.WorkspaceRevision != prepared.WorkspaceRevision || proof.EventCursor != prepared.EventCursor || proof.ConsumedAt != consumed.ConsumedAt || proof.PreparedAt != prepared.PreparedAt || strings.TrimSpace(proof.ReleasedAt) == "" || proof.SourceResumedAt != "" { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: server returned a changed handback proof", ErrCheckpointNonConverged) + } + if _, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(proof.ReleasedAt)); err != nil { + return CheckpointSealOwnership{}, health, fmt.Errorf("%w: server returned malformed handback time", ErrCheckpointNonConverged) + } + return proof, health, nil +} + +func (s *Syncer) settleCheckpointOutbox(ctx context.Context) error { + for { + if err := s.flushOutboxRecords(ctx, nil, true, true); err != nil { + return err + } + outbox := s.summarizeOutbox() + if outbox.Pending == 0 && outbox.NeedsAttention == 0 { + return nil + } + if outbox.NeedsAttention > 0 { + return fmt.Errorf("outbox pending=%d needsAttention=%d", outbox.Pending, outbox.NeedsAttention) + } + select { + case <-ctx.Done(): + return fmt.Errorf("outbox pending=%d needsAttention=%d: %w", outbox.Pending, outbox.NeedsAttention, ctx.Err()) + case <-time.After(25 * time.Millisecond): + } + } +} + +func checkpointScopeGrantsOpsRead(scopes []string) bool { + for _, scope := range scopes { + switch strings.TrimSpace(scope) { + case "ops:read", "ops:*", "*:*", "relayfile:ops:read", "relayfile:ops:*", "relayfile:*:*": + return true + } + } + return false +} + +// VerifyCheckpoint proves that a stopped destination mount contains exactly +// the bytes/revisions named by a consumed server seal. Relayfile owns both +// halves of the proof: this process recomputes the canonical local digest and +// the server re-attests the consumed receipt against current durable state. +// Callers receive an exit-0 style verdict; they must not duplicate the digest +// algorithm or infer convergence from elapsed sync time. +func (s *Syncer) VerifyCheckpoint(ctx context.Context, expected CheckpointSeal) (CheckpointVerification, error) { + var verification CheckpointVerification + if ctx == nil { + ctx = context.Background() + } + if err := validateConsumedCheckpointReceipt(expected); err != nil { + return verification, fmt.Errorf("%w: %v", ErrCheckpointNonConverged, err) + } + if err := ctx.Err(); err != nil { + return verification, fmt.Errorf("%w: %v", ErrCheckpointNonConverged, err) + } + if _, ok := ctx.Deadline(); !ok { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 30*time.Second) + defer cancel() + } + if s.remoteRoot != "/" || expected.Root != "/" || expected.WorkspaceID != s.workspace { + return verification, fmt.Errorf("%w: destination verification v1 requires the exact full workspace root", ErrCheckpointUnmanagedRoot) + } + verifier, ok := s.client.(checkpointSealVerifierClient) + if !ok { + return verification, fmt.Errorf("%w: server/client checkpoint verification contract unavailable", ErrCheckpointNonConverged) + } + if err := s.assertManagedCheckpointRoot(); err != nil { + return verification, err + } + if err := s.assertMountRootInvariant(); err != nil { + return verification, err + } + + s.localMutationMu.Lock() + defer s.localMutationMu.Unlock() + s.mu.Lock() + defer s.mu.Unlock() + if err := s.loadState(); err != nil { + return verification, fmt.Errorf("%w: load private mount state: %v", ErrCheckpointUnmanagedRoot, err) + } + if !s.state.BootstrapComplete || s.state.LastSuccessfulReconcileAt == "" || s.state.LastError != nil { + return verification, fmt.Errorf("%w: destination mount is not successfully bootstrapped", ErrCheckpointNonConverged) + } + if len(s.state.QuarantinedPaths) > 0 || len(s.state.SkippedMaterializations) > 0 || s.state.IncrementalBacklogDraining || s.state.IncrementalCheckpoint != nil || len(s.state.IncrementalReadNotReadySince) > 0 { + return verification, fmt.Errorf("%w: destination has quarantined, skipped, or incomplete remote state", ErrCheckpointNonConverged) + } + + health, err := s.checkpointVerificationHealthLocked() + if err != nil { + return verification, fmt.Errorf("%w: inspect destination health: %v", ErrCheckpointNonConverged, err) + } + verification.Health = health + if health.PendingWriteback != 0 || health.Conflicts != 0 || health.OutboxPending != 0 || health.OutboxNeedsAttention { + return verification, fmt.Errorf("%w: pendingWriteback=%d conflicts=%d outboxPending=%d outboxNeedsAttention=%t", ErrCheckpointNonConverged, health.PendingWriteback, health.Conflicts, health.OutboxPending, health.OutboxNeedsAttention) + } + + firstDigest, err := s.checkpointLocalDigestLocked() + if err != nil { + return verification, err + } + s.runCheckpointTestHook("verify-after-first-scan") + secondDigest, err := s.checkpointLocalDigestLocked() + if err != nil || secondDigest != firstDigest { + return verification, fmt.Errorf("%w: destination changed during local verification", ErrCheckpointConcurrentMutation) + } + verification.Observed.Digest = secondDigest + verification.Observed.EventCursor = normalizedCheckpointEventCursor(s.state.EventsCursor) + if secondDigest != expected.Digest || verification.Observed.EventCursor != expected.EventCursor { + return verification, fmt.Errorf("%w: local digest/cursor does not match consumed receipt", ErrCheckpointNonConverged) + } + + serverSeal, err := verifier.VerifyCheckpointSeal(ctx, s.workspace, checkpointVerifyRequestFromSeal(expected)) + if err != nil { + return verification, fmt.Errorf("%w: server re-attestation failed: %w", ErrCheckpointNonConverged, err) + } + if !sameCheckpointReceiptIdentity(serverSeal, expected) || serverSeal.SealToken != "" { + return verification, fmt.Errorf("%w: server returned a changed or bearer-bearing verification receipt", ErrCheckpointNonConverged) + } + s.runCheckpointTestHook("verify-after-rpc") + closingDigest, err := s.checkpointLocalDigestLocked() + if err != nil || closingDigest != secondDigest { + return verification, fmt.Errorf("%w: destination changed after server re-attestation", ErrCheckpointConcurrentMutation) + } + verification.Observed.WorkspaceRevision = serverSeal.WorkspaceRevision + return verification, nil +} + +// VerifyCheckpointOwnership is the source-side admission gate after the server +// has durably returned ownership. Process health alone is insufficient: a +// restarted source can be healthy while it is still missing the destination's +// final turn. This method proves the source's tracked local bytes/revisions and +// event cursor exactly match the server-issued resume proof before Cloud admits +// another turn. +func (s *Syncer) VerifyCheckpointOwnership(ctx context.Context, proof CheckpointSealOwnership) (CheckpointVerification, error) { + var verification CheckpointVerification + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return verification, fmt.Errorf("%w: %v", ErrCheckpointNonConverged, err) + } + if proof.Status != "source-resumed" || strings.TrimSpace(proof.SealID) == "" || proof.WorkspaceID != s.workspace || proof.Root != "/" || s.remoteRoot != "/" || + !checkpointSessionPattern.MatchString(strings.TrimSpace(proof.SessionID)) || proof.Generation == 0 || !validCheckpointDigestString(proof.Digest) || + !checkpointRevisionPattern.MatchString(strings.TrimSpace(proof.WorkspaceRevision)) || !checkpointEventCursorPattern.MatchString(strings.TrimSpace(proof.EventCursor)) || + strings.TrimSpace(proof.ReleasedAt) == "" || strings.TrimSpace(proof.SourceResumedAt) == "" { + return verification, fmt.Errorf("%w: malformed or mismatched source-resume proof", ErrCheckpointNonConverged) + } + for _, raw := range []string{proof.ReleasedAt, proof.SourceResumedAt} { + if _, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(raw)); err != nil { + return verification, fmt.Errorf("%w: malformed source-resume proof timestamp", ErrCheckpointNonConverged) + } + } + if strings.TrimSpace(proof.ConsumedAt) != "" { + if _, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(proof.ConsumedAt)); err != nil { + return verification, fmt.Errorf("%w: malformed source-resume consume timestamp", ErrCheckpointNonConverged) + } + } + if err := s.assertManagedCheckpointRoot(); err != nil { + return verification, err + } + if err := s.assertMountRootInvariant(); err != nil { + return verification, err + } + + s.localMutationMu.Lock() + defer s.localMutationMu.Unlock() + s.mu.Lock() + defer s.mu.Unlock() + if err := s.loadState(); err != nil { + return verification, fmt.Errorf("%w: load private mount state: %v", ErrCheckpointUnmanagedRoot, err) + } + if !s.state.BootstrapComplete || s.state.LastSuccessfulReconcileAt == "" || s.state.LastError != nil { + return verification, fmt.Errorf("%w: resumed source mount is not successfully reconciled", ErrCheckpointNonConverged) + } + if len(s.state.QuarantinedPaths) > 0 || len(s.state.SkippedMaterializations) > 0 || s.state.IncrementalBacklogDraining || s.state.IncrementalCheckpoint != nil || len(s.state.IncrementalReadNotReadySince) > 0 { + return verification, fmt.Errorf("%w: resumed source has quarantined, skipped, or incomplete remote state", ErrCheckpointNonConverged) + } + health, err := s.checkpointVerificationHealthLocked() + if err != nil { + return verification, fmt.Errorf("%w: inspect resumed source health: %v", ErrCheckpointNonConverged, err) + } + verification.Health = health + if health.PendingWriteback != 0 || health.Conflicts != 0 || health.OutboxPending != 0 || health.OutboxNeedsAttention { + return verification, fmt.Errorf("%w: resumed source pendingWriteback=%d conflicts=%d outboxPending=%d outboxNeedsAttention=%t", ErrCheckpointNonConverged, health.PendingWriteback, health.Conflicts, health.OutboxPending, health.OutboxNeedsAttention) + } + firstDigest, err := s.checkpointLocalDigestLocked() + if err != nil { + return verification, err + } + s.runCheckpointTestHook("resume-proof-after-first-scan") + secondDigest, err := s.checkpointLocalDigestLocked() + if err != nil || firstDigest != secondDigest { + return verification, fmt.Errorf("%w: resumed source changed during admission verification", ErrCheckpointConcurrentMutation) + } + verification.Observed = CheckpointObservedState{ + Digest: secondDigest, WorkspaceRevision: proof.WorkspaceRevision, + EventCursor: normalizedCheckpointEventCursor(s.state.EventsCursor), + } + if verification.Observed.Digest != proof.Digest || verification.Observed.EventCursor != proof.EventCursor { + return verification, fmt.Errorf("%w: resumed source digest/cursor does not match ownership proof", ErrCheckpointNonConverged) + } + return verification, nil +} + +func validateConsumedCheckpointReceipt(seal CheckpointSeal) error { + if strings.TrimSpace(seal.SealID) == "" || strings.TrimSpace(seal.SealToken) != "" || strings.TrimSpace(seal.WorkspaceID) == "" || seal.Root != "/" || + !checkpointSessionPattern.MatchString(strings.TrimSpace(seal.SessionID)) || seal.Generation == 0 || !validCheckpointDigestString(seal.Digest) || !checkpointRevisionPattern.MatchString(strings.TrimSpace(seal.WorkspaceRevision)) || !checkpointEventCursorPattern.MatchString(strings.TrimSpace(seal.EventCursor)) || strings.TrimSpace(seal.ConsumedAt) == "" { + return errors.New("receipt must be an exact consumed full-root seal without sealToken") + } + for _, raw := range []string{seal.IssuedAt, seal.ExpiresAt, seal.ConsumedAt} { + if _, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(raw)); err != nil { + return errors.New("receipt timestamps must be RFC3339") + } + } + return nil +} + +func normalizedCheckpointEventCursor(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "0" + } + return value +} + +func validCheckpointDigestString(value string) bool { + value = strings.TrimSpace(value) + if len(value) != len("sha256:")+sha256.Size*2 || !strings.HasPrefix(value, "sha256:") { + return false + } + _, err := hex.DecodeString(strings.TrimPrefix(value, "sha256:")) + return err == nil +} + +func checkpointVerifyRequestFromSeal(seal CheckpointSeal) CheckpointSealVerifyRequest { + return CheckpointSealVerifyRequest{ + SealID: seal.SealID, Root: seal.Root, SessionID: seal.SessionID, Generation: seal.Generation, + Digest: seal.Digest, WorkspaceRevision: seal.WorkspaceRevision, EventCursor: seal.EventCursor, + IssuedAt: seal.IssuedAt, ExpiresAt: seal.ExpiresAt, ConsumedAt: seal.ConsumedAt, + } +} + +func sameCheckpointReceiptIdentity(left, right CheckpointSeal) bool { + return left.SealID == right.SealID && left.WorkspaceID == right.WorkspaceID && left.Root == right.Root && left.SessionID == right.SessionID && left.Generation == right.Generation && + left.Digest == right.Digest && left.WorkspaceRevision == right.WorkspaceRevision && left.EventCursor == right.EventCursor && left.IssuedAt == right.IssuedAt && left.ExpiresAt == right.ExpiresAt && left.ConsumedAt == right.ConsumedAt +} + +func (s *Syncer) checkpointVerificationHealthLocked() (CheckpointVerificationHealth, error) { + var health CheckpointVerificationHealth + for remotePath, tracked := range s.state.Files { + if isUnderRemoteRoot(s.remoteRoot, remotePath) && (tracked.Dirty || tracked.DeletePending || tracked.Denied || tracked.WriteDenied) { + health.PendingWriteback++ + } + } + _, conflicts, err := s.listConflictArtifacts() + if err != nil { + return health, err + } + health.Conflicts = conflicts + pending, err := s.listPendingOutboxRecords() + if err != nil { + return health, err + } + health.OutboxPending = len(pending) + failed, err := countCheckpointJSONFiles(s.outboxFailedDir()) + if err != nil { + return health, err + } + attention, err := countCheckpointSuffixFiles(s.outboxAttentionDir(), ".marker") + if err != nil { + return health, err + } + health.OutboxNeedsAttention = failed > 0 || attention > 0 + return health, nil +} + +func countCheckpointJSONFiles(dir string) (int, error) { + return countCheckpointSuffixFiles(dir, ".json") +} + +func countCheckpointSuffixFiles(dir, suffix string) (int, error) { + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return 0, nil + } + if err != nil { + return 0, err + } + count := 0 + for _, entry := range entries { + if !entry.IsDir() && strings.HasSuffix(entry.Name(), suffix) { + count++ + } + } + return count, nil +} + +func (s *Syncer) assertManagedCheckpointRoot() error { + private, err := os.Stat(s.stateFile) + if err != nil || !private.Mode().IsRegular() { + return fmt.Errorf("%w: private mount state is missing", ErrCheckpointUnmanagedRoot) + } + payload, err := os.ReadFile(s.publicStatePath) + if err != nil { + return fmt.Errorf("%w: public mount identity is missing", ErrCheckpointUnmanagedRoot) + } + var state publicState + if err := json.Unmarshal(payload, &state); err != nil { + return fmt.Errorf("%w: malformed public mount identity", ErrCheckpointUnmanagedRoot) + } + if strings.TrimSpace(state.LocalRoot) == "" || strings.TrimSpace(state.WorkspaceID) == "" || strings.TrimSpace(state.RemoteRoot) == "" { + return fmt.Errorf("%w: public mount identity is incomplete", ErrCheckpointUnmanagedRoot) + } + wantRoot, err := filepath.Abs(s.localRoot) + if err != nil { + return fmt.Errorf("%w: resolve local root", ErrCheckpointUnmanagedRoot) + } + gotRoot, err := filepath.Abs(strings.TrimSpace(state.LocalRoot)) + if err != nil || state.WorkspaceID != s.workspace || normalizeRemotePath(state.RemoteRoot) != s.remoteRoot || filepath.Clean(gotRoot) != filepath.Clean(wantRoot) { + return fmt.Errorf("%w: mount identity does not match workspace/root", ErrCheckpointUnmanagedRoot) + } + return nil +} + +func (s *Syncer) checkpointLocalDigestLocked() (string, error) { + localFiles, err := s.scanLocalFiles() + if err != nil { + return "", fmt.Errorf("%w: scan local root: %v", ErrCheckpointNonConverged, err) + } + entries := make([]relayfile.CheckpointDigestEntry, 0, len(localFiles)) + seen := make(map[string]struct{}, len(localFiles)) + for remotePath, snapshot := range localFiles { + if !isUnderRemoteRoot(s.remoteRoot, remotePath) || snapshot.SkipWriteback { + return "", fmt.Errorf("%w: unsealable local path %s", ErrCheckpointNonConverged, remotePath) + } + tracked, ok := s.state.Files[remotePath] + if !ok || tracked.Denied || tracked.WriteDenied || tracked.Dirty || tracked.DeletePending || strings.TrimSpace(tracked.Revision) == "" || strings.TrimSpace(tracked.Hash) == "" || tracked.Hash != snapshot.Hash { + return "", fmt.Errorf("%w: local path %s is not tracked at its durable hash/revision (tracked=%t dirty=%t deletePending=%t denied=%t writeDenied=%t hashMatch=%t revisionPresent=%t)", ErrCheckpointNonConverged, remotePath, ok, tracked.Dirty, tracked.DeletePending, tracked.Denied, tracked.WriteDenied, tracked.Hash == snapshot.Hash, strings.TrimSpace(tracked.Revision) != "") + } + seen[remotePath] = struct{}{} + entries = append(entries, relayfile.CheckpointDigestEntry{Path: remotePath, Revision: tracked.Revision, ContentHash: snapshot.Hash}) + } + for remotePath, tracked := range s.state.Files { + if !isUnderRemoteRoot(s.remoteRoot, remotePath) { + continue + } + if tracked.Denied || tracked.WriteDenied || tracked.Dirty || tracked.DeletePending { + return "", fmt.Errorf("%w: tracked path %s is denied or unsettled", ErrCheckpointNonConverged, remotePath) + } + if _, ok := seen[remotePath]; !ok { + return "", fmt.Errorf("%w: tracked durable path %s is missing locally", ErrCheckpointNonConverged, remotePath) + } + } + digest, err := relayfile.ComputeCheckpointDigest(s.remoteRoot, entries) + if err != nil { + return "", fmt.Errorf("%w: canonical digest: %v", ErrCheckpointNonConverged, err) + } + return digest, nil +} + +func (s *Syncer) runCheckpointTestHook(stage string) { + if s.checkpointTestHook != nil { + s.checkpointTestHook(stage) + } +} + // HandleLocalChange routes a local filesystem event to the appropriate // writeback action. // diff --git a/internal/mountsync/syncer_test.go b/internal/mountsync/syncer_test.go index c899cc80..41ec45a7 100644 --- a/internal/mountsync/syncer_test.go +++ b/internal/mountsync/syncer_test.go @@ -8334,6 +8334,708 @@ func TestAtomicTempPatternHidesTempForDotPrefixedTarget(t *testing.T) { } } +func TestCheckpointAndSealDetectsConcurrentAppendBeforeIssuance(t *testing.T) { + client := &fakeClient{} + syncer, localPath := newManagedCheckpointSyncer(t, client) + mutated := false + syncer.checkpointTestHook = func(stage string) { + if stage == "after-first-scan" && !mutated { + mutated = true + if err := os.WriteFile(localPath, []byte("turn one\nturn two\n"), 0o644); err != nil { + t.Fatalf("concurrent append: %v", err) + } + } + } + _, err := syncer.CheckpointAndSeal(context.Background(), CheckpointAndSealOptions{SessionID: "thread-append", Generation: 1}) + if !errors.Is(err, ErrCheckpointConcurrentMutation) { + t.Fatalf("checkpoint error = %v, want concurrent mutation", err) + } + if client.checkpointIssueCalls != 0 { + t.Fatalf("server seal was issued before local stability: %d calls", client.checkpointIssueCalls) + } +} + +func TestCheckpointAndSealDetectsConcurrentAppendAfterIssuance(t *testing.T) { + client := &fakeClient{} + syncer, localPath := newManagedCheckpointSyncer(t, client) + syncer.checkpointTestHook = func(stage string) { + if stage == "after-issue" { + if err := os.WriteFile(localPath, []byte("turn one\nlate append\n"), 0o644); err != nil { + t.Fatalf("late concurrent append: %v", err) + } + } + } + _, err := syncer.CheckpointAndSeal(context.Background(), CheckpointAndSealOptions{SessionID: "thread-late", Generation: 2}) + if !errors.Is(err, ErrCheckpointConcurrentMutation) { + t.Fatalf("checkpoint error = %v, want concurrent mutation", err) + } + if client.checkpointIssueCalls != 1 { + t.Fatalf("server issue calls = %d, want 1", client.checkpointIssueCalls) + } +} + +func TestCheckpointAndSealHonorsDeadlineAndRemoteNonConvergence(t *testing.T) { + t.Run("deadline", func(t *testing.T) { + client := &fakeClient{checkpointIssueFunc: func(ctx context.Context, _ string, _ CheckpointSealRequest) (CheckpointSeal, error) { + <-ctx.Done() + return CheckpointSeal{}, ctx.Err() + }} + syncer, _ := newManagedCheckpointSyncer(t, client) + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) + defer cancel() + started := time.Now() + _, err := syncer.CheckpointAndSeal(ctx, CheckpointAndSealOptions{SessionID: "thread-timeout", Generation: 3}) + if !errors.Is(err, ErrCheckpointNonConverged) || !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("deadline error = %v", err) + } + if elapsed := time.Since(started); elapsed > time.Second { + t.Fatalf("checkpoint ignored deadline: %s", elapsed) + } + }) + + t.Run("server divergence", func(t *testing.T) { + client := &fakeClient{checkpointIssueFunc: func(context.Context, string, CheckpointSealRequest) (CheckpointSeal, error) { + return CheckpointSeal{}, &HTTPError{StatusCode: http.StatusConflict, Code: "checkpoint_diverged", Message: "server digest differs"} + }} + syncer, _ := newManagedCheckpointSyncer(t, client) + _, err := syncer.CheckpointAndSeal(context.Background(), CheckpointAndSealOptions{SessionID: "thread-diverged", Generation: 4}) + if !errors.Is(err, ErrCheckpointNonConverged) || !strings.Contains(err.Error(), "checkpoint_diverged") { + t.Fatalf("divergence error = %v", err) + } + }) +} + +func TestCheckpointAndSealRejectsUnmanagedOrMismatchedRoot(t *testing.T) { + localRoot := t.TempDir() + syncer, err := NewSyncer(&fakeClient{}, SyncerOptions{WorkspaceID: "ws_unmanaged", RemoteRoot: "/sessions", LocalRoot: localRoot, StateDir: t.TempDir()}) + if err != nil { + t.Fatalf("new unmanaged syncer: %v", err) + } + if _, err := syncer.CheckpointAndSeal(context.Background(), CheckpointAndSealOptions{SessionID: "thread-unmanaged", Generation: 1}); !errors.Is(err, ErrCheckpointUnmanagedRoot) { + t.Fatalf("unmanaged root error = %v", err) + } + + managed, _ := newManagedCheckpointSyncer(t, &fakeClient{}) + payload, err := os.ReadFile(managed.publicStatePath) + if err != nil { + t.Fatalf("read public state: %v", err) + } + var public map[string]any + if err := json.Unmarshal(payload, &public); err != nil { + t.Fatalf("decode public state: %v", err) + } + public["workspaceId"] = "ws_other" + payload, err = json.Marshal(public) + if err != nil { + t.Fatalf("encode altered public state: %v", err) + } + if err := os.WriteFile(managed.publicStatePath, payload, 0o600); err != nil { + t.Fatalf("write altered public state: %v", err) + } + if _, err := managed.CheckpointAndSeal(context.Background(), CheckpointAndSealOptions{SessionID: "thread-mismatch", Generation: 1}); !errors.Is(err, ErrCheckpointUnmanagedRoot) { + t.Fatalf("mismatched root error = %v", err) + } +} + +func TestCheckpointAndSealReturnsBoundAuthoritativeSeal(t *testing.T) { + client := &fakeClient{} + syncer, _ := newManagedCheckpointSyncer(t, client) + seal, err := syncer.CheckpointAndSeal(context.Background(), CheckpointAndSealOptions{SessionID: "thread-success", Generation: 9, TTLSeconds: 45}) + if err != nil { + t.Fatalf("checkpoint and seal: %v", err) + } + if seal.WorkspaceID != "ws_checkpoint" || seal.Root != "/" || seal.SessionID != "thread-success" || seal.Generation != 9 || seal.SealToken == "" || !strings.HasPrefix(seal.Digest, "sha256:") { + t.Fatalf("bound seal = %+v", seal) + } + if len(client.checkpointIssueRequests) != 1 || !checkpointSessionPattern.MatchString(client.checkpointIssueRequests[0].IssuanceIdempotencyKey) { + t.Fatalf("checkpoint request lacks stable issuance identity: %+v", client.checkpointIssueRequests) + } + wantKey := checkpointIssuanceIdempotencyKey("ws_checkpoint", "/", "thread-success", 9, seal.Digest, 45) + if client.checkpointIssueRequests[0].IssuanceIdempotencyKey != wantKey { + t.Fatalf("issuance key = %q, want %q", client.checkpointIssueRequests[0].IssuanceIdempotencyKey, wantKey) + } +} + +func TestCheckpointAndSealRejectsInvalidIdentityBeforeDraining(t *testing.T) { + client := &fakeClient{} + syncer, _ := newManagedCheckpointSyncer(t, client) + for name, options := range map[string]CheckpointAndSealOptions{ + "bad session": {SessionID: "bad session", Generation: 1}, + "zero generation": {SessionID: "thread-valid"}, + "negative ttl": {SessionID: "thread-valid", Generation: 1, TTLSeconds: -1}, + "oversize ttl": {SessionID: "thread-valid", Generation: 1, TTLSeconds: 301}, + } { + t.Run(name, func(t *testing.T) { + if _, err := syncer.CheckpointAndSeal(context.Background(), options); !errors.Is(err, ErrCheckpointNonConverged) { + t.Fatalf("invalid identity error = %v", err) + } + }) + } + if client.checkpointIssueCalls != 0 || client.writeFileCalls != 0 || client.bulkWriteCalls != 0 { + t.Fatalf("invalid input reached drain/server path: issue=%d write=%d bulk=%d", client.checkpointIssueCalls, client.writeFileCalls, client.bulkWriteCalls) + } +} + +func TestCheckpointAndHandbackRequireOpsReadBeforeDrain(t *testing.T) { + client := &fakeClient{} + syncer, _ := newManagedCheckpointSyncer(t, client) + syncer.scopes = []string{"fs:read", "fs:write", "sync:trigger"} + if _, err := syncer.CheckpointAndSeal(context.Background(), CheckpointAndSealOptions{SessionID: "thread-scope", Generation: 1}); !errors.Is(err, ErrCheckpointNonConverged) || !strings.Contains(err.Error(), "ops:read") { + t.Fatalf("checkpoint scope error = %v", err) + } + receipt := consumedCheckpointReceiptForTest(t, syncer) + if _, _, err := syncer.HandbackCheckpoint(context.Background(), receipt, "cutover-scope", "handback-scope"); !errors.Is(err, ErrCheckpointNonConverged) || !strings.Contains(err.Error(), "ops:read") { + t.Fatalf("handback scope error = %v", err) + } + if client.checkpointIssueCalls != 0 || client.checkpointHandbackCalls != 0 || client.writeFileCalls != 0 || client.bulkWriteCalls != 0 { + t.Fatalf("missing ops:read reached drain/server: %+v", client) + } +} + +func TestCheckpointAndSealRejectsNonQuiescentMountBeforeDrain(t *testing.T) { + for name, mutate := range map[string]func(*Syncer){ + "active watcher": func(syncer *Syncer) { + syncer.localWatcherActive = true + }, + "incremental checkpoint": func(syncer *Syncer) { + syncer.state.IncrementalCheckpoint = &incrementalCheckpoint{Cursor: "evt_1"} + if err := syncer.saveStateWithoutLocalScan(); err != nil { + t.Fatalf("persist incremental checkpoint: %v", err) + } + }, + "read not ready marker": func(syncer *Syncer) { + syncer.state.IncrementalReadNotReadySince = map[string]string{"/transcript.jsonl": time.Now().UTC().Format(time.RFC3339Nano)} + if err := syncer.saveStateWithoutLocalScan(); err != nil { + t.Fatalf("persist read-not-ready marker: %v", err) + } + }, + } { + t.Run(name, func(t *testing.T) { + client := &fakeClient{} + syncer, _ := newManagedCheckpointSyncer(t, client) + mutate(syncer) + _, err := syncer.CheckpointAndSeal(context.Background(), CheckpointAndSealOptions{SessionID: "thread-quiescence", Generation: 1}) + if !errors.Is(err, ErrCheckpointNonConverged) { + t.Fatalf("non-quiescent checkpoint error = %v", err) + } + if client.checkpointIssueCalls != 0 || client.writeFileCalls != 0 || client.bulkWriteCalls != 0 { + t.Fatalf("non-quiescent mount reached drain/server: issue=%d write=%d bulk=%d", client.checkpointIssueCalls, client.writeFileCalls, client.bulkWriteCalls) + } + }) + } +} + +func TestCheckpointAndSealRejectsPreexistingConflictBeforeIssuance(t *testing.T) { + client := &fakeClient{} + syncer, _ := newManagedCheckpointSyncer(t, client) + artifact := conflictArtifactPath(syncer.conflictsDir, "/transcript.jsonl", "rev_1") + if err := os.MkdirAll(filepath.Dir(artifact), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(artifact, []byte("unresolved local conflict\n"), 0o600); err != nil { + t.Fatal(err) + } + _, err := syncer.CheckpointAndSeal(context.Background(), CheckpointAndSealOptions{SessionID: "thread-conflict", Generation: 1}) + if !errors.Is(err, ErrCheckpointNonConverged) || !strings.Contains(err.Error(), "conflicts=1") { + t.Fatalf("pre-existing conflict error = %v", err) + } + if client.checkpointIssueCalls != 0 { + t.Fatalf("server seal issued with pre-existing conflict: %d", client.checkpointIssueCalls) + } +} + +func TestHandbackCheckpointDrainsAndReturnsAuthoritativeRelease(t *testing.T) { + client := &fakeClient{} + syncer, _ := newManagedCheckpointSyncer(t, client) + receipt := consumedCheckpointReceiptForTest(t, syncer) + localDestinationWrite := filepath.Join(syncer.localRoot, "destination.txt") + if err := os.WriteFile(localDestinationWrite, []byte("destination turn\n"), 0o644); err != nil { + t.Fatal(err) + } + proof, health, err := syncer.HandbackCheckpoint(context.Background(), receipt, "cutover-job-one", "handback-job-one") + if err != nil { + t.Fatalf("handback: %v", err) + } + if client.checkpointHandbackCalls != 2 || client.checkpointHandbackRequest.Phase != CheckpointHandbackPhaseCommit || client.writeFileCalls+client.bulkWriteCalls == 0 { + t.Fatalf("handback calls=%d writes=%d bulkWrites=%d", client.checkpointHandbackCalls, client.writeFileCalls, client.bulkWriteCalls) + } + if proof.Status != "released" || proof.Digest != client.checkpointHandbackRequest.ExpectedDigest || !checkpointEventCursorPattern.MatchString(proof.EventCursor) || proof.SourceResumedAt != "" { + t.Fatalf("handback proof=%+v request=%+v", proof, client.checkpointHandbackRequest) + } + if health != (CheckpointVerificationHealth{}) { + t.Fatalf("handback health=%+v", health) + } +} + +func TestHandbackCheckpointAcceptsReleasedPrepareReplayAfterLostCommitResponse(t *testing.T) { + client := &fakeClient{} + syncer, _ := newManagedCheckpointSyncer(t, client) + receipt := consumedCheckpointReceiptForTest(t, syncer) + preparedAt := time.Now().UTC().Add(-time.Second).Format(time.RFC3339Nano) + releasedAt := time.Now().UTC().Format(time.RFC3339Nano) + client.checkpointHandbackProof = CheckpointSealOwnership{ + SealID: receipt.SealID, WorkspaceID: receipt.WorkspaceID, Root: receipt.Root, + SessionID: receipt.SessionID, Generation: receipt.Generation, Status: "released", + Digest: receipt.Digest, WorkspaceRevision: "rev_3", EventCursor: "evt_3", + ConsumedAt: receipt.ConsumedAt, PreparedAt: preparedAt, ReleasedAt: releasedAt, + } + proof, health, err := syncer.HandbackCheckpoint(context.Background(), receipt, "cutover-lost-commit-response", "handback-lost-commit-response") + if err != nil { + t.Fatalf("recover released handback: %v", err) + } + if proof != client.checkpointHandbackProof || health != (CheckpointVerificationHealth{}) { + t.Fatalf("released replay proof=%+v health=%+v", proof, health) + } + if client.checkpointHandbackCalls != 1 || client.checkpointHandbackRequest.Phase != CheckpointHandbackPhasePrepare { + t.Fatalf("released replay calls=%d phase=%q", client.checkpointHandbackCalls, client.checkpointHandbackRequest.Phase) + } +} + +func TestHandbackCheckpointFailsBeforeReleaseOnMutationOrPendingHealth(t *testing.T) { + t.Run("concurrent mutation", func(t *testing.T) { + client := &fakeClient{} + syncer, localPath := newManagedCheckpointSyncer(t, client) + receipt := consumedCheckpointReceiptForTest(t, syncer) + syncer.checkpointTestHook = func(stage string) { + if stage == "handback-after-first-scan" { + if err := os.WriteFile(localPath, []byte("late destination write\n"), 0o644); err != nil { + t.Fatal(err) + } + } + } + if _, _, err := syncer.HandbackCheckpoint(context.Background(), receipt, "cutover-job-two", "handback-job-two"); !errors.Is(err, ErrCheckpointConcurrentMutation) { + t.Fatalf("mutation error=%v", err) + } + if client.checkpointHandbackCalls != 0 { + t.Fatalf("mutating handback reached server: %d", client.checkpointHandbackCalls) + } + }) + + t.Run("mutation during prepare never commits release", func(t *testing.T) { + client := &fakeClient{} + syncer, localPath := newManagedCheckpointSyncer(t, client) + receipt := consumedCheckpointReceiptForTest(t, syncer) + syncer.checkpointTestHook = func(stage string) { + if stage == "handback-after-prepare" { + if err := os.WriteFile(localPath, []byte("callback append during prepare\n"), 0o644); err != nil { + t.Fatal(err) + } + } + } + if _, _, err := syncer.HandbackCheckpoint(context.Background(), receipt, "cutover-job-prepare-race", "handback-job-prepare-race"); !errors.Is(err, ErrCheckpointConcurrentMutation) { + t.Fatalf("prepare race error=%v", err) + } + if client.checkpointHandbackCalls != 1 || client.checkpointHandbackRequest.Phase != CheckpointHandbackPhasePrepare { + t.Fatalf("unsafe handback calls=%d lastPhase=%q", client.checkpointHandbackCalls, client.checkpointHandbackRequest.Phase) + } + }) + + t.Run("append inside commit callback releases remotely but fails fenced locally", func(t *testing.T) { + client := &fakeClient{} + syncer, localPath := newManagedCheckpointSyncer(t, client) + receipt := consumedCheckpointReceiptForTest(t, syncer) + preparedAt := time.Now().UTC().Add(-time.Second).Format(time.RFC3339Nano) + releasedAt := time.Now().UTC().Format(time.RFC3339Nano) + authoritativeDigest := "" + released := false + client.checkpointHandbackFunc = func(_ context.Context, workspaceID string, request CheckpointSealHandbackRequest) (CheckpointSealOwnership, error) { + status := "prepared" + proofReleasedAt := "" + if authoritativeDigest == "" { + authoritativeDigest = request.ExpectedDigest + } + if released { + status = "released" + proofReleasedAt = releasedAt + } else if request.Phase == CheckpointHandbackPhaseCommit { + file, err := os.OpenFile(localPath, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteString("append during commit callback\n"); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + released = true + status = "released" + proofReleasedAt = releasedAt + } + return CheckpointSealOwnership{ + SealID: request.SealID, WorkspaceID: workspaceID, Root: request.Root, + SessionID: request.SessionID, Generation: request.Generation, Status: status, + Digest: authoritativeDigest, WorkspaceRevision: "rev_3", EventCursor: "evt_3", + ConsumedAt: request.ConsumedAt, PreparedAt: preparedAt, ReleasedAt: proofReleasedAt, + }, nil + } + + proof, _, err := syncer.HandbackCheckpoint(context.Background(), receipt, "cutover-job-commit-race", "handback-job-commit-race") + if !errors.Is(err, ErrCheckpointConcurrentMutation) || proof != (CheckpointSealOwnership{}) || !released { + t.Fatalf("commit append proof=%+v released=%v err=%v", proof, released, err) + } + if client.checkpointHandbackCalls != 2 || client.checkpointHandbackRequest.Phase != CheckpointHandbackPhaseCommit { + t.Fatalf("commit append calls=%d lastPhase=%q", client.checkpointHandbackCalls, client.checkpointHandbackRequest.Phase) + } + + // The same durable handback identity can be retried, but the released + // server proof excludes the appended bytes. The retry must remain fenced + // and must never attempt a second commit. + if retryProof, _, retryErr := syncer.HandbackCheckpoint(context.Background(), receipt, "cutover-job-commit-race", "handback-job-commit-race"); !errors.Is(retryErr, ErrCheckpointNonConverged) || retryProof != (CheckpointSealOwnership{}) { + t.Fatalf("commit append retry proof=%+v err=%v", retryProof, retryErr) + } + if client.checkpointHandbackCalls != 3 || client.checkpointHandbackRequest.Phase != CheckpointHandbackPhasePrepare { + t.Fatalf("commit append retry calls=%d lastPhase=%q", client.checkpointHandbackCalls, client.checkpointHandbackRequest.Phase) + } + }) + + t.Run("active watcher is not safe handback quiescence", func(t *testing.T) { + client := &fakeClient{} + syncer, _ := newManagedCheckpointSyncer(t, client) + receipt := consumedCheckpointReceiptForTest(t, syncer) + syncer.localWatcherActive = true + if _, _, err := syncer.HandbackCheckpoint(context.Background(), receipt, "cutover-job-watcher", "handback-job-watcher"); !errors.Is(err, ErrCheckpointNonConverged) || !strings.Contains(err.Error(), "watcherless") { + t.Fatalf("watcher quiescence error=%v", err) + } + if client.checkpointHandbackCalls != 0 { + t.Fatalf("active watcher reached server: %d", client.checkpointHandbackCalls) + } + }) + + t.Run("pre-existing conflict", func(t *testing.T) { + client := &fakeClient{} + syncer, _ := newManagedCheckpointSyncer(t, client) + receipt := consumedCheckpointReceiptForTest(t, syncer) + artifact := conflictArtifactPath(syncer.conflictsDir, "/transcript.jsonl", "rev_1") + if err := os.MkdirAll(filepath.Dir(artifact), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(artifact, []byte("conflict\n"), 0o600); err != nil { + t.Fatal(err) + } + _, health, err := syncer.HandbackCheckpoint(context.Background(), receipt, "cutover-job-three", "handback-job-three") + if !errors.Is(err, ErrCheckpointNonConverged) || health.Conflicts != 1 { + t.Fatalf("conflict health=%+v err=%v", health, err) + } + if client.checkpointHandbackCalls != 0 { + t.Fatalf("conflicted handback reached server: %d", client.checkpointHandbackCalls) + } + }) +} + +func TestCheckpointOwnershipEndToEndHandbackPreservesDestinationTurn(t *testing.T) { + store := relayfile.NewStore() + t.Cleanup(store.Close) + workspaceID := "ws_checkpoint_handoff_e2e" + if _, err := store.WriteFile(relayfile.WriteRequest{WorkspaceID: workspaceID, Path: "/seed.txt", IfMatch: "0", Content: "seed\n"}); err != nil { + t.Fatal(err) + } + api := httptest.NewServer(newMountsyncAPIHandler(t, store)) + defer api.Close() + sourceToken := mustMountsyncTestJWT(t, "dev-secret", workspaceID, "local-source", []string{"fs:read", "fs:write", "sync:trigger", "ops:read"}, time.Now().Add(time.Hour)) + destinationPrincipal := "live-teleport-e2e-proof" + destinationToken := mustMountsyncTestJWT(t, "dev-secret", workspaceID, destinationPrincipal, []string{"fs:read", "fs:write", "sync:trigger", "ops:read"}, time.Now().Add(time.Hour)) + sourceClient := NewHTTPClient(api.URL, sourceToken, api.Client()) + destinationClient := NewHTTPClient(api.URL, destinationToken, api.Client()) + + newE2ESyncer := func(localRoot string, client *HTTPClient) *Syncer { + syncer, err := NewSyncer(client, SyncerOptions{ + WorkspaceID: workspaceID, RemoteRoot: "/", LocalRoot: localRoot, + StateDir: t.TempDir(), RootCtx: context.Background(), FullPullEvery: -1, WebSocket: boolPtr(false), + }) + if err != nil { + t.Fatalf("new syncer: %v", err) + } + if err := syncer.SyncOnce(context.Background()); err != nil { + t.Fatalf("initial sync: %v", err) + } + return syncer + } + + sourceRoot := t.TempDir() + source := newE2ESyncer(sourceRoot, sourceClient) + if err := os.WriteFile(filepath.Join(sourceRoot, "source-turn.txt"), []byte("source turn\n"), 0o644); err != nil { + t.Fatal(err) + } + issued, err := source.CheckpointAndSeal(context.Background(), CheckpointAndSealOptions{SessionID: "thread-e2e", Generation: 1}) + if err != nil { + t.Fatalf("source checkpoint: %v", err) + } + consumerKey := "cutover-e2e-one" + consumed, err := destinationClient.ConsumeCheckpointSeal(context.Background(), workspaceID, CheckpointSealConsumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, + Generation: issued.Generation, ConsumerIdempotencyKey: consumerKey, + }) + if err != nil { + t.Fatalf("destination consume: %v", err) + } + if consumed.SealToken != "" || consumed.ConsumedAt == "" { + t.Fatalf("unsafe consumed receipt: %+v", consumed) + } + if _, err := sourceClient.ResumeCheckpointSeal(context.Background(), workspaceID, CheckpointSealResumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, + Generation: issued.Generation, ResumeIdempotencyKey: "source-resume-e2e", + }); err == nil { + t.Fatal("source resumed before destination handback") + } else { + var httpErr *HTTPError + if !errors.As(err, &httpErr) || httpErr.Code != "checkpoint_handback_required" { + t.Fatalf("premature source resume error = %v", err) + } + } + + destinationRoot := t.TempDir() + destination := newE2ESyncer(destinationRoot, destinationClient) + if _, err := destination.VerifyCheckpoint(context.Background(), consumed); err != nil { + t.Fatalf("destination verify: %v", err) + } + if err := os.WriteFile(filepath.Join(destinationRoot, "destination-turn.txt"), []byte("destination turn\n"), 0o644); err != nil { + t.Fatal(err) + } + proof, health, err := destination.HandbackCheckpoint(context.Background(), consumed, consumerKey, "handback-e2e-one") + if err != nil { + t.Fatalf("destination handback: %v", err) + } + if proof.Status != "released" || proof.Digest == consumed.Digest || health != (CheckpointVerificationHealth{}) { + t.Fatalf("handback proof=%+v health=%+v", proof, health) + } + resumed, err := sourceClient.ResumeCheckpointSeal(context.Background(), workspaceID, CheckpointSealResumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, + Generation: issued.Generation, ResumeIdempotencyKey: "source-resume-e2e", + }) + if err != nil || resumed.Status != "source-resumed" || resumed.Digest != proof.Digest { + t.Fatalf("source resume proof=%+v err=%v", resumed, err) + } + if _, statErr := os.Stat(filepath.Join(sourceRoot, "destination-turn.txt")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("destination turn materialized before admission check: %v", statErr) + } + if _, err := source.VerifyCheckpointOwnership(context.Background(), resumed); !errors.Is(err, ErrCheckpointNonConverged) { + t.Fatalf("source admitted before pulling destination turn: %v", err) + } + if _, err := os.Stat(filepath.Join(sourceRoot, "destination-turn.txt")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("destination turn materialized before source reconcile: %v", err) + } + source.forceFullReconcile = true + if err := source.Reconcile(context.Background()); err != nil { + t.Fatalf("source reconcile after handback: %v", err) + } + assertLocalFileContent(t, filepath.Join(sourceRoot, "destination-turn.txt"), "destination turn\n") + if _, err := source.VerifyCheckpointOwnership(context.Background(), resumed); !errors.Is(err, ErrCheckpointNonConverged) { + t.Fatalf("source admitted before durable event cursor caught up: %v", err) + } + if err := source.SyncOnce(context.Background()); err != nil { + t.Fatalf("source incremental cursor catch-up after handback: %v", err) + } + admission, err := source.VerifyCheckpointOwnership(context.Background(), resumed) + if err != nil || admission.Observed.Digest != resumed.Digest || admission.Observed.EventCursor != resumed.EventCursor || admission.Observed.WorkspaceRevision != resumed.WorkspaceRevision || admission.Health != (CheckpointVerificationHealth{}) { + t.Fatalf("source admission=%+v err=%v", admission, err) + } + assertLocalFileContent(t, filepath.Join(sourceRoot, "source-turn.txt"), "source turn\n") + assertLocalFileContent(t, filepath.Join(sourceRoot, "destination-turn.txt"), "destination turn\n") +} + +func TestVerifyCheckpointRequiresLocalExactnessAndServerReattestation(t *testing.T) { + t.Run("empty local cursor normalizes to canonical zero", func(t *testing.T) { + client := &fakeClient{} + syncer, _ := newManagedCheckpointSyncer(t, client) + receipt := consumedCheckpointReceiptForTest(t, syncer) + receipt.EventCursor = "0" + client.checkpointVerifySeal = receipt + syncer.state.EventsCursor = "" + if err := syncer.saveStateWithoutLocalScan(); err != nil { + t.Fatal(err) + } + verification, err := syncer.VerifyCheckpoint(context.Background(), receipt) + if err != nil { + t.Fatalf("verify canonical zero cursor: %v", err) + } + if verification.Observed.EventCursor != "0" || client.checkpointVerifyCalls != 1 { + t.Fatalf("verification=%+v calls=%d", verification, client.checkpointVerifyCalls) + } + }) + + t.Run("delete-latest workspace revision is server-attested", func(t *testing.T) { + client := &fakeClient{} + syncer, _ := newManagedCheckpointSyncer(t, client) + receipt := consumedCheckpointReceiptForTest(t, syncer) + // A delete-only latest mutation advances the server workspace revision + // beyond every surviving file revision. The verifier must not compare + // this field to the mount's max surviving revision. + syncer.state.LastAppliedRevision = "rev_1" + receipt.WorkspaceRevision = "rev_2" + client.checkpointVerifySeal = receipt + if err := syncer.saveStateWithoutLocalScan(); err != nil { + t.Fatal(err) + } + verification, err := syncer.VerifyCheckpoint(context.Background(), receipt) + if err != nil { + t.Fatalf("verify: %v", err) + } + if client.checkpointVerifyCalls != 1 || verification.Observed.WorkspaceRevision != "rev_2" || verification.Observed.Digest != receipt.Digest || verification.Observed.EventCursor != receipt.EventCursor { + t.Fatalf("verification=%+v calls=%d", verification, client.checkpointVerifyCalls) + } + }) + + t.Run("local divergence fails before server", func(t *testing.T) { + client := &fakeClient{} + syncer, localPath := newManagedCheckpointSyncer(t, client) + receipt := consumedCheckpointReceiptForTest(t, syncer) + if err := os.WriteFile(localPath, []byte("diverged\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := syncer.VerifyCheckpoint(context.Background(), receipt); !errors.Is(err, ErrCheckpointNonConverged) { + t.Fatalf("local divergence error = %v", err) + } + if client.checkpointVerifyCalls != 0 { + t.Fatalf("divergent local state reached server: %d", client.checkpointVerifyCalls) + } + }) + + t.Run("stale cursor and pending writeback fail closed", func(t *testing.T) { + client := &fakeClient{} + syncer, _ := newManagedCheckpointSyncer(t, client) + receipt := consumedCheckpointReceiptForTest(t, syncer) + syncer.state.EventsCursor = "evt_stale" + if err := syncer.saveStateWithoutLocalScan(); err != nil { + t.Fatal(err) + } + if _, err := syncer.VerifyCheckpoint(context.Background(), receipt); !errors.Is(err, ErrCheckpointNonConverged) { + t.Fatalf("stale cursor error = %v", err) + } + syncer.state.EventsCursor = receipt.EventCursor + tracked := syncer.state.Files["/transcript.jsonl"] + tracked.Dirty = true + syncer.state.Files["/transcript.jsonl"] = tracked + if err := syncer.saveStateWithoutLocalScan(); err != nil { + t.Fatal(err) + } + verification, err := syncer.VerifyCheckpoint(context.Background(), receipt) + if !errors.Is(err, ErrCheckpointNonConverged) || verification.Health.PendingWriteback != 1 { + t.Fatalf("pending verification=%+v err=%v", verification, err) + } + }) + + t.Run("server unavailable or changed response fails closed", func(t *testing.T) { + client := &fakeClient{checkpointVerifyErr: errors.New("server unavailable")} + syncer, _ := newManagedCheckpointSyncer(t, client) + receipt := consumedCheckpointReceiptForTest(t, syncer) + if _, err := syncer.VerifyCheckpoint(context.Background(), receipt); !errors.Is(err, ErrCheckpointNonConverged) || !strings.Contains(err.Error(), "server unavailable") { + t.Fatalf("server unavailable error = %v", err) + } + client.checkpointVerifyErr = nil + changed := receipt + changed.WorkspaceRevision = "rev_999" + client.checkpointVerifySeal = changed + if _, err := syncer.VerifyCheckpoint(context.Background(), receipt); !errors.Is(err, ErrCheckpointNonConverged) { + t.Fatalf("changed re-attestation error = %v", err) + } + }) + + t.Run("mutation during server re-attestation fails the closing scan", func(t *testing.T) { + client := &fakeClient{} + syncer, localPath := newManagedCheckpointSyncer(t, client) + receipt := consumedCheckpointReceiptForTest(t, syncer) + client.checkpointVerifyFunc = func(_ context.Context, workspaceID string, request CheckpointSealVerifyRequest) (CheckpointSeal, error) { + if err := os.WriteFile(localPath, []byte("callback append during verify RPC\n"), 0o644); err != nil { + t.Fatal(err) + } + return CheckpointSeal{ + SealID: request.SealID, WorkspaceID: workspaceID, Root: request.Root, + SessionID: request.SessionID, Generation: request.Generation, + Digest: request.Digest, WorkspaceRevision: request.WorkspaceRevision, + EventCursor: request.EventCursor, IssuedAt: request.IssuedAt, + ExpiresAt: request.ExpiresAt, ConsumedAt: request.ConsumedAt, + }, nil + } + if _, err := syncer.VerifyCheckpoint(context.Background(), receipt); !errors.Is(err, ErrCheckpointConcurrentMutation) || !strings.Contains(err.Error(), "after server re-attestation") { + t.Fatalf("closing scan error=%v", err) + } + if client.checkpointVerifyCalls != 1 { + t.Fatalf("verification RPC calls=%d", client.checkpointVerifyCalls) + } + }) + + t.Run("issued bearer is rejected", func(t *testing.T) { + client := &fakeClient{} + syncer, _ := newManagedCheckpointSyncer(t, client) + receipt := consumedCheckpointReceiptForTest(t, syncer) + receipt.SealToken = "must-not-cross-destination-verifier" + if _, err := syncer.VerifyCheckpoint(context.Background(), receipt); !errors.Is(err, ErrCheckpointNonConverged) { + t.Fatalf("bearer receipt error = %v", err) + } + if client.checkpointVerifyCalls != 0 { + t.Fatalf("bearer receipt reached server: %d", client.checkpointVerifyCalls) + } + }) + + t.Run("non-native revision and cursor are rejected before server", func(t *testing.T) { + for name, mutate := range map[string]func(*CheckpointSeal){ + "bare revision": func(receipt *CheckpointSeal) { receipt.WorkspaceRevision = "12" }, + "bare cursor": func(receipt *CheckpointSeal) { receipt.EventCursor = "12" }, + } { + t.Run(name, func(t *testing.T) { + client := &fakeClient{} + syncer, _ := newManagedCheckpointSyncer(t, client) + receipt := consumedCheckpointReceiptForTest(t, syncer) + mutate(&receipt) + if _, err := syncer.VerifyCheckpoint(context.Background(), receipt); !errors.Is(err, ErrCheckpointNonConverged) { + t.Fatalf("non-native receipt error = %v", err) + } + if client.checkpointVerifyCalls != 0 { + t.Fatalf("invalid receipt reached server: %d", client.checkpointVerifyCalls) + } + }) + } + }) +} + +func consumedCheckpointReceiptForTest(t *testing.T, syncer *Syncer) CheckpointSeal { + t.Helper() + digest, err := syncer.checkpointLocalDigestLocked() + if err != nil { + t.Fatalf("checkpoint digest: %v", err) + } + now := time.Now().UTC() + syncer.state.EventsCursor = "evt_2" + syncer.state.LastSuccessfulReconcileAt = now.Format(time.RFC3339Nano) + if err := syncer.saveStateWithoutLocalScan(); err != nil { + t.Fatalf("persist verification state: %v", err) + } + return CheckpointSeal{ + SealID: "cps_consumed", WorkspaceID: syncer.workspace, Root: "/", SessionID: "thread-verify", Generation: 1, + Digest: digest, WorkspaceRevision: "rev_1", EventCursor: "evt_2", + IssuedAt: now.Add(-time.Minute).Format(time.RFC3339Nano), ExpiresAt: now.Add(time.Minute).Format(time.RFC3339Nano), ConsumedAt: now.Format(time.RFC3339Nano), + } +} + +func newManagedCheckpointSyncer(t *testing.T, client *fakeClient) (*Syncer, string) { + t.Helper() + localRoot := t.TempDir() + localPath := filepath.Join(localRoot, "transcript.jsonl") + content := []byte("turn one\n") + if err := os.WriteFile(localPath, content, 0o644); err != nil { + t.Fatalf("write checkpoint fixture: %v", err) + } + remotePath := "/transcript.jsonl" + client.files = map[string]RemoteFile{remotePath: { + Path: remotePath, Revision: "rev_1", ContentType: "application/octet-stream", Content: string(content), ContentHash: hashBytes(content), + }} + syncer, err := NewSyncer(client, SyncerOptions{ + WorkspaceID: "ws_checkpoint", RemoteRoot: "/", LocalRoot: localRoot, + StateDir: t.TempDir(), RootCtx: context.Background(), FullPullEvery: -1, + }) + if err != nil { + t.Fatalf("new checkpoint syncer: %v", err) + } + syncer.state.Files[remotePath] = trackedFile{Revision: "rev_1", ContentType: detectContentType(localPath), Hash: hashBytes(content)} + syncer.state.BootstrapComplete = true + if err := syncer.saveState(); err != nil { + t.Fatalf("persist managed checkpoint state: %v", err) + } + return syncer, localPath +} + type fakeClient struct { mu sync.Mutex files map[string]RemoteFile @@ -8368,6 +9070,80 @@ type fakeClient struct { readFileErrByPath map[string]error mergeFileCalls int mergeFileFunc func(ctx context.Context, workspaceID, path, strategy, baseRevision, baseContent, content, contentType string) (MergeResult, error) + checkpointIssueCalls int + checkpointIssueRequests []CheckpointSealRequest + checkpointIssueFunc func(context.Context, string, CheckpointSealRequest) (CheckpointSeal, error) + checkpointVerifyCalls int + checkpointVerifyFunc func(context.Context, string, CheckpointSealVerifyRequest) (CheckpointSeal, error) + checkpointVerifySeal CheckpointSeal + checkpointVerifyErr error + checkpointHandbackCalls int + checkpointHandbackFunc func(context.Context, string, CheckpointSealHandbackRequest) (CheckpointSealOwnership, error) + checkpointHandbackRequest CheckpointSealHandbackRequest + checkpointHandbackProof CheckpointSealOwnership + checkpointHandbackErr error + checkpointHandbackPreparedAt string +} + +func (c *fakeClient) IssueCheckpointSeal(ctx context.Context, workspaceID string, request CheckpointSealRequest) (CheckpointSeal, error) { + c.checkpointIssueCalls++ + c.checkpointIssueRequests = append(c.checkpointIssueRequests, request) + if c.checkpointIssueFunc != nil { + return c.checkpointIssueFunc(ctx, workspaceID, request) + } + return CheckpointSeal{ + SealID: "cps_fake", SealToken: "fake-one-use-token", WorkspaceID: workspaceID, + Root: request.Root, SessionID: request.SessionID, Generation: request.Generation, + Digest: request.ExpectedDigest, WorkspaceRevision: "rev_2", EventCursor: "evt_2", + IssuedAt: time.Now().UTC().Format(time.RFC3339Nano), ExpiresAt: time.Now().UTC().Add(time.Minute).Format(time.RFC3339Nano), + }, nil +} + +func (c *fakeClient) VerifyCheckpointSeal(ctx context.Context, workspaceID string, request CheckpointSealVerifyRequest) (CheckpointSeal, error) { + c.checkpointVerifyCalls++ + if c.checkpointVerifyFunc != nil { + return c.checkpointVerifyFunc(ctx, workspaceID, request) + } + if c.checkpointVerifyErr != nil { + return CheckpointSeal{}, c.checkpointVerifyErr + } + if c.checkpointVerifySeal.SealID != "" { + return c.checkpointVerifySeal, nil + } + return CheckpointSeal{ + SealID: request.SealID, WorkspaceID: "ws_checkpoint", Root: request.Root, SessionID: request.SessionID, Generation: request.Generation, + Digest: request.Digest, WorkspaceRevision: request.WorkspaceRevision, EventCursor: request.EventCursor, + IssuedAt: request.IssuedAt, ExpiresAt: request.ExpiresAt, ConsumedAt: request.ConsumedAt, + }, nil +} + +func (c *fakeClient) HandbackCheckpointSeal(ctx context.Context, workspaceID string, request CheckpointSealHandbackRequest) (CheckpointSealOwnership, error) { + c.checkpointHandbackCalls++ + c.checkpointHandbackRequest = request + if c.checkpointHandbackFunc != nil { + return c.checkpointHandbackFunc(ctx, workspaceID, request) + } + if c.checkpointHandbackErr != nil { + return CheckpointSealOwnership{}, c.checkpointHandbackErr + } + if c.checkpointHandbackProof.SealID != "" { + return c.checkpointHandbackProof, nil + } + if c.checkpointHandbackPreparedAt == "" { + c.checkpointHandbackPreparedAt = time.Now().UTC().Format(time.RFC3339Nano) + } + status := "prepared" + releasedAt := "" + if request.Phase == CheckpointHandbackPhaseCommit { + status = "released" + releasedAt = time.Now().UTC().Format(time.RFC3339Nano) + } + return CheckpointSealOwnership{ + SealID: request.SealID, WorkspaceID: workspaceID, Root: request.Root, + SessionID: request.SessionID, Generation: request.Generation, Status: status, + Digest: request.ExpectedDigest, WorkspaceRevision: "rev_3", EventCursor: "evt_3", + ConsumedAt: request.ConsumedAt, PreparedAt: c.checkpointHandbackPreparedAt, ReleasedAt: releasedAt, + }, nil } // requestedReadCalls returns the cumulative number of ReadFile calls made diff --git a/internal/mountsync/types.go b/internal/mountsync/types.go index 23047a15..e254526a 100644 --- a/internal/mountsync/types.go +++ b/internal/mountsync/types.go @@ -17,6 +17,30 @@ type BulkWriteResult = relayfile.BulkWriteResult type OperationStatus = relayfile.OperationStatus +type CheckpointSeal = relayfile.CheckpointSeal + +type CheckpointSealRequest = relayfile.CheckpointSealRequest + +type CheckpointSealConsumeRequest = relayfile.CheckpointSealConsumeRequest + +type CheckpointSealConsumeRecoveryRequest = relayfile.CheckpointSealConsumeRecoveryRequest + +type CheckpointSealVerifyRequest = relayfile.CheckpointSealVerifyRequest + +type CheckpointSealHandbackRequest = relayfile.CheckpointSealHandbackRequest + +type CheckpointSealResumeRequest = relayfile.CheckpointSealResumeRequest + +type CheckpointSealOwnership = relayfile.CheckpointSealOwnership + +const DefaultCheckpointSealTTL = relayfile.DefaultCheckpointSealTTL + +const MaxCheckpointSealTTL = relayfile.MaxCheckpointSealTTL + +const CheckpointHandbackPhasePrepare = relayfile.CheckpointHandbackPhasePrepare + +const CheckpointHandbackPhaseCommit = relayfile.CheckpointHandbackPhaseCommit + type BulkWriteResponse struct { Written int `json:"written"` ErrorCount int `json:"errorCount"` diff --git a/internal/mountsync/watcher.go b/internal/mountsync/watcher.go index 21fb3595..6e259cd3 100644 --- a/internal/mountsync/watcher.go +++ b/internal/mountsync/watcher.go @@ -50,6 +50,24 @@ type LocalChange struct { Op fsnotify.Op } +type localChangeTimer interface { + Stop() bool + Reset(time.Duration) bool +} + +type localChangeClock interface { + Now() time.Time + AfterFunc(time.Duration, func()) localChangeTimer +} + +type realtimeLocalChangeClock struct{} + +func (realtimeLocalChangeClock) Now() time.Time { return time.Now() } + +func (realtimeLocalChangeClock) AfterFunc(delay time.Duration, callback func()) localChangeTimer { + return time.AfterFunc(delay, callback) +} + // LocalChangeBatcher collapses the near-simultaneous per-path callbacks emitted // by FileWatcher into one ordered batch. The per-file debounce has already let // editor rename/write/chmod sequences settle; this short cross-file window is @@ -59,21 +77,31 @@ type LocalChangeBatcher struct { window time.Duration maxWait time.Duration onBatch func([]LocalChange) + clock localChangeClock pending map[string]LocalChange - timer *time.Timer + timer localChangeTimer started time.Time + lastAdd time.Time closed bool wg sync.WaitGroup } func NewLocalChangeBatcher(window time.Duration, onBatch func([]LocalChange)) *LocalChangeBatcher { + return newLocalChangeBatcherWithClock(window, onBatch, realtimeLocalChangeClock{}) +} + +func newLocalChangeBatcherWithClock(window time.Duration, onBatch func([]LocalChange), clock localChangeClock) *LocalChangeBatcher { if window <= 0 { window = defaultLocalChangeBatchWindow } + if clock == nil { + clock = realtimeLocalChangeClock{} + } return &LocalChangeBatcher{ window: window, maxWait: 10 * window, onBatch: onBatch, + clock: clock, pending: make(map[string]LocalChange), } } @@ -95,35 +123,51 @@ func (b *LocalChangeBatcher) Add(relativePath string, op fsnotify.Op) { change.RelativePath = relativePath change.Op |= op b.pending[relativePath] = change + now := b.clock.Now() + b.lastAdd = now if b.timer != nil { // Flush after one quiet window so callbacks spread across a multi-file // save stay in one bulk request. Cap the total wait so sustained churn // cannot starve writeback indefinitely. - if b.timer.Stop() { - delay := b.window - if remaining := b.maxWait - time.Since(b.started); remaining < delay { - delay = remaining - } - if delay <= 0 { - delay = time.Nanosecond - } - b.timer.Reset(delay) + delay := b.nextFlushDelayLocked(now) + // Reset must be unconditional. For an AfterFunc timer, false means the + // callback has expired or is already running; Reset schedules the next + // callback in exactly that case. Gating Reset on Stop's result loses the + // quiet-window extension when Add races the callback. + if delay <= 0 { + delay = time.Nanosecond } + b.timer.Reset(delay) return } b.wg.Add(1) - b.started = time.Now() - b.timer = time.AfterFunc(b.window, b.flush) + b.started = now + b.timer = b.clock.AfterFunc(b.window, b.flush) } func (b *LocalChangeBatcher) flush() { - defer b.wg.Done() b.mu.Lock() + // Reset may race an already-running AfterFunc callback. Recheck the + // authoritative last-add deadline under the mutex before draining; a stale + // callback must yield to the newly extended quiet window. + if b.timer == nil { + b.mu.Unlock() + return + } + if !b.closed && len(b.pending) > 0 { + if delay := b.nextFlushDelayLocked(b.clock.Now()); delay > 0 { + b.timer.Reset(delay) + b.mu.Unlock() + return + } + } b.timer = nil b.started = time.Time{} + b.lastAdd = time.Time{} if b.closed || len(b.pending) == 0 { b.pending = make(map[string]LocalChange) b.mu.Unlock() + b.wg.Done() return } paths := make([]string, 0, len(b.pending)) @@ -138,11 +182,20 @@ func (b *LocalChangeBatcher) flush() { b.pending = make(map[string]LocalChange) onBatch := b.onBatch b.mu.Unlock() + defer b.wg.Done() if onBatch != nil { onBatch(changes) } } +func (b *LocalChangeBatcher) nextFlushDelayLocked(now time.Time) time.Duration { + delay := b.window - now.Sub(b.lastAdd) + if remaining := b.maxWait - now.Sub(b.started); remaining < delay { + delay = remaining + } + return delay +} + func (b *LocalChangeBatcher) Close() { if b == nil { return @@ -155,9 +208,9 @@ func (b *LocalChangeBatcher) Close() { } b.closed = true if b.timer != nil && b.timer.Stop() { + b.timer = nil b.wg.Done() } - b.timer = nil b.pending = nil b.mu.Unlock() b.wg.Wait() diff --git a/internal/relayfile/checkpoint_seal.go b/internal/relayfile/checkpoint_seal.go new file mode 100644 index 00000000..95058969 --- /dev/null +++ b/internal/relayfile/checkpoint_seal.go @@ -0,0 +1,979 @@ +package relayfile + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + pathpkg "path" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +const ( + DefaultCheckpointSealTTL = 60 * time.Second + MaxCheckpointSealTTL = 5 * time.Minute + CheckpointConsumeReplayRetention = 24 * time.Hour +) + +var ( + ErrCheckpointDiverged = errors.New("checkpoint digest does not match durable workspace state") + ErrCheckpointExpired = errors.New("checkpoint seal expired") + ErrCheckpointReplay = errors.New("checkpoint seal already consumed") + ErrCheckpointStale = errors.New("checkpoint seal is stale") + ErrCheckpointGenerationStale = errors.New("checkpoint generation is not newer than the last issued generation") + ErrCheckpointIssuanceConflict = errors.New("checkpoint issuance idempotency key is bound to a different request or issuer") + ErrCheckpointConsumerConflict = errors.New("checkpoint consumer idempotency key is bound to a different seal or identity") + ErrCheckpointUnconsumed = errors.New("checkpoint seal has not been consumed") + ErrCheckpointHandbackRequired = errors.New("checkpoint ownership has not been released by the destination") + ErrCheckpointHandbackUnprepared = errors.New("checkpoint handback has not been prepared") + ErrCheckpointHandbackConflict = errors.New("checkpoint handback idempotency key is bound to a different release") + ErrCheckpointResumeConflict = errors.New("checkpoint source resume idempotency key is bound to a different claim") + ErrCheckpointAdminConflict = errors.New("checkpoint administrative reconciliation identity conflicts with durable state") + checkpointSessionPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$`) + checkpointRevisionPattern = regexp.MustCompile(`^(?:0|rev_[0-9]+)$`) + checkpointEventCursorPattern = regexp.MustCompile(`^(?:0|evt_[0-9]+)$`) +) + +const ( + CheckpointHandbackPhasePrepare = "prepare" + CheckpointHandbackPhaseCommit = "commit" +) + +type CheckpointDigestEntry struct { + Path string `json:"path"` + Revision string `json:"revision"` + ContentHash string `json:"contentHash"` +} + +type CheckpointSealRequest struct { + Root string `json:"root"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + ExpectedDigest string `json:"expectedDigest"` + TTLSeconds int `json:"ttlSeconds,omitempty"` + IssuanceIdempotencyKey string `json:"issuanceIdempotencyKey"` + Issuer string `json:"-"` +} + +type CheckpointSealConsumeRequest struct { + SealToken string `json:"sealToken"` + Root string `json:"root"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + ConsumerIdempotencyKey string `json:"consumerIdempotencyKey"` + ConsumerPrincipal string `json:"-"` +} + +// CheckpointSealConsumeRecoveryRequest recovers the safe, tokenless consume +// receipt when the consume response was lost. Cloud persists this stable +// identity before the first consume but intentionally never persists the +// bearer token. +type CheckpointSealConsumeRecoveryRequest struct { + Root string `json:"root"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + ConsumerIdempotencyKey string `json:"consumerIdempotencyKey"` + ConsumerPrincipal string `json:"-"` +} + +// CheckpointSealVerifyRequest deliberately excludes SealToken. Destination +// convergence is proven only after Cloud has consumed the one-use bearer, and +// neither the request nor the response may reintroduce it. +type CheckpointSealVerifyRequest struct { + SealID string `json:"sealId"` + Root string `json:"root"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + Digest string `json:"digest"` + WorkspaceRevision string `json:"workspaceRevision"` + EventCursor string `json:"eventCursor"` + IssuedAt string `json:"issuedAt"` + ExpiresAt string `json:"expiresAt"` + ConsumedAt string `json:"consumedAt"` + ConsumerPrincipal string `json:"-"` +} + +// CheckpointSealHandbackRequest is one phase of the destination's final +// ownership-release assertion. Prepare durably binds the stopped destination's +// drained digest without releasing ownership. Commit releases only that exact +// prepared state after the destination performs its closing local scan. The +// original consumer key proves that the same cutover attempt which acquired the +// seal is releasing it. It deliberately contains no seal token. +type CheckpointSealHandbackRequest struct { + Phase string `json:"phase"` + SealID string `json:"sealId"` + Root string `json:"root"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + ConsumedAt string `json:"consumedAt"` + ConsumerIdempotencyKey string `json:"consumerIdempotencyKey"` + HandbackIdempotencyKey string `json:"handbackIdempotencyKey"` + ExpectedDigest string `json:"expectedDigest"` + ConsumerPrincipal string `json:"-"` +} + +// CheckpointSealResumeRequest is the stopped source's ownership claim. The +// original one-use token remains the source proof; it can only be used here +// after the destination has released ownership. +type CheckpointSealResumeRequest struct { + SealToken string `json:"sealToken"` + Root string `json:"root"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + ResumeIdempotencyKey string `json:"resumeIdempotencyKey"` +} + +// CheckpointSealOwnership is a tokenless authoritative handback/resume proof. +type CheckpointSealOwnership struct { + SealID string `json:"sealId"` + WorkspaceID string `json:"workspaceId"` + Root string `json:"root"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + Status string `json:"status"` + Digest string `json:"digest"` + WorkspaceRevision string `json:"workspaceRevision"` + EventCursor string `json:"eventCursor"` + ConsumedAt string `json:"consumedAt,omitempty"` + PreparedAt string `json:"preparedAt,omitempty"` + ReleasedAt string `json:"releasedAt,omitempty"` + SourceResumedAt string `json:"sourceResumedAt,omitempty"` +} + +type CheckpointSeal struct { + SealID string `json:"sealId"` + SealToken string `json:"sealToken,omitempty"` + WorkspaceID string `json:"workspaceId"` + Root string `json:"root"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + Digest string `json:"digest"` + WorkspaceRevision string `json:"workspaceRevision"` + EventCursor string `json:"eventCursor"` + IssuedAt string `json:"issuedAt"` + ExpiresAt string `json:"expiresAt"` + ConsumedAt string `json:"consumedAt,omitempty"` +} + +type checkpointSealRecord struct { + CheckpointSeal + IssuanceKeyHash string `json:"issuanceKeyHash,omitempty"` + IssuanceRequestHash string `json:"issuanceRequestHash,omitempty"` + TokenHash string `json:"tokenHash"` + Issuer string `json:"issuer,omitempty"` + ConsumerKeyHash string `json:"consumerKeyHash,omitempty"` + ConsumerPrincipal string `json:"consumerPrincipal,omitempty"` + HandbackKeyHash string `json:"handbackKeyHash,omitempty"` + HandbackDigest string `json:"handbackDigest,omitempty"` + HandbackRevision string `json:"handbackRevision,omitempty"` + HandbackEventCursor string `json:"handbackEventCursor,omitempty"` + HandbackPreparedAt string `json:"handbackPreparedAt,omitempty"` + HandbackReleasedAt string `json:"handbackReleasedAt,omitempty"` + SourceResumeKeyHash string `json:"sourceResumeKeyHash,omitempty"` + SourceResumedAt string `json:"sourceResumedAt,omitempty"` + AdminReconcileKeyHash string `json:"adminReconcileKeyHash,omitempty"` + AdminReconciledAt string `json:"adminReconciledAt,omitempty"` + IdempotencyExpiresAt string `json:"idempotencyExpiresAt,omitempty"` +} + +type checkpointConsumerBinding struct { + TokenHash string `json:"tokenHash"` + WorkspaceID string `json:"workspaceId"` + Root string `json:"root"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + Principal string `json:"principal"` + ReplayUntil string `json:"replayUntil"` +} + +type CheckpointSealRetentionRecord struct { + SealID string `json:"sealId"` + WorkspaceID string `json:"workspaceId"` + Root string `json:"root"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + OwnershipStatus string `json:"ownershipStatus"` + IssuedAt string `json:"issuedAt"` + ExpiresAt string `json:"expiresAt"` + ConsumedAt string `json:"consumedAt,omitempty"` + HandbackReleasedAt string `json:"handbackReleasedAt,omitempty"` + SourceResumedAt string `json:"sourceResumedAt,omitempty"` + AdminReconciledAt string `json:"adminReconciledAt,omitempty"` +} + +type CheckpointSealRetentionSummary struct { + GeneratedAt string `json:"generatedAt"` + UnresumedTotal int `json:"unresumedTotal"` + UnresumedByWorkspace map[string]int `json:"unresumedByWorkspace"` + Records []CheckpointSealRetentionRecord `json:"records"` +} + +type CheckpointSealAdminReconcileRequest struct { + WorkspaceID string `json:"workspaceId"` + Root string `json:"root"` + SessionID string `json:"sessionId"` + Generation uint64 `json:"generation"` + ExpectedOwnershipStatus string `json:"expectedOwnershipStatus"` + ReconciliationIdempotencyKey string `json:"reconciliationIdempotencyKey"` + ConfirmSourceReady bool `json:"confirmSourceReady"` +} + +func NormalizeCheckpointRoot(raw string) (string, error) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" || strings.IndexByte(trimmed, 0) >= 0 || !strings.HasPrefix(trimmed, "/") { + return "", ErrInvalidInput + } + cleaned := pathpkg.Clean(trimmed) + if cleaned != trimmed && !(trimmed != "/" && strings.TrimSuffix(trimmed, "/") == cleaned) { + return "", ErrInvalidInput + } + if cleaned == "." || !strings.HasPrefix(cleaned, "/") { + return "", ErrInvalidInput + } + return cleaned, nil +} + +func ComputeCheckpointDigest(root string, entries []CheckpointDigestEntry) (string, error) { + normalizedRoot, err := NormalizeCheckpointRoot(root) + if err != nil { + return "", err + } + canonical := append([]CheckpointDigestEntry(nil), entries...) + sort.Slice(canonical, func(i, j int) bool { return canonical[i].Path < canonical[j].Path }) + h := sha256.New() + writeDigestField(h, normalizedRoot) + lastPath := "" + for _, entry := range canonical { + entryPath, err := NormalizeCheckpointRoot(strings.TrimSpace(entry.Path)) + if err != nil || !withinBase(normalizedRoot, entryPath) || entryPath == normalizedRoot { + return "", ErrInvalidInput + } + revision := strings.TrimSpace(entry.Revision) + contentHash := strings.TrimSpace(entry.ContentHash) + if revision == "" || contentHash == "" || entryPath == lastPath { + return "", ErrInvalidInput + } + lastPath = entryPath + writeDigestField(h, entryPath) + writeDigestField(h, revision) + writeDigestField(h, contentHash) + } + return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil +} + +type digestWriter interface { + Write([]byte) (int, error) +} + +func writeDigestField(w digestWriter, value string) { + _, _ = w.Write([]byte(strconv.Itoa(len(value)))) + _, _ = w.Write([]byte{':'}) + _, _ = w.Write([]byte(value)) + _, _ = w.Write([]byte{0}) +} + +func (s *Store) IssueCheckpointSeal(workspaceID string, req CheckpointSealRequest, now time.Time) (CheckpointSeal, error) { + workspaceID = strings.TrimSpace(workspaceID) + sessionID := strings.TrimSpace(req.SessionID) + issuanceKey := strings.TrimSpace(req.IssuanceIdempotencyKey) + issuer := strings.TrimSpace(req.Issuer) + root, err := NormalizeCheckpointRoot(req.Root) + if err != nil || workspaceID == "" || !checkpointSessionPattern.MatchString(sessionID) || req.Generation == 0 { + return CheckpointSeal{}, ErrInvalidInput + } + expectedDigest := strings.TrimSpace(req.ExpectedDigest) + if !strings.HasPrefix(expectedDigest, "sha256:") || len(expectedDigest) != len("sha256:")+sha256.Size*2 { + return CheckpointSeal{}, ErrInvalidInput + } + if _, err := hex.DecodeString(strings.TrimPrefix(expectedDigest, "sha256:")); err != nil { + return CheckpointSeal{}, ErrInvalidInput + } + if req.TTLSeconds < 0 || req.TTLSeconds > int(MaxCheckpointSealTTL/time.Second) { + return CheckpointSeal{}, ErrInvalidInput + } + ttl := DefaultCheckpointSealTTL + if req.TTLSeconds > 0 { + ttl = time.Duration(req.TTLSeconds) * time.Second + } + issuanceRequestHash := checkpointIssuanceRequestHash(workspaceID, root, sessionID, req.Generation, expectedDigest, int(ttl/time.Second)) + issuanceKeyHash := "" + if issuanceKey != "" && !checkpointSessionPattern.MatchString(issuanceKey) { + return CheckpointSeal{}, ErrInvalidInput + } + if issuanceKey != "" { + issuanceKeyHash = checkpointTokenHash(issuanceKey) + } + persistedIssuanceRequestHash := issuanceRequestHash + if issuanceKeyHash == "" { + persistedIssuanceRequestHash = "" + } + now = now.UTC() + + s.mu.Lock() + defer s.mu.Unlock() + s.purgeCheckpointSealsLocked(now) + if oldTokenHash, replay, ok := s.checkpointSealByIssuanceKeyLocked(issuanceKeyHash); issuanceKeyHash != "" && ok { + if replay.IssuanceRequestHash != issuanceRequestHash || replay.Issuer != issuer { + return CheckpointSeal{}, ErrCheckpointIssuanceConflict + } + if replay.ConsumedAt != "" || replay.HandbackReleasedAt != "" || replay.SourceResumedAt != "" { + return CheckpointSeal{}, ErrCheckpointReplay + } + expiresAt, parseErr := time.Parse(time.RFC3339Nano, replay.ExpiresAt) + if parseErr != nil || !now.Before(expiresAt) { + return CheckpointSeal{}, ErrCheckpointExpired + } + digest, workspaceRevision, cursor, stateErr := s.checkpointStateLocked(workspaceID, root) + if stateErr != nil { + return CheckpointSeal{}, stateErr + } + if digest != replay.Digest || workspaceRevision != replay.WorkspaceRevision || cursor != replay.EventCursor { + return CheckpointSeal{}, ErrCheckpointStale + } + // A response-loss retry cannot recover a plaintext bearer from its + // stored hash. Rotate it atomically while preserving the seal identity + // and immutable attestation; the lost token becomes unusable. + token, tokenErr := newCheckpointToken() + if tokenErr != nil { + return CheckpointSeal{}, tokenErr + } + rotated := replay + rotated.TokenHash = checkpointTokenHash(token) + delete(s.checkpointSeals, oldTokenHash) + s.checkpointSeals[rotated.TokenHash] = rotated + if saveErr := s.saveLocked(); saveErr != nil { + delete(s.checkpointSeals, rotated.TokenHash) + s.checkpointSeals[oldTokenHash] = replay + return CheckpointSeal{}, saveErr + } + response := rotated.CheckpointSeal + response.SealToken = token + return response, nil + } + key := checkpointGenerationKey(workspaceID, root, sessionID) + previousGeneration := s.checkpointGenerations[key] + if req.Generation <= previousGeneration { + return CheckpointSeal{}, ErrCheckpointGenerationStale + } + digest, workspaceRevision, cursor, err := s.checkpointStateLocked(workspaceID, root) + if err != nil { + return CheckpointSeal{}, err + } + if digest != expectedDigest { + return CheckpointSeal{}, ErrCheckpointDiverged + } + token, err := newCheckpointToken() + if err != nil { + return CheckpointSeal{}, err + } + sealID := "cps_" + token[:16] + record := checkpointSealRecord{ + CheckpointSeal: CheckpointSeal{ + SealID: sealID, + WorkspaceID: workspaceID, + Root: root, + SessionID: sessionID, + Generation: req.Generation, + Digest: digest, + WorkspaceRevision: workspaceRevision, + EventCursor: cursor, + IssuedAt: now.Format(time.RFC3339Nano), + ExpiresAt: now.Add(ttl).Format(time.RFC3339Nano), + }, + IssuanceKeyHash: issuanceKeyHash, + IssuanceRequestHash: persistedIssuanceRequestHash, + TokenHash: checkpointTokenHash(token), + Issuer: issuer, + } + if s.checkpointSeals == nil { + s.checkpointSeals = map[string]checkpointSealRecord{} + } + if s.checkpointGenerations == nil { + s.checkpointGenerations = map[string]uint64{} + } + s.checkpointSeals[record.TokenHash] = record + s.checkpointGenerations[key] = req.Generation + if err := s.saveLocked(); err != nil { + delete(s.checkpointSeals, record.TokenHash) + if previousGeneration == 0 { + delete(s.checkpointGenerations, key) + } else { + s.checkpointGenerations[key] = previousGeneration + } + return CheckpointSeal{}, err + } + response := record.CheckpointSeal + response.SealToken = token + return response, nil +} + +func (s *Store) ConsumeCheckpointSeal(workspaceID string, req CheckpointSealConsumeRequest, now time.Time) (CheckpointSeal, error) { + workspaceID = strings.TrimSpace(workspaceID) + sessionID := strings.TrimSpace(req.SessionID) + consumerKey := strings.TrimSpace(req.ConsumerIdempotencyKey) + consumerPrincipal := strings.TrimSpace(req.ConsumerPrincipal) + root, err := NormalizeCheckpointRoot(req.Root) + if err != nil || workspaceID == "" || !checkpointSessionPattern.MatchString(sessionID) || req.Generation == 0 || strings.TrimSpace(req.SealToken) == "" || !checkpointSessionPattern.MatchString(consumerKey) || consumerPrincipal == "" { + return CheckpointSeal{}, ErrInvalidInput + } + now = now.UTC() + tokenHash := checkpointTokenHash(strings.TrimSpace(req.SealToken)) + consumerKeyHash := checkpointTokenHash(consumerKey) + s.mu.Lock() + defer s.mu.Unlock() + s.purgeCheckpointSealsLocked(now) + if binding, exists := s.checkpointConsumerKeys[consumerKeyHash]; exists && !checkpointConsumerBindingMatches(binding, tokenHash, workspaceID, root, sessionID, req.Generation, consumerPrincipal) { + return CheckpointSeal{}, ErrCheckpointConsumerConflict + } + record, ok := s.checkpointSeals[tokenHash] + if !ok { + return CheckpointSeal{}, ErrNotFound + } + if record.WorkspaceID != workspaceID || record.Root != root || record.SessionID != sessionID || record.Generation != req.Generation { + return CheckpointSeal{}, ErrInvalidInput + } + if record.HandbackReleasedAt != "" || record.SourceResumedAt != "" { + return CheckpointSeal{}, ErrCheckpointReplay + } + if record.ConsumedAt != "" { + if record.ConsumerKeyHash == consumerKeyHash && record.ConsumerPrincipal == consumerPrincipal { + return record.CheckpointSeal, nil + } + return CheckpointSeal{}, ErrCheckpointReplay + } + expiresAt, parseErr := time.Parse(time.RFC3339Nano, record.ExpiresAt) + if parseErr != nil || !now.Before(expiresAt) { + return CheckpointSeal{}, ErrCheckpointExpired + } + digest, workspaceRevision, cursor, err := s.checkpointStateLocked(workspaceID, root) + if err != nil { + return CheckpointSeal{}, err + } + if digest != record.Digest || workspaceRevision != record.WorkspaceRevision || cursor != record.EventCursor { + return CheckpointSeal{}, ErrCheckpointStale + } + record.ConsumedAt = now.Format(time.RFC3339Nano) + record.ConsumerKeyHash = consumerKeyHash + record.ConsumerPrincipal = consumerPrincipal + record.IdempotencyExpiresAt = now.Add(CheckpointConsumeReplayRetention).Format(time.RFC3339Nano) + s.checkpointSeals[tokenHash] = record + if s.checkpointConsumerKeys == nil { + s.checkpointConsumerKeys = map[string]checkpointConsumerBinding{} + } + s.checkpointConsumerKeys[consumerKeyHash] = checkpointConsumerBinding{ + TokenHash: tokenHash, WorkspaceID: workspaceID, Root: root, + SessionID: sessionID, Generation: req.Generation, + Principal: consumerPrincipal, + ReplayUntil: record.IdempotencyExpiresAt, + } + if err := s.saveLocked(); err != nil { + record.ConsumedAt = "" + record.ConsumerKeyHash = "" + record.ConsumerPrincipal = "" + record.IdempotencyExpiresAt = "" + s.checkpointSeals[tokenHash] = record + delete(s.checkpointConsumerKeys, consumerKeyHash) + return CheckpointSeal{}, err + } + return record.CheckpointSeal, nil +} + +// RecoverConsumedCheckpointSeal returns only the durable tokenless consume +// receipt bound to a controller-persisted consumer identity. NotFound is an +// authoritative statement that no matching consume committed. +func (s *Store) RecoverConsumedCheckpointSeal(workspaceID string, req CheckpointSealConsumeRecoveryRequest, now time.Time) (CheckpointSeal, error) { + workspaceID = strings.TrimSpace(workspaceID) + sessionID := strings.TrimSpace(req.SessionID) + consumerKey := strings.TrimSpace(req.ConsumerIdempotencyKey) + consumerPrincipal := strings.TrimSpace(req.ConsumerPrincipal) + root, err := NormalizeCheckpointRoot(req.Root) + if err != nil || workspaceID == "" || !checkpointSessionPattern.MatchString(sessionID) || req.Generation == 0 || !checkpointSessionPattern.MatchString(consumerKey) || consumerPrincipal == "" { + return CheckpointSeal{}, ErrInvalidInput + } + consumerKeyHash := checkpointTokenHash(consumerKey) + s.mu.Lock() + defer s.mu.Unlock() + s.purgeCheckpointSealsLocked(now.UTC()) + binding, ok := s.checkpointConsumerKeys[consumerKeyHash] + if !ok { + return CheckpointSeal{}, ErrNotFound + } + if binding.WorkspaceID != workspaceID || binding.Root != root || binding.SessionID != sessionID || binding.Generation != req.Generation || binding.Principal != consumerPrincipal { + return CheckpointSeal{}, ErrCheckpointConsumerConflict + } + record, ok := s.checkpointSeals[binding.TokenHash] + if !ok || record.ConsumerKeyHash != consumerKeyHash || record.ConsumerPrincipal != consumerPrincipal || record.ConsumedAt == "" { + return CheckpointSeal{}, ErrNotFound + } + response := record.CheckpointSeal + response.SealToken = "" + return response, nil +} + +// VerifyConsumedCheckpointSeal re-attests that an exact consumed seal still +// describes current durable workspace state. It is intentionally read-only: +// the destination's local verifier supplies the independent filesystem proof, +// while this method prevents a caller from inventing or mutating a receipt and +// handles workspace revisions (including delete-only revisions) authoritatively. +func (s *Store) VerifyConsumedCheckpointSeal(workspaceID string, req CheckpointSealVerifyRequest, now time.Time) (CheckpointSeal, error) { + workspaceID = strings.TrimSpace(workspaceID) + consumerPrincipal := strings.TrimSpace(req.ConsumerPrincipal) + root, err := NormalizeCheckpointRoot(req.Root) + if err != nil || workspaceID == "" || strings.TrimSpace(req.SealID) == "" || !checkpointSessionPattern.MatchString(strings.TrimSpace(req.SessionID)) || req.Generation == 0 || consumerPrincipal == "" { + return CheckpointSeal{}, ErrInvalidInput + } + if !validCheckpointDigest(req.Digest) || !checkpointRevisionPattern.MatchString(strings.TrimSpace(req.WorkspaceRevision)) || !checkpointEventCursorPattern.MatchString(strings.TrimSpace(req.EventCursor)) || strings.TrimSpace(req.ConsumedAt) == "" { + return CheckpointSeal{}, ErrInvalidInput + } + for _, raw := range []string{req.IssuedAt, req.ExpiresAt, req.ConsumedAt} { + if _, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(raw)); err != nil { + return CheckpointSeal{}, ErrInvalidInput + } + } + + s.mu.Lock() + defer s.mu.Unlock() + s.purgeCheckpointSealsLocked(now.UTC()) + var record checkpointSealRecord + found := false + for _, candidate := range s.checkpointSeals { + if candidate.SealID == strings.TrimSpace(req.SealID) { + record = candidate + found = true + break + } + } + if !found { + return CheckpointSeal{}, ErrNotFound + } + if strings.TrimSpace(record.ConsumedAt) == "" { + return CheckpointSeal{}, ErrCheckpointUnconsumed + } + if strings.TrimSpace(record.HandbackReleasedAt) != "" || strings.TrimSpace(record.SourceResumedAt) != "" { + return CheckpointSeal{}, ErrCheckpointReplay + } + if record.ConsumerPrincipal != consumerPrincipal { + return CheckpointSeal{}, ErrCheckpointConsumerConflict + } + if record.WorkspaceID != workspaceID || record.Root != root || record.SessionID != strings.TrimSpace(req.SessionID) || record.Generation != req.Generation || + record.Digest != strings.TrimSpace(req.Digest) || record.WorkspaceRevision != strings.TrimSpace(req.WorkspaceRevision) || record.EventCursor != strings.TrimSpace(req.EventCursor) || + record.IssuedAt != strings.TrimSpace(req.IssuedAt) || record.ExpiresAt != strings.TrimSpace(req.ExpiresAt) || record.ConsumedAt != strings.TrimSpace(req.ConsumedAt) { + return CheckpointSeal{}, ErrCheckpointStale + } + digest, workspaceRevision, cursor, err := s.checkpointStateLocked(workspaceID, root) + if err != nil { + return CheckpointSeal{}, err + } + if digest != record.Digest || workspaceRevision != record.WorkspaceRevision || cursor != record.EventCursor { + return CheckpointSeal{}, ErrCheckpointStale + } + response := record.CheckpointSeal + response.SealToken = "" + return response, nil +} + +// HandbackCheckpointSeal prepares or commits a consumed destination handback. +// Prepare persists the exact durable state while ownership remains with the +// destination. Commit releases ownership only if that prepared state is still +// current. Exact retries return the durable phase result; changed retry +// identities fail closed. +func (s *Store) HandbackCheckpointSeal(workspaceID string, req CheckpointSealHandbackRequest, now time.Time) (CheckpointSealOwnership, error) { + workspaceID = strings.TrimSpace(workspaceID) + phase := strings.TrimSpace(req.Phase) + sealID := strings.TrimSpace(req.SealID) + sessionID := strings.TrimSpace(req.SessionID) + consumerKey := strings.TrimSpace(req.ConsumerIdempotencyKey) + consumerPrincipal := strings.TrimSpace(req.ConsumerPrincipal) + handbackKey := strings.TrimSpace(req.HandbackIdempotencyKey) + consumedAt := strings.TrimSpace(req.ConsumedAt) + expectedDigest := strings.TrimSpace(req.ExpectedDigest) + root, err := NormalizeCheckpointRoot(req.Root) + if err != nil || (phase != CheckpointHandbackPhasePrepare && phase != CheckpointHandbackPhaseCommit) || workspaceID == "" || sealID == "" || !checkpointSessionPattern.MatchString(sessionID) || req.Generation == 0 || + !checkpointSessionPattern.MatchString(consumerKey) || !checkpointSessionPattern.MatchString(handbackKey) || consumerPrincipal == "" || !validCheckpointDigest(expectedDigest) || consumedAt == "" { + return CheckpointSealOwnership{}, ErrInvalidInput + } + if _, err := time.Parse(time.RFC3339Nano, consumedAt); err != nil { + return CheckpointSealOwnership{}, ErrInvalidInput + } + now = now.UTC() + consumerKeyHash := checkpointTokenHash(consumerKey) + handbackKeyHash := checkpointTokenHash(handbackKey) + + s.mu.Lock() + defer s.mu.Unlock() + s.purgeCheckpointSealsLocked(now) + tokenHash, record, ok := s.checkpointSealByIDLocked(sealID) + if !ok { + return CheckpointSealOwnership{}, ErrNotFound + } + if record.ConsumedAt == "" { + return CheckpointSealOwnership{}, ErrCheckpointUnconsumed + } + if record.WorkspaceID != workspaceID || record.Root != root || record.SessionID != sessionID || record.Generation != req.Generation || record.ConsumedAt != consumedAt { + return CheckpointSealOwnership{}, ErrCheckpointStale + } + if record.ConsumerKeyHash != consumerKeyHash || record.ConsumerPrincipal != consumerPrincipal { + return CheckpointSealOwnership{}, ErrCheckpointConsumerConflict + } + if record.HandbackReleasedAt != "" { + if record.HandbackKeyHash != handbackKeyHash || record.HandbackDigest != expectedDigest { + return CheckpointSealOwnership{}, ErrCheckpointHandbackConflict + } + return checkpointOwnershipFromRecord(record, "released"), nil + } + if record.SourceResumedAt != "" { + return CheckpointSealOwnership{}, ErrCheckpointResumeConflict + } + digest, revision, cursor, err := s.checkpointStateLocked(workspaceID, root) + if err != nil { + return CheckpointSealOwnership{}, err + } + + if record.HandbackPreparedAt != "" { + if record.HandbackKeyHash != handbackKeyHash || record.HandbackDigest != expectedDigest { + return CheckpointSealOwnership{}, ErrCheckpointHandbackConflict + } + if digest != record.HandbackDigest || revision != record.HandbackRevision || cursor != record.HandbackEventCursor { + return CheckpointSealOwnership{}, ErrCheckpointDiverged + } + if phase == CheckpointHandbackPhasePrepare { + return checkpointOwnershipFromRecord(record, "prepared"), nil + } + previous := record + record.HandbackReleasedAt = now.Format(time.RFC3339Nano) + record.IdempotencyExpiresAt = now.Add(CheckpointConsumeReplayRetention).Format(time.RFC3339Nano) + s.checkpointSeals[tokenHash] = record + if err := s.saveLocked(); err != nil { + s.checkpointSeals[tokenHash] = previous + return CheckpointSealOwnership{}, err + } + return checkpointOwnershipFromRecord(record, "released"), nil + } + + if phase == CheckpointHandbackPhaseCommit { + return CheckpointSealOwnership{}, ErrCheckpointHandbackUnprepared + } + if digest != expectedDigest { + return CheckpointSealOwnership{}, ErrCheckpointDiverged + } + previous := record + record.HandbackKeyHash = handbackKeyHash + record.HandbackDigest = digest + record.HandbackRevision = revision + record.HandbackEventCursor = cursor + record.HandbackPreparedAt = now.Format(time.RFC3339Nano) + record.IdempotencyExpiresAt = now.Add(CheckpointConsumeReplayRetention).Format(time.RFC3339Nano) + s.checkpointSeals[tokenHash] = record + if err := s.saveLocked(); err != nil { + s.checkpointSeals[tokenHash] = previous + return CheckpointSealOwnership{}, err + } + return checkpointOwnershipFromRecord(record, "prepared"), nil +} + +// ResumeCheckpointSeal returns ownership to the stopped source. An +// unconsumed seal may be cancelled directly; once a destination consumed it, +// the destination's durable handback is mandatory. The original token is the +// source proof and never appears in the response. +func (s *Store) ResumeCheckpointSeal(workspaceID string, req CheckpointSealResumeRequest, now time.Time) (CheckpointSealOwnership, error) { + workspaceID = strings.TrimSpace(workspaceID) + sessionID := strings.TrimSpace(req.SessionID) + resumeKey := strings.TrimSpace(req.ResumeIdempotencyKey) + root, err := NormalizeCheckpointRoot(req.Root) + if err != nil || workspaceID == "" || !checkpointSessionPattern.MatchString(sessionID) || req.Generation == 0 || + strings.TrimSpace(req.SealToken) == "" || !checkpointSessionPattern.MatchString(resumeKey) { + return CheckpointSealOwnership{}, ErrInvalidInput + } + now = now.UTC() + tokenHash := checkpointTokenHash(strings.TrimSpace(req.SealToken)) + resumeKeyHash := checkpointTokenHash(resumeKey) + s.mu.Lock() + defer s.mu.Unlock() + s.purgeCheckpointSealsLocked(now) + record, ok := s.checkpointSeals[tokenHash] + if !ok { + return CheckpointSealOwnership{}, ErrNotFound + } + if record.WorkspaceID != workspaceID || record.Root != root || record.SessionID != sessionID || record.Generation != req.Generation { + return CheckpointSealOwnership{}, ErrInvalidInput + } + if record.SourceResumedAt != "" { + if record.SourceResumeKeyHash != resumeKeyHash { + return CheckpointSealOwnership{}, ErrCheckpointResumeConflict + } + return checkpointOwnershipFromRecord(record, "source-resumed"), nil + } + if record.ConsumedAt != "" && record.HandbackReleasedAt == "" { + return CheckpointSealOwnership{}, ErrCheckpointHandbackRequired + } + previous := record + if record.HandbackReleasedAt == "" { + digest, revision, cursor, stateErr := s.checkpointStateLocked(workspaceID, root) + if stateErr != nil { + return CheckpointSealOwnership{}, stateErr + } + record.HandbackDigest = digest + record.HandbackRevision = revision + record.HandbackEventCursor = cursor + record.HandbackReleasedAt = now.Format(time.RFC3339Nano) + } + record.SourceResumeKeyHash = resumeKeyHash + record.SourceResumedAt = now.Format(time.RFC3339Nano) + record.IdempotencyExpiresAt = now.Add(CheckpointConsumeReplayRetention).Format(time.RFC3339Nano) + s.checkpointSeals[tokenHash] = record + if err := s.saveLocked(); err != nil { + s.checkpointSeals[tokenHash] = previous + return CheckpointSealOwnership{}, err + } + return checkpointOwnershipFromRecord(record, "source-resumed"), nil +} + +func (s *Store) checkpointSealByIDLocked(sealID string) (string, checkpointSealRecord, bool) { + for tokenHash, record := range s.checkpointSeals { + if record.SealID == sealID { + return tokenHash, record, true + } + } + return "", checkpointSealRecord{}, false +} + +func (s *Store) checkpointSealByIssuanceKeyLocked(keyHash string) (string, checkpointSealRecord, bool) { + if keyHash == "" { + return "", checkpointSealRecord{}, false + } + for tokenHash, record := range s.checkpointSeals { + if record.IssuanceKeyHash == keyHash { + return tokenHash, record, true + } + } + return "", checkpointSealRecord{}, false +} + +func checkpointOwnershipFromRecord(record checkpointSealRecord, status string) CheckpointSealOwnership { + return CheckpointSealOwnership{ + SealID: record.SealID, WorkspaceID: record.WorkspaceID, Root: record.Root, + SessionID: record.SessionID, Generation: record.Generation, Status: status, + Digest: record.HandbackDigest, WorkspaceRevision: record.HandbackRevision, + EventCursor: record.HandbackEventCursor, ConsumedAt: record.ConsumedAt, + PreparedAt: record.HandbackPreparedAt, ReleasedAt: record.HandbackReleasedAt, + SourceResumedAt: record.SourceResumedAt, + } +} + +func validCheckpointDigest(value string) bool { + value = strings.TrimSpace(value) + if !strings.HasPrefix(value, "sha256:") || len(value) != len("sha256:")+sha256.Size*2 { + return false + } + _, err := hex.DecodeString(strings.TrimPrefix(value, "sha256:")) + return err == nil +} + +func checkpointConsumerBindingMatches(binding checkpointConsumerBinding, tokenHash, workspaceID, root, sessionID string, generation uint64, principal string) bool { + return binding.TokenHash == tokenHash && binding.WorkspaceID == workspaceID && binding.Root == root && binding.SessionID == sessionID && binding.Generation == generation && binding.Principal == principal +} + +// GetCheckpointSealRetentionSummary exposes bounded operational visibility +// without returning bearer material or internal hashes. Unresumed records are +// intentionally retained fail-closed until an ordinary source resume or an +// explicit break-glass administrative reconciliation proves source readiness. +func (s *Store) GetCheckpointSealRetentionSummary(workspaceID string, now time.Time) CheckpointSealRetentionSummary { + workspaceID = strings.TrimSpace(workspaceID) + now = now.UTC() + s.mu.Lock() + defer s.mu.Unlock() + s.purgeCheckpointSealsLocked(now) + summary := CheckpointSealRetentionSummary{ + GeneratedAt: now.Format(time.RFC3339Nano), + UnresumedByWorkspace: map[string]int{}, + Records: []CheckpointSealRetentionRecord{}, + } + for _, record := range s.checkpointSeals { + if record.SourceResumedAt != "" || (workspaceID != "" && record.WorkspaceID != workspaceID) { + continue + } + summary.UnresumedTotal++ + summary.UnresumedByWorkspace[record.WorkspaceID]++ + summary.Records = append(summary.Records, checkpointRetentionRecord(record)) + } + sort.Slice(summary.Records, func(i, j int) bool { + if summary.Records[i].WorkspaceID != summary.Records[j].WorkspaceID { + return summary.Records[i].WorkspaceID < summary.Records[j].WorkspaceID + } + if summary.Records[i].IssuedAt != summary.Records[j].IssuedAt { + return summary.Records[i].IssuedAt < summary.Records[j].IssuedAt + } + return summary.Records[i].SealID < summary.Records[j].SealID + }) + return summary +} + +// ReconcileCheckpointSealSource is a break-glass administrative path. It can +// resolve only states where destination ownership is absent (never a consumed, +// unreleased seal), requires an exact durable identity/status fence and an +// explicit assertion that the source is ready, and is idempotent across lost +// responses. The record becomes a bounded tombstone; any consumer-key binding +// is removed immediately and the tombstone expires after replay retention. +func (s *Store) ReconcileCheckpointSealSource(sealID string, req CheckpointSealAdminReconcileRequest, now time.Time) (CheckpointSealRetentionRecord, error) { + sealID = strings.TrimSpace(sealID) + workspaceID := strings.TrimSpace(req.WorkspaceID) + sessionID := strings.TrimSpace(req.SessionID) + expectedStatus := strings.TrimSpace(req.ExpectedOwnershipStatus) + reconciliationKey := strings.TrimSpace(req.ReconciliationIdempotencyKey) + root, err := NormalizeCheckpointRoot(req.Root) + if err != nil || sealID == "" || workspaceID == "" || !checkpointSessionPattern.MatchString(sessionID) || req.Generation == 0 || + (expectedStatus != "unconsumed" && expectedStatus != "released") || !checkpointSessionPattern.MatchString(reconciliationKey) || !req.ConfirmSourceReady { + return CheckpointSealRetentionRecord{}, ErrInvalidInput + } + now = now.UTC() + reconciliationKeyHash := checkpointTokenHash(reconciliationKey) + + s.mu.Lock() + defer s.mu.Unlock() + s.purgeCheckpointSealsLocked(now) + tokenHash, record, ok := s.checkpointSealByIDLocked(sealID) + if !ok { + return CheckpointSealRetentionRecord{}, ErrNotFound + } + if record.WorkspaceID != workspaceID || record.Root != root || record.SessionID != sessionID || record.Generation != req.Generation { + return CheckpointSealRetentionRecord{}, ErrCheckpointAdminConflict + } + if record.SourceResumedAt != "" { + if record.AdminReconcileKeyHash == reconciliationKeyHash { + return checkpointRetentionRecord(record), nil + } + return CheckpointSealRetentionRecord{}, ErrCheckpointAdminConflict + } + status := checkpointOwnershipStatus(record) + if status == "consumed" { + return CheckpointSealRetentionRecord{}, ErrCheckpointHandbackRequired + } + if status != expectedStatus { + return CheckpointSealRetentionRecord{}, ErrCheckpointAdminConflict + } + previous := record + var previousBinding checkpointConsumerBinding + hadBinding := false + if record.ConsumerKeyHash != "" { + previousBinding, hadBinding = s.checkpointConsumerKeys[record.ConsumerKeyHash] + delete(s.checkpointConsumerKeys, record.ConsumerKeyHash) + } + reconciledAt := now.Format(time.RFC3339Nano) + record.SourceResumedAt = reconciledAt + record.AdminReconciledAt = reconciledAt + record.AdminReconcileKeyHash = reconciliationKeyHash + record.IdempotencyExpiresAt = now.Add(CheckpointConsumeReplayRetention).Format(time.RFC3339Nano) + s.checkpointSeals[tokenHash] = record + if err := s.saveLocked(); err != nil { + s.checkpointSeals[tokenHash] = previous + if hadBinding { + s.checkpointConsumerKeys[record.ConsumerKeyHash] = previousBinding + } + return CheckpointSealRetentionRecord{}, err + } + return checkpointRetentionRecord(record), nil +} + +func checkpointRetentionRecord(record checkpointSealRecord) CheckpointSealRetentionRecord { + return CheckpointSealRetentionRecord{ + SealID: record.SealID, WorkspaceID: record.WorkspaceID, Root: record.Root, + SessionID: record.SessionID, Generation: record.Generation, + OwnershipStatus: checkpointOwnershipStatus(record), IssuedAt: record.IssuedAt, + ExpiresAt: record.ExpiresAt, ConsumedAt: record.ConsumedAt, + HandbackReleasedAt: record.HandbackReleasedAt, SourceResumedAt: record.SourceResumedAt, + AdminReconciledAt: record.AdminReconciledAt, + } +} + +func checkpointOwnershipStatus(record checkpointSealRecord) string { + if record.SourceResumedAt != "" { + return "source-resumed" + } + if record.HandbackReleasedAt != "" { + return "released" + } + if record.ConsumedAt != "" { + return "consumed" + } + return "unconsumed" +} + +func (s *Store) purgeCheckpointSealsLocked(now time.Time) { + for tokenHash, record := range s.checkpointSeals { + // Every seal represents a stopped source until that source explicitly + // resumes. Expiry only closes destination consume; it is never evidence + // that source ownership was restored. Likewise, destination handback does + // not prove the source came back. Retain both unconsumed and released + // records indefinitely until SourceResumedAt is durable. + if record.SourceResumedAt == "" { + continue + } + replayUntil, err := time.Parse(time.RFC3339Nano, record.IdempotencyExpiresAt) + if err == nil && now.Before(replayUntil) { + continue + } + delete(s.checkpointSeals, tokenHash) + if record.ConsumerKeyHash != "" { + delete(s.checkpointConsumerKeys, record.ConsumerKeyHash) + } + } +} + +func (s *Store) checkpointStateLocked(workspaceID, root string) (digest, workspaceRevision, cursor string, err error) { + entries := []CheckpointDigestEntry{} + workspaceRevision = s.currentWorkspaceRevisionLocked(workspaceID) + cursor = "0" + ws, ok := s.workspaces[workspaceID] + if ok { + paths := make([]string, 0, len(ws.Files)) + for filePath := range ws.Files { + if withinBase(root, filePath) && normalizePath(filePath) != root { + paths = append(paths, normalizePath(filePath)) + } + } + sort.Strings(paths) + for _, filePath := range paths { + file := ws.Files[filePath] + entries = append(entries, CheckpointDigestEntry{Path: filePath, Revision: file.Revision, ContentHash: storedContentHashForFile(file)}) + } + if len(ws.Events) > 0 { + cursor = strings.TrimSpace(ws.Events[len(ws.Events)-1].EventID) + } + } + digest, err = ComputeCheckpointDigest(root, entries) + return digest, workspaceRevision, cursor, err +} + +func checkpointGenerationKey(workspaceID, root, sessionID string) string { + return strings.Join([]string{strings.TrimSpace(workspaceID), root, strings.TrimSpace(sessionID)}, "\x00") +} + +func checkpointIssuanceRequestHash(workspaceID, root, sessionID string, generation uint64, expectedDigest string, ttlSeconds int) string { + h := sha256.New() + for _, value := range []string{ + strings.TrimSpace(workspaceID), root, strings.TrimSpace(sessionID), + strconv.FormatUint(generation, 10), strings.TrimSpace(expectedDigest), strconv.Itoa(ttlSeconds), + } { + writeDigestField(h, value) + } + return hex.EncodeToString(h.Sum(nil)) +} + +func newCheckpointToken() (string, error) { + var raw [32]byte + if _, err := rand.Read(raw[:]); err != nil { + return "", fmt.Errorf("generate checkpoint seal token: %w", err) + } + return base64.RawURLEncoding.EncodeToString(raw[:]), nil +} + +func checkpointTokenHash(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/relayfile/checkpoint_seal_test.go b/internal/relayfile/checkpoint_seal_test.go new file mode 100644 index 00000000..64b2f3d6 --- /dev/null +++ b/internal/relayfile/checkpoint_seal_test.go @@ -0,0 +1,934 @@ +package relayfile + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +const checkpointTestConsumerPrincipal = "cloud-dashboard-observer" + +func TestCheckpointIssuanceResponseLossRotatesBearerAcrossRestart(t *testing.T) { + stateFile := filepath.Join(t.TempDir(), "relayfile-state.json") + now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + first := NewStoreWithOptions(StoreOptions{StateFile: stateFile, DisableWorkers: true}) + seedCheckpointFile(t, first, "ws_issue_retry", "/transcript.jsonl", "turn one\n") + digest := checkpointDigestForStore(t, first, "ws_issue_retry", "/") + request := CheckpointSealRequest{ + Root: "/", SessionID: "thread-issue-retry", Generation: 9, + ExpectedDigest: digest, TTLSeconds: 60, + IssuanceIdempotencyKey: "source-issue-attempt-9", Issuer: "LocalController", + } + issued, err := first.IssueCheckpointSeal("ws_issue_retry", request, now) + if err != nil { + t.Fatalf("initial issue: %v", err) + } + first.Close() + + second := NewStoreWithOptions(StoreOptions{StateFile: stateFile, DisableWorkers: true}) + defer second.Close() + recovered, err := second.IssueCheckpointSeal("ws_issue_retry", request, now.Add(time.Second)) + if err != nil { + t.Fatalf("response-loss recovery: %v", err) + } + if recovered.SealID != issued.SealID || recovered.Digest != issued.Digest || recovered.IssuedAt != issued.IssuedAt || recovered.ExpiresAt != issued.ExpiresAt { + t.Fatalf("retry changed immutable attestation: first=%+v retry=%+v", issued, recovered) + } + if recovered.SealToken == "" || recovered.SealToken == issued.SealToken { + t.Fatalf("retry did not rotate the lost bearer: first=%q retry=%q", issued.SealToken, recovered.SealToken) + } + changedRequest := request + changedRequest.TTLSeconds = 61 + if _, err := second.IssueCheckpointSeal("ws_issue_retry", changedRequest, now.Add(2*time.Second)); !errors.Is(err, ErrCheckpointIssuanceConflict) { + t.Fatalf("changed request error = %v, want issuance conflict", err) + } + if _, err := second.ConsumeCheckpointSeal("ws_issue_retry", CheckpointSealConsumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: request.SessionID, Generation: request.Generation, + ConsumerIdempotencyKey: "consume-old-lost-token", ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(2*time.Second)); !errors.Is(err, ErrNotFound) { + t.Fatalf("lost bearer remained usable after retry rotation: %v", err) + } + if _, err := second.ConsumeCheckpointSeal("ws_issue_retry", CheckpointSealConsumeRequest{ + SealToken: recovered.SealToken, Root: "/", SessionID: request.SessionID, Generation: request.Generation, + ConsumerIdempotencyKey: "consume-recovered-token", ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(2*time.Second)); err != nil { + t.Fatalf("rotated bearer was not consumable: %v", err) + } + + persisted, err := os.ReadFile(stateFile) + if err != nil { + t.Fatalf("read persisted state: %v", err) + } + for _, secret := range []string{request.IssuanceIdempotencyKey, issued.SealToken, recovered.SealToken} { + if strings.Contains(string(persisted), secret) { + t.Fatalf("persisted state leaked plaintext issuance material %q", secret) + } + } + + changed := request + changed.Issuer = "OtherController" + if _, err := second.IssueCheckpointSeal("ws_issue_retry", changed, now.Add(3*time.Second)); !errors.Is(err, ErrCheckpointIssuanceConflict) { + t.Fatalf("changed issuer error = %v, want issuance conflict", err) + } +} + +func TestCheckpointRetentionVisibilityAndBreakGlassReconciliation(t *testing.T) { + stateFile := filepath.Join(t.TempDir(), "relayfile-state.json") + now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + store := NewStoreWithOptions(StoreOptions{StateFile: stateFile, DisableWorkers: true}) + seedCheckpointFile(t, store, "ws_admin_reconcile", "/transcript.jsonl", "turn one\n") + digest := checkpointDigestForStore(t, store, "ws_admin_reconcile", "/") + issued, err := store.IssueCheckpointSeal("ws_admin_reconcile", CheckpointSealRequest{ + Root: "/", SessionID: "thread-admin-reconcile", Generation: 1, + ExpectedDigest: digest, IssuanceIdempotencyKey: "issue-admin-reconcile-1", Issuer: "LocalController", + }, now) + if err != nil { + t.Fatalf("issue: %v", err) + } + consumeKey := "consume-admin-reconcile-1" + consumed, err := store.ConsumeCheckpointSeal("ws_admin_reconcile", CheckpointSealConsumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, Generation: issued.Generation, + ConsumerIdempotencyKey: consumeKey, ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(time.Second)) + if err != nil { + t.Fatalf("consume: %v", err) + } + handback := CheckpointSealHandbackRequest{ + Phase: CheckpointHandbackPhasePrepare, SealID: consumed.SealID, Root: "/", + SessionID: consumed.SessionID, Generation: consumed.Generation, ConsumedAt: consumed.ConsumedAt, + ConsumerIdempotencyKey: consumeKey, HandbackIdempotencyKey: "handback-admin-reconcile-1", + ExpectedDigest: digest, ConsumerPrincipal: checkpointTestConsumerPrincipal, + } + if _, err := store.HandbackCheckpointSeal("ws_admin_reconcile", handback, now.Add(2*time.Second)); err != nil { + t.Fatalf("prepare handback: %v", err) + } + handback.Phase = CheckpointHandbackPhaseCommit + if _, err := store.HandbackCheckpointSeal("ws_admin_reconcile", handback, now.Add(3*time.Second)); err != nil { + t.Fatalf("commit handback: %v", err) + } + + summary := store.GetCheckpointSealRetentionSummary("", now.Add(4*time.Second)) + if summary.UnresumedTotal != 1 || summary.UnresumedByWorkspace["ws_admin_reconcile"] != 1 || len(summary.Records) != 1 || summary.Records[0].OwnershipStatus != "released" { + t.Fatalf("unexpected retention visibility: %+v", summary) + } + payload, err := json.Marshal(summary) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(payload), issued.SealToken) || strings.Contains(string(payload), consumeKey) { + t.Fatalf("retention visibility leaked bearer/idempotency material: %s", payload) + } + + reconcile := CheckpointSealAdminReconcileRequest{ + WorkspaceID: "ws_admin_reconcile", Root: "/", SessionID: consumed.SessionID, + Generation: consumed.Generation, ExpectedOwnershipStatus: "released", + ReconciliationIdempotencyKey: "admin-reconcile-source-1", ConfirmSourceReady: true, + } + reconciled, err := store.ReconcileCheckpointSealSource(consumed.SealID, reconcile, now.Add(5*time.Second)) + if err != nil || reconciled.OwnershipStatus != "source-resumed" || reconciled.AdminReconciledAt == "" { + t.Fatalf("administrative reconciliation = %+v err=%v", reconciled, err) + } + replayed, err := store.ReconcileCheckpointSealSource(consumed.SealID, reconcile, now.Add(6*time.Second)) + if err != nil || replayed.AdminReconciledAt != reconciled.AdminReconciledAt { + t.Fatalf("reconciliation response-loss replay = %+v err=%v", replayed, err) + } + changedKey := reconcile + changedKey.ReconciliationIdempotencyKey = "admin-reconcile-source-2" + if _, err := store.ReconcileCheckpointSealSource(consumed.SealID, changedKey, now.Add(6*time.Second)); !errors.Is(err, ErrCheckpointAdminConflict) { + t.Fatalf("changed reconciliation key error = %v", err) + } + store.mu.Lock() + _, consumerBindingRetained := store.checkpointConsumerKeys[checkpointTokenHash(consumeKey)] + store.mu.Unlock() + if consumerBindingRetained { + t.Fatal("administrative reconciliation retained the associated consumer binding") + } + if got := store.GetCheckpointSealRetentionSummary("ws_admin_reconcile", now.Add(7*time.Second)); got.UnresumedTotal != 0 || len(got.Records) != 0 { + t.Fatalf("reconciled seal remained in unresumed metrics: %+v", got) + } + store.Close() + + restarted := NewStoreWithOptions(StoreOptions{StateFile: stateFile, DisableWorkers: true}) + defer restarted.Close() + replayed, err = restarted.ReconcileCheckpointSealSource(consumed.SealID, reconcile, now.Add(8*time.Second)) + if err != nil || replayed.AdminReconciledAt != reconciled.AdminReconciledAt { + t.Fatalf("restart reconciliation replay = %+v err=%v", replayed, err) + } + restarted.GetCheckpointSealRetentionSummary("", now.Add(CheckpointConsumeReplayRetention+6*time.Second)) + restarted.mu.Lock() + _, _, retained := restarted.checkpointSealByIDLocked(consumed.SealID) + restarted.mu.Unlock() + if retained { + t.Fatal("reconciled tombstone survived beyond bounded replay retention") + } +} + +func TestCheckpointAdminReconciliationCannotOverrideConsumedOwnership(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + defer store.Close() + now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + seedCheckpointFile(t, store, "ws_admin_owned", "/transcript.jsonl", "turn one\n") + digest := checkpointDigestForStore(t, store, "ws_admin_owned", "/") + issued, err := store.IssueCheckpointSeal("ws_admin_owned", CheckpointSealRequest{ + Root: "/", SessionID: "thread-admin-owned", Generation: 1, ExpectedDigest: digest, + IssuanceIdempotencyKey: "issue-admin-owned-1", Issuer: "LocalController", + }, now) + if err != nil { + t.Fatal(err) + } + if _, err := store.ConsumeCheckpointSeal("ws_admin_owned", CheckpointSealConsumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, Generation: issued.Generation, + ConsumerIdempotencyKey: "consume-admin-owned-1", ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(time.Second)); err != nil { + t.Fatal(err) + } + _, err = store.ReconcileCheckpointSealSource(issued.SealID, CheckpointSealAdminReconcileRequest{ + WorkspaceID: "ws_admin_owned", Root: "/", SessionID: issued.SessionID, Generation: issued.Generation, + ExpectedOwnershipStatus: "released", ReconciliationIdempotencyKey: "admin-reconcile-owned-1", ConfirmSourceReady: true, + }, now.Add(2*time.Second)) + if !errors.Is(err, ErrCheckpointHandbackRequired) { + t.Fatalf("consumed ownership override error = %v, want handback required", err) + } +} + +func TestCheckpointHandbackReportsUnconsumedBeforeReceiptMismatch(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + defer store.Close() + now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + seedCheckpointFile(t, store, "ws_unconsumed_code", "/transcript.jsonl", "turn\n") + digest := checkpointDigestForStore(t, store, "ws_unconsumed_code", "/") + issued, err := store.IssueCheckpointSeal("ws_unconsumed_code", CheckpointSealRequest{ + Root: "/", SessionID: "thread-unconsumed-code", Generation: 1, ExpectedDigest: digest, + IssuanceIdempotencyKey: "issue-unconsumed-code-1", Issuer: "LocalController", + }, now) + if err != nil { + t.Fatal(err) + } + _, err = store.HandbackCheckpointSeal("ws_unconsumed_code", CheckpointSealHandbackRequest{ + Phase: CheckpointHandbackPhasePrepare, SealID: issued.SealID, Root: "/", + SessionID: issued.SessionID, Generation: issued.Generation, + ConsumedAt: now.Add(time.Second).Format(time.RFC3339Nano), + ConsumerIdempotencyKey: "consume-unconsumed-code-1", HandbackIdempotencyKey: "handback-unconsumed-code-1", + ExpectedDigest: digest, ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(2*time.Second)) + if !errors.Is(err, ErrCheckpointUnconsumed) { + t.Fatalf("handback error = %v, want checkpoint unconsumed", err) + } +} + +func TestCheckpointSealOwnershipOmitsEmptyConsumedAt(t *testing.T) { + payload, err := json.Marshal(CheckpointSealOwnership{Status: "source-resumed"}) + if err != nil { + t.Fatal(err) + } + var wire map[string]any + if err := json.Unmarshal(payload, &wire); err != nil { + t.Fatal(err) + } + if _, present := wire["consumedAt"]; present { + t.Fatalf("empty consumedAt must be omitted to match the optional OpenAPI/TypeScript contract: %s", payload) + } + + payload, err = json.Marshal(CheckpointSealOwnership{Status: "released", ConsumedAt: "2026-08-23T12:00:00Z"}) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(payload, &wire); err != nil { + t.Fatal(err) + } + if wire["consumedAt"] != "2026-08-23T12:00:00Z" { + t.Fatalf("non-empty consumedAt was not preserved: %s", payload) + } +} + +func TestCheckpointSealIsOneUseAndIdentityBound(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + defer store.Close() + seedCheckpointFile(t, store, "ws_seal", "/sessions/transcript.jsonl", "first\n") + digest := checkpointDigestForStore(t, store, "ws_seal", "/sessions") + now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + seal, err := store.IssueCheckpointSeal("ws_seal", CheckpointSealRequest{ + Root: "/sessions", + SessionID: "thread-123", + Generation: 7, + ExpectedDigest: digest, + TTLSeconds: 30, + }, now) + if err != nil { + t.Fatalf("issue seal: %v", err) + } + if seal.SealToken == "" || seal.Digest != digest || seal.EventCursor == "" { + t.Fatalf("incomplete server seal: %+v", seal) + } + for name, req := range map[string]CheckpointSealConsumeRequest{ + "root": {SealToken: seal.SealToken, Root: "/other", SessionID: "thread-123", Generation: 7, ConsumerIdempotencyKey: "acquire-mismatch-root", ConsumerPrincipal: checkpointTestConsumerPrincipal}, + "session": {SealToken: seal.SealToken, Root: "/sessions", SessionID: "thread-456", Generation: 7, ConsumerIdempotencyKey: "acquire-mismatch-session", ConsumerPrincipal: checkpointTestConsumerPrincipal}, + "generation": {SealToken: seal.SealToken, Root: "/sessions", SessionID: "thread-123", Generation: 8, ConsumerIdempotencyKey: "acquire-mismatch-generation", ConsumerPrincipal: checkpointTestConsumerPrincipal}, + } { + t.Run(name, func(t *testing.T) { + if _, err := store.ConsumeCheckpointSeal("ws_seal", req, now.Add(time.Second)); !errors.Is(err, ErrInvalidInput) { + t.Fatalf("identity mismatch error = %v, want invalid input", err) + } + }) + } + consume := CheckpointSealConsumeRequest{SealToken: seal.SealToken, Root: "/sessions", SessionID: "thread-123", Generation: 7, ConsumerIdempotencyKey: "acquire-one", ConsumerPrincipal: checkpointTestConsumerPrincipal} + if _, err := store.ConsumeCheckpointSeal("ws_seal", consume, now.Add(time.Second)); err != nil { + t.Fatalf("consume seal: %v", err) + } + if replay, err := store.ConsumeCheckpointSeal("ws_seal", consume, now.Add(2*time.Second)); err != nil || replay.ConsumedAt == "" { + t.Fatalf("exact idempotent replay = %+v, err=%v", replay, err) + } + differentPrincipal := consume + differentPrincipal.ConsumerPrincipal = "other-authenticated-agent" + if _, err := store.ConsumeCheckpointSeal("ws_seal", differentPrincipal, now.Add(2*time.Second)); !errors.Is(err, ErrCheckpointConsumerConflict) { + t.Fatalf("different authenticated principal error = %v, want consumer conflict", err) + } + differentConsumer := consume + differentConsumer.ConsumerIdempotencyKey = "acquire-two" + if _, err := store.ConsumeCheckpointSeal("ws_seal", differentConsumer, now.Add(2*time.Second)); !errors.Is(err, ErrCheckpointReplay) { + t.Fatalf("different consumer replay error = %v, want checkpoint replay", err) + } + if _, err := store.IssueCheckpointSeal("ws_seal", CheckpointSealRequest{ + Root: "/sessions", SessionID: "thread-123", Generation: 7, ExpectedDigest: digest, + }, now.Add(3*time.Second)); !errors.Is(err, ErrCheckpointGenerationStale) { + t.Fatalf("stale generation error = %v", err) + } +} + +func TestCheckpointSealRejectsDivergenceExpiryAndRemoteMutation(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + defer store.Close() + seedCheckpointFile(t, store, "ws_stale", "/sessions/transcript.jsonl", "first\n") + now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + if _, err := store.IssueCheckpointSeal("ws_stale", CheckpointSealRequest{ + Root: "/sessions", SessionID: "thread-stale", Generation: 1, + ExpectedDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, now); !errors.Is(err, ErrCheckpointDiverged) { + t.Fatalf("mismatched caller digest error = %v", err) + } + digest := checkpointDigestForStore(t, store, "ws_stale", "/sessions") + expiring, err := store.IssueCheckpointSeal("ws_stale", CheckpointSealRequest{ + Root: "/sessions", SessionID: "thread-expire", Generation: 1, ExpectedDigest: digest, TTLSeconds: 1, + }, now) + if err != nil { + t.Fatalf("issue expiring: %v", err) + } + if _, err := store.ConsumeCheckpointSeal("ws_stale", CheckpointSealConsumeRequest{ + SealToken: expiring.SealToken, Root: "/sessions", SessionID: "thread-expire", Generation: 1, ConsumerIdempotencyKey: "acquire-expire", ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(time.Second)); !errors.Is(err, ErrCheckpointExpired) { + t.Fatalf("expiry error = %v", err) + } + + stale, err := store.IssueCheckpointSeal("ws_stale", CheckpointSealRequest{ + Root: "/sessions", SessionID: "thread-stale", Generation: 2, ExpectedDigest: digest, + }, now) + if err != nil { + t.Fatalf("issue stale candidate: %v", err) + } + seedCheckpointFile(t, store, "ws_stale", "/sessions/after.json", "changed") + if _, err := store.ConsumeCheckpointSeal("ws_stale", CheckpointSealConsumeRequest{ + SealToken: stale.SealToken, Root: "/sessions", SessionID: "thread-stale", Generation: 2, ConsumerIdempotencyKey: "acquire-stale", ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(time.Second)); !errors.Is(err, ErrCheckpointStale) { + t.Fatalf("remote mutation error = %v, want stale", err) + } +} + +func TestCheckpointSealSurvivesDaemonRestart(t *testing.T) { + stateFile := filepath.Join(t.TempDir(), "relayfile-state.json") + now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + first := NewStoreWithOptions(StoreOptions{StateFile: stateFile, DisableWorkers: true}) + seedCheckpointFile(t, first, "ws_restart", "/sessions/transcript.jsonl", "durable\n") + digest := checkpointDigestForStore(t, first, "ws_restart", "/sessions") + seal, err := first.IssueCheckpointSeal("ws_restart", CheckpointSealRequest{ + Root: "/sessions", SessionID: "thread-restart", Generation: 3, ExpectedDigest: digest, + }, now) + if err != nil { + t.Fatalf("issue before restart: %v", err) + } + first.Close() + + second := NewStoreWithOptions(StoreOptions{StateFile: stateFile, DisableWorkers: true}) + defer second.Close() + if _, err := second.ConsumeCheckpointSeal("ws_restart", CheckpointSealConsumeRequest{ + SealToken: seal.SealToken, Root: "/sessions", SessionID: "thread-restart", Generation: 3, ConsumerIdempotencyKey: "acquire-restart", ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(time.Second)); err != nil { + t.Fatalf("consume after restart: %v", err) + } + if _, err := second.IssueCheckpointSeal("ws_restart", CheckpointSealRequest{ + Root: "/sessions", SessionID: "thread-restart", Generation: 3, ExpectedDigest: digest, + }, now.Add(2*time.Second)); !errors.Is(err, ErrCheckpointGenerationStale) { + t.Fatalf("generation replay after restart error = %v", err) + } +} + +func TestCheckpointConsumeResponseLossIsIdempotentAcrossExpiryAndRestart(t *testing.T) { + stateFile := filepath.Join(t.TempDir(), "relayfile-state.json") + now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + first := NewStoreWithOptions(StoreOptions{StateFile: stateFile, DisableWorkers: true}) + seedCheckpointFile(t, first, "ws_idempotent", "/sessions/transcript.jsonl", "durable\n") + digest := checkpointDigestForStore(t, first, "ws_idempotent", "/sessions") + seal, err := first.IssueCheckpointSeal("ws_idempotent", CheckpointSealRequest{ + Root: "/sessions", SessionID: "thread-idempotent", Generation: 1, + ExpectedDigest: digest, TTLSeconds: 1, + }, now) + if err != nil { + t.Fatalf("issue seal: %v", err) + } + consume := CheckpointSealConsumeRequest{ + SealToken: seal.SealToken, Root: "/sessions", SessionID: "thread-idempotent", + Generation: 1, ConsumerIdempotencyKey: "cloud-acquire-attempt-123", ConsumerPrincipal: checkpointTestConsumerPrincipal, + } + consumed, err := first.ConsumeCheckpointSeal("ws_idempotent", consume, now.Add(500*time.Millisecond)) + if err != nil { + t.Fatalf("first consume: %v", err) + } + first.Close() + + second := NewStoreWithOptions(StoreOptions{StateFile: stateFile, DisableWorkers: true}) + defer second.Close() + replayed, err := second.ConsumeCheckpointSeal("ws_idempotent", consume, now.Add(10*time.Minute)) + if err != nil { + t.Fatalf("response-loss replay after seal expiry/restart: %v", err) + } + if replayed.ConsumedAt != consumed.ConsumedAt || replayed.SealID != consumed.SealID { + t.Fatalf("replayed result changed: first=%+v replay=%+v", consumed, replayed) + } + recovered, err := second.RecoverConsumedCheckpointSeal("ws_idempotent", CheckpointSealConsumeRecoveryRequest{ + Root: consume.Root, SessionID: consume.SessionID, Generation: consume.Generation, + ConsumerIdempotencyKey: consume.ConsumerIdempotencyKey, ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(25*time.Hour)) + if err != nil || recovered.SealToken != "" || recovered.SealID != consumed.SealID || recovered.ConsumedAt != consumed.ConsumedAt { + t.Fatalf("tokenless consume recovery after lease cap = %+v err=%v", recovered, err) + } + if _, err := second.RecoverConsumedCheckpointSeal("ws_idempotent", CheckpointSealConsumeRecoveryRequest{ + Root: "/other", SessionID: consume.SessionID, Generation: consume.Generation, + ConsumerIdempotencyKey: consume.ConsumerIdempotencyKey, ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(25*time.Hour)); !errors.Is(err, ErrCheckpointConsumerConflict) { + t.Fatalf("changed recovery identity error = %v", err) + } + if _, err := second.RecoverConsumedCheckpointSeal("ws_idempotent", CheckpointSealConsumeRecoveryRequest{ + Root: consume.Root, SessionID: consume.SessionID, Generation: consume.Generation, + ConsumerIdempotencyKey: consume.ConsumerIdempotencyKey, ConsumerPrincipal: "other-authenticated-agent", + }, now.Add(25*time.Hour)); !errors.Is(err, ErrCheckpointConsumerConflict) { + t.Fatalf("changed recovery principal error = %v", err) + } + if _, err := second.RecoverConsumedCheckpointSeal("ws_idempotent", CheckpointSealConsumeRecoveryRequest{ + Root: consume.Root, SessionID: consume.SessionID, Generation: consume.Generation, + ConsumerIdempotencyKey: "unknown-consumer", ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(25*time.Hour)); !errors.Is(err, ErrNotFound) { + t.Fatalf("unknown recovery error = %v", err) + } + + changedIdentity := consume + changedIdentity.Root = "/other" + if _, err := second.ConsumeCheckpointSeal("ws_idempotent", changedIdentity, now.Add(11*time.Minute)); !errors.Is(err, ErrCheckpointConsumerConflict) { + t.Fatalf("same consumer key with changed identity error = %v", err) + } + + otherSeal, err := second.IssueCheckpointSeal("ws_idempotent", CheckpointSealRequest{ + Root: "/sessions", SessionID: "thread-other", Generation: 1, ExpectedDigest: digest, + }, now.Add(12*time.Minute)) + if err != nil { + t.Fatalf("issue other seal: %v", err) + } + if _, err := second.ConsumeCheckpointSeal("ws_idempotent", CheckpointSealConsumeRequest{ + SealToken: otherSeal.SealToken, Root: "/sessions", SessionID: "thread-other", + Generation: 1, ConsumerIdempotencyKey: consume.ConsumerIdempotencyKey, ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(13*time.Minute)); !errors.Is(err, ErrCheckpointConsumerConflict) { + t.Fatalf("consumer key reused for another seal error = %v", err) + } +} + +func TestCheckpointConsumedOwnershipIsRetainedUntilExplicitHandback(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + defer store.Close() + seedCheckpointFile(t, store, "ws_retention", "/sessions/transcript.jsonl", "durable\n") + digest := checkpointDigestForStore(t, store, "ws_retention", "/sessions") + now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + seal, err := store.IssueCheckpointSeal("ws_retention", CheckpointSealRequest{ + Root: "/sessions", SessionID: "thread-retention", Generation: 1, ExpectedDigest: digest, + }, now) + if err != nil { + t.Fatalf("issue: %v", err) + } + consume := CheckpointSealConsumeRequest{ + SealToken: seal.SealToken, Root: "/sessions", SessionID: "thread-retention", + Generation: 1, ConsumerIdempotencyKey: "cloud-retention-attempt", ConsumerPrincipal: checkpointTestConsumerPrincipal, + } + if _, err := store.ConsumeCheckpointSeal("ws_retention", consume, now.Add(time.Second)); err != nil { + t.Fatalf("consume: %v", err) + } + if replayed, err := store.ConsumeCheckpointSeal("ws_retention", consume, now.Add(time.Second+CheckpointConsumeReplayRetention)); err != nil || replayed.ConsumedAt == "" { + t.Fatalf("active ownership replay after diagnostic retention = %+v err=%v", replayed, err) + } +} + +func TestCheckpointVerifyReattestsConsumedSealWhenLatestRevisionIsDeletion(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + defer store.Close() + keep := seedCheckpointFileResult(t, store, "ws_verify_delete", "/keep.txt", "keep\n") + deleted := seedCheckpointFileResult(t, store, "ws_verify_delete", "/delete.txt", "delete\n") + if _, err := store.DeleteFile(DeleteRequest{WorkspaceID: "ws_verify_delete", Path: "/delete.txt", IfMatch: deleted.TargetRevision}); err != nil { + t.Fatalf("delete latest file: %v", err) + } + digest := checkpointDigestForStore(t, store, "ws_verify_delete", "/") + now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + issued, err := store.IssueCheckpointSeal("ws_verify_delete", CheckpointSealRequest{ + Root: "/", SessionID: "thread-delete", Generation: 1, ExpectedDigest: digest, + }, now) + if err != nil { + t.Fatalf("issue: %v", err) + } + if issued.WorkspaceRevision == keep.TargetRevision { + t.Fatalf("fixture did not create delete-only revision: workspace=%q surviving=%q", issued.WorkspaceRevision, keep.TargetRevision) + } + unconsumedRequest := checkpointVerifyRequest(issued) + unconsumedRequest.ConsumedAt = now.Add(time.Second).Format(time.RFC3339Nano) + if _, err := store.VerifyConsumedCheckpointSeal("ws_verify_delete", unconsumedRequest, now); !errors.Is(err, ErrCheckpointUnconsumed) { + t.Fatalf("unconsumed verification error = %v", err) + } + consumed, err := store.ConsumeCheckpointSeal("ws_verify_delete", CheckpointSealConsumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, Generation: issued.Generation, + ConsumerIdempotencyKey: "cloud-delete-proof", ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(time.Second)) + if err != nil { + t.Fatalf("consume: %v", err) + } + verified, err := store.VerifyConsumedCheckpointSeal("ws_verify_delete", checkpointVerifyRequest(consumed), now.Add(2*time.Second)) + if err != nil { + t.Fatalf("verify consumed delete-latest seal: %v", err) + } + if verified.SealToken != "" || verified.WorkspaceRevision != consumed.WorkspaceRevision || verified.ConsumedAt != consumed.ConsumedAt { + t.Fatalf("verification leaked token or changed identity: %+v", verified) + } + + tampered := checkpointVerifyRequest(consumed) + tampered.EventCursor = "evt_999" + if _, err := store.VerifyConsumedCheckpointSeal("ws_verify_delete", tampered, now.Add(2*time.Second)); !errors.Is(err, ErrCheckpointStale) { + t.Fatalf("tampered receipt error = %v", err) + } + seedCheckpointFile(t, store, "ws_verify_delete", "/after.txt", "after\n") + if _, err := store.VerifyConsumedCheckpointSeal("ws_verify_delete", checkpointVerifyRequest(consumed), now.Add(3*time.Second)); !errors.Is(err, ErrCheckpointStale) { + t.Fatalf("post-consume remote mutation error = %v", err) + } +} + +func TestCheckpointSealUsesCanonicalZeroRevisionAndCursorForEmptyWorkspace(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + defer store.Close() + digest := checkpointDigestForStore(t, store, "ws_empty_checkpoint", "/") + seal, err := store.IssueCheckpointSeal("ws_empty_checkpoint", CheckpointSealRequest{ + Root: "/", SessionID: "thread-empty", Generation: 1, ExpectedDigest: digest, + }, time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatalf("issue empty-workspace seal: %v", err) + } + if seal.WorkspaceRevision != "0" || seal.EventCursor != "0" { + t.Fatalf("empty-workspace wire state = revision %q cursor %q, want canonical zeroes", seal.WorkspaceRevision, seal.EventCursor) + } +} + +func TestCheckpointHandbackIsConsumerBoundDurableAndGatesSourceResume(t *testing.T) { + stateFile := filepath.Join(t.TempDir(), "relayfile-state.json") + now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + first := NewStoreWithOptions(StoreOptions{StateFile: stateFile, DisableWorkers: true}) + seedCheckpointFile(t, first, "ws_handback", "/transcript.jsonl", "source turn\n") + digest := checkpointDigestForStore(t, first, "ws_handback", "/") + issued, err := first.IssueCheckpointSeal("ws_handback", CheckpointSealRequest{ + Root: "/", SessionID: "thread-handback", Generation: 4, ExpectedDigest: digest, + }, now) + if err != nil { + t.Fatalf("issue: %v", err) + } + consumerKey := "cutover-job-4" + consumed, err := first.ConsumeCheckpointSeal("ws_handback", CheckpointSealConsumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, Generation: issued.Generation, + ConsumerIdempotencyKey: consumerKey, ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(time.Second)) + if err != nil { + t.Fatalf("consume: %v", err) + } + resumeRequest := CheckpointSealResumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, Generation: issued.Generation, + ResumeIdempotencyKey: "source-resume-job-4", + } + if _, err := first.ResumeCheckpointSeal("ws_handback", resumeRequest, now.Add(2*time.Second)); !errors.Is(err, ErrCheckpointHandbackRequired) { + t.Fatalf("premature source resume error = %v, want handback required", err) + } + seedCheckpointFile(t, first, "ws_handback", "/destination.txt", "destination turn\n") + finalDigest, _, finalCursor := checkpointStateForTest(t, first, "ws_handback", "/") + handback := CheckpointSealHandbackRequest{ + Phase: CheckpointHandbackPhasePrepare, + SealID: issued.SealID, Root: "/", SessionID: issued.SessionID, Generation: issued.Generation, + ConsumedAt: consumed.ConsumedAt, ConsumerIdempotencyKey: consumerKey, + HandbackIdempotencyKey: "handback-job-4", ExpectedDigest: finalDigest, ConsumerPrincipal: checkpointTestConsumerPrincipal, + } + wrongConsumer := handback + wrongConsumer.ConsumerIdempotencyKey = "cutover-job-other" + if _, err := first.HandbackCheckpointSeal("ws_handback", wrongConsumer, now.Add(3*time.Second)); !errors.Is(err, ErrCheckpointConsumerConflict) { + t.Fatalf("wrong-consumer handback error = %v", err) + } + wrongPrincipal := handback + wrongPrincipal.ConsumerPrincipal = "other-authenticated-agent" + if _, err := first.HandbackCheckpointSeal("ws_handback", wrongPrincipal, now.Add(3*time.Second)); !errors.Is(err, ErrCheckpointConsumerConflict) { + t.Fatalf("wrong-principal handback error = %v", err) + } + validButDiverged := handback + validButDiverged.ExpectedDigest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + if _, err := first.HandbackCheckpointSeal("ws_handback", validButDiverged, now.Add(3*time.Second)); !errors.Is(err, ErrCheckpointDiverged) { + t.Fatalf("diverged handback digest error = %v, want checkpoint diverged", err) + } + diverged := handback + diverged.ExpectedDigest = "sha256:" + string(make([]byte, 64)) + if _, err := first.HandbackCheckpointSeal("ws_handback", diverged, now.Add(3*time.Second)); !errors.Is(err, ErrInvalidInput) { + t.Fatalf("malformed handback digest error = %v", err) + } + commitWithoutPrepare := handback + commitWithoutPrepare.Phase = CheckpointHandbackPhaseCommit + if _, err := first.HandbackCheckpointSeal("ws_handback", commitWithoutPrepare, now.Add(3*time.Second)); !errors.Is(err, ErrCheckpointHandbackUnprepared) { + t.Fatalf("commit without prepare error = %v", err) + } + prepared, err := first.HandbackCheckpointSeal("ws_handback", handback, now.Add(4*time.Second)) + if err != nil { + t.Fatalf("prepare handback: %v", err) + } + if prepared.Status != "prepared" || prepared.Digest != finalDigest || prepared.EventCursor != finalCursor || prepared.PreparedAt == "" || prepared.ReleasedAt != "" || prepared.SourceResumedAt != "" { + t.Fatalf("handback preparation = %+v", prepared) + } + replayed, err := first.HandbackCheckpointSeal("ws_handback", handback, now.Add(5*time.Second)) + if err != nil || replayed != prepared { + t.Fatalf("prepare replay = %+v err=%v, want %+v", replayed, err, prepared) + } + if _, err := first.ResumeCheckpointSeal("ws_handback", resumeRequest, now.Add(5*time.Second)); !errors.Is(err, ErrCheckpointHandbackRequired) { + t.Fatalf("prepared handback released ownership early: %v", err) + } + // Simulate a destination crash after the durable prepare response. The exact + // commit must remain recoverable after reopening the store. + first.Close() + + second := NewStoreWithOptions(StoreOptions{StateFile: stateFile, DisableWorkers: true}) + defer second.Close() + handback.Phase = CheckpointHandbackPhaseCommit + proof, err := second.HandbackCheckpointSeal("ws_handback", handback, now.Add(6*time.Second)) + if err != nil { + t.Fatalf("commit handback after reopen: %v", err) + } + if proof.Status != "released" || proof.Digest != finalDigest || proof.EventCursor != finalCursor || proof.PreparedAt != prepared.PreparedAt || proof.ReleasedAt == "" || proof.SourceResumedAt != "" { + t.Fatalf("handback proof = %+v", proof) + } + commitReplay, err := second.HandbackCheckpointSeal("ws_handback", handback, now.Add(7*time.Second)) + if err != nil || commitReplay != proof { + t.Fatalf("commit replay = %+v err=%v, want %+v", commitReplay, err, proof) + } + handback.Phase = CheckpointHandbackPhasePrepare + releasedPrepareReplay, err := second.HandbackCheckpointSeal("ws_handback", handback, now.Add(7*time.Second)) + if err != nil || releasedPrepareReplay != proof { + t.Fatalf("released prepare replay = %+v err=%v, want %+v", releasedPrepareReplay, err, proof) + } + handback.Phase = CheckpointHandbackPhaseCommit + changedHandback := handback + changedHandback.HandbackIdempotencyKey = "handback-job-changed" + if _, err := second.HandbackCheckpointSeal("ws_handback", changedHandback, now.Add(7*time.Second)); !errors.Is(err, ErrCheckpointHandbackConflict) { + t.Fatalf("changed handback replay error = %v", err) + } + resumed, err := second.ResumeCheckpointSeal("ws_handback", resumeRequest, now.Add(8*time.Second)) + if err != nil { + t.Fatalf("resume after durable handback: %v", err) + } + if resumed.Status != "source-resumed" || resumed.Digest != proof.Digest || resumed.SourceResumedAt == "" { + t.Fatalf("source resume proof = %+v", resumed) + } + resumedReplay, err := second.ResumeCheckpointSeal("ws_handback", resumeRequest, now.Add(9*time.Second)) + if err != nil || resumedReplay != resumed { + t.Fatalf("resume replay = %+v err=%v, want %+v", resumedReplay, err, resumed) + } + changedResume := resumeRequest + changedResume.ResumeIdempotencyKey = "source-resume-changed" + if _, err := second.ResumeCheckpointSeal("ws_handback", changedResume, now.Add(9*time.Second)); !errors.Is(err, ErrCheckpointResumeConflict) { + t.Fatalf("changed resume replay error = %v", err) + } + if _, err := second.ConsumeCheckpointSeal("ws_handback", CheckpointSealConsumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, Generation: issued.Generation, + ConsumerIdempotencyKey: consumerKey, ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(8*time.Second)); !errors.Is(err, ErrCheckpointReplay) { + t.Fatalf("destination reacquire after handback error = %v", err) + } +} + +func TestCheckpointUnconsumedSealCanBeCancelledBySource(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + defer store.Close() + now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + digest := checkpointDigestForStore(t, store, "ws_cancel", "/") + issued, err := store.IssueCheckpointSeal("ws_cancel", CheckpointSealRequest{ + Root: "/", SessionID: "thread-cancel", Generation: 1, ExpectedDigest: digest, + }, now) + if err != nil { + t.Fatal(err) + } + proof, err := store.ResumeCheckpointSeal("ws_cancel", CheckpointSealResumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, + Generation: issued.Generation, ResumeIdempotencyKey: "source-cancel-one", + }, now.Add(10*time.Minute)) + if err != nil { + t.Fatalf("cancel expired unconsumed seal: %v", err) + } + if proof.Status != "source-resumed" || proof.ConsumedAt != "" || proof.SourceResumedAt == "" { + t.Fatalf("cancel proof = %+v", proof) + } +} + +func TestCheckpointHandbackCommitRejectsDurableChangeAfterPrepare(t *testing.T) { + now := time.Date(2026, 8, 23, 14, 0, 0, 0, time.UTC) + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + defer store.Close() + seedCheckpointFile(t, store, "ws_handback_prepare_race", "/source.txt", "source turn\n") + digest := checkpointDigestForStore(t, store, "ws_handback_prepare_race", "/") + issued, err := store.IssueCheckpointSeal("ws_handback_prepare_race", CheckpointSealRequest{ + Root: "/", SessionID: "thread-handback-prepare-race", Generation: 1, ExpectedDigest: digest, + }, now) + if err != nil { + t.Fatal(err) + } + consumed, err := store.ConsumeCheckpointSeal("ws_handback_prepare_race", CheckpointSealConsumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, Generation: issued.Generation, + ConsumerIdempotencyKey: "consume-handback-prepare-race", ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(time.Second)) + if err != nil { + t.Fatal(err) + } + handback := CheckpointSealHandbackRequest{ + Phase: CheckpointHandbackPhasePrepare, + SealID: consumed.SealID, Root: "/", SessionID: consumed.SessionID, Generation: consumed.Generation, + ConsumedAt: consumed.ConsumedAt, ConsumerIdempotencyKey: "consume-handback-prepare-race", + HandbackIdempotencyKey: "handback-prepare-race", ExpectedDigest: digest, ConsumerPrincipal: checkpointTestConsumerPrincipal, + } + prepared, err := store.HandbackCheckpointSeal("ws_handback_prepare_race", handback, now.Add(2*time.Second)) + if err != nil || prepared.Status != "prepared" { + t.Fatalf("prepare=%+v err=%v", prepared, err) + } + seedCheckpointFile(t, store, "ws_handback_prepare_race", "/late.txt", "late callback bytes\n") + handback.Phase = CheckpointHandbackPhaseCommit + if _, err := store.HandbackCheckpointSeal("ws_handback_prepare_race", handback, now.Add(3*time.Second)); !errors.Is(err, ErrCheckpointDiverged) { + t.Fatalf("changed durable state commit error=%v", err) + } + if _, err := store.ResumeCheckpointSeal("ws_handback_prepare_race", CheckpointSealResumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, Generation: issued.Generation, + ResumeIdempotencyKey: "resume-handback-prepare-race", + }, now.Add(4*time.Second)); !errors.Is(err, ErrCheckpointHandbackRequired) { + t.Fatalf("diverged prepare released ownership: %v", err) + } +} + +func TestCheckpointConsumedOwnershipSurvivesPastGatewayLeaseCap(t *testing.T) { + stateFile := filepath.Join(t.TempDir(), "relayfile-state.json") + now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + first := NewStoreWithOptions(StoreOptions{StateFile: stateFile, DisableWorkers: true}) + digest := checkpointDigestForStore(t, first, "ws_long_owner", "/") + issued, err := first.IssueCheckpointSeal("ws_long_owner", CheckpointSealRequest{ + Root: "/", SessionID: "thread-long-owner", Generation: 1, ExpectedDigest: digest, + }, now) + if err != nil { + t.Fatal(err) + } + consumerKey := "cutover-long-owner" + consumed, err := first.ConsumeCheckpointSeal("ws_long_owner", CheckpointSealConsumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, + Generation: issued.Generation, ConsumerIdempotencyKey: consumerKey, ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(time.Second)) + if err != nil { + t.Fatal(err) + } + first.Close() + + second := NewStoreWithOptions(StoreOptions{StateFile: stateFile, DisableWorkers: true}) + defer second.Close() + handback := CheckpointSealHandbackRequest{ + Phase: CheckpointHandbackPhasePrepare, + SealID: consumed.SealID, Root: "/", SessionID: consumed.SessionID, Generation: consumed.Generation, + ConsumedAt: consumed.ConsumedAt, ConsumerIdempotencyKey: consumerKey, + HandbackIdempotencyKey: "handback-after-cap", ExpectedDigest: digest, ConsumerPrincipal: checkpointTestConsumerPrincipal, + } + prepared, err := second.HandbackCheckpointSeal("ws_long_owner", handback, now.Add(25*time.Hour)) + if err != nil { + t.Fatalf("prepare handback after 24h lease cap: %v", err) + } + if prepared.Status != "prepared" { + t.Fatalf("late handback preparation = %+v", prepared) + } + handback.Phase = CheckpointHandbackPhaseCommit + proof, err := second.HandbackCheckpointSeal("ws_long_owner", handback, now.Add(25*time.Hour+time.Second)) + if err != nil || proof.Status != "released" { + t.Fatalf("late handback proof = %+v", proof) + } +} + +func TestCheckpointStoppedSourceOwnershipSurvivesPastDiagnosticRetention(t *testing.T) { + now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + + t.Run("destination handed back but source stayed offline", func(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + defer store.Close() + seedCheckpointFile(t, store, "ws_offline_after_handback", "/turn.txt", "destination turn\n") + digest := checkpointDigestForStore(t, store, "ws_offline_after_handback", "/") + issued, err := store.IssueCheckpointSeal("ws_offline_after_handback", CheckpointSealRequest{ + Root: "/", SessionID: "thread-offline-handback", Generation: 1, ExpectedDigest: digest, + }, now) + if err != nil { + t.Fatal(err) + } + consumed, err := store.ConsumeCheckpointSeal("ws_offline_after_handback", CheckpointSealConsumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, Generation: issued.Generation, + ConsumerIdempotencyKey: "consume-offline-handback", ConsumerPrincipal: checkpointTestConsumerPrincipal, + }, now.Add(time.Second)) + if err != nil { + t.Fatal(err) + } + handback := CheckpointSealHandbackRequest{ + Phase: CheckpointHandbackPhasePrepare, + SealID: issued.SealID, Root: "/", SessionID: issued.SessionID, Generation: issued.Generation, + ConsumedAt: consumed.ConsumedAt, ConsumerIdempotencyKey: "consume-offline-handback", + HandbackIdempotencyKey: "handback-offline-source", ExpectedDigest: digest, ConsumerPrincipal: checkpointTestConsumerPrincipal, + } + if _, err := store.HandbackCheckpointSeal("ws_offline_after_handback", handback, now.Add(2*time.Second)); err != nil { + t.Fatal(err) + } + handback.Phase = CheckpointHandbackPhaseCommit + if _, err := store.HandbackCheckpointSeal("ws_offline_after_handback", handback, now.Add(3*time.Second)); err != nil { + t.Fatal(err) + } + proof, err := store.ResumeCheckpointSeal("ws_offline_after_handback", CheckpointSealResumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, Generation: issued.Generation, + ResumeIdempotencyKey: "resume-after-25-hours", + }, now.Add(25*time.Hour)) + if err != nil || proof.Status != "source-resumed" { + t.Fatalf("resume after 25h offline proof=%+v err=%v", proof, err) + } + }) + + t.Run("unconsumed expired seal still identifies stopped source", func(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + defer store.Close() + digest := checkpointDigestForStore(t, store, "ws_offline_unconsumed", "/") + issued, err := store.IssueCheckpointSeal("ws_offline_unconsumed", CheckpointSealRequest{ + Root: "/", SessionID: "thread-offline-unconsumed", Generation: 1, ExpectedDigest: digest, TTLSeconds: 1, + }, now) + if err != nil { + t.Fatal(err) + } + proof, err := store.ResumeCheckpointSeal("ws_offline_unconsumed", CheckpointSealResumeRequest{ + SealToken: issued.SealToken, Root: "/", SessionID: issued.SessionID, Generation: issued.Generation, + ResumeIdempotencyKey: "resume-unconsumed-after-25-hours", + }, now.Add(25*time.Hour)) + if err != nil || proof.Status != "source-resumed" || proof.ConsumedAt != "" { + t.Fatalf("resume expired unconsumed seal proof=%+v err=%v", proof, err) + } + }) +} + +func TestCheckpointDigestRejectsMalformedRootAndDuplicatePath(t *testing.T) { + for _, root := range []string{"", "sessions", "/sessions/../other", "/sessions//nested"} { + if _, err := ComputeCheckpointDigest(root, nil); !errors.Is(err, ErrInvalidInput) { + t.Fatalf("root %q error = %v, want invalid input", root, err) + } + } + entries := []CheckpointDigestEntry{ + {Path: "/sessions/a", Revision: "rev_1", ContentHash: "a"}, + {Path: "/sessions/a", Revision: "rev_2", ContentHash: "b"}, + } + if _, err := ComputeCheckpointDigest("/sessions", entries); !errors.Is(err, ErrInvalidInput) { + t.Fatalf("duplicate path error = %v", err) + } +} + +func TestCheckpointSealCanonicalizesSessionBeforeGenerationCheck(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + defer store.Close() + seedCheckpointFile(t, store, "ws_canonical", "/sessions/transcript.jsonl", "first\n") + digest := checkpointDigestForStore(t, store, "ws_canonical", "/sessions") + now := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + if _, err := store.IssueCheckpointSeal("ws_canonical", CheckpointSealRequest{ + Root: "/sessions", SessionID: " thread-123 ", Generation: 1, ExpectedDigest: digest, + }, now); err != nil { + t.Fatalf("issue canonical session: %v", err) + } + if _, err := store.IssueCheckpointSeal("ws_canonical", CheckpointSealRequest{ + Root: "/sessions", SessionID: "thread-123", Generation: 1, ExpectedDigest: digest, + }, now.Add(time.Second)); !errors.Is(err, ErrCheckpointGenerationStale) { + t.Fatalf("whitespace variant replay error = %v, want stale generation", err) + } +} + +func TestCheckpointSealRejectsMalformedDigestAndTTL(t *testing.T) { + store := NewStoreWithOptions(StoreOptions{DisableWorkers: true}) + defer store.Close() + for name, req := range map[string]CheckpointSealRequest{ + "non-hex digest": { + Root: "/sessions", SessionID: "thread-123", Generation: 1, + ExpectedDigest: "sha256:zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", + }, + "negative ttl": { + Root: "/sessions", SessionID: "thread-123", Generation: 1, + ExpectedDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", TTLSeconds: -1, + }, + "oversize ttl": { + Root: "/sessions", SessionID: "thread-123", Generation: 1, + ExpectedDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", TTLSeconds: 301, + }, + } { + t.Run(name, func(t *testing.T) { + if _, err := store.IssueCheckpointSeal("ws_input", req, time.Now()); !errors.Is(err, ErrInvalidInput) { + t.Fatalf("error = %v, want invalid input", err) + } + }) + } +} + +func seedCheckpointFile(t *testing.T, store *Store, workspaceID, path, content string) { + t.Helper() + _ = seedCheckpointFileResult(t, store, workspaceID, path, content) +} + +func seedCheckpointFileResult(t *testing.T, store *Store, workspaceID, path, content string) WriteResult { + t.Helper() + result, err := store.WriteFile(WriteRequest{WorkspaceID: workspaceID, Path: path, IfMatch: "*", Content: content}) + if err != nil { + t.Fatalf("seed %s: %v", path, err) + } + return result +} + +func checkpointVerifyRequest(seal CheckpointSeal) CheckpointSealVerifyRequest { + return CheckpointSealVerifyRequest{ + SealID: seal.SealID, Root: seal.Root, SessionID: seal.SessionID, Generation: seal.Generation, + Digest: seal.Digest, WorkspaceRevision: seal.WorkspaceRevision, EventCursor: seal.EventCursor, + IssuedAt: seal.IssuedAt, ExpiresAt: seal.ExpiresAt, ConsumedAt: seal.ConsumedAt, + ConsumerPrincipal: checkpointTestConsumerPrincipal, + } +} + +func checkpointDigestForStore(t *testing.T, store *Store, workspaceID, root string) string { + t.Helper() + store.mu.RLock() + defer store.mu.RUnlock() + digest, _, _, err := store.checkpointStateLocked(workspaceID, root) + if err != nil { + t.Fatalf("checkpoint state: %v", err) + } + return digest +} + +func checkpointStateForTest(t *testing.T, store *Store, workspaceID, root string) (string, string, string) { + t.Helper() + store.mu.RLock() + defer store.mu.RUnlock() + digest, revision, cursor, err := store.checkpointStateLocked(workspaceID, root) + if err != nil { + t.Fatalf("checkpoint state: %v", err) + } + return digest, revision, cursor +} diff --git a/internal/relayfile/store.go b/internal/relayfile/store.go index 62e3f7c2..57c32843 100644 --- a/internal/relayfile/store.go +++ b/internal/relayfile/store.go @@ -553,6 +553,9 @@ type Store struct { envelopeAttempts map[string]int envelopeNextAttempt map[string]time.Time deadLetters map[string]EnvelopeDeadLetter + checkpointSeals map[string]checkpointSealRecord + checkpointGenerations map[string]uint64 + checkpointConsumerKeys map[string]checkpointConsumerBinding providerWrite ProviderWriteFunc providerWriteConfigured bool providerWriteAction ProviderWriteActionFunc @@ -631,19 +634,22 @@ type providerIngressCounter struct { } type persistedState struct { - RevCounter uint64 `json:"revCounter"` - OpCounter uint64 `json:"opCounter"` - EventCounter uint64 `json:"eventCounter"` - Workspaces map[string]*workspaceState `json:"workspaces"` - Forks map[string]*forkState `json:"forks,omitempty"` - EnvelopesByID map[string]WebhookEnvelopeRequest `json:"envelopesById"` - DeliveryIndex map[string]string `json:"deliveryIndex"` - ProcessedEnvs map[string]bool `json:"processedEnvs"` - IngressByWorkspace map[string]ingressCounter `json:"ingressByWorkspace"` - EnvelopeAttempts map[string]int `json:"envelopeAttempts"` - EnvelopeNextAttempt map[string]time.Time `json:"envelopeNextAttempt"` - DeadLetters map[string]EnvelopeDeadLetter `json:"deadLetters"` - Suppressions map[string]time.Time `json:"suppressions"` + RevCounter uint64 `json:"revCounter"` + OpCounter uint64 `json:"opCounter"` + EventCounter uint64 `json:"eventCounter"` + Workspaces map[string]*workspaceState `json:"workspaces"` + Forks map[string]*forkState `json:"forks,omitempty"` + EnvelopesByID map[string]WebhookEnvelopeRequest `json:"envelopesById"` + DeliveryIndex map[string]string `json:"deliveryIndex"` + ProcessedEnvs map[string]bool `json:"processedEnvs"` + IngressByWorkspace map[string]ingressCounter `json:"ingressByWorkspace"` + EnvelopeAttempts map[string]int `json:"envelopeAttempts"` + EnvelopeNextAttempt map[string]time.Time `json:"envelopeNextAttempt"` + DeadLetters map[string]EnvelopeDeadLetter `json:"deadLetters"` + Suppressions map[string]time.Time `json:"suppressions"` + CheckpointSeals map[string]checkpointSealRecord `json:"checkpointSeals,omitempty"` + CheckpointGenerations map[string]uint64 `json:"checkpointGenerations,omitempty"` + CheckpointConsumerKeys map[string]checkpointConsumerBinding `json:"checkpointConsumerKeys,omitempty"` } type StateBackend interface { @@ -971,6 +977,9 @@ func NewStoreWithOptions(opts StoreOptions) *Store { envelopeAttempts: map[string]int{}, envelopeNextAttempt: map[string]time.Time{}, deadLetters: map[string]EnvelopeDeadLetter{}, + checkpointSeals: map[string]checkpointSealRecord{}, + checkpointGenerations: map[string]uint64{}, + checkpointConsumerKeys: map[string]checkpointConsumerBinding{}, revisionProvenance: map[string][]revisionProvenanceEntry{}, providerWrite: writer, providerWriteConfigured: legacyWriterConfigured, @@ -4699,6 +4708,15 @@ func (s *Store) loadFromDisk() error { if snapshot.Suppressions != nil { s.suppressions = snapshot.Suppressions } + if snapshot.CheckpointSeals != nil { + s.checkpointSeals = snapshot.CheckpointSeals + } + if snapshot.CheckpointGenerations != nil { + s.checkpointGenerations = snapshot.CheckpointGenerations + } + if snapshot.CheckpointConsumerKeys != nil { + s.checkpointConsumerKeys = snapshot.CheckpointConsumerKeys + } s.revCounter = snapshot.RevCounter s.opCounter = snapshot.OpCounter s.eventCounter = snapshot.EventCounter @@ -4710,19 +4728,22 @@ func (s *Store) saveLocked() error { return nil } snapshot := persistedState{ - RevCounter: s.revCounter, - OpCounter: s.opCounter, - EventCounter: s.eventCounter, - Workspaces: s.workspaces, - Forks: s.forks, - EnvelopesByID: s.envelopesByID, - DeliveryIndex: s.deliveryIndex, - ProcessedEnvs: s.processedEnvs, - IngressByWorkspace: s.ingressByWS, - EnvelopeAttempts: s.envelopeAttempts, - EnvelopeNextAttempt: s.envelopeNextAttempt, - DeadLetters: s.deadLetters, - Suppressions: s.suppressions, + RevCounter: s.revCounter, + OpCounter: s.opCounter, + EventCounter: s.eventCounter, + Workspaces: s.workspaces, + Forks: s.forks, + EnvelopesByID: s.envelopesByID, + DeliveryIndex: s.deliveryIndex, + ProcessedEnvs: s.processedEnvs, + IngressByWorkspace: s.ingressByWS, + EnvelopeAttempts: s.envelopeAttempts, + EnvelopeNextAttempt: s.envelopeNextAttempt, + DeadLetters: s.deadLetters, + Suppressions: s.suppressions, + CheckpointSeals: s.checkpointSeals, + CheckpointGenerations: s.checkpointGenerations, + CheckpointConsumerKeys: s.checkpointConsumerKeys, } return s.stateBackend.Save(&snapshot) } diff --git a/openapi/relayfile-v1.openapi.yaml b/openapi/relayfile-v1.openapi.yaml index 0f049750..96e44fd6 100644 --- a/openapi/relayfile-v1.openapi.yaml +++ b/openapi/relayfile-v1.openapi.yaml @@ -1291,6 +1291,313 @@ paths: '500': $ref: '#/components/responses/InternalError' + /v1/workspaces/{workspaceId}/sync/checkpoint-seals: + post: + tags: [Sync] + operationId: issueCheckpointSeal + summary: Atomically certify a caller digest against durable workspace state + description: >- + Computes the canonical digest, workspace revision, and event cursor on + the server under the workspace mutation lock. The caller digest is only + a convergence assertion and is never copied into the seal. Issuance + requires sync:trigger plus fs:read and fs:write authority for the full + requested root. Generations are monotonic per workspace/root/session. + security: + - BearerAuth: [sync:trigger, fs:read, fs:write] + parameters: + - $ref: '#/components/parameters/WorkspaceId' + - $ref: '#/components/parameters/CorrelationId' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CheckpointSealRequest' + responses: + '201': + description: One-use checkpoint seal issued from durable server state + content: + application/json: + schema: + $ref: '#/components/schemas/CheckpointSeal' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '409': + description: Digest divergence, stale generation, or conflicting issuance retry identity + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '413': + $ref: '#/components/responses/PayloadTooLarge' + '429': + $ref: '#/components/responses/RateLimited' + '500': + $ref: '#/components/responses/InternalError' + + /v1/workspaces/{workspaceId}/sync/checkpoint-seals/consume: + post: + tags: [Sync] + operationId: consumeCheckpointSeal + summary: Verify and consume a one-use checkpoint seal + description: >- + Recomputes durable digest, workspace revision, and event cursor before + atomically marking the seal consumed. Expired, replayed, identity- + mismatched, or remotely stale seals fail closed. A first successful + consume is replayable with the exact same consumer idempotency key and + bound identity until handback; consumed+unreleased ownership records are + never time-purged. Response-loss retries return the same consumed result. + Reusing that key for another seal or identity conflicts, while a + different key cannot reuse an already-consumed seal. + security: + - BearerAuth: [sync:trigger, fs:read, fs:write] + parameters: + - $ref: '#/components/parameters/WorkspaceId' + - $ref: '#/components/parameters/CorrelationId' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CheckpointSealConsumeRequest' + responses: + '200': + description: Seal verified against current durable state and consumed + content: + application/json: + schema: + $ref: '#/components/schemas/CheckpointSeal' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '409': + description: Seal expired, replayed, or stale against durable state + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '413': + $ref: '#/components/responses/PayloadTooLarge' + '429': + $ref: '#/components/responses/RateLimited' + '500': + $ref: '#/components/responses/InternalError' + + /v1/workspaces/{workspaceId}/sync/checkpoint-seals/recover-consume: + post: + tags: [Sync] + operationId: recoverConsumedCheckpointSeal + summary: Recover a committed consume after response loss without the seal token + description: >- + Looks up the durable tokenless consume receipt by the exact stable + consumer idempotency key and bound workspace/root/session/generation. + The authenticated AgentName must equal the principal that performed the + consume; refreshed credentials for that same principal are accepted. + A 404 authoritatively means no matching consume committed. + security: + - BearerAuth: [sync:trigger, fs:read] + parameters: + - $ref: '#/components/parameters/WorkspaceId' + - $ref: '#/components/parameters/CorrelationId' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CheckpointSealConsumeRecoveryRequest' + responses: + '200': + description: Exact durable tokenless consumed receipt + content: + application/json: + schema: + $ref: '#/components/schemas/CheckpointSeal' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '409': + description: Consumer identity or authenticated principal conflicts + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '413': + $ref: '#/components/responses/PayloadTooLarge' + '429': + $ref: '#/components/responses/RateLimited' + '500': + $ref: '#/components/responses/InternalError' + + /v1/workspaces/{workspaceId}/sync/checkpoint-seals/verify: + post: + tags: [Sync] + operationId: verifyConsumedCheckpointSeal + summary: Re-attest a consumed checkpoint seal against current durable state + description: >- + Requires the exact consumed receipt identity and independently recomputes + the durable digest, workspace revision, and event cursor under the server + lock. The one-use sealToken is neither accepted nor returned. Destination + clients combine this durable re-attestation with Relayfile's local + canonical digest verification; callers must not reproduce the digest + algorithm outside Relayfile. The authenticated AgentName must equal the + principal that consumed the seal. + security: + - BearerAuth: [sync:trigger, fs:read] + parameters: + - $ref: '#/components/parameters/WorkspaceId' + - $ref: '#/components/parameters/CorrelationId' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CheckpointSealVerifyRequest' + responses: + '200': + description: Consumed seal identity and current durable state match exactly + content: + application/json: + schema: + $ref: '#/components/schemas/CheckpointSeal' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '409': + description: Receipt is unconsumed, stale, tampered, or no longer matches durable state + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '413': + $ref: '#/components/responses/PayloadTooLarge' + '429': + $ref: '#/components/responses/RateLimited' + '500': + $ref: '#/components/responses/InternalError' + + /v1/workspaces/{workspaceId}/sync/checkpoint-seals/handback: + post: + tags: [Sync] + operationId: handbackCheckpointSeal + summary: Drain and release consumed destination ownership + description: >- + The original destination consumer first prepares ownership handback + with its exact consumer idempotency key plus a separately stable + handback key, then commits that same prepared state after a closing + local scan. Prepare does not release ownership. The authenticated + AgentName must equal the principal that consumed it. The server + independently recomputes the final digest, workspace revision, and + event cursor under the mutation lock in both phases. + Consumed+unreleased ownership records do not expire; only an explicit + commit permits source resume. The sealToken is neither accepted nor + returned. + security: + - BearerAuth: [sync:trigger, fs:read, fs:write] + parameters: + - $ref: '#/components/parameters/WorkspaceId' + - $ref: '#/components/parameters/CorrelationId' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CheckpointSealHandbackRequest' + responses: + '200': + description: Handback durably prepared or destination ownership released with authoritative final state + content: + application/json: + schema: + $ref: '#/components/schemas/CheckpointSealOwnership' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '409': + description: Consumer mismatch, divergent final state, or conflicting handback retry + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '413': + $ref: '#/components/responses/PayloadTooLarge' + '429': + $ref: '#/components/responses/RateLimited' + '500': + $ref: '#/components/responses/InternalError' + + /v1/workspaces/{workspaceId}/sync/checkpoint-seals/resume: + post: + tags: [Sync] + operationId: resumeCheckpointSealSource + summary: Return released checkpoint ownership to the stopped source + description: >- + The stopped source presents the original one-use sealToken and a stable + resume idempotency key. Unconsumed seals can be cancelled directly; + consumed seals require an explicit destination handback first. Exact + retries are idempotent and the response never returns the sealToken. + security: + - BearerAuth: [sync:trigger, fs:read, fs:write] + parameters: + - $ref: '#/components/parameters/WorkspaceId' + - $ref: '#/components/parameters/CorrelationId' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CheckpointSealResumeRequest' + responses: + '200': + description: Source ownership resumed or unconsumed seal cancelled + content: + application/json: + schema: + $ref: '#/components/schemas/CheckpointSealOwnership' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '409': + description: Destination has not handed back or resume retry identity conflicts + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '413': + $ref: '#/components/responses/PayloadTooLarge' + '429': + $ref: '#/components/responses/RateLimited' + '500': + $ref: '#/components/responses/InternalError' + /v1/workspaces/{workspaceId}/sync/status: get: tags: [Sync] @@ -1820,6 +2127,93 @@ paths: '500': $ref: '#/components/responses/InternalError' + /v1/admin/checkpoint-seals: + get: + tags: [Admin] + operationId: getAdminCheckpointSealRetention + summary: Inspect fail-closed checkpoint seals awaiting source resume + description: >- + Returns a bearer-free per-workspace count and deterministic record list + for checkpoint seals whose source has not durably resumed. Records are + never removed merely because a lease or seal expired. + security: + - BearerAuth: [admin:read] + - BearerAuth: [admin:replay] + parameters: + - $ref: '#/components/parameters/CorrelationId' + - name: workspaceId + in: query + required: false + schema: + type: string + responses: + '200': + description: Unresumed checkpoint ownership metrics and records + content: + application/json: + schema: + $ref: '#/components/schemas/CheckpointSealRetentionSummary' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '500': + $ref: '#/components/responses/InternalError' + + /v1/admin/checkpoint-seals/{sealId}/reconcile-source: + post: + tags: [Admin] + operationId: reconcileAdminCheckpointSealSource + summary: Break-glass reconciliation after externally proving source readiness + description: >- + Requires admin:replay, an exact durable identity/status fence, a stable + idempotency key, and confirmSourceReady=true. Consumed and unreleased + destination ownership can never be overridden here. Eligible + unconsumed or destination-released records become bounded + source-resumed tombstones and their consumer-key binding is removed. + security: + - BearerAuth: [admin:replay] + parameters: + - $ref: '#/components/parameters/CorrelationId' + - name: sealId + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CheckpointSealAdminReconcileRequest' + responses: + '200': + description: Exact reconciled source-resumed tombstone + content: + application/json: + schema: + $ref: '#/components/schemas/CheckpointSealRetentionRecord' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '409': + description: Identity/status conflict or destination ownership remains unreleased + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '413': + $ref: '#/components/responses/PayloadTooLarge' + '500': + $ref: '#/components/responses/InternalError' + /v1/admin/replay/envelope/{envelopeId}: post: tags: [Admin] @@ -3318,6 +3712,351 @@ components: type: string maxLength: 1000 + CheckpointSealRetentionRecord: + type: object + additionalProperties: false + required: [sealId, workspaceId, root, sessionId, generation, ownershipStatus, issuedAt, expiresAt] + properties: + sealId: + type: string + workspaceId: + type: string + root: + type: string + sessionId: + type: string + generation: + type: integer + format: uint64 + minimum: 1 + maximum: 18446744073709551615 + ownershipStatus: + type: string + enum: [unconsumed, consumed, released, source-resumed] + issuedAt: + type: string + format: date-time + expiresAt: + type: string + format: date-time + consumedAt: + type: string + format: date-time + handbackReleasedAt: + type: string + format: date-time + sourceResumedAt: + type: string + format: date-time + adminReconciledAt: + type: string + format: date-time + + CheckpointSealRetentionSummary: + type: object + additionalProperties: false + required: [generatedAt, unresumedTotal, unresumedByWorkspace, records] + properties: + generatedAt: + type: string + format: date-time + unresumedTotal: + type: integer + minimum: 0 + unresumedByWorkspace: + type: object + additionalProperties: + type: integer + minimum: 0 + records: + type: array + items: + $ref: '#/components/schemas/CheckpointSealRetentionRecord' + + CheckpointSealAdminReconcileRequest: + type: object + additionalProperties: false + required: [workspaceId, root, sessionId, generation, expectedOwnershipStatus, reconciliationIdempotencyKey, confirmSourceReady] + properties: + workspaceId: + type: string + root: + type: string + pattern: '^/' + sessionId: + type: string + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$' + generation: + type: integer + format: uint64 + minimum: 1 + maximum: 18446744073709551615 + expectedOwnershipStatus: + type: string + enum: [unconsumed, released] + reconciliationIdempotencyKey: + type: string + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$' + confirmSourceReady: + type: boolean + const: true + + CheckpointSealRequest: + type: object + additionalProperties: false + required: [root, sessionId, generation, expectedDigest, issuanceIdempotencyKey] + properties: + root: + type: string + pattern: '^/' + sessionId: + type: string + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$' + generation: + type: integer + format: uint64 + minimum: 1 + maximum: 18446744073709551615 + expectedDigest: + type: string + pattern: '^sha256:[0-9a-f]{64}$' + description: Local convergence assertion; the server independently computes and seals its own digest. + issuanceIdempotencyKey: + type: string + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$' + description: >- + Stable source-attempt identity persisted before issuance. If a + response is lost, an exact retry preserves the seal attestation, + atomically rotates its one-use bearer, and invalidates the lost + token. Reusing the key for another request or issuer conflicts. + ttlSeconds: + type: integer + minimum: 0 + maximum: 300 + + CheckpointSealConsumeRequest: + type: object + additionalProperties: false + required: [sealToken, root, sessionId, generation, consumerIdempotencyKey] + properties: + sealToken: + type: string + minLength: 32 + root: + type: string + pattern: '^/' + sessionId: + type: string + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$' + generation: + type: integer + format: uint64 + minimum: 1 + maximum: 18446744073709551615 + consumerIdempotencyKey: + type: string + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$' + description: Stable Cloud acquire-attempt id persisted before the first consume request. + + CheckpointSealConsumeRecoveryRequest: + type: object + additionalProperties: false + required: [root, sessionId, generation, consumerIdempotencyKey] + properties: + root: + type: string + pattern: '^/' + sessionId: + type: string + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$' + generation: + type: integer + format: uint64 + minimum: 1 + maximum: 18446744073709551615 + consumerIdempotencyKey: + type: string + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$' + + CheckpointSealVerifyRequest: + type: object + additionalProperties: false + required: [sealId, root, sessionId, generation, digest, workspaceRevision, eventCursor, issuedAt, expiresAt, consumedAt] + properties: + sealId: + type: string + minLength: 1 + root: + type: string + pattern: '^/' + sessionId: + type: string + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$' + generation: + type: integer + format: uint64 + minimum: 1 + maximum: 18446744073709551615 + digest: + type: string + pattern: '^sha256:[0-9a-f]{64}$' + workspaceRevision: + type: string + pattern: '^(0|rev_[0-9]+)$' + eventCursor: + type: string + pattern: '^(0|evt_[0-9]+)$' + issuedAt: + type: string + format: date-time + expiresAt: + type: string + format: date-time + consumedAt: + type: string + format: date-time + + CheckpointSealHandbackRequest: + type: object + additionalProperties: false + required: [phase, sealId, root, sessionId, generation, consumedAt, consumerIdempotencyKey, handbackIdempotencyKey, expectedDigest] + properties: + phase: + type: string + enum: [prepare, commit] + sealId: + type: string + minLength: 1 + root: + type: string + pattern: '^/' + sessionId: + type: string + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$' + generation: + type: integer + format: uint64 + minimum: 1 + maximum: 18446744073709551615 + consumedAt: + type: string + format: date-time + consumerIdempotencyKey: + type: string + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$' + handbackIdempotencyKey: + type: string + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$' + expectedDigest: + type: string + pattern: '^sha256:[0-9a-f]{64}$' + + CheckpointSealResumeRequest: + type: object + additionalProperties: false + required: [sealToken, root, sessionId, generation, resumeIdempotencyKey] + properties: + sealToken: + type: string + minLength: 32 + root: + type: string + pattern: '^/' + sessionId: + type: string + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$' + generation: + type: integer + format: uint64 + minimum: 1 + maximum: 18446744073709551615 + resumeIdempotencyKey: + type: string + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$' + + CheckpointSealOwnership: + type: object + additionalProperties: false + required: [sealId, workspaceId, root, sessionId, generation, status, digest, workspaceRevision, eventCursor] + properties: + sealId: + type: string + workspaceId: + type: string + root: + type: string + sessionId: + type: string + generation: + type: integer + format: uint64 + minimum: 1 + maximum: 18446744073709551615 + status: + type: string + enum: [prepared, released, source-resumed] + digest: + type: string + pattern: '^sha256:[0-9a-f]{64}$' + workspaceRevision: + type: string + pattern: '^(0|rev_[0-9]+)$' + eventCursor: + type: string + pattern: '^(0|evt_[0-9]+)$' + consumedAt: + type: string + format: date-time + preparedAt: + type: string + format: date-time + releasedAt: + type: string + format: date-time + sourceResumedAt: + type: string + format: date-time + + CheckpointSeal: + type: object + additionalProperties: false + required: [sealId, workspaceId, root, sessionId, generation, digest, workspaceRevision, eventCursor, issuedAt, expiresAt] + properties: + sealId: + type: string + sealToken: + type: string + description: Opaque one-use bearer returned only at issuance; omitted after consume. + workspaceId: + type: string + root: + type: string + sessionId: + type: string + generation: + type: integer + format: uint64 + minimum: 1 + maximum: 18446744073709551615 + digest: + type: string + pattern: '^sha256:[0-9a-f]{64}$' + workspaceRevision: + type: string + pattern: '^(0|rev_[0-9]+)$' + eventCursor: + type: string + pattern: '^(0|evt_[0-9]+)$' + issuedAt: + type: string + format: date-time + expiresAt: + type: string + format: date-time + consumedAt: + type: string + format: date-time + SyncStatusResponse: type: object additionalProperties: false diff --git a/packages/sdk/parity.json b/packages/sdk/parity.json index f978017d..2c292230 100644 --- a/packages/sdk/parity.json +++ b/packages/sdk/parity.json @@ -154,6 +154,28 @@ "RelayFileClient" ] }, + { + "id": "checkpoint-seals", + "status": "ts-only", + "summary": "Authoritative checkpoint ownership, one-use consume, destination handback, and source resume for live migration.", + "tsExports": [ + "CheckpointSeal", + "CheckpointSealOwnershipStatus", + "CheckpointSealOwnership", + "CheckpointSealRetentionRecord", + "CheckpointSealRetentionSummary", + "ConsumedCheckpointSeal", + "ConsumeCheckpointSealInput", + "GetAdminCheckpointSealRetentionOptions", + "HandbackCheckpointSealInput", + "IssueCheckpointSealInput", + "RecoverConsumedCheckpointSealInput", + "ReconcileAdminCheckpointSealSourceInput", + "ResumeCheckpointSealInput", + "VerifyCheckpointSealInput" + ], + "pyExports": [] + }, { "id": "on-write-subscriptions", "status": "both", @@ -442,6 +464,7 @@ "status": "ts-only", "summary": "Node process launcher and optional native relayfile-mount binary resolution.", "tsExports": [ + "CheckpointAndSealInput", "defaultMountLauncher", "ensureRelayfileMount", "getRelayfileMountBinaryPath" diff --git a/packages/sdk/typescript/src/client.test.ts b/packages/sdk/typescript/src/client.test.ts index 0094e21d..93ff38e0 100644 --- a/packages/sdk/typescript/src/client.test.ts +++ b/packages/sdk/typescript/src/client.test.ts @@ -1416,6 +1416,227 @@ describe("RelayFileClient — existing methods", () => { }); }); + describe("checkpoint seals", () => { + const seal = { + sealId: "cps_123", + sealToken: "opaque-token", + workspaceId: "ws_acme", + root: "/sessions", + sessionId: "thread-1", + generation: 4, + digest: `sha256:${"a".repeat(64)}`, + workspaceRevision: "rev_9", + eventCursor: "evt_9", + issuedAt: "2026-08-23T12:00:00Z", + expiresAt: "2026-08-23T12:01:00Z", + }; + + it("issues using a convergence assertion while preserving the server response", async () => { + const f = mockFetch(seal, 201); + const client = makeClient(f); + await expect(client.issueCheckpointSeal({ + workspaceId: "ws_acme", + root: "/sessions", + sessionId: "thread-1", + generation: 4, + expectedDigest: seal.digest, + issuanceIdempotencyKey: "source-issue-attempt-1", + ttlSeconds: 60, + })).resolves.toEqual(seal); + expect(f.mock.calls[0]![0]).toContain("/v1/workspaces/ws_acme/sync/checkpoint-seals"); + const init = f.mock.calls[0]![1] as RequestInit; + expect(JSON.parse(init.body as string)).toEqual({ + root: "/sessions", + sessionId: "thread-1", + generation: 4, + expectedDigest: seal.digest, + issuanceIdempotencyKey: "source-issue-attempt-1", + ttlSeconds: 60, + }); + }); + + it("reuses the persisted issuance key on a response-loss retry", async () => { + const fetchImpl = vi.fn() + .mockResolvedValueOnce(new Response(JSON.stringify({ + code: "internal_error", + message: "response lost" + }), { + status: 500, + headers: { "content-type": "application/json" } + })) + .mockResolvedValueOnce(new Response(JSON.stringify(seal), { + status: 201, + headers: { "content-type": "application/json" } + })) as unknown as typeof fetch; + const client = new RelayFileClient({ + baseUrl: "https://relay.test", + token: "tok_test", + fetchImpl, + retry: { maxRetries: 1, baseDelayMs: 1, maxDelayMs: 1, jitterRatio: 0 } + }); + + await expect(client.issueCheckpointSeal({ + workspaceId: "ws_acme", + root: "/sessions", + sessionId: "thread-1", + generation: 4, + expectedDigest: seal.digest, + issuanceIdempotencyKey: "source-issue-attempt-1", + ttlSeconds: 60 + })).resolves.toEqual(seal); + + expect(fetchImpl).toHaveBeenCalledTimes(2); + const first = JSON.parse((fetchImpl.mock.calls[0]![1] as RequestInit).body as string); + const second = JSON.parse((fetchImpl.mock.calls[1]![1] as RequestInit).body as string); + expect(second).toEqual(first); + expect(second.issuanceIdempotencyKey).toBe("source-issue-attempt-1"); + }); + + it("consumes by opaque token and bound identity without a caller digest", async () => { + const consumed = { ...seal, sealToken: undefined, consumedAt: "2026-08-23T12:00:05Z" }; + const f = mockFetch(consumed); + const client = makeClient(f); + await expect(client.consumeCheckpointSeal({ + workspaceId: "ws_acme", + sealToken: "opaque-token", + root: "/sessions", + sessionId: "thread-1", + generation: 4, + consumerIdempotencyKey: "cloud-acquire-attempt-1", + })).resolves.toEqual(consumed); + expect(f.mock.calls[0]![0]).toContain("/v1/workspaces/ws_acme/sync/checkpoint-seals/consume"); + const init = f.mock.calls[0]![1] as RequestInit; + expect(JSON.parse(init.body as string)).toEqual({ + sealToken: "opaque-token", + root: "/sessions", + sessionId: "thread-1", + generation: 4, + consumerIdempotencyKey: "cloud-acquire-attempt-1", + }); + }); + + it("recovers a committed consume by stable identity without the seal token", async () => { + const consumed = { ...seal, sealToken: undefined, consumedAt: "2026-08-23T12:00:05Z" }; + const f = mockFetch(consumed); + const client = makeClient(f); + await expect(client.recoverConsumedCheckpointSeal({ + workspaceId: "ws_acme", + root: seal.root, + sessionId: seal.sessionId, + generation: seal.generation, + consumerIdempotencyKey: "cloud-acquire-attempt-1", + })).resolves.toEqual(consumed); + expect(f.mock.calls[0]![0]).toContain("/v1/workspaces/ws_acme/sync/checkpoint-seals/recover-consume"); + const init = f.mock.calls[0]![1] as RequestInit; + expect(JSON.parse(init.body as string)).toEqual({ + root: seal.root, + sessionId: seal.sessionId, + generation: seal.generation, + consumerIdempotencyKey: "cloud-acquire-attempt-1", + }); + }); + + it("re-attests only the safe consumed receipt and never sends sealToken", async () => { + const consumed = { ...seal, sealToken: undefined, consumedAt: "2026-08-23T12:00:05Z" }; + const f = mockFetch(consumed); + const client = makeClient(f); + await expect(client.verifyCheckpointSeal({ + workspaceId: "ws_acme", + receipt: consumed, + })).resolves.toEqual(consumed); + expect(f.mock.calls[0]![0]).toContain("/v1/workspaces/ws_acme/sync/checkpoint-seals/verify"); + const init = f.mock.calls[0]![1] as RequestInit; + expect(JSON.parse(init.body as string)).toEqual({ + sealId: seal.sealId, + root: seal.root, + sessionId: seal.sessionId, + generation: seal.generation, + digest: seal.digest, + workspaceRevision: seal.workspaceRevision, + eventCursor: seal.eventCursor, + issuedAt: seal.issuedAt, + expiresAt: seal.expiresAt, + consumedAt: consumed.consumedAt, + }); + }); + + it("hands back with the original consumer identity and final convergence assertion", async () => { + const consumed = { ...seal, sealToken: undefined, consumedAt: "2026-08-23T12:00:05Z" }; + const proof = { + sealId: seal.sealId, + workspaceId: seal.workspaceId, + root: seal.root, + sessionId: seal.sessionId, + generation: seal.generation, + status: "released" as const, + digest: `sha256:${"b".repeat(64)}`, + workspaceRevision: "rev_10", + eventCursor: "evt_10", + consumedAt: consumed.consumedAt, + releasedAt: "2026-08-23T13:00:00Z", + }; + const f = mockFetch(proof); + const client = makeClient(f); + await expect(client.handbackCheckpointSeal({ + workspaceId: "ws_acme", + phase: "commit", + receipt: consumed, + consumerIdempotencyKey: "cloud-acquire-attempt-1", + handbackIdempotencyKey: "cloud-handback-attempt-1", + expectedDigest: proof.digest, + })).resolves.toEqual(proof); + expect(f.mock.calls[0]![0]).toContain("/v1/workspaces/ws_acme/sync/checkpoint-seals/handback"); + const init = f.mock.calls[0]![1] as RequestInit; + expect(JSON.parse(init.body as string)).toEqual({ + phase: "commit", + sealId: seal.sealId, + root: seal.root, + sessionId: seal.sessionId, + generation: seal.generation, + consumedAt: consumed.consumedAt, + consumerIdempotencyKey: "cloud-acquire-attempt-1", + handbackIdempotencyKey: "cloud-handback-attempt-1", + expectedDigest: proof.digest, + }); + }); + + it("resumes source ownership with the original token without echoing it", async () => { + const proof = { + sealId: seal.sealId, + workspaceId: seal.workspaceId, + root: seal.root, + sessionId: seal.sessionId, + generation: seal.generation, + status: "source-resumed" as const, + digest: seal.digest, + workspaceRevision: seal.workspaceRevision, + eventCursor: seal.eventCursor, + consumedAt: "2026-08-23T12:00:05Z", + releasedAt: "2026-08-23T13:00:00Z", + sourceResumedAt: "2026-08-23T13:00:05Z", + }; + const f = mockFetch(proof); + const client = makeClient(f); + await expect(client.resumeCheckpointSeal({ + workspaceId: "ws_acme", + sealToken: "opaque-token", + root: seal.root, + sessionId: seal.sessionId, + generation: seal.generation, + resumeIdempotencyKey: "source-resume-attempt-1", + })).resolves.toEqual(proof); + expect(f.mock.calls[0]![0]).toContain("/v1/workspaces/ws_acme/sync/checkpoint-seals/resume"); + const init = f.mock.calls[0]![1] as RequestInit; + expect(JSON.parse(init.body as string)).toEqual({ + sealToken: "opaque-token", + root: seal.root, + sessionId: seal.sessionId, + generation: seal.generation, + resumeIdempotencyKey: "source-resume-attempt-1", + }); + }); + }); + // ---- exportWorkspace ---- describe("exportWorkspace", () => { it("wraps JSON export arrays in a files object", async () => { @@ -1735,6 +1956,75 @@ describe("RelayFileClient — existing methods", () => { const url = f.mock.calls[0]![0] as string; expect(url).toContain("/v1/admin/replay/envelope/env_1"); }); + + it("gets bearer-free checkpoint retention metrics by workspace", async () => { + const payload = { + generatedAt: "2026-08-23T12:00:00Z", + unresumedTotal: 1, + unresumedByWorkspace: { ws_acme: 1 }, + records: [{ + sealId: "cps_1", + workspaceId: "ws_acme", + root: "/", + sessionId: "thread-1", + generation: 1, + ownershipStatus: "unconsumed", + issuedAt: "2026-08-23T11:59:00Z", + expiresAt: "2026-08-23T12:00:00Z" + }] + } as const; + const f = mockFetch(payload); + const client = makeClient(f); + + await expect(client.getAdminCheckpointSealRetention({ + workspaceId: "ws_acme", + correlationId: "corr-retention" + })).resolves.toEqual(payload); + expect(f.mock.calls[0]![0]).toBe( + "https://relay.test/v1/admin/checkpoint-seals?workspaceId=ws_acme" + ); + }); + + it("posts an explicitly fenced checkpoint source reconciliation", async () => { + const payload = { + sealId: "cps_1", + workspaceId: "ws_acme", + root: "/", + sessionId: "thread-1", + generation: 1, + ownershipStatus: "source-resumed", + issuedAt: "2026-08-23T11:59:00Z", + expiresAt: "2026-08-23T12:00:00Z", + sourceResumedAt: "2026-08-23T12:05:00Z", + adminReconciledAt: "2026-08-23T12:05:00Z" + } as const; + const f = mockFetch(payload); + const client = makeClient(f); + + await expect(client.reconcileAdminCheckpointSealSource({ + sealId: "cps_1", + workspaceId: "ws_acme", + root: "/", + sessionId: "thread-1", + generation: 1, + expectedOwnershipStatus: "unconsumed", + reconciliationIdempotencyKey: "admin-reconcile-1", + confirmSourceReady: true + })).resolves.toEqual(payload); + expect(f.mock.calls[0]![0]).toBe( + "https://relay.test/v1/admin/checkpoint-seals/cps_1/reconcile-source" + ); + expect(JSON.parse((f.mock.calls[0]![1] as RequestInit).body as string)) + .toEqual({ + workspaceId: "ws_acme", + root: "/", + sessionId: "thread-1", + generation: 1, + expectedOwnershipStatus: "unconsumed", + reconciliationIdempotencyKey: "admin-reconcile-1", + confirmSourceReady: true + }); + }); }); describe("forks", () => { diff --git a/packages/sdk/typescript/src/client.ts b/packages/sdk/typescript/src/client.ts index 96abfe04..ea2667ba 100644 --- a/packages/sdk/typescript/src/client.ts +++ b/packages/sdk/typescript/src/client.ts @@ -3,6 +3,19 @@ import { type AdminSyncStatusResponse, type BulkWriteInput, type BulkWriteResponse, + type CheckpointSeal, + type CheckpointSealRetentionRecord, + type CheckpointSealRetentionSummary, + type GetAdminCheckpointSealRetentionOptions, + type ReconcileAdminCheckpointSealSourceInput, + type ConsumedCheckpointSeal, + type IssueCheckpointSealInput, + type ConsumeCheckpointSealInput, + type RecoverConsumedCheckpointSealInput, + type VerifyCheckpointSealInput, + type CheckpointSealOwnership, + type HandbackCheckpointSealInput, + type ResumeCheckpointSealInput, type BackendStatusResponse, type AckResponse, type CommitForkInput, @@ -1651,6 +1664,113 @@ export class RelayFileClient { return result; } + async issueCheckpointSeal(input: IssueCheckpointSealInput): Promise { + return this.request({ + method: "POST", + path: `/v1/workspaces/${encodeURIComponent(input.workspaceId)}/sync/checkpoint-seals`, + correlationId: input.correlationId, + body: { + root: input.root, + sessionId: input.sessionId, + generation: input.generation, + expectedDigest: input.expectedDigest, + issuanceIdempotencyKey: input.issuanceIdempotencyKey, + ttlSeconds: input.ttlSeconds + }, + signal: input.signal + }); + } + + async consumeCheckpointSeal(input: ConsumeCheckpointSealInput): Promise { + return this.request({ + method: "POST", + path: `/v1/workspaces/${encodeURIComponent(input.workspaceId)}/sync/checkpoint-seals/consume`, + correlationId: input.correlationId, + body: { + sealToken: input.sealToken, + root: input.root, + sessionId: input.sessionId, + generation: input.generation, + consumerIdempotencyKey: input.consumerIdempotencyKey + }, + signal: input.signal + }); + } + + async recoverConsumedCheckpointSeal(input: RecoverConsumedCheckpointSealInput): Promise { + return this.request({ + method: "POST", + path: `/v1/workspaces/${encodeURIComponent(input.workspaceId)}/sync/checkpoint-seals/recover-consume`, + correlationId: input.correlationId, + body: { + root: input.root, + sessionId: input.sessionId, + generation: input.generation, + consumerIdempotencyKey: input.consumerIdempotencyKey + }, + signal: input.signal + }); + } + + async verifyCheckpointSeal(input: VerifyCheckpointSealInput): Promise { + const receipt = input.receipt; + return this.request({ + method: "POST", + path: `/v1/workspaces/${encodeURIComponent(input.workspaceId)}/sync/checkpoint-seals/verify`, + correlationId: input.correlationId, + body: { + sealId: receipt.sealId, + root: receipt.root, + sessionId: receipt.sessionId, + generation: receipt.generation, + digest: receipt.digest, + workspaceRevision: receipt.workspaceRevision, + eventCursor: receipt.eventCursor, + issuedAt: receipt.issuedAt, + expiresAt: receipt.expiresAt, + consumedAt: receipt.consumedAt + }, + signal: input.signal + }); + } + + async handbackCheckpointSeal(input: HandbackCheckpointSealInput): Promise { + const receipt = input.receipt; + return this.request({ + method: "POST", + path: `/v1/workspaces/${encodeURIComponent(input.workspaceId)}/sync/checkpoint-seals/handback`, + correlationId: input.correlationId, + body: { + phase: input.phase, + sealId: receipt.sealId, + root: receipt.root, + sessionId: receipt.sessionId, + generation: receipt.generation, + consumedAt: receipt.consumedAt, + consumerIdempotencyKey: input.consumerIdempotencyKey, + handbackIdempotencyKey: input.handbackIdempotencyKey, + expectedDigest: input.expectedDigest + }, + signal: input.signal + }); + } + + async resumeCheckpointSeal(input: ResumeCheckpointSealInput): Promise { + return this.request({ + method: "POST", + path: `/v1/workspaces/${encodeURIComponent(input.workspaceId)}/sync/checkpoint-seals/resume`, + correlationId: input.correlationId, + body: { + sealToken: input.sealToken, + root: input.root, + sessionId: input.sessionId, + generation: input.generation, + resumeIdempotencyKey: input.resumeIdempotencyKey + }, + signal: input.signal + }); + } + async deleteFile(input: DeleteFileInput): Promise { const query = buildQuery({ path: input.path, forkId: input.forkId }); const result = await this.request({ @@ -2096,6 +2216,38 @@ export class RelayFileClient { }); } + async getAdminCheckpointSealRetention( + options: GetAdminCheckpointSealRetentionOptions = {} + ): Promise { + const query = buildQuery({ workspaceId: options.workspaceId }); + return this.request({ + method: "GET", + path: `/v1/admin/checkpoint-seals${query}`, + correlationId: options.correlationId, + signal: options.signal + }); + } + + async reconcileAdminCheckpointSealSource( + input: ReconcileAdminCheckpointSealSourceInput + ): Promise { + return this.request({ + method: "POST", + path: `/v1/admin/checkpoint-seals/${encodeURIComponent(input.sealId)}/reconcile-source`, + correlationId: input.correlationId, + body: { + workspaceId: input.workspaceId, + root: input.root, + sessionId: input.sessionId, + generation: input.generation, + expectedOwnershipStatus: input.expectedOwnershipStatus, + reconciliationIdempotencyKey: input.reconciliationIdempotencyKey, + confirmSourceReady: input.confirmSourceReady + }, + signal: input.signal + }); + } + async getSyncStatus(workspaceId: string, options: GetSyncStatusOptions = {}): Promise { const query = buildQuery({ provider: options.provider }); return this.request({ diff --git a/packages/sdk/typescript/src/index.ts b/packages/sdk/typescript/src/index.ts index a49d2f05..1c88e5c3 100644 --- a/packages/sdk/typescript/src/index.ts +++ b/packages/sdk/typescript/src/index.ts @@ -46,6 +46,7 @@ export { type AgentWorkspaceScopedInviteOptions, type ConnectIntegrationOptions, type ConnectIntegrationResult, + type CheckpointAndSealInput, type CreateWorkspaceOptions, type JoinWorkspaceOptions, type MountLauncher, @@ -151,6 +152,20 @@ export type { BulkWriteFile, BulkWriteInput, BulkWriteResponse, + CheckpointSeal, + CheckpointSealOwnershipStatus, + CheckpointSealRetentionRecord, + CheckpointSealRetentionSummary, + GetAdminCheckpointSealRetentionOptions, + ReconcileAdminCheckpointSealSourceInput, + ConsumedCheckpointSeal, + IssueCheckpointSealInput, + ConsumeCheckpointSealInput, + RecoverConsumedCheckpointSealInput, + VerifyCheckpointSealInput, + CheckpointSealOwnership, + HandbackCheckpointSealInput, + ResumeCheckpointSealInput, CancelDurableResourceSubscriptionOptions, ChangeLogQueryResult, ChangeEvent, diff --git a/packages/sdk/typescript/src/mount-launcher.test.ts b/packages/sdk/typescript/src/mount-launcher.test.ts index b99f0c81..f5df9eb6 100644 --- a/packages/sdk/typescript/src/mount-launcher.test.ts +++ b/packages/sdk/typescript/src/mount-launcher.test.ts @@ -46,6 +46,49 @@ function createMountEnv(localDir: string, mode: "poll" | "fuse" = "poll") { } } +function createCheckpointMountEnv( + localDir: string, + mode: "poll" | "fuse" = "poll" +) { + return { + ...createMountEnv(localDir, mode), + RELAYFILE_REMOTE_PATH: "/", + RELAYFILE_MOUNT_SCOPES: + "fs:read fs:write sync:trigger sync:read ops:read" + } +} + +async function writeReadyState(localDir: string, mode: "poll" | "fuse" = "poll") { + await mkdir(path.join(localDir, ".relay"), { recursive: true }) + await writeFile( + path.join(localDir, ".relay", "state.json"), + JSON.stringify({ + mode, + intervalMs: 30_000, + lastReconcileAt: new Date().toISOString(), + providers: [{ status: "ready" }] + }), + "utf8" + ) +} + +function validCheckpointSeal(overrides: Record = {}) { + return { + sealId: "seal_123", + sealToken: "one-use-secret", + workspaceId: "ws_123", + root: "/", + sessionId: "session-123", + generation: 7, + digest: `sha256:${"a".repeat(64)}`, + workspaceRevision: "rev_123", + eventCursor: "evt_123", + issuedAt: "2026-08-23T10:00:00.000Z", + expiresAt: "2026-08-23T10:01:00.000Z", + ...overrides + } +} + describe("default mount launcher", () => { beforeEach(() => { vi.restoreAllMocks() @@ -247,6 +290,325 @@ describe("default mount launcher", () => { } }) + it("stops a ready poll daemon before running the one-shot checkpoint command", async () => { + const tempRoot = await mkdtemp( + path.join(os.tmpdir(), "relayfile-default-launcher-checkpoint-") + ) + const localDir = path.join(tempRoot, "mirror") + const daemon = new FakeChildProcess() + const checkpoint = new FakeChildProcess() + const events: string[] = [] + const spawnImpl = vi.fn().mockImplementation((_command, args: string[]) => { + if (spawnImpl.mock.calls.length === 1) { + events.push("daemon-started") + return daemon as never + } + events.push(`checkpoint-started:${daemon.killSignals.join(",")}`) + queueMicrotask(() => { + checkpoint.stdout.write(`${JSON.stringify(validCheckpointSeal())}\n`) + checkpoint.exitCode = 0 + checkpoint.emit("exit", 0, null) + }) + return checkpoint as never + }) + const launcher = createDefaultMountLauncher({ spawnImpl }) + + try { + await writeReadyState(localDir) + const instance = await launcher.start({ + env: createCheckpointMountEnv(localDir), + readyTimeoutMs: 50 + }) + await instance.ready + + await expect(instance.checkpointAndSeal?.({ + sessionId: "", + generation: 7 + })).rejects.toMatchObject({ code: "checkpoint_seal_invalid_input" }) + await expect(instance.checkpointAndSeal?.({ + generation: 7 + } as never)).rejects.toMatchObject({ code: "checkpoint_seal_invalid_input" }) + expect(daemon.killSignals).toEqual([]) + + const seal = await instance.checkpointAndSeal?.({ + sessionId: "session-123", + generation: 7, + timeoutMs: 250, + ttlSeconds: 45 + }) + + expect(seal).toMatchObject(validCheckpointSeal()) + expect(events).toEqual(["daemon-started", "checkpoint-started:SIGTERM"]) + expect(spawnImpl).toHaveBeenNthCalledWith( + 2, + expect.any(String), + [ + "--checkpoint-and-seal", + "--checkpoint-session", "session-123", + "--checkpoint-generation", "7", + "--checkpoint-seal-ttl", "45s", + "--timeout", "250ms" + ], + expect.objectContaining({ + cwd: localDir, + stdio: ["ignore", "pipe", "pipe"] + }) + ) + } finally { + await rm(tempRoot, { recursive: true, force: true }) + } + }) + + it("rejects FUSE checkpoints before unmounting the source", async () => { + const tempRoot = await mkdtemp( + path.join(os.tmpdir(), "relayfile-default-launcher-checkpoint-fuse-") + ) + const localDir = path.join(tempRoot, "mirror") + const daemon = new FakeChildProcess() + const spawnImpl = vi.fn().mockReturnValue(daemon as never) + const launcher = createDefaultMountLauncher({ spawnImpl }) + + try { + await writeReadyState(localDir, "fuse") + const instance = await launcher.start({ + env: createCheckpointMountEnv(localDir, "fuse"), + readyTimeoutMs: 50 + }) + await instance.ready + + await expect(instance.checkpointAndSeal?.({ + sessionId: "session-123", + generation: 7 + })).rejects.toMatchObject({ code: "checkpoint_seal_mode_unavailable" }) + expect(instance.stopped).toBe(false) + expect(daemon.killSignals).toEqual([]) + expect(spawnImpl).toHaveBeenCalledTimes(1) + await instance.stop() + } finally { + await rm(tempRoot, { recursive: true, force: true }) + } + }) + + it("rejects pull-only checkpoints before stopping the source", async () => { + const tempRoot = await mkdtemp( + path.join(os.tmpdir(), "relayfile-default-launcher-checkpoint-pull-only-") + ) + const localDir = path.join(tempRoot, "mirror") + const daemon = new FakeChildProcess() + const spawnImpl = vi.fn().mockReturnValue(daemon as never) + const launcher = createDefaultMountLauncher({ spawnImpl }) + + try { + await writeReadyState(localDir) + const instance = await launcher.start({ + env: { + ...createCheckpointMountEnv(localDir), + RELAYFILE_MOUNT_SYNC_MODE: "pull-only" + }, + readyTimeoutMs: 50 + }) + await instance.ready + + await expect(instance.checkpointAndSeal?.({ + sessionId: "session-123", + generation: 7 + })).rejects.toMatchObject({ code: "checkpoint_seal_mode_unavailable" }) + expect(instance.stopped).toBe(false) + expect(daemon.killSignals).toEqual([]) + expect(spawnImpl).toHaveBeenCalledTimes(1) + await instance.stop() + } finally { + await rm(tempRoot, { recursive: true, force: true }) + } + }) + + it("rejects a non-root checkpoint before stopping the source", async () => { + const tempRoot = await mkdtemp( + path.join(os.tmpdir(), "relayfile-default-launcher-checkpoint-root-") + ) + const localDir = path.join(tempRoot, "mirror") + const daemon = new FakeChildProcess() + const spawnImpl = vi.fn().mockReturnValue(daemon as never) + const launcher = createDefaultMountLauncher({ spawnImpl }) + + try { + await writeReadyState(localDir) + const instance = await launcher.start({ + env: { + ...createMountEnv(localDir), + RELAYFILE_MOUNT_SCOPES: + "fs:read fs:write sync:trigger sync:read ops:read" + }, + readyTimeoutMs: 50 + }) + await instance.ready + + await expect(instance.checkpointAndSeal?.({ + sessionId: "session-123", + generation: 7 + })).rejects.toMatchObject({ code: "checkpoint_seal_root_unavailable" }) + expect(instance.stopped).toBe(false) + expect(daemon.killSignals).toEqual([]) + expect(spawnImpl).toHaveBeenCalledTimes(1) + await instance.stop() + } finally { + await rm(tempRoot, { recursive: true, force: true }) + } + }) + + it("checks inherited effective mount configuration before stopping the source", async () => { + const tempRoot = await mkdtemp( + path.join(os.tmpdir(), "relayfile-default-launcher-checkpoint-inherited-root-") + ) + const localDir = path.join(tempRoot, "mirror") + const daemon = new FakeChildProcess() + const spawnImpl = vi.fn().mockReturnValue(daemon as never) + const launcher = createDefaultMountLauncher({ spawnImpl }) + const previousRemotePath = process.env.RELAYFILE_REMOTE_PATH + const { RELAYFILE_REMOTE_PATH: _omittedRemotePath, ...inheritedEnv } = + createCheckpointMountEnv(localDir) + + try { + process.env.RELAYFILE_REMOTE_PATH = "/inherited-scope" + await writeReadyState(localDir) + const instance = await launcher.start({ + env: inheritedEnv, + readyTimeoutMs: 50 + }) + await instance.ready + + await expect(instance.checkpointAndSeal?.({ + sessionId: "session-123", + generation: 7 + })).rejects.toMatchObject({ code: "checkpoint_seal_root_unavailable" }) + expect(instance.stopped).toBe(false) + expect(daemon.killSignals).toEqual([]) + expect(spawnImpl).toHaveBeenCalledTimes(1) + await instance.stop() + } finally { + if (previousRemotePath === undefined) { + delete process.env.RELAYFILE_REMOTE_PATH + } else { + process.env.RELAYFILE_REMOTE_PATH = previousRemotePath + } + await rm(tempRoot, { recursive: true, force: true }) + } + }) + + it.each([ + ["missing ops", "fs:read fs:write sync:trigger"], + ["missing trigger", "fs:read fs:write ops:read"], + [ + "narrow filesystem grants", + "relayfile:fs:read:/notion/** relayfile:fs:write:/notion/** sync:trigger ops:read" + ], + [ + "exact root-node grants", + "relayfile:fs:read:/ relayfile:fs:write:/ sync:trigger ops:read" + ], + [ + "colon-bearing narrow filesystem grants", + "relayfile:fs:read:/:secret relayfile:fs:write:/:secret sync:trigger ops:read" + ] + ])("rejects %s checkpoint scopes before stopping the source", async (_name, scopes) => { + const tempRoot = await mkdtemp( + path.join(os.tmpdir(), "relayfile-default-launcher-checkpoint-scopes-") + ) + const localDir = path.join(tempRoot, "mirror") + const daemon = new FakeChildProcess() + const spawnImpl = vi.fn().mockReturnValue(daemon as never) + const launcher = createDefaultMountLauncher({ spawnImpl }) + + try { + await writeReadyState(localDir) + const instance = await launcher.start({ + env: { + ...createCheckpointMountEnv(localDir), + RELAYFILE_MOUNT_SCOPES: scopes + }, + readyTimeoutMs: 50 + }) + await instance.ready + + await expect(instance.checkpointAndSeal?.({ + sessionId: "session-123", + generation: 7 + })).rejects.toMatchObject({ code: "checkpoint_seal_scope_unavailable" }) + expect(instance.stopped).toBe(false) + expect(daemon.killSignals).toEqual([]) + expect(spawnImpl).toHaveBeenCalledTimes(1) + await instance.stop() + } finally { + await rm(tempRoot, { recursive: true, force: true }) + } + }) + + it("rejects a server seal bound to a different workspace", async () => { + const tempRoot = await mkdtemp( + path.join(os.tmpdir(), "relayfile-default-launcher-checkpoint-binding-") + ) + const localDir = path.join(tempRoot, "mirror") + const daemon = new FakeChildProcess() + const checkpoint = new FakeChildProcess() + const spawnImpl = vi.fn().mockImplementation(() => { + if (spawnImpl.mock.calls.length === 1) return daemon as never + queueMicrotask(() => { + checkpoint.stdout.write(JSON.stringify(validCheckpointSeal({ workspaceId: "ws_other" }))) + checkpoint.exitCode = 0 + checkpoint.emit("exit", 0, null) + }) + return checkpoint as never + }) + const launcher = createDefaultMountLauncher({ spawnImpl }) + + try { + await writeReadyState(localDir) + const instance = await launcher.start({ + env: createCheckpointMountEnv(localDir), + readyTimeoutMs: 50 + }) + await instance.ready + + await expect(instance.checkpointAndSeal?.({ + sessionId: "session-123", + generation: 7 + })).rejects.toMatchObject({ code: "checkpoint_seal_invalid_output" }) + } finally { + await rm(tempRoot, { recursive: true, force: true }) + } + }) + + it("kills a checkpoint command that exceeds its deadline", async () => { + const tempRoot = await mkdtemp( + path.join(os.tmpdir(), "relayfile-default-launcher-checkpoint-timeout-") + ) + const localDir = path.join(tempRoot, "mirror") + const daemon = new FakeChildProcess() + const checkpoint = new FakeChildProcess() + const spawnImpl = vi.fn().mockImplementation(() => + spawnImpl.mock.calls.length === 1 ? daemon as never : checkpoint as never + ) + const launcher = createDefaultMountLauncher({ spawnImpl }) + + try { + await writeReadyState(localDir) + const instance = await launcher.start({ + env: createCheckpointMountEnv(localDir), + readyTimeoutMs: 50 + }) + await instance.ready + + await expect(instance.checkpointAndSeal?.({ + sessionId: "session-123", + generation: 7, + timeoutMs: 5 + })).rejects.toMatchObject({ code: "checkpoint_seal_timeout" }) + expect(checkpoint.killSignals).toContain("SIGKILL") + } finally { + await rm(tempRoot, { recursive: true, force: true }) + } + }) + it("translates fuse-mode startup failures to MountModeUnavailableError", async () => { const tempRoot = await mkdtemp( path.join(os.tmpdir(), "relayfile-default-launcher-fuse-") diff --git a/packages/sdk/typescript/src/mount-launcher.ts b/packages/sdk/typescript/src/mount-launcher.ts index 6688c6e3..c2195b31 100644 --- a/packages/sdk/typescript/src/mount-launcher.ts +++ b/packages/sdk/typescript/src/mount-launcher.ts @@ -23,18 +23,23 @@ import { import type { MountLocalLayout, MountLauncher, + CheckpointCapableMountLauncherInstance, MountLauncherInstance, MountLauncherStart, MountMode, MountSyncMode, + CheckpointAndSealInput, MountedWorkspaceStatus, ReadMountedWorkspaceStatusInput } from "./setup-types.js" +import type { CheckpointSeal } from "./types.js" const DEFAULT_READY_POLL_INTERVAL_MS = 250 const DEFAULT_STOP_TIMEOUT_MS = 10_000 const LOG_ROTATION_MAX_BYTES = 10 * 1024 * 1024 const LOG_ROTATION_FILES = 3 +const DEFAULT_CHECKPOINT_TIMEOUT_MS = 30_000 +const MAX_CHECKPOINT_OUTPUT_BYTES = 1024 * 1024 const FUSE_UNAVAILABLE_SIGNATURE = "fuse mode is not available in this build" interface DefaultMountLauncherOptions { @@ -149,13 +154,18 @@ async function startRelayfileMount( outputBuffer, input, localDir: mountLocalDir, + command, + effectiveEnv, + cwd: input.cwd ?? mountLocalDir, + spawnImpl: options.spawnImpl ?? spawn, now: options.now ?? Date.now, readyPollIntervalMs: options.readyPollIntervalMs ?? DEFAULT_READY_POLL_INTERVAL_MS }) } -class RelayfileMountProcessInstance implements MountLauncherInstance { +class RelayfileMountProcessInstance + implements CheckpointCapableMountLauncherInstance { readonly pid?: number readonly ready: Promise @@ -167,10 +177,19 @@ class RelayfileMountProcessInstance implements MountLauncherInstance { private readonly localDir: string private readonly now: () => number private readonly readyPollIntervalMs: number + private readonly command: string + private readonly effectiveEnv: NodeJS.ProcessEnv + private readonly cwd: string + private readonly spawnImpl: typeof spawn private exited = false private stopping?: Promise private readyResolved = false + private checkpointPromise?: Promise + + get stopped(): boolean { + return this.exited + } constructor(input: { child: ChildProcess @@ -179,6 +198,10 @@ class RelayfileMountProcessInstance implements MountLauncherInstance { outputBuffer: string[] input: MountLauncherStart localDir: string + command: string + effectiveEnv: NodeJS.ProcessEnv + cwd: string + spawnImpl: typeof spawn now: () => number readyPollIntervalMs: number }) { @@ -188,6 +211,10 @@ class RelayfileMountProcessInstance implements MountLauncherInstance { this.outputBuffer = input.outputBuffer this.input = input.input this.localDir = input.localDir + this.command = input.command + this.effectiveEnv = input.effectiveEnv + this.cwd = input.cwd + this.spawnImpl = input.spawnImpl this.pid = input.child.pid ?? undefined this.now = input.now this.readyPollIntervalMs = input.readyPollIntervalMs @@ -223,6 +250,63 @@ class RelayfileMountProcessInstance implements MountLauncherInstance { await this.stopping } + async checkpointAndSeal(input: CheckpointAndSealInput): Promise { + this.validateCheckpointPreconditions(input) + if (!this.checkpointPromise) { + this.checkpointPromise = this.performCheckpointAndSeal(input) + } + return this.checkpointPromise + } + + private validateCheckpointPreconditions(input: CheckpointAndSealInput): void { + validateCheckpointInput(input) + if (normalizeMountMode(this.effectiveEnv.RELAYFILE_MOUNT_MODE) !== "poll") { + throw new RelayfileSetupError( + "checkpointAndSeal currently requires a poll-mode mount; a FUSE mount must remain running until daemon checkpoint IPC is available.", + "checkpoint_seal_mode_unavailable" + ) + } + if (normalizeMountSyncMode(this.effectiveEnv.RELAYFILE_MOUNT_SYNC_MODE) === "pull-only") { + throw new RelayfileSetupError( + "checkpointAndSeal cannot drain a pull-only mount.", + "checkpoint_seal_mode_unavailable" + ) + } + if (normalizeRemotePath(this.effectiveEnv.RELAYFILE_REMOTE_PATH ?? "/") !== "/") { + throw new RelayfileSetupError( + "checkpointAndSeal v1 requires a full-root (/) mount.", + "checkpoint_seal_root_unavailable" + ) + } + const scopes = (this.effectiveEnv.RELAYFILE_MOUNT_SCOPES ?? "") + .split(/\s+/) + .map((scope) => scope.trim()) + .filter(Boolean) + if ( + !scopeGrants(scopes, "sync", "trigger") || + !scopeGrants(scopes, "ops", "read") || + !scopeGrantsFullRoot(scopes, "read") || + !scopeGrantsFullRoot(scopes, "write") + ) { + throw new RelayfileSetupError( + "checkpointAndSeal requires sync:trigger, ops:read, and full-root fs:read/fs:write scopes.", + "checkpoint_seal_scope_unavailable" + ) + } + } + + private async performCheckpointAndSeal(input: CheckpointAndSealInput): Promise { + await this.ready + await this.stop() + return runCheckpointSealProcess({ + command: this.command, + cwd: this.cwd, + env: this.effectiveEnv, + spawnImpl: this.spawnImpl, + input + }) + } + private async waitForReady(): Promise { const startedAt = this.now() const timeoutAt = startedAt + this.input.readyTimeoutMs @@ -294,6 +378,224 @@ class RelayfileMountProcessInstance implements MountLauncherInstance { } } +async function runCheckpointSealProcess(input: { + command: string + cwd: string + env: NodeJS.ProcessEnv + spawnImpl: typeof spawn + input: CheckpointAndSealInput +}): Promise { + const { sessionId, generation, timeoutMs, ttlSeconds } = validateCheckpointInput(input.input) + const workspaceId = input.env.RELAYFILE_WORKSPACE?.trim() ?? "" + const root = normalizeRemotePath(input.env.RELAYFILE_REMOTE_PATH) + + const args = [ + "--checkpoint-and-seal", + "--checkpoint-session", sessionId, + "--checkpoint-generation", String(generation), + "--checkpoint-seal-ttl", `${ttlSeconds}s`, + "--timeout", `${timeoutMs}ms` + ] + const child = input.spawnImpl(input.command, args, { + cwd: input.cwd, + env: input.env, + stdio: ["ignore", "pipe", "pipe"] + }) + + return new Promise((resolve, reject) => { + let stdout = "" + let stderr = "" + let settled = false + let timer: ReturnType | undefined + const finish = (error?: Error, seal?: CheckpointSeal): void => { + if (settled) return + settled = true + if (timer) clearTimeout(timer) + input.input.signal?.removeEventListener("abort", onAbort) + if (error) reject(error) + else resolve(seal!) + } + const append = (current: string, chunk: unknown): string => { + const next = current + (Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk)) + if (Buffer.byteLength(next) > MAX_CHECKPOINT_OUTPUT_BYTES) { + child.kill("SIGKILL") + finish(new RelayfileSetupError("relayfile-mount checkpoint output exceeded 1 MiB.", "checkpoint_seal_invalid_output")) + } + return next + } + child.stdout?.on("data", (chunk) => { stdout = append(stdout, chunk) }) + child.stderr?.on("data", (chunk) => { stderr = append(stderr, chunk) }) + child.once("error", (error) => { + finish(new RelayfileSetupError(`relayfile-mount checkpoint failed to start: ${error.message}`, "checkpoint_seal_failed")) + }) + child.once("exit", (code, signal) => { + if (settled) return + if (code !== 0) { + const detail = stderr.trim().slice(-2_000) + finish(new RelayfileSetupError( + `relayfile-mount checkpoint failed (${signal ?? `exit ${code ?? "unknown"}`})${detail ? `: ${detail}` : ""}`, + "checkpoint_seal_failed" + )) + return + } + let parsed: unknown + try { + parsed = JSON.parse(stdout.trim()) + } catch { + finish(new RelayfileSetupError("relayfile-mount returned malformed checkpoint JSON.", "checkpoint_seal_invalid_output")) + return + } + try { + finish( + undefined, + validateCheckpointSeal(parsed, workspaceId, root, sessionId, generation) + ) + } catch (error) { + finish(error instanceof Error ? error : new Error(String(error))) + } + }) + const onAbort = (): void => { + child.kill("SIGKILL") + finish(new CloudAbortError("checkpointAndSeal")) + } + input.input.signal?.addEventListener("abort", onAbort, { once: true }) + timer = setTimeout(() => { + child.kill("SIGKILL") + finish(new RelayfileSetupError(`checkpointAndSeal timed out after ${timeoutMs}ms.`, "checkpoint_seal_timeout")) + }, timeoutMs) + timer.unref?.() + }) +} + +function validateCheckpointInput(input: CheckpointAndSealInput): { + sessionId: string + generation: number + timeoutMs: number + ttlSeconds: number +} { + const sessionId = + typeof input.sessionId === "string" ? input.sessionId.trim() : "" + const generation = input.generation + const timeoutMs = input.timeoutMs ?? DEFAULT_CHECKPOINT_TIMEOUT_MS + const ttlSeconds = input.ttlSeconds ?? 60 + if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/.test(sessionId)) { + throw new RelayfileSetupError( + "checkpointAndSeal requires a valid sessionId.", + "checkpoint_seal_invalid_input" + ) + } + if ( + !Number.isSafeInteger(generation) || generation <= 0 || + !Number.isSafeInteger(timeoutMs) || timeoutMs <= 0 || + !Number.isSafeInteger(ttlSeconds) || ttlSeconds <= 0 || ttlSeconds > 300 + ) { + throw new RelayfileSetupError( + "checkpointAndSeal requires positive safe generation/timeout values and ttlSeconds <= 300.", + "checkpoint_seal_invalid_input" + ) + } + if (input.signal?.aborted) { + throw new CloudAbortError("checkpointAndSeal") + } + return { sessionId, generation, timeoutMs, ttlSeconds } +} + +function scopeGrants( + scopes: string[], + resource: string, + action: string +): boolean { + const bare = `${resource}:${action}` + return scopes.some((scope) => { + if (scope === bare) return true + const [plane, grantedResource, grantedAction] = scope.split(":", 4) + return ( + (plane === "relayfile" || plane === "*") && + (grantedResource === resource || grantedResource === "*") && + (grantedAction === action || grantedAction === "*") + ) + }) +} + +function scopeGrantsFullRoot( + scopes: string[], + action: "read" | "write" +): boolean { + const relevantNarrowScopes: string[] = [] + for (const scope of scopes) { + const parsed = parseStructuredScope(scope) + if (!parsed) continue + const { plane, resource, action: grantedAction, path: scopePath } = parsed + const actionMatches = + grantedAction === action || + grantedAction === "*" || + grantedAction === "manage" + if (!actionMatches) continue + if ( + (plane === "relayfile" || plane === "*") && + (resource === "fs" || resource === "*") + ) { + if (scopePath === undefined || scopePath.trim() === "*") return true + relevantNarrowScopes.push(scopePath.trim()) + continue + } + if (plane === "workspace") { + if (scopePath === undefined || scopePath.trim() === "*") return true + relevantNarrowScopes.push(scopePath.trim()) + } + } + if (relevantNarrowScopes.some((scopePath) => scopePath === "/**")) { + return true + } + return relevantNarrowScopes.length === 0 && scopes.includes(`fs:${action}`) +} + +function parseStructuredScope(scope: string): { + plane: string + resource: string + action: string + path?: string +} | null { + const first = scope.indexOf(":") + const second = first < 0 ? -1 : scope.indexOf(":", first + 1) + if (first < 1 || second <= first + 1) return null + const third = scope.indexOf(":", second + 1) + return { + plane: scope.slice(0, first), + resource: scope.slice(first + 1, second), + action: scope.slice(second + 1, third < 0 ? undefined : third), + path: third < 0 ? undefined : scope.slice(third + 1) + } +} + +function validateCheckpointSeal( + value: unknown, + workspaceId: string, + root: string, + sessionId: string, + generation: number +): CheckpointSeal { + if (!value || typeof value !== "object") { + throw new RelayfileSetupError("relayfile-mount returned an invalid checkpoint seal.", "checkpoint_seal_invalid_output") + } + const seal = value as Partial + const strings = [seal.sealId, seal.sealToken, seal.workspaceId, seal.root, seal.digest, seal.workspaceRevision, seal.eventCursor, seal.issuedAt, seal.expiresAt] + if ( + strings.some((field) => typeof field !== "string") || + seal.sealToken!.trim() === "" || + seal.workspaceId !== workspaceId || + normalizeRemotePath(seal.root) !== root || + seal.sessionId !== sessionId || + seal.generation !== generation || + !seal.digest!.startsWith("sha256:") || + Number.isNaN(Date.parse(seal.issuedAt!)) || + Number.isNaN(Date.parse(seal.expiresAt!)) + ) { + throw new RelayfileSetupError("relayfile-mount returned an unbound or incomplete checkpoint seal.", "checkpoint_seal_invalid_output") + } + return seal as CheckpointSeal +} + function pipeChildOutput( child: ChildProcess, logStream: NodeJS.WritableStream, diff --git a/packages/sdk/typescript/src/setup-types.ts b/packages/sdk/typescript/src/setup-types.ts index 01998136..f0b74fa9 100644 --- a/packages/sdk/typescript/src/setup-types.ts +++ b/packages/sdk/typescript/src/setup-types.ts @@ -1,5 +1,6 @@ import type { WorkspaceHandle } from "./setup.js" import type { AccessTokenProvider } from "./client.js" +import type { CheckpointSeal } from "./types.js" export const WORKSPACE_INTEGRATION_PROVIDERS = [ "github", @@ -148,6 +149,14 @@ export interface MountedWorkspaceStatus { pendingConflicts?: number } +export interface CheckpointAndSealInput { + sessionId: string + generation: number + timeoutMs?: number + ttlSeconds?: number + signal?: AbortSignal +} + export interface ReadMountedWorkspaceStatusInput { localDir: string workspaceId: string @@ -173,6 +182,7 @@ export interface MountedWorkspaceHandle { env(): Record status(): Promise + checkpointAndSeal(input: CheckpointAndSealInput): Promise stop(): Promise } @@ -199,13 +209,33 @@ export interface MountLauncherStart { background?: boolean } -export interface MountLauncherInstance { +interface MountLauncherInstanceBase { pid?: number ready: Promise status(): Promise stop(): Promise } +/** A launcher without checkpoint support may report stop state optionally. */ +export interface BasicMountLauncherInstance extends MountLauncherInstanceBase { + readonly stopped?: boolean + checkpointAndSeal?: never +} + +/** + * Checkpoint-capable launchers must report whether the physical daemon stopped. + * The SDK uses this signal to retire a failed checkpoint safely. + */ +export interface CheckpointCapableMountLauncherInstance + extends MountLauncherInstanceBase { + readonly stopped: boolean + checkpointAndSeal(input: CheckpointAndSealInput): Promise +} + +export type MountLauncherInstance = + | BasicMountLauncherInstance + | CheckpointCapableMountLauncherInstance + export interface MountLauncher { start(input: MountLauncherStart): Promise } diff --git a/packages/sdk/typescript/src/setup.test.ts b/packages/sdk/typescript/src/setup.test.ts index dd181e5f..6145d2b0 100644 --- a/packages/sdk/typescript/src/setup.test.ts +++ b/packages/sdk/typescript/src/setup.test.ts @@ -112,6 +112,7 @@ function createLauncherStub( pid: number ready: Promise status: ReturnType + checkpointAndSeal: ReturnType stop: ReturnType } readyControl: ReturnType> @@ -119,6 +120,7 @@ function createLauncherStub( const readyControl = deferred() const instance = { pid: 4321, + stopped: false, ready: readyControl.promise, status: vi.fn().mockResolvedValue({ ready: true, @@ -127,6 +129,7 @@ function createLauncherStub( suggestedRefreshAt: null, ...status } satisfies MountedWorkspaceStatus), + checkpointAndSeal: vi.fn().mockResolvedValue(validCheckpointSeal()), stop: vi.fn().mockResolvedValue(undefined) } const launcher: MountLauncher = { @@ -135,6 +138,22 @@ function createLauncherStub( return { launcher, instance, readyControl } } +function validCheckpointSeal() { + return { + sealId: "seal_123", + sealToken: "one-use-secret", + workspaceId: "ws_123", + root: "/notion", + sessionId: "session-123", + generation: 7, + digest: `sha256:${"a".repeat(64)}`, + workspaceRevision: "rev_123", + eventCursor: "evt_123", + issuedAt: "2026-08-23T10:00:00.000Z", + expiresAt: "2026-08-23T10:01:00.000Z" + } +} + async function flushPromises(times = 3): Promise { for (let index = 0; index < times; index += 1) { await Promise.resolve() @@ -1248,6 +1267,99 @@ describe("RelayfileSetup", () => { } }) + it("delegates checkpointAndSeal to the owned launcher and retires readiness", async () => { + const tempRoot = await mkdtemp( + path.join(os.tmpdir(), "relayfile-sdk-mount-checkpoint-") + ) + const localDir = path.join(tempRoot, "mirror") + queueFetch( + makeJoinResponse("rf_jwt_joined"), + makeMountSessionResponse() + ) + const { launcher, instance, readyControl } = createLauncherStub() + + try { + const setup = new RelayfileSetup() + const workspace = await setup.joinWorkspace("ws_123") + readyControl.resolve() + const handle = await setup.mountWorkspace({ + workspace, + localDir, + remotePath: "/notion", + mode: "poll", + launcher + }) + + await expect(handle.checkpointAndSeal({ + sessionId: "session-123", + generation: 7, + ttlSeconds: 60 + })).resolves.toEqual(validCheckpointSeal()) + expect(instance.checkpointAndSeal).toHaveBeenCalledWith({ + sessionId: "session-123", + generation: 7, + ttlSeconds: 60 + }) + expect(handle.ready).toBe(false) + } finally { + await rm(tempRoot, { recursive: true, force: true }) + } + }) + + it("rejects a checkpoint-capable launcher that cannot prove it stopped", async () => { + const tempRoot = await mkdtemp( + path.join(os.tmpdir(), "relayfile-sdk-mount-checkpoint-contract-") + ) + const localDir = path.join(tempRoot, "mirror") + const checkpointAndSeal = vi.fn().mockResolvedValue(validCheckpointSeal()) + const stop = vi.fn().mockResolvedValue(undefined) + const launcher = { + start: vi.fn().mockResolvedValue({ + pid: 4321, + ready: Promise.resolve(), + status: vi.fn().mockResolvedValue({ + ready: true, + mode: "poll", + expiresAt: null, + suggestedRefreshAt: null + } satisfies MountedWorkspaceStatus), + checkpointAndSeal, + stop + }) + } as unknown as MountLauncher + queueFetch( + makeJoinResponse("rf_jwt_joined"), + makeMountSessionResponse() + ) + + try { + const setup = new RelayfileSetup() + const workspace = await setup.joinWorkspace("ws_123") + const handle = await setup.mountWorkspace({ + workspace, + localDir, + remotePath: "/notion", + mode: "poll", + launcher + }) + + await expect(handle.checkpointAndSeal({ + sessionId: "session-123", + generation: 7, + ttlSeconds: 60 + })).rejects.toMatchObject({ + code: "checkpoint_seal_launcher_contract_invalid" + }) + expect(checkpointAndSeal).not.toHaveBeenCalled() + expect(handle.ready).toBe(true) + + await handle.stop() + expect(stop).toHaveBeenCalledTimes(1) + } finally { + await rm(tempRoot, { recursive: true, force: true }) + } + }) + it.each(["pull-only", "write-only"] as const)( "passes explicit local layout and %s sync mode only to the local mount launcher", async (syncMode) => { @@ -1648,6 +1760,192 @@ describe("RelayfileSetup", () => { } }) + it.each([ + { + name: "invalid input", + mode: "poll" as const, + syncMode: "mirror" as const, + code: "checkpoint_seal_invalid_input" + }, + { + name: "FUSE mode", + mode: "fuse" as const, + syncMode: "mirror" as const, + code: "checkpoint_seal_mode_unavailable" + }, + { + name: "pull-only mode", + mode: "poll" as const, + syncMode: "pull-only" as const, + code: "checkpoint_seal_mode_unavailable" + } + ])( + "ensureMountedWorkspace preserves its shared handle when checkpoint rejects $name before stop", + async ({ mode, syncMode, code }) => { + const tempRoot = await mkdtemp( + path.join(os.tmpdir(), "relayfile-sdk-ensure-checkpoint-preflight-") + ) + const localDir = path.join(tempRoot, "mirror") + const fetchMock = queueFetch( + makeJoinResponse("rf_jwt_joined"), + makeMountSessionResponse({ mode, syncMode }) + ) + let stopped = false + const instance = { + pid: 4321, + get stopped() { + return stopped + }, + ready: Promise.resolve(), + status: vi.fn().mockResolvedValue({ + ready: true, + mode, + expiresAt: null, + suggestedRefreshAt: null + } satisfies MountedWorkspaceStatus), + checkpointAndSeal: vi.fn().mockRejectedValue(Object.assign( + new Error("checkpoint rejected before daemon stop"), + { code } + )), + stop: vi.fn().mockImplementation(async () => { + stopped = true + }) + } + const launcher: MountLauncher = { + start: vi.fn().mockResolvedValue(instance) + } + const ensureInput = { + localDir, + mode, + syncMode, + verifyProvider: false, + launcher + } + + try { + const setup = new RelayfileSetup() + const workspace = await setup.joinWorkspace("ws_123") + const first = await setup.ensureMountedWorkspace({ + workspace, + ...ensureInput + }) + const checkpointInput = { + sessionId: + mode === "poll" && syncMode === "mirror" ? "" : "session-123", + generation: 7 + } + + await expect(first.checkpointAndSeal(checkpointInput)) + .rejects.toMatchObject({ code }) + + expect(first.ready).toBe(true) + expect(instance.stop).not.toHaveBeenCalled() + const reused = await setup.ensureMountedWorkspace({ + workspace, + ...ensureInput + }) + expect(reused).toBe(first) + expect(launcher.start).toHaveBeenCalledTimes(1) + expect(fetchMock).toHaveBeenCalledTimes(2) + await expect(reused.checkpointAndSeal(checkpointInput)) + .rejects.toMatchObject({ code }) + expect(instance.checkpointAndSeal).toHaveBeenCalledTimes(2) + expect(reused.ready).toBe(true) + + await reused.stop() + expect(instance.stop).toHaveBeenCalledTimes(1) + } finally { + await rm(tempRoot, { recursive: true, force: true }) + } + } + ) + + it("ensureMountedWorkspace evicts after a checkpoint failure with a confirmed daemon stop", async () => { + const tempRoot = await mkdtemp( + path.join(os.tmpdir(), "relayfile-sdk-ensure-checkpoint-stopped-") + ) + const localDir = path.join(tempRoot, "mirror") + const fetchMock = queueFetch( + makeJoinResponse("rf_jwt_joined"), + makeMountSessionResponse(), + makeMountSessionResponse({ + relayfileToken: "rf_mount_after_checkpoint_failure" + }) + ) + let firstStopped = false + const instances = [ + { + pid: 4321, + get stopped() { + return firstStopped + }, + ready: Promise.resolve(), + status: vi.fn().mockResolvedValue({ + ready: true, + mode: "poll", + expiresAt: null, + suggestedRefreshAt: null + }), + checkpointAndSeal: vi.fn().mockImplementation(async () => { + firstStopped = true + throw Object.assign(new Error("seal command failed after stop"), { + code: "checkpoint_seal_failed" + }) + }), + stop: vi.fn().mockImplementation(async () => { + firstStopped = true + }) + }, + { + pid: 4322, + stopped: false, + ready: Promise.resolve(), + status: vi.fn().mockResolvedValue({ + ready: true, + mode: "poll", + expiresAt: null, + suggestedRefreshAt: null + }), + checkpointAndSeal: vi.fn().mockResolvedValue(validCheckpointSeal()), + stop: vi.fn().mockResolvedValue(undefined) + } + ] + const launcher: MountLauncher = { + start: vi.fn() + .mockResolvedValueOnce(instances[0]) + .mockResolvedValueOnce(instances[1]) + } + + try { + const setup = new RelayfileSetup() + const workspace = await setup.joinWorkspace("ws_123") + const first = await setup.ensureMountedWorkspace({ + workspace, + localDir, + verifyProvider: false, + launcher + }) + await expect(first.checkpointAndSeal({ + sessionId: "session-123", + generation: 7 + })).rejects.toMatchObject({ code: "checkpoint_seal_failed" }) + expect(first.ready).toBe(false) + + const restarted = await setup.ensureMountedWorkspace({ + workspace, + localDir, + verifyProvider: false, + launcher + }) + expect(restarted).not.toBe(first) + expect(launcher.start).toHaveBeenCalledTimes(2) + expect(fetchMock).toHaveBeenCalledTimes(3) + await restarted.stop() + } finally { + await rm(tempRoot, { recursive: true, force: true }) + } + }) + it("lets the creating waiter abort without cancelling shared physical startup", async () => { const tempRoot = await mkdtemp( path.join(os.tmpdir(), "relayfile-sdk-ensure-waiter-abort-") @@ -1972,6 +2270,74 @@ describe("RelayfileSetup", () => { } }) + it("fences an in-flight health refresh before checkpoint can stop the mount", async () => { + vi.useFakeTimers() + vi.setSystemTime("2026-05-09T10:00:00.000Z") + const tempRoot = await mkdtemp( + path.join(os.tmpdir(), "relayfile-sdk-supervised-checkpoint-refresh-") + ) + const localDir = path.join(tempRoot, "mirror") + queueFetch( + makeJoinResponse("rf_jwt_joined"), + makeMountSessionResponse({ + suggestedRefreshAt: "2026-05-09T11:00:00.000Z" + }) + ) + let releaseStatus!: (status: MountedWorkspaceStatus) => void + const pendingStatus = new Promise((resolve) => { + releaseStatus = resolve + }) + let stopped = false + const checkpointAndSeal = vi.fn().mockImplementation(async () => { + stopped = true + return validCheckpointSeal() + }) + const launcher: MountLauncher = { + start: vi.fn().mockResolvedValue({ + pid: 4321, + get stopped() { + return stopped + }, + ready: Promise.resolve(), + status: vi.fn().mockReturnValue(pendingStatus), + checkpointAndSeal, + stop: vi.fn().mockImplementation(async () => { + stopped = true + }) + }) + } + + try { + const setup = new RelayfileSetup({ retry: { maxRetries: 0 } }) + const handle = await setup.ensureMountedWorkspace({ + workspaceId: "ws_123", + localDir, + verifyProvider: false, + launcher, + healthCheckIntervalMs: 1_000 + }) + + await vi.advanceTimersByTimeAsync(1_000) + const checkpoint = handle.checkpointAndSeal({ + sessionId: "session-refresh-race", + generation: 8 + }) + releaseStatus({ + ready: false, + mode: "poll", + expiresAt: null, + suggestedRefreshAt: null + }) + + await expect(checkpoint).resolves.toEqual(validCheckpointSeal()) + expect(checkpointAndSeal).toHaveBeenCalledTimes(1) + expect(launcher.start).toHaveBeenCalledTimes(1) + expect(handle.ready).toBe(false) + } finally { + await rm(tempRoot, { recursive: true, force: true }) + } + }) + it("mounted handle status reads .relay/state.json and keeps expiresAt fields stable", async () => { const tempRoot = await mkdtemp( path.join(os.tmpdir(), "relayfile-sdk-status-state-file-") diff --git a/packages/sdk/typescript/src/setup.ts b/packages/sdk/typescript/src/setup.ts index c2521dd0..6ad4c576 100644 --- a/packages/sdk/typescript/src/setup.ts +++ b/packages/sdk/typescript/src/setup.ts @@ -32,6 +32,7 @@ import { type AgentWorkspaceScopedInviteOptions, type ConnectIntegrationOptions, type ConnectIntegrationResult, + type CheckpointAndSealInput, type CreateWorkspaceOptions, type JoinWorkspaceOptions, type MountLauncher, @@ -55,6 +56,7 @@ import { type WorkspaceIntegrationProvider, type WorkspacePermissions } from "./setup-types.js" +import type { CheckpointSeal } from "./types.js" import { RELAYFILE_SDK_VERSION } from "./version.js" export { RELAYFILE_SDK_VERSION } from "./version.js" @@ -76,6 +78,17 @@ const DEFAULT_MOUNT_SYNC_MODE: MountSyncMode = "mirror" const DEFAULT_MOUNT_HEALTH_INTERVAL_MS = 30_000 const DEFAULT_MOUNT_REFRESH_RETRY_BASE_MS = 1_000 const DEFAULT_MOUNT_REFRESH_RETRY_MAX_MS = 60_000 +const confirmedMountStop = Symbol("confirmedMountStop") + +interface ConfirmedMountStopHandle { + [confirmedMountStop](): boolean +} + +function hasConfirmedMountStop(handle: MountedWorkspaceHandle): boolean { + const candidate = handle as MountedWorkspaceHandle & + Partial + return candidate[confirmedMountStop]?.() === true +} const TOKEN_REFRESH_AGE_MS = 55 * 60 * 1000 const nodeOnlyMountLauncher: MountLauncher = { @@ -1155,6 +1168,8 @@ class MountedWorkspaceHandleImpl implements MountedWorkspaceHandle { private readySnapshot = true private stopPromise?: Promise + private checkpointPromise?: Promise + private checkpointStopped = false constructor(input: { mountSession: MountSessionResult @@ -1220,6 +1235,48 @@ class MountedWorkspaceHandleImpl implements MountedWorkspaceHandle { await this.stopPromise } + async checkpointAndSeal(input: CheckpointAndSealInput): Promise { + if (!this.checkpointPromise) { + if (this.probeOnly || !this.launcherInstance?.checkpointAndSeal) { + throw new RelayfileSetupError( + "checkpointAndSeal requires a locally managed relayfile-mount launcher.", + "checkpoint_seal_unavailable" + ) + } + if (typeof this.launcherInstance.stopped !== "boolean") { + throw new RelayfileSetupError( + "checkpointAndSeal requires a launcher that reports physical stop state.", + "checkpoint_seal_launcher_contract_invalid" + ) + } + this.checkpointPromise = this.performCheckpointAndSeal(input) + } + return this.checkpointPromise + } + + [confirmedMountStop](): boolean { + return this.checkpointStopped + } + + private async performCheckpointAndSeal(input: CheckpointAndSealInput): Promise { + try { + const seal = await this.launcherInstance!.checkpointAndSeal!(input) + this.checkpointStopped = true + this.readySnapshot = false + return seal + } catch (error) { + if (this.launcherInstance?.stopped === true) { + this.checkpointStopped = true + this.readySnapshot = false + } else { + // A launcher precondition can reject before touching the daemon. Keep + // both readiness and retryability when no physical stop was confirmed. + this.checkpointPromise = undefined + } + throw error + } + } + private async performStop(): Promise { this.readySnapshot = false if (this.probeOnly || !this.launcherInstance) { @@ -1278,6 +1335,16 @@ class SharedMountedWorkspaceHandle implements MountedWorkspaceHandle { return this.mounted.status() } + async checkpointAndSeal(input: CheckpointAndSealInput): Promise { + try { + return await this.mounted.checkpointAndSeal(input) + } finally { + if (hasConfirmedMountStop(this.mounted)) { + this.onStopped() + } + } + } + async stop(): Promise { if (!this.stopPromise) this.stopPromise = this.performStop() await this.stopPromise @@ -1304,6 +1371,8 @@ class SupervisedMountedWorkspaceHandle implements MountedWorkspaceHandle { private refreshPromise?: Promise private stopPromise?: Promise private stopped = false + private checkpointing = false + private supervisorEpoch = 0 private supervisorReady = true private refreshAttempts = 0 @@ -1378,8 +1447,51 @@ class SupervisedMountedWorkspaceHandle implements MountedWorkspaceHandle { await this.stopPromise } + async checkpointAndSeal(input: CheckpointAndSealInput): Promise { + this.checkpointing = true + this.supervisorEpoch += 1 + this.supervisorReady = false + if (this.timer) clearTimeout(this.timer) + this.timer = undefined + await this.refreshPromise?.catch(() => {}) + try { + const seal = await this.mounted.checkpointAndSeal(input) + this.retireAfterCheckpointStop() + return seal + } catch (error) { + if (hasConfirmedMountStop(this.mounted)) { + this.retireAfterCheckpointStop() + } else { + // Validation/mode failures occur before the daemon is stopped. Restore + // the supervisor with a fresh epoch and leave the shared ensured + // handle reusable. Work from the fenced epoch cannot replace it. + this.checkpointing = false + this.supervisorEpoch += 1 + this.supervisorReady = this.mounted.ready + this.schedule() + } + throw error + } + } + + [confirmedMountStop](): boolean { + return this.stopped && hasConfirmedMountStop(this.mounted) + } + + private retireAfterCheckpointStop(): void { + this.stopped = true + this.checkpointing = false + this.supervisorEpoch += 1 + this.supervisorReady = false + this.signal?.removeEventListener("abort", this.handleAbort) + if (this.timer) clearTimeout(this.timer) + this.timer = undefined + } + private async performStop(): Promise { this.stopped = true + this.checkpointing = false + this.supervisorEpoch += 1 this.supervisorReady = false this.signal?.removeEventListener("abort", this.handleAbort) if (this.timer) clearTimeout(this.timer) @@ -1393,7 +1505,8 @@ class SupervisedMountedWorkspaceHandle implements MountedWorkspaceHandle { } private schedule(delayOverrideMs?: number): void { - if (this.stopped || this.timer) return + if (this.stopped || this.checkpointing || this.timer) return + const epoch = this.supervisorEpoch const suggestedRefreshAtMs = Date.parse(this.mounted.suggestedRefreshAt ?? "") const untilRefresh = Number.isFinite(suggestedRefreshAtMs) ? Math.max(0, suggestedRefreshAtMs - Date.now()) @@ -1401,20 +1514,24 @@ class SupervisedMountedWorkspaceHandle implements MountedWorkspaceHandle { const delayMs = delayOverrideMs ?? Math.min(this.healthCheckIntervalMs, untilRefresh) this.timer = setTimeout(() => { this.timer = undefined - this.refreshPromise = this.checkAndRefresh().finally(() => { - this.refreshPromise = undefined + if (!this.isActiveEpoch(epoch)) return + const refresh = this.checkAndRefresh(epoch) + const tracked = refresh.finally(() => { + if (this.refreshPromise === tracked) this.refreshPromise = undefined }) + this.refreshPromise = tracked }, Math.max(1_000, delayMs)) this.timer.unref?.() } - private async checkAndRefresh(): Promise { - if (this.stopped) return + private async checkAndRefresh(epoch: number): Promise { + if (!this.isActiveEpoch(epoch)) return try { const suggestedRefreshAtMs = Date.parse(this.mounted.suggestedRefreshAt ?? "") const refreshDue = Number.isFinite(suggestedRefreshAtMs) && Date.now() >= suggestedRefreshAtMs if (!refreshDue) { const status = await this.mounted.status() + if (!this.isActiveEpoch(epoch)) return if (status.ready) { this.supervisorReady = true this.refreshAttempts = 0 @@ -1422,8 +1539,8 @@ class SupervisedMountedWorkspaceHandle implements MountedWorkspaceHandle { return } } - await this.replaceMount() - if (this.stopped) return + await this.replaceMount(epoch) + if (!this.isActiveEpoch(epoch)) return this.supervisorReady = true this.refreshAttempts = 0 this.emit({ @@ -1434,7 +1551,7 @@ class SupervisedMountedWorkspaceHandle implements MountedWorkspaceHandle { }) this.schedule() } catch (error) { - if (this.stopped) return + if (!this.isActiveEpoch(epoch)) return this.supervisorReady = false this.refreshAttempts += 1 const retryInMs = Math.min( @@ -1453,19 +1570,23 @@ class SupervisedMountedWorkspaceHandle implements MountedWorkspaceHandle { } } - private async replaceMount(): Promise { + private async replaceMount(epoch: number): Promise { const previous = this.mounted - if (this.stopped) return + if (!this.isActiveEpoch(epoch)) return await previous.stop().catch(() => {}) - if (this.stopped) return + if (!this.isActiveEpoch(epoch)) return const replacement = await this.launch() - if (this.stopped) { + if (!this.isActiveEpoch(epoch)) { await replacement.stop().catch(() => {}) return } this.mounted = replacement } + private isActiveEpoch(epoch: number): boolean { + return !this.stopped && !this.checkpointing && epoch === this.supervisorEpoch + } + private emit(event: MountSupervisorEvent): void { void Promise.resolve(this.onEvent?.(event)).catch(() => {}) } diff --git a/packages/sdk/typescript/src/types.ts b/packages/sdk/typescript/src/types.ts index b9f1505e..5d1ae848 100644 --- a/packages/sdk/typescript/src/types.ts +++ b/packages/sdk/typescript/src/types.ts @@ -140,6 +140,152 @@ export interface BulkWriteResponse { correlationId: string; } +export interface CheckpointSeal { + sealId: string; + sealToken?: string; + workspaceId: string; + root: string; + sessionId: string; + generation: number; + digest: string; + workspaceRevision: string; + eventCursor: string; + issuedAt: string; + expiresAt: string; + consumedAt?: string; +} + +export type CheckpointSealOwnershipStatus = + | "unconsumed" + | "consumed" + | "released" + | "source-resumed"; + +export interface CheckpointSealRetentionRecord { + sealId: string; + workspaceId: string; + root: string; + sessionId: string; + generation: number; + ownershipStatus: CheckpointSealOwnershipStatus; + issuedAt: string; + expiresAt: string; + consumedAt?: string; + handbackReleasedAt?: string; + sourceResumedAt?: string; + adminReconciledAt?: string; +} + +export interface CheckpointSealRetentionSummary { + generatedAt: string; + unresumedTotal: number; + unresumedByWorkspace: Record; + records: CheckpointSealRetentionRecord[]; +} + +export interface GetAdminCheckpointSealRetentionOptions { + workspaceId?: string; + correlationId?: string; + signal?: AbortSignal; +} + +export interface ReconcileAdminCheckpointSealSourceInput { + sealId: string; + workspaceId: string; + root: string; + sessionId: string; + generation: number; + expectedOwnershipStatus: "unconsumed" | "released"; + reconciliationIdempotencyKey: string; + confirmSourceReady: true; + correlationId?: string; + signal?: AbortSignal; +} + +export interface ConsumedCheckpointSeal extends Omit { + sealToken?: never; + consumedAt: string; +} + +export interface IssueCheckpointSealInput { + workspaceId: string; + root: string; + sessionId: string; + generation: number; + expectedDigest: string; + /** Stable per-attempt key. Response-loss retries rotate the lost bearer safely. */ + issuanceIdempotencyKey: string; + ttlSeconds?: number; + correlationId?: string; + signal?: AbortSignal; +} + +export interface ConsumeCheckpointSealInput { + workspaceId: string; + sealToken: string; + root: string; + sessionId: string; + generation: number; + consumerIdempotencyKey: string; + correlationId?: string; + signal?: AbortSignal; +} + +export interface RecoverConsumedCheckpointSealInput { + workspaceId: string; + root: string; + sessionId: string; + generation: number; + consumerIdempotencyKey: string; + correlationId?: string; + signal?: AbortSignal; +} + +export interface VerifyCheckpointSealInput { + workspaceId: string; + receipt: ConsumedCheckpointSeal; + correlationId?: string; + signal?: AbortSignal; +} + +export interface CheckpointSealOwnership { + sealId: string; + workspaceId: string; + root: string; + sessionId: string; + generation: number; + status: "prepared" | "released" | "source-resumed"; + digest: string; + workspaceRevision: string; + eventCursor: string; + consumedAt?: string; + preparedAt?: string; + releasedAt?: string; + sourceResumedAt?: string; +} + +export interface HandbackCheckpointSealInput { + workspaceId: string; + phase: "prepare" | "commit"; + receipt: ConsumedCheckpointSeal; + consumerIdempotencyKey: string; + handbackIdempotencyKey: string; + expectedDigest: string; + correlationId?: string; + signal?: AbortSignal; +} + +export interface ResumeCheckpointSealInput { + workspaceId: string; + sealToken: string; + root: string; + sessionId: string; + generation: number; + resumeIdempotencyKey: string; + correlationId?: string; + signal?: AbortSignal; +} + export interface FileQueryItem { path: string; revision: string;