closed: client bulk request hardening (not the #180 fix)

.trajectories/completed/2026-08/traj_hhthwa7xxikz.json

@@ -0,0 +1,78 @@
+{
+  "id": "traj_hhthwa7xxikz",
+  "version": 1,
+  "task": {
+    "title": "Fix #445 SQL variable limit data loss",
+    "source": {
+      "system": "plain",
+      "id": "AgentWorkforce/relayfile#445"
+    }
+  },
+  "status": "completed",
+  "startedAt": "2026-08-24T09:12:49.140Z",
+  "completedAt": "2026-08-24T09:35:03.059Z",
+  "agents": [
+    {
+      "name": "default",
+      "role": "lead",
+      "joinedAt": "2026-08-24T09:28:38.146Z"
+    }
+  ],
+  "chapters": [
+    {
+      "id": "chap_iocnaxzx9nvs",
+      "title": "Work",
+      "agentName": "default",
+      "startedAt": "2026-08-24T09:28:38.146Z",
+      "endedAt": "2026-08-24T09:35:03.059Z",
+      "events": [
+        {
+          "ts": 1787563718150,
+          "type": "decision",
+          "content": "Cap bulk writes at 200 files across durable outbox and client boundaries: Cap bulk writes at 200 files across durable outbox and client boundaries",
+          "raw": {
+            "question": "Cap bulk writes at 200 files across durable outbox and client boundaries",
+            "chosen": "Cap bulk writes at 200 files across durable outbox and client boundaries",
+            "alternatives": [],
+            "reasoning": "The outbox maintenance path chunks only by 8 MiB, so 1,454 small records can be reassembled into one request. A fixed 200-file cap stays well below SQLite's conservative 999-variable limit, preserves byte chunking, and protects foreground, maintenance, and direct client callers."
+          },
+          "significance": "high"
+        },
+        {
+          "ts": 1787564102654,
+          "type": "reflection",
+          "content": "Reproduced the production signature with 1,001 small workspace files; count chunking is green across Go mount, direct Go HTTP, TypeScript SDK, and sync/async Python SDK paths.",
+          "raw": {
+            "focalPoints": [
+              "production-reproduction",
+              "sibling-paths",
+              "response-aggregation"
+            ],
+            "adjustments": "Kept the existing byte cap and added the same 200-file ceiling at both durable-outbox and direct client boundaries; left circuit breaker and HTTP contract unchanged.",
+            "confidence": 0.9
+          },
+          "significance": "high",
+          "tags": [
+            "focal:production-reproduction",
+            "focal:sibling-paths",
+            "focal:response-aggregation",
+            "confidence:0.9"
+          ]
+        }
+      ]
+    }
+  ],
+  "retrospective": {
+    "summary": "Fixed #445 by capping bulk write requests at 200 files, covering durable mount outbox maintenance and all public client bulk-write methods; added over-limit red/green regression coverage and verified all Go/TypeScript/Python suites.",
+    "approach": "Standard approach",
+    "confidence": 0.9
+  },
+  "commits": [],
+  "filesChanged": [],
+  "projectId": "/Users/khaliqgant/Projects/AgentWorkforce/relayfile/.lanes/relayfile-445-sql-variable-limit",
+  "tags": [],
+  "_trace": {
+    "startRef": "fecb9ce404a47e25d18303d5c4ae80139eba2562",
+    "endRef": "fecb9ce404a47e25d18303d5c4ae80139eba2562"
+  }
+}

.trajectories/completed/2026-08/traj_hhthwa7xxikz.md

@@ -0,0 +1,33 @@
+# Trajectory: Fix #445 SQL variable limit data loss
+
+> **Status:** ✅ Completed
+> **Task:** AgentWorkforce/relayfile#445
+> **Confidence:** 90%
+> **Started:** August 24, 2026 at 11:12 AM
+> **Completed:** August 24, 2026 at 11:35 AM
+
+---
+
+## Summary
+
+Fixed #445 by capping bulk write requests at 200 files, covering durable mount outbox maintenance and all public client bulk-write methods; added over-limit red/green regression coverage and verified all Go/TypeScript/Python suites.
+
+**Approach:** Standard approach
+
+---
+
+## Key Decisions
+
+### Cap bulk writes at 200 files across durable outbox and client boundaries
+- **Chose:** Cap bulk writes at 200 files across durable outbox and client boundaries
+- **Reasoning:** The outbox maintenance path chunks only by 8 MiB, so 1,454 small records can be reassembled into one request. A fixed 200-file cap stays well below SQLite's conservative 999-variable limit, preserves byte chunking, and protects foreground, maintenance, and direct client callers.
+
+---
+
+## Chapters
+
+### 1. Work
+*Agent: default*
+
+- Cap bulk writes at 200 files across durable outbox and client boundaries: Cap bulk writes at 200 files across durable outbox and client boundaries
+- Reproduced the production signature with 1,001 small workspace files; count chunking is green across Go mount, direct Go HTTP, TypeScript SDK, and sync/async Python SDK paths.

.trajectories/index.json

@@ -1,6 +1,6 @@
 {
   "version": 1,
-  "lastUpdated": "2026-08-23T14:45:33.239Z",
+  "lastUpdated": "2026-08-24T09:35:03.379Z",
   "trajectories": {
     "traj_4pvrlmqfnzng": {
       "title": "Review PR #278 in AgentWorkforce/relayfile",
@@ -197,6 +197,13 @@
       "startedAt": "2026-08-23T14:42:40.801Z",
       "completedAt": "2026-08-23T14:45:33.103Z",
       "path": ".trajectories/completed/2026-08/traj_2ng1fbz1wsxb.json"
+    },
+    "traj_hhthwa7xxikz": {
+      "title": "Fix #445 SQL variable limit data loss",
+      "status": "completed",
+      "startedAt": "2026-08-24T09:12:49.140Z",
+      "completedAt": "2026-08-24T09:35:03.059Z",
+      "path": ".trajectories/completed/2026-08/traj_hhthwa7xxikz.json"
     }
   }
 }

internal/mountsync/http_client_test.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -460,3 +461,73 @@ func TestHTTPClientWriteFilesBulkRejectsEmptyBatch(t *testing.T) {
 		t.Fatalf("expected empty batch to fail before issuing a request, got %d calls", atomic.LoadInt32(&calls))
 	}
 }
+
+func TestHTTPClientWriteFilesBulkChunksLargeCallerSlice(t *testing.T) {
+	const fileCount = 1001
+
+	var calls atomic.Int32
+	var largestBatch atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1/workspaces/ws_large_bulk/fs/bulk" {
+			t.Fatalf("unexpected path %q", r.URL.Path)
+		}
+		var payload struct {
+			Files []BulkWriteFile `json:"files"`
+		}
+		if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
+			t.Fatalf("decode request body failed: %v", err)
+		}
+		batchSize := int32(len(payload.Files))
+		for previous := largestBatch.Load(); batchSize > previous; previous = largestBatch.Load() {
+			if largestBatch.CompareAndSwap(previous, batchSize) {
+				break
+			}
+		}
+		call := calls.Add(1)
+		if len(payload.Files) > 999 {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusInternalServerError)
+			_ = json.NewEncoder(w).Encode(map[string]string{
+				"code":    "internal_error",
+				"message": "too many SQL variables at offset 412: SQLITE_ERROR",
+			})
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(BulkWriteResponse{
+			Written: len(payload.Files),
+			Results: []BulkWriteResult{{
+				Path:     payload.Files[0].Path,
+				Revision: fmt.Sprintf("rev_%d", call),
+			}},
+			CorrelationID: fmt.Sprintf("corr_%d", call),
+		})
+	}))
+	defer server.Close()
+
+	files := make([]BulkWriteFile, 0, fileCount)
+	for idx := 0; idx < fileCount; idx++ {
+		files = append(files, BulkWriteFile{
+			Path:    fmt.Sprintf("/notion/Docs/File%04d.md", idx+1),
+			Content: "small",
+		})
+	}
+	client := NewHTTPClient(server.URL, "token", server.Client())
+	response, err := client.WriteFilesBulk(context.Background(), "ws_large_bulk", files)
+	if err != nil {
+		t.Fatalf("large bulk write failed: %v", err)
+	}
+	if response.Written != fileCount || response.ErrorCount != 0 {
+		t.Fatalf("unexpected aggregate response: %+v", response)
+	}
+	wantCalls := int32((fileCount + maxBulkWriteFilesPerRequest - 1) / maxBulkWriteFilesPerRequest)
+	if got := calls.Load(); got != wantCalls {
+		t.Fatalf("bulk request count = %d, want %d", got, wantCalls)
+	}
+	if got := len(response.Results); got != int(wantCalls) {
+		t.Fatalf("aggregate result count = %d, want %d", got, wantCalls)
+	}
+	if got := largestBatch.Load(); got > maxBulkWriteFilesPerRequest {
+		t.Fatalf("largest bulk request = %d, safe cap = %d", got, maxBulkWriteFilesPerRequest)
+	}
+}

internal/mountsync/syncer.go

@@ -77,7 +77,15 @@ func (e *SchemaValidationError) Is(target error) bool {
 	return target == ErrSchemaValidation
 }
 
-const defaultBulkFlushThreshold = 256
+const (
+	defaultBulkFlushThreshold = 256
+	// maxBulkWriteFilesPerRequest keeps caller-sized /fs/bulk payloads well
+	// below SQLite's conservative 999-variable build limit. The cloud storage
+	// layer may bind one variable per file while committing a bulk mutation, so
+	// the byte-size cap alone is insufficient for large workspaces of small
+	// files.
+	maxBulkWriteFilesPerRequest = 200
+)
 
 // The WebSocket server may inline file payloads up to 1 MiB. nhooyr's client
 // default is only 32 KiB, which would disconnect real-time mounts whenever a
@@ -755,6 +763,25 @@ func (c *HTTPClient) WriteFilesBulk(ctx context.Context, workspaceID string, fil
 	if len(files) == 0 {
 		return BulkWriteResponse{}, ErrEmptyBulkWrite
 	}
+	var combined BulkWriteResponse
+	for start := 0; start < len(files); start += maxBulkWriteFilesPerRequest {
+		end := min(start+maxBulkWriteFilesPerRequest, len(files))
+		response, err := c.writeFilesBulkBatch(ctx, workspaceID, files[start:end])
+		if err != nil {
+			return combined, err
+		}
+		combined.Written += response.Written
+		combined.ErrorCount += response.ErrorCount
+		combined.Errors = append(combined.Errors, response.Errors...)
+		combined.Results = append(combined.Results, response.Results...)
+		if strings.TrimSpace(response.CorrelationID) != "" {
+			combined.CorrelationID = response.CorrelationID
+		}
+	}
+	return combined, nil
+}
+
+func (c *HTTPClient) writeFilesBulkBatch(ctx context.Context, workspaceID string, files []BulkWriteFile) (BulkWriteResponse, error) {
 	body := struct {
 		Files []BulkWriteFile `json:"files"`
 	}{
@@ -3882,14 +3909,13 @@ func chunkOutboxRecords(records []outboxRecord, maxBytes int64) [][]outboxRecord
 	if len(records) == 0 {
 		return nil
 	}
-	if maxBytes <= 0 {
-		return [][]outboxRecord{records}
-	}
 	chunks := make([][]outboxRecord, 0, 1)
 	current := make([]outboxRecord, 0, len(records))
 	for _, record := range records {
 		candidate := append(append([]outboxRecord(nil), current...), record)
-		if len(current) > 0 && bulkWriteRequestSize(outboxRecordsAsBulkFiles(candidate)) > maxBytes {
+		exceedsFileLimit := len(current) >= maxBulkWriteFilesPerRequest
+		exceedsByteLimit := maxBytes > 0 && bulkWriteRequestSize(outboxRecordsAsBulkFiles(candidate)) > maxBytes
+		if len(current) > 0 && (exceedsFileLimit || exceedsByteLimit) {
 			chunks = append(chunks, append([]outboxRecord(nil), current...))
 			current = current[:0]
 		}

internal/mountsync/syncer_test.go

@@ -5510,17 +5510,77 @@ func TestBulkWrite_ChunkAtThreshold(t *testing.T) {
 		t.Fatalf("chunked sync failed: %v", err)
 	}
 
-	if client.bulkWriteCalls != 2 {
-		t.Fatalf("expected two bulk write calls, got %d", client.bulkWriteCalls)
+	if client.bulkWriteCalls != 3 {
+		t.Fatalf("expected three bulk write calls, got %d", client.bulkWriteCalls)
+	}
+	if got := len(client.bulkWriteBatches); got != 3 {
+		t.Fatalf("expected three recorded bulk batches, got %d", got)
+	}
+	if got := len(client.bulkWriteBatches[0]); got != 200 {
+		t.Fatalf("expected first batch size 200, got %d", got)
+	}
+	if got := len(client.bulkWriteBatches[1]); got != 56 {
+		t.Fatalf("expected second batch size 56, got %d", got)
+	}
+	if got := len(client.bulkWriteBatches[2]); got != 44 {
+		t.Fatalf("expected third batch size 44, got %d", got)
+	}
+}
+
+func TestBulkWrite_LargeWorkspaceStaysBelowSQLVariableLimit(t *testing.T) {
+	const (
+		fileCount        = 1001
+		sqlVariableLimit = 999
+	)
+
+	client := &fakeClient{
+		files: map[string]RemoteFile{},
+		bulkWriteResponseFunc: func(_ context.Context, _ string, files []BulkWriteFile) (BulkWriteResponse, error) {
+			if len(files) > sqlVariableLimit {
+				return BulkWriteResponse{}, &HTTPError{
+					StatusCode: http.StatusInternalServerError,
+					Code:       "internal_error",
+					Message:    "too many SQL variables at offset 412: SQLITE_ERROR",
+				}
+			}
+			return BulkWriteResponse{}, nil
+		},
+	}
+	localDir := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(localDir, "Docs"), 0o755); err != nil {
+		t.Fatalf("mkdir docs failed: %v", err)
 	}
-	if got := len(client.bulkWriteBatches); got != 2 {
-		t.Fatalf("expected two recorded bulk batches, got %d", got)
+	for idx := 0; idx < fileCount; idx++ {
+		name := fmt.Sprintf("File%04d.md", idx+1)
+		path := filepath.Join(localDir, "Docs", name)
+		if err := os.WriteFile(path, []byte(name), 0o644); err != nil {
+			t.Fatalf("seed file %s failed: %v", name, err)
+		}
 	}
-	if got := len(client.bulkWriteBatches[0]); got != 256 {
-		t.Fatalf("expected first batch size 256, got %d", got)
+
+	syncer, err := NewSyncer(client, SyncerOptions{
+		WorkspaceID: "ws_mount_sql_variable_limit",
+		RemoteRoot:  "/notion",
+		LocalRoot:   localDir,
+	})
+	if err != nil {
+		t.Fatalf("new syncer failed: %v", err)
 	}
-	if got := len(client.bulkWriteBatches[1]); got != 44 {
-		t.Fatalf("expected second batch size 44, got %d", got)
+	// Reproduce the maintenance path that can collect every small due outbox
+	// record into one request instead of stopping at the normal foreground
+	// discovery threshold.
+	syncer.bulkFlushThreshold = fileCount
+
+	if err := syncer.SyncOnce(context.Background()); err != nil {
+		t.Fatalf("large-workspace sync failed: %v", err)
+	}
+	if got := len(client.files); got != fileCount {
+		t.Fatalf("remote file count = %d, want %d", got, fileCount)
+	}
+	for idx, batch := range client.bulkWriteBatches {
+		if len(batch) > maxBulkWriteFilesPerRequest {
+			t.Fatalf("batch %d contains %d files, over safe cap %d", idx, len(batch), maxBulkWriteFilesPerRequest)
+		}
 	}
 }