From 60731df644451535c735d4ca2679573380069775 Mon Sep 17 00:00:00 2001
From: Ayman Hamed <ayman3000@gmail.com>
Date: Tue, 28 Jul 2026 13:31:54 +0300
Subject: [PATCH] =?UTF-8?q?test(tools):=20gated=20live=20test=20=E2=80=94?=
 =?UTF-8?q?=20multi-stage=20loop=20with=20a=20real=20goal=20verifier?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Stresses the full agentic loop end-to-end against glm-5.2:cloud: the agent
writes a Python script, runs it, and a goal verifier that ACTUALLY EXECUTES
the script requires exact output (FIB10=55) — forcing self-repair if wrong.
Combines FileWriteTool + ShellTool + autonomousMode + verifyCompletion.
Passed 3/3.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 .../SwiftAgentKitToolsTests.swift             | 69 +++++++++++++++++++
 1 file changed, 69 insertions(+)

diff --git a/Tests/SwiftAgentKitToolsTests/SwiftAgentKitToolsTests.swift b/Tests/SwiftAgentKitToolsTests/SwiftAgentKitToolsTests.swift
index d332737..a20d59a 100644
--- a/Tests/SwiftAgentKitToolsTests/SwiftAgentKitToolsTests.swift
+++ b/Tests/SwiftAgentKitToolsTests/SwiftAgentKitToolsTests.swift
@@ -208,3 +208,72 @@ func liveAgentChainsWriteThenRun() async throws {
     #expect(answer.contains("CHAIN_OK_7"))
     #expect(FileManager.default.fileExists(atPath: script))
 }
+
+// MARK: - Live: hard multi-stage loop with a real goal verifier (gated)
+
+/// Run a shell command and capture stdout+stderr (used by the verifier).
+private func shellCapture(_ command: String) -> String {
+    let p = Process()
+    p.executableURL = URL(fileURLWithPath: "/bin/zsh")
+    p.arguments = ["-lc", command]
+    let pipe = Pipe(); p.standardOutput = pipe; p.standardError = pipe
+    try? p.run()
+    let data = pipe.fileHandleForReading.readDataToEndOfFile()
+    p.waitUntilExit()
+    return String(data: data, encoding: .utf8) ?? ""
+}
+
+actor RetryCounter { private(set) var n = 0; func bump() { n += 1 } }
+
+/// Stresses the full loop: write code -> run it -> a goal verifier RUNS the
+/// script and requires exact output -> if wrong, the agent self-repairs and
+/// retries until correct. The verifier is deterministic (it actually executes
+/// the script), so a wrong first attempt forces iteration.
+///   SAK_LIVE_TESTS=1 swift test --filter liveAgentSelfRepairsToExactOutput
+@Test(.enabled(if: ProcessInfo.processInfo.environment["SAK_LIVE_TESTS"] == "1"))
+func liveAgentSelfRepairsToExactOutput() async throws {
+    let dir = tempDir()
+    let script = dir.appendingPathComponent("fib.py").path
+    let retries = RetryCounter()
+
+    let agent = Agent(config: AgentConfig(
+        provider: OllamaProvider(configuration: OllamaProvider.local(model: "glm-5.2:cloud")),
+        model: "glm-5.2:cloud",
+        systemPrompt: """
+        You are a hands-on coding assistant. When a task needs several steps, keep \
+        going until it is FULLY done — never stop after one step or hand back \
+        mid-task. After writing a script, RUN it, check the actual output, and if \
+        it is wrong, fix the script and run it again until it is correct.
+        """,
+        maxTurns: 12,
+        tools: [FileWriteTool(), ShellTool(), FileReadTool()],
+        autonomousMode: true,
+        maxVerificationRetries: 4
+    ))
+
+    // Goal verifier: actually execute the script and require the exact line.
+    var callbacks = AgentCallbacks()
+    callbacks.verifyCompletion = { _, _, _ in
+        await retries.bump()
+        guard FileManager.default.fileExists(atPath: script) else {
+            return .unsatisfied(reason: "The script \(script) does not exist yet — create it.")
+        }
+        let out = shellCapture("python3 \(script)")
+        if out.contains("FIB10=55") { return .satisfied }
+        return .unsatisfied(reason:
+            "Running the script did not print the required line FIB10=55. Actual output was:\n\(out)\n"
+            + "Fix \(script) so it prints exactly FIB10=55, then run it again.")
+    }
+    agent.callbacks = callbacks
+
+    _ = try await agent.run(
+        "Write a Python script at \(script) that computes the 10th Fibonacci number "
+        + "(with fib(0)=0, fib(1)=1, so the 10th is 55) and prints exactly the line "
+        + "FIB10=55. Then run it and make sure the output is correct.")
+
+    // The goal is verifiably met: running the script prints the exact line.
+    let finalOut = shellCapture("python3 \(script)")
+    #expect(finalOut.contains("FIB10=55"))
+    // The verifier ran at least once (proving the goal-gate engaged).
+    #expect(await retries.n >= 1)
+}