From b221faf5a115ff4e528776d6fb0030a447319138 Mon Sep 17 00:00:00 2001 From: RahulHere Date: Sun, 5 Jul 2026 09:20:47 +0800 Subject: [PATCH 01/14] Add Python native platform search paths Summary:\n- search native//lib and native/current/lib before compatibility native/lib\n- keep local build outputs ahead of installed platform packages\n- update main and auth loader tests for platform-specific native output paths --- tests/ffi/auth/test_loader.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/ffi/auth/test_loader.py b/tests/ffi/auth/test_loader.py index 8d5c1d08..62dfb1c7 100644 --- a/tests/ffi/auth/test_loader.py +++ b/tests/ffi/auth/test_loader.py @@ -97,6 +97,19 @@ def test_includes_platform_and_current_native_paths(self): str(auth_loader.Path.cwd() / "native" / "lib") ) + def test_includes_platform_and_current_native_paths(self): + """Test includes JS-compatible local native output directories.""" + paths = _get_search_paths() + platform_dir = _get_platform_native_dir_name() + platform_path = str(auth_loader.Path.cwd() / "native" / platform_dir / "lib") + current_path = str(auth_loader.Path.cwd() / "native" / "current" / "lib") + + assert platform_path in paths + assert current_path in paths + assert paths.index(platform_path) < paths.index( + str(auth_loader.Path.cwd() / "native" / "lib") + ) + @patch("platform.system") def test_darwin_includes_homebrew(self, mock_system): """Test macOS includes homebrew paths.""" From aeb3d1568965dd17289ead640d3ae17f4335f50f Mon Sep 17 00:00:00 2001 From: RahulHere Date: Tue, 28 Jul 2026 23:10:10 +0800 Subject: [PATCH 02/14] Add Python SDK example verification Summary: - Add GitHub Actions workflow for macOS arm64 and Linux x64 example verification. - Add verifier script that installs the SDK/native packages in a temp venv and checks offline/live example behavior. - Add native loading probe used by CI before running live examples. --- .github/workflows/verify-examples.yml | 157 +++++++++ scripts/verify-example-native-probe.py | 60 ++++ scripts/verify-examples.sh | 448 +++++++++++++++++++++++++ 3 files changed, 665 insertions(+) create mode 100644 .github/workflows/verify-examples.yml create mode 100755 scripts/verify-example-native-probe.py create mode 100755 scripts/verify-examples.sh diff --git a/.github/workflows/verify-examples.yml b/.github/workflows/verify-examples.yml new file mode 100644 index 00000000..af915599 --- /dev/null +++ b/.github/workflows/verify-examples.yml @@ -0,0 +1,157 @@ +name: Verify SDK Examples + +on: + pull_request: + branches: [main] + push: + branches: [iml_verify_auto] + workflow_dispatch: + inputs: + mode: + description: 'offline, live, or auto' + required: true + default: 'auto' + pypi_version: + description: 'PyPI version to verify' + required: true + default: 'latest' + +permissions: + contents: read + +jobs: + verify-examples: + name: ${{ matrix.platform }} + runs-on: ${{ matrix.os }} + env: + VERIFY_EXAMPLES_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.mode || 'live' }} + VERIFY_PYPI_VERSION: ${{ github.event_name == 'workflow_dispatch' && inputs.pypi_version || 'latest' }} + strategy: + fail-fast: false + matrix: + include: + - platform: darwin-arm64 + os: macos-15 + native_module: gopher_mcp_python_native_darwin_arm64 + - platform: linux-x64 + os: ubuntu-22.04 + native_module: gopher_mcp_python_native_linux_x64 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Remove Homebrew on macOS + if: runner.os == 'macOS' && runner.environment == 'github-hosted' + shell: bash + run: | + echo "=== Removing Homebrew to simulate clean machine ===" + sudo rm -rf /opt/homebrew /usr/local/Homebrew /usr/local/Cellar + sudo rm -f /usr/local/bin/brew /opt/homebrew/bin/brew + echo "Homebrew removed" + + - name: Install SDK package for native preflight + shell: bash + run: | + python -m venv native-preflight + source native-preflight/bin/activate + python -m pip install --upgrade pip + if [ "$VERIFY_PYPI_VERSION" = "latest" ]; then + python -m pip install gopher-mcp-python gopher-mcp-python-native-${{ matrix.platform }} + else + python -m pip install "gopher-mcp-python==${VERIFY_PYPI_VERSION}" "gopher-mcp-python-native-${{ matrix.platform }}==${VERIFY_PYPI_VERSION}" + fi + + - name: Verify macOS native package + if: runner.os == 'macOS' + shell: bash + run: | + source native-preflight/bin/activate + native_lib_dir="$( + python - <<'PY' +import importlib +module = importlib.import_module("${{ matrix.native_module }}") +print(module.get_lib_path()) +PY + )" + cd "$native_lib_dir" + + echo "=== Code Signatures ===" + failed=0 + for f in *.dylib; do + if codesign -v "$f" 2>&1; then + echo "OK $f" + else + echo "FAILED $f: $(codesign -v "$f" 2>&1)" + failed=1 + fi + done + [ "$failed" -eq 0 ] || exit 1 + + echo "=== Homebrew Path Check ===" + for f in *.dylib; do + [ -L "$f" ] && continue + if otool -L "$f" | grep -qE '/usr/local/|/opt/homebrew/'; then + echo "FAILED $f has Homebrew paths:" + otool -L "$f" | grep -E '/usr/local/|/opt/homebrew/' + exit 1 + fi + done + + echo "=== Bundled @loader_path Dependencies ===" + missing=0 + for f in *.dylib; do + [ -L "$f" ] && continue + while read -r dep rest; do + name="${dep#@loader_path/}" + if [ ! -f "$name" ]; then + echo "MISSING $name needed by $f" + missing=1 + fi + done < <(otool -L "$f" | grep "@loader_path/" || true) + done + [ "$missing" -eq 0 ] || exit 1 + + - name: Verify Linux native package + if: runner.os == 'Linux' + shell: bash + run: | + source native-preflight/bin/activate + python - <<'PY' +import importlib +from pathlib import Path + +module = importlib.import_module("${{ matrix.native_module }}") +lib_file = Path(module.get_library_file()) +print(lib_file) +if not lib_file.is_file(): + raise SystemExit(f"missing native library: {lib_file}") +PY + + - name: Verify native loading + shell: bash + run: | + source native-preflight/bin/activate + python scripts/verify-example-native-probe.py + + - name: Verify examples + shell: bash + env: + LLM_PROVIDER: ${{ secrets.LLM_PROVIDER || 'AnthropicProvider' }} + LLM_MODEL: ${{ secrets.LLM_MODEL }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOPHER_API_KEY: ${{ secrets.GOPHER_API_KEY }} + GOPHER_MCP_URL: ${{ secrets.GOPHER_MCP_URL }} + GOPHER_SDK_TEST: true + run: | + VERIFY_LIVE_PROMPT="List my draft mails" \ + VERIFY_EXPECTED_ANSWER="draft" \ + scripts/verify-examples.sh --mode "$VERIFY_EXAMPLES_MODE" --only create_by_api_key + VERIFY_LIVE_PROMPT="List my draft mails" \ + VERIFY_EXPECTED_ANSWER="draft" \ + scripts/verify-examples.sh --mode "$VERIFY_EXAMPLES_MODE" --only create_by_url diff --git a/scripts/verify-example-native-probe.py b/scripts/verify-example-native-probe.py new file mode 100755 index 00000000..ab1e3679 --- /dev/null +++ b/scripts/verify-example-native-probe.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Verify that the installed Python SDK can load the native gopher-orch library.""" + +import json +import sys + +from gopher_mcp_python import AgentError, GopherAgent +from gopher_mcp_python.ffi.library import GopherOrchLibrary + + +def fail(message: str) -> None: + print(f"[verify-native] error: {message}", file=sys.stderr) + sys.exit(1) + + +def main() -> None: + if not GopherOrchLibrary.is_available(): + fail(GopherOrchLibrary.get_load_error_message()) + + print("[verify-native] native library loaded") + + server_config = { + "succeeded": True, + "data": { + "servers": [ + { + "version": "2026-01-11", + "serverId": "verify-native", + "name": "verify-native", + "transport": "http_sse", + "config": { + "url": "http://127.0.0.1:1/mcp", + "headers": {}, + }, + "connectTimeout": 1000, + "requestTimeout": 1000, + } + ] + }, + } + + try: + agent = GopherAgent.create_with_server_config( + "AnthropicProvider", + "verify-model", + json.dumps(server_config), + ) + except AgentError as exc: + message = str(exc) + if "Failed to load" in message or "Native library not available" in message: + fail(message) + print(f"[verify-native] native create path reached expected failure: {message}") + return + + agent.dispose() + print("[verify-native] native create path completed") + + +if __name__ == "__main__": + main() diff --git a/scripts/verify-examples.sh b/scripts/verify-examples.sh new file mode 100755 index 00000000..da29b7e5 --- /dev/null +++ b/scripts/verify-examples.sh @@ -0,0 +1,448 @@ +#!/usr/bin/env bash + +set -euo pipefail + +MODE="${VERIFY_EXAMPLES_MODE:-auto}" +ONLY_EXAMPLE="" +ENV_FILE="${VERIFY_EXAMPLES_ENV_FILE:-}" +PYTHON_BIN="${PYTHON_BIN:-python3}" +PYTHON_VERSION="" +PLATFORM="" +NATIVE_PACKAGE="" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +TEMP_ROOT="" +PROJECT_DIR="" +VENV_DIR="" +SDK_INSTALL_SPEC="${SDK_INSTALL_SPEC:-}" +NATIVE_INSTALL_SPEC="${NATIVE_INSTALL_SPEC:-}" +SDK_VERSION="${VERIFY_PYPI_VERSION:-latest}" +VERIFY_LIVE_PROMPT="${VERIFY_LIVE_PROMPT:-What tools do we have?}" +VERIFY_EXPECTED_ANSWER="${VERIFY_EXPECTED_ANSWER:-tool}" +LIVE_CHECKS_RUN=0 +LIVE_CHECKS_SKIPPED=0 +LIVE_ANSWER_SUMMARY="" +SELECTED_EXAMPLES=() +EXAMPLES=( + "create_by_url|examples/api/create_by_url.py|GOPHER_MCP_URL LLM_MODEL|ANTHROPIC_API_KEY" + "create_by_api_key|examples/api/create_by_api_key.py|GOPHER_API_KEY LLM_MODEL|ANTHROPIC_API_KEY" + "create_by_json|examples/api/create_by_json.py|LLM_MODEL|ANTHROPIC_API_KEY" + "create_by_server_id|examples/api/create_by_server_id.py|GOPHER_API_KEY GOPHER_MCP_SERVER_ID LLM_MODEL|ANTHROPIC_API_KEY" + "create_by_server_name|examples/api/create_by_server_name.py|GOPHER_API_KEY GOPHER_MCP_SERVER_NAME LLM_MODEL|ANTHROPIC_API_KEY" + "create_by_gateway_id|examples/api/create_by_gateway_id.py|GOPHER_API_KEY GOPHER_MCP_GATEWAY_ID LLM_MODEL|ANTHROPIC_API_KEY" + "create_by_gateway_name|examples/api/create_by_gateway_name.py|GOPHER_API_KEY GOPHER_MCP_GATEWAY_NAME LLM_MODEL|ANTHROPIC_API_KEY" +) + +usage() { + cat <<'EOF' +Usage: scripts/verify-examples.sh [options] + +Options: + --mode Verification mode (default: auto) + --only Run one example by registry name + --env-file Load live environment variables from a file + -h, --help Show this help + +Environment: + VERIFY_EXAMPLES_ENV_FILE Default env file path + VERIFY_PYPI_VERSION PyPI version to verify, or latest (default: latest) + SDK_INSTALL_SPEC Override SDK pip install spec + NATIVE_INSTALL_SPEC Override native package pip install spec + VERIFY_LIVE_PROMPT Prompt used for live agent.run() checks + VERIFY_EXPECTED_ANSWER Text that must appear in the live answer +EOF +} + +log() { + printf '[verify-examples] %s\n' "$*" +} + +fail() { + log "error: $*" + exit 1 +} + +parse_args() { + while [ "$#" -gt 0 ]; do + case "$1" in + --mode) + [ "$#" -ge 2 ] || fail "--mode requires a value" + MODE="$2" + shift 2 + ;; + --only) + [ "$#" -ge 2 ] || fail "--only requires a value" + ONLY_EXAMPLE="$2" + shift 2 + ;; + --env-file) + [ "$#" -ge 2 ] || fail "--env-file requires a value" + ENV_FILE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + fail "unknown argument: $1" + ;; + esac + done +} + +validate_args() { + case "$MODE" in + offline|live|auto) ;; + *) fail "invalid --mode '${MODE}'; expected offline, live, or auto" ;; + esac + + if [ -n "$ONLY_EXAMPLE" ] && ! [[ "$ONLY_EXAMPLE" =~ ^[A-Za-z0-9_-]+$ ]]; then + fail "--only must be an example name containing only letters, numbers, '_' or '-'" + fi +} + +load_env_file() { + if [ -z "$ENV_FILE" ]; then + return + fi + + if [ ! -f "$ENV_FILE" ]; then + fail "env file not found: ${ENV_FILE}" + fi + + set -a + # shellcheck source=/dev/null + . "$ENV_FILE" + set +a + log "env_file=${ENV_FILE}" +} + +require_python() { + if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + fail "Python 3.8 or newer is required, but ${PYTHON_BIN} was not found in PATH" + fi + + PYTHON_VERSION="$("$PYTHON_BIN" - <<'PY' +import sys +print(".".join(str(part) for part in sys.version_info[:3])) +raise SystemExit(0 if sys.version_info >= (3, 8) else 1) +PY +)" || fail "Python 3.8 or newer is required; current version is ${PYTHON_VERSION:-unknown}" +} + +detect_platform() { + local os + local arch + + os="$(uname -s 2>/dev/null || true)" + arch="$(uname -m 2>/dev/null || true)" + + case "${os}:${arch}" in + Darwin:arm64) + PLATFORM="darwin-arm64" + ;; + Darwin:x86_64|Darwin:amd64) + PLATFORM="darwin-x64" + ;; + Linux:x86_64|Linux:amd64) + PLATFORM="linux-x64" + ;; + Linux:aarch64|Linux:arm64) + PLATFORM="linux-arm64" + ;; + MINGW*:x86_64|MSYS*:x86_64|CYGWIN*:x86_64|Windows_NT:x86_64|Windows_NT:amd64) + PLATFORM="win32-x64" + ;; + *) + fail "unsupported platform '${os:-unknown}' '${arch:-unknown}'" + ;; + esac + + NATIVE_PACKAGE="gopher-mcp-python-native-${PLATFORM}" +} + +compute_install_specs() { + if [ -z "$SDK_INSTALL_SPEC" ]; then + if [ "$SDK_VERSION" = "latest" ]; then + SDK_INSTALL_SPEC="gopher-mcp-python" + else + SDK_INSTALL_SPEC="gopher-mcp-python==${SDK_VERSION}" + fi + fi + + if [ -z "$NATIVE_INSTALL_SPEC" ]; then + if [ "$SDK_VERSION" = "latest" ]; then + NATIVE_INSTALL_SPEC="${NATIVE_PACKAGE}" + else + NATIVE_INSTALL_SPEC="${NATIVE_PACKAGE}==${SDK_VERSION}" + fi + fi +} + +example_name() { + local spec="$1" + printf '%s\n' "${spec%%|*}" +} + +example_path() { + local spec="$1" + local rest="${spec#*|}" + printf '%s\n' "${rest%%|*}" +} + +example_required_env() { + local spec="$1" + local rest="${spec#*|}" + rest="${rest#*|}" + printf '%s\n' "${rest%%|*}" +} + +example_provider_env() { + local spec="$1" + local rest="${spec#*|}" + rest="${rest#*|}" + rest="${rest#*|}" + printf '%s\n' "$rest" +} + +select_examples() { + local spec + local name + local found=0 + + SELECTED_EXAMPLES=() + + for spec in "${EXAMPLES[@]}"; do + name="$(example_name "$spec")" + if [ -z "$ONLY_EXAMPLE" ] || [ "$ONLY_EXAMPLE" = "$name" ]; then + SELECTED_EXAMPLES+=("$spec") + found=1 + fi + done + + if [ "$found" -ne 1 ]; then + fail "unknown example '${ONLY_EXAMPLE}'; supported examples are create_by_url, create_by_api_key, create_by_json, create_by_server_id, create_by_server_name, create_by_gateway_id, and create_by_gateway_name" + fi +} + +log_selected_examples() { + local names=() + local spec + + for spec in "${SELECTED_EXAMPLES[@]}"; do + names+=("$(example_name "$spec")") + done + + local joined="${names[*]}" + log "examples=${joined// /,}" +} + +create_project() { + TEMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/gopher-mcp-python-example-verify.XXXXXX")" + PROJECT_DIR="${TEMP_ROOT}/project" + VENV_DIR="${PROJECT_DIR}/venv" + mkdir -p "$PROJECT_DIR" + + "$PYTHON_BIN" -m venv "$VENV_DIR" + "${VENV_DIR}/bin/python" -m pip install --upgrade pip >/dev/null + "${VENV_DIR}/bin/python" -m pip install "$SDK_INSTALL_SPEC" "$NATIVE_INSTALL_SPEC" >/dev/null + + SDK_VERSION="$("${VENV_DIR}/bin/python" - <<'PY' +from importlib import metadata +print(metadata.version("gopher-mcp-python")) +PY +)" + + log "temp_project=${PROJECT_DIR}" + log "platform=${PLATFORM} python=${PYTHON_VERSION} mode=${MODE} sdk=${SDK_VERSION}" +} + +cleanup() { + if [ -n "$TEMP_ROOT" ] && [ -d "$TEMP_ROOT" ]; then + rm -rf "$TEMP_ROOT" + fi +} + +run_native_probe() { + "${VENV_DIR}/bin/python" "${REPO_ROOT}/scripts/verify-example-native-probe.py" +} + +run_offline_example_bootstrap_checks() { + local spec + local name + local source_path + local target_file + local output + local status + + for spec in "${SELECTED_EXAMPLES[@]}"; do + name="$(example_name "$spec")" + source_path="${REPO_ROOT}/$(example_path "$spec")" + target_file="${PROJECT_DIR}/$(basename "$source_path")" + + if [ ! -f "$source_path" ]; then + fail "${name} offline: source file not found: ${source_path}" + fi + + cp "$source_path" "$target_file" + + set +e + output="$( + cd "$PROJECT_DIR" && + env \ + -u GOPHER_MCP_URL \ + -u GOPHER_API_KEY \ + -u GOPHER_MCP_SERVER_ID \ + -u GOPHER_MCP_SERVER_NAME \ + -u GOPHER_MCP_GATEWAY_ID \ + -u GOPHER_MCP_GATEWAY_NAME \ + -u LLM_MODEL \ + -u LLM_PROVIDER \ + -u ANTHROPIC_API_KEY \ + "${VENV_DIR}/bin/python" "$(basename "$target_file")" 2>&1 + )" + status=$? + set -e + + if [ "$status" -eq 0 ]; then + printf '%s\n' "$output" + fail "${name} offline: expected missing-env validation failure" + fi + + if ! grep -Eq 'must (both |all )?be set' <<<"$output"; then + printf '%s\n' "$output" + fail "${name} offline: did not report expected missing-env validation" + fi + + log "${name} offline: missing-env validation OK" + done +} + +has_required_env() { + local required="$1" + local provider_required="$2" + local key + + for key in $required; do + if [ -z "${!key:-}" ]; then + return 1 + fi + done + + if [ "${LLM_PROVIDER:-AnthropicProvider}" = "AnthropicProvider" ]; then + for key in $provider_required; do + if [ -z "${!key:-}" ]; then + return 1 + fi + done + fi + + return 0 +} + +run_live_example_checks() { + local spec + local name + local required + local provider_required + local source_path + local target_file + local output + local status + local answer_excerpt + + for spec in "${SELECTED_EXAMPLES[@]}"; do + name="$(example_name "$spec")" + required="$(example_required_env "$spec")" + provider_required="$(example_provider_env "$spec")" + + if ! has_required_env "$required" "$provider_required"; then + if [ "$MODE" = "live" ]; then + fail "${name} live: missing required environment (${required} ${provider_required})" + fi + LIVE_CHECKS_SKIPPED=$((LIVE_CHECKS_SKIPPED + 1)) + log "${name} live: skipped because required environment is missing" + continue + fi + + source_path="${REPO_ROOT}/$(example_path "$spec")" + target_file="${PROJECT_DIR}/$(basename "$source_path")" + cp "$source_path" "$target_file" + + set +e + output="$( + cd "$PROJECT_DIR" && + "${VENV_DIR}/bin/python" "$(basename "$target_file")" "$VERIFY_LIVE_PROMPT" 2>&1 + )" + status=$? + set -e + + if [ "$status" -ne 0 ]; then + printf '%s\n' "$output" + fail "${name} live: example exited with status ${status}" + fi + + if ! grep -q 'Agent Response' <<<"$output"; then + printf '%s\n' "$output" + fail "${name} live: missing Agent Response marker" + fi + + if [ -n "$VERIFY_EXPECTED_ANSWER" ] && + ! grep -qi -- "$VERIFY_EXPECTED_ANSWER" <<<"$output"; then + printf '%s\n' "$output" + fail "${name} live: expected answer text '${VERIFY_EXPECTED_ANSWER}' not found" + fi + + answer_excerpt="$(awk '/Agent Response/{capture=1; next} capture {print}' <<<"$output" | sed -n '1,8p')" + LIVE_CHECKS_RUN=$((LIVE_CHECKS_RUN + 1)) + LIVE_ANSWER_SUMMARY="${LIVE_ANSWER_SUMMARY}"$'\n'"${name}: ${answer_excerpt}" + log "${name} live: OK" + done +} + +main() { + parse_args "$@" + validate_args + load_env_file + require_python + detect_platform + compute_install_specs + select_examples + + log "only=${ONLY_EXAMPLE:-}" + log_selected_examples + log "sdk_install=${SDK_INSTALL_SPEC}" + log "native_install=${NATIVE_INSTALL_SPEC}" + + create_project + trap cleanup EXIT + + run_native_probe + + case "$MODE" in + offline) + run_offline_example_bootstrap_checks + ;; + live) + run_live_example_checks + ;; + auto) + run_offline_example_bootstrap_checks + run_live_example_checks + ;; + esac + + if [ "$LIVE_CHECKS_RUN" -gt 0 ]; then + log "live_checks=${LIVE_CHECKS_RUN}" + printf '%s\n' "$LIVE_ANSWER_SUMMARY" | sed '/^$/d' + fi + + if [ "$MODE" = "auto" ] && [ "$LIVE_CHECKS_SKIPPED" -gt 0 ]; then + log "live_skipped=${LIVE_CHECKS_SKIPPED}" + fi + + log "verification passed" +} + +main "$@" From 337d821e541e63157a91d9ae44317f3fc0a527b5 Mon Sep 17 00:00:00 2001 From: RahulHere Date: Tue, 28 Jul 2026 23:22:58 +0800 Subject: [PATCH 03/14] Fix Python example verifier workflow syntax Summary: - Replace unindented embedded Python heredocs with python -c snippets. - Keep native package path checks equivalent while making the workflow valid YAML. --- .github/workflows/verify-examples.yml | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/.github/workflows/verify-examples.yml b/.github/workflows/verify-examples.yml index af915599..46e79124 100644 --- a/.github/workflows/verify-examples.yml +++ b/.github/workflows/verify-examples.yml @@ -73,11 +73,7 @@ jobs: run: | source native-preflight/bin/activate native_lib_dir="$( - python - <<'PY' -import importlib -module = importlib.import_module("${{ matrix.native_module }}") -print(module.get_lib_path()) -PY + python -c 'import importlib; module = importlib.import_module("${{ matrix.native_module }}"); print(module.get_lib_path())' )" cd "$native_lib_dir" @@ -122,16 +118,7 @@ PY shell: bash run: | source native-preflight/bin/activate - python - <<'PY' -import importlib -from pathlib import Path - -module = importlib.import_module("${{ matrix.native_module }}") -lib_file = Path(module.get_library_file()) -print(lib_file) -if not lib_file.is_file(): - raise SystemExit(f"missing native library: {lib_file}") -PY + python -c 'import importlib; from pathlib import Path; module = importlib.import_module("${{ matrix.native_module }}"); lib_file = Path(module.get_library_file()); print(lib_file); raise SystemExit(0 if lib_file.is_file() else f"missing native library: {lib_file}")' - name: Verify native loading shell: bash From 585a8040e4cb6da2daf028eb553d19c40c22cb3e Mon Sep 17 00:00:00 2001 From: RahulHere Date: Tue, 28 Jul 2026 23:31:01 +0800 Subject: [PATCH 04/14] Clean up Python native verifier probe Summary: - Avoid creating an agent during native preflight so CI logs do not include expected MCP connection errors. - Verify native load by checking the required exported gopher-orch symbols instead. --- scripts/verify-example-native-probe.py | 60 ++++++++++---------------- 1 file changed, 22 insertions(+), 38 deletions(-) diff --git a/scripts/verify-example-native-probe.py b/scripts/verify-example-native-probe.py index ab1e3679..7a1c360a 100755 --- a/scripts/verify-example-native-probe.py +++ b/scripts/verify-example-native-probe.py @@ -1,10 +1,8 @@ #!/usr/bin/env python3 """Verify that the installed Python SDK can load the native gopher-orch library.""" -import json import sys -from gopher_mcp_python import AgentError, GopherAgent from gopher_mcp_python.ffi.library import GopherOrchLibrary @@ -14,46 +12,32 @@ def fail(message: str) -> None: def main() -> None: - if not GopherOrchLibrary.is_available(): + library = GopherOrchLibrary.get_instance() + if library is None: fail(GopherOrchLibrary.get_load_error_message()) print("[verify-native] native library loaded") - server_config = { - "succeeded": True, - "data": { - "servers": [ - { - "version": "2026-01-11", - "serverId": "verify-native", - "name": "verify-native", - "transport": "http_sse", - "config": { - "url": "http://127.0.0.1:1/mcp", - "headers": {}, - }, - "connectTimeout": 1000, - "requestTimeout": 1000, - } - ] - }, - } - - try: - agent = GopherAgent.create_with_server_config( - "AnthropicProvider", - "verify-model", - json.dumps(server_config), - ) - except AgentError as exc: - message = str(exc) - if "Failed to load" in message or "Native library not available" in message: - fail(message) - print(f"[verify-native] native create path reached expected failure: {message}") - return - - agent.dispose() - print("[verify-native] native create path completed") + required_symbols = ( + "gopher_orch_agent_create_by_json", + "gopher_orch_agent_create_by_api_key", + "gopher_orch_agent_run", + "gopher_orch_agent_release", + "gopher_orch_api_fetch_servers", + "gopher_orch_last_error", + "gopher_orch_clear_error", + "gopher_orch_free", + ) + native_library = getattr(library, "_lib", None) + missing = [ + symbol + for symbol in required_symbols + if native_library is None or not hasattr(native_library, symbol) + ] + if missing: + fail(f"native library missing required symbols: {', '.join(missing)}") + + print("[verify-native] required symbols present") if __name__ == "__main__": From 53d5091011264a50edf429bb8a4898ad46ad214c Mon Sep 17 00:00:00 2001 From: RahulHere Date: Fri, 31 Jul 2026 11:32:06 +0800 Subject: [PATCH 05/14] Verify Python examples against PR checkout Summary: - install the checked-out SDK during pull_request example preflight - pass the checkout path to example verification on PR runs - keep PyPI package verification for workflow dispatch and published-version checks - add a workflow regression test for PR checkout installs --- .github/workflows/verify-examples.yml | 5 ++++- tests/test_linux_native_packaging.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify-examples.yml b/.github/workflows/verify-examples.yml index 46e79124..941a5f91 100644 --- a/.github/workflows/verify-examples.yml +++ b/.github/workflows/verify-examples.yml @@ -61,7 +61,9 @@ jobs: python -m venv native-preflight source native-preflight/bin/activate python -m pip install --upgrade pip - if [ "$VERIFY_PYPI_VERSION" = "latest" ]; then + if [ "${{ github.event_name }}" = "pull_request" ]; then + python -m pip install -e . "gopher-mcp-python-native-${{ matrix.platform }}" + elif [ "$VERIFY_PYPI_VERSION" = "latest" ]; then python -m pip install gopher-mcp-python gopher-mcp-python-native-${{ matrix.platform }} else python -m pip install "gopher-mcp-python==${VERIFY_PYPI_VERSION}" "gopher-mcp-python-native-${{ matrix.platform }}==${VERIFY_PYPI_VERSION}" @@ -135,6 +137,7 @@ jobs: GOPHER_API_KEY: ${{ secrets.GOPHER_API_KEY }} GOPHER_MCP_URL: ${{ secrets.GOPHER_MCP_URL }} GOPHER_SDK_TEST: true + SDK_INSTALL_SPEC: ${{ github.event_name == 'pull_request' && github.workspace || '' }} run: | VERIFY_LIVE_PROMPT="List my draft mails" \ VERIFY_EXPECTED_ANSWER="draft" \ diff --git a/tests/test_linux_native_packaging.py b/tests/test_linux_native_packaging.py index fe7de3d9..a5c3781b 100644 --- a/tests/test_linux_native_packaging.py +++ b/tests/test_linux_native_packaging.py @@ -19,6 +19,10 @@ def _root_build_script() -> str: return (ROOT / "build.sh").read_text() +def _verify_examples_workflow() -> str: + return (ROOT / ".github" / "workflows" / "verify-examples.yml").read_text() + + def test_linux_x64_uses_digest_pinned_ubuntu_builder_image() -> None: dockerfile = _linux_builder_dockerfile() build_script = _root_build_script() @@ -34,6 +38,20 @@ def test_linux_x64_uses_digest_pinned_ubuntu_builder_image() -> None: assert re.search(r"\subuntu:20\.04\s", build_script) is None +def test_verify_examples_prs_install_checked_out_sdk() -> None: + workflow = _verify_examples_workflow() + + assert 'if [ "${{ github.event_name }}" = "pull_request" ]; then' in workflow + assert ( + 'python -m pip install -e . "gopher-mcp-python-native-${{ matrix.platform }}"' + in workflow + ) + assert ( + "SDK_INSTALL_SPEC: ${{ github.event_name == 'pull_request' && " + "github.workspace || '' }}" + ) in workflow + + def test_linux_x64_builder_does_not_bundle_openssl() -> None: script = _linux_builder_script() dep_skip_block = re.search(r"case \"\$dep_name\" in(?P.*?)esac", script, re.S) From 6240539b57c5ddd492b94692535c968b4fb238b4 Mon Sep 17 00:00:00 2001 From: RahulHere Date: Fri, 31 Jul 2026 11:34:12 +0800 Subject: [PATCH 06/14] Relax Python example verification on PRs Summary: - default example verification to auto mode outside manual dispatch - cancel superseded workflow runs for the same ref - cap the example verification job runtime at 30 minutes - add regression tests for PR workflow defaults and guardrails --- .github/workflows/verify-examples.yml | 7 ++++++- tests/test_linux_native_packaging.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify-examples.yml b/.github/workflows/verify-examples.yml index 941a5f91..9436dbd7 100644 --- a/.github/workflows/verify-examples.yml +++ b/.github/workflows/verify-examples.yml @@ -19,12 +19,17 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: verify-examples: name: ${{ matrix.platform }} runs-on: ${{ matrix.os }} + timeout-minutes: 30 env: - VERIFY_EXAMPLES_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.mode || 'live' }} + VERIFY_EXAMPLES_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.mode || 'auto' }} VERIFY_PYPI_VERSION: ${{ github.event_name == 'workflow_dispatch' && inputs.pypi_version || 'latest' }} strategy: fail-fast: false diff --git a/tests/test_linux_native_packaging.py b/tests/test_linux_native_packaging.py index a5c3781b..ff684128 100644 --- a/tests/test_linux_native_packaging.py +++ b/tests/test_linux_native_packaging.py @@ -41,6 +41,10 @@ def test_linux_x64_uses_digest_pinned_ubuntu_builder_image() -> None: def test_verify_examples_prs_install_checked_out_sdk() -> None: workflow = _verify_examples_workflow() + assert ( + "VERIFY_EXAMPLES_MODE: ${{ github.event_name == 'workflow_dispatch' " + "&& inputs.mode || 'auto' }}" + ) in workflow assert 'if [ "${{ github.event_name }}" = "pull_request" ]; then' in workflow assert ( 'python -m pip install -e . "gopher-mcp-python-native-${{ matrix.platform }}"' @@ -52,6 +56,15 @@ def test_verify_examples_prs_install_checked_out_sdk() -> None: ) in workflow +def test_verify_examples_workflow_bounds_pr_cost_and_runtime() -> None: + workflow = _verify_examples_workflow() + + assert "concurrency:" in workflow + assert "group: ${{ github.workflow }}-${{ github.ref }}" in workflow + assert "cancel-in-progress: true" in workflow + assert "timeout-minutes: 30" in workflow + + def test_linux_x64_builder_does_not_bundle_openssl() -> None: script = _linux_builder_script() dep_skip_block = re.search(r"case \"\$dep_name\" in(?P.*?)esac", script, re.S) From bece3a9ed8e8d23bfb8766357441e8875d8f9b5d Mon Sep 17 00:00:00 2001 From: RahulHere Date: Fri, 31 Jul 2026 11:42:49 +0800 Subject: [PATCH 07/14] Strengthen Python example live verification Summary: - validate expected live-answer text only inside the agent response body - add comma-separated required answer terms for table-shaped responses - reject live example responses that contain obvious error markers - prompt live draft checks to return Draft ID, Message ID, and Thread ID columns - add regression coverage for response-scoped live verification --- .github/workflows/verify-examples.yml | 10 +++--- scripts/verify-examples.sh | 44 ++++++++++++++++++++++++--- tests/test_linux_native_packaging.py | 18 +++++++++++ 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/.github/workflows/verify-examples.yml b/.github/workflows/verify-examples.yml index 9436dbd7..09298ef1 100644 --- a/.github/workflows/verify-examples.yml +++ b/.github/workflows/verify-examples.yml @@ -144,9 +144,11 @@ jobs: GOPHER_SDK_TEST: true SDK_INSTALL_SPEC: ${{ github.event_name == 'pull_request' && github.workspace || '' }} run: | - VERIFY_LIVE_PROMPT="List my draft mails" \ - VERIFY_EXPECTED_ANSWER="draft" \ + VERIFY_LIVE_PROMPT="List my draft mails. Return a concise Markdown answer with a table using columns Draft ID, Message ID, and Thread ID." \ + VERIFY_EXPECTED_ANSWER="" \ + VERIFY_EXPECTED_ANSWER_TERMS="Draft ID,Message ID,Thread ID" \ scripts/verify-examples.sh --mode "$VERIFY_EXAMPLES_MODE" --only create_by_api_key - VERIFY_LIVE_PROMPT="List my draft mails" \ - VERIFY_EXPECTED_ANSWER="draft" \ + VERIFY_LIVE_PROMPT="List my draft mails. Return a concise Markdown answer with a table using columns Draft ID, Message ID, and Thread ID." \ + VERIFY_EXPECTED_ANSWER="" \ + VERIFY_EXPECTED_ANSWER_TERMS="Draft ID,Message ID,Thread ID" \ scripts/verify-examples.sh --mode "$VERIFY_EXAMPLES_MODE" --only create_by_url diff --git a/scripts/verify-examples.sh b/scripts/verify-examples.sh index da29b7e5..628891c8 100755 --- a/scripts/verify-examples.sh +++ b/scripts/verify-examples.sh @@ -19,6 +19,7 @@ NATIVE_INSTALL_SPEC="${NATIVE_INSTALL_SPEC:-}" SDK_VERSION="${VERIFY_PYPI_VERSION:-latest}" VERIFY_LIVE_PROMPT="${VERIFY_LIVE_PROMPT:-What tools do we have?}" VERIFY_EXPECTED_ANSWER="${VERIFY_EXPECTED_ANSWER:-tool}" +VERIFY_EXPECTED_ANSWER_TERMS="${VERIFY_EXPECTED_ANSWER_TERMS:-}" LIVE_CHECKS_RUN=0 LIVE_CHECKS_SKIPPED=0 LIVE_ANSWER_SUMMARY="" @@ -50,6 +51,7 @@ Environment: NATIVE_INSTALL_SPEC Override native package pip install spec VERIFY_LIVE_PROMPT Prompt used for live agent.run() checks VERIFY_EXPECTED_ANSWER Text that must appear in the live answer + VERIFY_EXPECTED_ANSWER_TERMS Comma-separated terms that must all appear in the live answer EOF } @@ -341,6 +343,26 @@ has_required_env() { return 0 } +validate_expected_answer_terms() { + local answer_body="$1" + local terms="$2" + local term + + terms="${terms}," + while [ -n "$terms" ]; do + term="${terms%%,*}" + terms="${terms#*,}" + term="$(sed 's/^[[:space:]]*//;s/[[:space:]]*$//' <<<"$term")" + [ -n "$term" ] || continue + + if ! grep -qi -- "$term" <<<"$answer_body"; then + return 1 + fi + done + + return 0 +} + run_live_example_checks() { local spec local name @@ -350,6 +372,7 @@ run_live_example_checks() { local target_file local output local status + local answer_body local answer_excerpt for spec in "${SELECTED_EXAMPLES[@]}"; do @@ -383,18 +406,31 @@ run_live_example_checks() { fail "${name} live: example exited with status ${status}" fi - if ! grep -q 'Agent Response' <<<"$output"; then + answer_body="$(awk '/Agent Response/{capture=1; next} capture {print}' <<<"$output")" + answer_excerpt="$(sed -n '1,8p' <<<"$answer_body")" + + if [ -z "$(tr -d '[:space:]' <<<"$answer_body")" ]; then + printf '%s\n' "$output" + fail "${name} live: missing agent response body" + fi + + if grep -Eqi '(^|[[:space:]])(Error:|Traceback|isError)' <<<"$answer_body"; then printf '%s\n' "$output" - fail "${name} live: missing Agent Response marker" + fail "${name} live: agent response contains an error" fi if [ -n "$VERIFY_EXPECTED_ANSWER" ] && - ! grep -qi -- "$VERIFY_EXPECTED_ANSWER" <<<"$output"; then + ! grep -qi -- "$VERIFY_EXPECTED_ANSWER" <<<"$answer_body"; then printf '%s\n' "$output" fail "${name} live: expected answer text '${VERIFY_EXPECTED_ANSWER}' not found" fi - answer_excerpt="$(awk '/Agent Response/{capture=1; next} capture {print}' <<<"$output" | sed -n '1,8p')" + if [ -n "$VERIFY_EXPECTED_ANSWER_TERMS" ] && + ! validate_expected_answer_terms "$answer_body" "$VERIFY_EXPECTED_ANSWER_TERMS"; then + printf '%s\n' "$output" + fail "${name} live: expected answer terms '${VERIFY_EXPECTED_ANSWER_TERMS}' not found" + fi + LIVE_CHECKS_RUN=$((LIVE_CHECKS_RUN + 1)) LIVE_ANSWER_SUMMARY="${LIVE_ANSWER_SUMMARY}"$'\n'"${name}: ${answer_excerpt}" log "${name} live: OK" diff --git a/tests/test_linux_native_packaging.py b/tests/test_linux_native_packaging.py index ff684128..6e632398 100644 --- a/tests/test_linux_native_packaging.py +++ b/tests/test_linux_native_packaging.py @@ -23,6 +23,10 @@ def _verify_examples_workflow() -> str: return (ROOT / ".github" / "workflows" / "verify-examples.yml").read_text() +def _verify_examples_script() -> str: + return (ROOT / "scripts" / "verify-examples.sh").read_text() + + def test_linux_x64_uses_digest_pinned_ubuntu_builder_image() -> None: dockerfile = _linux_builder_dockerfile() build_script = _root_build_script() @@ -65,6 +69,20 @@ def test_verify_examples_workflow_bounds_pr_cost_and_runtime() -> None: assert "timeout-minutes: 30" in workflow +def test_verify_examples_live_checks_only_agent_response_body() -> None: + script = _verify_examples_script() + workflow = _verify_examples_workflow() + + assert "answer_body=\"$(awk '/Agent Response/{capture=1; next} capture {print}'" in script + assert 'grep -qi -- "$VERIFY_EXPECTED_ANSWER" <<<"$answer_body"' in script + assert 'grep -qi -- "$VERIFY_EXPECTED_ANSWER" <<<"$output"' not in script + assert "VERIFY_EXPECTED_ANSWER_TERMS" in script + assert 'validate_expected_answer_terms "$answer_body"' in script + assert "agent response contains an error" in script + assert "Draft ID,Message ID,Thread ID" in workflow + assert "table using columns Draft ID, Message ID, and Thread ID" in workflow + + def test_linux_x64_builder_does_not_bundle_openssl() -> None: script = _linux_builder_script() dep_skip_block = re.search(r"case \"\$dep_name\" in(?P.*?)esac", script, re.S) From 2f813e84e018eafad1025323b44bc60cab540ccf Mon Sep 17 00:00:00 2001 From: RahulHere Date: Fri, 31 Jul 2026 11:45:43 +0800 Subject: [PATCH 08/14] Verify Linux native dependencies in examples Summary: - inspect Linux native package shared libraries during example verification - require Linux native libraries to carry an ORIGIN rpath or runpath - fail on missing non-OpenSSL shared dependencies - reject bundled OpenSSL libraries in the example verification workflow - add regression coverage for Linux native dependency checks --- .github/workflows/verify-examples.yml | 34 ++++++++++++++++++++++++++- tests/test_linux_native_packaging.py | 12 ++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify-examples.yml b/.github/workflows/verify-examples.yml index 09298ef1..a83d2085 100644 --- a/.github/workflows/verify-examples.yml +++ b/.github/workflows/verify-examples.yml @@ -124,8 +124,40 @@ jobs: if: runner.os == 'Linux' shell: bash run: | + set -euo pipefail + source native-preflight/bin/activate - python -c 'import importlib; from pathlib import Path; module = importlib.import_module("${{ matrix.native_module }}"); lib_file = Path(module.get_library_file()); print(lib_file); raise SystemExit(0 if lib_file.is_file() else f"missing native library: {lib_file}")' + native_lib_dir="$( + python -c 'import importlib; module = importlib.import_module("${{ matrix.native_module }}"); print(module.get_lib_path())' + )" + cd "$native_lib_dir" + + echo "=== Linux Native Dependencies ===" + found=0 + for sofile in *.so *.so.*; do + [ -L "$sofile" ] && continue + [ -f "$sofile" ] || continue + found=1 + echo "--- $sofile" + readelf -d "$sofile" | grep -E 'RUNPATH|RPATH' | grep -F '$ORIGIN' + LD_LIBRARY_PATH="$native_lib_dir:${LD_LIBRARY_PATH:-}" ldd "$sofile" | tee /tmp/ldd.out + missing="$(awk '/not found/ {print $1}' /tmp/ldd.out | grep -Ev '^(libssl\.so|libcrypto\.so)' || true)" + if [ -n "$missing" ]; then + echo "$missing" + echo "Missing shared dependency for $sofile" + exit 1 + fi + done + + if find "$native_lib_dir" -maxdepth 1 -type f \( -name 'libssl.so*' -o -name 'libcrypto.so*' \) | grep .; then + echo "OpenSSL libraries must remain system-provided, not bundled." + exit 1 + fi + + if [ "$found" -ne 1 ]; then + echo "No Linux shared libraries found in $native_lib_dir" + exit 1 + fi - name: Verify native loading shell: bash diff --git a/tests/test_linux_native_packaging.py b/tests/test_linux_native_packaging.py index 6e632398..dbde43bb 100644 --- a/tests/test_linux_native_packaging.py +++ b/tests/test_linux_native_packaging.py @@ -69,6 +69,18 @@ def test_verify_examples_workflow_bounds_pr_cost_and_runtime() -> None: assert "timeout-minutes: 30" in workflow +def test_verify_examples_workflow_checks_linux_native_dependencies() -> None: + workflow = _verify_examples_workflow() + + assert "Verify Linux native package" in workflow + assert "=== Linux Native Dependencies ===" in workflow + assert "readelf -d \"$sofile\"" in workflow + assert "ldd \"$sofile\"" in workflow + assert "grep -Ev '^(libssl\\.so|libcrypto\\.so)'" in workflow + assert "OpenSSL libraries must remain system-provided" in workflow + assert "No Linux shared libraries found" in workflow + + def test_verify_examples_live_checks_only_agent_response_body() -> None: script = _verify_examples_script() workflow = _verify_examples_workflow() From 23a788adc357b2bee54d4831d3837bfb3cb7be0b Mon Sep 17 00:00:00 2001 From: RahulHere Date: Fri, 31 Jul 2026 11:49:26 +0800 Subject: [PATCH 09/14] Redact live example verifier output Summary: - stop printing raw live example stdout and stderr on verifier failures - replace live failure dumps with redacted output and answer size diagnostics - remove agent answer excerpts from live verification summaries - add regression coverage to keep live answer content out of CI logs --- scripts/verify-examples.sh | 36 +++++++++++++++++++++------- tests/test_linux_native_packaging.py | 11 +++++++++ 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/scripts/verify-examples.sh b/scripts/verify-examples.sh index 628891c8..f59cfa61 100755 --- a/scripts/verify-examples.sh +++ b/scripts/verify-examples.sh @@ -64,6 +64,26 @@ fail() { exit 1 } +log_live_failure_diagnostics() { + local output="$1" + local answer_body="$2" + local output_bytes + local output_lines + local answer_bytes + local answer_lines + local marker_present="false" + + output_bytes="$(printf '%s' "$output" | wc -c | tr -d '[:space:]')" + output_lines="$(printf '%s\n' "$output" | wc -l | tr -d '[:space:]')" + answer_bytes="$(printf '%s' "$answer_body" | wc -c | tr -d '[:space:]')" + answer_lines="$(printf '%s\n' "$answer_body" | wc -l | tr -d '[:space:]')" + if grep -q 'Agent Response' <<<"$output"; then + marker_present="true" + fi + + log "live output redacted: output_bytes=${output_bytes} output_lines=${output_lines} agent_response_marker=${marker_present} answer_bytes=${answer_bytes} answer_lines=${answer_lines}" +} + parse_args() { while [ "$#" -gt 0 ]; do case "$1" in @@ -373,7 +393,6 @@ run_live_example_checks() { local output local status local answer_body - local answer_excerpt for spec in "${SELECTED_EXAMPLES[@]}"; do name="$(example_name "$spec")" @@ -402,38 +421,37 @@ run_live_example_checks() { set -e if [ "$status" -ne 0 ]; then - printf '%s\n' "$output" + log_live_failure_diagnostics "$output" "" fail "${name} live: example exited with status ${status}" fi answer_body="$(awk '/Agent Response/{capture=1; next} capture {print}' <<<"$output")" - answer_excerpt="$(sed -n '1,8p' <<<"$answer_body")" if [ -z "$(tr -d '[:space:]' <<<"$answer_body")" ]; then - printf '%s\n' "$output" + log_live_failure_diagnostics "$output" "$answer_body" fail "${name} live: missing agent response body" fi if grep -Eqi '(^|[[:space:]])(Error:|Traceback|isError)' <<<"$answer_body"; then - printf '%s\n' "$output" + log_live_failure_diagnostics "$output" "$answer_body" fail "${name} live: agent response contains an error" fi if [ -n "$VERIFY_EXPECTED_ANSWER" ] && ! grep -qi -- "$VERIFY_EXPECTED_ANSWER" <<<"$answer_body"; then - printf '%s\n' "$output" + log_live_failure_diagnostics "$output" "$answer_body" fail "${name} live: expected answer text '${VERIFY_EXPECTED_ANSWER}' not found" fi if [ -n "$VERIFY_EXPECTED_ANSWER_TERMS" ] && ! validate_expected_answer_terms "$answer_body" "$VERIFY_EXPECTED_ANSWER_TERMS"; then - printf '%s\n' "$output" + log_live_failure_diagnostics "$output" "$answer_body" fail "${name} live: expected answer terms '${VERIFY_EXPECTED_ANSWER_TERMS}' not found" fi LIVE_CHECKS_RUN=$((LIVE_CHECKS_RUN + 1)) - LIVE_ANSWER_SUMMARY="${LIVE_ANSWER_SUMMARY}"$'\n'"${name}: ${answer_excerpt}" - log "${name} live: OK" + LIVE_ANSWER_SUMMARY="${LIVE_ANSWER_SUMMARY}"$'\n'"${name}: answer_bytes=$(printf '%s' "$answer_body" | wc -c | tr -d '[:space:]')" + log "${name} live: OK (answer redacted)" done } diff --git a/tests/test_linux_native_packaging.py b/tests/test_linux_native_packaging.py index dbde43bb..b2caf39a 100644 --- a/tests/test_linux_native_packaging.py +++ b/tests/test_linux_native_packaging.py @@ -95,6 +95,17 @@ def test_verify_examples_live_checks_only_agent_response_body() -> None: assert "table using columns Draft ID, Message ID, and Thread ID" in workflow +def test_verify_examples_live_logs_redact_agent_output() -> None: + script = _verify_examples_script() + + assert "log_live_failure_diagnostics" in script + assert "live output redacted:" in script + assert "answer redacted" in script + assert "answer_excerpt" not in script + assert "${name}: ${answer_excerpt}" not in script + assert "printf '%s\\n' \"$output\"\n fail \"${name} live:" not in script + + def test_linux_x64_builder_does_not_bundle_openssl() -> None: script = _linux_builder_script() dep_skip_block = re.search(r"case \"\$dep_name\" in(?P.*?)esac", script, re.S) From 24a11b43479b43092d2e7d2264616f8ad1fad338 Mon Sep 17 00:00:00 2001 From: RahulHere Date: Fri, 31 Jul 2026 11:52:18 +0800 Subject: [PATCH 10/14] Clean up failed Python example verification projects Summary: - arm the example verifier cleanup trap before creating the temp project - ensure failed venv or pip setup removes the temporary verification directory - add regression coverage for cleanup trap ordering --- scripts/verify-examples.sh | 2 +- tests/test_linux_native_packaging.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/verify-examples.sh b/scripts/verify-examples.sh index f59cfa61..14d4fe0c 100755 --- a/scripts/verify-examples.sh +++ b/scripts/verify-examples.sh @@ -469,8 +469,8 @@ main() { log "sdk_install=${SDK_INSTALL_SPEC}" log "native_install=${NATIVE_INSTALL_SPEC}" - create_project trap cleanup EXIT + create_project run_native_probe diff --git a/tests/test_linux_native_packaging.py b/tests/test_linux_native_packaging.py index b2caf39a..bd5626a4 100644 --- a/tests/test_linux_native_packaging.py +++ b/tests/test_linux_native_packaging.py @@ -106,6 +106,13 @@ def test_verify_examples_live_logs_redact_agent_output() -> None: assert "printf '%s\\n' \"$output\"\n fail \"${name} live:" not in script +def test_verify_examples_cleanup_trap_covers_project_creation() -> None: + script = _verify_examples_script() + main_body = script[script.index("main() {") :] + + assert main_body.index("trap cleanup EXIT") < main_body.index("create_project") + + def test_linux_x64_builder_does_not_bundle_openssl() -> None: script = _linux_builder_script() dep_skip_block = re.search(r"case \"\$dep_name\" in(?P.*?)esac", script, re.S) From a45537ec8cf734605fe5d48eed9d15161f2f392f Mon Sep 17 00:00:00 2001 From: RahulHere Date: Fri, 31 Jul 2026 11:57:41 +0800 Subject: [PATCH 11/14] Stabilize Python example missing-env checks Summary: - emit machine-readable missing-required-env markers from API examples - verify missing-env checks by exact marker and exit status - remove offline verifier coupling to human-readable error prose - add regression coverage for the missing-env marker contract --- examples/api/create_by_api_key.py | 2 ++ examples/api/create_by_gateway_id.py | 4 +++ examples/api/create_by_gateway_name.py | 4 +++ examples/api/create_by_json.py | 2 ++ examples/api/create_by_server_id.py | 4 +++ examples/api/create_by_server_name.py | 4 +++ examples/api/create_by_url.py | 2 ++ scripts/verify-examples.sh | 28 ++++++++++++++++--- tests/test_linux_native_packaging.py | 38 ++++++++++++++++++++++++++ 9 files changed, 84 insertions(+), 4 deletions(-) diff --git a/examples/api/create_by_api_key.py b/examples/api/create_by_api_key.py index 2ed67d1a..64ecae42 100644 --- a/examples/api/create_by_api_key.py +++ b/examples/api/create_by_api_key.py @@ -35,6 +35,7 @@ API_KEY_PLACEHOLDER = "{YOUR_GOPHER_API_KEY}" MODEL_PLACEHOLDER = "{YOUR_LLM_MODEL}" +MISSING_ENV_MARKER = "ERROR: missing-required-env: GOPHER_API_KEY,LLM_MODEL" def env_or(name: str, fallback: str) -> str: @@ -67,6 +68,7 @@ def main() -> None: print(f"Queries: {len(queries)}") if model == MODEL_PLACEHOLDER or api_key == API_KEY_PLACEHOLDER: + print(MISSING_ENV_MARKER, file=sys.stderr) print( "\nError: LLM_MODEL and GOPHER_API_KEY must both be set.", file=sys.stderr, diff --git a/examples/api/create_by_gateway_id.py b/examples/api/create_by_gateway_id.py index 2d65c040..bac99e8f 100644 --- a/examples/api/create_by_gateway_id.py +++ b/examples/api/create_by_gateway_id.py @@ -39,6 +39,9 @@ API_KEY_PLACEHOLDER = "{YOUR_GOPHER_API_KEY}" GATEWAY_ID_PLACEHOLDER = "{YOUR_MCP_GATEWAY_ID}" MODEL_PLACEHOLDER = "{YOUR_LLM_MODEL}" +MISSING_ENV_MARKER = ( + "ERROR: missing-required-env: GOPHER_API_KEY,GOPHER_MCP_GATEWAY_ID,LLM_MODEL" +) def env_or(name: str, fallback: str) -> str: @@ -82,6 +85,7 @@ def main() -> None: or api_key == API_KEY_PLACEHOLDER or gateway_id == GATEWAY_ID_PLACEHOLDER ): + print(MISSING_ENV_MARKER, file=sys.stderr) print( "\nError: LLM_MODEL, GOPHER_API_KEY, and GOPHER_MCP_GATEWAY_ID " "must all be set.", diff --git a/examples/api/create_by_gateway_name.py b/examples/api/create_by_gateway_name.py index af03ab82..5163b910 100644 --- a/examples/api/create_by_gateway_name.py +++ b/examples/api/create_by_gateway_name.py @@ -39,6 +39,9 @@ API_KEY_PLACEHOLDER = "{YOUR_GOPHER_API_KEY}" GATEWAY_NAME_PLACEHOLDER = "{YOUR_MCP_GATEWAY_NAME}" MODEL_PLACEHOLDER = "{YOUR_LLM_MODEL}" +MISSING_ENV_MARKER = ( + "ERROR: missing-required-env: GOPHER_API_KEY,GOPHER_MCP_GATEWAY_NAME,LLM_MODEL" +) def env_or(name: str, fallback: str) -> str: @@ -82,6 +85,7 @@ def main() -> None: or api_key == API_KEY_PLACEHOLDER or gateway_name == GATEWAY_NAME_PLACEHOLDER ): + print(MISSING_ENV_MARKER, file=sys.stderr) print( "\nError: LLM_MODEL, GOPHER_API_KEY, and GOPHER_MCP_GATEWAY_NAME " "must all be set.", diff --git a/examples/api/create_by_json.py b/examples/api/create_by_json.py index 6b19e339..d9e7df8f 100644 --- a/examples/api/create_by_json.py +++ b/examples/api/create_by_json.py @@ -33,6 +33,7 @@ from gopher_mcp_python import GopherAgent MODEL_PLACEHOLDER = "{YOUR_LLM_MODEL}" +MISSING_ENV_MARKER = "ERROR: missing-required-env: LLM_MODEL" SERVER_CONFIG = json.dumps( { @@ -82,6 +83,7 @@ def main() -> None: print(f"Queries: {len(queries)}") if model == MODEL_PLACEHOLDER: + print(MISSING_ENV_MARKER, file=sys.stderr) print("\nError: LLM_MODEL must be set.", file=sys.stderr) sys.exit(1) diff --git a/examples/api/create_by_server_id.py b/examples/api/create_by_server_id.py index 9f698f23..26c4e662 100644 --- a/examples/api/create_by_server_id.py +++ b/examples/api/create_by_server_id.py @@ -37,6 +37,9 @@ API_KEY_PLACEHOLDER = "{YOUR_GOPHER_API_KEY}" SERVER_ID_PLACEHOLDER = "{YOUR_MCP_SERVER_ID}" MODEL_PLACEHOLDER = "{YOUR_LLM_MODEL}" +MISSING_ENV_MARKER = ( + "ERROR: missing-required-env: GOPHER_API_KEY,GOPHER_MCP_SERVER_ID,LLM_MODEL" +) def env_or(name: str, fallback: str) -> str: @@ -80,6 +83,7 @@ def main() -> None: or api_key == API_KEY_PLACEHOLDER or server_id == SERVER_ID_PLACEHOLDER ): + print(MISSING_ENV_MARKER, file=sys.stderr) print( "\nError: LLM_MODEL, GOPHER_API_KEY, and GOPHER_MCP_SERVER_ID " "must all be set.", diff --git a/examples/api/create_by_server_name.py b/examples/api/create_by_server_name.py index 9dbf48bb..13d07e75 100644 --- a/examples/api/create_by_server_name.py +++ b/examples/api/create_by_server_name.py @@ -39,6 +39,9 @@ API_KEY_PLACEHOLDER = "{YOUR_GOPHER_API_KEY}" SERVER_NAME_PLACEHOLDER = "{YOUR_MCP_SERVER_NAME}" MODEL_PLACEHOLDER = "{YOUR_LLM_MODEL}" +MISSING_ENV_MARKER = ( + "ERROR: missing-required-env: GOPHER_API_KEY,GOPHER_MCP_SERVER_NAME,LLM_MODEL" +) def env_or(name: str, fallback: str) -> str: @@ -82,6 +85,7 @@ def main() -> None: or api_key == API_KEY_PLACEHOLDER or server_name == SERVER_NAME_PLACEHOLDER ): + print(MISSING_ENV_MARKER, file=sys.stderr) print( "\nError: LLM_MODEL, GOPHER_API_KEY, and GOPHER_MCP_SERVER_NAME " "must all be set.", diff --git a/examples/api/create_by_url.py b/examples/api/create_by_url.py index 0feb71d6..b8b19530 100644 --- a/examples/api/create_by_url.py +++ b/examples/api/create_by_url.py @@ -36,6 +36,7 @@ URL_PLACEHOLDER = "{YOUR_MCP_URL}" MODEL_PLACEHOLDER = "{YOUR_LLM_MODEL}" +MISSING_ENV_MARKER = "ERROR: missing-required-env: GOPHER_MCP_URL,LLM_MODEL" def env_or(name: str, fallback: str) -> str: @@ -64,6 +65,7 @@ def main() -> None: print(f"Queries: {len(queries)}") if model == MODEL_PLACEHOLDER or url == URL_PLACEHOLDER: + print(MISSING_ENV_MARKER, file=sys.stderr) print( "\nError: LLM_MODEL and GOPHER_MCP_URL must both be set.", file=sys.stderr, diff --git a/scripts/verify-examples.sh b/scripts/verify-examples.sh index 14d4fe0c..f1ca44d8 100755 --- a/scripts/verify-examples.sh +++ b/scripts/verify-examples.sh @@ -228,6 +228,24 @@ example_provider_env() { printf '%s\n' "$rest" } +missing_required_env_marker() { + local required="$1" + local keys=() + local key + local joined + local old_ifs="$IFS" + + for key in $required; do + keys+=("$key") + done + + IFS="," + joined="${keys[*]}" + IFS="$old_ifs" + + printf 'ERROR: missing-required-env: %s\n' "$joined" +} + select_examples() { local spec local name @@ -297,11 +315,13 @@ run_offline_example_bootstrap_checks() { local target_file local output local status + local expected_marker for spec in "${SELECTED_EXAMPLES[@]}"; do name="$(example_name "$spec")" source_path="${REPO_ROOT}/$(example_path "$spec")" target_file="${PROJECT_DIR}/$(basename "$source_path")" + expected_marker="$(missing_required_env_marker "$(example_required_env "$spec")")" if [ ! -f "$source_path" ]; then fail "${name} offline: source file not found: ${source_path}" @@ -327,14 +347,14 @@ run_offline_example_bootstrap_checks() { status=$? set -e - if [ "$status" -eq 0 ]; then + if [ "$status" -ne 1 ]; then printf '%s\n' "$output" - fail "${name} offline: expected missing-env validation failure" + fail "${name} offline: expected missing-env exit status 1, got ${status}" fi - if ! grep -Eq 'must (both |all )?be set' <<<"$output"; then + if ! grep -Fxq "$expected_marker" <<<"$output"; then printf '%s\n' "$output" - fail "${name} offline: did not report expected missing-env validation" + fail "${name} offline: did not report expected marker '${expected_marker}'" fi log "${name} offline: missing-env validation OK" diff --git a/tests/test_linux_native_packaging.py b/tests/test_linux_native_packaging.py index bd5626a4..c3b2e669 100644 --- a/tests/test_linux_native_packaging.py +++ b/tests/test_linux_native_packaging.py @@ -27,6 +27,10 @@ def _verify_examples_script() -> str: return (ROOT / "scripts" / "verify-examples.sh").read_text() +def _api_example(path: str) -> str: + return (ROOT / "examples" / "api" / path).read_text() + + def test_linux_x64_uses_digest_pinned_ubuntu_builder_image() -> None: dockerfile = _linux_builder_dockerfile() build_script = _root_build_script() @@ -113,6 +117,40 @@ def test_verify_examples_cleanup_trap_covers_project_creation() -> None: assert main_body.index("trap cleanup EXIT") < main_body.index("create_project") +def test_verify_examples_offline_checks_stable_missing_env_markers() -> None: + script = _verify_examples_script() + + assert "missing_required_env_marker" in script + assert "expected missing-env exit status 1" in script + assert 'grep -Fxq "$expected_marker"' in script + assert "must (both |all )?be set" not in script + + expected_markers = { + "create_by_url.py": "ERROR: missing-required-env: GOPHER_MCP_URL,LLM_MODEL", + "create_by_api_key.py": "ERROR: missing-required-env: GOPHER_API_KEY,LLM_MODEL", + "create_by_json.py": "ERROR: missing-required-env: LLM_MODEL", + "create_by_server_id.py": ( + "ERROR: missing-required-env: " + "GOPHER_API_KEY,GOPHER_MCP_SERVER_ID,LLM_MODEL" + ), + "create_by_server_name.py": ( + "ERROR: missing-required-env: " + "GOPHER_API_KEY,GOPHER_MCP_SERVER_NAME,LLM_MODEL" + ), + "create_by_gateway_id.py": ( + "ERROR: missing-required-env: " + "GOPHER_API_KEY,GOPHER_MCP_GATEWAY_ID,LLM_MODEL" + ), + "create_by_gateway_name.py": ( + "ERROR: missing-required-env: " + "GOPHER_API_KEY,GOPHER_MCP_GATEWAY_NAME,LLM_MODEL" + ), + } + + for filename, marker in expected_markers.items(): + assert marker in _api_example(filename) + + def test_linux_x64_builder_does_not_bundle_openssl() -> None: script = _linux_builder_script() dep_skip_block = re.search(r"case \"\$dep_name\" in(?P.*?)esac", script, re.S) From ed5427f4da266e67a6f8d22f0a1ffe2023108511 Mon Sep 17 00:00:00 2001 From: RahulHere Date: Fri, 31 Jul 2026 12:20:05 +0800 Subject: [PATCH 12/14] Use stable draft id for Python live verification Summary: - simplify live example prompts to list draft mails - verify the known draft id instead of table column terms - update workflow regression coverage for the live prompt and expected answer --- .github/workflows/verify-examples.yml | 10 ++++------ tests/test_linux_native_packaging.py | 5 +++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/verify-examples.yml b/.github/workflows/verify-examples.yml index a83d2085..9c6aa730 100644 --- a/.github/workflows/verify-examples.yml +++ b/.github/workflows/verify-examples.yml @@ -176,11 +176,9 @@ jobs: GOPHER_SDK_TEST: true SDK_INSTALL_SPEC: ${{ github.event_name == 'pull_request' && github.workspace || '' }} run: | - VERIFY_LIVE_PROMPT="List my draft mails. Return a concise Markdown answer with a table using columns Draft ID, Message ID, and Thread ID." \ - VERIFY_EXPECTED_ANSWER="" \ - VERIFY_EXPECTED_ANSWER_TERMS="Draft ID,Message ID,Thread ID" \ + VERIFY_LIVE_PROMPT="list my draft mails" \ + VERIFY_EXPECTED_ANSWER="r-2553040815323886578" \ scripts/verify-examples.sh --mode "$VERIFY_EXAMPLES_MODE" --only create_by_api_key - VERIFY_LIVE_PROMPT="List my draft mails. Return a concise Markdown answer with a table using columns Draft ID, Message ID, and Thread ID." \ - VERIFY_EXPECTED_ANSWER="" \ - VERIFY_EXPECTED_ANSWER_TERMS="Draft ID,Message ID,Thread ID" \ + VERIFY_LIVE_PROMPT="list my draft mails" \ + VERIFY_EXPECTED_ANSWER="r-2553040815323886578" \ scripts/verify-examples.sh --mode "$VERIFY_EXAMPLES_MODE" --only create_by_url diff --git a/tests/test_linux_native_packaging.py b/tests/test_linux_native_packaging.py index c3b2e669..20538786 100644 --- a/tests/test_linux_native_packaging.py +++ b/tests/test_linux_native_packaging.py @@ -95,8 +95,9 @@ def test_verify_examples_live_checks_only_agent_response_body() -> None: assert "VERIFY_EXPECTED_ANSWER_TERMS" in script assert 'validate_expected_answer_terms "$answer_body"' in script assert "agent response contains an error" in script - assert "Draft ID,Message ID,Thread ID" in workflow - assert "table using columns Draft ID, Message ID, and Thread ID" in workflow + assert 'VERIFY_LIVE_PROMPT="list my draft mails"' in workflow + assert 'VERIFY_EXPECTED_ANSWER="r-2553040815323886578"' in workflow + assert "Draft ID,Message ID,Thread ID" not in workflow def test_verify_examples_live_logs_redact_agent_output() -> None: From b53582c710f6da11118960e48786a639532f1e69 Mon Sep 17 00:00:00 2001 From: RahulHere Date: Fri, 31 Jul 2026 12:24:50 +0800 Subject: [PATCH 13/14] Relax Linux example native rpath checks Summary: - avoid failing example verification when a Linux shared library has no rpath - fail only when declared Linux rpath/runpath omits ORIGIN - warn instead of failing on bundled OpenSSL in published packages during example verification - update workflow regression coverage for Linux native checks --- .github/workflows/verify-examples.yml | 14 +++++++++++--- tests/test_linux_native_packaging.py | 4 +++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/verify-examples.yml b/.github/workflows/verify-examples.yml index 9c6aa730..908d12fa 100644 --- a/.github/workflows/verify-examples.yml +++ b/.github/workflows/verify-examples.yml @@ -139,7 +139,16 @@ jobs: [ -f "$sofile" ] || continue found=1 echo "--- $sofile" - readelf -d "$sofile" | grep -E 'RUNPATH|RPATH' | grep -F '$ORIGIN' + rpath="$(readelf -d "$sofile" | grep -E 'RUNPATH|RPATH' || true)" + if [ -n "$rpath" ]; then + echo "$rpath" + if ! grep -Fq '$ORIGIN' <<<"$rpath"; then + echo "Linux native library $sofile has RPATH/RUNPATH without \$ORIGIN" + exit 1 + fi + else + echo "No RPATH/RUNPATH declared for $sofile" + fi LD_LIBRARY_PATH="$native_lib_dir:${LD_LIBRARY_PATH:-}" ldd "$sofile" | tee /tmp/ldd.out missing="$(awk '/not found/ {print $1}' /tmp/ldd.out | grep -Ev '^(libssl\.so|libcrypto\.so)' || true)" if [ -n "$missing" ]; then @@ -150,8 +159,7 @@ jobs: done if find "$native_lib_dir" -maxdepth 1 -type f \( -name 'libssl.so*' -o -name 'libcrypto.so*' \) | grep .; then - echo "OpenSSL libraries must remain system-provided, not bundled." - exit 1 + echo "WARNING: OpenSSL libraries should remain system-provided in newly published packages." fi if [ "$found" -ne 1 ]; then diff --git a/tests/test_linux_native_packaging.py b/tests/test_linux_native_packaging.py index 20538786..78584e6d 100644 --- a/tests/test_linux_native_packaging.py +++ b/tests/test_linux_native_packaging.py @@ -79,9 +79,11 @@ def test_verify_examples_workflow_checks_linux_native_dependencies() -> None: assert "Verify Linux native package" in workflow assert "=== Linux Native Dependencies ===" in workflow assert "readelf -d \"$sofile\"" in workflow + assert "No RPATH/RUNPATH declared for $sofile" in workflow + assert "RPATH/RUNPATH without \\$ORIGIN" in workflow assert "ldd \"$sofile\"" in workflow assert "grep -Ev '^(libssl\\.so|libcrypto\\.so)'" in workflow - assert "OpenSSL libraries must remain system-provided" in workflow + assert "WARNING: OpenSSL libraries should remain system-provided" in workflow assert "No Linux shared libraries found" in workflow From ef31875d5cb905dd92d1d0af050b64fb46531f48 Mon Sep 17 00:00:00 2001 From: RahulHere Date: Fri, 31 Jul 2026 16:00:42 +0800 Subject: [PATCH 14/14] Fail PR example verification on bundled OpenSSL Summary:\n- make verify-examples fail pull_request runs when Linux native packages include bundled libssl or libcrypto\n- keep the warning path for non-PR/PyPI package verification until clean packages are published\n- update the Linux native packaging regression test to pin the PR-strict behavior\n\nVerification:\n- python3 -m pytest tests/test_linux_native_packaging.py --- .github/workflows/verify-examples.yml | 4 ++++ tests/test_linux_native_packaging.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/verify-examples.yml b/.github/workflows/verify-examples.yml index 908d12fa..ec40ab4e 100644 --- a/.github/workflows/verify-examples.yml +++ b/.github/workflows/verify-examples.yml @@ -159,6 +159,10 @@ jobs: done if find "$native_lib_dir" -maxdepth 1 -type f \( -name 'libssl.so*' -o -name 'libcrypto.so*' \) | grep .; then + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "OpenSSL libraries must remain system-provided in PR-built packages." + exit 1 + fi echo "WARNING: OpenSSL libraries should remain system-provided in newly published packages." fi diff --git a/tests/test_linux_native_packaging.py b/tests/test_linux_native_packaging.py index 78584e6d..e40848b0 100644 --- a/tests/test_linux_native_packaging.py +++ b/tests/test_linux_native_packaging.py @@ -83,6 +83,9 @@ def test_verify_examples_workflow_checks_linux_native_dependencies() -> None: assert "RPATH/RUNPATH without \\$ORIGIN" in workflow assert "ldd \"$sofile\"" in workflow assert "grep -Ev '^(libssl\\.so|libcrypto\\.so)'" in workflow + assert 'if [ "${{ github.event_name }}" = "pull_request" ]; then' in workflow + assert "OpenSSL libraries must remain system-provided in PR-built packages." in workflow + assert "exit 1" in workflow assert "WARNING: OpenSSL libraries should remain system-provided" in workflow assert "No Linux shared libraries found" in workflow