diff --git a/.github/workflows/verify-examples.yml b/.github/workflows/verify-examples.yml new file mode 100644 index 00000000..ec40ab4e --- /dev/null +++ b/.github/workflows/verify-examples.yml @@ -0,0 +1,196 @@ +name: Verify SDK Examples + +on: + pull_request: + branches: [main] + push: + branches: [iml_verify_auto] + workflow_dispatch: + inputs: + mode: + description: 'offline, live, or auto' + required: true + default: 'auto' + pypi_version: + description: 'PyPI version to verify' + required: true + default: 'latest' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify-examples: + name: ${{ matrix.platform }} + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + env: + VERIFY_EXAMPLES_MODE: ${{ github.event_name == 'workflow_dispatch' && inputs.mode || 'auto' }} + VERIFY_PYPI_VERSION: ${{ github.event_name == 'workflow_dispatch' && inputs.pypi_version || 'latest' }} + strategy: + fail-fast: false + matrix: + include: + - platform: darwin-arm64 + os: macos-15 + native_module: gopher_mcp_python_native_darwin_arm64 + - platform: linux-x64 + os: ubuntu-22.04 + native_module: gopher_mcp_python_native_linux_x64 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Remove Homebrew on macOS + if: runner.os == 'macOS' && runner.environment == 'github-hosted' + shell: bash + run: | + echo "=== Removing Homebrew to simulate clean machine ===" + sudo rm -rf /opt/homebrew /usr/local/Homebrew /usr/local/Cellar + sudo rm -f /usr/local/bin/brew /opt/homebrew/bin/brew + echo "Homebrew removed" + + - name: Install SDK package for native preflight + shell: bash + run: | + python -m venv native-preflight + source native-preflight/bin/activate + python -m pip install --upgrade pip + if [ "${{ github.event_name }}" = "pull_request" ]; then + python -m pip install -e . "gopher-mcp-python-native-${{ matrix.platform }}" + elif [ "$VERIFY_PYPI_VERSION" = "latest" ]; then + python -m pip install gopher-mcp-python gopher-mcp-python-native-${{ matrix.platform }} + else + python -m pip install "gopher-mcp-python==${VERIFY_PYPI_VERSION}" "gopher-mcp-python-native-${{ matrix.platform }}==${VERIFY_PYPI_VERSION}" + fi + + - name: Verify macOS native package + if: runner.os == 'macOS' + shell: bash + run: | + source native-preflight/bin/activate + native_lib_dir="$( + python -c 'import importlib; module = importlib.import_module("${{ matrix.native_module }}"); print(module.get_lib_path())' + )" + cd "$native_lib_dir" + + echo "=== Code Signatures ===" + failed=0 + for f in *.dylib; do + if codesign -v "$f" 2>&1; then + echo "OK $f" + else + echo "FAILED $f: $(codesign -v "$f" 2>&1)" + failed=1 + fi + done + [ "$failed" -eq 0 ] || exit 1 + + echo "=== Homebrew Path Check ===" + for f in *.dylib; do + [ -L "$f" ] && continue + if otool -L "$f" | grep -qE '/usr/local/|/opt/homebrew/'; then + echo "FAILED $f has Homebrew paths:" + otool -L "$f" | grep -E '/usr/local/|/opt/homebrew/' + exit 1 + fi + done + + echo "=== Bundled @loader_path Dependencies ===" + missing=0 + for f in *.dylib; do + [ -L "$f" ] && continue + while read -r dep rest; do + name="${dep#@loader_path/}" + if [ ! -f "$name" ]; then + echo "MISSING $name needed by $f" + missing=1 + fi + done < <(otool -L "$f" | grep "@loader_path/" || true) + done + [ "$missing" -eq 0 ] || exit 1 + + - name: Verify Linux native package + if: runner.os == 'Linux' + shell: bash + run: | + set -euo pipefail + + source native-preflight/bin/activate + native_lib_dir="$( + python -c 'import importlib; module = importlib.import_module("${{ matrix.native_module }}"); print(module.get_lib_path())' + )" + cd "$native_lib_dir" + + echo "=== Linux Native Dependencies ===" + found=0 + for sofile in *.so *.so.*; do + [ -L "$sofile" ] && continue + [ -f "$sofile" ] || continue + found=1 + echo "--- $sofile" + rpath="$(readelf -d "$sofile" | grep -E 'RUNPATH|RPATH' || true)" + if [ -n "$rpath" ]; then + echo "$rpath" + if ! grep -Fq '$ORIGIN' <<<"$rpath"; then + echo "Linux native library $sofile has RPATH/RUNPATH without \$ORIGIN" + exit 1 + fi + else + echo "No RPATH/RUNPATH declared for $sofile" + fi + LD_LIBRARY_PATH="$native_lib_dir:${LD_LIBRARY_PATH:-}" ldd "$sofile" | tee /tmp/ldd.out + missing="$(awk '/not found/ {print $1}' /tmp/ldd.out | grep -Ev '^(libssl\.so|libcrypto\.so)' || true)" + if [ -n "$missing" ]; then + echo "$missing" + echo "Missing shared dependency for $sofile" + exit 1 + fi + done + + if find "$native_lib_dir" -maxdepth 1 -type f \( -name 'libssl.so*' -o -name 'libcrypto.so*' \) | grep .; then + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "OpenSSL libraries must remain system-provided in PR-built packages." + exit 1 + fi + echo "WARNING: OpenSSL libraries should remain system-provided in newly published packages." + fi + + if [ "$found" -ne 1 ]; then + echo "No Linux shared libraries found in $native_lib_dir" + exit 1 + fi + + - name: Verify native loading + shell: bash + run: | + source native-preflight/bin/activate + python scripts/verify-example-native-probe.py + + - name: Verify examples + shell: bash + env: + LLM_PROVIDER: ${{ secrets.LLM_PROVIDER || 'AnthropicProvider' }} + LLM_MODEL: ${{ secrets.LLM_MODEL }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOPHER_API_KEY: ${{ secrets.GOPHER_API_KEY }} + GOPHER_MCP_URL: ${{ secrets.GOPHER_MCP_URL }} + GOPHER_SDK_TEST: true + SDK_INSTALL_SPEC: ${{ github.event_name == 'pull_request' && github.workspace || '' }} + run: | + VERIFY_LIVE_PROMPT="list my draft mails" \ + VERIFY_EXPECTED_ANSWER="r-2553040815323886578" \ + scripts/verify-examples.sh --mode "$VERIFY_EXAMPLES_MODE" --only create_by_api_key + VERIFY_LIVE_PROMPT="list my draft mails" \ + VERIFY_EXPECTED_ANSWER="r-2553040815323886578" \ + scripts/verify-examples.sh --mode "$VERIFY_EXAMPLES_MODE" --only create_by_url diff --git a/examples/api/create_by_api_key.py b/examples/api/create_by_api_key.py index 2ed67d1a..64ecae42 100644 --- a/examples/api/create_by_api_key.py +++ b/examples/api/create_by_api_key.py @@ -35,6 +35,7 @@ API_KEY_PLACEHOLDER = "{YOUR_GOPHER_API_KEY}" MODEL_PLACEHOLDER = "{YOUR_LLM_MODEL}" +MISSING_ENV_MARKER = "ERROR: missing-required-env: GOPHER_API_KEY,LLM_MODEL" def env_or(name: str, fallback: str) -> str: @@ -67,6 +68,7 @@ def main() -> None: print(f"Queries: {len(queries)}") if model == MODEL_PLACEHOLDER or api_key == API_KEY_PLACEHOLDER: + print(MISSING_ENV_MARKER, file=sys.stderr) print( "\nError: LLM_MODEL and GOPHER_API_KEY must both be set.", file=sys.stderr, diff --git a/examples/api/create_by_gateway_id.py b/examples/api/create_by_gateway_id.py index 2d65c040..bac99e8f 100644 --- a/examples/api/create_by_gateway_id.py +++ b/examples/api/create_by_gateway_id.py @@ -39,6 +39,9 @@ API_KEY_PLACEHOLDER = "{YOUR_GOPHER_API_KEY}" GATEWAY_ID_PLACEHOLDER = "{YOUR_MCP_GATEWAY_ID}" MODEL_PLACEHOLDER = "{YOUR_LLM_MODEL}" +MISSING_ENV_MARKER = ( + "ERROR: missing-required-env: GOPHER_API_KEY,GOPHER_MCP_GATEWAY_ID,LLM_MODEL" +) def env_or(name: str, fallback: str) -> str: @@ -82,6 +85,7 @@ def main() -> None: or api_key == API_KEY_PLACEHOLDER or gateway_id == GATEWAY_ID_PLACEHOLDER ): + print(MISSING_ENV_MARKER, file=sys.stderr) print( "\nError: LLM_MODEL, GOPHER_API_KEY, and GOPHER_MCP_GATEWAY_ID " "must all be set.", diff --git a/examples/api/create_by_gateway_name.py b/examples/api/create_by_gateway_name.py index af03ab82..5163b910 100644 --- a/examples/api/create_by_gateway_name.py +++ b/examples/api/create_by_gateway_name.py @@ -39,6 +39,9 @@ API_KEY_PLACEHOLDER = "{YOUR_GOPHER_API_KEY}" GATEWAY_NAME_PLACEHOLDER = "{YOUR_MCP_GATEWAY_NAME}" MODEL_PLACEHOLDER = "{YOUR_LLM_MODEL}" +MISSING_ENV_MARKER = ( + "ERROR: missing-required-env: GOPHER_API_KEY,GOPHER_MCP_GATEWAY_NAME,LLM_MODEL" +) def env_or(name: str, fallback: str) -> str: @@ -82,6 +85,7 @@ def main() -> None: or api_key == API_KEY_PLACEHOLDER or gateway_name == GATEWAY_NAME_PLACEHOLDER ): + print(MISSING_ENV_MARKER, file=sys.stderr) print( "\nError: LLM_MODEL, GOPHER_API_KEY, and GOPHER_MCP_GATEWAY_NAME " "must all be set.", diff --git a/examples/api/create_by_json.py b/examples/api/create_by_json.py index 6b19e339..d9e7df8f 100644 --- a/examples/api/create_by_json.py +++ b/examples/api/create_by_json.py @@ -33,6 +33,7 @@ from gopher_mcp_python import GopherAgent MODEL_PLACEHOLDER = "{YOUR_LLM_MODEL}" +MISSING_ENV_MARKER = "ERROR: missing-required-env: LLM_MODEL" SERVER_CONFIG = json.dumps( { @@ -82,6 +83,7 @@ def main() -> None: print(f"Queries: {len(queries)}") if model == MODEL_PLACEHOLDER: + print(MISSING_ENV_MARKER, file=sys.stderr) print("\nError: LLM_MODEL must be set.", file=sys.stderr) sys.exit(1) diff --git a/examples/api/create_by_server_id.py b/examples/api/create_by_server_id.py index 9f698f23..26c4e662 100644 --- a/examples/api/create_by_server_id.py +++ b/examples/api/create_by_server_id.py @@ -37,6 +37,9 @@ API_KEY_PLACEHOLDER = "{YOUR_GOPHER_API_KEY}" SERVER_ID_PLACEHOLDER = "{YOUR_MCP_SERVER_ID}" MODEL_PLACEHOLDER = "{YOUR_LLM_MODEL}" +MISSING_ENV_MARKER = ( + "ERROR: missing-required-env: GOPHER_API_KEY,GOPHER_MCP_SERVER_ID,LLM_MODEL" +) def env_or(name: str, fallback: str) -> str: @@ -80,6 +83,7 @@ def main() -> None: or api_key == API_KEY_PLACEHOLDER or server_id == SERVER_ID_PLACEHOLDER ): + print(MISSING_ENV_MARKER, file=sys.stderr) print( "\nError: LLM_MODEL, GOPHER_API_KEY, and GOPHER_MCP_SERVER_ID " "must all be set.", diff --git a/examples/api/create_by_server_name.py b/examples/api/create_by_server_name.py index 9dbf48bb..13d07e75 100644 --- a/examples/api/create_by_server_name.py +++ b/examples/api/create_by_server_name.py @@ -39,6 +39,9 @@ API_KEY_PLACEHOLDER = "{YOUR_GOPHER_API_KEY}" SERVER_NAME_PLACEHOLDER = "{YOUR_MCP_SERVER_NAME}" MODEL_PLACEHOLDER = "{YOUR_LLM_MODEL}" +MISSING_ENV_MARKER = ( + "ERROR: missing-required-env: GOPHER_API_KEY,GOPHER_MCP_SERVER_NAME,LLM_MODEL" +) def env_or(name: str, fallback: str) -> str: @@ -82,6 +85,7 @@ def main() -> None: or api_key == API_KEY_PLACEHOLDER or server_name == SERVER_NAME_PLACEHOLDER ): + print(MISSING_ENV_MARKER, file=sys.stderr) print( "\nError: LLM_MODEL, GOPHER_API_KEY, and GOPHER_MCP_SERVER_NAME " "must all be set.", diff --git a/examples/api/create_by_url.py b/examples/api/create_by_url.py index 0feb71d6..b8b19530 100644 --- a/examples/api/create_by_url.py +++ b/examples/api/create_by_url.py @@ -36,6 +36,7 @@ URL_PLACEHOLDER = "{YOUR_MCP_URL}" MODEL_PLACEHOLDER = "{YOUR_LLM_MODEL}" +MISSING_ENV_MARKER = "ERROR: missing-required-env: GOPHER_MCP_URL,LLM_MODEL" def env_or(name: str, fallback: str) -> str: @@ -64,6 +65,7 @@ def main() -> None: print(f"Queries: {len(queries)}") if model == MODEL_PLACEHOLDER or url == URL_PLACEHOLDER: + print(MISSING_ENV_MARKER, file=sys.stderr) print( "\nError: LLM_MODEL and GOPHER_MCP_URL must both be set.", file=sys.stderr, diff --git a/scripts/verify-example-native-probe.py b/scripts/verify-example-native-probe.py new file mode 100755 index 00000000..7a1c360a --- /dev/null +++ b/scripts/verify-example-native-probe.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Verify that the installed Python SDK can load the native gopher-orch library.""" + +import sys + +from gopher_mcp_python.ffi.library import GopherOrchLibrary + + +def fail(message: str) -> None: + print(f"[verify-native] error: {message}", file=sys.stderr) + sys.exit(1) + + +def main() -> None: + library = GopherOrchLibrary.get_instance() + if library is None: + fail(GopherOrchLibrary.get_load_error_message()) + + print("[verify-native] native library loaded") + + required_symbols = ( + "gopher_orch_agent_create_by_json", + "gopher_orch_agent_create_by_api_key", + "gopher_orch_agent_run", + "gopher_orch_agent_release", + "gopher_orch_api_fetch_servers", + "gopher_orch_last_error", + "gopher_orch_clear_error", + "gopher_orch_free", + ) + native_library = getattr(library, "_lib", None) + missing = [ + symbol + for symbol in required_symbols + if native_library is None or not hasattr(native_library, symbol) + ] + if missing: + fail(f"native library missing required symbols: {', '.join(missing)}") + + print("[verify-native] required symbols present") + + +if __name__ == "__main__": + main() diff --git a/scripts/verify-examples.sh b/scripts/verify-examples.sh new file mode 100755 index 00000000..f1ca44d8 --- /dev/null +++ b/scripts/verify-examples.sh @@ -0,0 +1,522 @@ +#!/usr/bin/env bash + +set -euo pipefail + +MODE="${VERIFY_EXAMPLES_MODE:-auto}" +ONLY_EXAMPLE="" +ENV_FILE="${VERIFY_EXAMPLES_ENV_FILE:-}" +PYTHON_BIN="${PYTHON_BIN:-python3}" +PYTHON_VERSION="" +PLATFORM="" +NATIVE_PACKAGE="" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +TEMP_ROOT="" +PROJECT_DIR="" +VENV_DIR="" +SDK_INSTALL_SPEC="${SDK_INSTALL_SPEC:-}" +NATIVE_INSTALL_SPEC="${NATIVE_INSTALL_SPEC:-}" +SDK_VERSION="${VERIFY_PYPI_VERSION:-latest}" +VERIFY_LIVE_PROMPT="${VERIFY_LIVE_PROMPT:-What tools do we have?}" +VERIFY_EXPECTED_ANSWER="${VERIFY_EXPECTED_ANSWER:-tool}" +VERIFY_EXPECTED_ANSWER_TERMS="${VERIFY_EXPECTED_ANSWER_TERMS:-}" +LIVE_CHECKS_RUN=0 +LIVE_CHECKS_SKIPPED=0 +LIVE_ANSWER_SUMMARY="" +SELECTED_EXAMPLES=() +EXAMPLES=( + "create_by_url|examples/api/create_by_url.py|GOPHER_MCP_URL LLM_MODEL|ANTHROPIC_API_KEY" + "create_by_api_key|examples/api/create_by_api_key.py|GOPHER_API_KEY LLM_MODEL|ANTHROPIC_API_KEY" + "create_by_json|examples/api/create_by_json.py|LLM_MODEL|ANTHROPIC_API_KEY" + "create_by_server_id|examples/api/create_by_server_id.py|GOPHER_API_KEY GOPHER_MCP_SERVER_ID LLM_MODEL|ANTHROPIC_API_KEY" + "create_by_server_name|examples/api/create_by_server_name.py|GOPHER_API_KEY GOPHER_MCP_SERVER_NAME LLM_MODEL|ANTHROPIC_API_KEY" + "create_by_gateway_id|examples/api/create_by_gateway_id.py|GOPHER_API_KEY GOPHER_MCP_GATEWAY_ID LLM_MODEL|ANTHROPIC_API_KEY" + "create_by_gateway_name|examples/api/create_by_gateway_name.py|GOPHER_API_KEY GOPHER_MCP_GATEWAY_NAME LLM_MODEL|ANTHROPIC_API_KEY" +) + +usage() { + cat <<'EOF' +Usage: scripts/verify-examples.sh [options] + +Options: + --mode Verification mode (default: auto) + --only Run one example by registry name + --env-file Load live environment variables from a file + -h, --help Show this help + +Environment: + VERIFY_EXAMPLES_ENV_FILE Default env file path + VERIFY_PYPI_VERSION PyPI version to verify, or latest (default: latest) + SDK_INSTALL_SPEC Override SDK pip install spec + NATIVE_INSTALL_SPEC Override native package pip install spec + VERIFY_LIVE_PROMPT Prompt used for live agent.run() checks + VERIFY_EXPECTED_ANSWER Text that must appear in the live answer + VERIFY_EXPECTED_ANSWER_TERMS Comma-separated terms that must all appear in the live answer +EOF +} + +log() { + printf '[verify-examples] %s\n' "$*" +} + +fail() { + log "error: $*" + exit 1 +} + +log_live_failure_diagnostics() { + local output="$1" + local answer_body="$2" + local output_bytes + local output_lines + local answer_bytes + local answer_lines + local marker_present="false" + + output_bytes="$(printf '%s' "$output" | wc -c | tr -d '[:space:]')" + output_lines="$(printf '%s\n' "$output" | wc -l | tr -d '[:space:]')" + answer_bytes="$(printf '%s' "$answer_body" | wc -c | tr -d '[:space:]')" + answer_lines="$(printf '%s\n' "$answer_body" | wc -l | tr -d '[:space:]')" + if grep -q 'Agent Response' <<<"$output"; then + marker_present="true" + fi + + log "live output redacted: output_bytes=${output_bytes} output_lines=${output_lines} agent_response_marker=${marker_present} answer_bytes=${answer_bytes} answer_lines=${answer_lines}" +} + +parse_args() { + while [ "$#" -gt 0 ]; do + case "$1" in + --mode) + [ "$#" -ge 2 ] || fail "--mode requires a value" + MODE="$2" + shift 2 + ;; + --only) + [ "$#" -ge 2 ] || fail "--only requires a value" + ONLY_EXAMPLE="$2" + shift 2 + ;; + --env-file) + [ "$#" -ge 2 ] || fail "--env-file requires a value" + ENV_FILE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + fail "unknown argument: $1" + ;; + esac + done +} + +validate_args() { + case "$MODE" in + offline|live|auto) ;; + *) fail "invalid --mode '${MODE}'; expected offline, live, or auto" ;; + esac + + if [ -n "$ONLY_EXAMPLE" ] && ! [[ "$ONLY_EXAMPLE" =~ ^[A-Za-z0-9_-]+$ ]]; then + fail "--only must be an example name containing only letters, numbers, '_' or '-'" + fi +} + +load_env_file() { + if [ -z "$ENV_FILE" ]; then + return + fi + + if [ ! -f "$ENV_FILE" ]; then + fail "env file not found: ${ENV_FILE}" + fi + + set -a + # shellcheck source=/dev/null + . "$ENV_FILE" + set +a + log "env_file=${ENV_FILE}" +} + +require_python() { + if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + fail "Python 3.8 or newer is required, but ${PYTHON_BIN} was not found in PATH" + fi + + PYTHON_VERSION="$("$PYTHON_BIN" - <<'PY' +import sys +print(".".join(str(part) for part in sys.version_info[:3])) +raise SystemExit(0 if sys.version_info >= (3, 8) else 1) +PY +)" || fail "Python 3.8 or newer is required; current version is ${PYTHON_VERSION:-unknown}" +} + +detect_platform() { + local os + local arch + + os="$(uname -s 2>/dev/null || true)" + arch="$(uname -m 2>/dev/null || true)" + + case "${os}:${arch}" in + Darwin:arm64) + PLATFORM="darwin-arm64" + ;; + Darwin:x86_64|Darwin:amd64) + PLATFORM="darwin-x64" + ;; + Linux:x86_64|Linux:amd64) + PLATFORM="linux-x64" + ;; + Linux:aarch64|Linux:arm64) + PLATFORM="linux-arm64" + ;; + MINGW*:x86_64|MSYS*:x86_64|CYGWIN*:x86_64|Windows_NT:x86_64|Windows_NT:amd64) + PLATFORM="win32-x64" + ;; + *) + fail "unsupported platform '${os:-unknown}' '${arch:-unknown}'" + ;; + esac + + NATIVE_PACKAGE="gopher-mcp-python-native-${PLATFORM}" +} + +compute_install_specs() { + if [ -z "$SDK_INSTALL_SPEC" ]; then + if [ "$SDK_VERSION" = "latest" ]; then + SDK_INSTALL_SPEC="gopher-mcp-python" + else + SDK_INSTALL_SPEC="gopher-mcp-python==${SDK_VERSION}" + fi + fi + + if [ -z "$NATIVE_INSTALL_SPEC" ]; then + if [ "$SDK_VERSION" = "latest" ]; then + NATIVE_INSTALL_SPEC="${NATIVE_PACKAGE}" + else + NATIVE_INSTALL_SPEC="${NATIVE_PACKAGE}==${SDK_VERSION}" + fi + fi +} + +example_name() { + local spec="$1" + printf '%s\n' "${spec%%|*}" +} + +example_path() { + local spec="$1" + local rest="${spec#*|}" + printf '%s\n' "${rest%%|*}" +} + +example_required_env() { + local spec="$1" + local rest="${spec#*|}" + rest="${rest#*|}" + printf '%s\n' "${rest%%|*}" +} + +example_provider_env() { + local spec="$1" + local rest="${spec#*|}" + rest="${rest#*|}" + rest="${rest#*|}" + printf '%s\n' "$rest" +} + +missing_required_env_marker() { + local required="$1" + local keys=() + local key + local joined + local old_ifs="$IFS" + + for key in $required; do + keys+=("$key") + done + + IFS="," + joined="${keys[*]}" + IFS="$old_ifs" + + printf 'ERROR: missing-required-env: %s\n' "$joined" +} + +select_examples() { + local spec + local name + local found=0 + + SELECTED_EXAMPLES=() + + for spec in "${EXAMPLES[@]}"; do + name="$(example_name "$spec")" + if [ -z "$ONLY_EXAMPLE" ] || [ "$ONLY_EXAMPLE" = "$name" ]; then + SELECTED_EXAMPLES+=("$spec") + found=1 + fi + done + + if [ "$found" -ne 1 ]; then + fail "unknown example '${ONLY_EXAMPLE}'; supported examples are create_by_url, create_by_api_key, create_by_json, create_by_server_id, create_by_server_name, create_by_gateway_id, and create_by_gateway_name" + fi +} + +log_selected_examples() { + local names=() + local spec + + for spec in "${SELECTED_EXAMPLES[@]}"; do + names+=("$(example_name "$spec")") + done + + local joined="${names[*]}" + log "examples=${joined// /,}" +} + +create_project() { + TEMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/gopher-mcp-python-example-verify.XXXXXX")" + PROJECT_DIR="${TEMP_ROOT}/project" + VENV_DIR="${PROJECT_DIR}/venv" + mkdir -p "$PROJECT_DIR" + + "$PYTHON_BIN" -m venv "$VENV_DIR" + "${VENV_DIR}/bin/python" -m pip install --upgrade pip >/dev/null + "${VENV_DIR}/bin/python" -m pip install "$SDK_INSTALL_SPEC" "$NATIVE_INSTALL_SPEC" >/dev/null + + SDK_VERSION="$("${VENV_DIR}/bin/python" - <<'PY' +from importlib import metadata +print(metadata.version("gopher-mcp-python")) +PY +)" + + log "temp_project=${PROJECT_DIR}" + log "platform=${PLATFORM} python=${PYTHON_VERSION} mode=${MODE} sdk=${SDK_VERSION}" +} + +cleanup() { + if [ -n "$TEMP_ROOT" ] && [ -d "$TEMP_ROOT" ]; then + rm -rf "$TEMP_ROOT" + fi +} + +run_native_probe() { + "${VENV_DIR}/bin/python" "${REPO_ROOT}/scripts/verify-example-native-probe.py" +} + +run_offline_example_bootstrap_checks() { + local spec + local name + local source_path + local target_file + local output + local status + local expected_marker + + for spec in "${SELECTED_EXAMPLES[@]}"; do + name="$(example_name "$spec")" + source_path="${REPO_ROOT}/$(example_path "$spec")" + target_file="${PROJECT_DIR}/$(basename "$source_path")" + expected_marker="$(missing_required_env_marker "$(example_required_env "$spec")")" + + if [ ! -f "$source_path" ]; then + fail "${name} offline: source file not found: ${source_path}" + fi + + cp "$source_path" "$target_file" + + set +e + output="$( + cd "$PROJECT_DIR" && + env \ + -u GOPHER_MCP_URL \ + -u GOPHER_API_KEY \ + -u GOPHER_MCP_SERVER_ID \ + -u GOPHER_MCP_SERVER_NAME \ + -u GOPHER_MCP_GATEWAY_ID \ + -u GOPHER_MCP_GATEWAY_NAME \ + -u LLM_MODEL \ + -u LLM_PROVIDER \ + -u ANTHROPIC_API_KEY \ + "${VENV_DIR}/bin/python" "$(basename "$target_file")" 2>&1 + )" + status=$? + set -e + + if [ "$status" -ne 1 ]; then + printf '%s\n' "$output" + fail "${name} offline: expected missing-env exit status 1, got ${status}" + fi + + if ! grep -Fxq "$expected_marker" <<<"$output"; then + printf '%s\n' "$output" + fail "${name} offline: did not report expected marker '${expected_marker}'" + fi + + log "${name} offline: missing-env validation OK" + done +} + +has_required_env() { + local required="$1" + local provider_required="$2" + local key + + for key in $required; do + if [ -z "${!key:-}" ]; then + return 1 + fi + done + + if [ "${LLM_PROVIDER:-AnthropicProvider}" = "AnthropicProvider" ]; then + for key in $provider_required; do + if [ -z "${!key:-}" ]; then + return 1 + fi + done + fi + + return 0 +} + +validate_expected_answer_terms() { + local answer_body="$1" + local terms="$2" + local term + + terms="${terms}," + while [ -n "$terms" ]; do + term="${terms%%,*}" + terms="${terms#*,}" + term="$(sed 's/^[[:space:]]*//;s/[[:space:]]*$//' <<<"$term")" + [ -n "$term" ] || continue + + if ! grep -qi -- "$term" <<<"$answer_body"; then + return 1 + fi + done + + return 0 +} + +run_live_example_checks() { + local spec + local name + local required + local provider_required + local source_path + local target_file + local output + local status + local answer_body + + for spec in "${SELECTED_EXAMPLES[@]}"; do + name="$(example_name "$spec")" + required="$(example_required_env "$spec")" + provider_required="$(example_provider_env "$spec")" + + if ! has_required_env "$required" "$provider_required"; then + if [ "$MODE" = "live" ]; then + fail "${name} live: missing required environment (${required} ${provider_required})" + fi + LIVE_CHECKS_SKIPPED=$((LIVE_CHECKS_SKIPPED + 1)) + log "${name} live: skipped because required environment is missing" + continue + fi + + source_path="${REPO_ROOT}/$(example_path "$spec")" + target_file="${PROJECT_DIR}/$(basename "$source_path")" + cp "$source_path" "$target_file" + + set +e + output="$( + cd "$PROJECT_DIR" && + "${VENV_DIR}/bin/python" "$(basename "$target_file")" "$VERIFY_LIVE_PROMPT" 2>&1 + )" + status=$? + set -e + + if [ "$status" -ne 0 ]; then + log_live_failure_diagnostics "$output" "" + fail "${name} live: example exited with status ${status}" + fi + + answer_body="$(awk '/Agent Response/{capture=1; next} capture {print}' <<<"$output")" + + if [ -z "$(tr -d '[:space:]' <<<"$answer_body")" ]; then + log_live_failure_diagnostics "$output" "$answer_body" + fail "${name} live: missing agent response body" + fi + + if grep -Eqi '(^|[[:space:]])(Error:|Traceback|isError)' <<<"$answer_body"; then + log_live_failure_diagnostics "$output" "$answer_body" + fail "${name} live: agent response contains an error" + fi + + if [ -n "$VERIFY_EXPECTED_ANSWER" ] && + ! grep -qi -- "$VERIFY_EXPECTED_ANSWER" <<<"$answer_body"; then + log_live_failure_diagnostics "$output" "$answer_body" + fail "${name} live: expected answer text '${VERIFY_EXPECTED_ANSWER}' not found" + fi + + if [ -n "$VERIFY_EXPECTED_ANSWER_TERMS" ] && + ! validate_expected_answer_terms "$answer_body" "$VERIFY_EXPECTED_ANSWER_TERMS"; then + log_live_failure_diagnostics "$output" "$answer_body" + fail "${name} live: expected answer terms '${VERIFY_EXPECTED_ANSWER_TERMS}' not found" + fi + + LIVE_CHECKS_RUN=$((LIVE_CHECKS_RUN + 1)) + LIVE_ANSWER_SUMMARY="${LIVE_ANSWER_SUMMARY}"$'\n'"${name}: answer_bytes=$(printf '%s' "$answer_body" | wc -c | tr -d '[:space:]')" + log "${name} live: OK (answer redacted)" + done +} + +main() { + parse_args "$@" + validate_args + load_env_file + require_python + detect_platform + compute_install_specs + select_examples + + log "only=${ONLY_EXAMPLE:-}" + log_selected_examples + log "sdk_install=${SDK_INSTALL_SPEC}" + log "native_install=${NATIVE_INSTALL_SPEC}" + + trap cleanup EXIT + create_project + + run_native_probe + + case "$MODE" in + offline) + run_offline_example_bootstrap_checks + ;; + live) + run_live_example_checks + ;; + auto) + run_offline_example_bootstrap_checks + run_live_example_checks + ;; + esac + + if [ "$LIVE_CHECKS_RUN" -gt 0 ]; then + log "live_checks=${LIVE_CHECKS_RUN}" + printf '%s\n' "$LIVE_ANSWER_SUMMARY" | sed '/^$/d' + fi + + if [ "$MODE" = "auto" ] && [ "$LIVE_CHECKS_SKIPPED" -gt 0 ]; then + log "live_skipped=${LIVE_CHECKS_SKIPPED}" + fi + + log "verification passed" +} + +main "$@" diff --git a/tests/ffi/auth/test_loader.py b/tests/ffi/auth/test_loader.py index 8d5c1d08..62dfb1c7 100644 --- a/tests/ffi/auth/test_loader.py +++ b/tests/ffi/auth/test_loader.py @@ -97,6 +97,19 @@ def test_includes_platform_and_current_native_paths(self): str(auth_loader.Path.cwd() / "native" / "lib") ) + def test_includes_platform_and_current_native_paths(self): + """Test includes JS-compatible local native output directories.""" + paths = _get_search_paths() + platform_dir = _get_platform_native_dir_name() + platform_path = str(auth_loader.Path.cwd() / "native" / platform_dir / "lib") + current_path = str(auth_loader.Path.cwd() / "native" / "current" / "lib") + + assert platform_path in paths + assert current_path in paths + assert paths.index(platform_path) < paths.index( + str(auth_loader.Path.cwd() / "native" / "lib") + ) + @patch("platform.system") def test_darwin_includes_homebrew(self, mock_system): """Test macOS includes homebrew paths.""" diff --git a/tests/test_linux_native_packaging.py b/tests/test_linux_native_packaging.py index fe7de3d9..e40848b0 100644 --- a/tests/test_linux_native_packaging.py +++ b/tests/test_linux_native_packaging.py @@ -19,6 +19,18 @@ def _root_build_script() -> str: return (ROOT / "build.sh").read_text() +def _verify_examples_workflow() -> str: + return (ROOT / ".github" / "workflows" / "verify-examples.yml").read_text() + + +def _verify_examples_script() -> str: + return (ROOT / "scripts" / "verify-examples.sh").read_text() + + +def _api_example(path: str) -> str: + return (ROOT / "examples" / "api" / path).read_text() + + def test_linux_x64_uses_digest_pinned_ubuntu_builder_image() -> None: dockerfile = _linux_builder_dockerfile() build_script = _root_build_script() @@ -34,6 +46,117 @@ def test_linux_x64_uses_digest_pinned_ubuntu_builder_image() -> None: assert re.search(r"\subuntu:20\.04\s", build_script) is None +def test_verify_examples_prs_install_checked_out_sdk() -> None: + workflow = _verify_examples_workflow() + + assert ( + "VERIFY_EXAMPLES_MODE: ${{ github.event_name == 'workflow_dispatch' " + "&& inputs.mode || 'auto' }}" + ) in workflow + assert 'if [ "${{ github.event_name }}" = "pull_request" ]; then' in workflow + assert ( + 'python -m pip install -e . "gopher-mcp-python-native-${{ matrix.platform }}"' + in workflow + ) + assert ( + "SDK_INSTALL_SPEC: ${{ github.event_name == 'pull_request' && " + "github.workspace || '' }}" + ) in workflow + + +def test_verify_examples_workflow_bounds_pr_cost_and_runtime() -> None: + workflow = _verify_examples_workflow() + + assert "concurrency:" in workflow + assert "group: ${{ github.workflow }}-${{ github.ref }}" in workflow + assert "cancel-in-progress: true" in workflow + assert "timeout-minutes: 30" in workflow + + +def test_verify_examples_workflow_checks_linux_native_dependencies() -> None: + workflow = _verify_examples_workflow() + + assert "Verify Linux native package" in workflow + assert "=== Linux Native Dependencies ===" in workflow + assert "readelf -d \"$sofile\"" in workflow + assert "No RPATH/RUNPATH declared for $sofile" in workflow + assert "RPATH/RUNPATH without \\$ORIGIN" in workflow + assert "ldd \"$sofile\"" in workflow + assert "grep -Ev '^(libssl\\.so|libcrypto\\.so)'" in workflow + assert 'if [ "${{ github.event_name }}" = "pull_request" ]; then' in workflow + assert "OpenSSL libraries must remain system-provided in PR-built packages." in workflow + assert "exit 1" in workflow + assert "WARNING: OpenSSL libraries should remain system-provided" in workflow + assert "No Linux shared libraries found" in workflow + + +def test_verify_examples_live_checks_only_agent_response_body() -> None: + script = _verify_examples_script() + workflow = _verify_examples_workflow() + + assert "answer_body=\"$(awk '/Agent Response/{capture=1; next} capture {print}'" in script + assert 'grep -qi -- "$VERIFY_EXPECTED_ANSWER" <<<"$answer_body"' in script + assert 'grep -qi -- "$VERIFY_EXPECTED_ANSWER" <<<"$output"' not in script + assert "VERIFY_EXPECTED_ANSWER_TERMS" in script + assert 'validate_expected_answer_terms "$answer_body"' in script + assert "agent response contains an error" in script + assert 'VERIFY_LIVE_PROMPT="list my draft mails"' in workflow + assert 'VERIFY_EXPECTED_ANSWER="r-2553040815323886578"' in workflow + assert "Draft ID,Message ID,Thread ID" not in workflow + + +def test_verify_examples_live_logs_redact_agent_output() -> None: + script = _verify_examples_script() + + assert "log_live_failure_diagnostics" in script + assert "live output redacted:" in script + assert "answer redacted" in script + assert "answer_excerpt" not in script + assert "${name}: ${answer_excerpt}" not in script + assert "printf '%s\\n' \"$output\"\n fail \"${name} live:" not in script + + +def test_verify_examples_cleanup_trap_covers_project_creation() -> None: + script = _verify_examples_script() + main_body = script[script.index("main() {") :] + + assert main_body.index("trap cleanup EXIT") < main_body.index("create_project") + + +def test_verify_examples_offline_checks_stable_missing_env_markers() -> None: + script = _verify_examples_script() + + assert "missing_required_env_marker" in script + assert "expected missing-env exit status 1" in script + assert 'grep -Fxq "$expected_marker"' in script + assert "must (both |all )?be set" not in script + + expected_markers = { + "create_by_url.py": "ERROR: missing-required-env: GOPHER_MCP_URL,LLM_MODEL", + "create_by_api_key.py": "ERROR: missing-required-env: GOPHER_API_KEY,LLM_MODEL", + "create_by_json.py": "ERROR: missing-required-env: LLM_MODEL", + "create_by_server_id.py": ( + "ERROR: missing-required-env: " + "GOPHER_API_KEY,GOPHER_MCP_SERVER_ID,LLM_MODEL" + ), + "create_by_server_name.py": ( + "ERROR: missing-required-env: " + "GOPHER_API_KEY,GOPHER_MCP_SERVER_NAME,LLM_MODEL" + ), + "create_by_gateway_id.py": ( + "ERROR: missing-required-env: " + "GOPHER_API_KEY,GOPHER_MCP_GATEWAY_ID,LLM_MODEL" + ), + "create_by_gateway_name.py": ( + "ERROR: missing-required-env: " + "GOPHER_API_KEY,GOPHER_MCP_GATEWAY_NAME,LLM_MODEL" + ), + } + + for filename, marker in expected_markers.items(): + assert marker in _api_example(filename) + + def test_linux_x64_builder_does_not_bundle_openssl() -> None: script = _linux_builder_script() dep_skip_block = re.search(r"case \"\$dep_name\" in(?P.*?)esac", script, re.S)