diff --git a/.github/workflows/test-backup.yml b/.github/workflows/test-backup.yml index dcd899d..f22f406 100644 --- a/.github/workflows/test-backup.yml +++ b/.github/workflows/test-backup.yml @@ -24,4 +24,8 @@ jobs: - name: "Build local shiv package" run: ./scripts/build_shiv_package.sh - name: "Run backup tests" + env: + # "Enable debug logging" on a re-run sets runner.debug; pass it through + # so the test script turns on xtrace and dumps its environment. + STACK_SCRIPT_DEBUG: ${{ runner.debug }} run: ./tests/backup/run-test.sh diff --git a/.github/workflows/test-database.yml b/.github/workflows/test-database.yml index 82d7b50..cb0f294 100644 --- a/.github/workflows/test-database.yml +++ b/.github/workflows/test-database.yml @@ -30,4 +30,8 @@ jobs: - name: "Build local shiv package" run: ./scripts/build_shiv_package.sh - name: "Run k8s deploy tests" + env: + # "Enable debug logging" on a re-run sets runner.debug; pass it through + # so the test script turns on xtrace and dumps its environment. + STACK_SCRIPT_DEBUG: ${{ runner.debug }} run: ./tests/database/run-test.sh diff --git a/.github/workflows/test-deploy-k3s.yml b/.github/workflows/test-deploy-k3s.yml index e7d4697..be72d98 100644 --- a/.github/workflows/test-deploy-k3s.yml +++ b/.github/workflows/test-deploy-k3s.yml @@ -57,4 +57,5 @@ jobs: STACK_IMAGE_REGISTRY_USER: ${{ secrets.K3S_TEST_REGISTRY_TOKEN }} STACK_IMAGE_REGISTRY_TOKEN: ${{ secrets.K3S_TEST_REGISTRY_TOKEN }} LETSENCRYPT_EMAIL: ${{ vars.K3S_TEST_LETSENCRYPT_EMAIL }} + STACK_SCRIPT_DEBUG: ${{ runner.debug }} run: MACHINE_SSH_KEY_FILE=$HOME/.ssh/k3s-test-key ./tests/k3s-deploy/run-k3s-deploy-test.sh diff --git a/.github/workflows/test-deploy-k8s.yml b/.github/workflows/test-deploy-k8s.yml index 3147916..b132e48 100644 --- a/.github/workflows/test-deploy-k8s.yml +++ b/.github/workflows/test-deploy-k8s.yml @@ -30,4 +30,8 @@ jobs: - name: "Build local shiv package" run: ./scripts/build_shiv_package.sh - name: "Run k8s deploy tests" + env: + # "Enable debug logging" on a re-run sets runner.debug; pass it through + # so the test script turns on xtrace and dumps its environment. + STACK_SCRIPT_DEBUG: ${{ runner.debug }} run: ./tests/k8s-deploy/run-deploy-test.sh diff --git a/.github/workflows/test-deploy.yml b/.github/workflows/test-deploy.yml index 6dfcdde..b85a891 100644 --- a/.github/workflows/test-deploy.yml +++ b/.github/workflows/test-deploy.yml @@ -24,4 +24,8 @@ jobs: - name: "Build local shiv package" run: ./scripts/build_shiv_package.sh - name: "Run deploy tests" + env: + # "Enable debug logging" on a re-run sets runner.debug; pass it through + # so the test script turns on xtrace and dumps its environment. + STACK_SCRIPT_DEBUG: ${{ runner.debug }} run: ./tests/deploy/run-deploy-test.sh diff --git a/.github/workflows/test-skill.yml b/.github/workflows/test-skill.yml index 07b51da..fdb5871 100644 --- a/.github/workflows/test-skill.yml +++ b/.github/workflows/test-skill.yml @@ -24,4 +24,8 @@ jobs: - name: "Build local shiv package" run: ./scripts/build_shiv_package.sh - name: "Run skill quickstart test" + env: + # "Enable debug logging" on a re-run sets runner.debug; pass it through + # so the test script turns on xtrace and dumps its environment. + STACK_SCRIPT_DEBUG: ${{ runner.debug }} run: ./tests/skill/run-skill-test.sh diff --git a/.github/workflows/test-static-content.yml b/.github/workflows/test-static-content.yml index d8d7ee6..22668dc 100644 --- a/.github/workflows/test-static-content.yml +++ b/.github/workflows/test-static-content.yml @@ -24,4 +24,8 @@ jobs: - name: "Build local shiv package" run: ./scripts/build_shiv_package.sh - name: "Run static content tests" + env: + # "Enable debug logging" on a re-run sets runner.debug; pass it through + # so the test script turns on xtrace and dumps its environment. + STACK_SCRIPT_DEBUG: ${{ runner.debug }} run: ./tests/static-content-test/run-static-content-test.sh diff --git a/.github/workflows/test-webapp.yml b/.github/workflows/test-webapp.yml index 1bcef59..e9625c1 100644 --- a/.github/workflows/test-webapp.yml +++ b/.github/workflows/test-webapp.yml @@ -24,4 +24,8 @@ jobs: - name: "Build local shiv package" run: ./scripts/build_shiv_package.sh - name: "Run webapp tests" + env: + # "Enable debug logging" on a re-run sets runner.debug; pass it through + # so the test script turns on xtrace and dumps its environment. + STACK_SCRIPT_DEBUG: ${{ runner.debug }} run: ./tests/webapp-test/run-webapp-test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index de4f71d..ec99de6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,4 +24,8 @@ jobs: - name: "Build local shiv package" run: ./scripts/build_shiv_package.sh - name: "Run smoke tests" + env: + # "Enable debug logging" on a re-run sets runner.debug; pass it through + # so the test script turns on xtrace and dumps its environment. + STACK_SCRIPT_DEBUG: ${{ runner.debug }} run: ./tests/smoke-test/run-smoke-test.sh diff --git a/CLAUDE.md b/CLAUDE.md index 5a6ab16..b61db68 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,10 +54,23 @@ The integration tests are **bash shell scripts**, not pytest. They require Docke ./tests/deploy/run-deploy-test.sh # Kubernetes deployment tests -./tests/k8s-deploy/run-k8s-deploy-test.sh +./tests/k8s-deploy/run-deploy-test.sh ``` -The smoke tests require building a shiv package first (`./scripts/build_shiv_package.sh`). +Run them from the repo root. By default each one tests the most recently built +shiv package in `./package` (`./scripts/build_shiv_package.sh`); pass `from-path` +to test the `stack` on your PATH instead, or set `TEST_TARGET_STACK` (e.g. to +`"uv run stack"`). + +Set `STACK_SCRIPT_DEBUG` to turn on xtrace and an environment dump. In CI it is +wired to `runner.debug`, so ticking "Enable debug logging" when re-running a +failed job turns it on for that run. + +Shared helpers live in `tests/lib/common.sh`, which every test script sources as +its first line: the debug preamble, target selection, test-directory setup, and +the `wait_for_*` / teardown helpers. Put anything used by more than one test +there — these scripts were written by copying each other, and each copied helper +eventually drifted from its siblings. ## Source Layout diff --git a/tests/backup/run-test.sh b/tests/backup/run-test.sh index 8dc9d8e..4a544fb 100755 --- a/tests/backup/run-test.sh +++ b/tests/backup/run-test.sh @@ -14,21 +14,11 @@ # NOTE: this fetches the test stacks and the backup stack from GitHub, so the # `test-backup-stack` additions in bozemanpass/stack-test-stacks and the # bozemanpass/backup-stack repo must be pushed for this to run. -set -e -if [ -n "$STACK_SCRIPT_DEBUG" ]; then - set -x -fi +source "$( dirname -- "${BASH_SOURCE[0]}" )/../lib/common.sh" -if ! command -v docker &> /dev/null; then - echo "Error: 'docker' is not installed or not available on the PATH" - exit 1 -fi +require_commands docker -if [ "$1" == "from-path" ]; then - TEST_TARGET_STACK="stack" -else - TEST_TARGET_STACK=$( ls -t1 ./package/stack* | head -1 ) -fi +select_test_target "$@" app_stack="test-backup-stack" backup_stack="backup" @@ -43,60 +33,10 @@ export STACK_BACKUP_S3_BUCKET=stack-backups payload="backup-test-payload-$$" # a value unique to this run -# Run a command inside a deployment container. The stack `exec` wraps the command in -# `sh -c`, so the whole command must be passed as a single argument. -dexec () { $TEST_TARGET_STACK manage --dir "$test_deployment_dir" exec "$1" "$2"; } - -# Containers write into the bind-mounted volume dirs as root, so the resulting files cannot -# be removed by the (non-root) host user. Remove such a dir via a throwaway container. -force_rm () { - if [ -d "$1" ]; then - docker run --rm -v "$(dirname "$1")":/w alpine rm -rf "/w/$(basename "$1")" || rm -rf "$1" - fi -} - -# Dump container logs on failure. The backup container's own output does not reveal why the -# S3 store (SeaweedFS) rejected a request, so capture every service's logs - especially s3 - -# before the deployment is torn down. -dump_diagnostics () { - echo "===================== FAILURE DIAGNOSTICS =====================" - echo "----- ps -----" - $TEST_TARGET_STACK manage --dir "$test_deployment_dir" ps || true - echo "----- container logs (last 200 lines per service) -----" - $TEST_TARGET_STACK manage --dir "$test_deployment_dir" logs -n 200 || true - echo "==============================================================" -} - -cleanup_exit () { - dump_diagnostics - $TEST_TARGET_STACK manage --dir "$test_deployment_dir" stop --delete-volumes || true - exit 1 -} - -wait_for_pods_started () { - for i in {1..50}; do - local ps_output - ps_output=$( $TEST_TARGET_STACK manage --dir "$test_deployment_dir" ps ) - if [[ "$ps_output" == *"id:"* ]]; then - return - fi - sleep 5 - done - echo "waiting for pods to start: FAILED" - cleanup_exit -} - -STACK_TEST_DIR=~/stack-test/backup-test-dir -export STACK_REPO_BASE_DIR=${STACK_TEST_DIR}/repo-base-dir -echo "Testing this package: $TEST_TARGET_STACK" -$TEST_TARGET_STACK version -echo "Using test directory: $STACK_TEST_DIR" -force_rm "$STACK_TEST_DIR" -mkdir -p "$STACK_REPO_BASE_DIR" +setup_test_dir backup-test-dir # Force a rebuild of the backup image so the test exercises current sources. -existing=$(docker image ls -q --filter=reference=bozemanpass/backup | uniq) -if [ -n "$existing" ]; then docker image rm -f ${existing} || true; fi +remove_local_images bozemanpass/backup # Fetch and prepare the stacks. $TEST_TARGET_STACK fetch repo github.com/bozemanpass/stack-test-stacks @@ -120,22 +60,21 @@ $TEST_TARGET_STACK init --stack ${backup_stack} --output "$test_backup_spec" \ --config AWS_SECRET_ACCESS_KEY=test-secret-key # Deploy, mixing in the backup stack. +stop_deployment_on_exit "$test_deployment_dir" $TEST_TARGET_STACK deploy \ --spec-file "$test_backup_spec" \ --spec-file "$test_app_spec" \ --deployment-dir "$test_deployment_dir" if [ ! -d "$test_deployment_dir" ]; then - echo "deploy create test: deployment directory not present" - echo "deploy create test: FAILED" - exit 1 + fail "deploy create test: FAILED - deployment directory not present" fi echo "deploy create test: passed" $TEST_TARGET_STACK manage --dir "$test_deployment_dir" start -wait_for_pods_started +wait_for_containers_started # 1. Write a known payload into the app's data volume (via the app). -dexec app "echo ${payload} > /data/payload.txt" +deployment_exec app "echo ${payload} > /data/payload.txt" echo "wrote payload: ${payload}" # 2. Take a backup. backup.sh's ensure_repo already waits for the S3 store to finish @@ -145,48 +84,43 @@ echo "wrote payload: ${payload}" # hiccup; genuine unavailability fails promptly. backed_up= for i in {1..3}; do - if dexec backup "/scripts/backup.sh"; then backed_up=1; break; fi + if deployment_exec backup "/scripts/backup.sh"; then backed_up=1; break; fi echo "backup attempt ${i} failed, retrying" sleep 5 done if [ -z "$backed_up" ]; then - echo "Backup test: FAILED" - cleanup_exit + fail "Backup test: FAILED" fi echo "Backup test: passed" # 3. Simulate data loss by wiping the app volume (through the backup container's rw mount). -dexec backup "rm -rf /backup/app-data/*" -gone=$( dexec backup "ls /backup/app-data" || true ) +deployment_exec backup "rm -rf /backup/app-data/*" +gone=$( deployment_exec backup "ls /backup/app-data" || true ) if [[ "$gone" == *"payload.txt"* ]]; then - echo "Simulate data loss: FAILED (payload still present)" - cleanup_exit + fail "Simulate data loss: FAILED (payload still present)" fi echo "Simulate data loss: passed (payload gone)" # 4. Restore from the latest snapshot. -dexec backup "/scripts/restore.sh latest" +deployment_exec backup "/scripts/restore.sh latest" # 5. Assert the payload came back, reading it through the app. -restored=$( dexec app "cat /data/payload.txt" || true ) +restored=$( deployment_exec app "cat /data/payload.txt" || true ) if [[ "$restored" == *"$payload"* ]]; then echo "Restore content test: passed" else - echo "Restore content test: FAILED (expected '${payload}', got '${restored}')" - cleanup_exit + fail "Restore content test: FAILED (expected '${payload}', got '${restored}')" fi # 6. Assert the excluded s3 store volume was NOT mounted into / captured by the backup. -listing=$( dexec backup "ls /backup" || true ) +listing=$( deployment_exec backup "ls /backup" || true ) if [[ "$listing" == *"s3-data"* ]]; then - echo "Exclude annotation test: FAILED (s3-data was backed up)" - cleanup_exit + fail "Exclude annotation test: FAILED (s3-data was backed up)" fi if [[ "$listing" != *"app-data"* ]]; then - echo "Exclude annotation test: FAILED (app-data missing from backup)" - cleanup_exit + fail "Exclude annotation test: FAILED (app-data missing from backup)" fi echo "Exclude annotation test: passed (s3-data excluded, app-data backed up)" -$TEST_TARGET_STACK manage --dir "$test_deployment_dir" stop --delete-volumes +# The registered teardown stops the deployment and deletes its volumes. echo "Test passed" diff --git a/tests/database/run-test.sh b/tests/database/run-test.sh index eba5e8b..a13dd71 100755 --- a/tests/database/run-test.sh +++ b/tests/database/run-test.sh @@ -1,93 +1,19 @@ #!/usr/bin/env bash -set -e -if [ -n "$STACK_SCRIPT_DEBUG" ]; then - set -x - # Dump environment variables for debugging - echo "Environment variables:" - env -fi +source "$( dirname -- "${BASH_SOURCE[0]}" )/../lib/common.sh" -if ! command -v kind &> /dev/null; then - echo "Error: 'kind' is not installed or not available on the PATH" - exit 1 -fi +require_commands kind -if [ "$1" == "from-path" ]; then - TEST_TARGET_STACK="stack" -else - TEST_TARGET_STACK=$( ls -t1 ./package/stack* | head -1 ) -fi +select_test_target "$@" stack="test-database-stack" spec_file=${stack}-spec.yml deployment_dir=${stack}-deployment -# Helper functions: TODO move into a separate file -wait_for_pods_started () { - for i in {1..50} - do - local ps_output=$( $TEST_TARGET_STACK manage --dir $test_deployment_dir ps ) - - if [[ "$ps_output" == *"id:"* ]]; then - # if ready, return - return - else - # if not ready, wait - sleep 5 - fi - done - # Timed out, error exit - echo "waiting for pods to start: FAILED" - delete_cluster_exit -} - -wait_for_test_complete () { - for i in {1..50} - do - - local log_output=$( $TEST_TARGET_STACK manage --dir $test_deployment_dir logs ) - - if [[ "${log_output}" == *"Database test client: test complete"* ]]; then - # if ready, return - return - else - # if not ready, wait - sleep 5 - fi - done - # Timed out, error exit - echo "waiting for test complete: FAILED" - delete_cluster_exit -} - - -delete_cluster_exit () { - $TEST_TARGET_STACK manage --dir $test_deployment_dir stop --delete-volumes - exit 1 -} - -# We make a directory within which our test will create files -STACK_TEST_DIR=~/stack-test/database-test-dir -# Set a non-default repo dir -export STACK_REPO_BASE_DIR=${STACK_TEST_DIR}/repo-base-dir -echo "Testing this package: $TEST_TARGET_STACK" -echo "Test version command" -reported_version_string=$( $TEST_TARGET_STACK version ) -echo "Version reported is: ${reported_version_string}" -echo "Using test directory: $STACK_TEST_DIR" -rm -rf $STACK_TEST_DIR -mkdir -p $STACK_TEST_DIR -mkdir -p $STACK_REPO_BASE_DIR +setup_test_dir database-test-dir # We must delete any instances of the test-container in the local registory # otherwise we'll skip building it below -existing_test_images=$(docker image ls -q --filter=reference=bozemanpass/test-database-client | uniq) -if [ -n "$existing_test_images" ]; then - docker image rm -f ${existing_test_images} -fi -existing_test_images=$(docker image ls -q --filter=reference=bozemanpass/test-database-container | uniq) -if [ -n "$existing_test_images" ]; then - docker image rm -f ${existing_test_images} -fi +remove_local_images bozemanpass/test-database-client +remove_local_images bozemanpass/test-database-container # Fetch the test stacks echo "Fetching test stack repo into: $STACK_REPO_BASE_DIR" $TEST_TARGET_STACK fetch repo github.com/bozemanpass/stack-test-stacks @@ -99,35 +25,31 @@ test_deployment_spec=$STACK_TEST_DIR/${spec_file} $TEST_TARGET_STACK init --stack ${stack} --deploy-to k8s-kind --output $test_deployment_spec # Check the file now exists if [ ! -f "$test_deployment_spec" ]; then - echo "deploy init test: spec file not present" - echo "deploy init test: FAILED" - exit 1 + fail "deploy init test: FAILED - spec file not present" fi echo "deploy init test: passed" # Switch to a full path for the data dir so it gets provisioned as a host bind mounted volume and preserved beyond cluster lifetime sed -i "s|^\(\s*db-data:$\)$|\1 ${test_deployment_dir}/data/db-data|" $test_deployment_spec +stop_deployment_on_exit $test_deployment_dir $TEST_TARGET_STACK deploy --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir # Check the deployment dir exists if [ ! -d "$test_deployment_dir" ]; then - echo "deploy create test: deployment directory not present" - echo "deploy create test: FAILED" - exit 1 + fail "deploy create test: FAILED - deployment directory not present" fi echo "deploy create test: passed" # Try to start the deployment $TEST_TARGET_STACK manage --dir $test_deployment_dir start -wait_for_pods_started +wait_for_containers_started # Check logs command works -wait_for_test_complete +wait_for_log_content "Database test client: test complete" log_output_1=$( $TEST_TARGET_STACK manage --dir $test_deployment_dir logs ) if [[ "$log_output_1" == *"Database test client: test data does not exist"* ]]; then echo "Create database content test: passed" else - echo "Create database content test: FAILED" - delete_cluster_exit + fail "Create database content test: FAILED" fi # Stop then start again and check the volume was preserved @@ -135,17 +57,15 @@ $TEST_TARGET_STACK manage --dir $test_deployment_dir stop # Sleep a bit just in case sleep 20 $TEST_TARGET_STACK manage --dir $test_deployment_dir start -wait_for_pods_started -wait_for_test_complete +wait_for_containers_started +wait_for_log_content "Database test client: test complete" log_output_2=$( $TEST_TARGET_STACK manage --dir $test_deployment_dir logs ) if [[ "$log_output_2" == *"Database test client: test data already exists"* ]]; then echo "Retain database content test: passed" else - echo "Retain database content test: FAILED" - delete_cluster_exit + fail "Retain database content test: FAILED" fi -# Stop and clean up -$TEST_TARGET_STACK manage --dir $test_deployment_dir stop --delete-volumes +# The registered teardown stops the deployment and deletes its volumes. echo "Test passed" diff --git a/tests/deploy/run-deploy-test.sh b/tests/deploy/run-deploy-test.sh index 2fd390e..3667a56 100755 --- a/tests/deploy/run-deploy-test.sh +++ b/tests/deploy/run-deploy-test.sh @@ -1,200 +1,66 @@ #!/usr/bin/env bash -set -e -if [ -n "$STACK_SCRIPT_DEBUG" ]; then - set -x -fi - -# Check for required utilities -if ! command -v jq &> /dev/null; then - echo "Error: jq is not installed." - echo "Please install jq to run this test script." - exit 1 -fi +source "$( dirname -- "${BASH_SOURCE[0]}" )/../lib/common.sh" -# Dump environment variables for debugging -echo "Environment variables:" -env - -delete_cluster_exit () { - $TEST_TARGET_SO manage --dir $test_deployment_dir stop --delete-volumes -} - -trap delete_cluster_exit EXIT - -add_todo() { - set +e - - local running=0 - local check=0 - local check_limit=10 - - url=$1 - title=$2 - - try=0 - rc=1 - - while [ $rc -ne 0 ] && [ $try -lt 10 ]; do - try=$((try + 1)) - curl "$url" \ - --fail-with-body \ - -H 'Accept: application/json, text/plain, */*' \ - -H 'Accept-Language: en-US,en;q=0.9' \ - -H 'Connection: keep-alive' \ - -H 'Content-Type: application/json' \ - -H 'Origin: http://localhost' \ - -H 'Referer: http://localhost/' \ - -H 'Sec-Fetch-Dest: empty' \ - -H 'Sec-Fetch-Mode: cors' \ - -H 'Sec-Fetch-Site: same-site' \ - -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36 Edg/135.0.0.0' \ - -H 'sec-ch-ua: "Microsoft Edge";v="135", "Not-A.Brand";v="8", "Chromium";v="135"' \ - -H 'sec-ch-ua-mobile: ?0' \ - -H 'sec-ch-ua-platform: "Windows"' \ - --data-raw "{\"title\":\"$title\",\"completed\":false}" - rc=$? - - if [ $rc -ne 0 ]; then - echo "Error adding todo, retrying..." - sleep 5 - fi - done - - set -e - - return $rc -} - - -# Fetch a URL until its body contains the expected text. "status" reports a -# container as running as soon as Docker starts it, which is not the same as the -# frontend being ready to serve, so a single-shot fetch here is a race. -wait_for_content () { - set +e - - local url=$1 - local expected=$2 - local body="" - - local try=0 - local rc=1 - - while [ $rc -ne 0 ] && [ $try -lt 20 ]; do - try=$((try + 1)) - body=$(curl -s "$url") - echo "$body" | grep -q "$expected" - rc=$? - - if [ $rc -ne 0 ]; then - echo "Waiting for $expected at $url..." - sleep 5 - fi - done - - set -e - - if [ $rc -ne 0 ]; then - echo "deploy http: failed - $expected not found at $url" - echo "last response body was:" - echo "$body" - exit 1 - fi -} - -wait_for_running () { - # Check that all services are running - local how_many=$1 - local running=0 - local check=0 - local check_limit=10 - while [ $running -lt $how_many ] && [ $check -lt $check_limit ]; do - check=$((check + 1)) - running=$($TEST_TARGET_SO manage --dir $test_deployment_dir status | grep -ic "running") - if [ $running -lt $how_many ]; then - echo "deploy manage start: Waiting for services to start..." - sleep 5 - fi - done - - if [ $running -lt $how_many ]; then - echo "deploy manage start: failed - not all services started" - exit 1 - fi -} +require_commands jq export STACK_USE_BUILTIN_STACK=true # Test basic stack deploy echo "Running stack deploy test" -# Bit of a hack, test the most recent package -TEST_TARGET_SO=$( ls -t1 ./package/stack* | head -1 ) -# Set a non-default repo dir -STACK_TEST_DIR=~/stack-test/deploy-test-dir -export STACK_REPO_BASE_DIR=${STACK_TEST_DIR}/repo-base-dir -echo "Testing this package: $TEST_TARGET_SO" -echo "Test version command" -reported_version_string=$( $TEST_TARGET_SO version ) -echo "Version reported is: ${reported_version_string}" -echo "Cloning repositories into: $STACK_REPO_BASE_DIR" -rm -rf $STACK_TEST_DIR -mkdir -p $STACK_REPO_BASE_DIR +select_test_target "$@" +setup_test_dir deploy-test-dir # Test bringing the test container up and down # with and without volume removal STACK_NAME="todo" -$TEST_TARGET_SO fetch repo bozemanpass/example-todo-list -$TEST_TARGET_SO prepare --stack $STACK_NAME +$TEST_TARGET_STACK fetch repo bozemanpass/example-todo-list +$TEST_TARGET_STACK prepare --stack $STACK_NAME # Basic test of creating a deployment # Deployment artifacts live outside the repo base dir, so the deployment's copy # of the stack files is not seen when resolving stacks by name. test_deployment_dir=$STACK_TEST_DIR/test-deployment-dir test_deployment_spec=$STACK_TEST_DIR/test-deployment-spec.yml -$TEST_TARGET_SO init --stack $STACK_NAME --output $test_deployment_spec --map-ports-to-host localhost-same +$TEST_TARGET_STACK init --stack $STACK_NAME --output $test_deployment_spec --map-ports-to-host localhost-same # Check the file now exists if [ ! -f "$test_deployment_spec" ]; then - echo "deploy init test: spec file not present" - echo "deploy init test: FAILED" - exit 1 + fail "deploy init test: FAILED - spec file not present" fi echo "deploy init test: passed" -$TEST_TARGET_SO deploy --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir +stop_deployment_on_exit $test_deployment_dir +$TEST_TARGET_STACK deploy --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir # Check the deployment dir exists if [ ! -d "$test_deployment_dir" ]; then - echo "deploy deploy test: deployment directory not present" - echo "deploy deploy test: FAILED" - exit 1 + fail "deploy deploy test: FAILED - deployment directory not present" fi echo "deploy create test: passed" # Start -$TEST_TARGET_SO manage --dir $test_deployment_dir start +$TEST_TARGET_STACK manage --dir $test_deployment_dir start wait_for_running 3 # Add a todo todo_title="79b06705-b402-431a-83a3-a634392d2754" -add_todo http://localhost:5000 "$todo_title" +add_todo http://localhost:5000 "$todo_title" http://localhost # Check that it exists if [ "$todo_title" != "$(curl -s http://localhost:5000 | jq -r '.[] | select(.id == 1) | .title')" ]; then - echo "deploy storage: failed - todo $todo_title not found" - exit 1 + fail "deploy storage: failed - todo $todo_title not found" fi # Stop the stack (don't delete volumes) -$TEST_TARGET_SO manage --dir $test_deployment_dir stop +$TEST_TARGET_STACK manage --dir $test_deployment_dir stop # Restart the stack -$TEST_TARGET_SO manage --dir $test_deployment_dir start +$TEST_TARGET_STACK manage --dir $test_deployment_dir start # Check that all services are running wait_for_running 3 # Check that it is still viewable if [ "$todo_title" != "$(curl -s http://localhost:5000 | jq -r '.[] | select(.id == 1) | .title')" ]; then - echo "deploy storage: failed - todo $todo_title not found after restart" - exit 1 + fail "deploy storage: failed - todo $todo_title not found after restart" fi echo "deploy storage: passed" diff --git a/tests/k3s-deploy/run-k3s-deploy-test.sh b/tests/k3s-deploy/run-k3s-deploy-test.sh index 39ae7a2..eb86467 100755 --- a/tests/k3s-deploy/run-k3s-deploy-test.sh +++ b/tests/k3s-deploy/run-k3s-deploy-test.sh @@ -47,11 +47,7 @@ # (default: the main branch) # MACHINE_CMD The machine command to run (default: machine) # -set -e - -if [ -n "$STACK_SCRIPT_DEBUG" ]; then - set -x -fi +source "$( dirname -- "${BASH_SOURCE[0]}" )/../lib/common.sh" script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" @@ -67,12 +63,7 @@ MACHINE_NEW_USER=stacktest # cert-manager etc. (seconds). PROVISION_TIMEOUT=1800 -for cmd in "$MACHINE_CMD" jq ssh docker; do - if ! command -v "$cmd" &> /dev/null; then - echo "Error: $cmd is not installed." - exit 1 - fi -done +require_commands "$MACHINE_CMD" jq ssh docker missing="" for var in MACHINE_DO_TOKEN MACHINE_SSH_KEY_NAME MACHINE_SSH_KEY_FILE MACHINE_DNS_ZONE \ @@ -206,9 +197,16 @@ if ! grep -q "$machine_fqdn" "$kube_config"; then fi echo "Fetched kubeconfig" -# So that push-images can push to the registry. +# So that push-images can push to the registry. The token is an argument here, +# so under STACK_SCRIPT_DEBUG it would be echoed verbatim by xtrace; suppress +# tracing across the login and restore whatever it was afterward. (Heredoc +# bodies and pipe contents are not traced, so the machine config and the +# kubeconfig need no such guard.) +xtrace_was_set="" +case $- in *x*) xtrace_was_set=1; set +x ;; esac echo "$STACK_IMAGE_REGISTRY_TOKEN" | docker login "${STACK_IMAGE_REGISTRY%%/*}" \ --username "$STACK_IMAGE_REGISTRY_USER" --password-stdin +if [ -n "$xtrace_was_set" ]; then set -x; fi # The test hostname must resolve locally before the HTTP checks can pass (the # authoritative record was just created; Let's Encrypt resolves it diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh index b91d566..06a7a86 100755 --- a/tests/k8s-deploy/run-deploy-test.sh +++ b/tests/k8s-deploy/run-deploy-test.sh @@ -1,16 +1,7 @@ #!/usr/bin/env bash -set -e +source "$( dirname -- "${BASH_SOURCE[0]}" )/../lib/common.sh" -if [ -n "$STACK_SCRIPT_DEBUG" ]; then - set -x -fi - -# Check for required utilities -if ! command -v jq &> /dev/null; then - echo "Error: jq is not installed." - echo "Please install jq to run this test script." - exit 1 -fi +require_commands jq # Determine if we're testing against a remote k8s cluster # Set STACK_K8S_REMOTE=true to enable remote mode, which also requires: @@ -19,8 +10,7 @@ fi # STACK_K8S_HOSTNAME - hostname of the remote cluster if [ "$STACK_K8S_REMOTE" = "true" ]; then if [ -z "$STACK_KUBE_CONFIG" ] || [ -z "$STACK_IMAGE_REGISTRY" ] || [ -z "$STACK_K8S_HOSTNAME" ]; then - echo "Error: Remote k8s mode requires STACK_KUBE_CONFIG, STACK_IMAGE_REGISTRY, and STACK_K8S_HOSTNAME" - exit 1 + fail "Error: Remote k8s mode requires STACK_KUBE_CONFIG, STACK_IMAGE_REGISTRY, and STACK_K8S_HOSTNAME" fi DEPLOY_TO="k8s" TEST_HOSTNAME="$STACK_K8S_HOSTNAME" @@ -31,179 +21,21 @@ else TEST_SCHEME="http" fi -# Dump environment variables for debugging -echo "Environment variables:" -env - -delete_cluster_exit () { - if [ -d "$test_deployment_dir" ]; then - $TEST_TARGET_SO manage --dir $test_deployment_dir stop --delete-volumes - fi -} - -wait_for_pods_started () { - for i in {1..50} - do - local ps_output=$( $TEST_TARGET_SO manage --dir $test_deployment_dir ps ) - - if [[ "$ps_output" == *"Running containers:"* ]]; then - # if ready, return - return - else - # if not ready, wait - sleep 5 - fi - done - # Timed out, error exit - echo "waiting for pods to start: FAILED" - exit 1 -} - -wait_for_log_output () { - for i in {1..50} - do - - local log_output=$( $TEST_TARGET_SO manage --dir $test_deployment_dir logs ) - - if [[ ! -z "$log_output" ]]; then - # if ready, return - return - else - # if not ready, wait - sleep 5 - fi - done - # Timed out, error exit - echo "waiting for pods log content: FAILED" - exit 1 -} - -wait_for_running () { - set +e - - # Check that all services are running (and ready -- "status" only reports a - # pod as Running once its containers pass their readiness probes). - how_many=$1 - local running=0 - local check=0 - # Against a real cluster the images are pulled from a real registry over the - # network, which can take minutes on a cold node, so allow ~5 minutes here. - local check_limit=60 - while [ $running -lt $how_many ] && [ $check -lt $check_limit ]; do - check=$((check + 1)) - running=$($TEST_TARGET_SO manage --dir $test_deployment_dir status | grep -ic "running") - if [ $running -lt $how_many ]; then - echo "deploy manage start: Waiting for services to start..." - sleep 5 - fi - done - - if [ $running -lt $how_many ]; then - echo "deploy manage start: failed - not all services started" - exit 1 - fi - - set -e -} - -add_todo() { - set +e - - local running=0 - local check=0 - local check_limit=10 - - url=$1 - title=$2 - - try=0 - rc=1 - - while [ $rc -ne 0 ] && [ $try -lt 10 ]; do - try=$((try + 1)) - curl "$url" \ - --fail-with-body \ - -H 'Accept: application/json, text/plain, */*' \ - -H 'Accept-Language: en-US,en;q=0.9' \ - -H 'Connection: keep-alive' \ - -H 'Content-Type: application/json' \ - -H "Origin: ${TEST_SCHEME}://${TEST_HOSTNAME}" \ - -H "Referer: ${TEST_SCHEME}://${TEST_HOSTNAME}/" \ - -H 'Sec-Fetch-Dest: empty' \ - -H 'Sec-Fetch-Mode: cors' \ - -H 'Sec-Fetch-Site: same-site' \ - -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36 Edg/135.0.0.0' \ - -H 'sec-ch-ua: "Microsoft Edge";v="135", "Not-A.Brand";v="8", "Chromium";v="135"' \ - -H 'sec-ch-ua-mobile: ?0' \ - -H 'sec-ch-ua-platform: "Windows"' \ - --data-raw "{\"title\":\"$title\",\"completed\":false}" - rc=$? - - if [ $rc -ne 0 ]; then - echo "Error adding todo, retrying..." - sleep 5 - fi - done - - set -e - - return $rc -} - -# Fetch a URL until its body contains the expected text. Even once the pods are -# ready the ingress/gateway needs a moment to route to the new endpoints, so a -# single-shot fetch here is a race. -wait_for_content () { - set +e - - url=$1 - expected=$2 - - local try=0 - local rc=1 - - while [ $rc -ne 0 ] && [ $try -lt 20 ]; do - try=$((try + 1)) - curl -s "$url" | grep -q "$expected" - rc=$? - - if [ $rc -ne 0 ]; then - echo "Waiting for $expected at $url..." - sleep 5 - fi - done - - set -e - - if [ $rc -ne 0 ]; then - echo "deploy http: failed - $expected not found at $url" - exit 1 - fi -} +# Whether kind or a real cluster, the images are pulled over the network, which +# can take minutes on a cold node, so allow ~5 minutes for services to come up. +START_CHECK_LIMIT=60 # Test basic stack deploy echo "Running stack deploy test" -# Bit of a hack, test the most recent package -TEST_TARGET_SO=$( ls -t1 ./package/stack* | head -1 ) -# We make a directory within which our test will create files -STACK_TEST_DIR=~/stack-test/k8s-test-dir -# Set a non-default repo dir -export STACK_REPO_BASE_DIR=${STACK_TEST_DIR}/repo-base-dir -echo "Testing this package: $TEST_TARGET_SO" -echo "Test version command" -reported_version_string=$( $TEST_TARGET_SO version ) -echo "Version reported is: ${reported_version_string}" -echo "Cloning repositories into: $STACK_REPO_BASE_DIR" -rm -rf $STACK_TEST_DIR -mkdir -p $STACK_TEST_DIR -mkdir -p $STACK_REPO_BASE_DIR +select_test_target "$@" +setup_test_dir k8s-test-dir # Test bringing the test container up and down # with and without volume removal STACK_NAME="todo" -$TEST_TARGET_SO fetch repo bozemanpass/example-todo-list -$TEST_TARGET_SO prepare --stack $STACK_NAME +$TEST_TARGET_STACK fetch repo bozemanpass/example-todo-list +$TEST_TARGET_STACK prepare --stack $STACK_NAME # Basic test of creating a deployment test_deployment_dir=$STACK_TEST_DIR/test-deployment-dir @@ -214,42 +46,37 @@ init_args="$init_args --config REACT_APP_API_URL=${TEST_SCHEME}://${TEST_HOSTNAM if [ "$STACK_K8S_REMOTE" = "true" ]; then init_args="$init_args --kube-config $STACK_KUBE_CONFIG --image-registry $STACK_IMAGE_REGISTRY" fi -$TEST_TARGET_SO init $init_args +$TEST_TARGET_STACK init $init_args # Check the file now exists if [ ! -f "$test_deployment_spec" ]; then - echo "deploy init test: spec file not present" - echo "deploy init test: FAILED" - exit 1 + fail "deploy init test: FAILED - spec file not present" fi echo "deploy init test: passed" -$TEST_TARGET_SO deploy --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir +stop_deployment_on_exit $test_deployment_dir +$TEST_TARGET_STACK deploy --spec-file $test_deployment_spec --deployment-dir $test_deployment_dir # Check the deployment dir exists if [ ! -d "$test_deployment_dir" ]; then - echo "deploy deploy test: deployment directory not present" - echo "deploy deploy test: FAILED" - exit 1 + fail "deploy create test: FAILED - deployment directory not present" fi echo "deploy create test: passed" # Push images to remote registry if needed if [ "$STACK_K8S_REMOTE" = "true" ]; then - $TEST_TARGET_SO manage --dir $test_deployment_dir push-images + $TEST_TARGET_STACK manage --dir $test_deployment_dir push-images fi # Start -$TEST_TARGET_SO manage --dir $test_deployment_dir start -wait_for_running 3 +$TEST_TARGET_STACK manage --dir $test_deployment_dir start +wait_for_running 3 $START_CHECK_LIMIT # Add a todo todo_title="79b06705-b402-431a-83a3-a634392d2754" -add_todo ${TEST_SCHEME}://${TEST_HOSTNAME}/api/todos "$todo_title" - +add_todo ${TEST_SCHEME}://${TEST_HOSTNAME}/api/todos "$todo_title" ${TEST_SCHEME}://${TEST_HOSTNAME} # Check that it exists if [ "$todo_title" != "$(curl -s ${TEST_SCHEME}://${TEST_HOSTNAME}/api/todos | jq -r '.[] | select(.id == 1) | .title')" ]; then - echo "deploy storage: failed - todo $todo_title not found" - exit 1 + fail "deploy storage: failed - todo $todo_title not found" fi # The built frontend references its JS bundle as /assets/index-.js, so @@ -257,6 +84,4 @@ fi wait_for_content ${TEST_SCHEME}://${TEST_HOSTNAME} '/assets/index-' echo "deploy http: passed" -delete_cluster_exit - echo "Test passed" diff --git a/tests/k8s-deployment-control/run-test.sh b/tests/k8s-deployment-control/run-test.sh index 587a8d1..8070a28 100755 --- a/tests/k8s-deployment-control/run-test.sh +++ b/tests/k8s-deployment-control/run-test.sh @@ -1,10 +1,9 @@ #!/usr/bin/env bash set -e if [ -n "$STACK_SCRIPT_DEBUG" ]; then - set -x - # Dump environment variables for debugging - echo "Environment variables:" - env + set -x + echo "Environment variables:" + env fi if [ "$1" == "from-path" ]; then diff --git a/tests/lib/common.sh b/tests/lib/common.sh new file mode 100644 index 0000000..b059391 --- /dev/null +++ b/tests/lib/common.sh @@ -0,0 +1,390 @@ +#!/usr/bin/env bash +# +# Shared helpers for the integration test scripts -- the bash suites under +# tests/, not the pytest suite in tests/unit. +# +# Source this as the first thing a test script does: +# +# source "$( dirname -- "${BASH_SOURCE[0]}" )/../lib/common.sh" +# +# Sourcing it turns on `set -e`, applies STACK_SCRIPT_DEBUG (xtrace plus an +# environment dump), and defines the helpers below. The scripts still expect to +# be run from the repo root, since the package under test is found in ./package. +# +# These scripts grew by copying each other, and every copied helper eventually +# drifted: a fix would land in one copy and not its siblings (the retry limit +# raised for slow image pulls in one wait loop, the response-body dump added to +# one HTTP wait, a `ps` marker that the product stopped printing). So anything +# needed by more than one test belongs here, and divergence between tests should +# be a parameter rather than a second copy. + +set -e + +if [ -n "$STACK_SCRIPT_DEBUG" ]; then + set -x + echo "Environment variables:" + env +fi + +# The deployment the helpers below act on; set by stop_deployment_on_exit. +TEST_DEPLOYMENT_DIR="" + +# Optionally the name of a function run during teardown, for a test that has +# cleanup of its own to do (a scratch registry container, say). +TEST_EXTRA_CLEANUP="" + +# Containers started via start_container, stopped at exit. +TEST_CONTAINER_IDS="" + +# --- reporting --------------------------------------------------------------- + +# Report a failure and exit non-zero. Teardown registered with +# stop_deployment_on_exit still runs, so a test never needs to call its own +# cleanup function by hand before failing. +fail () { + echo "$@" + exit 1 +} + +# --- preconditions ----------------------------------------------------------- + +# Check that the utilities a test needs are on the PATH. +require_commands () { + local cmd + for cmd in "$@"; do + if ! command -v "$cmd" &> /dev/null; then + fail "Error: '$cmd' is not installed or not available on the PATH" + fi + done +} + +# --- the package under test -------------------------------------------------- + +# Pick the stack executable to test, setting TEST_TARGET_STACK, and report it +# along with the version it claims. Pass the test script's arguments through: +# +# select_test_target "$@" +# +# In precedence order: an inherited TEST_TARGET_STACK (so a developer can point +# a test at "uv run stack"), then "from-path" as the first argument, then the +# most recently built shiv package in ./package. +select_test_target () { + if [ -z "$TEST_TARGET_STACK" ]; then + if [ "$1" == "from-path" ]; then + TEST_TARGET_STACK="stack" + else + TEST_TARGET_STACK=$( ls -t1 ./package/stack* | head -1 ) + fi + fi + echo "Testing this package: $TEST_TARGET_STACK" + echo "Version reported is: $( $TEST_TARGET_STACK version )" +} + +# --- test directories -------------------------------------------------------- + +# Remove a directory even when it holds root-owned files. Containers write into +# bind-mounted volume dirs as root, so a previous run's data cannot always be +# removed by the (non-root) host user; fall back to a throwaway container. +force_rm () { + if [ -e "$1" ]; then + rm -rf "$1" 2> /dev/null || true + fi + if [ -e "$1" ]; then + docker run --rm -v "$( dirname "$1" )":/w alpine rm -rf "/w/$( basename "$1" )" + fi +} + +# Create a clean working directory for the test under ~/stack-test and put the +# stack tool's repo base dir inside it. Sets STACK_TEST_DIR and exports +# STACK_REPO_BASE_DIR. Each test passes its own directory name, so tests do not +# share state with each other. +setup_test_dir () { + STACK_TEST_DIR=~/stack-test/"$1" + export STACK_REPO_BASE_DIR=${STACK_TEST_DIR}/repo-base-dir + echo "Using test directory: $STACK_TEST_DIR" + force_rm "$STACK_TEST_DIR" + mkdir -p "$STACK_REPO_BASE_DIR" +} + +# --- container images -------------------------------------------------------- + +# Remove every local copy of an image, so that what follows exercises a real +# build or pull instead of silently reusing whatever happened to be present. +# Deliberately fails loudly: if the image survives, the test that follows is not +# testing what it claims to. +remove_local_images () { + local existing + existing=$( docker image ls -q --filter=reference="$1" | uniq ) + if [ -n "$existing" ]; then + docker image rm -f ${existing} + fi +} + +# --- deployment lifecycle ---------------------------------------------------- + +# Register teardown for a deployment, and tell the wait helpers below which +# deployment they are acting on. From here on any exit -- success, `fail`, or an +# error under `set -e` -- stops the deployment and deletes its volumes. +# +# On a failing exit the containers' logs are dumped first: the deployment is +# about to be destroyed, and in CI those logs are usually the only evidence of +# what went wrong. +stop_deployment_on_exit () { + TEST_DEPLOYMENT_DIR="$1" + trap _test_exit_handler EXIT +} + +_test_exit_handler () { + local rc=$? + trap - EXIT + set +e + if [ -n "$TEST_DEPLOYMENT_DIR" ] && [ -d "$TEST_DEPLOYMENT_DIR" ]; then + if [ $rc -ne 0 ]; then + dump_diagnostics + fi + $TEST_TARGET_STACK manage --dir "$TEST_DEPLOYMENT_DIR" stop --delete-volumes + fi + if [ -n "$TEST_CONTAINER_IDS" ]; then + docker stop $TEST_CONTAINER_IDS > /dev/null 2>&1 + fi + if [ -n "$TEST_EXTRA_CLEANUP" ]; then + $TEST_EXTRA_CLEANUP + fi + exit $rc +} + +# Start a container with `docker run` and remember it, setting CONTAINER_ID. +# Pass the usual docker run arguments: +# +# start_container -p 3000:80 -d "$image" +# +# The container is stopped at exit however the test ends. Without that, a test +# that fails mid-way leaves its port bound and the *next* test fails with a +# confusing "port is already allocated". Stopping it by hand earlier is fine. +start_container () { + CONTAINER_ID=$( docker run "$@" ) + TEST_CONTAINER_IDS="$TEST_CONTAINER_IDS $CONTAINER_ID" + trap _test_exit_handler EXIT +} + +# Report what a failing deployment was doing, before it is torn down. +dump_diagnostics () { + echo "===================== FAILURE DIAGNOSTICS =====================" + echo "----- ps -----" + $TEST_TARGET_STACK manage --dir "$TEST_DEPLOYMENT_DIR" ps || true + echo "----- container logs (last 200 lines per service) -----" + $TEST_TARGET_STACK manage --dir "$TEST_DEPLOYMENT_DIR" logs -n 200 || true + echo "==============================================================" +} + +# Run a command inside one of the deployment's containers. `exec` wraps the +# command in `sh -c`, so the whole command goes in a single argument: +# +# deployment_exec app "echo hello > /data/file" +deployment_exec () { + $TEST_TARGET_STACK manage --dir "$TEST_DEPLOYMENT_DIR" exec "$1" "$2" +} + +# --- waiting ----------------------------------------------------------------- + +# Wait until `manage status` reports at least $1 services running. $2 overrides +# the number of 5-second checks (default 10): against a real cluster the images +# are pulled from a real registry over the network, which can take minutes on a +# cold node, so those tests ask for more. +# +# "status" reports a container as running only once it is actually ready -- on +# k8s once its readiness probes pass -- so this is a readiness wait, not just a +# started wait. +wait_for_running () { + local how_many=$1 + local check_limit=${2:-10} + local running=0 + local check=0 + while [ $running -lt $how_many ] && [ $check -lt $check_limit ]; do + check=$((check + 1)) + # grep -c exits non-zero when the count is zero, which is a normal + # outcome early on, so do not let it trip `set -e`. + running=$( $TEST_TARGET_STACK manage --dir "$TEST_DEPLOYMENT_DIR" status | grep -ic "running" ) || true + if [ $running -lt $how_many ]; then + echo "waiting for services to start ($running/$how_many)..." + sleep 5 + fi + done + if [ $running -lt $how_many ]; then + fail "waiting for services to start: FAILED - $running of $how_many running" + fi +} + +# Wait until `manage ps` lists the deployment's containers. ps prints one +# "id: ..." line per container for both targets (ps_operation in +# src/stack/deploy/deploy.py), so that is the marker to match. $1 overrides the +# number of 5-second checks (default 50). +wait_for_containers_started () { + local check_limit=${1:-50} + local check=0 + local ps_output + while [ $check -lt $check_limit ]; do + check=$((check + 1)) + ps_output=$( $TEST_TARGET_STACK manage --dir "$TEST_DEPLOYMENT_DIR" ps ) + if [[ "$ps_output" == *"id:"* ]]; then + return + fi + sleep 5 + done + fail "waiting for containers to start: FAILED" +} + +# Wait until `manage logs` output contains $1 -- or, with no argument, until it +# produces any output at all. $2 overrides the number of 5-second checks +# (default 50). +wait_for_log_content () { + local expected=$1 + local check_limit=${2:-50} + local check=0 + local log_output + while [ $check -lt $check_limit ]; do + check=$((check + 1)) + log_output=$( $TEST_TARGET_STACK manage --dir "$TEST_DEPLOYMENT_DIR" logs ) + if [ -z "$expected" ]; then + if [ -n "$log_output" ]; then + return + fi + elif [[ "$log_output" == *"$expected"* ]]; then + return + fi + sleep 5 + done + fail "waiting for log content '${expected}': FAILED" +} + +# POST a todo to the example todo app serving at $1, retrying until it is +# accepted. $3 is the origin the request claims to come from -- the app's CORS +# handling rejects a mismatch, so it is "http://localhost" for a compose +# deployment and the scheme+host of the cluster for k8s. +# +# The browser-shaped headers are deliberate: the app is fronted by CORS and +# content-type checks, and a bare curl does not get past them. +add_todo () { + local url=$1 + local title=$2 + local origin=$3 + local try=0 + local rc=1 + + while [ $rc -ne 0 ] && [ $try -lt 10 ]; do + try=$((try + 1)) + rc=0 + curl "$url" \ + --fail-with-body \ + -H 'Accept: application/json, text/plain, */*' \ + -H 'Accept-Language: en-US,en;q=0.9' \ + -H 'Connection: keep-alive' \ + -H 'Content-Type: application/json' \ + -H "Origin: ${origin}" \ + -H "Referer: ${origin}/" \ + -H 'Sec-Fetch-Dest: empty' \ + -H 'Sec-Fetch-Mode: cors' \ + -H 'Sec-Fetch-Site: same-site' \ + -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36 Edg/135.0.0.0' \ + -H 'sec-ch-ua: "Microsoft Edge";v="135", "Not-A.Brand";v="8", "Chromium";v="135"' \ + -H 'sec-ch-ua-mobile: ?0' \ + -H 'sec-ch-ua-platform: "Windows"' \ + --data-raw "{\"title\":\"$title\",\"completed\":false}" || rc=$? + + if [ $rc -ne 0 ]; then + echo "Error adding todo, retrying..." + sleep 5 + fi + done + + if [ $rc -ne 0 ]; then + fail "add todo: FAILED - could not add '$title' at $url" + fi +} + +# Fetch $1 until its body contains $2. $3 overrides the number of 5-second +# attempts (default 20). +# +# A container being ready is not the same as its server being ready to serve -- +# and on k8s the ingress/gateway needs a moment to route to the new endpoints -- +# so a single-shot fetch here is a race. The last response body is printed on +# failure, without which a CI log says only that the text was not found. +wait_for_content () { + local url=$1 + local expected=$2 + local tries=${3:-20} + local try=0 + local body="" + while [ $try -lt $tries ]; do + try=$((try + 1)) + body=$( curl -s "$url" ) || true + if echo "$body" | grep -q "$expected"; then + return + fi + echo "Waiting for $expected at $url..." + sleep 5 + done + echo "last response body was:" + echo "$body" + fail "http: FAILED - $expected not found at $url" +} + +# --- fetching and asserting -------------------------------------------------- + +# Fetch $1 into the file $2, retrying while the server comes up. Any further +# arguments are passed to wget (e.g. -m to follow links, so that an assertion can +# look at content the page pulls in rather than just the page itself). +# +# Returns wget's exit status instead of aborting, so a caller that expects a +# fetch to fail can capture it: +# +# rc=0; fetch_url "$url" out.html || rc=$? +fetch_url () { + local url=$1 + local out=$2 + shift 2 + wget --tries 20 --retry-connrefused --waitretry=3 -O "$out" "$@" "$url" +} + +# Assert that $2 (an extended regular expression) appears in the file $1, +# reporting "