diff --git a/.github/workflows/beaker_acceptance.yml b/.github/workflows/beaker_acceptance.yml index a18f9c8..252a2ee 100644 --- a/.github/workflows/beaker_acceptance.yml +++ b/.github/workflows/beaker_acceptance.yml @@ -121,6 +121,7 @@ jobs: acceptance-working-dir: ${{ steps.set-defaults.outputs.acceptance_working_dir }} acceptance-pre-suite: ${{ steps.set-defaults.outputs.acceptance_pre_suite }} acceptance-tests: ${{ steps.set-defaults.outputs.acceptance_tests }} + acceptance-post-suite: ${{ steps.set-defaults.outputs.acceptance_post_suite }} beaker-options: ${{ steps.set-defaults.outputs.beaker_options }} steps: - name: Checkout platform config @@ -353,6 +354,8 @@ jobs: ${{ needs.set-project-defaults.outputs.acceptance-pre-suite || '[]' }} TESTS_ARRAY: |- ${{ needs.set-project-defaults.outputs.acceptance-tests || '[]' }} + POST_SUITE_ARRAY: |- + ${{ needs.set-project-defaults.outputs.acceptance-post-suite || '[]' }} TAGS: ${{ matrix.tags && toJson(matrix.tags) || '[]' }} BEAKER_OPTIONS: |- ${{ needs.set-project-defaults.outputs.beaker-options || '{}' }} @@ -375,6 +378,7 @@ jobs: "type": "aio", "pre_suite": ${PRE_SUITE_ARRAY}, "tests": ${TESTS_ARRAY}, + "post_suite": ${POST_SUITE_ARRAY}, "test_tag_and": ${TAGS} }, ${BEAKER_OPTIONS} diff --git a/.github/workflows/build_ezbake.yml b/.github/workflows/build_ezbake.yml index e147e07..306746d 100644 --- a/.github/workflows/build_ezbake.yml +++ b/.github/workflows/build_ezbake.yml @@ -20,8 +20,12 @@ on: Branch/tag from ezbake that will be used for openvoxdb/server builds. type: string required: false - ezbake-ver: - description: 'The version specified in project.clj in the given ezbake-ref. Will default to the version found in project.clj in the repo if not specified.' + ezbake-2-ver: + description: 'Version for the ezbake 2 build group used for older OSes.' + type: string + required: false + ezbake-4-ver: + description: 'Version for the ezbake 4 build group used for newer OSes.' type: string required: false branch: @@ -38,17 +42,15 @@ env: # https://github.com/boto/boto3/issues/4398#issuecomment-2619946229 AWS_REQUEST_CHECKSUM_CALCULATION: "WHEN_REQUIRED" AWS_RESPONSE_CHECKSUM_VALIDATION: "WHEN_REQUIRED" - EZBAKE_BRANCH: ${{ inputs.ezbake-ref }} jobs: - build: - runs-on: ubuntu-24.04 + build-ezbake-2: + name: ezbake 2.x build for legacy OSes + runs-on: ubuntu-26.04 timeout-minutes: 600 strategy: fail-fast: false steps: - # This checks out the release branch, because the vox:build task will - # check out the tag itself. - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -58,19 +60,30 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: OpenVoxProject/shared-actions - ref: main + ref: ezbake4 # switch to main before merge! path: .shared-actions sparse-checkout: platforms.json sparse-checkout-cone-mode: false - - name: Resolve platform lists + - name: Resolve platform lists for ezbake 2 env: DEB_PLATFORMS: ${{ inputs.deb_platform_list }} RPM_PLATFORMS: ${{ inputs.rpm_platform_list }} BRANCH: ${{ inputs.branch }} + EZBAKE_2_VER: ${{ inputs.ezbake-2-ver }} run: | - echo "DEB_PLATFORMS=${DEB_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake-deb" | join(",")' .shared-actions/platforms.json)}" >> $GITHUB_ENV - echo "RPM_PLATFORMS=${RPM_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake-rpm" | join(",")' .shared-actions/platforms.json)}" >> $GITHUB_ENV + EZBAKE_2_VER="${EZBAKE_2_VER:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake2-version"' .shared-actions/platforms.json)}" + + echo "EZBAKE_BRANCH=${{ inputs.ezbake-ref }}" >> $GITHUB_ENV + echo "EZBAKE_VERSION=${EZBAKE_2_VER}" >> $GITHUB_ENV + DEB_PLATFORMS="${DEB_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake2-deb" | join(",")' .shared-actions/platforms.json)}" + echo "DEB_PLATFORMS=$DEB_PLATFORMS" >> $GITHUB_ENV + RPM_PLATFORMS="${RPM_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake2-rpm" | join(",")' .shared-actions/platforms.json)}" + echo "RPM_PLATFORMS=$RPM_PLATFORMS" >> $GITHUB_ENV + + echo "::notice title=ezbake 2 EZBAKE_BRANCH::${{ inputs.ezbake-ref }}" + echo "::notice title=ezbake 2 EZBAKE_VERSION::$EZBAKE_2_VER" + echo "::notice title=ezbake 2 PLATFORMS::$DEB_PLATFORMS,$RPM_PLATFORMS" - name: Install Ruby uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 @@ -82,7 +95,7 @@ jobs: uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: /tmp/ezbake-builder.tar - key: docker-ezbake-${{ hashFiles('Dockerfile') }} + key: docker-ezbake-${{ hashFiles('Dockerfile') }}-legacy # same key as in the fips workflow - name: Load or build Docker image run: | @@ -97,17 +110,99 @@ jobs: if: always() && steps.docker-cache.outputs.cache-hit != 'true' with: path: /tmp/ezbake-builder.tar - key: docker-ezbake-${{ hashFiles('Dockerfile') }} + key: docker-ezbake-${{ hashFiles('Dockerfile') }}-legacy - name: Update awscli run: | python -m pip install --upgrade awscli - - name: Set EZBAKE_VERSION if provided - if: ${{ inputs.ezbake-ver != '' }} + - name: Run build script + env: + BUILD_REF: ${{ inputs.ref }} + run: | + rm -rf output + bundle exec rake "vox:build[${BUILD_REF}]" + + - name: Configure OSUOSL CA Certificate for S3 + uses: $/.github/actions/setup-osuosl-ca + + - name: Upload output to S3 + run: | + bundle exec rake vox:upload + + build-ezbake-4: + name: ezbake 4.x build for modern OSes + runs-on: ubuntu-26.04 + if: inputs.branch == 'main' + timeout-minutes: 600 + strategy: + fail-fast: false + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.branch }} + + - name: Checkout platform config + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: OpenVoxProject/shared-actions + ref: ezbake4 # switch to main before merge! + path: .shared-actions + sparse-checkout: platforms.json + sparse-checkout-cone-mode: false + + - name: Resolve platform lists for ezbake 4 env: - EZBAKE_VER: ${{ inputs.ezbake-ver }} - run: echo "EZBAKE_VERSION=${EZBAKE_VER}" >> $GITHUB_ENV + DEB_PLATFORMS: ${{ inputs.deb_platform_list }} + RPM_PLATFORMS: ${{ inputs.rpm_platform_list }} + BRANCH: ${{ inputs.branch }} + EZBAKE_4_VER: ${{ inputs.ezbake-4-ver }} + run: | + EZBAKE_4_VER="${EZBAKE_4_VER:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake4-version"' .shared-actions/platforms.json)}" + + echo "EZBAKE_BRANCH=${{ inputs.ezbake-ref }}" >> $GITHUB_ENV + echo "EZBAKE_VERSION=${EZBAKE_4_VER}" >> $GITHUB_ENV + DEB_PLATFORMS="${DEB_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake4-deb" | join(",")' .shared-actions/platforms.json)}" + echo "DEB_PLATFORMS=$DEB_PLATFORMS" >> $GITHUB_ENV + RPM_PLATFORMS="${RPM_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake4-rpm" | join(",")' .shared-actions/platforms.json)}" + echo "RPM_PLATFORMS=$RPM_PLATFORMS" >> $GITHUB_ENV + + echo "::notice title=ezbake 4 EZBAKE_BRANCH::${{ inputs.ezbake-ref }}" + echo "::notice title=ezbake 4 EZBAKE_VERSION::$EZBAKE_4_VER" + echo "::notice title=ezbake 4 PLATFORMS::$DEB_PLATFORMS,$RPM_PLATFORMS" + + - name: Install Ruby + uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 + with: + ruby-version: '4.0' + bundler-cache: true + + # we need to cache the container per-ezbake version because ezbake is checked out inside of that container + - id: docker-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: /tmp/ezbake-builder.tar + key: docker-ezbake-${{ hashFiles('Dockerfile') }}-modern + + - name: Load or build Docker image + run: | + if [[ -f /tmp/ezbake-builder.tar ]]; then + docker load -i /tmp/ezbake-builder.tar + else + docker build -t ezbake-builder . + docker save ezbake-builder -o /tmp/ezbake-builder.tar + fi + + - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: always() && steps.docker-cache.outputs.cache-hit != 'true' + with: + path: /tmp/ezbake-builder.tar + key: docker-ezbake-${{ hashFiles('Dockerfile') }}-modern + + - name: Update awscli + run: | + python -m pip install --upgrade awscli - name: Run build script env: diff --git a/.github/workflows/build_ezbake_fips.yml b/.github/workflows/build_ezbake_fips.yml index 23a8b42..bb10078 100644 --- a/.github/workflows/build_ezbake_fips.yml +++ b/.github/workflows/build_ezbake_fips.yml @@ -17,7 +17,7 @@ on: type: string required: false ezbake-ver: - description: 'The version specified in project.clj in the given ezbake-ref. Will default to the version found in project.clj in the repo if not specified.' + description: 'Version for the ezbake 2 build group used for older OSes.' type: string required: false branch: @@ -39,7 +39,7 @@ env: jobs: build: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 600 strategy: fail-fast: false @@ -55,7 +55,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: OpenVoxProject/shared-actions - ref: main + ref: ezbake4 # rename to main before merge path: .shared-actions sparse-checkout: platforms.json sparse-checkout-cone-mode: false @@ -65,7 +65,16 @@ jobs: RPM_PLATFORMS: ${{ inputs.rpm_platform_list }} BRANCH: ${{ inputs.branch }} run: | - echo "RPM_PLATFORMS=${RPM_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake-fips-rpm" | join(",")' .shared-actions/platforms.json)}" >> $GITHUB_ENV + EZBAKE_VERSION="${EZBAKE_VER:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake2-version"' .shared-actions/platforms.json)}" + + echo "EZBAKE_BRANCH=${{ inputs.ezbake-ref }}" >> $GITHUB_ENV + echo "EZBAKE_VERSION=${EZBAKE_VERSION}" >> $GITHUB_ENV + RPM_PLATFORMS="${RPM_PLATFORMS:-$(jq -r --arg branch "${BRANCH}" '.[$branch]."ezbake-fips-rpm" | join(",")' .shared-actions/platforms.json)}" + echo "RPM_PLATFORMS=$RPM_PLATFORMS" >> $GITHUB_ENV + + echo "::notice title=ezbake 2 EZBAKE_BRANCH::${{ inputs.ezbake-ref }}" + echo "::notice title=ezbake 2 EZBAKE_VERSION::$EZBAKE_VERSION" + echo "::notice title=ezbake 2 PLATFORMS::$RPM_PLATFORMS" - name: Install Ruby uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 @@ -77,7 +86,7 @@ jobs: uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: /tmp/ezbake-builder.tar - key: docker-ezbake-${{ hashFiles('Dockerfile') }} + key: docker-ezbake-${{ hashFiles('Dockerfile') }}-legacy # same key as in non-fips ezbake 2 - name: Load or build Docker image run: | @@ -92,18 +101,12 @@ jobs: if: always() && steps.docker-cache.outputs.cache-hit != 'true' with: path: /tmp/ezbake-builder.tar - key: docker-ezbake-${{ hashFiles('Dockerfile') }} + key: docker-ezbake-${{ hashFiles('Dockerfile') }}-legacy - name: Update awscli run: | python -m pip install --upgrade awscli - - name: Set EZBAKE_VERSION if provided - if: ${{ inputs.ezbake-ver != '' }} - env: - EZBAKE_VER: ${{ inputs.ezbake-ver }} - run: echo "EZBAKE_VERSION=${EZBAKE_VER}" >> $GITHUB_ENV - - name: Run build script env: BUILD_REF: ${{ inputs.ref }} diff --git a/platforms.json b/platforms.json index c4f6418..da95c89 100644 --- a/platforms.json +++ b/platforms.json @@ -1,5 +1,7 @@ { "main": { + "ezbake2-version": "2.9.1", + "ezbake4-version": "4.0.0", "vanagon": [ "amazon-2023-aarch64", "amazon-2023-x86_64", @@ -35,20 +37,24 @@ "ubuntu-26.04-armhf", "windows-msys2-x64" ], - "ezbake-deb": [ - "ubuntu-22.04", - "ubuntu-24.04", - "ubuntu-26.04", - "debian-13" + "ezbake2-deb": [ + "ubuntu-22.04" ], - "ezbake-rpm": [ + "ezbake2-rpm": [ "amazon-2023", "el-8", "el-9", + "sles-15" + ], + "ezbake4-deb": [ + "ubuntu-24.04", + "ubuntu-26.04", + "debian-13" + ], + "ezbake4-rpm": [ "el-10", "fedora-43", "fedora-44", - "sles-15", "sles-16" ], "ezbake-fips-rpm": [ @@ -57,6 +63,8 @@ ] }, "8.x": { + "ezbake2-version": "2.9.1", + "ezbake4-version": "4.0.0", "vanagon": [ "amazon-2-aarch64", "amazon-2-x86_64", @@ -102,16 +110,16 @@ "ubuntu-26.04-armhf", "windows-all-x64" ], - "ezbake-deb": [ + "ezbake2-deb": [ "ubuntu-22.04", "ubuntu-24.04", "ubuntu-25.04", - "ubuntu-26.04", "debian-11", "debian-12", + "ubuntu-26.04", "debian-13" ], - "ezbake-rpm": [ + "ezbake2-rpm": [ "amazon-2", "amazon-2023", "el-8", diff --git a/scripts/project-config.sh b/scripts/project-config.sh index d93f3d4..4482bf4 100755 --- a/scripts/project-config.sh +++ b/scripts/project-config.sh @@ -78,18 +78,26 @@ map_platforms=' intersect='. as $uni | $uni | map(select(. as $u | ($set | any(. == $u))))' # Which build list gates this suite: the agent (vanagon) list for the agent -# suites, or the combined server/db (ezbake) list otherwise. +# suites, or the combined server/db (ezbake) list otherwise. The platform file +# uses both the legacy ezbake2 keys and the current ezbake4 keys depending on +# the branch, so we must accept both shapes instead of assuming one stale name. case "$suite" in openvox|openvox-agent) build_list=$(jq -c --arg b "$branch" '.[$b].vanagon' "$platforms") ;; openvox-server|openvoxdb) - build_list=$(jq -c --arg b "$branch" '(.[$b]["ezbake-deb"] + .[$b]["ezbake-rpm"])' "$platforms") ;; + build_list=$(jq -c --arg b "$branch" ' + (.[$b]["ezbake4-deb"] // [] + .[$b]["ezbake4-rpm"] // [] + + .[$b]["ezbake2-deb"] // [] + .[$b]["ezbake2-rpm"] // []) + ' "$platforms") ;; *) echo "Unknown suite '$suite'" >&2; exit 1 ;; esac # The server/db build list is always needed to decide when a fall back is needed # to test an agent platform that does not have a corresponding server build. -server_list=$(jq -c --arg b "$branch" '(.[$b]["ezbake-deb"] + .[$b]["ezbake-rpm"])' "$platforms") +server_list=$(jq -c --arg b "$branch" ' + (.[$b]["ezbake4-deb"] // [] + .[$b]["ezbake4-rpm"] // [] + + .[$b]["ezbake2-deb"] // [] + .[$b]["ezbake2-rpm"] // []) +' "$platforms") # Translate platforms.json OS names to nested_vms tuples. build_os=$(jq -c "$map_platforms" <<<"$build_list") diff --git a/scripts/project_defaults.json b/scripts/project_defaults.json index a128851..95cd060 100644 --- a/scripts/project_defaults.json +++ b/scripts/project_defaults.json @@ -11,6 +11,9 @@ "acceptance-tests": [ "tests" ], + "acceptance-post-suite": [ + "teardown/common/099_Archive_Logs.rb" + ], "acceptance-shard-tags": [ "shard:group1", "shard:group2",