From 20a3ef995d57b5e4048bcb95e36e65bd677bf9b8 Mon Sep 17 00:00:00 2001 From: cpendery Date: Fri, 7 Aug 2026 09:14:15 -0700 Subject: [PATCH] ci: reduce release complexity Signed-off-by: cpendery --- .github/scripts/release/package-node.mjs | 5 +- .github/scripts/release/publish-npm.mjs | 63 ------ .github/scripts/release/smoke-node.mjs | 74 ------- .github/scripts/release/smoke-python.py | 75 ------- .github/scripts/release/utils.mjs | 49 ----- .github/scripts/release/verify-versions.mjs | 79 ------- .github/workflows/release.yml | 195 +++++++++--------- bindings/js/test/package.integration.test.mjs | 30 +++ 8 files changed, 135 insertions(+), 435 deletions(-) delete mode 100644 .github/scripts/release/publish-npm.mjs delete mode 100644 .github/scripts/release/smoke-node.mjs delete mode 100644 .github/scripts/release/smoke-python.py delete mode 100644 .github/scripts/release/utils.mjs delete mode 100644 .github/scripts/release/verify-versions.mjs create mode 100644 bindings/js/test/package.integration.test.mjs diff --git a/.github/scripts/release/package-node.mjs b/.github/scripts/release/package-node.mjs index 91cbd34..6762e38 100644 --- a/.github/scripts/release/package-node.mjs +++ b/.github/scripts/release/package-node.mjs @@ -1,13 +1,14 @@ +import { execFileSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; -import { runNpm } from "./utils.mjs"; - const bindingsDirectory = path.resolve("bindings/js"); const nativePackagesDirectory = path.join(bindingsDirectory, "npm"); const outputDirectory = path.resolve("package-artifacts/npm"); const packagePath = path.join(bindingsDirectory, "package.json"); +const runNpm = (args) => execFileSync("npm", args, { stdio: "inherit" }); + const nativePackages = fs .readdirSync(nativePackagesDirectory, { withFileTypes: true }) .filter((entry) => entry.isDirectory()) diff --git a/.github/scripts/release/publish-npm.mjs b/.github/scripts/release/publish-npm.mjs deleted file mode 100644 index 59e32ed..0000000 --- a/.github/scripts/release/publish-npm.mjs +++ /dev/null @@ -1,63 +0,0 @@ -import { - listPackageTarballs, - readPackageManifest, - runNpm, - spawnNpm, -} from "./utils.mjs"; - -const packages = listPackageTarballs("npm-packages").map((tarball) => ({ - tarball, - manifest: readPackageManifest(tarball), -})); - -function publishIfMissing({ tarball, manifest }) { - const packageVersion = `${manifest.name}@${manifest.version}`; - const result = spawnNpm(["view", packageVersion, "version"], { - stdio: "ignore", - }); - - if (result.error) { - throw result.error; - } - if (result.status === 0) { - console.log(`${packageVersion} is already published`); - return; - } - if (result.signal) { - throw new Error( - `npm view ${packageVersion} terminated with ${result.signal}`, - ); - } - - runNpm([ - "publish", - tarball, - "--access", - "public", - "--provenance", - "--tag", - "latest", - ]); -} - -const nativePackages = packages.filter(({ manifest }) => - manifest.name.startsWith("@microsoft/shell-use-"), -); -const rootPackage = packages.filter( - ({ manifest }) => manifest.name === "@microsoft/shell-use", -); - -if ( - nativePackages.length !== 8 || - rootPackage.length !== 1 -) { - throw new Error( - `Expected eight native packages and @microsoft/shell-use; found ${packages - .map(({ manifest }) => manifest.name) - .join(", ")}`, - ); -} - -for (const packageArtifact of [...nativePackages, ...rootPackage]) { - publishIfMissing(packageArtifact); -} diff --git a/.github/scripts/release/smoke-node.mjs b/.github/scripts/release/smoke-node.mjs deleted file mode 100644 index 31b3c11..0000000 --- a/.github/scripts/release/smoke-node.mjs +++ /dev/null @@ -1,74 +0,0 @@ -import fs from "node:fs"; -import path from "node:path"; -import { spawnSync } from "node:child_process"; -import { createRequire } from "node:module"; -import { pathToFileURL } from "node:url"; - -import { - listPackageTarballs, - readPackageManifest, - runNpm, -} from "./utils.mjs"; - -const packages = listPackageTarballs("npm-packages").map((tarball) => ({ - tarball, - manifest: readPackageManifest(tarball), -})); - -function findPackage(name) { - const matches = packages.filter(({ manifest }) => manifest.name === name); - if (matches.length !== 1) { - throw new Error(`Expected one ${name} package, found ${matches.length}`); - } - return matches[0]; -} - -const rootPackage = findPackage("@microsoft/shell-use"); -const platformPackage = findPackage("@microsoft/shell-use-linux-x64-gnu"); -const smokeDirectory = path.resolve("smoke"); - -for (const { manifest } of [rootPackage, platformPackage]) { - if (manifest.bin !== undefined) { - throw new Error(`${manifest.name} unexpectedly declares a CLI executable`); - } -} - -fs.mkdirSync(smokeDirectory, { recursive: true }); -runNpm(["init", "-y"], { - cwd: smokeDirectory, - stdio: ["ignore", "ignore", "inherit"], -}); -runNpm(["install", "--ignore-scripts", platformPackage.tarball], { - cwd: smokeDirectory, -}); -runNpm( - [ - "install", - "--ignore-scripts", - "--omit=optional", - rootPackage.tarball, - ], - { cwd: smokeDirectory }, -); - -process.env.SHELL_USE_BIN = path.join(smokeDirectory, "missing-shell-use"); -if (process.platform !== "win32") { - process.env.PATH = "/usr/bin:/bin"; -} -const cliProbe = spawnSync("shell-use", ["--version"], { stdio: "ignore" }); -if (!cliProbe.error || cliProbe.error.code !== "ENOENT") { - throw new Error("shell-use CLI unexpectedly available in smoke PATH"); -} - -const requireFromSmoke = createRequire(path.join(smokeDirectory, "package.json")); -const packageEntry = requireFromSmoke.resolve(rootPackage.manifest.name); -const { ShellUse } = await import(pathToFileURL(packageEntry).href); -const session = ShellUse.ephemeral("release-smoke"); -try { - await session.open(); - await session.submit("echo release-smoke"); - await session.waitCommand(); - await session.expectText("release-smoke", { strict: false }); -} finally { - await session.closeQuiet(); -} diff --git a/.github/scripts/release/smoke-python.py b/.github/scripts/release/smoke-python.py deleted file mode 100644 index 0c05d32..0000000 --- a/.github/scripts/release/smoke-python.py +++ /dev/null @@ -1,75 +0,0 @@ -import asyncio -import os -import shutil -import subprocess -import sys -import venv -from pathlib import Path - - -async def smoke_test(): - from shell_use import ShellUse - - async with ShellUse.ephemeral("release-smoke") as session: - await session.open() - await session.submit("echo release-smoke") - await session.wait_command() - await session.expect_text("release-smoke", strict=False) - - -def main(): - if "--run-smoke" in sys.argv: - asyncio.run(smoke_test()) - return - - wheels = sorted(Path("dist").glob("*.whl")) - if len(wheels) != 1: - raise RuntimeError(f"Expected one Python wheel in dist, found {len(wheels)}") - wheel = wheels[0] - if "abi3" not in wheel.name: - raise RuntimeError(f"Expected an abi3 wheel, found {wheel.name}") - - smoke_directory = Path("smoke") - venv.create(smoke_directory, with_pip=True) - python = smoke_directory / ( - "Scripts/python.exe" if os.name == "nt" else "bin/python" - ) - - subprocess.run( - [ - python, - "-m", - "pip", - "install", - "--disable-pip-version-check", - wheel, - ], - check=True, - ) - - runtime_env = os.environ.copy() - runtime_env["SHELL_USE_BIN"] = str( - (smoke_directory / "missing-shell-use").resolve() - ) - if os.name == "nt": - runtime_path = [str(python.parent)] - system_root = runtime_env.get("SystemRoot") - if system_root: - runtime_path.extend( - [str(Path(system_root) / "System32"), str(Path(system_root))] - ) - else: - runtime_path = [str(python.parent), "/usr/bin", "/bin"] - runtime_env["PATH"] = os.pathsep.join(runtime_path) - if shutil.which("shell-use", path=runtime_env["PATH"]) is not None: - raise RuntimeError("shell-use CLI unexpectedly available in smoke PATH") - - subprocess.run( - [python, Path(__file__).resolve(), "--run-smoke"], - check=True, - env=runtime_env, - ) - - -if __name__ == "__main__": - main() diff --git a/.github/scripts/release/utils.mjs b/.github/scripts/release/utils.mjs deleted file mode 100644 index 0a39dd0..0000000 --- a/.github/scripts/release/utils.mjs +++ /dev/null @@ -1,49 +0,0 @@ -import { execFileSync, spawnSync } from "node:child_process"; -import fs from "node:fs"; -import path from "node:path"; - -export function run(command, args, options = {}) { - const { stdio = "inherit", ...rest } = options; - execFileSync(command, args, { ...rest, stdio }); -} - -function npmInvocation(args) { - if (process.platform === "win32") { - return { - command: process.env.ComSpec ?? "cmd.exe", - args: ["/d", "/s", "/c", "npm", ...args], - }; - } - return { command: "npm", args }; -} - -export function runNpm(args, options = {}) { - const invocation = npmInvocation(args); - run(invocation.command, invocation.args, options); -} - -export function spawnNpm(args, options = {}) { - const invocation = npmInvocation(args); - return spawnSync(invocation.command, invocation.args, options); -} - -export function listPackageTarballs(directory) { - return fs - .readdirSync(directory, { withFileTypes: true }) - .filter((entry) => entry.isFile() && entry.name.endsWith(".tgz")) - .map((entry) => path.resolve(directory, entry.name)) - .sort(); -} - -export function readPackageManifest(tarball) { - const absoluteTarball = path.resolve(tarball); - const contents = execFileSync( - "tar", - ["-xOf", path.basename(absoluteTarball), "package/package.json"], - { - cwd: path.dirname(absoluteTarball), - encoding: "utf8", - }, - ); - return JSON.parse(contents); -} diff --git a/.github/scripts/release/verify-versions.mjs b/.github/scripts/release/verify-versions.mjs deleted file mode 100644 index 6609ba7..0000000 --- a/.github/scripts/release/verify-versions.mjs +++ /dev/null @@ -1,79 +0,0 @@ -import fs from "node:fs"; - -const releaseTag = process.env.RELEASE_TAG; -if (!releaseTag) { - throw new Error("RELEASE_TAG is required"); -} - -const expected = releaseTag.replace(/^v/, ""); - -function read(file) { - return fs.readFileSync(file, "utf8"); -} - -function matchedVersion(file, pattern) { - const version = read(file).match(pattern)?.[1]; - if (!version) { - throw new Error(`Could not read a version from ${file}`); - } - return version; -} - -const jsPackage = JSON.parse(read("bindings/js/package.json")); -const jsPackageLock = JSON.parse(read("bindings/js/package-lock.json")); -const versions = { - "Cargo.toml [workspace.package]": matchedVersion( - "Cargo.toml", - /^\[workspace\.package\]\s*$[\s\S]*?^\s*version\s*=\s*"([^"]+)"/m, - ), - "bindings/js/package.json": jsPackage.version, - "bindings/js/package-lock.json": jsPackageLock.version, - "bindings/js/package-lock.json packages['']": - jsPackageLock.packages?.[""]?.version, - "bindings/js/src/version.ts": matchedVersion( - "bindings/js/src/version.ts", - /VERSION\s*=\s*"([^"]+)"/, - ), - "bindings/python/pyproject.toml": matchedVersion( - "bindings/python/pyproject.toml", - /^\s*version\s*=\s*"([^"]+)"/m, - ), - "bindings/python/src/shell_use/_config.py": matchedVersion( - "bindings/python/src/shell_use/_config.py", - /VERSION\s*=\s*"([^"]+)"/, - ), -}; - -for (const [file, version] of Object.entries(versions)) { - if (version !== expected) { - throw new Error(`${file} has version ${version}; expected ${expected}`); - } -} - -const workspaceVersionManifests = [ - "crates/shell-use/Cargo.toml", - "crates/shell-use-cli/Cargo.toml", - "bindings/js/Cargo.toml", - "bindings/python/native/Cargo.toml", -]; -for (const file of workspaceVersionManifests) { - if (!/^\s*version\.workspace\s*=\s*true\s*$/m.test(read(file))) { - throw new Error(`${file} must inherit workspace.package.version`); - } -} - -const nativeLoader = read("bindings/js/native/index.js"); -const loaderVersions = new Set( - [...nativeLoader.matchAll(/bindingPackageVersion !== '([^']+)'/g)].map( - ([, version]) => version, - ), -); -if (loaderVersions.size !== 1 || !loaderVersions.has(expected)) { - throw new Error( - `bindings/js/native/index.js has package versions ${[...loaderVersions].join(", ") || "none"}; expected ${expected}`, - ); -} - -console.log( - `Verified ${expected} across release metadata; Rust packages inherit the workspace version.`, -); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aadd4cf..9c1c232 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,12 +33,25 @@ jobs: rustup toolchain install stable --profile default rustup default stable - - uses: actions/setup-node@v4 - with: - node-version: "24" + - run: sudo apt-get update && sudo apt-get install -y ripgrep - name: Verify release versions - run: node .github/scripts/release/verify-versions.mjs + shell: bash + run: | + version="${RELEASE_TAG#v}" + version_files=( + Cargo.toml + bindings/js/package.json + bindings/js/package-lock.json + bindings/js/src/version.ts + bindings/python/pyproject.toml + bindings/python/src/shell_use/_config.py + bindings/js/native/index.js + ) + diff -u \ + <(printf '%s\n' "${version_files[@]}" | sort) \ + <(rg -l --path-separator / --fixed-strings "$version" \ + "${version_files[@]}" | sort) - name: Verify shell-use crate package run: cargo publish --locked --package shell-use --dry-run @@ -221,6 +234,28 @@ jobs: - name: Pack Node packages run: node .github/scripts/release/package-node.mjs + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Test packed Node package with Node 20 + shell: bash + run: | + node="$(command -v node)" + version="${RELEASE_TAG#v}" + mkdir package-test + cd package-test + npm init -y >/dev/null + npm install --ignore-scripts \ + "$GITHUB_WORKSPACE/package-artifacts/npm/microsoft-shell-use-linux-x64-gnu-${version}.tgz" + npm install --ignore-scripts --omit=optional \ + "$GITHUB_WORKSPACE/package-artifacts/npm/microsoft-shell-use-${version}.tgz" + cd "$GITHUB_WORKSPACE" + PATH="/usr/bin:/bin" \ + SHELL_USE_BIN="$GITHUB_WORKSPACE/package-test/missing-shell-use" \ + SHELL_USE_TEST_PACKAGE_ROOT="$GITHUB_WORKSPACE/package-test" \ + "$node" --test bindings/js/test/package.integration.test.mjs + - uses: actions/upload-artifact@v4 with: name: npm-packages @@ -277,31 +312,45 @@ jobs: manylinux: ${{ matrix.compatibility || 'auto' }} args: --release --locked --out dist --compatibility pypi - - uses: actions/upload-artifact@v4 - with: - name: python-wheel-${{ matrix.target }} - path: bindings/python/dist/*.whl - if-no-files-found: error - - build-python-sdist: - needs: verify - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ env.RELEASE_TAG }} + - name: Build Python sdist + if: matrix.target == 'x86_64-unknown-linux-gnu' + working-directory: bindings/python + run: | + python -m pip install --disable-pip-version-check "maturin>=1.5,<2" + maturin sdist --manifest-path native/Cargo.toml --out dist - uses: actions/setup-python@v5 + if: matrix.target == 'x86_64-unknown-linux-gnu' with: - python-version: "3.11" + python-version: "3.8" - - run: python -m pip install --disable-pip-version-check "maturin>=1.5,<2" + - name: Test Python wheel with Python 3.8 + if: matrix.target == 'x86_64-unknown-linux-gnu' + shell: bash + run: | + shopt -s nullglob + wheels=(bindings/python/dist/*.whl) + test "${#wheels[@]}" -eq 1 + [[ "${wheels[0]}" == *abi3* ]] + python -m venv package-test-python-38 + package-test-python-38/bin/python -m pip install \ + --disable-pip-version-check bindings/python/dist/*.whl + ! PATH="$GITHUB_WORKSPACE/package-test-python-38/bin:/usr/bin:/bin" \ + command -v shell-use + PATH="$GITHUB_WORKSPACE/package-test-python-38/bin:/usr/bin:/bin" \ + SHELL_USE_BIN="$GITHUB_WORKSPACE/package-test-python-38/missing-shell-use" \ + "$GITHUB_WORKSPACE/package-test-python-38/bin/python" \ + bindings/python/tests/test_integration.py \ + IntegrationTests.test_echo_roundtrip - - name: Build Python sdist - working-directory: bindings/python - run: maturin sdist --manifest-path native/Cargo.toml --out dist + - uses: actions/upload-artifact@v4 + with: + name: python-wheel-${{ matrix.target }} + path: bindings/python/dist/*.whl + if-no-files-found: error - uses: actions/upload-artifact@v4 + if: matrix.target == 'x86_64-unknown-linux-gnu' with: name: python-sdist path: bindings/python/dist/*.tar.gz @@ -309,9 +358,9 @@ jobs: publish-npm: needs: - - smoke-node + - package-node - release - if: always() && needs.smoke-node.result == 'success' && (needs.release.result == 'success' || github.event_name == 'workflow_dispatch') + if: always() && needs.package-node.result == 'success' && (needs.release.result == 'success' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest environment: npm permissions: @@ -333,14 +382,36 @@ jobs: registry-url: https://registry.npmjs.org - name: Publish native and root packages - run: node .github/scripts/release/publish-npm.mjs + shell: bash + run: | + version="${RELEASE_TAG#v}" + packages=( + shell-use-darwin-arm64 + shell-use-darwin-x64 + shell-use-linux-arm64-gnu + shell-use-linux-arm64-musl + shell-use-linux-x64-gnu + shell-use-linux-x64-musl + shell-use-win32-arm64-msvc + shell-use-win32-x64-msvc + shell-use + ) + + for package in "${packages[@]}"; do + name="@microsoft/$package" + tarball="npm-packages/microsoft-${package}-${version}.tgz" + if npm view "$name@$version" version >/dev/null 2>&1; then + echo "$name@$version is already published" + else + npm publish "$tarball" --access public --provenance --tag latest + fi + done publish-pypi: needs: - - smoke-python - - build-python-sdist + - build-python - release - if: always() && needs.smoke-python.result == 'success' && needs.build-python-sdist.result == 'success' && (needs.release.result == 'success' || github.event_name == 'workflow_dispatch') + if: always() && needs.build-python.result == 'success' && (needs.release.result == 'success' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest environment: name: pypi @@ -399,18 +470,13 @@ jobs: release: needs: - build-cli - - smoke-node - - smoke-python - - build-python-sdist + - package-node + - build-python if: github.event_name == 'push' runs-on: ubuntu-latest permissions: contents: write steps: - - uses: actions/checkout@v4 - with: - ref: ${{ env.RELEASE_TAG }} - - uses: actions/download-artifact@v4 with: path: artifacts @@ -419,65 +485,8 @@ jobs: - env: GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} run: | gh release create "$RELEASE_TAG" \ --generate-notes \ artifacts/* - - smoke-node: - name: Smoke Node (${{ matrix.name }}) - needs: package-node - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - name: Node 20 - version: "20" - - name: Node current - version: "node" - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ env.RELEASE_TAG }} - - - uses: actions/download-artifact@v4 - with: - name: npm-packages - path: npm-packages - - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.version }} - - - name: Smoke packed root and platform packages without the CLI - run: node .github/scripts/release/smoke-node.mjs - - smoke-python: - name: Smoke Python (${{ matrix.name }}) - needs: build-python - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - name: Python 3.8 - version: "3.8" - - name: Python current - version: "3.14" - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ env.RELEASE_TAG }} - - - uses: actions/download-artifact@v4 - with: - name: python-wheel-x86_64-unknown-linux-gnu - path: dist - - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.version }} - - - name: Smoke the built abi3 wheel without the CLI - run: python .github/scripts/release/smoke-python.py diff --git a/bindings/js/test/package.integration.test.mjs b/bindings/js/test/package.integration.test.mjs new file mode 100644 index 0000000..ae7f5a0 --- /dev/null +++ b/bindings/js/test/package.integration.test.mjs @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createRequire } from "node:module"; +import { join } from "node:path"; +import { test } from "node:test"; +import { pathToFileURL } from "node:url"; + +const packageRoot = process.env.SHELL_USE_TEST_PACKAGE_ROOT; + +if (packageRoot) { + test("packed binding works without the CLI", async () => { + const requireFromPackage = createRequire(join(packageRoot, "package.json")); + const packageEntry = requireFromPackage.resolve("@microsoft/shell-use"); + assert.equal( + spawnSync("shell-use", ["--version"], { stdio: "ignore" }).error?.code, + "ENOENT", + ); + + const { ShellUse } = await import(pathToFileURL(packageEntry).href); + const session = ShellUse.ephemeral("release-integration"); + try { + await session.open(); + await session.submit("echo release-integration"); + await session.waitCommand(); + await session.expectText("release-integration", { strict: false }); + } finally { + await session.closeQuiet(); + } + }); +}