diff --git a/.github/check-clean-tree.sh b/.github/check-clean-tree.sh new file mode 100755 index 00000000..1410fcdc --- /dev/null +++ b/.github/check-clean-tree.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# Fails when the test run modified a file under version control. +# +# The tests run Bundler against fixture Gemfiles. A Bundler that has been +# pointed at the wrong lockfile - `bundle exec` exports BUNDLE_LOCKFILE, and +# children inherit it - rewrites OCRAN's own Gemfile.lock with a fixture's +# dependencies. The test suite stays green; what dies is the next step that +# runs `bundle exec`, with a frozen-mode error that names neither the test +# that did it nor the fact that a test did it at all. +# +# Untracked files are ignored on purpose: the builds leave stubs and packed +# executables behind. So are vendor/ and .bundle/, which are checked in but +# describe the machine rather than the project: `bundle install` rewrites the +# vendored bundle (shebangs and line endings differ per platform) and +# ruby/setup-ruby writes `deployment: true` into the config, both before +# anything of ours has run. +set -eu + +dirty=$(git status --porcelain --untracked-files=no \ + -- . ':(exclude)vendor' ':(exclude).bundle') +if [ -n "$dirty" ]; then + echo "::error::the test run modified files under version control" + echo "$dirty" + git diff --stat -- . ':(exclude)vendor' ':(exclude).bundle' + exit 1 +fi + +echo "working tree clean" diff --git a/.github/workflows/test-on-push.yml b/.github/workflows/test-on-push.yml index 4b9afe20..74aa7d0a 100644 --- a/.github/workflows/test-on-push.yml +++ b/.github/workflows/test-on-push.yml @@ -2,7 +2,12 @@ name: Run Tests on: push: branches: - - '*' + # '**' and not '*': a single star stops at a slash, so every branch + # named the way this project names them - fix/..., feat/... - was + # never tested until it landed on master, which is where the last two + # merges turned red. + - '**' + workflow_dispatch: jobs: run-tests-linux: strategy: @@ -32,6 +37,10 @@ jobs: bundle exec rake build bundle exec rake test + - name: Fail if the test run left the working tree dirty + shell: bash + run: bash .github/check-clean-tree.sh + - name: Build zlib fixture for cross-distro test env: OCRAN_DEBUG: "0" @@ -66,6 +75,10 @@ jobs: bundle exec rake build bundle exec rake test + - name: Fail if the test run left the working tree dirty + shell: bash + run: bash .github/check-clean-tree.sh + - name: Build zlib fixture for cross-system test run: ruby -Ilib exe/ocran test/fixtures/zlib/zlib.rb --output zlib_packed @@ -195,6 +208,10 @@ jobs: bundle exec rake build bundle exec rake test + - name: Fail if the test run left the working tree dirty + shell: bash + run: bash .github/check-clean-tree.sh + - name: Build zlib fixture for portability test run: bundle exec ruby -Ilib exe/ocran test/fixtures/zlib/zlib.rb --output zlib_packed.exe diff --git a/test/test_ocra.rb b/test/test_ocra.rb index 6d54a994..16e983e9 100644 --- a/test/test_ocra.rb +++ b/test/test_ocra.rb @@ -41,6 +41,24 @@ def exe_name(base) # Path to test fixtures. FixturePath = File.expand_path(File.join(File.dirname(__FILE__), 'fixtures')) + # OCRAN's own lockfile and its content before any test ran. The tests run + # Bundler against fixture Gemfiles, and a Bundler that has been pointed at + # the wrong lockfile rewrites this one instead of the fixture's. Nothing in + # the test run notices - the damage surfaces in whatever runs `bundle exec` + # afterwards, which in CI is a build step several minutes later. + OwnLockfile = File.join(OcranRoot, "Gemfile.lock") + OwnLockfileContent = File.exist?(OwnLockfile) ? File.binread(OwnLockfile) : nil + + # Asserts that running Bundler has not damaged OCRAN's own bundle. + def assert_own_lockfile_intact(what) + return if OwnLockfileContent.nil? + + assert OwnLockfileContent == File.binread(OwnLockfile), + "#{what} rewrote OCRAN's own lockfile (#{OwnLockfile}). " \ + "Bundler was run with a foreign Gemfile but the lockfile of " \ + "this bundle; restore it with `git checkout Gemfile.lock`." + end + # Create a pristine environment to test built executables. Files are # copied and the PATH environment is set to the minimal. Yields to # the block, then cleans up. @@ -134,10 +152,23 @@ def initialize(*args) # The environment `bundle exec` sets for the given Gemfile, or nil when # that bundle cannot be used here (Bundler missing, gems not installed), # so callers can skip rather than fail. + # + # BUNDLE_LOCKFILE is cleared rather than left alone. `bundle exec` exports + # the absolute path of the lockfile it used, and this suite itself runs + # under `bundle exec rake test`, so every child inherits OCRAN's own + # Gemfile.lock by name. Bundler believes that variable over the Gemfile it + # is handed, so resolving a fixture Gemfile here would write the fixture's + # dependencies into OCRAN's lockfile - leaving the tests green and every + # later `bundle exec` in the same job dead in frozen mode. Unset, Bundler + # puts the lockfile next to the Gemfile, which for a fixture means inside + # the temporary directory it was copied to. def bundle_exec_env(gemfile) dump = 'ENV.each { |name, value| puts "#{name}=#{value}" }' - output, status = capture_system({ "BUNDLE_GEMFILE" => gemfile.to_s }, + output, status = capture_system({ "BUNDLE_GEMFILE" => gemfile.to_s, + "BUNDLE_LOCKFILE" => nil }, "bundle", "exec", "ruby", "-e", dump) + assert_own_lockfile_intact("bundle exec against #{gemfile}") + return nil unless status&.success? env = output.lines.filter_map { |line|