[CMS-420] Fix, clean up and expand bcms test suite

.github/workflows/ci.yml

@@ -0,0 +1,208 @@
+name: CI
+
+on:
+  push:
+    branches: [master, develop]
+  pull_request:
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Suite (Ruby 2.7.8, Postgres 15)
+    # Pinned, not ubuntu-latest: ruby/setup-ruby has no prebuilt 2.7.8 for 24.04.
+    # If 22.04 is retired, move to `container: ruby:2.7.8-bullseye` with
+    # PGHOST: postgres rather than chasing runner images.
+    runs-on: ubuntu-22.04
+    timeout-minutes: 45
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports: ['5432:5432']
+        options: >-
+          --health-cmd pg_isready --health-interval 10s
+          --health-timeout 5s --health-retries 5
+
+    env:
+      # test/dummy/config/database.yml defines only `test`. Anything that boots
+      # the dummy app in `development` -- db:migrate and db:seed both do, via
+      # db:install -- aborts with ActiveRecord::AdapterNotSpecified.
+      RAILS_ENV: test
+      # database.yml specifies no host, so libpq falls back to PGHOST. That
+      # points it at the service container instead of a unix socket that does
+      # not exist on the runner, without editing the tracked config.
+      PGHOST: localhost
+      PGPORT: '5432'
+      PGUSER: postgres
+      # Rails deprecations print regardless; this adds the Ruby-level ones.
+      RUBYOPT: '-W:deprecated'
+      # features/support/open_on_first_failure.rb pops the first failed page
+      # open via Capybara's save_and_open_page -> Launchy -> xdg-open. Useful at
+      # a desk, useless on a headless runner, where it only prints "Couldn't
+      # find a suitable web browser!". Nothing here drives a real browser: there
+      # are no @javascript scenarios and no Capybara driver is ever selected.
+      launch_on_failure: 'false'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '2.7.8'
+          bundler: '1.17.3'
+          bundler-cache: true
+
+      # Travis passed --without development. Not carried over: `rake` is
+      # declared in the :development group and the whole chain is rake-driven.
+      #
+      # `rake` alone, deliberately -- not features:all. The default cucumber
+      # profile excludes @cli, @known-bug and @missing-feature, and those
+      # exclusions are the point of the tags. The excluded scenarios are
+      # measured by the non-blocking job below so the number stays visible.
+      # `ci:test` runs coverage:check after the suite.
+      - name: Full suite (units, spec, functionals, orphans, features)
+        run: bundle exec rake
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: coverage
+          path: coverage/
+          retention-days: 30
+
+  excluded-features:
+    name: Cucumber @cli and @known-bug (reporting only)
+    runs-on: ubuntu-22.04
+    timeout-minutes: 45
+    # Everything the default profile filters out, measured but not gating.
+    # Phase 0 baseline: @cli is 7/34 -- `rails new` fails inside aruba and 15 of
+    # the 27 failures are downstream of that one command. @known-bug is 2
+    # failing scenarios that are tagged precisely because they are known to
+    # fail. This job exists so both numbers stay visible per-run instead of
+    # living only in docs/rails-upgrade/phase-0-baseline.md. Make it gating
+    # once the numbers reach zero.
+    continue-on-error: true
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports: ['5432:5432']
+        options: >-
+          --health-cmd pg_isready --health-interval 10s
+          --health-timeout 5s --health-retries 5
+
+    env:
+      RAILS_ENV: test
+      PGHOST: localhost
+      PGPORT: '5432'
+      PGUSER: postgres
+      # This job is the one that actually trips it: the After hook is
+      # After('~@cli'), so it fires on the @known-bug failures, not the @cli
+      # ones. See the note in the test job.
+      launch_on_failure: 'false'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '2.7.8'
+          bundler: '1.17.3'
+          bundler-cache: true
+
+      # This job gets its own empty Postgres container. The `test` job only has
+      # a database because ci:test starts with db:drop, db:create:all,
+      # db:install -- the cucumber tasks assume one already exists, and
+      # project:ensure_db_exists only checks that database.yml is present, not
+      # that the database is. The same prefix ci:test uses -- including the
+      # drop, which has nothing to drop on a fresh container but keeps the step
+      # rerunnable: db:seed is not idempotent and fails on an already-seeded
+      # database with "Name has already been taken".
+      - name: Create and seed the test database
+        run: bundle exec rake db:drop db:create:all db:install
+
+      - name: Cucumber - @cli features
+        run: bundle exec rake features:cli
+
+      - name: Cucumber - @known-bug scenarios
+        if: always()
+        run: bundle exec rake features:known-bugs
+
+  next-rails:
+    name: Rails 5.0 (Gemfile.next)
+    runs-on: ubuntu-22.04
+    timeout-minutes: 45
+    # GATING as of Phase 2, and currently RED on purpose.
+    #
+    # Phase 2 took this job from "754 unit tests / 2 failures / 323 errors" down
+    # to five named application defects by borrowing two Phase 3 fixes; the
+    # harness itself is done. What is left is not harness work and no amount of
+    # it will help:
+    #   - 2x ActiveRecord::StaleObjectError on Cms::Page
+    #   - skip_callback :redirect_to_cms_site (content_controller.rb:11) -- 5.0
+    #     raises when the callback is not defined, and this one is a *load*
+    #     error, so it takes the whole functional suite down with it
+    #   - PublishableTestCase#test_publish_on_save
+    #   - PortletTest#test_.blacklist
+    #
+    # Those belong to Phase 3. This job was made gating anyway, deliberately, so
+    # the red is visible in the merge path rather than filed away in a report.
+    # It goes green when Phase 3 lands, not before.
+    # See docs/rails-upgrade/phase-2-harness-report.md.
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports: ['5432:5432']
+        options: >-
+          --health-cmd pg_isready --health-interval 10s
+          --health-timeout 5s --health-retries 5
+
+    env:
+      BUNDLE_GEMFILE: Gemfile.next
+      RAILS_ENV: test
+      PGHOST: localhost
+      PGPORT: '5432'
+      PGUSER: postgres
+      launch_on_failure: 'false'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '2.7.8'
+          bundler: '1.17.3'
+          bundler-cache: true
+
+      # Assert the version rather than trusting BUNDLE_GEMFILE. test/dummy's
+      # boot.rb used to reassign BUNDLE_GEMFILE unconditionally, which made every
+      # spawned test process silently fall back to 4.2 -- a job like this one
+      # would have reported green while testing the wrong Rails for four phases.
+      # If this step ever fails, dual-boot is broken, not Rails 5.
+      - name: Assert the bundle really is Rails 5.0
+        run: |
+          bundle exec ruby -e '
+            require "./test/dummy/config/environment"
+            v = Rails.version
+            abort "Expected Rails 5.0.x, booted #{v} -- dual-boot is broken" unless v.start_with?("5.0.")
+            puts "Booted Rails #{v}"
+          '
+
+      - name: Create and seed the test database
+        run: bundle exec rake db:drop db:create:all db:install
+
+      # `if: always()` went with continue-on-error: with the job gating, a step
+      # that runs regardless of whether the database step succeeded only turns
+      # one clear failure into two confusing ones.
+      - name: Suite
+        run: bundle exec rake

.simplecov

@@ -1,3 +1,38 @@
+# The suite runs as five separate processes (units, spec, functionals, orphans,
+# features) that merge through coverage/.resultset.json. SimpleCov discards any stored
+# result older than merge_timeout, which defaults to 600s -- so on a full run,
+# which takes longer than that, the earliest suites silently drop out and the
+# reported percentage covers only whatever finished inside the window.
 SimpleCov.start 'rails' do
-  # Config goes here
+  merge_timeout 3600
+
+  # Gated as of Phase 3: coverage:check compares this against COVERAGE_MINIMUM_BRANCH,
+  # which defaults to the 70.83% measured on a cleared resultset once the Phase 3 diff
+  # had landed. See lib/tasks/core_tasks.rake.
+  enable_coverage :branch
+
+  # Each suite must name itself. Left to CommandGuesser, two suites can guess the
+  # same name and overwrite each other's entry in the resultset. Set by the
+  # Rakefile, one prerequisite task per suite.
+  command_name ENV['COVERAGE_SUITE'] if ENV['COVERAGE_SUITE']
+
+  # Generator *templates* are copied into a user's application, not executed
+  # here. demo.seeds.rb alone is 249 counted lines -- 13.5% of every missed line
+  # in the report -- and it is a seed script: loading it would run it.
+  #
+  # Block filters, not regexes. This was once a hard constraint: SimpleCov 0.12's
+  # parse_filter accepted only a String, an Array, a Filter or a block and raised
+  # ArgumentError on a Regexp, which defaults.rb rescued around `load .simplecov`
+  # -- so a regex filter did not fail loudly, it abandoned the rest of this file
+  # with one line on stderr. 0.22 accepts regexes and the constraint is gone; the
+  # block form is kept because it works and rewriting it would buy nothing.
+  # Match on the absolute path; that is what #filename returns.
+  add_filter { |src| src.filename.include?("/lib/generators/") && src.filename.include?("/templates/") }
+  add_filter { |src| src.filename.include?("/lib/templates/") }
+
+  # The generator *classes* stay in the denominator. Excluding them was the
+  # standing recommendation, on the grounds that the @cli cucumber features
+  # cover them out of process where SimpleCov cannot see it. Phase 0 measured
+  # those features: 7 of 34 scenarios pass. The 0% is a measurement gap sitting
+  # on top of a real testing gap, and hiding it would misreport the second one.
 end

Gemfile

@@ -1,3 +1,11 @@
+# Dual-boot (next_rails). Gemfile.next is a *symlink* to this file, so both
+# bundles evaluate the same Gemfile and this is the only thing that tells them
+# apart. Note that browsercms.gemspec cannot call this -- Bundler evaluates the
+# gemspec in a Gem::Specification context, so it keys off BUNDLE_GEMFILE instead.
+def next?
+  File.basename(__FILE__) == "Gemfile.next"
+end
+
 source 'http://rubygems.org'
 
 ruby '2.7.8'
@@ -7,7 +15,7 @@ gemspec
 # gem 'query_reviewer' # Enable for performance tuning
 
 gem 'puma', '~> 4'
-gem 'railties', '~> 4.2'
+gem 'railties', next? ? '~> 5.0.0' : '~> 4.2'
 # Uncomment to confirm that older versions work (for compaitiblity with Spree 2.2.4/bcms_spree)
 # gem 'paperclip', '~> 3.4.1'
 # For testing behavior in production
@@ -17,36 +25,68 @@ end
 
 group :development do
   gem 'rake'
+  # Dual-boot tooling for the Rails upgrade: supplies `next_rails --init`
+  # (Gemfile.next) and `bundle_report compatibility`. A maintainer tool, so it
+  # belongs here and not in browsercms.gemspec.
+  gem 'next_rails'
   # gem 'debugger'
   # gem 'quiet_assets'
 end
 group :test, :development do
-  gem 'minitest'
-  gem 'minitest-rails'
+  # Rails 5.0's Rails::TestUnitReporter (railties/lib/rails/test_unit/reporter.rb)
+  # calls result.method, which predates Minitest::Result -- introduced in minitest
+  # 5.11. On a newer minitest the reporter raises NameError while formatting the
+  # *first* failure and takes the whole run down, so nothing is reportable.
+  # Pinned for the next bundle only; revisit when the hop passes 5.1.
+  if next?
+    gem 'minitest', '~> 5.10.3'
+  else
+    gem 'minitest'
+  end
+  # minitest-rails removed: it capped railties ~> 4.1 and nothing used it --
+  # every reference in test/minitest_helper.rb was already commented out. The
+  # cheapest of the six Rails 5 blockers. (minitest_helper.rb itself stays; 19
+  # test files require it.)
   gem 'minitest-reporters'
   gem 'yard'
   gem 'bluecloth'
   gem 'pry'
-  gem 'awesome_print'
+  # Not auto-required: requiring it monkeypatches Array#grep with an
+  # implementation that does `"str" =~ SomeClass`, which floods Ruby 2.7 with
+  # "deprecated Object#=~ is called on Class" warnings from every
+  # ActiveRecord where(hash) call. Run `require 'awesome_print'` in a console
+  # when you actually want `ap`.
+  gem 'awesome_print', require: false
 end
 
 group :test do
   gem 'pg'
   gem 'sass-rails'
-  gem 'simplecov', require: false
+  gem 'simplecov', '~> 0.22.0', require: false
 
-  gem 'poltergeist'
   gem 'm'
 
   gem 'single_test'
-  gem 'factory_girl_rails'
-  gem 'mocha', require: false
+  # 4.2 has assert_template and assigns built in; 5.0 extracted them. The gem
+  # cannot resolve on 4.2 (it needs actionpack >= 5.0.1.rc1), so this is one of
+  # the few places a next? branch is not a smell -- it is the only expressible
+  # form. Criterion 12 is about test *code*, not the Gemfile.
+  gem 'rails-controller-testing' if next?
+  gem 'factory_bot_rails', '~> 5.2'
+  # 1.x, not 2.x: `mocha/minitest` exists from 1.5.0, so 1.x satisfies the Rails 5
+  # requirement in full. Mocha 2.0 removes the legacy entry points *and* changes
+  # any_instance and configuration behaviour across 109 call sites -- a
+  # modernisation with no Rails deadline. See phase-2-implementation-plan.md D4.
+  gem 'mocha', '~> 1.16', require: false
 
   # Cucumber and dependencies
   gem 'capybara'
   gem 'cucumber-rails', require: false
   gem 'database_cleaner'
   gem 'launchy'
+
+  # ruby-prof needs this config for installation on modern macos
+  # bundle config --global build.ruby-prof --with-cflags="-Wno-incompatible-pointer-types"
   gem 'ruby-prof'
   gem 'aruba', '= 0.14.14'
   gem 'loofah', '= 2.19.1'