Check ruby test coverage before merging PRs

.github/workflows/ci.yml

@@ -87,4 +87,20 @@ jobs:
       - name: Run cucumber
         env:
           RAILS_ENV: test
+          TEST_COVERAGE_EXCLUDED_PATHS: **/*
         run: bundle exec rails cucumber
+
+      - name: Store test coverage statistics
+        uses: actions/upload-artifact@v4
+        with:
+          name: cucumber-test-coverage
+          path: coverage/statistics.txt
+          retention-days: 1
+
+  test-coverage:
+    name: Check test coverage
+    needs:
+      - test-ruby
+      - pact-tests
+      - integration-tests
+    uses: ./.github/workflows/test-coverage.yml

.github/workflows/pact-verify.yml

@@ -39,6 +39,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       RAILS_ENV: test
+      TEST_COVERAGE_EXCLUDED_PATHS: **/*
     steps:
       - uses: actions/checkout@v4
         with:
@@ -58,3 +59,10 @@ jobs:
           path: tmp/pacts
       - if: inputs.pact_artifact != ''
         run: bundle exec rake pact:verify:at[tmp/pacts/${{ inputs.pact_artifact_file_to_verify }}]
+
+      - name: Store test coverage statistics
+        uses: actions/upload-artifact@v4
+        with:
+          name: pact-test-coverage
+          path: coverage/statistics.txt
+          retention-days: 1

.github/workflows/rspec.yml

@@ -47,3 +47,10 @@ jobs:
           RAILS_ENV: test
           GOVUK_CONTENT_SCHEMAS_PATH: vendor/publishing-api/content_schemas
         run: bundle exec rake spec
+
+      - name: Store test coverage statistics
+        uses: actions/upload-artifact@v4
+        with:
+          name: rspec-test-coverage
+          path: coverage/statistics.txt
+          retention-days: 1

.github/workflows/test-coverage.yml

@@ -0,0 +1,50 @@
+name: Check test coverage
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: 'The branch, tag or SHA to checkout'
+        required: false
+        type: string
+
+jobs:
+  check_ruby_test_coverage:
+    name: Check Ruby test coverage
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          repository: alphagov/collections
+          ref: ${{ inputs.ref || github.ref }}
+
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+
+      - name: Retrieve rspec test coverage
+        uses: actions/download-artifact@v4
+        with:
+          name: rspec-test-coverage
+          path: rspec-test-coverage
+
+      - name: Retrieve pact test coverage
+        uses: actions/download-artifact@v4
+        with:
+          name: pact-test-coverage
+          path: pact-test-coverage
+
+      - name: Retrieve cucumber test coverage
+        uses: actions/download-artifact@v4
+        with:
+          name: cucumber-test-coverage
+          path: cucumber-test-coverage
+
+      - name: Check Ruby test coverage
+        run: >-
+          bundle exec ruby -e 'require_relative "test/test_coverage.rb"; TestCoverage.check_test_coverage'
+          rspec-test-coverage/statistics.txt
+          pact-test-coverage/statistics.txt
+          cucumber-test-coverage/statistics.txt

features/support/env.rb

@@ -1,9 +1,5 @@
-if ENV["USE_SIMPLECOV"]
-  require "simplecov"
-  require "simplecov-rcov"
-  SimpleCov.formatter = SimpleCov::Formatter::RcovFormatter
-  SimpleCov.start "rails"
-end
+require_relative "../../spec/test_coverage"
+TestCoverage.start
 
 # Duplicated in test_helper.rb
 ENV["GOVUK_WEBSITE_ROOT"] = "http://www.test.gov.uk"

spec/service_consumers/pact_helper.rb

@@ -1,5 +1,7 @@
 ENV["PACT_DO_NOT_TRACK"] = "true"
 
+require_relative "../test_coverage"
+TestCoverage.start
 require "pact/provider/rspec"
 require "webmock/rspec"
 require "gds_api"

spec/spec_helper.rb

@@ -1,9 +1,5 @@
-if ENV["USE_SIMPLECOV"]
-  require "simplecov"
-  require "simplecov-rcov"
-  SimpleCov.formatter = SimpleCov::Formatter::RcovFormatter
-  SimpleCov.start
-end
+require_relative "test_coverage"
+TestCoverage.start
 
 if ENV["USE_I18N_COVERAGE"]
   require "i18n/coverage"

spec/test_coverage.rb

@@ -0,0 +1,94 @@
+require "simplecov"
+require "simplecov-rcov"
+
+class TestCoverage
+  class << self
+    def start
+      return if @started
+
+      @started = true
+      SimpleCov.formatter = SimpleCov::Formatter::RcovFormatter
+      SimpleCov.at_exit do
+        SimpleCov.result.format!
+        store_coverage_percentage
+      end
+      SimpleCov.start "rails" do
+        add_filter do |source_file|
+          filter(source_file)
+        end
+      end
+    end
+
+    def check_test_coverage
+      covered, missed = ARGF
+        .readlines
+        .collect { |line| line.split(" ").collect(&:to_f) }
+        .reduce { |sums, stats| [sums[0] + stats[0], sums[1] + stats[1]] }
+      percentage = (covered / (covered + missed)) * 100
+      puts "Total test coverage percentage is #{percentage}."
+      quit(percentage >= 95)
+    end
+
+  private
+
+    def store_coverage_percentage
+      statistics = SimpleCov.result.coverage_statistics[:line]
+      File.write(
+        Rails.root.join("coverage/statistics.txt"),
+        "#{statistics.covered} #{statistics.missed}",
+      )
+    end
+
+    def file_pattern_to_pathnames(pattern)
+      pattern = pattern.to_s
+      if pattern.present?
+        Rails.root.glob(pattern)
+      else
+        []
+      end
+    end
+
+    def generate_included_pathnames
+      included_paths = ENV["TEST_COVERAGE_INCLUDED_PATHS"]
+      pathnames = file_pattern_to_pathnames(included_paths)
+      if pathnames.present?
+        puts "The following pattern has been included in test coverage: #{included_paths}"
+      end
+      pathnames
+    end
+
+    def generate_excluded_pathnames
+      excluded_paths = ENV["TEST_COVERAGE_EXCLUDED_PATHS"]
+      pathnames = file_pattern_to_pathnames(excluded_paths)
+      if pathnames.present?
+        puts "The following pattern has been excluded from test coverage: #{excluded_paths}"
+      end
+      pathnames
+    end
+
+    def included_pathnames
+      @included_pathnames ||= generate_included_pathnames
+    end
+
+    def excluded_pathnames
+      @excluded_pathnames ||= generate_excluded_pathnames
+    end
+
+    def filter(source_file)
+      pathname = Pathname.new(File.absolute_path(source_file.filename))
+      !(
+        (
+          included_pathnames.empty? ||
+          included_pathnames.include?(pathname)
+        ) && (
+          excluded_pathnames.empty? ||
+          !excluded_pathnames.include?(pathname)
+        )
+      )
+    end
+
+    def quit(is_success)
+      exit(is_success)
+    end
+  end
+end

spec/test_coverage_test.rb

@@ -0,0 +1,143 @@
+require "test_helper"
+
+class TestCoverageTest < ActiveSupport::TestCase
+  context ".generate_included_pathnames" do
+    should "only return matching pathnames" do
+      ClimateControl.modify TEST_COVERAGE_INCLUDED_PATHS: "{Gemfile,test/{test_cover*,test_helper.??}}" do
+        pathnames = nil
+        assert_output "The following pattern has been included in test coverage: {Gemfile,test/{test_cover*,test_helper.??}}\n" do
+          pathnames = TestCoverage.send(:generate_included_pathnames)
+        end
+        assert_includes pathnames, Rails.root.join("Gemfile")
+        assert_includes pathnames, Rails.root.join("test/test_coverage.rb")
+        assert_includes pathnames, Rails.root.join("test/test_coverage_test.rb")
+        assert_includes pathnames, Rails.root.join("test/test_helper.rb")
+        assert_not_includes pathnames, Rails.root.join("Gemfile.lock")
+      end
+    end
+  end
+
+  context ".generate_excluded_pathnames" do
+    should "only return matching pathnames" do
+      ClimateControl.modify TEST_COVERAGE_EXCLUDED_PATHS: "{Gemfile,test/{test_cover*,test_helper.??}}" do
+        pathnames = nil
+        assert_output "The following pattern has been excluded from test coverage: {Gemfile,test/{test_cover*,test_helper.??}}\n" do
+          pathnames = TestCoverage.send(:generate_excluded_pathnames)
+        end
+        assert_includes pathnames, Rails.root.join("Gemfile")
+        assert_includes pathnames, Rails.root.join("test/test_coverage.rb")
+        assert_includes pathnames, Rails.root.join("test/test_coverage_test.rb")
+        assert_includes pathnames, Rails.root.join("test/test_helper.rb")
+        assert_not_includes pathnames, Rails.root.join("Gemfile.lock")
+      end
+    end
+  end
+
+  context ".filter" do
+    setup do
+      @first_pathname = Rails.root.join("tmp/first_file.rb")
+      @first_source_file = mock
+      @first_source_file.stubs(:filename).returns(@first_pathname.to_s)
+      @second_pathname = Rails.root.join("tmp/second_file.rb")
+      @second_source_file = mock
+      @second_source_file.stubs(:filename).returns(@second_pathname.to_s)
+      @third_pathname = Rails.root.join("/tmp/third_file.rb")
+      @third_source_file = mock
+      @third_source_file.stubs(:filename).returns(@third_pathname.to_s)
+    end
+
+    context "when no included and no excluded paths specified" do
+      setup do
+        TestCoverage.stubs(:included_pathnames).returns([])
+        TestCoverage.stubs(:excluded_pathnames).returns([])
+      end
+
+      should "not filter anything" do
+        assert_not TestCoverage.send(:filter, @first_source_file)
+      end
+    end
+
+    context "when included path specified" do
+      setup do
+        TestCoverage.stubs(:included_pathnames).returns([@first_pathname])
+        TestCoverage.stubs(:excluded_pathnames).returns([])
+      end
+
+      should "filter not included files" do
+        assert_not TestCoverage.send(:filter, @first_source_file)
+        assert TestCoverage.send(:filter, @second_source_file)
+      end
+    end
+
+    context "when excluded path specified" do
+      setup do
+        TestCoverage.stubs(:included_pathnames).returns([])
+        TestCoverage.stubs(:excluded_pathnames).returns([@first_pathname])
+      end
+
+      should "filter excluded files" do
+        assert TestCoverage.send(:filter, @first_source_file)
+        assert_not TestCoverage.send(:filter, @second_source_file)
+      end
+    end
+
+    context "when different included and excluded paths specified" do
+      setup do
+        TestCoverage.stubs(:included_pathnames).returns([@first_pathname])
+        TestCoverage.stubs(:excluded_pathnames).returns([@second_pathname])
+      end
+
+      should "filter non-included files" do
+        assert_not TestCoverage.send(:filter, @first_source_file)
+        assert TestCoverage.send(:filter, @second_source_file)
+        assert TestCoverage.send(:filter, @third_source_file)
+      end
+    end
+
+    context "when the same included and excluded paths" do
+      setup do
+        TestCoverage.stubs(:included_pathnames).returns([@first_pathname])
+        TestCoverage.stubs(:excluded_pathnames).returns([@first_pathname])
+      end
+
+      should "filter all files" do
+        assert TestCoverage.send(:filter, @first_source_file)
+        assert TestCoverage.send(:filter, @second_source_file)
+      end
+    end
+  end
+
+  context ".check_test_coverage" do
+    context "when code coverage percentage below threshold" do
+      setup do
+        ARGF.expects(:readlines).once.returns([
+          "85 5",
+          "5 5",
+        ])
+      end
+
+      should "exit unsuccessfully" do
+        assert_output "Total test coverage percentage is 90.0.\n" do
+          TestCoverage.expects(:quit).with(false)
+          TestCoverage.check_test_coverage
+        end
+      end
+    end
+
+    context "when code coverage percentage above threshold" do
+      setup do
+        ARGF.expects(:readlines).once.returns([
+          "85 2",
+          "11 2",
+        ])
+      end
+
+      should "exit successfully" do
+        assert_output "Total test coverage percentage is 96.0.\n" do
+          TestCoverage.expects(:quit).with(true)
+          TestCoverage.check_test_coverage
+        end
+      end
+    end
+  end
+end