diff --git a/docs/configuration.md b/docs/configuration.md index 6849b2269..259fffbaa 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -11,7 +11,7 @@ The includes key takes a list of strings to add to ruby's `$LOAD_PATH`. This is ```yml --- includes: - - lib +- lib ``` Additional includes can be added by providing the `-I` or `--include` option to the CLI. @@ -23,7 +23,7 @@ The requires key takes a list of ruby files to be required. This is how mutant l ```yml --- requires: - - my_app +- my_app ``` Additional requires can be added by providing the `-r` or `--require` option to the CLI. @@ -63,5 +63,38 @@ See mutant's configuration file, [mutant.yml](/mutant.yml), for a complete examp #### `mutation_timeout` Specify the maximum time, in seconds, a mutation gets analysed. -Past this time the mutation analysis gets terminated and the result is currently being -reported as uncovered. + +```yml +--- +# Control the maximum time per mutation spend on analysis. +# Unit is in fractional seconds. +# +# Default absent. +# +# Absent value: No limit on per mutation analysis time. +# Present value: Limit per mutation analysis time to specified value in seconds. +mutation_timeout: 1.0 +``` + +Use `timeout` setting under `coverage_criteria` in the config file to control +if timeouts are allowed to cover mutations. + +#### `coverage_criteria` + +A configuration file only setting to control which criteria apply to determine mutation coverage. + +```yml +--- +coverage_criteria: + # Control the timeout criteria, defaults to `false`: + # * `true` - mutant will consider timeouts as covering mutations. + # * `false` mutant will ignore timeouts when determining mutation coverage. + timeout: false + # Control the test result criteria, # defaults to `true` + # * `true` mutant will consider failing tests covering mutations. + # * `false` mutant will ignore test results when determining mutation coverage. + # Hint: You probably do not want to touch the default. + test_result: true +``` + +At this point there is no CLI equivalent for these settings. diff --git a/lib/mutant.rb b/lib/mutant.rb index 5b2680d1f..21aa423c0 100644 --- a/lib/mutant.rb +++ b/lib/mutant.rb @@ -179,6 +179,7 @@ module Mutant require 'mutant/reporter/cli' require 'mutant/reporter/cli/printer' require 'mutant/reporter/cli/printer/config' +require 'mutant/reporter/cli/printer/coverage_result' require 'mutant/reporter/cli/printer/env' require 'mutant/reporter/cli/printer/env_progress' require 'mutant/reporter/cli/printer/env_result' @@ -223,6 +224,7 @@ module Mutant # Reopen class to initialize constant to avoid dep circle class Config DEFAULT = new( + coverage_criteria: Config::CoverageCriteria::DEFAULT, expression_parser: Expression::Parser.new([ Expression::Method, Expression::Methods, diff --git a/lib/mutant/config.rb b/lib/mutant/config.rb index 16cc2afae..79b5cfd1f 100644 --- a/lib/mutant/config.rb +++ b/lib/mutant/config.rb @@ -7,6 +7,7 @@ module Mutant # to current environment is being represented by the Mutant::Env object. class Config include Adamantium::Flat, Anima.new( + :coverage_criteria, :expression_parser, :fail_fast, :includes, @@ -24,25 +25,6 @@ class Config define_method(:"#{name}?") { public_send(name) } end - TRANSFORM = Transform::Sequence.new( - [ - Transform::Exception.new(SystemCallError, :read.to_proc), - Transform::Exception.new(YAML::SyntaxError, YAML.method(:safe_load)), - Transform::Hash.new( - optional: [ - Transform::Hash::Key.new('fail_fast', Transform::BOOLEAN), - Transform::Hash::Key.new('includes', Transform::STRING_ARRAY), - Transform::Hash::Key.new('integration', Transform::STRING), - Transform::Hash::Key.new('jobs', Transform::INTEGER), - Transform::Hash::Key.new('mutation_timeout', Transform::FLOAT), - Transform::Hash::Key.new('requires', Transform::STRING_ARRAY) - ], - required: [] - ), - Transform::Hash::Symbolize.new - ] - ) - MORE_THAN_ONE_CONFIG_FILE = <<~'MESSAGE' Found more than one candidate for use as implicit config file: %s MESSAGE @@ -53,6 +35,32 @@ class Config mutant.yml ].freeze + private_constant(*constants(false)) + + class CoverageCriteria + include Anima.new(:timeout, :test_result) + + DEFAULT = new( + timeout: false, + test_result: true + ) + + TRANSFORM = + Transform::Sequence.new( + [ + Transform::Hash.new( + optional: [ + Transform::Hash::Key.new('timeout', Transform::BOOLEAN), + Transform::Hash::Key.new('test_result', Transform::BOOLEAN) + ], + required: [] + ), + Transform::Hash::Symbolize.new, + ->(value) { Either::Right.new(DEFAULT.with(**value)) } + ] + ) + end # CoverageCriteria + # Merge with other config # # @param [Config] other @@ -71,8 +79,6 @@ def merge(other) ) end - private_constant(*constants(false)) - # Load config file # # @param [World] world @@ -106,5 +112,27 @@ def self.load_contents(path) def self.env DEFAULT.with(jobs: Etc.nprocessors) end + + TRANSFORM = Transform::Sequence.new( + [ + Transform::Exception.new(SystemCallError, :read.to_proc), + Transform::Exception.new(YAML::SyntaxError, YAML.method(:safe_load)), + Transform::Hash.new( + optional: [ + Transform::Hash::Key.new('coverage_criteria', CoverageCriteria::TRANSFORM), + Transform::Hash::Key.new('fail_fast', Transform::BOOLEAN), + Transform::Hash::Key.new('includes', Transform::STRING_ARRAY), + Transform::Hash::Key.new('integration', Transform::STRING), + Transform::Hash::Key.new('jobs', Transform::INTEGER), + Transform::Hash::Key.new('mutation_timeout', Transform::FLOAT), + Transform::Hash::Key.new('requires', Transform::STRING_ARRAY) + ], + required: [] + ), + Transform::Hash::Symbolize.new + ] + ) + + private_constant(:TRANSFORM) end # Config end # Mutant diff --git a/lib/mutant/isolation.rb b/lib/mutant/isolation.rb index 37bb2a0e3..539a16e6f 100644 --- a/lib/mutant/isolation.rb +++ b/lib/mutant/isolation.rb @@ -18,7 +18,7 @@ class Result # Test for successful result # # @return [Boolean] - def success? + def valid_value? timeout.nil? && exception.nil? && (process_status.nil? || process_status.success?) end end # Result diff --git a/lib/mutant/reporter/cli/printer/coverage_result.rb b/lib/mutant/reporter/cli/printer/coverage_result.rb new file mode 100644 index 000000000..f444c324a --- /dev/null +++ b/lib/mutant/reporter/cli/printer/coverage_result.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Mutant + class Reporter + class CLI + class Printer + # Reporter for mutation coverage results + class CoverageResult < self + # Run report printer + # + # @return [undefined] + def run + visit(MutationResult, object.mutation_result) + end + end # Printer + end # Coverage + end # CLI + end # Reporter +end # Mutant diff --git a/lib/mutant/reporter/cli/printer/env_progress.rb b/lib/mutant/reporter/cli/printer/env_progress.rb index 2d32b9889..68a4b434d 100644 --- a/lib/mutant/reporter/cli/printer/env_progress.rb +++ b/lib/mutant/reporter/cli/printer/env_progress.rb @@ -10,6 +10,7 @@ class EnvProgress < self :amount_mutation_results, :amount_mutations_alive, :amount_mutations_killed, + :amount_timeouts, :coverage, :env, :killtime, @@ -21,6 +22,7 @@ class EnvProgress < self [:info, 'Results: %s', :amount_mutation_results], [:info, 'Kills: %s', :amount_mutations_killed], [:info, 'Alive: %s', :amount_mutations_alive ], + [:info, 'Timeouts: %s', :amount_timeouts ], [:info, 'Runtime: %0.2fs', :runtime ], [:info, 'Killtime: %0.2fs', :killtime ], [:info, 'Overhead: %0.2f%%', :overhead_percent ], diff --git a/lib/mutant/reporter/cli/printer/subject_result.rb b/lib/mutant/reporter/cli/printer/subject_result.rb index 70079a497..4445ed7f0 100644 --- a/lib/mutant/reporter/cli/printer/subject_result.rb +++ b/lib/mutant/reporter/cli/printer/subject_result.rb @@ -7,7 +7,7 @@ class Printer # Subject result printer class SubjectResult < self - delegate :subject, :alive_mutation_results, :tests + delegate :subject, :uncovered_results, :tests # Run report printer # @@ -17,7 +17,7 @@ def run tests.each do |test| puts("- #{test.identification}") end - visit_collection(MutationResult, alive_mutation_results) + visit_collection(CoverageResult, uncovered_results) end end # SubjectResult diff --git a/lib/mutant/result.rb b/lib/mutant/result.rb index 2408bb1a5..f0bb78006 100644 --- a/lib/mutant/result.rb +++ b/lib/mutant/result.rb @@ -4,8 +4,8 @@ module Mutant # Namespace and mixin module for results module Result - # Coverage mixin - module Coverage + # CoverageMetric mixin + module CoverageMetric FULL_COVERAGE = Rational(1).freeze private_constant(*constants(false)) @@ -19,7 +19,7 @@ def coverage Rational(amount_mutations_killed, amount_mutation_results) end end - end # Coverage + end # CoverageMetric # Class level mixin module ClassMethods @@ -39,6 +39,13 @@ def sum(name, collection) end memoize(name) end + + # Delegate a method to child + def delegate(name, target) + define_method(name) do + public_send(target).public_send(name) + end + end end # ClassMethods private_constant(*constants(false)) @@ -68,7 +75,7 @@ def self.included(host) # Env result object class Env - include Coverage, Result, Anima.new( + include CoverageMetric, Result, Anima.new( :env, :runtime, :subject_results @@ -92,6 +99,7 @@ def failed_subject_results sum :amount_mutation_results, :subject_results sum :amount_mutations_alive, :subject_results sum :amount_mutations_killed, :subject_results + sum :amount_timeouts, :subject_results sum :killtime, :subject_results # Amount of mutations @@ -137,35 +145,42 @@ def initialize # Subject result class Subject - include Coverage, Result, Anima.new( - :mutation_results, + include CoverageMetric, Result, Anima.new( + :coverage_results, :subject, :tests ) - sum :killtime, :mutation_results - sum :runtime, :mutation_results + sum :killtime, :coverage_results + sum :runtime, :coverage_results # Test if subject was processed successful # # @return [Boolean] def success? - alive_mutation_results.empty? + uncovered_results.empty? end # Alive mutations # - # @return [Array] - def alive_mutation_results - mutation_results.reject(&:success?) + # @return [Array] + def uncovered_results + coverage_results.reject(&:success?) end - memoize :alive_mutation_results + memoize :uncovered_results # Amount of mutations # # @return [Integer] def amount_mutation_results - mutation_results.length + coverage_results.length + end + + # Amount of mutations + # + # @return [Integer] + def amount_timeouts + coverage_results.count(&:timeout?) end # Amount of mutations @@ -179,25 +194,49 @@ def amount_mutations # # @return [Integer] def amount_mutations_killed - killed_mutation_results.length + covered_results.length end # Number of alive mutations # # @return [Integer] def amount_mutations_alive - alive_mutation_results.length + uncovered_results.length end private - def killed_mutation_results - mutation_results.select(&:success?) + def covered_results + coverage_results.select(&:success?) end - memoize :killed_mutation_results + memoize :covered_results end # Subject + # Coverage of a mutation against criteria + class Coverage + include Result, Anima.new( + :mutation_result, + :criteria_result + ) + + delegate :killtime, :mutation_result + delegate :runtime, :mutation_result + delegate :success?, :criteria_result + delegate :timeout?, :mutation_result + end # Coverage + + class CoverageCriteria + include Result, Anima.new(*Config::CoverageCriteria.anima.attribute_names) + + # Test if one coverage criteria indicates success + # + # @return [Boolean] + def success? + test_result || timeout + end + end + # Mutation result class Mutation include Result, Anima.new( @@ -206,25 +245,39 @@ class Mutation :runtime ) + # Create mutation criteria results + # + # @praam [Result::CoverageCriteria] + def criteria_result(coverage_criteria) + CoverageCriteria.new( + test_result: coverage_criteria.test_result && test_result_success?, + timeout: coverage_criteria.timeout && timeout? + ) + end + # Time the tests had been running # # @return [Float] def killtime - if isolation_result.success? - isolation_result.value.runtime - else - 0.0 - end + isolation_result.value&.runtime || 0.0 end + # Test for timeout + # + # @return [Boolean] + def timeout? + !isolation_result.timeout.nil? + end + + private + # Test if mutation was handled successfully # # @return [Boolean] - def success? - isolation_result.success? && - mutation.class.success?(isolation_result.value) + def test_result_success? + isolation_result.valid_value? && mutation.class.success?(isolation_result.value) end - memoize :success? + memoize :test_result_success? end # Mutation end # Result diff --git a/lib/mutant/runner/sink.rb b/lib/mutant/runner/sink.rb index 7de1d42fe..094a2c4e0 100644 --- a/lib/mutant/runner/sink.rb +++ b/lib/mutant/runner/sink.rb @@ -42,7 +42,7 @@ def result(mutation_result) @subject_results[subject] = Result::Subject.new( subject: subject, - mutation_results: previous_mutation_results(subject) + [mutation_result], + coverage_results: previous_coverage_results(subject).dup << coverage_result(mutation_result), tests: env.selections.fetch(subject) ) @@ -51,9 +51,16 @@ def result(mutation_result) private - def previous_mutation_results(subject) + def coverage_result(mutation_result) + Result::Coverage.new( + mutation_result: mutation_result, + criteria_result: mutation_result.criteria_result(env.config.coverage_criteria) + ) + end + + def previous_coverage_results(subject) subject_result = @subject_results.fetch(subject) { return EMPTY_ARRAY } - subject_result.mutation_results + subject_result.coverage_results end end # Sink diff --git a/mutant.yml b/mutant.yml index 6ab4067ac..6b9216a75 100644 --- a/mutant.yml +++ b/mutant.yml @@ -2,7 +2,9 @@ includes: - lib integration: rspec -mutation_timeout: 5.0 +mutation_timeout: 0.1 requires: - mutant - mutant/meta +coverage_criteria: + timeout: true diff --git a/scripts/devloop.sh b/scripts/devloop.sh index 2019748fa..540d91f5c 100755 --- a/scripts/devloop.sh +++ b/scripts/devloop.sh @@ -1,5 +1,4 @@ while inotifywait **/*.rb Gemfile Gemfile.shared mutant.gemspec; do - bundle exec rspec spec/unit -fd --fail-fast --order default \ - && bundle exec ./mutant.sh --since master --fail-fast -- 'Mutant*' \ + bundle exec ./mutant.sh --since master --fail-fast -- 'Mutant*' \ && bundle exec rubocop done diff --git a/spec/support/shared_context.rb b/spec/support/shared_context.rb index e429266f3..7b46cae49 100644 --- a/spec/support/shared_context.rb +++ b/spec/support/shared_context.rb @@ -147,6 +147,34 @@ def setup_shared_context ) end + let(:mutation_a_coverage_result) do + Mutant::Result::Coverage.new( + mutation_result: mutation_a_result, + criteria_result: mutation_a_criteria_result + ) + end + + let(:mutation_b_coverage_result) do + Mutant::Result::Coverage.new( + mutation_result: mutation_b_result, + criteria_result: mutation_b_criteria_result + ) + end + + let(:mutation_a_criteria_result) do + Mutant::Result::CoverageCriteria.new( + test_result: true, + timeout: false + ) + end + + let(:mutation_b_criteria_result) do + Mutant::Result::CoverageCriteria.new( + test_result: true, + timeout: false + ) + end + let(:mutation_a_isolation_result) do Mutant::Isolation::Result.new( exception: nil, @@ -187,7 +215,7 @@ def setup_shared_context Mutant::Result::Subject.new( subject: subject_a, tests: [test_a], - mutation_results: [mutation_a_result, mutation_b_result] + coverage_results: [mutation_a_coverage_result, mutation_b_coverage_result] ) end end diff --git a/spec/unit/mutant/config_spec.rb b/spec/unit/mutant/config_spec.rb index e3f92d9ec..1cf68aae7 100644 --- a/spec/unit/mutant/config_spec.rb +++ b/spec/unit/mutant/config_spec.rb @@ -141,6 +141,20 @@ def expect_value(value) include_examples 'overwrite value' end + context 'merging coverage criteria' do + let(:key) { :coverage_criteria } + + let(:original_value) do + instance_double(Mutant::Config::CoverageCriteria, 'config') + end + + let(:other_value) do + instance_double(Mutant::Config::CoverageCriteria, 'other') + end + + include_examples 'overwrite value' + end + context 'merging isolation' do let(:key) { :isolation } let(:original_value) { instance_double(Mutant::Isolation, 'config') } diff --git a/spec/unit/mutant/isolation/result_spec.rb b/spec/unit/mutant/isolation/result_spec.rb index 2a022e228..9226c6533 100644 --- a/spec/unit/mutant/isolation/result_spec.rb +++ b/spec/unit/mutant/isolation/result_spec.rb @@ -11,7 +11,7 @@ ) end - describe '#success?' do + describe '#valid_value?' do let(:exception) { nil } let(:timeout) { nil } let(:process_success) { true } @@ -24,7 +24,7 @@ end def apply - object.success? + object.valid_value? end context 'no contraindications' do diff --git a/spec/unit/mutant/reporter/cli/printer/env_progress_spec.rb b/spec/unit/mutant/reporter/cli/printer/env_progress_spec.rb index d3943d7b0..56b40c7ac 100644 --- a/spec/unit/mutant/reporter/cli/printer/env_progress_spec.rb +++ b/spec/unit/mutant/reporter/cli/printer/env_progress_spec.rb @@ -7,7 +7,7 @@ describe '.call' do context 'without progress' do - with(:subject_a_result) { { mutation_results: [] } } + with(:subject_a_result) { { coverage_results: [] } } it_reports <<~'STR' Mutant environment: @@ -24,6 +24,7 @@ Results: 0 Kills: 0 Alive: 0 + Timeouts: 0 Runtime: 4.00s Killtime: 0.00s Overhead: Inf% @@ -48,6 +49,7 @@ Results: 2 Kills: 2 Alive: 0 + Timeouts: 0 Runtime: 4.00s Killtime: 2.00s Overhead: 100.00% @@ -57,7 +59,7 @@ end context 'on partial coverage' do - with(:mutation_a_test_result) { { passed: true } } + with(:mutation_a_criteria_result) { { test_result: false } } it_reports <<~'STR' Mutant environment: @@ -74,6 +76,7 @@ Results: 2 Kills: 1 Alive: 1 + Timeouts: 0 Runtime: 4.00s Killtime: 2.00s Overhead: 100.00% diff --git a/spec/unit/mutant/reporter/cli/printer/env_result_spec.rb b/spec/unit/mutant/reporter/cli/printer/env_result_spec.rb index 56235e0be..0143a20a7 100644 --- a/spec/unit/mutant/reporter/cli/printer/env_result_spec.rb +++ b/spec/unit/mutant/reporter/cli/printer/env_result_spec.rb @@ -3,7 +3,7 @@ RSpec.describe Mutant::Reporter::CLI::Printer::EnvResult do setup_shared_context - with(:mutation_a_test_result) { { passed: true } } + with(:mutation_a_criteria_result) { { test_result: false } } let(:reportable) { env_result } @@ -31,6 +31,7 @@ Results: 2 Kills: 1 Alive: 1 + Timeouts: 0 Runtime: 4.00s Killtime: 2.00s Overhead: 100.00% diff --git a/spec/unit/mutant/reporter/cli/printer/status_progressive_spec.rb b/spec/unit/mutant/reporter/cli/printer/status_progressive_spec.rb index 6b167a66d..4ab4ca9a3 100644 --- a/spec/unit/mutant/reporter/cli/printer/status_progressive_spec.rb +++ b/spec/unit/mutant/reporter/cli/printer/status_progressive_spec.rb @@ -27,7 +27,7 @@ with(:status) { { active_jobs: [job_b, job_a].to_set } } context 'on failure' do - with(:mutation_a_test_result) { { passed: true } } + with(:mutation_a_criteria_result) { { test_result: false } } it_reports(<<~REPORT) progress: 02/02 alive: 1 runtime: 4.00s killtime: 2.00s mutations/s: 0.50 diff --git a/spec/unit/mutant/reporter/cli/printer/subject_result_spec.rb b/spec/unit/mutant/reporter/cli/printer/subject_result_spec.rb index 902f9c3d2..eeacca622 100644 --- a/spec/unit/mutant/reporter/cli/printer/subject_result_spec.rb +++ b/spec/unit/mutant/reporter/cli/printer/subject_result_spec.rb @@ -14,7 +14,7 @@ end context 'on partial coverage' do - with(:mutation_a_test_result) { { passed: true } } + with(:mutation_a_criteria_result) { { test_result: false } } it_reports <<~'STR' subject-a @@ -29,7 +29,7 @@ end context 'without results' do - with(:subject_a_result) { { mutation_results: [] } } + with(:subject_a_result) { { coverage_results: [] } } it_reports <<~'STR' subject-a diff --git a/spec/unit/mutant/reporter/cli_spec.rb b/spec/unit/mutant/reporter/cli_spec.rb index 33635b22d..dc226b1f9 100644 --- a/spec/unit/mutant/reporter/cli_spec.rb +++ b/spec/unit/mutant/reporter/cli_spec.rb @@ -95,6 +95,7 @@ def self.it_reports(expected_content) Results: 2 Kills: 2 Alive: 0 + Timeouts: 0 Runtime: 4.00s Killtime: 2.00s Overhead: 100.00% @@ -126,7 +127,7 @@ def self.it_reports(expected_content) end context 'when mutation is NOT successful' do - with(:mutation_a_test_result) { { passed: true } } + with(:mutation_a_criteria_result) { { test_result: false } } it_reports "progress: 02/02 alive: 1 runtime: 4.00s killtime: 2.00s mutations/s: 0.50\n" end diff --git a/spec/unit/mutant/result/class_methods_spec.rb b/spec/unit/mutant/result/class_methods_spec.rb index 747ff0c43..45e5c4e26 100644 --- a/spec/unit/mutant/result/class_methods_spec.rb +++ b/spec/unit/mutant/result/class_methods_spec.rb @@ -15,12 +15,14 @@ extend Mutant::Result.const_get(:ClassMethods) sum :length, :collection + + delegate :to_set, :collection end end - describe '#sum' do - let(:object) { infected_class.new(collection) } + let(:object) { infected_class.new(collection) } + describe '#sum' do def apply object.length end @@ -48,4 +50,12 @@ def apply it { should be(3) } end end + + describe '#delegate' do + let(:collection) { [1, 2] } + + subject { object.to_set } + + it { should eql(Set.new([1, 2])) } + end end diff --git a/spec/unit/mutant/result/coverage_criteria_spec.rb b/spec/unit/mutant/result/coverage_criteria_spec.rb new file mode 100644 index 000000000..88658628a --- /dev/null +++ b/spec/unit/mutant/result/coverage_criteria_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +RSpec.describe Mutant::Result::CoverageCriteria do + let(:object) do + described_class.new( + test_result: test_result, + timeout: timeout + ) + end + + let(:timeout) { false } + let(:test_result) { false } + + describe '#success?' do + def apply + object.success? + end + + context 'on no success criteria set' do + it 'returns false' do + expect(apply).to be(false) + end + end + + context 'on timeout criteria set' do + let(:timeout) { true } + + it 'returns true' do + expect(apply).to be(true) + end + end + + context 'on test result criteria set' do + let(:test_result) { true } + + it 'returns true' do + expect(apply).to be(true) + end + end + end +end diff --git a/spec/unit/mutant/result/mutation_spec.rb b/spec/unit/mutant/result/mutation_spec.rb index 631b044af..5807e3ef8 100644 --- a/spec/unit/mutant/result/mutation_spec.rb +++ b/spec/unit/mutant/result/mutation_spec.rb @@ -28,27 +28,15 @@ ) end - shared_examples_for 'unsuccessful isolation' do - let(:isolation_result) do - Mutant::Isolation::Result.new( - exception: RuntimeError.new, - log: '', - process_status: nil, - timeout: nil, - value: test_result - ) - end - end - describe '#killtime' do subject { object.killtime } - context 'if isolation is successful' do + context 'if isolation reports runtime' do it { should eql(1.0) } end - context 'if isolation is not successful' do - include_context 'unsuccessful isolation' + context 'if isolation does not report runtime' do + let(:test_result) { nil } it { should eql(0.0) } end @@ -60,23 +48,77 @@ it { should eql(2.0) } end - describe '#success?' do - subject { object.success? } + describe '#criteria_result' do + def apply + object.criteria_result(coverage_criteria) + end + + let(:coverage_criteria) do + Mutant::Config::CoverageCriteria::DEFAULT + end + + before do + allow(mutation.class).to receive_messages(success?: true) + end + + context 'when timeouts cover mutations' do + let(:coverage_criteria) { super().with(timeout: true) } + + context 'on isolation result with timeouts' do + let(:isolation_result) { super().with(timeout: 1.0) } + + it 'covers timeout' do + expect(apply.timeout).to be(true) + end + end + + context 'on isolation result without timeouts' do + it 'does not cover timeout' do + expect(apply.timeout).to be(false) + end + end + end + + context 'when timeouts do not cover mutations' do + context 'on isolation result with timeouts' do + let(:isolation_result) { super().with(timeout: 1.0) } + + it 'does not cover timeout' do + expect(apply.timeout).to be(false) + end + end + end + + context 'when test results cover mutations' do + context 'on valid isolation result' do + it 'passes test_results criteria' do + expect(apply.test_result).to be(true) + end + + it 'calculates mutation class speific pass state' do + apply - context 'if isolation is successful' do - before do - expect(mutation.class).to receive(:success?) - .with(test_result) - .and_return(true) + expect(mutation.class).to have_received(:success?).with(test_result) + end end - it { should be(true) } + context 'on invalid isolation results' do + let(:isolation_result) { super().with(exception: RuntimeError.new) } + + it 'does not pass test_results criteria' do + expect(apply.test_result).to be(false) + end + end end - context 'if isolation is not successful' do - include_context 'unsuccessful isolation' + context 'when test results do not cover mutations' do + let(:coverage_criteria) { super().with(test_result: false) } - it { should be(false) } + context 'on valid isolation result' do + it 'does not pass test_results criteria' do + expect(apply.test_result).to be(false) + end + end end end end diff --git a/spec/unit/mutant/result/subject_spec.rb b/spec/unit/mutant/result/subject_spec.rb index b05d13f1c..4bd1fa19f 100644 --- a/spec/unit/mutant/result/subject_spec.rb +++ b/spec/unit/mutant/result/subject_spec.rb @@ -4,7 +4,7 @@ let(:object) do described_class.new( subject: mutation_subject, - mutation_results: mutation_results, + coverage_results: coverage_results, tests: [] ) end @@ -12,37 +12,37 @@ let(:mutation_subject) do instance_double( Mutant::Subject, - mutations: mutation_results.map { instance_double(Mutant::Mutation) } + mutations: coverage_results.map { instance_double(Mutant::Mutation) } ) end shared_context 'full coverage' do - let(:mutation_results) do + let(:coverage_results) do [ - instance_double(Mutant::Result::Mutation, success?: true) + instance_double(Mutant::Result::Coverage, success?: true) ] end end shared_context 'partial coverage' do - let(:mutation_results) do + let(:coverage_results) do [ - instance_double(Mutant::Result::Mutation, success?: false), - instance_double(Mutant::Result::Mutation, success?: true) + instance_double(Mutant::Result::Coverage, success?: false), + instance_double(Mutant::Result::Coverage, success?: true) ] end end shared_context 'no coverage' do - let(:mutation_results) do + let(:coverage_results) do [ - instance_double(Mutant::Result::Mutation, success?: false) + instance_double(Mutant::Result::Coverage, success?: false) ] end end shared_context 'no results' do - let(:mutation_results) { [] } + let(:coverage_results) { [] } end describe '#coverage' do diff --git a/spec/unit/mutant/runner/sink_spec.rb b/spec/unit/mutant/runner/sink_spec.rb index e65b12030..a86f3644f 100644 --- a/spec/unit/mutant/runner/sink_spec.rb +++ b/spec/unit/mutant/runner/sink_spec.rb @@ -54,7 +54,7 @@ context 'one result' do include_context 'one result' - with(:subject_a_result) { { mutation_results: [mutation_a_result] } } + with(:subject_a_result) { { coverage_results: [mutation_a_coverage_result] } } let(:expected_status) do Mutant::Result::Env.new(