Add timeout as configurable coverage criteria

Changelog.md

@@ -1,10 +1,20 @@
-# v0.10.7 unreleased
+# v0.10.7 2020-11-22
+
+* Add support for external mutation timeouts.
+
+  New config file settings. `mutation_timeout` and `coverage_criteria`
+  to control timeouts and coverage conditions.
+
+  - [#1105](https://github.com/mbj/mutant/pull/1105)
+  - [#1118](https://github.com/mbj/mutant/pull/1118)
+
+* Improve CLI reporting to be less noisy:
+  - [#1117](https://github.com/mbj/mutant/pull/1117)
+  - [#1106](https://github.com/mbj/mutant/pull/1106)
 
-* Add support for external mutation timeouts. [#1105](https://github.com/mbj/mutant/pull/1105)
 * Fix crash on static send mutation. [#1108](https://github.com/mbj/mutant/pull/1108)
-* Change CLI report format to print diff last.
-  This should be more user friendly on incremental / interactive
-  operation.
+
+* Add more verbose configuration [documentation](https://github.com/mbj/mutant/blob/master/docs/configuration.md).
 
 # v0.10.6 2020-11-06
 

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    mutant (0.10.6)
+    mutant (0.10.7)
       abstract_type (~> 0.0.7)
       adamantium (~> 0.2.0)
       anima (~> 0.3.1)

docs/configuration.md

@@ -11,7 +11,7 @@ The includes key takes a list of strings to add to ruby's `$LOAD_PATH`. This is
 ```yml
 ---
 includes:
-  - lib
+- lib
 ```
 
 Additional includes can be added by providing the `-I` or `--include` option to the CLI.
@@ -23,7 +23,7 @@ The requires key takes a list of ruby files to be required. This is how mutant l
 ```yml
 ---
 requires:
-  - my_app
+- my_app
 ```
 
 Additional requires can be added by providing the `-r` or `--require` option to the CLI.
@@ -63,5 +63,38 @@ See mutant's configuration file, [mutant.yml](/mutant.yml), for a complete examp
 #### `mutation_timeout`
 
 Specify the maximum time, in seconds, a mutation gets analysed.
-Past this time the mutation analysis gets terminated and the result is currently being
-reported as uncovered.
+
+```yml
+---
+# Control the maximum time per mutation spend on analysis.
+# Unit is in fractional seconds.
+#
+# Default absent.
+#
+# Absent value: No limit on per mutation analysis time.
+# Present value: Limit per mutation analysis time to specified value in seconds.
+mutation_timeout: 1.0
+```
+
+Use `timeout` setting under `coverage_criteria` in the config file to control
+if timeouts are allowed to cover mutations.
+
+#### `coverage_criteria`
+
+A configuration file only setting to control which criteria apply to determine mutation coverage.
+
+```yml
+---
+coverage_criteria:
+  # Control the timeout criteria, defaults to `false`:
+  # * `true` - mutant will consider timeouts as covering mutations.
+  # * `false` mutant will ignore timeouts when determining mutation coverage.
+  timeout: false
+  # Control the test result criteria, # defaults to `true`
+  # * `true` mutant will consider failing tests covering mutations.
+  # * `false` mutant will ignore test results when determining mutation coverage.
+  # Hint: You probably do not want to touch the default.
+  test_result: true
+```
+
+At this point there is no CLI equivalent for these settings.

lib/mutant.rb

@@ -179,6 +179,7 @@ module Mutant
 require 'mutant/reporter/cli'
 require 'mutant/reporter/cli/printer'
 require 'mutant/reporter/cli/printer/config'
+require 'mutant/reporter/cli/printer/coverage_result'
 require 'mutant/reporter/cli/printer/env'
 require 'mutant/reporter/cli/printer/env_progress'
 require 'mutant/reporter/cli/printer/env_result'
@@ -223,6 +224,7 @@ module Mutant
   # Reopen class to initialize constant to avoid dep circle
   class Config
     DEFAULT = new(
+      coverage_criteria: Config::CoverageCriteria::DEFAULT,
       expression_parser: Expression::Parser.new([
         Expression::Method,
         Expression::Methods,

lib/mutant/cli/command/run.rb

@@ -35,7 +35,7 @@ def zombie?
 
         def initialize(attributes)
           super(attributes)
-          @config = Config::DEFAULT
+          @config = Config.env
         end
 
         def execute
@@ -49,7 +49,7 @@ def execute
         end
 
         def expand(file_config)
-          @config = Config.env.merge(file_config).merge(@config)
+          @config = @config.merge(file_config)
         end
 
         def soft_fail(result)

lib/mutant/config.rb

@@ -7,6 +7,7 @@ module Mutant
   # to current environment is being represented by the Mutant::Env object.
   class Config
     include Adamantium::Flat, Anima.new(
+      :coverage_criteria,
       :expression_parser,
       :fail_fast,
       :includes,
@@ -24,32 +25,6 @@ class Config
       define_method(:"#{name}?") { public_send(name) }
     end
 
-    boolean = Transform::Boolean.new
-    float   = Transform::Primitive.new(Float)
-    integer = Transform::Primitive.new(Integer)
-    string  = Transform::Primitive.new(String)
-
-    string_array = Transform::Array.new(string)
-
-    TRANSFORM = Transform::Sequence.new(
-      [
-        Transform::Exception.new(SystemCallError, :read.to_proc),
-        Transform::Exception.new(YAML::SyntaxError, YAML.method(:safe_load)),
-        Transform::Hash.new(
-          optional: [
-            Transform::Hash::Key.new('fail_fast',        boolean),
-            Transform::Hash::Key.new('includes',         string_array),
-            Transform::Hash::Key.new('integration',      string),
-            Transform::Hash::Key.new('jobs',             integer),
-            Transform::Hash::Key.new('mutation_timeout', float),
-            Transform::Hash::Key.new('requires',         string_array)
-          ],
-          required: []
-        ),
-        Transform::Hash::Symbolize.new
-      ]
-    )
-
     MORE_THAN_ONE_CONFIG_FILE = <<~'MESSAGE'
       Found more than one candidate for use as implicit config file: %s
     MESSAGE
@@ -60,6 +35,32 @@ class Config
       mutant.yml
     ].freeze
 
+    private_constant(*constants(false))
+
+    class CoverageCriteria
+      include Anima.new(:timeout, :test_result)
+
+      DEFAULT = new(
+        timeout:     false,
+        test_result: true
+      )
+
+      TRANSFORM =
+        Transform::Sequence.new(
+          [
+            Transform::Hash.new(
+              optional: [
+                Transform::Hash::Key.new('timeout',     Transform::BOOLEAN),
+                Transform::Hash::Key.new('test_result', Transform::BOOLEAN)
+              ],
+              required: []
+            ),
+            Transform::Hash::Symbolize.new,
+            ->(value) { Either::Right.new(DEFAULT.with(**value)) }
+          ]
+        )
+    end # CoverageCriteria
+
     # Merge with other config
     #
     # @param [Config] other
@@ -68,18 +69,16 @@ class Config
     def merge(other)
       other.with(
         fail_fast:        fail_fast || other.fail_fast,
-        includes:         includes + other.includes,
+        includes:         other.includes + includes,
         jobs:             other.jobs || jobs,
         integration:      other.integration || integration,
         mutation_timeout: other.mutation_timeout || mutation_timeout,
         matcher:          matcher.merge(other.matcher),
-        requires:         requires + other.requires,
+        requires:         other.requires + requires,
         zombie:           zombie || other.zombie
       )
     end
 
-    private_constant(*constants(false))
-
     # Load config file
     #
     # @param [World] world
@@ -113,5 +112,27 @@ def self.load_contents(path)
     def self.env
       DEFAULT.with(jobs: Etc.nprocessors)
     end
+
+    TRANSFORM = Transform::Sequence.new(
+      [
+        Transform::Exception.new(SystemCallError, :read.to_proc),
+        Transform::Exception.new(YAML::SyntaxError, YAML.method(:safe_load)),
+        Transform::Hash.new(
+          optional: [
+            Transform::Hash::Key.new('coverage_criteria', CoverageCriteria::TRANSFORM),
+            Transform::Hash::Key.new('fail_fast',         Transform::BOOLEAN),
+            Transform::Hash::Key.new('includes',          Transform::STRING_ARRAY),
+            Transform::Hash::Key.new('integration',       Transform::STRING),
+            Transform::Hash::Key.new('jobs',              Transform::INTEGER),
+            Transform::Hash::Key.new('mutation_timeout',  Transform::FLOAT),
+            Transform::Hash::Key.new('requires',          Transform::STRING_ARRAY)
+          ],
+          required: []
+        ),
+        Transform::Hash::Symbolize.new
+      ]
+    )
+
+    private_constant(:TRANSFORM)
   end # Config
 end # Mutant

lib/mutant/isolation.rb

@@ -18,7 +18,7 @@ class Result
       # Test for successful result
       #
       # @return [Boolean]
-      def success?
+      def valid_value?
         timeout.nil? && exception.nil? && (process_status.nil? || process_status.success?)
       end
     end # Result

lib/mutant/reporter/cli/printer/coverage_result.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Mutant
+  class Reporter
+    class CLI
+      class Printer
+        # Reporter for mutation coverage results
+        class CoverageResult < self
+          # Run report printer
+          #
+          # @return [undefined]
+          def run
+            visit(MutationResult, object.mutation_result)
+          end
+        end # Printer
+      end # Coverage
+    end # CLI
+  end # Reporter
+end # Mutant

lib/mutant/reporter/cli/printer/env_progress.rb

@@ -10,6 +10,7 @@ class EnvProgress < self
             :amount_mutation_results,
             :amount_mutations_alive,
             :amount_mutations_killed,
+            :amount_timeouts,
             :coverage,
             :env,
             :killtime,
@@ -21,6 +22,7 @@ class EnvProgress < self
             [:info,   'Results:         %s',      :amount_mutation_results],
             [:info,   'Kills:           %s',      :amount_mutations_killed],
             [:info,   'Alive:           %s',      :amount_mutations_alive ],
+            [:info,   'Timeouts:        %s',      :amount_timeouts        ],
             [:info,   'Runtime:         %0.2fs',  :runtime                ],
             [:info,   'Killtime:        %0.2fs',  :killtime               ],
             [:info,   'Overhead:        %0.2f%%', :overhead_percent       ],

lib/mutant/reporter/cli/printer/subject_result.rb

@@ -7,7 +7,7 @@ class Printer
         # Subject result printer
         class SubjectResult < self
 
-          delegate :subject, :alive_mutation_results, :tests
+          delegate :subject, :uncovered_results, :tests
 
           # Run report printer
           #
@@ -17,7 +17,7 @@ def run
             tests.each do |test|
               puts("- #{test.identification}")
             end
-            visit_collection(MutationResult, alive_mutation_results)
+            visit_collection(CoverageResult, uncovered_results)
           end
 
         end # SubjectResult