Add timeout as configurable coverage criteria

docs/configuration.md

@@ -11,7 +11,7 @@ The includes key takes a list of strings to add to ruby's `$LOAD_PATH`. This is
 ```yml
 ---
 includes:
-  - lib
+- lib
 ```
 
 Additional includes can be added by providing the `-I` or `--include` option to the CLI.
@@ -23,7 +23,7 @@ The requires key takes a list of ruby files to be required. This is how mutant l
 ```yml
 ---
 requires:
-  - my_app
+- my_app
 ```
 
 Additional requires can be added by providing the `-r` or `--require` option to the CLI.
@@ -63,5 +63,38 @@ See mutant's configuration file, [mutant.yml](/mutant.yml), for a complete examp
 #### `mutation_timeout`
 
 Specify the maximum time, in seconds, a mutation gets analysed.
-Past this time the mutation analysis gets terminated and the result is currently being
-reported as uncovered.
+
+```yml
+---
+# Control the maximum time per mutation spend on analysis.
+# Unit is in fractional seconds.
+#
+# Default absent.
+#
+# Absent value: No limit on per mutation analysis time.
+# Present value: Limit per mutation analysis time to specified value in seconds.
+mutation_timeout: 1.0
+```
+
+Use `timeout` setting under `coverage_criteria` in the config file to control
+if timeouts are allowed to cover mutations.
+
+#### `coverage_criteria`
+
+A configuration file only setting to control which criteria apply to determine mutation coverage.
+
+```yml
+---
+coverage_criteria:
+  # Control the timeout criteria, defaults to `false`:
+  # * `true` - mutant will consider timeouts as covering mutations.
+  # * `false` mutant will ignore timeouts when determining mutation coverage.
+  timeout: false
+  # Control the test result criteria, # defaults to `true`
+  # * `true` mutant will consider failing tests covering mutations.
+  # * `false` mutant will ignore test results when determining mutation coverage.
+  # Hint: You probably do not want to touch the default.
+  test_result: true
+```
+
+At this point there is no CLI equivalent for these settings.

lib/mutant.rb

@@ -179,6 +179,7 @@ module Mutant
 require 'mutant/reporter/cli'
 require 'mutant/reporter/cli/printer'
 require 'mutant/reporter/cli/printer/config'
+require 'mutant/reporter/cli/printer/coverage_result'
 require 'mutant/reporter/cli/printer/env'
 require 'mutant/reporter/cli/printer/env_progress'
 require 'mutant/reporter/cli/printer/env_result'
@@ -223,6 +224,7 @@ module Mutant
   # Reopen class to initialize constant to avoid dep circle
   class Config
     DEFAULT = new(
+      coverage_criteria: Config::CoverageCriteria::DEFAULT,
       expression_parser: Expression::Parser.new([
         Expression::Method,
         Expression::Methods,

lib/mutant/config.rb

@@ -7,6 +7,7 @@ module Mutant
   # to current environment is being represented by the Mutant::Env object.
   class Config
     include Adamantium::Flat, Anima.new(
+      :coverage_criteria,
       :expression_parser,
       :fail_fast,
       :includes,
@@ -24,25 +25,6 @@ class Config
       define_method(:"#{name}?") { public_send(name) }
     end
 
-    TRANSFORM = Transform::Sequence.new(
-      [
-        Transform::Exception.new(SystemCallError, :read.to_proc),
-        Transform::Exception.new(YAML::SyntaxError, YAML.method(:safe_load)),
-        Transform::Hash.new(
-          optional: [
-            Transform::Hash::Key.new('fail_fast',        Transform::BOOLEAN),
-            Transform::Hash::Key.new('includes',         Transform::STRING_ARRAY),
-            Transform::Hash::Key.new('integration',      Transform::STRING),
-            Transform::Hash::Key.new('jobs',             Transform::INTEGER),
-            Transform::Hash::Key.new('mutation_timeout', Transform::FLOAT),
-            Transform::Hash::Key.new('requires',         Transform::STRING_ARRAY)
-          ],
-          required: []
-        ),
-        Transform::Hash::Symbolize.new
-      ]
-    )
-
     MORE_THAN_ONE_CONFIG_FILE = <<~'MESSAGE'
       Found more than one candidate for use as implicit config file: %s
     MESSAGE
@@ -53,6 +35,32 @@ class Config
       mutant.yml
     ].freeze
 
+    private_constant(*constants(false))
+
+    class CoverageCriteria
+      include Anima.new(:timeout, :test_result)
+
+      DEFAULT = new(
+        timeout:     false,
+        test_result: true
+      )
+
+      TRANSFORM =
+        Transform::Sequence.new(
+          [
+            Transform::Hash.new(
+              optional: [
+                Transform::Hash::Key.new('timeout',     Transform::BOOLEAN),
+                Transform::Hash::Key.new('test_result', Transform::BOOLEAN)
+              ],
+              required: []
+            ),
+            Transform::Hash::Symbolize.new,
+            ->(value) { Either::Right.new(DEFAULT.with(**value)) }
+          ]
+        )
+    end # CoverageCriteria
+
     # Merge with other config
     #
     # @param [Config] other
@@ -71,8 +79,6 @@ def merge(other)
       )
     end
 
-    private_constant(*constants(false))
-
     # Load config file
     #
     # @param [World] world
@@ -106,5 +112,27 @@ def self.load_contents(path)
     def self.env
       DEFAULT.with(jobs: Etc.nprocessors)
     end
+
+    TRANSFORM = Transform::Sequence.new(
+      [
+        Transform::Exception.new(SystemCallError, :read.to_proc),
+        Transform::Exception.new(YAML::SyntaxError, YAML.method(:safe_load)),
+        Transform::Hash.new(
+          optional: [
+            Transform::Hash::Key.new('coverage_criteria', CoverageCriteria::TRANSFORM),
+            Transform::Hash::Key.new('fail_fast',         Transform::BOOLEAN),
+            Transform::Hash::Key.new('includes',          Transform::STRING_ARRAY),
+            Transform::Hash::Key.new('integration',       Transform::STRING),
+            Transform::Hash::Key.new('jobs',              Transform::INTEGER),
+            Transform::Hash::Key.new('mutation_timeout',  Transform::FLOAT),
+            Transform::Hash::Key.new('requires',          Transform::STRING_ARRAY)
+          ],
+          required: []
+        ),
+        Transform::Hash::Symbolize.new
+      ]
+    )
+
+    private_constant(:TRANSFORM)
   end # Config
 end # Mutant

lib/mutant/isolation.rb

@@ -18,7 +18,7 @@ class Result
       # Test for successful result
       #
       # @return [Boolean]
-      def success?
+      def valid_value?
         timeout.nil? && exception.nil? && (process_status.nil? || process_status.success?)
       end
     end # Result

lib/mutant/reporter/cli/printer/coverage_result.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Mutant
+  class Reporter
+    class CLI
+      class Printer
+        # Reporter for mutation coverage results
+        class CoverageResult < self
+          # Run report printer
+          #
+          # @return [undefined]
+          def run
+            visit(MutationResult, object.mutation_result)
+          end
+        end # Printer
+      end # Coverage
+    end # CLI
+  end # Reporter
+end # Mutant

lib/mutant/reporter/cli/printer/env_progress.rb

@@ -10,6 +10,7 @@ class EnvProgress < self
             :amount_mutation_results,
             :amount_mutations_alive,
             :amount_mutations_killed,
+            :amount_timeouts,
             :coverage,
             :env,
             :killtime,
@@ -21,6 +22,7 @@ class EnvProgress < self
             [:info,   'Results:         %s',      :amount_mutation_results],
             [:info,   'Kills:           %s',      :amount_mutations_killed],
             [:info,   'Alive:           %s',      :amount_mutations_alive ],
+            [:info,   'Timeouts:        %s',      :amount_timeouts        ],
             [:info,   'Runtime:         %0.2fs',  :runtime                ],
             [:info,   'Killtime:        %0.2fs',  :killtime               ],
             [:info,   'Overhead:        %0.2f%%', :overhead_percent       ],

lib/mutant/reporter/cli/printer/subject_result.rb

@@ -7,7 +7,7 @@ class Printer
         # Subject result printer
         class SubjectResult < self
 
-          delegate :subject, :alive_mutation_results, :tests
+          delegate :subject, :uncovered_results, :tests
 
           # Run report printer
           #
@@ -17,7 +17,7 @@ def run
             tests.each do |test|
               puts("- #{test.identification}")
             end
-            visit_collection(MutationResult, alive_mutation_results)
+            visit_collection(CoverageResult, uncovered_results)
           end
 
         end # SubjectResult