Add no_op_evaluator function (#282)

kolena/workflow/__init__.py

@@ -47,6 +47,7 @@
 from .evaluator_function import BasicEvaluatorFunction
 from .evaluator_function import TestCases
 from .evaluator_function import EvaluationResults
+from .evaluator_function import no_op_evaluator
 from .test_run import TestRun
 from .test_run import test
 from .define_workflow import define_workflow
@@ -86,6 +87,7 @@
     "BasicEvaluatorFunction",
     "TestCases",
     "EvaluationResults",
+    "no_op_evaluator",
     "TestRun",
     "test",
     "define_workflow",

kolena/workflow/evaluator_function.py

@@ -246,3 +246,31 @@ def _test_sample_key(ts: TestSample) -> str:
 def _is_configured(evaluator: BasicEvaluatorFunction) -> bool:
     param_values = list(signature(evaluator).parameters.values())
     return len(param_values) == 5 and issubclass(param_values[4].annotation, EvaluatorConfiguration)
+
+
+def no_op_evaluator(
+    test_samples: List[TestSample],
+    ground_truths: List[GroundTruth],
+    inferences: List[Inference],
+    test_cases: TestCases,
+) -> EvaluationResults:
+    """
+    A no-op implementation of the Kolena [`Evaluator`][kolena.workflow.Evaluator] that will bypass evaluation but
+    make [`Inference`][kolena.workflow.Inference]s accessible in the platform.
+
+    ```python
+    from kolena.workflow import no_op_evaluator
+    from kolena.workflow import test
+
+    test(model, test_suite, no_op_evaluator)
+    ```
+    """
+    test_sample_metrics = [BaseMetricsTestSample() for _ in test_samples]
+    test_case_metrics = [
+        (tc, MetricsTestCase())
+        for tc, *_ in test_cases.iter(test_samples, ground_truths, inferences, test_sample_metrics)
+    ]
+    return EvaluationResults(
+        metrics_test_sample=list(zip(test_samples, test_sample_metrics)),
+        metrics_test_case=test_case_metrics,
+    )

tests/integration/workflow/test_test_run.py

@@ -35,6 +35,7 @@
 from kolena.workflow import Inference
 from kolena.workflow import MetricsTestCase
 from kolena.workflow import MetricsTestSample
+from kolena.workflow import no_op_evaluator
 from kolena.workflow import test
 from kolena.workflow import TestCase
 from kolena.workflow import TestCases
@@ -237,7 +238,6 @@ class MarkCrashedDummyEvaluator(DummyEvaluator):
 def test__evaluator__unconfigured(
     dummy_model: Model,
     dummy_test_suites: List[TestSuite],
-    dummy_test_samples: List[DummyTestSample],
 ) -> None:
     class UnconfiguredDummyEvaluator(DummyEvaluator):
         def compute_test_suite_metrics(
@@ -351,16 +351,22 @@ def compute_test_case_metrics(
 def test__test__function_evaluator(
     dummy_model: Model,
     dummy_test_suites: List[TestSuite],
-    dummy_test_samples: List[DummyTestSample],
 ) -> None:
     test(dummy_model, dummy_test_suites[0], dummy_evaluator_function)
     TestRun(dummy_model, dummy_test_suites[0], dummy_evaluator_function)
 
 
+def test__test__noop_evaluator(
+    dummy_model: Model,
+    dummy_test_suites: List[TestSuite],
+) -> None:
+    test(dummy_model, dummy_test_suites[0], no_op_evaluator)
+    TestRun(dummy_model, dummy_test_suites[0], no_op_evaluator)
+
+
 def test__test__function_evaluator__with_skip(
     dummy_model: Model,
     dummy_test_suites: List[TestSuite],
-    dummy_test_samples: List[DummyTestSample],
 ) -> None:
     config = [DummyConfiguration(value="skip")]
     test(dummy_model, dummy_test_suites[0], dummy_evaluator_function_with_config, config)