Fix acceptance test and snapshots

With EC-224 and the change in [1] we time-bombed the acceptance tests. This modifies the scenario slightly so a different rule, one emitting a warning, is used. I believe the original intent of the test is maintained. [1] enterprise-contract/ec-policies#793
zregvart · Jan 11, 2024 · e3c03a7 · e3c03a7
1 parent 1aa98b8
commit e3c03a7
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 24 deletions.
diff --git a/features/__snapshots__/task_validate_image.snap b/features/__snapshots__/task_validate_image.snap
@@ -81,27 +81,30 @@ components:
       description: The image signature matches available signing materials.
       title: Image signature check passed
     msg: Pass
+  warnings:
   - metadata:
-      code: test.no_skipped_tests
+      code: test.no_test_warnings
       collections:
       - redhat
       depends_on:
       - test.test_data_found
-      description: Produce a violation if any tests have their result set to "SKIPPED".
-        A skipped result means a pre-requirement for executing the test was not met,
-        e.g. a license key for executing a scanner was not provided. The result type
-        is configurable by the "skipped_tests_results" key in the rule data.
-      effective_on: "${TIMESTAMP}"
-      title: No tests were skipped
-    msg: Pass
+      description: Produce a warning if any tests have their result set to "WARNING".
+        The result type is configurable by the "warned_tests_results" key in the rule
+        data.
+      solution: There is a task with result 'TEST_OUTPUT' that returned a result of
+        'WARNING'. You can find which test resulted in 'WARNING' by examining the
+        'result' key in the 'TEST_OUTPUT'.
+      term: <NAMELESS>
+      title: No tests produced warnings
+    msg: Test "<NAMELESS>" returned a warning
 ec-version: ${EC_VERSION}
 effective-time: "${TIMESTAMP}"
 key: |
 ${__known_PUBLIC_KEY}
 policy:
   configuration:
     include:
-    - test.no_skipped_tests
+    - test.no_test_warnings
   publicKey: |
 ${____known_PUBLIC_KEY}
   sources:
@@ -143,27 +146,30 @@ components:
       description: The image signature matches available signing materials.
       title: Image signature check passed
     msg: Pass
+  warnings:
   - metadata:
-      code: test.no_skipped_tests
+      code: test.no_test_warnings
       collections:
       - redhat
       depends_on:
       - test.test_data_found
-      description: Produce a violation if any tests have their result set to "SKIPPED".
-        A skipped result means a pre-requirement for executing the test was not met,
-        e.g. a license key for executing a scanner was not provided. The result type
-        is configurable by the "skipped_tests_results" key in the rule data.
-      effective_on: "${TIMESTAMP}"
-      title: No tests were skipped
-    msg: Pass
+      description: Produce a warning if any tests have their result set to "WARNING".
+        The result type is configurable by the "warned_tests_results" key in the rule
+        data.
+      solution: There is a task with result 'TEST_OUTPUT' that returned a result of
+        'WARNING'. You can find which test resulted in 'WARNING' by examining the
+        'result' key in the 'TEST_OUTPUT'.
+      term: <NAMELESS>
+      title: No tests produced warnings
+    msg: Test "<NAMELESS>" returned a warning
 ec-version: ${EC_VERSION}
 effective-time: "${TIMESTAMP}"
 key: |
 ${__known_PUBLIC_KEY}
 policy:
   configuration:
     include:
-    - test.no_skipped_tests
+    - test.no_test_warnings
   publicKey: |
 ${____known_PUBLIC_KEY}
   sources:
@@ -388,13 +394,13 @@ TUF_MIRROR not set. Skipping TUF root initialization.
 
 [Strict with warnings:results - 1]
 {
-  "TEST_OUTPUT": "{\"timestamp\":\"${TIMESTAMP}\",\"namespace\":\"\",\"successes\":4,\"failures\":0,\"warnings\":0,\"result\":\"SUCCESS\"}\n"
+  "TEST_OUTPUT": "{\"timestamp\":\"${TIMESTAMP}\",\"namespace\":\"\",\"successes\":3,\"failures\":0,\"warnings\":1,\"result\":\"WARNING\"}\n"
 }
 ---
 
 [Non strict with warnings:results - 1]
 {
-  "TEST_OUTPUT": "{\"timestamp\":\"${TIMESTAMP}\",\"namespace\":\"\",\"successes\":4,\"failures\":0,\"warnings\":0,\"result\":\"SUCCESS\"}\n"
+  "TEST_OUTPUT": "{\"timestamp\":\"${TIMESTAMP}\",\"namespace\":\"\",\"successes\":3,\"failures\":0,\"warnings\":1,\"result\":\"WARNING\"}\n"
 }
 ---
 

diff --git a/features/task_validate_image.feature b/features/task_validate_image.feature
@@ -102,7 +102,7 @@ Feature: Verify Enterprise Contract Tekton Tasks
       And an image named "acceptance/non-strict-with-warnings"
       And a valid image signature of "acceptance/non-strict-with-warnings" image signed by the "known" key
       And a valid attestation of "acceptance/non-strict-with-warnings" signed by the "known" key, patched with
-      | [{"op": "add", "path": "/predicate/buildConfig", "value": {}},{"op": "add", "path": "/predicate/buildConfig/tasks", "value": [{"name":"skipped","results":[{"name":"TEST_OUTPUT","type":"string","value":"{\"result\":\"SKIPPED\"}"}]}]}] |
+      | [{"op": "add", "path": "/predicate/buildConfig", "value": {}},{"op": "add", "path": "/predicate/buildConfig/tasks", "value": [{"name":"skipped","results":[{"name":"TEST_OUTPUT","type":"string","value":"{\"result\":\"WARNING\"}"}]}]}] |
       And a cluster policy with content:
       ```
       {
@@ -116,7 +116,7 @@ Feature: Verify Enterprise Contract Tekton Tasks
         ],
         "configuration": {
           "include": [
-            "test.no_skipped_tests"
+            "test.no_test_warnings"
           ]
         }
       }
@@ -136,7 +136,7 @@ Feature: Verify Enterprise Contract Tekton Tasks
       And an image named "acceptance/strict-with-warnings"
       And a valid image signature of "acceptance/strict-with-warnings" image signed by the "known" key
       And a valid attestation of "acceptance/strict-with-warnings" signed by the "known" key, patched with
-      | [{"op": "add", "path": "/predicate/buildConfig", "value": {}},{"op": "add", "path": "/predicate/buildConfig/tasks", "value": [{"name":"skipped","results":[{"name":"TEST_OUTPUT","type":"string","value":"{\"result\":\"SKIPPED\"}"}]}]}] |
+      | [{"op": "add", "path": "/predicate/buildConfig", "value": {}},{"op": "add", "path": "/predicate/buildConfig/tasks", "value": [{"name":"skipped","results":[{"name":"TEST_OUTPUT","type":"string","value":"{\"result\":\"WARNING\"}"}]}]}] |
       And a cluster policy with content:
       ```
       {
@@ -150,7 +150,7 @@ Feature: Verify Enterprise Contract Tekton Tasks
         ],
         "configuration": {
           "include": [
-            "test.no_skipped_tests"
+            "test.no_test_warnings"
           ]
         }
       }