From 282056b373ae0c008b1fcc2c2ee19d1df5e163fb Mon Sep 17 00:00:00 2001 From: Binbin Li Date: Tue, 6 Jun 2023 11:31:08 +0800 Subject: [PATCH] test: switch to splitted bats test (#870) --- Makefile | 6 +- test/bats/test.bats | 445 -------------------------------------------- 2 files changed, 4 insertions(+), 447 deletions(-) delete mode 100644 test/bats/test.bats diff --git a/Makefile b/Makefile index 8f571d705..03596402f 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,8 @@ NOTATION_VERSION ?= 1.0.0-rc.3 ORAS_VERSION ?= 1.0.0-rc.2 HELM_VERSION ?= 3.9.2 -BATS_TESTS_FILE ?= test/bats/test.bats +BATS_BASE_TESTS_FILE ?= test/bats/base-test.bats +BATS_PLUGIN_TESTS_FILE ?= test/bats/plugin-test.bats BATS_CLI_TESTS_FILE ?= test/bats/cli-test.bats BATS_VERSION ?= 1.7.0 SYFT_VERSION ?= v0.76.0 @@ -121,7 +122,8 @@ delete-gatekeeper: .PHONY: test-e2e test-e2e: generate-rotation-certs - bats -t ${BATS_TESTS_FILE} + bats -t ${BATS_BASE_TESTS_FILE} + bats -t ${BATS_PLUGIN_TESTS_FILE} .PHONY: test-e2e-cli test-e2e-cli: e2e-dependencies e2e-create-local-registry e2e-notaryv2-setup e2e-notation-leaf-cert-setup e2e-cosign-setup e2e-licensechecker-setup e2e-sbom-setup e2e-schemavalidator-setup diff --git a/test/bats/test.bats b/test/bats/test.bats deleted file mode 100644 index 75c74510e..000000000 --- a/test/bats/test.bats +++ /dev/null @@ -1,445 +0,0 @@ -#!/usr/bin/env bats - -load helpers - -BATS_TESTS_DIR=${BATS_TESTS_DIR:-test/bats/tests} -WAIT_TIME=60 -SLEEP_TIME=1 - -@test "crd version test" { - run kubectl delete verifiers.config.ratify.deislabs.io/verifier-notary - assert_success - run kubectl apply -f ./config/samples/config_v1alpha1_verifier_notary.yaml - assert_success - run bash -c "kubectl get verifiers.config.ratify.deislabs.io/verifier-notary -o yaml | grep 'apiVersion: config.ratify.deislabs.io/v1beta1'" - assert_success - - run kubectl delete stores.config.ratify.deislabs.io/store-oras - assert_success - run kubectl apply -f ./config/samples/config_v1alpha1_store_oras_http.yaml - assert_success - run bash -c "kubectl get stores.config.ratify.deislabs.io/store-oras -o yaml | grep 'apiVersion: config.ratify.deislabs.io/v1beta1'" - assert_success -} - -@test "notary test" { - teardown() { - echo "cleaning up" - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod demo --namespace default --force --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod demo-oci-image --namespace default --force --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod demo1 --namespace default --force --ignore-not-found=true' - } - run kubectl apply -f ./library/default/template.yaml - assert_success - sleep 5 - run kubectl apply -f ./library/default/samples/constraint.yaml - assert_success - sleep 5 - # validate certificate store status property shows success - run bash -c "kubectl get certificatestores.config.ratify.deislabs.io/ratify-notary-inline-cert -n gatekeeper-system -o yaml | grep 'issuccess: true'" - assert_success - run kubectl run demo --namespace default --image=registry:5000/notation:signed - assert_success - # notation signature with OCI Artifact manifest format - if [[ $IS_OCI_1_1 == "true" ]]; then - run kubectl run demo-oci-image --namespace default --image=registry:5000/notation:ociartifact - assert_success - fi - run kubectl run demo1 --namespace default --image=registry:5000/notation:unsigned - assert_failure -} - -@test "cosign test" { - teardown() { - echo "cleaning up" - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod cosign-demo-key --namespace default --force --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod cosign-demo-unsigned --namespace default --force --ignore-not-found=true' - } - run kubectl apply -f ./library/default/template.yaml - assert_success - sleep 5 - run kubectl apply -f ./library/default/samples/constraint.yaml - assert_success - sleep 5 - - run kubectl run cosign-demo-key --namespace default --image=registry:5000/cosign:signed-key - assert_success - - run kubectl run cosign-demo-unsigned --namespace default --image=registry:5000/cosign:unsigned - assert_failure -} - -@test "cosign keyless test" { - teardown() { - echo "cleaning up" - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod cosign-demo-keyless --namespace default --force --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete verifiers.config.ratify.deislabs.io/verifier-cosign --namespace default --ignore-not-found=true' - } - - # use imperative command to guarantee useHttp is updated - run kubectl replace -f ./config/samples/config_v1beta1_verifier_cosign_keyless.yaml - sleep 5 - - run kubectl replace -f ./config/samples/config_v1beta1_store_oras.yaml - sleep 5 - - run kubectl run cosign-demo-keyless --namespace default --image=wabbitnetworks.azurecr.io/test/cosign-image:signed-keyless - assert_success - - run kubectl replace -f ./config/samples/config_v1beta1_verifier_cosign.yaml - sleep 5 - run kubectl replace -f ./config/samples/config_v1beta1_store_oras_http.yaml -} - -@test "licensechecker test" { - teardown() { - echo "cleaning up" - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod license-checker --namespace default --force --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod license-checker2 --namespace default --force --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod license-checker-oci-image --namespace default --force --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete verifiers.config.ratify.deislabs.io/verifier-license-checker --namespace default --ignore-not-found=true' - } - - run kubectl apply -f ./library/default/template.yaml - assert_success - sleep 5 - run kubectl apply -f ./library/default/samples/constraint.yaml - assert_success - sleep 5 - run kubectl apply -f ./config/samples/config_v1beta1_verifier_partial_licensechecker.yaml - sleep 5 - run kubectl run license-checker --namespace default --image=registry:5000/licensechecker:v0 - assert_failure - - run kubectl apply -f ./config/samples/config_v1beta1_verifier_complete_licensechecker.yaml - # wait for the httpserver cache to be invalidated - sleep 15 - run kubectl run license-checker2 --namespace default --image=registry:5000/licensechecker:v0 - assert_success - # licensechecker artifact with OCI Artifact manifest format - if [[ $IS_OCI_1_1 == "true" ]]; then - run kubectl run license-checker-oci-image --namespace default --image=registry:5000/licensechecker:ociartifact - assert_success - fi -} - -@test "sbom verifier test" { - teardown() { - echo "cleaning up" - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod sbom --namespace default --force --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod sbom-oci-image --namespace default --force --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod sbom2 --namespace default --force --ignore-not-found=true' - } - - run kubectl apply -f ./library/default/template.yaml - assert_success - sleep 5 - run kubectl apply -f ./library/default/samples/constraint.yaml - assert_success - sleep 5 - - run kubectl apply -f ./config/samples/config_v1beta1_verifier_sbom.yaml - sleep 5 - run kubectl run sbom --namespace default --image=registry:5000/sbom:v0 - assert_success - # sbom with OCI Artifact manifest format - if [[ $IS_OCI_1_1 == "true" ]]; then - run kubectl run sbom-oci-image --namespace default --image=registry:5000/sbom:ociartifact - assert_success - fi - - run kubectl delete verifiers.config.ratify.deislabs.io/verifier-sbom - assert_success - # wait for the httpserver cache to be invalidated - sleep 15 - run kubectl run sbom2 --namespace default --image=registry:5000/sbom:v0 - assert_failure -} - -@test "schemavalidator verifier test" { - skip "Skipping test for now until expected usage/configuration of this plugin can be verified" - teardown() { - echo "cleaning up" - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete verifiers.config.ratify.deislabs.io/verifier-license-checker --namespace default --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete verifiers.config.ratify.deislabs.io/verifier-sbom --namespace default --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete verifiers.config.ratify.deislabs.io/verifier-schemavalidator --namespace default --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod schemavalidator --namespace default --force --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod schemavalidator-oci-image --namespace default --force --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod schemavalidator2 --namespace default --force --ignore-not-found=true' - } - - run kubectl apply -f ./library/default/template.yaml - assert_success - sleep 5 - run kubectl apply -f ./library/default/samples/constraint.yaml - assert_success - sleep 5 - - run kubectl apply -f ./config/samples/config_v1beta1_verifier_schemavalidator.yaml - sleep 5 - run kubectl run schemavalidator --namespace default --image=registry:5000/schemavalidator:v0 - assert_success - # schemavalidator with OCI Artifact manifest format - if [[ $IS_OCI_1_1 == "true" ]]; then - run kubectl run schemavalidator-oci-image --namespace default --image=registry:5000/schemavalidator:ociartifact - assert_success - fi - - run kubectl apply -f ./config/samples/config_v1beta1_verifier_schemavalidator_bad.yaml - assert_success - # wait for the httpserver cache to be invalidated - sleep 15 - run kubectl run schemavalidator2 --namespace default --image=registry:5000/schemavalidator:v0 - assert_failure -} - -@test "sbom/notary/cosign/licensechecker/schemavalidator verifiers test" { - teardown() { - echo "cleaning up" - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete verifiers.config.ratify.deislabs.io/verifier-license-checker --namespace default --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete verifiers.config.ratify.deislabs.io/verifier-sbom --namespace default --ignore-not-found=true' - # Skipping test for now until expected usage/configuration of this plugin can be verified - # wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete verifiers.config.ratify.deislabs.io/verifier-schemavalidator --namespace default --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod all-in-one --namespace default --force --ignore-not-found=true' - } - - run kubectl apply -f ./library/default/template.yaml - assert_success - sleep 5 - run kubectl apply -f ./library/default/samples/constraint.yaml - assert_success - sleep 5 - - run kubectl apply -f ./config/samples/config_v1beta1_verifier_cosign.yaml - sleep 5 - run kubectl apply -f ./config/samples/config_v1beta1_verifier_sbom.yaml - sleep 5 - run kubectl apply -f ./config/samples/config_v1beta1_verifier_complete_licensechecker.yaml - - # Skipping test for now until expected usage/configuration of this plugin can be verified - # sleep 5 - # run kubectl apply -f ./config/samples/config_v1beta1_verifier_schemavalidator.yaml - # sleep 5 - - # wait for the httpserver cache to be invalidated - sleep 15 - run kubectl run all-in-one --namespace default --image=registry:5000/all:v0 - assert_success -} - -@test "validate crd add, replace and delete" { - teardown() { - echo "cleaning up" - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod crdtest --namespace default --force --ignore-not-found=true' - } - - echo "adding license checker, delete notary verifier and validate deployment fails due to missing notary verifier" - run kubectl apply -f ./config/samples/config_v1beta1_verifier_complete_licensechecker.yaml - assert_success - run kubectl delete verifiers.config.ratify.deislabs.io/verifier-notary - assert_success - # wait for the httpserver cache to be invalidated - sleep 15 - run kubectl run crdtest --namespace default --image=registry:5000/notation:signed - assert_failure - - echo "Add notary verifier and validate deployment succeeds" - run kubectl apply -f ./config/samples/config_v1beta1_verifier_notary.yaml - assert_success - - # wait for the httpserver cache to be invalidated - sleep 15 - run kubectl run crdtest --namespace default --image=registry:5000/notation:signed - assert_success -} - -@test "configmap update test" { - skip "Skipping test for now as we are no longer watching for configfile update in a k8 environment. This test ensures we are watching config file updates in a non-kub scenario" - run kubectl apply -f ./library/default/template.yaml - assert_success - sleep 5 - run kubectl apply -f ./library/default/samples/constraint.yaml - assert_success - sleep 5 - run kubectl run demo2 --image=registry:5000/notation:signed - assert_success - - run kubectl get configmaps ratify-configuration --namespace=gatekeeper-system -o yaml >currentConfig.yaml - run kubectl delete -f ./library/default/samples/constraint.yaml - - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl replace --namespace=gatekeeper-system -f ${BATS_TESTS_DIR}/configmap/invalidconfigmap.yaml" - echo "Waiting for 150 second for configuration update" - sleep 150 - - run kubectl apply -f ./library/default/samples/constraint.yaml - assert_success - run kubectl run demo3 --image=registry:5000/notation:signed - echo "Current time after validate : $(date +"%T")" - assert_failure - - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl replace --namespace=gatekeeper-system -f currentConfig.yaml" -} - -@test "dynamic plugins disabled test" { - teardown() { - echo "cleaning up" - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete verifiers.config.ratify.deislabs.io/verifier-dynamic --namespace default --ignore-not-found=true' - } - - start=$(date --iso-8601=seconds) - latestpod=$(kubectl -n gatekeeper-system get pod -l=app.kubernetes.io/name=ratify --sort-by=.metadata.creationTimestamp -o=name | tail -n 1) - - run kubectl apply -f ./config/samples/config_v1beta1_verifier_dynamic.yaml - sleep 5 - - run bash -c "kubectl -n gatekeeper-system logs $latestpod --since-time=$start | grep 'dynamic plugins are currently disabled'" - assert_success -} - -@test "validate mutation tag to digest" { - teardown() { - echo "cleaning up" - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod mutate-demo --namespace default --ignore-not-found=true' - } - - run kubectl apply -f ./library/default/template.yaml - assert_success - sleep 5 - run kubectl apply -f ./library/default/samples/constraint.yaml - assert_success - sleep 5 - run kubectl run mutate-demo --namespace default --image=registry:5000/notation:signed - assert_success - result=$(kubectl get pod mutate-demo --namespace default -o json | jq -r ".spec.containers[0].image" | grep @sha) - assert_mutate_success -} - -@test "validate inline cert provider" { - teardown() { - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete certificatestores.config.ratify.deislabs.io/certstore-inline --namespace default --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod demo-alternate --namespace default --force --ignore-not-found=true' - - # restore the original notary verifier for other tests - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl apply -f ./config/samples/config_v1beta1_verifier_notary.yaml' - } - - # configure the default template/constraint - run kubectl apply -f ./library/default/template.yaml - assert_success - run kubectl apply -f ./library/default/samples/constraint.yaml - assert_success - - # verify that the image cannot be run due to an invalid cert - run kubectl run demo-alternate --namespace default --image=registry:5000/notation:signed-alternate - assert_failure - - # add the alternate certificate as an inline certificate store - cat ~/.config/notation/truststore/x509/ca/alternate-cert/alternate-cert.crt | sed 's/^/ /g' >>./test/bats/tests/config/config_v1beta1_certstore_inline.yaml - run kubectl apply -f ./test/bats/tests/config/config_v1beta1_certstore_inline.yaml - assert_success - sed -i '9,$d' ./test/bats/tests/config/config_v1beta1_certstore_inline.yaml - - # configure the notary verifier to use the inline certificate store - run kubectl apply -f ./test/bats/tests/config/config_v1beta1_verifier_notary.yaml - assert_success - sleep 10 - - # verify that the image can now be run - run kubectl run demo-alternate --namespace default --image=registry:5000/notation:signed-alternate - assert_success -} - -@test "validate k8 secrets ORAS auth provider" { - teardown() { - echo "cleaning up" - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod demo --namespace default --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod demo1 --namespace default --ignore-not-found=true' - } - - run kubectl apply -f ./library/default/template.yaml - assert_success - sleep 5 - run kubectl apply -f ./library/default/samples/constraint.yaml - assert_success - sleep 5 - # apply store CRD with k8 secret auth provier enabled - run kubectl apply -f ./config/samples/config_v1beta1_store_oras_k8secretAuth.yaml - assert_success - sleep 5 - run kubectl run demo --namespace default --image=registry:5000/notation:signed - assert_success - run kubectl run demo1 --namespace default --image=registry:5000/notation:unsigned - assert_failure -} - -@test "validate image signed by leaf cert" { - teardown() { - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete certificatestores.config.ratify.deislabs.io/certstore-inline --namespace default --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod demo-leaf --namespace default --force --ignore-not-found=true' - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod demo-leaf2 --namespace default --force --ignore-not-found=true' - - # restore the original notary verifier for other tests - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl apply -f ./config/samples/config_v1beta1_verifier_notary.yaml' - } - - # configure the default template/constraint - run kubectl apply -f ./library/default/template.yaml - assert_success - run kubectl apply -f ./library/default/samples/constraint.yaml - assert_success - - # add the root certificate as an inline certificate store - cat ~/.config/notation/truststore/x509/ca/leaf-test/root.crt | sed 's/^/ /g' >>./test/bats/tests/config/config_v1beta1_certstore_inline.yaml - run kubectl apply -f ./test/bats/tests/config/config_v1beta1_certstore_inline.yaml - assert_success - sed -i '9,$d' ./test/bats/tests/config/config_v1beta1_certstore_inline.yaml - - # configure the notary verifier to use the inline certificate store - run kubectl apply -f ./test/bats/tests/config/config_v1beta1_verifier_notary.yaml - assert_success - - # verify that the image can be run with a root cert - run kubectl run demo-leaf --namespace default --image=registry:5000/notation:leafSigned - assert_success - - # add the root certificate as an inline certificate store - cat ~/.config/notation/truststore/x509/ca/leaf-test/leaf.crt | sed 's/^/ /g' >>./test/bats/tests/config/config_v1beta1_certstore_inline.yaml - run kubectl apply -f ./test/bats/tests/config/config_v1beta1_certstore_inline.yaml - assert_success - sed -i '9,$d' ./test/bats/tests/config/config_v1beta1_certstore_inline.yaml - - # wait for the httpserver cache to be invalidated - sleep 15 - # verify that the image cannot be run with a leaf cert - run kubectl run demo-leaf2 --namespace default --image=registry:5000/notation:leafSigned - assert_failure -} - -@test "validate ratify/gatekeeper tls cert rotation" { - teardown() { - wait_for_process ${WAIT_TIME} ${SLEEP_TIME} 'kubectl delete pod demo --namespace default --force --ignore-not-found=true' - } - - # update Providers to use the new CA - run kubectl get Provider ratify-mutation-provider -o json | jq --arg ca "$(cat .staging/rotation/ca.crt | base64)" '.spec.caBundle=$ca' | kubectl replace -f - - run kubectl get Provider ratify-provider -o json | jq --arg ca "$(cat .staging/rotation/ca.crt | base64)" '.spec.caBundle=$ca' | kubectl replace -f - - - # update the ratify tls secret to use the new tls cert and key - run kubectl get secret ratify-tls -n gatekeeper-system -o json | jq --arg cert "$(cat .staging/rotation/server.crt | base64)" --arg key "$(cat .staging/rotation/server.key | base64)" '.data["tls.key"]=$key | .data["tls.crt"]=$cert'| kubectl replace -f - - - # update the gatekeeper webhook server tls secret to use the new cert bundle - run kubectl get Secret gatekeeper-webhook-server-cert -n gatekeeper-system -o json | jq --arg caCert "$(cat .staging/rotation/gatekeeper/ca.crt | base64)" --arg caKey "$(cat .staging/rotation/gatekeeper/ca.key | base64)" --arg tlsCert "$(cat .staging/rotation/gatekeeper/server.crt | base64)" --arg tlsKey "$(cat .staging/rotation/gatekeeper/server.key | base64)" '.data["ca.crt"]=$caCert | .data["ca.key"]=$caKey | .data["tls.crt"]=$tlsCert | .data["tls.key"]=$tlsKey' | kubectl replace -f - - - # volume projection can take up to 90 seconds - sleep 100 - - # verify that the verification succeeds - run kubectl apply -f ./library/default/template.yaml - assert_success - sleep 5 - run kubectl apply -f ./library/default/samples/constraint.yaml - assert_success - sleep 5 - run kubectl run demo --namespace default --image=registry:5000/notation:signed - assert_success -}