feat: add cert rotator (ratify-project#869)

Signed-off-by: Binbin Li <libinbin@microsoft.com>

.github/workflows/build-pr.yml

@@ -90,12 +90,13 @@ jobs:
     name: "Build and run e2e Test"
     runs-on: ubuntu-latest
     timeout-minutes: 25
+    continue-on-error: true
     permissions:
       contents: read
     strategy:
       matrix:
         KUBERNETES_VERSION: ["1.25.8", "1.26.3"]
-        GATEKEEPER_VERSION: ["3.10.0", "3.11.0"]
+        GATEKEEPER_VERSION: ["3.11.0", "3.12.0"]
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
@@ -114,7 +115,7 @@ jobs:
         run: |
           make e2e-deploy-gatekeeper GATEKEEPER_VERSION=${{ matrix.GATEKEEPER_VERSION }}
           make e2e-deploy-ratify GATEKEEPER_VERSION=${{ matrix.GATEKEEPER_VERSION }}
-          make test-e2e
+          make test-e2e GATEKEEPER_VERSION=${{ matrix.GATEKEEPER_VERSION }}
       - name: Save logs
         if: ${{ always() }}
         run: |
@@ -146,7 +147,7 @@ jobs:
     strategy:
       matrix:
         KUBERNETES_VERSION: ["1.24.10", "1.25.6"]
-        GATEKEEPER_VERSION: ["3.10.0", "3.11.0"]
+        GATEKEEPER_VERSION: ["3.11.0", "3.12.0"]
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3

Makefile

@@ -27,6 +27,7 @@ BATS_CLI_TESTS_FILE ?= test/bats/cli-test.bats
 BATS_VERSION ?= 1.7.0
 SYFT_VERSION ?= v0.76.0
 ALPINE_IMAGE ?= alpine@sha256:93d5a28ff72d288d69b5997b8ba47396d2cbb62a72b5d87cd3351094b5d578a0
+CERT_ROTATION_ENABLED ?= false
 
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.24.2
@@ -123,7 +124,7 @@ delete-gatekeeper:
 
 .PHONY: test-e2e
 test-e2e: generate-rotation-certs
-	bats -t ${BATS_BASE_TESTS_FILE}
+	EXPIRING_CERT_DIR=.staging/rotation/expiring-certs GATEKEEPER_VERSION=${GATEKEEPER_VERSION} bats -t ${BATS_BASE_TESTS_FILE}
 	bats -t ${BATS_PLUGIN_TESTS_FILE}
 
 .PHONY: test-e2e-cli
@@ -140,9 +141,11 @@ generate-certs:
 generate-rotation-certs:
 	mkdir -p .staging/rotation
 	mkdir -p .staging/rotation/gatekeeper
+	mkdir -p .staging/rotation/expiring-certs
 
 	./scripts/generate-gk-tls-certs.sh .staging/rotation/gatekeeper ${GATEKEEPER_NAMESPACE}
 	./scripts/generate-tls-certs.sh .staging/rotation ${GATEKEEPER_NAMESPACE}
+	./scripts/generate-tls-certs.sh .staging/rotation/expiring-certs ${GATEKEEPER_NAMESPACE} 1
 
 install-bats:
 	# Download and install bats
@@ -451,8 +454,11 @@ e2e-deploy-base-ratify: e2e-notaryv2-setup e2e-notation-leaf-cert-setup e2e-inli
 	--set image.crdRepository=localbuildcrd \
 	--set image.tag=test \
 	--set gatekeeper.version=${GATEKEEPER_VERSION} \
+	--set featureFlags.RATIFY_CERT_ROTATION=${CERT_ROTATION_ENABLED} \
 	--set-file provider.tls.crt=${CERT_DIR}/server.crt \
 	--set-file provider.tls.key=${CERT_DIR}/server.key \
+	--set-file provider.tls.caCert=${CERT_DIR}/ca.crt \
+    --set-file provider.tls.caKey=${CERT_DIR}/ca.key \
 	--set provider.tls.cabundle="$(shell cat ${CERT_DIR}/ca.crt | base64 | tr -d '\n')" \
 	--set notaryCert="$$(cat ~/.config/notation/localkeys/ratify-bats-test.crt)" \
 	--set oras.useHttp=true \
@@ -462,7 +468,9 @@ e2e-deploy-base-ratify: e2e-notaryv2-setup e2e-notation-leaf-cert-setup e2e-inli
 
 	rm mount_config.json
 
-e2e-deploy-ratify: e2e-notaryv2-setup e2e-notation-leaf-cert-setup e2e-cosign-setup e2e-cosign-setup e2e-licensechecker-setup e2e-sbom-setup e2e-schemavalidator-setup e2e-inlinecert-setup e2e-build-crd-image
+e2e-deploy-ratify: e2e-notaryv2-setup e2e-notation-leaf-cert-setup e2e-cosign-setup e2e-cosign-setup e2e-licensechecker-setup e2e-sbom-setup e2e-schemavalidator-setup e2e-inlinecert-setup e2e-build-crd-image e2e-build-local-ratify-image e2e-helm-deploy-ratify
+
+e2e-build-local-ratify-image:
 	docker build --progress=plain --no-cache \
 	--build-arg build_cosign=true \
 	--build-arg build_sbom=true \
@@ -472,6 +480,7 @@ e2e-deploy-ratify: e2e-notaryv2-setup e2e-notation-leaf-cert-setup e2e-cosign-se
 	-t localbuild:test .
 	kind load docker-image --name kind localbuild:test
 
+e2e-helm-deploy-ratify:
 	printf "{\n\t\"auths\": {\n\t\t\"registry:5000\": {\n\t\t\t\"auth\": \"`echo "${TEST_REGISTRY_USERNAME}:${TEST_REGISTRY_PASSWORD}" | tr -d '\n' | base64 -i -w 0`\"\n\t\t}\n\t}\n}" > mount_config.json
 
 	./.staging/helm/linux-amd64/helm install ${RATIFY_NAME} \
@@ -480,8 +489,11 @@ e2e-deploy-ratify: e2e-notaryv2-setup e2e-notation-leaf-cert-setup e2e-cosign-se
 	--set image.crdRepository=localbuildcrd \
 	--set image.tag=test \
 	--set gatekeeper.version=${GATEKEEPER_VERSION} \
+	--set featureFlags.RATIFY_CERT_ROTATION=${CERT_ROTATION_ENABLED} \
 	--set-file provider.tls.crt=${CERT_DIR}/server.crt \
 	--set-file provider.tls.key=${CERT_DIR}/server.key \
+	--set-file provider.tls.caCert=${CERT_DIR}/ca.crt \
+    --set-file provider.tls.caKey=${CERT_DIR}/ca.key \
 	--set provider.tls.cabundle="$(shell cat ${CERT_DIR}/ca.crt | base64 | tr -d '\n')" \
 	--set notaryCert="$$(cat ~/.config/notation/localkeys/ratify-bats-test.crt)" \
 	--set cosign.key="$$(cat .staging/cosign/cosign.pub)" \
@@ -490,6 +502,7 @@ e2e-deploy-ratify: e2e-notaryv2-setup e2e-notation-leaf-cert-setup e2e-cosign-se
 	--set logLevel=debug
 
 	rm mount_config.json
+
 e2e-aks:
 	./scripts/azure-ci-test.sh ${KUBERNETES_VERSION} ${GATEKEEPER_VERSION} ${TENANT_ID} ${GATEKEEPER_NAMESPACE} ${CERT_DIR}
 

README.md

@@ -72,7 +72,8 @@ curl -sSLO https://raw.githubusercontent.com/deislabs/ratify/main/test/testdata/
 helm install ratify \
     ratify/ratify --atomic \
     --namespace gatekeeper-system \
-    --set-file notaryCert=./notary.crt
+    --set-file notaryCert=./notary.crt \
+    --set featureFlags.RATIFY_CERT_ROTATION=true
 ```
 
 - Option 2: Install ratify with charts from your local branch.  
@@ -83,7 +84,8 @@ cd ratify
 helm install ratify \
     ./charts/ratify --atomic \
     --namespace gatekeeper-system \
-    --set-file notaryCert=./test/testdata/notary.crt
+    --set-file notaryCert=./test/testdata/notary.crt \
+    --set featureFlags.RATIFY_CERT_ROTATION=true
 ```
 
 ### Step 3: See Ratify in action

charts/ratify/templates/_helpers.tpl

@@ -108,13 +108,18 @@ Choose the certificate/key pair to enable TLS for HTTP server
 */}}
 {{- define "ratify.tlsSecret" -}}
 {{- $top := index . 0 -}}
-{{- if and $top.Values.provider.tls.crt $top.Values.provider.tls.key $top.Values.provider.tls.cabundle }}
+{{- if and $top.Values.provider.tls.crt $top.Values.provider.tls.key $top.Values.provider.tls.cabundle $top.Values.provider.tls.caCert $top.Values.provider.tls.caKey }}
 tls.crt: {{ $top.Values.provider.tls.crt | b64enc | quote }}  
 tls.key: {{ $top.Values.provider.tls.key | b64enc | quote }}
+ca.crt: {{ $top.Values.provider.tls.caCert | b64enc | quote }}
+ca.key: {{ $top.Values.provider.tls.caKey | b64enc | quote }}
 {{- else }}
 {{- $cert := index . 1 -}}
+{{- $ca := index . 2 -}}
 tls.crt: {{ $cert.Cert | b64enc | quote }}
 tls.key: {{ $cert.Key | b64enc | quote }}
+ca.crt: {{ $ca.Cert | b64enc | quote }}
+ca.key: {{ $ca.Key | b64enc | quote }}
 {{- end }}
 {{- end }}
 

charts/ratify/templates/deployment.yaml

@@ -114,6 +114,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            - name: RATIFY_NAME
+              value: {{ include "ratify.fullname" . }}
           {{- range $k, $v := .Values.featureFlags }}
             - name: {{ $k }}
               value: {{ $v | ternary 1 0 | quote }}

charts/ratify/templates/provider.yaml

@@ -9,6 +9,9 @@
 kind: Provider
 metadata:
   name: ratify-provider
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-weight: "5"
 spec:
   url: https://{{ include "ratify.fullname" .}}.{{ .Release.Namespace }}:6001/ratify/gatekeeper/v1/verify
   timeout: {{ required "You must provide .Values.provider.timeout.validationTimeoutSeconds" .Values.provider.timeout.validationTimeoutSeconds }}
@@ -20,6 +23,9 @@ spec:
 kind: Provider
 metadata:
   name: ratify-mutation-provider
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-weight: "5"
 spec:
   url: https://{{ include "ratify.fullname" .}}.{{ .Release.Namespace }}:6001/ratify/gatekeeper/v1/mutate
   timeout: {{ required "You must provide .Values.provider.timeout.mutationTimeoutSeconds" .Values.provider.timeout.mutationTimeoutSeconds }}
@@ -31,5 +37,8 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: {{ include "ratify.fullname" . }}-tls
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-weight: "5"
 data:
-  {{ include "ratify.tlsSecret" (list . $cert) | nindent 2}}
+  {{ include "ratify.tlsSecret" (list . $cert $ca) | nindent 2}}

charts/ratify/templates/ratify_manager-role-clusterrole.yaml

@@ -83,4 +83,24 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - externaldata.gatekeeper.sh
+  resources:
+  - providers
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - create
   {{- end }}

charts/ratify/values.yaml

@@ -71,6 +71,8 @@ provider:
   tls:
     crt: "" # crt used by ratify (httpserver), please provide your own crt
     key: "" # key used by ratify (httpserver), please provide your own key
+    caCert: "" # CA crt used by ratify (httpserver), please provide your own CA crt
+    caKey: "" # CA key used by ratify (httpserver), please provide your own CA key
     cabundle: "" # base64 encoded CA bundle used for the 'caBundle' property for the ratify provider within gatekeeper
   timeout:
     # timeout values must match gatekeeper webhook timeouts
@@ -108,4 +110,7 @@ crds:
 
 # See https://github.com/deislabs/ratify/blob/main/docs/reference/usage.md for a list of available feature flags
 featureFlags:
-  # RATIFY_FEATURE_NAME: true
+  # RATIFY_FEATURE_NAME: true
+
+  # RATIFY_CERT_ROTATION enables rotation for TLS certificates.
+  RATIFY_CERT_ROTATION: false

cmd/ratify/cmd/serve.go

@@ -75,9 +75,10 @@ func NewCmdServe(argv ...string) *cobra.Command {
 func serve(opts serveCmdOptions) error {
 	// in crd mode, the manager gets latest store/verifier from crd and pass on to the http server
 	if opts.enableCrdManager {
+		tlsWatcherReady := make(chan struct{})
 		logrus.Infof("starting crd manager")
-		go manager.StartManager()
-		manager.StartServer(opts.httpServerAddress, opts.configFilePath, opts.certDirectory, opts.caCertFile, opts.cacheSize, opts.cacheTTL, opts.metricsEnabled, opts.metricsType, opts.metricsPort)
+		go manager.StartManager(tlsWatcherReady)
+		manager.StartServer(opts.httpServerAddress, opts.configFilePath, opts.certDirectory, opts.caCertFile, opts.cacheSize, opts.cacheTTL, opts.metricsEnabled, opts.metricsType, opts.metricsPort, tlsWatcherReady)
 
 		return nil
 	}
@@ -93,7 +94,7 @@ func serve(opts serveCmdOptions) error {
 			return err
 		}
 		logrus.Infof("starting server at" + opts.httpServerAddress)
-		if err := server.Run(); err != nil {
+		if err := server.Run(nil); err != nil {
 			return err
 		}
 	}

docs/reference/usage.md

@@ -20,3 +20,21 @@ Ratify may roll out new features behind feature flags, which are activated by se
 A value of `1` indicates the feature is active; any other value disables the flag.
 
 - `RATIFY_DYNAMIC_PLUGINS`: (disabled) Enables Ratify to download plugins at runtime from an OCI registry by setting `source` on the plugin config
+
+- `RATIFY_CERT_ROTATION`: (disabled) Enables Ratify to rotate TLS certificates automatically when they are about to expire. See [cert-controller](https://github.com/open-policy-agent/cert-controller) for more details on the implementation. The cert-controller checks the validation of certificates every 12 hours, if the certificate is expiring in 90 days, cert-controller will generate a new certificate that is valid for 10 years. Notes: as this [post](https://ahmet.im/blog/kubernetes-secret-volumes-delay/) pointed out, it may take Kubernetes 60-90 seconds to progagate changes to Secrets on the mounted volumes. If you provided invalid/expired certificates/keys during the service startup, it may take up to 90 seconds for the service to rotate the certificates and get to actual working state with mounted certs.
+
+Notes: the root CA certificate generated by cert-controller will have the Subject field like:
+
+  `Subject: O = Ratify, CN = ratify.gatekeeper-system`
+
+  and x509v3 extentions field like:
+  ```
+  X509v3 extensions:
+    X509v3 Subject Alternative Name: 
+      DNS:ratify.gatekeeper-system
+    X509v3 Basic Constraints: critical
+      CA:TRUE
+    X509v3 Key Usage: critical
+      Digital Signature, Key Encipherment, Certificate Sign
+  ```
+So if you want to generate your own root CA certificate, make sure it has the same Subject and x509v3 extensions fields.

go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/notaryproject/notation-core-go v1.0.0-rc.4
 	github.com/notaryproject/notation-go v1.0.0-rc.6
-	github.com/open-policy-agent/frameworks/constraint v0.0.0-20220627162905-95c012350402
+	github.com/open-policy-agent/frameworks/constraint v0.0.0-20230201235642-777dc99a6669
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.0-rc2
 	github.com/pkg/errors v0.9.1
@@ -42,7 +42,7 @@ require (
 	google.golang.org/protobuf v1.30.0
 	k8s.io/api v0.26.5
 	k8s.io/apimachinery v0.26.5
-	k8s.io/client-go v0.25.10
+	k8s.io/client-go v0.26.1
 	oras.land/oras-go/v2 v2.2.0
 )
 
@@ -79,7 +79,8 @@ require (
 	github.com/digitorus/pkcs7 v0.0.0-20221212123742-001c36b64ec3 // indirect
 	github.com/digitorus/timestamp v0.0.0-20221019182153-ef3b63b79b31 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
-	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/frankban/quicktest v1.14.4 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
@@ -132,7 +133,6 @@ require (
 	github.com/docker/docker v23.0.3+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/fxamacker/cbor/v2 v2.4.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-chi/chi v4.1.2+incompatible // indirect
@@ -183,6 +183,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
+	github.com/open-policy-agent/cert-controller v0.7.0
 	github.com/open-policy-agent/opa v0.51.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
@@ -238,12 +239,12 @@ require (
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/apiextensions-apiserver v0.24.2 // indirect
-	k8s.io/component-base v0.24.2 // indirect
+	k8s.io/apiextensions-apiserver v0.26.1 // indirect
+	k8s.io/component-base v0.26.1 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
 	k8s.io/utils v0.0.0-20230115233650-391b47cb4029 // indirect
-	sigs.k8s.io/controller-runtime v0.12.3
+	sigs.k8s.io/controller-runtime v0.14.2
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect