[NET-11297] Purge on disable (#4378)

* purge services on disable of sync catalog * gofumpt * update tests for sync catalog * Added changelog * create separate jobs to handle deregistering services when disabling sync catalog * Check error * fix tests * rename field in values file, remove references to pod security policy * remove psp * added bats testing
hashicorp · Oct 21, 2024 · f31a6ba · f31a6ba
1 parent bf41f34
commit f31a6ba
Show file tree

Hide file tree

Showing 9 changed files with 650 additions and 209 deletions.
diff --git a/.changelog/4378.txt b/.changelog/4378.txt
@@ -0,0 +1,3 @@
+```release-note:enhancement
+catalog-sync: Added field to helm chart to purge all services registered with catalog-sync from consul on disabling of catalog-sync.
+```
diff --git a/charts/consul/templates/sync-catalog-cleanup-on-uninstall-job.yaml b/charts/consul/templates/sync-catalog-cleanup-on-uninstall-job.yaml
@@ -0,0 +1,174 @@
+{{- if .Values.syncCatalog.cleanupNodeOnRemoval }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "consul.fullname" . }}-sync-catalog-cleanup-on-uninstall
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "consul.name" . }}
+    chart: {{ template "consul.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    component: sync-catalog-cleanup
+    {{- if .Values.global.extraLabels }}
+      {{- toYaml .Values.global.extraLabels | nindent 4 }}
+    {{- end }}
+  annotations:
+    "helm.sh/hook": pre-delete
+    "helm.sh/hook-weight": "0"
+    "helm.sh/hook-delete-policy": hook-succeeded,hook-failed
+spec:
+  template:
+    metadata:
+      name: {{ template "consul.fullname" . }}-sync-catalog-cleanup-on-uninstall
+      labels:
+        app: {{ template "consul.name" . }}
+        chart: {{ template "consul.chart" . }}
+        release: {{ .Release.Name }}
+        component: sync-catalog-cleanup
+        {{- if .Values.syncCatalog.extraLabels }}
+          {{- toYaml .Values.syncCatalog.extraLabels | nindent 8 }}
+        {{- end }}
+        {{- if .Values.global.extraLabels }}
+          {{- toYaml .Values.global.extraLabels | nindent 8 }}
+        {{- end }}
+      annotations:
+        "consul.hashicorp.com/connect-inject": "false"
+        "consul.hashicorp.com/mesh-inject": "false"
+        {{- if .Values.syncCatalog.annotations }}
+        {{- tpl .Values.syncCatalog.annotations . | nindent 8 }}
+        {{- end }}
+        {{- if (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) }}
+        "vault.hashicorp.com/agent-init-first": "true"
+        "vault.hashicorp.com/agent-inject": "true"
+        "vault.hashicorp.com/role": {{ .Values.global.secretsBackend.vault.consulCARole }}
+        "vault.hashicorp.com/agent-inject-secret-serverca.crt": {{ .Values.global.tls.caCert.secretName }}
+        "vault.hashicorp.com/agent-inject-template-serverca.crt": {{ template "consul.serverTLSCATemplate" . }}
+        {{- if and .Values.global.secretsBackend.vault.ca.secretName .Values.global.secretsBackend.vault.ca.secretKey }}
+        "vault.hashicorp.com/agent-extra-secret": "{{ .Values.global.secretsBackend.vault.ca.secretName }}"
+        "vault.hashicorp.com/ca-cert": "/vault/custom/{{ .Values.global.secretsBackend.vault.ca.secretKey }}"
+        {{- end }}
+        {{- if .Values.global.secretsBackend.vault.agentAnnotations }}
+        {{ tpl .Values.global.secretsBackend.vault.agentAnnotations . | nindent 8 | trim }}
+        {{- end }}
+       {{- if (and (.Values.global.secretsBackend.vault.vaultNamespace) (not (hasKey (default "" .Values.global.secretsBackend.vault.agentAnnotations | fromYaml) "vault.hashicorp.com/namespace")))}}
+        "vault.hashicorp.com/namespace": "{{ .Values.global.secretsBackend.vault.vaultNamespace }}"
+        {{- end }}
+        {{- end }}
+        {{- if .Values.syncCatalog.metrics.enabled | default .Values.global.metrics.enabled }}
+        "prometheus.io/scrape": "true"
+        {{- if not (hasKey (default "" .Values.syncCatalog.annotations | fromYaml) "prometheus.io/path")}}
+        "prometheus.io/path": {{ default "/metrics" .Values.syncCatalog.metrics.path }} 
+        {{- end }}
+        "prometheus.io/port": {{ .Values.syncCatalog.metrics.port | default "20300" | quote }}
+        {{- end }}
+    spec:
+      restartPolicy: Never
+      serviceAccountName: {{ template "consul.fullname" . }}-sync-catalog-cleanup
+      containers:
+      - name: sync-catalog-cleanup-job
+        image: "{{ default .Values.global.imageK8S .Values.syncCatalog.image }}"
+        {{ template "consul.imagePullPolicy" . }}
+        {{- include "consul.restrictedSecurityContext" . | nindent 8 }}
+        env:
+        {{- include "consul.consulK8sConsulServerEnvVars" . | nindent 8 }}
+        {{- if .Values.global.acls.manageSystemACLs }}
+        - name: CONSUL_LOGIN_AUTH_METHOD
+          {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter .Values.global.enableConsulNamespaces }}
+          value: {{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }}
+          {{- else }}
+          value: {{ template "consul.fullname" . }}-k8s-component-auth-method
+          {{- end }}
+        - name: CONSUL_LOGIN_DATACENTER
+          {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter .Values.global.enableConsulNamespaces }}
+          value: {{ .Values.global.federation.primaryDatacenter }}
+          {{- else }}
+          value: {{ .Values.global.datacenter }}
+          {{- end }}
+        - name: CONSUL_LOGIN_META
+          value: "component=sync-catalog,pod=$(NAMESPACE)/$(POD_NAME)"
+        {{- end }}
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        {{- if (and .Values.syncCatalog.aclSyncToken.secretName .Values.syncCatalog.aclSyncToken.secretKey) }}
+        - name: CONSUL_ACL_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.syncCatalog.aclSyncToken.secretName }}
+              key: {{ .Values.syncCatalog.aclSyncToken.secretKey }}
+        {{- end }}
+        volumeMounts:
+        {{- if .Values.global.tls.enabled }}
+        {{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }}
+        - name: consul-ca-cert
+          mountPath: /consul/tls/ca
+          readOnly: true
+        {{- end }}
+        {{- end }}
+        command:
+        - "/bin/sh"
+        - "-ec"
+        - |
+          exec consul-k8s-control-plane sync-catalog \
+            -log-level={{ default .Values.global.logLevel .Values.syncCatalog.logLevel }} \
+            -log-json={{ .Values.global.logJSON }} \
+            -k8s-default-sync={{ .Values.syncCatalog.default }} \
+            {{- if .Values.global.adminPartitions.enabled }}
+            -partition={{ .Values.global.adminPartitions.name }} \
+            {{- end }}
+            {{- if .Values.syncCatalog.consulNodeName }}
+            -consul-node-name={{ .Values.syncCatalog.consulNodeName }} \
+            {{- end }}
+            {{- if .Values.syncCatalog.metrics.enabled | default .Values.global.metrics.enabled }}
+            -enable-metrics \
+            {{- end }}
+            {{- if .Values.syncCatalog.metrics.path }}
+            -metrics-path={{ .Values.syncCatalog.metrics.path }} \
+            {{- end }}
+            {{- if .Values.syncCatalog.metrics.port }}
+            -metrics-port={{ .Values.syncCatalog.metrics.port }} \
+            {{- end }}
+            -prometheus-retention-time={{ .Values.global.metrics.agentMetricsRetentionTime }} \
+            -purge-k8s-services-from-node
+        {{- with .Values.syncCatalog.resources }}
+        resources:
+        {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- if or (eq (.Values.syncCatalog.metrics.enabled | toString) "-") .Values.syncCatalog.metrics.enabled .Values.global.metrics.enabled }}
+        ports:
+        - name: prometheus
+          containerPort: {{ .Values.syncCatalog.metrics.port | default "20300" | int }}
+        {{- end }}
+      {{- if .Values.syncCatalog.priorityClassName }}
+      priorityClassName: {{ .Values.syncCatalog.priorityClassName | quote }}
+      {{- end }}
+      {{- if .Values.syncCatalog.nodeSelector }}
+      nodeSelector:
+        {{ tpl .Values.syncCatalog.nodeSelector . | indent 8 | trim }}
+      {{- end }}
+      {{- if .Values.syncCatalog.affinity }}
+      affinity:
+        {{ tpl .Values.syncCatalog.affinity . | indent 8 | trim }}
+      {{- end }}
+      {{- if .Values.syncCatalog.tolerations }}
+      tolerations:
+        {{ tpl .Values.syncCatalog.tolerations . | indent 8 | trim }}
+      {{- end }}
+      volumes:
+      {{- if .Values.global.tls.enabled }}
+      {{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }}
+      - name: consul-ca-cert
+        secret:
+          {{- if .Values.global.tls.caCert.secretName }}
+          secretName: {{ .Values.global.tls.caCert.secretName }}
+          {{- else }}
+          secretName: {{ template "consul.fullname" . }}-ca-cert
+          {{- end }}
+          items:
+          - key: {{ default "tls.crt" .Values.global.tls.caCert.secretKey }}
+            path: tls.crt
+      {{- end }}
+      {{- end }}
+{{- end }}
diff --git a/charts/consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml b/charts/consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml
@@ -0,0 +1,175 @@
+{{- $syncCatalogEnabled := (or (and (ne (.Values.syncCatalog.enabled | toString) "-") .Values.syncCatalog.enabled) (and (eq (.Values.syncCatalog.enabled | toString) "-") .Values.global.enabled)) }}
+{{- if and (not $syncCatalogEnabled) .Values.syncCatalog.cleanupNodeOnRemoval .Release.IsUpgrade }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "consul.fullname" . }}-sync-catalog-cleanup-on-upgrade
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "consul.name" . }}
+    chart: {{ template "consul.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    component: sync-catalog-cleanup
+    {{- if .Values.global.extraLabels }}
+      {{- toYaml .Values.global.extraLabels | nindent 4 }}
+    {{- end }}
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "0"
+    "helm.sh/hook-delete-policy": hook-succeeded,hook-failed
+spec:
+  template:
+    metadata:
+      name: {{ template "consul.fullname" . }}-sync-catalog-cleanup-on-upgrade
+      labels:
+        app: {{ template "consul.name" . }}
+        chart: {{ template "consul.chart" . }}
+        release: {{ .Release.Name }}
+        component: sync-catalog-cleanup
+        {{- if .Values.syncCatalog.extraLabels }}
+          {{- toYaml .Values.syncCatalog.extraLabels | nindent 8 }}
+        {{- end }}
+        {{- if .Values.global.extraLabels }}
+          {{- toYaml .Values.global.extraLabels | nindent 8 }}
+        {{- end }}
+      annotations:
+        "consul.hashicorp.com/connect-inject": "false"
+        "consul.hashicorp.com/mesh-inject": "false"
+        {{- if .Values.syncCatalog.annotations }}
+        {{- tpl .Values.syncCatalog.annotations . | nindent 8 }}
+        {{- end }}
+        {{- if (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) }}
+        "vault.hashicorp.com/agent-init-first": "true"
+        "vault.hashicorp.com/agent-inject": "true"
+        "vault.hashicorp.com/role": {{ .Values.global.secretsBackend.vault.consulCARole }}
+        "vault.hashicorp.com/agent-inject-secret-serverca.crt": {{ .Values.global.tls.caCert.secretName }}
+        "vault.hashicorp.com/agent-inject-template-serverca.crt": {{ template "consul.serverTLSCATemplate" . }}
+        {{- if and .Values.global.secretsBackend.vault.ca.secretName .Values.global.secretsBackend.vault.ca.secretKey }}
+        "vault.hashicorp.com/agent-extra-secret": "{{ .Values.global.secretsBackend.vault.ca.secretName }}"
+        "vault.hashicorp.com/ca-cert": "/vault/custom/{{ .Values.global.secretsBackend.vault.ca.secretKey }}"
+        {{- end }}
+        {{- if .Values.global.secretsBackend.vault.agentAnnotations }}
+        {{ tpl .Values.global.secretsBackend.vault.agentAnnotations . | nindent 8 | trim }}
+        {{- end }}
+       {{- if (and (.Values.global.secretsBackend.vault.vaultNamespace) (not (hasKey (default "" .Values.global.secretsBackend.vault.agentAnnotations | fromYaml) "vault.hashicorp.com/namespace")))}}
+        "vault.hashicorp.com/namespace": "{{ .Values.global.secretsBackend.vault.vaultNamespace }}"
+        {{- end }}
+        {{- end }}
+        {{- if .Values.syncCatalog.metrics.enabled | default .Values.global.metrics.enabled }}
+        "prometheus.io/scrape": "true"
+        {{- if not (hasKey (default "" .Values.syncCatalog.annotations | fromYaml) "prometheus.io/path")}}
+        "prometheus.io/path": {{ default "/metrics" .Values.syncCatalog.metrics.path }} 
+        {{- end }}
+        "prometheus.io/port": {{ .Values.syncCatalog.metrics.port | default "20300" | quote }}
+        {{- end }}
+    spec:
+      restartPolicy: Never
+      serviceAccountName: {{ template "consul.fullname" . }}-sync-catalog-cleanup
+      containers:
+      - name: sync-catalog-cleanup-job
+        image: "{{ default .Values.global.imageK8S .Values.syncCatalog.image }}"
+        {{ template "consul.imagePullPolicy" . }}
+        {{- include "consul.restrictedSecurityContext" . | nindent 8 }}
+        env:
+        {{- include "consul.consulK8sConsulServerEnvVars" . | nindent 8 }}
+        {{- if .Values.global.acls.manageSystemACLs }}
+        - name: CONSUL_LOGIN_AUTH_METHOD
+          {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter .Values.global.enableConsulNamespaces }}
+          value: {{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }}
+          {{- else }}
+          value: {{ template "consul.fullname" . }}-k8s-component-auth-method
+          {{- end }}
+        - name: CONSUL_LOGIN_DATACENTER
+          {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter .Values.global.enableConsulNamespaces }}
+          value: {{ .Values.global.federation.primaryDatacenter }}
+          {{- else }}
+          value: {{ .Values.global.datacenter }}
+          {{- end }}
+        - name: CONSUL_LOGIN_META
+          value: "component=sync-catalog,pod=$(NAMESPACE)/$(POD_NAME)"
+        {{- end }}
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        {{- if (and .Values.syncCatalog.aclSyncToken.secretName .Values.syncCatalog.aclSyncToken.secretKey) }}
+        - name: CONSUL_ACL_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Values.syncCatalog.aclSyncToken.secretName }}
+              key: {{ .Values.syncCatalog.aclSyncToken.secretKey }}
+        {{- end }}
+        volumeMounts:
+        {{- if .Values.global.tls.enabled }}
+        {{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }}
+        - name: consul-ca-cert
+          mountPath: /consul/tls/ca
+          readOnly: true
+        {{- end }}
+        {{- end }}
+        command:
+        - "/bin/sh"
+        - "-ec"
+        - |
+          exec consul-k8s-control-plane sync-catalog \
+            -log-level={{ default .Values.global.logLevel .Values.syncCatalog.logLevel }} \
+            -log-json={{ .Values.global.logJSON }} \
+            -k8s-default-sync={{ .Values.syncCatalog.default }} \
+            {{- if .Values.global.adminPartitions.enabled }}
+            -partition={{ .Values.global.adminPartitions.name }} \
+            {{- end }}
+            {{- if .Values.syncCatalog.consulNodeName }}
+            -consul-node-name={{ .Values.syncCatalog.consulNodeName }} \
+            {{- end }}
+            {{- if .Values.syncCatalog.metrics.enabled | default .Values.global.metrics.enabled }}
+            -enable-metrics \
+            {{- end }}
+            {{- if .Values.syncCatalog.metrics.path }}
+            -metrics-path={{ .Values.syncCatalog.metrics.path }} \
+            {{- end }}
+            {{- if .Values.syncCatalog.metrics.port }}
+            -metrics-port={{ .Values.syncCatalog.metrics.port }} \
+            {{- end }}
+            -prometheus-retention-time={{ .Values.global.metrics.agentMetricsRetentionTime }} \
+            -purge-k8s-services-from-node
+        {{- with .Values.syncCatalog.resources }}
+        resources:
+        {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- if or (eq (.Values.syncCatalog.metrics.enabled | toString) "-") .Values.syncCatalog.metrics.enabled .Values.global.metrics.enabled }}
+        ports:
+        - name: prometheus
+          containerPort: {{ .Values.syncCatalog.metrics.port | default "20300" | int }}
+        {{- end }}
+      {{- if .Values.syncCatalog.priorityClassName }}
+      priorityClassName: {{ .Values.syncCatalog.priorityClassName | quote }}
+      {{- end }}
+      {{- if .Values.syncCatalog.nodeSelector }}
+      nodeSelector:
+        {{ tpl .Values.syncCatalog.nodeSelector . | indent 8 | trim }}
+      {{- end }}
+      {{- if .Values.syncCatalog.affinity }}
+      affinity:
+        {{ tpl .Values.syncCatalog.affinity . | indent 8 | trim }}
+      {{- end }}
+      {{- if .Values.syncCatalog.tolerations }}
+      tolerations:
+        {{ tpl .Values.syncCatalog.tolerations . | indent 8 | trim }}
+      {{- end }}
+      volumes:
+      {{- if .Values.global.tls.enabled }}
+      {{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }}
+      - name: consul-ca-cert
+        secret:
+          {{- if .Values.global.tls.caCert.secretName }}
+          secretName: {{ .Values.global.tls.caCert.secretName }}
+          {{- else }}
+          secretName: {{ template "consul.fullname" . }}-ca-cert
+          {{- end }}
+          items:
+          - key: {{ default "tls.crt" .Values.global.tls.caCert.secretKey }}
+            path: tls.crt
+      {{- end }}
+      {{- end }}
+{{- end }}
diff --git a/charts/consul/templates/sync-catalog-cleanup-serviceaccount.yaml b/charts/consul/templates/sync-catalog-cleanup-serviceaccount.yaml
@@ -0,0 +1,23 @@
+{{- if .Values.syncCatalog.cleanupNodeOnRemoval }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "consul.fullname" . }}-sync-catalog-cleanup
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "consul.name" . }}
+    chart: {{ template "consul.chart" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    component: sync-catalog-cleanup
+  {{- if .Values.syncCatalog.serviceAccount.annotations }}
+  annotations:
+    {{ tpl .Values.syncCatalog.serviceAccount.annotations . | nindent 4 | trim }}
+  {{- end }}
+{{- with .Values.global.imagePullSecrets }}
+imagePullSecrets:
+{{- range . }}
+  - name: {{ .name }}
+{{- end }}
+{{- end }}
+{{- end }}
diff --git a/charts/consul/templates/sync-catalog-deployment.yaml b/charts/consul/templates/sync-catalog-deployment.yaml
@@ -1,4 +1,5 @@
-{{- if (or (and (ne (.Values.syncCatalog.enabled | toString) "-") .Values.syncCatalog.enabled) (and (eq (.Values.syncCatalog.enabled | toString) "-") .Values.global.enabled)) }}
+{{- $syncCatalogEnabled := (or (and (ne (.Values.syncCatalog.enabled | toString) "-") .Values.syncCatalog.enabled) (and (eq (.Values.syncCatalog.enabled | toString) "-") .Values.global.enabled)) }}
+{{- if $syncCatalogEnabled }}
 {{- template "consul.reservedNamesFailer" (list .Values.syncCatalog.consulNamespaces.consulDestinationNamespace "syncCatalog.consulNamespaces.consulDestinationNamespace") }}
 {{ template "consul.validateRequiredCloudSecretsExist" . }}
 {{ template "consul.validateCloudSecretKeys" . }}

diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml
@@ -2090,6 +2090,10 @@ syncCatalog:
   # global.enabled.
   enabled: false
 
+  # True if you want to deregister all services in this cluster from consul that have been registered by the catalog sync when the `enabled` flag is set to false.
+  # @type: boolean
+  cleanupNodeOnRemoval: false
+
   # The name of the Docker image (including any tag) for consul-k8s-control-plane
   # to run the sync program.
   # @type: string