From 51a26a5ac205f649c2c5510f1c03e9200824260d Mon Sep 17 00:00:00 2001 From: jm96441n Date: Tue, 1 Oct 2024 13:23:08 -0400 Subject: [PATCH 01/11] purge services on disable of sync catalog --- .../templates/sync-catalog-deployment.yaml | 4 +- charts/consul/values.yaml | 4 ++ .../subcommand/sync-catalog/command.go | 68 ++++++++++++------- 3 files changed, 52 insertions(+), 24 deletions(-) diff --git a/charts/consul/templates/sync-catalog-deployment.yaml b/charts/consul/templates/sync-catalog-deployment.yaml index 94260b5e44..8a54dd83b0 100644 --- a/charts/consul/templates/sync-catalog-deployment.yaml +++ b/charts/consul/templates/sync-catalog-deployment.yaml @@ -1,4 +1,5 @@ -{{- if (or (and (ne (.Values.syncCatalog.enabled | toString) "-") .Values.syncCatalog.enabled) (and (eq (.Values.syncCatalog.enabled | toString) "-") .Values.global.enabled)) }} +{{ - $syncCatalogEnabled := (or (and (ne (.Values.syncCatalog.enabled | toString) "-") .Values.syncCatalog.enabled) (and (eq (.Values.syncCatalog.enabled | toString) "-") .Values.global.enabled)) }} +{{- if (or $syncCatalogEnabled .Values.syncCatalog.purgeServicesOnDisable) }} {{- template "consul.reservedNamesFailer" (list .Values.syncCatalog.consulNamespaces.consulDestinationNamespace "syncCatalog.consulNamespaces.consulDestinationNamespace") }} {{ template "consul.validateRequiredCloudSecretsExist" . }} {{ template "consul.validateCloudSecretKeys" . }} @@ -213,6 +214,7 @@ spec: -metrics-port={{ .Values.syncCatalog.metrics.port }} \ {{- end }} -prometheus-retention-time={{ .Values.global.metrics.agentMetricsRetentionTime }} \ + -purgeServicesOnDisable={{ and (not $syncCatalogEnabled) .Values.syncCatalog.purgeServicesOnDisable }} \ livenessProbe: httpGet: path: /health/ready diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml index 4a6aa20060..66115bad7a 100644 --- a/charts/consul/values.yaml +++ b/charts/consul/values.yaml @@ -2090,6 +2090,10 @@ syncCatalog: # global.enabled. enabled: false + # True if you want to deregister all services in this cluster from consul that have been registered by the catalog sync when the `enabled` flag is set to false. + # @type: boolean + purgeServicesOnDisable: false + # The name of the Docker image (including any tag) for consul-k8s-control-plane # to run the sync program. # @type: string diff --git a/control-plane/subcommand/sync-catalog/command.go b/control-plane/subcommand/sync-catalog/command.go index b5d2c9d5af..12e6d47ac3 100644 --- a/control-plane/subcommand/sync-catalog/command.go +++ b/control-plane/subcommand/sync-catalog/command.go @@ -19,6 +19,7 @@ import ( "github.com/armon/go-metrics/prometheus" mapset "github.com/deckarep/golang-set" "github.com/hashicorp/consul-server-connection-manager/discovery" + "github.com/hashicorp/consul/api" "github.com/hashicorp/go-hclog" "github.com/mitchellh/cli" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -33,7 +34,6 @@ import ( "github.com/hashicorp/consul-k8s/control-plane/helper/controller" "github.com/hashicorp/consul-k8s/control-plane/subcommand" "github.com/hashicorp/consul-k8s/control-plane/subcommand/common" - metricsutil "github.com/hashicorp/consul-k8s/control-plane/subcommand/common" "github.com/hashicorp/consul-k8s/control-plane/subcommand/flags" ) @@ -42,27 +42,29 @@ import ( type Command struct { UI cli.Ui - flags *flag.FlagSet - consul *flags.ConsulFlags - k8s *flags.K8SFlags - flagListen string - flagToConsul bool - flagToK8S bool - flagConsulDomain string - flagConsulK8STag string - flagConsulNodeName string - flagK8SDefault bool - flagK8SServicePrefix string - flagConsulServicePrefix string - flagK8SSourceNamespace string - flagK8SWriteNamespace string - flagConsulWritePeriod time.Duration - flagSyncClusterIPServices bool - flagSyncLBEndpoints bool - flagNodePortSyncType string - flagAddK8SNamespaceSuffix bool - flagLogLevel string - flagLogJSON bool + flags *flag.FlagSet + consul *flags.ConsulFlags + k8s *flags.K8SFlags + flagListen string + flagToConsul bool + flagToK8S bool + flagConsulDomain string + flagConsulK8STag string + flagConsulNodeName string + flagK8SDefault bool + flagK8SServicePrefix string + flagConsulServicePrefix string + flagK8SSourceNamespace string + flagK8SWriteNamespace string + flagConsulWritePeriod time.Duration + flagSyncClusterIPServices bool + flagSyncLBEndpoints bool + flagNodePortSyncType string + flagAddK8SNamespaceSuffix bool + flagLogLevel string + flagLogJSON bool + flagPurgeK8SServicesFromNode bool + flagFilter string // Flags to support namespaces flagEnableNamespaces bool // Use namespacing on all components @@ -150,6 +152,11 @@ func (c *Command) init() { "\"debug\", \"info\", \"warn\", and \"error\".") c.flags.BoolVar(&c.flagLogJSON, "log-json", false, "Enable or disable JSON output format for logging.") + c.flags.BoolVar(&c.flagPurgeK8SServicesFromNode, "purge-k8s-services-from-node", false, + "Specifies if services should be purged from the Consul node. If set, all K8S services will be removed from the Consul node.") + c.flags.StringVar(&c.flagFilter, "filter", "", + "Specifies the expression used to filter the services on the Consul node that will be deregistered. "+ + "The syntax for this filter is the same as the syntax used in the List Services for Node API in the Consul catalog.") c.flags.Var((*flags.AppendSliceValue)(&c.flagAllowK8sNamespacesList), "allow-k8s-namespace", "K8s namespaces to explicitly allow. May be specified multiple times.") @@ -436,6 +443,21 @@ func (c *Command) Run(args []string) int { } } +// remove all k8s services from Consul. +func (c *Command) removeAllK8SServicesFromConsulNode(consulClient *api.Client) error { + _, err := consulClient.Catalog().Deregister(&api.CatalogDeregistration{ + Node: c.flagConsulNodeName, + Partition: c.consul.Partition, + }, nil) + if err != nil { + c.UI.Error(fmt.Sprintf("unable to deregister all K8S services from Consul: %s", err)) + return err + } + + c.UI.Info("All K8S services were deregistered from Consul") + return nil +} + func (c *Command) handleReady(rw http.ResponseWriter, _ *http.Request) { if !c.ready { c.UI.Error("[GET /health/ready] sync catalog controller is not yet ready") @@ -482,7 +504,7 @@ func (c *Command) validateFlags() error { } if c.flagMetricsPort != "" { - if _, valid := metricsutil.ParseScrapePort(c.flagMetricsPort); !valid { + if _, valid := common.ParseScrapePort(c.flagMetricsPort); !valid { return errors.New("-metrics-port must be a valid unprivileged port number") } } From d822d778cfa1feb4e62a44d77ae25c83164470fc Mon Sep 17 00:00:00 2001 From: jm96441n Date: Tue, 1 Oct 2024 13:37:08 -0400 Subject: [PATCH 02/11] gofumpt From 496f20df6fe47bd8047c3cf525b8abb7dd041e84 Mon Sep 17 00:00:00 2001 From: jm96441n Date: Tue, 1 Oct 2024 14:37:23 -0400 Subject: [PATCH 03/11] update tests for sync catalog --- .../subcommand/sync-catalog/command.go | 13 ++ .../subcommand/sync-catalog/command_test.go | 148 ++++++++++++++++++ 2 files changed, 161 insertions(+) diff --git a/control-plane/subcommand/sync-catalog/command.go b/control-plane/subcommand/sync-catalog/command.go index 12e6d47ac3..36bb996f2f 100644 --- a/control-plane/subcommand/sync-catalog/command.go +++ b/control-plane/subcommand/sync-catalog/command.go @@ -275,6 +275,19 @@ func (c *Command) Run(args []string) int { } c.ready = true + if c.flagPurgeK8SServicesFromNode { + consulClient, err := consul.NewClientFromConnMgr(consulConfig, c.connMgr) + if err != nil { + c.UI.Error(fmt.Sprintf("unable to instantiate consul client: %s", err)) + return 1 + } + if err := c.removeAllK8SServicesFromConsulNode(consulClient); err != nil { + c.UI.Error(fmt.Sprintf("unable to remove all K8S services: %s", err)) + return 1 + } + return 0 + } + // Convert allow/deny lists to sets allowSet := flags.ToSet(c.flagAllowK8sNamespacesList) denySet := flags.ToSet(c.flagDenyK8sNamespacesList) diff --git a/control-plane/subcommand/sync-catalog/command_test.go b/control-plane/subcommand/sync-catalog/command_test.go index ca2aca4e37..94f5452ad9 100644 --- a/control-plane/subcommand/sync-catalog/command_test.go +++ b/control-plane/subcommand/sync-catalog/command_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/armon/go-metrics/prometheus" + "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/hashicorp/go-hclog" "github.com/mitchellh/cli" @@ -557,6 +558,153 @@ func TestRun_ToConsulChangingFlags(t *testing.T) { } } +// Test services could be de-registered from Consul. +func TestRemoveAllK8SServicesFromConsul(t *testing.T) { + testCases := map[string]struct { + nodeName string + partitionName string + }{ + "default Node name in default partition": { + nodeName: "k8s-sync", + partitionName: "default", + }, + "non-default Node name in default partition": { + nodeName: "custom-node", + partitionName: "default", + }, + "default Node name in non-default partition": { + nodeName: "k8s-sync", + partitionName: "part-1", + }, + "non-default Node name in non-default partition": { + nodeName: "custom-node", + partitionName: "part-1", + }, + } + + for name, tc := range testCases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + var err error + + k8s, testClient := completeSetup(t) + consulClient := testClient.APIClient + + // Create a mock reader to simulate user input + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + logger: hclog.New(&hclog.LoggerOptions{ + Name: t.Name(), + Level: hclog.Debug, + }), + flagAllowK8sNamespacesList: []string{"*"}, + connMgr: testClient.Watcher, + } + + otherPartitionName := "the-other-one" + + // create another partition and register 2 services there to the same node to show that we won't delete them + _, _, err = consulClient.Partitions().Create(context.Background(), &api.Partition{Name: otherPartitionName}, nil) + _, err = consulClient.Catalog().Register( + &api.CatalogRegistration{ + Node: tc.nodeName, + Address: "5.5.5.5", + Service: &api.AgentService{ + ID: "other-service-1", + Service: "other-service-1", + Tags: []string{"other-k8s-cluster"}, + Meta: map[string]string{}, + Port: 0, + Address: "5.5.5.5", + Partition: otherPartitionName, + }, + Partition: otherPartitionName, + }, + &api.WriteOptions{Partition: otherPartitionName}, + ) + require.NoError(t, err) + + _, err = consulClient.Catalog().Register( + &api.CatalogRegistration{ + Node: tc.nodeName, + Address: "6.6.6.6", + Service: &api.AgentService{ + ID: "other-service-2", + Service: "other-service-2", + Tags: []string{"other-k8s-cluster"}, + Meta: map[string]string{}, + Port: 0, + Address: "6.6.6.6", + Partition: otherPartitionName, + }, + Partition: otherPartitionName, + }, + &api.WriteOptions{Partition: otherPartitionName}, + ) + require.NoError(t, err) + + // create partition if it does not exist + if tc.partitionName != "default" { + p, _, err := consulClient.Partitions().Create(context.Background(), &api.Partition{Name: tc.partitionName}, nil) + require.NoError(t, err) + require.NotNil(t, p) + } + + // create two services in k8s + _, err = k8s.CoreV1().Services("bar").Create(context.Background(), lbService("foo", "1.1.1.1"), metav1.CreateOptions{}) + require.NoError(t, err) + + _, err = k8s.CoreV1().Services("baz").Create(context.Background(), lbService("foo", "2.2.2.2"), metav1.CreateOptions{}) + require.NoError(t, err) + + longRunningChan := runCommandAsynchronously(&cmd, []string{ + "-addresses", "127.0.0.1", + "-http-port", strconv.Itoa(testClient.Cfg.HTTPPort), + "-consul-write-interval", "100ms", + "-partition", tc.partitionName, + "-consul-node-name", tc.nodeName, + "-add-k8s-namespace-suffix", + }) + + // check that the two K8s services have been synced into Consul + retry.Run(t, func(r *retry.R) { + svc, _, err := consulClient.Catalog().Service("foo-bar", "k8s", &api.QueryOptions{Partition: tc.partitionName}) + require.NoError(r, err) + require.Len(r, svc, 1) + require.Equal(r, "1.1.1.1", svc[0].ServiceAddress) + svc, _, err = consulClient.Catalog().Service("foo-baz", "k8s", &api.QueryOptions{Partition: tc.partitionName}) + require.NoError(r, err) + require.Len(r, svc, 1) + require.Equal(r, "2.2.2.2", svc[0].ServiceAddress) + }) + + defer stopCommand(t, &cmd, longRunningChan) + + exitChan := runCommandAsynchronously(&cmd, []string{ + "-addresses", "127.0.0.1", + "-http-port", strconv.Itoa(testClient.Cfg.HTTPPort), + "-purge-k8s-services-from-node=true", + "-partition", tc.partitionName, + "-consul-node-name", tc.nodeName, + }) + stopCommand(t, &cmd, exitChan) + + retry.Run(t, func(r *retry.R) { + serviceList, _, err := consulClient.Catalog().NodeServiceList(tc.nodeName, &api.QueryOptions{AllowStale: false, Partition: tc.partitionName}) + require.NoError(r, err) + require.Len(r, serviceList.Services, 0) + otherPartitionServiceList, _, err := consulClient.Catalog().NodeServiceList(tc.nodeName, &api.QueryOptions{AllowStale: false, Partition: otherPartitionName}) + require.NoError(r, err) + require.Len(r, otherPartitionServiceList.Services, 2) + }) + }) + } +} + // Set up test consul agent and fake kubernetes cluster client. func completeSetup(t *testing.T) (*fake.Clientset, *test.TestServerClient) { k8s := fake.NewSimpleClientset() From 1e59ffc447882687e0dd032067da97b3a3ebc5f2 Mon Sep 17 00:00:00 2001 From: jm96441n Date: Thu, 10 Oct 2024 10:37:33 -0400 Subject: [PATCH 04/11] Added changelog --- .changelog/4378.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/4378.txt diff --git a/.changelog/4378.txt b/.changelog/4378.txt new file mode 100644 index 0000000000..30a4287816 --- /dev/null +++ b/.changelog/4378.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +catalog-sync: Added field to helm chart to purge all services registered with catalog-sync from consul on disabling of catalog-sync. +``` From 9cefd6e1e160ec2c029ad478a5efc40e9bf35e57 Mon Sep 17 00:00:00 2001 From: jm96441n Date: Thu, 10 Oct 2024 16:55:27 -0400 Subject: [PATCH 05/11] create separate jobs to handle deregistering services when disabling sync catalog --- ...sync-catalog-cleanup-on-uninstall-job.yaml | 123 +++++++++++++++++ .../sync-catalog-cleanup-on-upgrade-job.yaml | 124 ++++++++++++++++++ ...ync-catalog-cleanup-podsecuritypolicy.yaml | 32 +++++ .../sync-catalog-cleanup-serviceaccount.yaml | 19 +++ .../templates/sync-catalog-deployment.yaml | 5 +- 5 files changed, 300 insertions(+), 3 deletions(-) create mode 100644 charts/consul/templates/sync-catalog-cleanup-on-uninstall-job.yaml create mode 100644 charts/consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml create mode 100644 charts/consul/templates/sync-catalog-cleanup-podsecuritypolicy.yaml create mode 100644 charts/consul/templates/sync-catalog-cleanup-serviceaccount.yaml diff --git a/charts/consul/templates/sync-catalog-cleanup-on-uninstall-job.yaml b/charts/consul/templates/sync-catalog-cleanup-on-uninstall-job.yaml new file mode 100644 index 0000000000..0e4c6c3fff --- /dev/null +++ b/charts/consul/templates/sync-catalog-cleanup-on-uninstall-job.yaml @@ -0,0 +1,123 @@ +{{- if .Values.syncCatalog.purgeServicesOnDisable }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "consul.fullname" . }}-sync-catalog-cleanup-on-uninstall + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "consul.name" . }} + chart: {{ template "consul.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + component: sync-catalog-cleanup + {{- if .Values.global.extraLabels }} + {{- toYaml .Values.global.extraLabels | nindent 4 }} + {{- end }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed +spec: + template: + metadata: + name: {{ template "consul.fullname" . }}-sync-catalog-cleanup-on-uninstall + labels: + app: {{ template "consul.name" . }} + chart: {{ template "consul.chart" . }} + release: {{ .Release.Name }} + component: sync-catalog-cleanup + {{- if .Values.global.extraLabels }} + {{- toYaml .Values.global.extraLabels | nindent 8 }} + {{- end }} + annotations: + "consul.hashicorp.com/connect-inject": "false" + "consul.hashicorp.com/mesh-inject": "false" + spec: + restartPolicy: Never + serviceAccountName: {{ template "consul.fullname" . }}-sync-catalog-cleanup + containers: + - name: sync-catalog-cleanup-job + image: "{{ default .Values.global.imageK8S .Values.syncCatalog.image }}" + {{ template "consul.imagePullPolicy" . }} + {{- include "consul.restrictedSecurityContext" . | nindent 8 }} + env: + {{- include "consul.consulK8sConsulServerEnvVars" . | nindent 8 }} + {{- if .Values.global.acls.manageSystemACLs }} + - name: CONSUL_LOGIN_AUTH_METHOD + {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter .Values.global.enableConsulNamespaces }} + value: {{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }} + {{- else }} + value: {{ template "consul.fullname" . }}-k8s-component-auth-method + {{- end }} + - name: CONSUL_LOGIN_DATACENTER + {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter .Values.global.enableConsulNamespaces }} + value: {{ .Values.global.federation.primaryDatacenter }} + {{- else }} + value: {{ .Values.global.datacenter }} + {{- end }} + - name: CONSUL_LOGIN_META + value: "component=sync-catalog,pod=$(NAMESPACE)/$(POD_NAME)" + {{- end }} + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if (and .Values.syncCatalog.aclSyncToken.secretName .Values.syncCatalog.aclSyncToken.secretKey) }} + - name: CONSUL_ACL_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Values.syncCatalog.aclSyncToken.secretName }} + key: {{ .Values.syncCatalog.aclSyncToken.secretKey }} + {{- end }} + volumeMounts: + {{- if .Values.global.tls.enabled }} + {{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }} + - name: consul-ca-cert + mountPath: /consul/tls/ca + readOnly: true + {{- end }} + {{- end }} + command: + - "/bin/sh" + - "-ec" + - | + exec consul-k8s-control-plane sync-catalog \ + -log-level={{ default .Values.global.logLevel .Values.syncCatalog.logLevel }} \ + -log-json={{ .Values.global.logJSON }} \ + -k8s-default-sync={{ .Values.syncCatalog.default }} \ + {{- if .Values.global.adminPartitions.enabled }} + -partition={{ .Values.global.adminPartitions.name }} \ + {{- end }} + {{- if .Values.syncCatalog.metrics.enabled | default .Values.global.metrics.enabled }} + -enable-metrics \ + {{- end }} + {{- if .Values.syncCatalog.metrics.path }} + -metrics-path={{ .Values.syncCatalog.metrics.path }} \ + {{- end }} + {{- if .Values.syncCatalog.metrics.port }} + -metrics-port={{ .Values.syncCatalog.metrics.port }} \ + {{- end }} + -prometheus-retention-time={{ .Values.global.metrics.agentMetricsRetentionTime }} \ + {{- if .Release.IsUpgrade }} + -purge-k8s-services-from-node + {{- end }} + {{- if .Values.global.acls.tolerations }} + tolerations: + {{ tpl .Values.global.acls.tolerations . | indent 8 | trim }} + {{- end }} + volumes: + {{- if .Values.global.tls.enabled }} + {{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }} + - name: consul-ca-cert + secret: + {{- if .Values.global.tls.caCert.secretName }} + secretName: {{ .Values.global.tls.caCert.secretName }} + {{- else }} + secretName: {{ template "consul.fullname" . }}-ca-cert + {{- end }} + items: + - key: {{ default "tls.crt" .Values.global.tls.caCert.secretKey }} + path: tls.crt + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml b/charts/consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml new file mode 100644 index 0000000000..f9dfd1cccc --- /dev/null +++ b/charts/consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml @@ -0,0 +1,124 @@ +{{- $syncCatalogEnabled := (or (and (ne (.Values.syncCatalog.enabled | toString) "-") .Values.syncCatalog.enabled) (and (eq (.Values.syncCatalog.enabled | toString) "-") .Values.global.enabled)) }} +{{- if and (not $syncCatalogEnabled) .Values.syncCatalog.purgeServicesOnDisable }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "consul.fullname" . }}-sync-catalog-cleanup-on-upgrade + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "consul.name" . }} + chart: {{ template "consul.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + component: sync-catalog-cleanup + {{- if .Values.global.extraLabels }} + {{- toYaml .Values.global.extraLabels | nindent 4 }} + {{- end }} + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed +spec: + template: + metadata: + name: {{ template "consul.fullname" . }}-sync-catalog-cleanup-on-upgrade + labels: + app: {{ template "consul.name" . }} + chart: {{ template "consul.chart" . }} + release: {{ .Release.Name }} + component: sync-catalog-cleanup + {{- if .Values.global.extraLabels }} + {{- toYaml .Values.global.extraLabels | nindent 8 }} + {{- end }} + annotations: + "consul.hashicorp.com/connect-inject": "false" + "consul.hashicorp.com/mesh-inject": "false" + spec: + restartPolicy: Never + serviceAccountName: {{ template "consul.fullname" . }}-sync-catalog-cleanup + containers: + - name: sync-catalog-cleanup-job + image: "{{ default .Values.global.imageK8S .Values.syncCatalog.image }}" + {{ template "consul.imagePullPolicy" . }} + {{- include "consul.restrictedSecurityContext" . | nindent 8 }} + env: + {{- include "consul.consulK8sConsulServerEnvVars" . | nindent 8 }} + {{- if .Values.global.acls.manageSystemACLs }} + - name: CONSUL_LOGIN_AUTH_METHOD + {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter .Values.global.enableConsulNamespaces }} + value: {{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }} + {{- else }} + value: {{ template "consul.fullname" . }}-k8s-component-auth-method + {{- end }} + - name: CONSUL_LOGIN_DATACENTER + {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter .Values.global.enableConsulNamespaces }} + value: {{ .Values.global.federation.primaryDatacenter }} + {{- else }} + value: {{ .Values.global.datacenter }} + {{- end }} + - name: CONSUL_LOGIN_META + value: "component=sync-catalog,pod=$(NAMESPACE)/$(POD_NAME)" + {{- end }} + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if (and .Values.syncCatalog.aclSyncToken.secretName .Values.syncCatalog.aclSyncToken.secretKey) }} + - name: CONSUL_ACL_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Values.syncCatalog.aclSyncToken.secretName }} + key: {{ .Values.syncCatalog.aclSyncToken.secretKey }} + {{- end }} + volumeMounts: + {{- if .Values.global.tls.enabled }} + {{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }} + - name: consul-ca-cert + mountPath: /consul/tls/ca + readOnly: true + {{- end }} + {{- end }} + command: + - "/bin/sh" + - "-ec" + - | + exec consul-k8s-control-plane sync-catalog \ + -log-level={{ default .Values.global.logLevel .Values.syncCatalog.logLevel }} \ + -log-json={{ .Values.global.logJSON }} \ + -k8s-default-sync={{ .Values.syncCatalog.default }} \ + {{- if .Values.global.adminPartitions.enabled }} + -partition={{ .Values.global.adminPartitions.name }} \ + {{- end }} + {{- if .Values.syncCatalog.metrics.enabled | default .Values.global.metrics.enabled }} + -enable-metrics \ + {{- end }} + {{- if .Values.syncCatalog.metrics.path }} + -metrics-path={{ .Values.syncCatalog.metrics.path }} \ + {{- end }} + {{- if .Values.syncCatalog.metrics.port }} + -metrics-port={{ .Values.syncCatalog.metrics.port }} \ + {{- end }} + -prometheus-retention-time={{ .Values.global.metrics.agentMetricsRetentionTime }} \ + {{- if .Release.IsUpgrade }} + -purge-k8s-services-from-node + {{- end }} + {{- if .Values.global.acls.tolerations }} + tolerations: + {{ tpl .Values.global.acls.tolerations . | indent 8 | trim }} + {{- end }} + volumes: + {{- if .Values.global.tls.enabled }} + {{- if not (or (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) .Values.global.secretsBackend.vault.enabled) }} + - name: consul-ca-cert + secret: + {{- if .Values.global.tls.caCert.secretName }} + secretName: {{ .Values.global.tls.caCert.secretName }} + {{- else }} + secretName: {{ template "consul.fullname" . }}-ca-cert + {{- end }} + items: + - key: {{ default "tls.crt" .Values.global.tls.caCert.secretKey }} + path: tls.crt + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/consul/templates/sync-catalog-cleanup-podsecuritypolicy.yaml b/charts/consul/templates/sync-catalog-cleanup-podsecuritypolicy.yaml new file mode 100644 index 0000000000..c3c847b4ae --- /dev/null +++ b/charts/consul/templates/sync-catalog-cleanup-podsecuritypolicy.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.syncCatalog.purgeServicesOnDisable .Values.global.enablePodSecurityPolicies }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "consul.fullname" . }}-sync-catalog-cleanup + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "consul.name" . }} + chart: {{ template "consul.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + component: sync-catalog-cleanup +spec: + privileged: false + allowPrivilegeEscalation: false + # This is redundant with non-root + disallow privilege escalation, + # but we can provide it for defense in depth. + requiredDropCapabilities: + - ALL + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/consul/templates/sync-catalog-cleanup-serviceaccount.yaml b/charts/consul/templates/sync-catalog-cleanup-serviceaccount.yaml new file mode 100644 index 0000000000..162cdf6905 --- /dev/null +++ b/charts/consul/templates/sync-catalog-cleanup-serviceaccount.yaml @@ -0,0 +1,19 @@ +{{- if .Values.syncCatalog.purgeServicesOnDisable }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "consul.fullname" . }}-sync-catalog-cleanup + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "consul.name" . }} + chart: {{ template "consul.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + component: sync-catalog-cleanup +{{- with .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range . }} + - name: {{ .name }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/consul/templates/sync-catalog-deployment.yaml b/charts/consul/templates/sync-catalog-deployment.yaml index 8a54dd83b0..5e69110810 100644 --- a/charts/consul/templates/sync-catalog-deployment.yaml +++ b/charts/consul/templates/sync-catalog-deployment.yaml @@ -1,5 +1,5 @@ -{{ - $syncCatalogEnabled := (or (and (ne (.Values.syncCatalog.enabled | toString) "-") .Values.syncCatalog.enabled) (and (eq (.Values.syncCatalog.enabled | toString) "-") .Values.global.enabled)) }} -{{- if (or $syncCatalogEnabled .Values.syncCatalog.purgeServicesOnDisable) }} +{{- $syncCatalogEnabled := (or (and (ne (.Values.syncCatalog.enabled | toString) "-") .Values.syncCatalog.enabled) (and (eq (.Values.syncCatalog.enabled | toString) "-") .Values.global.enabled)) }} +{{- if $syncCatalogEnabled }} {{- template "consul.reservedNamesFailer" (list .Values.syncCatalog.consulNamespaces.consulDestinationNamespace "syncCatalog.consulNamespaces.consulDestinationNamespace") }} {{ template "consul.validateRequiredCloudSecretsExist" . }} {{ template "consul.validateCloudSecretKeys" . }} @@ -214,7 +214,6 @@ spec: -metrics-port={{ .Values.syncCatalog.metrics.port }} \ {{- end }} -prometheus-retention-time={{ .Values.global.metrics.agentMetricsRetentionTime }} \ - -purgeServicesOnDisable={{ and (not $syncCatalogEnabled) .Values.syncCatalog.purgeServicesOnDisable }} \ livenessProbe: httpGet: path: /health/ready From 20c63219a5c7273f8b6a030003231865376ec825 Mon Sep 17 00:00:00 2001 From: jm96441n Date: Thu, 17 Oct 2024 11:22:01 -0400 Subject: [PATCH 06/11] Check error --- control-plane/subcommand/sync-catalog/command_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/control-plane/subcommand/sync-catalog/command_test.go b/control-plane/subcommand/sync-catalog/command_test.go index 94f5452ad9..8c06c80f12 100644 --- a/control-plane/subcommand/sync-catalog/command_test.go +++ b/control-plane/subcommand/sync-catalog/command_test.go @@ -609,6 +609,8 @@ func TestRemoveAllK8SServicesFromConsul(t *testing.T) { // create another partition and register 2 services there to the same node to show that we won't delete them _, _, err = consulClient.Partitions().Create(context.Background(), &api.Partition{Name: otherPartitionName}, nil) + require.NoError(t, err) + _, err = consulClient.Catalog().Register( &api.CatalogRegistration{ Node: tc.nodeName, From aed70ff428d3d42a5816176b4e4a76c15b526734 Mon Sep 17 00:00:00 2001 From: jm96441n Date: Mon, 21 Oct 2024 12:33:44 -0400 Subject: [PATCH 07/11] fix tests --- .../sync-catalog/command_ent_test.go | 150 +++++++++++++++++- .../subcommand/sync-catalog/command_test.go | 85 ++++------ 2 files changed, 178 insertions(+), 57 deletions(-) diff --git a/control-plane/subcommand/sync-catalog/command_ent_test.go b/control-plane/subcommand/sync-catalog/command_ent_test.go index f8bdccda6b..231b2755c6 100644 --- a/control-plane/subcommand/sync-catalog/command_ent_test.go +++ b/control-plane/subcommand/sync-catalog/command_ent_test.go @@ -272,7 +272,6 @@ func TestRun_ToConsulMirroringNamespaces(t *testing.T) { } } - }) }) } @@ -706,3 +705,152 @@ func TestRun_ToConsulNamespacesACLs(t *testing.T) { }) } } + +// Test services could be de-registered from Consul. +func TestRemoveAllK8SServicesFromConsulWithPartitions(t *testing.T) { + testCases := map[string]struct { + nodeName string + partitionName string + }{ + "default Node name in default partition": { + nodeName: "k8s-sync", + partitionName: "default", + }, + "non-default Node name in default partition": { + nodeName: "custom-node", + partitionName: "default", + }, + "default Node name in non-default partition": { + nodeName: "k8s-sync", + partitionName: "part-1", + }, + "non-default Node name in non-default partition": { + nodeName: "custom-node", + partitionName: "part-1", + }, + } + + for name, tc := range testCases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + var err error + + k8s, testClient := completeSetup(t) + consulClient := testClient.APIClient + + // Create a mock reader to simulate user input + // Run the command. + ui := cli.NewMockUi() + cmd := Command{ + UI: ui, + clientset: k8s, + logger: hclog.New(&hclog.LoggerOptions{ + Name: t.Name(), + Level: hclog.Debug, + }), + flagAllowK8sNamespacesList: []string{"*"}, + connMgr: testClient.Watcher, + } + + otherPartitionName := "the-other-one" + + // create another partition and register 2 services there to the same node to show that we won't delete them + _, _, err = consulClient.Partitions().Create(context.Background(), &api.Partition{Name: otherPartitionName}, nil) + require.NoError(t, err) + + _, err = consulClient.Catalog().Register( + &api.CatalogRegistration{ + Node: tc.nodeName, + Address: "5.5.5.5", + Service: &api.AgentService{ + ID: "other-service-1", + Service: "other-service-1", + Tags: []string{"other-k8s-cluster"}, + Meta: map[string]string{}, + Port: 0, + Address: "5.5.5.5", + Partition: otherPartitionName, + }, + Partition: otherPartitionName, + }, + &api.WriteOptions{Partition: otherPartitionName}, + ) + require.NoError(t, err) + + _, err = consulClient.Catalog().Register( + &api.CatalogRegistration{ + Node: tc.nodeName, + Address: "6.6.6.6", + Service: &api.AgentService{ + ID: "other-service-2", + Service: "other-service-2", + Tags: []string{"other-k8s-cluster"}, + Meta: map[string]string{}, + Port: 0, + Address: "6.6.6.6", + Partition: otherPartitionName, + }, + Partition: otherPartitionName, + }, + &api.WriteOptions{Partition: otherPartitionName}, + ) + require.NoError(t, err) + + // create partition if it does not exist + if tc.partitionName != "default" { + p, _, err := consulClient.Partitions().Create(context.Background(), &api.Partition{Name: tc.partitionName}, nil) + require.NoError(t, err) + require.NotNil(t, p) + } + + // create two services in k8s + _, err = k8s.CoreV1().Services("bar").Create(context.Background(), lbService("foo", "1.1.1.1"), metav1.CreateOptions{}) + require.NoError(t, err) + + _, err = k8s.CoreV1().Services("baz").Create(context.Background(), lbService("foo", "2.2.2.2"), metav1.CreateOptions{}) + require.NoError(t, err) + + longRunningChan := runCommandAsynchronously(&cmd, []string{ + "-addresses", "127.0.0.1", + "-http-port", strconv.Itoa(testClient.Cfg.HTTPPort), + "-consul-write-interval", "100ms", + "-partition", tc.partitionName, + "-consul-node-name", tc.nodeName, + "-add-k8s-namespace-suffix", + }) + + // check that the two K8s services have been synced into Consul + retry.Run(t, func(r *retry.R) { + svc, _, err := consulClient.Catalog().Service("foo-bar", "k8s", &api.QueryOptions{Partition: tc.partitionName}) + require.NoError(r, err) + require.Len(r, svc, 1) + require.Equal(r, "1.1.1.1", svc[0].ServiceAddress) + svc, _, err = consulClient.Catalog().Service("foo-baz", "k8s", &api.QueryOptions{Partition: tc.partitionName}) + require.NoError(r, err) + require.Len(r, svc, 1) + require.Equal(r, "2.2.2.2", svc[0].ServiceAddress) + }) + + defer stopCommand(t, &cmd, longRunningChan) + + exitChan := runCommandAsynchronously(&cmd, []string{ + "-addresses", "127.0.0.1", + "-http-port", strconv.Itoa(testClient.Cfg.HTTPPort), + "-purge-k8s-services-from-node=true", + "-partition", tc.partitionName, + "-consul-node-name", tc.nodeName, + }) + stopCommand(t, &cmd, exitChan) + + retry.Run(t, func(r *retry.R) { + serviceList, _, err := consulClient.Catalog().NodeServiceList(tc.nodeName, &api.QueryOptions{AllowStale: false, Partition: tc.partitionName}) + require.NoError(r, err) + require.Len(r, serviceList.Services, 0) + otherPartitionServiceList, _, err := consulClient.Catalog().NodeServiceList(tc.nodeName, &api.QueryOptions{AllowStale: false, Partition: otherPartitionName}) + require.NoError(r, err) + require.Len(r, otherPartitionServiceList.Services, 2) + }) + }) + } +} diff --git a/control-plane/subcommand/sync-catalog/command_test.go b/control-plane/subcommand/sync-catalog/command_test.go index 8c06c80f12..d4d011b7e0 100644 --- a/control-plane/subcommand/sync-catalog/command_test.go +++ b/control-plane/subcommand/sync-catalog/command_test.go @@ -561,27 +561,18 @@ func TestRun_ToConsulChangingFlags(t *testing.T) { // Test services could be de-registered from Consul. func TestRemoveAllK8SServicesFromConsul(t *testing.T) { testCases := map[string]struct { - nodeName string - partitionName string + nodeToDeregisterName string }{ "default Node name in default partition": { - nodeName: "k8s-sync", - partitionName: "default", + nodeToDeregisterName: "k8s-sync", }, "non-default Node name in default partition": { - nodeName: "custom-node", - partitionName: "default", - }, - "default Node name in non-default partition": { - nodeName: "k8s-sync", - partitionName: "part-1", - }, - "non-default Node name in non-default partition": { - nodeName: "custom-node", - partitionName: "part-1", + nodeToDeregisterName: "custom-node", }, } + otherNodeName := "other-node" + for name, tc := range testCases { tc := tc t.Run(name, func(t *testing.T) { @@ -605,57 +596,40 @@ func TestRemoveAllK8SServicesFromConsul(t *testing.T) { connMgr: testClient.Watcher, } - otherPartitionName := "the-other-one" - - // create another partition and register 2 services there to the same node to show that we won't delete them - _, _, err = consulClient.Partitions().Create(context.Background(), &api.Partition{Name: otherPartitionName}, nil) - require.NoError(t, err) - _, err = consulClient.Catalog().Register( &api.CatalogRegistration{ - Node: tc.nodeName, + Node: otherNodeName, Address: "5.5.5.5", Service: &api.AgentService{ - ID: "other-service-1", - Service: "other-service-1", - Tags: []string{"other-k8s-cluster"}, - Meta: map[string]string{}, - Port: 0, - Address: "5.5.5.5", - Partition: otherPartitionName, + ID: "other-service-1", + Service: "other-service-1", + Tags: []string{"other-k8s-cluster"}, + Meta: map[string]string{}, + Port: 0, + Address: "5.5.5.5", }, - Partition: otherPartitionName, }, - &api.WriteOptions{Partition: otherPartitionName}, + &api.WriteOptions{}, ) require.NoError(t, err) _, err = consulClient.Catalog().Register( &api.CatalogRegistration{ - Node: tc.nodeName, + Node: otherNodeName, Address: "6.6.6.6", Service: &api.AgentService{ - ID: "other-service-2", - Service: "other-service-2", - Tags: []string{"other-k8s-cluster"}, - Meta: map[string]string{}, - Port: 0, - Address: "6.6.6.6", - Partition: otherPartitionName, + ID: "other-service-2", + Service: "other-service-2", + Tags: []string{"other-k8s-cluster"}, + Meta: map[string]string{}, + Port: 0, + Address: "6.6.6.6", }, - Partition: otherPartitionName, }, - &api.WriteOptions{Partition: otherPartitionName}, + &api.WriteOptions{}, ) require.NoError(t, err) - // create partition if it does not exist - if tc.partitionName != "default" { - p, _, err := consulClient.Partitions().Create(context.Background(), &api.Partition{Name: tc.partitionName}, nil) - require.NoError(t, err) - require.NotNil(t, p) - } - // create two services in k8s _, err = k8s.CoreV1().Services("bar").Create(context.Background(), lbService("foo", "1.1.1.1"), metav1.CreateOptions{}) require.NoError(t, err) @@ -667,18 +641,17 @@ func TestRemoveAllK8SServicesFromConsul(t *testing.T) { "-addresses", "127.0.0.1", "-http-port", strconv.Itoa(testClient.Cfg.HTTPPort), "-consul-write-interval", "100ms", - "-partition", tc.partitionName, - "-consul-node-name", tc.nodeName, + "-consul-node-name", tc.nodeToDeregisterName, "-add-k8s-namespace-suffix", }) // check that the two K8s services have been synced into Consul retry.Run(t, func(r *retry.R) { - svc, _, err := consulClient.Catalog().Service("foo-bar", "k8s", &api.QueryOptions{Partition: tc.partitionName}) + svc, _, err := consulClient.Catalog().Service("foo-bar", "k8s", &api.QueryOptions{}) require.NoError(r, err) require.Len(r, svc, 1) require.Equal(r, "1.1.1.1", svc[0].ServiceAddress) - svc, _, err = consulClient.Catalog().Service("foo-baz", "k8s", &api.QueryOptions{Partition: tc.partitionName}) + svc, _, err = consulClient.Catalog().Service("foo-baz", "k8s", &api.QueryOptions{}) require.NoError(r, err) require.Len(r, svc, 1) require.Equal(r, "2.2.2.2", svc[0].ServiceAddress) @@ -690,18 +663,18 @@ func TestRemoveAllK8SServicesFromConsul(t *testing.T) { "-addresses", "127.0.0.1", "-http-port", strconv.Itoa(testClient.Cfg.HTTPPort), "-purge-k8s-services-from-node=true", - "-partition", tc.partitionName, - "-consul-node-name", tc.nodeName, + "-consul-node-name", tc.nodeToDeregisterName, }) stopCommand(t, &cmd, exitChan) retry.Run(t, func(r *retry.R) { - serviceList, _, err := consulClient.Catalog().NodeServiceList(tc.nodeName, &api.QueryOptions{AllowStale: false, Partition: tc.partitionName}) + serviceList, _, err := consulClient.Catalog().NodeServiceList(tc.nodeToDeregisterName, &api.QueryOptions{AllowStale: false}) require.NoError(r, err) require.Len(r, serviceList.Services, 0) - otherPartitionServiceList, _, err := consulClient.Catalog().NodeServiceList(tc.nodeName, &api.QueryOptions{AllowStale: false, Partition: otherPartitionName}) + + otherNodeServiceList, _, err := consulClient.Catalog().NodeServiceList(otherNodeName, &api.QueryOptions{AllowStale: false}) require.NoError(r, err) - require.Len(r, otherPartitionServiceList.Services, 2) + require.Len(r, otherNodeServiceList.Services, 2) }) }) } From 7609190f1af2e6aac44937f6d2ca490111cfd66c Mon Sep 17 00:00:00 2001 From: jm96441n Date: Mon, 21 Oct 2024 15:27:46 -0400 Subject: [PATCH 08/11] rename field in values file, remove references to pod security policy --- .../templates/sync-catalog-cleanup-on-uninstall-job.yaml | 4 +--- .../consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml | 2 +- .../templates/sync-catalog-cleanup-podsecuritypolicy.yaml | 2 +- .../consul/templates/sync-catalog-cleanup-serviceaccount.yaml | 2 +- charts/consul/values.yaml | 2 +- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/charts/consul/templates/sync-catalog-cleanup-on-uninstall-job.yaml b/charts/consul/templates/sync-catalog-cleanup-on-uninstall-job.yaml index 0e4c6c3fff..626b325985 100644 --- a/charts/consul/templates/sync-catalog-cleanup-on-uninstall-job.yaml +++ b/charts/consul/templates/sync-catalog-cleanup-on-uninstall-job.yaml @@ -1,4 +1,4 @@ -{{- if .Values.syncCatalog.purgeServicesOnDisable }} +{{- if .Values.syncCatalog.cleanupNodeOnRemoval }} apiVersion: batch/v1 kind: Job metadata: @@ -98,9 +98,7 @@ spec: -metrics-port={{ .Values.syncCatalog.metrics.port }} \ {{- end }} -prometheus-retention-time={{ .Values.global.metrics.agentMetricsRetentionTime }} \ - {{- if .Release.IsUpgrade }} -purge-k8s-services-from-node - {{- end }} {{- if .Values.global.acls.tolerations }} tolerations: {{ tpl .Values.global.acls.tolerations . | indent 8 | trim }} diff --git a/charts/consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml b/charts/consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml index f9dfd1cccc..a14d4a9a9f 100644 --- a/charts/consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml +++ b/charts/consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml @@ -1,5 +1,5 @@ {{- $syncCatalogEnabled := (or (and (ne (.Values.syncCatalog.enabled | toString) "-") .Values.syncCatalog.enabled) (and (eq (.Values.syncCatalog.enabled | toString) "-") .Values.global.enabled)) }} -{{- if and (not $syncCatalogEnabled) .Values.syncCatalog.purgeServicesOnDisable }} +{{- if and (not $syncCatalogEnabled) .Values.syncCatalog.cleanupNodeOnRemoval }} apiVersion: batch/v1 kind: Job metadata: diff --git a/charts/consul/templates/sync-catalog-cleanup-podsecuritypolicy.yaml b/charts/consul/templates/sync-catalog-cleanup-podsecuritypolicy.yaml index c3c847b4ae..7050250653 100644 --- a/charts/consul/templates/sync-catalog-cleanup-podsecuritypolicy.yaml +++ b/charts/consul/templates/sync-catalog-cleanup-podsecuritypolicy.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.syncCatalog.purgeServicesOnDisable .Values.global.enablePodSecurityPolicies }} +{{- if and .Values.syncCatalog.cleanupNodeOnRemoval }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: diff --git a/charts/consul/templates/sync-catalog-cleanup-serviceaccount.yaml b/charts/consul/templates/sync-catalog-cleanup-serviceaccount.yaml index 162cdf6905..b4bd141ed7 100644 --- a/charts/consul/templates/sync-catalog-cleanup-serviceaccount.yaml +++ b/charts/consul/templates/sync-catalog-cleanup-serviceaccount.yaml @@ -1,4 +1,4 @@ -{{- if .Values.syncCatalog.purgeServicesOnDisable }} +{{- if .Values.syncCatalog.cleanupNodeOnRemoval }} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml index 66115bad7a..0415904c8a 100644 --- a/charts/consul/values.yaml +++ b/charts/consul/values.yaml @@ -2092,7 +2092,7 @@ syncCatalog: # True if you want to deregister all services in this cluster from consul that have been registered by the catalog sync when the `enabled` flag is set to false. # @type: boolean - purgeServicesOnDisable: false + cleanupNodeOnRemoval: false # The name of the Docker image (including any tag) for consul-k8s-control-plane # to run the sync program. From 23072d9bbf2b7d8f1a5cb92f09e74f34cfcbc349 Mon Sep 17 00:00:00 2001 From: jm96441n Date: Mon, 21 Oct 2024 15:38:03 -0400 Subject: [PATCH 09/11] remove psp --- ...ync-catalog-cleanup-podsecuritypolicy.yaml | 32 ------------------- 1 file changed, 32 deletions(-) delete mode 100644 charts/consul/templates/sync-catalog-cleanup-podsecuritypolicy.yaml diff --git a/charts/consul/templates/sync-catalog-cleanup-podsecuritypolicy.yaml b/charts/consul/templates/sync-catalog-cleanup-podsecuritypolicy.yaml deleted file mode 100644 index 7050250653..0000000000 --- a/charts/consul/templates/sync-catalog-cleanup-podsecuritypolicy.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if and .Values.syncCatalog.cleanupNodeOnRemoval }} -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: {{ template "consul.fullname" . }}-sync-catalog-cleanup - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "consul.name" . }} - chart: {{ template "consul.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - component: sync-catalog-cleanup -spec: - privileged: false - allowPrivilegeEscalation: false - # This is redundant with non-root + disallow privilege escalation, - # but we can provide it for defense in depth. - requiredDropCapabilities: - - ALL - hostNetwork: false - hostIPC: false - hostPID: false - runAsUser: - rule: 'RunAsAny' - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'RunAsAny' - fsGroup: - rule: 'RunAsAny' - readOnlyRootFilesystem: false -{{- end }} From a0d8b91f05ef853a5653393647d00948329cea2c Mon Sep 17 00:00:00 2001 From: jm96441n Date: Mon, 21 Oct 2024 16:33:17 -0400 Subject: [PATCH 10/11] added bats testing --- ...sync-catalog-cleanup-on-uninstall-job.yaml | 57 ++++++++++++++++- .../sync-catalog-cleanup-on-upgrade-job.yaml | 61 +++++++++++++++++-- .../sync-catalog-cleanup-serviceaccount.yaml | 4 ++ 3 files changed, 115 insertions(+), 7 deletions(-) diff --git a/charts/consul/templates/sync-catalog-cleanup-on-uninstall-job.yaml b/charts/consul/templates/sync-catalog-cleanup-on-uninstall-job.yaml index 626b325985..955042dd0a 100644 --- a/charts/consul/templates/sync-catalog-cleanup-on-uninstall-job.yaml +++ b/charts/consul/templates/sync-catalog-cleanup-on-uninstall-job.yaml @@ -26,12 +26,42 @@ spec: chart: {{ template "consul.chart" . }} release: {{ .Release.Name }} component: sync-catalog-cleanup + {{- if .Values.syncCatalog.extraLabels }} + {{- toYaml .Values.syncCatalog.extraLabels | nindent 8 }} + {{- end }} {{- if .Values.global.extraLabels }} {{- toYaml .Values.global.extraLabels | nindent 8 }} {{- end }} annotations: "consul.hashicorp.com/connect-inject": "false" "consul.hashicorp.com/mesh-inject": "false" + {{- if .Values.syncCatalog.annotations }} + {{- tpl .Values.syncCatalog.annotations . | nindent 8 }} + {{- end }} + {{- if (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) }} + "vault.hashicorp.com/agent-init-first": "true" + "vault.hashicorp.com/agent-inject": "true" + "vault.hashicorp.com/role": {{ .Values.global.secretsBackend.vault.consulCARole }} + "vault.hashicorp.com/agent-inject-secret-serverca.crt": {{ .Values.global.tls.caCert.secretName }} + "vault.hashicorp.com/agent-inject-template-serverca.crt": {{ template "consul.serverTLSCATemplate" . }} + {{- if and .Values.global.secretsBackend.vault.ca.secretName .Values.global.secretsBackend.vault.ca.secretKey }} + "vault.hashicorp.com/agent-extra-secret": "{{ .Values.global.secretsBackend.vault.ca.secretName }}" + "vault.hashicorp.com/ca-cert": "/vault/custom/{{ .Values.global.secretsBackend.vault.ca.secretKey }}" + {{- end }} + {{- if .Values.global.secretsBackend.vault.agentAnnotations }} + {{ tpl .Values.global.secretsBackend.vault.agentAnnotations . | nindent 8 | trim }} + {{- end }} + {{- if (and (.Values.global.secretsBackend.vault.vaultNamespace) (not (hasKey (default "" .Values.global.secretsBackend.vault.agentAnnotations | fromYaml) "vault.hashicorp.com/namespace")))}} + "vault.hashicorp.com/namespace": "{{ .Values.global.secretsBackend.vault.vaultNamespace }}" + {{- end }} + {{- end }} + {{- if .Values.syncCatalog.metrics.enabled | default .Values.global.metrics.enabled }} + "prometheus.io/scrape": "true" + {{- if not (hasKey (default "" .Values.syncCatalog.annotations | fromYaml) "prometheus.io/path")}} + "prometheus.io/path": {{ default "/metrics" .Values.syncCatalog.metrics.path }} + {{- end }} + "prometheus.io/port": {{ .Values.syncCatalog.metrics.port | default "20300" | quote }} + {{- end }} spec: restartPolicy: Never serviceAccountName: {{ template "consul.fullname" . }}-sync-catalog-cleanup @@ -88,6 +118,9 @@ spec: {{- if .Values.global.adminPartitions.enabled }} -partition={{ .Values.global.adminPartitions.name }} \ {{- end }} + {{- if .Values.syncCatalog.consulNodeName }} + -consul-node-name={{ .Values.syncCatalog.consulNodeName }} \ + {{- end }} {{- if .Values.syncCatalog.metrics.enabled | default .Values.global.metrics.enabled }} -enable-metrics \ {{- end }} @@ -99,9 +132,29 @@ spec: {{- end }} -prometheus-retention-time={{ .Values.global.metrics.agentMetricsRetentionTime }} \ -purge-k8s-services-from-node - {{- if .Values.global.acls.tolerations }} + {{- with .Values.syncCatalog.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if or (eq (.Values.syncCatalog.metrics.enabled | toString) "-") .Values.syncCatalog.metrics.enabled .Values.global.metrics.enabled }} + ports: + - name: prometheus + containerPort: {{ .Values.syncCatalog.metrics.port | default "20300" | int }} + {{- end }} + {{- if .Values.syncCatalog.priorityClassName }} + priorityClassName: {{ .Values.syncCatalog.priorityClassName | quote }} + {{- end }} + {{- if .Values.syncCatalog.nodeSelector }} + nodeSelector: + {{ tpl .Values.syncCatalog.nodeSelector . | indent 8 | trim }} + {{- end }} + {{- if .Values.syncCatalog.affinity }} + affinity: + {{ tpl .Values.syncCatalog.affinity . | indent 8 | trim }} + {{- end }} + {{- if .Values.syncCatalog.tolerations }} tolerations: - {{ tpl .Values.global.acls.tolerations . | indent 8 | trim }} + {{ tpl .Values.syncCatalog.tolerations . | indent 8 | trim }} {{- end }} volumes: {{- if .Values.global.tls.enabled }} diff --git a/charts/consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml b/charts/consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml index a14d4a9a9f..2349c513d4 100644 --- a/charts/consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml +++ b/charts/consul/templates/sync-catalog-cleanup-on-upgrade-job.yaml @@ -1,5 +1,5 @@ {{- $syncCatalogEnabled := (or (and (ne (.Values.syncCatalog.enabled | toString) "-") .Values.syncCatalog.enabled) (and (eq (.Values.syncCatalog.enabled | toString) "-") .Values.global.enabled)) }} -{{- if and (not $syncCatalogEnabled) .Values.syncCatalog.cleanupNodeOnRemoval }} +{{- if and (not $syncCatalogEnabled) .Values.syncCatalog.cleanupNodeOnRemoval .Release.IsUpgrade }} apiVersion: batch/v1 kind: Job metadata: @@ -27,12 +27,42 @@ spec: chart: {{ template "consul.chart" . }} release: {{ .Release.Name }} component: sync-catalog-cleanup + {{- if .Values.syncCatalog.extraLabels }} + {{- toYaml .Values.syncCatalog.extraLabels | nindent 8 }} + {{- end }} {{- if .Values.global.extraLabels }} {{- toYaml .Values.global.extraLabels | nindent 8 }} {{- end }} annotations: "consul.hashicorp.com/connect-inject": "false" "consul.hashicorp.com/mesh-inject": "false" + {{- if .Values.syncCatalog.annotations }} + {{- tpl .Values.syncCatalog.annotations . | nindent 8 }} + {{- end }} + {{- if (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) }} + "vault.hashicorp.com/agent-init-first": "true" + "vault.hashicorp.com/agent-inject": "true" + "vault.hashicorp.com/role": {{ .Values.global.secretsBackend.vault.consulCARole }} + "vault.hashicorp.com/agent-inject-secret-serverca.crt": {{ .Values.global.tls.caCert.secretName }} + "vault.hashicorp.com/agent-inject-template-serverca.crt": {{ template "consul.serverTLSCATemplate" . }} + {{- if and .Values.global.secretsBackend.vault.ca.secretName .Values.global.secretsBackend.vault.ca.secretKey }} + "vault.hashicorp.com/agent-extra-secret": "{{ .Values.global.secretsBackend.vault.ca.secretName }}" + "vault.hashicorp.com/ca-cert": "/vault/custom/{{ .Values.global.secretsBackend.vault.ca.secretKey }}" + {{- end }} + {{- if .Values.global.secretsBackend.vault.agentAnnotations }} + {{ tpl .Values.global.secretsBackend.vault.agentAnnotations . | nindent 8 | trim }} + {{- end }} + {{- if (and (.Values.global.secretsBackend.vault.vaultNamespace) (not (hasKey (default "" .Values.global.secretsBackend.vault.agentAnnotations | fromYaml) "vault.hashicorp.com/namespace")))}} + "vault.hashicorp.com/namespace": "{{ .Values.global.secretsBackend.vault.vaultNamespace }}" + {{- end }} + {{- end }} + {{- if .Values.syncCatalog.metrics.enabled | default .Values.global.metrics.enabled }} + "prometheus.io/scrape": "true" + {{- if not (hasKey (default "" .Values.syncCatalog.annotations | fromYaml) "prometheus.io/path")}} + "prometheus.io/path": {{ default "/metrics" .Values.syncCatalog.metrics.path }} + {{- end }} + "prometheus.io/port": {{ .Values.syncCatalog.metrics.port | default "20300" | quote }} + {{- end }} spec: restartPolicy: Never serviceAccountName: {{ template "consul.fullname" . }}-sync-catalog-cleanup @@ -89,6 +119,9 @@ spec: {{- if .Values.global.adminPartitions.enabled }} -partition={{ .Values.global.adminPartitions.name }} \ {{- end }} + {{- if .Values.syncCatalog.consulNodeName }} + -consul-node-name={{ .Values.syncCatalog.consulNodeName }} \ + {{- end }} {{- if .Values.syncCatalog.metrics.enabled | default .Values.global.metrics.enabled }} -enable-metrics \ {{- end }} @@ -99,12 +132,30 @@ spec: -metrics-port={{ .Values.syncCatalog.metrics.port }} \ {{- end }} -prometheus-retention-time={{ .Values.global.metrics.agentMetricsRetentionTime }} \ - {{- if .Release.IsUpgrade }} -purge-k8s-services-from-node - {{- end }} - {{- if .Values.global.acls.tolerations }} + {{- with .Values.syncCatalog.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- if or (eq (.Values.syncCatalog.metrics.enabled | toString) "-") .Values.syncCatalog.metrics.enabled .Values.global.metrics.enabled }} + ports: + - name: prometheus + containerPort: {{ .Values.syncCatalog.metrics.port | default "20300" | int }} + {{- end }} + {{- if .Values.syncCatalog.priorityClassName }} + priorityClassName: {{ .Values.syncCatalog.priorityClassName | quote }} + {{- end }} + {{- if .Values.syncCatalog.nodeSelector }} + nodeSelector: + {{ tpl .Values.syncCatalog.nodeSelector . | indent 8 | trim }} + {{- end }} + {{- if .Values.syncCatalog.affinity }} + affinity: + {{ tpl .Values.syncCatalog.affinity . | indent 8 | trim }} + {{- end }} + {{- if .Values.syncCatalog.tolerations }} tolerations: - {{ tpl .Values.global.acls.tolerations . | indent 8 | trim }} + {{ tpl .Values.syncCatalog.tolerations . | indent 8 | trim }} {{- end }} volumes: {{- if .Values.global.tls.enabled }} diff --git a/charts/consul/templates/sync-catalog-cleanup-serviceaccount.yaml b/charts/consul/templates/sync-catalog-cleanup-serviceaccount.yaml index b4bd141ed7..c49852b14c 100644 --- a/charts/consul/templates/sync-catalog-cleanup-serviceaccount.yaml +++ b/charts/consul/templates/sync-catalog-cleanup-serviceaccount.yaml @@ -10,6 +10,10 @@ metadata: heritage: {{ .Release.Service }} release: {{ .Release.Name }} component: sync-catalog-cleanup + {{- if .Values.syncCatalog.serviceAccount.annotations }} + annotations: + {{ tpl .Values.syncCatalog.serviceAccount.annotations . | nindent 4 | trim }} + {{- end }} {{- with .Values.global.imagePullSecrets }} imagePullSecrets: {{- range . }} From e2b5746a299241e97fa83f2c9444935054fe2698 Mon Sep 17 00:00:00 2001 From: jm96441n Date: Tue, 22 Oct 2024 12:06:47 -0400 Subject: [PATCH 11/11] added bats testing --- ...sync-catalog-cleanup-on-uninstall-job.bats | 839 ++++++++++++++++ .../sync-catalog-cleanup-on-upgrade-job.bats | 896 ++++++++++++++++++ .../sync-catalog-cleanup-serviceaccount.bats | 75 ++ 3 files changed, 1810 insertions(+) create mode 100644 charts/consul/test/unit/sync-catalog-cleanup-on-uninstall-job.bats create mode 100644 charts/consul/test/unit/sync-catalog-cleanup-on-upgrade-job.bats create mode 100644 charts/consul/test/unit/sync-catalog-cleanup-serviceaccount.bats diff --git a/charts/consul/test/unit/sync-catalog-cleanup-on-uninstall-job.bats b/charts/consul/test/unit/sync-catalog-cleanup-on-uninstall-job.bats new file mode 100644 index 0000000000..f7f4935ec3 --- /dev/null +++ b/charts/consul/test/unit/sync-catalog-cleanup-on-uninstall-job.bats @@ -0,0 +1,839 @@ +#!/usr/bin/env bats + +load _helpers + +target=templates/sync-catalog-cleanup-on-uninstall-job.yaml + +@test "syncCatalogCleanupJob/Uninstall: disabled by default" { + cd $(chart_dir) + assert_empty helm template \ + -s $target \ + . +} + +@test "syncCatalogCleanupJob/Uninstall: enable with syncCatalog.cleanupNodeOnRemoval true" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq -s 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# image + +@test "syncCatalogCleanupJob/Uninstall: image defaults to global.imageK8S" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'global.imageK8S=bar' \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[0].image' | tee /dev/stderr) + [ "${actual}" = "bar" ] +} + +@test "syncCatalogCleanupJob/Uninstall: image can be overridden with server.image" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'global.imageK8S=foo' \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.image=bar' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[0].image' | tee /dev/stderr) + [ "${actual}" = "bar" ] +} + +@test "syncCatalogCleanupJob/Uninstall: consul env defaults" { + cd $(chart_dir) + local env=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].env[]' | tee /dev/stderr) + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_ADDRESSES").value' | tee /dev/stderr) + [ "${actual}" = "release-name-consul-server.default.svc" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_GRPC_PORT").value' | tee /dev/stderr) + [ "${actual}" = "8502" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_HTTP_PORT").value' | tee /dev/stderr) + [ "${actual}" = "8500" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_DATACENTER").value' | tee /dev/stderr) + [ "${actual}" = "dc1" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_API_TIMEOUT").value' | tee /dev/stderr) + [ "${actual}" = "5s" ] +} + +#-------------------------------------------------------------------- +# consulNodeName + +@test "syncCatalogCleanupJob/Uninstall: consulNodeName defaults to k8s-sync" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command | any(contains("-consul-node-name=k8s-sync"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "syncCatalogCleanupJob/Uninstall: consulNodeName set to empty" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.consulNodeName=' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command | any(contains("-consul-node-name"))' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + +@test "syncCatalogCleanupJob/Uninstall: can specify consulNodeName" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.consulNodeName=aNodeName' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command | any(contains("-consul-node-name=aNodeName"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# serviceAccount + +@test "syncCatalogCleanupJob/Uninstall: serviceAccount set when sync enabled" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.serviceAccountName | contains("sync-catalog-cleanup")' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# aclSyncToken + +@test "syncCatalogCleanupJob/Uninstall: aclSyncToken disabled when secretName is missing" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.aclSyncToken.secretKey=bar' \ + . | tee /dev/stderr | + yq '[.spec.template.spec.containers[0].env[].name] | any(contains("CONSUL_ACL_TOKEN"))' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + +@test "syncCatalogCleanupJob/Uninstall: aclSyncToken disabled when secretKey is missing" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.aclSyncToken.secretName=foo' \ + . | tee /dev/stderr | + yq '[.spec.template.spec.containers[0].env[].name] | any(contains("CONSUL_ACL_TOKEN"))' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + +@test "syncCatalogCleanupJob/Uninstall: aclSyncToken enabled when secretName and secretKey is provided" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.aclSyncToken.secretName=foo' \ + --set 'syncCatalog.aclSyncToken.secretKey=bar' \ + . | tee /dev/stderr | + yq '[.spec.template.spec.containers[0].env[].name] | any(contains("CONSUL_ACL_TOKEN"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# affinity + +@test "syncCatalogCleanupJob/Uninstall: affinity not set by default" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.affinity == null' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "syncCatalogCleanupJob/Uninstall: affinity can be set" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.affinity=foobar' \ + . | tee /dev/stderr | + yq '.spec.template.spec | .affinity == "foobar"' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# nodeSelector + +@test "syncCatalogCleanupJob/Uninstall: nodeSelector is not set by default" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.nodeSelector' | tee /dev/stderr) + [ "${actual}" = "null" ] +} + +@test "syncCatalogCleanupJob/Uninstall: nodeSelector is not set by default with sync enabled" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.nodeSelector' | tee /dev/stderr) + [ "${actual}" = "null" ] +} + +@test "syncCatalogCleanupJob/Uninstall: specified nodeSelector" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.nodeSelector=testing' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.nodeSelector' | tee /dev/stderr) + [ "${actual}" = "testing" ] +} + +#-------------------------------------------------------------------- +# tolerations + +@test "syncCatalogCleanupJob/Uninstall: tolerations not set by default" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.tolerations == null' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "syncCatalogCleanupJob/Uninstall: tolerations can be set" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.tolerations=foobar' \ + . | tee /dev/stderr | + yq '.spec.template.spec | .tolerations == "foobar"' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# global.acls.manageSystemACLs + +@test "syncCatalogCleanupJob/Uninstall: ACL auth method env vars are set when acls are enabled" { + cd $(chart_dir) + local env=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.acls.manageSystemACLs=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].env[]' | tee /dev/stderr) + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_LOGIN_AUTH_METHOD").value' | tee /dev/stderr) + [ "${actual}" = "release-name-consul-k8s-component-auth-method" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_LOGIN_DATACENTER").value' | tee /dev/stderr) + [ "${actual}" = "dc1" ] + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_LOGIN_META").value' | tee /dev/stderr) + [ "${actual}" = 'component=sync-catalog,pod=$(NAMESPACE)/$(POD_NAME)' ] +} + +@test "syncCatalogCleanupJob/Uninstall: sets global auth method and primary datacenter when federation and acls and namespaces are enabled" { + cd $(chart_dir) + local env=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.acls.manageSystemACLs=true' \ + --set 'global.federation.enabled=true' \ + --set 'global.federation.primaryDatacenter=dc1' \ + --set 'global.datacenter=dc2' \ + --set 'global.enableConsulNamespaces=true' \ + --set 'global.tls.enabled=true' \ + --set 'meshGateway.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].env[]' | tee /dev/stderr) + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_LOGIN_AUTH_METHOD").value' | tee /dev/stderr) + [ "${actual}" = "release-name-consul-k8s-component-auth-method-dc2" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_LOGIN_DATACENTER").value' | tee /dev/stderr) + [ "${actual}" = "dc1" ] +} + +@test "syncCatalogCleanupJob/Uninstall: sets default login partition and acls and partitions are enabled" { + cd $(chart_dir) + local env=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.acls.manageSystemACLs=true' \ + --set 'global.adminPartitions.enabled=true' \ + --set 'global.enableConsulNamespaces=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].env[]' | tee /dev/stderr) + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_LOGIN_PARTITION").value' | tee /dev/stderr) + [ "${actual}" = "default" ] +} + +@test "syncCatalogCleanupJob/Uninstall: sets non-default login partition and acls and partitions are enabled" { + cd $(chart_dir) + local env=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.acls.manageSystemACLs=true' \ + --set 'global.adminPartitions.enabled=true' \ + --set 'global.adminPartitions.name=foo' \ + --set 'global.enableConsulNamespaces=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].env[]' | tee /dev/stderr) + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_LOGIN_PARTITION").value' | tee /dev/stderr) + [ "${actual}" = "foo" ] +} + +#-------------------------------------------------------------------- +# global.tls.enabled + +@test "syncCatalogCleanupJob/Uninstall: sets Consul environment variables when global.tls.enabled" { + cd $(chart_dir) + local env=$(helm template \ + -s $target \ + --set 'client.enabled=true' \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[0].env[]' | tee /dev/stderr) + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_HTTP_PORT").value' | tee /dev/stderr) + [ "${actual}" = "8501" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_USE_TLS").value' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_CACERT_FILE").value' | tee /dev/stderr) + [ "${actual}" = "/consul/tls/ca/tls.crt" ] +} + +@test "syncCatalogCleanupJob/Uninstall: can overwrite CA secret with the provided one" { + cd $(chart_dir) + local ca_cert_volume=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo-ca-cert' \ + --set 'global.tls.caCert.secretKey=key' \ + --set 'global.tls.caKey.secretName=foo-ca-key' \ + --set 'global.tls.caKey.secretKey=key' \ + . | tee /dev/stderr | + yq '.spec.template.spec.volumes[] | select(.name=="consul-ca-cert")' | tee /dev/stderr) + + # check that the provided ca cert secret is attached as a volume + local actual + actual=$(echo $ca_cert_volume | jq -r '.secret.secretName' | tee /dev/stderr) + [ "${actual}" = "foo-ca-cert" ] + + # check that the volume uses the provided secret key + actual=$(echo $ca_cert_volume | jq -r '.secret.items[0].key' | tee /dev/stderr) + [ "${actual}" = "key" ] +} + +@test "syncCatalogCleanupJob/Uninstall: consul-ca-cert volumeMount is added when TLS is enabled" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "consul-ca-cert") | length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "syncCatalogCleanupJob/Uninstall: consul-ca-cert volume is not added if externalServers.enabled=true and externalServers.useSystemRoots=true" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.enableAutoEncrypt=true' \ + --set 'externalServers.enabled=true' \ + --set 'externalServers.hosts[0]=foo.com' \ + --set 'externalServers.useSystemRoots=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.volumes[] | select(.name == "consul-ca-cert")' | tee /dev/stderr) + [ "${actual}" = "" ] +} + +#-------------------------------------------------------------------- +# resources + +@test "syncCatalogCleanupJob/Uninstall: default resources" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq -rc '.spec.template.spec.containers[0].resources' | tee /dev/stderr) + [ "${actual}" = '{"limits":{"cpu":"50m","memory":"50Mi"},"requests":{"cpu":"50m","memory":"50Mi"}}' ] +} + +@test "syncCatalogCleanupJob/Uninstall: can set resources" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.resources.requests.memory=100Mi' \ + --set 'syncCatalog.resources.requests.cpu=100m' \ + --set 'syncCatalog.resources.limits.memory=200Mi' \ + --set 'syncCatalog.resources.limits.cpu=200m' \ + . | tee /dev/stderr | + yq -rc '.spec.template.spec.containers[0].resources' | tee /dev/stderr) + [ "${actual}" = '{"limits":{"cpu":"200m","memory":"200Mi"},"requests":{"cpu":"100m","memory":"100Mi"}}' ] +} + +#-------------------------------------------------------------------- +# extraLabels + +@test "syncCatalogCleanupJob/Uninstall: no extra labels defined by default" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.labels | del(."app") | del(."chart") | del(."release") | del(."component")' | tee /dev/stderr) + [ "${actual}" = "{}" ] +} + +@test "syncCatalogCleanupJob/Uninstall: can set extra labels" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.extraLabels.foo=bar' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.labels.foo' | tee /dev/stderr) + + [ "${actual}" = "bar" ] +} + +@test "syncCatalogCleanupJob/Uninstall: extra global labels can be set" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.extraLabels.foo=bar' \ + . | tee /dev/stderr) + local actualBar=$(echo "${actual}" | yq -r '.metadata.labels.foo' | tee /dev/stderr) + [ "${actualBar}" = "bar" ] + local actualTemplateBar=$(echo "${actual}" | yq -r '.spec.template.metadata.labels.foo' | tee /dev/stderr) + [ "${actualTemplateBar}" = "bar" ] +} + +@test "syncCatalogCleanupJob/Uninstall: multiple extra global labels can be set" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.extraLabels.foo=bar' \ + --set 'global.extraLabels.baz=qux' \ + . | tee /dev/stderr) + local actualFoo=$(echo "${actual}" | yq -r '.metadata.labels.foo' | tee /dev/stderr) + local actualBaz=$(echo "${actual}" | yq -r '.metadata.labels.baz' | tee /dev/stderr) + [ "${actualFoo}" = "bar" ] + [ "${actualBaz}" = "qux" ] + local actualTemplateFoo=$(echo "${actual}" | yq -r '.spec.template.metadata.labels.foo' | tee /dev/stderr) + local actualTemplateBaz=$(echo "${actual}" | yq -r '.spec.template.metadata.labels.baz' | tee /dev/stderr) + [ "${actualTemplateFoo}" = "bar" ] + [ "${actualTemplateBaz}" = "qux" ] +} + +#-------------------------------------------------------------------- +# annotations + +@test "syncCatalogCleanupJob/Uninstall: no annotations defined by default" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.annotations | + del(."consul.hashicorp.com/connect-inject") | + del(."consul.hashicorp.com/mesh-inject")' | + tee /dev/stderr) + [ "${actual}" = "{}" ] +} + +@test "syncCatalogCleanupJob/Uninstall: annotations can be set" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.annotations=foo: bar' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.annotations.foo' | tee /dev/stderr) + [ "${actual}" = "bar" ] +} + +@test "syncCatalogCleanupJob/Uninstall: metrics annotations can be set" { + cd $(chart_dir) + local object=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.metrics.enabled=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.annotations | + del(."consul.hashicorp.com/connect-inject") | + del(."consul.hashicorp.com/mesh-inject")' | + tee /dev/stderr) + + # Annotations to check + annotations=("prometheus.io/scrape" "prometheus.io/path" "prometheus.io/port") + + # Check each annotation + for annotation in "${annotations[@]}"; do + actual=$(echo "$object" | yq -r "has(\"$annotation\")") + [ "$actual" = "true" ] + done +} + +#-------------------------------------------------------------------- +# logLevel + +@test "syncCatalogCleanupJob/Uninstall: logLevel info by default from global" { + cd $(chart_dir) + local cmd=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-log-level=info"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "syncCatalogCleanupJob/Uninstall: logLevel can be overridden" { + cd $(chart_dir) + local cmd=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.logLevel=debug' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-log-level=debug"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# Vault + +@test "syncCatalogCleanupJob/Uninstall: configures server CA to come from vault when vault is enabled" { + cd $(chart_dir) + local object=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=test' \ + --set 'global.secretsBackend.vault.consulServerRole=foo' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + # Check annotations + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-init-first"]' | tee /dev/stderr) + [ "${actual}" = "true" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-inject"]' | tee /dev/stderr) + [ "${actual}" = "true" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/role"]' | tee /dev/stderr) + [ "${actual}" = "carole" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-inject-secret-serverca.crt"]' | tee /dev/stderr) + [ "${actual}" = "foo" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-inject-template-serverca.crt"]' | tee /dev/stderr) + [ "${actual}" = $'{{- with secret \"foo\" -}}\n{{- .Data.certificate -}}\n{{- end -}}' ] + + actual=$(echo $object | jq -r '.spec.volumes[] | select( .name == "consul-ca-cert")' | tee /dev/stderr) + [ "${actual}" = "" ] + + actual=$(echo $object | jq -r '.spec.containers[0].volumeMounts[] | select( .name == "consul-ca-cert")' | tee /dev/stderr) + [ "${actual}" = "" ] +} + +@test "syncCatalogCleanupJob/Uninstall: vault namespace annotations is set when global.secretsBackend.vault.vaultNamespace is set" { + cd $(chart_dir) + local cmd=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=bar' \ + --set 'global.secretsBackend.vault.consulCARole=test' \ + --set 'global.secretsBackend.vault.vaultNamespace=vns' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.tls.enableAutoEncrypt=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata' | tee /dev/stderr) + + local actual="$(echo $cmd | + yq -r '.annotations["vault.hashicorp.com/namespace"]' | tee /dev/stderr)" + [ "${actual}" = "vns" ] +} + +@test "syncCatalogCleanupJob/Uninstall: correct vault namespace annotations is set when global.secretsBackend.vault.vaultNamespace is set and agentAnnotations are also set without vaultNamespace annotation" { + cd $(chart_dir) + local cmd=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=bar' \ + --set 'global.secretsBackend.vault.consulCARole=test' \ + --set 'global.secretsBackend.vault.vaultNamespace=vns' \ + --set 'global.secretsBackend.vault.agentAnnotations=vault.hashicorp.com/agent-extra-secret: bar' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.tls.enableAutoEncrypt=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata' | tee /dev/stderr) + + local actual="$(echo $cmd | + yq -r '.annotations["vault.hashicorp.com/namespace"]' | tee /dev/stderr)" + [ "${actual}" = "vns" ] +} + +@test "syncCatalogCleanupJob/Uninstall: correct vault namespace annotations is set when global.secretsBackend.vault.vaultNamespace is set and agentAnnotations are also set with vaultNamespace annotation" { + cd $(chart_dir) + local cmd=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=bar' \ + --set 'global.secretsBackend.vault.consulCARole=test' \ + --set 'global.secretsBackend.vault.vaultNamespace=vns' \ + --set 'global.secretsBackend.vault.agentAnnotations=vault.hashicorp.com/namespace: bar' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.tls.enableAutoEncrypt=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata' | tee /dev/stderr) + + local actual="$(echo $cmd | + yq -r '.annotations["vault.hashicorp.com/namespace"]' | tee /dev/stderr)" + [ "${actual}" = "bar" ] +} + +@test "syncCatalogCleanupJob/Uninstall: vault CA is not configured by default" { + cd $(chart_dir) + local object=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=test' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/agent-extra-secret")') + [ "${actual}" = "false" ] + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/ca-cert")') + [ "${actual}" = "false" ] +} + +@test "syncCatalogCleanupJob/Uninstall: vault CA is not configured when secretName is set but secretKey is not" { + cd $(chart_dir) + local object=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=test' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + --set 'global.secretsBackend.vault.ca.secretName=ca' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/agent-extra-secret")') + [ "${actual}" = "false" ] + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/ca-cert")') + [ "${actual}" = "false" ] +} + +@test "syncCatalogCleanupJob/Uninstall: vault CA is not configured when secretKey is set but secretName is not" { + cd $(chart_dir) + local object=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=test' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + --set 'global.secretsBackend.vault.ca.secretKey=tls.crt' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/agent-extra-secret")') + [ "${actual}" = "false" ] + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/ca-cert")') + [ "${actual}" = "false" ] +} + +@test "syncCatalogCleanupJob/Uninstall: vault CA is configured when both secretName and secretKey are set" { + cd $(chart_dir) + local object=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=test' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + --set 'global.secretsBackend.vault.ca.secretName=ca' \ + --set 'global.secretsBackend.vault.ca.secretKey=tls.crt' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.metadata.annotations."vault.hashicorp.com/agent-extra-secret"') + [ "${actual}" = "ca" ] + local actual=$(echo $object | yq -r '.metadata.annotations."vault.hashicorp.com/ca-cert"') + [ "${actual}" = "/vault/custom/tls.crt" ] +} + +#-------------------------------------------------------------------- +# Vault agent annotations + +@test "syncCatalogCleanupJob/Uninstall: no vault agent annotations defined by default" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=test' \ + --set 'global.secretsBackend.vault.consulServerRole=foo' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.annotations | + del(."consul.hashicorp.com/connect-inject") | + del(."consul.hashicorp.com/mesh-inject") | + del(."vault.hashicorp.com/agent-inject") | + del(."vault.hashicorp.com/role")' | + tee /dev/stderr) + [ "${actual}" = "{}" ] +} + +@test "syncCatalogCleanupJob/Uninstall: vault agent annotations can be set" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=test' \ + --set 'global.secretsBackend.vault.consulServerRole=foo' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + --set 'global.secretsBackend.vault.agentAnnotations=foo: bar' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.annotations.foo' | tee /dev/stderr) + [ "${actual}" = "bar" ] +} diff --git a/charts/consul/test/unit/sync-catalog-cleanup-on-upgrade-job.bats b/charts/consul/test/unit/sync-catalog-cleanup-on-upgrade-job.bats new file mode 100644 index 0000000000..01c7a4f45e --- /dev/null +++ b/charts/consul/test/unit/sync-catalog-cleanup-on-upgrade-job.bats @@ -0,0 +1,896 @@ +#!/usr/bin/env bats + +load _helpers + +target=templates/sync-catalog-cleanup-on-upgrade-job.yaml + +@test "syncCatalogCleanupJob/Upgrade: disabled by default" { + cd $(chart_dir) + assert_empty helm template \ + -s $target \ + . +} + +@test "syncCatalogCleanupJob/Upgrade: disabled with syncCatalog.cleanupNodeOnRemoval true and syncCatalog.enabled true and Release.IsUpgrade true" { + cd $(chart_dir) + assert_empty helm template \ + -s $target \ + --set 'syncCatalog.enabled=true' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --is-upgrade \ + . +} + +@test "syncCatalogCleanupJob/Upgrade: enable with syncCatalog.cleanupNodeOnRemoval true and syncCatalog.enabled false and Release.IsUpgrade true" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --is-upgrade \ + . | tee /dev/stderr | + yq -s 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# image + +@test "syncCatalogCleanupJob/Upgrade: image defaults to global.imageK8S" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'global.imageK8S=bar' \ + --set 'syncCatalog.enabled=false' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --is-upgrade \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[0].image' | tee /dev/stderr) + [ "${actual}" = "bar" ] +} + +@test "syncCatalogCleanupJob/Upgrade: image can be overridden with server.image" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'global.imageK8S=foo' \ + --set 'syncCatalog.enabled=false' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.image=bar' \ + --is-upgrade \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[0].image' | tee /dev/stderr) + [ "${actual}" = "bar" ] +} + +@test "syncCatalogCleanupJob/Upgrade: consul env defaults" { + cd $(chart_dir) + local env=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --is-upgrade \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].env[]' | tee /dev/stderr) + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_ADDRESSES").value' | tee /dev/stderr) + [ "${actual}" = "release-name-consul-server.default.svc" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_GRPC_PORT").value' | tee /dev/stderr) + [ "${actual}" = "8502" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_HTTP_PORT").value' | tee /dev/stderr) + [ "${actual}" = "8500" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_DATACENTER").value' | tee /dev/stderr) + [ "${actual}" = "dc1" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_API_TIMEOUT").value' | tee /dev/stderr) + [ "${actual}" = "5s" ] +} + +#-------------------------------------------------------------------- +# consulNodeName + +@test "syncCatalogCleanupJob/Upgrade: consulNodeName defaults to k8s-sync" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --is-upgrade \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command | any(contains("-consul-node-name=k8s-sync"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "syncCatalogCleanupJob/Upgrade: consulNodeName set to empty" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --is-upgrade \ + --set 'syncCatalog.consulNodeName=' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command | any(contains("-consul-node-name"))' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + +@test "syncCatalogCleanupJob/Upgrade: can specify consulNodeName" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --is-upgrade \ + --set 'syncCatalog.consulNodeName=aNodeName' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command | any(contains("-consul-node-name=aNodeName"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# serviceAccount + +@test "syncCatalogCleanupJob/Upgrade: serviceAccount set when sync enabled" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --is-upgrade \ + . | tee /dev/stderr | + yq '.spec.template.spec.serviceAccountName | contains("sync-catalog-cleanup")' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# aclSyncToken + +@test "syncCatalogCleanupJob/Upgrade: aclSyncToken disabled when secretName is missing" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --is-upgrade \ + --set 'syncCatalog.aclSyncToken.secretKey=bar' \ + . | tee /dev/stderr | + yq '[.spec.template.spec.containers[0].env[].name] | any(contains("CONSUL_ACL_TOKEN"))' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + +@test "syncCatalogCleanupJob/Upgrade: aclSyncToken disabled when secretKey is missing" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --is-upgrade \ + --set 'syncCatalog.aclSyncToken.secretName=foo' \ + . | tee /dev/stderr | + yq '[.spec.template.spec.containers[0].env[].name] | any(contains("CONSUL_ACL_TOKEN"))' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + +@test "syncCatalogCleanupJob/Upgrade: aclSyncToken enabled when secretName and secretKey is provided" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --is-upgrade \ + --set 'syncCatalog.aclSyncToken.secretName=foo' \ + --set 'syncCatalog.aclSyncToken.secretKey=bar' \ + . | tee /dev/stderr | + yq '[.spec.template.spec.containers[0].env[].name] | any(contains("CONSUL_ACL_TOKEN"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# affinity + +@test "syncCatalogCleanupJob/Upgrade: affinity not set by default" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --is-upgrade \ + . | tee /dev/stderr | + yq '.spec.template.spec.affinity == null' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "syncCatalogCleanupJob/Upgrade: affinity can be set" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.affinity=foobar' \ + . | tee /dev/stderr | + yq '.spec.template.spec | .affinity == "foobar"' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# nodeSelector + +@test "syncCatalogCleanupJob/Upgrade: nodeSelector is not set by default" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.nodeSelector' | tee /dev/stderr) + [ "${actual}" = "null" ] +} + +@test "syncCatalogCleanupJob/Upgrade: nodeSelector is not set by default with sync enabled" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.nodeSelector' | tee /dev/stderr) + [ "${actual}" = "null" ] +} + +@test "syncCatalogCleanupJob/Upgrade: specified nodeSelector" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.nodeSelector=testing' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.nodeSelector' | tee /dev/stderr) + [ "${actual}" = "testing" ] +} + +#-------------------------------------------------------------------- +# tolerations + +@test "syncCatalogCleanupJob/Upgrade: tolerations not set by default" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.tolerations == null' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "syncCatalogCleanupJob/Upgrade: tolerations can be set" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.tolerations=foobar' \ + . | tee /dev/stderr | + yq '.spec.template.spec | .tolerations == "foobar"' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# global.acls.manageSystemACLs + +@test "syncCatalogCleanupJob/Upgrade: ACL auth method env vars are set when acls are enabled" { + cd $(chart_dir) + local env=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.acls.manageSystemACLs=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].env[]' | tee /dev/stderr) + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_LOGIN_AUTH_METHOD").value' | tee /dev/stderr) + [ "${actual}" = "release-name-consul-k8s-component-auth-method" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_LOGIN_DATACENTER").value' | tee /dev/stderr) + [ "${actual}" = "dc1" ] + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_LOGIN_META").value' | tee /dev/stderr) + [ "${actual}" = 'component=sync-catalog,pod=$(NAMESPACE)/$(POD_NAME)' ] +} + +@test "syncCatalogCleanupJob/Upgrade: sets global auth method and primary datacenter when federation and acls and namespaces are enabled" { + cd $(chart_dir) + local env=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.acls.manageSystemACLs=true' \ + --set 'global.federation.enabled=true' \ + --set 'global.federation.primaryDatacenter=dc1' \ + --set 'global.datacenter=dc2' \ + --set 'global.enableConsulNamespaces=true' \ + --set 'global.tls.enabled=true' \ + --set 'meshGateway.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].env[]' | tee /dev/stderr) + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_LOGIN_AUTH_METHOD").value' | tee /dev/stderr) + [ "${actual}" = "release-name-consul-k8s-component-auth-method-dc2" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_LOGIN_DATACENTER").value' | tee /dev/stderr) + [ "${actual}" = "dc1" ] +} + +@test "syncCatalogCleanupJob/Upgrade: sets default login partition and acls and partitions are enabled" { + cd $(chart_dir) + local env=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.acls.manageSystemACLs=true' \ + --set 'global.adminPartitions.enabled=true' \ + --set 'global.enableConsulNamespaces=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].env[]' | tee /dev/stderr) + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_LOGIN_PARTITION").value' | tee /dev/stderr) + [ "${actual}" = "default" ] +} + +@test "syncCatalogCleanupJob/Upgrade: sets non-default login partition and acls and partitions are enabled" { + cd $(chart_dir) + local env=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.acls.manageSystemACLs=true' \ + --set 'global.adminPartitions.enabled=true' \ + --set 'global.adminPartitions.name=foo' \ + --set 'global.enableConsulNamespaces=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].env[]' | tee /dev/stderr) + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_LOGIN_PARTITION").value' | tee /dev/stderr) + [ "${actual}" = "foo" ] +} + +#-------------------------------------------------------------------- +# global.tls.enabled + +@test "syncCatalogCleanupJob/Upgrade: sets Consul environment variables when global.tls.enabled" { + cd $(chart_dir) + local env=$(helm template \ + -s $target \ + --set 'client.enabled=true' \ + --is-upgrade \ + --set 'syncCatalog.enabled=false' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[0].env[]' | tee /dev/stderr) + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_HTTP_PORT").value' | tee /dev/stderr) + [ "${actual}" = "8501" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_USE_TLS").value' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(echo "$env" | + jq -r '. | select( .name == "CONSUL_CACERT_FILE").value' | tee /dev/stderr) + [ "${actual}" = "/consul/tls/ca/tls.crt" ] +} + +@test "syncCatalogCleanupJob/Upgrade: can overwrite CA secret with the provided one" { + cd $(chart_dir) + local ca_cert_volume=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo-ca-cert' \ + --set 'global.tls.caCert.secretKey=key' \ + --set 'global.tls.caKey.secretName=foo-ca-key' \ + --set 'global.tls.caKey.secretKey=key' \ + . | tee /dev/stderr | + yq '.spec.template.spec.volumes[] | select(.name=="consul-ca-cert")' | tee /dev/stderr) + + # check that the provided ca cert secret is attached as a volume + local actual + actual=$(echo $ca_cert_volume | jq -r '.secret.secretName' | tee /dev/stderr) + [ "${actual}" = "foo-ca-cert" ] + + # check that the volume uses the provided secret key + actual=$(echo $ca_cert_volume | jq -r '.secret.items[0].key' | tee /dev/stderr) + [ "${actual}" = "key" ] +} + +@test "syncCatalogCleanupJob/Upgrade: consul-ca-cert volumeMount is added when TLS is enabled" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].volumeMounts[] | select(.name == "consul-ca-cert") | length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "syncCatalogCleanupJob/Upgrade: consul-ca-cert volume is not added if externalServers.enabled=true and externalServers.useSystemRoots=true" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.enableAutoEncrypt=true' \ + --set 'externalServers.enabled=true' \ + --set 'externalServers.hosts[0]=foo.com' \ + --set 'externalServers.useSystemRoots=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.volumes[] | select(.name == "consul-ca-cert")' | tee /dev/stderr) + [ "${actual}" = "" ] +} + +#-------------------------------------------------------------------- +# resources + +@test "syncCatalogCleanupJob/Upgrade: default resources" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq -rc '.spec.template.spec.containers[0].resources' | tee /dev/stderr) + [ "${actual}" = '{"limits":{"cpu":"50m","memory":"50Mi"},"requests":{"cpu":"50m","memory":"50Mi"}}' ] +} + +@test "syncCatalogCleanupJob/Upgrade: can set resources" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.resources.requests.memory=100Mi' \ + --set 'syncCatalog.resources.requests.cpu=100m' \ + --set 'syncCatalog.resources.limits.memory=200Mi' \ + --set 'syncCatalog.resources.limits.cpu=200m' \ + . | tee /dev/stderr | + yq -rc '.spec.template.spec.containers[0].resources' | tee /dev/stderr) + [ "${actual}" = '{"limits":{"cpu":"200m","memory":"200Mi"},"requests":{"cpu":"100m","memory":"100Mi"}}' ] +} + +#-------------------------------------------------------------------- +# extraLabels + +@test "syncCatalogCleanupJob/Upgrade: no extra labels defined by default" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.labels | del(."app") | del(."chart") | del(."release") | del(."component")' | tee /dev/stderr) + [ "${actual}" = "{}" ] +} + +@test "syncCatalogCleanupJob/Upgrade: can set extra labels" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.extraLabels.foo=bar' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.labels.foo' | tee /dev/stderr) + + [ "${actual}" = "bar" ] +} + +@test "syncCatalogCleanupJob/Upgrade: extra global labels can be set" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.extraLabels.foo=bar' \ + . | tee /dev/stderr) + local actualBar=$(echo "${actual}" | yq -r '.metadata.labels.foo' | tee /dev/stderr) + [ "${actualBar}" = "bar" ] + local actualTemplateBar=$(echo "${actual}" | yq -r '.spec.template.metadata.labels.foo' | tee /dev/stderr) + [ "${actualTemplateBar}" = "bar" ] +} + +@test "syncCatalogCleanupJob/Upgrade: multiple extra global labels can be set" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.extraLabels.foo=bar' \ + --set 'global.extraLabels.baz=qux' \ + . | tee /dev/stderr) + local actualFoo=$(echo "${actual}" | yq -r '.metadata.labels.foo' | tee /dev/stderr) + local actualBaz=$(echo "${actual}" | yq -r '.metadata.labels.baz' | tee /dev/stderr) + [ "${actualFoo}" = "bar" ] + [ "${actualBaz}" = "qux" ] + local actualTemplateFoo=$(echo "${actual}" | yq -r '.spec.template.metadata.labels.foo' | tee /dev/stderr) + local actualTemplateBaz=$(echo "${actual}" | yq -r '.spec.template.metadata.labels.baz' | tee /dev/stderr) + [ "${actualTemplateFoo}" = "bar" ] + [ "${actualTemplateBaz}" = "qux" ] +} + +#-------------------------------------------------------------------- +# annotations + +@test "syncCatalogCleanupJob/Upgrade: no annotations defined by default" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.annotations | + del(."consul.hashicorp.com/connect-inject") | + del(."consul.hashicorp.com/mesh-inject")' | + tee /dev/stderr) + [ "${actual}" = "{}" ] +} + +@test "syncCatalogCleanupJob/Upgrade: annotations can be set" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'syncCatalog.annotations=foo: bar' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.annotations.foo' | tee /dev/stderr) + [ "${actual}" = "bar" ] +} + +@test "syncCatalogCleanupJob/Upgrade: metrics annotations can be set" { + cd $(chart_dir) + local object=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --is-upgrade \ + --set 'syncCatalog.metrics.enabled=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.annotations | + del(."consul.hashicorp.com/connect-inject") | + del(."consul.hashicorp.com/mesh-inject")' | + tee /dev/stderr) + + # Annotations to check + annotations=("prometheus.io/scrape" "prometheus.io/path" "prometheus.io/port") + + # Check each annotation + for annotation in "${annotations[@]}"; do + actual=$(echo "$object" | yq -r "has(\"$annotation\")") + [ "$actual" = "true" ] + done +} + +#-------------------------------------------------------------------- +# logLevel + +@test "syncCatalogCleanupJob/Upgrade: logLevel info by default from global" { + cd $(chart_dir) + local cmd=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-log-level=info"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "syncCatalogCleanupJob/Upgrade: logLevel can be overridden" { + cd $(chart_dir) + local cmd=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.logLevel=debug' \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-log-level=debug"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# Vault + +@test "syncCatalogCleanupJob/Upgrade: configures server CA to come from vault when vault is enabled" { + cd $(chart_dir) + local object=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=test' \ + --set 'global.secretsBackend.vault.consulServerRole=foo' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + # Check annotations + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-init-first"]' | tee /dev/stderr) + [ "${actual}" = "true" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-inject"]' | tee /dev/stderr) + [ "${actual}" = "true" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/role"]' | tee /dev/stderr) + [ "${actual}" = "carole" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-inject-secret-serverca.crt"]' | tee /dev/stderr) + [ "${actual}" = "foo" ] + local actual + actual=$(echo $object | jq -r '.metadata.annotations["vault.hashicorp.com/agent-inject-template-serverca.crt"]' | tee /dev/stderr) + [ "${actual}" = $'{{- with secret \"foo\" -}}\n{{- .Data.certificate -}}\n{{- end -}}' ] + + actual=$(echo $object | jq -r '.spec.volumes[] | select( .name == "consul-ca-cert")' | tee /dev/stderr) + [ "${actual}" = "" ] + + actual=$(echo $object | jq -r '.spec.containers[0].volumeMounts[] | select( .name == "consul-ca-cert")' | tee /dev/stderr) + [ "${actual}" = "" ] +} + +@test "syncCatalogCleanupJob/Upgrade: vault namespace annotations is set when global.secretsBackend.vault.vaultNamespace is set" { + cd $(chart_dir) + local cmd=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=bar' \ + --set 'global.secretsBackend.vault.consulCARole=test' \ + --set 'global.secretsBackend.vault.vaultNamespace=vns' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.tls.enableAutoEncrypt=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata' | tee /dev/stderr) + + local actual="$(echo $cmd | + yq -r '.annotations["vault.hashicorp.com/namespace"]' | tee /dev/stderr)" + [ "${actual}" = "vns" ] +} + +@test "syncCatalogCleanupJob/Upgrade: correct vault namespace annotations is set when global.secretsBackend.vault.vaultNamespace is set and agentAnnotations are also set without vaultNamespace annotation" { + cd $(chart_dir) + local cmd=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=bar' \ + --set 'global.secretsBackend.vault.consulCARole=test' \ + --set 'global.secretsBackend.vault.vaultNamespace=vns' \ + --set 'global.secretsBackend.vault.agentAnnotations=vault.hashicorp.com/agent-extra-secret: bar' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.tls.enableAutoEncrypt=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata' | tee /dev/stderr) + + local actual="$(echo $cmd | + yq -r '.annotations["vault.hashicorp.com/namespace"]' | tee /dev/stderr)" + [ "${actual}" = "vns" ] +} + +@test "syncCatalogCleanupJob/Upgrade: correct vault namespace annotations is set when global.secretsBackend.vault.vaultNamespace is set and agentAnnotations are also set with vaultNamespace annotation" { + cd $(chart_dir) + local cmd=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=bar' \ + --set 'global.secretsBackend.vault.consulCARole=test' \ + --set 'global.secretsBackend.vault.vaultNamespace=vns' \ + --set 'global.secretsBackend.vault.agentAnnotations=vault.hashicorp.com/namespace: bar' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.tls.enableAutoEncrypt=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata' | tee /dev/stderr) + + local actual="$(echo $cmd | + yq -r '.annotations["vault.hashicorp.com/namespace"]' | tee /dev/stderr)" + [ "${actual}" = "bar" ] +} + +@test "syncCatalogCleanupJob/Upgrade: vault CA is not configured by default" { + cd $(chart_dir) + local object=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=test' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/agent-extra-secret")') + [ "${actual}" = "false" ] + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/ca-cert")') + [ "${actual}" = "false" ] +} + +@test "syncCatalogCleanupJob/Upgrade: vault CA is not configured when secretName is set but secretKey is not" { + cd $(chart_dir) + local object=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=test' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + --set 'global.secretsBackend.vault.ca.secretName=ca' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/agent-extra-secret")') + [ "${actual}" = "false" ] + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/ca-cert")') + [ "${actual}" = "false" ] +} + +@test "syncCatalogCleanupJob/Upgrade: vault CA is not configured when secretKey is set but secretName is not" { + cd $(chart_dir) + local object=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=test' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + --set 'global.secretsBackend.vault.ca.secretKey=tls.crt' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/agent-extra-secret")') + [ "${actual}" = "false" ] + local actual=$(echo $object | yq -r '.metadata.annotations | has("vault.hashicorp.com/ca-cert")') + [ "${actual}" = "false" ] +} + +@test "syncCatalogCleanupJob/Upgrade: vault CA is configured when both secretName and secretKey are set" { + cd $(chart_dir) + local object=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=foo' \ + --set 'global.secretsBackend.vault.consulServerRole=test' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + --set 'global.secretsBackend.vault.ca.secretName=ca' \ + --set 'global.secretsBackend.vault.ca.secretKey=tls.crt' \ + . | tee /dev/stderr | + yq -r '.spec.template' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.metadata.annotations."vault.hashicorp.com/agent-extra-secret"') + [ "${actual}" = "ca" ] + local actual=$(echo $object | yq -r '.metadata.annotations."vault.hashicorp.com/ca-cert"') + [ "${actual}" = "/vault/custom/tls.crt" ] +} + +#-------------------------------------------------------------------- +# Vault agent annotations + +@test "syncCatalogCleanupJob/Upgrade: no vault agent annotations defined by default" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=test' \ + --set 'global.secretsBackend.vault.consulServerRole=foo' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.annotations | + del(."consul.hashicorp.com/connect-inject") | + del(."consul.hashicorp.com/mesh-inject") | + del(."vault.hashicorp.com/agent-inject") | + del(."vault.hashicorp.com/role")' | + tee /dev/stderr) + [ "${actual}" = "{}" ] +} + +@test "syncCatalogCleanupJob/Upgrade: vault agent annotations can be set" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.enabled=false' \ + --is-upgrade \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.tls.enabled=true' \ + --set 'global.secretsBackend.vault.enabled=true' \ + --set 'global.secretsBackend.vault.consulClientRole=test' \ + --set 'global.secretsBackend.vault.consulServerRole=foo' \ + --set 'global.tls.caCert.secretName=foo' \ + --set 'global.secretsBackend.vault.consulCARole=carole' \ + --set 'global.secretsBackend.vault.agentAnnotations=foo: bar' \ + . | tee /dev/stderr | + yq -r '.spec.template.metadata.annotations.foo' | tee /dev/stderr) + [ "${actual}" = "bar" ] +} diff --git a/charts/consul/test/unit/sync-catalog-cleanup-serviceaccount.bats b/charts/consul/test/unit/sync-catalog-cleanup-serviceaccount.bats new file mode 100644 index 0000000000..8362eb87c8 --- /dev/null +++ b/charts/consul/test/unit/sync-catalog-cleanup-serviceaccount.bats @@ -0,0 +1,75 @@ +#!/usr/bin/env bats + +load _helpers + +target=templates/sync-catalog-cleanup-serviceaccount.yaml + +@test "syncCatalogCleanup/ServiceAccount: disabled by default" { + cd $(chart_dir) + assert_empty helm template \ + -s $target \ + . +} + +@test "syncCatalogCleanup/ServiceAccount: disabled with cleanup disabled" { + cd $(chart_dir) + assert_empty helm template \ + -s $target \ + --set 'syncCatalog.cleanupNodeOnRemoval=false' \ + . +} + +@test "syncCatalogCleanup/ServiceAccount: enabled with cleanup enabled" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# global.imagePullSecrets + +@test "syncCatalogCleanup/ServiceAccount: can set image pull secrets" { + cd $(chart_dir) + local object=$(helm template \ + -s $target \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set 'global.imagePullSecrets[0].name=my-secret' \ + --set 'global.imagePullSecrets[1].name=my-secret2' \ + . | tee /dev/stderr) + + local actual=$(echo "$object" | + yq -r '.imagePullSecrets[0].name' | tee /dev/stderr) + [ "${actual}" = "my-secret" ] + + local actual=$(echo "$object" | + yq -r '.imagePullSecrets[1].name' | tee /dev/stderr) + [ "${actual}" = "my-secret2" ] +} + +#-------------------------------------------------------------------- +# syncCatalog.serviceAccount.annotations + +@test "syncCatalogCleanup/ServiceAccount: no annotations by default" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + . | tee /dev/stderr | + yq '.metadata.annotations | length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + +@test "syncCatalogCleanup/ServiceAccount: annotations when enabled" { + cd $(chart_dir) + local actual=$(helm template \ + -s $target \ + --set 'syncCatalog.cleanupNodeOnRemoval=true' \ + --set "syncCatalog.serviceAccount.annotations=foo: bar" \ + . | tee /dev/stderr | + yq -r '.metadata.annotations.foo' | tee /dev/stderr) + [ "${actual}" = "bar" ] +}