diff --git a/deploy/charts/burrito/templates/controllers.yaml b/deploy/charts/burrito/templates/controllers.yaml index aaca1cb6..2f1f4365 100644 --- a/deploy/charts/burrito/templates/controllers.yaml +++ b/deploy/charts/burrito/templates/controllers.yaml @@ -1,5 +1,6 @@ {{ $configChecksum := (include (print $.Template.BasePath "/config.yaml") . | sha256sum) }} {{ $sshKnownHostsChecksum := (include (print $.Template.BasePath "/ssh-known-hosts.yaml") . | sha256sum) }} +{{- $baseEnv := list (dict "name" "BURRITO_CONTROLLER_MAINNAMESPACE" "valueFrom" (dict "fieldRef" (dict "fieldPath" "metadata.namespace"))) -}} {{- with mergeOverwrite (deepCopy .Values.global) .Values.controllers }} apiVersion: apps/v1 @@ -52,7 +53,7 @@ spec: resources: {{- toYaml .deployment.resources | nindent 12 }} env: - {{- toYaml .deployment.env | nindent 12 }} + {{- concat $baseEnv .deployment.env | toYaml | nindent 12}} envFrom: {{- toYaml .deployment.envFrom | nindent 12 }} volumeMounts: @@ -118,4 +119,22 @@ subjects: - kind: ServiceAccount name: burrito-controllers namespace: {{ $.Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: burrito-controllers-secrets + namespace: {{ $.Release.Namespace }} + labels: + {{- toYaml .metadata.labels | nindent 4 }} + annotations: + {{- toYaml .metadata.annotations | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: burrito-controllers-secrets +subjects: + - kind: ServiceAccount + name: burrito-controllers + namespace: {{ $.Release.Namespace }} {{- end }} diff --git a/deploy/charts/burrito/templates/hermitcrab.yaml b/deploy/charts/burrito/templates/hermitcrab.yaml new file mode 100644 index 00000000..44cf688f --- /dev/null +++ b/deploy/charts/burrito/templates/hermitcrab.yaml @@ -0,0 +1,109 @@ +{{- if .Values.config.burrito.hermitcrab.enabled }} +{{- with mergeOverwrite (deepCopy .Values.global) .Values.hermitcrab }} +{{- if .persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: burrito-hermitcrab + annotations: + {{- toYaml .metadata.annotations | nindent 4 }} + labels: + {{- toYaml .metadata.labels | nindent 4 }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .persistence.size }} + {{- if .persistence.storageClassName }} + storageClassName: {{ .persistence.storageClassName }} + {{- end }} +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: burrito-hermitcrab +spec: + selector: + {{- toYaml .metadata.labels | nindent 4 }} + ports: + - name: http + port: 80 + targetPort: http + - name: https + port: 443 + targetPort: https +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: burrito-hermitcrab + labels: + {{- toYaml .metadata.labels | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- toYaml .metadata.labels | nindent 6 }} + template: + metadata: + labels: + {{- toYaml .metadata.labels | nindent 8 }} + spec: + automountServiceAccountToken: false + containers: + - name: hermitcrab + image: "{{ .deployment.image.repository }}:{{ .deployment.image.tag }}" + imagePullPolicy: {{ .deployment.image.pullPolicy }} + resources: + {{- toYaml .deployment.resources | nindent 12 }} + env: + {{- toYaml .deployment.env | nindent 12 }} + envFrom: + {{- toYaml .deployment.envFrom | nindent 12 }} + ports: + {{- toYaml .deployment.ports | nindent 12 }} + livenessProbe: + {{- toYaml .deployment.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .deployment.readinessProbe | nindent 12 }} + volumeMounts: + {{- if .persistence.enabled }} + - name: data + mountPath: /var/run/hermitcrab + {{- end }} + {{- if .tls.certManager.use }} + - name: burrito-hermitcrab-tls + mountPath: /etc/hermitcrab/tls + {{- end }} + {{- if .deployment.extraVolumeMounts }} + {{- toYaml .deployment.extraVolumeMounts | nindent 12 }} + {{- end }} + volumes: + {{- if .persistence.enabled }} + - name: data + persistentVolumeClaim: + claimName: burrito-hermitcrab + {{- end }} + {{- if .tls.certManager.use }} + - name: burrito-hermitcrab-tls + secret: + secretName: {{ $.Values.config.burrito.hermitcrab.certificateSecretName }} + {{- end }} + {{- if .deployment.extraVolumes }} + {{- toYaml .deployment.extraVolumes | nindent 8 }} + {{- end }} +--- +{{- if .tls.certManager.use }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: burrito-hermitcrab + labels: + {{- toYaml .metadata.labels | nindent 4 }} +spec: + {{- toYaml .tls.certManager.certificate.spec | nindent 4 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/deploy/charts/burrito/templates/rbac.yaml b/deploy/charts/burrito/templates/rbac.yaml index 2481d70c..fc198e98 100644 --- a/deploy/charts/burrito/templates/rbac.yaml +++ b/deploy/charts/burrito/templates/rbac.yaml @@ -155,6 +155,14 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - list + - create + - update - apiGroups: - config.terraform.padok.cloud resources: @@ -272,3 +280,21 @@ rules: - update - patch - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: controllers + app.kubernetes.io/name: burrito-controllers + app.kubernetes.io/part-of: burrito + name: burrito-controllers-secrets +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - watch + - list + - get diff --git a/deploy/charts/burrito/values.yaml b/deploy/charts/burrito/values.yaml index 510ad124..fdffa8d5 100644 --- a/deploy/charts/burrito/values.yaml +++ b/deploy/charts/burrito/values.yaml @@ -51,6 +51,9 @@ config: # -- Prefer override with the BURRITO_CONTROLLER_GITLABCONFIG_APITOKEN environment variable apiToken: "" url: "" + hermitcrab: + enabled: false + certificateSecretName: burrito-hermitcrab-tls # Burrito server configuration server: @@ -90,6 +93,66 @@ redis: port: 6379 targetPort: 6379 +hermitcrab: + metadata: + labels: + app.kubernetes.io/component: hermitcrab + app.kubernetes.io/name: burrito-hermitcrab + persistence: + enabled: true + size: 1Gi + tls: + certManager: + use: false + certificate: + spec: {} + + deployment: + image: + pullPolicy: Always + repository: sealio/hermitcrab + tag: main + env: + - name: SERVER_TLS_CERT_FILE + value: /etc/hermitcrab/tls/tls.crt + - name: SERVER_TLS_PRIVATE_KEY_FILE + value: /etc/hermitcrab/tls/tls.key + resources: + limits: + cpu: '2' + memory: '4Gi' + requests: + cpu: '500m' + memory: '512Mi' + ports: + - name: http + containerPort: 80 + - name: https + containerPort: 443 + startupProbe: + failureThreshold: 10 + periodSeconds: 5 + httpGet: + port: 80 + path: /readyz + readinessProbe: + failureThreshold: 3 + timeoutSeconds: 5 + periodSeconds: 5 + httpGet: + port: 80 + path: /readyz + livenessProbe: + failureThreshold: 10 + timeoutSeconds: 5 + periodSeconds: 10 + httpGet: + httpHeaders: + - name: "User-Agent" + value: "" + port: 80 + path: /livez + global: metadata: labels: @@ -140,6 +203,7 @@ controllers: initialDelaySeconds: 15 periodSeconds: 20 envFrom: [] + env: [] service: enabled: false diff --git a/docs/operator-manual/provider-caching.md b/docs/operator-manual/provider-caching.md new file mode 100644 index 00000000..ec94c68c --- /dev/null +++ b/docs/operator-manual/provider-caching.md @@ -0,0 +1,96 @@ +# Caching Terraform Providers + +By caching Terraform providers, Burrito can avoid downloading them from outside the cluster every time a runner initializes a Terraform layer. This can significantly reduce the ingress traffic to the infrastructure running Burrito. + +The Burrito Helm chart is packaged with [Hermitcrab](https://github.com/seal-io/hermitcrab), which leverages the [Provider Network Mirror Protocol](https://developer.hashicorp.com/terraform/internals/provider-network-mirror-protocol) from Terraform to cache providers. + +## 1. Activate Hermitcrab on Burrito + +Hermitcrab is available to use with Burrito when using the Helm chart. +Set the `config.burrito.hermitcrab` parameter to true in your values file to activate Hermitcrab. + +As the Provider Network Mirror Protocol only supports HTTPS traffic, it is required to provide Burrito runners & the Hermitcrab server with some TLS configuration. By default, the Helm chart expects a secret named `burrito-hermitcrab-tls` to contain TLS configuration: `ca.crt`, `tls.crt`, and `tls.key`. + +### Option 1: Use Cert-Manager + +The Helm chart is packaged with Cert-Manager configuration to use for Burrito/Hermitcrab TLS encryption. +Assuming that Cert-Manager is installed on your cluster, set the `hermitcrab.tls.certmanager.use` parameter to `true`. This setting adds a Cert-Manager Certificate resource to be used with Burrito. +Provide Certificate spec with the `hermitcrab.tls.certmanager.spec` value. You **must** set the `secretName` value to the same value specified in `config.burrito.hermitcrab.certificateSecretName` (default `burrito-hermitcrab-tls`) + +#### Example configuration with a self-signed issuer + +Deploy Cert-Manager resources to generate self-signed certificates: + +```yaml +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: selfsigned-issuer +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: my-selfsigned-ca + namespace: cert-manager +spec: + isCA: true + commonName: my-selfsigned-ca + secretName: root-secret + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: selfsigned-issuer + kind: ClusterIssuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: my-ca-issuer +spec: + ca: + secretName: root-secret +``` + +Update the Helm chart values to create a self-signed certificate: + +```yaml +config: + burrito: + hermitcrab: + enabled: true +... +hermitcrab: + tls: + certManager: + use: true + certificate: + spec: + secretName: burrito-hermitcrab-tls + commonName: burrito-hermitcrab.burrito.svc.cluster.local + dnsNames: + - burrito-hermitcrab.burrito.svc.cluster.local + issuerRef: + name: my-ca-issuer + kind: ClusterIssuer +``` + +Burrito runners should now use Hermitcrab as a network mirror for caching providers. + +### Option 2: Mount a custom certificate + +If Hermitcrab is activated using the Helm chart, Burrito expects a secret named `burrito-hermitcrab-tls` to contain TLS configuration: `ca.crt`, `tls.crt`, and `tls.key`. +Assuming that Cert-Manager is installed on your cluster, set the `tls.certManager.use` value to true and specify an Issuer or ClusterIssuer with `tls.certManager.certificate.issuer.kind` and `tls.certManager.certificate.issuer.name` values. +This will create a [Certificate](https://cert-manager.io/docs/usage/certificate/) custom resource that will be used to ensure TLS between runners and Hermitcrab. + +#### Server side + +Mount your custom certificate to `/etc/hermitcrab/tls/tls.crt` and the private key to `/etc/hermitcrab/tls/tls.key` by using the `hermitcrab.deployment.extraVolumeMounts` and `hermitcrab.deployment.extraVolumeMounts` values. +Check out [the Hermitcrab documentation](https://github.com/seal-io/hermitcrab/blob/main/README.md#usage) for more information about injecting TLS Configuration. + +#### Runner side + +If Hermitcrab is activated using the Helm chart, the Burrito controller expects a secret named `burrito-hermitcrab-tls` to contain client TLS configuration in the `ca.crt` key. This private certificate will be trusted by Burrito runners. diff --git a/internal/burrito/config/config.go b/internal/burrito/config/config.go index d163419c..92b2611b 100644 --- a/internal/burrito/config/config.go +++ b/internal/burrito/config/config.go @@ -16,6 +16,7 @@ type Config struct { Controller ControllerConfig `mapstructure:"controller"` Redis Redis `mapstructure:"redis"` Server ServerConfig `mapstructure:"server"` + Hermitcrab HermitcrabConfig `mapstructure:"hermitcrab"` } type WebhookConfig struct { @@ -32,6 +33,7 @@ type WebhookGitlabConfig struct { } type ControllerConfig struct { + MainNamespace string `mapstructure:"mainNamespace"` Namespaces []string `mapstructure:"namespaces"` Timers ControllerTimers `mapstructure:"timers"` TerraformMaxRetries int `mapstructure:"terraformMaxRetries"` @@ -94,6 +96,11 @@ type Redis struct { Database int `mapstructure:"database"` } +type HermitcrabConfig struct { + Enabled bool `mapstructure:"enabled"` + CertificateSecretName string `mapstructure:"certificateSecretName"` +} + type ServerConfig struct { Addr string `mapstructure:"addr"` Webhook WebhookConfig `mapstructure:"webhook"` diff --git a/internal/controllers/manager.go b/internal/controllers/manager.go index 771c53d1..b5bda1d4 100644 --- a/internal/controllers/manager.go +++ b/internal/controllers/manager.go @@ -81,7 +81,7 @@ func (c *Controllers) Exec() { LeaderElection: c.config.Controller.LeaderElection.Enabled, LeaderElectionID: c.config.Controller.LeaderElection.ID, Cache: cache.Options{ - Namespaces: c.config.Controller.Namespaces, + Namespaces: append(c.config.Controller.Namespaces, c.config.Controller.MainNamespace), }, }) if err != nil { diff --git a/internal/controllers/terraformrun/pod.go b/internal/controllers/terraformrun/pod.go index 85c5f01c..e02d5815 100644 --- a/internal/controllers/terraformrun/pod.go +++ b/internal/controllers/terraformrun/pod.go @@ -7,7 +7,9 @@ import ( configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" "github.com/padok-team/burrito/internal/burrito/config" "github.com/padok-team/burrito/internal/version" + log "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" @@ -52,9 +54,78 @@ func (r *Reconciler) GetLinkedPods(run *configv1alpha1.TerraformRun) (*corev1.Po return list, nil } +func (r *Reconciler) ensureHermitcrabSecret(tenantNamespace string) error { + secret := &corev1.Secret{} + err := r.Client.Get(context.Background(), client.ObjectKey{Namespace: r.Config.Controller.MainNamespace, + Name: r.Config.Hermitcrab.CertificateSecretName}, secret) + if err != nil { + return err + } + if _, ok := secret.Data["ca.crt"]; !ok { + return fmt.Errorf("ca.crt not found in secret %s", r.Config.Hermitcrab.CertificateSecretName) + } + secret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.Config.Hermitcrab.CertificateSecretName, + Namespace: tenantNamespace, + }, + Data: map[string][]byte{ + "ca.crt": secret.Data["ca.crt"], + }, + } + err = r.Client.Create(context.Background(), secret) + if err != nil && apierrors.IsAlreadyExists(err) { + err = r.Client.Update(context.Background(), secret) + if err != nil { + return err + } + } else if err != nil { + return err + } + log.Infof("hermitcrab certificate is available in namespace %s", tenantNamespace) + return nil +} + func (r *Reconciler) getPod(run *configv1alpha1.TerraformRun, layer *configv1alpha1.TerraformLayer, repository *configv1alpha1.TerraformRepository) corev1.Pod { defaultSpec := defaultPodSpec(r.Config, layer, repository) + if r.Config.Hermitcrab.Enabled { + err := r.ensureHermitcrabSecret(layer.Namespace) + if err != nil { + log.Errorf("failed to ensure HermitCrab secret in namespace %s: %s", layer.Namespace, err) + } else { + defaultSpec.Volumes = append(defaultSpec.Volumes, corev1.Volume{ + Name: "hermitcrab-ca-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: r.Config.Hermitcrab.CertificateSecretName, + Items: []corev1.KeyToPath{ + { + Key: "ca.crt", + Path: "hermitcrab-ca.crt", + }, + }, + }, + }, + }) + defaultSpec.Containers[0].VolumeMounts = append(defaultSpec.Containers[0].VolumeMounts, corev1.VolumeMount{ + MountPath: "/etc/ssl/certs/hermitcrab-ca.crt", + Name: "hermitcrab-ca-cert", + SubPath: "hermitcrab-ca.crt", + }) + + defaultSpec.Containers[0].Env = append(defaultSpec.Containers[0].Env, + corev1.EnvVar{ + Name: "HERMITCRAB_ENABLED", + Value: "true", + }, + corev1.EnvVar{ + Name: "HERMITCRAB_URL", + Value: fmt.Sprintf("https://burrito-hermitcrab.%s.svc.cluster.local/v1/providers/", r.Config.Controller.MainNamespace), + }, + ) + } + } switch Action(run.Spec.Action) { case PlanAction: defaultSpec.Containers[0].Env = append(defaultSpec.Containers[0].Env, corev1.EnvVar{ diff --git a/internal/runner/runner.go b/internal/runner/runner.go index ec0a4fd1..b537e800 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -199,6 +199,15 @@ func (r *Runner) init() error { } log.Infof("binaries successfully installed") + if os.Getenv("HERMITCRAB_ENABLED") == "true" { + log.Infof("Hermitcrab configuration detected, creating network mirror configuration...") + err := createNetworkMirrorConfig(os.Getenv("HERMITCRAB_URL")) + if err != nil { + log.Errorf("error creating network mirror configuration: %s", err) + } + log.Infof("network mirror configuration created") + } + workingDir := fmt.Sprintf("%s/%s", WorkingDir, r.layer.Spec.Path) log.Infof("Launching terraform init in %s", workingDir) err = r.exec.Init(workingDir) @@ -324,3 +333,22 @@ func getDiff(plan *tfjson.Plan) (bool, string) { } return diff, fmt.Sprintf("Plan: %d to create, %d to update, %d to delete", create, update, delete) } + +func createNetworkMirrorConfig(endpoint string) error { + terraformrcContent := fmt.Sprintf(` +provider_installation { + network_mirror { + url = "%s" + } +}`, endpoint) + filePath := fmt.Sprintf("%s/config.tfrc", WorkingDir) + err := os.WriteFile(filePath, []byte(terraformrcContent), 0644) + if err != nil { + return err + } + err = os.Setenv("TF_CLI_CONFIG_FILE", filePath) + if err != nil { + return err + } + return nil +}