From f63ace69866ba1e51798475c91764fd55d5f43d1 Mon Sep 17 00:00:00 2001 From: Silenio Quarti Date: Fri, 10 Jan 2025 14:54:05 -0500 Subject: [PATCH] CAA: add support to look up imagePullSecrets for pods This PR initializes auth.json with the imagePullSecrets listed on the pod and service account. Fixes: #2231 Signed-off-by: Silenio Quarti --- .../install/overlays/aws/kustomization.yaml | 4 - .../install/overlays/azure/kustomization.yaml | 4 - .../overlays/docker/kustomization.yaml | 4 - .../install/overlays/gcp/kustomization.yaml | 4 - .../ibmcloud-powervs/kustomization.yaml | 4 - .../overlays/ibmcloud/kustomization.yaml | 4 - .../overlays/libvirt/kustomization.yaml | 4 - .../overlays/vsphere/kustomization.yaml | 4 - .../install/rbac/peer-pod.yaml | 6 ++ .../install/yamls/caa-pod.yaml | 7 -- .../pkg/adaptor/cloud/cloud.go | 18 ++-- .../pkg/adaptor/k8sops/node.go | 91 +++++++++++++++++++ .../test/e2e/assessment_runner.go | 17 ++-- .../provisioner/docker/provision_common.go | 4 - .../ibmcloud/provision_kustomize.go | 5 - .../test/provisioner/kustomize.go | 21 ----- .../provisioner/libvirt/provision_common.go | 4 - 17 files changed, 114 insertions(+), 91 deletions(-) diff --git a/src/cloud-api-adaptor/install/overlays/aws/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/aws/kustomization.yaml index f42fc09caf..8f2a652aea 100644 --- a/src/cloud-api-adaptor/install/overlays/aws/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/aws/kustomization.yaml @@ -45,10 +45,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: -- name: auth-json-secret - namespace: confidential-containers-system - files: - #- auth.json # set - path to auth.json pull credentials file - name: peer-pods-secret namespace: confidential-containers-system # This file should look like this (w/o quotes!): diff --git a/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml index 9692e6302d..8d8216a103 100644 --- a/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml @@ -51,10 +51,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: -- name: auth-json-secret - namespace: confidential-containers-system - files: - #- auth.json # set - path to auth.json pull credentials file - name: peer-pods-secret namespace: confidential-containers-system # This file should look like this (w/o quotes!): diff --git a/src/cloud-api-adaptor/install/overlays/docker/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/docker/kustomization.yaml index 1c8392a351..44b0bcd117 100644 --- a/src/cloud-api-adaptor/install/overlays/docker/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/docker/kustomization.yaml @@ -36,10 +36,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: - - name: auth-json-secret - namespace: confidential-containers-system - # files: - #- auth.json # set - path to auth.json pull credentials file - name: peer-pods-secret namespace: confidential-containers-system literals: diff --git a/src/cloud-api-adaptor/install/overlays/gcp/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/gcp/kustomization.yaml index 77823af107..d44e71334b 100644 --- a/src/cloud-api-adaptor/install/overlays/gcp/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/gcp/kustomization.yaml @@ -33,10 +33,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: -- name: auth-json-secret - namespace: confidential-containers-system - files: - #- auth.json # set - path to auth.json pull credentials file - name: peer-pods-secret namespace: confidential-containers-system files: diff --git a/src/cloud-api-adaptor/install/overlays/ibmcloud-powervs/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/ibmcloud-powervs/kustomization.yaml index 452b4be15a..77c8752823 100644 --- a/src/cloud-api-adaptor/install/overlays/ibmcloud-powervs/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/ibmcloud-powervs/kustomization.yaml @@ -44,10 +44,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: -- name: auth-json-secret - namespace: confidential-containers-system - files: - #- auth.json # set - path to auth.json pull credentials file - name: peer-pods-secret namespace: confidential-containers-system # the below file should be in this format diff --git a/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml index 341271f9e6..f232f21b6a 100644 --- a/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/ibmcloud/kustomization.yaml @@ -42,10 +42,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: -- name: auth-json-secret - namespace: confidential-containers-system - files: - #- auth.json # set - path to auth.json pull credentials file - name: peer-pods-secret namespace: confidential-containers-system literals: diff --git a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml index 5de1893ac4..f6db97ee31 100644 --- a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml @@ -40,10 +40,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: -- name: auth-json-secret - namespace: confidential-containers-system - files: - #- auth.json # set - path to auth.json pull credentials file - name: ssh-key-secret namespace: confidential-containers-system files: # key generation example: ssh-keygen -f ./id_rsa -N "" && sudo cat id_rsa.pub >> /root/.ssh/authorized_keys diff --git a/src/cloud-api-adaptor/install/overlays/vsphere/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/vsphere/kustomization.yaml index 14f5d1ffa5..90e4b6b210 100644 --- a/src/cloud-api-adaptor/install/overlays/vsphere/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/vsphere/kustomization.yaml @@ -57,10 +57,6 @@ configMapGenerator: ##TLS_SETTINGS secretGenerator: -- name: auth-json-secret - namespace: confidential-containers-system - files: - #- auth.json # set - path to auth.json pull credentials file - name: peer-pods-secret namespace: confidential-containers-system literals: diff --git a/src/cloud-api-adaptor/install/rbac/peer-pod.yaml b/src/cloud-api-adaptor/install/rbac/peer-pod.yaml index 79bf50c393..f033c4b216 100644 --- a/src/cloud-api-adaptor/install/rbac/peer-pod.yaml +++ b/src/cloud-api-adaptor/install/rbac/peer-pod.yaml @@ -14,6 +14,12 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "list"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] +- apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get", "list"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/src/cloud-api-adaptor/install/yamls/caa-pod.yaml b/src/cloud-api-adaptor/install/yamls/caa-pod.yaml index d66371d4d2..b15d1e3712 100644 --- a/src/cloud-api-adaptor/install/yamls/caa-pod.yaml +++ b/src/cloud-api-adaptor/install/yamls/caa-pod.yaml @@ -48,9 +48,6 @@ spec: periodSeconds: 20 initialDelaySeconds: 20 volumeMounts: - - name: auth-json - mountPath: "/root/containers/" # hardcoded - readOnly: true - mountPath: /root/.ssh/ name: ssh readOnly: true @@ -73,10 +70,6 @@ spec: nodeSelector: node.kubernetes.io/worker: "" volumes: - - name: auth-json - secret: - secretName: auth-json-secret - optional: true # failing? - name: ssh secret: defaultMode: 384 diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go index 8a88eb9cd8..5f439492b6 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go @@ -36,8 +36,7 @@ import ( ) const ( - SrcAuthfilePath = "/root/containers/auth.json" - Version = "0.0.0" + Version = "0.0.0" ) type InitData struct { @@ -253,16 +252,13 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r agentProxy := s.proxyFactory.New(serverName, socketPath) - var authJSON []byte - _, err = os.Stat(SrcAuthfilePath) + // Look up image pull secrets for the pod + authJSON, err := k8sops.GetImagePullSecrets(pod, namespace) if err != nil { - logger.Printf("credential file %s is not present, skipping image auth config", SrcAuthfilePath) - } else { - authJSON, err = os.ReadFile(SrcAuthfilePath) - if err != nil { - return nil, fmt.Errorf("error reading %s: %v", SrcAuthfilePath, err) - } - logger.Printf("configure agent to use credentials file %s", SrcAuthfilePath) + return nil, fmt.Errorf("error reading image pull secrets: %w", err) + } + if authJSON != nil { + logger.Printf("successfully retrieved pod image pull secrets for %s/%s", namespace, pod) } daemonConfig := forwarder.Config{ diff --git a/src/cloud-api-adaptor/pkg/adaptor/k8sops/node.go b/src/cloud-api-adaptor/pkg/adaptor/k8sops/node.go index c55282aee5..3538ccdd94 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/k8sops/node.go +++ b/src/cloud-api-adaptor/pkg/adaptor/k8sops/node.go @@ -81,6 +81,97 @@ func RemoveExtendedResources() error { return nil } +// Auths contains Registries with credentials +type Auths struct { + Registries Registries `json:"auths"` +} + +// Registries contains credentials for hosts +type Registries map[string]Auth + +// Auth contains credentials for a given host +type Auth struct { + Auth string `json:"auth"` +} + +// GetImagePullSecrets gets image pull secrets for the specified pod +func GetImagePullSecrets(podName string, namespace string) ([]byte, error) { + + config, err := getKubeConfig() + if err != nil { + return nil, fmt.Errorf("failed to get k8s config: %v", err) + } + + cli, err := getClient(config) + if err != nil { + return nil, fmt.Errorf("failed to get k8s client: %v", err) + } + + pod, err := cli.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + accountName := pod.Spec.ServiceAccountName + if accountName == "" { + accountName = "default" + } + serviceaAccount, err := cli.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), accountName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + auths := Auths{} + auths.Registries = make(map[string]Auth) + for _, secret := range serviceaAccount.ImagePullSecrets { + err := getAuths(cli, namespace, secret.Name, &auths) + if err != nil { + return nil, err + } + } + for _, secret := range pod.Spec.ImagePullSecrets { + err := getAuths(cli, namespace, secret.Name, &auths) + if err != nil { + return nil, err + } + } + + if len(auths.Registries) > 0 { + authJSON, err := json.Marshal(auths) + if err != nil { + return nil, err + } + return authJSON, nil + } + return nil, nil +} + +// getAuths get auth credentials from specified docker secret +func getAuths(cli *k8sclient.Clientset, namespace string, secretName string, auths *Auths) error { + secret, err := cli.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err != nil { + return err + } + registries := Registries{} + if secretData, ok := secret.Data[".dockerconfigjson"]; ok { + auths := Auths{} + err := json.Unmarshal(secretData, &auths) + if err != nil { + return err + } + registries = auths.Registries + } else if secretData, ok := secret.Data[".dockercfg"]; ok { + err = json.Unmarshal(secretData, ®istries) + if err != nil { + return err + } + } + for registry, creds := range registries { + auths.Registries[registry] = creds + } + return nil +} + // patchNodeStatus patches the status of a node func patchNodeStatus(c *k8sclient.Clientset, nodeName string, patches []jsonPatch) error { if len(patches) > 0 { diff --git a/src/cloud-api-adaptor/test/e2e/assessment_runner.go b/src/cloud-api-adaptor/test/e2e/assessment_runner.go index d2cf345b36..5d3e82c0a6 100644 --- a/src/cloud-api-adaptor/test/e2e/assessment_runner.go +++ b/src/cloud-api-adaptor/test/e2e/assessment_runner.go @@ -186,13 +186,16 @@ func writeAuthSecret(client klient.Client, ctx context.Context) error { return nil } - providerName := os.Getenv("CLOUD_PROVIDER") - // this path is relative to ./test/e2e - authFilePath := "../../install/overlays/" + providerName + "/auth.json" - authfile, err := os.ReadFile(authFilePath) - if err != nil { - return err - } + template := `{ + "auths": { + "%s": { + "auth": "%s" + } + } + }` + cred := os.Getenv("REGISTRY_CREDENTIAL_ENCODED") + registryName := strings.Split(os.Getenv("AUTHENTICATED_REGISTRY_IMAGE"), "/")[0] + authfile := []byte(fmt.Sprintf(template, registryName, cred)) secretData := map[string][]byte{v1.DockerConfigJsonKey: authfile} secret = NewSecret(E2eNamespace, DEFAULT_AUTH_SECRET, secretData, v1.SecretTypeDockerConfigJson) diff --git a/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go index 67656e1aef..374c2df96d 100644 --- a/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go @@ -257,10 +257,6 @@ func (lio *DockerInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, } } - if err := lio.Overlay.SetAuthJsonSecretIfApplicable(); err != nil { - return err - } - if err := lio.Overlay.YamlReload(); err != nil { return err } diff --git a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go index 7b2f5e3701..51abb803fb 100644 --- a/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go +++ b/src/cloud-api-adaptor/test/provisioner/ibmcloud/provision_kustomize.go @@ -8,7 +8,6 @@ import ( "encoding/json" "io" "net/http" - "path/filepath" "strings" pv "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner" @@ -168,10 +167,6 @@ func (lio *IBMCloudInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config } } - if err = lio.Overlay.SetAuthJsonSecretIfApplicable(); err != nil { - return err - } - if err = lio.Overlay.YamlReload(); err != nil { return err } diff --git a/src/cloud-api-adaptor/test/provisioner/kustomize.go b/src/cloud-api-adaptor/test/provisioner/kustomize.go index 84412365d8..7fa9c315a2 100644 --- a/src/cloud-api-adaptor/test/provisioner/kustomize.go +++ b/src/cloud-api-adaptor/test/provisioner/kustomize.go @@ -430,24 +430,3 @@ func setSecretGeneratorLiteral(k *ktypes.Kustomization, secretName string, key s return nil } - -func (kh *KustomizeOverlay) SetAuthJsonSecretIfApplicable() error { - if cred := os.Getenv("REGISTRY_CREDENTIAL_ENCODED"); cred != "" { - registryName := strings.Split(os.Getenv("AUTHENTICATED_REGISTRY_IMAGE"), "/")[0] - template := `{ - "auths": { - "%s": { - "auth": "%s" - } - } - }` - authJSON := fmt.Sprintf(template, registryName, cred) - if err := os.WriteFile(filepath.Join(kh.ConfigDir, "auth.json"), []byte(authJSON), 0644); err != nil { - return err - } - if err := kh.SetKustomizeSecretGeneratorFile("auth-json-secret", "auth.json"); err != nil { - return err - } - } - return nil -} diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go index eed55669ec..02925b91eb 100644 --- a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go @@ -387,10 +387,6 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, } } - if err = lio.Overlay.SetAuthJsonSecretIfApplicable(); err != nil { - return err - } - if err = lio.Overlay.YamlReload(); err != nil { return err }