From b2c16eba0c5d3134d3125cf4fa671f9a5ffbb109 Mon Sep 17 00:00:00 2001 From: Qi Feng Huo Date: Thu, 15 Aug 2024 17:42:15 +0800 Subject: [PATCH 1/5] initdata: migrate key release test cases to initdata - migrate key release test cases to initdata - remove AA_KBC_PARAMS and aaKBCParams - use allow-all rego policy to make key release test run correctly Fixes: #1985 Signed-off-by: Qi Feng Huo --- .../cmd/cloud-api-adaptor/main.go | 6 +- src/cloud-api-adaptor/docs/addnewprovider.md | 3 +- src/cloud-api-adaptor/docs/initdata.md | 1 - src/cloud-api-adaptor/entrypoint.sh | 1 - .../install/overlays/azure/kustomization.yaml | 1 - .../overlays/libvirt/kustomization.yaml | 1 - src/cloud-api-adaptor/pkg/aa/config.go | 51 ---------- src/cloud-api-adaptor/pkg/aa/config_test.go | 36 ------- .../pkg/adaptor/cloud/cloud.go | 42 ++------- .../pkg/adaptor/cloud/cloud_test.go | 4 +- .../pkg/adaptor/cloud/types.go | 1 - src/cloud-api-adaptor/pkg/adaptor/server.go | 3 +- src/cloud-api-adaptor/pkg/cdh/config.go | 48 ---------- src/cloud-api-adaptor/pkg/cdh/config_test.go | 42 --------- .../pkg/userdata/provision.go | 6 +- src/cloud-api-adaptor/test/e2e/azure_test.go | 6 +- src/cloud-api-adaptor/test/e2e/common.go | 65 +++++++++++++ .../test/e2e/common_suite.go | 12 +-- src/cloud-api-adaptor/test/e2e/docker_test.go | 5 +- .../test/e2e/libvirt_test.go | 9 +- src/cloud-api-adaptor/test/e2e/main_test.go | 5 +- .../provisioner/azure/provision_common.go | 2 +- .../provisioner/docker/provision_common.go | 2 +- .../provisioner/libvirt/provision_common.go | 13 ++- .../test/provisioner/trustee_kbs.go | 7 ++ .../test/tools/provisioner-cli/main.go | 4 +- .../util/cloudinit/cloudconfig.go | 3 +- .../util/cloudinit/cloudconfig_test.go | 93 ------------------- 28 files changed, 115 insertions(+), 357 deletions(-) delete mode 100644 src/cloud-api-adaptor/pkg/aa/config.go delete mode 100644 src/cloud-api-adaptor/pkg/aa/config_test.go delete mode 100644 src/cloud-api-adaptor/pkg/cdh/config.go delete mode 100644 src/cloud-api-adaptor/pkg/cdh/config_test.go diff --git a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go index b103dc06d..9cfb86ee7 100644 --- a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go +++ b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go @@ -24,8 +24,7 @@ import ( ) const ( - programName = "cloud-api-adaptor" - AA_KBC_PARAMS_DEFAULT = "cc_kbc::http://127.0.0.1:8080" + programName = "cloud-api-adaptor" ) type daemonConfig struct { @@ -122,7 +121,6 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) { flags.StringVar(&cfg.networkConfig.HostInterface, "host-interface", "", "Host Interface") flags.IntVar(&cfg.networkConfig.VXLANPort, "vxlan-port", vxlan.DefaultVXLANPort, "VXLAN UDP port number (VXLAN tunnel mode only") flags.IntVar(&cfg.networkConfig.VXLANMinID, "vxlan-min-id", vxlan.DefaultVXLANMinID, "Minimum VXLAN ID (VXLAN tunnel mode only") - flags.StringVar(&cfg.serverConfig.AAKBCParams, "aa-kbc-params", "", "attestation-agent KBC parameters") flags.BoolVar(&cfg.serverConfig.EnableCloudConfigVerify, "cloud-config-verify", false, "Enable cloud config verify - should use it for production") cloud.ParseCmd(flags) @@ -142,8 +140,6 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) { cfg.serverConfig.SecureCommsInbounds = secureCommsInbounds cfg.serverConfig.SecureCommsOutbounds = secureCommsOutbounds cfg.serverConfig.SecureCommsKbsAddress = secureCommsKbsAddr - - cfg.serverConfig.AAKBCParams = AA_KBC_PARAMS_DEFAULT } else { if !disableTLS { cfg.serverConfig.TLSConfig = &tlsConfig diff --git a/src/cloud-api-adaptor/docs/addnewprovider.md b/src/cloud-api-adaptor/docs/addnewprovider.md index 8a2e6bb5a..de023f318 100644 --- a/src/cloud-api-adaptor/docs/addnewprovider.md +++ b/src/cloud-api-adaptor/docs/addnewprovider.md @@ -282,7 +282,6 @@ optionals+="" [[ "${CERT_FILE}" ]] && [[ "${CERT_KEY}" ]] && optionals+="-cert-file ${CERT_FILE} -cert-key ${CERT_KEY} " [[ "${TLS_SKIP_VERIFY}" ]] && optionals+="-tls-skip-verify " [[ "${PROXY_TIMEOUT}" ]] && optionals+="-proxy-timeout ${PROXY_TIMEOUT} " -[[ "${AA_KBC_PARAMS}" ]] && optionals+="-aa-kbc-params ${AA_KBC_PARAMS} " [[ "${FORWARDER_PORT}" ]] && optionals+="-forwarder-port ${FORWARDER_PORT} " [[ "${CLOUD_CONFIG_VERIFY}" == "true" ]] && optionals+="-cloud-config-verify " @@ -396,7 +395,7 @@ cloud-api-adaptor version v0.8.2-dev cloud-api-adaptor: starting Cloud API Adaptor daemon for "libvirt" 2024/04/17 04:34:56 [adaptor/cloud/libvirt] libvirt config: &libvirt.Config{URI:"qemu+ssh://root@192.168.122.1/system?no_verify=1", PoolName:"default", NetworkName:"default", DataDir:"/opt/data-dir", DisableCVM:true, VolName:"podvm-base.qcow2", LaunchSecurity:"", Firmware:"/usr/share/edk2/ovmf/OVMF_CODE.fd"} 2024/04/17 04:34:56 [adaptor/cloud/libvirt] Created libvirt connection -2024/04/17 04:34:56 [adaptor] server config: &adaptor.ServerConfig{TLSConfig:(*tlsutil.TLSConfig)(0xc0000d4080), SocketPath:"/run/peerpod/hypervisor.sock", CriSocketPath:"", PauseImage:"", PodsDir:"/run/peerpod/pods", ForwarderPort:"15150", ProxyTimeout:300000000000, AAKBCParams:"", EnableCloudConfigVerify:false} +2024/04/17 04:34:56 [adaptor] server config: &adaptor.ServerConfig{TLSConfig:(*tlsutil.TLSConfig)(0xc0000d4080), SocketPath:"/run/peerpod/hypervisor.sock", CriSocketPath:"", PauseImage:"", PodsDir:"/run/peerpod/pods", ForwarderPort:"15150", ProxyTimeout:300000000000, EnableCloudConfigVerify:false} 2024/04/17 04:34:56 [util/k8sops] initialized PeerPodService 2024/04/17 04:34:56 [probe/probe] Using port: 8000 2024/04/17 04:34:56 [adaptor] server started diff --git a/src/cloud-api-adaptor/docs/initdata.md b/src/cloud-api-adaptor/docs/initdata.md index 883595a1e..02733ba3c 100644 --- a/src/cloud-api-adaptor/docs/initdata.md +++ b/src/cloud-api-adaptor/docs/initdata.md @@ -2,7 +2,6 @@ The document describes the implementation of the [initdata](https://github.com/confidential-containers/trustee/blob/main/kbs/docs/initdata.md) spec in PeerPods. -Initdata is used when `AA_KBC_PARAMS` is not set at the moment, the plan is to remove `AA_KBC_PARAMS` support after `initdata` function works completely. ## Initdata example diff --git a/src/cloud-api-adaptor/entrypoint.sh b/src/cloud-api-adaptor/entrypoint.sh index 113c2ca9d..b520405ab 100755 --- a/src/cloud-api-adaptor/entrypoint.sh +++ b/src/cloud-api-adaptor/entrypoint.sh @@ -18,7 +18,6 @@ optionals+="" [[ "${CERT_FILE}" ]] && [[ "${CERT_KEY}" ]] && optionals+="-cert-file ${CERT_FILE} -cert-key ${CERT_KEY} " [[ "${TLS_SKIP_VERIFY}" ]] && optionals+="-tls-skip-verify " [[ "${PROXY_TIMEOUT}" ]] && optionals+="-proxy-timeout ${PROXY_TIMEOUT} " -[[ "${AA_KBC_PARAMS}" ]] && optionals+="-aa-kbc-params ${AA_KBC_PARAMS} " [[ "${FORWARDER_PORT}" ]] && optionals+="-forwarder-port ${FORWARDER_PORT} " [[ "${CLOUD_CONFIG_VERIFY}" == "true" ]] && optionals+="-cloud-config-verify " [[ "${SECURE_COMMS}" == "true" ]] && optionals+="-secure-comms " diff --git a/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml index 4131d98a8..ba6cdc0e5 100644 --- a/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml @@ -33,7 +33,6 @@ configMapGenerator: # /subscriptions//resourceGroups//providers/Microsoft.Compute/images/ - AZURE_IMAGE_ID="" #set - SSH_USERNAME="" #set peer pod vm admin user name - - AA_KBC_PARAMS="" #set KBC params for podvm #- DISABLECVM="" # Uncomment it if you want a generic VM #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 diff --git a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml index 4f19d79e7..c66667a90 100644 --- a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml @@ -24,7 +24,6 @@ configMapGenerator: - LIBVIRT_POOL="default" # set - DISABLECVM="true" # set as false to enable confidential VM - SECURE_COMMS="false" # set as true to enable Secure Comms - - AA_KBC_PARAMS="" #set KBC params for podvm #- LIBVIRT_LAUNCH_SECURITY="" #sev or s390-pv #- LIBVIRT_FIRMWARE="" # Uncomment and set if you want to change the firmware path. Defaults to /usr/share/edk2/ovmf/OVMF_CODE.fd #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2 diff --git a/src/cloud-api-adaptor/pkg/aa/config.go b/src/cloud-api-adaptor/pkg/aa/config.go deleted file mode 100644 index a7e089c81..000000000 --- a/src/cloud-api-adaptor/pkg/aa/config.go +++ /dev/null @@ -1,51 +0,0 @@ -package aa - -import ( - "fmt" - "strings" - - toml "github.com/pelletier/go-toml/v2" -) - -const ( - ConfigFilePath = "/run/peerpod/aa.toml" -) - -type AAConfig struct { - TokenCfg struct { - CocoAs struct { - URL string `toml:"url"` - } `toml:"coco_as"` - Kbs struct { - URL string `toml:"url"` - } `toml:"kbs"` - } `toml:"token_configs"` -} - -func parseAAKBCParams(aaKBCParams string) (string, error) { - parts := strings.SplitN(aaKBCParams, "::", 2) - if len(parts) != 2 { - return "", fmt.Errorf("Invalid aa-kbs-params input: %s", aaKBCParams) - } - _, url := parts[0], parts[1] - return url, nil -} - -func CreateConfigFile(aaKBCParams string) (string, error) { - url, err := parseAAKBCParams(aaKBCParams) - if err != nil { - return "", err - } - - config := AAConfig{} - // Assume KBS and AS has same endpoint - // Need a new parameter in addition to aaKBCParams if deploy AS and KBS separately. - config.TokenCfg.CocoAs.URL = url - config.TokenCfg.Kbs.URL = url - - bytes, err := toml.Marshal(config) - if err != nil { - return "", err - } - return string(bytes), nil -} diff --git a/src/cloud-api-adaptor/pkg/aa/config_test.go b/src/cloud-api-adaptor/pkg/aa/config_test.go deleted file mode 100644 index 2a6f53ea0..000000000 --- a/src/cloud-api-adaptor/pkg/aa/config_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package aa - -import ( - "testing" -) - -func Test_parseAAKBCParams(t *testing.T) { - url, err := parseAAKBCParams("cc_kbc::http://127.0.0.1:8080") - if err != nil { - t.Error(err) - } - - expected := "http://127.0.0.1:8080" - if url != expected { - t.Errorf("Expected %s, got %s", expected, url) - } -} - -func TestConfigFile(t *testing.T) { - refcfg := `[token_configs] -[token_configs.coco_as] -url = 'http://127.0.0.1:8080' - -[token_configs.kbs] -url = 'http://127.0.0.1:8080' -` - - config, err := CreateConfigFile("cc_kbc::http://127.0.0.1:8080") - if err != nil { - t.Error(err) - } - - if config != refcfg { - t.Errorf("Expected: \n%s, got: \n%s", refcfg, config) - } -} diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go index 988b47433..36e41fe72 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go @@ -19,11 +19,9 @@ import ( "github.com/containerd/containerd/pkg/cri/annotations" pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" - "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/aa" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/k8sops" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/proxy" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/agent" - "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/cdh" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/wnssh" @@ -35,7 +33,9 @@ import ( const ( SrcAuthfilePath = "/root/containers/auth.json" + AaFilePath = "/run/peerpod/aa.toml" AuthFilePath = "/run/peerpod/auth.json" + CdhFilePath = "/run/peerpod/cdh.toml" InitdataPath = "/run/peerpod/initdata" Version = "0.0.0" ) @@ -79,8 +79,7 @@ func (s *cloudService) removeSandbox(id sandboxID) error { } func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNode podnetwork.WorkerNode, - secureComms bool, secureCommsInbounds, secureCommsOutbounds, kbsAddress, podsDir, daemonPort, aaKBCParams, sshport string, -) Service { + secureComms bool, secureCommsInbounds, secureCommsOutbounds, kbsAddress, podsDir, daemonPort, sshport string) Service { var err error var sshClient *wnssh.SshClient @@ -100,7 +99,6 @@ func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNo podsDir: podsDir, daemonPort: daemonPort, workerNode: workerNode, - aaKBCParams: aaKBCParams, sshClient: sshClient, } s.cond = sync.NewCond(&s.mutex) @@ -292,39 +290,13 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r } } - if s.aaKBCParams != "" { // Keep AA_KBC_PARAMS support as it is used by e2e test, KBS is dynamic k8s service in e2e test - logger.Printf("aaKBCParams: %s, support cc_kbc::*", s.aaKBCParams) - toml, err := cdh.CreateConfigFile(s.aaKBCParams) - if err != nil { - return nil, fmt.Errorf("creating CDH config: %w", err) - } - cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{ - Path: cdh.ConfigFilePath, - Content: toml, - }) - - toml, err = aa.CreateConfigFile(s.aaKBCParams) - if err != nil { - return nil, fmt.Errorf("creating attestation agent config: %w", err) - } - cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{ - Path: aa.ConfigFilePath, - Content: toml, - }) - } - initdataStr := util.GetInitdataFromAnnotation(req.Annotations) logger.Printf("initdata: %s", initdataStr) if initdataStr != "" { - if s.aaKBCParams != "" { - logger.Printf("Initdata ignored because AA_KBC_PARAMS set") - } else { - logger.Printf("Set and use initdata when no AA_KBC_PARAMS") - cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{ - Path: InitdataPath, - Content: initdataStr, - }) - } + cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{ + Path: InitdataPath, + Content: initdataStr, + }) } sandbox := &sandbox{ diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go index a8030ef72..d751d880b 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go @@ -117,7 +117,7 @@ func TestCloudService(t *testing.T) { podsDir: dir, } - s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, false, "", "", "", dir, forwarder.DefaultListenPort, "", "") + s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, false, "", "", "", dir, forwarder.DefaultListenPort, "") assert.NotNil(t, s) @@ -172,7 +172,7 @@ func TestCloudServiceWithSecureComms(t *testing.T) { podsDir: dir, } - s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, true, "", "", "127.0.0.1:9009", dir, forwarder.DefaultListenPort, "", sshport) + s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, true, "", "", "127.0.0.1:9009", dir, forwarder.DefaultListenPort, sshport) assert.NotNil(t, s) diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go index edb23d702..7e7295340 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go @@ -35,7 +35,6 @@ type cloudService struct { daemonPort string mutex sync.Mutex ppService *k8sops.PeerPodService - aaKBCParams string sshClient *wnssh.SshClient } diff --git a/src/cloud-api-adaptor/pkg/adaptor/server.go b/src/cloud-api-adaptor/pkg/adaptor/server.go index 8b58a888b..75643658b 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/server.go +++ b/src/cloud-api-adaptor/pkg/adaptor/server.go @@ -39,7 +39,6 @@ type ServerConfig struct { PodsDir string ForwarderPort string ProxyTimeout time.Duration - AAKBCParams string EnableCloudConfigVerify bool SecureComms bool SecureCommsInbounds string @@ -71,7 +70,7 @@ func NewServer(provider provider.Provider, cfg *ServerConfig, workerNode podnetw agentFactory := proxy.NewFactory(cfg.PauseImage, cfg.TLSConfig, cfg.ProxyTimeout) cloudService := cloud.NewService(provider, agentFactory, workerNode, - cfg.SecureComms, cfg.SecureCommsInbounds, cfg.SecureCommsOutbounds, cfg.SecureCommsKbsAddress, cfg.PodsDir, cfg.ForwarderPort, cfg.AAKBCParams, sshutil.SSHPORT) + cfg.SecureComms, cfg.SecureCommsInbounds, cfg.SecureCommsOutbounds, cfg.SecureCommsKbsAddress, cfg.PodsDir, cfg.ForwarderPort, sshutil.SSHPORT) vmInfoService := vminfo.NewService(cloudService) return &server{ diff --git a/src/cloud-api-adaptor/pkg/cdh/config.go b/src/cloud-api-adaptor/pkg/cdh/config.go deleted file mode 100644 index 0cc405a9c..000000000 --- a/src/cloud-api-adaptor/pkg/cdh/config.go +++ /dev/null @@ -1,48 +0,0 @@ -package cdh - -import ( - "fmt" - "strings" - - "github.com/pelletier/go-toml/v2" -) - -const ( - ConfigFilePath = "/run/peerpod/cdh.toml" - Socket = "unix:///run/confidential-containers/cdh.sock" -) - -type Credential struct{} - -type Config struct { - Socket string `toml:"socket"` - KBC KBCConfig `toml:"kbc"` - Credentials []Credential `toml:"credentials"` -} - -type KBCConfig struct { - Name string `toml:"name"` - URL string `toml:"url"` -} - -func parseAAKBCParams(aaKBCParams string) (*Config, error) { - parts := strings.SplitN(aaKBCParams, "::", 2) - if len(parts) != 2 { - return nil, fmt.Errorf("Invalid aa-kbs-params input: %s", aaKBCParams) - } - name, url := parts[0], parts[1] - kbcConfig := KBCConfig{name, url} - return &Config{Socket, kbcConfig, []Credential{}}, nil -} - -func CreateConfigFile(aaKBCParams string) (string, error) { - config, err := parseAAKBCParams(aaKBCParams) - if err != nil { - return "", err - } - bytes, err := toml.Marshal(config) - if err != nil { - return "", err - } - return string(bytes), nil -} diff --git a/src/cloud-api-adaptor/pkg/cdh/config_test.go b/src/cloud-api-adaptor/pkg/cdh/config_test.go deleted file mode 100644 index d3eb093a4..000000000 --- a/src/cloud-api-adaptor/pkg/cdh/config_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package cdh - -import ( - "fmt" - "testing" - - "github.com/pelletier/go-toml/v2" -) - -func TestCDHConfigFileFromAAKBCParams(t *testing.T) { - refdoc := ` -socket = "%s" -credentials = [] -[kbc] -name = "cc_kbc" -url = "http://1.2.3.4:8080" -` - refdoc = fmt.Sprintf(refdoc, Socket) - var refcfg Config - err := toml.Unmarshal([]byte(refdoc), &refcfg) - if err != nil { - panic(err) - } - - config, err := parseAAKBCParams("cc_kbc::http://1.2.3.4:8080") - if err != nil { - t.Error(err) - } - - if config.KBC.Name != refcfg.KBC.Name { - t.Errorf("Expected %s, got %s", refcfg.KBC.Name, config.KBC.Name) - } - if config.KBC.URL != refcfg.KBC.URL { - t.Errorf("Expected %s, got %s", refcfg.KBC.URL, config.KBC.URL) - } - if config.Socket != refcfg.Socket { - t.Errorf("Expected %s, got %s", refcfg.Socket, config.Socket) - } - if len(config.Credentials) != 0 { - t.Errorf("Expected empty credentials array") - } -} diff --git a/src/cloud-api-adaptor/pkg/userdata/provision.go b/src/cloud-api-adaptor/pkg/userdata/provision.go index cc5f2909e..f7ae922d4 100644 --- a/src/cloud-api-adaptor/pkg/userdata/provision.go +++ b/src/cloud-api-adaptor/pkg/userdata/provision.go @@ -14,10 +14,8 @@ import ( "time" "github.com/avast/retry-go/v4" - "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/aa" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/agent" - "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/cdh" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/aws" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/azure" @@ -33,8 +31,8 @@ const ( ) var logger = log.New(log.Writer(), "[userdata/provision] ", log.LstdFlags|log.Lmsgprefix) -var WriteFilesList = []string{aa.ConfigFilePath, cdh.ConfigFilePath, agent.ConfigFilePath, forwarder.DefaultConfigPath, cloud.AuthFilePath, cloud.InitdataPath} -var InitdDataFilesList = []string{aa.ConfigFilePath, cdh.ConfigFilePath, PolicyPath} +var WriteFilesList = []string{cloud.AaFilePath, cloud.CdhFilePath, agent.ConfigFilePath, forwarder.DefaultConfigPath, cloud.AuthFilePath, cloud.InitdataPath} +var InitdDataFilesList = []string{cloud.AaFilePath, cloud.CdhFilePath, PolicyPath} type Config struct { fetchTimeout int diff --git a/src/cloud-api-adaptor/test/e2e/azure_test.go b/src/cloud-api-adaptor/test/e2e/azure_test.go index bceabd372..bd0384639 100644 --- a/src/cloud-api-adaptor/test/e2e/azure_test.go +++ b/src/cloud-api-adaptor/test/e2e/azure_test.go @@ -129,7 +129,8 @@ func TestKbsKeyRelease(t *testing.T) { t.Skip("Skipping kbs related test as kbs is not deployed") } t.Parallel() - DoTestKbsKeyRelease(t, testEnv, assert) + kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint() + DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint) } func TestRemoteAttestation(t *testing.T) { @@ -145,5 +146,6 @@ func TestTrusteeOperatorKeyReleaseForSpecificKey(t *testing.T) { t.Skip("Skipping kbs related test as Trustee Operator is not deployed") } t.Parallel() - DoTestTrusteeOperatorKeyReleaseForSpecificKey(t, testEnv, assert) + kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint() + DoTestTrusteeOperatorKeyReleaseForSpecificKey(t, testEnv, assert, kbsEndpoint) } diff --git a/src/cloud-api-adaptor/test/e2e/common.go b/src/cloud-api-adaptor/test/e2e/common.go index 2adfc7c6d..10d61d419 100644 --- a/src/cloud-api-adaptor/test/e2e/common.go +++ b/src/cloud-api-adaptor/test/e2e/common.go @@ -5,6 +5,7 @@ package e2e import ( b64 "encoding/base64" + "fmt" "net" "os" "testing" @@ -28,6 +29,61 @@ const BUSYBOX_IMAGE = "quay.io/prometheus/busybox:latest" const WAIT_DEPLOYMENT_AVAILABLE_TIMEOUT = time.Second * 180 const DEFAULT_AUTH_SECRET = "auth-json-secret-default" +var testInitdata string = `algorithm = "sha384" +version = "0.1.0" + +[data] +"aa.toml" = ''' +[token_configs] +[token_configs.coco_as] +url = '%s' + +[token_configs.kbs] +url = '%s' +''' + +"cdh.toml" = ''' +socket = 'unix:///run/confidential-containers/cdh.sock' +credentials = [] + +[kbc] +name = 'cc_kbc' +url = '%s' +''' + +"policy.rego" = ''' +package agent_policy + +import future.keywords.in +import future.keywords.every + +import input + +# Default values, returned by OPA when rules cannot be evaluated to true. +default CopyFileRequest := true +default CreateContainerRequest := true +default CreateSandboxRequest := true +default DestroySandboxRequest := true +default ExecProcessRequest := true +default GetOOMEventRequest := true +default GuestDetailsRequest := true +default OnlineCPUMemRequest := true +default PullImageRequest := true +default ReadStreamRequest := true +default RemoveContainerRequest := true +default RemoveStaleVirtiofsShareMountsRequest := true +default SignalProcessRequest := true +default StartContainerRequest := true +default StatsContainerRequest := true +default TtyWinResizeRequest := true +default UpdateEphemeralMountsRequest := true +default UpdateInterfaceRequest := true +default UpdateRoutesRequest := true +default WaitProcessRequest := true +default WriteStreamRequest := true +''' +` + func isTestWithKbs() bool { return os.Getenv("TEST_KBS") == "yes" || os.Getenv("TEST_KBS") == "true" } @@ -218,6 +274,15 @@ func NewBusyboxPodWithName(namespace, podName string) *corev1.Pod { return NewPod(namespace, podName, "busybox", BUSYBOX_IMAGE, WithCommand([]string{"/bin/sh", "-c", "sleep 3600"})) } +func NewBusyboxPodWithNameWithInitdata(namespace, podName string, kbsEndpoint string) *corev1.Pod { + initdata := fmt.Sprintf(testInitdata, kbsEndpoint, kbsEndpoint, kbsEndpoint) + b64Data := b64.StdEncoding.EncodeToString([]byte(initdata)) + annotationData := map[string]string{ + "io.katacontainers.config.runtime.cc_init_data": b64Data, + } + return NewPod(namespace, podName, "busybox", BUSYBOX_IMAGE, WithCommand([]string{"/bin/sh", "-c", "sleep 3600"}), WithAnnotations(annotationData)) +} + func NewPodWithPolicy(namespace, podName, policyFilePath string) *corev1.Pod { containerName := "busybox" imageName := BUSYBOX_IMAGE diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go index 1fd624da9..e55a863ac 100644 --- a/src/cloud-api-adaptor/test/e2e/common_suite.go +++ b/src/cloud-api-adaptor/test/e2e/common_suite.go @@ -576,9 +576,9 @@ func DoTestPodsMTLSCommunication(t *testing.T, e env.Environment, assert CloudAs // DoTestKbsKeyRelease and DoTestKbsKeyReleaseForFailure should be run in a single test case if you're chaining opa in kbs // as test cases might be run in parallel -func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert) { +func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint string) { t.Log("Do test kbs key release") - pod := NewBusyboxPodWithName(E2eNamespace, "kbs-key-release") + pod := NewBusyboxPodWithNameWithInitdata(E2eNamespace, "kbs-key-release", kbsEndpoint) testCommands := []TestCommand{ { Command: []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/reponame/workload_key/key.bin"}, @@ -600,9 +600,9 @@ func DoTestKbsKeyRelease(t *testing.T, e env.Environment, assert CloudAssert) { // DoTestKbsKeyRelease and DoTestKbsKeyReleaseForFailure should be run in a single test case if you're chaining opa in kbs // as test cases might be run in parallel -func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert CloudAssert) { +func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint string) { t.Log("Do test kbs key release failure case") - pod := NewBusyboxPodWithName(E2eNamespace, "kbs-failure") + pod := NewBusyboxPodWithNameWithInitdata(E2eNamespace, "kbs-failure", kbsEndpoint) testCommands := []TestCommand{ { Command: []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/reponame/workload_key/key.bin"}, @@ -631,9 +631,9 @@ func DoTestKbsKeyReleaseForFailure(t *testing.T, e env.Environment, assert Cloud } // Test to check for specific key value from Trustee Operator Deployment -func DoTestTrusteeOperatorKeyReleaseForSpecificKey(t *testing.T, e env.Environment, assert CloudAssert) { +func DoTestTrusteeOperatorKeyReleaseForSpecificKey(t *testing.T, e env.Environment, assert CloudAssert, kbsEndpoint string) { t.Log("Do test Trustee operator key release for specific key") - pod := NewBusyboxPodWithName(E2eNamespace, "op-key-release") + pod := NewBusyboxPodWithNameWithInitdata(E2eNamespace, "op-key-release", kbsEndpoint) testCommands := []TestCommand{ { Command: []string{"wget", "-q", "-O-", "http://127.0.0.1:8006/cdh/resource/default/kbsres1/key1"}, diff --git a/src/cloud-api-adaptor/test/e2e/docker_test.go b/src/cloud-api-adaptor/test/e2e/docker_test.go index 0438b451d..8ae45f643 100644 --- a/src/cloud-api-adaptor/test/e2e/docker_test.go +++ b/src/cloud-api-adaptor/test/e2e/docker_test.go @@ -103,9 +103,10 @@ func TestDockerKbsKeyRelease(t *testing.T) { } keyBrokerService.SetSampleSecretKey() keyBrokerService.EnableKbsCustomizedResourcePolicy("deny_all.rego") + kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint() assert := DockerAssert{} t.Parallel() - DoTestKbsKeyReleaseForFailure(t, testEnv, assert) + DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint) keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego") - DoTestKbsKeyRelease(t, testEnv, assert) + DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint) } diff --git a/src/cloud-api-adaptor/test/e2e/libvirt_test.go b/src/cloud-api-adaptor/test/e2e/libvirt_test.go index 52b37e81a..4586e60cf 100644 --- a/src/cloud-api-adaptor/test/e2e/libvirt_test.go +++ b/src/cloud-api-adaptor/test/e2e/libvirt_test.go @@ -111,21 +111,22 @@ func TestLibvirtKbsKeyRelease(t *testing.T) { _ = keyBrokerService.SetSampleSecretKey() _ = keyBrokerService.EnableKbsCustomizedResourcePolicy("allow_all.rego") _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("deny_all.rego") + kbsEndpoint, _ := keyBrokerService.GetCachedKbsEndpoint() assert := LibvirtAssert{} t.Parallel() - DoTestKbsKeyReleaseForFailure(t, testEnv, assert) + DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint) if isTestWithKbsIBMSE() { t.Log("KBS with ibmse cases") // the allow_*_.rego file is created by follow document // https://github.com/confidential-containers/trustee/blob/main/deps/verifier/src/se/README.md#set-attestation-policy _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_wrong_image_tag.rego") - DoTestKbsKeyReleaseForFailure(t, testEnv, assert) + DoTestKbsKeyReleaseForFailure(t, testEnv, assert, kbsEndpoint) _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_with_correct_claims.rego") - DoTestKbsKeyRelease(t, testEnv, assert) + DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint) } else { t.Log("KBS normal cases") _ = keyBrokerService.EnableKbsCustomizedAttestationPolicy("allow_all.rego") - DoTestKbsKeyRelease(t, testEnv, assert) + DoTestKbsKeyRelease(t, testEnv, assert, kbsEndpoint) } } diff --git a/src/cloud-api-adaptor/test/e2e/main_test.go b/src/cloud-api-adaptor/test/e2e/main_test.go index fef5fff8a..8363b4711 100644 --- a/src/cloud-api-adaptor/test/e2e/main_test.go +++ b/src/cloud-api-adaptor/test/e2e/main_test.go @@ -152,7 +152,6 @@ func TestMain(m *testing.M) { } } - var kbsparams string if shouldDeployKbs { log.Info("Deploying kbs") if keyBrokerService, err = pv.NewKeyBrokerService(props["CLUSTER_NAME"], cfg); err != nil { @@ -167,8 +166,7 @@ func TestMain(m *testing.M) { return ctx, err } - kbsparams = "cc_kbc::" + kbsEndpoint - log.Infof("KBS PARAMS: %s", kbsparams) + log.Infof("kbsEndpoint: %s", kbsEndpoint) } if podvmImage != "" { @@ -186,7 +184,6 @@ func TestMain(m *testing.M) { } props = provisioner.GetProperties(ctx, cfg) - props["AA_KBC_PARAMS"] = kbsparams log.Info("Deploy the Cloud API Adaptor") if err = cloudAPIAdaptor.Deploy(ctx, cfg, props); err != nil { return ctx, err diff --git a/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go b/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go index cc4bcc84c..c00ad5f58 100644 --- a/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go @@ -381,7 +381,7 @@ func (p *AzureCloudProvisioner) UploadPodvm(imagePath string, ctx context.Contex func isAzureKustomizeConfigMapKey(key string) bool { switch key { - case "CLOUD_PROVIDER", "AZURE_SUBSCRIPTION_ID", "AZURE_REGION", "AZURE_INSTANCE_SIZE", "AZURE_RESOURCE_GROUP", "AZURE_SUBNET_ID", "AZURE_IMAGE_ID", "SSH_USERNAME", "AA_KBC_PARAMS", "TAGS": + case "CLOUD_PROVIDER", "AZURE_SUBSCRIPTION_ID", "AZURE_REGION", "AZURE_INSTANCE_SIZE", "AZURE_RESOURCE_GROUP", "AZURE_SUBNET_ID", "AZURE_IMAGE_ID", "SSH_USERNAME", "TAGS": return true default: return false diff --git a/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go index f715500f1..fb92e3b55 100644 --- a/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go @@ -206,7 +206,7 @@ func NewDockerInstallOverlay(installDir, provider string) (pv.InstallOverlay, er func isDockerKustomizeConfigMapKey(key string) bool { switch key { - case "CLOUD_PROVIDER", "DOCKER_HOST", "DOCKER_API_VERSION", "DOCKER_PODVM_IMAGE", "DOCKER_NETWORK_NAME", "AA_KBC_PARAMS": + case "CLOUD_PROVIDER", "DOCKER_HOST", "DOCKER_API_VERSION", "DOCKER_PODVM_IMAGE", "DOCKER_NETWORK_NAME": return true default: return false diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go index d6532d6e9..69cfd3a34 100644 --- a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go @@ -320,13 +320,12 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, // Mapping the internal properties to ConfigMapGenerator properties and their default values. mapProps := map[string][2]string{ - "network": {"default", "LIBVIRT_NET"}, - "storage": {"default", "LIBVIRT_POOL"}, - "pause_image": {"", "PAUSE_IMAGE"}, - "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, - "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, - "vxlan_port": {"", "VXLAN_PORT"}, - "AA_KBC_PARAMS": {"", "AA_KBC_PARAMS"}, + "network": {"default", "LIBVIRT_NET"}, + "storage": {"default", "LIBVIRT_POOL"}, + "pause_image": {"", "PAUSE_IMAGE"}, + "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, + "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, + "vxlan_port": {"", "VXLAN_PORT"}, } for k, v := range mapProps { diff --git a/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go b/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go index 7229fcdca..a628b1ffb 100644 --- a/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go +++ b/src/cloud-api-adaptor/test/provisioner/trustee_kbs.go @@ -393,6 +393,13 @@ func getNodeIPForSvc(deploymentName string, service corev1.Service, cfg *envconf return "", fmt.Errorf("Node IP not found for Service %s", service.Name) } +func (p *KeyBrokerService) GetCachedKbsEndpoint() (string, error) { + if p.endpoint != "" { + return p.endpoint, nil + } + return "", fmt.Errorf("KeyBrokerService not found") +} + func (p *KeyBrokerService) GetKbsEndpoint(ctx context.Context, cfg *envconf.Config) (string, error) { client, err := cfg.NewClient() if err != nil { diff --git a/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go b/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go index c97836a4e..8348d48a0 100644 --- a/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go +++ b/src/cloud-api-adaptor/test/tools/provisioner-cli/main.go @@ -109,11 +109,9 @@ func main() { log.Fatal(err) } - kbsparams := "cc_kbc::" + kbsEndpoint - log.Infof("KBS PARAMS: %s", kbsparams) + log.Infof("keyBrokerService: %s", kbsEndpoint) props = provisioner.GetProperties(context.TODO(), cfg) - props["AA_KBC_PARAMS"] = kbsparams } cloudAPIAdaptor, err := pv.NewCloudAPIAdaptor(cloudProvider, installDirectory) diff --git a/src/cloud-providers/util/cloudinit/cloudconfig.go b/src/cloud-providers/util/cloudinit/cloudconfig.go index 49d23f13a..b5cb7ba82 100644 --- a/src/cloud-providers/util/cloudinit/cloudconfig.go +++ b/src/cloud-providers/util/cloudinit/cloudconfig.go @@ -12,8 +12,7 @@ import ( ) const ( - DefaultAuthfileLimit = 12288 // TODO: use a whole userdata limit mechanism instead of limiting authfile - DefaultAAKBCParamsPath = "/etc/attestation-agent/kbc-params.json" + DefaultAuthfileLimit = 12288 // TODO: use a whole userdata limit mechanism instead of limiting authfile ) // https://cloudinit.readthedocs.io/en/latest/topics/format.html#cloud-config-data diff --git a/src/cloud-providers/util/cloudinit/cloudconfig_test.go b/src/cloud-providers/util/cloudinit/cloudconfig_test.go index 50641af33..9543509d2 100644 --- a/src/cloud-providers/util/cloudinit/cloudconfig_test.go +++ b/src/cloud-providers/util/cloudinit/cloudconfig_test.go @@ -136,96 +136,3 @@ func TestUserDataWithDaemonAndAuth(t *testing.T) { } } - -// Test userData with a daemon.json file, an auth.json file and -// kbc-params. -// The test should verify that the config has the daemon.json, auth.json and kbc-params -// files in the write_files section. -func TestUserDataWithDaemonAndAuthAndAAKBCParams(t *testing.T) { - testDaemonConfigJson := `{ - "pod-network": { - "podip": "10.244.0.19/24", - "pod-hw-addr": "0e:8f:62:f3:81:ad", - "interface": "eth0", - "worker-node-ip": "10.224.0.4/16", - "tunnel-type": "vxlan", - "routes": [ - { - "Dst": "", - "GW": "10.244.0.1", - "Dev": "eth0" - } - ], - "mtu": 1500, - "index": 1, - "vxlan-port": 8472, - "vxlan-id": 555001, - "dedicated": false - }, - "pod-namespace": "default", - "pod-name": "nginx-866fdb5bfb-b98nw", - "tls-server-key": "-----BEGIN PRIVATE KEY-----\n....\n-----END PRIVATE KEY-----\n", - "tls-server-cert": "-----BEGIN CERTIFICATE-----\n....\n-----END CERTIFICATE-----\n", - "tls-client-ca": "-----BEGIN CERTIFICATE-----\n....\n-----END CERTIFICATE-----\n", - "aa-kbc-params": "cc_kbc::http://192.168.100.2:8080" - }` - - // Create a variable to hold sample base64 encoded string which is the auth.json - // file - testAuthJson := `{ - "auths": { - "myregistry.io": { - "auth": "dXNlcjpwYXNzd29yZAo" - } - } - }` - - testResourcesJson := AuthJSONToResourcesJSON(string(testAuthJson)) - - // Create a CloudConfig struct - cloudConfig := &CloudConfig{ - WriteFiles: []WriteFile{ - {Path: forwarderConfigPath, Content: string(testDaemonConfigJson)}, - {Path: authJSONPath, Content: testResourcesJson}, - }, - } - - // Generate userData from cloudConfig - userData, err := cloudConfig.Generate() - if err != nil { - t.Fatalf("Expect no error, got %v", err) - } - - // Pretty print the userData - fmt.Printf("userData: %s\n", userData) - - // Verify that the userData has the daemon.json, auth.json and kbc-params files - // in the write_files section - if !strings.Contains(userData, forwarderConfigPath) { - t.Fatalf("Expect %q, got %q", forwarderConfigPath, userData) - } - - if !strings.Contains(userData, authJSONPath) { - t.Fatalf("Expect %q, got %q", authJSONPath, userData) - } - - var output CloudConfig - - if err := yaml.Unmarshal([]byte(userData), &output); err != nil { - t.Fatalf("Expect no error, got %v", err) - } - - // Pretty print the userData output - fmt.Printf("userData: %s\n", output) - - // Verify that the output yaml has the testDaemonConfigJson, testb64AuthJson and testAAKBCParams contents - // in the write_files section - if !strings.Contains(output.WriteFiles[0].Content, testDaemonConfigJson) { - t.Fatalf("Expect %q, got %q", testDaemonConfigJson, output.WriteFiles[0].Content) - } - - if !strings.Contains(output.WriteFiles[1].Content, testResourcesJson) { - t.Fatalf("Expect %q, got %q", testResourcesJson, output.WriteFiles[1].Content) - } - -} From e4fe685a9562695f99e483b6f1b051db0d3b4115 Mon Sep 17 00:00:00 2001 From: Qi Feng Huo Date: Mon, 19 Aug 2024 17:08:02 +0800 Subject: [PATCH 2/5] initdata: add global-initdata in configmap - add global-initdata in configmap and parameters Fixes: #1985 Signed-off-by: Qi Feng Huo --- .../cmd/cloud-api-adaptor/main.go | 1 + src/cloud-api-adaptor/docs/addnewprovider.md | 1 + src/cloud-api-adaptor/docs/initdata.md | 19 ++++++++++++++ src/cloud-api-adaptor/entrypoint.sh | 1 + .../install/overlays/azure/kustomization.yaml | 1 + .../overlays/libvirt/kustomization.yaml | 1 + .../pkg/adaptor/cloud/cloud.go | 25 ++++++++++++------- .../pkg/adaptor/cloud/cloud_test.go | 4 +-- .../pkg/adaptor/cloud/types.go | 21 ++++++++-------- src/cloud-api-adaptor/pkg/adaptor/server.go | 3 ++- .../provisioner/azure/provision_common.go | 2 +- .../provisioner/docker/provision_common.go | 2 +- .../provisioner/libvirt/provision_common.go | 13 +++++----- 13 files changed, 64 insertions(+), 30 deletions(-) diff --git a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go index 9cfb86ee7..aea2617c5 100644 --- a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go +++ b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go @@ -121,6 +121,7 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) { flags.StringVar(&cfg.networkConfig.HostInterface, "host-interface", "", "Host Interface") flags.IntVar(&cfg.networkConfig.VXLANPort, "vxlan-port", vxlan.DefaultVXLANPort, "VXLAN UDP port number (VXLAN tunnel mode only") flags.IntVar(&cfg.networkConfig.VXLANMinID, "vxlan-min-id", vxlan.DefaultVXLANMinID, "Minimum VXLAN ID (VXLAN tunnel mode only") + flags.StringVar(&cfg.serverConfig.GlobalInitdata, "global-initdata", "", "Default initdata for all Pods") flags.BoolVar(&cfg.serverConfig.EnableCloudConfigVerify, "cloud-config-verify", false, "Enable cloud config verify - should use it for production") cloud.ParseCmd(flags) diff --git a/src/cloud-api-adaptor/docs/addnewprovider.md b/src/cloud-api-adaptor/docs/addnewprovider.md index de023f318..d9f31efb6 100644 --- a/src/cloud-api-adaptor/docs/addnewprovider.md +++ b/src/cloud-api-adaptor/docs/addnewprovider.md @@ -282,6 +282,7 @@ optionals+="" [[ "${CERT_FILE}" ]] && [[ "${CERT_KEY}" ]] && optionals+="-cert-file ${CERT_FILE} -cert-key ${CERT_KEY} " [[ "${TLS_SKIP_VERIFY}" ]] && optionals+="-tls-skip-verify " [[ "${PROXY_TIMEOUT}" ]] && optionals+="-proxy-timeout ${PROXY_TIMEOUT} " +[[ "${GLOBAL_INITDATA}" ]] && optionals+="-global-initdata ${GLOBAL_INITDATA} " [[ "${FORWARDER_PORT}" ]] && optionals+="-forwarder-port ${FORWARDER_PORT} " [[ "${CLOUD_CONFIG_VERIFY}" == "true" ]] && optionals+="-cloud-config-verify " diff --git a/src/cloud-api-adaptor/docs/initdata.md b/src/cloud-api-adaptor/docs/initdata.md index 02733ba3c..3f0752ae4 100644 --- a/src/cloud-api-adaptor/docs/initdata.md +++ b/src/cloud-api-adaptor/docs/initdata.md @@ -174,5 +174,24 @@ allow if { } ``` +## Global initdata +If all of your applications(Pods) are using same initdata, it's convenient you set the `GLOBAL_INITDATA` in configmap `peer-pods-cm`, so that you don't need add initdata annotation in each Pod yaml. For example, for libvirt provider, it looks like: +``` +kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml +apiVersion: v1 +data: + CLOUD_CONFIG_VERIFY: "false" + CLOUD_PROVIDER: libvirt + DISABLECVM: "true" + ENABLE_CLOUD_PROVIDER_EXTERNAL_PLUGIN: "false" + GLOBAL_INITDATA: YWxnb3JpdGhtID0gInNoYTM4NCIKdmVyc2lvbiA9ICIwLjEuMCIKCltkYXRhXQoiYWEudG9tbCIgPSAnJycKW3Rva2VuX2NvbmZpZ3NdClt0b2tlbl9jb25maWdzLmNvY29fYXNdCnVybCA9ICdodHRwOi8vMTI3LjAuMC4xOjgwODAnCgpbdG9rZW5fY29uZmlncy5rYnNdCnVybCA9ICdodHRwOi8vMTI3LjAuMC4xOjgwODAnCmNlcnQgPSAiIiIKLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURsakNDQW42Z0F3SUJBZ0lVUi9VTmgxM0dGYW00ZW1nbHVkdHlwZS9TOUJJd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2RURUxNQWtHQTFVRUJoTUNRMDR4RVRBUEJnTlZCQWdNQ0Zwb1pXcHBZVzVuTVJFd0R3WURWUVFIREFoSQpZVzVuZW1odmRURVJNQThHQTFVRUNnd0lRVUZUTFZSRlUxUXhGREFTQmdOVkJBc01DMFJsZG1Wc2IzQnRaVzUwCk1SY3dGUVlEVlFRRERBNUJRVk10VkVWVFZDMUlWRlJRVXpBZUZ3MHlOREF6TVRnd056QXpOVE5hRncweU5UQXoKTVRnd056QXpOVE5hTUhVeEN6QUpCZ05WQkFZVEFrTk9NUkV3RHdZRFZRUUlEQWhhYUdWcWFXRnVaekVSTUE4RwpBMVVFQnd3SVNHRnVaM3BvYjNVeEVUQVBCZ05WQkFvTUNFRkJVeTFVUlZOVU1SUXdFZ1lEVlFRTERBdEVaWFpsCmJHOXdiV1Z1ZERFWE1CVUdBMVVFQXd3T1FVRlRMVlJGVTFRdFNGUlVVRk13Z2dFaU1BMEdDU3FHU0liM0RRRUIKQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUURmcDFhQnI2TGlOUkJsSlVjREdjQWJjVUNQRzZVenl3dFZJYzgrY29tUwpheS8vZ3d6MkFrRG1GVnZxd0k0YmRwL05VQ3dTQzZTaEh6eHNyQ0VpYWdSS3RBM2FmL2NrTTdoT2tiNFM2dS81CmV3SEhGY0w2WU9VcCtOT0g1L2RTTHJGSExqZXQwZHQ0TGt5TkJQZTdtS0F5Q0pYZmlYM3diMjV3SUJCMFRmYTAKcDVWb0t6d1dlRFFCeDdhWDhUS2JHNi9GWklpT1hHWmRsMjRER0FSaXFFM1hpZlg3REg5aVZaMlYyUkw5KzNXWQowNUdFVE5GUEt0Y3JOd1R5OFN0OC9Ic1dWeGpBekdGemY3NUxieXM5RmYzSk1Ec2c5elF6Z2NKSnpZV2lzeGxZCmczQ21uYkVOUDBlb0hTNFdqUWxUVXlZMG10bk93b2RvNFZkZjhaT2tVNHdKQWdNQkFBR2pIakFjTUJvR0ExVWQKRVFRVE1CR0NDV3h2WTJGc2FHOXpkSWNFZndBQUFUQU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFLVzMyc3BpaQp0MkpCN0MxSXZZcEp3NW1RNWJoSWxsZEUwaUI1cndXdk5idURnUHJnZlRJNHhpWDVzdW1kSHcrUDIrR1U5S1hGCm5Xa0ZSWjlXLzI2eEZyVmdHSVMvYTA3YUk3eHJscDBPaisxdU85MVVoQ0wzSGhNRS8wdFBDNnoxaWFGZVpwOFkKVDF0TG5hZnFpR2lUaEZVZ3ZnNlBLdDg2ZW5YNjB2R2FUWTdzc2xSbGdiRHI5c0FpL05EU1M3VTFQdml1QzZ5bwp5Smk3QkRpUlN4N0tyTUdMc2NRK0FLS28yUkYxTUx6bEpNYTFrSVpmdktEQlhGelJkNjFLNUlqRFJRNEhRaHdYCkRZRWJRdm9aSWtVVGMxZ0JVV0RjQVVTNXp0YkpnOUxDYjlXVnR2VVRxVFAybEd1TnltT3Zkc3VYcStzQVpoOWIKTTlRYUMxbXpRL09TdGc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCiIiIgonJycKCiJjZGgudG9tbCIgID0gJycnCnNvY2tldCA9ICd1bml4Oi8vL3J1bi9jb25maWRlbnRpYWwtY29udGFpbmVycy9jZGguc29jaycKY3JlZGVudGlhbHMgPSBbXQoKW2tiY10KbmFtZSA9ICdjY19rYmMnCnVybCA9ICdodHRwOi8vMS4yLjMuNDo4MDgwJwprYnNfY2VydCA9ICIiIgotLS0tLUJFR0lOIENFUlRJRklDQVRFLS0tLS0KTUlJRlREQ0NBdnVnQXdJQkFnSUJBREJHQmdrcWhraUc5dzBCQVFvd09hQVBNQTBHQ1dDR1NBRmxBd1FDQWdVQQpvUnd3R2dZSktvWklodmNOQVFFSU1BMEdDV0NHU0FGbEF3UUNBZ1VBb2dNQ0FUQ2pBd0lCQVRCN01SUXdFZ1lEClZRUUxEQXRGYm1kcGJtVmxjbWx1WnpFTE1Ba0dBMVVFQmhNQ1ZWTXhGREFTQmdOVkJBY01DMU5oYm5SaElFTnMKWVhKaE1Rc3dDUVlEVlFRSURBSkRRVEVmTUIwR0ExVUVDZ3dXUVdSMllXNWpaV1FnVFdsamNtOGdSR1YyYVdObApjekVTTUJBR0ExVUVBd3dKVTBWV0xVMXBiR0Z1TUI0WERUSXpNREV5TkRFM05UZ3lObG9YRFRNd01ERXlOREUzCk5UZ3lObG93ZWpFVU1CSUdBMVVFQ3d3TFJXNW5hVzVsWlhKcGJtY3hDekFKQmdOVkJBWVRBbFZUTVJRd0VnWUQKVlFRSERBdFRZVzUwWVNCRGJHRnlZVEVMTUFrR0ExVUVDQXdDUTBFeEh6QWRCZ05WQkFvTUZrRmtkbUZ1WTJWawpJRTFwWTNKdklFUmxkbWxqWlhNeEVUQVBCZ05WQkFNTUNGTkZWaTFXUTBWTE1IWXdFQVlIS29aSXpqMENBUVlGCks0RUVBQ0lEWWdBRXhtRzFaYnVvQVFLOTNVU1J5WlFjc3lvYmZiYUFFb0tFRUxmL2pLMzljT1ZKdDF0NHM4M1cKWE0zcnFJYlM3cUhVSFF3L0ZHeU92ZGFFVXM1K3d3eHBDV2ZEbm1KTUFRK2N0Z1pxZ0RFS2gxTnFsT3V1S2NLcQoyWUFXRTVjVEg3c0hvNElCRmpDQ0FSSXdFQVlKS3dZQkJBR2NlQUVCQkFNQ0FRQXdGd1lKS3dZQkJBR2NlQUVDCkJBb1dDRTFwYkdGdUxVSXdNQkVHQ2lzR0FRUUJuSGdCQXdFRUF3SUJBekFSQmdvckJnRUVBWng0QVFNQ0JBTUMKQVFBd0VRWUtLd1lCQkFHY2VBRURCQVFEQWdFQU1CRUdDaXNHQVFRQm5IZ0JBd1VFQXdJQkFEQVJCZ29yQmdFRQpBWng0QVFNR0JBTUNBUUF3RVFZS0t3WUJCQUdjZUFFREJ3UURBZ0VBTUJFR0Npc0dBUVFCbkhnQkF3TUVBd0lCCkNEQVJCZ29yQmdFRUFaeDRBUU1JQkFNQ0FYTXdUUVlKS3dZQkJBR2NlQUVFQkVERGhDZWpEVXg2K2RsdmVoVzUKY21tQ1dtVExkcUkxTC8xZEdCRmRpYTFIUDQ2TUM4MmFYWktHWVN1dFNxMzdSQ1lnV2p1ZVQrcUNNQkUxb1hEawpkMUpPTUVZR0NTcUdTSWIzRFFFQkNqQTVvQTh3RFFZSllJWklBV1VEQkFJQ0JRQ2hIREFhQmdrcWhraUc5dzBCCkFRZ3dEUVlKWUlaSUFXVURCQUlDQlFDaUF3SUJNS01EQWdFQkE0SUNBUUFDZ0NhaTl4OERBV3pYLzJJZWxOV20KaXR1RUJTaXE5QzllRG5CRWNrUVlpa0FoUGFzZmFnbm9XRkF0S3UvWldUS0hpK0JNYmhLd3N3QlM4VzBHMXl3aQpjVVdHbHppZ0k0dGR4eGYxWUJKeUNvVFNOc3NTYkttSWg1amVtQmZydklCbzF5RWQrZTU2WkpNZGhOOGUreFdVCmJ2b3ZVQzIvN0RsNzZmekFhQUNMU29yWlV2NVhQSndLWHdFT0hvN0ZJY1JFam9abitmS2pKVG5tZFhjZTBMRDYKOVJIcityK2NleUU3OWdtSzMxYkk5RFlpSm9MNExlR2RYWjNnTU9WRFIxT25Eb3M1bE9CY1YrcXVKNkp1anBnSApkOWczU2E3RHU3cHVzRDlGZGFwOThvY1pzbFJmRmpGaS8vMllkVk00TUticTZJd3BZTkIrMlBDRUtOQzdTZmJPCk5nWllKdVBabk0vd1ZpRVMvY1A3TVpOSjFLVUtCSTl5aDZUbWxTc1paT2NsR0p2ck9zQlppbVRYcEFUamROTXQKY2x1S3dxQVVVellRbVU3YmYyVE1kT1h5QTlpSDV3SXBqMWtXR0UxVnVGQURUS0lMa1RjNkx6THpPV0NvZkx4ZgpvbmhUdFNEdHpJdi91ZWw1NDdHWnFxK3JWUnZtSWllRXVFdkRFVHd1b29rZlY2cXUzRC85S3VTcjl4aXpubUVnCnh5bnVkL2Y1MjVqcHBKTWNEL29mYlF4VVp1R0t2YjNmM3p5K2FMeHFpZG9YN2djYTJYZDlqeVV5NVkvODMrWk4KYno0UFp4ODFVSnpYVkk5QUJFaDgveGlsQVRoMVp4T2VQVEJKak43bGdyMGxYdEtZalYvNDN5eXhnVVlyWE5aUwpvTFNHMmRMQ0s5bWpqcmFQamF1MzRRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQoiIiIKJycnCgoicG9saWN5LnJlZ28iID0gJycnCnBhY2thZ2UgYWdlbnRfcG9saWN5CgppbXBvcnQgZnV0dXJlLmtleXdvcmRzLmluCmltcG9ydCBmdXR1cmUua2V5d29yZHMuZXZlcnkKCmltcG9ydCBpbnB1dAoKIyBEZWZhdWx0IHZhbHVlcywgcmV0dXJuZWQgYnkgT1BBIHdoZW4gcnVsZXMgY2Fubm90IGJlIGV2YWx1YXRlZCB0byB0cnVlLgpkZWZhdWx0IENvcHlGaWxlUmVxdWVzdCA6PSBmYWxzZQpkZWZhdWx0IENyZWF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gZmFsc2UKZGVmYXVsdCBDcmVhdGVTYW5kYm94UmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgRGVzdHJveVNhbmRib3hSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBFeGVjUHJvY2Vzc1JlcXVlc3QgOj0gZmFsc2UKZGVmYXVsdCBHZXRPT01FdmVudFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEd1ZXN0RGV0YWlsc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IE9ubGluZUNQVU1lbVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFB1bGxJbWFnZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlYWRTdHJlYW1SZXF1ZXN0IDo9IGZhbHNlCmRlZmF1bHQgUmVtb3ZlQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVtb3ZlU3RhbGVWaXJ0aW9mc1NoYXJlTW91bnRzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2lnbmFsUHJvY2Vzc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXJ0Q29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RhdHNDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBUdHlXaW5SZXNpemVSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVFcGhlbWVyYWxNb3VudHNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVJbnRlcmZhY2VSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVSb3V0ZXNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBXYWl0UHJvY2Vzc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFdyaXRlU3RyZWFtUmVxdWVzdCA6PSBmYWxzZQonJyc= + LIBVIRT_NET: default + LIBVIRT_POOL: default + LIBVIRT_URI: qemu+ssh://root@192.168.122.1/system?no_verify=1 + LIBVIRT_VOL_NAME: podvm-base.qcow2 + SECURE_COMMS: "false" +kind: ConfigMap +``` + ## TODO A large policy bodies that cannot be provisioned via IMDS user-data, the limitation depends on providers IMDS limitation. We need add checking and limitations according to test result future. diff --git a/src/cloud-api-adaptor/entrypoint.sh b/src/cloud-api-adaptor/entrypoint.sh index b520405ab..f36e3c265 100755 --- a/src/cloud-api-adaptor/entrypoint.sh +++ b/src/cloud-api-adaptor/entrypoint.sh @@ -18,6 +18,7 @@ optionals+="" [[ "${CERT_FILE}" ]] && [[ "${CERT_KEY}" ]] && optionals+="-cert-file ${CERT_FILE} -cert-key ${CERT_KEY} " [[ "${TLS_SKIP_VERIFY}" ]] && optionals+="-tls-skip-verify " [[ "${PROXY_TIMEOUT}" ]] && optionals+="-proxy-timeout ${PROXY_TIMEOUT} " +[[ "${GLOBAL_INITDATA}" ]] && optionals+="-global-initdata ${GLOBAL_INITDATA} " [[ "${FORWARDER_PORT}" ]] && optionals+="-forwarder-port ${FORWARDER_PORT} " [[ "${CLOUD_CONFIG_VERIFY}" == "true" ]] && optionals+="-cloud-config-verify " [[ "${SECURE_COMMS}" == "true" ]] && optionals+="-secure-comms " diff --git a/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml index ba6cdc0e5..6f45fcd4b 100644 --- a/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml @@ -33,6 +33,7 @@ configMapGenerator: # /subscriptions//resourceGroups//providers/Microsoft.Compute/images/ - AZURE_IMAGE_ID="" #set - SSH_USERNAME="" #set peer pod vm admin user name + - GLOBAL_INITDATA="" # set default initdata for podvm #- DISABLECVM="" # Uncomment it if you want a generic VM #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 diff --git a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml index c66667a90..6a7fb47ad 100644 --- a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml @@ -24,6 +24,7 @@ configMapGenerator: - LIBVIRT_POOL="default" # set - DISABLECVM="true" # set as false to enable confidential VM - SECURE_COMMS="false" # set as true to enable Secure Comms + - GLOBAL_INITDATA="" # set default initdata for podvm #- LIBVIRT_LAUNCH_SECURITY="" #sev or s390-pv #- LIBVIRT_FIRMWARE="" # Uncomment and set if you want to change the firmware path. Defaults to /usr/share/edk2/ovmf/OVMF_CODE.fd #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2 diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go index 36e41fe72..ffed48e96 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go @@ -79,7 +79,7 @@ func (s *cloudService) removeSandbox(id sandboxID) error { } func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNode podnetwork.WorkerNode, - secureComms bool, secureCommsInbounds, secureCommsOutbounds, kbsAddress, podsDir, daemonPort, sshport string) Service { + secureComms bool, secureCommsInbounds, secureCommsOutbounds, kbsAddress, podsDir, daemonPort, globalInitdata, sshport string) Service { var err error var sshClient *wnssh.SshClient @@ -93,13 +93,14 @@ func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNo } s := &cloudService{ - provider: provider, - proxyFactory: proxyFactory, - sandboxes: map[sandboxID]*sandbox{}, - podsDir: podsDir, - daemonPort: daemonPort, - workerNode: workerNode, - sshClient: sshClient, + provider: provider, + proxyFactory: proxyFactory, + sandboxes: map[sandboxID]*sandbox{}, + podsDir: podsDir, + daemonPort: daemonPort, + globalInitdata: globalInitdata, + workerNode: workerNode, + sshClient: sshClient, } s.cond = sync.NewCond(&s.mutex) s.ppService, err = k8sops.NewPeerPodService() @@ -291,7 +292,13 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r } initdataStr := util.GetInitdataFromAnnotation(req.Annotations) - logger.Printf("initdata: %s", initdataStr) + logger.Printf("initdata in Pod annotation: %s", initdataStr) + + if initdataStr == "" { + logger.Printf("initdata in pod annotation is empty, use global initdata: %s", s.globalInitdata) + initdataStr = s.globalInitdata + } + if initdataStr != "" { cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{ Path: InitdataPath, diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go index d751d880b..a8030ef72 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go @@ -117,7 +117,7 @@ func TestCloudService(t *testing.T) { podsDir: dir, } - s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, false, "", "", "", dir, forwarder.DefaultListenPort, "") + s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, false, "", "", "", dir, forwarder.DefaultListenPort, "", "") assert.NotNil(t, s) @@ -172,7 +172,7 @@ func TestCloudServiceWithSecureComms(t *testing.T) { podsDir: dir, } - s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, true, "", "", "127.0.0.1:9009", dir, forwarder.DefaultListenPort, sshport) + s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, true, "", "", "127.0.0.1:9009", dir, forwarder.DefaultListenPort, "", sshport) assert.NotNil(t, s) diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go index 7e7295340..4bb4747f9 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go @@ -26,16 +26,17 @@ type Service interface { } type cloudService struct { - provider provider.Provider - proxyFactory proxy.Factory - workerNode podnetwork.WorkerNode - sandboxes map[sandboxID]*sandbox - cond *sync.Cond - podsDir string - daemonPort string - mutex sync.Mutex - ppService *k8sops.PeerPodService - sshClient *wnssh.SshClient + provider provider.Provider + proxyFactory proxy.Factory + workerNode podnetwork.WorkerNode + sandboxes map[sandboxID]*sandbox + cond *sync.Cond + podsDir string + daemonPort string + mutex sync.Mutex + ppService *k8sops.PeerPodService + globalInitdata string + sshClient *wnssh.SshClient } type sandboxID string diff --git a/src/cloud-api-adaptor/pkg/adaptor/server.go b/src/cloud-api-adaptor/pkg/adaptor/server.go index 75643658b..049fae799 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/server.go +++ b/src/cloud-api-adaptor/pkg/adaptor/server.go @@ -39,6 +39,7 @@ type ServerConfig struct { PodsDir string ForwarderPort string ProxyTimeout time.Duration + GlobalInitdata string EnableCloudConfigVerify bool SecureComms bool SecureCommsInbounds string @@ -70,7 +71,7 @@ func NewServer(provider provider.Provider, cfg *ServerConfig, workerNode podnetw agentFactory := proxy.NewFactory(cfg.PauseImage, cfg.TLSConfig, cfg.ProxyTimeout) cloudService := cloud.NewService(provider, agentFactory, workerNode, - cfg.SecureComms, cfg.SecureCommsInbounds, cfg.SecureCommsOutbounds, cfg.SecureCommsKbsAddress, cfg.PodsDir, cfg.ForwarderPort, sshutil.SSHPORT) + cfg.SecureComms, cfg.SecureCommsInbounds, cfg.SecureCommsOutbounds, cfg.SecureCommsKbsAddress, cfg.PodsDir, cfg.ForwarderPort, cfg.GlobalInitdata, sshutil.SSHPORT) vmInfoService := vminfo.NewService(cloudService) return &server{ diff --git a/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go b/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go index c00ad5f58..0a2fe6858 100644 --- a/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go @@ -381,7 +381,7 @@ func (p *AzureCloudProvisioner) UploadPodvm(imagePath string, ctx context.Contex func isAzureKustomizeConfigMapKey(key string) bool { switch key { - case "CLOUD_PROVIDER", "AZURE_SUBSCRIPTION_ID", "AZURE_REGION", "AZURE_INSTANCE_SIZE", "AZURE_RESOURCE_GROUP", "AZURE_SUBNET_ID", "AZURE_IMAGE_ID", "SSH_USERNAME", "TAGS": + case "CLOUD_PROVIDER", "AZURE_SUBSCRIPTION_ID", "AZURE_REGION", "AZURE_INSTANCE_SIZE", "AZURE_RESOURCE_GROUP", "AZURE_SUBNET_ID", "AZURE_IMAGE_ID", "SSH_USERNAME", "GLOBAL_INITDATA", "TAGS": return true default: return false diff --git a/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go index fb92e3b55..9b7c6b243 100644 --- a/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go @@ -206,7 +206,7 @@ func NewDockerInstallOverlay(installDir, provider string) (pv.InstallOverlay, er func isDockerKustomizeConfigMapKey(key string) bool { switch key { - case "CLOUD_PROVIDER", "DOCKER_HOST", "DOCKER_API_VERSION", "DOCKER_PODVM_IMAGE", "DOCKER_NETWORK_NAME": + case "CLOUD_PROVIDER", "DOCKER_HOST", "DOCKER_API_VERSION", "DOCKER_PODVM_IMAGE", "DOCKER_NETWORK_NAME", "GLOBAL_INITDATA": return true default: return false diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go index 69cfd3a34..832da0a56 100644 --- a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go @@ -320,12 +320,13 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, // Mapping the internal properties to ConfigMapGenerator properties and their default values. mapProps := map[string][2]string{ - "network": {"default", "LIBVIRT_NET"}, - "storage": {"default", "LIBVIRT_POOL"}, - "pause_image": {"", "PAUSE_IMAGE"}, - "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, - "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, - "vxlan_port": {"", "VXLAN_PORT"}, + "network": {"default", "LIBVIRT_NET"}, + "storage": {"default", "LIBVIRT_POOL"}, + "pause_image": {"", "PAUSE_IMAGE"}, + "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, + "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, + "vxlan_port": {"", "VXLAN_PORT"}, + "GLOBAL_INITDATA": {"", "GLOBAL_INITDATA"}, } for k, v := range mapProps { From b91fa2628c53a20c0635895db89548f90422869d Mon Sep 17 00:00:00 2001 From: Qi Feng Huo Date: Wed, 21 Aug 2024 09:56:54 +0800 Subject: [PATCH 3/5] initdata: rename variables - rename GLOBAL_INITDATA to INITDATA - rename CdhFilePath to CDHConfigPath - rename AaFilePath to AAConfigPath Fixes: #1985 Signed-off-by: Qi Feng Huo --- .../cmd/cloud-api-adaptor/main.go | 2 +- src/cloud-api-adaptor/docs/addnewprovider.md | 2 +- src/cloud-api-adaptor/docs/initdata.md | 4 +-- src/cloud-api-adaptor/entrypoint.sh | 2 +- .../install/overlays/azure/kustomization.yaml | 2 +- .../overlays/libvirt/kustomization.yaml | 2 +- .../pkg/adaptor/cloud/cloud.go | 26 +++++++++---------- .../pkg/adaptor/cloud/types.go | 22 ++++++++-------- src/cloud-api-adaptor/pkg/adaptor/server.go | 4 +-- .../pkg/userdata/provision.go | 4 +-- .../provisioner/azure/provision_common.go | 2 +- .../provisioner/docker/provision_common.go | 2 +- .../provisioner/libvirt/provision_common.go | 14 +++++----- 13 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go index aea2617c5..c37185056 100644 --- a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go +++ b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go @@ -121,7 +121,7 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) { flags.StringVar(&cfg.networkConfig.HostInterface, "host-interface", "", "Host Interface") flags.IntVar(&cfg.networkConfig.VXLANPort, "vxlan-port", vxlan.DefaultVXLANPort, "VXLAN UDP port number (VXLAN tunnel mode only") flags.IntVar(&cfg.networkConfig.VXLANMinID, "vxlan-min-id", vxlan.DefaultVXLANMinID, "Minimum VXLAN ID (VXLAN tunnel mode only") - flags.StringVar(&cfg.serverConfig.GlobalInitdata, "global-initdata", "", "Default initdata for all Pods") + flags.StringVar(&cfg.serverConfig.Initdata, "initdata", "", "Default initdata for all Pods") flags.BoolVar(&cfg.serverConfig.EnableCloudConfigVerify, "cloud-config-verify", false, "Enable cloud config verify - should use it for production") cloud.ParseCmd(flags) diff --git a/src/cloud-api-adaptor/docs/addnewprovider.md b/src/cloud-api-adaptor/docs/addnewprovider.md index d9f31efb6..c1d3b1eec 100644 --- a/src/cloud-api-adaptor/docs/addnewprovider.md +++ b/src/cloud-api-adaptor/docs/addnewprovider.md @@ -282,7 +282,7 @@ optionals+="" [[ "${CERT_FILE}" ]] && [[ "${CERT_KEY}" ]] && optionals+="-cert-file ${CERT_FILE} -cert-key ${CERT_KEY} " [[ "${TLS_SKIP_VERIFY}" ]] && optionals+="-tls-skip-verify " [[ "${PROXY_TIMEOUT}" ]] && optionals+="-proxy-timeout ${PROXY_TIMEOUT} " -[[ "${GLOBAL_INITDATA}" ]] && optionals+="-global-initdata ${GLOBAL_INITDATA} " +[[ "${INITDATA}" ]] && optionals+="-global-initdata ${INITDATA} " [[ "${FORWARDER_PORT}" ]] && optionals+="-forwarder-port ${FORWARDER_PORT} " [[ "${CLOUD_CONFIG_VERIFY}" == "true" ]] && optionals+="-cloud-config-verify " diff --git a/src/cloud-api-adaptor/docs/initdata.md b/src/cloud-api-adaptor/docs/initdata.md index 3f0752ae4..816064204 100644 --- a/src/cloud-api-adaptor/docs/initdata.md +++ b/src/cloud-api-adaptor/docs/initdata.md @@ -175,7 +175,7 @@ allow if { ``` ## Global initdata -If all of your applications(Pods) are using same initdata, it's convenient you set the `GLOBAL_INITDATA` in configmap `peer-pods-cm`, so that you don't need add initdata annotation in each Pod yaml. For example, for libvirt provider, it looks like: +If all of your applications(Pods) are using same initdata, it's convenient you set the `INITDATA` in configmap `peer-pods-cm`, so that you don't need add initdata annotation in each Pod yaml. For example, for libvirt provider, it looks like: ``` kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml apiVersion: v1 @@ -184,7 +184,7 @@ data: CLOUD_PROVIDER: libvirt DISABLECVM: "true" ENABLE_CLOUD_PROVIDER_EXTERNAL_PLUGIN: "false" - GLOBAL_INITDATA: YWxnb3JpdGhtID0gInNoYTM4NCIKdmVyc2lvbiA9ICIwLjEuMCIKCltkYXRhXQoiYWEudG9tbCIgPSAnJycKW3Rva2VuX2NvbmZpZ3NdClt0b2tlbl9jb25maWdzLmNvY29fYXNdCnVybCA9ICdodHRwOi8vMTI3LjAuMC4xOjgwODAnCgpbdG9rZW5fY29uZmlncy5rYnNdCnVybCA9ICdodHRwOi8vMTI3LjAuMC4xOjgwODAnCmNlcnQgPSAiIiIKLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURsakNDQW42Z0F3SUJBZ0lVUi9VTmgxM0dGYW00ZW1nbHVkdHlwZS9TOUJJd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2RURUxNQWtHQTFVRUJoTUNRMDR4RVRBUEJnTlZCQWdNQ0Zwb1pXcHBZVzVuTVJFd0R3WURWUVFIREFoSQpZVzVuZW1odmRURVJNQThHQTFVRUNnd0lRVUZUTFZSRlUxUXhGREFTQmdOVkJBc01DMFJsZG1Wc2IzQnRaVzUwCk1SY3dGUVlEVlFRRERBNUJRVk10VkVWVFZDMUlWRlJRVXpBZUZ3MHlOREF6TVRnd056QXpOVE5hRncweU5UQXoKTVRnd056QXpOVE5hTUhVeEN6QUpCZ05WQkFZVEFrTk9NUkV3RHdZRFZRUUlEQWhhYUdWcWFXRnVaekVSTUE4RwpBMVVFQnd3SVNHRnVaM3BvYjNVeEVUQVBCZ05WQkFvTUNFRkJVeTFVUlZOVU1SUXdFZ1lEVlFRTERBdEVaWFpsCmJHOXdiV1Z1ZERFWE1CVUdBMVVFQXd3T1FVRlRMVlJGVTFRdFNGUlVVRk13Z2dFaU1BMEdDU3FHU0liM0RRRUIKQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUURmcDFhQnI2TGlOUkJsSlVjREdjQWJjVUNQRzZVenl3dFZJYzgrY29tUwpheS8vZ3d6MkFrRG1GVnZxd0k0YmRwL05VQ3dTQzZTaEh6eHNyQ0VpYWdSS3RBM2FmL2NrTTdoT2tiNFM2dS81CmV3SEhGY0w2WU9VcCtOT0g1L2RTTHJGSExqZXQwZHQ0TGt5TkJQZTdtS0F5Q0pYZmlYM3diMjV3SUJCMFRmYTAKcDVWb0t6d1dlRFFCeDdhWDhUS2JHNi9GWklpT1hHWmRsMjRER0FSaXFFM1hpZlg3REg5aVZaMlYyUkw5KzNXWQowNUdFVE5GUEt0Y3JOd1R5OFN0OC9Ic1dWeGpBekdGemY3NUxieXM5RmYzSk1Ec2c5elF6Z2NKSnpZV2lzeGxZCmczQ21uYkVOUDBlb0hTNFdqUWxUVXlZMG10bk93b2RvNFZkZjhaT2tVNHdKQWdNQkFBR2pIakFjTUJvR0ExVWQKRVFRVE1CR0NDV3h2WTJGc2FHOXpkSWNFZndBQUFUQU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFLVzMyc3BpaQp0MkpCN0MxSXZZcEp3NW1RNWJoSWxsZEUwaUI1cndXdk5idURnUHJnZlRJNHhpWDVzdW1kSHcrUDIrR1U5S1hGCm5Xa0ZSWjlXLzI2eEZyVmdHSVMvYTA3YUk3eHJscDBPaisxdU85MVVoQ0wzSGhNRS8wdFBDNnoxaWFGZVpwOFkKVDF0TG5hZnFpR2lUaEZVZ3ZnNlBLdDg2ZW5YNjB2R2FUWTdzc2xSbGdiRHI5c0FpL05EU1M3VTFQdml1QzZ5bwp5Smk3QkRpUlN4N0tyTUdMc2NRK0FLS28yUkYxTUx6bEpNYTFrSVpmdktEQlhGelJkNjFLNUlqRFJRNEhRaHdYCkRZRWJRdm9aSWtVVGMxZ0JVV0RjQVVTNXp0YkpnOUxDYjlXVnR2VVRxVFAybEd1TnltT3Zkc3VYcStzQVpoOWIKTTlRYUMxbXpRL09TdGc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCiIiIgonJycKCiJjZGgudG9tbCIgID0gJycnCnNvY2tldCA9ICd1bml4Oi8vL3J1bi9jb25maWRlbnRpYWwtY29udGFpbmVycy9jZGguc29jaycKY3JlZGVudGlhbHMgPSBbXQoKW2tiY10KbmFtZSA9ICdjY19rYmMnCnVybCA9ICdodHRwOi8vMS4yLjMuNDo4MDgwJwprYnNfY2VydCA9ICIiIgotLS0tLUJFR0lOIENFUlRJRklDQVRFLS0tLS0KTUlJRlREQ0NBdnVnQXdJQkFnSUJBREJHQmdrcWhraUc5dzBCQVFvd09hQVBNQTBHQ1dDR1NBRmxBd1FDQWdVQQpvUnd3R2dZSktvWklodmNOQVFFSU1BMEdDV0NHU0FGbEF3UUNBZ1VBb2dNQ0FUQ2pBd0lCQVRCN01SUXdFZ1lEClZRUUxEQXRGYm1kcGJtVmxjbWx1WnpFTE1Ba0dBMVVFQmhNQ1ZWTXhGREFTQmdOVkJBY01DMU5oYm5SaElFTnMKWVhKaE1Rc3dDUVlEVlFRSURBSkRRVEVmTUIwR0ExVUVDZ3dXUVdSMllXNWpaV1FnVFdsamNtOGdSR1YyYVdObApjekVTTUJBR0ExVUVBd3dKVTBWV0xVMXBiR0Z1TUI0WERUSXpNREV5TkRFM05UZ3lObG9YRFRNd01ERXlOREUzCk5UZ3lObG93ZWpFVU1CSUdBMVVFQ3d3TFJXNW5hVzVsWlhKcGJtY3hDekFKQmdOVkJBWVRBbFZUTVJRd0VnWUQKVlFRSERBdFRZVzUwWVNCRGJHRnlZVEVMTUFrR0ExVUVDQXdDUTBFeEh6QWRCZ05WQkFvTUZrRmtkbUZ1WTJWawpJRTFwWTNKdklFUmxkbWxqWlhNeEVUQVBCZ05WQkFNTUNGTkZWaTFXUTBWTE1IWXdFQVlIS29aSXpqMENBUVlGCks0RUVBQ0lEWWdBRXhtRzFaYnVvQVFLOTNVU1J5WlFjc3lvYmZiYUFFb0tFRUxmL2pLMzljT1ZKdDF0NHM4M1cKWE0zcnFJYlM3cUhVSFF3L0ZHeU92ZGFFVXM1K3d3eHBDV2ZEbm1KTUFRK2N0Z1pxZ0RFS2gxTnFsT3V1S2NLcQoyWUFXRTVjVEg3c0hvNElCRmpDQ0FSSXdFQVlKS3dZQkJBR2NlQUVCQkFNQ0FRQXdGd1lKS3dZQkJBR2NlQUVDCkJBb1dDRTFwYkdGdUxVSXdNQkVHQ2lzR0FRUUJuSGdCQXdFRUF3SUJBekFSQmdvckJnRUVBWng0QVFNQ0JBTUMKQVFBd0VRWUtLd1lCQkFHY2VBRURCQVFEQWdFQU1CRUdDaXNHQVFRQm5IZ0JBd1VFQXdJQkFEQVJCZ29yQmdFRQpBWng0QVFNR0JBTUNBUUF3RVFZS0t3WUJCQUdjZUFFREJ3UURBZ0VBTUJFR0Npc0dBUVFCbkhnQkF3TUVBd0lCCkNEQVJCZ29yQmdFRUFaeDRBUU1JQkFNQ0FYTXdUUVlKS3dZQkJBR2NlQUVFQkVERGhDZWpEVXg2K2RsdmVoVzUKY21tQ1dtVExkcUkxTC8xZEdCRmRpYTFIUDQ2TUM4MmFYWktHWVN1dFNxMzdSQ1lnV2p1ZVQrcUNNQkUxb1hEawpkMUpPTUVZR0NTcUdTSWIzRFFFQkNqQTVvQTh3RFFZSllJWklBV1VEQkFJQ0JRQ2hIREFhQmdrcWhraUc5dzBCCkFRZ3dEUVlKWUlaSUFXVURCQUlDQlFDaUF3SUJNS01EQWdFQkE0SUNBUUFDZ0NhaTl4OERBV3pYLzJJZWxOV20KaXR1RUJTaXE5QzllRG5CRWNrUVlpa0FoUGFzZmFnbm9XRkF0S3UvWldUS0hpK0JNYmhLd3N3QlM4VzBHMXl3aQpjVVdHbHppZ0k0dGR4eGYxWUJKeUNvVFNOc3NTYkttSWg1amVtQmZydklCbzF5RWQrZTU2WkpNZGhOOGUreFdVCmJ2b3ZVQzIvN0RsNzZmekFhQUNMU29yWlV2NVhQSndLWHdFT0hvN0ZJY1JFam9abitmS2pKVG5tZFhjZTBMRDYKOVJIcityK2NleUU3OWdtSzMxYkk5RFlpSm9MNExlR2RYWjNnTU9WRFIxT25Eb3M1bE9CY1YrcXVKNkp1anBnSApkOWczU2E3RHU3cHVzRDlGZGFwOThvY1pzbFJmRmpGaS8vMllkVk00TUticTZJd3BZTkIrMlBDRUtOQzdTZmJPCk5nWllKdVBabk0vd1ZpRVMvY1A3TVpOSjFLVUtCSTl5aDZUbWxTc1paT2NsR0p2ck9zQlppbVRYcEFUamROTXQKY2x1S3dxQVVVellRbVU3YmYyVE1kT1h5QTlpSDV3SXBqMWtXR0UxVnVGQURUS0lMa1RjNkx6THpPV0NvZkx4ZgpvbmhUdFNEdHpJdi91ZWw1NDdHWnFxK3JWUnZtSWllRXVFdkRFVHd1b29rZlY2cXUzRC85S3VTcjl4aXpubUVnCnh5bnVkL2Y1MjVqcHBKTWNEL29mYlF4VVp1R0t2YjNmM3p5K2FMeHFpZG9YN2djYTJYZDlqeVV5NVkvODMrWk4KYno0UFp4ODFVSnpYVkk5QUJFaDgveGlsQVRoMVp4T2VQVEJKak43bGdyMGxYdEtZalYvNDN5eXhnVVlyWE5aUwpvTFNHMmRMQ0s5bWpqcmFQamF1MzRRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQoiIiIKJycnCgoicG9saWN5LnJlZ28iID0gJycnCnBhY2thZ2UgYWdlbnRfcG9saWN5CgppbXBvcnQgZnV0dXJlLmtleXdvcmRzLmluCmltcG9ydCBmdXR1cmUua2V5d29yZHMuZXZlcnkKCmltcG9ydCBpbnB1dAoKIyBEZWZhdWx0IHZhbHVlcywgcmV0dXJuZWQgYnkgT1BBIHdoZW4gcnVsZXMgY2Fubm90IGJlIGV2YWx1YXRlZCB0byB0cnVlLgpkZWZhdWx0IENvcHlGaWxlUmVxdWVzdCA6PSBmYWxzZQpkZWZhdWx0IENyZWF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gZmFsc2UKZGVmYXVsdCBDcmVhdGVTYW5kYm94UmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgRGVzdHJveVNhbmRib3hSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBFeGVjUHJvY2Vzc1JlcXVlc3QgOj0gZmFsc2UKZGVmYXVsdCBHZXRPT01FdmVudFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEd1ZXN0RGV0YWlsc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IE9ubGluZUNQVU1lbVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFB1bGxJbWFnZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlYWRTdHJlYW1SZXF1ZXN0IDo9IGZhbHNlCmRlZmF1bHQgUmVtb3ZlQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVtb3ZlU3RhbGVWaXJ0aW9mc1NoYXJlTW91bnRzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2lnbmFsUHJvY2Vzc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXJ0Q29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RhdHNDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBUdHlXaW5SZXNpemVSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVFcGhlbWVyYWxNb3VudHNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVJbnRlcmZhY2VSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVSb3V0ZXNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBXYWl0UHJvY2Vzc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFdyaXRlU3RyZWFtUmVxdWVzdCA6PSBmYWxzZQonJyc= + INITDATA: YWxnb3JpdGhtID0gInNoYTM4NCIKdmVyc2lvbiA9ICIwLjEuMCIKCltkYXRhXQoiYWEudG9tbCIgPSAnJycKW3Rva2VuX2NvbmZpZ3NdClt0b2tlbl9jb25maWdzLmNvY29fYXNdCnVybCA9ICdodHRwOi8vMTI3LjAuMC4xOjgwODAnCgpbdG9rZW5fY29uZmlncy5rYnNdCnVybCA9ICdodHRwOi8vMTI3LjAuMC4xOjgwODAnCmNlcnQgPSAiIiIKLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURsakNDQW42Z0F3SUJBZ0lVUi9VTmgxM0dGYW00ZW1nbHVkdHlwZS9TOUJJd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2RURUxNQWtHQTFVRUJoTUNRMDR4RVRBUEJnTlZCQWdNQ0Zwb1pXcHBZVzVuTVJFd0R3WURWUVFIREFoSQpZVzVuZW1odmRURVJNQThHQTFVRUNnd0lRVUZUTFZSRlUxUXhGREFTQmdOVkJBc01DMFJsZG1Wc2IzQnRaVzUwCk1SY3dGUVlEVlFRRERBNUJRVk10VkVWVFZDMUlWRlJRVXpBZUZ3MHlOREF6TVRnd056QXpOVE5hRncweU5UQXoKTVRnd056QXpOVE5hTUhVeEN6QUpCZ05WQkFZVEFrTk9NUkV3RHdZRFZRUUlEQWhhYUdWcWFXRnVaekVSTUE4RwpBMVVFQnd3SVNHRnVaM3BvYjNVeEVUQVBCZ05WQkFvTUNFRkJVeTFVUlZOVU1SUXdFZ1lEVlFRTERBdEVaWFpsCmJHOXdiV1Z1ZERFWE1CVUdBMVVFQXd3T1FVRlRMVlJGVTFRdFNGUlVVRk13Z2dFaU1BMEdDU3FHU0liM0RRRUIKQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUURmcDFhQnI2TGlOUkJsSlVjREdjQWJjVUNQRzZVenl3dFZJYzgrY29tUwpheS8vZ3d6MkFrRG1GVnZxd0k0YmRwL05VQ3dTQzZTaEh6eHNyQ0VpYWdSS3RBM2FmL2NrTTdoT2tiNFM2dS81CmV3SEhGY0w2WU9VcCtOT0g1L2RTTHJGSExqZXQwZHQ0TGt5TkJQZTdtS0F5Q0pYZmlYM3diMjV3SUJCMFRmYTAKcDVWb0t6d1dlRFFCeDdhWDhUS2JHNi9GWklpT1hHWmRsMjRER0FSaXFFM1hpZlg3REg5aVZaMlYyUkw5KzNXWQowNUdFVE5GUEt0Y3JOd1R5OFN0OC9Ic1dWeGpBekdGemY3NUxieXM5RmYzSk1Ec2c5elF6Z2NKSnpZV2lzeGxZCmczQ21uYkVOUDBlb0hTNFdqUWxUVXlZMG10bk93b2RvNFZkZjhaT2tVNHdKQWdNQkFBR2pIakFjTUJvR0ExVWQKRVFRVE1CR0NDV3h2WTJGc2FHOXpkSWNFZndBQUFUQU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFLVzMyc3BpaQp0MkpCN0MxSXZZcEp3NW1RNWJoSWxsZEUwaUI1cndXdk5idURnUHJnZlRJNHhpWDVzdW1kSHcrUDIrR1U5S1hGCm5Xa0ZSWjlXLzI2eEZyVmdHSVMvYTA3YUk3eHJscDBPaisxdU85MVVoQ0wzSGhNRS8wdFBDNnoxaWFGZVpwOFkKVDF0TG5hZnFpR2lUaEZVZ3ZnNlBLdDg2ZW5YNjB2R2FUWTdzc2xSbGdiRHI5c0FpL05EU1M3VTFQdml1QzZ5bwp5Smk3QkRpUlN4N0tyTUdMc2NRK0FLS28yUkYxTUx6bEpNYTFrSVpmdktEQlhGelJkNjFLNUlqRFJRNEhRaHdYCkRZRWJRdm9aSWtVVGMxZ0JVV0RjQVVTNXp0YkpnOUxDYjlXVnR2VVRxVFAybEd1TnltT3Zkc3VYcStzQVpoOWIKTTlRYUMxbXpRL09TdGc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCiIiIgonJycKCiJjZGgudG9tbCIgID0gJycnCnNvY2tldCA9ICd1bml4Oi8vL3J1bi9jb25maWRlbnRpYWwtY29udGFpbmVycy9jZGguc29jaycKY3JlZGVudGlhbHMgPSBbXQoKW2tiY10KbmFtZSA9ICdjY19rYmMnCnVybCA9ICdodHRwOi8vMS4yLjMuNDo4MDgwJwprYnNfY2VydCA9ICIiIgotLS0tLUJFR0lOIENFUlRJRklDQVRFLS0tLS0KTUlJRlREQ0NBdnVnQXdJQkFnSUJBREJHQmdrcWhraUc5dzBCQVFvd09hQVBNQTBHQ1dDR1NBRmxBd1FDQWdVQQpvUnd3R2dZSktvWklodmNOQVFFSU1BMEdDV0NHU0FGbEF3UUNBZ1VBb2dNQ0FUQ2pBd0lCQVRCN01SUXdFZ1lEClZRUUxEQXRGYm1kcGJtVmxjbWx1WnpFTE1Ba0dBMVVFQmhNQ1ZWTXhGREFTQmdOVkJBY01DMU5oYm5SaElFTnMKWVhKaE1Rc3dDUVlEVlFRSURBSkRRVEVmTUIwR0ExVUVDZ3dXUVdSMllXNWpaV1FnVFdsamNtOGdSR1YyYVdObApjekVTTUJBR0ExVUVBd3dKVTBWV0xVMXBiR0Z1TUI0WERUSXpNREV5TkRFM05UZ3lObG9YRFRNd01ERXlOREUzCk5UZ3lObG93ZWpFVU1CSUdBMVVFQ3d3TFJXNW5hVzVsWlhKcGJtY3hDekFKQmdOVkJBWVRBbFZUTVJRd0VnWUQKVlFRSERBdFRZVzUwWVNCRGJHRnlZVEVMTUFrR0ExVUVDQXdDUTBFeEh6QWRCZ05WQkFvTUZrRmtkbUZ1WTJWawpJRTFwWTNKdklFUmxkbWxqWlhNeEVUQVBCZ05WQkFNTUNGTkZWaTFXUTBWTE1IWXdFQVlIS29aSXpqMENBUVlGCks0RUVBQ0lEWWdBRXhtRzFaYnVvQVFLOTNVU1J5WlFjc3lvYmZiYUFFb0tFRUxmL2pLMzljT1ZKdDF0NHM4M1cKWE0zcnFJYlM3cUhVSFF3L0ZHeU92ZGFFVXM1K3d3eHBDV2ZEbm1KTUFRK2N0Z1pxZ0RFS2gxTnFsT3V1S2NLcQoyWUFXRTVjVEg3c0hvNElCRmpDQ0FSSXdFQVlKS3dZQkJBR2NlQUVCQkFNQ0FRQXdGd1lKS3dZQkJBR2NlQUVDCkJBb1dDRTFwYkdGdUxVSXdNQkVHQ2lzR0FRUUJuSGdCQXdFRUF3SUJBekFSQmdvckJnRUVBWng0QVFNQ0JBTUMKQVFBd0VRWUtLd1lCQkFHY2VBRURCQVFEQWdFQU1CRUdDaXNHQVFRQm5IZ0JBd1VFQXdJQkFEQVJCZ29yQmdFRQpBWng0QVFNR0JBTUNBUUF3RVFZS0t3WUJCQUdjZUFFREJ3UURBZ0VBTUJFR0Npc0dBUVFCbkhnQkF3TUVBd0lCCkNEQVJCZ29yQmdFRUFaeDRBUU1JQkFNQ0FYTXdUUVlKS3dZQkJBR2NlQUVFQkVERGhDZWpEVXg2K2RsdmVoVzUKY21tQ1dtVExkcUkxTC8xZEdCRmRpYTFIUDQ2TUM4MmFYWktHWVN1dFNxMzdSQ1lnV2p1ZVQrcUNNQkUxb1hEawpkMUpPTUVZR0NTcUdTSWIzRFFFQkNqQTVvQTh3RFFZSllJWklBV1VEQkFJQ0JRQ2hIREFhQmdrcWhraUc5dzBCCkFRZ3dEUVlKWUlaSUFXVURCQUlDQlFDaUF3SUJNS01EQWdFQkE0SUNBUUFDZ0NhaTl4OERBV3pYLzJJZWxOV20KaXR1RUJTaXE5QzllRG5CRWNrUVlpa0FoUGFzZmFnbm9XRkF0S3UvWldUS0hpK0JNYmhLd3N3QlM4VzBHMXl3aQpjVVdHbHppZ0k0dGR4eGYxWUJKeUNvVFNOc3NTYkttSWg1amVtQmZydklCbzF5RWQrZTU2WkpNZGhOOGUreFdVCmJ2b3ZVQzIvN0RsNzZmekFhQUNMU29yWlV2NVhQSndLWHdFT0hvN0ZJY1JFam9abitmS2pKVG5tZFhjZTBMRDYKOVJIcityK2NleUU3OWdtSzMxYkk5RFlpSm9MNExlR2RYWjNnTU9WRFIxT25Eb3M1bE9CY1YrcXVKNkp1anBnSApkOWczU2E3RHU3cHVzRDlGZGFwOThvY1pzbFJmRmpGaS8vMllkVk00TUticTZJd3BZTkIrMlBDRUtOQzdTZmJPCk5nWllKdVBabk0vd1ZpRVMvY1A3TVpOSjFLVUtCSTl5aDZUbWxTc1paT2NsR0p2ck9zQlppbVRYcEFUamROTXQKY2x1S3dxQVVVellRbVU3YmYyVE1kT1h5QTlpSDV3SXBqMWtXR0UxVnVGQURUS0lMa1RjNkx6THpPV0NvZkx4ZgpvbmhUdFNEdHpJdi91ZWw1NDdHWnFxK3JWUnZtSWllRXVFdkRFVHd1b29rZlY2cXUzRC85S3VTcjl4aXpubUVnCnh5bnVkL2Y1MjVqcHBKTWNEL29mYlF4VVp1R0t2YjNmM3p5K2FMeHFpZG9YN2djYTJYZDlqeVV5NVkvODMrWk4KYno0UFp4ODFVSnpYVkk5QUJFaDgveGlsQVRoMVp4T2VQVEJKak43bGdyMGxYdEtZalYvNDN5eXhnVVlyWE5aUwpvTFNHMmRMQ0s5bWpqcmFQamF1MzRRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQoiIiIKJycnCgoicG9saWN5LnJlZ28iID0gJycnCnBhY2thZ2UgYWdlbnRfcG9saWN5CgppbXBvcnQgZnV0dXJlLmtleXdvcmRzLmluCmltcG9ydCBmdXR1cmUua2V5d29yZHMuZXZlcnkKCmltcG9ydCBpbnB1dAoKIyBEZWZhdWx0IHZhbHVlcywgcmV0dXJuZWQgYnkgT1BBIHdoZW4gcnVsZXMgY2Fubm90IGJlIGV2YWx1YXRlZCB0byB0cnVlLgpkZWZhdWx0IENvcHlGaWxlUmVxdWVzdCA6PSBmYWxzZQpkZWZhdWx0IENyZWF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gZmFsc2UKZGVmYXVsdCBDcmVhdGVTYW5kYm94UmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgRGVzdHJveVNhbmRib3hSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBFeGVjUHJvY2Vzc1JlcXVlc3QgOj0gZmFsc2UKZGVmYXVsdCBHZXRPT01FdmVudFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEd1ZXN0RGV0YWlsc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IE9ubGluZUNQVU1lbVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFB1bGxJbWFnZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlYWRTdHJlYW1SZXF1ZXN0IDo9IGZhbHNlCmRlZmF1bHQgUmVtb3ZlQ29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgUmVtb3ZlU3RhbGVWaXJ0aW9mc1NoYXJlTW91bnRzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2lnbmFsUHJvY2Vzc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXJ0Q29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RhdHNDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBUdHlXaW5SZXNpemVSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVFcGhlbWVyYWxNb3VudHNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVJbnRlcmZhY2VSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBVcGRhdGVSb3V0ZXNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBXYWl0UHJvY2Vzc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFdyaXRlU3RyZWFtUmVxdWVzdCA6PSBmYWxzZQonJyc= LIBVIRT_NET: default LIBVIRT_POOL: default LIBVIRT_URI: qemu+ssh://root@192.168.122.1/system?no_verify=1 diff --git a/src/cloud-api-adaptor/entrypoint.sh b/src/cloud-api-adaptor/entrypoint.sh index f36e3c265..1783043b2 100755 --- a/src/cloud-api-adaptor/entrypoint.sh +++ b/src/cloud-api-adaptor/entrypoint.sh @@ -18,7 +18,7 @@ optionals+="" [[ "${CERT_FILE}" ]] && [[ "${CERT_KEY}" ]] && optionals+="-cert-file ${CERT_FILE} -cert-key ${CERT_KEY} " [[ "${TLS_SKIP_VERIFY}" ]] && optionals+="-tls-skip-verify " [[ "${PROXY_TIMEOUT}" ]] && optionals+="-proxy-timeout ${PROXY_TIMEOUT} " -[[ "${GLOBAL_INITDATA}" ]] && optionals+="-global-initdata ${GLOBAL_INITDATA} " +[[ "${INITDATA}" ]] && optionals+="-initdata ${INITDATA} " [[ "${FORWARDER_PORT}" ]] && optionals+="-forwarder-port ${FORWARDER_PORT} " [[ "${CLOUD_CONFIG_VERIFY}" == "true" ]] && optionals+="-cloud-config-verify " [[ "${SECURE_COMMS}" == "true" ]] && optionals+="-secure-comms " diff --git a/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml index 6f45fcd4b..221422a4f 100644 --- a/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/azure/kustomization.yaml @@ -33,7 +33,7 @@ configMapGenerator: # /subscriptions//resourceGroups//providers/Microsoft.Compute/images/ - AZURE_IMAGE_ID="" #set - SSH_USERNAME="" #set peer pod vm admin user name - - GLOBAL_INITDATA="" # set default initdata for podvm + - INITDATA="" # set default initdata for podvm #- DISABLECVM="" # Uncomment it if you want a generic VM #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789 diff --git a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml index 6a7fb47ad..a75dab375 100644 --- a/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml +++ b/src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml @@ -24,7 +24,7 @@ configMapGenerator: - LIBVIRT_POOL="default" # set - DISABLECVM="true" # set as false to enable confidential VM - SECURE_COMMS="false" # set as true to enable Secure Comms - - GLOBAL_INITDATA="" # set default initdata for podvm + - INITDATA="" # set default initdata for podvm #- LIBVIRT_LAUNCH_SECURITY="" #sev or s390-pv #- LIBVIRT_FIRMWARE="" # Uncomment and set if you want to change the firmware path. Defaults to /usr/share/edk2/ovmf/OVMF_CODE.fd #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2 diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go index ffed48e96..70080346f 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go @@ -33,9 +33,9 @@ import ( const ( SrcAuthfilePath = "/root/containers/auth.json" - AaFilePath = "/run/peerpod/aa.toml" + AAConfigPath = "/run/peerpod/aa.toml" AuthFilePath = "/run/peerpod/auth.json" - CdhFilePath = "/run/peerpod/cdh.toml" + CDHConfigPath = "/run/peerpod/cdh.toml" InitdataPath = "/run/peerpod/initdata" Version = "0.0.0" ) @@ -79,7 +79,7 @@ func (s *cloudService) removeSandbox(id sandboxID) error { } func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNode podnetwork.WorkerNode, - secureComms bool, secureCommsInbounds, secureCommsOutbounds, kbsAddress, podsDir, daemonPort, globalInitdata, sshport string) Service { + secureComms bool, secureCommsInbounds, secureCommsOutbounds, kbsAddress, podsDir, daemonPort, initdata, sshport string) Service { var err error var sshClient *wnssh.SshClient @@ -93,14 +93,14 @@ func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNo } s := &cloudService{ - provider: provider, - proxyFactory: proxyFactory, - sandboxes: map[sandboxID]*sandbox{}, - podsDir: podsDir, - daemonPort: daemonPort, - globalInitdata: globalInitdata, - workerNode: workerNode, - sshClient: sshClient, + provider: provider, + proxyFactory: proxyFactory, + sandboxes: map[sandboxID]*sandbox{}, + podsDir: podsDir, + daemonPort: daemonPort, + initdata: initdata, + workerNode: workerNode, + sshClient: sshClient, } s.cond = sync.NewCond(&s.mutex) s.ppService, err = k8sops.NewPeerPodService() @@ -295,8 +295,8 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r logger.Printf("initdata in Pod annotation: %s", initdataStr) if initdataStr == "" { - logger.Printf("initdata in pod annotation is empty, use global initdata: %s", s.globalInitdata) - initdataStr = s.globalInitdata + logger.Printf("initdata in pod annotation is empty, use global initdata: %s", s.initdata) + initdataStr = s.initdata } if initdataStr != "" { diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go index 4bb4747f9..6d71b6933 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go @@ -26,17 +26,17 @@ type Service interface { } type cloudService struct { - provider provider.Provider - proxyFactory proxy.Factory - workerNode podnetwork.WorkerNode - sandboxes map[sandboxID]*sandbox - cond *sync.Cond - podsDir string - daemonPort string - mutex sync.Mutex - ppService *k8sops.PeerPodService - globalInitdata string - sshClient *wnssh.SshClient + provider provider.Provider + proxyFactory proxy.Factory + workerNode podnetwork.WorkerNode + sandboxes map[sandboxID]*sandbox + cond *sync.Cond + podsDir string + daemonPort string + mutex sync.Mutex + ppService *k8sops.PeerPodService + initdata string + sshClient *wnssh.SshClient } type sandboxID string diff --git a/src/cloud-api-adaptor/pkg/adaptor/server.go b/src/cloud-api-adaptor/pkg/adaptor/server.go index 049fae799..036a644d3 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/server.go +++ b/src/cloud-api-adaptor/pkg/adaptor/server.go @@ -39,7 +39,7 @@ type ServerConfig struct { PodsDir string ForwarderPort string ProxyTimeout time.Duration - GlobalInitdata string + Initdata string EnableCloudConfigVerify bool SecureComms bool SecureCommsInbounds string @@ -71,7 +71,7 @@ func NewServer(provider provider.Provider, cfg *ServerConfig, workerNode podnetw agentFactory := proxy.NewFactory(cfg.PauseImage, cfg.TLSConfig, cfg.ProxyTimeout) cloudService := cloud.NewService(provider, agentFactory, workerNode, - cfg.SecureComms, cfg.SecureCommsInbounds, cfg.SecureCommsOutbounds, cfg.SecureCommsKbsAddress, cfg.PodsDir, cfg.ForwarderPort, cfg.GlobalInitdata, sshutil.SSHPORT) + cfg.SecureComms, cfg.SecureCommsInbounds, cfg.SecureCommsOutbounds, cfg.SecureCommsKbsAddress, cfg.PodsDir, cfg.ForwarderPort, cfg.Initdata, sshutil.SSHPORT) vmInfoService := vminfo.NewService(cloudService) return &server{ diff --git a/src/cloud-api-adaptor/pkg/userdata/provision.go b/src/cloud-api-adaptor/pkg/userdata/provision.go index f7ae922d4..7b92e8128 100644 --- a/src/cloud-api-adaptor/pkg/userdata/provision.go +++ b/src/cloud-api-adaptor/pkg/userdata/provision.go @@ -31,8 +31,8 @@ const ( ) var logger = log.New(log.Writer(), "[userdata/provision] ", log.LstdFlags|log.Lmsgprefix) -var WriteFilesList = []string{cloud.AaFilePath, cloud.CdhFilePath, agent.ConfigFilePath, forwarder.DefaultConfigPath, cloud.AuthFilePath, cloud.InitdataPath} -var InitdDataFilesList = []string{cloud.AaFilePath, cloud.CdhFilePath, PolicyPath} +var WriteFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, agent.ConfigFilePath, forwarder.DefaultConfigPath, cloud.AuthFilePath, cloud.InitdataPath} +var InitdDataFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, PolicyPath} type Config struct { fetchTimeout int diff --git a/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go b/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go index 0a2fe6858..2f2d2482a 100644 --- a/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/azure/provision_common.go @@ -381,7 +381,7 @@ func (p *AzureCloudProvisioner) UploadPodvm(imagePath string, ctx context.Contex func isAzureKustomizeConfigMapKey(key string) bool { switch key { - case "CLOUD_PROVIDER", "AZURE_SUBSCRIPTION_ID", "AZURE_REGION", "AZURE_INSTANCE_SIZE", "AZURE_RESOURCE_GROUP", "AZURE_SUBNET_ID", "AZURE_IMAGE_ID", "SSH_USERNAME", "GLOBAL_INITDATA", "TAGS": + case "CLOUD_PROVIDER", "AZURE_SUBSCRIPTION_ID", "AZURE_REGION", "AZURE_INSTANCE_SIZE", "AZURE_RESOURCE_GROUP", "AZURE_SUBNET_ID", "AZURE_IMAGE_ID", "SSH_USERNAME", "INITDATA", "TAGS": return true default: return false diff --git a/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go index 9b7c6b243..9438d0f67 100644 --- a/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go @@ -206,7 +206,7 @@ func NewDockerInstallOverlay(installDir, provider string) (pv.InstallOverlay, er func isDockerKustomizeConfigMapKey(key string) bool { switch key { - case "CLOUD_PROVIDER", "DOCKER_HOST", "DOCKER_API_VERSION", "DOCKER_PODVM_IMAGE", "DOCKER_NETWORK_NAME", "GLOBAL_INITDATA": + case "CLOUD_PROVIDER", "DOCKER_HOST", "DOCKER_API_VERSION", "DOCKER_PODVM_IMAGE", "DOCKER_NETWORK_NAME", "INITDATA": return true default: return false diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go index 832da0a56..f53b9e024 100644 --- a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go @@ -320,13 +320,13 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, // Mapping the internal properties to ConfigMapGenerator properties and their default values. mapProps := map[string][2]string{ - "network": {"default", "LIBVIRT_NET"}, - "storage": {"default", "LIBVIRT_POOL"}, - "pause_image": {"", "PAUSE_IMAGE"}, - "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, - "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, - "vxlan_port": {"", "VXLAN_PORT"}, - "GLOBAL_INITDATA": {"", "GLOBAL_INITDATA"}, + "network": {"default", "LIBVIRT_NET"}, + "storage": {"default", "LIBVIRT_POOL"}, + "pause_image": {"", "PAUSE_IMAGE"}, + "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, + "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, + "vxlan_port": {"", "VXLAN_PORT"}, + "INITDATA": {"", "INITDATA"}, } for k, v := range mapProps { From 7927e6035cdab0a13fdf17c03ab03d7e56151ce8 Mon Sep 17 00:00:00 2001 From: Qi Feng Huo Date: Wed, 21 Aug 2024 11:14:16 +0800 Subject: [PATCH 4/5] initdata: validate initdata by unmarshal it - Validate the initdata passed in both from configmap and annotation Fixes: #1985 Signed-off-by: Qi Feng Huo --- .../pkg/adaptor/cloud/cloud.go | 18 ++++++++++++++++++ .../pkg/userdata/provision.go | 8 +------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go index 70080346f..408af5a10 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go @@ -5,6 +5,7 @@ package cloud import ( "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -29,6 +30,7 @@ import ( provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers" putil "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util/cloudinit" + toml "github.com/pelletier/go-toml/v2" ) const ( @@ -40,6 +42,12 @@ const ( Version = "0.0.0" ) +type InitData struct { + Algorithm string `toml:"algorithm"` + Version string `toml:"version"` + Data map[string]string `toml:"data,omitempty"` +} + var logger = log.New(log.Writer(), "[adaptor/cloud] ", log.LstdFlags|log.Lmsgprefix) func (s *cloudService) addSandbox(sid sandboxID, sandbox *sandbox) error { @@ -300,6 +308,16 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r } if initdataStr != "" { + decodedBytes, err := base64.StdEncoding.DecodeString(initdataStr) + if err != nil { + return nil, fmt.Errorf("Error base64 decode initdata: %w", err) + } + initdata := InitData{} + err = toml.Unmarshal(decodedBytes, &initdata) + if err != nil { + return nil, fmt.Errorf("Error unmarshalling initdata: %w", err) + } + cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{ Path: InitdataPath, Content: initdataStr, diff --git a/src/cloud-api-adaptor/pkg/userdata/provision.go b/src/cloud-api-adaptor/pkg/userdata/provision.go index 7b92e8128..8801d8485 100644 --- a/src/cloud-api-adaptor/pkg/userdata/provision.go +++ b/src/cloud-api-adaptor/pkg/userdata/provision.go @@ -63,12 +63,6 @@ type CloudConfig struct { WriteFiles []WriteFile `yaml:"write_files"` } -type InitData struct { - Algorithm string `toml:"algorithm"` - Version string `toml:"version"` - Data map[string]string `toml:"data,omitempty"` -} - type UserDataProvider interface { GetUserData(ctx context.Context) ([]byte, error) GetRetryDelay() time.Duration @@ -225,7 +219,7 @@ func extractInitdataAndHash(cfg *Config) error { if err != nil { return fmt.Errorf("Error base64 decode initdata: %w", err) } - initdata := InitData{} + initdata := cloud.InitData{} err = toml.Unmarshal(decodedBytes, &initdata) if err != nil { return fmt.Errorf("Error unmarshalling initdata: %w", err) From b4c734cf9b603d699dec88258c9f956200ee32ad Mon Sep 17 00:00:00 2001 From: Qi Feng Huo Date: Mon, 2 Sep 2024 16:41:02 +0800 Subject: [PATCH 5/5] initdata: fix misc issues in md files Fixes: #1985 Signed-off-by: Qi Feng Huo --- src/cloud-api-adaptor/docs/addnewprovider.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cloud-api-adaptor/docs/addnewprovider.md b/src/cloud-api-adaptor/docs/addnewprovider.md index c1d3b1eec..c9209d78b 100644 --- a/src/cloud-api-adaptor/docs/addnewprovider.md +++ b/src/cloud-api-adaptor/docs/addnewprovider.md @@ -282,7 +282,7 @@ optionals+="" [[ "${CERT_FILE}" ]] && [[ "${CERT_KEY}" ]] && optionals+="-cert-file ${CERT_FILE} -cert-key ${CERT_KEY} " [[ "${TLS_SKIP_VERIFY}" ]] && optionals+="-tls-skip-verify " [[ "${PROXY_TIMEOUT}" ]] && optionals+="-proxy-timeout ${PROXY_TIMEOUT} " -[[ "${INITDATA}" ]] && optionals+="-global-initdata ${INITDATA} " +[[ "${INITDATA}" ]] && optionals+="-initdata ${INITDATA} " [[ "${FORWARDER_PORT}" ]] && optionals+="-forwarder-port ${FORWARDER_PORT} " [[ "${CLOUD_CONFIG_VERIFY}" == "true" ]] && optionals+="-cloud-config-verify "