podvm: add scratch-space for /run/container

This adds the configuration for an encrypted scratch space on an mkosi
image. At bootup a /dev/sda4 partition will be created and encrypted
with LUKS using an ephemeral key.

The partition will use the space available on the image volume. By
default the qcow2 image has 100mb allocated for this space. This amount
of space will only work for very small images, hence we do not mount the
scratch space to `/run/kata-container` by default.

If the kata-agent service units encounters a `/run/peerpod/mount-scratch`
file it will mount the encrypted partition `/dev/sda4` to
`/run/kata-containers`.

This file is provisioned by `process-user-data`, configured by the CAA
daemonset.

A new CAA parameter has been introduced that allows to specify the disk
size. If we have a disk size that is larger than 0 a cloud provider can
attempt to consider this size to create space for an encrypted
scratch partition that can be mounted to /run/container.

Signed-off-by: Magnus Kulke <magnuskulke@microsoft.com>

src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go

@@ -125,6 +125,7 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 		flags.StringVar(&cfg.serverConfig.Initdata, "initdata", "", "Default initdata for all Pods")
 		flags.BoolVar(&cfg.serverConfig.EnableCloudConfigVerify, "cloud-config-verify", false, "Enable cloud config verify - should use it for production")
 		flags.IntVar(&cfg.serverConfig.PeerPodsLimitPerNode, "peerpods-limit-per-node", 10, "peer pods limit per node (default=10)")
+		flags.Uint64Var(&cfg.serverConfig.RootVolumeSize, "root-volume-size", 0, "Root volume size in GB. Default is 0, which implies the default image disk size")
 
 		cloud.ParseCmd(flags)
 	})

src/cloud-api-adaptor/entrypoint.sh

@@ -77,6 +77,7 @@ azure() {
     [[ "${TAGS}" ]] && optionals+="-tags ${TAGS} " # Custom tags applied to pod vm
     [[ "${ENABLE_SECURE_BOOT}" == "true" ]] && optionals+="-enable-secure-boot "
     [[ "${USE_PUBLIC_IP}" == "true" ]] && optionals+="-use-public-ip "
+    [[ "${ROOT_VOLUME_SIZE}" ]] && optionals+="-root-volume-size ${ROOT_VOLUME_SIZE} " # OS disk size in GB
 
     set -x
     exec cloud-api-adaptor azure \

src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go

@@ -63,6 +63,7 @@ type ServerConfig struct {
 	SecureCommsPpOutbounds  string
 	SecureCommsKbsAddress   string
 	PeerPodsLimitPerNode    int
+	RootVolumeSize          uint64
 }
 
 var logger = log.New(log.Writer(), "[adaptor/cloud] ", log.LstdFlags|log.Lmsgprefix)
@@ -221,17 +222,20 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 	instanceType := util.GetInstanceTypeFromAnnotation(req.Annotations)
 
 	// Get Pod VM cpu and memory from annotations
-	vcpus, memory, gpus := util.GetPodvmResourcesFromAnnotation(req.Annotations)
+	resources := util.GetPodVMResourcesFromAnnotation(req.Annotations)
+
+	// Set caa-wide root volume size to resources if the storage has not been configured yet
+	if resources.Storage == 0 && s.serverConfig.RootVolumeSize > 0 {
+		resources.Storage = int64(s.serverConfig.RootVolumeSize)
+	}
 
 	// Get Pod VM image from annotations
 	image := util.GetImageFromAnnotation(req.Annotations)
 
 	// Pod VM spec
 	vmSpec := provider.InstanceTypeSpec{
 		InstanceType: instanceType,
-		VCPUs:        vcpus,
-		Memory:       memory,
-		GPUs:         gpus,
+		Resources:    resources,
 		Image:        image,
 	}
 
@@ -320,6 +324,14 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 		},
 	}
 
+	if s.serverConfig.RootVolumeSize > 0 {
+		// Write an empty file to indicate that we want to use available space as sandbox storage
+		cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{
+			Path:    UseScratchPath,
+			Content: "",
+		})
+	}
+
 	if authJSON != nil {
 		if len(authJSON) > cloudinit.DefaultAuthfileLimit {
 			logger.Printf("Credentials file is too large to be included in cloud-config")

src/cloud-api-adaptor/pkg/paths/paths.go

@@ -8,4 +8,5 @@ const (
 	AgentCfgPath     = "/run/peerpod/agent-config.toml"
 	ForwarderCfgPath = "/run/peerpod/daemon.json"
 	UserDataPath     = "/media/cidata/user-data"
+	UseScratchPath   = "/run/peerpod/mount-scratch"
 )

src/cloud-api-adaptor/pkg/userdata/provision.go

@@ -37,7 +37,7 @@ const (
 )
 
 var logger = log.New(log.Writer(), "[userdata/provision] ", log.LstdFlags|log.Lmsgprefix)
-var WriteFilesList = []string{AACfgPath, CDHCfgPath, ForwarderCfgPath, AuthFilePath, InitDataPath}
+var WriteFilesList = []string{AACfgPath, CDHCfgPath, ForwarderCfgPath, AuthFilePath, InitDataPath, UseScratchPath}
 var InitdDataFilesList = []string{AACfgPath, CDHCfgPath, PolicyPath}
 
 type Config struct {

src/cloud-api-adaptor/pkg/util/cloud.go

@@ -5,6 +5,7 @@ import (
 	"strconv"
 	"strings"
 
+	provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers"
 	cri "github.com/containerd/containerd/pkg/cri/annotations"
 	hypannotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
 )
@@ -44,48 +45,48 @@ func GetImageFromAnnotation(annotations map[string]string) string {
 }
 
 // Method to get vCPU, memory and gpus from annotations
-func GetPodvmResourcesFromAnnotation(annotations map[string]string) (int64, int64, int64) {
-
-	var vcpuInt, memoryInt, gpuInt int64
+func GetPodVMResourcesFromAnnotation(annotations map[string]string) provider.PodVMResources {
+	var vcpus, gpus, memory int64
 	var err error
 
 	vcpu, ok := annotations[hypannotations.DefaultVCPUs]
 	if ok {
-		vcpuInt, err = strconv.ParseInt(vcpu, 10, 64)
+		vcpus, err = strconv.ParseInt(vcpu, 10, 64)
 		if err != nil {
 			fmt.Printf("Error converting vcpu to int64. Defaulting to 0: %v\n", err)
-			vcpuInt = 0
+			vcpus = 0
 		}
 	} else {
-		vcpuInt = 0
+		vcpus = 0
 	}
 
-	memory, ok := annotations[hypannotations.DefaultMemory]
+	mem, ok := annotations[hypannotations.DefaultMemory]
 	if ok {
 		// Use strconv.ParseInt to convert string to int64
-		memoryInt, err = strconv.ParseInt(memory, 10, 64)
+		memory, err = strconv.ParseInt(mem, 10, 64)
 		if err != nil {
 			fmt.Printf("Error converting memory to int64. Defaulting to 0: %v\n", err)
-			memoryInt = 0
+			memory = 0
 		}
 
 	} else {
-		memoryInt = 0
+		memory = 0
 	}
 
 	gpu, ok := annotations[hypannotations.DefaultGPUs]
 	if ok {
-		gpuInt, err = strconv.ParseInt(gpu, 10, 64)
+		gpus, err = strconv.ParseInt(gpu, 10, 64)
 		if err != nil {
 			fmt.Printf("Error converting gpu to int64. Defaulting to 0: %v\n", err)
-			gpuInt = 0
+			gpus = 0
 		}
 	} else {
-		gpuInt = 0
+		gpus = 0
 	}
 
-	// Return vCPU, memory and GPU
-	return vcpuInt, memoryInt, gpuInt
+	storage := int64(0)
+
+	return provider.PodVMResources{VCPUs: vcpus, Memory: memory, GPUs: gpus, Storage: storage}
 }
 
 // Method to get initdata from annotation

src/cloud-api-adaptor/pkg/util/cloud_test.go

@@ -98,7 +98,8 @@ func TestGetPodvmResourcesFromAnnotation(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, got1, got2 := GetPodvmResourcesFromAnnotation(tt.args.annotations)
+			r := GetPodVMResourcesFromAnnotation(tt.args.annotations)
+			got, got1, got2 := r.VCPUs, r.Memory, r.GPUs
 			if got != tt.want {
 				t.Errorf("GetPodvmResourcesFromAnnotation() got = %v, want %v", got, tt.want)
 			}

src/cloud-api-adaptor/podvm-mkosi/Makefile

@@ -73,6 +73,7 @@ else ifeq ($(ARCH),s390x)
 else
 	mkosi --profile production.conf
 	qemu-img convert -f raw -O qcow2 build/system.raw build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2
+	qemu-img resize build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2 +10M
 endif
 
 PHONY: image-debug
@@ -90,6 +91,7 @@ else ifeq ($(ARCH),s390x)
 else
 	mkosi --profile debug.conf
 	qemu-img convert -f raw -O qcow2 build/system.raw build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2
+	qemu-img resize build/podvm-$(PODVM_DISTRO)-$(ARCH).qcow2 +10M
 endif
 
 PHONY: image-container

src/cloud-api-adaptor/podvm-mkosi/mkosi.presets/system/mkosi.conf.d/fedora.conf

@@ -7,7 +7,7 @@ Release=40
 
 [Content]
 CleanPackageMetadata=true
-SkeletonTrees=../../resources/binaries-tree
+SkeletonTrees=../../mkosi.skeleton-rootfs,../../resources/binaries-tree,
 Packages=
     kernel
     kernel-core
@@ -23,6 +23,7 @@ Packages=
     iptables
     afterburn
     neofetch
+    e2fsprogs
 
 RemoveFiles=/etc/issue
 RemoveFiles=/etc/issue.net

src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/etc/crypttab

@@ -0,0 +1 @@
+scratch /dev/disk/by-label/scratch - try-empty-password

src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton-rootfs/usr/lib/systemd/system/kata-agent.service.d/10-override.conf

@@ -0,0 +1,2 @@
+[Service]
+ExecStartPre=sh -c '[[ ! -f /run/peerpod/mount-scratch ]] || mount /dev/mapper/scratch /run/kata-containers'

src/cloud-api-adaptor/podvm-mkosi/mkosi.skeleton/usr/lib/repart.d/30-scratch.conf

@@ -0,0 +1,5 @@
+[Partition]
+Type=linux-generic
+Label=scratch
+Encrypt=key-file
+Format=ext4

src/cloud-providers/aws/provider.go

@@ -370,8 +370,10 @@ func (p *awsProvider) updateInstanceTypeSpecList() error {
 		if err != nil {
 			return err
 		}
+		resources := provider.NewPodVMResources(vcpus, memory)
+		resources.GPUs = gpuCount
 		instanceTypeSpecList = append(instanceTypeSpecList,
-			provider.InstanceTypeSpec{InstanceType: instanceType, VCPUs: vcpus, Memory: memory, GPUs: gpuCount})
+			provider.InstanceTypeSpec{InstanceType: instanceType, Resources: resources})
 	}
 
 	// Sort the instanceTypeSpecList and update the serviceConfig

src/cloud-providers/azure/provider.go

@@ -11,7 +11,6 @@ import (
 	"log"
 	"net/netip"
 	"os"
-	"path/filepath"
 	"regexp"
 	"strings"
 
@@ -35,24 +34,44 @@ const (
 type azureProvider struct {
 	azureClient   azcore.TokenCredential
 	serviceConfig *Config
+	sshKey        armcompute.SSHPublicKey
 }
 
 func NewProvider(config *Config) (provider.Provider, error) {
 
 	logger.Printf("azure config %+v", config.Redact())
 
-	// Clean the config.SSHKeyPath to avoid bad paths
-	config.SSHKeyPath = filepath.Clean(config.SSHKeyPath)
-
 	azureClient, err := NewAzureClient(*config)
 	if err != nil {
 		logger.Printf("creating azure client: %v", err)
 		return nil, err
 	}
 
+	// The podvm disk doesn't support sshd logins. keys can be baked
+	// into a debug-image. The ARM api mandates a pubkey, though.
+	sshPublicKeyPath := os.ExpandEnv(config.SSHKeyPath)
+	var pubkeyBytes []byte
+	if _, err := os.Stat(sshPublicKeyPath); err == nil {
+		pubkeyBytes, err = os.ReadFile(sshPublicKeyPath)
+		if err != nil {
+			err = fmt.Errorf("reading ssh public key file: %w", err)
+			logger.Printf("%v", err)
+			return nil, err
+		}
+	} else {
+		err = fmt.Errorf("ssh public key: %w", err)
+		logger.Printf("%v", err)
+		return nil, err
+	}
+	dummySSHKey := armcompute.SSHPublicKey{
+		Path:    to.Ptr(fmt.Sprintf("/home/%s/.ssh/authorized_keys", config.SSHUserName)),
+		KeyData: to.Ptr(string(pubkeyBytes)),
+	}
+
 	provider := &azureProvider{
 		azureClient:   azureClient,
 		serviceConfig: config,
+		sshKey:        dummySSHKey,
 	}
 
 	if err = provider.updateInstanceSizeSpecList(); err != nil {
@@ -218,28 +237,12 @@ func (p *azureProvider) CreateInstance(ctx context.Context, podName, sandboxID s
 	diskName := fmt.Sprintf("%s-disk", instanceName)
 	nicName := fmt.Sprintf("%s-net", instanceName)
 
-	// require ssh key for authentication on linux
-	sshPublicKeyPath := os.ExpandEnv(p.serviceConfig.SSHKeyPath)
-	var sshBytes []byte
-	if _, err := os.Stat(sshPublicKeyPath); err == nil {
-		sshBytes, err = os.ReadFile(sshPublicKeyPath)
-		if err != nil {
-			err = fmt.Errorf("reading ssh public key file: %w", err)
-			logger.Printf("%v", err)
-			return nil, err
-		}
-	} else {
-		err = fmt.Errorf("ssh public key: %w", err)
-		logger.Printf("%v", err)
-		return nil, err
-	}
-
 	if spec.Image != "" {
 		logger.Printf("Choosing %s from annotation as the Azure Image for the PodVM image", spec.Image)
 		p.serviceConfig.ImageId = spec.Image
 	}
 
-	vmParameters, err := p.getVMParameters(instanceSize, diskName, cloudConfigData, sshBytes, instanceName, nicName)
+	vmParameters, err := p.getVMParameters(instanceSize, diskName, cloudConfigData, instanceName, nicName, spec.Resources.Storage)
 	if err != nil {
 		return nil, err
 	}
@@ -350,7 +353,10 @@ func (p *azureProvider) updateInstanceSizeSpecList() error {
 		}
 		for _, vmSize := range nextResult.VirtualMachineSizeListResult.Value {
 			if util.Contains(instanceSizes, *vmSize.Name) {
-				instanceSizeSpecList = append(instanceSizeSpecList, provider.InstanceTypeSpec{InstanceType: *vmSize.Name, VCPUs: int64(*vmSize.NumberOfCores), Memory: int64(*vmSize.MemoryInMB)})
+				vcpus, memory := int64(*vmSize.NumberOfCores), int64(*vmSize.MemoryInMB)
+				resources := provider.NewPodVMResources(vcpus, memory)
+				instanceSizeSpec := provider.InstanceTypeSpec{InstanceType: *vmSize.Name, Resources: resources}
+				instanceSizeSpecList = append(instanceSizeSpecList, instanceSizeSpec)
 			}
 		}
 	}
@@ -371,7 +377,7 @@ func (p *azureProvider) getResourceTags() map[string]*string {
 	return tags
 }
 
-func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig string, sshBytes []byte, instanceName, nicName string) (*armcompute.VirtualMachine, error) {
+func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig, instanceName, nicName string, diskSize int64) (*armcompute.VirtualMachine, error) {
 	userDataB64 := base64.StdEncoding.EncodeToString([]byte(cloudConfig))
 
 	// Azure limits the base64 encrypted userData to 64KB.
@@ -416,6 +422,18 @@ func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig stri
 
 	networkConfig := p.buildNetworkConfig(nicName)
 
+	osDisk := armcompute.OSDisk{
+		Name:         to.Ptr(diskName),
+		CreateOption: to.Ptr(armcompute.DiskCreateOptionTypesFromImage),
+		Caching:      to.Ptr(armcompute.CachingTypesReadWrite),
+		DeleteOption: to.Ptr(armcompute.DiskDeleteOptionTypesDelete),
+		ManagedDisk:  managedDiskParams,
+	}
+
+	if diskSize > 0 {
+		osDisk.DiskSizeGB = to.Ptr(int32(diskSize))
+	}
+
 	vmParameters := armcompute.VirtualMachine{
 		Location: to.Ptr(p.serviceConfig.Region),
 		Properties: &armcompute.VirtualMachineProperties{
@@ -424,25 +442,15 @@ func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig stri
 			},
 			StorageProfile: &armcompute.StorageProfile{
 				ImageReference: imgRef,
-				OSDisk: &armcompute.OSDisk{
-					Name:         to.Ptr(diskName),
-					CreateOption: to.Ptr(armcompute.DiskCreateOptionTypesFromImage),
-					Caching:      to.Ptr(armcompute.CachingTypesReadWrite),
-					DeleteOption: to.Ptr(armcompute.DiskDeleteOptionTypesDelete),
-					ManagedDisk:  managedDiskParams,
-				},
+				OSDisk:         &osDisk,
 			},
 			OSProfile: &armcompute.OSProfile{
 				AdminUsername: to.Ptr(p.serviceConfig.SSHUserName),
 				ComputerName:  to.Ptr(instanceName),
 				LinuxConfiguration: &armcompute.LinuxConfiguration{
 					DisablePasswordAuthentication: to.Ptr(true),
-					//TBD: replace with a suitable mechanism to use precreated SSH key
 					SSH: &armcompute.SSHConfiguration{
-						PublicKeys: []*armcompute.SSHPublicKey{{
-							Path:    to.Ptr(fmt.Sprintf("/home/%s/.ssh/authorized_keys", p.serviceConfig.SSHUserName)),
-							KeyData: to.Ptr(string(sshBytes)),
-						}},
+						PublicKeys: []*armcompute.SSHPublicKey{&p.sshKey},
 					},
 				},
 			},

src/cloud-providers/ibmcloud/provider.go

@@ -332,7 +332,8 @@ func (p *ibmcloudVPCProvider) updateInstanceProfileSpecList() error {
 		if err != nil {
 			return err
 		}
-		instanceProfileSpecList = append(instanceProfileSpecList, provider.InstanceTypeSpec{InstanceType: profileType, VCPUs: vcpus, Memory: memory, Arch: arch})
+		resources := provider.NewPodVMResources(vcpus, memory)
+		instanceProfileSpecList = append(instanceProfileSpecList, provider.InstanceTypeSpec{InstanceType: profileType, Resources: resources, Arch: arch})
 	}
 
 	// Sort the instanceProfileSpecList and update the serviceConfig