libvirt: Enable multiple PodVM image scenario

src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go

@@ -203,11 +203,15 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 	// Get Pod VM cpu and memory from annotations
 	vcpus, memory := util.GetCPUAndMemoryFromAnnotation(req.Annotations)
 
+	// Get Pod VM image from annotations
+	image := util.GetImageFromAnnotation(req.Annotations)
+
 	// Pod VM spec
 	vmSpec := provider.InstanceTypeSpec{
 		InstanceType: instanceType,
 		VCPUs:        vcpus,
 		Memory:       memory,
+		Image:        image,
 	}
 
 	// TODO: server name is also generated in each cloud provider, and possibly inconsistent

src/cloud-api-adaptor/pkg/util/cloud.go

@@ -35,6 +35,14 @@ func GetInstanceTypeFromAnnotation(annotations map[string]string) string {
 	return annotations[hypannotations.MachineType]
 }
 
+// Method to get image from annotation
+func GetImageFromAnnotation(annotations map[string]string) string {
+	// The image annotation in Kata refers to image path
+	// For example image for Kata/Qemu refers to /hypervisor/image.img etc.
+	// We use the same annotation for Kata/remote to refer to image name
+	return annotations[hypannotations.ImagePath]
+}
+
 // Method to get vCPU and memory from annotations
 func GetCPUAndMemoryFromAnnotation(annotations map[string]string) (int64, int64) {
 

src/cloud-api-adaptor/pkg/util/cloud_test.go

@@ -114,3 +114,42 @@ func TestGetInstanceTypeFromAnnotation(t *testing.T) {
 		})
 	}
 }
+
+func TestGetImageFromAnnotation(t *testing.T) {
+	type args struct {
+		annotations map[string]string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		// Add test cases with annotations for only image name
+		{
+			name: "image name only",
+			args: args{
+				annotations: map[string]string{
+					hypannotations.ImagePath: "rhel9-os",
+				},
+			},
+			want: "rhel9-os",
+		},
+		// Add test cases with annotations for only image name with empty value
+		{
+			name: "image name only with empty value",
+			args: args{
+				annotations: map[string]string{
+					hypannotations.ImagePath: "",
+				},
+			},
+			want: "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := GetImageFromAnnotation(tt.args.annotations); got != tt.want {
+				t.Errorf("GetImageFromAnnotation() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

src/cloud-api-adaptor/test/e2e/assessment_helpers.go

@@ -525,3 +525,20 @@ func AddImagePullSecretToDefaultServiceAccount(ctx context.Context, client klien
 	}
 	return nil
 }
+
+func GetPodNamesByLabel(ctx context.Context, client klient.Client, t *testing.T, namespace string, labelName string, labelValue string) (*v1.PodList, error) {
+
+	clientset, err := kubernetes.NewForConfig(client.RESTConfig())
+	if err != nil {
+		t.Fatal(err)
+		return nil, err
+	}
+
+	pods, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelName + "=" + labelValue})
+	if err != nil {
+		t.Fatal(err)
+		return nil, err
+	}
+
+	return pods, nil
+}

src/cloud-api-adaptor/test/e2e/assessment_runner.go

@@ -78,6 +78,7 @@ type TestCase struct {
 	testInstanceTypes    InstanceValidatorFunctions
 	isNydusSnapshotter   bool
 	FailReason           string
+	alternateImageName   string
 }
 
 func (tc *TestCase) WithConfigMap(configMap *v1.ConfigMap) *TestCase {
@@ -135,6 +136,11 @@ func (tc *TestCase) WithInstanceTypes(testInstanceTypes InstanceValidatorFunctio
 	return tc
 }
 
+func (tc *TestCase) WithAlternateImage(alternateImageName string) *TestCase {
+	tc.alternateImageName = alternateImageName
+	return tc
+}
+
 func (pod *ExtraPod) WithTestCommands(TestCommands []TestCommand) *ExtraPod {
 	pod.testCommands = TestCommands
 	return pod
@@ -545,6 +551,25 @@ func (tc *TestCase) Run() {
 				}
 
 			}
+
+			if tc.alternateImageName != "" {
+				var caaPod v1.Pod
+				caaPod.Namespace = "confidential-containers-system"
+				expectedSuccessMessage := "Choosing " + tc.alternateImageName
+
+				pods, err := GetPodNamesByLabel(ctx, client, t, caaPod.Namespace, "app", "cloud-api-adaptor")
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				caaPod.Name = pods.Items[0].Name
+				LogString, err := ComparePodLogString(ctx, client, caaPod, expectedSuccessMessage)
+				if err != nil {
+					t.Logf("Output:%s", LogString)
+					t.Fatal(err)
+				}
+				t.Logf("PodVM was brought up using the alternate PodVM image %s", tc.alternateImageName)
+			}
 			return ctx
 		}).
 		Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {

src/cloud-api-adaptor/test/e2e/common_suite.go

@@ -406,6 +406,44 @@ func DoTestPodVMwithAnnotationsLargerCPU(t *testing.T, e env.Environment, assert
 	NewTestCase(t, e, "PodVMwithAnnotationsLargerCPU", assert, "Failed to Create PodVM with Annotations Larger CPU").WithPod(pod).WithInstanceTypes(testInstanceTypes).WithCustomPodState(v1.PodPending).Run()
 }
 
+func DoTestCreatePeerPodContainerWithValidAlternateImage(t *testing.T, e env.Environment, assert CloudAssert, alternateImageName string) {
+	podName := "annotations-valid-alternate-image"
+	containerName := "busybox"
+	imageName := BUSYBOX_IMAGE
+	annotationData := map[string]string{
+		"io.katacontainers.config.hypervisor.image": alternateImageName,
+	}
+	pod := NewPod(E2eNamespace, podName, containerName, imageName, WithCommand([]string{"/bin/sh", "-c", "sleep 3600"}), WithAnnotations(annotationData))
+
+	NewTestCase(t, e, "PodVMwithAnnotationsValidAlternateImage", assert, "PodVM created with an alternate image").WithPod(pod).WithAlternateImage(alternateImageName).Run()
+}
+
+func DoTestCreatePeerPodContainerWithInvalidAlternateImage(t *testing.T, e env.Environment, assert CloudAssert) {
+	podName := "annotations-invalid-alternate-image"
+	containerName := "busybox"
+	imageName := BUSYBOX_IMAGE
+	nonExistingImageName := "non-existing-image"
+	expectedErrorMessage := "Error in creating volume: Can't retrieve volume " + nonExistingImageName
+	annotationData := map[string]string{
+		"io.katacontainers.config.hypervisor.image": nonExistingImageName,
+	}
+	pod := NewPod(E2eNamespace, podName, containerName, imageName, WithCommand([]string{"/bin/sh", "-c", "sleep 3600"}), WithAnnotations(annotationData))
+
+	testInstanceTypes := InstanceValidatorFunctions{
+		testSuccessfn: IsStringEmpty,
+		testFailurefn: func(errorMsg error) bool {
+			if strings.Contains(errorMsg.Error(), expectedErrorMessage) {
+				t.Logf("Got Expected Error: %v", errorMsg.Error())
+				return true
+			} else {
+				t.Logf("Failed to Get Expected Error: %v", errorMsg.Error())
+				return false
+			}
+		},
+	}
+	NewTestCase(t, e, "PodVMwithAnnotationsInvalidAlternateImage", assert, "Failed to Create PodVM with a non-existent image").WithPod(pod).WithInstanceTypes(testInstanceTypes).WithCustomPodState(v1.PodPending).Run()
+}
+
 func DoTestPodToServiceCommunication(t *testing.T, e env.Environment, assert CloudAssert) {
 	clientPodName := "test-client"
 	clientContainerName := "busybox"

src/cloud-api-adaptor/test/e2e/libvirt_test.go

@@ -9,7 +9,7 @@ import (
 	"os"
 	"testing"
 
-	_ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/libvirt"
+	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/libvirt"
 )
 
 func TestLibvirtCreateSimplePod(t *testing.T) {
@@ -35,6 +35,16 @@ func TestLibvirtCreatePeerPodContainerWithExternalIPAccess(t *testing.T) {
 
 }
 
+func TestLibvirtCreatePeerPodContainerWithValidAlternateImage(t *testing.T) {
+	assert := LibvirtAssert{}
+	DoTestCreatePeerPodContainerWithValidAlternateImage(t, testEnv, assert, libvirt.AlternateVolumeName)
+}
+
+func TestLibvirtCreatePeerPodContainerWithInvalidAlternateImage(t *testing.T) {
+	assert := LibvirtAssert{}
+	DoTestCreatePeerPodContainerWithInvalidAlternateImage(t, testEnv, assert)
+}
+
 func TestLibvirtCreatePeerPodWithJob(t *testing.T) {
 	assert := LibvirtAssert{}
 	DoTestCreatePeerPodWithJob(t, testEnv, assert)

src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go

@@ -19,6 +19,8 @@ import (
 	"sigs.k8s.io/e2e-framework/pkg/envconf"
 )
 
+const AlternateVolumeName = "another-podvm-base.qcow2"
+
 // LibvirtProvisioner implements the CloudProvisioner interface for Libvirt.
 type LibvirtProvisioner struct {
 	conn         *libvirt.Connect // Libvirt connection
@@ -138,30 +140,35 @@ func (l *LibvirtProvisioner) CreateVPC(ctx context.Context, cfg *envconf.Config)
 		return fmt.Errorf("Storage pool '%s' not found. It should be created beforehand", l.storage)
 	}
 
-	// Create the podvm storage volume if it does not exist.
-	if _, err = sPool.LookupStorageVolByName(l.volumeName); err != nil {
-		volCfg := libvirtxml.StorageVolume{
-			Name: l.volumeName,
-			Capacity: &libvirtxml.StorageVolumeSize{
-				Unit:  "GiB",
-				Value: 20,
-			},
-			Allocation: &libvirtxml.StorageVolumeSize{
-				Unit:  "GiB",
-				Value: 2,
-			},
-			Target: &libvirtxml.StorageVolumeTarget{
-				Format: &libvirtxml.StorageVolumeTargetFormat{
-					Type: "qcow2",
+	// Create two volumes to test the multiple podvm image scenario.
+	lVolumes := [2]string{l.volumeName, AlternateVolumeName}
+
+	// Create the podvm storage volumes if it does not exist.
+	for _, volume := range lVolumes {
+		if _, err = sPool.LookupStorageVolByName(volume); err != nil {
+			volCfg := libvirtxml.StorageVolume{
+				Name: volume,
+				Capacity: &libvirtxml.StorageVolumeSize{
+					Unit:  "GiB",
+					Value: 20,
 				},
-			},
-		}
-		xml, err := volCfg.Marshal()
-		if err != nil {
-			return err
-		}
-		if _, err = sPool.StorageVolCreateXML(xml, libvirt.STORAGE_VOL_CREATE_PREALLOC_METADATA); err != nil {
-			return err
+				Allocation: &libvirtxml.StorageVolumeSize{
+					Unit:  "GiB",
+					Value: 2,
+				},
+				Target: &libvirtxml.StorageVolumeTarget{
+					Format: &libvirtxml.StorageVolumeTargetFormat{
+						Type: "qcow2",
+					},
+				},
+			}
+			xml, err := volCfg.Marshal()
+			if err != nil {
+				return err
+			}
+			if _, err = sPool.StorageVolCreateXML(xml, libvirt.STORAGE_VOL_CREATE_PREALLOC_METADATA); err != nil {
+				return err
+			}
 		}
 	}
 	return nil
@@ -211,52 +218,56 @@ func (l *LibvirtProvisioner) UploadPodvm(imagePath string, ctx context.Context,
 	}
 	length := fileStat.Size()
 
-	sVol, err := sPool.LookupStorageVolByName(l.volumeName)
-	if err != nil {
-		return err
-	}
+	lVolumes := [2]string{l.volumeName, AlternateVolumeName}
 
-	stream, err := l.conn.NewStream(0)
-	if err != nil {
-		return err
-	}
-
-	if err := sVol.Upload(stream, 0, uint64(length), libvirt.STORAGE_VOL_UPLOAD_SPARSE_STREAM); err != nil {
-		return err
-	}
+	for _, volume := range lVolumes {
+		sVol, err := sPool.LookupStorageVolByName(volume)
+		if err != nil {
+			return err
+		}
 
-	fileByteSlice, err := os.ReadFile(imagePath)
-	if err != nil {
-		return err
-	}
+		stream, err := l.conn.NewStream(0)
+		if err != nil {
+			return err
+		}
 
-	sent := 0
-	source := func(stream *libvirt.Stream, nbytes int) ([]byte, error) {
-		tosend := nbytes
-		if tosend > (len(fileByteSlice) - sent) {
-			tosend = len(fileByteSlice) - sent
+		if err := sVol.Upload(stream, 0, uint64(length), libvirt.STORAGE_VOL_UPLOAD_SPARSE_STREAM); err != nil {
+			return err
 		}
 
-		if tosend == 0 {
-			return []byte{}, nil
+		fileByteSlice, err := os.ReadFile(imagePath)
+		if err != nil {
+			return err
 		}
 
-		data := fileByteSlice[sent : sent+tosend]
-		sent += tosend
+		sent := 0
+		source := func(stream *libvirt.Stream, nbytes int) ([]byte, error) {
+			tosend := nbytes
+			if tosend > (len(fileByteSlice) - sent) {
+				tosend = len(fileByteSlice) - sent
+			}
 
-		return data, nil
-	}
+			if tosend == 0 {
+				return []byte{}, nil
+			}
 
-	if err := stream.SendAll(source); err != nil {
-		return err
-	}
+			data := fileByteSlice[sent : sent+tosend]
+			sent += tosend
 
-	if err := stream.Finish(); err != nil {
-		return err
-	}
+			return data, nil
+		}
 
-	if err := stream.Free(); err != nil {
-		return err
+		if err := stream.SendAll(source); err != nil {
+			return err
+		}
+
+		if err := stream.Finish(); err != nil {
+			return err
+		}
+
+		if err := stream.Free(); err != nil {
+			return err
+		}
 	}
 
 	return nil

src/cloud-providers/libvirt/provider.go

@@ -79,6 +79,14 @@ func (p *libvirtProvider) CreateInstance(ctx context.Context, podName, sandboxID
 	}
 	logger.Printf("LaunchSecurityType: %s", vm.launchSecurityType.String())
 
+	if spec.Image != "" {
+		logger.Printf("Choosing %s as libvirt volume for the PodVM image", spec.Image)
+		p.libvirtClient.volName = spec.Image
+	} else if spec.Image == "" && p.serviceConfig.VolName != p.libvirtClient.volName {
+		logger.Printf("Choosing the default %s as libvirt volume for the PodVM image", p.serviceConfig.VolName)
+		p.libvirtClient.volName = p.serviceConfig.VolName
+	}
+
 	result, err := CreateDomain(ctx, p.libvirtClient, vm)
 	if err != nil {
 		logger.Printf("failed to create an instance : %v", err)

src/cloud-providers/types.go

@@ -66,4 +66,5 @@ type InstanceTypeSpec struct {
 	Memory       int64
 	Arch         string
 	GPUs         int64
+	Image        string
 }