Merge pull request #404 from bpradipt/pvm-img-fix

Pod VM image handling fix

config/peerpods/podvm/README.md

@@ -1,46 +1,36 @@
 # Introduction
 
-This is a brief readme explaining the usage of the podvm-builder scripts and related files
+This is a brief readme explaining the usage of the podvm-builder scripts and
+related files.  The scripts and related manifest files are primarily used by
+the operator to generate a pod VM image.
 
-## Create PodVM image generation configuration
+## PodVM image generation configuration
 
 The configuration used for the podvm image generation is available in the following configmaps:
 
 - Azure: `azure-podvm-image-cm`
 - AWS: `aws-podvm-image-cm`
 
-Depending on the cloud provider (eg. aws or azure) create the respective
-configmaps. Please review and modify the settings in the configMap as required.
+If you want to change the default configuration, then depending on the cloud
+provider (eg. aws or azure) you'll need to pre-create the respective
+configmaps.  Please review and modify the settings in the configMap as
+required.  For example, if you need to add NVIDIA GPU drivers in the podvm
+image then set `ENABLE_NVIDIA_GPU: yes`. Likewise if you want to create image
+for confidential containers then set `CONFIDENTIAL_COMPUTE_ENABLED: yes`.
 
-For AWS
+Use the following command to create the configMap for AWS:
 
 ```sh
 kubectl apply -f aws-podvm-image-cm.yaml
 ```
 
-For Azure
+Use the following command to create the configMap for Azure:
 
 ```sh
 kubectl apply -f azure-podvm-image-cm.yaml
 ```
 
-## Create podvm image
-
-The podvm image is created in a Kubernetes job. To create the job run the following command
-
-```sh
-kubectl apply -f osc-podvm-create-job.yaml
-```
-
-On successful image creation, the podvm image details will be updated as an annotation in the `peer-pods-cm`
-under `openshift-sandboxed-containers-operator` namespace.
-
-The annotation key for AWS is `LATEST_AMI_ID` and for Azure it's `LATEST_IMAGE_ID`
-
-## Delete podvm image
-
-Update the IMAGE_ID for Azure or AMI_ID for AWS that you want to delete and then run the following command
-
-```sh
-kubectl delete -f osc-podvm-delete-job.yaml
-```
+Now when you create a KataConfig with `enablePeerPods: true` with empty
+`AZURE_IMAGE_ID` or `AWS_AMI_ID` in `peer-pods-cm`, then depending on the cloud
+provider configured, the operator will create the pod VM image based on the
+provided config.

config/peerpods/podvm/azure-podvm-image-cm.yaml

@@ -8,7 +8,13 @@ data:
   PODVM_DISTRO: rhel
 
   # Gallery
-  IMAGE_GALLERY_NAME: "PodVMGallery"
+  # Set the gallery name explicitly, otherwise it'll be set by the operator to
+  # PodVMGallery_${cluster-id}. If a gallery name is provided, the job will
+  # use the specific gallery name.
+  # Note that the gallery name must be unique across the subscription and not exceed 80 characters.
+  # Also, the allowed characters are English alphanumeric characters,
+  # with underscores and periods allowed in the middle
+  IMAGE_GALLERY_NAME: ""
 
   # Image definition
   IMAGE_DEFINITION_NAME: "podvm-image"

config/peerpods/podvm/azure-podvm-image-handler.sh

@@ -47,7 +47,9 @@ function verify_vars() {
     # Ensure that the image defintion variables are set
     [[ -z "${IMAGE_DEFINITION_PUBLISHER}" ]] && error_exit "IMAGE_DEFINITION_PUBLISHER is empty"
     [[ -z "${IMAGE_DEFINITION_OFFER}" ]] && error_exit "IMAGE_DEFINITION_OFFER is empty"
+
     [[ -z "${IMAGE_GALLERY_NAME}" ]] && error_exit "IMAGE_GALLERY_NAME is empty"
+
     [[ -z "${IMAGE_DEFINITION_SKU}" ]] && error_exit "IMAGE_DEFINITION_SKU is empty"
     [[ -z "${IMAGE_DEFINITION_OS_TYPE}" ]] && error_exit "IMAGE_DEFINITION_OS_TYPE is empty"
     [[ -z "${IMAGE_DEFINITION_OS_STATE}" ]] && error_exit "IMAGE_DEFINITION_OS_STATE is empty"
@@ -155,10 +157,15 @@ function create_image_gallery() {
     # If any error occurs, exit the script with an error message
 
     # Create the image gallery
+    echo "Creating image gallery ${IMAGE_GALLERY_NAME}"
+
     az sig create --resource-group "${AZURE_RESOURCE_GROUP}" \
         --gallery-name "${IMAGE_GALLERY_NAME}" ||
         error_exit "Failed to create Azure image gallery"
 
+    # Update peer-pods-cm configmap with the gallery name
+    add_image_gallery_annotation_to_peer_pods_cm
+
     echo "Azure image gallery created successfully"
 
 }
@@ -361,15 +368,6 @@ function create_or_update_image_configmap() {
         IMAGE_ID_LIST="${IMAGE_ID}"
     fi
 
-    # Create or update the value of the azure key in podvm-images configmap with all the images
-    # If any error occurs, exit the script with an error message
-    kubectl create configmap podvm-images \
-        -n openshift-sandboxed-containers-operator \
-        --from-literal=azure="${IMAGE_ID_LIST}" \
-        --dry-run=client -o yaml |
-        kubectl apply -f - ||
-        error_exit "Failed to create or update podvm-images configmap"
-
     echo "podvm-images configmap created or updated successfully"
 }
 
@@ -397,7 +395,7 @@ function recreate_image_configmap() {
 # Function to add the image id as annotation in the peer-pods-cm configmap
 
 function add_image_id_annotation_to_peer_pods_cm() {
-    echo "Adding image id to peer-pods-cm configmap"
+    echo "Adding image id annotation to peer-pods-cm configmap"
 
     # Check if the peer-pods-cm configmap exists
     if ! kubectl get configmap peer-pods-cm -n openshift-sandboxed-containers-operator >/dev/null 2>&1; then
@@ -413,6 +411,63 @@ function add_image_id_annotation_to_peer_pods_cm() {
     echo "Image id added as annotation to peer-pods-cm configmap successfully"
 }
 
+# Function to delete the LATEST_IMAGE_ID annotation from the peer-pods-cm configmap
+
+function delete_image_id_annotation_from_peer_pods_cm() {
+    echo "Deleting image id annotation from peer-pods-cm configmap"
+
+    # Check if the peer-pods-cm configmap exists
+    if ! kubectl get configmap peer-pods-cm -n openshift-sandboxed-containers-operator >/dev/null 2>&1; then
+        echo "peer-pods-cm configmap does not exist. Skipping deleting the image id"
+        return
+    fi
+
+    # Delete the image id annotation from peer-pods-cm configmap
+    kubectl annotate configmap peer-pods-cm -n openshift-sandboxed-containers-operator \
+        "LATEST_IMAGE_ID-" ||
+        error_exit "Failed to delete the image id annotation from peer-pods-cm configmap"
+
+    echo "Image id annotation deleted from peer-pods-cm configmap successfully"
+}
+
+# Function to add image gallery annotation to peer-pods-cm configmap
+
+function add_image_gallery_annotation_to_peer_pods_cm() {
+    echo "Adding IMAGE_GALLERY_NAME annotation to peer-pods-cm configmap"
+
+    # Check if the peer-pods-cm configmap exists
+    if ! kubectl get configmap peer-pods-cm -n openshift-sandboxed-containers-operator >/dev/null 2>&1; then
+        echo "peer-pods-cm configmap does not exist. Skipping adding the IMAGE_GALLERY_NAME annotation"
+        return
+    fi
+
+    # Add IMAGE_GALLERY_NAME annotation to peer-pods-cm configmap
+    kubectl annotate configmap peer-pods-cm -n openshift-sandboxed-containers-operator \
+        "IMAGE_GALLERY_NAME=${IMAGE_GALLERY_NAME}" ||
+        error_exit "Failed to add the IMAGE_GALLERY_NAME annotation to peer-pods-cm configmap"
+
+    echo "IMAGE_GALLERY_NAME annotation added to peer-pods-cm configmap successfully"
+}
+
+# Function to delete the image gallery annotation from peer-pods-cm configmap
+
+function delete_image_gallery_annotation_from_peer_pods_cm() {
+    echo "Deleting IMAGE_GALLERY_NAME annotation from peer-pods-cm configmap"
+
+    # Check if the peer-pods-cm configmap exists
+    if ! kubectl get configmap peer-pods-cm -n openshift-sandboxed-containers-operator >/dev/null 2>&1; then
+        echo "peer-pods-cm configmap does not exist. Skipping deleting the IMAGE_GALLERY_NAME annotation"
+        return
+    fi
+
+    # Delete the IMAGE_GALLERY_NAME annotation from peer-pods-cm configmap
+    kubectl annotate configmap peer-pods-cm -n openshift-sandboxed-containers-operator \
+        "IMAGE_GALLERY_NAME-" ||
+        error_exit "Failed to delete the IMAGE_GALLERY_NAME annotation from peer-pods-cm configmap"
+
+    echo "IMAGE_GALLERY_NAME annotation deleted from peer-pods-cm configmap successfully"
+}
+
 # Function to create the image in Azure
 # It's assumed you have already logged in to Azure
 # It's assumed that the gallery and image defintion exists
@@ -534,6 +589,7 @@ function delete_image_definition() {
 
 # Function to delete the image gallery from Azure
 # Accept force argument to delete the gallery even if image versions exist
+# IMAGE_GALLERY_NAME is assumed to be populated
 
 function delete_image_gallery() {
     echo "Deleting Azure image gallery"
@@ -553,6 +609,7 @@ function delete_image_gallery() {
     fi
 
     # Check if the gallery has any image versions
+    # This will set the IMAGE_ID_LIST variable
     get_all_image_ids
 
     # If the gallery has image versions, then skip deleting the gallery if "force" option is not passed
@@ -574,28 +631,15 @@ function delete_image_gallery() {
         --gallery-name "${IMAGE_GALLERY_NAME}" ||
         error_exit "Failed to delete the image gallery"
 
-    echo "Azure image gallery deleted successfully"
-}
-
-# Function to delete the image from Azure given the image name
-# Resource group is must
-# Input is of the form /subscriptions/<subscription-id>/resourceGroups/<resource-group>/providers/Microsoft.Compute/images/<image-name>
-
-function delete_image_using_name() {
-    echo "Deleting Azure image"
-    # If any error occurs, exit the script with an error message
-
-    # Delete the image
-    az image delete --resource-group "${AZURE_RESOURCE_GROUP}" \
-        --name "${IMAGE_NAME}" ||
-        error_exit "Failed to delete the image"
+    # Remove the image gallery annotation from peer-pods-cm configmap
+    delete_image_gallery_annotation_from_peer_pods_cm
 
-    echo "Azure image deleted successfully"
+    echo "Azure image gallery deleted successfully"
 }
 
 # Function to delete the image from Azure given the image id
-# Input is of the form /subscriptions/<subscription-id>/resourceGroups/<resource-group>/providers/Microsoft.Compute/images/<image-name>
-# or /subscriptions/<subscription-id>/resourceGroups/<resource-group>/providers/Microsoft.Compute/galleries/<gallery-name>/images/<image-name>/versions/<image-version>
+# Input is of the form
+# /subscriptions/<subscription-id>/resourceGroups/<resource-group>/providers/Microsoft.Compute/galleries/<gallery-name>/images/<image-name>/versions/<image-version>
 
 function delete_image_using_id() {
     echo "Deleting Azure image"
@@ -605,9 +649,12 @@ function delete_image_using_id() {
     [[ -z "${IMAGE_ID}" ]] && error_exit "IMAGE_ID is empty"
 
     # Delete the image
-    az image delete --ids "${IMAGE_ID}" ||
+    az sig image-version delete --ids "${IMAGE_ID}" ||
         error_exit "Failed to delete the image"
 
+    # Remove the image id annotation from peer-pods-cm configmap
+    delete_image_id_annotation_from_peer_pods_cm
+
     echo "Azure image deleted successfully"
 }
 

config/peerpods/podvm/podvm-builder.sh

@@ -26,8 +26,7 @@ function check_peer_pods_cm_exists() {
   fi
 }
 
-# function to create podvm image
-
+# Function to create podvm image
 function create_podvm_image() {
   case "${CLOUD_PROVIDER}" in
   azure)
@@ -87,9 +86,23 @@ function create_podvm_image() {
 # Function to delete podvm image
 # IMAGE_ID or AMI_ID is the input and expected to be set
 # These are checked in individual cloud provider scripts and if not set, the script will exit
+# Accepts two optional arguments
+# -f : force delete the image
+# -g : delete the image gallery
 
 function delete_podvm_image() {
 
+  local args=("$@")
+  local force=false
+  local delete_gallery=false
+
+  for ((i = 0; i < ${#args[@]}; i++)); do
+    case "${args[$i]}" in
+    -f) force=true ;;
+    -g) delete_gallery=true ;;
+    esac
+  done
+
   # Check for the existence of peer-pods-cm configmap. If not present, then exit
   if ! check_peer_pods_cm_exists; then
     echo "peer-pods-cm configmap does not exist. Skipping image deletion"
@@ -115,20 +128,26 @@ function delete_podvm_image() {
     # check if the AZURE_IMAGE_ID value in peer-pods-cm is same as the input IMAGE_ID
     # If yes, then don't delete the image unless force option is provided
     if [ "${AZURE_IMAGE_ID}" == "${IMAGE_ID}" ]; then
-      if [ "$1" != "-f" ]; then
+      if ! ${force}; then
         echo "AZURE_IMAGE_ID in peer-pods-cm is same as the input image to be deleted. Skipping the deletion of Azure image"
         exit 0
       fi
     fi
 
-    echo "Deleting Azure image"
+    echo "Deleting Azure image $IMAGE_ID"
     /scripts/azure-podvm-image-handler.sh -C
 
     # Update the peer-pods-cm configmap and remove the AZURE_IMAGE_ID value
     if [ "${UPDATE_PEERPODS_CM}" == "yes" ]; then
       kubectl patch configmap peer-pods-cm -n openshift-sandboxed-containers-operator --type merge -p "{\"data\":{\"AZURE_IMAGE_ID\":\"\"}}"
     fi
 
+    # If delete_gallery is set, then delete the image gallery
+    if ${delete_gallery}; then
+      echo "Deleting Azure image gallery (by force) since -g option is set"
+      delete_podvm_image_gallery -f
+    fi
+
     ;;
   aws)
     # If AMI_ID is not set, then exit
@@ -171,6 +190,8 @@ function delete_podvm_image() {
 }
 
 # Delete the podvm image gallery in Azure
+# It accepts an optional argument
+# -f : force delete the image gallery
 
 function delete_podvm_image_gallery() {
   echo "Deleting Azure image gallery"
@@ -180,7 +201,21 @@ function delete_podvm_image_gallery() {
     return
   fi
 
-  # Check if force option is passed
+  # Check if peer-pods-cm configmap exists
+  if ! check_peer_pods_cm_exists; then
+    echo "peer-pods-cm configmap does not exist. Skipping image gallery deletion"
+    exit 0
+  fi
+
+  # Get the IMAGE_GALLERY_NAME from the IMAGE_GALLERY_NAME annotation key in peer-pods-cm configmap
+  IMAGE_GALLERY_NAME=$(kubectl get configmap peer-pods-cm -n openshift-sandboxed-containers-operator -o jsonpath='{.metadata.annotations.IMAGE_GALLERY_NAME}')
+
+  # If IMAGE_GALLERY_NAME is not set, then exit
+  if [ -z "${IMAGE_GALLERY_NAME}" ]; then
+    echo "IMAGE_GALLERY_NAME is not set in peer-pods-cm. Skipping image gallery deletion"
+    exit 0
+  fi
+
   if [ "$1" == "-f" ]; then
     /scripts/azure-podvm-image-handler.sh -G force
   else
@@ -189,7 +224,7 @@ function delete_podvm_image_gallery() {
 }
 
 function display_usage() {
-  echo "Usage: $0 {create|delete [-f]|delete-gallery [-f]}"
+  echo "Usage: $0 {create|delete [-f] [-g]|delete-gallery [-f]}"
 }
 
 # Check if CLOUD_PROVIDER is set to azure or aws
@@ -214,7 +249,9 @@ create)
   create_podvm_image
   ;;
 delete)
-  delete_podvm_image "$2"
+  # Pass the arguments to delete_podvm_image function except the first argument
+  shift
+  delete_podvm_image "$@"
   ;;
 delete-gallery)
   delete_podvm_image_gallery "$2"