confidential-containers · bpradipt · Sep 9, 2024 · Sep 6, 2024 · Sep 6, 2024 · Sep 6, 2024
@@ -74,7 +74,6 @@ jobs:
       matrix:
         controller:
           - peerpod-ctrl
-          - peerpodconfig-ctrl
     defaults:
       run:
         working-directory: src/${{ matrix.controller }}

@@ -138,9 +138,6 @@ jobs:
           echo "::group::cloud-api-adaptor logs"
           kubectl logs -l app=cloud-api-adaptor -n confidential-containers-system
           echo "::endgroup::"
-          echo "::group::peerpodconfig-ctrl-caa-daemon logs"
-          kubectl logs ds/peerpodconfig-ctrl-caa-daemon -n confidential-containers-system
-          echo "::endgroup::"
 
           for pod in $(kubectl get pods -o name 2>/dev/null); do
             echo "::group::Describe $pod"

@@ -31,14 +31,7 @@ jobs:
     with:
       image_tags: latest,${{ github.sha }}
       git_ref: ${{ github.sha }}
-    secrets: inherit
-
-  peerpodconfig-ctrl:
-    uses: ./.github/workflows/peerpodconfig-ctrl_image.yaml
-    with:
-      image_tags: ${{ github.sha }},latest
-      git_ref: ${{ github.sha }}
-    secrets: inherit
+    secrets: inherit  
 
   webhook:
     uses: ./.github/workflows/webhook_image.yaml

@@ -31,13 +31,6 @@ jobs:
       git_ref: ${{ github.ref }}
     secrets: inherit
 
-  peerpodconfig-ctrl:
-    uses: ./.github/workflows/peerpodconfig-ctrl_image.yaml
-    with:
-      image_tags: ${{ github.event.release.tag_name }}
-      git_ref: ${{ github.ref }}
-    secrets: inherit
-
   podvm_builder:
     uses: ./.github/workflows/podvm_builder.yaml
     with:

@@ -26,9 +26,6 @@ The background and description of the components involved in 'peer pods' can be
 ## Cloud API Adaptor
 [cloud-api-adaptor](./src/cloud-api-adaptor/) Ability to create Kata pods using cloud provider APIs aka the peer-pods approach
 
-## PeerPodConfig controller
-[peerpodconfig-ctrl](./src/peerpodconfig-ctrl/) PeerPodConfig controller is watching the PeerPodConfig CRD object and manages the creation and deletion lifecycle of all required components to run peer pods.
-
 ## CSI Wrapper
 [csi-wrapper](./src/csi-wrapper/) CSI Wrapper solution for Peer Pod Storage
 

@@ -101,8 +101,6 @@ git tag src/csi-wrapper/v0.8.0-alpha.1 main
 git push upstream src/csi-wrapper/v0.8.0-alpha.1
 git tag src/peerpod-ctrl/v0.8.0-alpha.1 main
 git push upstream src/peerpod-ctrl/v0.8.0-alpha.1
-git tag src/peerpodconfig-ctrl/v0.8.0-alpha.1 main
-git push upstream src/peerpodconfig-ctrl/v0.8.0-alpha.1
 git tag src/webhook/v0.8.0-alpha.1 main
 git push upstream src/webhook/v0.8.0-alpha.1
 ```
@@ -121,9 +119,6 @@ Total 0 (delta 0), reused 0 (delta 0), pack-reused 0 (from 0)
 To github.com:confidential-containers/cloud-api-adaptor.git
  * [new tag]         src/peerpod-ctrl/v0.8.0-alpha.1 -> src/peerpod-ctrl/v0.8.0-alpha.1
 Total 0 (delta 0), reused 0 (delta 0), pack-reused 0 (from 0)
-To github.com:confidential-containers/cloud-api-adaptor.git
- * [new tag]         src/peerpodconfig-ctrl/v0.8.0-alpha.1 -> src/peerpodconfig-ctrl/v0.8.0-alpha.1
-Total 0 (delta 0), reused 0 (delta 0), pack-reused 0 (from 0)
 To github.com:confidential-containers/cloud-api-adaptor.git
  * [new tag]         src/webhook/v0.8.0-alpha.1 -> src/webhook/v0.8.0-alpha.1
 ```
@@ -183,8 +178,6 @@ git tag src/csi-wrapper/v0.8.0 main
 git push upstream src/csi-wrapper/v0.8.0
 git tag src/peerpod-ctrl/v0.8.0 main
 git push upstream src/peerpod-ctrl/v0.8.0
-git tag src/peerpodconfig-ctrl/v0.8.0 main
-git push upstream src/peerpodconfig-ctrl/v0.8.0
 git tag src/webhook/v0.8.0 main
 git push upstream src/webhook/v0.8.0
 ```
@@ -202,10 +195,7 @@ To github.com:confidential-containers/cloud-api-adaptor.git
 Total 0 (delta 0), reused 0 (delta 0), pack-reused 0 (from 0)
 To github.com:confidential-containers/cloud-api-adaptor.git
  * [new tag]         src/peerpod-ctrl/v0.8.0 -> src/peerpod-ctrl/v0.8.0
-Total 0 (delta 0), reused 0 (delta 0), pack-reused 0 (from 0)
-To github.com:confidential-containers/cloud-api-adaptor.git
- * [new tag]         src/peerpodconfig-ctrl/v0.8.0 -> src/peerpodconfig-ctrl/v0.8.0
-Total 0 (delta 0), reused 0 (delta 0), pack-reused 0 (from 0)
+Total 0 (delta 0), reused 0 (delta 0), pack-reused 0 (from 0) 
 To github.com:confidential-containers/cloud-api-adaptor.git
  * [new tag]         src/webhook/v0.8.0 -> src/webhook/v0.8.0
 ```

@@ -118,11 +118,6 @@ The [webhook](../src/webhook/) is an mutating admission controller that modifies
 remove all resources entries and replace it with peer-pod extended resource. This is needed as unlike a standard pod, a
 peer pod will not consume the worker node's resources in the same way.
 
-### peerpodconfig-ctrl
-
-The [peerpodconfig-ctrl](../src/peerpodconfig-ctrl/) is a Kubernetes controller that manages the creation and deletion of
-components required to run peer pods.
-
 ### peerpod-ctrl
 
 The [peerpod-ctrl](../src/peerpod-ctrl/) is a Kubernetes controller that is used to track the cloud provider resources for

@@ -11,7 +11,6 @@ install/
 ibmcloud/
 libvirt/
 peerpod-ctrl/
-peerpodconfig-ctrl/
 volumes/
 webwook/
 

@@ -123,6 +123,7 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 		flags.IntVar(&cfg.networkConfig.VXLANMinID, "vxlan-min-id", vxlan.DefaultVXLANMinID, "Minimum VXLAN ID (VXLAN tunnel mode only")
 		flags.StringVar(&cfg.serverConfig.Initdata, "initdata", "", "Default initdata for all Pods")
 		flags.BoolVar(&cfg.serverConfig.EnableCloudConfigVerify, "cloud-config-verify", false, "Enable cloud config verify - should use it for production")
+		flags.IntVar(&cfg.serverConfig.PeerPodsLimitPerNode, "peerpods-limit-per-node", 10, "peer pods limit per node (default=10)")
 
 		cloud.ParseCmd(flags)
 	})

@@ -25,6 +25,7 @@ optionals+=""
 [[ "${SECURE_COMMS_INBOUNDS}" ]] && optionals+="-secure-comms-inbounds ${SECURE_COMMS_INBOUNDS} "
 [[ "${SECURE_COMMS_OUTBOUNDS}" ]] && optionals+="-secure-comms-outbounds ${SECURE_COMMS_OUTBOUNDS} "
 [[ "${SECURE_COMMS_KBS_ADDR}" ]] && optionals+="-secure-comms-kbs ${SECURE_COMMS_KBS_ADDR} "
+[[ "${PEERPODS_LIMIT_PER_NODE}" ]] && optionals+="-peerpods-limit-per-node ${PEERPODS_LIMIT_PER_NODE} "
 
 test_vars() {
     for i in "$@"; do
@@ -86,18 +87,18 @@ azure() {
 }
 
 gcp() {
-test_vars GCP_CREDENTIALS GCP_PROJECT_ID GCP_ZONE PODVM_IMAGE_NAME
-
-[[ "${PODVM_IMAGE_NAME}" ]] && optionals+="-gcp-image-name ${PODVM_IMAGE_NAME} "
-[[ "${GCP_PROJECT_ID}" ]] && optionals+="-gcp-project-id ${GCP_PROJECT_ID} "
-[[ "${GCP_ZONE}" ]] && optionals+="-gcp-zone ${GCP_ZONE} " # if not set retrieved from IMDS
-[[ "${GCP_MACHINE_TYPE}" ]] && optionals+="-gcp-machine-type ${GCP_MACHINE_TYPE} " # default e2-medium
-[[ "${GCP_NETWORK}" ]] && optionals+="-gcp-network ${GCP_NETWORK} " # defaults to 'default'
-
-set -x
-exec cloud-api-adaptor gcp \
-	-pods-dir /run/peerpod/pods \
-	${optionals}
+    test_vars GCP_CREDENTIALS GCP_PROJECT_ID GCP_ZONE PODVM_IMAGE_NAME
+
+    [[ "${PODVM_IMAGE_NAME}" ]] && optionals+="-gcp-image-name ${PODVM_IMAGE_NAME} "
+    [[ "${GCP_PROJECT_ID}" ]] && optionals+="-gcp-project-id ${GCP_PROJECT_ID} "
+    [[ "${GCP_ZONE}" ]] && optionals+="-gcp-zone ${GCP_ZONE} "                         # if not set retrieved from IMDS
+    [[ "${GCP_MACHINE_TYPE}" ]] && optionals+="-gcp-machine-type ${GCP_MACHINE_TYPE} " # default e2-medium
+    [[ "${GCP_NETWORK}" ]] && optionals+="-gcp-network ${GCP_NETWORK} "                # defaults to 'default'
+
+    set -x
+    exec cloud-api-adaptor gcp \
+        -pods-dir /run/peerpod/pods \
+        ${optionals}
 }
 
 ibmcloud() {

@@ -34,6 +34,7 @@ configMapGenerator:
   #- USE_PUBLIC_IP="true" # Uncomment if you want to use public ip for podvm
   #- ROOT_VOLUME_SIZE="30" # Uncomment and set if you want to use a specific root volume size. Defaults to 30
   #- FORWARDER_PORT="" # Uncomment and set if you want to use a specific port for agent-protocol-forwarder. Defaults to 15150
+  #- PEERPODS_LIMIT_PER_NODE="10" # Max number of peer pods that can be created per node. Default is 10
 ##TLS_SETTINGS
   #- CACERT_FILE="/etc/certificates/ca.crt" # for TLS
   #- CERT_FILE="/etc/certificates/client.crt" # for TLS

@@ -40,6 +40,7 @@ configMapGenerator:
   #- AZURE_INSTANCE_SIZES="" # comma separated
   #- TAGS="" # Uncomment and add key1=value1,key2=value2 etc if you want to use specific tags for podvm
   #- FORWARDER_PORT="" # Uncomment and set if you want to use a specific port for agent-protocol-forwarder. Defaults to 15150
+  #- PEERPODS_LIMIT_PER_NODE="10" # Max number of peer pods that can be created per node. Default is 10
 ##TLS_SETTINGS
   #- CACERT_FILE="/etc/certificates/ca.crt" # for TLS
   #- CERT_FILE="/etc/certificates/client.crt" # for TLS

@@ -26,6 +26,7 @@ configMapGenerator:
     #- DOCKER_NETWORK_NAME="bridge" # Uncomment and set if you want to use a specific docker network
     #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image
     #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789
+    #- PEERPODS_LIMIT_PER_NODE="10" # Max number of peer pods that can be created per node. Default is 10
 ##TLS_SETTINGS
 #- CACERT_FILE="/etc/certificates/ca.crt" # for TLS
 #- CERT_FILE="/etc/certificates/client.crt" # for TLS

@@ -34,6 +34,7 @@ configMapGenerator:
   #- PROXY_TIMEOUT="" # Uncomment and set if you want to pass a specific timeout. Defaults to 5m
   #- USE_PUBLIC_IP="true" # Uncomment if you want to use public ip for podvm
   #- FORWARDER_PORT="" # Uncomment and set if you want to use a specific port for agent-protocol-forwarder. Defaults to 15150
+  #- PEERPODS_LIMIT_PER_NODE="10" # Max number of peer pods that can be created per node. Default is 10
 ##TLS_SETTINGS
   #- CACERT_FILE="/etc/certificates/ca.crt" # for TLS
   #- CERT_FILE="/etc/certificates/client.crt" # for TLS

@@ -32,6 +32,7 @@ configMapGenerator:
   - CRI_RUNTIME_ENDPOINT="/run/cri-runtime/containerd.sock"
   #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image
   #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789
+  #- PEERPODS_LIMIT_PER_NODE="10" # Max number of peer pods that can be created per node. Default is 10
 ##TLS_SETTINGS
   #- CACERT_FILE="/etc/certificates/ca.crt" # for TLS
   #- CERT_FILE="/etc/certificates/client.crt" # for TLS

@@ -30,6 +30,7 @@ configMapGenerator:
   #- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2
   #- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image
   #- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789
+  #- PEERPODS_LIMIT_PER_NODE="10" # Max number of peer pods that can be created per node. Default is 10
 ##TLS_SETTINGS
   #- CACERT_FILE="/etc/certificates/ca.crt" # for TLS
   #- CERT_FILE="/etc/certificates/client.crt" # for TLS

@@ -46,6 +46,7 @@ configMapGenerator:
   #- PAUSE_IMAGE=""    # Uncomment and set if you want to use a specific pause image
   #- VXLAN_PORT=""     # Uncomment and set to use "9000" or change if you want to use a specific vxlan port.
                        # Defaults to 4789.
+  #- PEERPODS_LIMIT_PER_NODE="10" # Max number of peer pods that can be created per node. Default is 10
 ##TLS_SETTINGS
   #- CACERT_FILE="/etc/certificates/ca.crt" # for TLS
   #- CERT_FILE="/etc/certificates/client.crt" # for TLS

@@ -73,6 +73,28 @@ roleRef:
   apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: node-status-patcher
+rules:
+- apiGroups: [""]
+  resources: ["nodes/status"]
+  verbs: ["patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: node-status-patcher
+subjects:
+- kind: ServiceAccount
+  name: cloud-api-adaptor
+  namespace: confidential-containers-system
+roleRef:
+  kind: ClusterRole
+  name: node-status-patcher
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   name: pp-secrets

@@ -87,7 +87,8 @@ func (s *cloudService) removeSandbox(id sandboxID) error {
 }
 
 func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNode podnetwork.WorkerNode,
-	secureComms bool, secureCommsInbounds, secureCommsOutbounds, kbsAddress, podsDir, daemonPort, initdata, sshport string) Service {
+	secureComms bool, secureCommsInbounds, secureCommsOutbounds, kbsAddress, podsDir,
+	daemonPort, initdata, sshport string) Service {
 	var err error
 	var sshClient *wnssh.SshClient