Issue #3935 - Feature Request: Enable cluster agent install to use re…

…mote image registry Signed-off-by: Le Zhang <zhangl@us.ibm.com>
open-horizon · Nov 10, 2023 · c6aa40a · c6aa40a
1 parent d3e88b7
commit c6aa40a
Show file tree

Hide file tree

Showing 6 changed files with 322 additions and 106 deletions.
diff --git a/agent-install/agent-install.sh b/agent-install/agent-install.sh
diff --git a/agent-install/agent-uninstall.sh b/agent-install/agent-uninstall.sh
@@ -8,6 +8,7 @@ DEPLOYMENT_NAME="agent"
 SERVICE_ACCOUNT_NAME="agent-service-account"
 CLUSTER_ROLE_BINDING_NAME="openhorizon-agent-cluster-rule"
 SECRET_NAME="openhorizon-agent-secrets"
+IMAGE_PULL_SECRET_NAME="registry-creds"
 IMAGE_REGISTRY_SECRET_NAME="openhorizon-agent-secrets-docker-cert"
 CONFIGMAP_NAME="openhorizon-agent-config"
 PVC_NAME="openhorizon-agent-pvc"
@@ -362,6 +363,7 @@ function deleteAgentResources() {
     log_info "Deleting secret..."
     $KUBECTL delete secret $SECRET_NAME -n $AGENT_NAMESPACE
     $KUBECTL delete secret $IMAGE_REGISTRY_SECRET_NAME -n $AGENT_NAMESPACE
+    $KUBECTL delete secret $IMAGE_PULL_SECRET_NAME -n $AGENT_NAMESPACE
     $KUBECTL delete secret ${SECRET_NAME}-backup -n $AGENT_NAMESPACE
     set -e
 

diff --git a/agent-install/k8s/auto-upgrade-cronjob-template.yml b/agent-install/k8s/auto-upgrade-cronjob-template.yml
@@ -2,6 +2,9 @@ apiVersion: __KubernetesApi__
 kind: CronJob
 metadata:
   name: auto-upgrade-cronjob
+  labels:
+    app: agent
+    openhorizon.org/component: agent
 spec:
   schedule: '*/1 * * * *'
   concurrencyPolicy: Forbid
@@ -11,11 +14,19 @@ spec:
     spec:
       backoffLimit: 0
       template:
+        metadata:
+          labels:
+            app: agent
+            openhorizon.org/component: agent
         spec:
           volumes:
           - name: agent-pvc-storage
             persistentVolumeClaim:
               claimName: openhorizon-agent-pvc
+          # START_REMOTE_ICR
+          imagePullSecrets:
+          - name: registry-creds
+          # END_REMOTE_ICR
           containers:
           - name: agent-auto-upgrade
             securityContext:
@@ -33,7 +44,7 @@ spec:
               - '-c'
               - >-
                 /usr/local/bin/auto-upgrade-cronjob.sh
-            imagePullPolicy: Always
+            imagePullPolicy: IfNotPresent
             volumeMounts:
             - mountPath: /var/horizon
               name: agent-pvc-storage

diff --git a/agent-install/k8s/deployment-template.yml b/agent-install/k8s/deployment-template.yml
@@ -3,15 +3,20 @@ kind: Deployment
 metadata:
   name: agent
   namespace: __AgentNameSpace__
+  labels:
+    app: agent
+    openhorizon.org/component: agent
 spec:
   replicas: 1
   selector:
     matchLabels:
       app: agent
+      openhorizon.org/component: agent
   template:
     metadata:
       labels:
         app: agent
+        openhorizon.org/component: agent
     spec:
       serviceAccountName: agent-service-account
       volumes:
@@ -29,7 +34,8 @@ spec:
       # START_NOT_FOR_OCP
       initContainers:
       - name: initcontainer
-        image: alpine:latest
+        image: __InitContainerImagePath__
+        imagePullPolicy: IfNotPresent
         securityContext:
           runAsNonRoot: false
         command:
@@ -44,10 +50,14 @@ spec:
           name: agent-pvc-storage
           subPath: horizon
       # END_NOT_FOR_OCP
+      # START_REMOTE_ICR
+      imagePullSecrets:
+      - name: registry-creds
+      # END_REMOTE_ICR
       containers:
       - name: anax
         image: __ImagePath__
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         volumeMounts:
         - mountPath: /etc/default/horizon
           subPath: horizon

diff --git a/clusterupgrade/cluster_upgrade_worker.go b/clusterupgrade/cluster_upgrade_worker.go
@@ -50,7 +50,8 @@ const (
 )
 
 const (
-	DEFAULT_CERT_PATH = "/etc/default/cert/"
+	DEFAULT_CERT_PATH                    = "/etc/default/cert/"
+	DEFAULT_IMAGE_REGISTRY_IN_DEPLOYMENT = "__ImageRegistryHost__"
 )
 
 const (
@@ -702,8 +703,9 @@ func checkAgentImage(kubeClient *KubeClient, workDir string) (bool, string, stri
 	}
 	glog.Infof(cuwlog(fmt.Sprintf("Get image %v from tar file, extracted image tag: %v", fullImageTag, imageTag)))
 
-	if currentAgentVersion != imageTag {
-		// push image to image registry
+	if currentAgentVersion != imageTag && !agentUseRemoteRegistry() {
+		// push image to image registry if use edge cluster local registry
+		// If AGENT_CLUSTER_IMAGE_REGISTRY_HOST env is not set, it means agent is using remote image registry, and no need to push image
 		imageRegistry := os.Getenv("AGENT_CLUSTER_IMAGE_REGISTRY_HOST")
 		if imageRegistry == "" {
 			return false, "", "", fmt.Errorf("failed to get edge cluster image registry host from environment veriable: %v", imageRegistry)
@@ -751,7 +753,6 @@ func checkAgentImage(kubeClient *KubeClient, workDir string) (bool, string, stri
 		}
 		glog.Infof(cuwlog(fmt.Sprintf("Successfully pushed image %v", newImageRepoWithTag)))
 	}
-
 	return (currentAgentVersion == imageTag), imageTag, currentAgentVersion, nil
 }
 
@@ -767,3 +768,12 @@ func checkAgentImageAgainstStatusFile(workDir string) (bool, error) {
 		return true, nil
 	}
 }
+
+func agentUseRemoteRegistry() bool {
+	useRemoteRegistry := false
+	imageRegistry := os.Getenv("AGENT_CLUSTER_IMAGE_REGISTRY_HOST")
+	if imageRegistry == DEFAULT_IMAGE_REGISTRY_IN_DEPLOYMENT {
+		useRemoteRegistry = true
+	}
+	return useRemoteRegistry
+}