Add support for a hosted mode deployment

The "pause" hack no longer works with the latest addon-framework library. The alternative, which is in this commit, is to return an error in the `Manifests` method when the pause annotation is set. It does spam the logs but since this isn't a setting that should be used in production, this seems okay until the following is addressed: open-cluster-management-io/addon-framework#80 Relates: stolostron/backlog#24362 Signed-off-by: mprahl <mprahl@users.noreply.github.com> (cherry picked from commit 12cfe6e3f51534f0d10edab037bbaf49d7f298e2)
stolostron · Jul 27, 2022 · 27adb05 · 27adb05
1 parent 9e47608
commit 27adb05
Show file tree

Hide file tree

Showing 39 changed files with 1,080 additions and 786 deletions.
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -26,6 +26,9 @@ jobs:
         kind:
           - 'v1.19.16'
           - 'latest'
+        hosted_mode:
+          - "true"
+          - "false"
     name: KinD tests
     steps:
     - name: Checkout 
@@ -40,60 +43,33 @@ jobs:
       with:
         go-version: ${{ matrix.go }}
 
-    - name: Verify modules
-      run: |
-        go mod verify
-
-    - name: Verify format
-      run: |
-        make fmt
-        git diff --exit-code
-        make lint
-
     - name: Create KinD Cluster (${{ matrix.kind }}) and deploy components
       env:
         KIND_VERSION: ${{ matrix.kind }}
+        HOSTED_MODE: ${{ matrix.hosted_mode }}
       run: |
         ./build/manage-clusters.sh
 
-    - name: Unit and Integration Tests
+    - if: matrix.hosted_mode == 'false'
+      name: E2E tests
       run: |
-        make test
+        export GOPATH=$(go env GOPATH)
+        make wait-for-work-agent
+        make e2e-test
 
-    - name: E2E Tests
+    - if: matrix.hosted_mode == 'true'
+      name: E2E tests hosted mode
       run: |
         export GOPATH=$(go env GOPATH)
         make wait-for-work-agent
-        make e2e-test
-    
+        make e2e-test-hosted-mode
+
     - name: Debug
       if: ${{ failure() }}
       run: |
         RUN_MODE="debug" ./build/manage-clusters.sh
 
-    - name: Gosec
-      if: |
-        github.event_name == 'push' &&
-        github.repository_owner == 'stolostron'  &&
-        matrix.kind == 'latest'
-      run: |
-        make gosec-scan
-
-    - name: SonarCloud Analysis
-      if: |
-        github.event_name == 'push' &&
-        github.repository_owner == 'stolostron'  &&
-        matrix.kind == 'latest'
-      uses: sonarsource/sonarcloud-github-action@master
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-      with:
-        projectBaseDir: governance-policy-addon-controller
-        args: >
-          --debug
-
-    - name: Clean up cluster
+    - name: Clean up clusters
       if: ${{ always() }}
       run: |
         RUN_MODE="delete" ./build/manage-clusters.sh
diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml
@@ -0,0 +1,66 @@
+name: Linting and Unit tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+defaults:
+  run:
+    shell: bash
+    working-directory: governance-policy-addon-controller
+
+jobs:
+  linting-and-unit-tests:
+    runs-on: ubuntu-latest
+    name: Linting and Unit tests
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        path: governance-policy-addon-controller
+        fetch-depth: 0 # Fetch all history for all tags and branches
+
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version-file: governance-policy-addon-controller/go.mod
+
+    - name: Verify modules
+      run: |
+        go mod verify
+
+    - name: Verify format
+      run: |
+        make fmt
+        git diff --exit-code
+        make lint
+
+    - name: Unit and Integration Tests
+      run: |
+        make test
+
+    - name: Gosec
+      if: |
+        github.event_name == 'push' &&
+        github.repository_owner == 'stolostron'  &&
+        matrix.kind == 'latest'
+      run: |
+        make gosec-scan
+
+    - name: SonarCloud Analysis
+      if: |
+        github.event_name == 'push' &&
+        github.repository_owner == 'stolostron'  &&
+        matrix.kind == 'latest'
+      uses: sonarsource/sonarcloud-github-action@master
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+      with:
+        projectBaseDir: governance-policy-addon-controller
+        args: >
+          --debug
diff --git a/Makefile b/Makefile
@@ -153,7 +153,10 @@ KUBEWAIT ?= $(PWD)/build/common/scripts/kubewait.sh
 
 KIND_NAME ?= policy-addon-ctrl1
 KIND_KUBECONFIG ?= $(PWD)/$(KIND_NAME).kubeconfig
-HUB_KUBECONFIG ?= $(PWD)/$(KIND_NAME).kubeconfig-internal
+KIND_KUBECONFIG_INTERNAL ?= $(PWD)/$(KIND_NAME).kubeconfig-internal
+HUB_KUBECONFIG ?= $(PWD)/policy-addon-ctrl1.kubeconfig
+HUB_KUBECONFIG_INTERNAL ?= $(PWD)/policy-addon-ctrl1.kubeconfig-internal
+HUB_CLUSTER_NAME ?= policy-addon-ctrl1
 MANAGED_CLUSTER_NAME ?= cluster1
 ifneq ($(KIND_VERSION), latest)
 	KIND_ARGS = --image kindest/node:$(KIND_VERSION)
@@ -168,33 +171,38 @@ $(KIND_KUBECONFIG):
 	@echo "creating cluster"
 	kind create cluster --name $(KIND_NAME) $(KIND_ARGS)
 	kind get kubeconfig --name $(KIND_NAME) > $(KIND_KUBECONFIG)
+	kind get kubeconfig --name $(KIND_NAME) --internal > $(KIND_KUBECONFIG_INTERNAL)
 	KUBECONFIG=$(KIND_KUBECONFIG) kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd-full/monitoring.coreos.com_servicemonitors.yaml
 
-$(HUB_KUBECONFIG):
-	@echo "fetching internal kubeconfig"
-	kind get kubeconfig --name $(KIND_NAME) --internal > $(HUB_KUBECONFIG)
-
 .PHONY: kind-delete-cluster
 kind-delete-cluster: ## Delete a kind cluster.
 	kind delete cluster --name $(KIND_NAME) || true
-	rm $(KIND_KUBECONFIG) || true
-	rm $(HUB_KUBECONFIG) || true
+	-rm $(KIND_KUBECONFIG)
+	-rm $(KIND_KUBECONFIG_INTERNAL)
 
 REGISTRATION_OPERATOR = $(PWD)/.go/registration-operator
 $(REGISTRATION_OPERATOR):
 	@mkdir -p .go
 	git clone --depth 1 https://github.com/open-cluster-management-io/registration-operator.git .go/registration-operator
+	# A workaround for https://github.com/open-cluster-management-io/registration-operator/pull/266
+	sed -i 's/-e "s,mode: Default,mode: Detached,"/-e "s,mode: Default,mode: Detached," -e "s,cluster1,$$(MANAGED_CLUSTER_NAME),"/' .go/registration-operator/Makefile
 
 .PHONY: kind-deploy-registration-operator-hub
-kind-deploy-registration-operator-hub: $(REGISTRATION_OPERATOR) $(KIND_KUBECONFIG) $(HUB_KUBECONFIG) ## Deploy the ocm registration operator to the kind cluster.
+kind-deploy-registration-operator-hub: $(REGISTRATION_OPERATOR) $(KIND_KUBECONFIG) ## Deploy the ocm registration operator to the kind cluster.
 	cd $(REGISTRATION_OPERATOR) && KUBECONFIG=$(KIND_KUBECONFIG) make deploy-hub
 	@printf "\n*** Pausing and waiting to let everything deploy ***\n\n"
 	KUBECONFIG=$(KIND_KUBECONFIG) $(KUBEWAIT) -r deploy/cluster-manager -n open-cluster-management -c condition=Available -m 90
 	KUBECONFIG=$(KIND_KUBECONFIG) $(KUBEWAIT) -r deploy/cluster-manager-placement-controller -n open-cluster-management-hub -c condition=Available -m 90
+	@echo installing Policy CRD on hub
+	KUBECONFIG=$(KIND_KUBECONFIG) kubectl apply -f https://raw.githubusercontent.com/stolostron/governance-policy-propagator/main/deploy/crds/policy.open-cluster-management.io_policies.yaml
 
 .PHONY: kind-deploy-registration-operator-managed
 kind-deploy-registration-operator-managed: $(REGISTRATION_OPERATOR) $(KIND_KUBECONFIG) ## Deploy the ocm registration operator to the kind cluster.
-	cd $(REGISTRATION_OPERATOR) && KUBECONFIG=$(KIND_KUBECONFIG) MANAGED_CLUSTER_NAME=$(MANAGED_CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG) make deploy-spoke
+	cd $(REGISTRATION_OPERATOR) && KUBECONFIG=$(KIND_KUBECONFIG) MANAGED_CLUSTER_NAME=$(MANAGED_CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) make deploy-spoke
+
+.PHONY: kind-deploy-registration-operator-managed-hosted
+kind-deploy-registration-operator-managed-hosted: $(REGISTRATION_OPERATOR) $(KIND_KUBECONFIG) ## Deploy the ocm registration operator to the kind cluster in hosted mode.
+	cd $(REGISTRATION_OPERATOR) && KUBECONFIG=$(HUB_KUBECONFIG) MANAGED_CLUSTER_NAME=$(MANAGED_CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) HOSTED_CLUSTER_MANAGER_NAME=$(HUB_CLUSTER_NAME) EXTERNAL_MANAGED_KUBECONFIG=$(KIND_KUBECONFIG_INTERNAL) make deploy-spoke-hosted
 
 .PHONY: kind-approve-cluster
 kind-approve-cluster: $(KIND_KUBECONFIG) ## Approve managed cluster in the kind cluster.
@@ -238,7 +246,11 @@ e2e-dependencies: ## Download ginkgo locally if necessary.
 
 .PHONY: e2e-test
 e2e-test: e2e-dependencies
-	$(GINKGO) -v --fail-fast --slow-spec-threshold=10s test/e2e
+	$(GINKGO) -v --label-filter="!hosted-mode" --fail-fast --slow-spec-threshold=10s test/e2e
+
+.PHONY: e2e-test-hosted-mode
+e2e-test-hosted-mode: e2e-dependencies
+	$(GINKGO) -v --label-filter="hosted-mode" --fail-fast --slow-spec-threshold=10s test/e2e
 
 .PHONY: e2e-debug
 e2e-debug: ## Collect debug logs from deployed clusters.

diff --git a/build/crd-sync.sh b/build/crd-sync.sh
@@ -50,13 +50,15 @@ $(cat .go/cert-policy-crd-v1.yaml)
 {{- end }}
 EOF
 
+addLabelsExpression='.metadata.labels += {"addon.open-cluster-management.io/hosted-manifest-location": "hosting"}'
+
 cat > pkg/addon/configpolicy/manifests/managedclusterchart/templates/policy.open-cluster-management.io_configurationpolicies_crd.yaml << EOF
 # Copyright Contributors to the Open Cluster Management project
 
 {{- if semverCompare "< 1.16.0" .Capabilities.KubeVersion.Version }}
-$(cat .go/config-policy-crd-v1beta1.yaml)
+$(yq e "$addLabelsExpression" .go/config-policy-crd-v1beta1.yaml)
 {{ else }}
-$(cat .go/config-policy-crd-v1.yaml)
+$(yq e "$addLabelsExpression" .go/config-policy-crd-v1.yaml)
 {{- end }}
 EOF
 
@@ -74,9 +76,9 @@ cat > pkg/addon/policyframework/manifests/managedclusterchart/templates/policy.o
 # Copyright Contributors to the Open Cluster Management project
 
 {{- if semverCompare "< 1.16.0" .Capabilities.KubeVersion.Version }}
-$(cat .go/policy-crd-v1beta1.yaml)
+$(yq e "$addLabelsExpression" .go/policy-crd-v1beta1.yaml)
 {{ else }}
-$(cat .go/policy-crd-v1.yaml)
+$(yq e "$addLabelsExpression" .go/policy-crd-v1.yaml)
 {{- end }}
 EOF
 

diff --git a/build/manage-clusters.sh b/build/manage-clusters.sh
@@ -8,6 +8,14 @@ if [[ -n "${MANAGED_CLUSTER_COUNT//[0-9]}" ]] || [[ "${MANAGED_CLUSTER_COUNT}" =
   echo "error: provided MANAGED_CLUSTER_COUNT is not a nonzero integer"
   exit 1
 fi
+
+if [[ "${MANAGED_CLUSTER_COUNT}" -gt 1 ]] && [[ "${HOSTED_MODE}" == "true" ]]; then
+  # This is a current limitation in the registration operator Makefile where the hosted mode Klusterlet object name
+  # is not customizable.
+  echo "error: provided MANAGED_CLUSTER_COUNT cannot be greater than 1 in hosted mode"
+  exit 1
+fi
+
 KIND_PREFIX=${KIND_PREFIX:-"policy-addon-ctrl"}
 CLUSTER_PREFIX=${CLUSTER_PREFIX:-"cluster"}
 
@@ -30,7 +38,6 @@ esac
 for i in $(seq 2 $((MANAGED_CLUSTER_COUNT+1))); do
   export KIND_NAME="${KIND_PREFIX}${i}"
   export MANAGED_CLUSTER_NAME="${CLUSTER_PREFIX}${i}"
-  export HUB_KUBECONFIG="${PWD}/${KIND_PREFIX}1.kubeconfig-internal"
   case ${RUN_MODE} in
     delete)
       make kind-delete-cluster
@@ -39,7 +46,12 @@ for i in $(seq 2 $((MANAGED_CLUSTER_COUNT+1))); do
       make e2e-debug
       ;;
     create)
-      make kind-deploy-registration-operator-managed
+      if [[ "${HOSTED_MODE}" == "true" ]]; then
+        make kind-deploy-registration-operator-managed-hosted
+      else
+        make kind-deploy-registration-operator-managed
+      fi
+
       # Approval takes place on the hub
       export KIND_NAME="${KIND_PREFIX}1"
       make kind-approve-cluster

diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -121,6 +121,17 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resourceNames:
@@ -203,19 +214,9 @@ rules:
   - manifestworks
   verbs:
   - create
+  - delete
   - get
   - list
-  - watch
-- apiGroups:
-  - work.open-cluster-management.io
-  resourceNames:
-  - addon-cert-policy-controller-deploy
-  - addon-config-policy-controller-deploy
-  - addon-governance-policy-framework-deploy
-  - addon-iam-policy-controller-deploy
-  resources:
-  - manifestworks
-  verbs:
-  - delete
   - patch
   - update
+  - watch