Adomerge packages (#237)

* coa first round (#3) * coa first round * enable manual trigger * fix otel dependency issue * fix api and k8s tests * add header --------- Co-authored-by: Jiawei Du <jiaweidu_js@163.com> * API done * tidy api/go.mod * fixes * enable more cases * refine ensureNamespace * refine tests * tidy k8s/go.mod * refine cases * verify * fix mage remove * refine go.mod in api & k8s * refine go.work and go.mod in tests * fix naming * temp * enable ginkogo test * fix deploymenthash and tested ginkgo tests * fix github.com/stretchr/objx * fix startup * testing * fix requeue due to update change * resolve a rebase issue * [Temp] Enable tests * Refine mage commands for tests * fix tests and github action * remove adomerge_staging branch * increase go test timeout to 1min * increase go test timeout to 5min * fix constant value * fix unnecessary changes * refine gatekeeper tests * Merge ADO changes to OSS * refine k8s magefile * fix gatekeeper tests * export symphony logs in integration test * fix requeue when deployment is in-progress * Remove unnecessary isDeploymentFinished, use summaryResult.State to check * Remove unnecessary isDeploymentFinished, use summaryResult.State to check * fix merge issues * first draft * checkpoint SAT * second version * Merge ADO K8S changes to OSS * honor OSS changes * expose delete sync delay for futuer override from helm charts * generate helm charts * fix typo in InstanceStatus * add debug logs for k8s target provider * refactor webhook and add metrics * refactor reconciliation policy to meet API spec * watch operationId change instead of annnotation change * webhook fixes * fix typo * refine reconciliation policy * refactor doc * remove version property in target/intance since we will have new version implementation * fix typo * fix dependency issues * clean up work * add ado suite test pipeline * resolve comments * resolve comments * resolve comments * resolve comments * fix helmTemplate * change timeout to 90 in local env mage file * upate cert manager chart version --------- Co-authored-by: Jiawei Du <jiaweidu_js@163.com> Co-authored-by: Jiawei Du <jiadu@microsoft.com>
eclipse-symphony · May 7, 2024 · 44b451c · 44b451c
1 parent b737375
commit 44b451c
Show file tree

Hide file tree

Showing 111 changed files with 13,597 additions and 3,148 deletions.
diff --git a/.github/workflows/suite.yml b/.github/workflows/suite.yml
@@ -0,0 +1,112 @@
+# This workflow will build a golang project
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
+
+name: suite
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+env:
+    ContainerRegistry: "ghcr.io"
+    ContainerRegistryRepo: "ghcr.io/eclipse-symphony"
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.21
+
+    - name: Set up custom GOPATH
+      run: |
+        mkdir -p /home/runner/go
+        echo "export GOPATH=/home/runner/go" >> $HOME/.bashrc
+        echo "export PATH=\$PATH:\$GOPATH/bin" >> $HOME/.bashrc
+        source $HOME/.bashrc
+
+    - name: Install make
+      run: sudo apt-get update && sudo apt-get install -y build-essential
+
+    - name: Check docker version and images
+      run: docker --version && docker images
+
+    - name: Install kubectl
+      run: |
+        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+        chmod +x kubectl
+        sudo mv ./kubectl /usr/local/bin/kubectl
+        kubectl version --client
+        kubectl config view
+
+    - name: Install Helm
+      run: |
+        curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+        chmod 700 get_helm.sh
+        ./get_helm.sh
+
+    - name: Install minikube
+      run: | 
+        curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
+        chmod +x minikube
+        sudo mv minikube /usr/local/bin/
+        minikube start
+        kubectl config view
+    
+    - name: Install Mage
+      run: |
+        cd ..
+        git clone https://github.com/magefile/mage
+        cd mage
+        go run bootstrap.go
+        cd ..
+
+    - name: Install Ginkgo
+      run: |
+        go install github.com/onsi/ginkgo/v2/ginkgo@v2.13.1
+        export PATH=$PATH:$(go env GOPATH)/bin
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ env.ContainerRegistry }}
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Build docker images
+      run: |
+        cd test/localenv/
+        mage build:all
+        mage cluster:up
+    
+    - name: Go work init
+      run: |
+        mv go.work.bk go.work
+
+    - name: Run ginkgo suite tests
+      run: |
+        cd test/integration/scenarios/06.ado/ 
+        ginkgo --cover --junit-report=junit-suite-tests.xml -r 
+      continue-on-error: true    
+
+    - name: Dump SymphonyLogs For ginkgo suite tests
+      run: |
+        cd test/localenv/
+        mage DumpSymphonyLogsForTest ginkgosuite
+      continue-on-error: true   
+
+    - name: Collect and upload symphony test results
+      uses: actions/upload-artifact@v2
+      with:
+        name: symphony-suite-result
+        path: |
+          test/integration/scenarios/06.ado/junit-suite-tests.xml
+          /tmp/symhony-integration-test-logs/**/*.log
+      continue-on-error: true
+      if: always()
diff --git a/api/Dockerfile b/api/Dockerfile
@@ -38,4 +38,4 @@ EXPOSE 8080
 EXPOSE 8081
 ENV LOG_LEVEL Debug
 # ENV CONFIG /symphony-api.json
-CMD exec /symphony-api -c $CONFIG -l $LOG_LEVEL
+CMD sh -c 'if [ -f /etc/pki/ca-trust/source/anchors/proxy-cert.crt ]; then update-ca-trust; fi && exec /symphony-api -c $CONFIG -l $LOG_LEVEL'
diff --git a/api/symphony-script-over-mqtt.json b/api/symphony-script-over-mqtt.json
@@ -26,7 +26,7 @@
               "providers.target": "script",
               "providers.state": "mem-state",
               "providers.config": "mock-config",  
-              "providers.secret": "mock-secret"  
+              "providers.secret": "mock-secret"
             },
             "providers": {
               "script": {

diff --git a/coa/pkg/apis/v1alpha2/bindings/http/jwt.go b/coa/pkg/apis/v1alpha2/bindings/http/jwt.go
@@ -89,19 +89,25 @@ func (j JWT) JWT(next fasthttp.RequestHandler) fasthttp.RequestHandler {
 		}
 		tokenStr := j.readAuthHeader(ctx)
 		if tokenStr == "" {
+			log.Errorf("JWT: Token is empty.\n")
 			ctx.Response.SetStatusCode(fasthttp.StatusForbidden)
 		} else {
 			if j.AuthServer == AuthServerKuberenetes {
+				log.Debugf("JWT: Validating token with k8s.\n")
 				err := j.validateServiceAccountToken(ctx, tokenStr)
 				if err != nil {
+					log.Errorf("JWT: Validate token with k8s failed. %s\n", err.Error())
 					ctx.Response.SetStatusCode(fasthttp.StatusForbidden)
 					return
 				}
 				next(ctx)
 			} else {
+				log.Debugf("JWT: Validating token with username plus pwd.\n")
 				_, roles, err := j.validateToken(tokenStr)
 				if err != nil {
+					log.Error("JWT: Validate token with user creds failed. %s\n", err.Error())
 					ctx.Response.SetStatusCode(fasthttp.StatusForbidden)
+					return
 				} else {
 					if j.EnableRBAC {
 						path := string(ctx.Path())
@@ -207,6 +213,7 @@ func (j *JWT) validateToken(tokenStr string) (map[string]interface{}, []string,
 func (j *JWT) validateServiceAccountToken(ctx *fasthttp.RequestCtx, tokenStr string) error {
 	clientset, err := getKubernetesClient()
 	if err != nil {
+		log.Errorf("JWT: Could not initialize Kubernetes client.\n")
 		return v1alpha2.NewCOAError(err, "Could not initialize Kubernetes client", v1alpha2.InternalError)
 	}
 	tokenReview := &v1.TokenReview{
@@ -217,11 +224,14 @@ func (j *JWT) validateServiceAccountToken(ctx *fasthttp.RequestCtx, tokenStr str
 			},
 		},
 	}
+
 	result, err := clientset.AuthenticationV1().TokenReviews().Create(ctx, tokenReview, metav1.CreateOptions{})
 	if err != nil {
+		log.Errorf("JWT: Token review using kubernetes api server failed. %s\n", err.Error())
 		return v1alpha2.NewCOAError(err, "Token review using kubernetes api server failed.", v1alpha2.InternalError)
 	}
 	if !result.Status.Authenticated {
+		log.Errorf("JWT: Validate token with k8s failed. K8s returned not authenticated.\n")
 		return v1alpha2.NewCOAError(nil, "Authentication failed.", v1alpha2.Unauthorized)
 	} else {
 		apiUsername, err := getApiServiceAccountUsername()
@@ -233,6 +243,7 @@ func (j *JWT) validateServiceAccountToken(ctx *fasthttp.RequestCtx, tokenStr str
 			return err
 		}
 		if result.Status.User.Username != apiUsername && result.Status.User.Username != controllerUsername {
+			log.Errorf("JWT: Validate token with k8s failed. K8s returned invalid username, %s\n", result.Status.User.Username)
 			return v1alpha2.NewCOAError(nil, "Authentication failed.", v1alpha2.Unauthorized)
 		}
 	}

diff --git a/docs/symphony-book/build_deployment/build.md b/docs/symphony-book/build_deployment/build.md
@@ -165,7 +165,7 @@ docker push ghcr.io/eclipse-symphony/symphony-k8s:latest
 ```bash
 cd k8s
 mage helmTemplate
-# Generated startup yaml will be updated in ../packages/helm/symphony/templates/symphony.yaml.
+# Generated startup yaml will be updated in ../packages/helm/symphony/templates/symphony-core/symphonyk8s.yaml.
 ```
 
 > **IMPORTANT**: With current Kustomize, empty `creationTimestamp` properties are inserted into the generated artifacts somehow, causing Helm chart to fail. You'll need to manually remove all occurrence of `creationTimestamp` properties with `null` or `"null"` from the artifacts, until a proper solution is found.

diff --git a/go.work.bk b/go.work.bk
@@ -1,7 +1,10 @@
 go 1.20
 
 use ./api
+
 use ./coa
+
 use ./k8s
+
 use ./cli
 use ./test/integration
diff --git a/go.work.sum b/go.work.sum
@@ -368,4 +368,4 @@ k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/kms v0.26.0/go.mod h1:ReC1IEGuxgfN+PDCIpR6w8+XMmDE7uJhxcCwMZFdIYc=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.33/go.mod h1:soWkSNf2tZC7aMibXEqVhCd73GOY5fJikn8qbdzemB0=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.33/go.mod h1:soWkSNf2tZC7aMibXEqVhCd73GOY5fJikn8qbdzemB0=
diff --git a/k8s/config/oss/default/manager_webhook_patch.yaml b/k8s/config/oss/default/manager_webhook_patch.yaml
@@ -18,6 +18,11 @@ spec:
           name: webhook-server
           protocol: TCP
         volumeMounts:
+        - mountPath: /var/run/secrets/tokens
+          name: symphony-api-token
+        - mountPath: '{{ include "symphony.apiServingCertsDir" . }}'
+          name: api-ca-cert
+          readOnly: true
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true

diff --git a/k8s/config/oss/helm/manager-patch.yaml b/k8s/config/oss/helm/manager-patch.yaml
@@ -28,8 +28,31 @@ spec:
           value: "{{ .Chart.AppVersion }}"
         - name: CONFIG_NAME
           value: '{{ include "symphony.fullname" . }}-manager-config'
+        - name: SERVICE_ACCOUNT_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.serviceAccountName
+        - name: USE_SERVICE_ACCOUNT_TOKENS
+          value: "true"
+        envFrom:
+        - configMapRef:
+            name: '{{ include "symphony.envConfigName" . }}'
       volumes:
       - name: cert
         secret:
           defaultMode: 420
-          secretName: '{{ include "symphony.fullname" . }}-webhook-server-cert'
+          secretName: '{{ include "symphony.fullname" . }}-webhook-server-cert'
+      - name: symphony-api-token
+        projected:
+          sources:
+          - serviceAccountToken:
+              audience: '{{ include "symphony.url" . }}'
+              expirationSeconds: 600
+              path: symphony-api-token
+      - name: api-ca-cert
+        secret:
+          defaultMode: 420
+          items:
+          - key: ca.crt
+            path: ca.crt
+          secretName: '{{ include "symphony.apiServingCertName" . }}'
diff --git a/k8s/magefile.go b/k8s/magefile.go
@@ -99,7 +99,7 @@ func Run() error {
 // Kustomize startup symphony yaml for helm chart.
 func HelmTemplate() error {
 	mg.Deps(ensureKustomize, Manifests)
-	return kustomize.Command("build config/oss/helm -o ../packages/helm/symphony/templates/symphony.yaml").Run()
+	return kustomize.Command("build config/oss/helm -o ../packages/helm/symphony/templates/symphony-core/symphonyk8s.yaml").Run()
 }
 
 // Install CRDs into the K8s cluster specified in ~/.kube/config.

diff --git a/packages/go.work b/packages/go.work
@@ -0,0 +1,6 @@
+go 1.19
+
+use (
+	./mage
+	./testutils
+)
diff --git a/packages/go.work.sum b/packages/go.work.sum
@@ -0,0 +1,11 @@
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60=
+github.com/cheggaaa/pb v2.0.7+incompatible h1:gLKifR1UkZ/kLkda5gC0K6c8g+jU2sINPtBeOiNlMhU=
+github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww=
+go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE=
+golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=
+google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M=
diff --git a/packages/helm/symphony/Chart.yaml b/packages/helm/symphony/Chart.yaml
@@ -8,7 +8,7 @@ version: "0.48.22"
 appVersion: "0.48.22"
 dependencies:
   - name: cert-manager
-    version: "1.4.0"
+    version: "1.13.1"
     repository: "https://charts.jetstack.io"
   - name: zipkin
     version: "0.7.0"

diff --git a/packages/helm/symphony/azure/metadata.json b/packages/helm/symphony/azure/metadata.json