From 2d792e4ddc9efe17f8fc806ed5aa6c0476be6426 Mon Sep 17 00:00:00 2001 From: Jiawei Du Date: Sat, 11 May 2024 09:10:12 +0800 Subject: [PATCH 01/11] override baseimage in build --- api/Dockerfile | 8 ++++++-- k8s/Dockerfile | 7 +++++-- packages/mage/mage.go | 12 ++++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index 5ec417b81..bb028b277 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -3,7 +3,11 @@ ## Licensed under the MIT license. ## SPDX-License-Identifier: MIT ## -FROM --platform=$BUILDPLATFORM golang:1.20.2-alpine AS build + +ARG BUILD_BASE_IMAGE=golang:1.20.2-alpine +ARG TARGET_BASE_IMAGE=ubuntu:latest + +FROM --platform=$BUILDPLATFORM ${BUILD_BASE_IMAGE} AS build ARG TARGETPLATFORM ARG BUILDPLATFORM @@ -21,7 +25,7 @@ WORKDIR /workspace/api RUN chmod +x pkg/apis/v1alpha1/providers/target/script/mock-*.sh RUN CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} GODEBUG=netdns=cgo go build -o /dist/symphony-api -FROM ubuntu:latest +FROM ${TARGET_BASE_IMAGE} RUN \ set -x \ && apt-get update \ diff --git a/k8s/Dockerfile b/k8s/Dockerfile index 4de547ff5..146c10e99 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -9,7 +9,10 @@ ## docker build -f k8s/Dockerfile . ## Or build with docker-compose file of k8s -FROM --platform=$BUILDPLATFORM golang:1.20.2-alpine AS builder +ARG BUILD_BASE_IMAGE=golang:1.20.2-alpine +ARG TARGET_BASE_IMAGE=mcr.microsoft.com/mirror/docker/library/alpine:3.16 + +FROM --platform=$BUILDPLATFORM ${BUILD_BASE_IMAGE} AS builder ARG TARGETPLATFORM ARG BUILDPLATFORM @@ -33,7 +36,7 @@ WORKDIR /k8s RUN CGO_ENABLED=1 mage generate operatorTest # Build RUN CGO_ENABLED=0 mage build -FROM mcr.microsoft.com/mirror/docker/library/alpine:3.16 AS manager +FROM ${TARGET_BASE_IMAGE} AS manager WORKDIR / COPY --from=builder /k8s/bin/manager . USER 65532:65532 diff --git a/packages/mage/mage.go b/packages/mage/mage.go index 7cf76eb2c..e0daa2387 100644 --- a/packages/mage/mage.go +++ b/packages/mage/mage.go @@ -310,6 +310,18 @@ func DockerBuild() error { return shellcmd.Command("docker-compose -f docker-compose.yaml build").Run() } +func DockerBuildWithOverrideImg(buildBaseImg string, targetBaseImg string) error { + var arg string + if buildBaseImg != "" { + arg += fmt.Sprintf(" --build-arg BUILD_BASE_IMAGE=%s", buildBaseImg) + } + if targetBaseImg != "" { + arg += fmt.Sprintf(" --build-arg TARGET_BASE_IMAGE=%s", targetBaseImg) + } + + return shellcmd.Command(fmt.Sprintf("docker-compose -f docker-compose.yaml build %s", arg)).Run() +} + // Run a command with | or other things that do not work in shellcmd func shellExec(cmd string, printCmdOrNot bool) error { if printCmdOrNot { From 17e7345dd23ef99a672ab77fb3f619737b6f2182 Mon Sep 17 00:00:00 2001 From: Jiawei Du Date: Sun, 19 May 2024 00:12:09 +0800 Subject: [PATCH 02/11] remove unused lines --- packages/helm/symphony/values.yaml | 1 - test/localenv/symphony-ghcr-values.yaml | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/helm/symphony/values.yaml b/packages/helm/symphony/values.yaml index d02981f21..ca6958d72 100644 --- a/packages/helm/symphony/values.yaml +++ b/packages/helm/symphony/values.yaml @@ -31,7 +31,6 @@ parent: username: admin password: siteId: hq -imagePrivateRegistryUrl: ghcr.io api: apiContainerPortHttp: 8080 apiContainerPortHttps: 8081 diff --git a/test/localenv/symphony-ghcr-values.yaml b/test/localenv/symphony-ghcr-values.yaml index 0f6ca204a..6df1a9e7a 100644 --- a/test/localenv/symphony-ghcr-values.yaml +++ b/test/localenv/symphony-ghcr-values.yaml @@ -9,5 +9,4 @@ symphonyImage: paiImage: pullPolicy: Never repository: ghcr.io/eclipse-symphony/symphony-api -installServiceExt: false -imagePrivateRegistryUrl: ghcr.io \ No newline at end of file +installServiceExt: false \ No newline at end of file From 77e25b71bcc9c78eb3e5ceb13aa73fea3e0794d9 Mon Sep 17 00:00:00 2001 From: Jiawei Du Date: Sun, 19 May 2024 00:30:52 +0800 Subject: [PATCH 03/11] make test params --- test/localenv/magefile.go | 75 +++++++++++++++++++++++++++++++-------- 1 file changed, 60 insertions(+), 15 deletions(-) diff --git a/test/localenv/magefile.go b/test/localenv/magefile.go index c85e739e5..8a74909fa 100644 --- a/test/localenv/magefile.go +++ b/test/localenv/magefile.go @@ -38,6 +38,47 @@ const ( GITHUB_PAT = "CR_PAT" ) +// global variables +func getContainerRegistry() string { + if os.Getenv("OSS_CONTAINER_REGISTRY") != "" { + return os.Getenv("OSS_CONTAINER_REGISTRY") + } else { + return OSS_CONTAINER_REGISTRY + } +} + +func getDockerTag() string { + if os.Getenv("DOCKER_TAG") != "" { + return os.Getenv("DOCKER_TAG") + } else { + return DOCKER_TAG + } +} + +func getNamespace() string { + if os.Getenv("NAMESPACE") != "" { + return os.Getenv("NAMESPACE") + } else { + return NAMESPACE + } +} + +func getReleaseName() string { + if os.Getenv("RELEASE_NAME") != "" { + return os.Getenv("RELEASE_NAME") + } else { + return RELEASE_NAME + } +} + +func getChartPath() string { + if os.Getenv("CHART_PATH") != "" { + return os.Getenv("CHART_PATH") + } else { + return CHART_PATH + } +} + var reWhiteSpace = regexp.MustCompile(`\n|\t| `) type Minikube mg.Namespace @@ -55,10 +96,10 @@ func (Cluster) Deploy() error { mg.Deps(ensureMinikubeUp) certsToVerify := []string{"symphony-api-serving-cert ", "symphony-serving-cert"} commands := []shellcmd.Command{ - shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml -f symphony-ghcr-values.yaml --set symphonyImage.tag=%s --set paiImage.tag=%s", RELEASE_NAME, CHART_PATH, NAMESPACE, DOCKER_TAG, DOCKER_TAG)), + shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml -f symphony-ghcr-values.yaml --set symphonyImage.tag=%s --set paiImage.tag=%s", getReleaseName(), getChartPath(), getNamespace(), getDockerTag(), getDockerTag())), } for _, cert := range certsToVerify { - commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, NAMESPACE))) + commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, getNamespace()))) } return shellcmd.RunAll(commands...) } @@ -70,10 +111,10 @@ func (Cluster) DeployWithSettings(values string) error { mg.Deps(ensureMinikubeUp) certsToVerify := []string{"symphony-api-serving-cert ", "symphony-serving-cert"} commands := []shellcmd.Command{ - shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml -f symphony-ghcr-values.yaml --set symphonyImage.tag=latest --set paiImage.tag=latest %s", RELEASE_NAME, CHART_PATH, NAMESPACE, values)), + shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml -f symphony-ghcr-values.yaml --set symphonyImage.tag=%s --set paiImage.tag=%s %s", getReleaseName(), getChartPath(), getNamespace(), getDockerTag(), getDockerTag(), values)), } for _, cert := range certsToVerify { - commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, NAMESPACE))) + commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, getNamespace()))) } return shellcmd.RunAll(commands...) } @@ -207,13 +248,13 @@ func Logs(logRootFolder string) error { apiLogFile := fmt.Sprintf("%s/api.log", logRootFolder) k8sLogFile := fmt.Sprintf("%s/k8s.log", logRootFolder) - err := shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-api' --all-containers -n %s > %s", NAMESPACE, apiLogFile), true) + err := shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-api' --all-containers -n %s > %s", getNamespace(), apiLogFile), true) if err != nil { return err } - err = shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-controller-manager' --all-containers -n %s > %s", NAMESPACE, k8sLogFile), true) + err = shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-controller-manager' --all-containers -n %s > %s", getNamespace(), k8sLogFile), true) return err } @@ -234,7 +275,7 @@ func DumpSymphonyLogsForTest(testName string) { // Uninstall all components, e.g. mage destroy all func Destroy(flags string) error { err := shellcmd.RunAll( - shellcmd.Command(fmt.Sprintf("helm uninstall %s -n %s --wait", RELEASE_NAME, NAMESPACE)), + shellcmd.Command(fmt.Sprintf("helm uninstall %s -n %s --wait", getReleaseName(), getNamespace())), ) if err != nil { return err @@ -284,12 +325,16 @@ func (Build) All() error { func (Build) Save() error { defer logTime(time.Now(), "build:save") - err := saveDockerImageToTarFile("symphony-k8s:latest.tar", "ghcr.io/eclipse-symphony/symphony-k8s:latest") + k8s_tar_file := fmt.Sprintf("symphony-k8s:%s.tar", getDockerTag()) + api_tar_file := fmt.Sprintf("symphony-api:%s.tar", getDockerTag()) + k8s_image_tag := fmt.Sprintf("%s/symphony-k8s:%s", getContainerRegistry(), getDockerTag()) + api_image_tag := fmt.Sprintf("%s/symphony-api:%s", getContainerRegistry(), getDockerTag()) + err := saveDockerImageToTarFile(k8s_tar_file, k8s_image_tag) if err != nil { return err } - err = saveDockerImageToTarFile("symphony-api:latest.tar", "ghcr.io/eclipse-symphony/symphony-api:latest") + err = saveDockerImageToTarFile(api_tar_file, api_image_tag) if err != nil { return err } @@ -379,8 +424,8 @@ func (Minikube) Stop() error { // Loads symphony component docker images onto the Minikube VM. func (Minikube) Load() error { return shellcmd.RunAll(load( - fmt.Sprintf("symphony-api:%s", DOCKER_TAG), - fmt.Sprintf("symphony-k8s:%s", DOCKER_TAG))...) + fmt.Sprintf("symphony-api:%s", getDockerTag()), + fmt.Sprintf("symphony-k8s:%s", getDockerTag()))...) } // Deletes the Minikube cluster from you dev box. @@ -456,8 +501,8 @@ func (Cluster) Status() { shellcmd.Command("kubectl get events -A").Run() fmt.Println("Describing failed pods") - dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl describe pod {} -n %s", NAMESPACE)) - dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl logs {} -n %s", NAMESPACE)) + dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl describe pod {} -n %s", getNamespace())) + dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl logs {} -n %s", getNamespace())) fmt.Println("**************************************************") } @@ -563,7 +608,7 @@ func runParallel(commands ...shellcmd.Command) error { func load(names ...string) []shellcmd.Command { loads := make([]shellcmd.Command, len(names)) for i, name := range names { - shellcmd.Command(fmt.Sprintf("docker image save -o %s.tar %s/%s", name, OSS_CONTAINER_REGISTRY, name)).Run() + shellcmd.Command(fmt.Sprintf("docker image save -o %s.tar %s/%s", name, getContainerRegistry(), name)).Run() loads[i] = shellcmd.Command(fmt.Sprintf( "minikube image load %s.tar", name, @@ -579,7 +624,7 @@ func pull(names ...string) []shellcmd.Command { for i, name := range names { loads[i] = shellcmd.Command(fmt.Sprintf( "docker pull %s/%s", - OSS_CONTAINER_REGISTRY, + getContainerRegistry(), name, )) } From e671cfa0408f4e40e767237d3f976018d39a4490 Mon Sep 17 00:00:00 2001 From: Jiawei Du Date: Sun, 19 May 2024 00:59:30 +0800 Subject: [PATCH 04/11] provide way to skip ghcrValueOptions in mage localenv --- test/localenv/magefile.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/test/localenv/magefile.go b/test/localenv/magefile.go index 8a74909fa..0b5402a5d 100644 --- a/test/localenv/magefile.go +++ b/test/localenv/magefile.go @@ -79,6 +79,17 @@ func getChartPath() string { } } +func skipGhcrValues() bool { + return os.Getenv("SKIP_GHCR_VALUES") == "true" +} + +func ghcrValuesOptions() string { + if skipGhcrValues() { + return "" + } + return "-f symphony-ghcr-values.yaml" +} + var reWhiteSpace = regexp.MustCompile(`\n|\t| `) type Minikube mg.Namespace @@ -96,7 +107,7 @@ func (Cluster) Deploy() error { mg.Deps(ensureMinikubeUp) certsToVerify := []string{"symphony-api-serving-cert ", "symphony-serving-cert"} commands := []shellcmd.Command{ - shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml -f symphony-ghcr-values.yaml --set symphonyImage.tag=%s --set paiImage.tag=%s", getReleaseName(), getChartPath(), getNamespace(), getDockerTag(), getDockerTag())), + shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml %s --set symphonyImage.tag=%s --set paiImage.tag=%s", getReleaseName(), getChartPath(), getNamespace(), ghcrValuesOptions(), getDockerTag(), getDockerTag())), } for _, cert := range certsToVerify { commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, getNamespace()))) @@ -111,7 +122,7 @@ func (Cluster) DeployWithSettings(values string) error { mg.Deps(ensureMinikubeUp) certsToVerify := []string{"symphony-api-serving-cert ", "symphony-serving-cert"} commands := []shellcmd.Command{ - shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml -f symphony-ghcr-values.yaml --set symphonyImage.tag=%s --set paiImage.tag=%s %s", getReleaseName(), getChartPath(), getNamespace(), getDockerTag(), getDockerTag(), values)), + shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml %s --set symphonyImage.tag=%s --set paiImage.tag=%s %s", getReleaseName(), getChartPath(), getNamespace(), ghcrValuesOptions(), getDockerTag(), getDockerTag(), values)), } for _, cert := range certsToVerify { commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, getNamespace()))) From 7561190b265d8e5a24a8ea8f807007bdf7747a48 Mon Sep 17 00:00:00 2001 From: Jiawei Du Date: Sun, 19 May 2024 01:06:36 +0800 Subject: [PATCH 05/11] fix typo --- .github/workflows/integration.yml | 2 +- .github/workflows/suite.yml | 2 +- test/localenv/magefile.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 35270cac7..24db91be6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -115,7 +115,7 @@ jobs: with: name: symphony-logs path: | - /tmp/symhony-integration-test-logs/**/*.log + /tmp/symphony-integration-test-logs/**/*.log continue-on-error: true if: always() diff --git a/.github/workflows/suite.yml b/.github/workflows/suite.yml index 874c3427a..7c85beb78 100644 --- a/.github/workflows/suite.yml +++ b/.github/workflows/suite.yml @@ -107,6 +107,6 @@ jobs: name: symphony-suite-result path: | test/integration/scenarios/06.ado/junit-suite-tests.xml - /tmp/symhony-integration-test-logs/**/*.log + /tmp/symphony-integration-test-logs/**/*.log continue-on-error: true if: always() \ No newline at end of file diff --git a/test/localenv/magefile.go b/test/localenv/magefile.go index 0b5402a5d..e7d1acbbe 100644 --- a/test/localenv/magefile.go +++ b/test/localenv/magefile.go @@ -276,7 +276,7 @@ func DumpSymphonyLogsForTest(testName string) { normalizedTestName = strings.Replace(normalizedTestName, " ", "_", -1) logFolderName := fmt.Sprintf("test_%s_%s", normalizedTestName, time.Now().Format("20060102150405")) - logRootFolder := fmt.Sprintf("/tmp/symhony-integration-test-logs/%s", logFolderName) + logRootFolder := fmt.Sprintf("/tmp/symphony-integration-test-logs/%s", logFolderName) _ = shellcmd.Command(fmt.Sprintf("mkdir -p %s", logRootFolder)).Run() From c5fc46e2fd16efd6d9c821c39813d46d7efe43f7 Mon Sep 17 00:00:00 2001 From: Jiawei Du Date: Sun, 19 May 2024 04:59:37 +0800 Subject: [PATCH 06/11] add print params --- test/localenv/magefile.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/localenv/magefile.go b/test/localenv/magefile.go index e7d1acbbe..5540cd1bd 100644 --- a/test/localenv/magefile.go +++ b/test/localenv/magefile.go @@ -38,6 +38,17 @@ const ( GITHUB_PAT = "CR_PAT" ) +func PrintParams() error { + fmt.Println("OSS_CONTAINER_REGISTRY: ", getContainerRegistry()) + fmt.Println("DOCKER_TAG: ", getDockerTag()) + fmt.Println("NAMESPACE: ", getNamespace()) + fmt.Println("RELEASE_NAME: ", getReleaseName()) + fmt.Println("CHART_PATH: ", getChartPath()) + fmt.Println("SKIP_GHCR_VALUES: ", skipGhcrValues()) + fmt.Println("GHCR_VALUES_OPTIONS: ", ghcrValuesOptions()) + return nil +} + // global variables func getContainerRegistry() string { if os.Getenv("OSS_CONTAINER_REGISTRY") != "" { From cdcd2e2367603f69645045dcfe62a2254c167109 Mon Sep 17 00:00:00 2001 From: Jiawei Du Date: Sun, 19 May 2024 06:01:31 +0800 Subject: [PATCH 07/11] add comments --- test/localenv/magefile.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/localenv/magefile.go b/test/localenv/magefile.go index 5540cd1bd..6f8224930 100644 --- a/test/localenv/magefile.go +++ b/test/localenv/magefile.go @@ -38,6 +38,7 @@ const ( GITHUB_PAT = "CR_PAT" ) +// Print parameters for mage local testing func PrintParams() error { fmt.Println("OSS_CONTAINER_REGISTRY: ", getContainerRegistry()) fmt.Println("DOCKER_TAG: ", getDockerTag()) From 5233978c07a0fa5b5ce2684980e97a3b427a9b08 Mon Sep 17 00:00:00 2001 From: Jiawei Du Date: Sun, 19 May 2024 08:01:41 +0800 Subject: [PATCH 08/11] provide options to build k8s image on mariner --- api/magefile.go | 14 +++++++++++++- k8s/Dockerfile | 20 ++++++++++++++++++-- packages/mage/mage.go | 17 ++++++++++++++--- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/api/magefile.go b/api/magefile.go index a4ee43585..3a377bb78 100644 --- a/api/magefile.go +++ b/api/magefile.go @@ -9,6 +9,7 @@ package main import ( + "fmt" "os" //mage:import @@ -33,10 +34,21 @@ func TestWithCoa() error { return nil } +func raceCheckSkipped() bool { + return os.Getenv("SKIP_RACE_CHECK") == "true" +} + +func raceOpt() string { + if raceCheckSkipped() { + return "" + } + return "-race" +} + func testHelper() error { if err := shellcmd.RunAll( "go clean -testcache", - "go test -race -timeout 5m -cover -coverprofile=coverage.out ./...", + fmt.Sprintf(`go test %s -timeout 5m -cover -coverprofile=coverage.out ./...`, raceOpt()), ); err != nil { return err } diff --git a/k8s/Dockerfile b/k8s/Dockerfile index 146c10e99..e49226098 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -21,8 +21,17 @@ ARG TARGETARCH ENV CGO_ENABLED=0 +ARG BUILD_BASE_IMAGE + # Install gcc, g++ and other necessary build tools -RUN apk add --no-cache gcc musl-dev +RUN if echo "${BUILD_BASE_IMAGE}" | grep "alpine"; then \ + apk add --no-cache gcc musl-dev; \ + elif echo "${BUILD_BASE_IMAGE}" | grep "mariner"; then \ + tdnf install -y gcc glibc-devel && tdnf clean all; \ + else \ + echo "Unsupported base image"; \ + exit 1; \ + fi RUN go install github.com/magefile/mage@latest WORKDIR / @@ -33,7 +42,14 @@ COPY packages/ packages/ COPY k8s/ k8s/ WORKDIR /k8s # Test -RUN CGO_ENABLED=1 mage generate operatorTest +# Mariner images doesn't support TSan which is the required for go test -race. +RUN if echo "${BUILD_BASE_IMAGE}" | grep "mariner"; then \ + export SKIP_RACE_CHECK=true; \ + CGO_ENABLED=1 mage generate operatorTest; \ + else \ + CGO_ENABLED=1 mage generate operatorTest; \ + fi + # Build RUN CGO_ENABLED=0 mage build FROM ${TARGET_BASE_IMAGE} AS manager diff --git a/packages/mage/mage.go b/packages/mage/mage.go index e0daa2387..82a4ca053 100644 --- a/packages/mage/mage.go +++ b/packages/mage/mage.go @@ -147,9 +147,20 @@ func docCfg() (func(), error) { return TmpFile(".gomarkdoc.yml", data.String()) } +func raceCheckSkipped() bool { + return os.Getenv("SKIP_RACE_CHECK") == "true" +} + +func raceOpt() string { + if raceCheckSkipped() { + return "" + } + return "-race" +} + // Test runs the unit tests. func Test() error { - return shellcmd.Command(`go test -race -timeout 5m -cover -coverprofile=coverage.out ./...`).Run() + return shellcmd.Command(fmt.Sprintf(`go test %s -timeout 5m -cover -coverprofile=coverage.out ./...`, raceOpt())).Run() } // TestRace runs unit tests without the test cache. @@ -165,7 +176,7 @@ func TestRace() error { func CleanTest() error { return shellcmd.RunAll( `go clean -testcache`, - `go test -race -timeout 5m -coverprofile=coverage.out ./...`, + shellcmd.Command(fmt.Sprintf(`go test %s -timeout 5m -cover -coverprofile=coverage.out ./...`, raceOpt())), ) } @@ -205,7 +216,7 @@ func UnitTest() error { bld := strings.Builder{} os.Setenv("GOUNIT", "true") defer os.Unsetenv("GOUNIT") - bld.WriteString("go test -v -cover -coverprofile=coverage.out -race -timeout 5m ./...") + bld.WriteString(fmt.Sprintf("go test -v -cover -coverprofile=coverage.out %s -timeout 5m ./...", raceOpt())) if isCI() { mg.Deps(ensureGoJUnit) bld.WriteString(" 2>&1 | bin/go-junit-report -set-exit-code -iocopy -out junit-unit-tests.xml") From 9196cd366a119602190def2287a6f9addb68adab Mon Sep 17 00:00:00 2001 From: Jiawei Du Date: Sun, 19 May 2024 08:34:29 +0800 Subject: [PATCH 09/11] fix typo --- api/magefile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/magefile.go b/api/magefile.go index 3a377bb78..24ea5e6f2 100644 --- a/api/magefile.go +++ b/api/magefile.go @@ -48,7 +48,7 @@ func raceOpt() string { func testHelper() error { if err := shellcmd.RunAll( "go clean -testcache", - fmt.Sprintf(`go test %s -timeout 5m -cover -coverprofile=coverage.out ./...`, raceOpt()), + fmt.Sprintf("go test %s -timeout 5m -cover -coverprofile=coverage.out ./...", raceOpt()), ); err != nil { return err } From caa96e4a2da4eb16f4fe588e2ad0680d93affd3f Mon Sep 17 00:00:00 2001 From: Jiawei Du Date: Sun, 19 May 2024 09:01:14 +0800 Subject: [PATCH 10/11] fix bugs --- api/magefile.go | 6 ++++-- test/localenv/magefile.go | 26 +++++++++++++------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/api/magefile.go b/api/magefile.go index 24ea5e6f2..71f6f7c05 100644 --- a/api/magefile.go +++ b/api/magefile.go @@ -46,9 +46,11 @@ func raceOpt() string { } func testHelper() error { + testClean := "go clean -testcache" + testCmd := fmt.Sprintf("go test %s -timeout 5m -cover -coverprofile=coverage.out ./...", raceOpt()) if err := shellcmd.RunAll( - "go clean -testcache", - fmt.Sprintf("go test %s -timeout 5m -cover -coverprofile=coverage.out ./...", raceOpt()), + shellcmd.Command(testClean), + shellcmd.Command(testCmd), ); err != nil { return err } diff --git a/test/localenv/magefile.go b/test/localenv/magefile.go index 6f8224930..435cb7279 100644 --- a/test/localenv/magefile.go +++ b/test/localenv/magefile.go @@ -42,7 +42,7 @@ const ( func PrintParams() error { fmt.Println("OSS_CONTAINER_REGISTRY: ", getContainerRegistry()) fmt.Println("DOCKER_TAG: ", getDockerTag()) - fmt.Println("NAMESPACE: ", getNamespace()) + fmt.Println("CHART_NAMESPACE: ", getChartNamespace()) fmt.Println("RELEASE_NAME: ", getReleaseName()) fmt.Println("CHART_PATH: ", getChartPath()) fmt.Println("SKIP_GHCR_VALUES: ", skipGhcrValues()) @@ -67,9 +67,9 @@ func getDockerTag() string { } } -func getNamespace() string { - if os.Getenv("NAMESPACE") != "" { - return os.Getenv("NAMESPACE") +func getChartNamespace() string { + if os.Getenv("CHART_NAMESPACE") != "" { + return os.Getenv("CHART_NAMESPACE") } else { return NAMESPACE } @@ -119,10 +119,10 @@ func (Cluster) Deploy() error { mg.Deps(ensureMinikubeUp) certsToVerify := []string{"symphony-api-serving-cert ", "symphony-serving-cert"} commands := []shellcmd.Command{ - shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml %s --set symphonyImage.tag=%s --set paiImage.tag=%s", getReleaseName(), getChartPath(), getNamespace(), ghcrValuesOptions(), getDockerTag(), getDockerTag())), + shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml %s --set symphonyImage.tag=%s --set paiImage.tag=%s", getReleaseName(), getChartPath(), getChartNamespace(), ghcrValuesOptions(), getDockerTag(), getDockerTag())), } for _, cert := range certsToVerify { - commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, getNamespace()))) + commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, getChartNamespace()))) } return shellcmd.RunAll(commands...) } @@ -134,10 +134,10 @@ func (Cluster) DeployWithSettings(values string) error { mg.Deps(ensureMinikubeUp) certsToVerify := []string{"symphony-api-serving-cert ", "symphony-serving-cert"} commands := []shellcmd.Command{ - shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml %s --set symphonyImage.tag=%s --set paiImage.tag=%s %s", getReleaseName(), getChartPath(), getNamespace(), ghcrValuesOptions(), getDockerTag(), getDockerTag(), values)), + shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml %s --set symphonyImage.tag=%s --set paiImage.tag=%s %s", getReleaseName(), getChartPath(), getChartNamespace(), ghcrValuesOptions(), getDockerTag(), getDockerTag(), values)), } for _, cert := range certsToVerify { - commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, getNamespace()))) + commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, getChartNamespace()))) } return shellcmd.RunAll(commands...) } @@ -271,13 +271,13 @@ func Logs(logRootFolder string) error { apiLogFile := fmt.Sprintf("%s/api.log", logRootFolder) k8sLogFile := fmt.Sprintf("%s/k8s.log", logRootFolder) - err := shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-api' --all-containers -n %s > %s", getNamespace(), apiLogFile), true) + err := shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-api' --all-containers -n %s > %s", getChartNamespace(), apiLogFile), true) if err != nil { return err } - err = shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-controller-manager' --all-containers -n %s > %s", getNamespace(), k8sLogFile), true) + err = shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-controller-manager' --all-containers -n %s > %s", getChartNamespace(), k8sLogFile), true) return err } @@ -298,7 +298,7 @@ func DumpSymphonyLogsForTest(testName string) { // Uninstall all components, e.g. mage destroy all func Destroy(flags string) error { err := shellcmd.RunAll( - shellcmd.Command(fmt.Sprintf("helm uninstall %s -n %s --wait", getReleaseName(), getNamespace())), + shellcmd.Command(fmt.Sprintf("helm uninstall %s -n %s --wait", getReleaseName(), getChartNamespace())), ) if err != nil { return err @@ -524,8 +524,8 @@ func (Cluster) Status() { shellcmd.Command("kubectl get events -A").Run() fmt.Println("Describing failed pods") - dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl describe pod {} -n %s", getNamespace())) - dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl logs {} -n %s", getNamespace())) + dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl describe pod {} -n %s", getChartNamespace())) + dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl logs {} -n %s", getChartNamespace())) fmt.Println("**************************************************") } From 09ee5999f54bfd92d9561a5b74ee1e693f713994 Mon Sep 17 00:00:00 2001 From: Jiawei Du Date: Mon, 20 May 2024 19:29:29 +0800 Subject: [PATCH 11/11] customize log root --- test/localenv/magefile.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/localenv/magefile.go b/test/localenv/magefile.go index 435cb7279..e3bc1938f 100644 --- a/test/localenv/magefile.go +++ b/test/localenv/magefile.go @@ -36,6 +36,7 @@ const ( DOCKER_TAG = "latest" CHART_PATH = "../../packages/helm/symphony" GITHUB_PAT = "CR_PAT" + LOG_ROOT = "/tmp/symphony-integration-test-logs" ) // Print parameters for mage local testing @@ -47,10 +48,19 @@ func PrintParams() error { fmt.Println("CHART_PATH: ", getChartPath()) fmt.Println("SKIP_GHCR_VALUES: ", skipGhcrValues()) fmt.Println("GHCR_VALUES_OPTIONS: ", ghcrValuesOptions()) + fmt.Println("LOG_ROOT: ", getLogRoot()) return nil } // global variables +func getLogRoot() string { + if os.Getenv("LOG_ROOT") != "" { + return os.Getenv("LOG_ROOT") + } else { + return LOG_ROOT + } +} + func getContainerRegistry() string { if os.Getenv("OSS_CONTAINER_REGISTRY") != "" { return os.Getenv("OSS_CONTAINER_REGISTRY") @@ -288,7 +298,7 @@ func DumpSymphonyLogsForTest(testName string) { normalizedTestName = strings.Replace(normalizedTestName, " ", "_", -1) logFolderName := fmt.Sprintf("test_%s_%s", normalizedTestName, time.Now().Format("20060102150405")) - logRootFolder := fmt.Sprintf("/tmp/symphony-integration-test-logs/%s", logFolderName) + logRootFolder := fmt.Sprintf("%s/%s", getLogRoot(), logFolderName) _ = shellcmd.Command(fmt.Sprintf("mkdir -p %s", logRootFolder)).Run()