diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 35270cac7..24db91be6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -115,7 +115,7 @@ jobs: with: name: symphony-logs path: | - /tmp/symhony-integration-test-logs/**/*.log + /tmp/symphony-integration-test-logs/**/*.log continue-on-error: true if: always() diff --git a/.github/workflows/suite.yml b/.github/workflows/suite.yml index 874c3427a..7c85beb78 100644 --- a/.github/workflows/suite.yml +++ b/.github/workflows/suite.yml @@ -107,6 +107,6 @@ jobs: name: symphony-suite-result path: | test/integration/scenarios/06.ado/junit-suite-tests.xml - /tmp/symhony-integration-test-logs/**/*.log + /tmp/symphony-integration-test-logs/**/*.log continue-on-error: true if: always() \ No newline at end of file diff --git a/api/Dockerfile b/api/Dockerfile index 5ec417b81..bb028b277 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -3,7 +3,11 @@ ## Licensed under the MIT license. ## SPDX-License-Identifier: MIT ## -FROM --platform=$BUILDPLATFORM golang:1.20.2-alpine AS build + +ARG BUILD_BASE_IMAGE=golang:1.20.2-alpine +ARG TARGET_BASE_IMAGE=ubuntu:latest + +FROM --platform=$BUILDPLATFORM ${BUILD_BASE_IMAGE} AS build ARG TARGETPLATFORM ARG BUILDPLATFORM @@ -21,7 +25,7 @@ WORKDIR /workspace/api RUN chmod +x pkg/apis/v1alpha1/providers/target/script/mock-*.sh RUN CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} GODEBUG=netdns=cgo go build -o /dist/symphony-api -FROM ubuntu:latest +FROM ${TARGET_BASE_IMAGE} RUN \ set -x \ && apt-get update \ diff --git a/api/magefile.go b/api/magefile.go index a4ee43585..71f6f7c05 100644 --- a/api/magefile.go +++ b/api/magefile.go @@ -9,6 +9,7 @@ package main import ( + "fmt" "os" //mage:import @@ -33,10 +34,23 @@ func TestWithCoa() error { return nil } +func raceCheckSkipped() bool { + return os.Getenv("SKIP_RACE_CHECK") == "true" +} + +func raceOpt() string { + if raceCheckSkipped() { + return "" + } + return "-race" +} + func testHelper() error { + testClean := "go clean -testcache" + testCmd := fmt.Sprintf("go test %s -timeout 5m -cover -coverprofile=coverage.out ./...", raceOpt()) if err := shellcmd.RunAll( - "go clean -testcache", - "go test -race -timeout 5m -cover -coverprofile=coverage.out ./...", + shellcmd.Command(testClean), + shellcmd.Command(testCmd), ); err != nil { return err } diff --git a/k8s/Dockerfile b/k8s/Dockerfile index 4de547ff5..e49226098 100644 --- a/k8s/Dockerfile +++ b/k8s/Dockerfile @@ -9,7 +9,10 @@ ## docker build -f k8s/Dockerfile . ## Or build with docker-compose file of k8s -FROM --platform=$BUILDPLATFORM golang:1.20.2-alpine AS builder +ARG BUILD_BASE_IMAGE=golang:1.20.2-alpine +ARG TARGET_BASE_IMAGE=mcr.microsoft.com/mirror/docker/library/alpine:3.16 + +FROM --platform=$BUILDPLATFORM ${BUILD_BASE_IMAGE} AS builder ARG TARGETPLATFORM ARG BUILDPLATFORM @@ -18,8 +21,17 @@ ARG TARGETARCH ENV CGO_ENABLED=0 +ARG BUILD_BASE_IMAGE + # Install gcc, g++ and other necessary build tools -RUN apk add --no-cache gcc musl-dev +RUN if echo "${BUILD_BASE_IMAGE}" | grep "alpine"; then \ + apk add --no-cache gcc musl-dev; \ + elif echo "${BUILD_BASE_IMAGE}" | grep "mariner"; then \ + tdnf install -y gcc glibc-devel && tdnf clean all; \ + else \ + echo "Unsupported base image"; \ + exit 1; \ + fi RUN go install github.com/magefile/mage@latest WORKDIR / @@ -30,10 +42,17 @@ COPY packages/ packages/ COPY k8s/ k8s/ WORKDIR /k8s # Test -RUN CGO_ENABLED=1 mage generate operatorTest +# Mariner images doesn't support TSan which is the required for go test -race. +RUN if echo "${BUILD_BASE_IMAGE}" | grep "mariner"; then \ + export SKIP_RACE_CHECK=true; \ + CGO_ENABLED=1 mage generate operatorTest; \ + else \ + CGO_ENABLED=1 mage generate operatorTest; \ + fi + # Build RUN CGO_ENABLED=0 mage build -FROM mcr.microsoft.com/mirror/docker/library/alpine:3.16 AS manager +FROM ${TARGET_BASE_IMAGE} AS manager WORKDIR / COPY --from=builder /k8s/bin/manager . USER 65532:65532 diff --git a/packages/helm/symphony/values.yaml b/packages/helm/symphony/values.yaml index d02981f21..ca6958d72 100644 --- a/packages/helm/symphony/values.yaml +++ b/packages/helm/symphony/values.yaml @@ -31,7 +31,6 @@ parent: username: admin password: siteId: hq -imagePrivateRegistryUrl: ghcr.io api: apiContainerPortHttp: 8080 apiContainerPortHttps: 8081 diff --git a/packages/mage/mage.go b/packages/mage/mage.go index 7cf76eb2c..82a4ca053 100644 --- a/packages/mage/mage.go +++ b/packages/mage/mage.go @@ -147,9 +147,20 @@ func docCfg() (func(), error) { return TmpFile(".gomarkdoc.yml", data.String()) } +func raceCheckSkipped() bool { + return os.Getenv("SKIP_RACE_CHECK") == "true" +} + +func raceOpt() string { + if raceCheckSkipped() { + return "" + } + return "-race" +} + // Test runs the unit tests. func Test() error { - return shellcmd.Command(`go test -race -timeout 5m -cover -coverprofile=coverage.out ./...`).Run() + return shellcmd.Command(fmt.Sprintf(`go test %s -timeout 5m -cover -coverprofile=coverage.out ./...`, raceOpt())).Run() } // TestRace runs unit tests without the test cache. @@ -165,7 +176,7 @@ func TestRace() error { func CleanTest() error { return shellcmd.RunAll( `go clean -testcache`, - `go test -race -timeout 5m -coverprofile=coverage.out ./...`, + shellcmd.Command(fmt.Sprintf(`go test %s -timeout 5m -cover -coverprofile=coverage.out ./...`, raceOpt())), ) } @@ -205,7 +216,7 @@ func UnitTest() error { bld := strings.Builder{} os.Setenv("GOUNIT", "true") defer os.Unsetenv("GOUNIT") - bld.WriteString("go test -v -cover -coverprofile=coverage.out -race -timeout 5m ./...") + bld.WriteString(fmt.Sprintf("go test -v -cover -coverprofile=coverage.out %s -timeout 5m ./...", raceOpt())) if isCI() { mg.Deps(ensureGoJUnit) bld.WriteString(" 2>&1 | bin/go-junit-report -set-exit-code -iocopy -out junit-unit-tests.xml") @@ -310,6 +321,18 @@ func DockerBuild() error { return shellcmd.Command("docker-compose -f docker-compose.yaml build").Run() } +func DockerBuildWithOverrideImg(buildBaseImg string, targetBaseImg string) error { + var arg string + if buildBaseImg != "" { + arg += fmt.Sprintf(" --build-arg BUILD_BASE_IMAGE=%s", buildBaseImg) + } + if targetBaseImg != "" { + arg += fmt.Sprintf(" --build-arg TARGET_BASE_IMAGE=%s", targetBaseImg) + } + + return shellcmd.Command(fmt.Sprintf("docker-compose -f docker-compose.yaml build %s", arg)).Run() +} + // Run a command with | or other things that do not work in shellcmd func shellExec(cmd string, printCmdOrNot bool) error { if printCmdOrNot { diff --git a/test/localenv/magefile.go b/test/localenv/magefile.go index c85e739e5..e3bc1938f 100644 --- a/test/localenv/magefile.go +++ b/test/localenv/magefile.go @@ -36,8 +36,82 @@ const ( DOCKER_TAG = "latest" CHART_PATH = "../../packages/helm/symphony" GITHUB_PAT = "CR_PAT" + LOG_ROOT = "/tmp/symphony-integration-test-logs" ) +// Print parameters for mage local testing +func PrintParams() error { + fmt.Println("OSS_CONTAINER_REGISTRY: ", getContainerRegistry()) + fmt.Println("DOCKER_TAG: ", getDockerTag()) + fmt.Println("CHART_NAMESPACE: ", getChartNamespace()) + fmt.Println("RELEASE_NAME: ", getReleaseName()) + fmt.Println("CHART_PATH: ", getChartPath()) + fmt.Println("SKIP_GHCR_VALUES: ", skipGhcrValues()) + fmt.Println("GHCR_VALUES_OPTIONS: ", ghcrValuesOptions()) + fmt.Println("LOG_ROOT: ", getLogRoot()) + return nil +} + +// global variables +func getLogRoot() string { + if os.Getenv("LOG_ROOT") != "" { + return os.Getenv("LOG_ROOT") + } else { + return LOG_ROOT + } +} + +func getContainerRegistry() string { + if os.Getenv("OSS_CONTAINER_REGISTRY") != "" { + return os.Getenv("OSS_CONTAINER_REGISTRY") + } else { + return OSS_CONTAINER_REGISTRY + } +} + +func getDockerTag() string { + if os.Getenv("DOCKER_TAG") != "" { + return os.Getenv("DOCKER_TAG") + } else { + return DOCKER_TAG + } +} + +func getChartNamespace() string { + if os.Getenv("CHART_NAMESPACE") != "" { + return os.Getenv("CHART_NAMESPACE") + } else { + return NAMESPACE + } +} + +func getReleaseName() string { + if os.Getenv("RELEASE_NAME") != "" { + return os.Getenv("RELEASE_NAME") + } else { + return RELEASE_NAME + } +} + +func getChartPath() string { + if os.Getenv("CHART_PATH") != "" { + return os.Getenv("CHART_PATH") + } else { + return CHART_PATH + } +} + +func skipGhcrValues() bool { + return os.Getenv("SKIP_GHCR_VALUES") == "true" +} + +func ghcrValuesOptions() string { + if skipGhcrValues() { + return "" + } + return "-f symphony-ghcr-values.yaml" +} + var reWhiteSpace = regexp.MustCompile(`\n|\t| `) type Minikube mg.Namespace @@ -55,10 +129,10 @@ func (Cluster) Deploy() error { mg.Deps(ensureMinikubeUp) certsToVerify := []string{"symphony-api-serving-cert ", "symphony-serving-cert"} commands := []shellcmd.Command{ - shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml -f symphony-ghcr-values.yaml --set symphonyImage.tag=%s --set paiImage.tag=%s", RELEASE_NAME, CHART_PATH, NAMESPACE, DOCKER_TAG, DOCKER_TAG)), + shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml %s --set symphonyImage.tag=%s --set paiImage.tag=%s", getReleaseName(), getChartPath(), getChartNamespace(), ghcrValuesOptions(), getDockerTag(), getDockerTag())), } for _, cert := range certsToVerify { - commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, NAMESPACE))) + commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, getChartNamespace()))) } return shellcmd.RunAll(commands...) } @@ -70,10 +144,10 @@ func (Cluster) DeployWithSettings(values string) error { mg.Deps(ensureMinikubeUp) certsToVerify := []string{"symphony-api-serving-cert ", "symphony-serving-cert"} commands := []shellcmd.Command{ - shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml -f symphony-ghcr-values.yaml --set symphonyImage.tag=latest --set paiImage.tag=latest %s", RELEASE_NAME, CHART_PATH, NAMESPACE, values)), + shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml %s --set symphonyImage.tag=%s --set paiImage.tag=%s %s", getReleaseName(), getChartPath(), getChartNamespace(), ghcrValuesOptions(), getDockerTag(), getDockerTag(), values)), } for _, cert := range certsToVerify { - commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, NAMESPACE))) + commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, getChartNamespace()))) } return shellcmd.RunAll(commands...) } @@ -207,13 +281,13 @@ func Logs(logRootFolder string) error { apiLogFile := fmt.Sprintf("%s/api.log", logRootFolder) k8sLogFile := fmt.Sprintf("%s/k8s.log", logRootFolder) - err := shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-api' --all-containers -n %s > %s", NAMESPACE, apiLogFile), true) + err := shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-api' --all-containers -n %s > %s", getChartNamespace(), apiLogFile), true) if err != nil { return err } - err = shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-controller-manager' --all-containers -n %s > %s", NAMESPACE, k8sLogFile), true) + err = shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-controller-manager' --all-containers -n %s > %s", getChartNamespace(), k8sLogFile), true) return err } @@ -224,7 +298,7 @@ func DumpSymphonyLogsForTest(testName string) { normalizedTestName = strings.Replace(normalizedTestName, " ", "_", -1) logFolderName := fmt.Sprintf("test_%s_%s", normalizedTestName, time.Now().Format("20060102150405")) - logRootFolder := fmt.Sprintf("/tmp/symhony-integration-test-logs/%s", logFolderName) + logRootFolder := fmt.Sprintf("%s/%s", getLogRoot(), logFolderName) _ = shellcmd.Command(fmt.Sprintf("mkdir -p %s", logRootFolder)).Run() @@ -234,7 +308,7 @@ func DumpSymphonyLogsForTest(testName string) { // Uninstall all components, e.g. mage destroy all func Destroy(flags string) error { err := shellcmd.RunAll( - shellcmd.Command(fmt.Sprintf("helm uninstall %s -n %s --wait", RELEASE_NAME, NAMESPACE)), + shellcmd.Command(fmt.Sprintf("helm uninstall %s -n %s --wait", getReleaseName(), getChartNamespace())), ) if err != nil { return err @@ -284,12 +358,16 @@ func (Build) All() error { func (Build) Save() error { defer logTime(time.Now(), "build:save") - err := saveDockerImageToTarFile("symphony-k8s:latest.tar", "ghcr.io/eclipse-symphony/symphony-k8s:latest") + k8s_tar_file := fmt.Sprintf("symphony-k8s:%s.tar", getDockerTag()) + api_tar_file := fmt.Sprintf("symphony-api:%s.tar", getDockerTag()) + k8s_image_tag := fmt.Sprintf("%s/symphony-k8s:%s", getContainerRegistry(), getDockerTag()) + api_image_tag := fmt.Sprintf("%s/symphony-api:%s", getContainerRegistry(), getDockerTag()) + err := saveDockerImageToTarFile(k8s_tar_file, k8s_image_tag) if err != nil { return err } - err = saveDockerImageToTarFile("symphony-api:latest.tar", "ghcr.io/eclipse-symphony/symphony-api:latest") + err = saveDockerImageToTarFile(api_tar_file, api_image_tag) if err != nil { return err } @@ -379,8 +457,8 @@ func (Minikube) Stop() error { // Loads symphony component docker images onto the Minikube VM. func (Minikube) Load() error { return shellcmd.RunAll(load( - fmt.Sprintf("symphony-api:%s", DOCKER_TAG), - fmt.Sprintf("symphony-k8s:%s", DOCKER_TAG))...) + fmt.Sprintf("symphony-api:%s", getDockerTag()), + fmt.Sprintf("symphony-k8s:%s", getDockerTag()))...) } // Deletes the Minikube cluster from you dev box. @@ -456,8 +534,8 @@ func (Cluster) Status() { shellcmd.Command("kubectl get events -A").Run() fmt.Println("Describing failed pods") - dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl describe pod {} -n %s", NAMESPACE)) - dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl logs {} -n %s", NAMESPACE)) + dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl describe pod {} -n %s", getChartNamespace())) + dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl logs {} -n %s", getChartNamespace())) fmt.Println("**************************************************") } @@ -563,7 +641,7 @@ func runParallel(commands ...shellcmd.Command) error { func load(names ...string) []shellcmd.Command { loads := make([]shellcmd.Command, len(names)) for i, name := range names { - shellcmd.Command(fmt.Sprintf("docker image save -o %s.tar %s/%s", name, OSS_CONTAINER_REGISTRY, name)).Run() + shellcmd.Command(fmt.Sprintf("docker image save -o %s.tar %s/%s", name, getContainerRegistry(), name)).Run() loads[i] = shellcmd.Command(fmt.Sprintf( "minikube image load %s.tar", name, @@ -579,7 +657,7 @@ func pull(names ...string) []shellcmd.Command { for i, name := range names { loads[i] = shellcmd.Command(fmt.Sprintf( "docker pull %s/%s", - OSS_CONTAINER_REGISTRY, + getContainerRegistry(), name, )) } diff --git a/test/localenv/symphony-ghcr-values.yaml b/test/localenv/symphony-ghcr-values.yaml index 0f6ca204a..6df1a9e7a 100644 --- a/test/localenv/symphony-ghcr-values.yaml +++ b/test/localenv/symphony-ghcr-values.yaml @@ -9,5 +9,4 @@ symphonyImage: paiImage: pullPolicy: Never repository: ghcr.io/eclipse-symphony/symphony-api -installServiceExt: false -imagePrivateRegistryUrl: ghcr.io \ No newline at end of file +installServiceExt: false \ No newline at end of file