Customize image build and integration test

.github/workflows/integration.yml

@@ -115,7 +115,7 @@ jobs:
       with:
         name: symphony-logs
         path: |
-          /tmp/symhony-integration-test-logs/**/*.log
+          /tmp/symphony-integration-test-logs/**/*.log
       continue-on-error: true
       if: always()
 

.github/workflows/suite.yml

@@ -107,6 +107,6 @@ jobs:
         name: symphony-suite-result
         path: |
           test/integration/scenarios/06.ado/junit-suite-tests.xml
-          /tmp/symhony-integration-test-logs/**/*.log
+          /tmp/symphony-integration-test-logs/**/*.log
       continue-on-error: true
       if: always()

api/Dockerfile

@@ -3,7 +3,11 @@
 ## Licensed under the MIT license.
 ## SPDX-License-Identifier: MIT
 ##
-FROM --platform=$BUILDPLATFORM golang:1.20.2-alpine AS build
+
+ARG BUILD_BASE_IMAGE=golang:1.20.2-alpine
+ARG TARGET_BASE_IMAGE=ubuntu:latest
+
+FROM --platform=$BUILDPLATFORM ${BUILD_BASE_IMAGE} AS build
 
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
@@ -21,7 +25,7 @@ WORKDIR /workspace/api
 RUN chmod +x pkg/apis/v1alpha1/providers/target/script/mock-*.sh
 RUN CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} GODEBUG=netdns=cgo go build -o /dist/symphony-api
 
-FROM ubuntu:latest
+FROM ${TARGET_BASE_IMAGE}
 RUN \
     set -x \
     && apt-get update \

api/magefile.go

@@ -9,6 +9,7 @@
 package main
 
 import (
+	"fmt"
 	"os"
 
 	//mage:import
@@ -33,10 +34,23 @@ func TestWithCoa() error {
 	return nil
 }
 
+func raceCheckSkipped() bool {
+	return os.Getenv("SKIP_RACE_CHECK") == "true"
+}
+
+func raceOpt() string {
+	if raceCheckSkipped() {
+		return ""
+	}
+	return "-race"
+}
+
 func testHelper() error {
+	testClean := "go clean -testcache"
+	testCmd := fmt.Sprintf("go test %s -timeout 5m -cover -coverprofile=coverage.out ./...", raceOpt())
 	if err := shellcmd.RunAll(
-		"go clean -testcache",
-		"go test -race -timeout 5m -cover -coverprofile=coverage.out ./...",
+		shellcmd.Command(testClean),
+		shellcmd.Command(testCmd),
 	); err != nil {
 		return err
 	}

k8s/Dockerfile

@@ -9,7 +9,10 @@
 ##   docker build -f k8s/Dockerfile .
 ## Or build with docker-compose file of k8s
 
-FROM --platform=$BUILDPLATFORM golang:1.20.2-alpine AS builder
+ARG BUILD_BASE_IMAGE=golang:1.20.2-alpine
+ARG TARGET_BASE_IMAGE=mcr.microsoft.com/mirror/docker/library/alpine:3.16
+
+FROM --platform=$BUILDPLATFORM ${BUILD_BASE_IMAGE} AS builder
 
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
@@ -18,8 +21,17 @@ ARG TARGETARCH
 
 ENV CGO_ENABLED=0
 
+ARG BUILD_BASE_IMAGE
+
 # Install gcc, g++ and other necessary build tools
-RUN apk add --no-cache gcc musl-dev
+RUN if echo "${BUILD_BASE_IMAGE}" | grep "alpine"; then \
+        apk add --no-cache gcc musl-dev; \
+    elif echo "${BUILD_BASE_IMAGE}" | grep "mariner"; then \
+        tdnf install -y gcc glibc-devel && tdnf clean all; \
+    else \
+        echo "Unsupported base image"; \
+        exit 1; \
+    fi
 
 RUN go install github.com/magefile/mage@latest
 WORKDIR /
@@ -30,10 +42,17 @@ COPY packages/ packages/
 COPY k8s/ k8s/
 WORKDIR /k8s
 # Test
-RUN CGO_ENABLED=1 mage generate operatorTest
+# Mariner images doesn't support TSan which is the required for go test -race.
+RUN if echo "${BUILD_BASE_IMAGE}" | grep "mariner"; then \
+        export SKIP_RACE_CHECK=true; \
+        CGO_ENABLED=1 mage generate operatorTest; \
+    else \
+        CGO_ENABLED=1 mage generate operatorTest; \
+    fi
+
 # Build
 RUN CGO_ENABLED=0 mage build
-FROM mcr.microsoft.com/mirror/docker/library/alpine:3.16 AS manager
+FROM ${TARGET_BASE_IMAGE} AS manager
 WORKDIR /
 COPY --from=builder /k8s/bin/manager .
 USER 65532:65532

packages/helm/symphony/values.yaml

@@ -31,7 +31,6 @@ parent:
   username: admin
   password:
 siteId: hq
-imagePrivateRegistryUrl: ghcr.io
 api:
   apiContainerPortHttp: 8080
   apiContainerPortHttps: 8081

packages/mage/mage.go

@@ -147,9 +147,20 @@ func docCfg() (func(), error) {
 	return TmpFile(".gomarkdoc.yml", data.String())
 }
 
+func raceCheckSkipped() bool {
+	return os.Getenv("SKIP_RACE_CHECK") == "true"
+}
+
+func raceOpt() string {
+	if raceCheckSkipped() {
+		return ""
+	}
+	return "-race"
+}
+
 // Test runs the unit tests.
 func Test() error {
-	return shellcmd.Command(`go test -race -timeout 5m -cover -coverprofile=coverage.out ./...`).Run()
+	return shellcmd.Command(fmt.Sprintf(`go test %s -timeout 5m -cover -coverprofile=coverage.out ./...`, raceOpt())).Run()
 }
 
 // TestRace runs unit tests without the test cache.
@@ -165,7 +176,7 @@ func TestRace() error {
 func CleanTest() error {
 	return shellcmd.RunAll(
 		`go clean -testcache`,
-		`go test -race -timeout 5m -coverprofile=coverage.out ./...`,
+		shellcmd.Command(fmt.Sprintf(`go test %s -timeout 5m -cover -coverprofile=coverage.out ./...`, raceOpt())),
 	)
 }
 
@@ -205,7 +216,7 @@ func UnitTest() error {
 	bld := strings.Builder{}
 	os.Setenv("GOUNIT", "true")
 	defer os.Unsetenv("GOUNIT")
-	bld.WriteString("go test -v -cover -coverprofile=coverage.out -race -timeout 5m ./...")
+	bld.WriteString(fmt.Sprintf("go test -v -cover -coverprofile=coverage.out %s -timeout 5m ./...", raceOpt()))
 	if isCI() {
 		mg.Deps(ensureGoJUnit)
 		bld.WriteString(" 2>&1 | bin/go-junit-report -set-exit-code -iocopy -out junit-unit-tests.xml")
@@ -310,6 +321,18 @@ func DockerBuild() error {
 	return shellcmd.Command("docker-compose -f docker-compose.yaml build").Run()
 }
 
+func DockerBuildWithOverrideImg(buildBaseImg string, targetBaseImg string) error {
+	var arg string
+	if buildBaseImg != "" {
+		arg += fmt.Sprintf(" --build-arg BUILD_BASE_IMAGE=%s", buildBaseImg)
+	}
+	if targetBaseImg != "" {
+		arg += fmt.Sprintf(" --build-arg TARGET_BASE_IMAGE=%s", targetBaseImg)
+	}
+
+	return shellcmd.Command(fmt.Sprintf("docker-compose -f docker-compose.yaml build %s", arg)).Run()
+}
+
 // Run a command with | or other things that do not work in shellcmd
 func shellExec(cmd string, printCmdOrNot bool) error {
 	if printCmdOrNot {

test/localenv/magefile.go

@@ -36,8 +36,82 @@ const (
 	DOCKER_TAG             = "latest"
 	CHART_PATH             = "../../packages/helm/symphony"
 	GITHUB_PAT             = "CR_PAT"
+	LOG_ROOT               = "/tmp/symphony-integration-test-logs"
 )
 
+// Print parameters for mage local testing
+func PrintParams() error {
+	fmt.Println("OSS_CONTAINER_REGISTRY: ", getContainerRegistry())
+	fmt.Println("DOCKER_TAG: ", getDockerTag())
+	fmt.Println("CHART_NAMESPACE: ", getChartNamespace())
+	fmt.Println("RELEASE_NAME: ", getReleaseName())
+	fmt.Println("CHART_PATH: ", getChartPath())
+	fmt.Println("SKIP_GHCR_VALUES: ", skipGhcrValues())
+	fmt.Println("GHCR_VALUES_OPTIONS: ", ghcrValuesOptions())
+	fmt.Println("LOG_ROOT: ", getLogRoot())
+	return nil
+}
+
+// global variables
+func getLogRoot() string {
+	if os.Getenv("LOG_ROOT") != "" {
+		return os.Getenv("LOG_ROOT")
+	} else {
+		return LOG_ROOT
+	}
+}
+
+func getContainerRegistry() string {
+	if os.Getenv("OSS_CONTAINER_REGISTRY") != "" {
+		return os.Getenv("OSS_CONTAINER_REGISTRY")
+	} else {
+		return OSS_CONTAINER_REGISTRY
+	}
+}
+
+func getDockerTag() string {
+	if os.Getenv("DOCKER_TAG") != "" {
+		return os.Getenv("DOCKER_TAG")
+	} else {
+		return DOCKER_TAG
+	}
+}
+
+func getChartNamespace() string {
+	if os.Getenv("CHART_NAMESPACE") != "" {
+		return os.Getenv("CHART_NAMESPACE")
+	} else {
+		return NAMESPACE
+	}
+}
+
+func getReleaseName() string {
+	if os.Getenv("RELEASE_NAME") != "" {
+		return os.Getenv("RELEASE_NAME")
+	} else {
+		return RELEASE_NAME
+	}
+}
+
+func getChartPath() string {
+	if os.Getenv("CHART_PATH") != "" {
+		return os.Getenv("CHART_PATH")
+	} else {
+		return CHART_PATH
+	}
+}
+
+func skipGhcrValues() bool {
+	return os.Getenv("SKIP_GHCR_VALUES") == "true"
+}
+
+func ghcrValuesOptions() string {
+	if skipGhcrValues() {
+		return ""
+	}
+	return "-f symphony-ghcr-values.yaml"
+}
+
 var reWhiteSpace = regexp.MustCompile(`\n|\t| `)
 
 type Minikube mg.Namespace
@@ -55,10 +129,10 @@ func (Cluster) Deploy() error {
 	mg.Deps(ensureMinikubeUp)
 	certsToVerify := []string{"symphony-api-serving-cert ", "symphony-serving-cert"}
 	commands := []shellcmd.Command{
-		shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml -f symphony-ghcr-values.yaml --set symphonyImage.tag=%s --set paiImage.tag=%s", RELEASE_NAME, CHART_PATH, NAMESPACE, DOCKER_TAG, DOCKER_TAG)),
+		shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml %s --set symphonyImage.tag=%s --set paiImage.tag=%s", getReleaseName(), getChartPath(), getChartNamespace(), ghcrValuesOptions(), getDockerTag(), getDockerTag())),
 	}
 	for _, cert := range certsToVerify {
-		commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, NAMESPACE)))
+		commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, getChartNamespace())))
 	}
 	return shellcmd.RunAll(commands...)
 }
@@ -70,10 +144,10 @@ func (Cluster) DeployWithSettings(values string) error {
 	mg.Deps(ensureMinikubeUp)
 	certsToVerify := []string{"symphony-api-serving-cert ", "symphony-serving-cert"}
 	commands := []shellcmd.Command{
-		shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml -f symphony-ghcr-values.yaml --set symphonyImage.tag=latest --set paiImage.tag=latest %s", RELEASE_NAME, CHART_PATH, NAMESPACE, values)),
+		shellcmd.Command(fmt.Sprintf("helm upgrade %s %s --install -n %s --create-namespace --wait -f ../../packages/helm/symphony/values.yaml %s --set symphonyImage.tag=%s --set paiImage.tag=%s %s", getReleaseName(), getChartPath(), getChartNamespace(), ghcrValuesOptions(), getDockerTag(), getDockerTag(), values)),
 	}
 	for _, cert := range certsToVerify {
-		commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, NAMESPACE)))
+		commands = append(commands, shellcmd.Command(fmt.Sprintf("kubectl wait --for=condition=ready certificates %s -n %s --timeout=90s", cert, getChartNamespace())))
 	}
 	return shellcmd.RunAll(commands...)
 }
@@ -207,13 +281,13 @@ func Logs(logRootFolder string) error {
 	apiLogFile := fmt.Sprintf("%s/api.log", logRootFolder)
 	k8sLogFile := fmt.Sprintf("%s/k8s.log", logRootFolder)
 
-	err := shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-api' --all-containers -n %s > %s", NAMESPACE, apiLogFile), true)
+	err := shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-api' --all-containers -n %s > %s", getChartNamespace(), apiLogFile), true)
 
 	if err != nil {
 		return err
 	}
 
-	err = shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-controller-manager' --all-containers -n %s > %s", NAMESPACE, k8sLogFile), true)
+	err = shellExec(fmt.Sprintf("kubectl logs 'deployment/symphony-controller-manager' --all-containers -n %s > %s", getChartNamespace(), k8sLogFile), true)
 
 	return err
 }
@@ -224,7 +298,7 @@ func DumpSymphonyLogsForTest(testName string) {
 	normalizedTestName = strings.Replace(normalizedTestName, " ", "_", -1)
 
 	logFolderName := fmt.Sprintf("test_%s_%s", normalizedTestName, time.Now().Format("20060102150405"))
-	logRootFolder := fmt.Sprintf("/tmp/symhony-integration-test-logs/%s", logFolderName)
+	logRootFolder := fmt.Sprintf("%s/%s", getLogRoot(), logFolderName)
 
 	_ = shellcmd.Command(fmt.Sprintf("mkdir -p %s", logRootFolder)).Run()
 
@@ -234,7 +308,7 @@ func DumpSymphonyLogsForTest(testName string) {
 // Uninstall all components, e.g. mage destroy all
 func Destroy(flags string) error {
 	err := shellcmd.RunAll(
-		shellcmd.Command(fmt.Sprintf("helm uninstall %s -n %s --wait", RELEASE_NAME, NAMESPACE)),
+		shellcmd.Command(fmt.Sprintf("helm uninstall %s -n %s --wait", getReleaseName(), getChartNamespace())),
 	)
 	if err != nil {
 		return err
@@ -284,12 +358,16 @@ func (Build) All() error {
 func (Build) Save() error {
 	defer logTime(time.Now(), "build:save")
 
-	err := saveDockerImageToTarFile("symphony-k8s:latest.tar", "ghcr.io/eclipse-symphony/symphony-k8s:latest")
+	k8s_tar_file := fmt.Sprintf("symphony-k8s:%s.tar", getDockerTag())
+	api_tar_file := fmt.Sprintf("symphony-api:%s.tar", getDockerTag())
+	k8s_image_tag := fmt.Sprintf("%s/symphony-k8s:%s", getContainerRegistry(), getDockerTag())
+	api_image_tag := fmt.Sprintf("%s/symphony-api:%s", getContainerRegistry(), getDockerTag())
+	err := saveDockerImageToTarFile(k8s_tar_file, k8s_image_tag)
 	if err != nil {
 		return err
 	}
 
-	err = saveDockerImageToTarFile("symphony-api:latest.tar", "ghcr.io/eclipse-symphony/symphony-api:latest")
+	err = saveDockerImageToTarFile(api_tar_file, api_image_tag)
 	if err != nil {
 		return err
 	}
@@ -379,8 +457,8 @@ func (Minikube) Stop() error {
 // Loads symphony component docker images onto the Minikube VM.
 func (Minikube) Load() error {
 	return shellcmd.RunAll(load(
-		fmt.Sprintf("symphony-api:%s", DOCKER_TAG),
-		fmt.Sprintf("symphony-k8s:%s", DOCKER_TAG))...)
+		fmt.Sprintf("symphony-api:%s", getDockerTag()),
+		fmt.Sprintf("symphony-k8s:%s", getDockerTag()))...)
 }
 
 // Deletes the Minikube cluster from you dev box.
@@ -456,8 +534,8 @@ func (Cluster) Status() {
 	shellcmd.Command("kubectl get events -A").Run()
 
 	fmt.Println("Describing failed pods")
-	dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl describe pod {} -n %s", NAMESPACE))
-	dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl logs {} -n %s", NAMESPACE))
+	dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl describe pod {} -n %s", getChartNamespace()))
+	dumpShellOutput(fmt.Sprintf("kubectl get pods --all-namespaces | grep -E 'CrashLoopBackOff|Error|ImagePullBackOff|InvalidImageName|Pending' | awk '{print $2}' | xargs -I {} kubectl logs {} -n %s", getChartNamespace()))
 	fmt.Println("**************************************************")
 }
 
@@ -563,7 +641,7 @@ func runParallel(commands ...shellcmd.Command) error {
 func load(names ...string) []shellcmd.Command {
 	loads := make([]shellcmd.Command, len(names))
 	for i, name := range names {
-		shellcmd.Command(fmt.Sprintf("docker image save -o %s.tar %s/%s", name, OSS_CONTAINER_REGISTRY, name)).Run()
+		shellcmd.Command(fmt.Sprintf("docker image save -o %s.tar %s/%s", name, getContainerRegistry(), name)).Run()
 		loads[i] = shellcmd.Command(fmt.Sprintf(
 			"minikube image load %s.tar",
 			name,
@@ -579,7 +657,7 @@ func pull(names ...string) []shellcmd.Command {
 	for i, name := range names {
 		loads[i] = shellcmd.Command(fmt.Sprintf(
 			"docker pull %s/%s",
-			OSS_CONTAINER_REGISTRY,
+			getContainerRegistry(),
 			name,
 		))
 	}