Align docker build to CAPI

.dockerignore

@@ -1,22 +1,39 @@
 .git
 .github
 .vscode
-
-.dockerignore
 .gitignore
 .golangci.yml
-*.out
 bin/
-hack/.bin/
-hack/tools/bin/
+**/*.yaml
+hack/
 out/
 docs/
-scripts/
+packaging/
+templates/
 **/*.md
-*.test
-cluster-api-provider-vsphere
-examples/provider-components/provider-components*.yaml
-test/
+tilt-settings.json
+tilt-settings.yaml
+tilt.d/
+Tiltfile
+**/.tiltbuild
+**/config/**/*.yaml
+**/config/**/*.yaml-e
+_artifacts
+Makefile
+**/Makefile
+
+# ignores changes to test-only code to avoid extra rebuilds
+test/e2e/**
+
+# We want to ignore any frequently modified files to avoid cache-busting the COPY ./ ./
+# Binaries for programs and plugins
+**/*.exe
+**/*.dll
+**/*.so
+**/*.dylib
+cmd/clusterctl/clusterctl/**
+**/bin/**
+**/out/**
 
 # go.work files
 go.work

Dockerfile

@@ -15,13 +15,23 @@
 # limitations under the License.
 
 # Build the manager binary
-ARG GOLANG_VERSION=golang:1.20.12
-FROM --platform=${BUILDPLATFORM} ${GOLANG_VERSION} as builder
+# Run this with docker build --build-arg builder_image=<golang:x.y.z>
+ARG builder_image
+
+# Build architecture
+ARG ARCH
+
+# Ignore Hadolint rule "Always tag the version of an image explicitly."
+# It's an invalid finding since the image is explicitly set in the Makefile.
+# https://github.com/hadolint/hadolint/wiki/DL3006
+# hadolint ignore=DL3006
+FROM ${builder_image} as builder
 WORKDIR /workspace
 
-# Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy
+# Run this with docker build --build-arg goproxy=$(go env GOPROXY) to override the goproxy
 ARG goproxy=https://proxy.golang.org
-ENV GOPROXY=${goproxy}
+# Run this with docker build --build-arg package=./test/infrastructure/vcsim
+ENV GOPROXY=$goproxy
 
 # Copy the Go Modules manifests
 COPY go.mod go.mod
@@ -32,22 +42,30 @@ COPY go.sum go.sum
 RUN --mount=type=cache,target=/go/pkg/mod \
     go mod download
 
+# Copy the sources
+COPY ./ ./
+
+# Cache the go build into the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    go build .
+
 # Build
-ARG TARGETOS
-ARG TARGETARCH
+ARG package=.
+ARG ARCH
 ARG ldflags
-RUN --mount=type=bind,target=. \
-    --mount=type=cache,target=/root/.cache/go-build \
+
+# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder
+RUN --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
-    CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
-    go build -a -ldflags "${ldflags} -extldflags '-static'" \
-    -o /out/manager .
+    CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
+    go build -trimpath -ldflags "${ldflags} -extldflags '-static'" \
+    -o manager ${package}
 
-# Copy the controller-manager into a thin image
-ARG TARGETPLATFORM
-FROM --platform=${TARGETPLATFORM} gcr.io/distroless/static:nonroot
+# Production image
+FROM gcr.io/distroless/static:nonroot-${ARCH}
 WORKDIR /
-COPY --from=builder /out/manager .
-# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying PSPs
+COPY --from=builder /workspace/manager .
+# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies
 USER 65532
 ENTRYPOINT ["/manager"]

Makefile

@@ -57,6 +57,7 @@ ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 BIN_DIR := bin
 BUILD_DIR := .build
 TEST_DIR := test
+VCSIM_DIR := test/infrastructure/vcsim
 TOOLS_DIR := hack/tools
 TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/$(BIN_DIR))
 FLAVOR_DIR := $(ROOT_DIR)/templates
@@ -488,15 +489,17 @@ DOCKER_BUILD_MODIFY_MANIFESTS ?= true
 
 .PHONY: docker-build
 docker-build: docker-pull-prerequisites ## Build the docker image for vsphere controller manager
-	DOCKER_BUILDKIT=1 docker build --platform linux/$(ARCH) --build-arg GOLANG_VERSION=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG)
+## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368)
+	cat ./Dockerfile | DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) --file -
 	@if [ "${DOCKER_BUILD_MODIFY_MANIFESTS}" = "true" ]; then \
   		$(MAKE) set-manifest-image MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./config/base/manager_image_patch.yaml"; \
 		$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./config/base/manager_pull_policy.yaml"; \
     fi
 
 .PHONY: docker-build-vcsim
 docker-build-vcsim: docker-pull-prerequisites ## Build the docker image for vcsim controller manager
-	DOCKER_BUILDKIT=1 docker build --platform linux/$(ARCH) --build-arg GOLANG_VERSION=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ldflags="$(LDFLAGS)" . -t $(VCSIM_CONTROLLER_IMG)-$(ARCH):$(TAG)
+## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368)
+	cat $(VCSIM_DIR)/Dockerfile | DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(VCSIM_CONTROLLER_IMG)-$(ARCH):$(TAG) --file -
 	@if [ "${DOCKER_BUILD_MODIFY_MANIFESTS}" = "true" ]; then \
   		$(MAKE) set-manifest-image MANIFEST_IMG=$(VCSIM_CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./test/infrastructure/vcsim/config/default/manager_image_patch.yaml"; \
 		$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./test/infrastructure/vcsim/config/default/manager_pull_policy.yaml"; \
@@ -539,17 +542,19 @@ test-cover: ## Run unit tests and generate a coverage report
 	go tool cover -html=coverage.out -o coverage.html
 
 .PHONY: test-integration
-test-integration: e2e-image ## Run integration tests
+test-integration: e2e-images ## Run integration tests
 test-integration: $(GINKGO) $(KUSTOMIZE) $(KIND)
 	time $(GINKGO) --output-dir="$(ARTIFACTS)" --junit-report="junit.integration_suite.1.xml" -v ./test/integration -- --config=$(INTEGRATION_CONF_FILE) --artifacts-folder="$(ARTIFACTS)"
 
-.PHONY: e2e-image
-e2e-image: ## Build the e2e manager image
-	docker buildx build --platform linux/$(ARCH) --output=type=docker \
-		--build-arg ldflags="$(LDFLAGS)" --tag="gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:dev" .
+.PHONY: e2e-images
+e2e-images: ## Build the e2e manager image
+	# please ensure the generated image name matches image names used in the E2E_CONF_FILE;
+    # also the same settings must exist in e2e.sh
+	$(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build
+	$(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build-vcsim
 
 .PHONY: e2e
-e2e: e2e-image generate-e2e-templates
+e2e: e2e-images generate-e2e-templates
 e2e: $(GINKGO) $(KUSTOMIZE) $(KIND) $(GOVC) ## Run e2e tests
 	@echo PATH="$(PATH)"
 	@echo

hack/e2e.sh

@@ -94,7 +94,7 @@ make envsubst
 # Only build and upload the image if we run tests which require it to save some $.
 if [[ -z "${GINKGO_FOCUS+x}" ]]; then
   # Save the docker image locally
-  make e2e-image
+  make e2e-images
   mkdir -p /tmp/images
   docker save gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:dev -o "$DOCKER_IMAGE_TAR"
 

test/e2e/config/vsphere.yaml

@@ -14,7 +14,7 @@ images:
     loadBehavior: tryLoad
   - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.6.1
     loadBehavior: tryLoad
-  - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:dev
+  - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-{ARCH}:dev
     loadBehavior: mustLoad
   - name: quay.io/jetstack/cert-manager-cainjector:v1.12.2
     loadBehavior: tryLoad
@@ -104,11 +104,7 @@ providers:
         # Use manifest from source files
         value: ../../../../cluster-api-provider-vsphere/config/default
         contract: v1beta1
-        replacements:
-          - old: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:main
-            new: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:dev
-          - old: "imagePullPolicy: Always"
-            new: "imagePullPolicy: IfNotPresent"
+        replacements: {}
         files:
           # Add a cluster template
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-conformance.yaml"

test/framework/vmoperator/vmoperator.go

@@ -41,7 +41,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
-	"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util"
 	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/session"
 )
 
@@ -342,7 +341,7 @@ func ReconcileDependencies(ctx context.Context, c client.Client, config Dependen
 		},
 		ClassRef: vmoprv1.ClassReference{
 			APIVersion: vmoprv1.SchemeGroupVersion.String(),
-			Kind:       util.TypeToKind(&vmoprv1.VirtualMachineClass{}),
+			Kind:       "VirtualMachineClass",
 			Name:       vmClass.Name,
 		},
 	}
@@ -430,7 +429,7 @@ func ReconcileDependencies(ctx context.Context, c client.Client, config Dependen
 		},
 		ContentSourceRef: vmoprv1.ContentSourceReference{
 			APIVersion: vmoprv1.SchemeGroupVersion.String(),
-			Kind:       util.TypeToKind(&vmoprv1.ContentSource{}),
+			Kind:       "ContentSource",
 			Name:       contentSource.Name,
 		},
 	}

test/infrastructure/vcsim/Dockerfile

@@ -49,7 +49,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \
 
 # This needs to build with the entire Cluster API context
 WORKDIR /workspace
-# Copy the sources (which includes the test/infrastructure/inmemory subdirectory)
+# Copy the sources (which includes the test/infrastructure/vcsim subdirectory)
 COPY ./ ./
 
 # Essentially, change directories into vcsim