introduces ensure-<dep> pattern for key host level deps

Common.mk

@@ -423,6 +423,8 @@ SKIP_CHECKSUM_VALIDATION?=false
 IN_DOCKER_TARGETS=all-attributions all-attributions-checksums all-checksums attribution attribution-checksums binaries checksums clean clean-go-cache
 PRUNE_BUILDCTL?=false
 GITHUB_TOKEN?=
+
+DEPENDENCY_TOOLS=buildctl helm jq skopeo tuftool yq
 ####################################################
 
 #################### LOGGING #######################
@@ -621,7 +623,7 @@ $(OUTPUT_DIR)/%TTRIBUTION.txt:
 $(OUTPUT_DIR)/%ttribution/go-license.csv: BINARY_TARGET=$(if $(filter .,$(*D)),,$(*D))
 $(OUTPUT_DIR)/%ttribution/go-license.csv: GO_MOD_PATH=$(if $(BINARY_TARGET),$(GO_MOD_TARGET_FOR_BINARY_$(call TO_UPPER,$(BINARY_TARGET))),$(word 1,$(UNIQ_GO_MOD_PATHS)))
 $(OUTPUT_DIR)/%ttribution/go-license.csv: LICENSE_PACKAGE_FILTER=$(GO_MOD_$(subst /,_,$(GO_MOD_PATH))_LICENSE_PACKAGE_FILTER)
-$(OUTPUT_DIR)/%ttribution/go-license.csv: $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH))	
+$(OUTPUT_DIR)/%ttribution/go-license.csv: ensure-jq $$(call GO_MOD_DOWNLOAD_TARGET_FROM_GO_MOD_PATH,$$(GO_MOD_PATH)) 
 	@echo -e $(call TARGET_START_LOG)
 	$(BASE_DIRECTORY)/build/lib/gather_licenses.sh $(REPO) $(MAKE_ROOT)/$(OUTPUT_DIR)/$(BINARY_TARGET) "$(LICENSE_PACKAGE_FILTER)" $(GO_MOD_PATH) $(GOLANG_VERSION) $(LICENSE_THRESHOLD)
 	@echo -e $(call TARGET_END_LOG)
@@ -744,19 +746,19 @@ clean-job-caches: $(and $(findstring presubmit,$(JOB_TYPE)),$(filter true,$(PRUN
 %/images/amd64 %/images/arm64: IMAGE_OUTPUT_TYPE?=oci
 %/images/amd64 %/images/arm64: IMAGE_OUTPUT?=dest=$(IMAGE_OUTPUT_DIR)/$(IMAGE_OUTPUT_NAME).tar
 
-%/images/push: $(BINARY_TARGETS) $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET)
+%/images/push: $(BINARY_TARGETS) $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET) ensure-buildctl ensure-buildkitd-host
 	@echo -e $(call TARGET_START_LOG)
 	$(BUILDCTL)
 	@echo -e $(call TARGET_END_LOG)
 
-%/images/amd64: $(BINARY_TARGETS) $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET)
+%/images/amd64: $(BINARY_TARGETS) $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET) ensure-buildctl ensure-buildkitd-host
 	@echo -e $(call TARGET_START_LOG)
 	@mkdir -p $(IMAGE_OUTPUT_DIR)
 	$(BUILDCTL)
 	$(WRITE_LOCAL_IMAGE_TAG)
 	@echo -e $(call TARGET_END_LOG)
 
-%/images/arm64: $(BINARY_TARGETS) $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET)
+%/images/arm64: $(BINARY_TARGETS) $(LICENSES_TARGETS_FOR_PREREQ) $(HANDLE_DEPENDENCIES_TARGET) ensure-buildctl ensure-buildkitd-host
 	@echo -e $(call TARGET_START_LOG)
 	@mkdir -p $(IMAGE_OUTPUT_DIR)
 	$(BUILDCTL)
@@ -776,10 +778,10 @@ clean-job-caches: $(and $(findstring presubmit,$(JOB_TYPE)),$(filter true,$(PRUN
 %/cgo/amd64: IMAGE_PLATFORMS=linux/amd64
 %/cgo/arm64: IMAGE_PLATFORMS=linux/arm64
 
-%/cgo/amd64:
+%/cgo/amd64: ensure-jq
 	$(if $(filter true, $(USE_DOCKER_FOR_CGO_BUILD)),$(CGO_DOCKER),$(BUILDCTL))
 
-%/cgo/arm64:
+%/cgo/arm64: ensure-jq
 	$(if $(filter true, $(USE_DOCKER_FOR_CGO_BUILD)),$(CGO_DOCKER),$(BUILDCTL))
 
 # As an attempt to see if using docker is more stable for cgo builds in Codebuild
@@ -792,7 +794,7 @@ binary-builder/cgo/%: USE_DOCKER_FOR_CGO_BUILD=$(shell command -v docker &> /dev
 %-useradd/images/export: IMAGE_BUILD_ARGS=IMAGE_USERADD_USER_ID IMAGE_USERADD_USER_NAME
 %-useradd/images/export: DOCKERFILE_FOLDER=$(BUILD_LIB)/docker/linux/useradd
 %-useradd/images/export: IMAGE_PLATFORMS=linux/amd64
-%-useradd/images/export:
+%-useradd/images/export: ensure-buildctl ensure-buildkitd-host
 	@mkdir -p $(IMAGE_OUTPUT_DIR)
 	$(BUILDCTL)
 
@@ -807,7 +809,7 @@ helm/pull:
 .PHONY: helm/build
 helm/build: $(LICENSES_TARGETS_FOR_PREREQ)
 helm/build: $(if $(filter true,$(REPO_NO_CLONE)),,$(HELM_GIT_CHECKOUT_TARGET))
-helm/build: $(if $(wildcard $(PROJECT_ROOT)/helm/patches),$(HELM_GIT_PATCH_TARGET),)
+helm/build: $(if $(wildcard $(PROJECT_ROOT)/helm/patches),$(HELM_GIT_PATCH_TARGET),) ensure-helm ensure-skopeo
 	@echo -e $(call TARGET_START_LOG)
 	$(BUILD_LIB)/helm_copy.sh $(HELM_SOURCE_REPOSITORY) $(HELM_DESTINATION_REPOSITORY) $(HELM_DIRECTORY) $(OUTPUT_DIR)
 	$(BUILD_LIB)/helm_require.sh $(HELM_SOURCE_IMAGE_REPO) $(HELM_DESTINATION_REPOSITORY) $(OUTPUT_DIR) $(IMAGE_TAG) $(HELM_TAG) $(PROJECT_ROOT) $(LATEST) $(HELM_USE_UPSTREAM_IMAGE) $(HELM_IMAGE_LIST)
@@ -817,7 +819,7 @@ helm/build: $(if $(wildcard $(PROJECT_ROOT)/helm/patches),$(HELM_GIT_PATCH_TARGE
 
 # Build helm chart and push to registry defined in IMAGE_REPO.
 .PHONY: helm/push
-helm/push: helm/build
+helm/push: helm/build ensure-helm
 	@echo -e $(call TARGET_START_LOG)
 	$(BUILD_LIB)/helm_push.sh $(IMAGE_REPO) $(HELM_DESTINATION_REPOSITORY) $(HELM_TAG) $(GIT_TAG) $(OUTPUT_DIR) $(LATEST)
 	@echo -e $(call TARGET_END_LOG)
@@ -988,6 +990,39 @@ github-rate-limit-%:
 		GH_PAGER='' gh api rate_limit; \
 	fi
 
+.PHONY: ensure-/
+ensure/%: CMD=$*
+ensure/%:
+	@if ! command -v "$(CMD)" &> /dev/null; then \
+		echo "'$(CMD)' is required for this target, please install."; \
+		exit 1; \
+	fi
+
+# needs to be defined explictly to avoid messing up usage as prereqs for targets with their own %/stems
+.PHONY: $(foreach tool,$(DEPENDENCY_TOOLS), ensure-$(tool))
+$(foreach tool,$(DEPENDENCY_TOOLS),$(eval ensure-$(tool): ensure/$(tool)))
+
+.PHONY: ensure-docker
+ensure-docker: ensure/docker
+	@if ! docker info > /dev/null 2>&1 ; then \
+		echo "Please ensure docker is running to make this target"; \
+		exit 1; \
+	fi
+
+.PHONY: ensure-buildkitd-host
+ensure-buildkitd-host:
+	@if [ -z "$${BUILDKIT_HOST:-}" ] && [ ! -S /run/buildkit/buildkitd.sock ]; then \
+		echo "Please set the 'BUILDKIT_HOST' environment variable."; \
+		echo "If you want to run buildkitd via a docker container use"; \
+		echo "export BUILDKIT_HOST=docker-container://buildkitd && make run-buildkit-and-registry"; \
+		exit 1; \
+	elif ! buildctl debug workers > /dev/null 2>&1; then \
+		echo "buildkit does not appear to be running."; \
+		echo "If you want to run buildkitd via a docker container use"; \
+		echo "make run-buildkit-and-registry"; \
+		exit 1; \
+	fi
+
 ## --------------------------------------
 ## Docker Helpers
 ## --------------------------------------

projects/aws/etcdadm-bootstrap-provider/Makefile

@@ -16,7 +16,7 @@ s3-artifacts: create-manifests
 
 .PHONY: create-manifests
 create-manifests: export PATH:=$(MAKE_ROOT)/$(OUTPUT_DIR):$(PATH)
-create-manifests: $(KUSTOMIZE_TARGET) tarballs
+create-manifests: $(KUSTOMIZE_TARGET) tarballs ensure-yq
 	build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(IMAGE_REPO) $(IMAGE_TAG)
 
 

projects/aws/etcdadm-controller/Makefile

@@ -16,7 +16,7 @@ s3-artifacts: create-manifests
 
 .PHONY: create-manifests
 create-manifests: export PATH:=$(MAKE_ROOT)/$(OUTPUT_DIR):$(PATH)
-create-manifests: $(KUSTOMIZE_TARGET) tarballs
+create-manifests: $(KUSTOMIZE_TARGET) tarballs ensure-yq
 	build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(IMAGE_REPO) $(IMAGE_TAG)
 
 

projects/kubernetes-sigs/cluster-api-provider-vsphere/Makefile

@@ -23,7 +23,7 @@ s3-artifacts: create-manifests
 $(GATHER_LICENSES_TARGETS): | $(FIX_LICENSES_VM_OPERATOR_TARGET)
 
 .PHONY: create-manifests
-create-manifests: tarballs
+create-manifests: tarballs ensure-yq
 	build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(IMAGE_REPO) $(IMAGE_TAG) $(GOLANG_VERSION)
 
 $(FIX_LICENSES_VM_OPERATOR_TARGET): | $(GO_MOD_DOWNLOAD_TARGETS)

projects/kubernetes-sigs/cluster-api/Makefile

@@ -43,7 +43,7 @@ $(GATHER_LICENSES_TARGETS): | $(FIX_LICENSES_GO_JSON_TARGET) $(FIX_LICENSES_TEST
 cluster-api-docker-controller/images/%: BASE_IMAGE_NAME=eks-distro-minimal-base
 
 .PHONY: create-manifests
-create-manifests: tarballs
+create-manifests: tarballs ensure-yq
 	build/create_manifests.sh $(REPO) $(OUTPUT_DIR) $(ARTIFACTS_PATH) $(GIT_TAG) $(IMAGE_REPO) $(IMAGE_TAG) $(GOLANG_VERSION)
 
 $(FIX_LICENSES_GO_JSON_TARGET): | $(GO_MOD_DOWNLOAD_TARGETS)

projects/kubernetes-sigs/image-builder/Makefile

@@ -218,7 +218,7 @@ setup-ami-share:
 	@echo -e $(call TARGET_END_LOG)
 
 .PHONY: setup-packer-configs-%
-setup-packer-configs-%: $(GIT_PATCH_TARGET)
+setup-packer-configs-%: $(GIT_PATCH_TARGET) ensure-jq ensure-yq
 	@echo -e $(call TARGET_START_LOG)
 	build/setup_packer_configs.sh $(RELEASE_BRANCH) $(IMAGE_FORMAT) $(IMAGE_OS) $(ARTIFACTS_BUCKET) $(FINAL_IMAGE_DIR) $(ADDITIONAL_PAUSE_$(RELEASE_BRANCH)_FROM) $(LATEST) $(IMAGE_BUILDER_DIR) $(IMAGE_OS_DIR) $(IMAGE_OS_VERSION)
 	@echo -e $(call TARGET_END_LOG)
@@ -247,7 +247,7 @@ $(ARTIFACTS_PATH)/bottlerocket/% upload-bottlerocket-%: IMAGE_OS=bottlerocket
 $(ARTIFACTS_PATH)/bottlerocket/%: IMAGE_OS_VERSION=$(word 1,$(subst /, ,$(*D)))
 $(ARTIFACTS_PATH)/bottlerocket/%: IMAGE_FORMAT=$(word 2,$(subst /, ,$(*D)))
 $(ARTIFACTS_PATH)/bottlerocket/%: export BOTTLEROCKET_ROOT_JSON_PATH=$(BOTTLEROCKET_DOWNLOAD_PATH)/root.json
-$(ARTIFACTS_PATH)/bottlerocket/%:
+$(ARTIFACTS_PATH)/bottlerocket/%: ensure-yq ensure-tuftool
 	@echo -e $(call TARGET_START_LOG)
 	mkdir -p $(@D) $(BOTTLEROCKET_DOWNLOAD_PATH)
 	# This configuration supports local installations and checksum validations of root.json file
@@ -311,14 +311,14 @@ release-ami-%: EXPORT_AMI_BUCKET?=$(ARTIFACTS_BUCKET)
 release-ami-%: AMI_S3_DST=$(EXPORT_AMI_BUCKET)/$(ARTIFACTS_UPLOAD_PATH)/ami/$(IMAGE_OS_DIR)
 release-ami-%: EXPORT_AMI_DST=$(AMI_S3_DST)/$(GIT_HASH)
 release-ami-%: LATEST_AMI_S3_URL=$(AMI_S3_DST)/$(LATEST)/ubuntu.raw
-release-ami-%: validate-supported-image-% setup-ami-share
+release-ami-%: validate-supported-image-% setup-ami-share ensure-jq
 	@echo -e $(call TARGET_START_LOG)
 	build/build_image.sh $(IMAGE_OS) $(IMAGE_OS_VERSION) $(RELEASE_BRANCH) $(IMAGE_FORMAT) $(ARTIFACTS_BUCKET) $(LATEST)
 	build/export-ami-to-s3.sh $(RELEASE_BRANCH) $(MANIFEST_OUTPUT) raw $(EXPORT_AMI_DST) $(LATEST_AMI_S3_URL)
 	@echo -e $(call TARGET_END_LOG)
 
 release-ova-%: IMAGE_FORMAT=ova
-release-ova-%: validate-supported-image-%
+release-ova-%: validate-supported-image-% ensure-jq
 	@echo -e $(call TARGET_START_LOG)
 	build/build_image.sh $(IMAGE_OS) $(IMAGE_OS_VERSION) $(RELEASE_BRANCH) $(IMAGE_FORMAT) $(ARTIFACTS_BUCKET) $(LATEST) $(IMAGE_OS_FIRMWARE)
 	@echo -e $(call TARGET_END_LOG)
@@ -334,7 +334,7 @@ release-cloudstack-%: release-image-build-on-metal-%
 # used for cloudstack and raw
 # clone the repo first since its scp'd to the temporary instance which runs kvm
 release-image-build-on-metal-%: IMAGE_FORMAT?=raw
-release-image-build-on-metal-%: validate-supported-image-% $(GIT_PATCH_TARGET)
+release-image-build-on-metal-%: validate-supported-image-% $(GIT_PATCH_TARGET) ensure-jq
 	@echo -e $(call TARGET_START_LOG)
 	build/build_image_on_metal.sh $(BASE_DIRECTORY) $(PROJECT_PATH) $(RELEASE_BRANCH) $(RAW_IMAGE_BUILD_AMI) $(RAW_IMAGE_BUILD_INSTANCE_TYPE) $(RAW_IMAGE_BUILD_KEY_NAME) $(IMAGE_OS) $(IMAGE_OS_VERSION) $(IMAGE_FORMAT) $(LATEST) $(FINAL_IMAGE_DIR) $(BRANCH_NAME)
 	@echo -e $(call TARGET_END_LOG)

projects/kubernetes-sigs/kind/Makefile

@@ -124,12 +124,12 @@ kind-node/images/push: LATEST_TAG=$(NODE_IMAGE_LATEST_TAG)
 kind-node/images/push: IMAGE_TARGET=node
 kind-node/images/push: $(KIND_NODE_BUILD_AMD64_TARGET) $(KIND_NODE_BUILD_ARM64_TARGET) $(ARM_ENV_CONF_TARGET)
 
-$(KIND_BASE_IMAGE_BUILD_ARGS):
+$(KIND_BASE_IMAGE_BUILD_ARGS): ensure-yq
 	@echo -e $(call TARGET_START_LOG)
 	build/base-image-build-args.sh $(RELEASE_BRANCH) $(ARTIFACTS_BUCKET) $@ $(LATEST)
 	@echo -e $(call TARGET_END_LOG)
 
-$(KIND_NODE_IMAGE_BUILD_ARGS):
+$(KIND_NODE_IMAGE_BUILD_ARGS): ensure-yq
 	@echo -e $(call TARGET_START_LOG)
 	build/node-image-build-args.sh $(RELEASE_BRANCH) $(KINDNETD_IMAGE_COMPONENT) $(IMAGE_REPO) "$(ARTIFACTS_BUCKET)" $(IMAGE_TAG) $(LATEST) $@
 	@echo -e $(call TARGET_END_LOG)
@@ -151,11 +151,11 @@ $(KIND_BASE_KUBEADM_OVERRIDE):
 # kind build node-image process, also extracts kindnetd manifest to push to s3
 # kind cli built for current host is needed to create the node image
 $(KIND_NODE_BUILD_AMD64_TARGET): KIND_CLI=./$(OUTPUT_BIN_DIR)/$(subst /,-,$(BUILDER_PLATFORM))/kind
-$(KIND_NODE_BUILD_AMD64_TARGET): $(KIND_NODE_IMAGE_BUILD_ARGS) $(ORGANIZE_BINARIES_AMD64_TARGET)
+$(KIND_NODE_BUILD_AMD64_TARGET): $(KIND_NODE_IMAGE_BUILD_ARGS) $(ORGANIZE_BINARIES_AMD64_TARGET) ensure-jq
 	$(MAKE) $(KIND_CLI) BINARY_PLATFORMS=$(BUILDER_PLATFORM)
 	build/build-kind-node-image.sh $(RELEASE_BRANCH) $(VERSIONED_BASE_IMAGE) amd64
 
-$(KIND_NODE_BUILD_ARM64_TARGET): $(KIND_NODE_IMAGE_BUILD_ARGS) $(ORGANIZE_BINARIES_ARM64_TARGET)
+$(KIND_NODE_BUILD_ARM64_TARGET): $(KIND_NODE_IMAGE_BUILD_ARGS) $(ORGANIZE_BINARIES_ARM64_TARGET) ensure-jq
 	build/build-kind-node-image.sh $(RELEASE_BRANCH) $(VERSIONED_BASE_IMAGE) arm64
 
 $(ORGANIZE_BINARIES_AMD64_TARGET): $(HANDLE_DEPENDENCIES_TARGET)