Update github.com/libgit2/git2go to v32

This includes changes to the `libgit2` package to address deprecations
and other small signature changes.

Signed-off-by: Hidde Beydals <hello@hidde.co>

.dockerignore

@@ -0,0 +1,10 @@
+# Excluding the libgit2 directory contents has two goals:
+#   1. The compiled dependencies should never be copied over to
+#      an image, but rather be build within to ensure it is build
+#      for the right environment, architecture, and set of
+#      dependencies
+#   2. It speeds up the "pre-build" step of the image by a lot,
+#      as the dependency files are not included in the build
+#      context
+hack/libgit2/**
+!hack/libgit2/CMakeLists.txt

.github/actions/run-tests/Dockerfile

@@ -1,14 +1,12 @@
 FROM golang:1.16-buster as builder
 
-# Up-to-date libgit2 dependencies are only available in
-# unstable, as libssh2 in testing/bullseye has been linked
-# against gcrypt which causes issues with PKCS* formats.
-# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271
-RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \
-    && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list
+# Build dependencies
 RUN set -eux; \
-    apt-get update \
-    && apt-get install -y libgit2-dev/unstable \
+    apt-get update && \
+    apt-get install -y --no-install-recommends \
+        cmake \
+        curl \
+        python3 \
     && apt-get clean \
     && apt-get autoremove --purge -y \
     && rm -rf /var/lib/apt/lists/*

.github/workflows/e2e.yaml

@@ -115,4 +115,5 @@ jobs:
           kubectl -n source-system get helmcharts -oyaml
           kubectl -n source-system get all
           kubectl -n source-system logs deploy/source-controller
+          kubectl -n source-system logs -p deploy/source-controller
           kubectl -n minio get all

.gitignore

@@ -15,3 +15,7 @@
 # vendor/
 bin/
 config/release/
+
+# Exclude all libgit2 related files, except for the CMakeLists instructions
+hack/libgit2/**
+!hack/libgit2/CMakeLists.txt

Dockerfile

@@ -1,61 +1,61 @@
 FROM golang:1.16-buster as builder
 
-# Up-to-date libgit2 dependencies are only available in
-# unstable, as libssh2 in testing/bullseye has been linked
-# against gcrypt which causes issues with PKCS* formats.
-# Explicitly listing all build dependencies is required because
-# they can only be automagically found for AMD64 builds.
-# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271
-RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \
-    && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list
+# Install depedencies required to built libgit2
 RUN set -eux; \
-    apt-get update \
-    && apt-get install -y \
-        libgit2-dev/unstable \
-        zlib1g-dev/unstable \
-        libssh2-1-dev/unstable \
-        libpcre3-dev/unstable \
+    apt-get update && \
+    apt-get install -y --no-install-recommends \
+        cmake \
+        curl \
+        python3 \
     && apt-get clean \
     && apt-get autoremove --purge -y \
     && rm -rf /var/lib/apt/lists/*
 
+# Configure workspace
 WORKDIR /workspace
 
-# copy api submodule
+# Static build libgit2 and other dependencies
+COPY hack/libgit2/CMakeLists.txt libgit2/
+RUN cd libgit2 \
+    && cmake -DBUILD_SHARED_LIBS:BOOL=OFF . \
+    && cmake --build .
+
+# Copy api submodule
 COPY api/ api/
 
-# copy modules manifests
+# Copy modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
 
-# cache modules
+# Cache modules
 RUN go mod download
 
-# copy source code
+# Copy source code
 COPY main.go main.go
 COPY controllers/ controllers/
 COPY pkg/ pkg/
 COPY internal/ internal/
 
-# build without specifing the arch
-RUN CGO_ENABLED=1 go build -o source-controller main.go
+# Build a binary with all C dependencies statically linked.
+# PKG_CONFIG_PATH is set to the result of our own C libary builds
+# to overwrite the git2go defaults that assume a submodule with a
+# specific path.
+RUN PKG_CONFIG_PATH="$PWD/libgit2/install/lib/pkgconfig" \
+    LD_LIBRARY_PATH="$PWD/libgit2/install/lib" \
+    go build -o source-controller \
+    -tags static,system_libgit2 \
+    main.go
 
 FROM debian:buster-slim as controller
 
 # link repo to the GitHub Container Registry image
 LABEL org.opencontainers.image.source="https://github.com/fluxcd/source-controller"
 
-# Up-to-date libgit2 dependencies are only available in
-# unstable, as libssh2 in testing/bullseye has been linked
-# against gcrypt which causes issues with PKCS* formats.
-# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271
-RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \
-    && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list
 RUN set -eux; \
     apt-get update \
     && apt-get install -y \
         ca-certificates \
-        libgit2-1.1 \
+        libc6 \
     && apt-get clean \
     && apt-get autoremove --purge -y \
     && rm -rf /var/lib/apt/lists/*

Makefile

@@ -3,8 +3,10 @@ IMG ?= fluxcd/source-controller:latest
 # Produce CRDs that work back to Kubernetes 1.16
 CRD_OPTIONS ?= crd:crdVersions=v1
 
-ENVTEST_BIN_VERSION?=1.19.2
-KUBEBUILDER_ASSETS?=$(shell $(SETUP_ENVTEST) use -i $(ENVTEST_BIN_VERSION) -p path)
+CONTROLLER_GEN_VERSION ?= v0.5.0
+GEN_API_REF_DOCS_VERSION ?= 0.3.0
+ENVTEST_BIN_VERSION ?= 1.19.2
+KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use -i $(ENVTEST_BIN_VERSION) -p path)
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -13,121 +15,168 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
-all: manager
+REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel)
+LIBGIT2_DIR := hack/libgit2
 
-# Run tests
-test: generate fmt vet manifests api-docs setup-envtest
-	KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) go test ./... -coverprofile cover.out
-	cd api; go test ./... -coverprofile cover.out
+LIBGIT2_SHARED_DIR := $(LIBGIT2_DIR)/shared
+LIBGIT2_SHARED_LIB := $(LIBGIT2_SHARED_DIR)/install/lib
+LIBGIT2_SHARED_BUILD := $(LIBGIT2_SHARED_LIB)/libgit2.so
+LIBGIT2_SHARED_PKG_CONFIG := $(LIBGIT2_SHARED_LIB)/pkgconfig
+
+LIBGIT2_STATIC_DIR := $(LIBGIT2_DIR)/static
+LIBGIT2_STATIC_LIB := $(LIBGIT2_STATIC_DIR)/install/lib
+LIBGIT2_STATIC_BUILD := $(LIBGIT2_STATIC_LIB)/libgit2.a
+LIBGIT2_STATIC_PKG_CONFIG := $(LIBGIT2_STATIC_LIB)/pkgconfig
+
+all: build
 
-# Build manager binary
-manager: generate fmt vet
+build: $(LIBGIT2_SHARED_BUILD) generate fmt vet ## Build manager binary
+	PKG_CONFIG_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_SHARED_PKG_CONFIG) \
+	LD_LIBRARY_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_SHARED_LIB) \
 	go build -o bin/manager main.go
 
-# Run against the configured Kubernetes cluster in ~/.kube/config
-run: generate fmt vet manifests
+build-static: $(LIBGIT2_STATIC_BUILD) generate fmt vet  ## Build static manager binary
+	PKG_CONFIG_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_STATIC_PKG_CONFIG) \
+	LD_LIBRARY_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_STATIC_LIB) \
+	go build -tags static,system_libgit2 -o bin/manager-static main.go
+
+test: $(LIBGIT2_SHARED_BUILD) test-api  ## Run tests
+	PKG_CONFIG_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_SHARED_PKG_CONFIG) \
+	LD_LIBRARY_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_SHARED_LIB) \
+	go test ./... -coverprofile cover.out
+
+test-static: $(LIBGIT2_STATIC_BUILD) test-api  ## Run static tests
+	PKG_CONFIG_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_STATIC_PKG_CONFIG) \
+	LD_LIBRARY_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_STATIC_LIB) \
+	go test -tags static,system_libgit2 ./... -coverprofile cover.out
+
+test-api: ## Run api tests
+	cd api; go test ./... -coverprofile cover.out
+
+run: $(LIBGIT2_SHARED_BUILD) generate fmt vet manifests  ## Run against the configured Kubernetes cluster in ~/.kube/config
+	PKG_CONFIG_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_SHARED_PKG_CONFIG) \
+	LD_LIBRARY_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_SHARED_LIB) \
 	go run ./main.go
 
-# Install CRDs into a cluster
-install: manifests
+run-static: $(LIBGIT2_STATIC_BUILD) generate fmt vet manifests  ## Static run against the configured Kubernetes cluster in ~/.kube/config
+	PKG_CONFIG_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_STATIC_PKG_CONFIG) \
+	LD_LIBRARY_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_STATIC_LIB) \
+	go run ./main.go
+
+install: manifests  ## Install CRDs into a cluster
 	kustomize build config/crd | kubectl apply -f -
 
-# Uninstall CRDs from a cluster
-uninstall: manifests
+uninstall: manifests  ## Uninstall CRDs from a cluster
 	kustomize build config/crd | kubectl delete -f -
 
-# Deploy controller in the configured Kubernetes cluster in ~/.kube/config
-deploy: manifests
+deploy: manifests  ## Deploy controller in the configured Kubernetes cluster in ~/.kube/config
 	cd config/manager && kustomize edit set image fluxcd/source-controller=${IMG}
 	kustomize build config/default | kubectl apply -f -
 
-# Deploy controller dev image in the configured Kubernetes cluster in ~/.kube/config
-dev-deploy:
+dev-deploy:  ## Deploy controller dev image in the configured Kubernetes cluster in ~/.kube/config
 	mkdir -p config/dev && cp config/default/* config/dev
 	cd config/dev && kustomize edit set image fluxcd/source-controller=${IMG}
 	kustomize build config/dev | kubectl apply -f -
 	rm -rf config/dev
 
-# Generate manifests e.g. CRD, RBAC etc.
-manifests: controller-gen
+manifests: controller-gen  ## Generate manifests, e.g. CRD, RBAC, etc.
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role paths="./..." output:crd:artifacts:config="config/crd/bases"
 	cd api; $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role paths="./..." output:crd:artifacts:config="../config/crd/bases"
 
-# Generate API reference documentation
-api-docs: gen-crd-api-reference-docs
+api-docs: gen-crd-api-reference-docs  ## Generate API reference documentation
 	$(API_REF_GEN) -api-dir=./api/v1beta1 -config=./hack/api-docs/config.json -template-dir=./hack/api-docs/template -out-file=./docs/api/source.md
 
-# Run go mod tidy
-tidy:
+tidy:  ## Run go mod tidy
 	go mod tidy
 	cd api; go mod tidy
 
-# Run go fmt against code
-fmt:
+fmt:  ## Run go fmt against code
 	go fmt ./...
 	cd api; go fmt ./...
 
-# Run go vet against code
-vet:
+vet:  ## Run go vet against code
 	go vet ./...
 	cd api; go vet ./...
 
-# Generate code
-generate: controller-gen
+generate: controller-gen  ## Generate API code
 	cd api; $(CONTROLLER_GEN) object:headerFile="../hack/boilerplate.go.txt" paths="./..."
 
-# Build the docker image
-docker-build:
+docker-build:  ## Build the docker image
 	docker build . -t ${IMG}
 
-# Push the docker image
-docker-push:
+docker-push:  ## Push docker image
 	docker push ${IMG}
 
-# Find or download controller-gen
-controller-gen:
+$(LIBGIT2_SHARED_BUILD): $(LIBGIT2_SHARED_DIR)
+	(cd $(LIBGIT2_SHARED_DIR) && cmake -DBUILD_SHARED_LIBS:BOOL=ON .. && cmake --build .)
+
+$(LIBGIT2_STATIC_BUILD): $(LIBGIT2_STATIC_DIR)
+	(cd $(LIBGIT2_STATIC_DIR) && cmake -DBUILD_SHARED_LIBS:BOOL=OFF .. && cmake --build .)
+
+$(LIBGIT2_SHARED_DIR):
+	mkdir -p "$@"
+
+$(LIBGIT2_STATIC_DIR):
+	mkdir -p "$@"
+
+.PHONY: clean
+clean:  ## Removes dependency builds and clears Go cache
+	rm -rf $(LIBGIT2_SHARED_DIR)
+	rm -rf $(LIBGIT2_STATIC_DIR)
+
+	# C-binding directives in Go are cached, which means that e.g.
+	# changes to $PKG_CONFIG_PATH do not have an effect until this cache
+	# is cleared and the package(s) is forced to be rebuild.
+	# This can also be done using `go <cmd> -a`, but this would make
+	# things slow for something that is not expected to change much,
+	# unless you are specifically dealing with build related changes.
+	go clean
+
+controller-gen: ## Find or download controller-gen
 ifeq (, $(shell which controller-gen))
 	@{ \
-	set -e ;\
-	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
-	cd $$CONTROLLER_GEN_TMP_DIR ;\
-	go mod init tmp ;\
-	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.5.0 ;\
-	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
+	set -e; \
+	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d); \
+	cd $$CONTROLLER_GEN_TMP_DIR; \
+	go mod init tmp; \
+	go get sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_GEN_VERSION); \
+	rm -rf $$CONTROLLER_GEN_TMP_DIR; \
 	}
 CONTROLLER_GEN=$(GOBIN)/controller-gen
 else
 CONTROLLER_GEN=$(shell which controller-gen)
 endif
 
-# Find or download gen-crd-api-reference-docs
-gen-crd-api-reference-docs:
+gen-crd-api-reference-docs:  ## Find or download gen-crd-api-reference-docs
 ifeq (, $(shell which gen-crd-api-reference-docs))
 	@{ \
-	set -e ;\
-	API_REF_GEN_TMP_DIR=$$(mktemp -d) ;\
-	cd $$API_REF_GEN_TMP_DIR ;\
-	go mod init tmp ;\
-	go get github.com/ahmetb/gen-crd-api-reference-docs@v0.3.0 ;\
-	rm -rf $$API_REF_GEN_TMP_DIR ;\
+	set -e; \
+	API_REF_GEN_TMP_DIR=$$(mktemp -d); \
+	cd $$API_REF_GEN_TMP_DIR; \
+	go mod init tmp; \
+	go get github.com/ahmetb/gen-crd-api-reference-docs@$(GEN_API_REF_DOCS_VERSION); \
+	rm -rf $$API_REF_GEN_TMP_DIR; \
 	}
 API_REF_GEN=$(GOBIN)/gen-crd-api-reference-docs
 else
 API_REF_GEN=$(shell which gen-crd-api-reference-docs)
 endif
 
-# Find or download setup-envtest
-setup-envtest:
+setup-envtest:  ## Find or download setup-envtest
 ifeq (, $(shell which setup-envtest))
 	@{ \
-	set -e ;\
-	SETUP_ENVTEST_TMP_DIR=$$(mktemp -d) ;\
-	cd $$SETUP_ENVTEST_TMP_DIR ;\
-	go mod init tmp ;\
-	go get sigs.k8s.io/controller-runtime/tools/setup-envtest@latest ;\
-	rm -rf $$SETUP_ENVTEST_TMP_DIR ;\
+	set -e; \
+	SETUP_ENVTEST_TMP_DIR=$$(mktemp -d); \
+	cd $$SETUP_ENVTEST_TMP_DIR; \
+	go mod init tmp; \
+	go get sigs.k8s.io/controller-runtime/tools/setup-envtest@latest; \
+	rm -rf $$SETUP_ENVTEST_TMP_DIR; \
 	}
 SETUP_ENVTEST=$(GOBIN)/setup-envtest
 else
 SETUP_ENVTEST=$(shell which setup-envtest)
 endif
+
+.PHONY: help
+help:  ## Display this help menu.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/go-git/go-billy/v5 v5.3.1
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/go-logr/logr v0.4.0
-	github.com/libgit2/git2go/v31 v31.4.14
+	github.com/libgit2/git2go/v32 v32.0.4
 	github.com/minio/minio-go/v7 v7.0.10
 	github.com/onsi/ginkgo v1.16.4
 	github.com/onsi/gomega v1.14.0

go.sum

@@ -545,8 +545,8 @@ github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6Fm
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
 github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/libgit2/git2go/v31 v31.4.14 h1:6GOd3965D9e/+gjxCwZF4eQ+vB9kKB4yKFqdQr6XZ2E=
-github.com/libgit2/git2go/v31 v31.4.14/go.mod h1:c/rkJcBcUFx6wHaT++UwNpKvIsmPNqCeQ/vzO4DrEec=
+github.com/libgit2/git2go/v32 v32.0.4 h1:qK2yWGh88K2Gh76E1+vUEsjKDfOAq0J2THKaoaFIPbA=
+github.com/libgit2/git2go/v32 v32.0.4/go.mod h1:FAA2ePV5PlLjw1ccncFIvu2v8hJSZVN5IzEn4lo/vwo=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=