From c1a094d9d30050c8af2a4d3de4078a3af49b397e Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 10 Sep 2021 12:39:40 +0200 Subject: [PATCH] Update github.com/libgit2/git2go to v32 This includes changes to the `libgit2` package to address deprecations and other small signature changes. Signed-off-by: Hidde Beydals --- .dockerignore | 10 ++ .github/actions/run-tests/Dockerfile | 14 +-- .github/workflows/e2e.yaml | 1 + .gitignore | 4 + Dockerfile | 54 ++++----- Makefile | 167 +++++++++++++++++---------- go.mod | 2 +- go.sum | 4 +- hack/libgit2/CMakeLists.txt | 76 ++++++++++++ internal/fs/rename.go | 1 + internal/fs/rename_windows.go | 1 + pkg/git/git.go | 2 +- pkg/git/libgit2/checkout.go | 12 +- pkg/git/libgit2/checkout_test.go | 11 +- pkg/git/libgit2/commit.go | 2 +- pkg/git/libgit2/transport.go | 28 ++--- pkg/git/libgit2/transport_test.go | 2 +- 17 files changed, 266 insertions(+), 125 deletions(-) create mode 100644 .dockerignore create mode 100644 hack/libgit2/CMakeLists.txt diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..21405c7aa --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +# Excluding the libgit2 directory contents has two goals: +# 1. The compiled dependencies should never be copied over to +# an image, but rather be build within to ensure it is build +# for the right environment, architecture, and set of +# dependencies +# 2. It speeds up the "pre-build" step of the image by a lot, +# as the dependency files are not included in the build +# context +hack/libgit2/** +!hack/libgit2/CMakeLists.txt diff --git a/.github/actions/run-tests/Dockerfile b/.github/actions/run-tests/Dockerfile index 1e8919567..cbc351d8c 100644 --- a/.github/actions/run-tests/Dockerfile +++ b/.github/actions/run-tests/Dockerfile @@ -1,14 +1,12 @@ FROM golang:1.16-buster as builder -# Up-to-date libgit2 dependencies are only available in -# unstable, as libssh2 in testing/bullseye has been linked -# against gcrypt which causes issues with PKCS* formats. -# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271 -RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \ - && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list +# Build dependencies RUN set -eux; \ - apt-get update \ - && apt-get install -y libgit2-dev/unstable \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + cmake \ + curl \ + python3 \ && apt-get clean \ && apt-get autoremove --purge -y \ && rm -rf /var/lib/apt/lists/* diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 085724d56..66a0e5b57 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -115,4 +115,5 @@ jobs: kubectl -n source-system get helmcharts -oyaml kubectl -n source-system get all kubectl -n source-system logs deploy/source-controller + kubectl -n source-system logs -p deploy/source-controller kubectl -n minio get all diff --git a/.gitignore b/.gitignore index 8f19ec807..c6dff83fc 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,7 @@ # vendor/ bin/ config/release/ + +# Exclude all libgit2 related files, except for the CMakeLists instructions +hack/libgit2/** +!hack/libgit2/CMakeLists.txt diff --git a/Dockerfile b/Dockerfile index 059a25bb3..d6ac6d750 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,61 +1,61 @@ FROM golang:1.16-buster as builder -# Up-to-date libgit2 dependencies are only available in -# unstable, as libssh2 in testing/bullseye has been linked -# against gcrypt which causes issues with PKCS* formats. -# Explicitly listing all build dependencies is required because -# they can only be automagically found for AMD64 builds. -# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271 -RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \ - && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list +# Install depedencies required to built libgit2 RUN set -eux; \ - apt-get update \ - && apt-get install -y \ - libgit2-dev/unstable \ - zlib1g-dev/unstable \ - libssh2-1-dev/unstable \ - libpcre3-dev/unstable \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + cmake \ + curl \ + python3 \ && apt-get clean \ && apt-get autoremove --purge -y \ && rm -rf /var/lib/apt/lists/* +# Configure workspace WORKDIR /workspace -# copy api submodule +# Static build libgit2 and other dependencies +COPY hack/libgit2/CMakeLists.txt libgit2/ +RUN cd libgit2 \ + && cmake -DBUILD_SHARED_LIBS:BOOL=OFF . \ + && cmake --build . + +# Copy api submodule COPY api/ api/ -# copy modules manifests +# Copy modules manifests COPY go.mod go.mod COPY go.sum go.sum -# cache modules +# Cache modules RUN go mod download -# copy source code +# Copy source code COPY main.go main.go COPY controllers/ controllers/ COPY pkg/ pkg/ COPY internal/ internal/ -# build without specifing the arch -RUN CGO_ENABLED=1 go build -o source-controller main.go +# Build a binary with all C dependencies statically linked. +# PKG_CONFIG_PATH is set to the result of our own C libary builds +# to overwrite the git2go defaults that assume a submodule with a +# specific path. +RUN PKG_CONFIG_PATH="$PWD/libgit2/install/lib/pkgconfig" \ + LD_LIBRARY_PATH="$PWD/libgit2/install/lib" \ + go build -o source-controller \ + -tags static,system_libgit2 \ + main.go FROM debian:buster-slim as controller # link repo to the GitHub Container Registry image LABEL org.opencontainers.image.source="https://github.com/fluxcd/source-controller" -# Up-to-date libgit2 dependencies are only available in -# unstable, as libssh2 in testing/bullseye has been linked -# against gcrypt which causes issues with PKCS* formats. -# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271 -RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \ - && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list RUN set -eux; \ apt-get update \ && apt-get install -y \ ca-certificates \ - libgit2-1.1 \ + libc6 \ && apt-get clean \ && apt-get autoremove --purge -y \ && rm -rf /var/lib/apt/lists/* diff --git a/Makefile b/Makefile index 8f1ecc800..d4275fc2a 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,10 @@ IMG ?= fluxcd/source-controller:latest # Produce CRDs that work back to Kubernetes 1.16 CRD_OPTIONS ?= crd:crdVersions=v1 -ENVTEST_BIN_VERSION?=1.19.2 -KUBEBUILDER_ASSETS?=$(shell $(SETUP_ENVTEST) use -i $(ENVTEST_BIN_VERSION) -p path) +CONTROLLER_GEN_VERSION ?= v0.5.0 +GEN_API_REF_DOCS_VERSION ?= 0.3.0 +ENVTEST_BIN_VERSION ?= 1.19.2 +KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use -i $(ENVTEST_BIN_VERSION) -p path) # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -13,121 +15,168 @@ else GOBIN=$(shell go env GOBIN) endif -all: manager +REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel) +LIBGIT2_DIR := hack/libgit2 -# Run tests -test: generate fmt vet manifests api-docs setup-envtest - KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) go test ./... -coverprofile cover.out - cd api; go test ./... -coverprofile cover.out +LIBGIT2_SHARED_DIR := $(LIBGIT2_DIR)/shared +LIBGIT2_SHARED_LIB := $(LIBGIT2_SHARED_DIR)/install/lib +LIBGIT2_SHARED_BUILD := $(LIBGIT2_SHARED_LIB)/libgit2.so +LIBGIT2_SHARED_PKG_CONFIG := $(LIBGIT2_SHARED_LIB)/pkgconfig + +LIBGIT2_STATIC_DIR := $(LIBGIT2_DIR)/static +LIBGIT2_STATIC_LIB := $(LIBGIT2_STATIC_DIR)/install/lib +LIBGIT2_STATIC_BUILD := $(LIBGIT2_STATIC_LIB)/libgit2.a +LIBGIT2_STATIC_PKG_CONFIG := $(LIBGIT2_STATIC_LIB)/pkgconfig + +all: build -# Build manager binary -manager: generate fmt vet +build: $(LIBGIT2_SHARED_BUILD) generate fmt vet ## Build manager binary + PKG_CONFIG_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_SHARED_PKG_CONFIG) \ + LD_LIBRARY_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_SHARED_LIB) \ go build -o bin/manager main.go -# Run against the configured Kubernetes cluster in ~/.kube/config -run: generate fmt vet manifests +build-static: $(LIBGIT2_STATIC_BUILD) generate fmt vet ## Build static manager binary + PKG_CONFIG_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_STATIC_PKG_CONFIG) \ + LD_LIBRARY_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_STATIC_LIB) \ + go build -tags static,system_libgit2 -o bin/manager-static main.go + +test: $(LIBGIT2_SHARED_BUILD) test-api ## Run tests + PKG_CONFIG_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_SHARED_PKG_CONFIG) \ + LD_LIBRARY_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_SHARED_LIB) \ + go test ./... -coverprofile cover.out + +test-static: $(LIBGIT2_STATIC_BUILD) test-api ## Run static tests + PKG_CONFIG_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_STATIC_PKG_CONFIG) \ + LD_LIBRARY_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_STATIC_LIB) \ + go test -tags static,system_libgit2 ./... -coverprofile cover.out + +test-api: ## Run api tests + cd api; go test ./... -coverprofile cover.out + +run: $(LIBGIT2_SHARED_BUILD) generate fmt vet manifests ## Run against the configured Kubernetes cluster in ~/.kube/config + PKG_CONFIG_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_SHARED_PKG_CONFIG) \ + LD_LIBRARY_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_SHARED_LIB) \ go run ./main.go -# Install CRDs into a cluster -install: manifests +run-static: $(LIBGIT2_STATIC_BUILD) generate fmt vet manifests ## Static run against the configured Kubernetes cluster in ~/.kube/config + PKG_CONFIG_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_STATIC_PKG_CONFIG) \ + LD_LIBRARY_PATH=$(REPOSITORY_ROOT)/$(LIBGIT2_STATIC_LIB) \ + go run ./main.go + +install: manifests ## Install CRDs into a cluster kustomize build config/crd | kubectl apply -f - -# Uninstall CRDs from a cluster -uninstall: manifests +uninstall: manifests ## Uninstall CRDs from a cluster kustomize build config/crd | kubectl delete -f - -# Deploy controller in the configured Kubernetes cluster in ~/.kube/config -deploy: manifests +deploy: manifests ## Deploy controller in the configured Kubernetes cluster in ~/.kube/config cd config/manager && kustomize edit set image fluxcd/source-controller=${IMG} kustomize build config/default | kubectl apply -f - -# Deploy controller dev image in the configured Kubernetes cluster in ~/.kube/config -dev-deploy: +dev-deploy: ## Deploy controller dev image in the configured Kubernetes cluster in ~/.kube/config mkdir -p config/dev && cp config/default/* config/dev cd config/dev && kustomize edit set image fluxcd/source-controller=${IMG} kustomize build config/dev | kubectl apply -f - rm -rf config/dev -# Generate manifests e.g. CRD, RBAC etc. -manifests: controller-gen +manifests: controller-gen ## Generate manifests, e.g. CRD, RBAC, etc. $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role paths="./..." output:crd:artifacts:config="config/crd/bases" cd api; $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role paths="./..." output:crd:artifacts:config="../config/crd/bases" -# Generate API reference documentation -api-docs: gen-crd-api-reference-docs +api-docs: gen-crd-api-reference-docs ## Generate API reference documentation $(API_REF_GEN) -api-dir=./api/v1beta1 -config=./hack/api-docs/config.json -template-dir=./hack/api-docs/template -out-file=./docs/api/source.md -# Run go mod tidy -tidy: +tidy: ## Run go mod tidy go mod tidy cd api; go mod tidy -# Run go fmt against code -fmt: +fmt: ## Run go fmt against code go fmt ./... cd api; go fmt ./... -# Run go vet against code -vet: +vet: ## Run go vet against code go vet ./... cd api; go vet ./... -# Generate code -generate: controller-gen +generate: controller-gen ## Generate API code cd api; $(CONTROLLER_GEN) object:headerFile="../hack/boilerplate.go.txt" paths="./..." -# Build the docker image -docker-build: +docker-build: ## Build the docker image docker build . -t ${IMG} -# Push the docker image -docker-push: +docker-push: ## Push docker image docker push ${IMG} -# Find or download controller-gen -controller-gen: +$(LIBGIT2_SHARED_BUILD): $(LIBGIT2_SHARED_DIR) + (cd $(LIBGIT2_SHARED_DIR) && cmake -DBUILD_SHARED_LIBS:BOOL=ON .. && cmake --build .) + +$(LIBGIT2_STATIC_BUILD): $(LIBGIT2_STATIC_DIR) + (cd $(LIBGIT2_STATIC_DIR) && cmake -DBUILD_SHARED_LIBS:BOOL=OFF .. && cmake --build .) + +$(LIBGIT2_SHARED_DIR): + mkdir -p "$@" + +$(LIBGIT2_STATIC_DIR): + mkdir -p "$@" + +.PHONY: clean +clean: ## Removes dependency builds and clears Go cache + rm -rf $(LIBGIT2_SHARED_DIR) + rm -rf $(LIBGIT2_STATIC_DIR) + + # C-binding directives in Go are cached, which means that e.g. + # changes to $PKG_CONFIG_PATH do not have an effect until this cache + # is cleared and the package(s) is forced to be rebuild. + # This can also be done using `go -a`, but this would make + # things slow for something that is not expected to change much, + # unless you are specifically dealing with build related changes. + go clean + +controller-gen: ## Find or download controller-gen ifeq (, $(shell which controller-gen)) @{ \ - set -e ;\ - CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ - cd $$CONTROLLER_GEN_TMP_DIR ;\ - go mod init tmp ;\ - go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.5.0 ;\ - rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ + set -e; \ + CONTROLLER_GEN_TMP_DIR=$$(mktemp -d); \ + cd $$CONTROLLER_GEN_TMP_DIR; \ + go mod init tmp; \ + go get sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_GEN_VERSION); \ + rm -rf $$CONTROLLER_GEN_TMP_DIR; \ } CONTROLLER_GEN=$(GOBIN)/controller-gen else CONTROLLER_GEN=$(shell which controller-gen) endif -# Find or download gen-crd-api-reference-docs -gen-crd-api-reference-docs: +gen-crd-api-reference-docs: ## Find or download gen-crd-api-reference-docs ifeq (, $(shell which gen-crd-api-reference-docs)) @{ \ - set -e ;\ - API_REF_GEN_TMP_DIR=$$(mktemp -d) ;\ - cd $$API_REF_GEN_TMP_DIR ;\ - go mod init tmp ;\ - go get github.com/ahmetb/gen-crd-api-reference-docs@v0.3.0 ;\ - rm -rf $$API_REF_GEN_TMP_DIR ;\ + set -e; \ + API_REF_GEN_TMP_DIR=$$(mktemp -d); \ + cd $$API_REF_GEN_TMP_DIR; \ + go mod init tmp; \ + go get github.com/ahmetb/gen-crd-api-reference-docs@$(GEN_API_REF_DOCS_VERSION); \ + rm -rf $$API_REF_GEN_TMP_DIR; \ } API_REF_GEN=$(GOBIN)/gen-crd-api-reference-docs else API_REF_GEN=$(shell which gen-crd-api-reference-docs) endif -# Find or download setup-envtest -setup-envtest: +setup-envtest: ## Find or download setup-envtest ifeq (, $(shell which setup-envtest)) @{ \ - set -e ;\ - SETUP_ENVTEST_TMP_DIR=$$(mktemp -d) ;\ - cd $$SETUP_ENVTEST_TMP_DIR ;\ - go mod init tmp ;\ - go get sigs.k8s.io/controller-runtime/tools/setup-envtest@latest ;\ - rm -rf $$SETUP_ENVTEST_TMP_DIR ;\ + set -e; \ + SETUP_ENVTEST_TMP_DIR=$$(mktemp -d); \ + cd $$SETUP_ENVTEST_TMP_DIR; \ + go mod init tmp; \ + go get sigs.k8s.io/controller-runtime/tools/setup-envtest@latest; \ + rm -rf $$SETUP_ENVTEST_TMP_DIR; \ } SETUP_ENVTEST=$(GOBIN)/setup-envtest else SETUP_ENVTEST=$(shell which setup-envtest) endif + +.PHONY: help +help: ## Display this help menu. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/go.mod b/go.mod index 2b0337332..8a0cb3aba 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/go-git/go-billy/v5 v5.3.1 github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v0.4.0 - github.com/libgit2/git2go/v31 v31.4.14 + github.com/libgit2/git2go/v32 v32.0.4 github.com/minio/minio-go/v7 v7.0.10 github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.14.0 diff --git a/go.sum b/go.sum index 477e3ddbc..c2e7a1cca 100644 --- a/go.sum +++ b/go.sum @@ -545,8 +545,8 @@ github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6Fm github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/libgit2/git2go/v31 v31.4.14 h1:6GOd3965D9e/+gjxCwZF4eQ+vB9kKB4yKFqdQr6XZ2E= -github.com/libgit2/git2go/v31 v31.4.14/go.mod h1:c/rkJcBcUFx6wHaT++UwNpKvIsmPNqCeQ/vzO4DrEec= +github.com/libgit2/git2go/v32 v32.0.4 h1:qK2yWGh88K2Gh76E1+vUEsjKDfOAq0J2THKaoaFIPbA= +github.com/libgit2/git2go/v32 v32.0.4/go.mod h1:FAA2ePV5PlLjw1ccncFIvu2v8hJSZVN5IzEn4lo/vwo= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= diff --git a/hack/libgit2/CMakeLists.txt b/hack/libgit2/CMakeLists.txt new file mode 100644 index 000000000..7cc0c33e1 --- /dev/null +++ b/hack/libgit2/CMakeLists.txt @@ -0,0 +1,76 @@ +cmake_minimum_required(VERSION 3.5.1) + +project(Superbuild) + +# Set a default build type if none was specified +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + message(STATUS "Setting build type to 'Release' as none was specified.") + set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE) + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS + "Debug" "Release" "MinSizeRel" "RelWithDebInfo") +endif() + +option(BUILD_SHARED_LIBS "Build with shared libs." ON) + +option(USE_EXTERNAL_INSTALL "If enabled, install dependencies to CMAKE_INSTALL_PREFIX." OFF) +mark_as_advanced(USE_EXTERNAL_INSTALL) + +include(ExternalProject) + +set_property(DIRECTORY PROPERTY EP_PREFIX ${Superbuild_BINARY_DIR}) +set(install_prefix ${Superbuild_BINARY_DIR}/install) +if (USE_EXTERNAL_INSTALL) + set(install_prefix ${CMAKE_INSTALL_PREFIX}) +else() + mark_as_advanced(CMAKE_INSTALL_PREFIX) +endif() + +set(default_cmake_args + "-DCMAKE_PREFIX_PATH:PATH=${install_prefix};${DRAKE_SUPERBUILD_PREFIX_PATH};${CMAKE_PREFIX_PATH}" + "-DCMAKE_INSTALL_PREFIX:PATH=${install_prefix}" + "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}" + "-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON" + "-DBUILD_SHARED_LIBS:BOOL=${BUILD_SHARED_LIBS}" + "-DENABLE_TESTING:BOOL=OFF" + "-DENABLE_PROGRAMS:BOOL=OFF" + "-DBUILD_DOCUMENTATION:BOOL=OFF" + "-DBUILD_EXAMPLE:BOOL=OFF" +) + +ExternalProject_Add(mbedtls + URL https://github.com/ARMmbed/mbedtls/archive/refs/tags/v2.27.0.tar.gz + URL_HASH SHA256=2a07856e541f0e5f6eaee4f78018c52f25bd244ed76f9020dea54a8b02cac6ea + PATCH_COMMAND + ./scripts/config.pl set MBEDTLS_THREADING_C && + ./scripts/config.pl set MBEDTLS_THREADING_PTHREAD && + ./scripts/config.pl set MBEDTLS_MD4_C + CMAKE_ARGS + ${default_cmake_args} + -DUSE_SHARED_MBEDTLS_LIBRARY:BOOL=${BUILD_SHARED_LIBS} +) + +ExternalProject_Add(libssh2 + DEPENDS mbedtls + URL https://github.com/libssh2/libssh2/archive/refs/tags/libssh2-1.10.0.tar.gz + URL_HASH SHA256=31469ccfc71a5247c926e3f0938e122cbb7a7a4a1bdf1cf2d3121f78b558d261 + CMAKE_ARGS + ${default_cmake_args} + -DCRYPTO_BACKEND:STRING=mbedTLS +) + +ExternalProject_Add(libgit2 + DEPENDS mbedtls + DEPENDS libssh2 + URL https://github.com/libgit2/libgit2/archive/109b4c887ffb63962c7017a66fc4a1f48becb48e.tar.gz + URL_HASH SHA256=bc4ef7d6628d2248995bbd86ad77eb96376d683e7121779c7abde480928ae21a + CMAKE_ARGS + ${default_cmake_args} + -DCMAKE_C_FLAGS:STRING="-fPIC" + -DBUILD_CLAR:BOOL:BOOL=OFF + -DBUILD_DEPRECATED_HARD:BOOL=ON + -DTHREADSAFE:BOOL=ON + -DUSE_BUNDLED_ZLIB:BOOL=ON + -DUSE_HTTPS:STRING=mbedTLS + -DUSE_HTTP_PARSER:STRING=builtin + -DREGEX_BACKEND:STRING=builtin +) diff --git a/internal/fs/rename.go b/internal/fs/rename.go index a1b4a411d..bad1f4778 100644 --- a/internal/fs/rename.go +++ b/internal/fs/rename.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !windows // +build !windows package fs diff --git a/internal/fs/rename_windows.go b/internal/fs/rename_windows.go index 3b5650573..fa9a0b4d9 100644 --- a/internal/fs/rename_windows.go +++ b/internal/fs/rename_windows.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build windows // +build windows package fs diff --git a/pkg/git/git.go b/pkg/git/git.go index 6ec7257ae..7e9e89079 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -20,7 +20,7 @@ import ( "context" "github.com/go-git/go-git/v5/plumbing/transport" - git2go "github.com/libgit2/git2go/v31" + git2go "github.com/libgit2/git2go/v32" corev1 "k8s.io/api/core/v1" ) diff --git a/pkg/git/libgit2/checkout.go b/pkg/git/libgit2/checkout.go index 01363f8fa..29cd8e917 100644 --- a/pkg/git/libgit2/checkout.go +++ b/pkg/git/libgit2/checkout.go @@ -24,7 +24,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/fluxcd/pkg/version" - git2go "github.com/libgit2/git2go/v31" + git2go "github.com/libgit2/git2go/v32" "github.com/fluxcd/pkg/gitutil" @@ -59,7 +59,7 @@ type CheckoutBranch struct { func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) { repo, err := git2go.Clone(url, path, &git2go.CloneOptions{ - FetchOptions: &git2go.FetchOptions{ + FetchOptions: git2go.FetchOptions{ DownloadTags: git2go.DownloadTagsNone, RemoteCallbacks: git2go.RemoteCallbacks{ CredentialsCallback: auth.CredCallback, @@ -88,7 +88,7 @@ type CheckoutTag struct { func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) { repo, err := git2go.Clone(url, path, &git2go.CloneOptions{ - FetchOptions: &git2go.FetchOptions{ + FetchOptions: git2go.FetchOptions{ DownloadTags: git2go.DownloadTagsAll, RemoteCallbacks: git2go.RemoteCallbacks{ CredentialsCallback: auth.CredCallback, @@ -132,7 +132,7 @@ type CheckoutCommit struct { func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, auth *git.Auth) (git.Commit, string, error) { repo, err := git2go.Clone(url, path, &git2go.CloneOptions{ - FetchOptions: &git2go.FetchOptions{ + FetchOptions: git2go.FetchOptions{ DownloadTags: git2go.DownloadTagsNone, RemoteCallbacks: git2go.RemoteCallbacks{ CredentialsCallback: auth.CredCallback, @@ -156,7 +156,7 @@ func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, auth *g if err != nil { return nil, "", fmt.Errorf("git worktree error: %w", err) } - err = repo.CheckoutTree(tree, &git2go.CheckoutOpts{ + err = repo.CheckoutTree(tree, &git2go.CheckoutOptions{ Strategy: git2go.CheckoutForce, }) if err != nil { @@ -177,7 +177,7 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, auth *g } repo, err := git2go.Clone(url, path, &git2go.CloneOptions{ - FetchOptions: &git2go.FetchOptions{ + FetchOptions: git2go.FetchOptions{ DownloadTags: git2go.DownloadTagsAll, RemoteCallbacks: git2go.RemoteCallbacks{ CredentialsCallback: auth.CredCallback, diff --git a/pkg/git/libgit2/checkout_test.go b/pkg/git/libgit2/checkout_test.go index 6de5484d8..c0c742661 100644 --- a/pkg/git/libgit2/checkout_test.go +++ b/pkg/git/libgit2/checkout_test.go @@ -25,14 +25,14 @@ import ( "path" "testing" - git2go "github.com/libgit2/git2go/v31" + git2go "github.com/libgit2/git2go/v32" "github.com/fluxcd/source-controller/pkg/git" ) func TestCheckoutTagSemVer_Checkout(t *testing.T) { - certCallback := func(cert *git2go.Certificate, valid bool, hostname string) git2go.ErrorCode { - return 0 + certCallback := func(cert *git2go.Certificate, valid bool, hostname string) error { + return nil } auth := &git.Auth{CertCallback: certCallback} @@ -57,9 +57,10 @@ func TestCheckoutTagSemVer_Checkout(t *testing.T) { if _, err := io.Copy(h, f); err != nil { t.Error(err) } + const expectedHash = "2bd1707542a11f987ee24698dcc095a9f57639f401133ef6a29da97bf8f3f302" fileHash := hex.EncodeToString(h.Sum(nil)) - if fileHash != "2bd1707542a11f987ee24698dcc095a9f57639f401133ef6a29da97bf8f3f302" { - t.Errorf("expected files not checked out. Expected hash %s, got %s", "2bd1707542a11f987ee24698dcc095a9f57639f401133ef6a29da97bf8f3f302", fileHash) + if fileHash != expectedHash { + t.Errorf("expected files not checked out. Expected hash %s, got %s", expectedHash, fileHash) } semVer := CheckoutSemVer{ diff --git a/pkg/git/libgit2/commit.go b/pkg/git/libgit2/commit.go index 1e459f319..cd09fff7e 100644 --- a/pkg/git/libgit2/commit.go +++ b/pkg/git/libgit2/commit.go @@ -23,7 +23,7 @@ import ( "golang.org/x/crypto/openpgp" - git2go "github.com/libgit2/git2go/v31" + git2go "github.com/libgit2/git2go/v32" corev1 "k8s.io/api/core/v1" ) diff --git a/pkg/git/libgit2/transport.go b/pkg/git/libgit2/transport.go index da3d04e92..1bf32d65c 100644 --- a/pkg/git/libgit2/transport.go +++ b/pkg/git/libgit2/transport.go @@ -29,7 +29,7 @@ import ( "net/url" "strings" - git2go "github.com/libgit2/git2go/v31" + git2go "github.com/libgit2/git2go/v32" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/knownhosts" corev1 "k8s.io/api/core/v1" @@ -66,8 +66,8 @@ func (s *BasicAuth) Method(secret corev1.Secret) (*git.Auth, error) { password = string(d) } if username != "" && password != "" { - credCallback = func(url string, usernameFromURL string, allowedTypes git2go.CredType) (*git2go.Cred, error) { - cred, err := git2go.NewCredUserpassPlaintext(username, password) + credCallback = func(url string, usernameFromURL string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) { + cred, err := git2go.NewCredentialUserpassPlaintext(username, password) if err != nil { return nil, err } @@ -77,11 +77,11 @@ func (s *BasicAuth) Method(secret corev1.Secret) (*git.Auth, error) { var certCallback git2go.CertificateCheckCallback if caFile, ok := secret.Data[git.CAFile]; ok { - certCallback = func(cert *git2go.Certificate, valid bool, hostname string) git2go.ErrorCode { + certCallback = func(cert *git2go.Certificate, valid bool, hostname string) error { roots := x509.NewCertPool() ok := roots.AppendCertsFromPEM(caFile) if !ok { - return git2go.ErrCertificate + return fmt.Errorf("failed to append data from '%s' to certificate pool", git.CAFile) } opts := x509.VerifyOptions{ @@ -90,9 +90,9 @@ func (s *BasicAuth) Method(secret corev1.Secret) (*git.Auth, error) { } _, err := cert.X509.Verify(opts) if err != nil { - return git2go.ErrCertificate + return err } - return git2go.ErrOk + return nil } } @@ -137,14 +137,14 @@ func (s *PublicKeyAuth) Method(secret corev1.Secret) (*git.Auth, error) { user = git.DefaultPublicKeyAuthUser } - credCallback := func(url string, usernameFromURL string, allowedTypes git2go.CredType) (*git2go.Cred, error) { - cred, err := git2go.NewCredSshKeyFromMemory(user, "", string(identity), string(password)) + credCallback := func(url string, usernameFromURL string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) { + cred, err := git2go.NewCredentialSSHKeyFromMemory(user, "", string(identity), string(password)) if err != nil { return nil, err } return cred, nil } - certCallback := func(cert *git2go.Certificate, valid bool, hostname string) git2go.ErrorCode { + certCallback := func(cert *git2go.Certificate, valid bool, hostname string) error { // First, attempt to split the configured host and port to validate // the port-less hostname given to the callback. host, _, err := net.SplitHostPort(s.host) @@ -157,20 +157,20 @@ func (s *PublicKeyAuth) Method(secret corev1.Secret) (*git.Auth, error) { // Check if the configured host matches the hostname given to // the callback. if host != hostname { - return git2go.ErrUser + return fmt.Errorf("'%s' does not match configured hostname '%s'", hostname, host) } // We are now certain that the configured host and the hostname // given to the callback match. Use the configured host (that - // includes the port), and normalize it so we can check if there + // includes the port), and normalize it, so we can check if there // is an entry for the hostname _and_ port. host = knownhosts.Normalize(s.host) for _, k := range kk { if k.matches(host, cert.Hostkey) { - return git2go.ErrOk + return nil } } - return git2go.ErrCertificate + return fmt.Errorf("could not find matching fingerprint for host '%s'", host) } return &git.Auth{CredCallback: credCallback, CertCallback: certCallback}, nil diff --git a/pkg/git/libgit2/transport_test.go b/pkg/git/libgit2/transport_test.go index 733fa0c96..4566e6875 100644 --- a/pkg/git/libgit2/transport_test.go +++ b/pkg/git/libgit2/transport_test.go @@ -21,7 +21,7 @@ import ( "reflect" "testing" - git2go "github.com/libgit2/git2go/v31" + git2go "github.com/libgit2/git2go/v32" corev1 "k8s.io/api/core/v1" "github.com/fluxcd/source-controller/pkg/git"