diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ea6c076..952ff05 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -17,121 +17,7 @@ permissions: id-token: write # needed for keyless signing jobs: - # changing this job name for some reason caused GitHub checks to - # hang at `Expected — Waiting for status to be reported`. build: - runs-on: ubuntu-latest - env: - PLATFORMS: linux/amd64,linux/arm/v7,linux/arm64 - services: - registry: - image: registry:2 - ports: - - 5000:5000 - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Unshallow - run: git fetch --prune --unshallow - - name: Setup Cosign - uses: sigstore/cosign-installer@main - - name: Set up QEMU - id: qemu - uses: docker/setup-qemu-action@v2 - with: - image: tonistiigi/binfmt:latest - platforms: ${{ env.PLATFORMS }} - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - with: - config-inline: | - [worker.oci] - max-parallelism = 4 - driver-opts: network=host - - name: Inspect builder - run: | - echo "Name: ${{ steps.buildx.outputs.name }}" - echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" - echo "Status: ${{ steps.buildx.outputs.status }}" - echo "Flags: ${{ steps.buildx.outputs.flags }}" - echo "Platforms: ${{ steps.buildx.outputs.platforms }}" - - name: Cache Docker layers - uses: actions/cache@v3 - id: cache - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-ghcache-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-ghcache- - - run: cat ./hack/static.sh - - name: Build candidate image - libgit2 compiled with libssh2 and openssl - id: build_candidate_libgit2_all - uses: docker/build-push-action@v3 - with: - context: . - file: Dockerfile - platforms: ${{ env.PLATFORMS }} - push: true - tags: localhost:5000/${{ github.repository_owner }}/golang-with-libgit2-all:latest - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - - name: Inspect candidate images - run: | - docker buildx imagetools inspect localhost:5000/${{ github.repository_owner }}/golang-with-libgit2-all:latest - - name: Test candidate images - id: test_candidate - run: | - IMG=localhost:5000/${{ github.repository_owner }}/golang-with-libgit2-all make test - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache - - name: Login to GHCR - if: github.event_name != 'pull_request' - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Compose release candidate metadata - golang-with-libgit2-all - id: meta_libgit2_all - uses: docker/metadata-action@v3 - if: github.event_name != 'pull_request' - with: - images: | - ghcr.io/${{ github.repository_owner }}/golang-with-libgit2-all - tags: | - type=semver,pattern=v{{version}} - type=sha - type=sha,format=long - - name: Release candidate image - golang-with-libgit2-all - id: release_candidate_libgit2_all - if: github.event_name != 'pull_request' - uses: docker/build-push-action@v3 - with: - context: . - file: Dockerfile - platforms: ${{ env.PLATFORMS }} - push: true - tags: ${{ steps.meta_libgit2_all.outputs.tags }} - labels: ${{ steps.meta_libgit2_all.outputs.labels }} - cache-from: type=local,src=/tmp/.buildx-cache - - name: Sign images - env: - COSIGN_EXPERIMENTAL: 1 - run: | - array=($(echo ${{ steps.meta_libgit2_all.outputs.tags }} | tr '\n' " ")) - for image in "${array[@]}" - do - echo "Signing $image" - cosign sign $image - done - - docker-build-libgit2-only: runs-on: ubuntu-latest env: PLATFORMS: linux/amd64,linux/arm/v7,linux/arm64 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 666a680..fd0c748 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -17,30 +17,6 @@ permissions: id-token: write # needed for keyless signing jobs: - linux-x86_64-all-libs: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Build static libraries - libgit2 compiled with libssh2 and openssl - run: | - TARGET_DIR=${GITHUB_WORKSPACE}/build/libgit2-linux-all-libs \ - BUILD_ROOT_DIR=${GITHUB_WORKSPACE}/libgit2/build/amd \ - ./hack/static.sh all - - mkdir -p ./libgit2-linux-all-libs/ - mv ${GITHUB_WORKSPACE}/build/libgit2-linux-all-libs/include ./libgit2-linux-all-libs/ - mv ${GITHUB_WORKSPACE}/build/libgit2-linux-all-libs/share ./libgit2-linux-all-libs/ - mv ${GITHUB_WORKSPACE}/build/libgit2-linux-all-libs/lib ./libgit2-linux-all-libs/ - mv ${GITHUB_WORKSPACE}/build/libgit2-linux-all-libs/lib64 ./libgit2-linux-all-libs/ - - tar -zcvf linux-x86_64-all-libs.tar.gz libgit2-linux-all-libs - rm -rf ${GITHUB_WORKSPACE}/build ${GITHUB_WORKSPACE}/libgit2 - - uses: actions/upload-artifact@v3 - with: - name: release-artifact - path: "*.tar.gz" - if-no-files-found: error linux-x86_64-libgit2-only: runs-on: ubuntu-latest @@ -65,88 +41,6 @@ jobs: path: "*.tar.gz" if-no-files-found: error - darwin-all-libs: - # This job builds and releases "universal libraries" that are - # supported by both darwin-amd64 and darwin-arm64. - # - # First builds in amd64, then cross-compile in arm64. Later combining - # both outcomes onto a single binary for each static library. - # - # `macos-11` has been picked as support for arm64 was only added on Xcode 12. - # Although some minor versions of Catalina 10.15 can support it, at the time - # of testing, GitHub's macos-10.15 did not seem to. - # Cross-compiling to arm64 on that runner consistently failed. - runs-on: macos-11 - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Build universal static libraries for Darwin - libgit2 compiled with libssh2 and openssl - run: | - # The amd64 is used as base, using the target dir name (darwin-all-libs) - # instead of the platform specific one, removes the need of replacing the - # path in the .pc files. - TARGET_DIR=${GITHUB_WORKSPACE}/build/darwin-all-libs \ - BUILD_ROOT_DIR=${GITHUB_WORKSPACE}/libgit2/build/amd \ - ./hack/static.sh all - - TARGET_DIR=${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64 \ - BUILD_ROOT_DIR=${GITHUB_WORKSPACE}/libgit2/build/arm \ - TARGET_ARCH=arm64 \ - CMAKE_APPLE_SILICON_PROCESSOR=arm64 \ - ./hack/static.sh all - - - LIBGIT2_SED="s;-L/Applications/Xcode.* ;;g" - LIBGIT2PC="${GITHUB_WORKSPACE}/build/darwin-all-libs/lib/pkgconfig/libgit2.pc" - - # pkgconfig includes absolute paths that are specific to the runner machine. - # We need to remove the absolute path for iconv, so when the libgit2 library - # is consumed pkgconfig will automatically try to find it through the default - # search paths on the target machine. - if command -v gsed &> /dev/null; then - gsed -i "${LIBGIT2_SED}" "${LIBGIT2PC}" - else - sed -i "" "${LIBGIT2_SED}" "${LIBGIT2PC}" - fi - - mkdir -p ./darwin-all-libs/lib - mv ${GITHUB_WORKSPACE}/build/darwin-all-libs/include ./darwin-all-libs/ - mv ${GITHUB_WORKSPACE}/build/darwin-all-libs/share ./darwin-all-libs/ - mv ${GITHUB_WORKSPACE}/build/darwin-all-libs/lib/cmake ./darwin-all-libs/lib/ - mv ${GITHUB_WORKSPACE}/build/darwin-all-libs/lib/engines-3 ./darwin-all-libs/lib/ - mv ${GITHUB_WORKSPACE}/build/darwin-all-libs/lib/ossl-modules ./darwin-all-libs/lib/ - mv ${GITHUB_WORKSPACE}/build/darwin-all-libs/lib/pkgconfig ./darwin-all-libs/lib/ - - libtool -static -o ./darwin-all-libs/lib/libcrypto.a \ - ${GITHUB_WORKSPACE}/build/darwin-all-libs/lib/libcrypto.a \ - ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libcrypto.a - - libtool -static -o ./darwin-all-libs/lib/libgit2.a \ - ${GITHUB_WORKSPACE}/build/darwin-all-libs/lib/libgit2.a \ - ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libgit2.a - - libtool -static -o ./darwin-all-libs/lib/libssh2.a \ - ${GITHUB_WORKSPACE}/build/darwin-all-libs/lib/libssh2.a \ - ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libssh2.a - - libtool -static -o ./darwin-all-libs/lib/libssl.a \ - ${GITHUB_WORKSPACE}/build/darwin-all-libs/lib/libssl.a \ - ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libssl.a - - libtool -static -o ./darwin-all-libs/lib/libz.a \ - ${GITHUB_WORKSPACE}/build/darwin-all-libs/lib/libz.a \ - ${GITHUB_WORKSPACE}/build/libgit2-darwin-arm64/lib/libz.a - - tar -zcvf darwin-all-libs.tar.gz darwin-all-libs - rm -rf ${GITHUB_WORKSPACE}/build ${GITHUB_WORKSPACE}/libgit2 - env: - MACOSX_DEPLOYMENT_TARGET: 10.15 - - uses: actions/upload-artifact@v3 - with: - name: release-artifact - path: "*.tar.gz" - if-no-files-found: error - # similar to darwin-all-libs, but only compiles libgit2. darwin-libgit2-only: runs-on: macos-11 @@ -165,7 +59,8 @@ jobs: CMAKE_APPLE_SILICON_PROCESSOR=arm64 \ ./hack/static.sh build_libgit2_only - LIBGIT2_SED="s;-L/Applications/Xcode.* ;;g" + LIBGIT2_WORKDIR_SED="s;${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64;${GITHUB_WORKSPACE}/build/darwin-libgit2-only;g" + LIBGIT2_XCODE_SED="s;-L/Applications/Xcode.* ;;g" LIBGIT2PC="${GITHUB_WORKSPACE}/build/libgit2-darwin-amd64/lib/pkgconfig/libgit2.pc" # pkgconfig includes absolute paths that are specific to the runner machine. @@ -173,9 +68,11 @@ jobs: # is consumed pkgconfig will automatically try to find it through the default # search paths on the target machine. if command -v gsed &> /dev/null; then - gsed -i "${LIBGIT2_SED}" "${LIBGIT2PC}" + gsed -i "${LIBGIT2_WORKDIR_SED}" "${LIBGIT2PC}" + gsed -i "${LIBGIT2_XCODE_SED}" "${LIBGIT2PC}" else - sed -i "" "${LIBGIT2_SED}" "${LIBGIT2PC}" + sed -i "" "${LIBGIT2_WORKDIR_SED}" "${LIBGIT2PC}" + sed -i "" "${LIBGIT2_XCODE_SED}" "${LIBGIT2PC}" fi mkdir ./darwin-libgit2-only @@ -200,9 +97,7 @@ jobs: runs-on: ubuntu-latest needs: [ - linux-x86_64-all-libs, linux-x86_64-libgit2-only, - darwin-all-libs, darwin-libgit2-only, ] if: ${{ always() && contains(join(needs.*.result, ','), 'success') }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 480567a..3f15f55 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,9 +14,6 @@ permissions: {} jobs: darwin-dev-test: - strategy: - matrix: - libgit_only: [true, false] runs-on: macos-11 steps: - name: Checkout @@ -26,13 +23,9 @@ jobs: rm -rf ${GITHUB_WORKSPACE}/build make dev-test env: - LIBGIT2_ONLY: ${{ matrix.libgit_only }} MACOSX_DEPLOYMENT_TARGET: 10.15 linux-dev-test: - strategy: - matrix: - libgit_only: [true, false] runs-on: ubuntu-latest steps: - name: Checkout @@ -41,5 +34,3 @@ jobs: run: | rm -rf ${GITHUB_WORKSPACE}/build make dev-test - env: - LIBGIT2_ONLY: ${{ matrix.libgit_only }} diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index ab9eb8f..0000000 --- a/Dockerfile +++ /dev/null @@ -1,83 +0,0 @@ -# This Dockerfile builds and packages libgit2 linked with openssl and libssh2. - -ARG BASE_VARIANT=alpine -ARG GO_VERSION=1.18 -ARG XX_VERSION=1.1.0 - -FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx - -FROM --platform=$BUILDPLATFORM ${BASE_VARIANT} AS build-base - -RUN apk add --no-cache \ - bash \ - curl \ - build-base \ - linux-headers \ - perl \ - cmake \ - pkgconfig \ - gcc \ - musl-dev \ - clang \ - lld - -COPY --from=xx / / - -FROM build-base AS build-cross - -ARG TARGETPLATFORM - -RUN xx-apk add --no-cache \ - build-base \ - pkgconfig \ - gcc \ - musl-dev \ - clang \ - lld \ - llvm \ - linux-headers - -WORKDIR /build -COPY hack/static.sh . - -ENV CC=xx-clang -ENV CXX=xx-clang++ - -RUN CHOST=$(xx-clang --print-target-triple) \ - ./static.sh build_libz - -RUN CHOST=$(xx-clang --print-target-triple) \ - ./static.sh build_openssl - -RUN export LIBRARY_PATH="/usr/local/$(xx-info triple)/lib:/usr/local/$(xx-info triple)/lib64:${LIBRARY_PATH}" && \ - export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig:/usr/local/$(xx-info triple)/lib64/pkgconfig" && \ - export OPENSSL_ROOT_DIR="/usr/local/$(xx-info triple)" && \ - export OPENSSL_CRYPTO_LIBRARY="/usr/local/$(xx-info triple)/lib64" && \ - export OPENSSL_INCLUDE_DIR="/usr/local/$(xx-info triple)/include/openssl" - -RUN ./static.sh build_libssh2 -RUN ./static.sh build_libgit2 - - -# trimmed removes all non necessary files (i.e. openssl binary). -FROM build-cross AS trimmed - -ARG TARGETPLATFORM -RUN mkdir -p /trimmed/usr/local/$(xx-info triple)/ && \ - mkdir -p /trimmed/usr/local/$(xx-info triple)/share - -RUN cp -r /usr/local/$(xx-info triple)/lib/ /trimmed/usr/local/$(xx-info triple)/ && \ - cp -r /usr/local/$(xx-info triple)/lib64/ /trimmed/usr/local/$(xx-info triple)/ | true && \ - cp -r /usr/local/$(xx-info triple)/include/ /trimmed/usr/local/$(xx-info triple)/ && \ - cp -r /usr/local/$(xx-info triple)/share/doc/ /trimmed/usr/local/$(xx-info triple)/share/ - -FROM scratch as libs-arm64 -COPY --from=trimmed /trimmed/ / - -FROM scratch as libs-amd64 -COPY --from=trimmed /trimmed/ / - -FROM scratch as libs-armv7 -COPY --from=trimmed /trimmed/ / - -FROM libs-$TARGETARCH$TARGETVARIANT as libs diff --git a/Dockerfile.test b/Dockerfile.test deleted file mode 100644 index 3b8e6d7..0000000 --- a/Dockerfile.test +++ /dev/null @@ -1,148 +0,0 @@ -# This Dockerfile builds libgit2 linked with openssl and libssh2; and tests it against git2go. - -ARG BASE_VARIANT=alpine -ARG GO_VERSION=1.18 -ARG XX_VERSION=1.1.0 - -FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx - -FROM --platform=$BUILDPLATFORM ${BASE_VARIANT} AS build-base - -RUN apk add --no-cache \ - bash \ - curl \ - build-base \ - linux-headers \ - perl \ - cmake \ - pkgconfig \ - gcc \ - musl-dev \ - clang \ - lld - -COPY --from=xx / / - -FROM build-base AS build-cross - -ARG TARGETPLATFORM - -RUN xx-apk add --no-cache \ - build-base \ - pkgconfig \ - gcc \ - musl-dev \ - clang \ - lld \ - llvm \ - linux-headers - -WORKDIR /build -COPY hack/static.sh . - -ENV CC=xx-clang -ENV CXX=xx-clang++ - -RUN CHOST=$(xx-clang --print-target-triple) \ - ./static.sh build_libz - -RUN CHOST=$(xx-clang --print-target-triple) \ - ./static.sh build_openssl - -RUN export LIBRARY_PATH="/usr/local/$(xx-info triple)/lib:/usr/local/$(xx-info triple)/lib64:${LIBRARY_PATH}" && \ - export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig:/usr/local/$(xx-info triple)/lib64/pkgconfig" && \ - export OPENSSL_ROOT_DIR="/usr/local/$(xx-info triple)" && \ - export OPENSSL_CRYPTO_LIBRARY="/usr/local/$(xx-info triple)/lib64" && \ - export OPENSSL_INCLUDE_DIR="/usr/local/$(xx-info triple)/include/openssl" - -RUN ./static.sh build_libssh2 -RUN ./static.sh build_libgit2 - -# trimmed removes all non necessary files (i.e. openssl binary). -FROM build-cross AS trimmed - -ARG TARGETPLATFORM -RUN mkdir -p /trimmed/usr/local/$(xx-info triple)/ && \ - mkdir -p /trimmed/usr/local/$(xx-info triple)/share - -RUN cp -r /usr/local/$(xx-info triple)/lib/ /trimmed/usr/local/$(xx-info triple)/ && \ - cp -r /usr/local/$(xx-info triple)/lib64/ /trimmed/usr/local/$(xx-info triple)/ | true && \ - cp -r /usr/local/$(xx-info triple)/include/ /trimmed/usr/local/$(xx-info triple)/ && \ - cp -r /usr/local/$(xx-info triple)/share/doc/ /trimmed/usr/local/$(xx-info triple)/share/ - -FROM scratch as libs-arm64 -COPY --from=trimmed /trimmed/ / - -FROM scratch as libs-amd64 -COPY --from=trimmed /trimmed/ / - -FROM scratch as libs-armv7 -COPY --from=trimmed /trimmed/ / - -FROM libs-$TARGETARCH$TARGETVARIANT as libs - -# Everything above this line is a copy from Dockefile. - -FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable - -FROM gostable AS go-linux - -# Build-base consists of build platform dependencies and xx. -# These will be used at current arch to yield execute the cross compilations. -FROM go-${TARGETOS} AS go-base - -RUN apk add clang lld pkgconfig - -COPY --from=xx / / - -# build-go-mod can still be cached at build platform architecture. -FROM go-base as build-go-mod - -WORKDIR /root/smoketest -COPY tests/smoketest/go.mod . -COPY tests/smoketest/go.sum . -RUN go mod download - -# Build stage install per target platform -# dependency and effectively cross compile the application. -FROM build-go-mod as build - -ARG TARGETPLATFORM - -# Some dependencies have to installed -# for the target platform: https://github.com/tonistiigi/xx#go--cgo -RUN xx-apk add musl-dev gcc clang lld - -WORKDIR /root/smoketest - -COPY tests/smoketest/main.go . -COPY --from=libs /usr/local/ /usr/local/ - -ENV CGO_ENABLED=1 -RUN export LIBRARY_PATH="/usr/local/$(xx-info triple):/usr/local/$(xx-info triple)/lib64" && \ - export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig:/usr/local/$(xx-info triple)/lib64/pkgconfig" && \ - export FLAGS="$(pkg-config --static --libs --cflags libssh2 openssl libgit2)" && \ - export CGO_LDFLAGS="${FLAGS} -static" && \ - xx-go build \ - -ldflags "-s -w" \ - -tags 'netgo,osusergo,static_build' \ - -o static-test-runner -trimpath main.go - - -# Ensure that the generated binary is valid for the target platform -RUN xx-verify --static static-test-runner - -# This can be deployed into a gcr.io/distroless/static, however -# the alpine has been chosen so it can run the static application -# using the `RUN` statement. -FROM ${BASE_VARIANT} - -RUN apk add git - -WORKDIR /root/smoketest -COPY --from=build \ - /root/smoketest/static-test-runner . - -ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt - -RUN /root/smoketest/static-test-runner diff --git a/Makefile b/Makefile index 7683edf..3b6909f 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,5 @@ TAG ?= latest -ifeq ($(LIBGIT2_ONLY),true) - IMG ?= ghcr.io/fluxcd/golang-with-libgit2-only - DOCKERFILE ?= Dockerfile.libgit2-only - TEST_DOCKERFILE ?= Dockerfile.test-libgit2-only -else - IMG ?= ghcr.io/fluxcd/golang-with-libgit2-all - DOCKERFILE ?= Dockerfile - TEST_DOCKERFILE ?= Dockerfile.test -endif - PLATFORMS ?= linux/amd64,linux/arm/v7,linux/arm64 BUILD_ARGS ?= @@ -22,26 +12,17 @@ LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib LIBGIT2_LIB64_PATH := $(LIBGIT2_PATH)/lib64 LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.a MUSL-CC = -LIBGIT2_ONLY ?= + +IMG ?= ghcr.io/fluxcd/golang-with-libgit2-only +DOCKERFILE ?= Dockerfile.libgit2-only +TEST_DOCKERFILE ?= Dockerfile.test-libgit2-only export CGO_ENABLED=1 -export LIBRARY_PATH=$(LIBGIT2_LIB_PATH):$(LIBGIT2_LIB64_PATH) -export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig:$(LIBGIT2_LIB64_PATH)/pkgconfig +export LIBRARY_PATH=$(LIBGIT2_LIB_PATH) +export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig +export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libgit2) export CGO_CFLAGS=-I$(LIBGIT2_PATH)/include - -ifeq ($(shell uname -s),Linux) - export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libssh2 openssl libgit2) -static -else - export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libgit2) -endif - -ifeq ($(shell uname -s),Linux) - MUSL-PREFIX=$(REPOSITORY_ROOT)/build/musl/$(shell uname -m)-linux-musl-native/bin/$(shell uname -m)-linux-musl - MUSL-CC=$(MUSL-PREFIX)-gcc - export CC=$(MUSL-PREFIX)-gcc - export CXX=$(MUSL-PREFIX)-g++ - export AR=$(MUSL-PREFIX)-ar -endif +export ALLOW_THREADLESS_LIBGIT2=true GO_STATIC_FLAGS=-tags 'netgo,osusergo,static_build' @@ -74,24 +55,9 @@ builder: # install qemu emulators docker run -it --rm --privileged tonistiigi/binfmt --install all -$(LIBGIT2): $(MUSL-CC) -ifeq ($(shell uname -s),Darwin) -ifeq ($(LIBGIT2_ONLY),true) +$(LIBGIT2): TARGET_DIR=$(TARGET_DIR) BUILD_ROOT_DIR=$(BUILD_ROOT_DIR) \ ./hack/static.sh build_libgit2_only -else - TARGET_DIR=$(TARGET_DIR) BUILD_ROOT_DIR=$(BUILD_ROOT_DIR) \ - ./hack/static.sh all -endif -else - IMG_TAG=$(IMG):$(TAG) ./hack/extract-libraries.sh -endif - -$(MUSL-CC): -ifneq ($(shell uname -s),Darwin) - ./hack/download-musl.sh -endif - # dev-test is a smoke test for development environment # consuming the libraries generated by this project. diff --git a/README.md b/README.md index cd7e843..aed6b81 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ Both of which can be used to build the [libgit2][] dependency chain for **AMD64, of Go projects that depend on [git2go][]. The `Makefile` is useful for development environments and will leverage OS specific packages to build `libgit2`. -The `static.sh` will build all `libgit2` dependencies from source using `musl` toolchain. This enables for a full -static binary with the freedom of configuring each of the dependencies in chain. +The `static.sh` builds a `libgit2` static library without linking it to either openssl or libssh2, resulting +in a static library that will largely depend on Smart Transport to work. Alternatively, the statically built libraries can be pulling from the produced images for Linux or from the github release artifacts for MacOS. diff --git a/hack/Makefile b/hack/Makefile deleted file mode 100644 index b89fb4f..0000000 --- a/hack/Makefile +++ /dev/null @@ -1,137 +0,0 @@ -# This determines if we are building while making use of xx. Ensuring -# the package directives target the right manager and inject (OS) vendor -# specific configurations. -IS_XX := $(shell command -v xx-info 2> /dev/null) -ifdef IS_XX -XX_VENDOR ?= $(shell xx-info vendor) -endif - -# Libgit2 build configuration flags. -INSTALL_PREFIX ?= /usr -ifndef IS_XX -INSTALL_LIBDIR ?= $(INSTALL_PREFIX)/lib -else -INSTALL_LIBDIR ?= $(INSTALL_PREFIX)/lib/$(shell xx-info triple) -endif -BUILD_TYPE ?= "RelWithDebInfo" -FLAGS ?= -USE_HTTPS ?= OpenSSL -USE_SSH ?= ON -USE_BUNDLED_ZLIB ?= OFF -BUILD_SHARED_LIBS ?= ON - -# Cmake version to be installed. -CMAKE_VERSION ?= 3.21.3 - -# Libgit2 version to be compiled and installed. -LIBGIT2_VERSION ?= 1.3.0 -# In some scenarios libgit2 needs to be checked out to a specific commit. -# This takes precedence over LIBGIT_VERSION if defined. -# Ref: https://github.com/libgit2/git2go/issues/834 -LIBGIT2_REVISION ?= - -# Set the download URL based on the above information. -ifeq (,$(LIBGIT2_REVISION)) -LIBGIT2_DOWNLOAD_URL ?= https://github.com/libgit2/libgit2/archive/refs/tags/v$(LIBGIT2_VERSION).tar.gz -else -LIBGIT2_DOWNLOAD_URL ?= https://github.com/libgit2/libgit2/archive/$(LIBGIT2_REVISION).tar.gz -endif - -# OS specific expected lib output. -LIBGIT2 := $(INSTALL_LIBDIR)/libgit2.so.$(LIBGIT2_VERSION) -ifeq (Darwin,$(shell uname -s)) - LIBGIT2 := $(INSTALL_LIBDIR)/libgit2.$(LIBGIT2_VERSION).dylib - HAS_BREW := $(shell brew --version 2>/dev/null) -endif - -cmake: -ifeq (debian,$(XX_VENDOR)) -cmake: - apt-get update && apt-get install -y clang cmake -endif -.PHONY: cmake - -base: -ifeq (debian,$(XX_VENDOR)) -base: - xx-apt update && xx-apt install -y binutils gcc libc6-dev dpkg-dev -endif -.PHONY: base - -dependencies: base -ifeq (debian,$(XX_VENDOR)) -# Install libssh2 for $TARGETPLATFORM from "bookworm", as the version in "bullseye" -# has been linked against gcrypt, which causes issues with PKCS* formats. -# We pull (sub)dependencies from there as well, to ensure all versions are aligned, -# and not accidentially linked to e.g. mbedTLS (which has limited support for -# certain key formats). -# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271 -# Ref: https://github.com/ARMmbed/mbedtls/issues/2452#issuecomment-802683144 -DEPENDENCIES = -ifneq ("OFF",$(USE_BUNDLED_ZLIB)) -DEPENDENCIES += zlib1g-dev -endif -ifneq ("OFF",$(USE_HTTPS)) -DEPENDENCIES += libssl-dev -endif -ifneq ("OFF",$(USE_SSH)) -DEPENDENCIES += libssh2-1-dev -endif -dependencies: -ifneq ("",$(DEPENDENCIES)) - set -e; \ - echo "deb http://deb.debian.org/debian bookworm main" > /etc/apt/sources.list.d/bookworm.list \ - && echo "deb-src http://deb.debian.org/debian bookworm main" /etc/apt/sources.list.d/bookworm.list \ - && xx-apt update \ - && xx-apt -t bookworm install -y $(DEPENDENCIES) -endif -endif -.PHONY: dependencies - -libgit2: $(LIBGIT2) -.PHONY: libgit2 - -ifdef HAS_BREW -ifneq ("OFF",$(USE_HTTPS)) -HAS_OPENSSL := $(shell brew --prefix openssl@1.1) -# NB: the OPENSSL_LDFLAGS ensures the path is included in the libgit2.pc -# file. As a standard brew installation doesn't appear to be system wide -# on most macOS instances, and we thus have to explicitly tell where -# it can be found. -ifdef HAS_OPENSSL - PKG_CONFIG_PATH := $(PKG_CONFIG_PATH):$(HAS_OPENSSL)/lib/pkgconfig - FLAGS += -DOPENSSL_LDFLAGS:STRING='-L $(HAS_OPENSSL)/lib' -endif -endif -ifneq ("OFF",$(USE_SSH)) -HAS_LIBSSH2 := $(shell brew --prefix libssh2) -ifdef HAS_LIBSSH2 - PKG_CONFIG_PATH := $(PKG_CONFIG_PATH):$(HAS_LIBSSH2)/lib/pkgconfig -endif -endif -endif - -$(LIBGIT2): - set -e; \ - SETUP_LIBGIT2_TMP_DIR=$$(mktemp -d) \ - && curl -L $(LIBGIT2_DOWNLOAD_URL) -o $$SETUP_LIBGIT2_TMP_DIR/archive.tar.gz \ - && mkdir -p $$SETUP_LIBGIT2_TMP_DIR/src \ - && tar xzf $$SETUP_LIBGIT2_TMP_DIR/archive.tar.gz --strip 1 -C $$SETUP_LIBGIT2_TMP_DIR/src \ - && PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) cmake -S $$SETUP_LIBGIT2_TMP_DIR/src -B $$SETUP_LIBGIT2_TMP_DIR/build \ - -DCMAKE_BUILD_TYPE:STRING=$(BUILD_TYPE)\ - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON \ - -DCMAKE_C_FLAGS=-fPIC \ - -DDEPRECATE_HARD=ON \ - -DCMAKE_INSTALL_PREFIX:PATH=$(INSTALL_PREFIX) \ - -DCMAKE_INSTALL_LIBDIR:PATH=$(INSTALL_LIBDIR) \ - -DBUILD_CLAR:BOOL:BOOL=OFF \ - -DTHREADSAFE:BOOL=ON \ - -DBUILD_SHARED_LIBS:BOOL=$(BUILD_SHARED_LIBS) \ - -DUSE_BUNDLED_ZLIB:BOOL=$(USE_BUNDLED_ZLIB) \ - -DUSE_HTTP_PARSER:STRING=builtin \ - -DREGEX_BACKEND:STRING=builtin \ - -DUSE_HTTPS:STRING=$(USE_HTTPS) \ - -DUSE_SSH:BOOL=$(USE_SSH) \ - $(FLAGS) \ - && cmake --build $$SETUP_LIBGIT2_TMP_DIR/build --target install \ - && rm -rf $$SETUP_LIBGIT2_TMP_DIR diff --git a/hack/download-musl.sh b/hack/download-musl.sh deleted file mode 100755 index de6b4a3..0000000 --- a/hack/download-musl.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -set -eoux pipefail - -MUSL_X86_64_FILENAME=x86_64-linux-musl-native.tgz -MUSL_X86_64_SHA512=44d441ad9aa11a06feddf3daa4c9f53ad7d9ca37af1f5a61379aca07793703d179410cea723c1b7fca94c4de19a321228bdb3656bc5cbdb5e3bea8e2d6dac6c7 -MUSL_AARCH64_FILENAME=aarch64-linux-musl-native.tgz -MUSL_AARCH64_SHA512=16d544e09845c9dbba50f29e0cb04dd661e17eb63c56acad6a67fd2a78aa7596b792477c7177d3cd56d408a27dc291a90507df882f2b099c0f25511ce08fd3b5 - -MUSL_FILENAME="${MUSL_X86_64_FILENAME}" -MUSL_SHA512="${MUSL_X86_64_SHA512}" -if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then - MUSL_FILENAME="${MUSL_AARCH64_FILENAME}" - MUSL_SHA512="${MUSL_AARCH64_SHA512}" -fi - -MUSL_AARCH64_URL="https://more.musl.cc/11.2.1/x86_64-linux-musl/${MUSL_FILENAME}" - -ROOT_DIR="$(git rev-parse --show-toplevel)" -MUSL_DIR="${ROOT_DIR}/build/musl" - -if [ ! -f "${MUSL_DIR}/bin" ]; then - TARGET_FILE="${MUSL_DIR}/${MUSL_FILENAME}" - mkdir -p "${MUSL_DIR}" - - echo "${MUSL_SHA512} ${TARGET_FILE}" - curl -o "${TARGET_FILE}" -LO "${MUSL_AARCH64_URL}" - if ! echo "${MUSL_SHA512} ${TARGET_FILE}" | sha512sum --check; then - echo "Checksum failed for ${MUSL_FILENAME}." - rm -rf "${MUSL_DIR}" - exit 1 - fi - - tar xzf "${TARGET_FILE}" -C "${MUSL_DIR}" - rm "${TARGET_FILE}" -fi diff --git a/hack/static.sh b/hack/static.sh index 7dde0c1..64a42b6 100755 --- a/hack/static.sh +++ b/hack/static.sh @@ -3,12 +3,6 @@ set -euxo pipefail LIBGIT2_URL="${LIBGIT2_URL:-https://github.com/libgit2/libgit2/archive/refs/tags/v1.3.2.tar.gz}" -OPENSSL_URL="${OPENSSL_URL:-https://github.com/openssl/openssl/archive/refs/tags/openssl-3.0.2.tar.gz}" -LIBSSH2_URL="${LIBSSH2_URL:-https://github.com/libssh2/libssh2/archive/refs/tags/libssh2-1.10.0.tar.gz}" - -# May be worth considering other forks/implementations that either -# provide better performance (i.e. intel/cloudflare) or that are better maintained. -LIBZ_URL="${LIBZ_URL:-https://github.com/madler/zlib/archive/refs/tags/v1.2.12.tar.gz}" TARGET_DIR="${TARGET_DIR:-/usr/local/$(xx-info triple)}" BUILD_ROOT_DIR="${BUILD_ROOT_DIR:-/build}" @@ -35,159 +29,6 @@ function download_source(){ rm "$2/source.tar.gz" } -function build_libz(){ - download_source "${LIBZ_URL}" "${SRC_DIR}/libz" - pushd "${SRC_DIR}/libz" - - # if target architecture is the same as current, no cross compiling is required - if [ "${TARGET_ARCH}" = "$(uname -m)" ]; then - ./configure --static --prefix="${TARGET_DIR}" - else - ./configure --static --prefix="${TARGET_DIR}" \ - --archs="-arch ${TARGET_ARCH}" - fi - - make install - - popd -} - -function build_openssl(){ - download_source "${OPENSSL_URL}" "${SRC_DIR}/openssl" - pushd "${SRC_DIR}/openssl" - - export OPENSSL_ROOT_DIR="${TARGET_DIR}" - export OPENSSL_LIBRARIES="${TARGET_DIR}/lib" - - export KERNEL_BITS=64 - target_arch="" - if [[ ! $OSTYPE == darwin* ]]; then - if [ "${TARGET_ARCH}" = "armv7l" ]; then - # openssl does not have a specific armv7l - # using generic32 instead. - target_arch="linux-generic32" - export KERNEL_BITS=32 - elif [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then - target_arch="linux-aarch64" - elif [ "${TARGET_ARCH}" = "x86_64" ]; then - target_arch="linux-x86_64" - fi - else - SUFFIX="" - if [ ! "${TARGET_ARCH}" = "$(uname -m)" ]; then - SUFFIX="-cc" - fi - - if [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then - target_arch="darwin64-arm64${SUFFIX}" - elif [ "${TARGET_ARCH}" = "x86_64" ]; then - target_arch="darwin64-x86_64${SUFFIX}" - fi - # if none of the above, let openssl figure it out. - fi - - ./Configure "${target_arch}" threads no-shared no-tests zlib -fPIC -DOPENSSL_PIC \ - --prefix="${TARGET_DIR}" \ - --with-zlib-include="${TARGET_DIR}/include" \ - --with-zlib-lib="${TARGET_DIR}/lib" \ - --openssldir=/etc/ssl - - make - make install_sw - - popd -} - -function build_libssh2(){ - download_source "${LIBSSH2_URL}" "${SRC_DIR}/libssh2" - - pushd "${SRC_DIR}/libssh2" - - mkdir -p build - pushd build - - OPENSSL_LIBRARIES="${TARGET_DIR}/lib" - if [ "${TARGET_ARCH}" = "x86_64" ] && [[ ! $OSTYPE == darwin* ]]; then - OPENSSL_LIBRARIES="${TARGET_DIR}/lib64" - fi - - # Set osx arch only when cross compiling on darwin - if [[ $OSTYPE == darwin* ]] && [ ! "${TARGET_ARCH}" = "$(uname -m)" ]; then - CMAKE_PARAMS=-DCMAKE_OSX_ARCHITECTURES="${TARGET_ARCH}" - fi - - # Building examples allow for validating against missing symbols at compilation time. - cmake "${CMAKE_PARAMS}" \ - -DCMAKE_C_COMPILER="${C_COMPILER}" \ - -DCMAKE_INSTALL_PREFIX="${TARGET_DIR}" \ - -DBUILD_SHARED_LIBS:BOOL=OFF \ - -DLINT:BOOL=OFF \ - -DBUILD_EXAMPLES:BOOL=ON \ - -DBUILD_TESTING:BOOL=OFF \ - -DCMAKE_C_FLAGS=-fPIC \ - -DCRYPTO_BACKEND=OpenSSL \ - -DENABLE_ZLIB_COMPRESSION:BOOL=ON \ - -DCMAKE_BUILD_TYPE="RelWithDebInfo" \ - -DZLIB_LIBRARY="${TARGET_DIR}/lib/libz.a" \ - -DOPENSSL_CRYPTO_LIBRARY="${OPENSSL_LIBRARIES}/libcrypto.a" \ - -DOPENSSL_SSL_LIBRARY="${OPENSSL_LIBRARIES}/libssl.a" \ - .. - - cmake --build . --target install - - popd - popd -} - -function build_libgit2(){ - download_source "${LIBGIT2_URL}" "${SRC_DIR}/libgit2" - - pushd "${SRC_DIR}/libgit2" - - mkdir -p build - - pushd build - - SSL_LIBRARY="${TARGET_DIR}/lib/libssl.a" - CRYPTO_LIBRARY="${TARGET_DIR}/lib/libcrypto.a" - if [[ ! $OSTYPE == darwin* ]] && [ "${TARGET_ARCH}" = "x86_64" ]; then - SSL_LIBRARY="${TARGET_DIR}/lib64/libssl.a" - CRYPTO_LIBRARY="${TARGET_DIR}/lib64/libcrypto.a" - fi - - # Set osx arch only when cross compiling on darwin - if [[ $OSTYPE == darwin* ]] && [ ! "${TARGET_ARCH}" = "$(uname -m)" ]; then - CMAKE_PARAMS=-DCMAKE_OSX_ARCHITECTURES="${TARGET_ARCH}" - fi - - cmake "${CMAKE_PARAMS}" \ - -DCMAKE_C_COMPILER="${C_COMPILER}" \ - -DCMAKE_INSTALL_PREFIX="${TARGET_DIR}" \ - -DTHREADSAFE:BOOL=ON \ - -DBUILD_CLAR:BOOL=OFF \ - -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON \ - -DCMAKE_C_FLAGS=-fPIC \ - -DUSE_SSH:BOOL=ON \ - -DHAVE_LIBSSH2_MEMORY_CREDENTIALS:BOOL=ON \ - -DDEPRECATE_HARD:BOOL=ON \ - -DUSE_BUNDLED_ZLIB:BOOL=ON \ - -DUSE_HTTPS:STRING=OpenSSL \ - -DREGEX_BACKEND:STRING=builtin \ - -DOPENSSL_SSL_LIBRARY="${SSL_LIBRARY}" \ - -DOPENSSL_CRYPTO_LIBRARY="${CRYPTO_LIBRARY}" \ - -DCMAKE_INCLUDE_PATH="${TARGET_DIR}/include" \ - -DCMAKE_LIBRARY_PATH="${TARGET_DIR}/lib" \ - -DCMAKE_PREFIX_PATH="${TARGET_DIR}" \ - -DCMAKE_BUILD_TYPE="RelWithDebInfo" \ - .. - - cmake --build . --target install - - popd - popd -} - function build_libgit2_only(){ download_source "${LIBGIT2_URL}" "${SRC_DIR}/libgit2" @@ -226,11 +67,4 @@ function build_libgit2_only(){ popd } -function all(){ - build_libz - build_openssl - build_libssh2 - build_libgit2 -} - "$@"