Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ld: dockerfile enhancements #189

Merged
merged 3 commits into from
Jan 13, 2025
Merged

ld: dockerfile enhancements #189

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
70b5b51
ld: use faster download source for binutils
crazy-max Dec 11, 2024
55756b3
ld: use heredocs and ADD to clone in dockerfile
crazy-max Dec 11, 2024
74dc16c
ld: skip verification on non-binary files
crazy-max Jan 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
263 changes: 171 additions & 92 deletions src/ld/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#syntax=docker/dockerfile:1.8
#syntax=docker/dockerfile:1
#check=error=true

ARG ALPINE_VERSION=3.21
Expand All @@ -16,41 +16,54 @@ RUN apk add --no-cache git clang lld cmake make python3 bash
COPY --link --from=xx / /
ARG LIBTAPI_VERSION
WORKDIR /work
RUN git clone https://github.com/tpoechtrager/apple-libtapi --depth 1 -b ${LIBTAPI_VERSION}
ADD "https://github.com/tpoechtrager/apple-libtapi.git#${LIBTAPI_VERSION}" apple-libtapi
WORKDIR apple-libtapi
RUN --mount=target=/tmp/libtapi-cmake-args.patch,source=libtapi-cmake-args.patch \
git apply /tmp/libtapi-cmake-args.patch
RUN apk add --no-cache gcc g++
RUN NOMAKE=1 CMAKE_EXTRA_ARGS="$(xx-clang --print-cmake-defines) -DCMAKE_CXX_FLAGS=-D_LARGEFILE64_SOURCE" ./build.sh && \
cd build && \
make -j $(nproc) clang-tblgen llvm-tblgen && \
cp -a bin/clang-tblgen bin/llvm-tblgen /usr/bin/ && \
cd .. && rm -rf build
RUN <<EOT
set -ex
NOMAKE=1 CMAKE_EXTRA_ARGS="$(xx-clang --print-cmake-defines) -DCMAKE_CXX_FLAGS=-D_LARGEFILE64_SOURCE" ./build.sh
cd build
make -j $(nproc) clang-tblgen llvm-tblgen
cp -a bin/clang-tblgen bin/llvm-tblgen /usr/bin/
cd ..
rm -rf build
EOT

FROM libtapi-base AS libtapi
ARG TARGETPLATFORM
RUN xx-apk add g++
RUN INSTALLPREFIX=/opt/libtapi/ \
CMAKE_EXTRA_ARGS="-DCLANG_TABLEGEN_EXE=/usr/bin/clang-tblgen -DLLVM_TABLEGEN=/usr/bin/llvm-tblgen -DCLANG_TABLEGEN=/usr/bin/clang-tblgen -DCMAKE_CXX_FLAGS=-D_LARGEFILE64_SOURCE $(xx-clang --print-cmake-defines)" \
./build.sh && ./install.sh && \
xx-verify /opt/libtapi/lib/libtapi.so && \
rm -rf build
RUN <<EOT
set -ex
export INSTALLPREFIX=/opt/libtapi/
CMAKE_EXTRA_ARGS="-DCLANG_TABLEGEN_EXE=/usr/bin/clang-tblgen -DLLVM_TABLEGEN=/usr/bin/llvm-tblgen -DCLANG_TABLEGEN=/usr/bin/clang-tblgen -DCMAKE_CXX_FLAGS=-D_LARGEFILE64_SOURCE $(xx-clang --print-cmake-defines)" ./build.sh
./install.sh
xx-verify /opt/libtapi/lib/libtapi.so
rm -rf build
EOT

FROM libtapi-base AS libtapi-static
ARG TARGETPLATFORM
RUN xx-apk add g++
RUN export INSTALLPREFIX=/opt/libtapi/ \
CMAKE_EXTRA_ARGS="-DCLANG_TABLEGEN_EXE=/usr/bin/clang-tblgen -DLLVM_TABLEGEN=/usr/bin/llvm-tblgen -DCLANG_TABLEGEN=/usr/bin/clang-tblgen -DCMAKE_CXX_FLAGS=-D_LARGEFILE64_SOURCE $(xx-clang --print-cmake-defines)" \
&& sed -i s/SHARED/STATIC/g src/tapi/tools/libtapi/CMakeLists.txt && \
./build.sh && cd build && make -j $(nproc) LLVMObject tapiCore LLVMSupport LLVMDemangle LLVMMC LLVMBinaryFormat tapiObjCMetadata tapiNoInits LLVMTextAPI install-tapi-headers && \
mkdir /opt/libtapi/lib && cp -a ./lib/*.a /opt/libtapi/lib/ && \
cd .. && rm -rf build
RUN <<EOT
set -x
export INSTALLPREFIX=/opt/libtapi/
sed -i s/SHARED/STATIC/g src/tapi/tools/libtapi/CMakeLists.txt
CMAKE_EXTRA_ARGS="-DCLANG_TABLEGEN_EXE=/usr/bin/clang-tblgen -DLLVM_TABLEGEN=/usr/bin/llvm-tblgen -DCLANG_TABLEGEN=/usr/bin/clang-tblgen -DCMAKE_CXX_FLAGS=-D_LARGEFILE64_SOURCE $(xx-clang --print-cmake-defines)" ./build.sh
cd build
make -j $(nproc) LLVMObject tapiCore LLVMSupport LLVMDemangle LLVMMC LLVMBinaryFormat tapiObjCMetadata tapiNoInits LLVMTextAPI install-tapi-headers
mkdir /opt/libtapi/lib
cp -a ./lib/*.a /opt/libtapi/lib/
cd ..
rm -rf build
EOT

FROM --platform=$BUILDPLATFORM alpine:${ALPINE_VERSION} AS aports
RUN apk add git
WORKDIR /work
ARG APORTS_VERSION
RUN git clone --depth 1 -b ${APORTS_VERSION} https://github.com/alpinelinux/aports.git
ADD "https://github.com/alpinelinux/aports.git#${APORTS_VERSION}" aports

FROM scratch AS patches-binutils
COPY --from=aports /work/aports/main/binutils/*.patch /
Expand All @@ -64,19 +77,20 @@ RUN apk add git clang lld cmake samurai patch
ARG LIBDISPATCH_VERSION
COPY --link --from=xx / /
WORKDIR /work
RUN --mount=target=/patches,from=patches-libdispatch \
git clone -b ${LIBDISPATCH_VERSION} --depth 1 https://github.com/apple/swift-corelibs-libdispatch.git && \
cd swift-corelibs-libdispatch && \
for f in /patches/*; do patch -p1 < $f; done
ADD "https://github.com/apple/swift-corelibs-libdispatch.git#${LIBDISPATCH_VERSION}" swift-corelibs-libdispatch
WORKDIR swift-corelibs-libdispatch
RUN --mount=target=/patches,from=patches-libdispatch \
for f in /patches/*; do patch -p1 < $f; done
ARG TARGETPLATFORM
RUN xx-apk add --no-cache gcc g++ musl-dev linux-headers bsd-compat-headers
RUN cmake $(xx-clang --print-cmake-defines) -G Ninja -B build -DCMAKE_INSTALL_PREFIX=/out/libdispatch -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=False -DCMAKE_BUILD_TYPE=MinSizeRel && \
cmake --build build && \
cmake --install build && \
rm -rf build
RUN ls -l /out/libdispatch/lib && \
[ -f /out/libdispatch/lib/libdispatch.a ] && [ -f /out/libdispatch/lib/libBlocksRuntime.a ]
RUN <<EOT
set -ex
cmake $(xx-clang --print-cmake-defines) -G Ninja -B build -DCMAKE_INSTALL_PREFIX=/out/libdispatch -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=False -DCMAKE_BUILD_TYPE=MinSizeRel
cmake --build build
cmake --install build
rm -rf build
EOT
RUN ls -l /out/libdispatch/lib && [ -f /out/libdispatch/lib/libdispatch.a ] && [ -f /out/libdispatch/lib/libBlocksRuntime.a ]

FROM scratch AS libdispatch
COPY --from=libdispatch-base /out/libdispatch/lib/*.a /lib/
Expand All @@ -85,9 +99,8 @@ FROM --platform=${BUILDPLATFORM} alpine:${ALPINE_VERSION} AS cctools-base
RUN apk add --no-cache git clang lld make llvm llvm-dev libdispatch-dev
COPY --link --from=xx / /
WORKDIR /work
ARG CCTOOLS_REPO=https://github.com/tpoechtrager/cctools-port
ARG CCTOOLS_VERSION
RUN git clone $CCTOOLS_REPO -b ${CCTOOLS_VERSION}
ADD "https://github.com/tpoechtrager/cctools-port.git#${CCTOOLS_VERSION}" cctools-port
WORKDIR cctools-port/cctools
ARG TARGETPLATFORM
RUN --mount=target=/libdispatch,from=libdispatch \
Expand All @@ -96,53 +109,65 @@ RUN --mount=target=/libdispatch,from=libdispatch \

FROM cctools-base AS lipo-base
ARG LIPO_CFLAGS="-Wl,-s -Os"
RUN export CFLAGS=${LIPO_CFLAGS} && \
./configure --host=$(xx-clang --print-target-triple) LDFLAGS=--static && \
make -C libmacho && make -C libstuff && make -C misc lipo && \
xx-verify --static misc/lipo
RUN <<EOT
set -ex
export CFLAGS=${LIPO_CFLAGS}
./configure --host=$(xx-clang --print-target-triple) LDFLAGS=--static
make -C libmacho
make -C libstuff
make -C misc lipo
xx-verify --static misc/lipo
EOT

FROM scratch AS lipo-static
COPY --from=lipo-base /work/cctools-port/cctools/misc/lipo /

FROM cctools-base AS codesign-allocate-base
ARG CODESIGN_CFLAGS="-Wl,-s -Os"
RUN export CFLAGS=${CODESIGN_CFLAGS} && \
./configure --host=$(xx-clang --print-target-triple) LDFLAGS=--static && \
make -C libmacho && make -C libstuff && make -C misc codesign_allocate && \
xx-verify --static misc/codesign_allocate

RUN <<EOT
export CFLAGS=${CODESIGN_CFLAGS}
./configure --host=$(xx-clang --print-target-triple) LDFLAGS=--static
make -C libmacho
make -C libstuff
make -C misc codesign_allocate
xx-verify --static misc/codesign_allocate
EOT
FROM scratch AS codesign-allocate-static
COPY --from=codesign-allocate-base /work/cctools-port/cctools/misc/codesign_allocate /

FROM cctools-base AS ld64-static-base
# for LTO
RUN apk add llvm-dev
ARG LD64_CXXFLAGS="-Wl,-s -Os"
RUN --mount=from=libtapi-static,source=/opt/libtapi,target=/opt/libtapi \
export CXXFLAGS=${LD64_CXXFLAGS} && ./configure --with-libtapi=/opt/libtapi --host=$(xx-clang --print-target-triple) &&\
sed -i 's/-ltapi/-ltapi -ltapiCore -ltapiObjCMetadata -lLLVMObject -lLLVMTextAPI -lLLVMSupport -lLLVMDemangle -lLLVMMC -lLLVMBinaryFormat --static/' ld64/src/ld/Makefile && \
make -j $(nproc) -C ld64 && \
xx-verify --static ld64/src/ld/ld

RUN --mount=from=libtapi-static,source=/opt/libtapi,target=/opt/libtapi <<EOT
set -ex
export CXXFLAGS=${LD64_CXXFLAGS}
./configure --with-libtapi=/opt/libtapi --host=$(xx-clang --print-target-triple)
sed -i 's/-ltapi/-ltapi -ltapiCore -ltapiObjCMetadata -lLLVMObject -lLLVMTextAPI -lLLVMSupport -lLLVMDemangle -lLLVMMC -lLLVMBinaryFormat --static/' ld64/src/ld/Makefile
make -j $(nproc) -C ld64
xx-verify --static ld64/src/ld/ld
EOT
FROM --platform=${BUILDPLATFORM} alpine:${ALPINE_VERSION} AS sigtool-base
RUN apk add --no-cache git clang lld cmake make pkgconf
COPY --from=xx / /
WORKDIR /work
ARG SIGTOOL_VERSION
RUN git clone https://github.com/thefloweringash/sigtool && \
cd sigtool && \
git checkout ${SIGTOOL_VERSION}
ADD "https://github.com/thefloweringash/sigtool.git#${SIGTOOL_VERSION}" sigtool
WORKDIR sigtool
RUN --mount=target=/tmp/sigtool-static.patch,source=sigtool-static.patch \
git apply /tmp/sigtool-static.patch
ARG TARGETPLATFORM
RUN xx-apk add --no-cache g++ openssl-dev openssl-libs-static
ARG SIGTOOL_CXXFLAGS="-Wl,-s -Os"
RUN export CXXFLAGS=${SIGTOOL_CXXFLAGS} && \
mkdir build && cd build && \
cmake -DBUILD_SHARED_LIBS=False $(xx-clang --print-cmake-defines) -DCMAKE_EXE_LINKER_FLAGS=-static .. && \
make -j $(nproc) && \
xx-verify --static ./sigtool ./codesign
RUN <<EOT
set -ex
export CXXFLAGS=${SIGTOOL_CXXFLAGS}
mkdir build
cd build
cmake -DBUILD_SHARED_LIBS=False $(xx-clang --print-cmake-defines) -DCMAKE_EXE_LINKER_FLAGS=-static ..
make -j $(nproc)
xx-verify --static ./sigtool ./codesign
EOT

FROM scratch AS sigtool-static
COPY --link --from=codesign-allocate-static / /
Expand Down Expand Up @@ -174,13 +199,15 @@ FROM cctools-base AS cctools-build
RUN apk add llvm-dev
ARG CCTOOLS_CXXFLAGS="-Wl,-s -Os"
ARG CCTOOLS_CFLAGS="-Wl,-s -Os"
RUN --mount=from=libtapi,source=/opt/libtapi,target=/opt/libtapi \
# copy to /usr/bin for clean rpath. TODO: try static build
cp -a /opt/libtapi/. /usr/ && \
export CFLAGS=${CCTOOLS_CFLAGS} CXXFLAGS=${CCTOOLS_CXXFLAGS} && \
./configure --prefix=/opt/cctools --with-libtapi=/opt/libtapi --host=$(xx-clang --print-target-triple) && \
make -j $(nproc) install && \
xx-verify /opt/cctools/bin/ld
RUN --mount=from=libtapi,source=/opt/libtapi,target=/opt/libtapi <<EOT
set -ex
# copy to /usr/bin for clean rpath. TODO: try static build
cp -a /opt/libtapi/. /usr/
export CFLAGS=${CCTOOLS_CFLAGS} CXXFLAGS=${CCTOOLS_CXXFLAGS}
./configure --prefix=/opt/cctools --with-libtapi=/opt/libtapi --host=$(xx-clang --print-target-triple)
make -j $(nproc) install
xx-verify /opt/cctools/bin/ld
EOT

FROM scratch AS cctools
COPY --link --from=libtapi /opt/libtapi/lib/*.so /usr/lib/
Expand All @@ -193,17 +220,22 @@ WORKDIR /work
COPY --link --from=xx / /
COPY --link --from=patches-binutils / patches/
ARG BINUTILS_VERSION
RUN wget https://ftp.gnu.org/gnu/binutils/binutils-${BINUTILS_VERSION}.tar.gz
RUN wget https://sourceware.org/pub/binutils/releases/binutils-${BINUTILS_VERSION}.tar.gz
ARG TARGETPLATFORM
# first build version for current architecture that is used then cross compiling
RUN export CC=xx-clang CXX=xx-clang++ LD=lld BINUTILS_TARGET=${TARGETPLATFORM} && unset TARGETPLATFORM && \
tar xf binutils-${BINUTILS_VERSION}.tar.gz && \
cd binutils-${BINUTILS_VERSION} && \
for f in ../patches/*; do patch -p1 < $f; done && \
./configure --disable-separate-code --libdir=/lib --prefix=/usr --disable-multilib --enable-deterministic-archives --target=$(TARGETPLATFORM=$BINUTILS_TARGET xx-info) --disable-nls --disable-gprofng && \
make -j $(nproc) && \
make install && \
cd .. && rm -rf binutils-${BINUTILS_VERSION}
RUN <<EOT
set -ex
export CC=xx-clang CXX=xx-clang++ LD=lld BINUTILS_TARGET=${TARGETPLATFORM}
unset TARGETPLATFORM
tar xf binutils-${BINUTILS_VERSION}.tar.gz
cd binutils-${BINUTILS_VERSION}
for f in ../patches/*; do patch -p1 < $f; done
./configure --disable-separate-code --libdir=/lib --prefix=/usr --disable-multilib --enable-deterministic-archives --target=$(TARGETPLATFORM=$BINUTILS_TARGET xx-info) --disable-nls --disable-gprofng
make -j $(nproc)
make install
cd ..
rm -rf binutils-${BINUTILS_VERSION}
EOT
RUN xx-apk add --no-cache musl-dev gcc g++ zlib-dev zstd-dev

FROM binutils-base0 AS ld-base
Expand All @@ -215,18 +247,42 @@ ARG LD_TARGET=${TARGETOS}-${TARGETARCH}${TARGETVARIANT}
ARG LD_CFLAGS="-Wl,-s -Os"
# BINUTILS_CONFIG defines extra options passed to binutils configure script
ARG BINUTILS_CONFIG=
RUN export CC=xx-clang CXX=xx-clang++ CFLAGS="$LD_CFLAGS --static" CXXFLAGS="$LD_CFLAGS" LD_TARGET_TRIPLE=$(TARGETPLATFORM= TARGETPAIR=$LD_TARGET xx-info) XX_CC_PREFER_LINKER=ld && \
tar xf binutils-${BINUTILS_VERSION}.tar.gz && \
cd binutils-${BINUTILS_VERSION} && \
for f in ../patches/*; do patch -p1 < $f; done && \
./configure --disable-separate-code --libdir=/lib --prefix=/ --disable-multilib --enable-deterministic-archives --target=$LD_TARGET_TRIPLE --host $(xx-clang --print-target-triple) --disable-nls --disable-gold --enable-relro --disable-plugins --with-pic --with-mmap --with-system-zlib --disable-gas --disable-elfcpp --disable-binutils --disable-gprofng $BINUTILS_CONFIG && \
make -j $(nproc) && \
make install && \
cd .. && rm -rf binutils-${BINUTILS_VERSION} && \
xx-verify --static /$LD_TARGET_TRIPLE/bin/ld && \
mkdir -p /out && mv /$LD_TARGET_TRIPLE/bin/ld /out/$LD_TARGET-ld && \
mkdir -p /out/ldscripts && mv /$LD_TARGET_TRIPLE/lib/ldscripts/* /out/ldscripts/

RUN <<EOT
set -ex
export CC=xx-clang CXX=xx-clang++ CFLAGS="$LD_CFLAGS --static" CXXFLAGS="$LD_CFLAGS" LD_TARGET_TRIPLE=$(TARGETPLATFORM= TARGETPAIR=$LD_TARGET xx-info) XX_CC_PREFER_LINKER=ld
tar xf binutils-${BINUTILS_VERSION}.tar.gz
cd binutils-${BINUTILS_VERSION}
for f in ../patches/*; do patch -p1 < $f; done
./configure \
--disable-separate-code \
--libdir=/lib \
--prefix=/ \
--disable-multilib \
--enable-deterministic-archives \
--target=$LD_TARGET_TRIPLE \
--host=$(xx-clang --print-target-triple) \
--disable-nls \
--disable-gold \
--enable-relro \
--disable-plugins \
--with-pic \
--with-mmap \
--with-system-zlib \
--disable-gas \
--disable-elfcpp \
--disable-binutils \
--disable-gprofng \
$BINUTILS_CONFIG
make -j $(nproc)
make install
cd ..
rm -rf binutils-${BINUTILS_VERSION}
xx-verify --static /$LD_TARGET_TRIPLE/bin/ld
mkdir -p /out
mv /$LD_TARGET_TRIPLE/bin/ld /out/$LD_TARGET-ld
mkdir -p /out/ldscripts
mv /$LD_TARGET_TRIPLE/lib/ldscripts/* /out/ldscripts/
EOT
FROM ld-base AS ld-tgz-base
ARG TARGETOS TARGETARCH TARGETVARIANT
ARG LD_TARGET
Expand All @@ -247,15 +303,38 @@ ARG BINUTILS_TARGET=${TARGETOS}-${TARGETARCH}${TARGETVARIANT}
ARG BINUTILS_CFLAGS="-Wl,-s -Os"
# BINUTILS_CONFIG defines extra options passed to binutils configure script
ARG BINUTILS_CONFIG=
RUN export CC=xx-clang CXX=xx-clang++ CFLAGS="$BINUTILS_CFLAGS" CXXFLAGS="$BINUTILS_CFLAGS" && \
tar xf binutils-${BINUTILS_VERSION}.tar.gz && \
cd binutils-${BINUTILS_VERSION} && \
for f in ../patches/*; do patch -p1 < $f; done && \
./configure --disable-separate-code --libdir=/lib --prefix=/out --disable-multilib --enable-deterministic-archives --target=$(TARGETPLATFORM= TARGETPAIR=$BINUTILS_TARGET xx-info) --host $(xx-clang --print-target-triple) --disable-nls --enable-gold --enable-relro --enable-plugins --with-pic --with-mmap --with-system-zlib --disable-gprofng $BINUTILS_CONFIG && \
make -j $(nproc) && \
make install && \
cd .. && rm -rf binutils-${BINUTILS_VERSION} && \
for f in /out/bin/*; do xx-verify $f; done

RUN <<EOT
set -ex
export CC=xx-clang CXX=xx-clang++ CFLAGS="$BINUTILS_CFLAGS" CXXFLAGS="$BINUTILS_CFLAGS"
tar xf binutils-${BINUTILS_VERSION}.tar.gz
cd binutils-${BINUTILS_VERSION}
for f in ../patches/*; do patch -p1 < $f; done
./configure \
--disable-separate-code \
--libdir=/lib \
--prefix=/out \
--disable-multilib \
--enable-deterministic-archives \
--target=$(TARGETPLATFORM= TARGETPAIR=$BINUTILS_TARGET xx-info) \
--host=$(xx-clang --print-target-triple) \
--disable-nls \
--enable-gold \
--enable-relro \
--enable-plugins \
--with-pic \
--with-mmap \
--with-system-zlib \
--disable-gprofng \
$BINUTILS_CONFIG
make -j $(nproc)
make install
cd ..
rm -rf binutils-${BINUTILS_VERSION}
for f in /out/bin/*; do
if ! file "$f" | grep -q 'ASCII text'; then
xx-verify $f
fi
done
EOT
FROM scratch AS binutils
COPY --from=binutils-base /out /usr/
Loading