From a1121eb503e2a4e14fdeba1068015f87e02f3bf3 Mon Sep 17 00:00:00 2001 From: Scott Todd Date: Mon, 26 Aug 2024 10:51:25 -0700 Subject: [PATCH] Add non-ghr version of cpubuilder dockerfile. (#4) I believe we can replace https://github.com/iree-org/iree/blob/main/build_tools/docker/dockerfiles/base.Dockerfile with this. At least, I tested the `linux_x64_clang_debug` workflow in IREE locally and was able to get it working using this Dockerfile and only minor edits to the workflow: ```diff --- a/.github/workflows/ci_linux_x64_clang_debug.yml +++ b/.github/workflows/ci_linux_x64_clang_debug.yml @@ -51,7 +51,7 @@ jobs: with: submodules: true - name: Install Python requirements - run: pip install -r ./runtime/bindings/python/iree/runtime/build_requirements.txt + run: python3 -m pip install -r ./runtime/bindings/python/iree/runtime/build_requirements.txt # Note: Not using ccache here. Debug builds need a lot of cache space # and caching only provides marginal build time improvements. - name: CMake - configure @@ -59,6 +59,7 @@ jobs: cmake \ -G Ninja \ -B ${BUILD_DIR} \ + -DPython3_EXECUTABLE="$(which python)" \ -DCMAKE_BUILD_TYPE=Debug \ -DIREE_BUILD_PYTHON_BINDINGS=ON \ -DIREE_ENABLE_LLD=ON \ ``` Command history for future-me (could go in docs): ```bash # Host sudo docker buildx build --file dockerfiles/cpubuilder_ubuntu_jammy_x86_64.dockerfile . # Note the hash here sudo docker run --rm --mount type=bind,source="/home/scott/dev/projects/iree",target=/iree,readonly -it --entrypoint /bin/bash ${HASH} # Docker python3 -m pip install -r /iree/runtime/bindings/python/iree/runtime/build_requirements.txt cmake \ -S /iree \ -G Ninja \ -B /build \ -DPython3_EXECUTABLE="$(which python)" \ -DCMAKE_BUILD_TYPE=Debug \ -DIREE_BUILD_PYTHON_BINDINGS=ON \ -DIREE_ENABLE_LLD=ON \ -DIREE_ENABLE_ASSERTIONS=ON # Configure mostly finished, good enough? -- Configuring done CMake Error: Cannot open file for write: /iree/.env.tmp86d6a CMake Error: : System Error: Read-only file system CMake Error in CMakeLists.txt: Could not open file for write in copy operation /iree/.env ``` For now the Dockerfiles for ghr and non-ghr versions both have large sections of code copy/pasted. Hopefully these will be edited infrequently and we don't need to overcomplicate that with multistage builds. The publish workflow also has duplicated code. That could use a matrix or reusable workflow. --- .../workflows/publish_cpubuilder_x86_64.yml | 37 ++++++++++++++++++- ...builder_ubuntu_jammy_ghr_x86_64.Dockerfile | 34 +++++------------ .../cpubuilder_ubuntu_jammy_x86_64.dockerfile | 32 ++++++++++++++++ dockerfiles/manylinux_x86_64.Dockerfile | 2 + 4 files changed, 80 insertions(+), 25 deletions(-) create mode 100644 dockerfiles/cpubuilder_ubuntu_jammy_x86_64.dockerfile diff --git a/.github/workflows/publish_cpubuilder_x86_64.yml b/.github/workflows/publish_cpubuilder_x86_64.yml index d6a69b6..37c9d41 100644 --- a/.github/workflows/publish_cpubuilder_x86_64.yml +++ b/.github/workflows/publish_cpubuilder_x86_64.yml @@ -4,10 +4,45 @@ on: push: branches: ['main'] paths: - - dockerfiles/cpubuilder*_ghr_x86_64.Dockerfile + - dockerfiles/cpubuilder*_x86_64.Dockerfile - .github/workflows/publish_cpubuilder_x86_64.yml jobs: + build-cpubuilder-ubuntu-jammy: + runs-on: ubuntu-latest + env: + REGISTRY: ghcr.io + IMAGE_NAME: iree-org/cpubuilder_ubuntu_jammy_x86_64 + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + file: dockerfiles/cpubuilder_ubuntu_jammy_x86_64.Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-cpubuilder-ubuntu-jammy-ghr: runs-on: ubuntu-latest env: diff --git a/dockerfiles/cpubuilder_ubuntu_jammy_ghr_x86_64.Dockerfile b/dockerfiles/cpubuilder_ubuntu_jammy_ghr_x86_64.Dockerfile index 61014c9..02cb306 100644 --- a/dockerfiles/cpubuilder_ubuntu_jammy_ghr_x86_64.Dockerfile +++ b/dockerfiles/cpubuilder_ubuntu_jammy_ghr_x86_64.Dockerfile @@ -1,45 +1,31 @@ # GitHub Actions Runner with IREE build deps. FROM docker.io/myoung34/github-runner:ubuntu-jammy -# Apt packages. +######## Basic apt packages ######## RUN apt update && \ apt install -y \ - wget python3.11 git unzip curl gnupg2 lsb-release && \ + wget python3.11-dev python3-pip git unzip curl gnupg2 lsb-release && \ update-alternatives --install /usr/local/bin/python python /usr/bin/python3.11 3 && \ update-alternatives --install /usr/local/bin/python3 python3 /usr/bin/python3.11 3 && \ apt install -y \ ccache clang-14 lld-14 libssl-dev ninja-build libxml2-dev llvm-dev pkg-config \ libcapstone-dev libtbb-dev libzstd-dev && \ - apt install -y \ - libboost-all-dev && \ apt clean && rm -rf /var/lib/apt/lists/* -# Python deps. -# It is expected the venvs get set up for real deployment needs. -# These are just basic deps for running automation. -RUN python -m pip --no-cache-dir install --upgrade pip && \ - python -m pip --no-cache-dir install \ - numpy==1.26.2 \ - requests==2.31.0 \ - setuptools==59.6.0 \ - PyYAML==5.4.1 - -# CMake. -# Version 3.23 has support for presets v4, needed for out of tree -# configurations. -RUN curl --silent --fail --show-error --location \ - "https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-x86_64.sh" \ - --output /cmake-installer.sh && \ - bash /cmake-installer.sh --skip-license --prefix=/usr && \ - rm -f /cmake-installer.sh +######## CMake ######## +WORKDIR /install-cmake +# Install our minimum supported CMake version. +ENV CMAKE_VERSION="3.23.2" +COPY build_tools/install_cmake.sh ./ +RUN ./install_cmake.sh "${CMAKE_VERSION}" && rm -rf /install-cmake +######## Build toolchain configuration ######## # Setup symlinks and alternatives. RUN ln -s /usr/bin/lld-14 /usr/bin/lld && \ ln -s /usr/bin/ld.lld-14 /usr/bin/ld.lld && \ ln -s /usr/bin/clang-14 /usr/bin/clang && \ ln -s /usr/bin/clang++-14 /usr/bin/clang++ - -# Environment. +# Default to using clang. This can be overriden to gcc as desired. ENV CC=clang ENV CXX=clang++ diff --git a/dockerfiles/cpubuilder_ubuntu_jammy_x86_64.dockerfile b/dockerfiles/cpubuilder_ubuntu_jammy_x86_64.dockerfile new file mode 100644 index 0000000..6864338 --- /dev/null +++ b/dockerfiles/cpubuilder_ubuntu_jammy_x86_64.dockerfile @@ -0,0 +1,32 @@ +# Stock Ubuntu with IREE build deps. +FROM ubuntu:jammy + +######## Basic apt packages ######## +RUN apt update && \ + apt install -y \ + wget python3.11-dev python3-pip git unzip curl gnupg2 lsb-release && \ + update-alternatives --install /usr/local/bin/python python /usr/bin/python3.11 3 && \ + update-alternatives --install /usr/local/bin/python3 python3 /usr/bin/python3.11 3 && \ + apt install -y \ + ccache clang-14 lld-14 libssl-dev ninja-build libxml2-dev llvm-dev pkg-config \ + libcapstone-dev libtbb-dev libzstd-dev && \ + apt clean && rm -rf /var/lib/apt/lists/* + +######## CMake ######## +WORKDIR /install-cmake +# Install our minimum supported CMake version. +ENV CMAKE_VERSION="3.23.2" +COPY build_tools/install_cmake.sh ./ +RUN ./install_cmake.sh "${CMAKE_VERSION}" && rm -rf /install-cmake + +######## Build toolchain configuration ######## +# Setup symlinks and alternatives. +RUN ln -s /usr/bin/lld-14 /usr/bin/lld && \ + ln -s /usr/bin/ld.lld-14 /usr/bin/ld.lld && \ + ln -s /usr/bin/clang-14 /usr/bin/clang && \ + ln -s /usr/bin/clang++-14 /usr/bin/clang++ +# Default to using clang. This can be overriden to gcc as desired. +ENV CC=clang +ENV CXX=clang++ + +WORKDIR / diff --git a/dockerfiles/manylinux_x86_64.Dockerfile b/dockerfiles/manylinux_x86_64.Dockerfile index ab9cac4..0bc90d9 100644 --- a/dockerfiles/manylinux_x86_64.Dockerfile +++ b/dockerfiles/manylinux_x86_64.Dockerfile @@ -35,3 +35,5 @@ RUN ./install_bazel.sh && rm -rf /install-bazel # We use the wildcard option to disable the checks. This was added # in git 2.35.3 RUN git config --global --add safe.directory '*' + +WORKDIR /