From fc0848ca3517702dd113a8571ab5aa3251c47df3 Mon Sep 17 00:00:00 2001 From: Colin Palmer <163194595+colinpalmer-pro@users.noreply.github.com> Date: Thu, 12 Sep 2024 22:07:32 -0400 Subject: [PATCH] 331 dockerhub rate limit - Explicitly state docker.io repo in base images, update fabric8 to use own .docker config location and test build2 --- .github/workflows/build2.yml | 120 ++++++++++++++++++ build-parent/pom.xml | 2 +- .../src/main/resources/docker/Dockerfile | 4 +- .../src/main/resources/docker/Dockerfile | 4 +- .../src/main/resources/docker/Dockerfile | 2 +- .../src/main/resources/docker/Dockerfile | 2 +- .../src/main/resources/docker/Dockerfile | 2 +- .../src/main/resources/docker/Dockerfile | 2 +- .../src/main/resources/docker/Dockerfile | 2 +- .../src/main/resources/docker/Dockerfile | 2 +- .../src/main/resources/docker/Dockerfile | 2 +- extensions/extensions-docker/pom.xml | 3 +- 12 files changed, 134 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/build2.yml diff --git a/.github/workflows/build2.yml b/.github/workflows/build2.yml new file mode 100644 index 000000000..1c0fbaf3d --- /dev/null +++ b/.github/workflows/build2.yml @@ -0,0 +1,120 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Build aissemble 2 + +on: + workflow_dispatch: + inputs: + buildBranch: + description: "Branch you want to build" + required: true + type: string + default: "331-dockerhub-rate-limit" + push: + branches: [ "331-dockerhub-rate-limit" ] + +jobs: + build: + runs-on: arc-runner-set-aissemble + env: + DOCKER_CONFIG: /home/runner/.docker + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.buildBranch }} + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Install required packages + run: | + sudo apt-get update + sudo apt install -y build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl \ + git libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev zip unzip \ + libpython3.11 + - name: Install Python + uses: gabrielfalcao/pyenv-action@v18 + with: + default: 3.11.4 + - name: Load m2 repository cache # Manually caching .m2 repo as the setup-java caching isn't falling back to older caches + id: cached-m2-repo + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven- + - name: Load m2 build cache + id: cached-m2-build + uses: actions/cache@v4 + with: + path: ~/.m2/build-cache + key: maven-build-cache-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-build-cache- + - name: Install Poetry + uses: snok/install-poetry@v1 + - name: Poetry cache + id: cached-poetry + uses: actions/cache@v4 + with: + path: ~/.cache/pypoetry + key: poetry-cache-${{ hashFiles('**/pom.xml') }} + restore-keys: | + poetry- + - name: Install Helm + run: | + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh + - name: Install Helm Unittest Plugin + run: | + echo "Updating helm unittest plugin to latest version..." + helm plugin install https://github.com/helm-unittest/helm-unittest.git + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + - name: Create Docker Builder Config File + run: sudo touch /etc/buildkitd.toml + # Generate the settings.xml for ghcr.io, pypi, & dev-pypi server profiles + - name: Create settings.xml + run: | + echo "ghcr.io${{ secrets.GHCR_IO_USERNAME }}${{ secrets.GHCR_IO_TOKEN }}pypi${{ secrets.PYPI_USERNAME }}${{ secrets.PYPI_TOKEN }}dev-pypi${{ secrets.TEST_PYPI_USERNAME }}${{ secrets.TEST_PYPI_TOKEN }} " > $HOME/.m2/settings.xml + # Run build with the gh-build profile + - name: Build aiSSEMBLE + run: | + ./mvnw -B clean install -Dmaven.build.cache.skipCache=true -U -file pom.xml -Pci,integration-test --settings $HOME/.m2/settings.xml + # Install Maven which is needed for archetype tests + - name: Set up Maven + uses: stCarolas/setup-maven@v5 + with: + maven-version: 3.9.9 + # Execute archetype tests + - name: Run Archetype Tests + run: | + ./mvnw -B clean install -Parchetype-test -pl :foundation-archetype + #NB: The following two explicit cache saves are necessary to ensure caches are saved on build failure, + # until https://github.com/actions/cache/issues/1315 is resolved + - name: Save m2 repository cache + id: save-m2-repo + uses: actions/cache/save@v4 + if: always() + with: + path: ~/.m2/repository + key: maven-${{ hashFiles('**/pom.xml') }} + - name: Save m2 build cache + id: save-m2-build + uses: actions/cache/save@v4 + if: always() + with: + path: ~/.m2/build-cache + key: maven-build-cache-${{ hashFiles('**/pom.xml') }} \ No newline at end of file diff --git a/build-parent/pom.xml b/build-parent/pom.xml index 94666e138..bd9ee7dd5 100644 --- a/build-parent/pom.xml +++ b/build-parent/pom.xml @@ -599,7 +599,7 @@ default - ~/.docker + ~/.docker/fabric8 ${docker.platforms} diff --git a/extensions/extensions-docker/aissemble-fastapi/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-fastapi/src/main/resources/docker/Dockerfile index 56572d4be..e454fb896 100644 --- a/extensions/extensions-docker/aissemble-fastapi/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-fastapi/src/main/resources/docker/Dockerfile @@ -1,6 +1,6 @@ # Script for creating base FastAPI Docker image -FROM python:3.11 +FROM docker.io/python:3.11 LABEL org.opencontainers.image.source="https://github.com/boozallen/aissemble" @@ -10,7 +10,7 @@ WORKDIR /app COPY ./src/main/resources/docker/requirements.txt /tmp/requirements.txt RUN pip install --no-cache-dir --upgrade -r /tmp/requirements.txt -# Custom start script to run fastAPI with specific +# Custom start script to run fastAPI with specific # module (defined using MODULE environment variable) COPY ./src/main/resources/docker/scripts/start.sh /start.sh RUN chmod +x /start.sh diff --git a/extensions/extensions-docker/aissemble-hive-service/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-hive-service/src/main/resources/docker/Dockerfile index 0c90b4aed..0d1b09f0a 100644 --- a/extensions/extensions-docker/aissemble-hive-service/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-hive-service/src/main/resources/docker/Dockerfile @@ -1,7 +1,7 @@ ARG METASTORE_VERSION -FROM apache/hive:${METASTORE_VERSION} AS appsource +FROM docker.io/apache/hive:${METASTORE_VERSION} AS appsource -FROM eclipse-temurin:17-jre AS final +FROM docker.io/eclipse-temurin:17-jre AS final LABEL org.opencontainers.image.source="https://github.com/boozallen/aissemble" diff --git a/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-agent/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-agent/src/main/resources/docker/Dockerfile index 1f502edf3..5ef79dde4 100644 --- a/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-agent/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-agent/src/main/resources/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM jenkins/ssh-agent:latest-debian-jdk11 +FROM docker.io/jenkins/ssh-agent:latest-debian-jdk11 LABEL org.opencontainers.image.source="https://github.com/boozallen/aissemble" diff --git a/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-controller/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-controller/src/main/resources/docker/Dockerfile index 1932638c7..8bee6d0b0 100644 --- a/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-controller/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-jenkins/aissemble-jenkins-controller/src/main/resources/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM jenkins/jenkins:lts-jdk11 +FROM docker.io/jenkins/jenkins:lts-jdk11 LABEL org.opencontainers.image.source="https://github.com/boozallen/aissemble" diff --git a/extensions/extensions-docker/aissemble-nvidia/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-nvidia/src/main/resources/docker/Dockerfile index ebed75a80..9f36255ad 100644 --- a/extensions/extensions-docker/aissemble-nvidia/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-nvidia/src/main/resources/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM nvidia/cuda:12.1.1-base-ubuntu22.04 +FROM docker.io/nvidia/cuda:12.1.1-base-ubuntu22.04 LABEL org.opencontainers.image.source="https://github.com/boozallen/aissemble" diff --git a/extensions/extensions-docker/aissemble-spark-operator/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-spark-operator/src/main/resources/docker/Dockerfile index e3e16af54..58988db9f 100644 --- a/extensions/extensions-docker/aissemble-spark-operator/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-spark-operator/src/main/resources/docker/Dockerfile @@ -16,7 +16,7 @@ ARG DOCKER_BASELINE_REPO_ID ARG VERSION_AISSEMBLE -FROM kubeflow/spark-operator:v1beta2-1.6.2-3.5.0 AS builder +FROM docker.io/kubeflow/spark-operator:v1beta2-1.6.2-3.5.0 AS builder # We would be able to use the kubeflow image directly, except that it is on Spark 3.5 instead of 3.4 FROM ${DOCKER_BASELINE_REPO_ID}boozallen/aissemble-spark:${VERSION_AISSEMBLE} diff --git a/extensions/extensions-docker/aissemble-spark/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-spark/src/main/resources/docker/Dockerfile index 7973dfc09..85a4b30c0 100644 --- a/extensions/extensions-docker/aissemble-spark/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-spark/src/main/resources/docker/Dockerfile @@ -1,7 +1,7 @@ # Script for creating base Spark Docker image # ARG SPARK_VERSION -FROM apache/spark-py:v${SPARK_VERSION} +FROM docker.io/apache/spark-py:v${SPARK_VERSION} LABEL org.opencontainers.image.source="https://github.com/boozallen/aissemble" diff --git a/extensions/extensions-docker/aissemble-vault/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-vault/src/main/resources/docker/Dockerfile index 08f09aee7..288957755 100644 --- a/extensions/extensions-docker/aissemble-vault/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-vault/src/main/resources/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.13 +FROM docker.io/alpine:3.13 LABEL org.opencontainers.image.source="https://github.com/boozallen/aissemble" diff --git a/extensions/extensions-docker/aissemble-versioning/src/main/resources/docker/Dockerfile b/extensions/extensions-docker/aissemble-versioning/src/main/resources/docker/Dockerfile index 029748a54..260d84e48 100644 --- a/extensions/extensions-docker/aissemble-versioning/src/main/resources/docker/Dockerfile +++ b/extensions/extensions-docker/aissemble-versioning/src/main/resources/docker/Dockerfile @@ -20,7 +20,7 @@ RUN --mount=type=cache,target=/.cache/pypoetry/ \ poetry bundle venv /opt/venv #HABUSHU_BUILDER_STAGE - HABUSHU GENERATED CODE (END) -FROM python:3.11 AS builder +FROM docker.io/python:3.11 AS builder # Download Maven ARG MAVEN_VERSION=3.9.6 diff --git a/extensions/extensions-docker/pom.xml b/extensions/extensions-docker/pom.xml index 7464d0fc7..397417134 100644 --- a/extensions/extensions-docker/pom.xml +++ b/extensions/extensions-docker/pom.xml @@ -26,7 +26,8 @@ ${group.fabric8.plugin} docker-maven-plugin - + +