Forward-merge branch-23.12 into branch-24.02

.github/actions/build-and-test-feature/action.yml

@@ -8,8 +8,6 @@ inputs:
   vault_host: {type: string, defaut: '', required: false}
   rw_sccache_bucket: {type: string, defaut: '', required: false}
   rw_sccache_region: {type: string, defaut: '', required: false}
-  ro_sccache_bucket: {type: string, defaut: '', required: false}
-  ro_sccache_region: {type: string, defaut: '', required: false}
 
 runs:
   using: composite
@@ -34,5 +32,3 @@ runs:
         vault_host: "${{ inputs.vault_host }}"
         rw_sccache_bucket: "${{ inputs.rw_sccache_bucket }}"
         rw_sccache_region: "${{ inputs.rw_sccache_region }}"
-        ro_sccache_bucket: "${{ inputs.ro_sccache_bucket }}"
-        ro_sccache_region: "${{ inputs.ro_sccache_region }}"

.github/workflows/build-and-test-feature.yml

@@ -25,6 +25,12 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
+          password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
+
       - name: ${{ inputs.name }}
         uses: ./.github/actions/build-and-test-feature
         with:
@@ -33,5 +39,3 @@ jobs:
           vault_host: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'https://vault.ops.k8s.rapids.ai' || '' }}"
           rw_sccache_bucket: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'rapids-sccache-devs' || '' }}"
           rw_sccache_region: "${{ vars.AWS_REGION }}"
-          ro_sccache_bucket: rapids-sccache-east
-          ro_sccache_region: "${{ vars.AWS_REGION }}"

.github/workflows/build-test-and-push-linux-image.yml

@@ -51,6 +51,12 @@ jobs:
           features: "${{ inputs.features }}"
           container_env: "${{ inputs.container_env }}"
 
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
+          password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
+
       - name: Build ${{ steps.json.outputs.tag }}-${{ matrix.arch }}
         uses: ./.github/actions/build-linux-image
         with:

.github/workflows/build-test-and-push-windows-image.yml

@@ -59,6 +59,12 @@ jobs:
           tag=${version}-cuda${cuda}-cl${cl}-${{ inputs.os }}${{ matrix.edition }}
           EOF
 
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
+          password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
+
       - name: Build ${{ steps.info.outputs.tag }}
         uses: ./.github/actions/build-windows-image
         with:
@@ -79,13 +85,6 @@ jobs:
           version: "${{ steps.info.outputs.version }}"
           edition: "${{ matrix.edition }}"
 
-      - if: inputs.push == 'true'
-        name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
-          password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
-
       - if: inputs.push == 'true'
         name: Push ${{ steps.info.outputs.tag }}
         shell: powershell

features/src/rapids-build-utils/devcontainer-feature.json

@@ -1,7 +1,7 @@
 {
   "name": "NVIDIA RAPIDS devcontainer build utilities",
   "id": "rapids-build-utils",
-  "version": "23.12.1",
+  "version": "23.12.3",
   "description": "A feature to install the RAPIDS devcontainer build utilities",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"

features/src/rapids-build-utils/install.sh

@@ -11,20 +11,21 @@ check_packages bc jq sudo wget gettext-base bash-completion ca-certificates;
 
 # Install yq if not installed
 if ! type yq >/dev/null 2>&1; then
-    YQ_VERSION=latest;
-    find_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq;
-
     YQ_BINARY="yq";
     YQ_BINARY+="_$(uname -s | tr '[:upper:]' '[:lower:]')";
     YQ_BINARY+="_${TARGETARCH:-$(dpkg --print-architecture | awk -F'-' '{print $NF}')}";
 
-    wget --no-hsts -q -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" \
-        | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/";
+    YQ_VERSION=latest;
+    find_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq;
+    while ! wget --no-hsts -q -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/"; do
+        echo "(!) YQ version ${YQ_VERSION} failed to download. Attempting to fall back one version to retry...";
+        find_prev_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq;
+    done
 fi
 
 # Install the rapids dependency file generator and conda-merge
 if type python >/dev/null 2>&1; then
-    python -m pip install rapids-dependency-file-generator conda-merge toml;
+    python -m pip install 'rapids-dependency-file-generator<1.14' conda-merge toml;
 fi
 
 # Install RAPIDS build utility scripts to /opt/

features/src/sccache/devcontainer-feature.json

@@ -1,14 +1,14 @@
 {
   "name": "sccache",
   "id": "sccache",
-  "version": "23.12.1",
+  "version": "23.12.2",
   "description": "A feature to install sccache",
   "options": {
     "version": {
       "type": "string",
       "proposals": [
         "latest",
-        "0.7.4",
+        "0.7.7",
         "0.4.0",
         "0.3.1",
         "0.3.0",

features/src/utils/devcontainer-feature.json

@@ -1,7 +1,7 @@
 {
   "name": "devcontainer-utils",
   "id": "utils",
-  "version": "23.12.2",
+  "version": "23.12.3",
   "description": "A feature to install RAPIDS devcontainer utility scripts",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"

features/src/utils/install.sh

@@ -26,15 +26,16 @@ fi
 
 # Install yq if not installed
 if ! type yq >/dev/null 2>&1; then
-    YQ_VERSION=latest;
-    find_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq;
-
     YQ_BINARY="yq";
     YQ_BINARY+="_$(uname -s | tr '[:upper:]' '[:lower:]')";
     YQ_BINARY+="_${TARGETARCH:-$(dpkg --print-architecture | awk -F'-' '{print $NF}')}";
 
-    wget --no-hsts -q -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" \
-        | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/";
+    YQ_VERSION=latest;
+    find_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq;
+    while ! wget --no-hsts -q -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/"; do
+        echo "(!) YQ version ${YQ_VERSION} failed to download. Attempting to fall back one version to retry...";
+        find_prev_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq;
+    done
 fi
 
 # Remove built-in anacron configs

features/src/utils/opt/devcontainer/bin/vault/s3/creds/persist.sh

@@ -38,7 +38,6 @@ store_s3_creds() {
     local region=;
     local no_bucket=;
     local no_region=;
-    local no_credentials=;
     local aws_access_key_id=;
     local aws_session_token=;
     local aws_secret_access_key=;
@@ -50,7 +49,6 @@ store_s3_creds() {
             region                            |
             no_bucket                         |
             no_region                         |
-            no_credentials                    |
             aws_access_key_id                 |
             aws_session_token                 |
             aws_secret_access_key             |
@@ -64,7 +62,6 @@ store_s3_creds() {
     reset_envvar "AWS_ACCESS_KEY_ID";
     reset_envvar "AWS_SESSION_TOKEN";
     reset_envvar "AWS_SECRET_ACCESS_KEY";
-    reset_envvar "SCCACHE_S3_NO_CREDENTIALS";
 
     mkdir -p ~/.aws;
     rm -f ~/.aws/{config,credentials};
@@ -92,41 +89,37 @@ ________EOF
     fi
 
     if test -f ~/.aws/config; then
-        cat <<________EOF > ~/.aws/config
+        cat <<________EOF > ~/.aws/config2 && mv ~/.aws/config{2,}
 [default]
 $(cat ~/.aws/config)
 ________EOF
     fi
 
-    if ! grep -qE "^$" <<< "${no_credentials:-}"; then
-        export_envvar "SCCACHE_S3_NO_CREDENTIALS" "1";
-    else
 
-        if ! grep -qE "^$" <<< "${aws_access_key_id:-}"; then
-            cat <<____________EOF >> ~/.aws/credentials
+    if ! grep -qE "^$" <<< "${aws_access_key_id:-}"; then
+        cat <<________EOF >> ~/.aws/credentials
 aws_access_key_id=${aws_access_key_id}
-____________EOF
-        fi
+________EOF
+    fi
 
-        if ! grep -qE "^$" <<< "${aws_session_token:-}"; then
-            cat <<____________EOF >> ~/.aws/credentials
+    if ! grep -qE "^$" <<< "${aws_session_token:-}"; then
+        cat <<________EOF >> ~/.aws/credentials
 aws_session_token=${aws_session_token}
-____________EOF
-        fi
+________EOF
+    fi
 
-        if ! grep -qE "^$" <<< "${aws_secret_access_key:-}"; then
-            cat <<____________EOF >> ~/.aws/credentials
+    if ! grep -qE "^$" <<< "${aws_secret_access_key:-}"; then
+        cat <<________EOF >> ~/.aws/credentials
 aws_secret_access_key=${aws_secret_access_key}
-____________EOF
-        fi
+________EOF
+    fi
 
-        if test -f ~/.aws/credentials; then
-            cat <<____________EOF > ~/.aws/credentials
+    if test -f ~/.aws/credentials; then
+        cat <<________EOF > ~/.aws/credentials2 && mv ~/.aws/credentials{2,}
 [default]
 $(cat ~/.aws/credentials)
-____________EOF
-            chmod 0600 ~/.aws/credentials;
-        fi
+________EOF
+        chmod 0600 ~/.aws/credentials;
     fi
 }
 

features/src/utils/opt/devcontainer/bin/vault/s3/creds/propagate.sh

@@ -16,7 +16,7 @@ ensure_s3_creds_have_propagated() {
 
     while true; do
 
-        if SCCACHE_NO_DAEMON=1 sccache --show-stats >/dev/null 2>&1; then
+        if sccache --start-server >/dev/null 2>&1; then
             if [ "${num_restarts}" -gt "0" ]; then echo "Success!"; fi
             exit 0;
         fi

features/src/utils/opt/devcontainer/bin/vault/s3/creds/test.sh

@@ -17,47 +17,25 @@ test_aws_creds() {
         fi
     fi
 
-    local bucket="${SCCACHE_BUCKET:-"$(grep 'bucket=' ~/.aws/config 2>/dev/null | sed 's/bucket=//' || echo)"}";
+    local bucket="${SCCACHE_BUCKET:-"$(sed -n 's/bucket=//p' ~/.aws/config 2>/dev/null)"}";
     if [ -z "${bucket:-}" ]; then exit 1; fi
 
-    local region="${SCCACHE_REGION:-"${AWS_DEFAULT_REGION:-"$(grep 'region=' ~/.aws/config 2>/dev/null | sed 's/region=//' || echo)"}"}";
-    local aws_access_key_id="${AWS_ACCESS_KEY_ID:-"$(grep 'aws_access_key_id=' ~/.aws/credentials 2>/dev/null | sed 's/aws_access_key_id=//' || echo)"}";
-    local aws_session_token="${AWS_SESSION_TOKEN:-"$(grep 'aws_session_token=' ~/.aws/credentials 2>/dev/null | sed 's/aws_session_token=//' || echo)"}";
-    local aws_secret_access_key="${AWS_SECRET_ACCESS_KEY:-"$(grep 'aws_secret_access_key=' ~/.aws/credentials 2>/dev/null | sed 's/aws_secret_access_key=//' || echo)"}";
+    local region="${SCCACHE_REGION:-"${AWS_DEFAULT_REGION:-"$(sed -n 's/region=//p' ~/.aws/config 2>/dev/null)"}"}";
+    local aws_access_key_id="${AWS_ACCESS_KEY_ID:-"$(sed -n 's/aws_access_key_id=//p' ~/.aws/credentials 2>/dev/null)"}";
+    local aws_session_token="${AWS_SESSION_TOKEN:-"$(sed -n 's/aws_session_token=//p' ~/.aws/credentials 2>/dev/null)"}";
+    local aws_secret_access_key="${AWS_SECRET_ACCESS_KEY:-"$(sed -n 's/aws_secret_access_key=//p' ~/.aws/credentials 2>/dev/null)"}";
 
     if test -n "$(pgrep sccache || echo)"; then
         sccache --stop-server >/dev/null 2>&1 || true;
     fi
 
-    export SCCACHE_NO_DAEMON="1";
-    export SCCACHE_BUCKET="${bucket:-}";
-    export SCCACHE_REGION="${region:-}";
-    export AWS_ACCESS_KEY_ID="${aws_access_key_id:-}";
-    export AWS_SESSION_TOKEN="${aws_session_token:-}";
-    export AWS_SECRET_ACCESS_KEY="${aws_secret_access_key:-}";
-
-    if ! sccache --show-stats 2>&1 | grep -qE 'Cache location \s+ s3'; then
-
-        export SCCACHE_S3_NO_CREDENTIALS="1";
-
-        export AWS_ACCESS_KEY_ID=;
-        export AWS_SESSION_TOKEN=;
-        export AWS_SECRET_ACCESS_KEY=;
-        export -n AWS_ACCESS_KEY_ID;
-        export -n AWS_SESSION_TOKEN;
-        export -n AWS_SECRET_ACCESS_KEY;
-        unset AWS_ACCESS_KEY_ID;
-        unset AWS_SESSION_TOKEN;
-        unset AWS_SECRET_ACCESS_KEY;
-
-        if sccache --show-stats 2>&1 | grep -qE 'Cache location \s+ s3'; then
-            exit 2;
-        fi
-
-        exit 1;
-    fi
-
-    exit 0;
+    SCCACHE_BUCKET="${bucket:-}" \
+    SCCACHE_REGION="${region:-}" \
+    AWS_ACCESS_KEY_ID="${aws_access_key_id:-}" \
+    AWS_SESSION_TOKEN="${aws_session_token:-}" \
+    AWS_SECRET_ACCESS_KEY="${aws_secret_access_key:-}" \
+    sccache --start-server >/dev/null 2>&1;
+    sccache --show-stats | grep -qE 'Cache location \s+ s3';
 }
 
 if test -n "${devcontainer_utils_debug:-}"; then

features/src/utils/opt/devcontainer/bin/vault/s3/init.sh

@@ -9,9 +9,9 @@ ____EOF
 
 s3_bucket_auth() {
     cat <<____EOF
-        --aws_access_key_id='$(grep 'aws_access_key_id=' ~/.aws/credentials 2>/dev/null | sed 's/aws_access_key_id=//' || echo)'
-        --aws_session_token='$(grep 'aws_session_token=' ~/.aws/credentials 2>/dev/null | sed 's/aws_session_token=//' || echo)'
-        --aws_secret_access_key='$(grep 'aws_secret_access_key=' ~/.aws/credentials 2>/dev/null | sed 's/aws_secret_access_key=//' || echo)'
+        --aws_access_key_id='$(sed -n 's/aws_access_key_id=//p' ~/.aws/credentials 2>/dev/null)'
+        --aws_session_token='$(sed -n 's/aws_session_token=//p' ~/.aws/credentials 2>/dev/null)'
+        --aws_secret_access_key='$(sed -n 's/aws_secret_access_key=//p' ~/.aws/credentials 2>/dev/null)'
 ____EOF
 }
 
@@ -25,7 +25,7 @@ init_vault_s3_creds() {
             && grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}" ; then
                 if test -n "${VAULT_HOST:-}"                  ; then
                     # Generate S3 creds if they don't exist (or are expired)
-                    if devcontainer-utils-vault-s3-creds-test 2>&1 >/dev/null\
+                    if devcontainer-utils-vault-s3-creds-test \
                     || devcontainer-utils-vault-s3-creds-generate; then
                         # Persist creds in ~/.aws dir
                         devcontainer-utils-vault-s3-creds-persist <<< "
@@ -37,34 +37,21 @@ init_vault_s3_creds() {
                     else
                         devcontainer-utils-vault-s3-creds-persist <<< "--no_bucket --no_region";
                     fi
+                elif devcontainer-utils-vault-s3-creds-test; then
+                    # bucket is read + write with the current credentials
+                    devcontainer-utils-vault-s3-creds-persist <<< "
+                        $(s3_bucket_args)
+                        $(s3_bucket_auth)
+                    ";
                 else
-                    # If credentials have been mounted in, ensure they're used
-                    case $(devcontainer-utils-vault-s3-creds-test; echo $?) in
-                        # bucket is read + write with the current credentials
-                        [0] )
-                            devcontainer-utils-vault-s3-creds-persist <<< "
-                                $(s3_bucket_args)
-                                $(s3_bucket_auth)
-                            ";;
-                        # bucket is read-only and should be accessed without credentials
-                        [2] )
-                            devcontainer-utils-vault-s3-creds-persist <<< "
-                                --no_credentials
-                                $(s3_bucket_args)
-                            ";;
-                          # bucket is inaccessible
-                          * )
-                            devcontainer-utils-vault-s3-creds-persist <<< "--no_bucket --no_region";;
-                    esac
+                    # bucket is inaccessible
+                    devcontainer-utils-vault-s3-creds-persist <<< "--no_bucket --no_region";
                 fi
-            elif devcontainer-utils-vault-s3-creds-propagate; then
-                # Block until the new temporary AWS S3 credentials propagate
-                echo -n "";
+            elif ! devcontainer-utils-vault-s3-creds-propagate; then
+                # bucket is inaccessible
+                devcontainer-utils-vault-s3-creds-persist <<< "--no_bucket --no_region";
             fi
         fi
-        . /etc/profile.d/*-devcontainer-utils.sh;
-        # start the sccache server
-        sccache --start-server >/dev/null 2>&1 || true;
     fi
 }
 

features/test/oneapi/scenarios.json

@@ -0,0 +1,10 @@
+{
+  "oneapi_2023_2_0": {
+    "image": "ubuntu:22.04",
+    "features": {
+      "oneapi": {
+        "version": "2023.2.0"
+      }
+    }
+  }
+}