diff --git a/.github/actions/gpu_setup/action.yml b/.github/actions/gpu_setup/action.yml new file mode 100644 index 0000000000..a09740e060 --- /dev/null +++ b/.github/actions/gpu_setup/action.yml @@ -0,0 +1,63 @@ +name: Setup Cuda +description: Setup Cuda on Hyperstack or GitHub instance + +inputs: + cuda-version: + description: Version of Cuda to use + required: true + gcc-version: + description: Version of GCC to use + required: true + cmake-version: + description: Version of cmake to use + default: 3.29.6 + github-instance: + description: Instance is hosted on GitHub + default: 'false' + +runs: + using: "composite" + steps: + # Mandatory on hyperstack since a bootable volume is not re-usable yet. + - name: Install dependencies + shell: bash + run: | + sudo apt update + curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo gpg --dearmour -o /etc/apt/trusted.gpg.d/kitware.gpg + sudo chmod 644 /etc/apt/trusted.gpg.d/kitware.gpg + echo 'deb [signed-by=/etc/apt/trusted.gpg.d/kitware.gpg] https://apt.kitware.com/ubuntu/ jammy main' | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null + sudo apt update + sudo apt install -y cmake cmake-format libclang-dev + + - name: Install CUDA + if: inputs.github-instance == 'true' + shell: bash + run: | + TOOLKIT_VERSION="$(echo ${{ inputs.cuda-version }} | sed 's/\(.*\)\.\(.*\)/\1-\2/')" + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb + sudo dpkg -i cuda-keyring_1.1-1_all.deb + sudo apt update + sudo apt -y install cuda-toolkit-${TOOLKIT_VERSION} + + - name: Export CUDA variables + shell: bash + run: | + CUDA_PATH=/usr/local/cuda-${{ inputs.cuda-version }} + echo "CUDA_PATH=$CUDA_PATH" >> "${GITHUB_ENV}" + echo "PATH=$PATH:$CUDA_PATH/bin" >> "${GITHUB_PATH}" + echo "LD_LIBRARY_PATH=$CUDA_PATH/lib64:$LD_LIBRARY_PATH" >> "${GITHUB_ENV}" + echo "CUDA_MODULE_LOADER=EAGER" >> "${GITHUB_ENV}" + + # Specify the correct host compilers + - name: Export gcc and g++ variables + shell: bash + run: | + { + echo "CC=/usr/bin/gcc-${{ inputs.gcc-version }}"; + echo "CXX=/usr/bin/g++-${{ inputs.gcc-version }}"; + echo "CUDAHOSTCXX=/usr/bin/g++-${{ inputs.gcc-version }}"; + } >> "${GITHUB_ENV}" + + - name: Check device is detected + shell: bash + run: nvidia-smi diff --git a/.github/actions/hyperstack_setup/action.yml b/.github/actions/hyperstack_setup/action.yml deleted file mode 100644 index 34315dbadf..0000000000 --- a/.github/actions/hyperstack_setup/action.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Setup Cuda -description: Setup Cuda on Hyperstack instance - -inputs: - cuda-version: - description: Version of Cuda to use - required: true - gcc-version: - description: Version of GCC to use - required: true - cmake-version: - description: Version of cmake to use - default: 3.29.6 - -runs: - using: "composite" - steps: - # Mandatory on hyperstack since a bootable volume is not re-usable yet. - - name: Install dependencies - shell: bash - run: | - sudo apt update - sudo apt install -y checkinstall zlib1g-dev libssl-dev libclang-dev - wget https://github.com/Kitware/CMake/releases/download/v${{ inputs.cmake-version }}/cmake-${{ inputs.cmake-version }}.tar.gz - tar -zxvf cmake-${{ inputs.cmake-version }}.tar.gz - cd cmake-${{ inputs.cmake-version }} - ./bootstrap - make -j"$(nproc)" - sudo make install - - - name: Export CUDA variables - shell: bash - run: | - CUDA_PATH=/usr/local/cuda-${{ inputs.cuda-version }} - echo "CUDA_PATH=$CUDA_PATH" >> "${GITHUB_ENV}" - echo "$CUDA_PATH/bin" >> "${GITHUB_PATH}" - echo "LD_LIBRARY_PATH=$CUDA_PATH/lib:$LD_LIBRARY_PATH" >> "${GITHUB_ENV}" - echo "CUDACXX=/usr/local/cuda-${{ inputs.cuda-version }}/bin/nvcc" >> "${GITHUB_ENV}" - - # Specify the correct host compilers - - name: Export gcc and g++ variables - shell: bash - run: | - { - echo "CC=/usr/bin/gcc-${{ inputs.gcc-version }}"; - echo "CXX=/usr/bin/g++-${{ inputs.gcc-version }}"; - echo "CUDAHOSTCXX=/usr/bin/g++-${{ inputs.gcc-version }}"; - echo "HOME=/home/ubuntu"; - } >> "${GITHUB_ENV}" - - - name: Check device is detected - shell: bash - run: nvidia-smi diff --git a/.github/workflows/aws_tfhe_backward_compat_tests.yml b/.github/workflows/aws_tfhe_backward_compat_tests.yml index f711e3f6ab..01a6a72538 100644 --- a/.github/workflows/aws_tfhe_backward_compat_tests.yml +++ b/.github/workflows/aws_tfhe_backward_compat_tests.yml @@ -11,53 +11,26 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} - REF: ${{ github.event.pull_request.head.sha || github.sha }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "large_ubuntu_16" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - paths: - - '**' - - '!.github/**' - - '!ci/**' jobs: - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (backward-compat-tests) - needs: check-user-permission runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -67,6 +40,13 @@ jobs: backend: aws profile: cpu-small + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + backward-compat-tests: name: Backward compatibility tests needs: [ setup-instance ] @@ -79,8 +59,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -131,8 +110,9 @@ jobs: needs: [ setup-instance, backward-compat-tests ] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/aws_tfhe_fast_tests.yml b/.github/workflows/aws_tfhe_fast_tests.yml index ab3ef8baff..2fb1c59eab 100644 --- a/.github/workflows/aws_tfhe_fast_tests.yml +++ b/.github/workflows/aws_tfhe_fast_tests.yml @@ -11,26 +11,16 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} - IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} - REF: ${{ github.event.pull_request.head.sha || github.sha }} + IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "large_ubuntu_64-22.04" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - paths: - - '**' - - '!.github/**' - - '!ci/**' jobs: should-run: @@ -69,8 +59,7 @@ jobs: with: fetch-depth: 0 persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Check for file changes id: changed-files @@ -137,34 +126,18 @@ jobs: run: | echo "any_changed=true" >> "$GITHUB_OUTPUT" - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (fast-tests) if: github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && needs.should-run.outputs.any_file_changed == 'true') - needs: [ should-run, check-user-permission ] + needs: should-run runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -174,6 +147,13 @@ jobs: backend: aws profile: cpu-big + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + fast-tests: name: Fast CPU tests needs: [ should-run, setup-instance ] @@ -186,8 +166,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -286,7 +265,7 @@ jobs: make test_zk - name: Slack Notification - if: ${{ failure() }} + if: ${{ failure() && env.SECRETS_AVAILABLE == 'true' }} continue-on-error: true uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 env: @@ -299,8 +278,9 @@ jobs: needs: [ setup-instance, fast-tests ] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/aws_tfhe_integer_tests.yml b/.github/workflows/aws_tfhe_integer_tests.yml index 6af7b62fce..be27d28b58 100644 --- a/.github/workflows/aws_tfhe_integer_tests.yml +++ b/.github/workflows/aws_tfhe_integer_tests.yml @@ -10,31 +10,20 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} # We clear the cache to reduce memory pressure because of the numerous processes of cargo # nextest TFHE_RS_CLEAR_IN_MEMORY_KEY_CACHE: "1" NO_BIG_PARAMS: FALSE - REF: ${{ github.event.pull_request.head.sha || github.sha }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "large_ubuntu_64-22.04" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: types: [ labeled ] - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - types: [ labeled ] - paths: - - '**' - - '!.github/**' - - '!ci/**' push: branches: - main @@ -43,7 +32,7 @@ jobs: should-run: if: (github.event_name == 'push' && github.repository == 'zama-ai/tfhe-rs') || - (github.event_name == 'pull_request_target' && contains(github.event.label.name, 'approved')) || + (github.event_name == 'pull_request' && contains(github.event.label.name, 'approved')) || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest permissions: @@ -57,8 +46,7 @@ jobs: with: fetch-depth: 0 persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Check for file changes id: changed-files @@ -75,26 +63,9 @@ jobs: - tfhe/src/integer/** - .github/workflows/aws_tfhe_integer_tests.yml - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (unsigned-integer-tests) - needs: [ should-run, check-user-permission ] + needs: should-run if: (github.event_name == 'push' && github.repository == 'zama-ai/tfhe-rs' && needs.should-run.outputs.integer_test == 'true') || (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') || @@ -102,10 +73,11 @@ jobs: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -115,6 +87,13 @@ jobs: backend: aws profile: cpu-big + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + unsigned-integer-tests: name: Unsigned integer tests needs: setup-instance @@ -127,8 +106,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: "false" - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -136,7 +114,7 @@ jobs: toolchain: stable - name: Should skip big parameters set - if: github.event_name == 'pull_request_target' + if: github.event_name == 'pull_request' run: | echo "NO_BIG_PARAMS=TRUE" >> "${GITHUB_ENV}" @@ -170,8 +148,9 @@ jobs: needs: [setup-instance, unsigned-integer-tests] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/aws_tfhe_signed_integer_tests.yml b/.github/workflows/aws_tfhe_signed_integer_tests.yml index ce5ffcdc6a..48047aaf18 100644 --- a/.github/workflows/aws_tfhe_signed_integer_tests.yml +++ b/.github/workflows/aws_tfhe_signed_integer_tests.yml @@ -10,31 +10,20 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} # We clear the cache to reduce memory pressure because of the numerous processes of cargo # nextest TFHE_RS_CLEAR_IN_MEMORY_KEY_CACHE: "1" NO_BIG_PARAMS: FALSE - REF: ${{ github.event.pull_request.head.sha || github.sha }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "large_ubuntu_64-22.04" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: types: [ labeled ] - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - types: [ labeled ] - paths: - - '**' - - '!.github/**' - - '!ci/**' push: branches: - main @@ -44,7 +33,7 @@ jobs: if: (github.event_name == 'push' && github.repository == 'zama-ai/tfhe-rs') || (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') || - ((github.event_name == 'pull_request_target' || github.event_name == 'pull_request_target') && contains(github.event.label.name, 'approved')) || + (github.event_name == 'pull_request' && contains(github.event.label.name, 'approved')) || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest permissions: @@ -58,8 +47,7 @@ jobs: with: fetch-depth: 0 persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Check for file changes id: changed-files @@ -76,26 +64,9 @@ jobs: - tfhe/src/integer/** - .github/workflows/aws_tfhe_signed_integer_tests.yml - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (unsigned-integer-tests) - needs: [ should-run, check-user-permission ] + needs: should-run if: (github.event_name == 'push' && github.repository == 'zama-ai/tfhe-rs' && needs.should-run.outputs.integer_test == 'true') || (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') || @@ -103,10 +74,11 @@ jobs: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -116,6 +88,13 @@ jobs: backend: aws profile: cpu-big + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + signed-integer-tests: name: Signed integer tests needs: setup-instance @@ -128,8 +107,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: "false" - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -137,7 +115,7 @@ jobs: toolchain: stable - name: Should skip big parameters set - if: github.event_name == 'pull_request_target' + if: github.event_name == 'pull_request' run: | echo "NO_BIG_PARAMS=TRUE" >> "${GITHUB_ENV}" @@ -175,8 +153,9 @@ jobs: needs: [setup-instance, signed-integer-tests] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/aws_tfhe_tests.yml b/.github/workflows/aws_tfhe_tests.yml index 9e2c8f144c..706dd6ca6d 100644 --- a/.github/workflows/aws_tfhe_tests.yml +++ b/.github/workflows/aws_tfhe_tests.yml @@ -10,28 +10,17 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} - IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} - REF: ${{ github.event.pull_request.head.sha || github.sha }} + IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "large_ubuntu_64-22.04" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: types: [ labeled ] - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - types: [ labeled ] - paths: - - '**' - - '!.github/**' - - '!ci/**' schedule: # Nightly tests @ 1AM after each work day - cron: "0 1 * * MON-FRI" @@ -79,8 +68,7 @@ jobs: with: fetch-depth: 0 persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Check for file changes id: changed-files @@ -147,34 +135,18 @@ jobs: run: | echo "any_changed=true" >> "$GITHUB_OUTPUT" - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (cpu-tests) - if: github.event_name != 'pull_request_target' || + if: github.event_name != 'pull_request' || (github.event.action == 'labeled' && github.event.label.name == 'approved' && needs.should-run.outputs.any_file_changed == 'true') - needs: [ should-run, check-user-permission ] + needs: should-run runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -184,10 +156,17 @@ jobs: backend: aws profile: cpu-big + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + cpu-tests: name: CPU tests - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.setup-instance.result != 'skipped') + if: github.event_name != 'pull_request' || + (github.event_name == 'pull_request' && needs.setup-instance.result != 'skipped') needs: [ should-run, setup-instance ] concurrency: group: ${{ github.workflow }}_${{github.event_name}}_${{ github.head_ref || github.ref }} @@ -198,8 +177,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -282,8 +260,9 @@ jobs: needs: [ setup-instance, cpu-tests ] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/aws_tfhe_wasm_tests.yml b/.github/workflows/aws_tfhe_wasm_tests.yml index 6292d8ab8c..e09d5855b1 100644 --- a/.github/workflows/aws_tfhe_wasm_tests.yml +++ b/.github/workflows/aws_tfhe_wasm_tests.yml @@ -10,56 +10,28 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} - REF: ${{ github.event.pull_request.head.sha || github.sha }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "large_ubuntu_16" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: types: [ labeled ] - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - types: [ labeled ] - paths: - - '**' - - '!.github/**' - - '!ci/**' jobs: - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (wasm-tests) - needs: check-user-permission if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.label.name, 'approved') }} runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -69,6 +41,13 @@ jobs: backend: aws profile: cpu-small + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + wasm-tests: name: WASM tests needs: setup-instance @@ -81,8 +60,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -151,8 +129,9 @@ jobs: needs: [ setup-instance, wasm-tests ] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/benchmark_gpu_4090.yml b/.github/workflows/benchmark_gpu_4090.yml index b97a0f2676..20b73bf9c1 100644 --- a/.github/workflows/benchmark_gpu_4090.yml +++ b/.github/workflows/benchmark_gpu_4090.yml @@ -11,53 +11,20 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} FAST_BENCH: TRUE - REF: ${{ github.event.pull_request.head.sha || github.sha }} on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: types: [ labeled ] - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - types: [ labeled ] - paths: - - '**' - - '!.github/**' - - '!ci/**' schedule: # Weekly benchmarks will be triggered each Friday at 9p.m. - cron: "0 21 * * 5" jobs: - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - cuda-integer-benchmarks: name: Cuda integer benchmarks (RTX 4090) - needs: check-user-permission if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs' || contains(github.event.label.name, '4090_bench') }} @@ -73,7 +40,6 @@ jobs: fetch-depth: 0 persist-credentials: 'false' token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} - name: Get benchmark details run: | @@ -220,7 +186,7 @@ jobs: remove_github_label: name: Remove 4090 bench label - if: ${{ always() && github.event_name == 'pull_request_target' }} + if: ${{ always() && github.event_name == 'pull_request' }} needs: [cuda-integer-benchmarks, cuda-core-crypto-benchmarks] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/benchmark_gpu_core_crypto.yml b/.github/workflows/benchmark_gpu_core_crypto.yml index f1aea7ae0b..7104ed83e2 100644 --- a/.github/workflows/benchmark_gpu_core_crypto.yml +++ b/.github/workflows/benchmark_gpu_core_crypto.yml @@ -57,7 +57,7 @@ jobs: token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - name: Setup Hyperstack dependencies - uses: ./.github/actions/hyperstack_setup + uses: ./.github/actions/gpu_setup with: cuda-version: ${{ matrix.cuda }} gcc-version: ${{ matrix.gcc }} diff --git a/.github/workflows/benchmark_gpu_erc20_common.yml b/.github/workflows/benchmark_gpu_erc20_common.yml index 0b1b0bbf0c..23e17fd5a7 100644 --- a/.github/workflows/benchmark_gpu_erc20_common.yml +++ b/.github/workflows/benchmark_gpu_erc20_common.yml @@ -84,7 +84,7 @@ jobs: token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - name: Setup Hyperstack dependencies - uses: ./.github/actions/hyperstack_setup + uses: ./.github/actions/gpu_setup with: cuda-version: ${{ matrix.cuda }} gcc-version: ${{ matrix.gcc }} diff --git a/.github/workflows/benchmark_gpu_integer_common.yml b/.github/workflows/benchmark_gpu_integer_common.yml index 10bb1d7076..3d7c82872e 100644 --- a/.github/workflows/benchmark_gpu_integer_common.yml +++ b/.github/workflows/benchmark_gpu_integer_common.yml @@ -154,7 +154,7 @@ jobs: token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - name: Setup Hyperstack dependencies - uses: ./.github/actions/hyperstack_setup + uses: ./.github/actions/gpu_setup with: cuda-version: ${{ matrix.cuda }} gcc-version: ${{ matrix.gcc }} diff --git a/.github/workflows/check_actor_permissions.yml b/.github/workflows/check_actor_permissions.yml deleted file mode 100644 index b2c4dcb751..0000000000 --- a/.github/workflows/check_actor_permissions.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Check if an actor is a collaborator and has write access -name: Check Actor Permissions - -on: - workflow_call: - inputs: - username: - type: string - default: ${{ github.triggering_actor }} - outputs: - is_authorized: - value: ${{ jobs.check-actor-permission.outputs.actor_authorized }} - secrets: - TOKEN: - required: true - -jobs: - check-actor-permission: - runs-on: ubuntu-latest - outputs: - actor_authorized: ${{ steps.check-access.outputs.require-result }} - steps: - - name: Get User Permission - id: check-access - uses: actions-cool/check-user-permission@7b90a27f92f3961b368376107661682c441f6103 # v2.3.0 - with: - require: write - username: ${{ inputs.username }} - env: - GITHUB_TOKEN: ${{ secrets.TOKEN }} - - - name: Check User Permission - if: ${{ !(inputs.username == 'dependabot[bot]' || inputs.username == 'cla-bot[bot]') && - steps.check-access.outputs.require-result == 'false' }} - run: | - echo "${{ inputs.username }} does not have permissions on this repo." - echo "Current permission level is ${{ steps.check-access.outputs.user-permission }}" - echo "Job originally triggered by ${{ github.actor }}" - exit 1 diff --git a/.github/workflows/check_ci_files_change.yml b/.github/workflows/check_ci_files_change.yml deleted file mode 100644 index f85feff3bd..0000000000 --- a/.github/workflows/check_ci_files_change.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Check if there is any change in CI files since last commit -name: Check changes in CI files - -on: - workflow_call: - inputs: - checkout_ref: - type: string - required: true - outputs: - ci_file_changed: - value: ${{ jobs.check-changes.outputs.ci_file_changed }} - secrets: - REPO_CHECKOUT_TOKEN: - required: true - -jobs: - check-changes: - runs-on: ubuntu-latest - permissions: - pull-requests: read - outputs: - ci_file_changed: ${{ steps.changed-files.outputs.ci_any_changed }} - steps: - - name: Checkout tfhe-rs - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - with: - fetch-depth: 0 - persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ inputs.checkout_ref }} - - - name: Check for file changes - id: changed-files - uses: tj-actions/changed-files@dcc7a0cba800f454d79fff4b993e8c3555bcc0a8 - with: - files_yaml: | - ci: - - .github/** - - ci/** diff --git a/.github/workflows/check_external_pr.yml b/.github/workflows/check_external_pr.yml deleted file mode 100644 index 4dc7b569ee..0000000000 --- a/.github/workflows/check_external_pr.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Check if a pull request fulfill pre-conditions to be accepted -name: Check PR from fork - -on: - pull_request_target: - paths: - - '.github/**' - - 'ci/**' - -jobs: - # Fail if the triggering actor is not part of Zama organization. - check-user-permission: - name: Check event user permissions - uses: ./.github/workflows/check_actor_permissions.yml - with: - username: ${{ github.event.pull_request.user.login }} - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - - write-comment: - name: Write PR comment - if: ${{ always() && needs.check-user-permission.outputs.is_authorized == 'false' }} - needs: check-user-permission - runs-on: ubuntu-latest - permissions: - pull-requests: write - steps: - - name: Write warning - uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b - with: - message: | - CI files have changed. Only Zama organization members are authorized to modify these files. diff --git a/.github/workflows/ci_lint.yml b/.github/workflows/ci_lint.yml index 5063a16635..1acf55f237 100644 --- a/.github/workflows/ci_lint.yml +++ b/.github/workflows/ci_lint.yml @@ -6,6 +6,7 @@ on: env: ACTIONLINT_VERSION: 1.6.27 + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} jobs: lint-check: @@ -16,7 +17,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Get actionlint run: | diff --git a/.github/workflows/csprng_randomness_tests.yml b/.github/workflows/csprng_randomness_tests.yml index 9f03fced26..54ab9e766a 100644 --- a/.github/workflows/csprng_randomness_tests.yml +++ b/.github/workflows/csprng_randomness_tests.yml @@ -10,56 +10,28 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} - REF: ${{ github.event.pull_request.head.sha || github.sha }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "large_ubuntu_16" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: types: [ labeled ] - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - types: [ labeled ] - paths: - - '**' - - '!.github/**' - - '!ci/**' jobs: - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (csprng-randomness-tests) - needs: check-user-permission if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.label.name, 'approved') }} runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -69,6 +41,13 @@ jobs: backend: aws profile: cpu-small + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + csprng-randomness-tests: name: CSPRNG randomness tests needs: setup-instance @@ -81,8 +60,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -107,8 +85,9 @@ jobs: needs: [ setup-instance, csprng-randomness-tests ] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/data_pr_close.yml b/.github/workflows/data_pr_close.yml index 35aefb85cd..7e323f7170 100644 --- a/.github/workflows/data_pr_close.yml +++ b/.github/workflows/data_pr_close.yml @@ -8,8 +8,6 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} PR_BRANCH: ${{ github.head_ref || github.ref_name }} CLOSE_TYPE: ${{ github.event.pull_request.merged && 'merge' || 'close' }} @@ -17,8 +15,6 @@ env: on: pull_request: types: [ closed ] - pull_request_target: - types: [ closed ] # The same pattern is used for jobs that use the github api: # - save the result of the API call in the env var "GH_API_RES". Since the var is multiline diff --git a/.github/workflows/gpu_4090_tests.yml b/.github/workflows/gpu_4090_tests.yml index 84c762f466..c467d7896e 100644 --- a/.github/workflows/gpu_4090_tests.yml +++ b/.github/workflows/gpu_4090_tests.yml @@ -11,52 +11,20 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} - REF: ${{ github.event.pull_request.head.sha || github.sha }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: types: [ labeled ] - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - types: [ labeled ] - paths: - - '**' - - '!.github/**' - - '!ci/**' schedule: # Nightly tests @ 1AM after each work day - cron: "0 1 * * MON-FRI" jobs: - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - cuda-tests-linux: name: CUDA tests (RTX 4090) - needs: check-user-permission if: github.event_name == 'workflow_dispatch' || contains(github.event.label.name, '4090_test') || (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') @@ -70,8 +38,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -103,7 +70,7 @@ jobs: make test_high_level_api_gpu - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 - if: ${{ always() && github.event_name == 'pull_request_target' }} + if: ${{ always() && github.event_name == 'pull_request' }} with: labels: 4090_test github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/gpu_fast_h100_tests.yml b/.github/workflows/gpu_fast_h100_tests.yml index 94aa26b55a..87231635c0 100644 --- a/.github/workflows/gpu_fast_h100_tests.yml +++ b/.github/workflows/gpu_fast_h100_tests.yml @@ -11,28 +11,17 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} - IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} - REF: ${{ github.event.pull_request.head.sha || github.sha }} + IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "gpu_ubuntu-22.04" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: types: [ labeled ] - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - types: [ labeled ] - paths: - - '**' - - '!.github/**' - - '!ci/**' jobs: should-run: @@ -47,8 +36,7 @@ jobs: with: fetch-depth: 0 persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Check for file changes id: changed-files @@ -72,35 +60,19 @@ jobs: - scripts/integer-tests.sh - ci/slab.toml - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (cuda-h100-tests) - needs: [ should-run, check-user-permission ] - if: github.event_name != 'pull_request_target' || + needs: should-run + if: github.event_name != 'pull_request' || (github.event.action != 'labeled' && needs.should-run.outputs.gpu_test == 'true') || (github.event.action == 'labeled' && github.event.label.name == 'approved' && needs.should-run.outputs.gpu_test == 'true') runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -110,11 +82,18 @@ jobs: backend: hyperstack profile: single-h100 + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + cuda-tests-linux: name: CUDA H100 tests needs: [ should-run, setup-instance ] - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.setup-instance.result != 'skipped') + if: github.event_name != 'pull_request' || + (github.event_name == 'pull_request' && needs.setup-instance.result != 'skipped') concurrency: group: ${{ github.workflow }}_${{ github.head_ref || github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} @@ -132,14 +111,14 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Setup Hyperstack dependencies - uses: ./.github/actions/hyperstack_setup + uses: ./.github/actions/gpu_setup with: cuda-version: ${{ matrix.cuda }} gcc-version: ${{ matrix.gcc }} + github-instance: ${{ env.SECRETS_AVAILABLE == 'false' }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -172,6 +151,7 @@ jobs: continue-on-error: true steps: - name: Send message + if: env.SECRETS_AVAILABLE == 'true' uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 env: SLACK_COLOR: ${{ needs.cuda-tests-linux.result }} @@ -183,8 +163,9 @@ jobs: needs: [ setup-instance, cuda-tests-linux ] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/gpu_fast_tests.yml b/.github/workflows/gpu_fast_tests.yml index 6fd13170ab..32b483e3a2 100644 --- a/.github/workflows/gpu_fast_tests.yml +++ b/.github/workflows/gpu_fast_tests.yml @@ -11,26 +11,16 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} - IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} - REF: ${{ github.event.pull_request.head.sha || github.sha }} + IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "gpu_ubuntu-22.04" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - paths: - - '**' - - '!.github/**' - - '!ci/**' jobs: should-run: @@ -45,8 +35,7 @@ jobs: with: fetch-depth: 0 persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Check for file changes id: changed-files @@ -70,34 +59,18 @@ jobs: - scripts/integer-tests.sh - ci/slab.toml - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (cuda-tests) - needs: [ should-run, check-user-permission ] + needs: should-run if: github.event_name == 'workflow_dispatch' || needs.should-run.outputs.gpu_test == 'true' runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -107,11 +80,18 @@ jobs: backend: hyperstack profile: gpu-test + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + cuda-tests-linux: name: CUDA tests needs: [ should-run, setup-instance ] - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.setup-instance.result != 'skipped') + if: github.event_name != 'pull_request' || + (github.event_name == 'pull_request' && needs.setup-instance.result != 'skipped') concurrency: group: ${{ github.workflow }}_${{ github.head_ref || github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} @@ -129,14 +109,14 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Setup Hyperstack dependencies - uses: ./.github/actions/hyperstack_setup + uses: ./.github/actions/gpu_setup with: cuda-version: ${{ matrix.cuda }} gcc-version: ${{ matrix.gcc }} + github-instance: ${{ env.SECRETS_AVAILABLE == 'false' }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -169,6 +149,7 @@ jobs: continue-on-error: true steps: - name: Send message + if: env.SECRETS_AVAILABLE == 'true' uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 env: SLACK_COLOR: ${{ needs.cuda-tests-linux.result }} @@ -180,8 +161,9 @@ jobs: needs: [ setup-instance, cuda-tests-linux ] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/gpu_full_h100_tests.yml b/.github/workflows/gpu_full_h100_tests.yml index c0dbc6a949..e4e2ee5625 100644 --- a/.github/workflows/gpu_full_h100_tests.yml +++ b/.github/workflows/gpu_full_h100_tests.yml @@ -68,7 +68,7 @@ jobs: token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - name: Setup Hyperstack dependencies - uses: ./.github/actions/hyperstack_setup + uses: ./.github/actions/gpu_setup with: cuda-version: ${{ matrix.cuda }} gcc-version: ${{ matrix.gcc }} diff --git a/.github/workflows/gpu_full_multi_gpu_tests.yml b/.github/workflows/gpu_full_multi_gpu_tests.yml index 78747143d7..fc5340d9c0 100644 --- a/.github/workflows/gpu_full_multi_gpu_tests.yml +++ b/.github/workflows/gpu_full_multi_gpu_tests.yml @@ -11,28 +11,17 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} - IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} - REF: ${{ github.event.pull_request.head.sha || github.sha }} + IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "gpu_ubuntu-22.04" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: types: [ labeled ] - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - types: [ labeled ] - paths: - - '**' - - '!.github/**' - - '!ci/**' jobs: should-run: @@ -47,8 +36,7 @@ jobs: with: fetch-depth: 0 persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Check for file changes id: changed-files @@ -72,35 +60,19 @@ jobs: - scripts/integer-tests.sh - ci/slab.toml - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (cuda-tests-multi-gpu) - needs: [ should-run, check-user-permission ] - if: github.event_name != 'pull_request_target' || + needs: should-run + if: github.event_name != 'pull_request' || (github.event.action != 'labeled' && needs.should-run.outputs.gpu_test == 'true') || (github.event.action == 'labeled' && github.event.label.name == 'approved' && needs.should-run.outputs.gpu_test == 'true') runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -110,11 +82,18 @@ jobs: backend: hyperstack profile: multi-gpu-test + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + cuda-tests-linux: name: CUDA multi-GPU tests needs: [ should-run, setup-instance ] - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.setup-instance.result != 'skipped') + if: github.event_name != 'pull_request' || + (github.event_name == 'pull_request' && needs.setup-instance.result != 'skipped') concurrency: group: ${{ github.workflow }}_${{ github.head_ref || github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} @@ -132,14 +111,14 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Setup Hyperstack dependencies - uses: ./.github/actions/hyperstack_setup + uses: ./.github/actions/gpu_setup with: cuda-version: ${{ matrix.cuda }} gcc-version: ${{ matrix.gcc }} + github-instance: ${{ env.SECRETS_AVAILABLE == 'false' }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -175,6 +154,7 @@ jobs: continue-on-error: true steps: - name: Send message + if: env.SECRETS_AVAILABLE == 'true' uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 env: SLACK_COLOR: ${{ needs.cuda-tests-linux.result }} @@ -186,8 +166,9 @@ jobs: needs: [ setup-instance, cuda-tests-linux ] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/gpu_integer_long_run_tests.yml b/.github/workflows/gpu_integer_long_run_tests.yml index 92080452a2..8581c45f41 100644 --- a/.github/workflows/gpu_integer_long_run_tests.yml +++ b/.github/workflows/gpu_integer_long_run_tests.yml @@ -59,7 +59,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - name: Setup Hyperstack dependencies - uses: ./.github/actions/hyperstack_setup + uses: ./.github/actions/gpu_setup with: cuda-version: ${{ matrix.cuda }} gcc-version: ${{ matrix.gcc }} diff --git a/.github/workflows/gpu_pcc.yml b/.github/workflows/gpu_pcc.yml index 3eb468d2f4..a25253d242 100644 --- a/.github/workflows/gpu_pcc.yml +++ b/.github/workflows/gpu_pcc.yml @@ -11,51 +11,24 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} - REF: ${{ github.event.pull_request.head.sha || github.sha }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "large_ubuntu_16-22.04" on: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - paths: - - '**' - - '!.github/**' - - '!ci/**' jobs: - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (cuda-pcc) - needs: check-user-permission runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -65,6 +38,13 @@ jobs: backend: aws profile: gpu-build + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + cuda-pcc: name: CUDA post-commit checks needs: setup-instance @@ -88,8 +68,17 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} + + - name: Install CUDA + if: env.SECRETS_AVAILABLE == 'false' + shell: bash + run: | + TOOLKIT_VERSION="$(echo ${{ matrix.cuda }} | sed 's/\(.*\)\.\(.*\)/\1-\2/')" + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb + sudo dpkg -i cuda-keyring_1.1-1_all.deb + sudo apt update + sudo apt -y install "cuda-toolkit-${TOOLKIT_VERSION}" cmake-format - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -112,7 +101,6 @@ jobs: echo "CC=/usr/bin/gcc-${{ matrix.gcc }}"; echo "CXX=/usr/bin/g++-${{ matrix.gcc }}"; echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}"; - echo "HOME=/home/ubuntu"; } >> "${GITHUB_ENV}" - name: Run fmt checks @@ -124,7 +112,7 @@ jobs: make pcc_gpu - name: Slack Notification - if: ${{ failure() }} + if: ${{ failure() && env.SECRETS_AVAILABLE == 'true' }} continue-on-error: true uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 env: @@ -137,8 +125,9 @@ jobs: needs: [ setup-instance, cuda-pcc ] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/gpu_signed_integer_classic_tests.yml b/.github/workflows/gpu_signed_integer_classic_tests.yml index 42db1fba16..4cb8aad898 100644 --- a/.github/workflows/gpu_signed_integer_classic_tests.yml +++ b/.github/workflows/gpu_signed_integer_classic_tests.yml @@ -11,28 +11,17 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} - IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} - REF: ${{ github.event.pull_request.head.sha || github.sha }} + IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "gpu_ubuntu-22.04" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: types: [ labeled ] - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - types: [ labeled ] - paths: - - '**' - - '!.github/**' - - '!ci/**' jobs: should-run: @@ -47,8 +36,7 @@ jobs: with: fetch-depth: 0 persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Check for file changes id: changed-files @@ -72,35 +60,19 @@ jobs: - scripts/integer-tests.sh - ci/slab.toml - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (cuda-signed-classic-tests) - needs: [ should-run, check-user-permission ] - if: github.event_name != 'pull_request_target' || + needs: should-run + if: github.event_name != 'pull_request' || (github.event.action != 'labeled' && needs.should-run.outputs.gpu_test == 'true') || (github.event.action == 'labeled' && github.event.label.name == 'approved' && needs.should-run.outputs.gpu_test == 'true') runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -110,11 +82,18 @@ jobs: backend: hyperstack profile: gpu-test + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + cuda-tests-linux: name: CUDA signed integer tests with classical PBS needs: [ should-run, setup-instance ] - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.setup-instance.result != 'skipped') + if: github.event_name != 'pull_request' || + (github.event_name == 'pull_request' && needs.setup-instance.result != 'skipped') concurrency: group: ${{ github.workflow }}_${{ github.head_ref || github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} @@ -132,14 +111,14 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Setup Hyperstack dependencies - uses: ./.github/actions/hyperstack_setup + uses: ./.github/actions/gpu_setup with: cuda-version: ${{ matrix.cuda }} gcc-version: ${{ matrix.gcc }} + github-instance: ${{ env.SECRETS_AVAILABLE == 'false' }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -158,6 +137,7 @@ jobs: continue-on-error: true steps: - name: Send message + if: env.SECRETS_AVAILABLE == 'true' uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 env: SLACK_COLOR: ${{ needs.cuda-tests-linux.result }} @@ -169,8 +149,9 @@ jobs: needs: [ setup-instance, cuda-tests-linux ] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/gpu_signed_integer_h100_tests.yml b/.github/workflows/gpu_signed_integer_h100_tests.yml index 584e5c3c63..dc3d900ef0 100644 --- a/.github/workflows/gpu_signed_integer_h100_tests.yml +++ b/.github/workflows/gpu_signed_integer_h100_tests.yml @@ -11,28 +11,18 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} - IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} - REF: ${{ github.event.pull_request.head.sha || github.sha }} + IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "gpu_ubuntu-22.04" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: types: [ labeled ] - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - types: [ labeled ] - paths: - - '**' - - '!.github/**' - - '!ci/**' + jobs: should-run: @@ -47,8 +37,7 @@ jobs: with: fetch-depth: 0 persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Check for file changes id: changed-files @@ -72,35 +61,19 @@ jobs: - scripts/integer-tests.sh - ci/slab.toml - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (cuda-h100-tests) - needs: [ should-run, check-user-permission ] - if: github.event_name != 'pull_request_target' || + needs: should-run + if: github.event_name != 'pull_request' || (github.event.action != 'labeled' && needs.should-run.outputs.gpu_test == 'true') || (github.event.action == 'labeled' && github.event.label.name == 'approved' && needs.should-run.outputs.gpu_test == 'true') runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -110,11 +83,18 @@ jobs: backend: hyperstack profile: single-h100 + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + cuda-tests-linux: name: CUDA H100 signed integer tests needs: [ should-run, setup-instance ] - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.setup-instance.result != 'skipped') + if: github.event_name != 'pull_request' || + (github.event_name == 'pull_request' && needs.setup-instance.result != 'skipped') concurrency: group: ${{ github.workflow }}_${{ github.head_ref || github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} @@ -132,14 +112,14 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Setup Hyperstack dependencies - uses: ./.github/actions/hyperstack_setup + uses: ./.github/actions/gpu_setup with: cuda-version: ${{ matrix.cuda }} gcc-version: ${{ matrix.gcc }} + github-instance: ${{ env.SECRETS_AVAILABLE == 'false' }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -158,6 +138,7 @@ jobs: continue-on-error: true steps: - name: Send message + if: env.SECRETS_AVAILABLE == 'true' uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 env: SLACK_COLOR: ${{ needs.cuda-tests-linux.result }} @@ -169,8 +150,9 @@ jobs: needs: [ setup-instance, cuda-tests-linux ] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/gpu_signed_integer_tests.yml b/.github/workflows/gpu_signed_integer_tests.yml index 65747e42fc..37dd2740dc 100644 --- a/.github/workflows/gpu_signed_integer_tests.yml +++ b/.github/workflows/gpu_signed_integer_tests.yml @@ -11,28 +11,18 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} FAST_TESTS: TRUE NIGHTLY_TESTS: FALSE - IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} - REF: ${{ github.event.pull_request.head.sha || github.sha }} + IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "gpu_ubuntu-22.04" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - paths: - - '**' - - '!.github/**' - - '!ci/**' schedule: # Nightly tests @ 1AM after each work day - cron: "0 1 * * MON-FRI" @@ -50,8 +40,7 @@ jobs: with: fetch-depth: 0 persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Check for file changes id: changed-files @@ -74,36 +63,19 @@ jobs: - '.github/workflows/gpu_signed_integer_tests.yml' - scripts/integer-tests.sh - ci/slab.toml - - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (cuda-signed-integer-tests) runs-on: ubuntu-latest - needs: [ should-run, check-user-permission ] + needs: should-run if: (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') || github.event_name == 'workflow_dispatch' || needs.should-run.outputs.gpu_test == 'true' outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -113,11 +85,18 @@ jobs: backend: hyperstack profile: gpu-test + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + cuda-signed-integer-tests: name: CUDA signed integer tests needs: [ should-run, setup-instance ] - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.setup-instance.result != 'skipped') + if: github.event_name != 'pull_request' || + (github.event_name == 'pull_request' && needs.setup-instance.result != 'skipped') concurrency: group: ${{ github.workflow }}_${{ github.head_ref || github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} @@ -135,14 +114,14 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Setup Hyperstack dependencies - uses: ./.github/actions/hyperstack_setup + uses: ./.github/actions/gpu_setup with: cuda-version: ${{ matrix.cuda }} gcc-version: ${{ matrix.gcc }} + github-instance: ${{ env.SECRETS_AVAILABLE == 'false' }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -180,8 +159,9 @@ jobs: needs: [ setup-instance, cuda-signed-integer-tests ] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/gpu_unsigned_integer_classic_tests.yml b/.github/workflows/gpu_unsigned_integer_classic_tests.yml index 80d50f0c5c..b5b05bd5fc 100644 --- a/.github/workflows/gpu_unsigned_integer_classic_tests.yml +++ b/.github/workflows/gpu_unsigned_integer_classic_tests.yml @@ -11,28 +11,18 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} - IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} - REF: ${{ github.event.pull_request.head.sha || github.sha }} + IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "gpu_ubuntu-22.04" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: types: [ labeled ] - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - types: [ labeled ] - paths: - - '**' - - '!.github/**' - - '!ci/**' + jobs: should-run: @@ -47,8 +37,7 @@ jobs: with: fetch-depth: 0 persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Check for file changes id: changed-files @@ -72,35 +61,19 @@ jobs: - scripts/integer-tests.sh - ci/slab.toml - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (cuda-unsigned-classic-tests) - needs: [ should-run, check-user-permission ] + needs: should-run if: github.event_name == 'workflow_dispatch' || (github.event.action != 'labeled' && needs.should-run.outputs.gpu_test == 'true') || (github.event.action == 'labeled' && github.event.label.name == 'approved' && needs.should-run.outputs.gpu_test == 'true') runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -110,11 +83,18 @@ jobs: backend: hyperstack profile: gpu-test + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + cuda-tests-linux: name: CUDA unsigned integer tests with classical PBS needs: [ should-run, setup-instance ] - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.setup-instance.result != 'skipped') + if: github.event_name != 'pull_request' || + (github.event_name == 'pull_request' && needs.setup-instance.result != 'skipped') concurrency: group: ${{ github.workflow }}_${{ github.head_ref || github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} @@ -132,14 +112,14 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Setup Hyperstack dependencies - uses: ./.github/actions/hyperstack_setup + uses: ./.github/actions/gpu_setup with: cuda-version: ${{ matrix.cuda }} gcc-version: ${{ matrix.gcc }} + github-instance: ${{ env.SECRETS_AVAILABLE == 'false' }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -158,6 +138,7 @@ jobs: continue-on-error: true steps: - name: Send message + if: env.SECRETS_AVAILABLE == 'true' uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 env: SLACK_COLOR: ${{ needs.cuda-tests-linux.result }} @@ -169,8 +150,9 @@ jobs: needs: [ setup-instance, cuda-tests-linux ] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/gpu_unsigned_integer_h100_tests.yml b/.github/workflows/gpu_unsigned_integer_h100_tests.yml index a01b0f2073..bf1f5d4513 100644 --- a/.github/workflows/gpu_unsigned_integer_h100_tests.yml +++ b/.github/workflows/gpu_unsigned_integer_h100_tests.yml @@ -11,28 +11,17 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} - IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} - REF: ${{ github.event.pull_request.head.sha || github.sha }} + IS_PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "gpu_ubuntu-22.04" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: types: [ labeled ] - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - types: [ labeled ] - paths: - - '**' - - '!.github/**' - - '!ci/**' jobs: should-run: @@ -47,8 +36,7 @@ jobs: with: fetch-depth: 0 persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Check for file changes id: changed-files @@ -72,35 +60,19 @@ jobs: - scripts/integer-tests.sh - ci/slab.toml - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (cuda-h100-tests) - needs: [ should-run, check-user-permission ] + needs: should-run if: github.event_name == 'workflow_dispatch' || (github.event.action != 'labeled' && needs.should-run.outputs.gpu_test == 'true') || (github.event.action == 'labeled' && github.event.label.name == 'approved' && needs.should-run.outputs.gpu_test == 'true') runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -110,11 +82,18 @@ jobs: backend: hyperstack profile: single-h100 + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + cuda-tests-linux: name: CUDA H100 unsigned integer tests needs: [ should-run, setup-instance ] - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.setup-instance.result != 'skipped') + if: github.event_name != 'pull_request' || + (github.event_name == 'pull_request' && needs.setup-instance.result != 'skipped') concurrency: group: ${{ github.workflow }}_${{ github.head_ref || github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} @@ -132,14 +111,14 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Setup Hyperstack dependencies - uses: ./.github/actions/hyperstack_setup + uses: ./.github/actions/gpu_setup with: cuda-version: ${{ matrix.cuda }} gcc-version: ${{ matrix.gcc }} + github-instance: ${{ env.SECRETS_AVAILABLE == 'false' }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -158,6 +137,7 @@ jobs: continue-on-error: true steps: - name: Send message + if: env.SECRETS_AVAILABLE == 'true' uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 env: SLACK_COLOR: ${{ needs.cuda-tests-linux.result }} @@ -169,8 +149,9 @@ jobs: needs: [ setup-instance, cuda-tests-linux ] runs-on: ubuntu-latest steps: - - name: Stop instance + - name: Stop remote instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/gpu_unsigned_integer_tests.yml b/.github/workflows/gpu_unsigned_integer_tests.yml index c548533cce..542a663dcb 100644 --- a/.github/workflows/gpu_unsigned_integer_tests.yml +++ b/.github/workflows/gpu_unsigned_integer_tests.yml @@ -11,29 +11,18 @@ env: SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} FAST_TESTS: TRUE NIGHTLY_TESTS: FALSE - REF: ${{ github.event.pull_request.head.sha || github.sha }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} + # Secrets will be available only to zama-ai organization members + SECRETS_AVAILABLE: ${{ secrets.JOB_SECRET != '' }} + EXTERNAL_CONTRIBUTION_RUNNER: "gpu_ubuntu-22.04" on: # Allows you to run this workflow manually from the Actions tab as an alternative. workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: types: [ labeled ] - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - types: [ labeled ] - paths: - - '**' - - '!.github/**' - - '!ci/**' schedule: # Nightly tests @ 1AM after each work day - cron: "0 1 * * MON-FRI" @@ -51,8 +40,7 @@ jobs: with: fetch-depth: 0 persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Check for file changes id: changed-files @@ -76,35 +64,19 @@ jobs: - scripts/integer-tests.sh - ci/slab.toml - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - setup-instance: name: Setup instance (cuda-unsigned-integer-tests) - needs: [ should-run, check-user-permission ] + needs: should-run if: (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') || github.event_name == 'workflow_dispatch' || needs.should-run.outputs.gpu_test == 'true' runs-on: ubuntu-latest outputs: - runner-name: ${{ steps.start-instance.outputs.label }} + runner-name: ${{ steps.start-remote-instance.outputs.label || steps.start-github-instance.outputs.runner_group }} steps: - - name: Start instance - id: start-instance + - name: Start remote instance + id: start-remote-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: start @@ -114,11 +86,18 @@ jobs: backend: hyperstack profile: gpu-test + # This instance will be spawned especially for pull-request from forked repository + - name: Start GitHub instance + id: start-github-instance + if: env.SECRETS_AVAILABLE == 'false' + run: | + echo "runner_group=${{ env.EXTERNAL_CONTRIBUTION_RUNNER }}" >> "$GITHUB_OUTPUT" + cuda-unsigned-integer-tests: name: CUDA unsigned integer tests needs: [ should-run, setup-instance ] - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.setup-instance.result != 'skipped') + if: github.event_name != 'pull_request' || + (github.event_name == 'pull_request' && needs.setup-instance.result != 'skipped') concurrency: group: ${{ github.workflow }}_${{ github.head_ref || github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} @@ -136,14 +115,14 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: 'false' - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Setup Hyperstack dependencies - uses: ./.github/actions/hyperstack_setup + uses: ./.github/actions/gpu_setup with: cuda-version: ${{ matrix.cuda }} gcc-version: ${{ matrix.gcc }} + github-instance: ${{ env.SECRETS_AVAILABLE == 'false' }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -170,6 +149,7 @@ jobs: continue-on-error: true steps: - name: Send message + if: env.SECRETS_AVAILABLE == 'true' uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 env: SLACK_COLOR: ${{ needs.cuda-unsigned-integer-tests.result }} @@ -183,6 +163,7 @@ jobs: steps: - name: Stop instance id: stop-instance + if: env.SECRETS_AVAILABLE == 'true' uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac with: mode: stop diff --git a/.github/workflows/m1_tests.yml b/.github/workflows/m1_tests.yml index 07c48ba382..d20c1a344e 100644 --- a/.github/workflows/m1_tests.yml +++ b/.github/workflows/m1_tests.yml @@ -2,20 +2,8 @@ name: Tests on M1 CPU on: workflow_dispatch: - # Trigger pull_request event on CI files to be able to test changes before merging to main branch. - # Workflow would fail if changes come from a forked repository since secrets are not available with this event. pull_request: types: [ labeled ] - paths: - - '.github/**' - - 'ci/**' - # General entry point for Zama's pull request as well as contribution from forks. - pull_request_target: - types: [ labeled ] - paths: - - '**' - - '!.github/**' - - '!ci/**' # Have a nightly build for M1 tests schedule: # * is a special character in YAML so you have to quote this string @@ -33,32 +21,14 @@ env: # We clear the cache to reduce memory pressure because of the numerous processes of cargo # nextest TFHE_RS_CLEAR_IN_MEMORY_KEY_CACHE: "1" - REF: ${{ github.event.pull_request.head.sha || github.sha }} + CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} concurrency: group: ${{ github.workflow }}_${{ github.head_ref || github.ref }} cancel-in-progress: true jobs: - check-ci-files: - uses: ./.github/workflows/check_ci_files_change.yml - with: - checkout_ref: ${{ github.event.pull_request.head.sha || github.sha }} - secrets: - REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }} - - # Fail if the triggering actor is not part of Zama organization. - # If pull_request_target is emitted and CI files have changed, skip this job. This would skip following jobs. - check-user-permission: - needs: check-ci-files - if: github.event_name != 'pull_request_target' || - (github.event_name == 'pull_request_target' && needs.check-ci-files.outputs.ci_file_changed == 'false') - uses: ./.github/workflows/check_actor_permissions.yml - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - cargo-builds-m1: - needs: check-user-permission if: ${{ (github.event_name == 'schedule' && github.repository == 'zama-ai/tfhe-rs') || github.event_name == 'workflow_dispatch' || contains(github.event.label.name, 'm1_test') }} @@ -70,8 +40,7 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: persist-credentials: "false" - token: ${{ secrets.REPO_CHECKOUT_TOKEN }} - ref: ${{ env.REF }} + token: ${{ env.CHECKOUT_TOKEN }} - name: Install latest stable uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 @@ -213,7 +182,7 @@ jobs: if: ${{ always() }} steps: - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 - if: ${{ github.event_name == 'pull_request_target' }} + if: ${{ github.event_name == 'pull_request' }} with: labels: m1_test github_token: ${{ secrets.GITHUB_TOKEN }} @@ -230,4 +199,4 @@ jobs: SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} MSG_MINIMAL: event,action url,commit - BRANCH: ${{ github.head_ref || github.ref }} + BRANCH: ${{ github.ref }}