Refactor benchmarks out of sycl e2e

.github/workflows/sycl-linux-benchmark.yml

@@ -0,0 +1,322 @@
+name: SYCL Benchmarking
+
+on:
+  workflow_call:
+    inputs:
+      runner:
+        type: string
+        required: True
+      image:
+        type: string
+        required: True
+      image_options:
+        type: string
+        required: True
+
+      target_devices:
+        type: string
+        required: True
+
+      ref:
+        type: string
+        required: True
+      merge_ref:
+        description: |
+          Commit-ish to merge post-checkout if non-empty. Must be reachable from
+          the default_branch input paramter.
+        type: string
+        default: 'FETCH_HEAD'
+        required: False
+
+      sycl_toolchain_artifact:
+        type: string
+        default: ''
+        required: False
+      sycl_toolchain_archive:
+        type: string
+        default: ''
+        required: False
+      sycl_toolchain_decompress_command:
+        type: string
+        default: ''
+        required: False
+
+      reset_gpu:
+        type: string
+        required: False
+      install_drivers:
+        type: string
+        required: False
+      use_dev_igc:
+        type: string
+        required: False
+      env:
+        type: string
+        default: '{}'
+        required: False
+
+  workflow_dispatch:
+    inputs:
+      runner:
+        type: choice
+        options:
+          - '["Linux", "gen12"]'
+          - '["amdgpu"]'
+          - '["Linux", "arc"]'
+      image:
+        description: |
+          Use option ending with ":build" for AMDGPU, ":latest" for the rest.
+        type: choice
+        options:
+          - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:latest'
+          - 'ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:build'
+      image_options:
+        description: |
+          Use option with "--device=/dev/kfd" for AMDGPU, without it for the rest.
+        type: choice
+        options:
+          - '-u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN'
+          - '-u 1001 --device=/dev/dri --device=/dev/kfd --privileged --cap-add SYS_ADMIN'
+      target_devices:
+        type: choice
+        options:
+          - 'opencl:cpu'
+          - 'opencl:gpu'
+          - 'opencl:fpga'
+          - 'ext_oneapi_level_zero:gpu'
+          - 'ext_oneapi_hip:gpu'
+      env:
+        description: |
+          Suggested variables: for E2E tests - LIT_FILTER, LIT_FILTER_OUT.
+          LIT_OPTS won't work as we redefine it as part of this workflow.
+
+          For SYCL CTS - CTS_TESTS_TO_BUILD to specify which categories to
+          build.
+
+          Format: '{"VAR1":"VAL1","VAR2":"VAL2",...}'
+        default: '{}'
+
+      install_drivers:
+        type: choice
+        options:
+          - false
+          - true
+
+      use_dev_igc:
+        type: choice
+        options:
+          - false
+          - true
+
+      sycl_toolchain_artifact:
+        type: string
+        default: ''
+        required: False
+      sycl_toolchain_archive:
+        type: string
+        default: ''
+        required: False
+      sycl_toolchain_decompress_command:
+        type: string
+        default: ''
+        required: False
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    if: inputs.sycl_toolchain_artifact == ''
+    uses: ./.github/workflows/sycl-linux-build.yml
+    with:
+      build_ref: ${{ github.sha }}
+      merge_ref: ''
+      build_cache_root: "/__w/"
+      build_artifact_suffix: "default"
+      build_cache_suffix: "default"
+      build_image: ${{ inputs.image }}
+
+  run_archive:
+    if: inputs.sycl_toolchain_archive != ''
+    runs-on: ${{ fromJSON(inputs.runner) }}
+    container:
+      image: ${{ inputs.image }}
+      options: ${{ inputs.image_options }}
+    env: ${{ fromJSON(inputs.env) }}
+    steps:
+    - name: Reset GPU
+      if: inputs.reset_gpu == 'true'
+      run: |
+        sudo mount -t debugfs none /sys/kernel/debug
+        sudo bash -c 'echo 1 > /sys/kernel/debug/dri/0/i915_wedged'
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.ref }}
+        sparse-checkout: |
+          devops
+    - name: Register cleanup after job is finished
+      uses: ./devops/actions/cleanup
+    - name: Install drivers
+      if: inputs.install_drivers == 'true'
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        if [ "${{ inputs.use_dev_igc }}" = "true" ]; then
+            # If libllvm14 is already installed (dev igc docker), still return true.
+            sudo apt-get install -yqq libllvm14 || true;
+        fi
+        sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.use_dev_igc == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all
+    - name: Source OneAPI TBB vars.sh
+      shell: bash
+      run: |
+        # https://github.com/actions/runner/issues/1964 prevents us from using
+        # the ENTRYPOINT in the image.
+        env | sort > env_before
+        if [ -e /runtimes/oneapi-tbb/env/vars.sh ]; then
+          source /runtimes/oneapi-tbb/env/vars.sh;
+        elif [ -e /opt/runtimes/oneapi-tbb/env/vars.sh ]; then
+          source /opt/runtimes/oneapi-tbb/env/vars.sh;
+        else
+          echo "no TBB vars in /opt/runtimes or /runtimes";
+        fi
+        env | sort > env_after
+        comm -13 env_before env_after >> $GITHUB_ENV
+        rm env_before env_after
+
+    - name: Download SYCL toolchain
+      if: inputs.sycl_toolchain_artifact != '' && github.event_name != 'workflow_run'
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ inputs.sycl_toolchain_artifact }}
+    - name: Download SYCL toolchain [workflow_run]
+      if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run'
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const name = '${{ inputs.sycl_toolchain_artifact }}'
+          let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+             owner: context.repo.owner,
+             repo: context.repo.repo,
+             run_id: context.payload.workflow_run.id,
+          });
+          let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+            return artifact.name == name
+          })[0];
+          let download = await github.rest.actions.downloadArtifact({
+             owner: context.repo.owner,
+             repo: context.repo.repo,
+             artifact_id: matchArtifact.id,
+             archive_format: 'zip',
+          });
+          let fs = require('fs');
+          fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/` + name + '.zip', Buffer.from(download.data));
+    - name: Unzip artifact [workflow_run]
+      if: inputs.sycl_toolchain_artifact != '' && github.event_name == 'workflow_run'
+      run: |
+        unzip ${{ inputs.sycl_toolchain_artifact }}.zip
+        rm ${{ inputs.sycl_toolchain_artifact }}.zip
+    - name: Extract/Setup SYCL toolchain
+      if: inputs.sycl_toolchain_artifact != ''
+      run: |
+        mkdir toolchain
+        tar -I '${{ inputs.sycl_toolchain_decompress_command }}' -xf ${{ inputs.sycl_toolchain_archive }} -C toolchain
+        rm -f ${{ inputs.sycl_toolchain_archive }}
+        echo PATH=$PWD/toolchain/bin/:$PATH >> $GITHUB_ENV
+        echo LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH >> $GITHUB_ENV
+    - run: which clang++ sycl-ls
+    - run: sycl-ls --verbose
+    - run: SYCL_UR_TRACE=1 sycl-ls
+    - run: |
+          if [ -f /usr/local/lib/igc/IGCTAG.txt ]; then
+            cat /usr/local/lib/igc/IGCTAG.txt
+          fi
+
+    - name: Run compute-benchmarks
+      id: run_benchmarks
+      if: success()
+      run: |
+        whereis clang++
+        whereis sycl-ls
+        clang++ --version
+        ls
+        ./devops/scripts/benchmarking/benchmark.sh
+    # - name: Upload sycl-bench microbenchmark results
+    #   if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success'
+    #   uses: actions/upload-artifact@v4
+    #   with:
+    #     name: sycl_benchmark_res_${{ steps.run_benchmarks.outputs.TIMESTAMP }}
+    #     path: ${{ steps.run_benchmarks.outputs.BENCHMARK_RESULTS }}
+    #     retention-days: 7
+
+  run_build:
+    needs: [build]
+    runs-on: ${{ fromJSON(inputs.runner) }}
+    container:
+      image: ${{ inputs.image }}
+      options: ${{ inputs.image_options }}
+    env: ${{ fromJSON(inputs.env) }}
+    steps:
+    - name: Reset GPU
+      if: inputs.reset_gpu == 'true'
+      run: |
+        sudo mount -t debugfs none /sys/kernel/debug
+        sudo bash -c 'echo 1 > /sys/kernel/debug/dri/0/i915_wedged'
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.ref }}
+        sparse-checkout: |
+          devops
+    - name: Register cleanup after job is finished
+      uses: ./devops/actions/cleanup
+    - name: Install drivers
+      if: inputs.install_drivers == 'true'
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        if [ "${{ inputs.use_dev_igc }}" = "true" ]; then
+            # If libllvm14 is already installed (dev igc docker), still return true.
+            sudo apt-get install -yqq libllvm14 || true;
+        fi
+        sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json ${{ inputs.use_dev_igc == 'true' && 'llvm/devops/dependencies-igc-dev.json --use-dev-igc' || '' }} --all
+    - name: Source OneAPI TBB vars.sh
+      shell: bash
+      run: |
+        # https://github.com/actions/runner/issues/1964 prevents us from using
+        # the ENTRYPOINT in the image.
+        env | sort > env_before
+        if [ -e /runtimes/oneapi-tbb/env/vars.sh ]; then
+          source /runtimes/oneapi-tbb/env/vars.sh;
+        elif [ -e /opt/runtimes/oneapi-tbb/env/vars.sh ]; then
+          source /opt/runtimes/oneapi-tbb/env/vars.sh;
+        else
+          echo "no TBB vars in /opt/runtimes or /runtimes";
+        fi
+        env | sort > env_after
+        comm -13 env_before env_after >> $GITHUB_ENV
+        rm env_before env_after
+
+    - run: ls
+    - run: which clang++ sycl-ls
+    - run: sycl-ls --verbose
+    - run: SYCL_UR_TRACE=1 sycl-ls
+    - run: |
+          if [ -f /usr/local/lib/igc/IGCTAG.txt ]; then
+            cat /usr/local/lib/igc/IGCTAG.txt
+          fi
+
+    # - name: Run compute-benchmarks
+    #   id: run_benchmarks
+    #   if: success()
+    #   run: |
+    #     whereis clang++
+    #     whereis sycl-ls
+    #     clang++ --version
+    #     ls
+    #     ./devops/scripts/benchmarking/benchmark.sh
+    # - name: Upload sycl-bench microbenchmark results
+    #   if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success'
+    #   uses: actions/upload-artifact@v4
+    #   with:
+    #     name: sycl_benchmark_res_${{ steps.run_benchmarks.outputs.TIMESTAMP }}
+    #     path: ${{ steps.run_benchmarks.outputs.BENCHMARK_RESULTS }}
+    #     retention-days: 7

.github/workflows/sycl-linux-run-tests.yml

@@ -112,7 +112,7 @@ on:
         options:
           - e2e
           - cts
-          - benchmark
+          #- benchmark
 
       env:
         description: |
@@ -214,22 +214,6 @@ jobs:
         comm -13 env_before env_after >> $GITHUB_ENV
         rm env_before env_after
 
-    - name: Build SYCL from head for benchmarking, if no artifact provided
-      id: build_for_benchmarking
-      if: inputs.tests_selector == 'benchmark' && inputs.sycl_toolchain_artifact == ''
-      uses: ./.github/workflows/sycl-linux-build.yml
-      with:
-        build_ref: ${{ github.sha }}
-        merge_ref: ''
-        build_cache_root: "/__w/"
-        build_artifact_suffix: "default"
-        build_cache_suffix: "default"
-
-    - name: Testing
-      shell: bash
-      run: |
-        ls
-
     - name: Download SYCL toolchain
       if: inputs.sycl_toolchain_artifact != '' && github.event_name != 'workflow_run'
       uses: actions/download-artifact@v4
@@ -381,15 +365,15 @@ jobs:
 
         exit $ret
 
-    - name: Run compute-benchmarks
-      id: run_benchmarks
-      if: inputs.tests_selector == 'benchmark' && success()
-      run: |
-        whereis clang++
-        whereis sycl-ls
-        clang++ --version
-        ls
-        ./devops/scripts/benchmarking/benchmark.sh
+    # - name: Run compute-benchmarks
+    #   id: run_benchmarks
+    #   if: inputs.tests_selector == 'benchmark' && success()
+    #   run: |
+    #     whereis clang++
+    #     whereis sycl-ls
+    #     clang++ --version
+    #     ls
+    #     ./devops/scripts/benchmarking/benchmark.sh
     # - name: Upload sycl-bench microbenchmark results
     #   if: inputs.tests_selector == 'benchmark' && steps.run_benchmarks.outcome == 'success'
     #   uses: actions/upload-artifact@v4