[ci] Revive CI, wip (#8606)

taichi-dev · Dec 18, 2024 · 1c0e3a6 · 1c0e3a6
1 parent e9f19b8
commit 1c0e3a6
Show file tree

Hide file tree

Showing 5 changed files with 389 additions and 367 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -120,51 +120,51 @@ jobs:
         env:
           PY: ${{ matrix.python }}
 
-  build_and_test_mac:
-    name: Build and Upload (macOS only)
-    needs: matrix_prep
-    strategy:
-      fail-fast: false
-      matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix_osx) }}
-    runs-on: [self-hosted, macos-11]
-    env:
-      PY: ${{ matrix.python }}
-      CXX: clang++
-      TAICHI_CMAKE_ARGS: >-
-        -DTI_WITH_VULKAN:BOOL=ON
-        -DTI_WITH_OPENGL:BOOL=OFF
-        -DTI_BUILD_TESTS:BOOL=ON
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: 'recursive'
-          fetch-depth: '0'
-
-      - name: Prepare Environment
-        run: |
-          . .github/workflows/scripts/common-utils.sh
-          prepare-build-cache
-
-      - name: Create Python Wheel
-        run: |
-          # Use the molten-vk v1.1.10 downloaded from taichi assets
-          brew uninstall molten-vk -f
-          .github/workflows/scripts/build.py $NIGHTLY
-        env:
-          PROJECT_NAME: ${{ matrix.name }}
-
-      - name: Archive Wheel Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: dist
-          path: dist/*.whl
-          retention-days: 20
-
-      - name: Test
-        run: .github/workflows/scripts/unix_test.sh
-        env:
-          TI_WANTED_ARCHS: 'cpu'
-          TI_SKIP_CPP_TESTS: Disabled because Vulkan is supported but not working on buildbot4
+  # build_and_test_mac:
+  #   name: Build and Upload (macOS only)
+  #   needs: matrix_prep
+  #   strategy:
+  #     fail-fast: false
+  #     matrix: ${{ fromJson(needs.matrix_prep.outputs.matrix_osx) }}
+  #   runs-on: [self-hosted, macos-11]
+  #   env:
+  #     PY: ${{ matrix.python }}
+  #     CXX: clang++
+  #     TAICHI_CMAKE_ARGS: >-
+  #       -DTI_WITH_VULKAN:BOOL=ON
+  #       -DTI_WITH_OPENGL:BOOL=OFF
+  #       -DTI_BUILD_TESTS:BOOL=ON
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #       with:
+  #         submodules: 'recursive'
+  #         fetch-depth: '0'
+
+  #     - name: Prepare Environment
+  #       run: |
+  #         . .github/workflows/scripts/common-utils.sh
+  #         prepare-build-cache
+
+  #     - name: Create Python Wheel
+  #       run: |
+  #         # Use the molten-vk v1.1.10 downloaded from taichi assets
+  #         brew uninstall molten-vk -f
+  #         .github/workflows/scripts/build.py $NIGHTLY
+  #       env:
+  #         PROJECT_NAME: ${{ matrix.name }}
+
+  #     - name: Archive Wheel Artifacts
+  #       uses: actions/upload-artifact@v3
+  #       with:
+  #         name: dist
+  #         path: dist/*.whl
+  #         retention-days: 20
+
+  #     - name: Test
+  #       run: .github/workflows/scripts/unix_test.sh
+  #       env:
+  #         TI_WANTED_ARCHS: 'cpu'
+  #         TI_SKIP_CPP_TESTS: Disabled because Vulkan is supported but not working on buildbot4
 
   build_and_test_m1:
     name: Build and Upload (Apple M1)
@@ -241,6 +241,8 @@ jobs:
         run: |
           git config --system core.longpaths true
           git submodule foreach 'git rev-parse HEAD > /dev/null 2>&1 || rm -rf $PWD'
+          . .github/workflows/scripts/common-utils.ps1
+          UnsetGitCachingProxy
 
       - uses: actions/checkout@v3
         with:
@@ -256,6 +258,8 @@ jobs:
         run: |
           $nightlyFlag = $null
           if ($env:NIGHTLY) { $nightlyFlag = "--nightly" }
+          . .github/workflows/scripts/common-utils.ps1
+          SetGitCachingProxy
           python .\.github\workflows\scripts\build.py $nightlyFlag
 
       - name: Archive Wheel Artifacts
@@ -268,6 +272,8 @@ jobs:
       - name: Test
         shell: pwsh
         run: |
+          . .github/workflows/scripts/common-utils.ps1
+          SetGitCachingProxy
           .\.github\workflows\scripts\win_test.ps1
         env:
           PY: ${{ matrix.python }}

diff --git a/.github/workflows/scripts/aot-demo.sh b/.github/workflows/scripts/aot-demo.sh
@@ -122,7 +122,6 @@ function build-and-test-headless-demo {
 
     export PATH=/android-sdk/platform-tools:$PATH
     grab-android-bot
-    trap release-android-bot EXIT
     adb connect $BOT
 
     # Clear temporary test folder

diff --git a/.github/workflows/scripts/common-utils.ps1 b/.github/workflows/scripts/common-utils.ps1
@@ -1,14 +1,16 @@
 function UnsetGitCachingProxy {
     Write-Host "Unsetting git caching proxy"
     git config --global --unset-all url.http://git-cdn-github.botmaster.tgr/.insteadOf
-    git config --global --unset-all url.http://git-cdn-gitlab.botmaster.tgr/.insteadOf
+    # git config --global --unset-all url.http://git-cdn-gitlab.botmaster.tgr/.insteadOf
 }
 
 function SetGitCachingProxy {
     Write-Host "Setting up git caching proxy"
     git config --global --add url.http://git-cdn-github.botmaster.tgr/.insteadOf https://github.com/
     git config --global --add url.http://git-cdn-github.botmaster.tgr/.insteadOf git@github.com:
-    git config --global --add url.http://git-cdn-gitlab.botmaster.tgr/.insteadOf https://gitlab.com/
+    # git config --global --add url.http://git-cdn-gitlab.botmaster.tgr/.insteadOf https://gitlab.com/
+    git config --global credential.helper "store --file $env:TEMP\.git-credentials"
+    "http://oauth2:${env:GITHUB_TOKEN}@git-cdn-github.botmaster.tgr" | Out-File -FilePath $env:TEMP\.git-credentials
 }
 
 if($env:TI_USE_GIT_CACHE) {
@@ -242,6 +244,7 @@ function CIDockerRun {
         -e PIP_CACHE_DIR=X:/pip-cache `
         -e GIT_ALTERNATE_OBJECT_DIRECTORIES=X:/git-cache/objects `
         -e TI_CI=1 `
+        -e GITHUB_TOKEN `
         @TiEnvs `
         -v (($env:LocalAppData -replace "\\", "/") + "/build-cache:X:") `
         @extraArgs `

diff --git a/.github/workflows/scripts/common-utils.sh b/.github/workflows/scripts/common-utils.sh
@@ -5,15 +5,18 @@ set -x
 function unset-git-caching-proxy {
     echo "Unsetting git caching proxy"
     git config --global --unset-all url.http://git-cdn-github.botmaster.tgr/.insteadOf || true
-    git config --global --unset-all url.http://git-cdn-gitlab.botmaster.tgr/.insteadOf || true
+    # git config --global --unset-all url.http://git-cdn-gitlab.botmaster.tgr/.insteadOf || true
+    rm -f ~/.git-credentials
 }
 
 function set-git-caching-proxy {
     trap unset-git-caching-proxy EXIT
     echo "Setting git caching proxy"
     git config --global --add url.http://git-cdn-github.botmaster.tgr/.insteadOf https://github.com/
     git config --global --add url.http://git-cdn-github.botmaster.tgr/.insteadOf git@github.com:
-    git config --global --add url.http://git-cdn-gitlab.botmaster.tgr/.insteadOf https://gitlab.com/
+    # git config --global --add url.http://git-cdn-gitlab.botmaster.tgr/.insteadOf https://gitlab.com/
+    git config --global credential.helper store
+    echo "http://oauth2:$GITHUB_TOKEN@git-cdn-github.botmaster.tgr" > ~/.git-credentials
 }
 
 if [ ! -z "$TI_USE_GIT_CACHE" ]; then
@@ -136,6 +139,7 @@ function ci-docker-run {
         -e SCCACHE_ROOT=/var/lib/sccache \
         -e CACHE_HOME=/var/lib/cache-home \
         -e GIT_ALTERNATE_OBJECT_DIRECTORIES=/var/lib/git-cache/objects \
+        -e GITHUB_TOKEN \
         -v $(readlink -f $CACHE_HOME):/var/lib/cache-home \
         -v $(readlink -f $CACHE_HOME/sccache):/var/lib/sccache \
         -v $(readlink -f $CACHE_HOME/git-cache):/var/lib/git-cache \
@@ -210,6 +214,7 @@ function grab-android-bot {
             export BOT_LOCK_KEY="android-bot-lock:$bot"
             LOCKED=$(redis-cli -h $REDIS_HOST --raw setnx $BOT_LOCK_KEY $BOT_LOCK_COOKIE)
             if [ $LOCKED -eq 1 ]; then
+                trap release-android-bot EXIT
                 redis-cli -h $REDIS_HOST --raw expire android-bot-lock:$bot 300 > /dev/null
                 break
             fi
@@ -247,7 +252,6 @@ function run-android-app {
 
     /android-sdk/platform-tools/adb shell am force-stop $(echo $ACTIVITY | sed 's#/.*$##g')
     /android-sdk/platform-tools/adb disconnect
-    release-android-bot
     if [ -s logcat.log ]; then
         echo "!!!!!!!!!!!!!! Something is wrong !!!!!!!!!!!!!!"
         exit 1