Feature/more caching (#1353)

* cache * dont install from cargo as its too slow * fix urls * fix urls * fix urls * fix urls * fix urls * add sccache to code cov * better sccache config in rust * change code cov path for sccache * change code cov path for sccache * change code cov path for sccache * change windows install * fix bad ubuntu path * add caching to python tests * setup and show stats for cache server * improvements to sscache * add cache to doc tests * change cache keys
Pometry · Oct 25, 2023 · 5119ca7 · 5119ca7
1 parent e0b6721
commit 5119ca7
Show file tree

Hide file tree

Showing 3 changed files with 126 additions and 17 deletions.
diff --git a/.github/workflows/code_coverage.yml b/.github/workflows/code_coverage.yml
@@ -12,9 +12,21 @@ jobs:
     if: ${{ !inputs.skip_tests }}
     name: Code Coverage
     runs-on: ubuntu-latest
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_CACHE_SIZE: 2G
+      SCCACHE_DIR: /home/runner/.cache/sccache
+      SCCACHE_PATH: /home/runner/.cache/sccache
     steps:
       - uses: actions/checkout@v3
         name: Checkout
+      - name: Cache sccache
+        uses: actions/cache@v3
+        with:
+          path: /home/runner/.cache/sccache
+          key: codecov-${{ runner.os }}-sccache-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            codecov-${{ runner.os }}-sccache-
       - name: Set up cargo cache
         uses: actions/cache@v3
         continue-on-error: false
@@ -26,30 +38,35 @@ jobs:
               ~/.cargo/git/db/
               target/debug
               target/release
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-coverage
-          restore-keys: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-coverage
+          key: codecov-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-coverage
+          restore-keys: codecov-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-coverage
       - uses: actions-rs/toolchain@v1
         name: Setup Rust
         with:
           toolchain: nightly
           override: true
           components: rustfmt, clippy
+      - name: Install sccache (Ubuntu)
+        run: |
+          curl -L https://github.com/mozilla/sccache/releases/download/v0.5.4/sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz | tar xz
+          sudo mv sccache-v0.5.4-x86_64-unknown-linux-musl/sccache /usr/local/bin/
+      - name: Start sccache server
+        run: sccache --start-server
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+          cache: 'pip'
       - name: Run Maturin develop
         uses: PyO3/maturin-action@v1
         with:
           working-directory: ./python
           command: build
+          sccache: true
       - name: Install Python dependencies
         run: |
           python -m pip install -q pytest networkx numpy seaborn pandas nbmake pytest-xdist matplotlib pyvis pytest-cov coverage
           python -m pip install target/wheels/raphtory-*.whl
-      - uses: actions-rs/cargo@v1
-        with:
-          command: clean
       - name: Run tests (rust)
         run: cargo test -p raphtory --features "io python"
         env:
@@ -73,4 +90,8 @@ jobs:
           verbose: true
           fail_ci_if_error: false
           files: ./coverage.xml, ${{ steps.coveragerust.outputs.report }}
-
+      - name: Print sccache stats
+        run: sccache --show-stats
+      - name: Stop sccache server
+        run: sccache --stop-server || true
+
diff --git a/.github/workflows/test_python_workflow.yml b/.github/workflows/test_python_workflow.yml
@@ -32,10 +32,38 @@ jobs:
       matrix:
         python: ${{ fromJson(needs.select-strategy.outputs.python-versions) }}
         os: [macos-latest, ubuntu-latest, windows-latest]
+        include:
+          - os: macos-latest
+            sccache-path: /Users/runner/Library/Caches/Mozilla.sccache
+          - os: ubuntu-latest
+            sccache-path: /home/runner/.cache/sccache
+          - os: windows-latest
+            sccache-path: C:\Users\runner\AppData\Local\Mozilla\sccache
     runs-on: '${{ matrix.os }}'
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_CACHE_SIZE: 2G
+      SCCACHE_DIR: ${{ matrix.sccache-path }}
+      SCCACHE_PATH: ${{ matrix.sccache-path }}
     steps:
+      - name: Setting sccache-path
+        run: echo "SCCACHE_PATH=${{ matrix.sccache-path }}"
+        if: matrix.os == 'macos-latest'
+      - name: Setting sccache-path
+        run: echo "SCCACHE_PATH=/home/runner/.cache/sccache"
+        if: matrix.os == 'ubuntu-latest'
+      - name: Setting sccache-path
+        run: echo "SCCACHE_PATH=C:\Users\runner\AppData\Local\Mozilla\sccache"
+        if: matrix.os == 'windows-latest'
       - uses: actions/checkout@v3
         name: Checkout
+      - name: Cache sccache
+        uses: actions/cache@v3
+        with:
+          path: ${{ matrix.sccache-path }}
+          key: pytest-${{ runner.os }}-sccache-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            pytest-${{ runner.os }}-sccache-
       - name: Set up cargo cache
         uses: actions/cache@v3
         continue-on-error: false
@@ -47,26 +75,41 @@ jobs:
             ~/.cargo/git/db/
             target/debug
             target/release
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          key: pytest-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: pytest-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
       - uses: actions-rs/toolchain@v1
         name: Setup Rust
         with:
           profile: minimal
           toolchain: stable
           override: true
           components: rustfmt, clippy
+      - name: Install sccache (macOS)
+        if: matrix.os == 'macos-latest'
+        run: brew install sccache
+      - name: Install sccache (Ubuntu)
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          curl -L https://github.com/mozilla/sccache/releases/download/v0.5.4/sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz | tar xz
+          sudo mv sccache-v0.5.4-x86_64-unknown-linux-musl/sccache /usr/local/bin/
+      - name: Install sccache (Windows)
+        if: matrix.os == 'windows-latest'
+        run: choco install sccache
+      - name: Start sccache server
+        run: sccache --start-server
       - name: Setup Python ${{ matrix.python }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
+          cache: 'pip'
       - name: Run Maturin develop
         uses: PyO3/maturin-action@v1
         with:
           working-directory: ./python
           command: build
           target: ${{ matrix.target }}
           args: --release
+          sccache: true
       - name: Install Python dependencies (Unix)
         if: "contains(matrix.os, 'Ubuntu') || contains(matrix.os, 'macOS')"
         run: |
@@ -88,3 +131,8 @@ jobs:
       - name: Run Python extension tests
         run: |
           cd examples/custom_python_extension/test && pytest .
+      - name: Print sccache stats
+        run: sccache --show-stats
+      - name: Stop sccache server
+        run: sccache --stop-server || true
+
diff --git a/.github/workflows/test_rust_workflow.yml b/.github/workflows/test_rust_workflow.yml
@@ -16,15 +16,22 @@ jobs:
       matrix:
         include:
           - os: macos-latest
+            sccache-path: /Users/runner/Library/Caches/Mozilla.sccache
           - os: ubuntu-latest
+            sccache-path: /home/runner/.cache/sccache
           - os: windows-latest
+            sccache-path: C:\Users\runner\AppData\Local\Mozilla\sccache
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_CACHE_SIZE: 2G
+      SCCACHE_DIR: ${{ matrix.sccache-path }}
     steps:
       - uses: actions/checkout@v3
         name: Checkout
       - name: Cache sccache
         uses: actions/cache@v3
         with:
-          path: ~/.cache/sccache
+          path: ${{ matrix.sccache-path }}
           key: ${{ runner.os }}-sccache-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: |
             ${{ runner.os }}-sccache-
@@ -46,20 +53,30 @@ jobs:
           toolchain: stable
           override: true
           components: rustfmt, clippy
-      - name: Install sccache
+      - name: Install sccache (macOS)
+        if: matrix.os == 'macos-latest'
+        run: brew install sccache
+      - name: Install sccache (Ubuntu)
+        if: matrix.os == 'ubuntu-latest'
         run: |
-          cargo install sccache
-      - name: Set up sccache
-        run: |
-          echo "SCCACHE_DIR=~/.cache/sccache" >> $GITHUB_ENV
-          echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+          curl -L https://github.com/mozilla/sccache/releases/download/v0.5.4/sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz | tar xz
+          sudo mv sccache-v0.5.4-x86_64-unknown-linux-musl/sccache /usr/local/bin/
+      - name: Install sccache (Windows)
+        if: matrix.os == 'windows-latest'
+        run: choco install sccache
+      - name: Start sccache server
+        run: sccache --start-server
       - name: Run Tests
         uses: actions-rs/cargo@v1
         env:
           RUSTFLAGS: -Awarnings
         with:
           command: test
           args: --all --no-default-features --features "raphtory/io raphtory/python"
+      - name: Print sccache stats
+        run: sccache --show-stats
+      - name: Stop sccache server
+        run: sccache --stop-server || true
   doc-test:
     if: ${{ !inputs.skip_tests }}
     name: "Doc tests"
@@ -68,9 +85,21 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_CACHE_SIZE: 2G
+      SCCACHE_DIR: /home/runner/.cache/sccache
+      SCCACHE_PATH: /home/runner/.cache/sccache
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
+      - name: Cache sccache
+        uses: actions/cache@v3
+        with:
+          path: /home/runner/.cache/sccache
+          key: doc-${{ runner.os }}-sccache-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            doc-${{ runner.os }}-sccache-
       - name: Set up cargo cache
         uses: actions/cache@v3
         continue-on-error: false
@@ -80,19 +109,26 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: ${{ runner.os }}-cargo-
+          key: doc-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: doc-${{ runner.os }}-cargo-
       - uses: actions-rs/toolchain@v1
         name: Setup Rust
         with:
           profile: minimal
           toolchain: stable
           override: true
           components: rustfmt, clippy
+      - name: Install sccache (Ubuntu)
+        run: |
+          curl -L https://github.com/mozilla/sccache/releases/download/v0.5.4/sccache-v0.5.4-x86_64-unknown-linux-musl.tar.gz | tar xz
+          sudo mv sccache-v0.5.4-x86_64-unknown-linux-musl/sccache /usr/local/bin/
+      - name: Start sccache server
+        run: sccache --start-server
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+          cache: 'pip'
       - name: Build Rust Docs
         uses: actions-rs/cargo@v1
         env:
@@ -106,3 +142,7 @@ jobs:
           sudo apt update && sudo apt install -y pandoc make python3-sphinx
           pip install -e python
           cd docs && python -m pip install -q -r requirements.txt && make html
+      - name: Print sccache stats
+        run: sccache --show-stats
+      - name: Stop sccache server
+        run: sccache --stop-server || true