feat: parallelise at hole level (#44)

* feat: parallelise at hole level

* fix(ci): move strategy to testbed job

* feat: output json results file

* fix(ci): install jq

* fix(ci): add missing `runs-on`

* fix(ci): add dependency to testbed job

* fix(ci): invalid artifact key name

* fix(ci): add missing i in fastapi key

* feat(ci): make CI run different # of threads per repo

* fix(ci): results.json not in markdown

* feat: round output values

* fix: avoid creating zombie processes

* fix: check on word instead of line

* feat: recreate holes for long CI

.github/workflows/test.yml

@@ -11,6 +11,22 @@ concurrency:
 
 jobs:
   testbed:
+    strategy:
+      matrix:
+        repo:
+          - { name: simple, key: simple, parallel: 8 }
+          - { name: mmaitre314/picklescan, key: picklescan, parallel: 8 }
+          - { name: huggingface/huggingface_hub, key: huggingface_hub, parallel: 8 }
+          - { name: tiangolo/fastapi, key: fastapi, parallel: 8 }
+          - { name: encode/starlette, key: starlette, parallel: 8 }
+          - { name: lancedb/lancedb, key: lancedb, parallel: 2 }
+          - { name: lancedb/lance, key: lance, parallel: 2 }
+          - { name: tkaitchuck/constrandom, key: constrandom, parallel: 8 }
+          - { name: jaemk/cached, key: cached, parallel: 4 }
+          - { name: smol-rs/async-executor, key: async-executor, parallel: 4 }
+          - { name: gcanti/io-ts, key: io-ts, parallel: 8 }
+          - { name: colinhacks/zod, key: zod, parallel: 8 }
+          - { name: helix-editor/helix, key: helix, parallel: 2 }
     runs-on: [self-hosted, intel-cpu, 8-cpu, ci]
     container:
       image: ubuntu:22.04
@@ -60,17 +76,118 @@ jobs:
         run: cargo build -r
 
       - name: Run testbed
-        run: cargo run --bin testbed -r -- --api-token $API_TOKEN -r `pwd`/crates/testbed/repositories-ci.yaml
+        run: 'cargo run --bin testbed -r -- --api-token $API_TOKEN -r `pwd`/crates/testbed/repositories-ci.yaml -f ${{ matrix.repo.name }} -p ${{ matrix.repo.parallel }}'
         if: github.event_name == 'push' || github.event_name == 'pull_request'
         env:
           API_TOKEN: ${{ secrets.API_TOKEN }}
 
       - name: Run testbed
-        run: cargo run --bin testbed -r -- --api-token $API_TOKEN
+        run: 'cargo run --bin testbed -r -- --api-token $API_TOKEN -f ${{ matrix.repo.name }} -p ${{ matrix.repo.parallel }}'
         if: github.event_name == 'workflow_dispatch'
         env:
           API_TOKEN: ${{ secrets.API_TOKEN }}
 
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: results-${{ matrix.repo.key }}
+          path: ./results.json
+
+  comment_results:
+    needs: [testbed]
+    runs-on: [self-hosted, intel-cpu, 8-cpu, ci]
+    container:
+      image: ubuntu:22.04
+    steps:
+      - name: Install dependencies
+        run: |
+          apt update
+          apt install -y jq
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-simple
+          path: results-simple
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-picklescan
+          path: results-picklescan
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-huggingface_hub
+          path: results-huggingface_hub
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-fastapi
+          path: results-fastapi
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-starlette
+          path: results-starlette
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-lancedb
+          path: results-lancedb
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-lance
+          path: results-lance
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-constrandom
+          path: results-constrandom
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-cached
+          path: results-cached
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-async-executor
+          path: results-async-executor
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-io-ts
+          path: results-io-ts
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-zod
+          path: results-zod
+
+      - uses: actions/download-artifact@v1
+        with:
+          name: results-helix
+          path: results-helix
+
+      - name: Display structure of downloaded files
+        run: ls -R
+
+      - name: output to markdown
+        run: |
+          cat > results.md  <<EOF
+          | Repository name | Source type | Average hole completion time (s) | Pass percentage |
+          | :-------------- | :---------- | -------------------------------: | --------------: |
+          EOF
+          cat **/results.json | jq -r '"| \(.[0].repo_name) | \(.[0].source_type) | \(.[0].avg_hole_completion_time_ms) | \(.[0].pass_percentage)% |"' >> results.md
+          cat >> results.md <<EOF
+
+          **Note:** The "hole completion time" represents the full process of:
+            - copying files from the setup cache directory
+            - replacing the code from the file with a completion from the model
+            - building the project
+            - running the tests
+          EOF
+
       - name: Find Comment
         uses: peter-evans/find-comment@v2
         id: fc

.gitignore

@@ -3,5 +3,5 @@ dist/
 target/
 .DS_Store
 __pycache__/
-results.md
+results.json
 .pytest_cache/

crates/llm-ls/src/document.rs

@@ -1,7 +1,6 @@
 use ropey::Rope;
 use tower_lsp::jsonrpc::Result;
 use tower_lsp::lsp_types::Range;
-use tracing::info;
 use tree_sitter::{InputEdit, Parser, Point, Tree};
 
 use crate::language_id::LanguageId;

crates/testbed/README.md

@@ -11,9 +11,10 @@ Here is a simplified pseudo code algorithm for testbed:
 read the repositories file
 read the holes file(s)
 for each repository
-  spawn a thread
-  setup the repository
   for each hole
+    spawn a thread
+    setup the repository -- only once for each repository
+    copy files from the setup cache to a new temp dir
     make the hole as specified by the file
     generate completions
     build the code
@@ -37,7 +38,7 @@ Before running testbed, you will need to generate a holes file for each reposito
 
 ### Setup
 
-testbed runs completions for each repository in parallel. It will first create a temporary directory, then copy or download the repository's source files to that location and finally run the setup commands.
+testbed runs hole completions in parallel. It will first, and only once per repository, create a temporary directory, then copy or download the repository's source files to that location and finally run the setup commands. Then for each subsequent completion it will copy the content of the "setup directory" to a new temporary directory so that work can be parallelised.
 
 Setup commands are useful to install dependencies.
 
@@ -60,9 +61,11 @@ build_args: ["-m", "compileall", "-q", "."]
 
 ### Runners
 
-testbed supports two test runners at the moment:
+testbed supports four test runners:
 - cargo
+- jest
 - pytest
+- vitest
 
 To configure your runner, you have the following options:
 ```yaml

crates/testbed/holes/async-executor.json

@@ -1 +1 @@
-[{"cursor":{"line":86,"character":2},"file":"src/lib.rs"},{"cursor":{"line":266,"character":3},"file":"src/lib.rs"},{"cursor":{"line":565,"character":6},"file":"src/lib.rs"},{"cursor":{"line":914,"character":12},"file":"src/lib.rs"},{"cursor":{"line":261,"character":3},"file":"src/lib.rs"},{"cursor":{"line":839,"character":4},"file":"src/lib.rs"},{"cursor":{"line":337,"character":4},"file":"src/lib.rs"},{"cursor":{"line":517,"character":0},"file":"src/lib.rs"},{"cursor":{"line":763,"character":12},"file":"src/lib.rs"},{"cursor":{"line":551,"character":1},"file":"src/lib.rs"},{"cursor":{"line":461,"character":14},"file":"src/lib.rs"},{"cursor":{"line":847,"character":0},"file":"src/lib.rs"},{"cursor":{"line":835,"character":14},"file":"src/lib.rs"},{"cursor":{"line":793,"character":4},"file":"src/lib.rs"},{"cursor":{"line":901,"character":13},"file":"src/lib.rs"},{"cursor":{"line":191,"character":3},"file":"src/lib.rs"},{"cursor":{"line":694,"character":7},"file":"src/lib.rs"},{"cursor":{"line":824,"character":5},"file":"src/lib.rs"},{"cursor":{"line":840,"character":3},"file":"src/lib.rs"},{"cursor":{"line":310,"character":0},"file":"src/lib.rs"},{"cursor":{"line":762,"character":7},"file":"src/lib.rs"},{"cursor":{"line":664,"character":2},"file":"src/lib.rs"},{"cursor":{"line":267,"character":1},"file":"src/lib.rs"},{"cursor":{"line":484,"character":12},"file":"src/lib.rs"},{"cursor":{"line":24,"character":4},"file":"src/lib.rs"},{"cursor":{"line":100,"character":4},"file":"src/lib.rs"},{"cursor":{"line":873,"character":9},"file":"src/lib.rs"},{"cursor":{"line":690,"character":2},"file":"src/lib.rs"},{"cursor":{"line":876,"character":13},"file":"src/lib.rs"},{"cursor":{"line":435,"character":4},"file":"src/lib.rs"},{"cursor":{"line":79,"character":11},"file":"src/lib.rs"},{"cursor":{"line":552,"character":4},"file":"src/lib.rs"},{"cursor":{"line":472,"character":0},"file":"src/lib.rs"},{"cursor":{"line":762,"character":0},"file":"src/lib.rs"},{"cursor":{"line":845,"character":5},"file":"src/lib.rs"},{"cursor":{"line":707,"character":4},"file":"src/lib.rs"},{"cursor":{"line":433,"character":10},"file":"src/lib.rs"},{"cursor":{"line":754,"character":1},"file":"src/lib.rs"},{"cursor":{"line":758,"character":12},"file":"src/lib.rs"},{"cursor":{"line":811,"character":8},"file":"src/lib.rs"},{"cursor":{"line":760,"character":13},"file":"src/lib.rs"},{"cursor":{"line":898,"character":8},"file":"src/lib.rs"},{"cursor":{"line":149,"character":3},"file":"src/lib.rs"},{"cursor":{"line":242,"character":14},"file":"src/lib.rs"},{"cursor":{"line":805,"character":6},"file":"src/lib.rs"},{"cursor":{"line":830,"character":4},"file":"src/lib.rs"},{"cursor":{"line":578,"character":4},"file":"src/lib.rs"},{"cursor":{"line":701,"character":11},"file":"src/lib.rs"},{"cursor":{"line":181,"character":10},"file":"src/lib.rs"},{"cursor":{"line":704,"character":14},"file":"src/lib.rs"},{"cursor":{"line":852,"character":8},"file":"src/lib.rs"},{"cursor":{"line":311,"character":1},"file":"src/lib.rs"},{"cursor":{"line":707,"character":6},"file":"src/lib.rs"},{"cursor":{"line":75,"character":0},"file":"src/lib.rs"},{"cursor":{"line":699,"character":1},"file":"src/lib.rs"},{"cursor":{"line":600,"character":0},"file":"src/lib.rs"},{"cursor":{"line":191,"character":3},"file":"src/lib.rs"},{"cursor":{"line":150,"character":1},"file":"src/lib.rs"},{"cursor":{"line":767,"character":10},"file":"src/lib.rs"},{"cursor":{"line":739,"character":0},"file":"src/lib.rs"},{"cursor":{"line":919,"character":3},"file":"src/lib.rs"},{"cursor":{"line":143,"character":6},"file":"src/lib.rs"},{"cursor":{"line":234,"character":5},"file":"src/lib.rs"},{"cursor":{"line":860,"character":6},"file":"src/lib.rs"},{"cursor":{"line":519,"character":0},"file":"src/lib.rs"},{"cursor":{"line":668,"character":14},"file":"src/lib.rs"},{"cursor":{"line":184,"character":7},"file":"src/lib.rs"},{"cursor":{"line":607,"character":10},"file":"src/lib.rs"},{"cursor":{"line":931,"character":5},"file":"src/lib.rs"},{"cursor":{"line":191,"character":0},"file":"src/lib.rs"},{"cursor":{"line":710,"character":6},"file":"src/lib.rs"},{"cursor":{"line":509,"character":1},"file":"src/lib.rs"},{"cursor":{"line":873,"character":11},"file":"src/lib.rs"},{"cursor":{"line":539,"character":12},"file":"src/lib.rs"},{"cursor":{"line":524,"character":9},"file":"src/lib.rs"},{"cursor":{"line":959,"character":1},"file":"src/lib.rs"},{"cursor":{"line":147,"character":11},"file":"src/lib.rs"},{"cursor":{"line":620,"character":2},"file":"src/lib.rs"},{"cursor":{"line":903,"character":6},"file":"src/lib.rs"},{"cursor":{"line":931,"character":13},"file":"src/lib.rs"},{"cursor":{"line":830,"character":3},"file":"src/lib.rs"},{"cursor":{"line":832,"character":11},"file":"src/lib.rs"},{"cursor":{"line":652,"character":3},"file":"src/lib.rs"},{"cursor":{"line":673,"character":7},"file":"src/lib.rs"},{"cursor":{"line":22,"character":9},"file":"src/lib.rs"},{"cursor":{"line":150,"character":6},"file":"src/lib.rs"},{"cursor":{"line":853,"character":13},"file":"src/lib.rs"},{"cursor":{"line":658,"character":13},"file":"src/lib.rs"},{"cursor":{"line":616,"character":0},"file":"src/lib.rs"},{"cursor":{"line":457,"character":12},"file":"src/lib.rs"},{"cursor":{"line":856,"character":10},"file":"src/lib.rs"},{"cursor":{"line":795,"character":11},"file":"src/lib.rs"},{"cursor":{"line":763,"character":3},"file":"src/lib.rs"},{"cursor":{"line":778,"character":6},"file":"src/lib.rs"},{"cursor":{"line":866,"character":1},"file":"src/lib.rs"},{"cursor":{"line":764,"character":4},"file":"src/lib.rs"},{"cursor":{"line":181,"character":9},"file":"src/lib.rs"},{"cursor":{"line":780,"character":11},"file":"src/lib.rs"},{"cursor":{"line":846,"character":5},"file":"src/lib.rs"},{"cursor":{"line":651,"character":2},"file":"src/lib.rs"}]
+[{"cursor":{"line":875,"character":11},"file":"src/lib.rs"},{"cursor":{"line":730,"character":4},"file":"src/lib.rs"},{"cursor":{"line":716,"character":4},"file":"src/lib.rs"},{"cursor":{"line":37,"character":14},"file":"src/lib.rs"},{"cursor":{"line":906,"character":9},"file":"src/lib.rs"},{"cursor":{"line":322,"character":9},"file":"src/lib.rs"},{"cursor":{"line":914,"character":1},"file":"src/lib.rs"},{"cursor":{"line":757,"character":1},"file":"src/lib.rs"},{"cursor":{"line":37,"character":13},"file":"src/lib.rs"},{"cursor":{"line":314,"character":10},"file":"src/lib.rs"},{"cursor":{"line":927,"character":2},"file":"src/lib.rs"},{"cursor":{"line":694,"character":7},"file":"src/lib.rs"},{"cursor":{"line":490,"character":6},"file":"src/lib.rs"},{"cursor":{"line":159,"character":2},"file":"src/lib.rs"},{"cursor":{"line":494,"character":0},"file":"src/lib.rs"},{"cursor":{"line":690,"character":4},"file":"src/lib.rs"},{"cursor":{"line":602,"character":0},"file":"src/lib.rs"},{"cursor":{"line":940,"character":1},"file":"src/lib.rs"},{"cursor":{"line":190,"character":1},"file":"src/lib.rs"},{"cursor":{"line":655,"character":4},"file":"src/lib.rs"},{"cursor":{"line":843,"character":0},"file":"src/lib.rs"},{"cursor":{"line":802,"character":9},"file":"src/lib.rs"},{"cursor":{"line":781,"character":13},"file":"src/lib.rs"},{"cursor":{"line":748,"character":9},"file":"src/lib.rs"},{"cursor":{"line":233,"character":8},"file":"src/lib.rs"},{"cursor":{"line":781,"character":9},"file":"src/lib.rs"},{"cursor":{"line":864,"character":12},"file":"src/lib.rs"},{"cursor":{"line":26,"character":0},"file":"src/lib.rs"},{"cursor":{"line":822,"character":11},"file":"src/lib.rs"},{"cursor":{"line":902,"character":1},"file":"src/lib.rs"},{"cursor":{"line":898,"character":0},"file":"src/lib.rs"},{"cursor":{"line":753,"character":0},"file":"src/lib.rs"},{"cursor":{"line":583,"character":1},"file":"src/lib.rs"},{"cursor":{"line":250,"character":1},"file":"src/lib.rs"},{"cursor":{"line":745,"character":2},"file":"src/lib.rs"},{"cursor":{"line":88,"character":0},"file":"src/lib.rs"},{"cursor":{"line":519,"character":3},"file":"src/lib.rs"},{"cursor":{"line":747,"character":2},"file":"src/lib.rs"},{"cursor":{"line":264,"character":12},"file":"src/lib.rs"},{"cursor":{"line":656,"character":0},"file":"src/lib.rs"},{"cursor":{"line":872,"character":4},"file":"src/lib.rs"},{"cursor":{"line":189,"character":5},"file":"src/lib.rs"},{"cursor":{"line":470,"character":0},"file":"src/lib.rs"},{"cursor":{"line":906,"character":7},"file":"src/lib.rs"},{"cursor":{"line":378,"character":0},"file":"src/lib.rs"},{"cursor":{"line":871,"character":14},"file":"src/lib.rs"},{"cursor":{"line":378,"character":9},"file":"src/lib.rs"},{"cursor":{"line":850,"character":8},"file":"src/lib.rs"},{"cursor":{"line":683,"character":4},"file":"src/lib.rs"},{"cursor":{"line":925,"character":0},"file":"src/lib.rs"},{"cursor":{"line":504,"character":14},"file":"src/lib.rs"},{"cursor":{"line":460,"character":12},"file":"src/lib.rs"},{"cursor":{"line":79,"character":7},"file":"src/lib.rs"},{"cursor":{"line":241,"character":8},"file":"src/lib.rs"},{"cursor":{"line":605,"character":8},"file":"src/lib.rs"},{"cursor":{"line":741,"character":11},"file":"src/lib.rs"},{"cursor":{"line":875,"character":11},"file":"src/lib.rs"},{"cursor":{"line":854,"character":13},"file":"src/lib.rs"},{"cursor":{"line":800,"character":2},"file":"src/lib.rs"},{"cursor":{"line":502,"character":13},"file":"src/lib.rs"},{"cursor":{"line":316,"character":6},"file":"src/lib.rs"},{"cursor":{"line":711,"character":14},"file":"src/lib.rs"},{"cursor":{"line":675,"character":4},"file":"src/lib.rs"},{"cursor":{"line":531,"character":4},"file":"src/lib.rs"},{"cursor":{"line":147,"character":2},"file":"src/lib.rs"},{"cursor":{"line":90,"character":3},"file":"src/lib.rs"},{"cursor":{"line":334,"character":6},"file":"src/lib.rs"},{"cursor":{"line":336,"character":7},"file":"src/lib.rs"},{"cursor":{"line":461,"character":5},"file":"src/lib.rs"},{"cursor":{"line":895,"character":0},"file":"src/lib.rs"},{"cursor":{"line":38,"character":14},"file":"src/lib.rs"},{"cursor":{"line":823,"character":8},"file":"src/lib.rs"},{"cursor":{"line":845,"character":3},"file":"src/lib.rs"},{"cursor":{"line":856,"character":14},"file":"src/lib.rs"},{"cursor":{"line":525,"character":2},"file":"src/lib.rs"},{"cursor":{"line":919,"character":6},"file":"src/lib.rs"},{"cursor":{"line":462,"character":10},"file":"src/lib.rs"},{"cursor":{"line":215,"character":3},"file":"src/lib.rs"},{"cursor":{"line":389,"character":10},"file":"src/lib.rs"},{"cursor":{"line":76,"character":0},"file":"src/lib.rs"},{"cursor":{"line":844,"character":10},"file":"src/lib.rs"},{"cursor":{"line":563,"character":6},"file":"src/lib.rs"},{"cursor":{"line":529,"character":14},"file":"src/lib.rs"},{"cursor":{"line":659,"character":3},"file":"src/lib.rs"},{"cursor":{"line":286,"character":1},"file":"src/lib.rs"},{"cursor":{"line":525,"character":4},"file":"src/lib.rs"},{"cursor":{"line":669,"character":3},"file":"src/lib.rs"},{"cursor":{"line":622,"character":9},"file":"src/lib.rs"},{"cursor":{"line":280,"character":3},"file":"src/lib.rs"},{"cursor":{"line":916,"character":0},"file":"src/lib.rs"},{"cursor":{"line":574,"character":9},"file":"src/lib.rs"},{"cursor":{"line":795,"character":0},"file":"src/lib.rs"},{"cursor":{"line":246,"character":2},"file":"src/lib.rs"},{"cursor":{"line":878,"character":8},"file":"src/lib.rs"},{"cursor":{"line":785,"character":4},"file":"src/lib.rs"},{"cursor":{"line":757,"character":10},"file":"src/lib.rs"},{"cursor":{"line":921,"character":11},"file":"src/lib.rs"},{"cursor":{"line":647,"character":11},"file":"src/lib.rs"},{"cursor":{"line":154,"character":6},"file":"src/lib.rs"},{"cursor":{"line":558,"character":1},"file":"src/lib.rs"}]