feat: setup results branch and folder to pull result files

repoqa/search_needle_function.py

@@ -163,7 +163,7 @@ def evaluate_model(
     result_dir: str = "results",
     languages: List[str] = None,
     caching: bool = False,  # if enabled, will cache the tasks which can be used to resume
-    system_message: str = "You are a helpful assistant good at code understanding.",
+    system_message: str = None,
 ):
     if backend is None:
         if base_url is not None:
@@ -178,7 +178,12 @@ def evaluate_model(
 
     # makedir if not exists
     os.makedirs(result_dir, exist_ok=True)
-    result_file = os.path.join(result_dir, f"{model.replace('/', '_slash_')}.jsonl")
+    context_size_dir = os.path.join(result_dir, f"ntoken_{code_context_size}")
+    os.makedirs(context_size_dir, exist_ok=True)
+    result_file = os.path.join(
+        context_size_dir,
+        f"{model.replace('/', '_slash_')}.jsonl",
+    )
 
     # resume from result_file
     if os.path.exists(result_file):
@@ -303,10 +308,6 @@ def evaluate_model(
             max_model_len=int(code_context_size * 1.25),  # Magic number
         )
 
-    if model == "mistralai/Mixtral-8x7B-Instruct-v0.1":
-        system_message = None
-        print(f"Warning: {model} does not support system message")
-
     if not system_message:
         print("🔥 System message is disabled")
         system_message = None

results/.gitignore

@@ -0,0 +1 @@
+*.zip

results/README.md

@@ -0,0 +1,7 @@
+## Pull evaluated results
+
+```bash
+gh release download dev-results --pattern "*.zip" --clobber
+# unzip all zip files
+unzip "*.zip"
+```