feat: experiment launcher now works with mixtral

instadeepai · Apr 20, 2024 · a87ee3d · a87ee3d
1 parent 5f49e22
commit a87ee3d
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 7 deletions.
diff --git a/manifest.yaml b/manifest.yaml
@@ -14,7 +14,7 @@ builder:
 spec:
   operator: tf
   image: debatellm
-  command: python experiments/evaluate.py
+  command: python scripts/launch_experiments.py
   tensorboard:
     enabled: false
 

diff --git a/scripts/launch_experiments.py b/scripts/launch_experiments.py
@@ -21,7 +21,7 @@
     {
         "system": "single_agent",
         "system.agents": gen_agent_config(
-            1, use_gpt=[False, True], prompt=["simple", "cot"]
+            1, use_gpt=True, prompt=["simple", "cot"]
         ),
     }
 )
@@ -36,7 +36,7 @@
             "system.num_aggregation_steps": 0,
             "system.agents": gen_agent_config(
                 1,
-                use_gpt=[False, True],
+                use_gpt=True,
                 prompt="er_simple",
                 is_es=True,
                 use_few_shot_examples=[False, True],
@@ -49,7 +49,7 @@
             "system.num_aggregation_steps": 0,
             "system.agents": gen_agent_config(
                 1,
-                use_gpt=[False, True],
+                use_gpt=True,
                 prompt="er_cot",
                 is_es=True,
                 use_few_shot_examples=[False, True],
@@ -68,7 +68,7 @@
             "system.num_aggregation_steps": 0,
             "system.agents": gen_agent_config(
                 1,
-                use_gpt=[False, True],
+                use_gpt=True,
                 prompt="er_simple",
                 is_es=True,
                 use_few_shot_examples=[False, True],
@@ -81,7 +81,7 @@
             "system.num_aggregation_steps": 0,
             "system.agents": gen_agent_config(
                 1,
-                use_gpt=[False, True],
+                use_gpt=True,
                 prompt="er_cot",
                 is_es=True,
                 use_few_shot_examples=[False, True],
@@ -211,4 +211,4 @@
         "pubmedqa",
         "mmlu",
     ]  # "medqa", "pubmedqa", "mmlu", "cosmosqa", "ciar", "gpqa", medmcqa
-run_experiments(exp_table, parallel_workers=4, shuffle=False, sort_by_dataset=False)
+run_experiments(exp_table, parallel_workers=2, shuffle=False, sort_by_dataset=False)