Merge pull request #718 from allenai/ot-fix-mmlu-bpb

Fix mmlu bpb bug only scoring answer=A questions
allenai · Sep 6, 2024 · 0b92077 · 0b92077
2 parents ca81901 + 138cfe9
commit 0b92077
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/olmo/eval/downstream.py b/olmo/eval/downstream.py
@@ -1365,6 +1365,9 @@ def doc_to_continuations(self, doc):
             return choices
 
     def doc_to_label(self, doc):
+        if self.metric_type in ["ce_loss", "bpb"]:
+            # Only the correct answer is provided for these metrics
+            return 0
         return doc["answer"]
 
     def doc_to_domain_conditional(self, doc):