[ENH] add internlm2-chat-20b-ppo (#207)

* add model config

* modify `huggingface_local_completion` to remove EOS

* add results & update leaderboard

* delete extra leaderboard csv

* add docstring of `remove_ending`

* remove reference_outputs.json

src/alpaca_eval/decoders/huggingface_local.py

@@ -32,6 +32,7 @@ def huggingface_local_completions(
     batch_size: int = 1,
     model_kwargs=None,
     cache_dir: Optional[str] = constants.DEFAULT_CACHE_DIR,
+    remove_ending: Optional[str] = None,
     is_fast_tokenizer: bool = True,
     adapters_name: Optional[str] = None,
     **kwargs,
@@ -58,6 +59,9 @@ def huggingface_local_completions(
     cache_dir : str, optional
         Directory to use for caching the model.
 
+    remove_ending : str, optional
+        The ending string to be removed from completions. Typically eos_token.
+
     kwargs :
         Additional kwargs to pass to `InferenceApi.__call__`.
     """
@@ -143,7 +147,10 @@ def huggingface_local_completions(
                 pad_token_id=tokenizer.pad_token_id,
             )
         ):
-            completions.append(out[0]["generated_text"])
+            generated_text = out[0]["generated_text"]
+            if remove_ending is not None and generated_text.endswith(remove_ending):
+                generated_text = generated_text[:-len(remove_ending)]
+            completions.append(generated_text)
 
     logging.info(f"Time for {n_examples} completions: {t}")
 

src/alpaca_eval/leaderboards/data_AlpacaEval_2/weighted_alpaca_eval_gpt4_turbo_leaderboard.csv

@@ -5,6 +5,7 @@ gpt4,23.576789314782605,1.275704201206918,179,618,8,805,22.732919254658384,minim
 gpt4_0314,22.07325892871952,1.246672549460114,172,627,6,805,21.73913043478261,verified,1371
 mistral-medium,21.855772543461345,1.2682402187234831,164,639,2,805,20.496894409937887,minimal,1500
 xwinlm-70b-v0.1,21.812957073994184,1.2303274476035724,166,635,4,805,20.869565217391305,community,1775
+internlm2-chat-20b-ppo,21.749154500562636,1.2443662409556862,170,632,3,805,21.304347826086957,community,2373
 evo-v2-7b,20.83411302254932,1.2159901798147486,158,644,3,805,19.81366459627329,community,1754
 pairrm-tulu-2-70b,18.63896296749771,1.1924966699965729,140,665,0,805,17.391304347826086,community,1607
 Mixtral-8x7B-Instruct-v0.1,18.25531762637268,1.1885585968848205,135,668,2,805,16.8944099378882,minimal,1465

src/alpaca_eval/models_configs/internlm2-chat-20b-ppo/configs.yaml

@@ -0,0 +1,15 @@
+internlm2-chat-20b-ppo:
+  prompt_template: "internlm2-chat-20b-ppo/prompt.txt"
+  fn_completions: "huggingface_local_completions"
+  completions_kwargs:
+    model_name: "internlm/internlm2-chat-20b"
+    model_kwargs:
+      torch_dtype: "bfloat16"
+      trust_remote_code: True
+    is_fast_tokenizer: False
+    max_new_tokens: 2048
+    batch_size: 32
+    eos_token_id: 92542
+    remove_ending: "[UNUSED_TOKEN_145]"
+  pretty_name: "InternLM2 Chat 20B ppo"
+  link: "https://huggingface.co/internlm/internlm2-chat-20b"

src/alpaca_eval/models_configs/internlm2-chat-20b-ppo/prompt.txt

@@ -0,0 +1,3 @@
+[UNUSED_TOKEN_146]user
+{instruction}[UNUSED_TOKEN_145]
+[UNUSED_TOKEN_146]assistant