-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathlm_eval.ps1
10 lines (10 loc) · 2.3 KB
/
lm_eval.ps1
1
2
3
4
5
6
7
8
9
10
lm_eval --model hf --model_args pretrained=../pythia-160m-deduped --tasks arc_easy,lambada_openai,sciq,piqa,boolq --device cuda:0 --batch_size 32 --output_path lm_eval_results/base_results.json;
lm_eval --model hf --model_args pretrained=faridlazuarda/pythia-160m-gqa-48-b12-g2-v2,backend=mlkv,revision=master --tasks arc_easy,lambada_openai,sciq,piqa,boolq --device cuda:0 --batch_size 32 --output_path lm_eval_results/gqa_48_results.json;
lm_eval --model hf --model_args pretrained=faridlazuarda/pythia-160m-mlkv-48-b12-g2-v9,backend=mlkv,revision=master --tasks arc_easy,lambada_openai,sciq,piqa,boolq --device cuda:0 --batch_size 32 --output_path lm_eval_results/mlkv_48_results.json;
lm_eval --model hf --model_args pretrained=faridlazuarda/pythia-160m-mqa-12-b12-g2-v1,backend=mlkv,revision=master --tasks arc_easy,lambada_openai,sciq,piqa,boolq --device cuda:0 --batch_size 32 --output_path lm_eval_results/mqa_12_results.json;
lm_eval --model hf --model_args pretrained=faridlazuarda/pythia-160m-mlkv-12-b12-g2-v1,backend=mlkv,revision=master --tasks arc_easy,lambada_openai,sciq,piqa,boolq --device cuda:0 --batch_size 32 --output_path lm_eval_results/mlkv_12_results.json;
lm_eval --model hf --model_args pretrained=faridlazuarda/pythia-160m-mlkv-6-b12-g2-v1,backend=mlkv,revision=master --tasks arc_easy,lambada_openai,sciq,piqa,boolq --device cuda:0 --batch_size 32 --output_path lm_eval_results/mlkv_6_results.json;
lm_eval --model hf --model_args pretrained=faridlazuarda/pythia-160m-mlkv-4-b12-g2-v1,backend=mlkv,revision=master --tasks arc_easy,lambada_openai,sciq,piqa,boolq --device cuda:0 --batch_size 32 --output_path lm_eval_results/mlkv_4_results.json;
lm_eval --model hf --model_args pretrained=faridlazuarda/pythia-160m-mlkv-2-b12-g2-v1,backend=mlkv,revision=master --tasks arc_easy,lambada_openai,sciq,piqa,boolq --device cuda:0 --batch_size 32 --output_path lm_eval_results/mlkv_2_results.json;
lm_eval --model hf --model_args pretrained=faridlazuarda/pythia-160m-mlkv-1-b12-g2-v1,backend=mlkv,revision=master --tasks arc_easy,lambada_openai,sciq,piqa,boolq --device cuda:0 --batch_size 32 --output_path lm_eval_results/mlkv_1_results.json;
lm_eval --model hf --model_args pretrained=pythia-160m-random --tasks arc_easy,lambada_openai,sciq,piqa,boolq --device cuda:0 --batch_size 32 --output_path lm_eval_results/random_results.json;