Test shortrope length 16

gkielian · Nov 9, 2023 · 4daf7a1 · 4daf7a1
1 parent b1d0e22
commit 4daf7a1
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/train.py b/train.py
@@ -57,8 +57,8 @@ def parse_args():
     # Positional Embedding variations
     model_group.add_argument('--use_rotary_embeddings', default=True, action=argparse.BooleanOptionalAction)
     model_group.add_argument("--rope_variant", type=str, default="shortrope", choices=["shortrope", "rope"])
-    model_group.add_argument("--shortrope_length", type=int, default="8", help="number of embeddings to use with rope, must be <= length, and be even")
-    model_group.add_argument('--use_abs_pos_embeddings', default=True, action=argparse.BooleanOptionalAction)
+    model_group.add_argument("--shortrope_length", type=int, default="16", help="number of embeddings to use with rope, must be <= length, and be even")
+    model_group.add_argument('--use_abs_pos_embeddings', default=False, action=argparse.BooleanOptionalAction)
 
     # Softmax variations
     model_group.add_argument('--use_softmax_variant', default=False, action=argparse.BooleanOptionalAction)