From 23416ae8f38f13c90c342609a590c020ff1a4157 Mon Sep 17 00:00:00 2001 From: Gregory Kielian Date: Fri, 17 Nov 2023 01:28:00 -0800 Subject: [PATCH] Further parameterize pos emb exploration script --- .../test_all_positional_embeddings.sh | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/explorations/test_all_positional_embeddings.sh b/explorations/test_all_positional_embeddings.sh index 11970d97d3..b407757af8 100644 --- a/explorations/test_all_positional_embeddings.sh +++ b/explorations/test_all_positional_embeddings.sh @@ -7,6 +7,8 @@ python3 data/shakespeare_char/prepare.py # Common settings max_iterations=3000 +dataset="shakespeare" +tensorboard_project="shkspr_tiktoken" # rope python3 train.py \ @@ -14,12 +16,12 @@ python3 train.py \ --eval_iters 200 \ --eval_interval 100 \ --log_interval 10 \ - --dataset "shakespeare_char" \ + --dataset "$dataset" \ --use_rotary_embeddings \ --no-use_abs_pos_embeddings \ --rope_variant "rope" \ --no-use_softmax_variant \ - --tensorboard_project "shkspr" \ + --tensorboard_project "$tensorboard_project" \ --tensorboard_run_name "rope" \ --block_size 256 \ --out_dir "shkspr_rope" \ @@ -31,11 +33,11 @@ python3 train.py \ --eval_iters 200 \ --eval_interval 100 \ --log_interval 10 \ - --dataset "shakespeare_char" \ + --dataset "$dataset" \ --no-use_rotary_embeddings \ --use_abs_pos_embeddings \ --no-use_softmax_variant \ - --tensorboard_project "shkspr" \ + --tensorboard_project "$tensorboard_project" \ --tensorboard_run_name "abs_pos" \ --block_size 256 \ --out_dir "shkspr_abs_pos" \ @@ -47,28 +49,28 @@ python3 train.py \ --eval_iters 200 \ --eval_interval 100 \ --log_interval 10 \ - --dataset "shakespeare_char" \ + --dataset "$dataset" \ --use_rotary_embeddings \ --rope_variant "rope" \ --use_abs_pos_embeddings \ --no-use_softmax_variant \ - --tensorboard_project "shkspr" \ + --tensorboard_project "$tensorboard_project" \ --tensorboard_run_name "rope_abs_pos" \ --block_size 256 \ --out_dir "shkspr_rope_abs_pos" \ --compile -no positional embeddings +# no positional embeddings python3 train.py \ --max_iters "$max_iterations" \ --eval_iters 200 \ --eval_interval 100 \ --log_interval 10 \ - --dataset "shakespeare_char" \ + --dataset "$dataset" \ --no-use_rotary_embeddings \ --no-use_abs_pos_embeddings \ --no-use_softmax_variant \ - --tensorboard_project "shkspr" \ + --tensorboard_project "$tensorboard_project" \ --tensorboard_run_name "no_pos_emb" \ --block_size 256 \ --out_dir "shkspr_nope" \ @@ -81,13 +83,13 @@ for i in {2..16..2}; do --eval_iters 200 \ --eval_interval 100 \ --log_interval 10 \ - --dataset "shakespeare_char" \ + --dataset "$dataset" \ --use_rotary_embeddings \ --rope_variant "shortrope" \ --shortrope_length "${i}" \ --no-use_abs_pos_embeddings \ --no-use_softmax_variant \ - --tensorboard_project "shkspr" \ + --tensorboard_project "$tensorboard_project" \ --tensorboard_run_name "shortrope_${i}" \ --block_size 256 \ --out_dir "shkspr_rope_abs_pos_${i}" \