Skip to content

Commit

Permalink
ADLR/megatron-lm!2514 - ci: Implement frozen-ckpt tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@ko3n1g
ko3n1g committed Jan 12, 2025
1 parent 726da58 commit b41bcba
Show file tree
Hide file tree
Showing 174 changed files with 647 additions and 320 deletions.
3 changes: 3 additions & 0 deletions .gitlab/stages/02.functional-tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ functional:configure:
- |
A100_CLUSTER=$([[ "$FUNCTIONAL_TEST_CLUSTER_A100" != "" ]] && echo $FUNCTIONAL_TEST_CLUSTER_A100 || echo $DEFAULT_A100_CLUSTER)
H100_CLUSTER=$([[ "$FUNCTIONAL_TEST_CLUSTER_H100" != "" ]] && echo $FUNCTIONAL_TEST_CLUSTER_H100 || echo $DEFAULT_H100_CLUSTER)
- |
RECORD_CHECKPOINTS=$([[ "$CI_MERGE_REQUEST_LABELS" == *"Record checkpoints"* ]] && echo "true" || echo "false")
- |
if [[ "$FUNCTIONAL_TEST_SCOPE" == "release" || "$FUNCTIONAL_TEST_SCOPE" == "pre-release" ]]; then
FUNCTIONAL_TEST_NAME=$(eval echo $FUNCTIONAL_TEST_NAME)
Expand All @@ -58,6 +60,7 @@ functional:configure:
"--container-image ${UTILITY_IMAGE}"
"--container-tag ${CI_PIPELINE_ID}"
"--dependent-job functional:configure"
"--record-checkpoints ${RECORD_CHECKPOINTS}"
)
- |
export PYTHONPATH=$(pwd)
Expand Down
2 changes: 1 addition & 1 deletion tests/functional_tests/python_test_utils/test_regular_pipeline.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ def test_regular_pipeline(
],
"num-zeros": [
common.DeterministicTest(),
common.ApproximateTest(atol_func=common.approximate_threshold(rtol=0.15), rtol=0),
common.ApproximateTest(atol_func=common.approximate_threshold(rtol=0.20), rtol=0),
],
}

Expand Down
2 changes: 1 addition & 1 deletion tests/functional_tests/python_test_utils/test_resume_checkpoint_pipeline.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def test_resume_checkpoint_pipeline(
],
"num-zeros": [
common.DeterministicTest(),
common.ApproximateTest(atol_func=common.approximate_threshold(rtol=0.15), rtol=0),
common.ApproximateTest(atol_func=common.approximate_threshold(rtol=0.20), rtol=0),
],
}

Expand Down
5 changes: 3 additions & 2 deletions tests/functional_tests/shell_test_utils/_run_training.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ MANDATORY_VARS=(
"TRAINING_PARAMS_PATH"
"OUTPUT_PATH"
"TENSORBOARD_PATH"
"CHECKPOINT_PATH"
"CHECKPOINT_SAVE_PATH"
"CHECKPOINT_LOAD_PATH"
"DATA_PATH"
"RUN_NUMBER"
"REPEAT"
Expand All @@ -40,7 +41,7 @@ done

# Envsubst model_params
cat $TRAINING_PARAMS_PATH | envsubst "$(env | cut -d= -f1 | sed -e 's/^/$/')" >$TRAINING_PARAMS_PATH.tmp
mv $TRAINING_PARAMS_PATH.tmp "$TRAINING_PARAMS_PATH"
TRAINING_PARAMS_PATH="$TRAINING_PARAMS_PATH.tmp"

# Pull env vars to export
ENV_VARS=$(yq '... comments="" | .ENV_VARS | to_entries | .[] | [.key + "=" + .value] | join(" ")' "$TRAINING_PARAMS_PATH")
Expand Down
54 changes: 46 additions & 8 deletions tests/functional_tests/shell_test_utils/run_ci_test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ MANDATORY_VARS=(
"GOLDEN_VALUES_PATH"
"OUTPUT_PATH"
"TENSORBOARD_PATH"
"CHECKPOINT_PATH"
"CHECKPOINT_SAVE_PATH"
"CHECKPOINT_LOAD_PATH"
"DATA_PATH"
"DATA_CACHE_PATH"
)
Expand All @@ -32,6 +33,13 @@ for mandatory_var in "${MANDATORY_VARS[@]}"; do
fi
done

TEST_TYPES=("regular" "ckpt-resume" "frozen-resume" "release")

mkdir -p $CHECKPOINT_SAVE_PATH
mkdir -p $CHECKPOINT_LOAD_PATH
_CHECKPOINT_LOAD_PATH=$CHECKPOINT_LOAD_PATH
_CHECKPOINT_SAVE_PATH=$CHECKPOINT_SAVE_PATH

SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
ROOT_DIR=$(realpath $SCRIPT_DIR/../../../)

Expand All @@ -47,23 +55,52 @@ TRAIN_ITERS=$(cat $TRAINING_PARAMS_PATH |

for i in $(seq 1 $N_REPEAT); do
if [[ $i -gt 1 ]]; then
rm -rf $CHECKPOINT_PATH/*
rm -rf $CHECKPOINT_SAVE_PATH/*
rm -rf /tmp/checkpoints/*
rm -rf $TENSORBOARD_PATH/*
fi

# Training
# First run never loads from a checkpoint
export RUN_NUMBER=1
export REPEAT=$i
export CHECKPOINT_SAVE_PATH=$_CHECKPOINT_SAVE_PATH
export CHECKPOINT_LOAD_PATH=/tmp/checkpoints/

bash $ROOT_DIR/tests/functional_tests/shell_test_utils/_run_training.sh

# Maybe checkpoint resume training
if [[ "$TEST_TYPE" = "frozen-resume" && -z "$(ls -A "$_CHECKPOINT_LOAD_PATH" 2>/dev/null)" ]]; then
echo "No frozen checkpoint found. Will skip second run."

export CHECKPOINT_SAVE_PATH=$_CHECKPOINT_SAVE_PATH
rm -rf "$CHECKPOINT_SAVE_PATH/iter_0000$TRAIN_ITERS"
echo $((TRAIN_ITERS / 2)) >$CHECKPOINT_SAVE_PATH/latest_checkpointed_iteration.txt
break
fi

if [[ "$TEST_TYPE" == "ckpt-resume" ]]; then
rm -rf "$CHECKPOINT_PATH/iter_0000$TRAIN_ITERS"
echo $((TRAIN_ITERS / 2)) >$CHECKPOINT_PATH/latest_checkpointed_iteration.txt
export CHECKPOINT_LOAD_PATH=$CHECKPOINT_SAVE_PATH

rm -rf "$CHECKPOINT_LOAD_PATH/iter_0000$TRAIN_ITERS"
echo $((TRAIN_ITERS / 2)) >$CHECKPOINT_LOAD_PATH/latest_checkpointed_iteration.txt

export RUN_NUMBER=2
bash $ROOT_DIR/tests/functional_tests/shell_test_utils/_run_training.sh
fi

if [[ "$TEST_TYPE" == "frozen-resume" ]]; then

# Checkpoint-resume tests load from prev run
export CHECKPOINT_LOAD_PATH=$_CHECKPOINT_LOAD_PATH
export CHECKPOINT_SAVE_PATH=/tmp/checkpoints/

export RUN_NUMBER=2
bash $ROOT_DIR/tests/functional_tests/shell_test_utils/_run_training.sh

export CHECKPOINT_SAVE_PATH=$_CHECKPOINT_SAVE_PATH
rm -rf "$CHECKPOINT_SAVE_PATH/iter_0000$TRAIN_ITERS"
echo $((TRAIN_ITERS / 2)) >$CHECKPOINT_SAVE_PATH/latest_checkpointed_iteration.txt
fi

# Save run results
export PYTHONPATH=$ROOT_DIR
if [[ "$TEST_TYPE" == "release" ]]; then
Expand Down Expand Up @@ -93,15 +130,16 @@ for i in $(seq 1 $N_REPEAT); do
--model-config-path ${TRAINING_PARAMS_PATH} \
$ALLOW_NONDETERMINISTIC_ALGO_ARG

if [[ "$TEST_TYPE" == "ckpt-resume" ]]; then
if [[ "$TEST_TYPE" == "ckpt-resume" || "$TEST_TYPE" == "frozen-resume" ]]; then
echo "Running pytest 1st vs 2nd run comparison"
pytest -s -o log_cli=true --log-cli-level=info $ROOT_DIR/tests/functional_tests/python_test_utils/test_resume_checkpoint_pipeline.py \
--tensorboard-path $TENSORBOARD_PATH \
--train-iters $TRAIN_ITERS \
--model-config-path ${TRAINING_PARAMS_PATH} \
$ALLOW_NONDETERMINISTIC_ALGO_ARG
fi

else
if [[ ! " ${TEST_TYPES[*]} " =~ " ${TEST_TYPE} " ]]; then
echo "Test type $TEST_TYPE not yet implemented."
fi
fi
Expand Down
4 changes: 2 additions & 2 deletions tests/functional_tests/test_cases/bert/bert_mr_mcore_tp2_pp2_dgx_a100_1N8G/model_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ MODEL_ARGS:
--train-iters: 50
--timing-log-level: 2
--lr-decay-iters: 990000
--save: ${CHECKPOINT_PATH}
--load: ${CHECKPOINT_PATH}
--save: ${CHECKPOINT_SAVE_PATH}
--load: ${CHECKPOINT_LOAD_PATH}
--data-path: ${DATA_PATH}/my-bert_00_text_sentence
--vocab-file: ${DATA_PATH}/vocab.txt
--split: 949,50,1
Expand Down
1 change: 1 addition & 0 deletions .../bert/bert_mr_mcore_tp2_pp2_frozen_resume_torch_dist_dgx_a100_1N8G/golden_values_dev.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.49569, "5": 10.48402, "10": 10.49933, "15": 10.46635, "20": 10.44782, "25": 10.34968, "30": 10.17276, "35": 10.04265, "40": 9.90757, "45": 9.75784, "50": 9.67694, "55": 9.55383, "60": 9.45452, "65": 9.42152, "70": 9.30114, "75": 9.3222, "80": 9.26181, "85": 9.2967, "90": 9.23351, "95": 9.23792, "100": 9.10613}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2254.0, "5": 2635.0, "10": 2165.0, "15": 2534.0, "20": 2227.0, "25": 2559.0, "30": 2905.0, "35": 3026.0, "40": 2314.0, "45": 3924.0, "50": 3557.0, "55": 3573.0, "60": 2689.0, "65": 3434.0, "70": 3935.0, "75": 5047.0, "80": 3601.0, "85": 4133.0, "90": 4603.0, "95": 4291.0, "100": 3165.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1754654208.0, "5": 1754654208.0, "10": 1754654208.0, "15": 1754654208.0, "20": 1754654208.0, "25": 1754654208.0, "30": 1754654208.0, "35": 1754654208.0, "40": 1754654208.0, "45": 1754654208.0, "50": 1754654208.0, "55": 1754654208.0, "60": 1754654208.0, "65": 1754654208.0, "70": 1754654208.0, "75": 1754654208.0, "80": 1754654208.0, "85": 1754654208.0, "90": 1754654208.0, "95": 1754654208.0, "100": 1754654208.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.27751, "5": 2.21085, "10": 1.21719, "15": 1.18434, "20": 1.21325, "25": 1.19624, "30": 1.45908, "35": 2.35681, "40": 1.19028, "45": 1.1919, "50": 2.30834, "55": 1.18751, "60": 1.20527, "65": 1.19149, "70": 1.20169, "75": 1.19, "80": 1.19246, "85": 1.1936, "90": 1.19995, "95": 2.10846, "100": 1.18558}}}
1 change: 1 addition & 0 deletions .../bert/bert_mr_mcore_tp2_pp2_frozen_resume_torch_dist_dgx_a100_1N8G/golden_values_lts.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.49574, "5": 10.48398, "10": 10.49943, "15": 10.4663, "20": 10.44775, "25": 10.34964, "30": 10.1728, "35": 10.04262, "40": 9.90767, "45": 9.75792, "50": 9.67684, "55": 9.55378, "60": 9.45458, "65": 9.42133, "70": 9.30109, "75": 9.32203, "80": 9.26184, "85": 9.29667, "90": 9.23332, "95": 9.23793, "100": 9.10611}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2182.0, "5": 2568.0, "10": 2108.0, "15": 2533.0, "20": 2151.0, "25": 2601.0, "30": 2801.0, "35": 3107.0, "40": 2294.0, "45": 3909.0, "50": 3482.0, "55": 3606.0, "60": 2653.0, "65": 3341.0, "70": 3849.0, "75": 5090.0, "80": 3613.0, "85": 4194.0, "90": 4618.0, "95": 4439.0, "100": 3224.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1754654208.0, "5": 1754654208.0, "10": 1754654208.0, "15": 1754654208.0, "20": 1754654208.0, "25": 1754654208.0, "30": 1754654208.0, "35": 1754654208.0, "40": 1754654208.0, "45": 1754654208.0, "50": 1754654208.0, "55": 1754654208.0, "60": 1754654208.0, "65": 1754654208.0, "70": 1754654208.0, "75": 1754654208.0, "80": 1754654208.0, "85": 1754654208.0, "90": 1754654208.0, "95": 1754654208.0, "100": 1754654208.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 12.95742, "5": 1.16734, "10": 2.45473, "15": 1.45839, "20": 1.51474, "25": 1.15989, "30": 1.14801, "35": 1.14584, "40": 1.15517, "45": 1.14468, "50": 1.14969, "55": 1.15684, "60": 1.14892, "65": 1.14737, "70": 1.30233, "75": 1.37176, "80": 1.1466, "85": 1.24468, "90": 1.15157, "95": 1.15026, "100": 1.15254}}}
46 changes: 46 additions & 0 deletions ...cases/bert/bert_mr_mcore_tp2_pp2_frozen_resume_torch_dist_dgx_a100_1N8G/model_config.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
ENV_VARS:
CUDA_DEVICE_MAX_CONNECTIONS: 1
NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
NCCL_ALGO: Tree
CUBLAS_WORKSPACE_CONFIG: :4096:8
MODEL_ARGS:
--num-layers: 24
--hidden-size: 1024
--num-attention-heads: 16
--log-params-norm: true
--log-num-zeros-in-grad: true
--log-validation-ppl-to-tensorboard: true
--log-timers-to-tensorboard: true
--log-memory-to-tensorboard: true
--tensorboard-dir: ${TENSORBOARD_PATH}
--micro-batch-size: 4
--global-batch-size: 128
--seq-length: 512
--max-position-embeddings: 512
--train-iters: 100
--timing-log-level: 2
--lr-decay-iters: 990000
--save: ${CHECKPOINT_SAVE_PATH}
--load: ${CHECKPOINT_LOAD_PATH}
--data-path: ${DATA_PATH}/my-bert_00_text_sentence
--vocab-file: ${DATA_PATH}/vocab.txt
--split: 949,50,1
--distributed-backend: nccl
--lr: 0.0001
--min-lr: 0.00001
--lr-warmup-fraction: 0.01
--log-interval: 1
--save-interval: 50
--eval-interval: 1000
--eval-iters: 10
--tensor-model-parallel-size: 2
--pipeline-model-parallel-size: 2
--deterministic-mode: true
--use-checkpoint-args: true
--use-checkpoint-opt_param-scheduler: true
--no-gradient-accumulation-fusion: true
--data-cache-path: ${DATA_CACHE_PATH}
--bf16: true
--ckpt-format: torch
--attention-backend: unfused
TEST_TYPE: frozen-resume
4 changes: 2 additions & 2 deletions ...al_tests/test_cases/bert/bert_mr_mcore_tp2_pp2_local_spec_dgx_a100_1N8G/model_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ MODEL_ARGS:
--train-iters: 50
--timing-log-level: 2
--lr-decay-iters: 990000
--save: ${CHECKPOINT_PATH}
--load: ${CHECKPOINT_PATH}
--save: ${CHECKPOINT_SAVE_PATH}
--load: ${CHECKPOINT_LOAD_PATH}
--data-path: ${DATA_PATH}/my-bert_00_text_sentence
--vocab-file: ${DATA_PATH}/vocab.txt
--split: 949,50,1
Expand Down
4 changes: 2 additions & 2 deletions ...s/test_cases/bert/bert_mr_mcore_tp2_pp2_resume_torch_dist_dgx_a100_1N8G/model_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ MODEL_ARGS:
--train-iters: 100
--timing-log-level: 2
--lr-decay-iters: 990000
--save: ${CHECKPOINT_PATH}
--load: ${CHECKPOINT_PATH}
--save: ${CHECKPOINT_SAVE_PATH}
--load: ${CHECKPOINT_LOAD_PATH}
--data-path: ${DATA_PATH}/my-bert_00_text_sentence
--vocab-file: ${DATA_PATH}/vocab.txt
--split: 949,50,1
Expand Down
4 changes: 2 additions & 2 deletions ...s/bert/bert_mr_mcore_tp2_pp2_resume_torch_dist_local_spec_dgx_a100_1N8G/model_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ MODEL_ARGS:
--train-iters: 100
--timing-log-level: 2
--lr-decay-iters: 990000
--save: ${CHECKPOINT_PATH}
--load: ${CHECKPOINT_PATH}
--save: ${CHECKPOINT_SAVE_PATH}
--load: ${CHECKPOINT_LOAD_PATH}
--data-path: ${DATA_PATH}/my-bert_00_text_sentence
--vocab-file: ${DATA_PATH}/vocab.txt
--split: 949,50,1
Expand Down
4 changes: 2 additions & 2 deletions tests/functional_tests/test_cases/bert/bert_mr_tp1_pp4_vp2_dgx_a100_1N8G/model_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ MODEL_ARGS:
--train-iters: 50
--timing-log-level: 2
--lr-decay-iters: 990000
--save: ${CHECKPOINT_PATH}
--load: ${CHECKPOINT_PATH}
--save: ${CHECKPOINT_SAVE_PATH}
--load: ${CHECKPOINT_LOAD_PATH}
--data-path: ${DATA_PATH}/my-bert_00_text_sentence
--vocab-file: ${DATA_PATH}/vocab.txt
--split: 949,50,1
Expand Down
4 changes: 2 additions & 2 deletions ...al_tests/test_cases/bert/bert_mr_tp1_pp4_vp2_resume_torch_dgx_a100_1N8G/model_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ MODEL_ARGS:
--train-iters: 100
--timing-log-level: 2
--lr-decay-iters: 990000
--save: ${CHECKPOINT_PATH}
--load: ${CHECKPOINT_PATH}
--save: ${CHECKPOINT_SAVE_PATH}
--load: ${CHECKPOINT_LOAD_PATH}
--data-path: ${DATA_PATH}/my-bert_00_text_sentence
--vocab-file: ${DATA_PATH}/vocab.txt
--split: 949,50,1
Expand Down
4 changes: 2 additions & 2 deletions tests/functional_tests/test_cases/bert/bert_mr_tp2_pp2_dgx_a100_1N8G/model_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ MODEL_ARGS:
--train-iters: 50
--timing-log-level: 2
--lr-decay-iters: 990000
--save: ${CHECKPOINT_PATH}
--load: ${CHECKPOINT_PATH}
--save: ${CHECKPOINT_SAVE_PATH}
--load: ${CHECKPOINT_LOAD_PATH}
--data-path: ${DATA_PATH}/my-bert_00_text_sentence
--vocab-file: ${DATA_PATH}/vocab.txt
--split: 949,50,1
Expand Down
4 changes: 2 additions & 2 deletions ...tional_tests/test_cases/bert/bert_mr_tp2_pp2_resume_torch_dgx_a100_1N8G/model_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ MODEL_ARGS:
--train-iters: 100
--timing-log-level: 2
--lr-decay-iters: 990000
--save: ${CHECKPOINT_PATH}
--load: ${CHECKPOINT_PATH}
--save: ${CHECKPOINT_SAVE_PATH}
--load: ${CHECKPOINT_LOAD_PATH}
--data-path: ${DATA_PATH}/my-bert_00_text_sentence
--vocab-file: ${DATA_PATH}/vocab.txt
--split: 949,50,1
Expand Down
6 changes: 3 additions & 3 deletions ...nctional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_mcore_tp1_pp2/model_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ MODEL_ARGS:
--train-iters: 50
--timing-log-level: 2
--lr-decay-iters: 990000
--save: ${CHECKPOINT_PATH}
--load: ${CHECKPOINT_PATH}
--save: ${CHECKPOINT_SAVE_PATH}
--load: ${CHECKPOINT_LOAD_PATH}
--data-path: ${DATA_PATH}/my-bert_00_text_sentence
--vocab-file: ${DATA_PATH}/vocab.txt
--split: 949,50,1
Expand All @@ -41,4 +41,4 @@ MODEL_ARGS:
--bf16: true
--ckpt-format: torch
--attention-backend: unfused
TEST_TYPE: regular
TEST_TYPE: regular
6 changes: 3 additions & 3 deletions ...onal_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_mcore_tp1_pp4_vp2/model_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ MODEL_ARGS:
--train-iters: 50
--timing-log-level: 2
--lr-decay-iters: 990000
--save: ${CHECKPOINT_PATH}
--load: ${CHECKPOINT_PATH}
--save: ${CHECKPOINT_SAVE_PATH}
--load: ${CHECKPOINT_LOAD_PATH}
--data-path: ${DATA_PATH}/my-bert_00_text_sentence
--vocab-file: ${DATA_PATH}/vocab.txt
--split: 949,50,1
Expand All @@ -42,4 +42,4 @@ MODEL_ARGS:
--bf16: true
--ckpt-format: torch
--attention-backend: unfused
TEST_TYPE: regular
TEST_TYPE: regular
6 changes: 3 additions & 3 deletions ...nctional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_mcore_tp4_pp1/model_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ MODEL_ARGS:
--train-iters: 50
--timing-log-level: 2
--lr-decay-iters: 990000
--save: ${CHECKPOINT_PATH}
--load: ${CHECKPOINT_PATH}
--save: ${CHECKPOINT_SAVE_PATH}
--load: ${CHECKPOINT_LOAD_PATH}
--data-path: ${DATA_PATH}/my-bert_00_text_sentence
--vocab-file: ${DATA_PATH}/vocab.txt
--split: 949,50,1
Expand All @@ -41,4 +41,4 @@ MODEL_ARGS:
--bf16: true
--ckpt-format: torch
--attention-backend: unfused
TEST_TYPE: regular
TEST_TYPE: regular
6 changes: 3 additions & 3 deletions tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_tp1_pp2/model_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ MODEL_ARGS:
--train-iters: 50
--timing-log-level: 2
--lr-decay-iters: 990000
--save: ${CHECKPOINT_PATH}
--load: ${CHECKPOINT_PATH}
--save: ${CHECKPOINT_SAVE_PATH}
--load: ${CHECKPOINT_LOAD_PATH}
--data-path: ${DATA_PATH}/my-bert_00_text_sentence
--vocab-file: ${DATA_PATH}/vocab.txt
--split: 949,50,1
Expand All @@ -44,4 +44,4 @@ MODEL_ARGS:
--apply-query-key-layer-scaling: true
--ckpt-format: torch
--attention-backend: unfused
TEST_TYPE: regular
TEST_TYPE: regular
6 changes: 3 additions & 3 deletions tests/functional_tests/test_cases/bert/bert_nightly_dgx_a100_1N8G_tp4_pp1/model_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ MODEL_ARGS:
--train-iters: 50
--timing-log-level: 2
--lr-decay-iters: 990000
--save: ${CHECKPOINT_PATH}
--load: ${CHECKPOINT_PATH}
--save: ${CHECKPOINT_SAVE_PATH}
--load: ${CHECKPOINT_LOAD_PATH}
--data-path: ${DATA_PATH}/my-bert_00_text_sentence
--vocab-file: ${DATA_PATH}/vocab.txt
--split: 949,50,1
Expand All @@ -44,4 +44,4 @@ MODEL_ARGS:
--apply-query-key-layer-scaling: true
--ckpt-format: torch
--attention-backend: unfused
TEST_TYPE: regular
TEST_TYPE: regular
4 changes: 2 additions & 2 deletions ...a100_1N8G_mcore_tp1_pp1_dist_optimizer_overlap_grad_reduce_param_gather/model_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ MODEL_ARGS:
--train-iters: 50
--timing-log-level: 2
--lr-decay-iters: 320000
--save: ${CHECKPOINT_PATH}
--load: ${CHECKPOINT_PATH}
--save: ${CHECKPOINT_SAVE_PATH}
--load: ${CHECKPOINT_LOAD_PATH}
--data-path: ${DATA_PATH}/my-gpt3_00_text_document
--vocab-file: ${DATA_PATH}/bpe/vocab.json
--merge-file: ${DATA_PATH}/bpe/merges.txt
Expand Down
Loading

0 comments on commit b41bcba

Please sign in to comment.