Merge branch 'ko3n1g/tests/repeats' into 'main'

tests: Allow running tests multiple times See merge request ADLR/megatron-lm!1941
NVIDIA · Aug 19, 2024 · 3bb2585 · 3bb2585
2 parents fb155f4 + 49af43e
commit 3bb2585
Show file tree

Hide file tree

Showing 44 changed files with 86 additions and 42 deletions.
diff --git a/tests/functional_tests/jet_recipes/bert.yaml b/tests/functional_tests/jet_recipes/bert.yaml
@@ -9,8 +9,6 @@ spec:
   nodes: 1
   gpus: 8
   platforms: dgx_a100
-  time_limit: 1200
-  scope: null
   artifacts: 
     /workspace/data/bert_data: text/the_pile/bert_shard00
   script: |-
@@ -32,6 +30,7 @@ spec:
 
 products:
   - scope: [mr]
+    time_limit: [1200]
     test_case: 
     - bert_mr_mcore_tp2_pp2_dgx_a100_1N8G
     - bert_mr_mcore_tp2_pp2_local_spec_dgx_a100_1N8G
@@ -42,6 +41,7 @@ products:
     - bert_mr_tp2_pp2_dgx_a100_1N8G
     - bert_mr_tp2_pp2_resume_torch_dgx_a100_1N8G
   - scope: [nightly]
+    time_limit: [12000]
     test_case:
     - bert_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2
     - bert_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp4_vp2

diff --git a/tests/functional_tests/jet_recipes/gpt.yaml b/tests/functional_tests/jet_recipes/gpt.yaml
@@ -105,7 +105,7 @@ products:
     - gpt3_mr_tp2_pp2_dgx_a100_1N8G
   - scope: [nightly]
     platforms: [dgx_a100]
-    time_limit: [1200]
+    time_limit: [12000]
     test_case:
     - gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp1_dist_optimizer_overlap_grad_reduce_param_gather
     - gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather

diff --git a/tests/functional_tests/shell_test_utils/run_ci_test.sh b/tests/functional_tests/shell_test_utils/run_ci_test.sh
@@ -34,45 +34,55 @@ done
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 ROOT_DIR=$(realpath $SCRIPT_DIR/../../../)
 
-# Training
-bash $ROOT_DIR/tests/functional_tests/shell_test_utils/_run_training.sh
-
 # Extract settings from params file
 TEST_TYPE=$(cat $TRAINING_PARAMS_PATH \
             | yq '.TEST_TYPE')
 NVTE_ALLOW_NONDETERMINISTIC_ALGO=$(cat $TRAINING_PARAMS_PATH \
                                    | yq '.ENV_VARS.NVTE_ALLOW_NONDETERMINISTIC_ALGO')
 SKIP_PYTEST=$(cat $TRAINING_PARAMS_PATH \
               | yq '.ENV_VARS.SKIP_PYTEST')
+N_REPEATS=$(cat $TRAINING_PARAMS_PATH \
+              | yq '.ENV_VARS.N_REPEATS //1')
+
+for i in $(seq 1 $N_REPEATS);
+do
+    rm -rf $CHECKPOINT_PATH/*
+    rm -rf $OUTPUT_PATH/*
 
-# Maybe checkpoint resume training
-if [[ "$TEST_TYPE" == "ckpt-resume" ]]; then 
-    rm -rf $CHECKPOINT_PATH/iter_0000100; 
-    echo 50 > $CHECKPOINT_PATH/latest_checkpointed_iteration.txt;
+    # Training
     bash $ROOT_DIR/tests/functional_tests/shell_test_utils/_run_training.sh
-fi
-
-# Save run results
-export PYTHONPATH=$ROOT_DIR
-python3 $ROOT_DIR/tests/functional_tests/python_test_utils/get_test_results_from_tensorboard_logs.py \
-    --logs-dir $TENSORBOARD_PATH \
-    --output-path ${OUTPUT_PATH}/$(basename $GOLDEN_VALUES_PATH)
-
-# Maybe run tests
-if [[ ${SKIP_PYTEST:-0} != 1 ]]; then
-    export NVTE_ALLOW_NONDETERMINISTIC_ALGO
-    export LOGS_DIR=$TENSORBOARD_PATH
-
-    if [[ "$TEST_TYPE" == "ckpt-resume" ]]; then
-        echo "Running pytest 1st vs 2nd run comparison"
-        pytest -s $ROOT_DIR/tests/functional_tests/python_test_utils/test_resume_checkpoint_pipeline.py
-
-    elif [[ "$TEST_TYPE" == "regular" ]]; then
-        echo "Running pytest checks against golden values"
-        export EXPECTED_METRICS_FILE=$GOLDEN_VALUES_PATH 
-        pytest -s $ROOT_DIR/tests/functional_tests/python_test_utils/test_ci_pipeline.py
-
-    else
-        echo "Test type $TEST_TYPE not yet implemented."
+
+    # Maybe checkpoint resume training
+    if [[ "$TEST_TYPE" == "ckpt-resume" ]]; then 
+        rm -rf $CHECKPOINT_PATH/iter_0000100; 
+        echo 50 > $CHECKPOINT_PATH/latest_checkpointed_iteration.txt;
+        bash $ROOT_DIR/tests/functional_tests/shell_test_utils/_run_training.sh
     fi
-fi
+
+    # Save run results
+    export PYTHONPATH=$ROOT_DIR
+    python3 $ROOT_DIR/tests/functional_tests/python_test_utils/get_test_results_from_tensorboard_logs.py \
+        --logs-dir $TENSORBOARD_PATH \
+        --output-path ${OUTPUT_PATH}/$(basename $GOLDEN_VALUES_PATH)
+
+    # Maybe run tests
+    if [[ ${SKIP_PYTEST:-0} != 1 ]]; then
+        export NVTE_ALLOW_NONDETERMINISTIC_ALGO
+        export LOGS_DIR=$TENSORBOARD_PATH
+
+        if [[ "$TEST_TYPE" == "ckpt-resume" ]]; then
+            echo "Running pytest 1st vs 2nd run comparison"
+            pytest -s $ROOT_DIR/tests/functional_tests/python_test_utils/test_resume_checkpoint_pipeline.py
+
+        elif [[ "$TEST_TYPE" == "regular" ]]; then
+            echo "Running pytest checks against golden values"
+            export EXPECTED_METRICS_FILE=$GOLDEN_VALUES_PATH 
+            pytest -s $ROOT_DIR/tests/functional_tests/python_test_utils/test_ci_pipeline.py
+
+        else
+            echo "Test type $TEST_TYPE not yet implemented."
+        fi
+    fi
+done
+
+
diff --git a/...nal_tests/test_cases/bert/bert_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2/model_config.yaml b/...nal_tests/test_cases/bert/bert_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 24
   --hidden-size: 1024

diff --git a/...tests/test_cases/bert/bert_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp4_vp2/model_config.yaml b/...tests/test_cases/bert/bert_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp4_vp2/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 24
   --hidden-size: 1024

diff --git a/...nal_tests/test_cases/bert/bert_345m_nightly_dgx_a100_1N8G_mcore_tp4_pp1/model_config.yaml b/...nal_tests/test_cases/bert/bert_345m_nightly_dgx_a100_1N8G_mcore_tp4_pp1/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 24
   --hidden-size: 1024

diff --git a/...unctional_tests/test_cases/bert/bert_345m_nightly_dgx_a100_1N8G_tp1_pp2/model_config.yaml b/...unctional_tests/test_cases/bert/bert_345m_nightly_dgx_a100_1N8G_tp1_pp2/model_config.yaml
@@ -4,6 +4,7 @@ ENV_VARS:
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
   NVTE_APPLY_QK_LAYER_SCALING: 1
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 24
   --hidden-size: 1024

diff --git a/...unctional_tests/test_cases/bert/bert_345m_nightly_dgx_a100_1N8G_tp4_pp1/model_config.yaml b/...unctional_tests/test_cases/bert/bert_345m_nightly_dgx_a100_1N8G_tp4_pp1/model_config.yaml
@@ -4,6 +4,7 @@ ENV_VARS:
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
   NVTE_APPLY_QK_LAYER_SCALING: 1
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 24
   --hidden-size: 1024

diff --git a/...a100_1N8G_mcore_tp1_pp1_dist_optimizer_overlap_grad_reduce_param_gather/model_config.yaml b/...a100_1N8G_mcore_tp1_pp1_dist_optimizer_overlap_grad_reduce_param_gather/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/model_config.yaml b/...1_pp1_resume_torch_dist_dist_optimizer_overlap_grad_reduce_param_gather/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...onal_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2/model_config.yaml b/...onal_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2_fp16/model_config.yaml b/...tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2_fp16/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...ses/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2_resume_torch_dist/model_config.yaml b/...ses/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp2_resume_torch_dist/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...onal_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp4/model_config.yaml b/...onal_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp4/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...ses/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp4_resume_torch_dist/model_config.yaml b/...ses/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp1_pp4_resume_torch_dist/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/..._dgx_a100_1N8G_mcore_tp2_pp2_ep2_resume_torch_dist_te_4experts2parallel/model_config.yaml b/..._dgx_a100_1N8G_mcore_tp2_pp2_ep2_resume_torch_dist_te_4experts2parallel/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/.../gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp2_pp2_ep2_te_4experts2parallel/model_config.yaml b/.../gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp2_pp2_ep2_te_4experts2parallel/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/..._345m_nightly_dgx_a100_1N8G_mcore_tp2_pp2_resume_torch_dist_te_2experts/model_config.yaml b/..._345m_nightly_dgx_a100_1N8G_mcore_tp2_pp2_resume_torch_dist_te_2experts/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...est_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp2_pp2_te_2experts/model_config.yaml b/...est_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp2_pp2_te_2experts/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...onal_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp4_pp1/model_config.yaml b/...onal_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp4_pp1/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...st_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp4_pp1_resume_torch/model_config.yaml b/...st_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp4_pp1_resume_torch/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...ses/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp4_pp1_resume_torch_dist/model_config.yaml b/...ses/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp4_pp1_resume_torch_dist/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...3_345m_nightly_dgx_a100_1N8G_tp1_pp1_dist_optimizer_overlap_grad_reduce/model_config.yaml b/...3_345m_nightly_dgx_a100_1N8G_tp1_pp1_dist_optimizer_overlap_grad_reduce/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...t_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp1_overlap_grad_reduce/model_config.yaml b/...t_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp1_overlap_grad_reduce/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp2/model_config.yaml b/...functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp2/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...sts/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp2_resume_torch/model_config.yaml b/...sts/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp2_resume_torch/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4/model_config.yaml b/...functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...t_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4_overlap_grad_reduce/model_config.yaml b/...t_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4_overlap_grad_reduce/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...sts/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4_resume_torch/model_config.yaml b/...sts/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4_resume_torch/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...ses/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4_vp1_overlap_grad_reduce/model_config.yaml b/...ses/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp1_pp4_vp1_overlap_grad_reduce/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...l_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_4experts/model_config.yaml b/...l_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_4experts/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...t_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_overlap_grad_reduce/model_config.yaml b/...t_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_overlap_grad_reduce/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_resume_torch_4experts/model_config.yaml b/...cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_resume_torch_4experts/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...pt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_resume_torch_overlap_grad_reduce/model_config.yaml b/...pt3_345m_nightly_dgx_a100_1N8G_tp2_pp2_resume_torch_overlap_grad_reduce/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp4_pp1/model_config.yaml b/...functional_tests/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp4_pp1/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...t_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp4_pp1_overlap_grad_reduce/model_config.yaml b/...t_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp4_pp1_overlap_grad_reduce/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512

diff --git a/...sts/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp4_pp1_resume_torch/model_config.yaml b/...sts/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_tp4_pp1_resume_torch/model_config.yaml
@@ -3,6 +3,7 @@ ENV_VARS:
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
+  N_REPEATS: 10
 MODEL_ARGS:
   --num-layers: 12
   --hidden-size: 512