diff --git a/.jenkins/lm-eval-harness/test_lm_eval_correctness.py b/.jenkins/lm-eval-harness/test_lm_eval_correctness.py
index 9c6d0ee48caf..421a949ab72e 100644
--- a/.jenkins/lm-eval-harness/test_lm_eval_correctness.py
+++ b/.jenkins/lm-eval-harness/test_lm_eval_correctness.py
@@ -19,7 +19,7 @@
 
 import vllm
 
-RTOL = 0.05
+RTOL = 0.06
 TEST_DATA_FILE = os.environ.get(
     "LM_EVAL_TEST_DATA_FILE",
     ".jenkins/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct.yaml")
diff --git a/.jenkins/test_config.yaml b/.jenkins/test_config.yaml
index 99ff97df8cd3..f90cdb354d4f 100644
--- a/.jenkins/test_config.yaml
+++ b/.jenkins/test_config.yaml
@@ -19,6 +19,6 @@ stages:
       - name: gsm8k_large_g3_tp2
         flavor: g3.s
         command: cd .jenkins/lm-eval-harness && bash run-tests.sh -c configs/models-large.txt -t 2
-      - name: gsm8k_large_g2_tp2
-        flavor: g2.s
-        command: cd .jenkins/lm-eval-harness && bash run-tests.sh -c configs/models-large.txt -t 2
+      - name: gsm8k_large_g2_tp4
+        flavor: g2.m
+        command: cd .jenkins/lm-eval-harness && bash run-tests.sh -c configs/models-large.txt -t 4