Fix Phi 3.5 MoE tests (#2684)

PR #2682 also fixed in issue in Phi MoE, but it changes the test
outputs a bit. Fix this.

integration-tests/models/__snapshots__/test_flash_phi35_moe/test_flash_phi35_moe.json

@@ -11,99 +11,99 @@
       },
       {
         "id": 338,
-        "logprob": -0.7133789,
+        "logprob": -0.6201172,
         "text": "is"
       },
       {
         "id": 16030,
-        "logprob": -13.9296875,
+        "logprob": -13.6484375,
         "text": "gradient"
       },
       {
         "id": 26815,
-        "logprob": -0.048919678,
+        "logprob": -0.003894806,
         "text": "descent"
       },
       {
         "id": 29973,
-        "logprob": -3.0078125,
+        "logprob": -2.6386719,
         "text": "?"
       },
       {
         "id": 13,
-        "logprob": -2.8105469,
+        "logprob": -6.46875,
         "text": "\n"
       },
       {
         "id": 13,
-        "logprob": -0.84521484,
+        "logprob": -6.6875,
         "text": "\n"
       }
     ],
     "seed": null,
     "tokens": [
       {
         "id": 25584,
-        "logprob": -0.017028809,
+        "logprob": -0.008979797,
         "special": false,
         "text": "Grad"
       },
       {
         "id": 993,
-        "logprob": -0.0027313232,
+        "logprob": -8.34465e-07,
         "special": false,
         "text": "ient"
       },
       {
         "id": 26815,
-        "logprob": -0.023254395,
+        "logprob": -0.0009407997,
         "special": false,
         "text": " descent"
       },
       {
         "id": 338,
-        "logprob": -2.0623207e-05,
+        "logprob": -0.0003838539,
         "special": false,
         "text": " is"
       },
       {
-        "id": 263,
-        "logprob": -0.5361328,
+        "id": 385,
+        "logprob": -0.24499512,
         "special": false,
-        "text": " a"
+        "text": " an"
       },
       {
-        "id": 937,
-        "logprob": -0.17578125,
+        "id": 13883,
+        "logprob": -0.010406494,
         "special": false,
-        "text": " first"
+        "text": " optimization"
       },
       {
-        "id": 29899,
-        "logprob": 0.0,
+        "id": 5687,
+        "logprob": -0.00024354458,
         "special": false,
-        "text": "-"
+        "text": " algorithm"
       },
       {
-        "id": 2098,
-        "logprob": -0.00011539459,
+        "id": 15574,
+        "logprob": -0.6582031,
         "special": false,
-        "text": "order"
+        "text": " commonly"
       },
       {
-        "id": 13883,
-        "logprob": -0.47436523,
+        "id": 1304,
+        "logprob": -0.00092840195,
         "special": false,
-        "text": " optimization"
+        "text": " used"
       },
       {
-        "id": 5687,
-        "logprob": -0.00027680397,
+        "id": 297,
+        "logprob": -0.19470215,
         "special": false,
-        "text": " algorithm"
+        "text": " in"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "Gradient descent is a first-order optimization algorithm"
+  "generated_text": "Gradient descent is an optimization algorithm commonly used in"
 }

integration-tests/models/__snapshots__/test_flash_phi35_moe/test_flash_phi35_moe_all_params.json

@@ -5,95 +5,95 @@
     "generated_tokens": 10,
     "prefill": [
       {
-        "id": 16030,
+        "id": 338,
         "logprob": null,
+        "text": "is"
+      },
+      {
+        "id": 16030,
+        "logprob": -13.328125,
         "text": "gradient"
       },
       {
         "id": 26815,
-        "logprob": -6.4960938,
+        "logprob": -0.24023438,
         "text": "descent"
       },
       {
         "id": 29973,
-        "logprob": -5.1484375,
+        "logprob": -3.1386719,
         "text": "?"
       },
       {
         "id": 13,
-        "logprob": -4.0351562,
-        "text": "\n"
-      },
-      {
-        "id": 13,
-        "logprob": -5.2265625,
+        "logprob": -3.0878906,
         "text": "\n"
       }
     ],
     "seed": 0,
     "tokens": [
       {
-        "id": 10994,
-        "logprob": -1.1542969,
+        "id": 25584,
+        "logprob": 0.0,
         "special": false,
-        "text": "Hello"
+        "text": "Grad"
       },
       {
-        "id": 29991,
+        "id": 993,
         "logprob": 0.0,
         "special": false,
-        "text": "!"
+        "text": "ient"
       },
       {
-        "id": 739,
+        "id": 2726,
         "logprob": 0.0,
         "special": false,
-        "text": " It"
+        "text": " Des"
       },
       {
-        "id": 2444,
-        "logprob": -0.42260742,
+        "id": 1760,
+        "logprob": 0.0,
         "special": false,
-        "text": " seems"
+        "text": "cent"
       },
       {
-        "id": 366,
-        "logprob": 0.0,
+        "id": 313,
+        "logprob": -0.12322998,
         "special": false,
-        "text": " you"
+        "text": " ("
       },
       {
-        "id": 29915,
+        "id": 29954,
         "logprob": 0.0,
         "special": false,
-        "text": "'"
+        "text": "G"
       },
       {
-        "id": 276,
-        "logprob": -0.9838867,
+        "id": 29928,
+        "logprob": 0.0,
         "special": false,
-        "text": "re"
+        "text": "D"
       },
       {
-        "id": 3211,
+        "id": 29897,
         "logprob": 0.0,
         "special": false,
-        "text": " address"
+        "text": ")"
       },
       {
-        "id": 292,
-        "logprob": 0.0,
+        "id": 338,
+        "logprob": -0.6040039,
         "special": false,
-        "text": "ing"
+        "text": " is"
       },
       {
-        "id": 263,
-        "logprob": -0.15124512,
+        "id": 385,
+        "logprob": -0.1796875,
         "special": false,
-        "text": " a"
+        "text": " an"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "What is gradient descent?\n\nHello! It seems you're addressing a"
+  "generated_text": "What is gradient descent?\nGradient Descent (GD) is an"
 }