Upstream sync

.gitlab-ci.yml

@@ -13,22 +13,28 @@ workflow:
         FUNCTIONAL_TEST: "no"
     - if: $CI_MERGE_REQUEST_LABELS =~ /Run tests/ && $CI_MERGE_REQUEST_TARGET_BRANCH_SHA != ""
       variables:
-        FUNCTIONAL_TEST: "yes"
-        FUNCTIONAL_TEST_SCOPE: mr
         UNIT_TEST_REPEAT: 5
         UNIT_TEST_TIMEOUT: 50
+        FUNCTIONAL_TEST: "yes"
+        FUNCTIONAL_TEST_SCOPE: mr
+        FUNCTIONAL_TEST_CLUSTER_A100: ""
+        FUNCTIONAL_TEST_CLUSTER_H100: ""
     - if: $CI_MERGE_REQUEST_LABELS =~ /Run nightly/ && $CI_MERGE_REQUEST_TARGET_BRANCH_SHA != ""
       variables:
-        FUNCTIONAL_TEST: "yes"
-        FUNCTIONAL_TEST_SCOPE: nightly
         UNIT_TEST_REPEAT: 5
         UNIT_TEST_TIMEOUT: 50
+        FUNCTIONAL_TEST: "yes"
+        FUNCTIONAL_TEST_SCOPE: nightly
+        FUNCTIONAL_TEST_CLUSTER_A100: ""
+        FUNCTIONAL_TEST_CLUSTER_H100: ""
     - if: $CI_MERGE_REQUEST_LABELS =~ /Run weekly/ && $CI_MERGE_REQUEST_TARGET_BRANCH_SHA != ""
       variables:
-        FUNCTIONAL_TEST: "yes"
-        FUNCTIONAL_TEST_SCOPE: weekly
         UNIT_TEST_REPEAT: 5
         UNIT_TEST_TIMEOUT: 50
+        FUNCTIONAL_TEST: "yes"
+        FUNCTIONAL_TEST_SCOPE: weekly
+        FUNCTIONAL_TEST_CLUSTER_A100: ""
+        FUNCTIONAL_TEST_CLUSTER_H100: ""
     - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_SHA != ""
       variables:
         FUNCTIONAL_TEST: "no"
@@ -58,29 +64,23 @@ variables:
       - "mr"
       - "nightly"
       - "weekly"
+      - "pre-release"
+      - "release"
     description: "Testsuite to run (only for FUNCTIONAL_TEST=yes)"
-  FUNCTIONAL_TEST_CLUSTER:
+  FUNCTIONAL_TEST_CLUSTER_A100:
     value: "dgxa100_dracooci"
     options:
       - "dgxa100_dracooci"
       - "dgxa100_dracooci-ord"
-      - "dgxh100_eos"
-    description: '"dgxa100_dracooci" for OCI-IAD, "dgxh100_eos" for EOS'
-  CONVERGENCE_TEST:
-    value: "no"
+    description: 'Cluster for A100 workloads'
+  FUNCTIONAL_TEST_CLUSTER_H100:
+    value: "dgxh100_eos"
     options:
-      - "yes"
-      - "no"
-    description: To run a convergence test
-  CONVERGENCE_TEST_SCOPE:
-    value: "release"
-    options:
-      - "release"
-      - "pre-release"
-    description: "Test suite to run (only for CONVERGENCE_TEST=yes)"
-  CONVERGENCE_TEST_RUN_NAME:
-    value: "pre-release-$$CI_PIPELINE_ID"
-    description: "Run directory of convergence test"
+      - "dgxh100_coreweave"
+      - "dgxh100_eos"
+    description: 'Cluster for H100 workloads'
+  FUNCTIONAL_TEST_NAME:
+    description: "Name of functional test run (only for pre-release and release)"
   PUBLISH: 
     value: "no"
     options: 
@@ -96,6 +96,7 @@ variables:
 
   # CI wide variables
   CI_MCORE_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/mcore_ci
+  CI_MCORE_DEV_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/mcore_ci_dev
   CI_NEMO_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/nemo_ci
   LINTING_IMAGE: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/mcore_linting
   UNIT_TEST_TIMEOUT: 15
@@ -105,5 +106,4 @@ include:
   - .gitlab/stages/00.pre.yml
   - .gitlab/stages/01.tests.yml
   - .gitlab/stages/02.functional-tests.yml
-  - .gitlab/stages/03.convergence-tests.yml
-  - .gitlab/stages/04.publish.yml
+  - .gitlab/stages/03.publish.yml

.gitlab/stages/00.pre.yml

@@ -1,6 +1,15 @@
 include:
   - template: Security/Secret-Detection.gitlab-ci.yml
 
+.pre_mr_rules:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_PROTECTED != "true"
+      allow_failure: true
+      when: always
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    - when: never
+  stage: .pre
+
 mirror_to_github:
   rules:
     - if: '$CI_COMMIT_REF_PROTECTED == "true" && $CI_PIPELINE_SOURCE == "push"'
@@ -35,14 +44,11 @@ create_ci_branches:
     GIT_STRATEGY: "clone"
   script:
     - git remote set-url origin "https://gitlab-ci-token:${PROJECT_ACCESS_TOKEN_MCORE}@${GITLAB_ENDPOINT}/adlr/megatron-lm.git"
-    - git switch --force-create $branch;
+    - git switch --force-create $branch
     - git push --force -u origin $branch
 
 label_merge_request:
-  rules: 
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-    - when: never
-  stage: .pre
+  extends: [.pre_mr_rules]
   image: golang:1.22
   tags:
     - mcore-docker-node-small
@@ -62,17 +68,18 @@ label_merge_request:
       curl --header "PRIVATE-TOKEN: ${PROJECT_ACCESS_TOKEN_MCORE}" --url "https://${GITLAB_ENDPOINT}/api/v4/projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}" --data-urlencode "add_labels=$LABELS" -X PUT
 
 clean_docker_node:
-  stage: .pre
+  extends: [.pre_mr_rules]
   image: docker:26.1.4-dind
   tags: 
     - ${node}
   parallel:
     matrix:
       - node: 8xL40S
       - node: mcore-docker-node-small
+      - node: mcore-docker-node-jet
   script:
     - export DOCKER_HOST='unix:///var/run/docker.sock'
-    - docker system prune -a --filter "until=48h" -f || true
+    - docker system prune -a --filter "until=36h" -f || true
 
 maybe_cherry_pick_commit:
   rules:
@@ -95,8 +102,13 @@ maybe_cherry_pick_commit:
     - git config --global user.email "mcore-bot@nvidia.com"
     - git config --global user.name "Mcore Bot"
     - |
-      LABELS=$(curl --header "PRIVATE-TOKEN: ${PROJECT_ACCESS_TOKEN_MCORE}" --url "https://${GITLAB_ENDPOINT}/api/v4/projects/${CI_PROJECT_ID}/merge_requests/${MR_ID}" | jq '.labels | join(",")' | tr -d '"')
-
+      MR=$(curl --header "PRIVATE-TOKEN: ${PROJECT_ACCESS_TOKEN_MCORE}" --url "https://${GITLAB_ENDPOINT}/api/v4/projects/${CI_PROJECT_ID}/merge_requests/${MR_ID}")
+
+      LABELS=$(echo -E $MR | jq '.labels | join(",")' | tr -d '"')
+      AUTHOR_ID=$(echo -E $MR | jq '.author.id' | tr -d '"')
+      AUTHOR_NAME=$(echo -E $MR | jq '.author.username' | tr -d '"')
+      TITLE=$(echo -E $MR | jq '.title' | tr -d '"')
+      MILESTONE_ID=$(echo -E $MR | jq '.milestone.id' | tr -d '"')
       TARGET_BRANCHES=$(echo "$LABELS" | grep -o 'core_[^,]*')
 
       if [[ $TARGET_BRANCHES == "" ]]; then
@@ -128,8 +140,11 @@ maybe_cherry_pick_commit:
             --url https://${GITLAB_ENDPOINT}/api/v4/projects/${CI_PROJECT_ID}/merge_requests \
             -d "source_branch=cherry-pick-$MR_ID-$RELEASE_BRANCH" \
             -d "target_branch=$RELEASE_BRANCH" \
-            -d "title=Cherry-pick $MR_ID into $RELEASE_BRANCH" \
-            -d "labels=cherry-pick"
+            -d "title=Cherry pick \`$TITLE ($MR_ID)\` into \`$RELEASE_BRANCH\`" \
+            -d "labels=cherry-pick" \
+            -d "reviewer_ids=$AUTHOR_ID" \
+            -d "milestone_id=$MILESTONE_ID" \
+            -d "description=[🤖]: Hi @$AUTHOR_NAME 👋,<br><br>we've cherry picked \`$TITLE ($MR_ID)\` into \`$RELEASE_BRANCH\` for you! 🚀<br><br>Please review and approve this cherry pick by your convenience\!"
 
         else
           URL=https://${GITLAB_ENDPOINT}/ADLR/megatron-lm/-/merge_requests/$MR_ID
@@ -154,10 +169,7 @@ maybe_cherry_pick_commit:
   interruptible: false
 
 check_milestone:
-  rules: 
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-    - when: never
-  stage: .pre
+  extends: [.pre_mr_rules]
   image: ${GITLAB_ENDPOINT}:5005/adlr/megatron-lm/mcore_ci:buildcache
   tags:
     - mcore-docker-node-small