WIP: Initial implementation of Crossgen2 CI pipeline, take dotnet#3

I have refactored my Crossgen2 CI pipeline change to not use SuperIlc for framework compilation. I intend to introduce that at a later time to basically simplify the change and make it easier to review and test. It's currently based on my preparatory change "CoreCLR pipeline optimization" which also introduces the logic to Crossgen framework assemblies as part of the run-test job. Thanks Tomas
trylek · Oct 29, 2019 · caf664e · caf664e
1 parent eae780c
commit caf664e
Show file tree

Hide file tree

Showing 11 changed files with 124 additions and 154 deletions.
diff --git a/build-test.cmd b/build-test.cmd
@@ -53,6 +53,7 @@ set __SkipNative=
 set __RuntimeId=
 set __TargetsWindows=1
 set __DoCrossgen=
+set __DoCrossgen2=
 set __CopyNativeTestBinaries=0
 set __CopyNativeProjectsAfterCombinedTestBuild=true
 set __SkipGenerateLayout=0
@@ -92,6 +93,7 @@ if /i "%1" == "buildtesthostonly"     (set __SkipNative=1&set __SkipManaged=1&se
 if /i "%1" == "buildagainstpackages"  (echo error: Remove /BuildAgainstPackages switch&&exit /b1)
 if /i "%1" == "skiprestorepackages"   (set __SkipRestorePackages=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "crossgen"              (set __DoCrossgen=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
+if /i "%1" == "crossgen2"             (set __DoCrossgen2=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "runtimeid"             (set __RuntimeId=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop)
 if /i "%1" == "targetsNonWindows"     (set __TargetsWindows=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "Exclude"               (set __Exclude=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop)
@@ -526,6 +528,17 @@ if defined __DoCrossgen (
     )
 )
 
+if defined __DoCrossgen2 (
+    set __CrossgenArg="/p:Crossgen2=true"
+    if "%__BuildArch%" == "x64" (
+        echo %__MsgPrefix%Running crossgen2 on framework assemblies
+        call :PrecompileFX
+    ) else (
+        echo "%__MsgPrefix%Crossgen2 only supported on x64, for now"
+    )
+)
+
+
 rd /s /q "%CORE_ROOT_STAGE%"
 
 REM =========================================================================================
@@ -614,9 +627,22 @@ echo "%2" | findstr /b "xunit." >nul && (
 set __CrossgenExe="%CORE_ROOT_STAGE%\crossgen.exe"
 if /i "%__BuildArch%" == "arm" ( set __CrossgenExe="%CORE_ROOT_STAGE%\x86\crossgen.exe" )
 if /i "%__BuildArch%" == "arm64" ( set __CrossgenExe="%CORE_ROOT_STAGE%\x64\crossgen.exe" )
+set __CrossgenExe=%__CrossgenExe%
+
+if defined __DoCrossgen2 (
+    set __CrossgenExe="%CORE_ROOT_STATE%\crossgen2\crossgen2.exe"
+)
+
+set __CrossgenOutputFile="%CORE_ROOT%\temp.ni.dll"
+
+if defined __Crossgen (
+    "!__CrossgenExe!" /Platform_Assemblies_Paths "!CORE_ROOT!" /in "%1" /out "!__CrossgenOutputFile" >nul 2>nul
+    set /a __exitCode = !errorlevel!
+) else (
+    "!CORE_ROOT_STAGE!\crossgen2\crossgen2 -r:"!CORE_ROOT!\*.dll" -O --inputbubble -out:"__CrossgenOutputFile" "%1" >nul 2>nul
+    set /a __exitCode = !errorlevel!
+)
 
-"%__CrossgenExe%" /Platform_Assemblies_Paths "%CORE_ROOT%" /in "%1" /out "%CORE_ROOT%/temp.ni.dll" >nul 2>nul
-set /a __exitCode = %errorlevel%
 if "%__exitCode%" == "-2146230517" (
     echo %2 is not a managed assembly.
     exit /b 0
@@ -629,7 +655,7 @@ if %__exitCode% neq 0 (
 
 REM Delete original .dll & replace it with the Crossgened .dll
 del %1
-ren "%CORE_ROOT%\temp.ni.dll" %2
+ren "%__CrossgenOutputFile%" %2
 
 echo Successfully precompiled %2
 exit /b 0

diff --git a/build-test.sh b/build-test.sh
@@ -166,20 +166,26 @@ generate_layout()
     fi
 
     # Precompile framework assemblies with crossgen if required
-    if [ $__DoCrossgen -ne 0 ]; then
+    if [ $__DoCrossgen -ne 0 || $__DoCrossgen2 -ne 0 ]; then
         precompile_coreroot_fx
     fi
 }
 
 precompile_coreroot_fx()
 {
-    echo "${__MsgPrefix}Running crossgen on framework assemblies in CORE_ROOT: '${CORE_ROOT}'"
+    local overlayDir=$CORE_ROOT
+    local action = crossgen
+    local compiler = $__CrossgenExe /Platform_Assemblies_Paths $overlayDir
+
+    if [ $__DoCrossgen2 -ne 0 ]; then
+        action = crossgen2
+        compiler = ${overlayDir}/crossgen2/crossgen2 -r:${overlayDir}/*.dll -O --inputbubble
+    fi
+
+    echo "${__MsgPrefix}Running ${action} on framework assemblies in CORE_ROOT: '${CORE_ROOT}'"
 
     # Read the exclusion file for this platform
     skipCrossGenFiles=($(read_array "$(dirname "$0")/tests/skipCrossGenFiles.${__BuildArch}.txt"))
-    skipCrossGenFiles+=('System.Runtime.WindowsRuntime.dll')
-
-    local overlayDir=$CORE_ROOT
 
     filesToPrecompile=$(find -L $overlayDir -maxdepth 1 -iname \*.dll -not -iname \*.ni.dll -not -iname \*-ms-win-\* -not -iname xunit.\* -type f)
     for fileToPrecompile in ${filesToPrecompile}
@@ -189,7 +195,7 @@ precompile_coreroot_fx()
                 continue
         fi
         echo Precompiling $filename
-        $__CrossgenExe /Platform_Assemblies_Paths $overlayDir $filename 1> $filename.stdout 2>$filename.stderr
+        $compiler $filename 1> $filename.stdout 2>$filename.stderr
         local exitCode=$?
         if [[ $exitCode != 0 ]]; then
             if grep -q -e '0x80131018' $filename.stderr; then

diff --git a/eng/pipelines/pr.yml b/eng/pipelines/pr.yml
@@ -13,175 +13,37 @@ jobs:
 #
 - template: /eng/checkout-job.yml
 
-#
-# Debug builds
-#
-- template: /eng/platform-matrix.yml
-  parameters:
-    jobTemplate: build-job.yml
-    buildConfig: debug
-    platforms:
-    - Windows_NT_x64
-    - Windows_NT_x86
-    jobParameters:
-      testGroup: innerloop
-
-#
-# Checked builds
-#
 - template: /eng/platform-matrix.yml
   parameters:
     jobTemplate: build-job.yml
     buildConfig: checked
     platforms:
-    - Linux_arm
-    - Linux_arm64
-    - Linux_musl_x64
     - Linux_x64
     - OSX_x64
-    - Windows_NT_arm
-    - Windows_NT_arm64
     - Windows_NT_x64
-    - Windows_NT_x86
     jobParameters:
       testGroup: innerloop
 
-#
-# Release builds
-#
-- template: /eng/platform-matrix.yml
-  parameters:
-    jobTemplate: build-job.yml
-    buildConfig: release
-    platforms:
-    - Linux_arm64
-    - Linux_musl_x64
-    - Linux_rhel6_x64
-    - OSX_x64
-    - Windows_NT_arm
-    - Windows_NT_arm64
-    - Windows_NT_x64
-    jobParameters:
-      testGroup: innerloop
-
-#
-# Checked test builds
-#
 - template: /eng/platform-matrix.yml
   parameters:
     jobTemplate: build-test-job.yml
     buildConfig: checked
     platforms:
-    - Linux_arm
-    - Linux_arm64
     - OSX_x64
-    - Windows_NT_arm
-    - Windows_NT_arm64
     - Windows_NT_x64
-    - Windows_NT_x86
-    helixQueueGroup: pr
     jobParameters:
       testGroup: innerloop
 
-#
-# Checked JIT test executions
-#
 - template: /eng/platform-matrix.yml
   parameters:
     jobTemplate: run-test-job.yml
     buildConfig: checked
     platforms:
-    - Linux_arm
-    - Linux_arm64
-    - Linux_musl_x64
     - Linux_x64
     - OSX_x64
-    - Windows_NT_arm
-    - Windows_NT_arm64
     - Windows_NT_x64
-    - Windows_NT_x86
-    helixQueueGroup: pr
-    jobParameters:
-      testGroup: innerloop
-
-#
-# Checked R2R test executions
-#
-- template: /eng/platform-matrix.yml
-  parameters:
-    jobTemplate: run-test-job.yml
-    buildConfig: checked
-    platforms:
-    - Linux_x64
-    - OSX_x64
-    - Windows_NT_x64
-    - Windows_NT_x86
-    helixQueueGroup: pr
     jobParameters:
       testGroup: innerloop
       readyToRun: true
-      displayNameArgs: R2R
-
-#
-# CoreFX test runs against CoreCLR
-#
-- template: /eng/platform-matrix.yml
-  parameters:
-    jobTemplate: run-test-job.yml
-    buildConfig: checked
-    platforms:
-    - Linux_x64
-    - Windows_NT_x64
-    helixQueueGroup: pr
-    testGroup: innerloop
-    jobParameters:
-      testGroup: innerloop
-      corefxTests: true
-      displayNameArgs: CoreFX
-
-#
-# Crossgen-comparison jobs
-#
-- template: /eng/platform-matrix.yml
-  parameters:
-    jobTemplate: crossgen-comparison-job.yml
-    buildConfig: checked
-    platforms:
-    - Linux_arm
-    helixQueueGroup: pr
-
-#
-# Release test builds
-#
-- template: /eng/platform-matrix.yml
-  parameters:
-    jobTemplate: build-test-job.yml
-    buildConfig: release
-    platforms:
-    - OSX_x64
-    helixQueueGroup: pr
-    jobParameters:
-      testGroup: innerloop
-
-#
-# Release test builds
-#
-- template: /eng/platform-matrix.yml
-  parameters:
-    jobTemplate: run-test-job.yml
-    buildConfig: release
-    platforms:
-    - Linux_musl_x64
-    helixQueueGroup: pr
-    jobParameters:
-      testGroup: innerloop
-
-#
-# Formatting
-#
-- template: /eng/platform-matrix.yml
-  parameters:
-    jobTemplate: format-job.yml
-    platforms:
-    - Linux_x64
-
+      crossgen2: true
+      displayNameArgs: R2R_CG2
diff --git a/eng/run-test-job.yml b/eng/run-test-job.yml
@@ -9,6 +9,7 @@ parameters:
   testGroup: ''
   crossrootfsDir: ''
   readyToRun: false
+  crossgen2: false
   helixQueues: ''
   # If true, run the corefx tests instead of the coreclr ones
   corefxTests: false
@@ -252,7 +253,8 @@ jobs:
           timeoutPerTestCollectionInMinutes: 300
           timeoutPerTestInMinutes: 90
 
-        runCrossGen: ${{ parameters.readyToRun }}
+        runCrossGen: ${{ and(parameters.readyToRun, not(parameters.crossgen2)) }}
+        runCrossGen2: ${{ and(parameters.readyToRun, parameters.crossgen2) }}
         runInUnloadableContext: ${{ parameters.runInUnloadableContext }}
 
         ${{ if eq(variables['System.TeamProject'], 'internal') }}:

diff --git a/eng/send-to-helix-step.yml b/eng/send-to-helix-step.yml
@@ -15,6 +15,7 @@ parameters:
   timeoutPerTestCollectionInMinutes: ''
   timeoutPerTestInMinutes: ''
   runCrossGen: ''
+  runCrossGen2: ''
   helixProjectArguments: ''
   runInUnloadableContext: ''
   longRunningGcTests: ''
@@ -44,6 +45,7 @@ steps:
       _HelixTargetQueues: ${{ join(',', parameters.helixQueues) }}
       _HelixType: ${{ parameters.helixType }}
       _RunCrossGen: ${{ parameters.runCrossGen }}
+      _RunCrossGen2: ${{ parameters.runCrossGen2 }}
       _RunInUnloadableContext: ${{ parameters.runInUnloadableContext }}
       _LongRunningGcTests: ${{ parameters.longRunningGcTests }}
       _GcSimulatorTests: ${{ parameters.gcSimulatorTests }}

diff --git a/eng/test-job.yml b/eng/test-job.yml
@@ -8,6 +8,7 @@ parameters:
   container: ''
   testGroup: ''
   readyToRun: false
+  crossgen2: false
   helixQueues: ''
   crossrootfsDir: ''
   # If true, run the corefx tests instead of the coreclr ones
@@ -48,6 +49,7 @@ jobs:
     testGroup: ${{ parameters.testGroup }}
     crossrootfsDir: ${{ parameters.crossrootfsDir }}
     readyToRun: ${{ parameters.readyToRun }}
+    crossgen2: ${{ parameters.crossgen2 }}
     helixQueues: ${{ parameters.helixQueues }}
     corefxTests: ${{ parameters.coreFxTests }}
     displayNameArgs: ${{ parameters.displayNameArgs }}

diff --git a/tests/runtest.cmd b/tests/runtest.cmd
@@ -76,8 +76,10 @@ if /i "%1" == "skipgeneratelayout"                      (set __SkipGenerateLayou
 if /i "%1" == "buildxunitwrappers"                      (set __BuildXunitWrappers=1&shift&goto Arg_Loop)
 if /i "%1" == "printlastresultsonly"                    (set __PrintLastResultsOnly=1&shift&goto Arg_Loop)
 if /i "%1" == "runcrossgentests"                        (set RunCrossGen=true&shift&goto Arg_Loop)
+if /i "%1" == "runcrossgen2tests"                       (set RunCrossGen2=true&shift&goto Arg_Loop)
 REM This test feature is currently intentionally undocumented
 if /i "%1" == "runlargeversionbubblecrossgentests"      (set RunCrossGen=true&set CrossgenLargeVersionBubble=true&shift&goto Arg_Loop)
+if /i "%1" == "runlargeversionbubblecrossgen2tests"     (set RunCrossGen2=true&set CrossgenLargeVersionBubble=true&shift&goto Arg_Loop)
 if /i "%1" == "link"                                    (set DoLink=true&set ILLINK=%2&shift&shift&goto Arg_Loop)
 REM tieredcompilation is on by default now, but setting this environment variable is harmless and I didn't want to break any automation that might be using it just yet
 if /i "%1" == "tieredcompilation"                       (set COMPLUS_TieredCompilation=1&shift&goto Arg_Loop)
@@ -176,6 +178,10 @@ if defined RunCrossGen (
     set __RuntestPyArgs=%__RuntestPyArgs% --run_crossgen_tests
 )
 
+if defined RunCrossGen2 (
+    set __RuntestPyArgs=%__RuntestPyArgs% --run_crossgen2_tests
+)
+
 if defined __DoCrossgen (
     set __RuntestPyArgs=%__RuntestPyArgs% --precompile_core_root
 )

diff --git a/tests/runtest.py b/tests/runtest.py
@@ -970,6 +970,11 @@ def run_tests(host_os,
         print("Setting RunCrossGen=true")
         os.environ["RunCrossGen"] = "true"
 
+    if run_crossgen2_tests:
+        print("Running tests R2R (Crossgen2)")
+        print("Setting RunCrossGen2=true")
+        os.environ["RunCrossGen2"] = "true"
+
     if large_version_bubble:
         print("Large Version Bubble enabled")
         os.environ["LargeVersionBubble"] = "true"

diff --git a/tests/runtest.sh b/tests/runtest.sh
@@ -24,6 +24,7 @@ function print_usage {
     echo '  --test-env                       : Script to set environment variables for tests'
     echo '  --crossgen                       : Precompiles the framework managed assemblies'
     echo '  --runcrossgentests               : Runs the ready to run tests' 
+    echo '  --runcrossgen2tests              : Runs the ready to run tests compiled with Crossgen2' 
     echo '  --jitstress=<n>                  : Runs the tests with COMPlus_JitStress=n'
     echo '  --jitstressregs=<n>              : Runs the tests with COMPlus_JitStressRegs=n'
     echo '  --jitminopts                     : Runs the tests with COMPlus_JITMinOpts=1'
@@ -328,6 +329,9 @@ do
         --runcrossgentests)
             export RunCrossGen=1
             ;;
+        --runcrossgen2tests)
+            export RunCrossGen2=1
+            ;;
         --corefxtests)
             export RunCoreFXTests=1
             ;;
@@ -547,6 +551,10 @@ if [ ! -z "$RunCrossGen" ]; then
     runtestPyArguments+=("--run_crossgen_tests")
 fi
 
+if [ ! -z "$RunCrossGen2" ]; then
+    runtestPyArguments+=("--run_crossgen2_tests")
+fi
+
 if (($doCrossgen!=0)); then
     runtestPyArguments+=("--precompile_core_root")
 fi