From caf664eaf9080f09ba04d8ad12b8d143b61d7c5f Mon Sep 17 00:00:00 2001 From: Tomas Rylek Date: Mon, 28 Oct 2019 12:54:00 -0700 Subject: [PATCH] WIP: Initial implementation of Crossgen2 CI pipeline, take #3 I have refactored my Crossgen2 CI pipeline change to not use SuperIlc for framework compilation. I intend to introduce that at a later time to basically simplify the change and make it easier to review and test. It's currently based on my preparatory change "CoreCLR pipeline optimization" which also introduces the logic to Crossgen framework assemblies as part of the run-test job. Thanks Tomas --- build-test.cmd | 32 +++++- build-test.sh | 18 ++-- eng/pipelines/pr.yml | 142 +------------------------- eng/run-test-job.yml | 4 +- eng/send-to-helix-step.yml | 2 + eng/test-job.yml | 2 + tests/runtest.cmd | 6 ++ tests/runtest.py | 5 + tests/runtest.sh | 8 ++ tests/src/CLRTest.CrossGen.targets | 54 +++++++++- tests/src/helixpublishwitharcade.proj | 5 + 11 files changed, 124 insertions(+), 154 deletions(-) diff --git a/build-test.cmd b/build-test.cmd index 04891975310d..a2fb8e82d005 100644 --- a/build-test.cmd +++ b/build-test.cmd @@ -53,6 +53,7 @@ set __SkipNative= set __RuntimeId= set __TargetsWindows=1 set __DoCrossgen= +set __DoCrossgen2= set __CopyNativeTestBinaries=0 set __CopyNativeProjectsAfterCombinedTestBuild=true set __SkipGenerateLayout=0 @@ -92,6 +93,7 @@ if /i "%1" == "buildtesthostonly" (set __SkipNative=1&set __SkipManaged=1&se if /i "%1" == "buildagainstpackages" (echo error: Remove /BuildAgainstPackages switch&&exit /b1) if /i "%1" == "skiprestorepackages" (set __SkipRestorePackages=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "crossgen" (set __DoCrossgen=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "crossgen2" (set __DoCrossgen2=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "runtimeid" (set __RuntimeId=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) if /i "%1" == "targetsNonWindows" (set __TargetsWindows=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "Exclude" (set __Exclude=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) @@ -526,6 +528,17 @@ if defined __DoCrossgen ( ) ) +if defined __DoCrossgen2 ( + set __CrossgenArg="/p:Crossgen2=true" + if "%__BuildArch%" == "x64" ( + echo %__MsgPrefix%Running crossgen2 on framework assemblies + call :PrecompileFX + ) else ( + echo "%__MsgPrefix%Crossgen2 only supported on x64, for now" + ) +) + + rd /s /q "%CORE_ROOT_STAGE%" REM ========================================================================================= @@ -614,9 +627,22 @@ echo "%2" | findstr /b "xunit." >nul && ( set __CrossgenExe="%CORE_ROOT_STAGE%\crossgen.exe" if /i "%__BuildArch%" == "arm" ( set __CrossgenExe="%CORE_ROOT_STAGE%\x86\crossgen.exe" ) if /i "%__BuildArch%" == "arm64" ( set __CrossgenExe="%CORE_ROOT_STAGE%\x64\crossgen.exe" ) +set __CrossgenExe=%__CrossgenExe% + +if defined __DoCrossgen2 ( + set __CrossgenExe="%CORE_ROOT_STATE%\crossgen2\crossgen2.exe" +) + +set __CrossgenOutputFile="%CORE_ROOT%\temp.ni.dll" + +if defined __Crossgen ( + "!__CrossgenExe!" /Platform_Assemblies_Paths "!CORE_ROOT!" /in "%1" /out "!__CrossgenOutputFile" >nul 2>nul + set /a __exitCode = !errorlevel! +) else ( + "!CORE_ROOT_STAGE!\crossgen2\crossgen2 -r:"!CORE_ROOT!\*.dll" -O --inputbubble -out:"__CrossgenOutputFile" "%1" >nul 2>nul + set /a __exitCode = !errorlevel! +) -"%__CrossgenExe%" /Platform_Assemblies_Paths "%CORE_ROOT%" /in "%1" /out "%CORE_ROOT%/temp.ni.dll" >nul 2>nul -set /a __exitCode = %errorlevel% if "%__exitCode%" == "-2146230517" ( echo %2 is not a managed assembly. exit /b 0 @@ -629,7 +655,7 @@ if %__exitCode% neq 0 ( REM Delete original .dll & replace it with the Crossgened .dll del %1 -ren "%CORE_ROOT%\temp.ni.dll" %2 +ren "%__CrossgenOutputFile%" %2 echo Successfully precompiled %2 exit /b 0 diff --git a/build-test.sh b/build-test.sh index 40c80bc246ab..49fc70bf775d 100755 --- a/build-test.sh +++ b/build-test.sh @@ -166,20 +166,26 @@ generate_layout() fi # Precompile framework assemblies with crossgen if required - if [ $__DoCrossgen -ne 0 ]; then + if [ $__DoCrossgen -ne 0 || $__DoCrossgen2 -ne 0 ]; then precompile_coreroot_fx fi } precompile_coreroot_fx() { - echo "${__MsgPrefix}Running crossgen on framework assemblies in CORE_ROOT: '${CORE_ROOT}'" + local overlayDir=$CORE_ROOT + local action = crossgen + local compiler = $__CrossgenExe /Platform_Assemblies_Paths $overlayDir + + if [ $__DoCrossgen2 -ne 0 ]; then + action = crossgen2 + compiler = ${overlayDir}/crossgen2/crossgen2 -r:${overlayDir}/*.dll -O --inputbubble + fi + + echo "${__MsgPrefix}Running ${action} on framework assemblies in CORE_ROOT: '${CORE_ROOT}'" # Read the exclusion file for this platform skipCrossGenFiles=($(read_array "$(dirname "$0")/tests/skipCrossGenFiles.${__BuildArch}.txt")) - skipCrossGenFiles+=('System.Runtime.WindowsRuntime.dll') - - local overlayDir=$CORE_ROOT filesToPrecompile=$(find -L $overlayDir -maxdepth 1 -iname \*.dll -not -iname \*.ni.dll -not -iname \*-ms-win-\* -not -iname xunit.\* -type f) for fileToPrecompile in ${filesToPrecompile} @@ -189,7 +195,7 @@ precompile_coreroot_fx() continue fi echo Precompiling $filename - $__CrossgenExe /Platform_Assemblies_Paths $overlayDir $filename 1> $filename.stdout 2>$filename.stderr + $compiler $filename 1> $filename.stdout 2>$filename.stderr local exitCode=$? if [[ $exitCode != 0 ]]; then if grep -q -e '0x80131018' $filename.stderr; then diff --git a/eng/pipelines/pr.yml b/eng/pipelines/pr.yml index 9d2acda4b882..7db804dbbfb5 100644 --- a/eng/pipelines/pr.yml +++ b/eng/pipelines/pr.yml @@ -13,175 +13,37 @@ jobs: # - template: /eng/checkout-job.yml -# -# Debug builds -# -- template: /eng/platform-matrix.yml - parameters: - jobTemplate: build-job.yml - buildConfig: debug - platforms: - - Windows_NT_x64 - - Windows_NT_x86 - jobParameters: - testGroup: innerloop - -# -# Checked builds -# - template: /eng/platform-matrix.yml parameters: jobTemplate: build-job.yml buildConfig: checked platforms: - - Linux_arm - - Linux_arm64 - - Linux_musl_x64 - Linux_x64 - OSX_x64 - - Windows_NT_arm - - Windows_NT_arm64 - Windows_NT_x64 - - Windows_NT_x86 jobParameters: testGroup: innerloop -# -# Release builds -# -- template: /eng/platform-matrix.yml - parameters: - jobTemplate: build-job.yml - buildConfig: release - platforms: - - Linux_arm64 - - Linux_musl_x64 - - Linux_rhel6_x64 - - OSX_x64 - - Windows_NT_arm - - Windows_NT_arm64 - - Windows_NT_x64 - jobParameters: - testGroup: innerloop - -# -# Checked test builds -# - template: /eng/platform-matrix.yml parameters: jobTemplate: build-test-job.yml buildConfig: checked platforms: - - Linux_arm - - Linux_arm64 - OSX_x64 - - Windows_NT_arm - - Windows_NT_arm64 - Windows_NT_x64 - - Windows_NT_x86 - helixQueueGroup: pr jobParameters: testGroup: innerloop -# -# Checked JIT test executions -# - template: /eng/platform-matrix.yml parameters: jobTemplate: run-test-job.yml buildConfig: checked platforms: - - Linux_arm - - Linux_arm64 - - Linux_musl_x64 - Linux_x64 - OSX_x64 - - Windows_NT_arm - - Windows_NT_arm64 - Windows_NT_x64 - - Windows_NT_x86 - helixQueueGroup: pr - jobParameters: - testGroup: innerloop - -# -# Checked R2R test executions -# -- template: /eng/platform-matrix.yml - parameters: - jobTemplate: run-test-job.yml - buildConfig: checked - platforms: - - Linux_x64 - - OSX_x64 - - Windows_NT_x64 - - Windows_NT_x86 - helixQueueGroup: pr jobParameters: testGroup: innerloop readyToRun: true - displayNameArgs: R2R - -# -# CoreFX test runs against CoreCLR -# -- template: /eng/platform-matrix.yml - parameters: - jobTemplate: run-test-job.yml - buildConfig: checked - platforms: - - Linux_x64 - - Windows_NT_x64 - helixQueueGroup: pr - testGroup: innerloop - jobParameters: - testGroup: innerloop - corefxTests: true - displayNameArgs: CoreFX - -# -# Crossgen-comparison jobs -# -- template: /eng/platform-matrix.yml - parameters: - jobTemplate: crossgen-comparison-job.yml - buildConfig: checked - platforms: - - Linux_arm - helixQueueGroup: pr - -# -# Release test builds -# -- template: /eng/platform-matrix.yml - parameters: - jobTemplate: build-test-job.yml - buildConfig: release - platforms: - - OSX_x64 - helixQueueGroup: pr - jobParameters: - testGroup: innerloop - -# -# Release test builds -# -- template: /eng/platform-matrix.yml - parameters: - jobTemplate: run-test-job.yml - buildConfig: release - platforms: - - Linux_musl_x64 - helixQueueGroup: pr - jobParameters: - testGroup: innerloop - -# -# Formatting -# -- template: /eng/platform-matrix.yml - parameters: - jobTemplate: format-job.yml - platforms: - - Linux_x64 - + crossgen2: true + displayNameArgs: R2R_CG2 diff --git a/eng/run-test-job.yml b/eng/run-test-job.yml index 7f02ed1d96ad..494bc7435fb5 100644 --- a/eng/run-test-job.yml +++ b/eng/run-test-job.yml @@ -9,6 +9,7 @@ parameters: testGroup: '' crossrootfsDir: '' readyToRun: false + crossgen2: false helixQueues: '' # If true, run the corefx tests instead of the coreclr ones corefxTests: false @@ -252,7 +253,8 @@ jobs: timeoutPerTestCollectionInMinutes: 300 timeoutPerTestInMinutes: 90 - runCrossGen: ${{ parameters.readyToRun }} + runCrossGen: ${{ and(parameters.readyToRun, not(parameters.crossgen2)) }} + runCrossGen2: ${{ and(parameters.readyToRun, parameters.crossgen2) }} runInUnloadableContext: ${{ parameters.runInUnloadableContext }} ${{ if eq(variables['System.TeamProject'], 'internal') }}: diff --git a/eng/send-to-helix-step.yml b/eng/send-to-helix-step.yml index 04058bc29192..7f8bb9a53116 100644 --- a/eng/send-to-helix-step.yml +++ b/eng/send-to-helix-step.yml @@ -15,6 +15,7 @@ parameters: timeoutPerTestCollectionInMinutes: '' timeoutPerTestInMinutes: '' runCrossGen: '' + runCrossGen2: '' helixProjectArguments: '' runInUnloadableContext: '' longRunningGcTests: '' @@ -44,6 +45,7 @@ steps: _HelixTargetQueues: ${{ join(',', parameters.helixQueues) }} _HelixType: ${{ parameters.helixType }} _RunCrossGen: ${{ parameters.runCrossGen }} + _RunCrossGen2: ${{ parameters.runCrossGen2 }} _RunInUnloadableContext: ${{ parameters.runInUnloadableContext }} _LongRunningGcTests: ${{ parameters.longRunningGcTests }} _GcSimulatorTests: ${{ parameters.gcSimulatorTests }} diff --git a/eng/test-job.yml b/eng/test-job.yml index f10cc0748129..4f4d4c211552 100644 --- a/eng/test-job.yml +++ b/eng/test-job.yml @@ -8,6 +8,7 @@ parameters: container: '' testGroup: '' readyToRun: false + crossgen2: false helixQueues: '' crossrootfsDir: '' # If true, run the corefx tests instead of the coreclr ones @@ -48,6 +49,7 @@ jobs: testGroup: ${{ parameters.testGroup }} crossrootfsDir: ${{ parameters.crossrootfsDir }} readyToRun: ${{ parameters.readyToRun }} + crossgen2: ${{ parameters.crossgen2 }} helixQueues: ${{ parameters.helixQueues }} corefxTests: ${{ parameters.coreFxTests }} displayNameArgs: ${{ parameters.displayNameArgs }} diff --git a/tests/runtest.cmd b/tests/runtest.cmd index f2c550d344b1..eaae39deddec 100644 --- a/tests/runtest.cmd +++ b/tests/runtest.cmd @@ -76,8 +76,10 @@ if /i "%1" == "skipgeneratelayout" (set __SkipGenerateLayou if /i "%1" == "buildxunitwrappers" (set __BuildXunitWrappers=1&shift&goto Arg_Loop) if /i "%1" == "printlastresultsonly" (set __PrintLastResultsOnly=1&shift&goto Arg_Loop) if /i "%1" == "runcrossgentests" (set RunCrossGen=true&shift&goto Arg_Loop) +if /i "%1" == "runcrossgen2tests" (set RunCrossGen2=true&shift&goto Arg_Loop) REM This test feature is currently intentionally undocumented if /i "%1" == "runlargeversionbubblecrossgentests" (set RunCrossGen=true&set CrossgenLargeVersionBubble=true&shift&goto Arg_Loop) +if /i "%1" == "runlargeversionbubblecrossgen2tests" (set RunCrossGen2=true&set CrossgenLargeVersionBubble=true&shift&goto Arg_Loop) if /i "%1" == "link" (set DoLink=true&set ILLINK=%2&shift&shift&goto Arg_Loop) REM tieredcompilation is on by default now, but setting this environment variable is harmless and I didn't want to break any automation that might be using it just yet if /i "%1" == "tieredcompilation" (set COMPLUS_TieredCompilation=1&shift&goto Arg_Loop) @@ -176,6 +178,10 @@ if defined RunCrossGen ( set __RuntestPyArgs=%__RuntestPyArgs% --run_crossgen_tests ) +if defined RunCrossGen2 ( + set __RuntestPyArgs=%__RuntestPyArgs% --run_crossgen2_tests +) + if defined __DoCrossgen ( set __RuntestPyArgs=%__RuntestPyArgs% --precompile_core_root ) diff --git a/tests/runtest.py b/tests/runtest.py index 379941b9a4b2..d1e49a5e33a9 100755 --- a/tests/runtest.py +++ b/tests/runtest.py @@ -970,6 +970,11 @@ def run_tests(host_os, print("Setting RunCrossGen=true") os.environ["RunCrossGen"] = "true" + if run_crossgen2_tests: + print("Running tests R2R (Crossgen2)") + print("Setting RunCrossGen2=true") + os.environ["RunCrossGen2"] = "true" + if large_version_bubble: print("Large Version Bubble enabled") os.environ["LargeVersionBubble"] = "true" diff --git a/tests/runtest.sh b/tests/runtest.sh index 47bd80c2c3da..95cb6010a1b4 100755 --- a/tests/runtest.sh +++ b/tests/runtest.sh @@ -24,6 +24,7 @@ function print_usage { echo ' --test-env : Script to set environment variables for tests' echo ' --crossgen : Precompiles the framework managed assemblies' echo ' --runcrossgentests : Runs the ready to run tests' + echo ' --runcrossgen2tests : Runs the ready to run tests compiled with Crossgen2' echo ' --jitstress= : Runs the tests with COMPlus_JitStress=n' echo ' --jitstressregs= : Runs the tests with COMPlus_JitStressRegs=n' echo ' --jitminopts : Runs the tests with COMPlus_JITMinOpts=1' @@ -328,6 +329,9 @@ do --runcrossgentests) export RunCrossGen=1 ;; + --runcrossgen2tests) + export RunCrossGen2=1 + ;; --corefxtests) export RunCoreFXTests=1 ;; @@ -547,6 +551,10 @@ if [ ! -z "$RunCrossGen" ]; then runtestPyArguments+=("--run_crossgen_tests") fi +if [ ! -z "$RunCrossGen2" ]; then + runtestPyArguments+=("--run_crossgen2_tests") +fi + if (($doCrossgen!=0)); then runtestPyArguments+=("--precompile_core_root") fi diff --git a/tests/src/CLRTest.CrossGen.targets b/tests/src/CLRTest.CrossGen.targets index 3747f275bf84..183ac51e2ded 100644 --- a/tests/src/CLRTest.CrossGen.targets +++ b/tests/src/CLRTest.CrossGen.targets @@ -47,8 +47,9 @@ if [ ! -z ${RunCrossGen+x} ]%3B then mkdir IL cp $(MSBuildProjectName).dll IL/$(MSBuildProjectName).dll mv $(MSBuildProjectName).dll $(MSBuildProjectName).org - echo $_DebuggerFullPath "$CORE_ROOT/crossgen" /Platform_Assemblies_Paths $CORE_ROOT%3A$PWD/IL%3A$PWD /in $(MSBuildProjectName).org /out $(MSBuildProjectName).dll - $_DebuggerFullPath "$CORE_ROOT/crossgen" /Platform_Assemblies_Paths $CORE_ROOT%3A$PWD/IL%3A$PWD /in $(MSBuildProjectName).org /out $(MSBuildProjectName).dll + __Command=$_DebuggerFullPath "$CORE_ROOT/crossgen" /Platform_Assemblies_Paths $CORE_ROOT/Crossgen-ret.out%3A$PWD /in $(MSBuildProjectName).org /out $(MSBuildProjectName).dll + echo $__Command + $__Command __cgExitCode=$? if [ $__cgExitCode -ne 0 ] then @@ -59,6 +60,28 @@ if [ ! -z ${RunCrossGen+x} ]%3B then fi ReleaseLock fi +fi +# CrossGen2 Script +if [ ! -z ${RunCrossGen2+x} ]%3B then + if [ ! -f $(MSBuildProjectName).org ]%3B then + TakeLock + if [ ! -f $(MSBuildProjectName).org ]%3B then + mkdir IL + cp $(MSBuildProjectName).dll IL/$(MSBuildProjectName).dll + mv $(MSBuildProjectName).dll $(MSBuildProjectName).org + __Command=$_DebuggerFullPath "$CORE_ROOT/crossgen2/crossgen2" -r:$CORE_ROOT/*.dll --targetarch=x64 -O --inputbubble -o:$(scriptPath)$(MSBuildProjectName).dll $(scriptPath)$(MSBuildProjectName).org + echo $__Command + $__Command + __cg2ExitCode=$? + if [ $__cg2ExitCode -ne 0 ] + then + echo Crossgen2 failed with exitcode: $__cg2ExitCode + ReleaseLock + exit 1 + fi + fi + ReleaseLock + fi fi ]]> @@ -85,8 +108,9 @@ if defined RunCrossGen ( mkdir IL copy $(MSBuildProjectName).dll IL\$(MSBuildProjectName).dll ren $(MSBuildProjectName).dll $(MSBuildProjectName).org - echo "%_DebuggerFullPath% %CORE_ROOT%\crossgen.exe" !OptionalArguments! /Platform_Assemblies_Paths %CORE_ROOT%%3B%25cd%25\IL%3B%25cd%25 /in $(MSBuildProjectName).org /out $(MSBuildProjectName).dll - %_DebuggerFullPath% "%CORE_ROOT%\crossgen.exe" !OptionalArguments! /Platform_Assemblies_Paths %CORE_ROOT%%3B%25cd%25\IL%3B%25cd%25 /in $(MSBuildProjectName).org /out $(MSBuildProjectName).dll + set __Command=!_DebuggerFullPath! "!CORE_ROOT!\crossgen.exe" !OptionalArguments! /Platform_Assemblies_Paths !CORE_ROOT!\Crossgen-ret.out%3B%25cd%25 /in $(scriptPath)\$(MSBuildProjectName).org /out %25scriptPath%25\$(MSBuildProjectName).dll + echo "!__Command!" + call !__Command! set CrossGenStatus=!ERRORLEVEL! ) call :ReleaseLock @@ -95,6 +119,28 @@ if defined RunCrossGen ( Exit /b 1 ) ) +) +REM CrossGen2 Script +if defined RunCrossGen2 ( + if defined LargeVersionBubble ( set OptionalArguments=!OptionalArguments! /largeversionbubble) + if not exist "$(MSBuildProjectName).org" ( + call :TakeLock + set CrossGen2Status=0 + if not exist "$(MSBuildProjectName).org" ( + mkdir IL + copy $(MSBuildProjectName).dll IL\$(MSBuildProjectName).dll + ren $(MSBuildProjectName).dll $(MSBuildProjectName).org + set __Command=!_DebuggerFullPath! "!CORE_ROOT!\crossgen2\crossgen2" %21scriptPath%21$(MSBuildProjectName).org -o:%21scriptPath%21$(MSBuildProjectName).dll --targetarch:x64 -O --inputbubble -r:!CORE_ROOT!\*.dll + echo "!__Command!" + call !__Command! + set CrossGen2Status=!ERRORLEVEL! + ) + call :ReleaseLock + IF NOT !CrossGen2Status!==0 ( + ECHO Crossgen2 failed with exitcode - !CrossGen2Status! + Exit /b 1 + ) + ) ) ]]> diff --git a/tests/src/helixpublishwitharcade.proj b/tests/src/helixpublishwitharcade.proj index 61af68f15648..eedd39c9cb4b 100644 --- a/tests/src/helixpublishwitharcade.proj +++ b/tests/src/helixpublishwitharcade.proj @@ -26,6 +26,7 @@ HelixType=$(_HelixType); PublishTestResults=$(_PublishTestResults); RunCrossGen=$(_RunCrossGen); + RunCrossGen2=$(_RunCrossGen2); LongRunningGCTests=$(_LongRunningGCTests); GcSimulatorTests=$(_GcSimulatorTests); RunInUnloadableContext=$(_RunInUnloadableContext); @@ -172,9 +173,11 @@ $(BuildType) $(BuildType)-$(Scenario) false + false false false R2R + R2R-CG2 $(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) @ $(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) $(Scenario) @ $([System.TimeSpan]::FromMinutes($(TimeoutPerTestInMinutes)).TotalMilliseconds) @@ -192,6 +195,7 @@ + @@ -206,6 +210,7 @@ +