From 3a6925c21afccf417eb81578686961e241c6ef20 Mon Sep 17 00:00:00 2001 From: Tomas Date: Mon, 11 Nov 2019 20:00:43 +0100 Subject: [PATCH 01/12] WIP: remaining changes to enable CoreCLR PR pipeline in the runtime repo --- NuGet.config | 1 + eng/helixcorefxtests.proj | 1 + eng/pipelines/coreclr/templates/build-test-job.yml | 2 +- .../coreclr/templates/crossgen-comparison-job.yml | 10 +++------- eng/pipelines/coreclr/templates/format-job.yml | 10 +++++++--- eng/pipelines/coreclr/templates/perf-job.yml | 4 ++-- eng/pipelines/coreclr/templates/run-test-job.yml | 2 +- eng/pipelines/coreclr/templates/xplat-pipeline-job.yml | 4 ++-- src/coreclr/build-test.sh | 2 +- src/coreclr/src/pal/tools/gen-buildsys.sh | 2 +- src/coreclr/tests/src/runtest.proj | 2 +- 11 files changed, 21 insertions(+), 19 deletions(-) diff --git a/NuGet.config b/NuGet.config index 7071908f1abac..4ea7e326f205a 100644 --- a/NuGet.config +++ b/NuGet.config @@ -13,6 +13,7 @@ + diff --git a/eng/helixcorefxtests.proj b/eng/helixcorefxtests.proj index b335ab42a4297..7c2153d6a0d5e 100644 --- a/eng/helixcorefxtests.proj +++ b/eng/helixcorefxtests.proj @@ -85,6 +85,7 @@ TODO: ProjectDir, RootBinDir, TestWorkingDir, and TargetsWindows are global properties set in dir.props, remove the property assignment here when we port to arcade. --> $(MSBuildThisFileDirectory)..\ + $(ProjectDir)\src\coreclr $(ProjectDir)bin\ $(RootBinDir)tests\$(__BuildOS).$(__BuildArch).$(__BuildType)\ true diff --git a/eng/pipelines/coreclr/templates/build-test-job.yml b/eng/pipelines/coreclr/templates/build-test-job.yml index 880ee2935feb8..851673b48cd34 100644 --- a/eng/pipelines/coreclr/templates/build-test-job.yml +++ b/eng/pipelines/coreclr/templates/build-test-job.yml @@ -67,7 +67,7 @@ jobs: displayName: Install native dependencies - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: # Necessary to install correct cmake version - - script: $(coreClrRepoRootDir)eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force + - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force displayName: Install native dependencies diff --git a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml index b10490379eae5..c045128391f09 100644 --- a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml @@ -83,12 +83,8 @@ jobs: # Populate Core_Root - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: ./build-test.sh $(buildConfig) $(archType) $(crossArg) generatelayoutonly - displayName: Populate Core_Root - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - script: build-test.cmd $(buildConfig) $(archType) generateLayoutOnly - displayName: Populate Core_Root + - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(buildConfig) $(archType) $(crossArg) generatelayoutonly + displayName: Populate Core_Root # Create baseline output on the host (x64) machine @@ -96,7 +92,7 @@ jobs: displayName: Create cross-platform crossgen baseline inputs: scriptSource: 'filePath' - scriptPath: $(Build.SourcesDirectory)/tests/scripts/crossgen_comparison.py + scriptPath: $(coreClrRepoRoot)/tests/scripts/crossgen_comparison.py ${{ if ne(parameters.osGroup, 'Windows_NT') }}: arguments: crossgen_framework diff --git a/eng/pipelines/coreclr/templates/format-job.yml b/eng/pipelines/coreclr/templates/format-job.yml index 30e90348dd5ad..55d0f35139994 100644 --- a/eng/pipelines/coreclr/templates/format-job.yml +++ b/eng/pipelines/coreclr/templates/format-job.yml @@ -10,7 +10,7 @@ parameters: ### Format job jobs: -- template: xplat-job.yml +- template: xplat-pipeline-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -32,8 +32,12 @@ jobs: displayName: Run tests/scripts/format.py inputs: scriptSource: 'filePath' - scriptPath: $(Build.SourcesDirectory)/tests/scripts/format.py - arguments: '-c $(Build.SourcesDirectory) -o $(osGroup) -a $(archType)' + ${{ if startsWith(variables['Build.DefinitionName'], 'runtime') }}: + scriptPath: $(Build.SourcesDirectory)/src/coreclr/tests/scripts/format.py + arguments: '-c $(Build.SourcesDirectory)/src/coreclr -o $(osGroup) -a $(archType)' + ${{ if not(startsWith(variables['Build.DefinitionName'], 'runtime')) }}: + scriptPath: $(Build.SourcesDirectory)/tests/scripts/format.py + arguments: '-c $(Build.SourcesDirectory) -o $(osGroup) -a $(archType)' - task: PublishBuildArtifacts@1 displayName: Publish format.patch inputs: diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index f0d598d718049..a6b0998105778 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -55,9 +55,9 @@ jobs: # Create Core_Root - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: ./build-test.sh ${{ parameters.buildConfig }} ${{ parameters.archType }} generatelayoutonly + - script: $(coreClrRepoRootDir)build-test$(scriptExt) ${{ parameters.buildConfig }} ${{ parameters.archType }} generatelayoutonly displayName: Create Core_Root - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: # TODO: add generatelayoutonly to build-test.cmd. - - script: build-test.cmd ${{ parameters.buildConfig }} ${{ parameters.archType }} skipmanaged skipnative + - script: $(coreClrRepoRootDir)build-test$(scriptExt) ${{ parameters.buildConfig }} ${{ parameters.archType }} skipmanaged skipnative displayName: Create Core_Root \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/run-test-job.yml b/eng/pipelines/coreclr/templates/run-test-job.yml index ddac9b17b6181..3b26e220a8a58 100644 --- a/eng/pipelines/coreclr/templates/run-test-job.yml +++ b/eng/pipelines/coreclr/templates/run-test-job.yml @@ -250,7 +250,7 @@ jobs: # Choose which tests to send to Helix: CoreFX or CoreCLR. ${{ if eq(parameters.corefxTests, true) }}: - helixProjectArguments: '$(coreClrRepoRoot)/eng/helixcorefxtests.proj' + helixProjectArguments: '$(Build.SourcesDirectory)/eng/helixcorefxtests.proj' ${{ if ne(parameters.corefxTests, true) }}: helixProjectArguments: '$(coreClrRepoRoot)/tests/src/helixpublishwitharcade.proj' diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index ffc806edf8714..ac96e979781ae 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -80,9 +80,9 @@ jobs: value: '/' - name: coreClrRepoRoot - value: '$(Build.SourcesDirectory)/src/coreclr + value: '$(Build.SourcesDirectory)/src/coreclr' - - ${{ if not(startsWith(variables['Build.DefinitionName'], 'runtime')) }}:' + - ${{ if not(startsWith(variables['Build.DefinitionName'], 'runtime')) }}: - name: coreClrRepoRoot value: '$(Build.SourcesDirectory)' diff --git a/src/coreclr/build-test.sh b/src/coreclr/build-test.sh index 9f178bfa84fad..ecf417a10f974 100755 --- a/src/coreclr/build-test.sh +++ b/src/coreclr/build-test.sh @@ -537,7 +537,7 @@ build_native_projects() extraCmakeArguments="$extraCmakeArguments -DCLR_CMAKE_ENABLE_CODE_COVERAGE=1" fi - nextCommand="\"$scriptDir/gen-buildsys.sh\" \"$__TestDir\" \"$intermediatesForBuild\" $platformArch $__BuildType $generator $extraCmakeArguments $__cmakeargs" + nextCommand="CONFIG_DIR=\"$__RepoRootDir/eng/common/cross\" \"$scriptDir/gen-buildsys.sh\" \"$__TestDir\" \"$intermediatesForBuild\" $platformArch $__BuildType $generator $extraCmakeArguments $__cmakeargs" echo "Invoking $nextCommand" eval $nextCommand diff --git a/src/coreclr/src/pal/tools/gen-buildsys.sh b/src/coreclr/src/pal/tools/gen-buildsys.sh index 41c49f032b4f9..f555119e33915 100755 --- a/src/coreclr/src/pal/tools/gen-buildsys.sh +++ b/src/coreclr/src/pal/tools/gen-buildsys.sh @@ -75,7 +75,7 @@ if [ "$CROSSCOMPILE" == "1" ]; then exit 1 fi export TARGET_BUILD_ARCH=$build_arch - cmake_extra_defines="$cmake_extra_defines -C ${__CoreClrDir}/tryrun.cmake" + cmake_extra_defines="$cmake_extra_defines -C ${__CoreClrDir}/src/pal/tools/tryrun.cmake" cmake_extra_defines="$cmake_extra_defines -DCMAKE_TOOLCHAIN_FILE=${__RepoRootDir}/eng/common/cross/toolchain.cmake" fi diff --git a/src/coreclr/tests/src/runtest.proj b/src/coreclr/tests/src/runtest.proj index c7b3483f293a6..420f865c9acda 100644 --- a/src/coreclr/tests/src/runtest.proj +++ b/src/coreclr/tests/src/runtest.proj @@ -26,7 +26,7 @@ $([System.IO.Path]::GetFullPath(%(Identity))) - + From 1da2ebe53d453dc429ed1bf7f537f74d69eb6a76 Mon Sep 17 00:00:00 2001 From: Tomas Date: Mon, 11 Nov 2019 22:39:08 +0100 Subject: [PATCH 02/12] Fix location of empty.csproj in Linux build.sh --- src/coreclr/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/build.sh b/src/coreclr/build.sh index d2d3dad1f5ee4..0446592217b29 100755 --- a/src/coreclr/build.sh +++ b/src/coreclr/build.sh @@ -235,7 +235,7 @@ build_native() __versionSourceFile="$intermediatesForBuild/version.c" if [ $__SkipGenerateVersion == 0 ]; then pwd - "$__RepoRootDir/eng/common/msbuild.sh" $__ArcadeScriptArgs $__ProjectRoot/eng/empty.csproj \ + "$__RepoRootDir/eng/common/msbuild.sh" $__ArcadeScriptArgs $__RepoRootDir/eng/empty.csproj \ /p:NativeVersionFile=$__versionSourceFile \ /t:GenerateNativeVersionFile /restore \ $__CommonMSBuildArgs $__UnprocessedBuildArgs @@ -1063,7 +1063,7 @@ __MsbuildDebugLogsDir="$__LogsDir/MsbuildDebugLogs" __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType" __PackagesBinDir="$__BinDir/.nuget" export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType" -export __ArtifactsIntermediatesDir="$__ProjectDir/artifacts/obj" +export __ArtifactsIntermediatesDir="$__RepoRootDir/artifacts/obj" __isMSBuildOnNETCoreSupported=0 __CrossComponentBinDir="$__BinDir" From cf32e9e3ec64ea195b98519b8ac9792d1fd9e9dc Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 01:03:20 +0100 Subject: [PATCH 03/12] Fix path to CoreFX.issues.rsp file; hopefully fix Linux builds on ARM --- eng/helixcorefxtests.proj | 2 +- src/coreclr/build.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/eng/helixcorefxtests.proj b/eng/helixcorefxtests.proj index 7c2153d6a0d5e..584b54f041581 100644 --- a/eng/helixcorefxtests.proj +++ b/eng/helixcorefxtests.proj @@ -85,7 +85,7 @@ TODO: ProjectDir, RootBinDir, TestWorkingDir, and TargetsWindows are global properties set in dir.props, remove the property assignment here when we port to arcade. --> $(MSBuildThisFileDirectory)..\ - $(ProjectDir)\src\coreclr + $(ProjectDir)\src\coreclr $(ProjectDir)bin\ $(RootBinDir)tests\$(__BuildOS).$(__BuildArch).$(__BuildType)\ true diff --git a/src/coreclr/build.sh b/src/coreclr/build.sh index 0446592217b29..f042e9ccc2c4b 100755 --- a/src/coreclr/build.sh +++ b/src/coreclr/build.sh @@ -273,8 +273,9 @@ build_native() extraCmakeArguments="$extraCmakeArguments -DCLR_CMAKE_ENABLE_CODE_COVERAGE=1" fi - echo "Invoking \"$scriptDir/gen-buildsys.sh\" \"$__ProjectRoot\" \"$intermediatesForBuild\" $platformArch $__BuildType $generator $scan_build $extraCmakeArguments $__cmakeargs" - source "$scriptDir/gen-buildsys.sh" "$__ProjectRoot" "$intermediatesForBuild" $platformArch $__BuildType $generator $scan_build "$extraCmakeArguments" "$__cmakeargs" + nextCommand="CONFIG_DIR=\"$__RepoRootDir/eng/common/cross\" \"$scriptDir/gen-buildsys.sh\" \"$__ProjectRoot\" \"$intermediatesForBuild\" $platformArch $__BuildType $generator $scan_build $extraCmakeArguments $__cmakeargs" + echo "Invoking $nextCommand" + eval $nextCommand if [ $? != 0 ]; then echo "${__ErrMsgPrefix}Failed to generate $message build project!" From 74ce55b755192373e95fb45dd07a5cb1113ccfa7 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 08:17:15 +0100 Subject: [PATCH 04/12] Two more path fixes --- eng/helixcorefxtests.proj | 2 +- src/coreclr/src/pal/tools/gen-buildsys.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/helixcorefxtests.proj b/eng/helixcorefxtests.proj index 584b54f041581..2ee3ac4290fcc 100644 --- a/eng/helixcorefxtests.proj +++ b/eng/helixcorefxtests.proj @@ -85,7 +85,7 @@ TODO: ProjectDir, RootBinDir, TestWorkingDir, and TargetsWindows are global properties set in dir.props, remove the property assignment here when we port to arcade. --> $(MSBuildThisFileDirectory)..\ - $(ProjectDir)\src\coreclr + $(ProjectDir)\src\coreclr\ $(ProjectDir)bin\ $(RootBinDir)tests\$(__BuildOS).$(__BuildArch).$(__BuildType)\ true diff --git a/src/coreclr/src/pal/tools/gen-buildsys.sh b/src/coreclr/src/pal/tools/gen-buildsys.sh index f555119e33915..41c49f032b4f9 100755 --- a/src/coreclr/src/pal/tools/gen-buildsys.sh +++ b/src/coreclr/src/pal/tools/gen-buildsys.sh @@ -75,7 +75,7 @@ if [ "$CROSSCOMPILE" == "1" ]; then exit 1 fi export TARGET_BUILD_ARCH=$build_arch - cmake_extra_defines="$cmake_extra_defines -C ${__CoreClrDir}/src/pal/tools/tryrun.cmake" + cmake_extra_defines="$cmake_extra_defines -C ${__CoreClrDir}/tryrun.cmake" cmake_extra_defines="$cmake_extra_defines -DCMAKE_TOOLCHAIN_FILE=${__RepoRootDir}/eng/common/cross/toolchain.cmake" fi From c2d075ae5174028718146904ad695e8e4a6a6396 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 09:36:08 +0100 Subject: [PATCH 05/12] Various pipeline fixes --- eng/helixcorefxtests.proj | 2 +- .../coreclr/templates/crossgen-comparison-job.yml | 8 ++++---- eng/pipelines/coreclr/templates/format-job.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/helixcorefxtests.proj b/eng/helixcorefxtests.proj index 2ee3ac4290fcc..f25f1cc0e19e9 100644 --- a/eng/helixcorefxtests.proj +++ b/eng/helixcorefxtests.proj @@ -150,7 +150,7 @@ - <_ProjectsToBuild Include="..\tests\testenvironment.proj"> + <_ProjectsToBuild Include="$(ProjectDir)tests\testenvironment.proj"> Scenario=$(Scenario);TestEnvFileName=$(TestEnvFilePath);TargetsWindows=$(TargetsWindows) diff --git a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml index c045128391f09..bf192cb3cef1a 100644 --- a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml @@ -40,14 +40,14 @@ jobs: value: $(osGroup).$(hostArchType)_$(archType).$(buildConfigUpper) - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - name: binDirectory - value: $(Build.SourcesDirectory)/bin + value: $(coreClrRepoRoot)/bin - name: productDirectory - value: $(Build.SourcesDirectory)/bin/Product + value: $(binDirectory)/Product - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - name: binDirectory - value: $(Build.SourcesDirectory)\bin + value: $(coreClrRepoRoot)\bin - name: productDirectory - value: $(Build.SourcesDirectory)\bin\Product + value: $(binDirectory\Product # Test job depends on the corresponding build job dependsOn: ${{ format('build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} diff --git a/eng/pipelines/coreclr/templates/format-job.yml b/eng/pipelines/coreclr/templates/format-job.yml index 55d0f35139994..f40733ffd3225 100644 --- a/eng/pipelines/coreclr/templates/format-job.yml +++ b/eng/pipelines/coreclr/templates/format-job.yml @@ -27,7 +27,7 @@ jobs: - task: DotNetCoreInstaller@0 inputs: packageType: 'sdk' - version: '2.1.402' + version: '3.1.100' - task: PythonScript@0 displayName: Run tests/scripts/format.py inputs: From fb555f5bbd69575e2b554145b7fd0c26aa8f4426 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 10:55:00 +0100 Subject: [PATCH 06/12] Crossgen comparison & format job fixes --- eng/pipelines/coreclr/templates/crossgen-comparison-job.yml | 2 +- eng/pipelines/coreclr/templates/format-job.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml index bf192cb3cef1a..a44820f01828e 100644 --- a/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen-comparison-job.yml @@ -122,7 +122,7 @@ jobs: Creator: $(Creator) WorkItemTimeout: 1:00 # 1 hour WorkItemDirectory: '$(binDirectory)' - CorrelationPayloadDirectory: '$(Build.SourcesDirectory)/tests/scripts' + CorrelationPayloadDirectory: '$(coreClrRepoRoot)/tests/scripts' ${{ if ne(parameters.osName, 'Windows_NT') }}: WorkItemCommand: chmod +x $HELIX_WORKITEM_PAYLOAD/Product/$(targetFlavor)/crossgen; diff --git a/eng/pipelines/coreclr/templates/format-job.yml b/eng/pipelines/coreclr/templates/format-job.yml index f40733ffd3225..fc09facd7e7c3 100644 --- a/eng/pipelines/coreclr/templates/format-job.yml +++ b/eng/pipelines/coreclr/templates/format-job.yml @@ -27,7 +27,7 @@ jobs: - task: DotNetCoreInstaller@0 inputs: packageType: 'sdk' - version: '3.1.100' + version: '3.1.100-preview3-010431' - task: PythonScript@0 displayName: Run tests/scripts/format.py inputs: From a414ceaf1136451b57bd96cf54da0b6833923cbc Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 11:12:56 +0100 Subject: [PATCH 07/12] More fixes for the formatting job --- eng/pipelines/coreclr/templates/format-job.yml | 12 ++++-------- src/coreclr/tests/scripts/format.py | 5 ++++- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/coreclr/templates/format-job.yml b/eng/pipelines/coreclr/templates/format-job.yml index fc09facd7e7c3..d7bd4cbbe4de1 100644 --- a/eng/pipelines/coreclr/templates/format-job.yml +++ b/eng/pipelines/coreclr/templates/format-job.yml @@ -27,21 +27,17 @@ jobs: - task: DotNetCoreInstaller@0 inputs: packageType: 'sdk' - version: '3.1.100-preview3-010431' + version: '2.1.402' - task: PythonScript@0 displayName: Run tests/scripts/format.py inputs: scriptSource: 'filePath' - ${{ if startsWith(variables['Build.DefinitionName'], 'runtime') }}: - scriptPath: $(Build.SourcesDirectory)/src/coreclr/tests/scripts/format.py - arguments: '-c $(Build.SourcesDirectory)/src/coreclr -o $(osGroup) -a $(archType)' - ${{ if not(startsWith(variables['Build.DefinitionName'], 'runtime')) }}: - scriptPath: $(Build.SourcesDirectory)/tests/scripts/format.py - arguments: '-c $(Build.SourcesDirectory) -o $(osGroup) -a $(archType)' + scriptPath: $(coreClrRepoRoot)/tests/scripts/format.py + arguments: '-c $(coreClrRepoRoot) -o $(osGroup) -a $(archType)' - task: PublishBuildArtifacts@1 displayName: Publish format.patch inputs: - PathtoPublish: '$(Build.SourcesDirectory)/format.patch' + PathtoPublish: '$(coreClrRepoRoot)/format.patch' ArtifactName: format.$(osGroup).$(archType).patch continueOnError: true condition: failed() diff --git a/src/coreclr/tests/scripts/format.py b/src/coreclr/tests/scripts/format.py index a20368a5905d6..08910d7dc8de8 100644 --- a/src/coreclr/tests/scripts/format.py +++ b/src/coreclr/tests/scripts/format.py @@ -157,9 +157,12 @@ def main(argv): os.chdir(current_dir) + patchFilePath = os.path.join(coreclr, "format.patch") + if returncode != 0: # Create a patch file - patchFile = open("format.patch", "w") + print("Creating patch file " + patchFilePath) + patchFile = open(patchFilePath, "w") proc = subprocess.Popen(["git", "diff", "--patch", "-U20"], env=my_env, stdout=patchFile) output,error = proc.communicate() From 14fb98deecfbf8805941067027f2cc4db4426db5 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 11:27:43 +0100 Subject: [PATCH 08/12] Introduce repo root variable in format.py --- src/coreclr/tests/scripts/format.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/coreclr/tests/scripts/format.py b/src/coreclr/tests/scripts/format.py index 08910d7dc8de8..39aaedfaefb70 100644 --- a/src/coreclr/tests/scripts/format.py +++ b/src/coreclr/tests/scripts/format.py @@ -64,6 +64,12 @@ def main(argv): return -1 coreclr = args.coreclr + repoRoot = os.path.join(coreclr, '..', '..') + + # Remove after repo consolidation + if not os.path.isfile(os.path.join(repoRoot, '.dotnet-runtime-placeholder')): + repoRoot = coreclr + platform = args.os arch = args.arch @@ -73,7 +79,7 @@ def main(argv): bootstrapFilename = "" - jitUtilsPath = os.path.join(coreclr, "jitutils") + jitUtilsPath = os.path.join(repoRoot, "jitutils") if os.path.isdir(jitUtilsPath): print("Deleting " + jitUtilsPath) @@ -86,7 +92,7 @@ def main(argv): bootstrapUrl = "https://raw.githubusercontent.com/dotnet/jitutils/master/" + bootstrapFilename - bootstrapPath = os.path.join(coreclr, bootstrapFilename) + bootstrapPath = os.path.join(repoRoot, bootstrapFilename) urlretrieve(bootstrapUrl, bootstrapPath) if not os.path.isfile(bootstrapPath): From 087e8fc62cb62d24b7bbd2f4110bdb0b5c32de03 Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 11:51:32 +0100 Subject: [PATCH 09/12] Try to fix Windows Formatting job by fixing slashes in path --- src/coreclr/tests/scripts/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/tests/scripts/format.py b/src/coreclr/tests/scripts/format.py index 39aaedfaefb70..b89f3e07d25ec 100644 --- a/src/coreclr/tests/scripts/format.py +++ b/src/coreclr/tests/scripts/format.py @@ -63,7 +63,7 @@ def main(argv): print('Bad path to coreclr') return -1 - coreclr = args.coreclr + coreclr = args.coreclr.replace('/', os.sep) repoRoot = os.path.join(coreclr, '..', '..') # Remove after repo consolidation From fe7217d57ff6adda236700fa5e28addf2305ff3c Mon Sep 17 00:00:00 2001 From: Tomas Date: Tue, 12 Nov 2019 15:09:14 +0100 Subject: [PATCH 10/12] Shorten intermediate paths for two interop test native components --- .../ReversePInvoke/PassingByOut/CMakeLists.txt | 9 +++++---- .../ReversePInvoke/PassingByRef/CMakeLists.txt | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByOut/CMakeLists.txt b/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByOut/CMakeLists.txt index f9639aa95aa7d..151d42f06c3d2 100644 --- a/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByOut/CMakeLists.txt +++ b/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByOut/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required (VERSION 2.6) -project (ReversePInvokePassingByOutNative) +project (RPIP_ByOutNative) include ("${CLR_INTEROP_TEST_ROOT}/Interop.cmake") include_directories("..") set(SOURCES @@ -7,7 +7,8 @@ set(SOURCES ) # Additional files to reference: # add the executable -add_library (ReversePInvokePassingByOutNative SHARED ${SOURCES}) -target_link_libraries(ReversePInvokePassingByOutNative ${LINK_LIBRARIES_ADDITIONAL}) +add_library (RPIP_ByOutNative SHARED ${SOURCES}) +set_property (TARGET RPIP_ByOutNative PROPERTY OUTPUT_NAME ReversePInvokePassingByOutNative) +target_link_libraries(RPIP_ByOutNative ${LINK_LIBRARIES_ADDITIONAL}) # add the install targets -install (TARGETS ReversePInvokePassingByOutNative DESTINATION bin) +install (TARGETS RPIP_ByOutNative DESTINATION bin) diff --git a/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByRef/CMakeLists.txt b/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByRef/CMakeLists.txt index 180d43c9676c7..3176cfc826e96 100644 --- a/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByRef/CMakeLists.txt +++ b/src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByRef/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required (VERSION 2.6) -project (ReversePInvokePassingByRefNative) +project (RPIP_ByRefNative) include ("${CLR_INTEROP_TEST_ROOT}/Interop.cmake") include_directories("..") set(SOURCES @@ -7,7 +7,8 @@ set(SOURCES ) # Additional files to reference: # add the executable -add_library (ReversePInvokePassingByRefNative SHARED ${SOURCES}) -target_link_libraries(ReversePInvokePassingByRefNative ${LINK_LIBRARIES_ADDITIONAL}) +add_library (RPIP_ByRefNative SHARED ${SOURCES}) +set_property (TARGET RPIP_ByRefNative PROPERTY OUTPUT_NAME ReversePInvokePassingByRefNative) +target_link_libraries(RPIP_ByRefNative ${LINK_LIBRARIES_ADDITIONAL}) # add the install targets -install (TARGETS ReversePInvokePassingByRefNative DESTINATION bin) +install (TARGETS RPIP_ByRefNative DESTINATION bin) From a84d432c4a77439b00108e7ab9afe9e7d359af84 Mon Sep 17 00:00:00 2001 From: jashook Date: Tue, 12 Nov 2019 10:58:16 -0800 Subject: [PATCH 11/12] Initial attempt to fix format jobs --- src/coreclr/tests/scripts/format.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/coreclr/tests/scripts/format.py b/src/coreclr/tests/scripts/format.py index b89f3e07d25ec..56d40c5d5e00c 100644 --- a/src/coreclr/tests/scripts/format.py +++ b/src/coreclr/tests/scripts/format.py @@ -64,11 +64,6 @@ def main(argv): return -1 coreclr = args.coreclr.replace('/', os.sep) - repoRoot = os.path.join(coreclr, '..', '..') - - # Remove after repo consolidation - if not os.path.isfile(os.path.join(repoRoot, '.dotnet-runtime-placeholder')): - repoRoot = coreclr platform = args.os arch = args.arch @@ -79,7 +74,7 @@ def main(argv): bootstrapFilename = "" - jitUtilsPath = os.path.join(repoRoot, "jitutils") + jitUtilsPath = os.path.join(coreclr, "jitutils") if os.path.isdir(jitUtilsPath): print("Deleting " + jitUtilsPath) @@ -92,7 +87,10 @@ def main(argv): bootstrapUrl = "https://raw.githubusercontent.com/dotnet/jitutils/master/" + bootstrapFilename - bootstrapPath = os.path.join(repoRoot, bootstrapFilename) + bootstrapPath = os.path.join(os.path.join(coreclr, "bin", "jitutils"), bootstrapFilename) + if not os.path.isdir(os.path.dirname(bootstrapPath)): + os.makedirs(os.path.dirname(bootstrapPath)) + urlretrieve(bootstrapUrl, bootstrapPath) if not os.path.isfile(bootstrapPath): @@ -119,7 +117,7 @@ def main(argv): # Run jit-format returncode = 0 - jitutilsBin = os.path.join(coreclr, "jitutils", "bin") + jitutilsBin = os.path.join(os.path.dirname(bootstrapPath), "jitutils", "bin") my_env["PATH"] = jitutilsBin + os.pathsep + my_env["PATH"] current_dir = os.getcwd() From f423e18b33beadd7ce135bcad4c34d267de17814 Mon Sep 17 00:00:00 2001 From: jashook Date: Wed, 13 Nov 2019 00:29:08 +0000 Subject: [PATCH 12/12] Change the cwd to the bootstrap location --- src/coreclr/tests/scripts/format.py | 37 +++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/src/coreclr/tests/scripts/format.py b/src/coreclr/tests/scripts/format.py index 56d40c5d5e00c..2946dcf47e7ef 100644 --- a/src/coreclr/tests/scripts/format.py +++ b/src/coreclr/tests/scripts/format.py @@ -20,6 +20,18 @@ import subprocess import shutil +class ChangeDir: + def __init__(self, dir): + self.dir = dir + self.cwd = None + + def __enter__(self): + self.cwd = os.getcwd() + os.chdir(self.dir) + + def __exit__(self, exc_type, exc_val, exc_tb): + os.chdir(self.cwd) + # Version specific imports if sys.version_info.major < 3: @@ -88,8 +100,12 @@ def main(argv): bootstrapUrl = "https://raw.githubusercontent.com/dotnet/jitutils/master/" + bootstrapFilename bootstrapPath = os.path.join(os.path.join(coreclr, "bin", "jitutils"), bootstrapFilename) - if not os.path.isdir(os.path.dirname(bootstrapPath)): - os.makedirs(os.path.dirname(bootstrapPath)) + + if os.path.isdir(os.path.dirname(bootstrapPath)): + shutil.rmtree(os.path.dirname(bootstrapPath)) + + assert not os.path.isdir(os.path.dirname(bootstrapPath)) + os.makedirs(os.path.dirname(bootstrapPath)) urlretrieve(bootstrapUrl, bootstrapPath) @@ -105,14 +121,15 @@ def main(argv): print(bootstrapPath) - # Run bootstrap - if platform == 'Linux' or platform == 'OSX': - print("Running bootstrap") - proc = subprocess.Popen(['bash', bootstrapPath], env=my_env) - output,error = proc.communicate() - elif platform == 'Windows_NT': - proc = subprocess.Popen([bootstrapPath], env=my_env) - output,error = proc.communicate() + with ChangeDir(os.path.dirname(bootstrapPath)): + # Run bootstrap + if platform == 'Linux' or platform == 'OSX': + print("Running bootstrap") + proc = subprocess.Popen(['bash', bootstrapPath], env=my_env) + output,error = proc.communicate() + elif platform == 'Windows_NT': + proc = subprocess.Popen([bootstrapPath], env=my_env) + output,error = proc.communicate() # Run jit-format