Remaining changes required for the CI pipeline in runtime repo.

NuGet.config

@@ -13,6 +13,7 @@
     <!-- TEST_RESTORE_SOURCES_INSERTION_LINE -->
     <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
     <add key="dotnet-coreclr" value="https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json" />
+    <add key="myget.org dotnet-core" value="https://dotnet.myget.org/F/dotnet-core/api/v3/index.json" />
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
   </packageSources>
   <disabledPackageSources>

eng/helixcorefxtests.proj

@@ -85,6 +85,7 @@
       TODO: ProjectDir, RootBinDir, TestWorkingDir, and TargetsWindows are global properties set in dir.props, remove the property assignment here when we port to arcade.
      -->
     <ProjectDir Condition="'$(__ProjectDir)'==''">$(MSBuildThisFileDirectory)..\</ProjectDir>
+    <ProjectDir Condition="Exists('$(ProjectDir).dotnet-runtime-placeholder')">$(ProjectDir)\src\coreclr\</ProjectDir>
     <RootBinDir Condition="'$(__RootBinDir)'==''">$(ProjectDir)bin\</RootBinDir>
     <TestWorkingDir Condition="'$(__TestWorkingDir)'==''">$(RootBinDir)tests\$(__BuildOS).$(__BuildArch).$(__BuildType)\</TestWorkingDir>
     <TargetsWindows Condition="'$(__BuildOS)' == 'Windows_NT'">true</TargetsWindows>
@@ -149,7 +150,7 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <_ProjectsToBuild Include="..\tests\testenvironment.proj">
+      <_ProjectsToBuild Include="$(ProjectDir)tests\testenvironment.proj">
         <Properties>Scenario=$(Scenario);TestEnvFileName=$(TestEnvFilePath);TargetsWindows=$(TargetsWindows)</Properties>
       </_ProjectsToBuild>
     </ItemGroup>

eng/pipelines/coreclr/templates/build-test-job.yml

@@ -67,7 +67,7 @@ jobs:
         displayName: Install native dependencies
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       # Necessary to install correct cmake version
-      - script: $(coreClrRepoRootDir)eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force
+      - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force
         displayName: Install native dependencies
 
 

eng/pipelines/coreclr/templates/crossgen-comparison-job.yml

@@ -40,14 +40,14 @@ jobs:
       value: $(osGroup).$(hostArchType)_$(archType).$(buildConfigUpper)
     - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
       - name: binDirectory
-        value: $(Build.SourcesDirectory)/bin
+        value: $(coreClrRepoRoot)/bin
       - name: productDirectory
-        value: $(Build.SourcesDirectory)/bin/Product
+        value: $(binDirectory)/Product
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       - name: binDirectory
-        value: $(Build.SourcesDirectory)\bin
+        value: $(coreClrRepoRoot)\bin
       - name: productDirectory
-        value: $(Build.SourcesDirectory)\bin\Product
+        value: $(binDirectory\Product
 
     # Test job depends on the corresponding build job
     dependsOn: ${{ format('build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
@@ -83,20 +83,16 @@ jobs:
 
 
     # Populate Core_Root
-    - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
-      - script: ./build-test.sh $(buildConfig) $(archType) $(crossArg) generatelayoutonly
-        displayName: Populate Core_Root
-    - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
-      - script: build-test.cmd $(buildConfig) $(archType) generateLayoutOnly
-        displayName: Populate Core_Root
+    - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(buildConfig) $(archType) $(crossArg) generatelayoutonly
+      displayName: Populate Core_Root
 
 
     # Create baseline output on the host (x64) machine
     - task: PythonScript@0
       displayName: Create cross-platform crossgen baseline
       inputs:
         scriptSource: 'filePath'
-        scriptPath: $(Build.SourcesDirectory)/tests/scripts/crossgen_comparison.py
+        scriptPath: $(coreClrRepoRoot)/tests/scripts/crossgen_comparison.py
         ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
           arguments:
             crossgen_framework
@@ -126,7 +122,7 @@ jobs:
           Creator: $(Creator)
         WorkItemTimeout: 1:00 # 1 hour
         WorkItemDirectory: '$(binDirectory)'
-        CorrelationPayloadDirectory: '$(Build.SourcesDirectory)/tests/scripts'
+        CorrelationPayloadDirectory: '$(coreClrRepoRoot)/tests/scripts'
         ${{ if ne(parameters.osName, 'Windows_NT') }}:
           WorkItemCommand:
             chmod +x     $HELIX_WORKITEM_PAYLOAD/Product/$(targetFlavor)/crossgen;

eng/pipelines/coreclr/templates/format-job.yml

@@ -10,7 +10,7 @@ parameters:
 
 ### Format job
 jobs:
-- template: xplat-job.yml
+- template: xplat-pipeline-job.yml
   parameters:
     buildConfig: ${{ parameters.buildConfig }}
     archType: ${{ parameters.archType }}
@@ -32,12 +32,12 @@ jobs:
       displayName: Run tests/scripts/format.py
       inputs:
         scriptSource: 'filePath'
-        scriptPath: $(Build.SourcesDirectory)/tests/scripts/format.py
-        arguments: '-c $(Build.SourcesDirectory) -o $(osGroup) -a $(archType)'
+        scriptPath: $(coreClrRepoRoot)/tests/scripts/format.py
+        arguments: '-c $(coreClrRepoRoot) -o $(osGroup) -a $(archType)'
     - task: PublishBuildArtifacts@1
       displayName: Publish format.patch
       inputs:
-        PathtoPublish: '$(Build.SourcesDirectory)/format.patch'
+        PathtoPublish: '$(coreClrRepoRoot)/format.patch'
         ArtifactName: format.$(osGroup).$(archType).patch
       continueOnError: true
       condition: failed()

eng/pipelines/coreclr/templates/perf-job.yml

@@ -55,9 +55,9 @@ jobs:
 
     # Create Core_Root
     - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
-      - script: ./build-test.sh ${{ parameters.buildConfig }} ${{ parameters.archType }} generatelayoutonly
+      - script: $(coreClrRepoRootDir)build-test$(scriptExt) ${{ parameters.buildConfig }} ${{ parameters.archType }} generatelayoutonly
         displayName: Create Core_Root
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       # TODO: add generatelayoutonly to build-test.cmd.
-      - script: build-test.cmd ${{ parameters.buildConfig }} ${{ parameters.archType }} skipmanaged skipnative
+      - script: $(coreClrRepoRootDir)build-test$(scriptExt) ${{ parameters.buildConfig }} ${{ parameters.archType }} skipmanaged skipnative
         displayName: Create Core_Root

eng/pipelines/coreclr/templates/run-test-job.yml

@@ -250,7 +250,7 @@ jobs:
 
         # Choose which tests to send to Helix: CoreFX or CoreCLR.
         ${{ if eq(parameters.corefxTests, true) }}:
-          helixProjectArguments: '$(coreClrRepoRoot)/eng/helixcorefxtests.proj'
+          helixProjectArguments: '$(Build.SourcesDirectory)/eng/helixcorefxtests.proj'
         ${{ if ne(parameters.corefxTests, true) }}:
           helixProjectArguments: '$(coreClrRepoRoot)/tests/src/helixpublishwitharcade.proj'
 

eng/pipelines/coreclr/templates/xplat-pipeline-job.yml

@@ -80,9 +80,9 @@ jobs:
         value: '/'
 
     - name: coreClrRepoRoot
-      value: '$(Build.SourcesDirectory)/src/coreclr
+      value: '$(Build.SourcesDirectory)/src/coreclr'
 
-    - ${{ if not(startsWith(variables['Build.DefinitionName'], 'runtime')) }}:'
+    - ${{ if not(startsWith(variables['Build.DefinitionName'], 'runtime')) }}:
 
       - name: coreClrRepoRoot
         value: '$(Build.SourcesDirectory)'

src/coreclr/build-test.sh

@@ -537,7 +537,7 @@ build_native_projects()
             extraCmakeArguments="$extraCmakeArguments -DCLR_CMAKE_ENABLE_CODE_COVERAGE=1"
         fi
 
-        nextCommand="\"$scriptDir/gen-buildsys.sh\" \"$__TestDir\" \"$intermediatesForBuild\" $platformArch $__BuildType $generator $extraCmakeArguments $__cmakeargs"
+        nextCommand="CONFIG_DIR=\"$__RepoRootDir/eng/common/cross\" \"$scriptDir/gen-buildsys.sh\" \"$__TestDir\" \"$intermediatesForBuild\" $platformArch $__BuildType $generator $extraCmakeArguments $__cmakeargs"
         echo "Invoking $nextCommand"
         eval $nextCommand
 

src/coreclr/build.sh

@@ -235,7 +235,7 @@ build_native()
         __versionSourceFile="$intermediatesForBuild/version.c"
         if [ $__SkipGenerateVersion == 0 ]; then
             pwd
-            "$__RepoRootDir/eng/common/msbuild.sh" $__ArcadeScriptArgs $__ProjectRoot/eng/empty.csproj \
+            "$__RepoRootDir/eng/common/msbuild.sh" $__ArcadeScriptArgs $__RepoRootDir/eng/empty.csproj \
                                                    /p:NativeVersionFile=$__versionSourceFile \
                                                    /t:GenerateNativeVersionFile /restore \
                                                    $__CommonMSBuildArgs $__UnprocessedBuildArgs
@@ -273,8 +273,9 @@ build_native()
             extraCmakeArguments="$extraCmakeArguments -DCLR_CMAKE_ENABLE_CODE_COVERAGE=1"
         fi
 
-        echo "Invoking \"$scriptDir/gen-buildsys.sh\" \"$__ProjectRoot\" \"$intermediatesForBuild\" $platformArch $__BuildType $generator $scan_build $extraCmakeArguments $__cmakeargs"
-        source "$scriptDir/gen-buildsys.sh" "$__ProjectRoot" "$intermediatesForBuild" $platformArch $__BuildType $generator $scan_build "$extraCmakeArguments" "$__cmakeargs"
+        nextCommand="CONFIG_DIR=\"$__RepoRootDir/eng/common/cross\" \"$scriptDir/gen-buildsys.sh\" \"$__ProjectRoot\" \"$intermediatesForBuild\" $platformArch $__BuildType $generator $scan_build $extraCmakeArguments $__cmakeargs"
+        echo "Invoking $nextCommand"
+        eval $nextCommand
 
         if [ $? != 0  ]; then
             echo "${__ErrMsgPrefix}Failed to generate $message build project!"
@@ -1063,7 +1064,7 @@ __MsbuildDebugLogsDir="$__LogsDir/MsbuildDebugLogs"
 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 __PackagesBinDir="$__BinDir/.nuget"
 export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
-export __ArtifactsIntermediatesDir="$__ProjectDir/artifacts/obj"
+export __ArtifactsIntermediatesDir="$__RepoRootDir/artifacts/obj"
 __isMSBuildOnNETCoreSupported=0
 __CrossComponentBinDir="$__BinDir"
 

src/coreclr/tests/scripts/format.py

@@ -20,6 +20,18 @@
 import subprocess
 import shutil
 
+class ChangeDir:
+    def __init__(self, dir):
+        self.dir = dir
+        self.cwd = None
+
+    def __enter__(self):
+        self.cwd = os.getcwd()
+        os.chdir(self.dir)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        os.chdir(self.cwd)
+
 # Version specific imports
 
 if sys.version_info.major < 3:
@@ -63,7 +75,8 @@ def main(argv):
         print('Bad path to coreclr')
         return -1
 
-    coreclr = args.coreclr
+    coreclr = args.coreclr.replace('/', os.sep)
+
     platform = args.os
     arch = args.arch
 
@@ -86,7 +99,14 @@ def main(argv):
 
     bootstrapUrl = "https://raw.githubusercontent.com/dotnet/jitutils/master/" + bootstrapFilename
 
-    bootstrapPath = os.path.join(coreclr, bootstrapFilename)
+    bootstrapPath = os.path.join(os.path.join(coreclr, "bin", "jitutils"), bootstrapFilename)
+
+    if os.path.isdir(os.path.dirname(bootstrapPath)):
+        shutil.rmtree(os.path.dirname(bootstrapPath))
+
+    assert not os.path.isdir(os.path.dirname(bootstrapPath))
+    os.makedirs(os.path.dirname(bootstrapPath))
+
     urlretrieve(bootstrapUrl, bootstrapPath)
 
     if not os.path.isfile(bootstrapPath):
@@ -101,19 +121,20 @@ def main(argv):
 
     print(bootstrapPath)
 
-    # Run bootstrap
-    if platform == 'Linux' or platform == 'OSX':
-        print("Running bootstrap")
-        proc = subprocess.Popen(['bash', bootstrapPath], env=my_env)
-        output,error = proc.communicate()
-    elif platform == 'Windows_NT':
-        proc = subprocess.Popen([bootstrapPath], env=my_env)
-        output,error = proc.communicate()
+    with ChangeDir(os.path.dirname(bootstrapPath)):
+        # Run bootstrap
+        if platform == 'Linux' or platform == 'OSX':
+            print("Running bootstrap")
+            proc = subprocess.Popen(['bash', bootstrapPath], env=my_env)
+            output,error = proc.communicate()
+        elif platform == 'Windows_NT':
+            proc = subprocess.Popen([bootstrapPath], env=my_env)
+            output,error = proc.communicate()
 
     # Run jit-format
 
     returncode = 0
-    jitutilsBin = os.path.join(coreclr, "jitutils", "bin")
+    jitutilsBin = os.path.join(os.path.dirname(bootstrapPath), "jitutils", "bin")
     my_env["PATH"] = jitutilsBin + os.pathsep + my_env["PATH"]
     current_dir = os.getcwd()
 
@@ -157,9 +178,12 @@ def main(argv):
 
     os.chdir(current_dir)
 
+    patchFilePath = os.path.join(coreclr, "format.patch")
+
     if returncode != 0:
         # Create a patch file
-        patchFile = open("format.patch", "w")
+        print("Creating patch file " + patchFilePath)
+        patchFile = open(patchFilePath, "w")
         proc = subprocess.Popen(["git", "diff", "--patch", "-U20"], env=my_env, stdout=patchFile)
         output,error = proc.communicate()
 

src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByOut/CMakeLists.txt

@@ -1,13 +1,14 @@
 cmake_minimum_required (VERSION 2.6) 
-project (ReversePInvokePassingByOutNative) 
+project (RPIP_ByOutNative) 
 include ("${CLR_INTEROP_TEST_ROOT}/Interop.cmake") 
 include_directories("..") 
 set(SOURCES 
     ReversePInvokePassingByOutNative.cpp 
 ) 
 # Additional files to reference: 
 # add the executable 
-add_library (ReversePInvokePassingByOutNative SHARED ${SOURCES}) 
-target_link_libraries(ReversePInvokePassingByOutNative ${LINK_LIBRARIES_ADDITIONAL}) 
+add_library (RPIP_ByOutNative SHARED ${SOURCES}) 
+set_property (TARGET RPIP_ByOutNative PROPERTY OUTPUT_NAME ReversePInvokePassingByOutNative)
+target_link_libraries(RPIP_ByOutNative ${LINK_LIBRARIES_ADDITIONAL}) 
 # add the install targets 
-install (TARGETS ReversePInvokePassingByOutNative DESTINATION bin) 
+install (TARGETS RPIP_ByOutNative DESTINATION bin) 

src/coreclr/tests/src/Interop/PInvoke/SizeParamIndex/ReversePInvoke/PassingByRef/CMakeLists.txt

@@ -1,13 +1,14 @@
 cmake_minimum_required (VERSION 2.6) 
-project (ReversePInvokePassingByRefNative) 
+project (RPIP_ByRefNative)
 include ("${CLR_INTEROP_TEST_ROOT}/Interop.cmake") 
 include_directories("..") 
 set(SOURCES 
     ReversePInvokePassingByRefNative.cpp 
 ) 
 # Additional files to reference: 
 # add the executable 
-add_library (ReversePInvokePassingByRefNative SHARED ${SOURCES}) 
-target_link_libraries(ReversePInvokePassingByRefNative ${LINK_LIBRARIES_ADDITIONAL}) 
+add_library (RPIP_ByRefNative SHARED ${SOURCES}) 
+set_property (TARGET RPIP_ByRefNative PROPERTY OUTPUT_NAME ReversePInvokePassingByRefNative)
+target_link_libraries(RPIP_ByRefNative ${LINK_LIBRARIES_ADDITIONAL}) 
 # add the install targets 
-install (TARGETS ReversePInvokePassingByRefNative DESTINATION bin) 
+install (TARGETS RPIP_ByRefNative DESTINATION bin) 

src/coreclr/tests/src/runtest.proj

@@ -26,7 +26,7 @@
       <AllTestDirsPaths Include="@(AllTestDirsNonCanonicalPaths)" >
         <Path>$([System.IO.Path]::GetFullPath(%(Identity)))</Path>
       </AllTestDirsPaths>
-      <SkipTestDirsPaths Include="$([System.IO.Path]::GetFullPath('$(XunitTestBinBase)%(_SkipTestDir.Identity)'))" />
+      <SkipTestDirsPaths Include="$([System.IO.Path]::GetFullPath('$(XunitTestBinBase)\%(_SkipTestDir.Identity)'))" />
       <NonExcludedTestDirectories Include="@(AllTestDirsPaths -> '%(Path)')" Exclude="@(SkipTestDirsPaths)" />
       <TopLevelDirectories Include="@(NonExcludedTestDirectories)" />
       <SecondLevel Include="$([System.IO.Directory]::GetDirectories(%(TopLevelDirectories.Identity)))" />