[PERF] Move performance testing logic into python scripts (#101143)

* Initial debugging work for python pipeline

* Use custom performance branch

* Remove template reference

* Move resources to perf.yml

* Ensure runtime repo is checked out to SourcesDirectory

* Try fix python run

* Set python in perf-job.yml

* More fixes

* More fixes

* Set most params in perf-job.yml

* Fix variable declaration

* Fix some bugs in setup parameters

* Ensure Helix Access Token is passed in

* Fix osGroup arg parsing

* Fix ios args and use local commit time

* Fix physical promotion yaml

* Fix project file path

* Ensure internal param is passed

* Ensure project file is passed in

* Try dynamically load performance repo

* Try use condition inside repositories definition

* Try put condition on repositories object

* Try use variables to conditionally create repository

* Fix conditional variable definition

* Just use ADO for performance repo

* Use updated repository name for scenarios jobs

* Use HELIX_WORKITEM_ROOT for performance dir

* Fix paths for android APKs

* Remove accidental submodule

* Use new project file location for arm64 jit job

* Move archive of work item dir to python

* Use github for performance repo checkout on public jobs

* Another attempt at conditional repository definition

* Another attempt at conditional repository

* Use multi repo checkout only for internal runs

* Ensure ios scenarios and wasm jobs are working

* Only copy files from dotnet-runtime directory on internal jobs

* Use branch instead of ref for performance repository clone

* Clone performance repo to correct folder in public jobs

eng/pipelines/common/variables.yml

@@ -58,6 +58,4 @@ variables:
                 ne(variables['isExtraPlatformsBuild'], true),
                 eq(variables['isRollingBuild'], true))) ]
 
-- template: /eng/pipelines/common/perf-variables.yml
-
-- template: /eng/common/${{ parameters.templatePath }}/variables/pool-providers.yml
+- template: /eng/common/${{ parameters.templatePath }}/variables/pool-providers.yml

eng/pipelines/coreclr/perf-non-wasm-jobs.yml

@@ -1,3 +1,6 @@
+parameters:
+  perfBranch: 'main'
+
 jobs:
 
 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'Schedule')) }}:
@@ -100,6 +103,7 @@ jobs:
   - template: /eng/pipelines/coreclr/templates/build-and-run-perf-ios-scenarios.yml
     parameters:
       hybridGlobalization: True
+      perfBranch: ${{ parameters.perfBranch }}
 
   # run android scenarios
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -112,10 +116,11 @@ jobs:
       jobParameters:
         testGroup: perf
         runtimeType: AndroidMono
-        projectFile: android_scenarios.proj
+        projectFile: $(Build.SourcesDirectory)/eng/testing/performance/android_scenarios.proj
         runKind: android_scenarios
         runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml
         logicalmachine: 'perfpixel4a'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run mono microbenchmarks perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -129,10 +134,11 @@ jobs:
         testGroup: perf
         liveLibrariesBuildConfig: Release
         runtimeType: mono
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro_mono
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run mono interpreter perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -147,10 +153,11 @@ jobs:
         liveLibrariesBuildConfig: Release
         runtimeType: mono
         codeGenType: 'Interpreter'
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro_mono
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run mono aot microbenchmarks perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -165,10 +172,11 @@ jobs:
         liveLibrariesBuildConfig: Release
         runtimeType: mono
         codeGenType: 'AOT'
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro_mono
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run coreclr perftiger microbenchmarks perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -184,10 +192,11 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run coreclr perftiger microbenchmarks no dynamic pgo perf jobs
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -200,11 +209,12 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
         pgoRunType: -NoDynamicPGO
+        perfBranch: ${{ parameters.perfBranch }}
 
   - template: /eng/pipelines/common/platform-matrix.yml
     parameters:
@@ -216,11 +226,12 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
         pgoRunType: --nodynamicpgo
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run coreclr perftiger microbenchmarks no R2R perf jobs
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -233,11 +244,12 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
         r2rRunType: -NoR2R
+        perfBranch: ${{ parameters.perfBranch }}
 
   - template: /eng/pipelines/common/platform-matrix.yml
     parameters:
@@ -249,11 +261,12 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
         r2rRunType: --nor2r
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run coreclr perfowl microbenchmarks perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -267,10 +280,11 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perfowl'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run coreclr perfviper microbenchmarks perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -284,10 +298,11 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perfviper'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run coreclr perfviper microbenchmarks perf jitoptrepeat jobs
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -301,11 +316,12 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perfviper'
         experimentName: 'jitoptrepeat'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # run coreclr crossgen perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -319,10 +335,11 @@ jobs:
       jobParameters:
         testGroup: perf
         liveLibrariesBuildConfig: Release
-        projectFile: crossgen_perf.proj
+        projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj
         runKind: crossgen_scenarios
         runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml
         logicalmachine: 'perftiger_crossgen'
+        perfBranch: ${{ parameters.perfBranch }}
 
   # build mono runtime packs
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -368,3 +385,4 @@ jobs:
               archiveExtension: '.tar.gz'
               archiveType: tar
               tarCompression: gz
+              perfBranch: ${{ parameters.perfBranch }}

eng/pipelines/coreclr/perf-wasm-jobs.yml

@@ -7,7 +7,7 @@ parameters:
   onlySanityCheck: false
   downloadSpecificBuild: null # { buildId, pipeline, branchName, project }
   collectHelixLogsScript: ''
-  perfForkToUse: {} # url, branch
+  perfBranch: 'main'
 
 jobs:
 
@@ -43,14 +43,14 @@ jobs:
         liveLibrariesBuildConfig: Release
         runtimeType: wasm
         codeGenType: 'wasm'
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
         javascriptEngine: 'javascriptcore'
         collectHelixLogsScript: ${{ parameters.collectHelixLogsScript }}
         downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
-        perfForkToUse: ${{ parameters.perfForkToUse }}
+        perfBranch: ${{ parameters.perfBranch }}
         #downloadSpecificBuild:
           #buildId: '1693181'
           #pipeline: 'perf-wasm'
@@ -69,14 +69,14 @@ jobs:
         livelibrariesbuildconfig: Release
         runtimetype: wasm
         codegentype: 'aot'
-        projectfile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runkind: micro
         runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
         javascriptengine: 'javascriptcore'
         collectHelixLogsScript: ${{ parameters.collectHelixLogsScript }}
         downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
-        perfForkToUse: ${{ parameters.perfForkToUse }}
+        perfBranch: ${{ parameters.perfBranch }}
 
 - ${{ if eq(parameters.runProfile, 'v8') }}:
   - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}:
@@ -110,7 +110,7 @@ jobs:
         liveLibrariesBuildConfig: Release
         runtimeType: wasm
         codeGenType: 'wasm'
-        projectFile: microbenchmarks.proj
+        projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
         runKind: micro
         runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
         logicalmachine: 'perftiger'
@@ -121,7 +121,7 @@ jobs:
         compare: ${{ parameters.compare }}
         onlySanityCheck: ${{ parameters.onlySanityCheck }}
         downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
-        perfForkToUse: ${{ parameters.perfForkToUse }}
+        perfBranch: ${{ parameters.perfBranch }}
 
   #run mono wasm aot microbenchmarks perf job
   # Disabled for runtime-wasm-perf on PRs due to https://github.com/dotnet/runtime/issues/95101
@@ -138,7 +138,7 @@ jobs:
           livelibrariesbuildconfig: Release
           runtimetype: wasm
           codegentype: 'aot'
-          projectfile: microbenchmarks.proj
+          projectFile: $(Agent.BuildDirectory)/performance/eng/performance/helix.proj
           runkind: micro
           runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml
           logicalmachine: 'perftiger'
@@ -149,7 +149,7 @@ jobs:
           compare: ${{ parameters.compare }}
           onlySanityCheck: ${{ parameters.onlySanityCheck }}
           downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
-          perfForkToUse: ${{ parameters.perfForkToUse }}
+          perfBranch: ${{ parameters.perfBranch }}
 
   # run mono wasm blazor perf job
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -163,14 +163,14 @@ jobs:
         testGroup: perf
         liveLibrariesBuildConfig: Release
         runtimeType: wasm
-        projectFile: blazor_perf.proj
+        projectFile: $(Build.SourcesDirectory)/eng/testing/performance/blazor_perf.proj
         runKind: blazor_scenarios
         runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml
         # For working with a newer sdk, and previous tfm (eg. 9.0 sdk, and net8.0 tfm)
-        #additionalSetupParameters: '--dotnetversions 8.0.0' # passed to performance-setup.sh
+        #additionalSetupParameters: '--dotnetversions 8.0.0' # passed to run-performance-job.py
         logicalmachine: 'perftiger'
         downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
-        perfForkToUse: ${{ parameters.perfForkToUse }}
+        perfBranch: ${{ parameters.perfBranch }}
 
 - ${{if or(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), in(variables['Build.DefinitionName'], 'runtime-wasm-perf')) }}:
   # run mono wasm blazor perf job
@@ -185,12 +185,12 @@ jobs:
         testGroup: perf
         liveLibrariesBuildConfig: Release
         runtimeType: wasm
-        projectFile: blazor_perf.proj
+        projectFile: $(Build.SourcesDirectory)/eng/testing/performance/blazor_perf.proj
         runKind: blazor_scenarios
         runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml
         # For working with a newer sdk, and previous tfm (eg. 9.0 sdk, and net8.0 tfm)
-        #additionalSetupParameters: '--dotnetversions 8.0.0' # passed to performance-setup.sh
+        #additionalSetupParameters: '--dotnetversions 8.0.0' # passed to run-performance-job.py
         logicalmachine: 'perftiger'
         downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }}
-        perfForkToUse: ${{ parameters.perfForkToUse }}
         hybridGlobalization: True
+        perfBranch: ${{ parameters.perfBranch }}

eng/pipelines/coreclr/perf.yml

@@ -1,3 +1,9 @@
+parameters:
+- name: perfBranch
+  displayName: Performance Repo Branch
+  type: string
+  default: 'main'
+
 trigger:
   batch: true
   branches:
@@ -19,7 +25,7 @@ trigger:
     - THIRD-PARTY-NOTICES.TXT
 
 variables:
-  - template: /eng/pipelines/common/variables.yml
+- template: /eng/pipelines/common/variables.yml
 
 #
 # For the 'schedule' case, only wasm/jsc perf jobs are run.
@@ -46,9 +52,12 @@ extends:
       - template: /eng/pipelines/coreclr/perf-wasm-jobs.yml
         parameters:
           collectHelixLogsScript: ${{ variables._wasmCollectHelixLogsScript }}
+          perfBranch: ${{ parameters.perfBranch }}
           #${{ and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')) }}:
           #  runProfile: 'non-v8'
           ${{ if ne(variables['System.TeamProject'], 'public') }}:
             runProfile: 'v8'
 
       - template: /eng/pipelines/coreclr/perf-non-wasm-jobs.yml
+        parameters:
+          perfBranch: ${{ parameters.perfBranch }}