Add benchmarking pipeline

eng/ci/host.benchmarks.yml

@@ -24,7 +24,7 @@ resources:
     ref: refs/tags/release
 
 variables:
-  - template: /eng/ci/templates/variables/crank.yml@self
+  - template: /eng/ci/templates/variables/benchmarks.yml@self
   - template: /ci/variables/cfs.yml@eng
 
 extends:

eng/ci/templates/official/jobs/run-benchmarks.yml

@@ -16,10 +16,12 @@ jobs:
   variables:
     runDescription: ${{ parameters.description }}
     functionApp: ${{ parameters.functionAppName }}
-    functionAppOutputPath: $(Build.ArtifactStagingDirectory)\FunctionApps\$(functionApp)
-    benchmarkResultsJsonPath: "$(Build.ArtifactStagingDirectory)\\BenchmarkResults\\$(functionApp).json"
+    functionAppOutputPath: $(Build.ArtifactStagingDirectory)/FunctionApps/$(functionApp)
+    benchmarkResultsJsonPath: "$(Build.ArtifactStagingDirectory)/BenchmarkResults/$(buildNumber)_$(functionApp).json"
     functionsWorkerRuntime: 'dotnet-isolated'
     crankAgentUrl: "http://localhost:5010"  # Default crank agent URL.
+    configFilePath: "./eng/perf/http.benchmarks.yml"
+    hostLocation: "./../../"
 
   templateContext:
     outputParentDirectory: $(Build.ArtifactStagingDirectory)
@@ -31,91 +33,70 @@ jobs:
 
   steps:
 
-  - checkout: self
-    path: sourcecode/host
-
   - task: AzureKeyVault@2
     condition: and(succeeded(), eq('${{ parameters.storeResultsInDatabase }}', true))
     inputs:
-      azureSubscription:  Azure-Functions-Host-CI-internal
+      azureSubscription: Azure-Functions-Host-CI-internal
       KeyVaultName: functions-perf-crank-kv
       SecretsFilter: BenchmarkResultsSqlConnectionString
       RunAsPreJob: false
 
-  - task: PowerShell@2
-    condition: and(succeeded(), eq('${{ parameters.storeResultsInDatabase }}', true))
-    displayName: Set environment variables
-    inputs:
-      targetType: 'inline'
-      script: |
-        $env:CRANK_SQL_CONNECTION_STRING = "$(BenchmarkResultsSqlConnectionString)"
-        Write-Host "##vso[task.setvariable variable=CRANK_SQL_CONNECTION_STRING]$env:CRANK_SQL_CONNECTION_STRING"
-
   - template: /eng/ci/templates/install-dotnet.yml@self
 
-  - task: CopyFiles@2
-    displayName: Copy benchmark app files to temp location
-    inputs:
-      SourceFolder: '$(Build.SourcesDirectory)\tools\Crank\BenchmarkApps\DotNet'
-      Contents: '**/*'
-      TargetFolder: '$(Build.ArtifactStagingDirectory)\BenchmarkApps\DotNet'
-      CleanTargetFolder: true
-
   - script: dotnet tool install -g Microsoft.Crank.Agent --version "0.2.0-*"
     displayName: Install Microsoft.Crank.Agent tool
+
   - task: PowerShell@2
     displayName: Start crank-agent
     inputs:
       targetType: 'inline'
       script: |
         Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent'
 
+  - task: CopyFiles@2
+    displayName: Copy benchmark apps to temp location
+    inputs:
+      SourceFolder: '$(Build.SourcesDirectory)/test/Performance/Apps'
+      Contents: '**/*'
+      TargetFolder: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps'
+      CleanTargetFolder: true
+
   - task: DotNetCoreCLI@2
     displayName: Publish $(functionApp) app
     inputs:
       command: publish
       publishWebProjects: false
       zipAfterPublish: false
       modifyOutputPath: false
-      projects: '$(Build.ArtifactStagingDirectory)\BenchmarkApps\DotNet\$(functionApp)\HelloHttp.csproj'
+      projects: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)/HelloHttp.csproj'
       arguments: -c Release -o $(functionAppOutputPath) -f net9.0
-      workingDirectory: $(Build.ArtifactStagingDirectory)\BenchmarkApps\DotNet\$(functionApp)
+      workingDirectory: $(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)
 
-  - script: dotnet tool install -g Microsoft.Crank.Controller  --version "0.2.0-*"
-    displayName: Install Microsoft.Crank.Controller 
+  - script: dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*"
+    displayName: Install Microsoft.Crank.Controller
 
   - task: PowerShell@2
     displayName: Run crank-controller
     inputs:
       targetType: 'inline'
       script: |
-        $workerRuntime = "$(functionsWorkerRuntime)"
-        $functionAppOutputPath = "$(functionAppOutputPath)"
-        $configFilePath = ".\..\host\eng\perf\http.benchmarks.yml"
-        $hostLocation = ".\..\..\..\host"
-
-        $crankArgs = "--config  $configFilePath --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-metadata --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(sourceVersion) --property buildNumber=$(buildNumber) --property buildId=$(buildId) --variable FunctionsWorkerRuntime=$workerRuntime --variable HostLocation=$hostLocation --variable FunctionAppPath=$functionAppOutputPath"
-        if ($env:CRANK_SQL_CONNECTION_STRING) {
-          $crankArgs += " --table HttpBenchmarks --sql CRANK_SQL_CONNECTION_STRING"
-        }
+        $crankArgs = "--config $(configFilePath) --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-metadata --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(sourceVersion) --property buildNumber=$(buildNumber) --property buildId=$(buildId) --variable FunctionsWorkerRuntime=$(functionsWorkerRuntime) --variable HostLocation=$(hostLocation) --variable FunctionAppPath=$(functionAppOutputPath)"
         $crankArgs += " ${{ parameters.additionalCrankArgs }}"
         $command = "crank $crankArgs"
+
+        if ('${{ parameters.storeResultsInDatabase }}' -eq 'true') {
+            $command += " --table HttpBenchmarks --sql `"$(BenchmarkResultsSqlConnectionString)`""
+        }
+
         Write-Host "Running command: $command"
         Invoke-Expression $command
 
   - task: PowerShell@2
-    displayName: Output logs
+    displayName: Functions host logs
     inputs:
       targetType: 'inline'
       script: |
         $url = "$(crankAgentUrl)/jobs/1/output"
         Write-Host "Making GET request to: $url to get logs"
-
-        try {
-            $response = Invoke-WebRequest -Uri $url -Method GET -UseBasicParsing
-            Write-Host $response.Content
-        } catch {
-            Write-Host "Failed to call the REST API."
-            Write-Host $_.Exception.Message
-            exit 1
-        }
+        $response = Invoke-WebRequest -Uri $url -Method GET -UseBasicParsing
+        Write-Host $response.Content

eng/ci/templates/variables/crank.yml → eng/ci/templates/variables/benchmarks.yml

@@ -6,7 +6,4 @@ variables:
 - name: sourceVersion
   value: $(Build.SourceVersion)
 - name: storeBenchmarkResultsInDatabase
-  ${{ if eq(variables['Build.Reason'], 'Schedule') }}:
-    value: true
-  ${{ else }}:
-    value: false
+  value: ${{ eq(variables['Build.Reason'], 'Schedule') }}

tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/Hello.cs → test/Performance/Apps/HelloHttpNet9/Hello.cs

@@ -5,19 +5,12 @@
 
 namespace HelloHttpNet9
 {
-    public class Hello
+    public sealed class Hello(ILogger<Hello> logger)
     {
-        private readonly ILogger<Hello> _logger;
-
-        public Hello(ILogger<Hello> logger)
-        {
-            _logger = logger;
-        }
-
         [Function("Hello")]
         public IActionResult Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequest req)
         {
-            _logger.LogInformation("C# HTTP trigger function processed a request.");
+            logger.LogInformation("C# HTTP trigger function processed a request.");
             return new OkObjectResult("Welcome to Azure Functions!");
         }
     }

tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/HelloHttp.csproj → test/Performance/Apps/HelloHttpNet9/HelloHttp.csproj

@@ -13,16 +13,4 @@
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" Version="2.0.0" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="2.0.0" />
   </ItemGroup>
-  <ItemGroup>
-    <None Update="host.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-    <None Update="local.settings.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <CopyToPublishDirectory>Never</CopyToPublishDirectory>
-    </None>
-  </ItemGroup>
-  <ItemGroup>
-    <Using Include="System.Threading.ExecutionContext" Alias="ExecutionContext"/>
-  </ItemGroup>
 </Project>

tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/Hello.cs → test/Performance/Apps/HelloHttpNet9NoProxy/Hello.cs

@@ -5,19 +5,12 @@
 
 namespace HelloHttpNet9
 {
-    public class Hello
+    public sealed class Hello(ILogger<Hello> logger)
     {
-        private readonly ILogger<Hello> _logger;
-
-        public Hello(ILogger<Hello> logger)
-        {
-            _logger = logger;
-        }
-
         [Function("Hello")]
         public HttpResponseData Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req)
         {
-            _logger.LogInformation("C# HTTP trigger function processed a request.");
+            logger.LogInformation("C# HTTP trigger function processed a request.");
 
             var response = req.CreateResponse(HttpStatusCode.OK);
             response.Headers.Add("Content-Type", "text/plain; charset=utf-8");

tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/HelloHttp.csproj → test/Performance/Apps/HelloHttpNet9NoProxy/HelloHttp.csproj

@@ -13,16 +13,4 @@
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.1.0" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="2.0.0" />
   </ItemGroup>
-  <ItemGroup>
-    <None Update="host.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-    <None Update="local.settings.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <CopyToPublishDirectory>Never</CopyToPublishDirectory>
-    </None>
-  </ItemGroup>
-  <ItemGroup>
-    <Using Include="System.Threading.ExecutionContext" Alias="ExecutionContext" />
-  </ItemGroup>
 </Project>