From 8a84f4eb665252126774abe7bb6e4e11a8a721a7 Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Fri, 10 Jan 2025 13:49:12 -0800 Subject: [PATCH 1/4] Adding benchmark pipeline changes. --- eng/ci/host.benchmarks.yml | 56 ++++++++ .../official/jobs/run-benchmarks.yml | 121 ++++++++++++++++++ eng/ci/templates/variables/crank.yml | 12 ++ eng/perf/http.benchmarks.yml | 37 ++++++ .../DotNet/HelloHttpNet9/Hello.cs | 24 ++++ .../DotNet/HelloHttpNet9/HelloHttp.csproj | 28 ++++ .../DotNet/HelloHttpNet9/Program.cs | 8 ++ .../DotNet/HelloHttpNet9/global.json | 6 + .../DotNet/HelloHttpNet9/host.json | 12 ++ .../DotNet/HelloHttpNet9NoProxy/Hello.cs | 29 +++++ .../HelloHttpNet9NoProxy/HelloHttp.csproj | 28 ++++ .../DotNet/HelloHttpNet9NoProxy/Program.cs | 7 + .../DotNet/HelloHttpNet9NoProxy/global.json | 6 + .../DotNet/HelloHttpNet9NoProxy/host.json | 12 ++ 14 files changed, 386 insertions(+) create mode 100644 eng/ci/host.benchmarks.yml create mode 100644 eng/ci/templates/official/jobs/run-benchmarks.yml create mode 100644 eng/ci/templates/variables/crank.yml create mode 100644 eng/perf/http.benchmarks.yml create mode 100644 tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/Hello.cs create mode 100644 tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/HelloHttp.csproj create mode 100644 tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/Program.cs create mode 100644 tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/global.json create mode 100644 tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/host.json create mode 100644 tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/Hello.cs create mode 100644 tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/HelloHttp.csproj create mode 100644 tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/Program.cs create mode 100644 tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/global.json create mode 100644 tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/host.json diff --git a/eng/ci/host.benchmarks.yml b/eng/ci/host.benchmarks.yml new file mode 100644 index 0000000000..827ec7977d --- /dev/null +++ b/eng/ci/host.benchmarks.yml @@ -0,0 +1,56 @@ +trigger: + batch: true + branches: + include: + - shkr/crank-on-1es-pool-agent + +# No PR triggers for now +pr: none + +schedules: + - cron: "*/15 * * * *" + displayName: Nightly Build + branches: + include: + - shkr/crank-on-1es-pool-agent + always: true + +resources: + repositories: + - repository: 1es + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + - repository: eng + type: git + name: engineering + ref: refs/tags/release + +variables: + - template: /eng/ci/templates/variables/crank.yml@self + - template: /ci/variables/cfs.yml@eng + +extends: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1es + parameters: + pool: + name: 1es-pool-azfunc-large + image: 1es-windows-2022 + os: windows + + stages: + - stage: HttpApps + displayName: Run Benchmarks + jobs: + - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self + parameters: + description: .NET9 Web Application + functionAppName: HelloHttpNet9 # App with ASP.NET Integration + additionalCrankArgs: $(AdditionalCrankArguments) # Pipeline variable to pass additional arguments to crank command. + storeResultsInDatabase: ${{ variables.storeBenchmarkResultsInDatabase }} + - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self + parameters: + description: .NET9 Worker Application + functionAppName: HelloHttpNet9NoProxy # App without ASP.NET Integration + additionalCrankArgs: $(AdditionalCrankArguments) + storeResultsInDatabase: ${{ variables.storeBenchmarkResultsInDatabase }} diff --git a/eng/ci/templates/official/jobs/run-benchmarks.yml b/eng/ci/templates/official/jobs/run-benchmarks.yml new file mode 100644 index 0000000000..3e80d86e2f --- /dev/null +++ b/eng/ci/templates/official/jobs/run-benchmarks.yml @@ -0,0 +1,121 @@ +parameters: +- name: functionAppName + type: string +- name: description + type: string +- name: storeResultsInDatabase + type: boolean + default: false +- name: additionalCrankArgs + type: string + default: '' + +jobs: +- job: ${{ parameters.functionAppName }} + + variables: + runDescription: ${{ parameters.description }} + functionApp: ${{ parameters.functionAppName }} + functionAppOutputPath: $(Build.ArtifactStagingDirectory)\FunctionApps\$(functionApp) + benchmarkResultsJsonPath: "$(Build.ArtifactStagingDirectory)\\BenchmarkResults\\$(functionApp).json" + functionsWorkerRuntime: 'dotnet-isolated' + crankAgentUrl: "http://localhost:5010" # Default crank agent URL. + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + displayName: Publish benchmark results + path: $(benchmarkResultsJsonPath) + artifact: 'BenchmarkResults_$(functionApp)' + + steps: + + - checkout: self + path: sourcecode/host + + - task: AzureKeyVault@2 + condition: and(succeeded(), eq('${{ parameters.storeResultsInDatabase }}', true)) + inputs: + azureSubscription: Azure-Functions-Host-CI-internal + KeyVaultName: functions-perf-crank-kv + SecretsFilter: BenchmarkResultsSqlConnectionString + RunAsPreJob: false + + - task: PowerShell@2 + condition: and(succeeded(), eq('${{ parameters.storeResultsInDatabase }}', true)) + displayName: Set environment variables + inputs: + targetType: 'inline' + script: | + $env:CRANK_SQL_CONNECTION_STRING = "$(BenchmarkResultsSqlConnectionString)" + Write-Host "##vso[task.setvariable variable=CRANK_SQL_CONNECTION_STRING]$env:CRANK_SQL_CONNECTION_STRING" + + - template: /eng/ci/templates/install-dotnet.yml@self + + - task: CopyFiles@2 + displayName: Copy benchmark app files to temp location + inputs: + SourceFolder: '$(Build.SourcesDirectory)\tools\Crank\BenchmarkApps\DotNet' + Contents: '**/*' + TargetFolder: '$(Build.ArtifactStagingDirectory)\BenchmarkApps\DotNet' + CleanTargetFolder: true + + - script: dotnet tool install -g Microsoft.Crank.Agent --version "0.2.0-*" + displayName: Install Microsoft.Crank.Agent tool + - task: PowerShell@2 + displayName: Start crank-agent + inputs: + targetType: 'inline' + script: | + Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent' + + - task: DotNetCoreCLI@2 + displayName: Publish $(functionApp) app + inputs: + command: publish + publishWebProjects: false + zipAfterPublish: false + modifyOutputPath: false + projects: '$(Build.ArtifactStagingDirectory)\BenchmarkApps\DotNet\$(functionApp)\HelloHttp.csproj' + arguments: -c Release -o $(functionAppOutputPath) -f net9.0 + workingDirectory: $(Build.ArtifactStagingDirectory)\BenchmarkApps\DotNet\$(functionApp) + + - script: dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*" + displayName: Install Microsoft.Crank.Controller + + - task: PowerShell@2 + displayName: Run crank-controller + inputs: + targetType: 'inline' + script: | + $workerRuntime = "$(functionsWorkerRuntime)" + $functionAppOutputPath = "$(functionAppOutputPath)" + $configFilePath = ".\..\host\eng\perf\http.benchmarks.yml" + $hostLocation = ".\..\..\..\host" + + $crankArgs = "--config $configFilePath --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-metadata --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(sourceVersion) --property buildNumber=$(buildNumber) --property buildId=$(buildId) --variable FunctionsWorkerRuntime=$workerRuntime --variable HostLocation=$hostLocation --variable FunctionAppPath=$functionAppOutputPath" + if ($env:CRANK_SQL_CONNECTION_STRING) { + $crankArgs += " --table HttpBenchmarks --sql CRANK_SQL_CONNECTION_STRING" + } + $crankArgs += " ${{ parameters.additionalCrankArgs }}" + $command = "crank $crankArgs" + Write-Host "Running command: $command" + Invoke-Expression $command + + - task: PowerShell@2 + displayName: Output logs + inputs: + targetType: 'inline' + script: | + $url = "$(crankAgentUrl)/jobs/1/output" + Write-Host "Making GET request to: $url to get logs" + + try { + $response = Invoke-WebRequest -Uri $url -Method GET -UseBasicParsing + Write-Host $response.Content + } catch { + Write-Host "Failed to call the REST API." + Write-Host $_.Exception.Message + exit 1 + } \ No newline at end of file diff --git a/eng/ci/templates/variables/crank.yml b/eng/ci/templates/variables/crank.yml new file mode 100644 index 0000000000..0556e5816f --- /dev/null +++ b/eng/ci/templates/variables/crank.yml @@ -0,0 +1,12 @@ +variables: +- name: buildId + value: $(Build.BuildId) +- name: buildNumber + value: $(Build.BuildNumber) +- name: sourceVersion + value: $(Build.SourceVersion) +- name: storeBenchmarkResultsInDatabase + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + value: true + ${{ else }}: + value: false \ No newline at end of file diff --git a/eng/perf/http.benchmarks.yml b/eng/perf/http.benchmarks.yml new file mode 100644 index 0000000000..906b42193b --- /dev/null +++ b/eng/perf/http.benchmarks.yml @@ -0,0 +1,37 @@ +imports: + - https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Bombardier/bombardier.yml + +jobs: + + server: + sources: + functionshost: + localFolder: '{{HostLocation}}' + project: functionshost/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj + readyStateText: 'Application started.' + environmentVariables: + FUNCTIONS_WORKER_RUNTIME: '{{FunctionsWorkerRuntime}}' + AzureWebJobsScriptRoot: '{{FunctionAppPath}}' + +scenarios: + hellohttp: + hostruntime: + job: server + + load: + job: bombardier + variables: + serverPort: 5000 + path: /api/hello + +profiles: + win2022: + variables: + serverAddress: localhost + jobs: + hostruntime: + endpoints: + - http://localhost:5010 + load: + endpoints: + - http://localhost:5010 diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/Hello.cs b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/Hello.cs new file mode 100644 index 0000000000..d7548698ac --- /dev/null +++ b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/Hello.cs @@ -0,0 +1,24 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; + +namespace HelloHttpNet9 +{ + public class Hello + { + private readonly ILogger _logger; + + public Hello(ILogger logger) + { + _logger = logger; + } + + [Function("Hello")] + public IActionResult Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequest req) + { + _logger.LogInformation("C# HTTP trigger function processed a request."); + return new OkObjectResult("Welcome to Azure Functions!"); + } + } +} diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/HelloHttp.csproj b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/HelloHttp.csproj new file mode 100644 index 0000000000..7fdf003e7a --- /dev/null +++ b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/HelloHttp.csproj @@ -0,0 +1,28 @@ + + + net9.0 + v4 + Exe + enable + enable + true + + + + + + + + + + PreserveNewest + + + PreserveNewest + Never + + + + + + \ No newline at end of file diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/Program.cs b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/Program.cs new file mode 100644 index 0000000000..f585a51485 --- /dev/null +++ b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/Program.cs @@ -0,0 +1,8 @@ +using Microsoft.Azure.Functions.Worker.Builder; +using Microsoft.Extensions.Hosting; + +var builder = FunctionsApplication.CreateBuilder(args); + +builder.ConfigureFunctionsWebApplication(); + +builder.Build().Run(); diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/global.json b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/global.json new file mode 100644 index 0000000000..652ca01f09 --- /dev/null +++ b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/global.json @@ -0,0 +1,6 @@ +{ + "sdk": { + "rollForward": "latestPatch", + "version": "9.0.100" + } +} \ No newline at end of file diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/host.json b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/host.json new file mode 100644 index 0000000000..d4f2d4120c --- /dev/null +++ b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/host.json @@ -0,0 +1,12 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + }, + "enableLiveMetricsFilters": true + } + } +} \ No newline at end of file diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/Hello.cs b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/Hello.cs new file mode 100644 index 0000000000..63af6bc29e --- /dev/null +++ b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/Hello.cs @@ -0,0 +1,29 @@ +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; +using System.Net; + +namespace HelloHttpNet9 +{ + public class Hello + { + private readonly ILogger _logger; + + public Hello(ILogger logger) + { + _logger = logger; + } + + [Function("Hello")] + public HttpResponseData Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req) + { + _logger.LogInformation("C# HTTP trigger function processed a request."); + + var response = req.CreateResponse(HttpStatusCode.OK); + response.Headers.Add("Content-Type", "text/plain; charset=utf-8"); + response.WriteString("Welcome to Azure Functions!"); + + return response; + } + } +} diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/HelloHttp.csproj b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/HelloHttp.csproj new file mode 100644 index 0000000000..94e73f6684 --- /dev/null +++ b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/HelloHttp.csproj @@ -0,0 +1,28 @@ + + + net9.0 + v4 + Exe + enable + enable + true + + + + + + + + + + PreserveNewest + + + PreserveNewest + Never + + + + + + \ No newline at end of file diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/Program.cs b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/Program.cs new file mode 100644 index 0000000000..51336f3d1b --- /dev/null +++ b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/Program.cs @@ -0,0 +1,7 @@ +using Microsoft.Extensions.Hosting; + +var host = new HostBuilder() + .ConfigureFunctionsWorkerDefaults() + .Build(); + +host.Run(); diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/global.json b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/global.json new file mode 100644 index 0000000000..652ca01f09 --- /dev/null +++ b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/global.json @@ -0,0 +1,6 @@ +{ + "sdk": { + "rollForward": "latestPatch", + "version": "9.0.100" + } +} \ No newline at end of file diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/host.json b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/host.json new file mode 100644 index 0000000000..ee5cf5f83f --- /dev/null +++ b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/host.json @@ -0,0 +1,12 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + }, + "enableLiveMetricsFilters": true + } + } +} \ No newline at end of file From 2c79281753691618d09d2f4a002adff5ab1be2c4 Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Mon, 13 Jan 2025 16:26:12 -0800 Subject: [PATCH 2/4] update schedule to midnight. --- eng/ci/host.benchmarks.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/eng/ci/host.benchmarks.yml b/eng/ci/host.benchmarks.yml index 827ec7977d..1c9eeaa64e 100644 --- a/eng/ci/host.benchmarks.yml +++ b/eng/ci/host.benchmarks.yml @@ -1,18 +1,15 @@ -trigger: - batch: true - branches: - include: - - shkr/crank-on-1es-pool-agent +# No triggers for code push to any branch. +trigger: none # No PR triggers for now pr: none schedules: - - cron: "*/15 * * * *" + - cron: "0 0 * * *" displayName: Nightly Build branches: include: - - shkr/crank-on-1es-pool-agent + - dev always: true resources: From c7538e00332b79430cddf218f36320fb23a536af Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Tue, 14 Jan 2025 11:12:25 -0800 Subject: [PATCH 3/4] Improvements --- .../official/jobs/run-benchmarks.yml | 58 ++++++++----------- eng/ci/templates/variables/crank.yml | 5 +- .../Performance/Apps}/HelloHttpNet9/Hello.cs | 0 .../Apps}/HelloHttpNet9/HelloHttp.csproj | 12 ---- .../Apps}/HelloHttpNet9/Program.cs | 0 .../Performance/Apps}/HelloHttpNet9/host.json | 0 .../Apps}/HelloHttpNet9NoProxy/Hello.cs | 0 .../HelloHttpNet9NoProxy/HelloHttp.csproj | 12 ---- .../Apps}/HelloHttpNet9NoProxy/Program.cs | 0 .../Apps}/HelloHttpNet9NoProxy/host.json | 0 .../Performance/Apps}/global.json | 0 .../DotNet/HelloHttpNet9NoProxy/global.json | 6 -- 12 files changed, 25 insertions(+), 68 deletions(-) rename {tools/Crank/BenchmarkApps/DotNet => test/Performance/Apps}/HelloHttpNet9/Hello.cs (100%) rename {tools/Crank/BenchmarkApps/DotNet => test/Performance/Apps}/HelloHttpNet9/HelloHttp.csproj (62%) rename {tools/Crank/BenchmarkApps/DotNet => test/Performance/Apps}/HelloHttpNet9/Program.cs (100%) rename {tools/Crank/BenchmarkApps/DotNet => test/Performance/Apps}/HelloHttpNet9/host.json (100%) rename {tools/Crank/BenchmarkApps/DotNet => test/Performance/Apps}/HelloHttpNet9NoProxy/Hello.cs (100%) rename {tools/Crank/BenchmarkApps/DotNet => test/Performance/Apps}/HelloHttpNet9NoProxy/HelloHttp.csproj (61%) rename {tools/Crank/BenchmarkApps/DotNet => test/Performance/Apps}/HelloHttpNet9NoProxy/Program.cs (100%) rename {tools/Crank/BenchmarkApps/DotNet => test/Performance/Apps}/HelloHttpNet9NoProxy/host.json (100%) rename {tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9 => test/Performance/Apps}/global.json (100%) delete mode 100644 tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/global.json diff --git a/eng/ci/templates/official/jobs/run-benchmarks.yml b/eng/ci/templates/official/jobs/run-benchmarks.yml index 3e80d86e2f..bbbec1c134 100644 --- a/eng/ci/templates/official/jobs/run-benchmarks.yml +++ b/eng/ci/templates/official/jobs/run-benchmarks.yml @@ -16,8 +16,8 @@ jobs: variables: runDescription: ${{ parameters.description }} functionApp: ${{ parameters.functionAppName }} - functionAppOutputPath: $(Build.ArtifactStagingDirectory)\FunctionApps\$(functionApp) - benchmarkResultsJsonPath: "$(Build.ArtifactStagingDirectory)\\BenchmarkResults\\$(functionApp).json" + functionAppOutputPath: $(Build.ArtifactStagingDirectory)/FunctionApps/$(functionApp) + benchmarkResultsJsonPath: "$(Build.ArtifactStagingDirectory)/BenchmarkResults/$(buildNumber)_$(functionApp).json" functionsWorkerRuntime: 'dotnet-isolated' crankAgentUrl: "http://localhost:5010" # Default crank agent URL. @@ -31,9 +31,6 @@ jobs: steps: - - checkout: self - path: sourcecode/host - - task: AzureKeyVault@2 condition: and(succeeded(), eq('${{ parameters.storeResultsInDatabase }}', true)) inputs: @@ -42,27 +39,11 @@ jobs: SecretsFilter: BenchmarkResultsSqlConnectionString RunAsPreJob: false - - task: PowerShell@2 - condition: and(succeeded(), eq('${{ parameters.storeResultsInDatabase }}', true)) - displayName: Set environment variables - inputs: - targetType: 'inline' - script: | - $env:CRANK_SQL_CONNECTION_STRING = "$(BenchmarkResultsSqlConnectionString)" - Write-Host "##vso[task.setvariable variable=CRANK_SQL_CONNECTION_STRING]$env:CRANK_SQL_CONNECTION_STRING" - - template: /eng/ci/templates/install-dotnet.yml@self - - task: CopyFiles@2 - displayName: Copy benchmark app files to temp location - inputs: - SourceFolder: '$(Build.SourcesDirectory)\tools\Crank\BenchmarkApps\DotNet' - Contents: '**/*' - TargetFolder: '$(Build.ArtifactStagingDirectory)\BenchmarkApps\DotNet' - CleanTargetFolder: true - - script: dotnet tool install -g Microsoft.Crank.Agent --version "0.2.0-*" displayName: Install Microsoft.Crank.Agent tool + - task: PowerShell@2 displayName: Start crank-agent inputs: @@ -70,6 +51,15 @@ jobs: script: | Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent' + # Moving projects to a temp location due to .NET8.X SDK specified in global.json at repo root. + - task: CopyFiles@2 + displayName: Copy benchmark apps to temp location + inputs: + SourceFolder: '$(Build.SourcesDirectory)/test/Performance/Apps' + Contents: '**/*' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps' + CleanTargetFolder: true + - task: DotNetCoreCLI@2 displayName: Publish $(functionApp) app inputs: @@ -77,9 +67,9 @@ jobs: publishWebProjects: false zipAfterPublish: false modifyOutputPath: false - projects: '$(Build.ArtifactStagingDirectory)\BenchmarkApps\DotNet\$(functionApp)\HelloHttp.csproj' + projects: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)/HelloHttp.csproj' arguments: -c Release -o $(functionAppOutputPath) -f net9.0 - workingDirectory: $(Build.ArtifactStagingDirectory)\BenchmarkApps\DotNet\$(functionApp) + workingDirectory: $(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp) - script: dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*" displayName: Install Microsoft.Crank.Controller @@ -89,22 +79,22 @@ jobs: inputs: targetType: 'inline' script: | - $workerRuntime = "$(functionsWorkerRuntime)" - $functionAppOutputPath = "$(functionAppOutputPath)" - $configFilePath = ".\..\host\eng\perf\http.benchmarks.yml" - $hostLocation = ".\..\..\..\host" - - $crankArgs = "--config $configFilePath --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-metadata --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(sourceVersion) --property buildNumber=$(buildNumber) --property buildId=$(buildId) --variable FunctionsWorkerRuntime=$workerRuntime --variable HostLocation=$hostLocation --variable FunctionAppPath=$functionAppOutputPath" - if ($env:CRANK_SQL_CONNECTION_STRING) { - $crankArgs += " --table HttpBenchmarks --sql CRANK_SQL_CONNECTION_STRING" - } + $configFilePath = "./eng/perf/http.benchmarks.yml" + $hostLocation = "./../../" + + $crankArgs = "--config $configFilePath --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-metadata --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(sourceVersion) --property buildNumber=$(buildNumber) --property buildId=$(buildId) --variable FunctionsWorkerRuntime=$(functionsWorkerRuntime) --variable HostLocation=$hostLocation --variable FunctionAppPath=$(functionAppOutputPath)" $crankArgs += " ${{ parameters.additionalCrankArgs }}" $command = "crank $crankArgs" + + if ('${{ parameters.storeResultsInDatabase }}' -eq 'true') { + $command += " --table HttpBenchmarks --sql `"$(BenchmarkResultsSqlConnectionString)`"" + } + Write-Host "Running command: $command" Invoke-Expression $command - task: PowerShell@2 - displayName: Output logs + displayName: Functions host logs inputs: targetType: 'inline' script: | diff --git a/eng/ci/templates/variables/crank.yml b/eng/ci/templates/variables/crank.yml index 0556e5816f..ef78a0537c 100644 --- a/eng/ci/templates/variables/crank.yml +++ b/eng/ci/templates/variables/crank.yml @@ -6,7 +6,4 @@ variables: - name: sourceVersion value: $(Build.SourceVersion) - name: storeBenchmarkResultsInDatabase - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: - value: true - ${{ else }}: - value: false \ No newline at end of file + value: ${{ eq(variables['Build.Reason'], 'Schedule') }} \ No newline at end of file diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/Hello.cs b/test/Performance/Apps/HelloHttpNet9/Hello.cs similarity index 100% rename from tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/Hello.cs rename to test/Performance/Apps/HelloHttpNet9/Hello.cs diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/HelloHttp.csproj b/test/Performance/Apps/HelloHttpNet9/HelloHttp.csproj similarity index 62% rename from tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/HelloHttp.csproj rename to test/Performance/Apps/HelloHttpNet9/HelloHttp.csproj index 7fdf003e7a..5b0003fa26 100644 --- a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/HelloHttp.csproj +++ b/test/Performance/Apps/HelloHttpNet9/HelloHttp.csproj @@ -13,16 +13,4 @@ - - - PreserveNewest - - - PreserveNewest - Never - - - - - \ No newline at end of file diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/Program.cs b/test/Performance/Apps/HelloHttpNet9/Program.cs similarity index 100% rename from tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/Program.cs rename to test/Performance/Apps/HelloHttpNet9/Program.cs diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/host.json b/test/Performance/Apps/HelloHttpNet9/host.json similarity index 100% rename from tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/host.json rename to test/Performance/Apps/HelloHttpNet9/host.json diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/Hello.cs b/test/Performance/Apps/HelloHttpNet9NoProxy/Hello.cs similarity index 100% rename from tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/Hello.cs rename to test/Performance/Apps/HelloHttpNet9NoProxy/Hello.cs diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/HelloHttp.csproj b/test/Performance/Apps/HelloHttpNet9NoProxy/HelloHttp.csproj similarity index 61% rename from tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/HelloHttp.csproj rename to test/Performance/Apps/HelloHttpNet9NoProxy/HelloHttp.csproj index 94e73f6684..616ed44d20 100644 --- a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/HelloHttp.csproj +++ b/test/Performance/Apps/HelloHttpNet9NoProxy/HelloHttp.csproj @@ -13,16 +13,4 @@ - - - PreserveNewest - - - PreserveNewest - Never - - - - - \ No newline at end of file diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/Program.cs b/test/Performance/Apps/HelloHttpNet9NoProxy/Program.cs similarity index 100% rename from tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/Program.cs rename to test/Performance/Apps/HelloHttpNet9NoProxy/Program.cs diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/host.json b/test/Performance/Apps/HelloHttpNet9NoProxy/host.json similarity index 100% rename from tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/host.json rename to test/Performance/Apps/HelloHttpNet9NoProxy/host.json diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/global.json b/test/Performance/Apps/global.json similarity index 100% rename from tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9/global.json rename to test/Performance/Apps/global.json diff --git a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/global.json b/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/global.json deleted file mode 100644 index 652ca01f09..0000000000 --- a/tools/Crank/BenchmarkApps/DotNet/HelloHttpNet9NoProxy/global.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "sdk": { - "rollForward": "latestPatch", - "version": "9.0.100" - } -} \ No newline at end of file From 37ed31fe76e67159f7f76412f09a9121539ed3ed Mon Sep 17 00:00:00 2001 From: Shyju Krishnankutty Date: Wed, 15 Jan 2025 06:58:35 -0800 Subject: [PATCH 4/4] Optimizations --- eng/ci/host.benchmarks.yml | 2 +- .../official/jobs/run-benchmarks.yml | 25 ++++++------------- .../variables/{crank.yml => benchmarks.yml} | 0 test/Performance/Apps/HelloHttpNet9/Hello.cs | 11 ++------ .../Apps/HelloHttpNet9NoProxy/Hello.cs | 11 ++------ 5 files changed, 13 insertions(+), 36 deletions(-) rename eng/ci/templates/variables/{crank.yml => benchmarks.yml} (100%) diff --git a/eng/ci/host.benchmarks.yml b/eng/ci/host.benchmarks.yml index 1c9eeaa64e..d364bbb169 100644 --- a/eng/ci/host.benchmarks.yml +++ b/eng/ci/host.benchmarks.yml @@ -24,7 +24,7 @@ resources: ref: refs/tags/release variables: - - template: /eng/ci/templates/variables/crank.yml@self + - template: /eng/ci/templates/variables/benchmarks.yml@self - template: /ci/variables/cfs.yml@eng extends: diff --git a/eng/ci/templates/official/jobs/run-benchmarks.yml b/eng/ci/templates/official/jobs/run-benchmarks.yml index bbbec1c134..5840dbdc23 100644 --- a/eng/ci/templates/official/jobs/run-benchmarks.yml +++ b/eng/ci/templates/official/jobs/run-benchmarks.yml @@ -20,6 +20,8 @@ jobs: benchmarkResultsJsonPath: "$(Build.ArtifactStagingDirectory)/BenchmarkResults/$(buildNumber)_$(functionApp).json" functionsWorkerRuntime: 'dotnet-isolated' crankAgentUrl: "http://localhost:5010" # Default crank agent URL. + configFilePath: "./eng/perf/http.benchmarks.yml" + hostLocation: "./../../" templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) @@ -34,7 +36,7 @@ jobs: - task: AzureKeyVault@2 condition: and(succeeded(), eq('${{ parameters.storeResultsInDatabase }}', true)) inputs: - azureSubscription: Azure-Functions-Host-CI-internal + azureSubscription: Azure-Functions-Host-CI-internal KeyVaultName: functions-perf-crank-kv SecretsFilter: BenchmarkResultsSqlConnectionString RunAsPreJob: false @@ -51,7 +53,6 @@ jobs: script: | Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent' - # Moving projects to a temp location due to .NET8.X SDK specified in global.json at repo root. - task: CopyFiles@2 displayName: Copy benchmark apps to temp location inputs: @@ -71,18 +72,15 @@ jobs: arguments: -c Release -o $(functionAppOutputPath) -f net9.0 workingDirectory: $(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp) - - script: dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*" - displayName: Install Microsoft.Crank.Controller + - script: dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*" + displayName: Install Microsoft.Crank.Controller - task: PowerShell@2 displayName: Run crank-controller inputs: targetType: 'inline' script: | - $configFilePath = "./eng/perf/http.benchmarks.yml" - $hostLocation = "./../../" - - $crankArgs = "--config $configFilePath --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-metadata --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(sourceVersion) --property buildNumber=$(buildNumber) --property buildId=$(buildId) --variable FunctionsWorkerRuntime=$(functionsWorkerRuntime) --variable HostLocation=$hostLocation --variable FunctionAppPath=$(functionAppOutputPath)" + $crankArgs = "--config $(configFilePath) --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-metadata --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(sourceVersion) --property buildNumber=$(buildNumber) --property buildId=$(buildId) --variable FunctionsWorkerRuntime=$(functionsWorkerRuntime) --variable HostLocation=$(hostLocation) --variable FunctionAppPath=$(functionAppOutputPath)" $crankArgs += " ${{ parameters.additionalCrankArgs }}" $command = "crank $crankArgs" @@ -100,12 +98,5 @@ jobs: script: | $url = "$(crankAgentUrl)/jobs/1/output" Write-Host "Making GET request to: $url to get logs" - - try { - $response = Invoke-WebRequest -Uri $url -Method GET -UseBasicParsing - Write-Host $response.Content - } catch { - Write-Host "Failed to call the REST API." - Write-Host $_.Exception.Message - exit 1 - } \ No newline at end of file + $response = Invoke-WebRequest -Uri $url -Method GET -UseBasicParsing + Write-Host $response.Content diff --git a/eng/ci/templates/variables/crank.yml b/eng/ci/templates/variables/benchmarks.yml similarity index 100% rename from eng/ci/templates/variables/crank.yml rename to eng/ci/templates/variables/benchmarks.yml diff --git a/test/Performance/Apps/HelloHttpNet9/Hello.cs b/test/Performance/Apps/HelloHttpNet9/Hello.cs index d7548698ac..0b11df3f28 100644 --- a/test/Performance/Apps/HelloHttpNet9/Hello.cs +++ b/test/Performance/Apps/HelloHttpNet9/Hello.cs @@ -5,19 +5,12 @@ namespace HelloHttpNet9 { - public class Hello + public sealed class Hello(ILogger logger) { - private readonly ILogger _logger; - - public Hello(ILogger logger) - { - _logger = logger; - } - [Function("Hello")] public IActionResult Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequest req) { - _logger.LogInformation("C# HTTP trigger function processed a request."); + logger.LogInformation("C# HTTP trigger function processed a request."); return new OkObjectResult("Welcome to Azure Functions!"); } } diff --git a/test/Performance/Apps/HelloHttpNet9NoProxy/Hello.cs b/test/Performance/Apps/HelloHttpNet9NoProxy/Hello.cs index 63af6bc29e..989a94b5c2 100644 --- a/test/Performance/Apps/HelloHttpNet9NoProxy/Hello.cs +++ b/test/Performance/Apps/HelloHttpNet9NoProxy/Hello.cs @@ -5,19 +5,12 @@ namespace HelloHttpNet9 { - public class Hello + public sealed class Hello(ILogger logger) { - private readonly ILogger _logger; - - public Hello(ILogger logger) - { - _logger = logger; - } - [Function("Hello")] public HttpResponseData Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req) { - _logger.LogInformation("C# HTTP trigger function processed a request."); + logger.LogInformation("C# HTTP trigger function processed a request."); var response = req.CreateResponse(HttpStatusCode.OK); response.Headers.Add("Content-Type", "text/plain; charset=utf-8");