diff --git a/eng/ci/host.benchmarks.yml b/eng/ci/host.benchmarks.yml index fcccfb8653..0df171bdb5 100644 --- a/eng/ci/host.benchmarks.yml +++ b/eng/ci/host.benchmarks.yml @@ -36,13 +36,26 @@ extends: template: v1/1ES.Unofficial.PipelineTemplate.yml@1es parameters: pool: - name: 1es-pool-azfunc-large - image: 1es-windows-2022 + name: 1es-pool-azfunc-benchmarking-large + image: 1es-windows-2022-benchmark-runner-vanilla os: windows stages: - - stage: HttpApps + - stage: Setup + displayName: Setup & start agents + jobs: + - template: /eng/ci/templates/official/jobs/setup-benchmark-agents.yml@self + parameters: + functionAppName: HelloHttpNet9 + agentName: HelloHttpNet9_APP + - template: /eng/ci/templates/official/jobs/setup-benchmark-agents.yml@self + parameters: + agentName: HelloHttpNet9NoProxy_APP + functionAppName: HelloHttpNet9NoProxy + + - stage: Run displayName: Run Benchmarks + dependsOn: [] # Force this stage to run independently and in parallel with other stages. jobs: - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self parameters: @@ -50,9 +63,11 @@ extends: functionAppName: HelloHttpNet9 # App with ASP.NET Integration additionalCrankArgs: $(AdditionalCrankArguments) # Pipeline variable to pass additional arguments to crank command. storeResultsInDatabase: ${{ variables.storeBenchmarkResultsInDatabase }} + agentName: HelloHttpNet9_APP - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self parameters: description: .NET9 Worker Application functionAppName: HelloHttpNet9NoProxy # App without ASP.NET Integration additionalCrankArgs: $(AdditionalCrankArguments) storeResultsInDatabase: ${{ variables.storeBenchmarkResultsInDatabase }} + agentName: HelloHttpNet9NoProxy_APP diff --git a/eng/ci/templates/official/jobs/run-benchmarks.yml b/eng/ci/templates/official/jobs/run-benchmarks.yml index 0af987efd2..a11f556199 100644 --- a/eng/ci/templates/official/jobs/run-benchmarks.yml +++ b/eng/ci/templates/official/jobs/run-benchmarks.yml @@ -9,22 +9,25 @@ parameters: - name: additionalCrankArgs type: string default: '' +- name: agentName + type: string jobs: - job: ${{ parameters.functionAppName }} variables: + agentName: ${{ parameters.agentName }} runDescription: ${{ parameters.description }} functionApp: ${{ parameters.functionAppName }} benchmarkArtifactName: benchmark_results_$(functionApp) functionAppOutputPath: $(Build.ArtifactStagingDirectory)/FunctionApps/$(functionApp) benchmarkResultsJsonPath: $(Build.ArtifactStagingDirectory)/BenchmarkResults/$(Build.BuildNumber)_$(functionApp).json functionsWorkerRuntime: 'dotnet-isolated' - crankAgentUrl: "http://localhost:5010" # Default crank agent URL. configFilePath: "./eng/perf/http.benchmarks.yml" hostLocation: "./../../" baselineBenchmarkResultFilePath: '' baselineBenchmarkResultsDownloadDir: $(Pipeline.Workspace)/BenchmarkBaselineResult + appAgentIpAddress: '' templateContext: inputs: @@ -55,8 +58,11 @@ jobs: - script: dotnet tool install -g Microsoft.Crank.Agent --version "0.2.0-*" displayName: Install Microsoft.Crank.Agent tool + - script: dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*" + displayName: Install Microsoft.Crank.Controller tool + - pwsh: Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent' - displayName: 'Start crank-agent' + displayName: Start crank-agent - task: CopyFiles@2 displayName: Copy benchmark apps to temp location @@ -77,11 +83,44 @@ jobs: arguments: -c Release -o $(functionAppOutputPath) -f net9.0 workingDirectory: $(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp) - - script: dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*" - displayName: Install Microsoft.Crank.Controller + - task: AzureCLI@2 + displayName: Get Remote Agent IP Address + inputs: + azureSubscription: $(ServiceConnection) + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $maxAttempts = 60 + $attempt = 0 + + do { + $Entity = az storage entity show ` + --account-name $(StorageAccount) ` + --auth-mode login ` + --table-name $(BenchmarkAgentInfoTableName) ` + --partition-key $(Build.BuildNumber) ` + --row-key $(agentName) ` + --select AgentIPAddress + + if ($Entity) { + $AgentIPAddress = ($Entity | ConvertFrom-Json).AgentIPAddress + Write-Host "##vso[task.setvariable variable=appAgentIpAddress]$AgentIPAddress" + break + } else { + Write-Host "Entity not found. Retrying in 30 seconds..." + Start-Sleep -Seconds 30 + $attempt++ + } + + if ($attempt -ge $maxAttempts) { + Write-Host "Maximum attempts reached. Exiting..." + exit 1 + } + } while (-not $Entity) - pwsh: | $crankArgs = "--config $(configFilePath) --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(Build.SourceVersion) --property buildNumber=$(Build.BuildNumber) --property buildId=$(Build.BuildId) --property sourceBranch=$(Build.SourceBranch) --variable FunctionsWorkerRuntime=$(functionsWorkerRuntime) --variable HostLocation=$(hostLocation) --variable FunctionAppPath=$(functionAppOutputPath)" + $crankArgs += " --variable CrankAgentAppVm=$(appAgentIpAddress) --variable CrankAgentLoadVm=localhost --variable AspNetUrls=http://$(appAgentIpAddress):5000" $crankArgs += " ${{ parameters.additionalCrankArgs }}" $command = "crank $crankArgs" @@ -95,7 +134,7 @@ jobs: # Retrieve function host logs - pwsh: | - $url = "$(crankAgentUrl)/jobs/1/output" + $url = "http://$(appAgentIpAddress):5010/jobs/1/output" Write-Host "Fetching logs from: $url" $response = Invoke-WebRequest -Uri $url -Method GET -UseBasicParsing Write-Host $response.Content diff --git a/eng/ci/templates/official/jobs/setup-benchmark-agents.yml b/eng/ci/templates/official/jobs/setup-benchmark-agents.yml new file mode 100644 index 0000000000..7c14a1a2c1 --- /dev/null +++ b/eng/ci/templates/official/jobs/setup-benchmark-agents.yml @@ -0,0 +1,111 @@ +parameters: +- name: functionAppName + type: string +- name: agentName + type: string +jobs: +- job: ${{ parameters.functionAppName }} + + variables: + agentName: ${{ parameters.agentName }} + functionApp: ${{ parameters.functionAppName }} + functionAppOutputPath: $(Build.ArtifactStagingDirectory)/FunctionApps/$(functionApp) + + steps: + + - template: /eng/ci/templates/install-dotnet.yml@self + + - script: dotnet tool install -g Microsoft.Crank.Agent --version "0.2.0-*" + displayName: Install Microsoft.Crank.Agent tool + + - pwsh: Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent' + displayName: Start crank agent + + - pwsh: | + dotnet --info + displayName: Print .NET info + + - task: CopyFiles@2 + displayName: Copy benchmark apps to temp location + inputs: + SourceFolder: '$(Build.SourcesDirectory)/test/Performance/Apps' + Contents: '**/*' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps' + CleanTargetFolder: true + + - task: DotNetCoreCLI@2 + displayName: Publish benchmark app + inputs: + command: publish + publishWebProjects: false + zipAfterPublish: false + modifyOutputPath: false + projects: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)/HelloHttp.csproj' + arguments: -c Release -o $(functionAppOutputPath) -f net9.0 -v n + workingDirectory: $(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp) + + - script: | + netsh advfirewall firewall add rule name="Open Port 5000" dir=in action=allow protocol=TCP localport=5000 + netsh advfirewall firewall add rule name="Open Port 5010" dir=in action=allow protocol=TCP localport=5010 + displayName: Open port 5000 and 5010 + + - task: PowerShell@2 + displayName: Get Agent IP Address + inputs: + targetType: 'inline' + script: | + $ipAddress = (Test-Connection -ComputerName (hostname) -Count 1).IPv4Address.IPAddressToString + Write-Host "IP Address: $ipAddress" + Write-Host "##vso[task.setvariable variable=agentIp]$ipAddress" + + - task: AzureCLI@2 + displayName: Persist Agent IP Address + inputs: + azureSubscription: $(ServiceConnection) + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + az storage entity insert ` + --auth-mode login ` + --account-name $(StorageAccount) ` + --table-name $(BenchmarkAgentInfoTableName) ` + --entity PartitionKey=$(Build.BuildNumber) RowKey=$(agentName) AgentName=$(agentName) AgentIPAddress=$(agentIp) + + - pwsh: | + $url = "http://localhost:5010/jobs/all" + Write-Host "Calling $url to check benchmark job status" + $maxAttempts = 60 + $attempt = 0 + + while ($attempt -lt $maxAttempts) { + $response = Invoke-WebRequest -Uri $url -Method Get -ErrorAction Stop + $data = $response.Content | ConvertFrom-Json + $completedJobCount = ($data | Where-Object { $_.state -eq "Deleted" }).Count + + if ($completedJobCount -gt 0) { + Write-Host "Found at least 1 job with 'state' = 'Deleted'. Exiting task." + exit 0 + } + + Write-Host "No completed jobs found. Polling again in 30 seconds..." + Start-Sleep -Seconds 30 + $attempt++ + } + + Write-Host "Maximum attempts reached. Exiting..." + exit 1 + displayName: Wait for job completion + + - task: AzureCLI@2 + displayName: Clean up storage + condition: always() + inputs: + azureSubscription: $(ServiceConnection) + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + az storage entity delete ` + --auth-mode login ` + --account-name $(StorageAccount) ` + --table-name $(BenchmarkAgentInfoTableName) ` + --partition-key $(Build.BuildNumber) --row-key=$(agentName) diff --git a/eng/perf/http.benchmarks.yml b/eng/perf/http.benchmarks.yml index 906b42193b..1c5eba1c6e 100644 --- a/eng/perf/http.benchmarks.yml +++ b/eng/perf/http.benchmarks.yml @@ -12,6 +12,7 @@ jobs: environmentVariables: FUNCTIONS_WORKER_RUNTIME: '{{FunctionsWorkerRuntime}}' AzureWebJobsScriptRoot: '{{FunctionAppPath}}' + ASPNETCORE_URLS: '{{ AspNetUrls }}' scenarios: hellohttp: @@ -27,11 +28,12 @@ scenarios: profiles: win2022: variables: - serverAddress: localhost + serverUri: http://{{ CrankAgentAppVm }} + serverPort: 5000 jobs: hostruntime: endpoints: - - http://localhost:5010 + - http://{{ CrankAgentAppVm }}:5010 load: endpoints: - - http://localhost:5010 + - http://{{ CrankAgentLoadVm }}:5010