Add benchmarking pipeline

eng/ci/host.benchmarks.yml

@@ -0,0 +1,53 @@
+# No triggers for code push to any branch.
+trigger: none
+
+# No PR triggers for now
+pr: none
+
+schedules:
+  - cron: "0 0 * * *"
+    displayName: Nightly Build
+    branches:
+      include:
+      - dev
+    always: true
+
+resources:
+  repositories:
+  - repository: 1es
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+  - repository: eng
+    type: git
+    name: engineering
+    ref: refs/tags/release
+
+variables:
+  - template: /eng/ci/templates/variables/benchmarks.yml@self
+  - template: /ci/variables/cfs.yml@eng
+
+extends:
+  template: v1/1ES.Unofficial.PipelineTemplate.yml@1es
+  parameters:
+    pool:
+      name: 1es-pool-azfunc-large
+      image: 1es-windows-2022
+      os: windows
+
+    stages:
+    - stage: HttpApps
+      displayName: Run Benchmarks
+      jobs:
+      - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self
+        parameters:
+          description: .NET9 Web Application
+          functionAppName: HelloHttpNet9  # App with ASP.NET Integration
+          additionalCrankArgs: $(AdditionalCrankArguments)  # Pipeline variable to pass additional arguments to crank command.
+          storeResultsInDatabase: ${{ variables.storeBenchmarkResultsInDatabase }}
+      - template: /eng/ci/templates/official/jobs/run-benchmarks.yml@self
+        parameters:
+          description: .NET9 Worker Application
+          functionAppName: HelloHttpNet9NoProxy # App without ASP.NET Integration
+          additionalCrankArgs: $(AdditionalCrankArguments)
+          storeResultsInDatabase: ${{ variables.storeBenchmarkResultsInDatabase }}

eng/ci/templates/official/jobs/run-benchmarks.yml

@@ -0,0 +1,102 @@
+parameters:
+- name: functionAppName
+  type: string
+- name: description
+  type: string
+- name: storeResultsInDatabase
+  type: boolean
+  default: false
+- name: additionalCrankArgs
+  type: string
+  default: ''
+
+jobs:
+- job: ${{ parameters.functionAppName }}
+
+  variables:
+    runDescription: ${{ parameters.description }}
+    functionApp: ${{ parameters.functionAppName }}
+    functionAppOutputPath: $(Build.ArtifactStagingDirectory)/FunctionApps/$(functionApp)
+    benchmarkResultsJsonPath: "$(Build.ArtifactStagingDirectory)/BenchmarkResults/$(buildNumber)_$(functionApp).json"
+    functionsWorkerRuntime: 'dotnet-isolated'
+    crankAgentUrl: "http://localhost:5010"  # Default crank agent URL.
+    configFilePath: "./eng/perf/http.benchmarks.yml"
+    hostLocation: "./../../"
+
+  templateContext:
+    outputParentDirectory: $(Build.ArtifactStagingDirectory)
+    outputs:
+    - output: pipelineArtifact
+      displayName: Publish benchmark results
+      path: $(benchmarkResultsJsonPath)
+      artifact: 'BenchmarkResults_$(functionApp)'
+
+  steps:
+
+  - task: AzureKeyVault@2
+    condition: and(succeeded(), eq('${{ parameters.storeResultsInDatabase }}', true))
+    inputs:
+      azureSubscription: Azure-Functions-Host-CI-internal
+      KeyVaultName: functions-perf-crank-kv
+      SecretsFilter: BenchmarkResultsSqlConnectionString
+      RunAsPreJob: false
+
+  - template: /eng/ci/templates/install-dotnet.yml@self
+
+  - script: dotnet tool install -g Microsoft.Crank.Agent --version "0.2.0-*"
+    displayName: Install Microsoft.Crank.Agent tool
+
+  - task: PowerShell@2
+    displayName: Start crank-agent
+    inputs:
+      targetType: 'inline'
+      script: |
+        Start-Process powershell -ArgumentList '-NoExit', '-Command', 'crank-agent'
+
+  - task: CopyFiles@2
+    displayName: Copy benchmark apps to temp location
+    inputs:
+      SourceFolder: '$(Build.SourcesDirectory)/test/Performance/Apps'
+      Contents: '**/*'
+      TargetFolder: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps'
+      CleanTargetFolder: true
+
+  - task: DotNetCoreCLI@2
+    displayName: Publish $(functionApp) app
+    inputs:
+      command: publish
+      publishWebProjects: false
+      zipAfterPublish: false
+      modifyOutputPath: false
+      projects: '$(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)/HelloHttp.csproj'
+      arguments: -c Release -o $(functionAppOutputPath) -f net9.0
+      workingDirectory: $(Build.ArtifactStagingDirectory)/PerformanceTestApps/$(functionApp)
+
+  - script: dotnet tool install -g Microsoft.Crank.Controller --version "0.2.0-*"
+    displayName: Install Microsoft.Crank.Controller
+
+  - task: PowerShell@2
+    displayName: Run crank-controller
+    inputs:
+      targetType: 'inline'
+      script: |
+        $crankArgs = "--config $(configFilePath) --scenario hellohttp --profile win2022 --load.options.reuseBuild true --description `"$(runDescription)`" --command-line-property --no-metadata --no-measurements --json $(benchmarkResultsJsonPath) --property sourceVersion=$(sourceVersion) --property buildNumber=$(buildNumber) --property buildId=$(buildId) --variable FunctionsWorkerRuntime=$(functionsWorkerRuntime) --variable HostLocation=$(hostLocation) --variable FunctionAppPath=$(functionAppOutputPath)"
+        $crankArgs += " ${{ parameters.additionalCrankArgs }}"
+        $command = "crank $crankArgs"
+
+        if ('${{ parameters.storeResultsInDatabase }}' -eq 'true') {
+            $command += " --table HttpBenchmarks --sql `"$(BenchmarkResultsSqlConnectionString)`""
+        }
+
+        Write-Host "Running command: $command"
+        Invoke-Expression $command
+
+  - task: PowerShell@2
+    displayName: Functions host logs
+    inputs:
+      targetType: 'inline'
+      script: |
+        $url = "$(crankAgentUrl)/jobs/1/output"
+        Write-Host "Making GET request to: $url to get logs"
+        $response = Invoke-WebRequest -Uri $url -Method GET -UseBasicParsing
+        Write-Host $response.Content

eng/ci/templates/variables/benchmarks.yml

@@ -0,0 +1,9 @@
+variables:
+- name: buildId
+  value: $(Build.BuildId)
+- name: buildNumber
+  value: $(Build.BuildNumber)
+- name: sourceVersion
+  value: $(Build.SourceVersion)
+- name: storeBenchmarkResultsInDatabase
+  value: ${{ eq(variables['Build.Reason'], 'Schedule') }}

eng/perf/http.benchmarks.yml

@@ -0,0 +1,37 @@
+imports:
+  - https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Bombardier/bombardier.yml
+
+jobs:
+
+  server:
+    sources:
+      functionshost:
+        localFolder: '{{HostLocation}}'
+    project: functionshost/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj
+    readyStateText: 'Application started.'
+    environmentVariables:
+      FUNCTIONS_WORKER_RUNTIME: '{{FunctionsWorkerRuntime}}'
+      AzureWebJobsScriptRoot: '{{FunctionAppPath}}'
+
+scenarios:
+  hellohttp:
+    hostruntime:
+      job: server
+
+    load:
+      job: bombardier
+      variables:
+        serverPort: 5000
+        path: /api/hello
+
+profiles:
+  win2022:
+    variables:
+      serverAddress: localhost
+    jobs: 
+      hostruntime:
+        endpoints: 
+          - http://localhost:5010
+      load:
+        endpoints: 
+          - http://localhost:5010

test/Performance/Apps/HelloHttpNet9/Hello.cs

@@ -0,0 +1,17 @@
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Extensions.Logging;
+
+namespace HelloHttpNet9
+{
+    public sealed class Hello(ILogger<Hello> logger)
+    {
+        [Function("Hello")]
+        public IActionResult Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequest req)
+        {
+            logger.LogInformation("C# HTTP trigger function processed a request.");
+            return new OkObjectResult("Welcome to Azure Functions!");
+        }
+    }
+}

test/Performance/Apps/HelloHttpNet9/HelloHttp.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net9.0</TargetFramework>
+    <AzureFunctionsVersion>v4</AzureFunctionsVersion>
+    <OutputType>Exe</OutputType>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <PublishReadyToRun>true</PublishReadyToRun>
+  </PropertyGroup>
+  <ItemGroup>
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+    <PackageReference Include="Microsoft.Azure.Functions.Worker" Version="2.0.0" />
+    <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" Version="2.0.0" />
+    <PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="2.0.0" />
+  </ItemGroup>
+</Project>

test/Performance/Apps/HelloHttpNet9/Program.cs

@@ -0,0 +1,8 @@
+using Microsoft.Azure.Functions.Worker.Builder;
+using Microsoft.Extensions.Hosting;
+
+var builder = FunctionsApplication.CreateBuilder(args);
+
+builder.ConfigureFunctionsWebApplication();
+
+builder.Build().Run();

test/Performance/Apps/HelloHttpNet9/host.json

@@ -0,0 +1,12 @@
+{
+    "version": "2.0",
+    "logging": {
+        "applicationInsights": {
+            "samplingSettings": {
+                "isEnabled": true,
+                "excludedTypes": "Request"
+            },
+            "enableLiveMetricsFilters": true
+        }
+    }
+}

test/Performance/Apps/HelloHttpNet9NoProxy/Hello.cs

@@ -0,0 +1,22 @@
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Azure.Functions.Worker.Http;
+using Microsoft.Extensions.Logging;
+using System.Net;
+
+namespace HelloHttpNet9
+{
+    public sealed class Hello(ILogger<Hello> logger)
+    {
+        [Function("Hello")]
+        public HttpResponseData Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req)
+        {
+            logger.LogInformation("C# HTTP trigger function processed a request.");
+
+            var response = req.CreateResponse(HttpStatusCode.OK);
+            response.Headers.Add("Content-Type", "text/plain; charset=utf-8");
+            response.WriteString("Welcome to Azure Functions!");
+
+            return response;
+        }
+    }
+}

test/Performance/Apps/HelloHttpNet9NoProxy/HelloHttp.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net9.0</TargetFramework>
+    <AzureFunctionsVersion>v4</AzureFunctionsVersion>
+    <OutputType>Exe</OutputType>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <PublishReadyToRun>true</PublishReadyToRun>
+  </PropertyGroup>
+  <ItemGroup>
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+    <PackageReference Include="Microsoft.Azure.Functions.Worker" Version="2.0.0" />
+    <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.1.0" />
+    <PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="2.0.0" />
+  </ItemGroup>
+</Project>

test/Performance/Apps/HelloHttpNet9NoProxy/Program.cs

@@ -0,0 +1,7 @@
+using Microsoft.Extensions.Hosting;
+
+var host = new HostBuilder()
+    .ConfigureFunctionsWorkerDefaults()
+    .Build();
+
+host.Run();

test/Performance/Apps/HelloHttpNet9NoProxy/host.json

@@ -0,0 +1,12 @@
+{
+    "version": "2.0",
+    "logging": {
+        "applicationInsights": {
+            "samplingSettings": {
+                "isEnabled": true,
+                "excludedTypes": "Request"
+            },
+            "enableLiveMetricsFilters": true
+        }
+    }
+}

test/Performance/Apps/global.json

@@ -0,0 +1,6 @@
+{
+  "sdk": {
+    "rollForward": "latestPatch",
+    "version": "9.0.100"
+  }
+}