diff --git a/.github/actions/perf/action.yml b/.github/actions/xskperf/action.yml similarity index 62% rename from .github/actions/perf/action.yml rename to .github/actions/xskperf/action.yml index 367de8af..9476d7ec 100644 --- a/.github/actions/perf/action.yml +++ b/.github/actions/xskperf/action.yml @@ -1,5 +1,5 @@ -name: 'Perf Tests' -description: 'Run Perf Tests' +name: 'XSK Perf Tests' +description: 'Run XSK Perf Tests' inputs: ref: required: false @@ -58,22 +58,27 @@ runs: path: ./${{ inputs.id }}/artifacts/bin - name: Run xskperfsuite (Generic, Native) shell: PowerShell - run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -RawResultsFile "./${{ inputs.id }}/artifacts/logs/xskperfsuite.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} + run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ inputs.ref != '' && format('_{0}', inputs.ref) || github.sha }} - name: Run xskperfsuite (TX-inspect, RX-inject) shell: PowerShell - run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Generic" -TxInspect -RxInject -RawResultsFile "./${{ inputs.id }}/artifacts/logs/xskperfsuite.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} + run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Generic" -TxInspect -RxInject -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ inputs.ref != '' && format('_{0}', inputs.ref) || github.sha }} - name: Run xskperfsuite (ZerocopySimulation) shell: PowerShell - run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -ZerocopySimulation -RawResultsFile "./${{ inputs.id }}/artifacts/logs/xskperfsuite.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} + run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -ZerocopySimulation -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ inputs.ref != '' && format('_{0}', inputs.ref) || github.sha }} - name: Run xskperfsuite (Winsock, RIO) shell: PowerShell - run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Winsock", "RIO" -Modes "RX", "TX" -RawResultsFile "./${{ inputs.id }}/artifacts/logs/xskperfsuite.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} + run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Winsock", "RIO" -Modes "RX", "TX" -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ inputs.ref != '' && format('_{0}', inputs.ref) || github.sha }} - name: Upload Logs uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 if: ${{ always() }} with: - name: logs_perf_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }} + name: logs_xskperf_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }} path: ./${{ inputs.id }}/artifacts/logs + - name: Upload Perf Data + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b + with: + name: perfdata_xskperf_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }} + path: ./${{ inputs.id }}/artifacts/perf - name: Check Drivers if: ${{ always() }} shell: PowerShell diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fdf8bd1..37232014 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -323,8 +323,8 @@ jobs: path: artifacts/logs if-no-files-found: ignore - perf_tests: - name: Perf Tests + xskperf_tests: + name: XSK Perf Tests needs: [build, build_base, resolve_base] env: ITERATIONS: ${{ (github.event_name != 'pull_request' && github.event_name != 'workflow_dispatch' ) && '8' || '3' }} @@ -351,12 +351,12 @@ jobs: sparse-checkout: | .github tools - - uses: ./.github/actions/perf + - uses: ./.github/actions/xskperf with: config: ${{ matrix.configuration }} os: ${{ matrix.windows }} platform: ${{ matrix.platform }} - - uses: ./.github/actions/perf + - uses: ./.github/actions/xskperf with: config: ${{ matrix.configuration }} os: ${{ matrix.windows }} @@ -365,7 +365,7 @@ jobs: id: "baseref" - name: Compare Results shell: PowerShell - run: tools\perfcompare.ps1 -DataFile1 baseref/artifacts/logs/xskperfsuite.json -DataFile2 artifacts/logs/xskperfsuite.json -Metric pps + run: tools\perfcompare.ps1 -DataFile1 baseref/artifacts/perf/xskperfsuite*.json -DataFile2 artifacts/perf/xskperfsuite*.json -Metric pps ring_perf_tests: name: Ring Perf Tests @@ -434,10 +434,10 @@ jobs: run: tools/prepare-machine.ps1 -ForPerfTest -Platform ${{ matrix.platform }} -RequireNoReboot -Verbose - name: Run rxfilter (drop, generic) shell: PowerShell - run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Generic + run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Generic -RawResultsFile artifacts/perf/rxfilterperf_win${{ matrix.windows }}_${{ matrix.configuration }}_${{ matrix.platform }}.json -CommitHash ${{ github.sha }} - name: Run rxfilter (drop, native) shell: PowerShell - run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Native + run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Native -RawResultsFile artifacts/perf/rxfilterperf_win${{ matrix.windows }}_${{ matrix.configuration }}_${{ matrix.platform }}.json -CommitHash ${{ github.sha }} - name: Upload Logs uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 if: ${{ always() }} @@ -445,10 +445,35 @@ jobs: name: logs_rxfilter_win${{ matrix.windows }}_${{ matrix.configuration }}_${{ matrix.platform }} path: artifacts/logs if-no-files-found: ignore + - name: Upload Perf Data + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b + with: + name: perfdata_rxfilter_win${{ matrix.windows }}_${{ matrix.configuration }}_${{ matrix.platform }} + path: artifacts/perf + if-no-files-found: ignore - name: Check Drivers shell: PowerShell run: tools/check-drivers.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose + perf_test_summary: + name: Perf Test Summary + needs: [xskperf_tests, rxfilter_perf_tests] + runs-on: windows-2022 + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - name: Download Artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 + with: + path: artifacts/perf + pattern: perfdata_* + merge-multiple: true + - name: Summarize Perf Data + shell: PowerShell + run: | + Install-Module -Name FormatMarkdownTable -Force -AllowClobber -Scope CurrentUser -Repository PSGallery -Verbose + tools/summarize-perf.ps1 -Files artifacts/perf/*.json | Format-MarkdownTableTableStyle -Property ScenarioName, Platform, CommitHash, MetricName, Average, Minimum, Maximum -DoNotCopyToClipboard -ShowMarkdown -HideStandardOutput >> $env:GITHUB_STEP_SUMMARY + downlevel_functional_tests: name: Downlevel Functional Tests needs: build @@ -607,7 +632,7 @@ jobs: Complete: name: Complete if: always() - needs: [build, build_allpackage, onebranch_build_validation, functional_tests, stress_tests, pktfuzz_tests, perf_tests, ring_perf_tests, rxfilter_perf_tests, xskfwdkm_test, downlevel_functional_tests, create_artifacts] + needs: [build, build_allpackage, onebranch_build_validation, functional_tests, stress_tests, pktfuzz_tests, xskperf_tests, ring_perf_tests, rxfilter_perf_tests, xskfwdkm_test, downlevel_functional_tests, create_artifacts] runs-on: ubuntu-latest permissions: {} # No need for any permissions. steps: diff --git a/tools/common.ps1 b/tools/common.ps1 index 7f9620d0..9b33287f 100644 --- a/tools/common.ps1 +++ b/tools/common.ps1 @@ -323,7 +323,7 @@ function Write-PerfDataSet { ) New-Item -ItemType Directory -Force -Path (Split-Path $File) | Out-Null - $DataSet | ConvertTo-Json -Depth 100 | Out-File -FilePath $File -Encoding utf8 + ConvertTo-Json -InputObject $DataSet -Depth 100 | Out-File -FilePath $File -Encoding utf8 } function New-PerfData { diff --git a/tools/rxfilterperf.ps1 b/tools/rxfilterperf.ps1 index 4260a7e0..27238155 100644 --- a/tools/rxfilterperf.ps1 +++ b/tools/rxfilterperf.ps1 @@ -21,7 +21,13 @@ param ( [switch]$Fndis = $false, [Parameter(Mandatory = $false)] - [string]$Action = "Drop" + [string]$Action = "Drop", + + [Parameter(Mandatory=$false)] + [string]$RawResultsFile = "", + + [Parameter(Mandatory=$false)] + [string]$CommitHash = "" ) Set-StrictMode -Version 'Latest' @@ -84,7 +90,26 @@ try { & $RootDir\tools\xdpmpratesim.ps1 -AdapterName XDPMP -RxFramesPerInterval 1000 - Write-Output "Filtered $(($EndPackets - $StartPackets)) packets in $Timeout seconds ($(($EndPackets - $StartPackets) / 1000 / $Timeout) Kpps)." + $Kpps = ($EndPackets - $StartPackets) / 1000 / $Timeout + Write-Output "Filtered $(($EndPackets - $StartPackets)) packets in $Timeout seconds ($Kpps Kpps)." + + if (![string]::IsNullOrEmpty($RawResultsFile)) { + $Results = New-PerfDataSet -Files $RawResultsFile + + [void]$Results.Add($( + New-PerfData -ScenarioName "RXFILTER-$XdpMode-$Action-$($QueueCount)Q" ` + -Platform $Platform -CommitHash $CommitHash -Metrics @( + @{ + Name = "pps" + Value = $Kpps + Scale = 1000 + } + ) + ) + ) + + Write-PerfDataSet -DataSet $Results -File $RawResultsFile + } } finally { if ($RxFilterProcess) { Stop-Process -InputObject $RxFilterProcess diff --git a/tools/summarize-perf.ps1 b/tools/summarize-perf.ps1 new file mode 100644 index 00000000..215fba87 --- /dev/null +++ b/tools/summarize-perf.ps1 @@ -0,0 +1,53 @@ +param ( + [Parameter(Mandatory=$true)] + [string[]]$Files +) + +Set-StrictMode -Version 'Latest' +$ErrorActionPreference = 'Stop' + +# Important paths. +$RootDir = Split-Path $PSScriptRoot -Parent +. $RootDir\tools\common.ps1 + +$Summary = @{} + +# First, merge metrics from all runs, indexed by scenario name, platform, and commit hash. +foreach ($File in Get-Item -Path $Files) { + $Data = New-PerfDataSet -Files $File + + foreach ($Run in $Data) { + $Key = [ordered]@{ + ScenarioName = $Run.ScenarioName + Platform = $Run.Platform + CommitHash = $Run.CommitHash + } + $KeyString = $Key | Out-String + + if (!$Summary.ContainsKey($KeyString)) { + $Summary[$KeyString] = @{ + Key = $Key + Metrics = @{} + } + } + + foreach ($Metric in $Run.Metrics) { + [array]$Summary[$KeyString].Metrics[$Metric.Name] += $Metric.Value + } + } +} + +# Second, flatten the merged metrics and summarize the results across runs. +$Summary.Values | ForEach-Object { + foreach ($Metric in $_.Metrics.GetEnumerator()) { + [pscustomobject]@{ + ScenarioName = $_.Key.ScenarioName + Platform = $_.Key.Platform + CommitHash = $_.Key.CommitHash + MetricName = $Metric.Key + Average = $Metric.Value | Measure-Object -Average | Select-Object -ExpandProperty Average + Minimum = $Metric.Value | Measure-Object -Minimum | Select-Object -ExpandProperty Minimum + Maximum = $Metric.Value | Measure-Object -Maximum | Select-Object -ExpandProperty Maximum + } + } +} | Sort-Object -Property ScenarioName, Platform, CommitHash, MetricName