From a1961ecd532794735c75cc4d15d51cea478c8bb9 Mon Sep 17 00:00:00 2001 From: Michael Friesen <3517159+mtfriesen@users.noreply.github.com> Date: Fri, 10 Jan 2025 14:18:55 -0500 Subject: [PATCH 01/12] WIP --- .github/workflows/ci.yml | 4 ++-- tools/common.ps1 | 27 +++++++++++++++++++++++++++ tools/rxfilterperf.ps1 | 29 +++++++++++++++++++++++++++-- tools/xskperfsuite.ps1 | 8 ++------ 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04fe7acb..a4c1c744 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -434,10 +434,10 @@ jobs: run: tools/prepare-machine.ps1 -ForPerfTest -Platform ${{ matrix.platform }} -RequireNoReboot -Verbose - name: Run rxfilter (drop, generic) shell: PowerShell - run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Generic + run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Generic -RawResultsFile artifacts/logs/rxfilterperf.json -CommitHash ${{ github.sha }} - name: Run rxfilter (drop, native) shell: PowerShell - run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Native + run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Native -RawResultsFile artifacts/logs/rxfilterperf.json -CommitHash ${{ github.sha }} - name: Upload Logs uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b if: ${{ always() }} diff --git a/tools/common.ps1 b/tools/common.ps1 index 2651c23e..70fc73c5 100644 --- a/tools/common.ps1 +++ b/tools/common.ps1 @@ -296,6 +296,33 @@ function Initiate-Bugcheck { & $NotMyFault -accepteula -bugcheck $Code } +function New-PerfDataSet { + param ( + [Parameter()] + [string[]]$Files = @() + ) + + $Results = [System.Collections.ArrayList]::new() + + foreach ($File in $Files) { + if (-not [string]::IsNullOrEmpty($File) -and (Test-Path $File)) { + $Results.AddRange($(Get-Content -Raw $File | ConvertFrom-Json)) + } + } +} + +function Write-PerfDataSet { + param ( + [Parameter()] + [object]$DataSet, + + [Parameter()] + [string]$File + ) + + Set-Content -Path $File -Value $($DataSet | ConvertTo-Json -Depth 100) +} + function New-PerfData { param ( [Parameter()] diff --git a/tools/rxfilterperf.ps1 b/tools/rxfilterperf.ps1 index 4260a7e0..e56e0dd1 100644 --- a/tools/rxfilterperf.ps1 +++ b/tools/rxfilterperf.ps1 @@ -21,7 +21,13 @@ param ( [switch]$Fndis = $false, [Parameter(Mandatory = $false)] - [string]$Action = "Drop" + [string]$Action = "Drop", + + [Parameter(Mandatory=$false)] + [string]$RawResultsFile = "", + + [Parameter(Mandatory=$false)] + [string]$CommitHash = "" ) Set-StrictMode -Version 'Latest' @@ -84,7 +90,26 @@ try { & $RootDir\tools\xdpmpratesim.ps1 -AdapterName XDPMP -RxFramesPerInterval 1000 - Write-Output "Filtered $(($EndPackets - $StartPackets)) packets in $Timeout seconds ($(($EndPackets - $StartPackets) / 1000 / $Timeout) Kpps)." + $Kpps = ($EndPackets - $StartPackets) / 1000 / $Timeout + Write-Output "Filtered $(($EndPackets - $StartPackets)) packets in $Timeout seconds ($Kpps Kpps)." + + if (![string]::IsNullOrEmpty($RawResultsFile)) { + $Results = New-PerfDataSet -Files $RawResultsFile + + [void]$Results.Add($( + New-PerfData -ScenarioName "RXFILTER-$XdpMode-$Action-$($QueueCount)Q" ` + -Platform $Platform ` -CommitHash $CommitHash -Metrics @( + @{ + Name = "pps" + Value = $Kpps + Scale = 1000 + } + ) + ) + ) + + Write-PerfDataSet -DataSet $Results -File $RawResultsFile + } } finally { if ($RxFilterProcess) { Stop-Process -InputObject $RxFilterProcess diff --git a/tools/xskperfsuite.ps1 b/tools/xskperfsuite.ps1 index e0862e6f..a0e44cae 100644 --- a/tools/xskperfsuite.ps1 +++ b/tools/xskperfsuite.ps1 @@ -205,11 +205,7 @@ try { } try { - $Results = [System.Collections.ArrayList]::new() - - if (-not [string]::IsNullOrEmpty($RawResultsFile) -and (Test-Path $RawResultsFile)) { - $Results.AddRange($(Get-Content -Raw $RawResultsFile | ConvertFrom-Json)) - } + $Results = New-PerfDataSet -Files $RawResultsFile for ($i = 0; $i -lt $Iterations; $i++) { $TmpFile = [System.IO.Path]::GetTempFileName() @@ -246,7 +242,7 @@ try { Write-Host $($Format -f $ScenarioName, [Math]::ceiling($avg), [Math]::ceiling($stddev)) if (-not [string]::IsNullOrEmpty($RawResultsFile)) { - Set-Content -Path $RawResultsFile -Value $($Results | ConvertTo-Json -Depth 100) + Write-PerfDataSet -DataSet $Results -File $RawResultsFile } } catch { Write-Error "$($PSItem.Exception.Message)`n$($PSItem.ScriptStackTrace)" From cde9d9e1875f0db6b542fc61a2dd6c733001de51 Mon Sep 17 00:00:00 2001 From: Michael Friesen <3517159+mtfriesen@users.noreply.github.com> Date: Fri, 10 Jan 2025 14:39:55 -0500 Subject: [PATCH 02/12] WIP --- .github/actions/perf/action.yml | 13 +++++++++---- .github/workflows/ci.yml | 27 +++++++++++++++++++++++++-- tools/common.ps1 | 2 ++ tools/summarize-perf.ps1 | 20 ++++++++++++++++++++ 4 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 tools/summarize-perf.ps1 diff --git a/.github/actions/perf/action.yml b/.github/actions/perf/action.yml index 367de8af..23592422 100644 --- a/.github/actions/perf/action.yml +++ b/.github/actions/perf/action.yml @@ -58,22 +58,27 @@ runs: path: ./${{ inputs.id }}/artifacts/bin - name: Run xskperfsuite (Generic, Native) shell: PowerShell - run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -RawResultsFile "./${{ inputs.id }}/artifacts/logs/xskperfsuite.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} + run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} - name: Run xskperfsuite (TX-inspect, RX-inject) shell: PowerShell - run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Generic" -TxInspect -RxInject -RawResultsFile "./${{ inputs.id }}/artifacts/logs/xskperfsuite.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} + run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Generic" -TxInspect -RxInject -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} - name: Run xskperfsuite (ZerocopySimulation) shell: PowerShell - run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -ZerocopySimulation -RawResultsFile "./${{ inputs.id }}/artifacts/logs/xskperfsuite.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} + run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -ZerocopySimulation -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} - name: Run xskperfsuite (Winsock, RIO) shell: PowerShell - run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Winsock", "RIO" -Modes "RX", "TX" -RawResultsFile "./${{ inputs.id }}/artifacts/logs/xskperfsuite.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} + run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Winsock", "RIO" -Modes "RX", "TX" -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} - name: Upload Logs uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 if: ${{ always() }} with: name: logs_perf_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }} path: ./${{ inputs.id }}/artifacts/logs + - name: Upload Perf Data + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b + with: + name: perfdata_xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }} + path: ./${{ inputs.id }}/artifacts/perf - name: Check Drivers if: ${{ always() }} shell: PowerShell diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4c1c744..af7fc12b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -434,10 +434,10 @@ jobs: run: tools/prepare-machine.ps1 -ForPerfTest -Platform ${{ matrix.platform }} -RequireNoReboot -Verbose - name: Run rxfilter (drop, generic) shell: PowerShell - run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Generic -RawResultsFile artifacts/logs/rxfilterperf.json -CommitHash ${{ github.sha }} + run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Generic -RawResultsFile artifacts/perf/rxfilterperf_win${{ matrix.windows }}_${{ matrix.configuration }}_${{ matrix.platform }}.json -CommitHash ${{ github.sha }} - name: Run rxfilter (drop, native) shell: PowerShell - run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Native -RawResultsFile artifacts/logs/rxfilterperf.json -CommitHash ${{ github.sha }} + run: tools/rxfilterperf.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose -Fndis -QueueCount 4 -Action Drop -XdpMode Native -RawResultsFile artifacts/perf/rxfilterperf_win${{ matrix.windows }}_${{ matrix.configuration }}_${{ matrix.platform }}.json -CommitHash ${{ github.sha }} - name: Upload Logs uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b if: ${{ always() }} @@ -445,10 +445,33 @@ jobs: name: logs_rxfilter_win${{ matrix.windows }}_${{ matrix.configuration }}_${{ matrix.platform }} path: artifacts/logs if-no-files-found: ignore + - name: Upload Perf Data + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b + with: + name: perfdata_rxfilter_win${{ matrix.windows }}_${{ matrix.configuration }}_${{ matrix.platform }} + path: artifacts/perf + if-no-files-found: ignore - name: Check Drivers shell: PowerShell run: tools/check-drivers.ps1 -Config ${{ matrix.configuration }} -Platform ${{ matrix.platform }} -Verbose + perf_test_summary: + name: Perf Test Summary + needs: [perf_tests, rxfilter_perf_tests] + runs-on: windows-2022 + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - name: Download Artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 + with: + path: artifacts/perf + pattern: perfdata_* + merge-multiple: true + - name: Summarize Perf Data + shell: PowerShell + run: tools/summarize-perf.ps1 -Files artifacts/perf/*.json + downlevel_functional_tests: name: Downlevel Functional Tests needs: build diff --git a/tools/common.ps1 b/tools/common.ps1 index 70fc73c5..fa7dd335 100644 --- a/tools/common.ps1 +++ b/tools/common.ps1 @@ -309,6 +309,8 @@ function New-PerfDataSet { $Results.AddRange($(Get-Content -Raw $File | ConvertFrom-Json)) } } + + return $Results } function Write-PerfDataSet { diff --git a/tools/summarize-perf.ps1 b/tools/summarize-perf.ps1 new file mode 100644 index 00000000..83758e75 --- /dev/null +++ b/tools/summarize-perf.ps1 @@ -0,0 +1,20 @@ +param ( + [Parameter(Mandatory=$true)] + [string[]]$Files +) + +Set-StrictMode -Version 'Latest' +$ErrorActionPreference = 'Stop' + +# Important paths. +$RootDir = Split-Path $PSScriptRoot -Parent +. $RootDir\tools\common.ps1 + +foreach ($File in $Files) { + $Data = New-PerfDataSet -Files $File + + foreach ($Scenario in $Data) { + $Metrics = $Scenario.Metrics | ForEach-Object { "$($_.Name)=$($_.Value)" } + Write-Output "$($Scenario.ScenarioName) ($($Scenario.Platform)) $Metrics" + } +} From e71e9b8c4517df89f7787f98a09af7134b4dd5d8 Mon Sep 17 00:00:00 2001 From: Michael Friesen <3517159+mtfriesen@users.noreply.github.com> Date: Fri, 10 Jan 2025 15:24:42 -0500 Subject: [PATCH 03/12] fix bugs --- .github/actions/{perf => xskperf}/action.yml | 8 ++++---- .github/workflows/ci.yml | 12 ++++++------ tools/common.ps1 | 5 +++-- 3 files changed, 13 insertions(+), 12 deletions(-) rename .github/actions/{perf => xskperf}/action.yml (92%) diff --git a/.github/actions/perf/action.yml b/.github/actions/xskperf/action.yml similarity index 92% rename from .github/actions/perf/action.yml rename to .github/actions/xskperf/action.yml index 23592422..4273a25b 100644 --- a/.github/actions/perf/action.yml +++ b/.github/actions/xskperf/action.yml @@ -1,5 +1,5 @@ -name: 'Perf Tests' -description: 'Run Perf Tests' +name: 'XSK Perf Tests' +description: 'Run XSK Perf Tests' inputs: ref: required: false @@ -72,12 +72,12 @@ runs: uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 if: ${{ always() }} with: - name: logs_perf_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }} + name: logs_xskperf_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }} path: ./${{ inputs.id }}/artifacts/logs - name: Upload Perf Data uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b with: - name: perfdata_xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }} + name: perfdata_xskperf_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }} path: ./${{ inputs.id }}/artifacts/perf - name: Check Drivers if: ${{ always() }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af7fc12b..a76ea43e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -323,8 +323,8 @@ jobs: path: artifacts/logs if-no-files-found: ignore - perf_tests: - name: Perf Tests + xskperf_tests: + name: XSK Perf Tests needs: [build, build_base, resolve_base] env: ITERATIONS: ${{ (github.event_name != 'pull_request' && github.event_name != 'workflow_dispatch' ) && '8' || '3' }} @@ -351,12 +351,12 @@ jobs: sparse-checkout: | .github tools - - uses: ./.github/actions/perf + - uses: ./.github/actions/xskperf with: config: ${{ matrix.configuration }} os: ${{ matrix.windows }} platform: ${{ matrix.platform }} - - uses: ./.github/actions/perf + - uses: ./.github/actions/xskperf with: config: ${{ matrix.configuration }} os: ${{ matrix.windows }} @@ -457,7 +457,7 @@ jobs: perf_test_summary: name: Perf Test Summary - needs: [perf_tests, rxfilter_perf_tests] + needs: [xskperf_tests, rxfilter_perf_tests] runs-on: windows-2022 steps: - name: Checkout repository @@ -630,7 +630,7 @@ jobs: Complete: name: Complete if: always() - needs: [build, build_allpackage, onebranch_build_validation, functional_tests, stress_tests, pktfuzz_tests, perf_tests, ring_perf_tests, rxfilter_perf_tests, xskfwdkm_test, downlevel_functional_tests, create_artifacts] + needs: [build, build_allpackage, onebranch_build_validation, functional_tests, stress_tests, pktfuzz_tests, xskperf_tests, ring_perf_tests, rxfilter_perf_tests, xskfwdkm_test, downlevel_functional_tests, create_artifacts] runs-on: ubuntu-latest permissions: {} # No need for any permissions. steps: diff --git a/tools/common.ps1 b/tools/common.ps1 index fa7dd335..7f9620d0 100644 --- a/tools/common.ps1 +++ b/tools/common.ps1 @@ -310,7 +310,7 @@ function New-PerfDataSet { } } - return $Results + return Write-Output -NoEnumerate $Results } function Write-PerfDataSet { @@ -322,7 +322,8 @@ function Write-PerfDataSet { [string]$File ) - Set-Content -Path $File -Value $($DataSet | ConvertTo-Json -Depth 100) + New-Item -ItemType Directory -Force -Path (Split-Path $File) | Out-Null + $DataSet | ConvertTo-Json -Depth 100 | Out-File -FilePath $File -Encoding utf8 } function New-PerfData { From 634eda69a4ef99f088c29e4f3b3c26812ce7b880 Mon Sep 17 00:00:00 2001 From: Michael Friesen <3517159+mtfriesen@users.noreply.github.com> Date: Fri, 10 Jan 2025 15:33:52 -0500 Subject: [PATCH 04/12] output to magic summary file --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a76ea43e..21a13fb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -470,7 +470,7 @@ jobs: merge-multiple: true - name: Summarize Perf Data shell: PowerShell - run: tools/summarize-perf.ps1 -Files artifacts/perf/*.json + run: tools/summarize-perf.ps1 -Files artifacts/perf/*.json >> $env:GITHUB_STEP_SUMMARY downlevel_functional_tests: name: Downlevel Functional Tests From 206e6353ec2193669a84bc3c31a9b5448448d3b4 Mon Sep 17 00:00:00 2001 From: Michael Friesen <3517159+mtfriesen@users.noreply.github.com> Date: Fri, 10 Jan 2025 16:14:04 -0500 Subject: [PATCH 05/12] fix rxfilterperf typo --- tools/rxfilterperf.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/rxfilterperf.ps1 b/tools/rxfilterperf.ps1 index e56e0dd1..27238155 100644 --- a/tools/rxfilterperf.ps1 +++ b/tools/rxfilterperf.ps1 @@ -98,7 +98,7 @@ try { [void]$Results.Add($( New-PerfData -ScenarioName "RXFILTER-$XdpMode-$Action-$($QueueCount)Q" ` - -Platform $Platform ` -CommitHash $CommitHash -Metrics @( + -Platform $Platform -CommitHash $CommitHash -Metrics @( @{ Name = "pps" Value = $Kpps From 91f0ec5e10dcea155f9df0b284b64a292bcfdf70 Mon Sep 17 00:00:00 2001 From: Michael Friesen <3517159+mtfriesen@users.noreply.github.com> Date: Mon, 13 Jan 2025 13:52:25 -0500 Subject: [PATCH 06/12] fix single data point JSON export --- tools/common.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/common.ps1 b/tools/common.ps1 index 7f9620d0..9b33287f 100644 --- a/tools/common.ps1 +++ b/tools/common.ps1 @@ -323,7 +323,7 @@ function Write-PerfDataSet { ) New-Item -ItemType Directory -Force -Path (Split-Path $File) | Out-Null - $DataSet | ConvertTo-Json -Depth 100 | Out-File -FilePath $File -Encoding utf8 + ConvertTo-Json -InputObject $DataSet -Depth 100 | Out-File -FilePath $File -Encoding utf8 } function New-PerfData { From 12d7cce2c9e59ffb52ea40ee4c4685b3596c3984 Mon Sep 17 00:00:00 2001 From: Michael Friesen <3517159+mtfriesen@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:51:31 -0500 Subject: [PATCH 07/12] update compare data path, too --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afe0fe31..0915c5a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -365,7 +365,7 @@ jobs: id: "baseref" - name: Compare Results shell: PowerShell - run: tools\perfcompare.ps1 -DataFile1 baseref/artifacts/logs/xskperfsuite.json -DataFile2 artifacts/logs/xskperfsuite.json -Metric pps + run: tools\perfcompare.ps1 -DataFile1 baseref/artifacts/perf/xskperfsuite.json -DataFile2 artifacts/perf/xskperfsuite.json -Metric pps ring_perf_tests: name: Ring Perf Tests From f5d5625da9fc99a12dfd12c5a076bde460b9505e Mon Sep 17 00:00:00 2001 From: Michael Friesen <3517159+mtfriesen@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:48:14 -0500 Subject: [PATCH 08/12] try using a wildcard --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0915c5a3..a47dffe8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -365,7 +365,7 @@ jobs: id: "baseref" - name: Compare Results shell: PowerShell - run: tools\perfcompare.ps1 -DataFile1 baseref/artifacts/perf/xskperfsuite.json -DataFile2 artifacts/perf/xskperfsuite.json -Metric pps + run: tools\perfcompare.ps1 -DataFile1 baseref/artifacts/perf/xskperfsuite*.json -DataFile2 artifacts/perf/xskperfsuite*.json -Metric pps ring_perf_tests: name: Ring Perf Tests From 29fb34f057430e0b3f3ee2dcd50a48dc889a2d20 Mon Sep 17 00:00:00 2001 From: Michael Friesen <3517159+mtfriesen@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:06:11 -0500 Subject: [PATCH 09/12] summarize better, and format as markdown in CI --- .github/workflows/ci.yml | 4 +++- tools/summarize-perf.ps1 | 41 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a47dffe8..8fe70143 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -470,7 +470,9 @@ jobs: merge-multiple: true - name: Summarize Perf Data shell: PowerShell - run: tools/summarize-perf.ps1 -Files artifacts/perf/*.json >> $env:GITHUB_STEP_SUMMARY + run: | + Install-Module -Name FormatMarkdownTable -Force -AllowClobber -Scope CurrentUser -Repository PSGallery -Verbose + tools/summarize-perf.ps1 -Files artifacts/perf/*.json | Format-MarkdownTableTableStyle -DoNotCopyToClipboard -Property ScenarioName, Platform, CommitHash, MetricName, Average, Minimum, Maximum >> $env:GITHUB_STEP_SUMMARY downlevel_functional_tests: name: Downlevel Functional Tests diff --git a/tools/summarize-perf.ps1 b/tools/summarize-perf.ps1 index 83758e75..215fba87 100644 --- a/tools/summarize-perf.ps1 +++ b/tools/summarize-perf.ps1 @@ -10,11 +10,44 @@ $ErrorActionPreference = 'Stop' $RootDir = Split-Path $PSScriptRoot -Parent . $RootDir\tools\common.ps1 -foreach ($File in $Files) { +$Summary = @{} + +# First, merge metrics from all runs, indexed by scenario name, platform, and commit hash. +foreach ($File in Get-Item -Path $Files) { $Data = New-PerfDataSet -Files $File - foreach ($Scenario in $Data) { - $Metrics = $Scenario.Metrics | ForEach-Object { "$($_.Name)=$($_.Value)" } - Write-Output "$($Scenario.ScenarioName) ($($Scenario.Platform)) $Metrics" + foreach ($Run in $Data) { + $Key = [ordered]@{ + ScenarioName = $Run.ScenarioName + Platform = $Run.Platform + CommitHash = $Run.CommitHash + } + $KeyString = $Key | Out-String + + if (!$Summary.ContainsKey($KeyString)) { + $Summary[$KeyString] = @{ + Key = $Key + Metrics = @{} + } + } + + foreach ($Metric in $Run.Metrics) { + [array]$Summary[$KeyString].Metrics[$Metric.Name] += $Metric.Value + } } } + +# Second, flatten the merged metrics and summarize the results across runs. +$Summary.Values | ForEach-Object { + foreach ($Metric in $_.Metrics.GetEnumerator()) { + [pscustomobject]@{ + ScenarioName = $_.Key.ScenarioName + Platform = $_.Key.Platform + CommitHash = $_.Key.CommitHash + MetricName = $Metric.Key + Average = $Metric.Value | Measure-Object -Average | Select-Object -ExpandProperty Average + Minimum = $Metric.Value | Measure-Object -Minimum | Select-Object -ExpandProperty Minimum + Maximum = $Metric.Value | Measure-Object -Maximum | Select-Object -ExpandProperty Maximum + } + } +} | Sort-Object -Property ScenarioName, Platform, CommitHash, MetricName From 2cb29f14335bc29c4bf71c773242f6372ebafa67 Mon Sep 17 00:00:00 2001 From: Michael Friesen <3517159+mtfriesen@users.noreply.github.com> Date: Tue, 14 Jan 2025 17:09:01 -0500 Subject: [PATCH 10/12] use actual markdown :D --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8fe70143..37232014 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -472,7 +472,7 @@ jobs: shell: PowerShell run: | Install-Module -Name FormatMarkdownTable -Force -AllowClobber -Scope CurrentUser -Repository PSGallery -Verbose - tools/summarize-perf.ps1 -Files artifacts/perf/*.json | Format-MarkdownTableTableStyle -DoNotCopyToClipboard -Property ScenarioName, Platform, CommitHash, MetricName, Average, Minimum, Maximum >> $env:GITHUB_STEP_SUMMARY + tools/summarize-perf.ps1 -Files artifacts/perf/*.json | Format-MarkdownTableTableStyle -Property ScenarioName, Platform, CommitHash, MetricName, Average, Minimum, Maximum -DoNotCopyToClipboard -ShowMarkdown -HideStandardOutput >> $env:GITHUB_STEP_SUMMARY downlevel_functional_tests: name: Downlevel Functional Tests From eb77ef6e528c8beea938cd0acd9396e906033403 Mon Sep 17 00:00:00 2001 From: Michael Friesen <3517159+mtfriesen@users.noreply.github.com> Date: Wed, 15 Jan 2025 08:11:45 -0500 Subject: [PATCH 11/12] fix up commit hash for baseref --- .github/actions/xskperf/action.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/actions/xskperf/action.yml b/.github/actions/xskperf/action.yml index 4273a25b..8914f6ef 100644 --- a/.github/actions/xskperf/action.yml +++ b/.github/actions/xskperf/action.yml @@ -58,16 +58,15 @@ runs: path: ./${{ inputs.id }}/artifacts/bin - name: Run xskperfsuite (Generic, Native) shell: PowerShell - run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} + run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ inputs.ref != '' && format('_{0}', inputs.ref) || github.sha }} - name: Run xskperfsuite (TX-inspect, RX-inject) shell: PowerShell - run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Generic" -TxInspect -RxInject -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} + run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Generic" -TxInspect -RxInject -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ inputs.ref != '' && format('_{0}', inputs.ref) || github.sha }} - name: Run xskperfsuite (ZerocopySimulation) shell: PowerShell - run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -ZerocopySimulation -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} - - name: Run xskperfsuite (Winsock, RIO) + run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -ZerocopySimulation -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ inputs.ref != '' && format('_{0}', inputs.ref) || github.sha }} shell: PowerShell - run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Winsock", "RIO" -Modes "RX", "TX" -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ github.sha }} + run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Winsock", "RIO" -Modes "RX", "TX" -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ inputs.ref != '' && format('_{0}', inputs.ref) || github.sha }} - name: Upload Logs uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 if: ${{ always() }} From a304bda31981afec75a04fa04832cd6d9bef7f4f Mon Sep 17 00:00:00 2001 From: Michael Friesen <3517159+mtfriesen@users.noreply.github.com> Date: Wed, 15 Jan 2025 08:42:06 -0500 Subject: [PATCH 12/12] undelete line --- .github/actions/xskperf/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/xskperf/action.yml b/.github/actions/xskperf/action.yml index 8914f6ef..9476d7ec 100644 --- a/.github/actions/xskperf/action.yml +++ b/.github/actions/xskperf/action.yml @@ -65,6 +65,7 @@ runs: - name: Run xskperfsuite (ZerocopySimulation) shell: PowerShell run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -ZerocopySimulation -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ inputs.ref != '' && format('_{0}', inputs.ref) || github.sha }} + - name: Run xskperfsuite (Winsock, RIO) shell: PowerShell run: ./${{ inputs.id }}/tools/xskperfsuite.ps1 -Verbose -Config ${{ inputs.config }} -Platform ${{ inputs.platform }} -Fndis -Iterations ${{ env.ITERATIONS }} -XdpModes "Winsock", "RIO" -Modes "RX", "TX" -RawResultsFile "./${{ inputs.id }}/artifacts/perf/xskperfsuite_win${{ inputs.os }}_${{ inputs.config }}_${{ inputs.platform }}${{ inputs.id != '' && format('_{0}', inputs.id) || '' }}.json" -XperfDirectory "./${{ inputs.id }}/artifacts/logs" -CommitHash ${{ inputs.ref != '' && format('_{0}', inputs.ref) || github.sha }} - name: Upload Logs