Prepare scenarios before sending to Helix (#2641)

Build vs measure: build creates completely self-contained packages (#2416)
dotnet · Oct 19, 2022 · 9247934 · 9247934
1 parent 4683d70
commit 9247934
Show file tree

Hide file tree

Showing 5 changed files with 123 additions and 37 deletions.
diff --git a/eng/performance/PreparePayloadWorkItems.targets b/eng/performance/PreparePayloadWorkItems.targets
@@ -0,0 +1,7 @@
+<Project>
+  <Target Name="PreparePayloadWorkItems" Inputs="@(PreparePayloadWorkItem)" Outputs="%(PreparePayloadWorkItem.Identity)">
+    <Message Text="Command: %(PreparePayloadWorkItem.Command)" />
+    <Exec Command="%(PreparePayloadWorkItem.Command)" WorkingDirectory="%(PreparePayloadWorkItem.WorkingDirectory)" />
+    <Exec Command="$(AfterPreparePayloadWorkItemCommand)" WorkingDirectory="%(PreparePayloadWorkItem.WorkingDirectory)" Condition="'$(AfterPreparePayloadWorkItemCommand)' != ''" />
+  </Target>
+</Project>
diff --git a/eng/performance/blazor_scenarios.proj b/eng/performance/blazor_scenarios.proj
@@ -71,4 +71,8 @@
     </HelixWorkItem>
   </ItemGroup>
 
+
+  <Import Project="PreparePayloadWorkItems.targets" />
+
+
 </Project>
diff --git a/eng/performance/scenarios.proj b/eng/performance/scenarios.proj
@@ -20,112 +20,158 @@
   </PropertyGroup>
 
   <ItemDefinitionGroup>
-      <HelixWorkItem>
+    <HelixWorkItem>
       <Timeout>4:00</Timeout>
     </HelixWorkItem>
   </ItemDefinitionGroup>
-
+
+  <PropertyGroup>
+    <AfterPreparePayloadWorkItemCommand>$(Python) post.py</AfterPreparePayloadWorkItemCommand>
+    <PreparePayloadOutDirectoryName>scenarios_out</PreparePayloadOutDirectoryName>
+    <PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' == 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\</PreparePayloadWorkItemBaseDirectory>
+    <PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' != 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/</PreparePayloadWorkItemBaseDirectory>
+  </PropertyGroup>
+
 
   <ItemGroup>
     <Scenario Include="Static Console Template">
-      <PayloadDirectory>$(ScenariosDir)staticconsoletemplate</PayloadDirectory>
+      <ScenarioDirectoryName>staticconsoletemplate</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
     </Scenario>
     <Scenario Include="Static VB Console Template">
-      <PayloadDirectory>$(ScenariosDir)staticvbconsoletemplate</PayloadDirectory>
+      <ScenarioDirectoryName>staticvbconsoletemplate</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
     </Scenario>
     <Scenario Include="Static F# Console Template">
-      <PayloadDirectory>$(ScenariosDir)staticfsconsoletemplate</PayloadDirectory>
+      <ScenarioDirectoryName>staticfsconsoletemplate</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
     </Scenario>
     <!-- <Scenario Include="Static Winforms Template" Condition="'$(PERFLAB_Framework)' != 'netcoreapp5.0'">
-      <PayloadDirectory>$(ScenariosDir)staticwinformstemplate</PayloadDirectory>
+      <ScenarioDirectoryName>staticwinformstemplate</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
     </Scenario> -->
     <Scenario Include="New Console Template">
-      <PayloadDirectory>$(ScenariosDir)emptyconsoletemplate</PayloadDirectory>
+      <ScenarioDirectoryName>emptyconsoletemplate</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
     </Scenario>
     <Scenario Include="New VB Console Template">
-      <PayloadDirectory>$(ScenariosDir)emptyvbconsoletemplate</PayloadDirectory>
+      <ScenarioDirectoryName>emptyvbconsoletemplate</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
     </Scenario>
     <UIScenario Include="WinForms Large" Condition="'$(TargetsWindows)' == 'true'">
-      <PayloadDirectory>$(ScenariosDir)windowsformslarge</PayloadDirectory>
+      <ScenarioDirectoryName>windowsformslarge</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
     </UIScenario>
     <UIScenario Include="WPF Template" Condition="'$(TargetsWindows)' == 'true'">
-      <PayloadDirectory>$(ScenariosDir)wpf</PayloadDirectory>
+      <ScenarioDirectoryName>wpf</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
     </UIScenario>
     <UIScenario Include="WPF SFC" Condition="'$(TargetsWindows)' == 'true'">
-      <PayloadDirectory>$(ScenariosDir)wpfsfc</PayloadDirectory>
+      <ScenarioDirectoryName>wpfsfc</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
     </UIScenario>
     <UIScenario Include="PaintDotNet" Condition="'$(TargetsWindows)' == 'true' And $(BUILD_REPOSITORY_PROVIDER) == 'TfsGit'">
-      <PayloadDirectory>$(ScenariosDir)paintdotnet</PayloadDirectory>
+      <ScenarioDirectoryName>paintdotnet</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
     </UIScenario>
+    <MAUIScenario Include="Desktop Maui" Condition="'$(TargetsWindows)' == 'true' And '$(Architecture)' != 'arm64' And '$(Architecture)' != 'arm'">
+      <ScenarioDirectoryName>mauidesktop</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+    </MAUIScenario>
+    <MAUIScenario Include="Desktop Maui Blazor" Condition="'$(TargetsWindows)' == 'true' And '$(Architecture)' != 'arm64' And '$(Architecture)' != 'arm'">
+      <ScenarioDirectoryName>mauiblazordesktop</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+    </MAUIScenario>
+  </ItemGroup>
+
+
+  <ItemGroup>
+    <PreparePayloadWorkItem Include="@(UIScenario)">
+      <Command>$(Python) pre.py publish -f $(PERFLAB_Framework) -c Release --windowsui -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)_fdd</Command>
+      <WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
+    </PreparePayloadWorkItem>
+    <PreparePayloadWorkItem Include="@(Scenario)">
+      <Command>$(Python) pre.py publish -f $(PERFLAB_Framework) -c Release -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)_fdd</Command>
+      <WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
+    </PreparePayloadWorkItem>
+    <PreparePayloadWorkItem Include="@(Scenario)">
+      <Command>$(Python) pre.py publish -f $(PERFLAB_Framework) -c Release -r $(RID) -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)_scd</Command>
+      <WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
+    </PreparePayloadWorkItem>
+    <PreparePayloadWorkItem Include="@(Scenario)">
+      <Command>$(Python) pre.py build -c Release -f $(PERFLAB_Framework) -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)_build</Command>
+      <WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
+    </PreparePayloadWorkItem>
+    <PreparePayloadWorkItem Include="@(MAUIScenario)">
+      <Command>$(Python) pre.py publish -c Release -f $(PERFLAB_Framework)-windows10.0.19041.0 -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName)_fdd</Command>
+      <WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
+    </PreparePayloadWorkItem>
   </ItemGroup>
 
 
   <!-- UI Startup FDD publish -->
   <ItemGroup>
     <HelixWorkItem Include="@(UIScenario -> 'Startup - %(Identity) - FDD Publish')">
-      <PreCommands>$(Python) pre.py publish -f $(PERFLAB_Framework) -c Release --windowsui</PreCommands>
+      <PreCommands Condition="'$(TargetsWindows)' == 'true'">xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)_fdd %HELIX_WORKITEM_ROOT%\pub /E /I /Y</PreCommands>
+      <PreCommands Condition="'$(TargetsWindows)' != 'true'">cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)_fdd $HELIX_WORKITEM_ROOT/pub</PreCommands>
       <Command>$(Python) test.py startup --scenario-name &quot;%(Identity)&quot;</Command>
     </HelixWorkItem>
   </ItemGroup>
 
   <!-- Startup FDD publish -->
   <ItemGroup>
     <HelixWorkItem Include="@(Scenario -> 'Startup - %(Identity) - FDD Publish')">
-      <PreCommands>$(Python) pre.py publish -f $(PERFLAB_Framework) -c Release</PreCommands>
+      <PreCommands Condition="'$(TargetsWindows)' == 'true'">xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)_fdd %HELIX_WORKITEM_ROOT%\pub /E /I /Y</PreCommands>
+      <PreCommands Condition="'$(TargetsWindows)' != 'true'">cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)_fdd $HELIX_WORKITEM_ROOT/pub</PreCommands>
       <Command>$(Python) test.py startup --scenario-name &quot;%(Identity)&quot;</Command>
     </HelixWorkItem>
   </ItemGroup>
 
   <!-- SOD FDD publish -->
   <ItemGroup>
     <HelixWorkItem Include="@(Scenario -> 'SOD - %(Identity) - FDD Publish')">
-      <PreCommands>$(Python) pre.py publish -f $(PERFLAB_Framework) -c Release</PreCommands>
+      <PreCommands Condition="'$(TargetsWindows)' == 'true'">xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)_fdd %HELIX_WORKITEM_ROOT%\pub /E /I /Y</PreCommands>
+      <PreCommands Condition="'$(TargetsWindows)' != 'true'">cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)_fdd $HELIX_WORKITEM_ROOT/pub</PreCommands>
       <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
     </HelixWorkItem>
   </ItemGroup>
 
   <!-- SOD SCD publish w/ RID -->
   <ItemGroup>
     <HelixWorkItem Include="@(Scenario -> 'SOD - %(Identity) - SCD Publish')">
-      <PreCommands>$(Python) pre.py publish -f $(PERFLAB_Framework) -c Release -r $(RID)</PreCommands>
+      <PreCommands Condition="'$(TargetsWindows)' == 'true'">xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)_scd %HELIX_WORKITEM_ROOT%\pub /E /I /Y</PreCommands>
+      <PreCommands Condition="'$(TargetsWindows)' != 'true'">cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)_scd $HELIX_WORKITEM_ROOT/pub</PreCommands>
       <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
     </HelixWorkItem>
   </ItemGroup>
 
   <!-- SOD build -->
   <ItemGroup>
     <HelixWorkItem Include="@(Scenario -> 'SOD - %(Identity) - Build')">
-      <PreCommands>$(Python) pre.py build -c Release -f $(PERFLAB_Framework)</PreCommands>
+      <PreCommands Condition="'$(TargetsWindows)' == 'true'">xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)_build %HELIX_WORKITEM_ROOT%\bin /E /I /Y</PreCommands>
+      <PreCommands Condition="'$(TargetsWindows)' != 'true'">cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)_build $HELIX_WORKITEM_ROOT/bin</PreCommands>
       <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
     </HelixWorkItem>
   </ItemGroup>
 
-  <ItemGroup Condition="'$(TargetsWindows)' == 'true' And '$(Architecture)' != 'arm64' And '$(Architecture)' != 'arm'">
-    <!-- SOD & Device Startup for Windows Desktop MAUI -->
-    <HelixWorkItem Include="SOD - Desktop Maui Unpackaged">
-      <PayloadDirectory>$(ScenariosDir)mauidesktop</PayloadDirectory>
-      <PreCommands>$(Python) pre.py publish -c Release -f $(PERFLAB_Framework)-windows10.0.19041.0</PreCommands>
-      <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
-    </HelixWorkItem>
-    <HelixWorkItem Include="Device Startup - Desktop Maui Default">
-      <PayloadDirectory>$(ScenariosDir)mauidesktop</PayloadDirectory>
-      <PreCommands>$(Python) pre.py publish -c Release -f $(PERFLAB_Framework)-windows10.0.19041.0</PreCommands>
-      <Command>$(Python) test.py startup --scenario-name &quot;%(Identity)&quot;</Command>
-    </HelixWorkItem>
-
-    <!-- SOD & Device Startup for Windows Desktop MAUI Blazor -->
-    <HelixWorkItem Include="SOD - Desktop Maui Blazor Unpackaged">
-      <PayloadDirectory>$(ScenariosDir)mauiblazordesktop</PayloadDirectory>
-      <PreCommands>$(Python) pre.py publish -c Release -f $(PERFLAB_Framework)-windows10.0.19041.0</PreCommands>
+  <!-- MAUI -->
+  <ItemGroup>
+    <HelixWorkItem Include="@(MAUIScenario -> 'SOD - %(Identity) Unpackaged')">
+      <PreCommands Condition="'$(TargetsWindows)' == 'true'">xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)_fdd %HELIX_WORKITEM_ROOT%\pub /E /I /Y</PreCommands>
+      <PreCommands Condition="'$(TargetsWindows)' != 'true'">cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)_fdd $HELIX_WORKITEM_ROOT/pub</PreCommands>
       <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
     </HelixWorkItem>
-    <HelixWorkItem Include="Device Startup - Desktop Maui Blazor Default">
-      <PayloadDirectory>$(ScenariosDir)mauiblazordesktop</PayloadDirectory>
-      <PreCommands>$(Python) pre.py publish -c Release -f $(PERFLAB_Framework)-windows10.0.19041.0</PreCommands>
+    <HelixWorkItem Include="@(MAUIScenario -> 'Device Startup - %(Identity) Default')">
+      <PreCommands Condition="'$(TargetsWindows)' == 'true'">xcopy %HELIX_CORRELATION_PAYLOAD%\$(PreparePayloadOutDirectoryName)\%(HelixWorkItem.ScenarioDirectoryName)_fdd %HELIX_WORKITEM_ROOT%\pub /E /I /Y</PreCommands>
+      <PreCommands Condition="'$(TargetsWindows)' != 'true'">cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName)_fdd $HELIX_WORKITEM_ROOT/pub</PreCommands>
       <Command>$(Python) test.py startup --scenario-name &quot;%(Identity)&quot;</Command>
     </HelixWorkItem>
   </ItemGroup>
 
+
+  <Import Project="PreparePayloadWorkItems.targets" />
+
+
   <!--
     This is useful for local testing to print the produced helix items
     To use this when you are changing how items are produced, uncomment the target

diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml
@@ -102,6 +102,18 @@ jobs:
             displayName: Build SOD tool
             env:
               PERFLAB_TARGET_FRAMEWORKS: $(PERFLAB_Framework)
+          - powershell: |
+              $(Python) -m pip install --upgrade pip
+              $(Python) -m pip install requests
+              .\src\scenarios\init.ps1
+              .\eng\common\msbuild.ps1 .\eng\performance\${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:.\artifacts\log\$(_BuildConfig)\PrepareWorkItemPayloads.binlog
+            displayName: Prepare scenarios
+            env:
+              CorrelationPayloadDirectory: $(CorrelationStaging)
+              Architecture: ${{ parameters.architecture }}
+              TargetsWindows: 'true'
+              WorkItemDirectory: $(Build.SourcesDirectory)
+              HelixTargetQueues: ${{ parameters.queue }}
           # Current version of PaintDotNet zip file is PDN7 built on 5/18/2022
           - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
             - powershell: |
@@ -126,6 +138,18 @@ jobs:
             displayName: Build SOD tool
             env:
               PERFLAB_TARGET_FRAMEWORKS: $(PERFLAB_Framework)
+          - script: |
+              $(Python) -m pip install --upgrade pip
+              $(Python) -m pip install requests
+              . ./src/scenarios/init.sh
+              ./eng/common/msbuild.sh ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog
+            displayName: Prepare scenarios
+            env:
+              CorrelationPayloadDirectory: $(CorrelationStaging)
+              Architecture: ${{ parameters.architecture }}
+              TargetsWindows: 'false'
+              WorkItemDirectory: $(Build.SourcesDirectory)
+              HelixTargetQueues: ${{ parameters.queue }}
         - template: /eng/performance/send-to-helix.yml
           parameters:
             HelixSource: '$(HelixSourcePrefix)/dotnet/performance/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/

diff --git a/eng/performance/sdk_scenarios.proj b/eng/performance/sdk_scenarios.proj
@@ -196,5 +196,10 @@
       <Timeout>4:00</Timeout>
     </HelixWorkItem>
   </ItemGroup>
+
+
+  <Import Project="PreparePayloadWorkItems.targets" />
+
+
 </Project>