Use sparse checkout for generate matrix job (#1452)

From a functional standpoint, the generate matrix job only takes a few seconds to run. Depending on the repository, the job can take a total time of 4 minutes before the test jobs are able to start, because azure pipelines does a clone of the entire repository (among other things). This PR makes a change to skip the azure pipelines checkout step, and add a lightweight git clone along with a sparse checkout of just the `eng` and service directory locations (which contain the matrix scripts and/or matrix configs).

This change reduces the total job runtime to around 20 seconds from 4 minutes (repo checkouts + auto-injected policy steps).

eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml

@@ -12,16 +12,40 @@ parameters:
   default: []
 - name: JobTemplatePath
   type: string
+# Set this to false to do a full checkout for private repositories with the azure pipelines service connection
+- name: SparseCheckout
+  type: boolean
+  default: true
+- name: SparseCheckoutPaths
+  type: object
+  default: []
+- name: Pool
+  type: string
+  default: azsdk-pool-mms-ubuntu-1804-general
+- name: OsVmImage
+  type: string
+  default: MMSUbuntu18.04
 
 jobs:
 - job: generate_matrix
   variables:
     displayNameFilter: $[ coalesce(variables.jobMatrixFilter, '.*') ]
   pool:
-    name: Azure Pipelines
-    vmImage: ubuntu-18.04
+    name: ${{ parameters.Pool }}
+    vmImage: ${{ parameters.OsVmImage }}
   displayName: Generate Job Matrix
   steps:
+    # Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
+    # as we require the github service connection to be loaded.
+    - ${{ if and(parameters.SparseCheckout, not(contains(variables['Build.DefinitionName'], '-pr - '))) }}:
+      - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+        parameters:
+          ${{ if ne(length(parameters.SparseCheckoutPaths), 0) }}:
+            Paths: ${{ parameters.SparseCheckoutPaths }}
+          ${{ if and(eq(length(parameters.SparseCheckoutPaths), 0), ne(parameters.AdditionalParameters.ServiceDirectory, '')) }}:
+            Paths:
+              - ${{ parameters.AdditionalParameters.ServiceDirectory }}
+
     - ${{ each config in parameters.MatrixConfigs }}:
       - ${{ if eq(config.GenerateVMJobs, 'true') }}:
         - task: Powershell@2

eng/common/pipelines/templates/steps/sparse-checkout.yml

@@ -0,0 +1,34 @@
+parameters:
+  - name: Paths
+    type: object
+    default: []
+  - name: Repositories
+    type: object
+    default:
+      - Name: $(Build.Repository.Name)
+        Commitish: $(Build.SourceVersion)
+        WorkingDirectory: $(System.DefaultWorkingDirectory)
+
+steps:
+  - checkout: none
+
+  - ${{ each repo in parameters.Repositories }}:
+    - pwsh: |
+        $dir = "${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }}"
+        New-Item $dir -ItemType Directory -Force
+
+    - pwsh: |
+        git clone --no-checkout --filter=tree:0 git://github.com/${{ repo.Name }} .
+        git sparse-checkout init
+        git sparse-checkout set eng
+      displayName: Init sparse checkout ${{ repo.Name }}
+      workingDirectory: ${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }}
+
+    - ${{ each path in parameters.Paths }}:
+      - pwsh: git sparse-checkout add ${{ path }}
+        displayName: Add sparse checkout path ${{ path }}
+        workingDirectory: ${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }}
+
+    - pwsh: git checkout ${{ repo.Commitish }}
+      displayName: Sparse checkout at ${{ repo.Commitish }}
+      workingDirectory: ${{ coalesce(repo.WorkingDirectory, format('{0}/{1}', '$(System.DefaultWorkingDirectory)', repo.Name)) }}

eng/common/scripts/job-matrix/samples/matrix-test.yml

@@ -7,7 +7,9 @@ jobs:
   - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml
     parameters:
       JobTemplatePath: /eng/common/scripts/job-matrix/samples/matrix-job-sample.yml
-      AdditionalParameters: []
+      AdditionalParameters: {}
+      Pool: Azure Pipelines
+      OsVmImage: ubuntu-18.04
       CloudConfig:
         SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
         Location: eastus2