Add Guardian SDL pipeline after internal builds

eng/common/templates/job/execute-sdl.yml

@@ -59,7 +59,9 @@ jobs:
   - checkout: self
     clean: true
 
-  - template: /eng/common/templates/post-build/setup-maestro-vars.yml
+  # If the template caller didn't provide an AzDO parameter, set them all up as Maestro vars.
+  - ${{ if not(and(parameters.AzDOProjectName, parameters.AzDOPipelineId, parameters.AzDOBuildId)) }}:
+    - template: /eng/common/templates/post-build/setup-maestro-vars.yml
 
   - ${{ if ne(parameters.downloadArtifacts, 'false')}}:
     - ${{ if ne(parameters.artifactNames, '') }}:

eng/compliance/Guardian/BinSkimConfig.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (c) Microsoft Corporation. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. -->
+<!--
+  To figure out what to put in this file, run this command:
+
+    {nuget-cache}\Microsoft.CodeAnalysis.BinSkim.{version}\tools\netcoreapp3.1\win-x64\BinSkim.exe
+      export-config {target-location}\BinSkimConfigFull.xml
+
+  Then, search in the resulting file to find the XML element with the configuration you want and
+  copy it into this file. For example, searching for "BA3001" (as seen in SDL tool logs) shows how
+  to disable "BA3001.EnablePositionIndependentExecutable.Options".
+-->
+<Properties Type="PropertiesDictionary">
+  <!-- Go doesn't build PIE executables by default: https://github.com/microsoft/go/issues/104 -->
+  <Properties Key="BA3001.EnablePositionIndependentExecutable.Options" Type="PropertiesDictionary">
+    <Property Key="RuleEnabled" Value="Disabled" Type="Driver.RuleEnabledState" />
+  </Properties>
+  <!-- Go has no stack protector, but it can be enabled for Cgo: https://github.com/microsoft/go/issues/104 -->
+  <Properties Key="BA3003.EnableStackProtector.Options" Type="PropertiesDictionary">
+    <Property Key="RuleEnabled" Value="Disabled" Type="Driver.RuleEnabledState" />
+  </Properties>
+  <!-- Go doesn't enable relro by default: https://github.com/microsoft/go/issues/104 -->
+  <Properties Key="BA3010.EnableReadOnlyRelocations.Options" Type="PropertiesDictionary">
+    <Property Key="RuleEnabled" Value="Disabled" Type="Driver.RuleEnabledState" />
+  </Properties>
+</Properties>

eng/compliance/Guardian/CredScanSuppressions.json

@@ -0,0 +1,17 @@
+{
+    "tool": "Credential Scanner",
+    "suppressions": [
+        {
+            "file": "example_test.go",
+            "_justification": "Public secret for testing purposes in an upstream source file."
+        },
+        {
+            "file": "example-key.pem",
+            "_justification": "Public secret for testing purposes in an upstream source file."
+        },
+        {
+            "file": "boring_test.go",
+            "_justification": "Public secret for testing purposes in an upstream source file."
+        }
+    ]
+}

eng/compliance/Guardian/README.md

@@ -0,0 +1,32 @@
+# Guardian
+
+Guardian is an internal Microsoft tool written in .NET that runs a suite of SDL (Security Development Lifecycle) tools. It also runs PoliCheck, which is not an SDL tool, but it is convenient to let Guardian run it and report the results.
+
+Internal rolling builds run Guardian and report results.
+
+The microsoft/go implementation of Guardian execution is based on [dotnet/arcade](https://github.com/dotnet/arcade). See [HowToAddSDLRunToPipeline.md](https://github.com/dotnet/arcade/blob/main/Documentation/HowToAddSDLRunToPipeline.md).
+
+# Running Guardian locally on Windows
+
+Microsoft internal auth is necessary to download the SDL tools.
+
+1. Create a temporary folder, e.g. `C:\temp\sdl`.
+1. Go to
+    https://dev.azure.com/SecurityTools/SecurityIntegration/_packaging?_a=package&feed=Guardian&package=Microsoft.Guardian.Cli&protocolType=NuGet
+    and download the desired version.
+1. Extract the `nupkg` file (it's just a `zip`) to a known location like `C:\temp\guardian`.
+1. Clone the Go repo into `C:\temp\sdl\src`.
+1. Place artifacts to validate into `C:\temp\sdl\artifacts`.
+    1. To validate a `zip` or `tar.gz`, extract it.
+1. Open a powershell terminal.
+    1. The build job uses `powershell`, not `pwsh`.
+1. Set `$env:BUILD_ARTIFACTSTAGINGDIRECTORY = "C:\temp\sdl"`
+1. Set `$env:BUILD_SOURCESDIRECTORY = "C:\temp\sdl\go"`
+1. In `C:\temp\sdl`, run:
+    ```powershell
+    & go\eng\compliance\Guardian\execute-go-sdl-tools.ps1 `
+        -GuardianCliLocation C:\temp\sdl\guardian\tools\guardian.cmd `
+        -WorkingDirectory C:\temp\sdl
+    ```
+
+Some steps (such as PoliCheck) may refuse to run locally due to lack of authentication, even if you have Microsoft internal auth. Those must be run in the internal rolling (official) build job. Running Guardian locally only confirms some basic functionality.

eng/compliance/Guardian/execute-go-sdl-tools.ps1

@@ -0,0 +1,83 @@
+# Copyright (c) Microsoft Corporation.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+$srcDir = $env:BUILD_SOURCESDIRECTORY
+# Microsoft-specific engineering tools and configuration.
+$engDirectory = Join-Path $srcDir "eng"
+# Microsoft-specific GitHub and GitHub Actions configuration.
+$dotGitHubDirectory = Join-Path $srcDir ".github"
+# Official build artifacts, downloaded from the build job that completed earlier.
+$downloadedArtifactsDirectory = Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY "artifacts"
+
+# Create a file for PoliCheck's ListFile option. The extension must be ".txt", and this file must
+# contain full paths, one per line, with no duplicates. The list should contain each microsoft/go
+# file but no upstream files. Sort and print it for debug purposes.
+$policheckFileList = (New-TemporaryFile).FullName + ".txt"
+(
+  Get-ChildItem -File -Recurse $srcDir `
+  | Where-Object {
+    # Submodule directory with upstream code.
+    -not $_.FullName.StartsWith((Join-Path $srcDir "go")) -and `
+    # SDL NuGet packages: ignore, not part of our code.
+    -not $_.FullName.StartsWith((Join-Path $srcDir ".packages")) } `
+  | ForEach-Object { $_.FullName } | Sort-Object `
+) -join "`r`n" > $policheckFileList
+
+Write-Host "--- List of files in PoliCheck file list:"
+Get-Content $policheckFileList | Write-Host
+Write-Host "---"
+
+& "$PSScriptRoot\..\..\common\sdl\execute-all-sdl-tools.ps1" `
+  -SourceToolsList @(
+    @{ Name="credscan"; Scenario="source" }
+  ) `
+  -ArtifactToolsList @(
+    @{ Name="credscan"; Scenario="artifacts" }
+  ) `
+  -CrScanAdditionalRunConfigParams @(
+    "SuppressionsPath < $engDirectory\compliance\Guardian\CredScanSuppressions.json"
+    "SuppressAsError < false"
+  ) `
+  -CustomToolsList @(
+    @{
+      Name="binskim"
+      Args=@(
+        # Point binskim at the artifact directory. Pass everything to binskim and let it decide what
+        # it needs to scan. For more information about the glob format, see
+        # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1378/Glob-Format
+        #
+        # Exclude "testdata" binaries because they are only used during testing, they do not pass
+        # "binskim" for various reasons, and they are checked into the upstream Go repository.
+        #
+        # Exclude infra dependencies in ".gdn" dir. We are not distributing these.
+        #
+        # Exclude all ".exe" files. BinSkim strongly expects PDB files for each one, but they don't
+        # exist for Go. See https://github.com/microsoft/go/issues/114
+        "Target < f|$downloadedArtifactsDirectory\**;-|**\testdata\*;-|.gdn\**;-|**\*.exe"
+        "ConfigPath < $engDirectory\compliance\Guardian\BinSkimConfig.xml"
+      )
+    }
+    @{
+      Name="codesign"
+      Args=@(
+        # Point codesign at the right location to find the artifacts that we've signed. However, we do
+        # not yet produce any artifacts that CodeSign knows how to verify, so don't fail if CodeSign
+        # fails to find anything.
+        "TargetDirectory < $downloadedArtifactsDirectory"
+        "targetFiles < f|**\*.dll;f|**\*.exe"
+        "failIfNoTargetsFound < false"
+      )
+    }
+    # Only point PoliCheck at directories we control, not directories from the upstream repo.
+    @{
+      Name="policheck"
+      Args=@(
+        # Target's default is ".", but we need to pass nothing instead. The Target and ListFile
+        # PoliCheck args are mutually exclusive.
+        "Target"
+        "ListFile < $policheckFileList"
+      )
+    }
+  ) `
+  @args

eng/pipeline/README.md

@@ -6,17 +6,15 @@ Pipeline definitions currently using each YAML file are:
 
 * [`pr-pipeline.yml`](pr-pipeline.yml) - Required PR check.
   * (Public) [microsoft-go](https://dev.azure.com/dnceng/public/_build?definitionId=1099)
-* [`pr-outerloop-pipeline.yml`](pr-outerloop-pipeline.yml) - Optional PR check.
-  Runs outerloop.
+* [`pr-outerloop-pipeline.yml`](pr-outerloop-pipeline.yml) - Optional PR check. Runs outerloop.
   * (Public) [microsoft-go-outerloop](https://dev.azure.com/dnceng/public/_build/index?definitionId=1100)
     * Comment `/azp run microsoft-go-outerloop` on a PR to run.
-* [`rolling-pipeline.yml`](rolling-pipeline.yml) - Triggers on merge, runs
-  innerloop + outerloop.
+* [`rolling-pipeline.yml`](rolling-pipeline.yml) - Triggers on merge, runs innerloop + outerloop.
   * (Internal) [microsoft-go-rolling](https://dev.azure.com/dnceng/internal/_build?definitionId=987)
-* [`rolling-internal-pipeline.yml`](rolling-internal-pipeline.yml) - Triggers on
-  merge. Builds, signs, and publishes. Runs innerloop, and won't publish if
-  innerloop fails.
+* [`rolling-internal-pipeline.yml`](rolling-internal-pipeline.yml) - Triggers on merge. Builds, signs, and publishes. Runs innerloop, and won't publish if innerloop fails.
   * (Internal) [microsoft-go](https://dev.azure.com/dnceng/internal/_build?definitionId=958)
+* [`rolling-internal-validation-pipeline.yml`](rolling-internal-validation-pipeline.yml) - Runs validation checks on internal build output.
+  * (Internal) [microsoft-go-validation](https://dev.azure.com/dnceng/internal/_build?definitionId=1166)
 
 The pipeline filenames are (mostly) based on the trigger scenario, not what they
 do. This means we can change their content later without worrying about

eng/pipeline/rolling-internal-validation-pipeline.yml

@@ -0,0 +1,69 @@
+# Copyright (c) Microsoft Corporation.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This pipeline runs after each internal rolling build of Go and validates compliance.
+#
+# This pipeline template runs automated SDL validation with internal-only tooling. It uses a job
+# template from dotnet/arcade that runs the Guardian suite of tools and reports the results to TSA
+# (Trust Services Automation).
+#
+# For more information, see:
+# https://microsoft.sharepoint.com/teams/managedlanguages/_layouts/OneNote.aspx?id=%2Fteams%2Fmanagedlanguages%2Ffiles%2FTeam%20Notebook%2FGoLang%20Team&wd=target%28Main.one%7C62B655D4-14E7-41D6-A063-0869C28D63FC%2FSDL%20Tools%7C3908F727-3751-4ACC-8C71-6CEB2DF277B4%2F%29
+
+trigger: none
+pr: none
+
+resources:
+  pipelines:
+    - pipeline: build
+      # The rolling pipeline and this validation pipeline share the same source repository. AzDO
+      # sees this and makes this pipeline's "checkout" steps download the same source code that was
+      # built by the microsoft-go pipeline:
+      # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/resources?view=azure-devops&tabs=schema#define-a-pipelines-resource
+      #
+      # This means we can have SDL scan the currently-checked-out source code as the way to scan the
+      # source code of the internal rolling build.
+      source: microsoft-go
+      trigger:
+        branches:
+          include:
+            # Validate all branches that may be released.
+            - microsoft/main
+            - microsoft/release-branch.*
+            - microsoft/dev.boringcrypto.go*
+
+stages:
+  - stage: SDLValidate
+    variables:
+      # TSA variables.
+      - group: go-sdl-validation
+    jobs:
+      # Run SDL validation tooling on sources and signed/complete artifacts.
+      - template: /eng/common/templates/job/execute-sdl.yml
+        parameters:
+          # Don't download any build artifacts: only pipeline artifacts.
+          downloadArtifacts: false
+          pipelineArtifactNames:
+            - Binaries Signed
+          extractArchiveArtifacts: true
+          enable: true
+          publishGuardianDirectoryToPipeline: true
+          # Specify that artifacts should be downloaded from the build that triggered this one.
+          AzDOProjectName: $(resources.pipeline.build.projectID)
+          AzDOPipelineId: $(resources.pipeline.build.pipelineID)
+          AzDOBuildId: $(resources.pipeline.build.runID)
+          # Use a wrapper script for the SDL tools to pass the Go-specific configuration.
+          executeAllSdlToolsScript: eng/compliance/Guardian/execute-go-sdl-tools.ps1
+          # Set up TSA publish and build break condition.
+          additionalParameters: >-
+            -TsaInstanceURL "$(TsaInstanceURL)"
+            -TsaProjectName "$(TsaProjectName)"
+            -TsaNotificationEmail "$(TsaNotificationEmail)"
+            -TsaCodebaseAdmin "$(TsaCodebaseAdmin)"
+            -TsaBugAreaPath "$(TsaBugAreaPath)"
+            -TsaIterationPath "$(TsaIterationPath)"
+            -TsaRepositoryName "$(TsaRepositoryName)"
+            -TsaCodebaseName "$(TsaCodebaseName)"
+            -TsaPublish $true
+            -BreakOnFailure $true