Onboard with OneLocBuild Task and Add Security checks (#631)

* Initial commit to move to one loc

* Fix templates reference

* Don't depend on validate

* try fake it

* Use loc branch variable

* One loc

* try fix path

* Fix StringResources filename

* Fix git add

* Don't use variable

* remove xliff

* test main

* Set main again

* Simplify policheck compliance

* Add loc to triggers

* Bump .NET version

* Update merge-translations-update.yml

* Update security_compliance.yml

eng/automation/LocProject.json

@@ -0,0 +1,23 @@
+{
+    "Projects": [
+      {
+        "LanguageSet": "VS_Main_Languages",
+        "LocItems": [
+            {
+                "SourceFile": "src\\Compatibility\\Core\\src\\iOS\\Resources\\StringResources.resx",
+                "LclFile": "loc\\{Lang}\\src\\Compatibility\\Core\\src\\iOS\\Resources\\StringResources.resx.lcl",
+                "CopyOption": "LangIDOnName",
+                "OutputPath": "src\\Compatibility\\Core\\src\\iOS\\Resources\\"
+            },
+            {
+                "SourceFile": "src\\Controls\\src\\Build.Tasks\\ErrorMessages.resx",
+                "LclFile": "loc\\{Lang}\\src\\Controls\\src\\Build.Tasks\\ErrorMessages.resx.lcl",
+                "CopyOption": "LangIDOnName",
+                "OutputPath": "src\\Controls\\src\\Build.Tasks\\xlf\\"
+            }
+        ],
+        "LssFiles": [],
+        "CloneLanguageSet": ""
+      }
+    ]
+}

eng/automation/scripts/Invoke-Git.psm1

@@ -0,0 +1,34 @@
+<#
+Copyright (c) Microsoft Corporation.  All rights reserved.
+ #>
+<#
+.SYNOPSIS
+Module for invoking git in a safe way that allows for stderr to be written to stdout.
+#>
+function Invoke-Git {
+    <#
+    .Synopsis
+    Wrapper function that deals with Powershell's peculiar error output when Git uses the error stream.
+    .Example
+    Invoke-Git ThrowError
+    $LASTEXITCODE
+    #>
+    [CmdletBinding()]
+    param(
+        [parameter(ValueFromRemainingArguments=$true)]
+        [string[]]$Arguments
+    )
+    & {
+        [CmdletBinding()]
+        param(
+            [parameter(ValueFromRemainingArguments=$true)]
+            [string[]]$InnerArgs
+        )
+        git.exe $InnerArgs
+    } -ErrorAction SilentlyContinue -ErrorVariable fail @Arguments
+    if ($fail) {
+        $fail.Exception
+    }
+}
+#Exports
+Export-ModuleMember -Function Invoke-Git

eng/pipelines/common/localization-handback.yml

@@ -0,0 +1,84 @@
+
+stages:
+  - stage: localization_handback
+    displayName: Localization Handback
+    dependsOn: []
+    condition: and(succeeded(), eq(variables.isLocBranch, true))
+
+    jobs:
+      - job : generate_resx
+        displayName: 'Process incoming translations'
+        pool:  $(HostedWinVS2019)
+        steps:
+          - checkout: self
+            persistCredentials: true
+            clean: true
+
+          - powershell: |
+              #-- Import Invoke-Git Module function
+              Import-Module $(Build.SourcesDirectory)\eng\automation\scripts\Invoke-Git.psm1 -Force -DisableNameChecking
+              git config user.email "vs-mobiletools-engineering-service2@microsoft.com"
+              git config user.name "VS Mobile Engineering Serice Account"
+              Invoke-Git checkout main
+              Invoke-Git merge origin/$(LocBranch) --no-commit
+            displayName: 'Merge loc Branch'
+
+          - task: cesve.one-loc-build.one-loc-build.OneLocBuild@2
+            displayName: 'Localization Build'
+            env:
+              SYSTEM_ACCESSTOKEN: $(System.AccessToken) 
+            inputs:
+              locProj: 'eng/automation/LocProject.json'
+              outDir: '$(Build.ArtifactStagingDirectory)'
+              packageSourceAuth: patAuth
+              patVariable: "$(OneLocBuildPAT)"
+              isCreatePrSelected: false
+              repoType: gitHub
+              prSourceBranchPrefix: $(LocBranchPrefix)
+              gitHubPatVariable: '$(GitHub.Token)'
+              gitHubPrMergeMethod: merge
+
+          - powershell: |
+              #-- Import Invoke-Git Module function
+              Import-Module $(Build.SourcesDirectory)\eng\automation\scripts\Invoke-Git.psm1 -Force -DisableNameChecking
+              #--Clear Stage
+              Invoke-Git reset HEAD -- .
+              #-- Create new branch
+              $branchName = "$(LocBranchPrefix)/$(new-guid)"
+              Invoke-Git checkout -B ${branchName}
+              Write-Host ("##vso[task.setvariable variable=HANDBACK_BRANCH_NAME;]$branchName")
+              #-- Stage Build Changes
+              Invoke-Git add .
+              #-- Only commit/push if there are changed files.
+              $changedFiles = $(git status --porcelain | Measure-Object | Select-Object -expand Count)
+              if ($changedFiles -gt 0)
+              {
+                #-- Commit Changes locally
+                Invoke-Git commit -m "[LOC_HB] string translations" --no-verify
+                #-- Push changes to VSTS
+                Invoke-Git push origin HEAD:${branchName}
+              }
+            displayName: 'Commit updates'
+
+          - powershell: |
+              $payload=@{
+                title = "[LOC_HB] checkin of localized string files"
+                head = "$(HANDBACK_BRANCH_NAME)"
+                base = "main"
+                maintainer_can_modify = $true
+              }
+
+              $headers = @{ Authorization = "token $(GitHub.Token)" }
+
+              # let it throw
+              $response = Invoke-WebRequest -UseBasicParsing -Method POST -Headers $headers -Uri "https://api.github.com/repos/dotnet/maui/pulls" -Body ($payload | ConvertTo-json)
+              $newPr =  $response.Content | ConvertFrom-Json
+
+              Write-Host "Response is $newPr"      
+            displayName: Open Pull Request
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              PathtoPublish: '$(Build.ArtifactStagingDirectory)'
+              ArtifactName: 'drop'
+              publishLocation: 'Container'

eng/pipelines/common/localization-handoff.yml

@@ -0,0 +1,27 @@
+stages:
+  - stage: localization_handoff
+    displayName: Localization Handoff
+    dependsOn: []
+    condition: and(succeeded(), eq(variables.isMainBranch, true))
+
+    jobs:
+      - job : generate_lci
+        displayName: 'Process outgoing strings'
+        pool:  $(HostedWinVS2019)
+        steps:
+
+          - task: cesve.one-loc-build.one-loc-build.OneLocBuild@2
+            displayName: 'Localization Build'
+            env:
+              SYSTEM_ACCESSTOKEN: $(System.AccessToken) 
+            inputs:
+              locProj: 'eng/automation/LocProject.json'
+              outDir: '$(Build.ArtifactStagingDirectory)'
+              packageSourceAuth: patAuth
+              patVariable: "$(OneLocBuildPAT)"
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              PathtoPublish: '$(Build.ArtifactStagingDirectory)'
+              ArtifactName: 'drop'
+              publishLocation: 'Container'

eng/pipelines/common/merge-translations-update.yml

@@ -0,0 +1,37 @@
+stages:
+  - stage: merge_translations_updates
+    displayName: 'Merge Translations Updates'
+    dependsOn: []
+    condition: and( succeeded(), and( eq(variables.isTargetMainBranch, true), variables.isLocPRBranch ) ) 
+
+    jobs:
+      - job : validate_merge
+        displayName: 'Validate and Merge Translations'
+        pool:
+          name: XamarinForms
+        variables:
+          pull_request_number: $(System.PullRequest.PullRequestNumber)
+        workspace:
+          clean: all
+
+        steps:
+          - powershell: |
+              Write-Host "Validating translations for PR# $(System.PullRequest.PullRequestNumber)"
+              $srcDir = "$(Build.SourcesDirectory)/maui"
+              $matches = Select-String -Path "$srcDir/**/*.resx" -Pattern '\[.*\]\s\(https:.*\)' -AllMatches
+              $matchCount = ($matches | Measure-Object -Line).Lines
+              Write-Host "Found $matchCount violations."
+              $matches | Write-Host
+              if( ($matches | Measure-Object -Line).Lines -eq 0 ) {
+                Write-Host "Quality Gate Succeeded."
+              } else {
+                throw "Quality Gate Failure : Check the logs for details."
+              }
+            displayName: 'Validate Markdown Urls'
+          - task: github-merge-pr@1
+            inputs:
+              github_token: $(GitHub.Token)
+              repository: 'dotnet/maui'
+              pr_number: $(pull_request_number)
+              merge_method: 'rebase'
+            displayName: Merge PR# $(System.PullRequest.PullRequestNumber)

eng/pipelines/common/security_compliance.yml

@@ -0,0 +1,43 @@
+stages:
+  - stage: security_compliance
+    displayName: Security and Compliance checks
+    jobs:
+    - job: run_policheck_security_compliance
+      displayName: 'Policheck And Credentials Compliance'
+      pool: $(HostedWinVS2019)
+      timeoutInMinutes: 60
+      cancelTimeoutInMinutes: 5
+      steps:
+      - checkout: self
+
+      - template: security/policheck/v1.yml@xamarin-templates
+
+      - template: security/xa-static-analysis/v2.yml@xamarin-templates
+        parameters:
+          credScanSuppressionsFile: $(System.DefaultWorkingDirectory)\eng\automation\CredScanSuppressions.json
+
+      - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3
+        displayName: Run AntiMalware (Defender) Scan
+        condition: succeededOrFailed()
+        inputs:
+          FileDirPath: $(System.DefaultWorkingDirectory)
+          EnableServices: true
+
+      - task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1
+        displayName: Create Security Analysis Report
+        condition: succeededOrFailed()
+        inputs:
+          AllTools: true
+
+      - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2
+        displayName: Publish Security Analysis Logs
+        condition: succeededOrFailed()
+        inputs:
+          ArtifactName: CodeAnalysisLogs
+
+      - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@1
+        displayName: Fail Job if Security Issues are Detected
+        condition: succeededOrFailed()
+        inputs:
+          AllTools: true
+

eng/pipelines/common/variables.yml

@@ -6,4 +6,23 @@ variables:
 - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE
   value: true
 - name: DOTNET_VERSION
-  value: 5.0.102
+  value: 5.0.201
+- name: LocBranchPrefix
+  value: 'loc-hb'
+- name: LocBranch
+  value: 'loc'
+- name: isMainBranch
+  value: $[eq(variables['Build.SourceBranch'], 'refs/heads/main')]
+- name: isLocBranch
+  value: $[eq(variables['Build.SourceBranch'], 'refs/heads/loc')]
+- name: isTargetMainBranch
+  value: $[eq(variables['System.PullRequest.TargetBranch'], 'refs/heads/main')]
+- name: isTargetLocBranch
+  value: $[eq(variables['System.PullRequest.TargetBranch'], 'refs/heads/loc')]
+- name: isLocPRBranch
+  value: $[startsWith(variables['System.PullRequest.SourceBranch'], 'refs/heads/loc-hb')]
+- name: isPullRequest
+  value: $[eq(variables['Build.Reason'], 'PullRequest')]
+# Common Agent Pools in use
+- name: HostedWinVS2019
+  value: Hosted Windows 2019 with VS2019

eng/pipelines/handlers.yml

@@ -45,6 +45,7 @@ schedules:
   branches:
     include:
     - main
+    - loc
 
 variables:
   - template: /eng/pipelines/common/variables.yml
@@ -337,8 +338,14 @@ stages:
           android: $(System.DefaultWorkingDirectory)/src/Core/tests/DeviceTests.Android/Core.DeviceTests.Android.csproj
           ios: $(System.DefaultWorkingDirectory)/src/Core/tests/DeviceTests.iOS/Core.DeviceTests.iOS.csproj
 
-  # only sign using the private server
-  - ${{ if eq(variables['System.TeamProject'], 'devdiv') }}:
+          stages:
+
+  - template: common/security_compliance.yml
+  - template: common/localization-handoff.yml                     # Process outgoing strings [Localization Handoff]
+  - template: common/localization-handback.yml                    # Process incoming translations and Create PR to main [Localization Handback]
+  - template: common/merge-translations-update.yml                # Validating incoming translations strings and merge PR [Localization Handback]
+
+  - ${{ if eq(variables['System.TeamProject'], 'devdiv') }}:      # Sign only using the private server
     - stage: nuget_signing
       dependsOn: pack_net6
       displayName: Sign Nuget