diff --git a/eng/pipelines/aggregate-reports.yml b/eng/pipelines/aggregate-reports.yml index a57a2ba686815..72032f968293a 100644 --- a/eng/pipelines/aggregate-reports.yml +++ b/eng/pipelines/aggregate-reports.yml @@ -17,12 +17,11 @@ jobs: pool: vmImage: 'ubuntu-18.04' steps: + - template: ../steps/cache-maven-repository.yml + # We `install` the code quality reports tooling into our local m2 cache separately from building the Maven project # reports. This means it is available as part of that, but also so that this is not documented in the project report. - template: templates/steps/install-reporting-tools.yml - parameters: - Options: --batch-mode - MavenOptions: $(MemoryOptions) $(LoggingOptions) - task: Maven@3 displayName: 'Build All Libraries' @@ -88,6 +87,9 @@ jobs: condition: succeededOrFailed() displayName: 'Publish Report Artifacts' artifact: reports + + - template: ../steps/upload-repository-on-failure.yml + - job: Compliance displayName: Analyze (no build) timeoutInMinutes: 120 @@ -142,7 +144,7 @@ jobs: endColumn = $_.locations.physicalLocation.region.endColumn level = $_.level } - } + } $csvRows | Export-Csv -Path "$(Pipeline.workspace)\.gdn\.r\CredScan\001\CredScan-matches.csv" -NoTypeInformation displayName: Add converted csv file for CredScan-matches.sarif - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3 diff --git a/eng/pipelines/templates/jobs/build-validate-pom.yml b/eng/pipelines/templates/jobs/build-validate-pom.yml index 88f100391dc36..4811b7eee14bd 100644 --- a/eng/pipelines/templates/jobs/build-validate-pom.yml +++ b/eng/pipelines/templates/jobs/build-validate-pom.yml @@ -41,13 +41,13 @@ jobs: - template: ../steps/cache-maven-repository.yml # HACK: Ideally I wouldn't need to do this but I need to figure out the right - # approach here around getting this dependency installed whilst targetting - # the parent POMs via the root aggregeate (given they aren't listed) - # anywhere in the depedency graph. + # approach here around getting this dependency installed whilst targeting + # the parent POMs via the root aggregate (given they aren't listed) + # anywhere in the dependency graph. # # The answer might be to finally strip out the references from # the parent POMs (something we want to do anyway) and then add them to - # the graph so that we can build off the root POM targetting the parent + # the graph so that we can build off the root POM targeting the parent # projects and then it would be able to find sdk-build-tools. But for now # I am more interested getting the end-to-end scenario working with this # hack rather than getting bogged down. diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index ca7a1c3c14394..f9b9e69cec999 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -22,7 +22,7 @@ parameters: default: '$(DefaultOptions) -T 2C -DskipTests -Dgpg.skip -DtrimStackTrace=false -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -Djacoco.skip=true' - name: TestOptions type: string - default: '$(DefaultOptions) -DtrimStackTrace=false -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -Dparallel-test-playback' + default: '$(DefaultTestOptions) -Dparallel-test-playback -DAZURE_TEST_DEBUG=$(IsDebug)' - name: Matrix type: object - name: CloudConfig @@ -62,12 +62,7 @@ jobs: ServiceDirectory: ${{ parameters.ServiceDirectory }} CheckoutRecordings: true - - template: ../steps/cache-maven-repository.yml - - template: ../steps/install-reporting-tools.yml - parameters: - Options: --batch-mode - MavenOptions: $(MemoryOptions) $(LoggingOptions) - ${{ parameters.PreTestSteps }} @@ -88,7 +83,7 @@ jobs: displayName: 'Build for non-From Source run' inputs: mavenPomFile: pom.xml - options: ${{ parameters.BuildOptions }} -pl $(PLSkipOptions)$(ProjectList) -am + options: ${{ parameters.BuildOptions }} -pl $(ProjectList) -am mavenOptions: '$(MemoryOptions) $(LoggingOptions)' javaHomeOption: 'JDKVersion' jdkVersionOption: $(JavaTestVersion) @@ -102,7 +97,7 @@ jobs: displayName: 'Run tests' inputs: mavenPomFile: pom.xml - options: ${{ parameters.TestOptions }} $(TestOptions) -pl $(PLSkipOptions)$(ProjectList) $(AdditionalOptions) -DAZURE_TEST_DEBUG=$(IsDebug) + options: ${{ parameters.TestOptions }} $(TestOptions) $(AdditionalOptions) mavenOptions: '$(MemoryOptions) $(LoggingOptions)' javaHomeOption: 'JDKVersion' jdkVersionOption: $(JavaTestVersion) @@ -152,7 +147,7 @@ jobs: mavenPomFile: ClientFromSourcePom.xml # For the From Source runs we don't want the -am switch as we don't care about running tests for our dependencies # but we do want the -amd switch because we want to run tests on things that depend on us. - options: ${{ parameters.TestOptions }} $(TestOptions) -pl $(PLSkipOptions)$(ProjectList) -amd -T 1C -DAZURE_TEST_DEBUG=$(IsDebug) + options: ${{ parameters.TestOptions }} $(TestOptions) -amd -T 1C mavenOptions: '$(MemoryOptions) $(LoggingOptions)' javaHomeOption: 'JDKVersion' jdkVersionOption: $(JavaTestVersion) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 8446ee41ed0bf..4eb4c94987351 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -92,31 +92,27 @@ jobs: parameters: ServiceDirectory: ${{parameters.ServiceDirectory}} - - ${{ each artifact in parameters.Artifacts }}: - - script: | - python3 --version - python3 eng/versioning/set_versions.py --build-type ${{parameters.SDKType}} --build-qualifier alpha.$(Build.BuildNumber) --artifact-id ${{artifact.name}} --group-id ${{artifact.groupId}} - condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) - displayName: Append dev package version suffix for ${{artifact.name}} - - - script: | + - pwsh: | + $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json python3 --version + + # Append dev package version suffix for each artifact + foreach ($artifact in $artifacts) { + python3 eng/versioning/set_versions.py --build-type ${{parameters.SDKType}} --build-qualifier alpha.$(Build.BuildNumber) --artifact-id $artifact.name --group-id $artifact.groupId + } + + # Set zero-dev-version for packages python3 eng/versioning/set_versions.py --set-dev-zero-version --build-type ${{parameters.SDKType}} --build-qualifier alpha.$(Build.BuildNumber) - condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) - displayName: Set zero-dev-version for packages - - script: | - python3 --version + # Apply version settings to repository python3 eng/versioning/update_versions.py --update-type library --build-type ${{parameters.SDKType}} --sr condition: and(succeeded(), eq(variables['SetDevVersion'],'true')) - displayName: Apply version settings to repository + displayName: Setup Dev versioning - script: | pip install markdown2 BeautifulSoup4 displayName: 'pip install markdown2 and BeautifulSoup4' - - template: ../steps/cache-maven-repository.yml - - task: Maven@3 displayName: 'Build and Package, JDK Version: $(JavaBuildVersion)' inputs: @@ -231,7 +227,7 @@ jobs: parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} - # Use BasePathLength of 38 instead of the default 49 as some GA'd files fail when the number is higher. + # Use BasePathLength of 38 instead of the default 49 as some released files fail when the number is higher. - template: /eng/common/pipelines/templates/steps/verify-path-length.yml parameters: SourceDirectory: $(Build.SourcesDirectory) @@ -248,9 +244,6 @@ jobs: # We `install` the code quality reports tooling into our local m2 cache separately from building the Maven project # reports. This means it is available as part of that, but also so that this is not documented in the project report. - template: ../steps/install-reporting-tools.yml - parameters: - Options: --batch-mode - MavenOptions: $(MemoryOptions) $(LoggingOptions) # maven dependency:tree needs to be able to resolve dependencies, so these should be installed. - task: Maven@3 diff --git a/eng/pipelines/templates/stages/archetype-sdk-tests.yml b/eng/pipelines/templates/stages/archetype-sdk-tests.yml index 0d170f7ea75aa..7a2ee0d0c6cac 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-tests.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-tests.yml @@ -62,8 +62,8 @@ parameters: - name: TestStepMavenInputs type: object default: - options: '$(DefaultOptions) -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -DtrimStackTrace=false -pl $(ProjectList)' - mavenOptions: '$(MemoryOptions) $(LoggingOptions)' + options: '$(DefaultTestOptions)' + mavenOptions: '$(DefaultTestMavenOptions)' javaHomeOption: 'JDKVersion' jdkVersionOption: '$(JavaTestVersion)' jdkArchitectureOption: 'x64' diff --git a/eng/pipelines/templates/stages/cosmos-sdk-client.yml b/eng/pipelines/templates/stages/cosmos-sdk-client.yml index 1e2c08eea53b6..a24de9ab2101c 100644 --- a/eng/pipelines/templates/stages/cosmos-sdk-client.yml +++ b/eng/pipelines/templates/stages/cosmos-sdk-client.yml @@ -34,8 +34,6 @@ stages: Artifacts: ${{ parameters.Artifacts }} AdditionalModules: ${{ parameters.AdditionalModules }} SDKType: ${{parameters.SDKType}} - PreTestSteps: - - template: ../steps/install-reporting-tools.yml MatrixConfigs: - Name: Cosmos_ci_test Path: eng/pipelines/templates/stages/platform-matrix.json @@ -73,7 +71,8 @@ stages: TimeoutInMinutes: 90 TestStepMavenInputs: goals: clean verify - options: '$(ProfileFlag) -Dgpg.skip -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -DtrimStackTrace=false $(AdditionalArgs) -pl $(ProjectList)' + options: '$(DefaultTestOptions) $(ProfileFlag) $(AdditionalArgs)' + mavenOptions: '$(DefaultTestMavenOptions)' mavenAuthenticateFeed: true jdkVersionOption: $(JavaTestVersion) @@ -117,10 +116,6 @@ stages: Write-Host "New Path environment variable: $updatedPath" displayName: 'Download and Extract Hadoop winutils and update env variables for Hadoop usage' - # We `install` separately from running `site:site site:stage` so that the `install` brings in the non-shipping-modules, - # but we don't include them in the Maven site commands (so that we don't generate reports for the non-shipping modules). - - template: ../steps/install-reporting-tools.yml - # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch. - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: - template: archetype-java-release.yml diff --git a/eng/pipelines/templates/steps/cache-maven-repository.yml b/eng/pipelines/templates/steps/cache-maven-repository.yml index 65f512bd06aa4..0923ededd1c9e 100644 --- a/eng/pipelines/templates/steps/cache-maven-repository.yml +++ b/eng/pipelines/templates/steps/cache-maven-repository.yml @@ -1,15 +1,6 @@ steps: -- script: | - echo "##vso[task.setvariable variable=Maven.RepositoryPath;]%USERPROFILE%\.m2\repository" - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - displayName: Detecting Maven repository on Windows -- script: | - echo "##vso[task.setvariable variable=Maven.RepositoryPath;]$HOME/.m2/repository" - condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) - displayName: Detecting Maven repository on Linux and macOS -- task: CacheBeta@0 +- task: Cache@2 inputs: - key: $(Agent.JobName)|$(CacheSalt)|$(Build.SourcesDirectory)/sdk/**/pom.xml|$(Build.SourcesDirectory)/pom*.xml - path: $(Maven.RepositoryPath) - displayName: 'Download/upload cache' - condition: and(succeeded(), ne(variables['EnableCaching'], 'false')) \ No newline at end of file + key: 'maven | $(CacheSalt) | $(Agent.OS) | $(Build.SourcesDirectory)/eng/versioning/external_dependencies.txt' + path: $(MAVEN_CACHE_FOLDER) + displayName: Cache Maven local repo \ No newline at end of file diff --git a/eng/pipelines/templates/steps/generate-project-list.yml b/eng/pipelines/templates/steps/generate-project-list.yml index 96e0de7898246..59e523153f214 100644 --- a/eng/pipelines/templates/steps/generate-project-list.yml +++ b/eng/pipelines/templates/steps/generate-project-list.yml @@ -1,31 +1,28 @@ # This job template uses the artifact list in the ci.yml file # to generate a variable at build time that can be passed into # the Maven -pl command-line switch to constrain what subset of -# projects are targetted based on the artifacts this pipeline +# projects are targeted based on the artifacts this pipeline # produces. The -am and -amd switches are then applied as # appropriate to constrain the full graph that is built depending # on whether we are doing a recorded test play back or a full live # test. steps: +- template: ../steps/cache-maven-repository.yml + - pwsh: | - Write-Host "##vso[task.setvariable variable=ProjectList;]" - displayName: Initialize project list variable + $artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json + $additionalModules = '${{ convertToJson(parameters.AdditionalModules) }}' | ConvertFrom-Json -# These artifacts are the principal outputs of the build. We source -# them from the artifact list. -- ${{each artifact in parameters.Artifacts }}: - - pwsh: | - Write-Host "##vso[task.setvariable variable=ProjectList;]$(ProjectList),${{artifact.groupId}}:${{artifact.name}}" - displayName: Append ${{artifact.groupId}}:${{artifact.name}} to project list + $projectList = @() + foreach ($artifact in $artifacts) { + $projectList += "$($artifact.groupId):$($artifact.name)" + } + foreach ($artifact in $additionalModules) { + $projectList += "$($artifact.groupId):$($artifact.name)" + } + $projects = $projectList -join ',' -# From time to time we might need to also build another module which -# we don't actually want to ship, so we list that here. We can't rely -# on the -amd switch to do this since is doesn't resolve dependencies -# AFTER dependents have been resolved, which means some projects might -# be missing from the resulting build graph. This works around that -# limitation in Maven. -- ${{each artifact in parameters.AdditionalModules }}: - - pwsh: | - Write-Host "##vso[task.setvariable variable=ProjectList;]$(ProjectList),${{artifact.groupId}}:${{artifact.name}}" - displayName: Append ${{artifact.groupId}}:${{artifact.name}} to project list + Write-Host "ProjectList = $projects" + Write-Host "##vso[task.setvariable variable=ProjectList;]$projects" + displayName: Initialize project list variable diff --git a/eng/pipelines/templates/steps/install-reporting-tools.yml b/eng/pipelines/templates/steps/install-reporting-tools.yml index f050a0d3c0616..124d9f7aa531a 100644 --- a/eng/pipelines/templates/steps/install-reporting-tools.yml +++ b/eng/pipelines/templates/steps/install-reporting-tools.yml @@ -1,7 +1,7 @@ parameters: - Options: '--batch-mode -Dmaven.wagon.http.pool=false --settings eng/settings.xml' - MavenOptions: '-Dorg.slf4j.simpleLogger.defaultLogLevel=error -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn' + Options: '$(DefaultOptions)' + MavenOptions: '$(MemoryOptions) $(LoggingOptions)' steps: - task: Maven@3 diff --git a/eng/pipelines/templates/steps/upload-repository-on-failure.yml b/eng/pipelines/templates/steps/upload-repository-on-failure.yml index 88c56e90cb1fd..1e2b6b7430573 100644 --- a/eng/pipelines/templates/steps/upload-repository-on-failure.yml +++ b/eng/pipelines/templates/steps/upload-repository-on-failure.yml @@ -3,3 +3,12 @@ steps: condition: or(eq(variables['CaptureRepositoryOnFailure'], 'true'), and(failed(), eq(variables['Build.Reason'],'Schedule'))) displayName: 'Capture repo state for analysis' artifact: repository-$(System.JobName) + +- pwsh: | + # Remove any installed packages that we build in the repo so we don't cache temporary builds + Remove-Item "$(MAVEN_CACHE_FOLDER)/com/azure" -Recurse -ErrorAction Ignore + Remove-Item "$(MAVEN_CACHE_FOLDER)/com/microsoft/azure" -Recurse -ErrorAction Ignore + + Write-Host "POM files left in the cache folder" + Get-ChildItem $env:MAVEN_CACHE_FOLDER -Recurse -Include *.pom | Select Name + displayName: Clean up maven local cache diff --git a/eng/pipelines/templates/variables/globals.yml b/eng/pipelines/templates/variables/globals.yml index 7f1c4b606ecc3..5d2107206bdc4 100644 --- a/eng/pipelines/templates/variables/globals.yml +++ b/eng/pipelines/templates/variables/globals.yml @@ -1,16 +1,20 @@ variables: DocWardenVersion: '0.7.1' - WagonOptions: '-Dmaven.wagon.rto=300000 -Dmaven.wagon.http.retryHandler.count=3' - DefaultOptions: '--batch-mode --fail-at-end --settings eng/settings.xml $(WagonOptions)' - LoggingOptions: '-Dorg.slf4j.simpleLogger.defaultLogLevel=error -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn' - MemoryOptions: '-Xmx3072m' - #Agent.Source.Git.ShallowFetchDepth: 1 - skipComponentGovernanceDetection: true - AdditionalOptions: '' - # format !:, !:,... no trailing comma is necessary - PLSkipOptions: '' # This is the default Java build version. It's the version used to build the shipping libraries, Spotbugs etc. JavaBuildVersion: '1.11' # This is the default Java test version. It's the version used when running tests. JavaTestVersion: '1.11' + + # Maven build/test options + MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository + # See https://github.com/actions/virtual-environments/issues/1499 for more info about the wagon options + WagonOptions: '-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120' + DefaultOptions: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) --batch-mode --fail-at-end --settings eng/settings.xml $(WagonOptions)' + LoggingOptions: '-Dorg.slf4j.simpleLogger.defaultLogLevel=error -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn' + MemoryOptions: '-Xmx3072m' + DefaultTestOptions: '$(DefaultOptions) -Dgpg.skip -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -DtrimStackTrace=false -pl $(ProjectList)' + DefaultTestMavenOptions: '$(MemoryOptions) $(LoggingOptions)' + AdditionalOptions: '' + + skipComponentGovernanceDetection: true IsDebug: $[coalesce(variables['System.Debug'], 'false')] diff --git a/sdk/cosmos/tests.yml b/sdk/cosmos/tests.yml index 5f852562bb973..363e53f3ccc4c 100644 --- a/sdk/cosmos/tests.yml +++ b/sdk/cosmos/tests.yml @@ -33,7 +33,8 @@ stages: - template: /eng/pipelines/templates/steps/install-reporting-tools.yml TestStepMavenInputs: goals: verify - options: '$(ProfileFlag) -Dgpg.skip $(AdditionalArgs) -pl $(ProjectList)' + options: '$(DefaultTestOptions) $(ProfileFlag) $(AdditionalArgs)' + mavenOptions: '$(DefaultTestMavenOptions)' TestResultsFiles: '**/junitreports/TEST-*.xml' - template: /eng/pipelines/templates/stages/archetype-sdk-tests.yml @@ -59,7 +60,8 @@ stages: - template: /eng/pipelines/templates/steps/install-reporting-tools.yml TestStepMavenInputs: goals: clean verify - options: '$(ProfileFlag) -Dgpg.skip -Dmaven.wagon.http.pool=false $(DefaultOptions) -Dmaven.javadoc.skip=true -Drevapi.skip=true -pl $(ProjectList)' + options: '$(DefaultTestOptions) $(ProfileFlag)' + mavenOptions: '$(DefaultTestMavenOptions)' - ${{ if or(eq(variables['Build.Reason'], 'Manual'), eq(variables['Build.Reason'], 'PullRequest')) }}: - template: /sdk/cosmos/ci.spark.databricks.yml diff --git a/sdk/spring/spring-test-template.yml b/sdk/spring/spring-test-template.yml index f1676f29451eb..486ea71bef354 100644 --- a/sdk/spring/spring-test-template.yml +++ b/sdk/spring/spring-test-template.yml @@ -101,7 +101,8 @@ stages: TestResourceDirectories: ${{ parameters.TestResourceDirectories }} Artifacts: ${{ parameters.Artifacts }} TestStepMavenInputs: - options: '-Dmaven.wagon.http.pool=false $(DefaultOptions) -Dmaven.javadoc.skip=true -Drevapi.skip=true -DskipSpringITs=false -pl $(ProjectList)' + options: '$(DefaultTestOptions) -DskipSpringITs=false' + mavenOptions: '$(DefaultTestMavenOptions)' goals: "verify" PreSteps: - script: | @@ -151,7 +152,9 @@ stages: inputs: mavenPomFile: pom.xml goals: verify - options: '-Dmaven.wagon.http.pool=false $(DefaultOptions) -Dmaven.javadoc.skip=true -Drevapi.skip=true -DskipSpringITs=false -pl $(ProjectList)' + options: '$(DefaultTestOptions) -DskipSpringITs=false' + mavenOptions: '$(DefaultTestMavenOptions)' + env: ${{ parameters.EnvVars }} diff --git a/sdk/storage/tests.yml b/sdk/storage/tests.yml index b2c7245adaa0c..7f60153d02223 100644 --- a/sdk/storage/tests.yml +++ b/sdk/storage/tests.yml @@ -45,7 +45,8 @@ stages: Location: canadacentral Clouds: 'Preview' TestStepMavenInputs: - options: '$(DefaultOptions) -T 1C -Dmaven.javadoc.skip=true -Dspotbugs.skip=true -Dcheckstyle.skip=true -Drevapi.skip=true -DtrimStackTrace=false -pl $(ProjectList)' + options: '$(DefaultTestOptions) -T 1C' + mavenOptions: '$(DefaultTestMavenOptions)' MatrixReplace: # Use dedicated storage pool in canadacentral with higher memory capacity - Pool=(.*)-general/$1-storage