Merge from upstream master

.github/CODEOWNERS

@@ -25,6 +25,9 @@
 # PRLabel: %Storage
 /sdk/storage/                                                        @amishra-dev @zezha-msft @annatisch @rakshith91 @xiafu-msft @tasherif-msft @kasobol-msft
 
+# PRLabel: %App Configuration
+/sdk/appconfiguration/                                               @xiangyan99
+
 /sdk/applicationinsights/azure-applicationinsights/                  @divya-jay @geneh @alongafni
 /sdk/loganalytics/azure-loganalytics/                                @divya-jay @geneh @alongafni
 
@@ -98,6 +101,9 @@
 # PRLabel: %Cognitive - Form Recognizer
 /sdk/formrecognizer/                                                 @kristapratico @iscai-msft @rakshith91
 
+# PRLabel: %Cognitive - Metrics Advisor
+/sdk/metricsadvisor/                                                 @xiangyan99 @kristapratico
+
 # PRLabel: %Tables
 /sdk/tables/                                                         @seankane-msft
 

README.md

@@ -36,7 +36,7 @@ Last stable versions of packages that have been provided for usage with Azure an
 
 ### Management: New Releases
 A new set of management libraries that follow the [Azure SDK Design Guidelines for Python](https://azure.github.io/azure-sdk/python/guidelines/) are now available. These new libraries provide a number of core capabilities that are shared amongst all Azure SDKs, including the intuitive Azure Identity library, an HTTP Pipeline with custom policies, error-handling, distributed tracing, and much more. 
-Documentation and code samples for these new libraries can be found [here](http://aka.ms/azsdk/python/mgmt). In addition, a migration guide that shows how to transition from older versions of libraries is located [here](https://azure.github.io/azure-sdk-for-python/mgmt_preview_quickstart.html#migration-guide). 
+Documentation and code samples for these new libraries can be found [here](http://aka.ms/azsdk/python/mgmt). In addition, a migration guide that shows how to transition from older versions of libraries is located [here](https://github.com/Azure/azure-sdk-for-python/blob/master/doc/sphinx/mgmt_quickstart.rst#migration-guide). 
 
 You can find the [most up to date list of all of the new packages on our page](https://azure.github.io/azure-sdk/releases/latest/mgmt/python.html)
 

doc/sphinx/python_mgmt_migration_guide.rst

@@ -102,7 +102,7 @@ To the show the code snippets for the change:
     from azure.identity import ClientSecretCredential
 
     credential = ClientSecretCredential(
-        tenant_id=tenant_id,
+        client_secret=client_secret,
         client_id=client_id,
         tenant_id=tenant_id
     )
@@ -167,3 +167,4 @@ If you have encountered an issue during migration, please file an issue
 via `Github
 Issues <https://github.com/Azure/azure-sdk-for-python/issues>`__ and
 make sure you add the "Preview" label to the issue
+

eng/.docsettings.yml

@@ -49,6 +49,7 @@ known_content_issues:
   - ['sdk/synapse/azure-synapse-artifacts/README.md', '#4554']
   - ['sdk/synapse/azure-synapse-nspkg/README.md', '#4554']
   - ['sdk/anomalydetector/azure-ai-anomalydetector/README.md', '#4554']
+  - ['sdk/metricsadvisor/azure-ai-metricsadvisor/README.md', '#4554']
   - ['sdk/applicationinsights/azure-applicationinsights/README.md', '#4554']
   - ['sdk/batch/azure-batch/README.md', '#4554']
   - ['sdk/cognitiveservices/azure-cognitiveservices-anomalydetector/README.md', '#4554']

eng/common/README.md

@@ -20,6 +20,6 @@ The 'Sync eng/common directory' PRs will be created in the language repositories
 3. More changes pushed to the **Tools PR**, will automatically triggered new pipeline runs in the respective **Sync PRs**. The **Sync PRs** are used to make sure the changes would not break any of the connected pipelines.
 4. Once satisfied with the changes;
     - First make sure all checks in the **Sync PRs** are green and approved. The **Tools PR** contains links to all the **Sync PRs**. If for some reason the PRs is blocked by a CI gate get someone with permission to override and manually merge the PR.
-    - To test the state of all the **Sync PRs**, you can download the `PRsCreated.txt` artifact from your `azure-sdk-tools - sync - eng-common` pipeline, then run `./eng/scripts/Verify-And-Merge.ps1 <path to PRsCreated.txt>` which will output the status of each associated PR.
+    - To test the state of all the **Sync PRs**, you can download the `PRsCreated.txt` artifact from your `azure-sdk-tools - sync - eng-common` pipeline, then run `./eng/scripts/Verify-And-Merge-PRs.ps1 <path to PRsCreated.txt>` which will output the status of each associated PR.
     - Next approve the `VerifyAndMerge` job for the `azure-sdk-tools - sync - eng-common` pipeline triggered by your **Tools PR** which will automatically merge all the **Sync PRs**. You need `azure-sdk` devops contributor permissions to reach the `azure-sdk-tools - sync - eng-common` pipeline.
-    - Finally merge the **Tools PR**. 
+    - Finally merge the **Tools PR**. 

eng/common/TestResources/New-TestResources.ps1

@@ -55,7 +55,7 @@ param (
     [string] $Location = '',
 
     [Parameter()]
-    [ValidateSet('AzureCloud', 'AzureUSGovernment', 'AzureChinaCloud')]
+    [ValidateSet('AzureCloud', 'AzureUSGovernment', 'AzureChinaCloud', 'Dogfood')]
     [string] $Environment = 'AzureCloud',
 
     [Parameter()]
@@ -141,6 +141,7 @@ if (!$Location) {
         'AzureCloud' = 'westus2';
         'AzureUSGovernment' = 'usgovvirginia';
         'AzureChinaCloud' = 'chinaeast2';
+        'Dogfood' = 'westus'
     }[$Environment]
 
     Write-Verbose "Location was not set. Using default location for environment: '$Location'"
@@ -512,6 +513,7 @@ is based on the cloud to which the template is being deployed:
 * AzureCloud -> 'westus2'
 * AzureUSGovernment -> 'usgovvirginia'
 * AzureChinaCloud -> 'chinaeast2'
+* Dogfood -> 'westus'
 
 .PARAMETER Environment
 Name of the cloud environment. The default is the Azure Public Cloud

eng/common/TestResources/New-TestResources.ps1.md

@@ -322,6 +322,7 @@ is based on the cloud to which the template is being deployed:
 * AzureCloud -\> 'westus2'
 * AzureUSGovernment -\> 'usgovvirginia'
 * AzureChinaCloud -\> 'chinaeast2'
+* Dogfood -\> 'westus'
 
 ```yaml
 Type: String

eng/common/TestResources/Remove-TestResources.ps1

@@ -43,7 +43,7 @@ param (
     [string] $ServiceDirectory,
 
     [Parameter()]
-    [ValidateSet('AzureCloud', 'AzureUSGovernment', 'AzureChinaCloud')]
+    [ValidateSet('AzureCloud', 'AzureUSGovernment', 'AzureChinaCloud', 'Dogfood')]
     [string] $Environment = 'AzureCloud',
 
     [Parameter()]

eng/common/TestResources/deploy-test-resources.yml

@@ -18,9 +18,7 @@ parameters:
 # }
 
 steps:
-  # New-TestResources command requires Az module
-  - pwsh: Install-Module -Name Az -Scope CurrentUser -AllowClobber -Force -Verbose
-    displayName: Install Azure PowerShell module
+  - template: /eng/common/TestResources/setup-az-modules.yml
 
   - pwsh: |
       $subscriptionConfiguration = @"

eng/common/TestResources/setup-az-modules.yml

@@ -0,0 +1,36 @@
+# Cloud Configuration will be splat into parameters of `Add-AzEnvironment`. It
+# should be JSON in the form (not all fields are required):
+# {
+#   "Name": "<environment name>",
+#   "PublishSettingsFileUrl": "<publish settings file url>",
+#   "ServiceEndpoint": "<service endpoint>",
+#   "ManagementPortalUrl": "<management portal url>",
+#   "ActiveDirectoryEndpoint": "<active directory endpoint>",
+#   "ActiveDirectoryServiceEndpointResourceId": "<active directory service endpoint resource id>",
+#   "ResourceManagerEndpoint": "<resource manager endpoint>",
+#   "GalleryEndpoint": "<gallery endpoint>",
+#   "GraphEndpoint": "<graph endpoint>",
+#   "GraphAudience": "<graph audience>",
+#   "AzureKeyVaultDnsSuffix": "<key vault suffix>",
+#   "AzureKeyVaultServiceEndpointResourceId": "<key vault service endpoint resource id>"
+# }
+
+steps:
+  - bash: sudo chown -R runner ~/.Azure
+    displayName: (MacOS) Grant access to ~/.Azure
+    condition: contains(variables['OSVmImage'], 'mac')
+
+  # New-TestResources command requires Az module
+  - pwsh: Install-Module -Name Az -Scope CurrentUser -AllowClobber -Force -Verbose
+    displayName: Install Azure PowerShell module
+
+  - task: Powershell@2
+    inputs:
+      displayName: Register Dogfood environment
+      targetType: inline
+      pwsh: true
+      script: |
+        $environmentSpec = @"
+          $(env-config-dogfood)
+        "@ | ConvertFrom-Json -AsHashtable;
+        Add-AzEnvironment @environmentSpec

eng/common/scripts/New-ReleaseAsset.ps1

@@ -0,0 +1,62 @@
+<#
+.SYNOPSIS
+Uploads the release asset and returns the resulting object from the upload
+
+.PARAMETER ReleaseTag
+Tag to look up release
+
+.PARAMETER AssetPath
+Location of the asset file to upload
+
+.PARAMETER GitHubRepo
+Name of the GitHub repo to search (of the form Azure/azure-sdk-for-cpp)
+
+#>
+
+param (
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $ReleaseTag,
+
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $AssetPath,
+
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $GitHubRepo,
+
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string] $GitHubPat
+)
+
+# Get information about release at $ReleaseTag
+$releaseInfoUrl = "https://api.github.com/repos/$GitHubRepo/releases/tags/$ReleaseTag"
+Write-Verbose "Requesting release info from $releaseInfoUrl"
+$release = Invoke-RestMethod `
+    -Uri  $releaseInfoUrl `
+    -Method GET
+
+$assetFilename = Split-Path $AssetPath -Leaf
+
+# Upload URL comes in the literal form (yes, those curly braces) of:
+# https://uploads.github.com/repos/Azure/azure-sdk-for-cpp/releases/123/assets{?name,label}
+# Converts to something like:
+# https://uploads.github.com/repos/Azure/azure-sdk-for-cpp/releases/123/assets?name=foo.tar.gz
+# Docs: https://docs.github.com/en/rest/reference/repos#get-a-release-by-tag-name
+$uploadUrl = $release.upload_url.Split('{')[0] + "?name=$assetFilename"
+
+Write-Verbose "Uploading $assetFilename to $uploadUrl"
+
+$asset = Invoke-RestMethod `
+    -Uri $uploadUrl `
+    -Method POST `
+    -InFile $AssetPath `
+    -Credential $credentials `
+    -Headers @{ Authorization = "token $GitHubPat" } `
+    -ContentType "application/gzip"
+
+Write-Verbose "Upload complete. Browser download URL: $($asset.browser_download_url)"
+
+return $asset

eng/common/scripts/Submit-PullRequest.ps1

@@ -58,7 +58,7 @@ $headers = @{
 
 $query = "state=open&head=${PROwner}:${PRBranch}&base=${BaseBranch}"
 
-function AddLabels([int] $prNumber, [string] $prLabelString)
+function AddLabels([int] $prNumber, [string] $prLabelString, [array] $existingLabels)
 {
   # Adding labels to the pr.
   if (-not $prLabelString) {
@@ -68,6 +68,12 @@ function AddLabels([int] $prNumber, [string] $prLabelString)
 
   # Parse the labels from string to array
   $prLabelArray = @($prLabelString.Split(",") | % { $_.Trim() } | ? { return $_ })
+  foreach ($label in $existingLabels) {
+    if ($prLabelArray -contains $label.name) {
+      continue
+    }
+    $prLabelArray += $label.name
+  }
   $prLabelUri = "https://api.github.com/repos/$RepoOwner/$RepoName/issues/$prNumber"
   $labelRequestData = @{
     labels = $prLabelArray
@@ -97,7 +103,7 @@ if ($resp.Count -gt 0) {
 
     # setting variable to reference the pull request by number
     Write-Host "##vso[task.setvariable variable=Submitted.PullRequest.Number]$($resp[0].number)"
-    AddLabels $resp[0].number $PRLabels
+    AddLabels $resp[0].number $PRLabels $resp[0].labels
 }
 else {
   $data = @{
@@ -124,5 +130,5 @@ else {
   # setting variable to reference the pull request by number
   Write-Host "##vso[task.setvariable variable=Submitted.PullRequest.Number]$($resp.number)"
 
-  AddLabels $resp.number $PRLabels
+  AddLabels $resp.number $PRLabels $resp.labels
 }

eng/common/scripts/logging.ps1

@@ -31,7 +31,7 @@ function LogDebug
 {
     if ($isDevOpsRun) 
     {
-        Write-Host "##vso[task.LogIssue type=debug;]$args"
+        Write-Host "[debug]$args"
     }
     else 
     {

eng/pipelines/templates/jobs/archetype-sdk-client.yml

@@ -111,7 +111,15 @@ jobs:
     pool:
       vmImage: '$(OSVmImage)'
 
-    steps:    
+    steps:
+    - task: PowerShell@2
+      displayName: Prep template pipeline for release
+      condition: and(succeeded(),eq(variables['TestPipeline'],'true'))
+      inputs:
+        pwsh: true
+        workingDirectory: $(Build.SourcesDirectory)
+        filePath: eng/scripts/SetTestPipelineVersion.ps1
+
     - pwsh: |
         $toxenvvar = "whl,sdist"
         if (('$(Build.Reason)' -eq 'Schedule') -and ('$(System.TeamProject)' -eq 'internal')) {

eng/pipelines/templates/stages/archetype-python-release.yml

@@ -28,6 +28,13 @@ stages:
                 deploy:
                   steps:
                     - checkout: self
+                    - task: PowerShell@2
+                      displayName: Prep template pipeline for release
+                      condition: and(succeeded(),eq(variables['TestPipeline'],'true'))
+                      inputs:
+                        pwsh: true
+                        workingDirectory: $(Build.SourcesDirectory)
+                        filePath: eng/scripts/SetTestPipelineVersion.ps1
                     - template: /eng/pipelines/templates/steps/stage-filtered-artifacts.yml
                       parameters:
                         SourceFolder: ${{parameters.ArtifactName}} 

eng/pipelines/templates/steps/build-artifacts.yml

@@ -5,6 +5,14 @@ parameters:
   BuildDocs: true
 
 steps:
+  - task: PowerShell@2
+    displayName: Prep template pipeline for release
+    condition: and(succeeded(),eq(variables['TestPipeline'],'true'))
+    inputs:
+      pwsh: true
+      workingDirectory: $(Build.SourcesDirectory)
+      filePath: eng/scripts/SetTestPipelineVersion.ps1
+
   - script: |
       echo "##vso[build.addbuildtag]Scheduled"
     displayName: 'Tag scheduled builds'

eng/scripts/SetTestPipelineVersion.ps1

@@ -0,0 +1,39 @@
+# Overides the project file and CHANGELOG.md for the template project using the next publishable version
+# This is to help with testing the release pipeline.
+. "${PSScriptRoot}\..\common\scripts\common.ps1"
+$latestTags = git tag -l "azure-template_*"
+$semVars = @()
+
+$versionFile = "${PSScriptRoot}\..\..\sdk\template\azure-template\azure\template\_version.py"
+$changeLogFile = "${PSScriptRoot}\..\..\sdk\template\azure-template\CHANGELOG.md"
+
+Foreach ($tags in $latestTags)
+{
+  $semVars += $tags.Replace("azure-template_", "")
+}
+
+$semVarsSorted = [AzureEngSemanticVersion]::SortVersionStrings($semVars)
+LogDebug "Last Published Version $($semVarsSorted[0])"
+
+$newVersion = [AzureEngSemanticVersion]::ParsePythonVersionString($semVarsSorted[0])
+$newVersion.IncrementAndSetToPrerelease()
+LogDebug "Version to publish [ $($newVersion.ToString()) ]"
+
+$versionFileContent = Get-Content -Path $versionFile
+$newVersionFile = @()
+
+Foreach ($line in $versionFileContent)
+{
+    if($line.StartsWith("VERSION"))
+    {
+        $line = 'VERSION = "{0}"' -F $newVersion.ToString()
+    }
+    $newVersionFile += $line
+}
+
+Set-Content -Path $versionFile -Value $newVersionFile
+Set-Content -Path $changeLogFile -Value @"
+# Release History
+## $($newVersion.ToString()) ($(Get-Date -f "yyyy-MM-dd"))
+- Test Release Pipeline
+"@

eng/tox/install_depend_packages.py

@@ -28,7 +28,7 @@
 logging.getLogger().setLevel(logging.INFO)
 
 MINIMUM_VERSION_SUPPORTED_OVERRIDE = {
-    'azure-common': '1.1.10',
+    'azure-common': '1.1.10'
 }
 
 def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir):

eng/tox/mypy_hard_failure_packages.py

@@ -10,5 +10,6 @@
   "azure-eventhub",
   "azure-servicebus",
   "azure-ai-textanalytics",
-  "azure-ai-formrecognizer"
+  "azure-ai-formrecognizer",
+  "azure-ai-metricsadvisor"
 ]

eng/tox/tox.ini

@@ -6,7 +6,7 @@ envlist = whl,sdist
 [tools]
 deps =
   -r ../../../eng/test_tools.txt
-  
+
 
 [coverage:paths]
 source =
@@ -20,6 +20,7 @@ deps =
   {[tools]deps}
 
 
+
 [packaging]
 pkgs = 
     wheel==0.34.2
@@ -215,7 +216,11 @@ commands =
 [testenv:mindependency]
 pre-deps =
   {[packaging]pkgs}
-deps = {[tools]deps}
+deps = 
+  azure-mgmt-keyvault<7.0.0
+  azure-mgmt-resource<15.0.0
+  azure-mgmt-storage<15.0.0
+  {[tools]deps}
 changedir = {toxinidir}
 passenv = *
 setenv =