Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python …

…into add_reprs * 'master' of https://github.com/Azure/azure-sdk-for-python: (71 commits) EG - more docs imrpovement (Azure#17079) [EventHubs] add logging.info to warn the usage of partition key of non-string type (Azure#17057) march release (Azure#16966) Release of Device Update for IoT Hub SDK for Python. (Azure#17005) Add Get-AllPackageInfoFromRepo (Azure#16947) Track1 package is incorrectly set as track2 (Azure#17075) [text analytics] add normalized_text (Azure#17074) Renaming with_token identity function (Azure#17066) Adapt to azure core's cloud event (Azure#17063) align perf tests with js (Azure#17069) [Perfstress][Storage] Added FileShare perf tests (Azure#15834) [formrecognizer] Adding custom forms perf test (Azure#16969) Fix LanguageShort typo (Azure#17068) sas creds updates (Azure#17065) [eventgrid] Fix Sample eh (Azure#17064) [Perfstress][Storage] Added Datalake perf tests (Azure#15861) [text analytics] Healthcare n-ary relations (Azure#16997) ServiceBus dict-representation acceptance and kwarg-update functionality (Azure#14807) [text analytics] add perf tests (Azure#17060) Add cloud event to core (Azure#16800) ...
iscai-msft · Mar 4, 2021 · f66bdb4 · f66bdb4
2 parents 9159595 + 4f18a58
commit f66bdb4
Show file tree

Hide file tree

Showing 1,707 changed files with 418,273 additions and 78,209 deletions.
diff --git a/ci_template.yml b/ci_template.yml
@@ -5,6 +5,7 @@ trigger:
   branches:
     include:
     - master
+    - main
     - hotfix/*
     - release/*
     - restapi*
@@ -16,6 +17,7 @@ pr:
   branches:
     include:
     - master
+    - main
     - feature/*
     - hotfix/*
     - release/*

diff --git a/doc/eng_sys_checks.md b/doc/eng_sys_checks.md
@@ -48,7 +48,7 @@ Analyze job in both nightly CI and pull request validation pipeline runs a set o
 Change log verification is added to ensure package has valid change log for current version. Guidelines to properly maintain the change log is documented [here](https://github.com/Azure/azure-sdk-for-python/blob/master/doc/)
 
 ## PR Validation Checks
-Each pull request runs various tests using `pytest` in addition to all the tests mentioned above in analyze check. Pull request validation performs 3 different types of test: `whl, sdist and depends`. Following section explains the purpose of each of these tests and how to execute them locally. All pull requests are validated on multiple python versions across different platforms and below is the test matrix for pull request.
+Each pull request runs various tests using `pytest` in addition to all the tests mentioned above in analyze check. Pull request validation performs 3 different types of test: `whl, sdist and depends`. The following section explains the purpose of each of these tests and how to execute them locally. All pull requests are validated on multiple python versions across different platforms. Find the test matrix below.
 
 
 |`Python  Version`|`Platform`  |
@@ -73,8 +73,8 @@ This test installs sdist of the package being tested and runs all tests cases in
 2. Run following command
    ``tox -e sdist -c ../../../eng/tox/tox.ini``
 
-####depends
-This test is to ensure all modules in the package being tested can be successfully imported. This is to ensure all package requirement is properly set in setup.py as well as to ensure modules are imported using valid namespace. This test install the package and it's required packages and executes `from <package-root-namespace> import *`. For e.g. `from azure.core import *`.
+#### depends
+The `depends` check ensures all modules in a target package can be successfully imported. Actually installing and importing will verify that all package requirements are properly set in setup.py and that the `__all__` set for the package is properly defined. This test installs the package and its required packages, then executes `from <package-root-namespace> import *`. For example from `azure-core`, the following would be invoked:  `from azure.core import *`.
 
 Following is the command to run this test environment locally.
 

diff --git a/eng/.docsettings.yml b/eng/.docsettings.yml
@@ -105,6 +105,7 @@ known_content_issues:
   - ['sdk/monitor/azure-monitor-opentelemetry-exporter/swagger/README.md', '#4554']
   - ['sdk/monitor/azure-monitor-opentelemetry-exporter/README.md', '#4554']
   - ['sdk/digitaltwins/azure-digitaltwins-core/swagger/README.md', '#4554']
+  - ['sdk/textanalytics/azure-ai-textanalytics/swagger/README.md', '#4554']
   - ['sdk/media/azure-media-nspkg/README.md', '#4554']
 
   # nspckg and common.
@@ -124,7 +125,9 @@ known_content_issues:
   - ['sdk/core/azure-mgmt-nspkg/README.rst', 'nspkg and common']
   - ['sdk/core/azure-nspkg/README.rst', 'nspkg and common']
   - ['sdk/keyvault/azure-keyvault-nspkg/README.md', 'nspkg and common']
+  - ['sdk/mixedreality/azure-mixedreality-nspkg/README.md', 'nspkg and common']
   - ['sdk/search/azure-search-nspkg/README.md', 'nspkg and common']
+  - ['sdk/search/azure-search-documents/README.md', 'nspkg and common']
   - ['sdk/storage/azure-storage-blob/samples/README.md', 'nspkg and common']
   - ['sdk/storage/azure-storage-file-datalake/samples/README.md', 'nspkg and common']
   - ['sdk/storage/azure-storage-blob-changefeed/samples/README.md', 'nspkg and common']
@@ -135,6 +138,7 @@ known_content_issues:
   - ['sdk/tables/azure-data-nspkg/README.md', 'nspkg and common']
   - ['sdk/cosmos/README.md', 'nspkg and common']
   - ['sdk/communication/azure-communication-nspkg/README.md', 'nspkg and common']
+  - ['sdk/iothub/azure-iot-nspkg/README.md', 'nspkg and common']
   # HISTORY.rst
   - ['sdk/core/azure/HISTORY.rst','nspkg and common']
   - ['sdk/digitaltwins/azure-digitaltwins-nspkg/README.md', 'nspkg and common']

diff --git a/eng/CredScanSuppression.json b/eng/CredScanSuppression.json
@@ -35,9 +35,13 @@
         {
             "file":[
                 "sdk/keyvault/azure-keyvault-certificates/tests/ca.key",
-                "sdk/identity/azure-identity/tests/ec-certificate.pem"
+                "sdk/identity/azure-identity/tests/ec-certificate.pem",
+                "sdk/core/azure-servicemanagement-legacy/tests/legacy_mgmt_settings_fake.py",
+                "sdk/storage/azure-storage-blob/tests/blob_settings_fake.py",
+                "sdk/storage/azure-storage-file-datalake/tests/data_lake_settings_fake.py",
+                "tools/azure-sdk-tools/devtools_testutils/mgmt_settings_fake.py"
             ],
             "_justification": "File contains private key used by test code."
-        }  
+        }
     ]
 }
diff --git a/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1 b/eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1
@@ -127,7 +127,6 @@ function FindPackageWorkItem($lang, $packageName, $version, $outputCommand = $tr
   $fields += "ID"
   $fields += "State"
   $fields += "System.AssignedTo"
-  $fields += "Microsoft.VSTS.Common.StateChangeDate"
   $fields += "Parent"
   $fields += "Language"
   $fields += "Package"
@@ -265,19 +264,6 @@ function CreateWorkItem($title, $type, $iteration, $area, $fields, $assignedTo,
   return $workItem
 }
 
-function ResetWorkItemState($workItem, $resetState = $null, $outputCommand = $true)
-{
-  if (!$resetState -or $resetState -eq "New") {
-    $resetState = "Next Release Unknown"
-  }
-  if ($workItem.fields["System.State"] -ne $resetState)
-  {
-    Write-Verbose "Resetting state for [$($workItem.id)] from '$($workItem.fields['System.State'])' to '$resetState'"
-    return UpdateWorkItem $workItem.id -state $resetState -outputCommand $outputCommand
-  }
-  return $workItem
-}
-
 function UpdateWorkItem($id, $fields, $title, $state, $assignedTo, $outputCommand = $true)
 {
   $parameters = $ReleaseDevOpsCommonParameters
@@ -305,6 +291,30 @@ function UpdatePackageWorkItemReleaseState($id, $state, $releaseType, $outputCom
   return UpdateWorkItem -id $id -state $state -fields $fields -outputCommand $outputCommand
 }
 
+function FindOrCreateClonePackageWorkItem($lang, $pkg, $verMajorMinor, $outputCommand = $false)
+{
+  $workItem = FindPackageWorkItem -lang $lang -packageName $pkg.Package -version $verMajorMinor -includeClosed $true -outputCommand $outputCommand
+
+  if (!$workItem) {
+    $latestVersionItem = FindLatestPackageWorkItem -lang $lang -packageName $pkg.Package -outputCommand $outputCommand
+    $assignedTo = "me"
+    if ($latestVersionItem) {
+      Write-Verbose "Copying data from latest matching [$($latestVersionItem.id)] with version $($latestVersionItem.fields["Custom.PackageVersionMajorMinor"])"
+      if ($latestVersionItem.fields["System.AssignedTo"]) {
+        $assignedTo = $latestVersionItem.fields["System.AssignedTo"]["uniqueName"]
+      }
+      $pkg.DisplayName = $latestVersionItem.fields["Custom.PackageDisplayName"]
+      $pkg.ServiceName = $latestVersionItem.fields["Custom.ServiceName"]
+      if (!$pkg.RepoPath -and $pkg.RepoPath -ne "NA" -and $pkg.fields["Custom.PackageRepoPath"]) {
+        $pkg.RepoPath = $pkg.fields["Custom.PackageRepoPath"]
+      }
+    }
+    $workItem = CreateOrUpdatePackageWorkItem $lang $pkg $verMajorMinor -existingItem $null -assignedTo $assignedTo -outputCommand $outputCommand
+  }
+
+  return $workItem
+}
+
 function CreateOrUpdatePackageWorkItem($lang, $pkg, $verMajorMinor, $existingItem, $assignedTo = $null, $outputCommand = $true)
 {
   if (!$lang -or !$pkg -or !$verMajorMinor) {
@@ -347,14 +357,18 @@ function CreateOrUpdatePackageWorkItem($lang, $pkg, $verMajorMinor, $existingIte
       Write-Host "At least field $changedField ($($existingItem.fields[$changedField])) changed so updating."
     }
 
-    $beforeState = $existingItem.fields["System.State"]
-
     if ($changedField) {
+      $beforeState = $existingItem.fields["System.State"]
+
       # Need to set to New to be able to update
       $existingItem = UpdateWorkItem -id $existingItem.id -fields $fields -title $title -state "New" -assignedTo $assignedTo -outputCommand $outputCommand
       Write-Host "[$($existingItem.id)]$lang - $pkgName($verMajorMinor) - Updated"
+
+      if ($beforeState -ne $existingItem.fields['System.State']) {
+        Write-Verbose "Resetting state for [$($existingItem.id)] from '$($existingItem.fields['System.State'])' to '$beforeState'"
+        $existingItem = UpdateWorkItem $existingItem.id -state $beforeState -outputCommand $outputCommand
+      }
     }
-    $existingItem = ResetWorkItemState $existingItem $beforeState -outputCommand $outputCommand
 
     $newparentItem = FindOrCreatePackageGroupParent $serviceName $pkgDisplayName -outputCommand $false
     UpdateWorkItemParent $existingItem $newParentItem -outputCommand $outputCommand
@@ -363,7 +377,6 @@ function CreateOrUpdatePackageWorkItem($lang, $pkg, $verMajorMinor, $existingIte
 
   $parentItem = FindOrCreatePackageGroupParent $serviceName $pkgDisplayName -outputCommand $false
   $workItem = CreateWorkItem $title "Package" "Release" "Release" $fields $assignedTo $parentItem.id -outputCommand $outputCommand
-  $workItem = ResetWorkItemState $workItem -outputCommand $outputCommand
   Write-Host "[$($workItem.id)]$lang - $pkgName($verMajorMinor) - Created"
   return $workItem
 }
@@ -752,25 +765,6 @@ function UpdatePackageVersions($pkgWorkItem, $plannedVersions, $shippedVersions)
   "value": "$shippedPackages"
 }
 "@
-
-    # If we shipped a version after we set "In Release" state then reset the state to "Next Release Unknown"
-    if ($pkgWorkItem.fields["System.State"] -eq "In Release")
-    {
-      $lastShippedDate = [DateTime]$newShippedVersions[0].Date
-      $markedInReleaseDate = ([DateTime]$pkgWorkItem.fields["Microsoft.VSTS.Common.StateChangeDate"])
-
-      # We just shipped so lets set the state to "Next Release Unknown"
-      if ($markedInReleaseDate -le $lastShippedDate)
-      {
-        $fieldUpdates += @'
-{
-  "op": "replace",
-  "path": "/fields/State",
-  "value": "Next Release Unknown"
-}
-'@
-      }
-    }
   }
 
   # Full merged version set

diff --git a/eng/common/scripts/Prepare-Release.ps1 b/eng/common/scripts/Prepare-Release.ps1
@@ -2,60 +2,67 @@
 
 [CmdletBinding()]
 param(
-    [Parameter(Mandatory=$true)]
-    [string]$PackageName,
-    [string]$ServiceDirectory,
-    [string]$ReleaseDate, # Pass Date in the form MM/dd/yyyy"
-    [string]$BuildType # For Java
+  [Parameter(Mandatory = $true)]
+  [string]$PackageName,
+  [string]$ServiceDirectory,
+  [string]$ReleaseDate # Pass Date in the form MM/dd/yyyy"
 )
+Set-StrictMode -Version 3
 
 . ${PSScriptRoot}\common.ps1
 
 function Get-ReleaseDay($baseDate)
 {
-    # Find first friday
-    while ($baseDate.DayOfWeek -ne 5)
-    {
-        $baseDate = $baseDate.AddDays(1)
-    }
-    
-    # Go to Tuesday
-    $baseDate = $baseDate.AddDays(4)
-
-    return $baseDate;
+  # Find first friday
+  while ($baseDate.DayOfWeek -ne 5)
+  {
+    $baseDate = $baseDate.AddDays(1)
+  }
+
+  # Go to Tuesday
+  $baseDate = $baseDate.AddDays(4)
+
+  return $baseDate;
 }
 
 $ErrorPreference = 'Stop'
 
 $packageProperties = Get-PkgProperties -PackageName $PackageName -ServiceDirectory $serviceDirectory
 
+if (!$packageProperties)
+{
+  Write-Error "Could not find a package with name [ $packageName ], please verify the package name matches the exact name."
+  exit 1
+}
+
+Write-Host "Package Name [ $($packageProperties.Name) ]"
 Write-Host "Source directory [ $serviceDirectory ]"
 
 if (!$ReleaseDate)
 {
-    $currentDate = Get-Date
-    $thisMonthReleaseDate = Get-ReleaseDay((Get-Date -Day 1));
-    $nextMonthReleaseDate = Get-ReleaseDay((Get-Date -Day 1).AddMonths(1));
-
-    if ($thisMonthReleaseDate -ge $currentDate)
-    {
-        # On track for this month release
-        $ParsedReleaseDate = $thisMonthReleaseDate
-    }
-    elseif ($currentDate.Day -lt 15)
-    {
-        # Catching up to this month release
-        $ParsedReleaseDate = $currentDate
-    }
-    else 
-    {
-        # Next month release
-        $ParsedReleaseDate = $nextMonthReleaseDate
-    }
+  $currentDate = Get-Date
+  $thisMonthReleaseDate = Get-ReleaseDay((Get-Date -Day 1));
+  $nextMonthReleaseDate = Get-ReleaseDay((Get-Date -Day 1).AddMonths(1));
+
+  if ($thisMonthReleaseDate -ge $currentDate)
+  {
+    # On track for this month release
+    $ParsedReleaseDate = $thisMonthReleaseDate
+  }
+  elseif ($currentDate.Day -lt 15)
+  {
+    # Catching up to this month release
+    $ParsedReleaseDate = $currentDate
+  }
+  else
+  {
+    # Next month release
+    $ParsedReleaseDate = $nextMonthReleaseDate
+  }
 }
 else
 {
-    $ParsedReleaseDate = [datetime]$ReleaseDate
+  $ParsedReleaseDate = [datetime]$ReleaseDate
 }
 
 $releaseDateString = $ParsedReleaseDate.ToString("MM/dd/yyyy")
@@ -70,40 +77,42 @@ $newVersion = Read-Host -Prompt "Input the new version, or press Enter to use us
 
 if (!$newVersion)
 {
-    $newVersion = $currentProjectVersion;
+  $newVersion = $currentProjectVersion;
 }
 
 $newVersionParsed = [AzureEngSemanticVersion]::ParseVersionString($newVersion)
 if ($null -eq $newVersionParsed)
 {
-    Write-Error "Invalid version $newVersion. Version must follow standard SemVer rules, see https://aka.ms/azsdk/engsys/packageversioning"
-    exit 1
+  Write-Error "Invalid version $newVersion. Version must follow standard SemVer rules, see https://aka.ms/azsdk/engsys/packageversioning"
+  exit 1
 }
 
 if (Test-Path "Function:SetPackageVersion")
 {
-    SetPackageVersion -PackageName $packageProperties.Name -Version $newVersion -ServiceDirectory $serviceDirectory -ReleaseDate $releaseDateString `
-    -BuildType $BuildType -GroupId $packageProperties.Group
+  SetPackageVersion -PackageName $packageProperties.Name -Version $newVersion -ServiceDirectory $serviceDirectory -ReleaseDate $releaseDateString `
+    -PackageProperties $packageProperties
 }
 else
 {
-    LogError "The function 'SetPackageVersion' was not found.`
+  LogError "The function 'SetPackageVersion' was not found.`
     Make sure it is present in eng/scripts/Language-Settings.ps1.`
     See https://github.com/Azure/azure-sdk-tools/blob/master/doc/common/common_engsys.md#code-structure"
-    exit 1
+  exit 1
 }
 
 &$EngCommonScriptsDir/Update-DevOps-Release-WorkItem.ps1 `
--language $LanguageDisplayName `
--packageName $packageProperties.Name `
--version $newVersion `
--plannedDate $releaseDateString `
--packageRepoPath $packageProperties.serviceDirectory
+  -language $LanguageDisplayName `
+  -packageName $packageProperties.Name `
+  -version $newVersion `
+  -plannedDate $releaseDateString `
+  -packageRepoPath $packageProperties.serviceDirectory `
+  -packageType $packageProperties.SDKType `
+  -packageNewLibrary $packageProperties.IsNewSDK
 
 git diff -s --exit-code $packageProperties.DirectoryPath
 if ($LASTEXITCODE -ne 0)
 {
-    git status
-    Write-Host "Some changes were made to the repo source" -ForegroundColor Green
-    Write-Host "Submit a pull request with the necessary changes to the repo" -ForegroundColor Green
-}
+  git status
+  Write-Host "Some changes were made to the repo source" -ForegroundColor Green
+  Write-Host "Submit a pull request with the necessary changes to the repo" -ForegroundColor Green
+}
diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1
@@ -25,12 +25,12 @@ if (Test-Path $EngScriptsLanguageSettings) {
   . $EngScriptsLanguageSettings
 }
 
-if (-not $LanguageShort)
+if (!(Get-Variable -Name "LanguageShort" -ValueOnly -ErrorAction "Ignore"))
 {
-  $LangaugeShort = $Language
+  $LanguageShort = $Language
 }
 
-if (-not $LanguageDisplayName)
+if (!(Get-Variable -Name "LanguageDisplayName" -ValueOnly -ErrorAction "Ignore"))
 {
   $LanguageDisplayName = $Language
 }

diff --git a/eng/common/scripts/logging.ps1 b/eng/common/scripts/logging.ps1
@@ -1,7 +1,4 @@
-if (-not $isDevOpsRun)
-{
-  $isDevOpsRun = ($null -ne $env:SYSTEM_TEAMPROJECTID)
-}
+$isDevOpsRun = ($null -ne $env:SYSTEM_TEAMPROJECTID)
 
 function LogWarning
 {
@@ -37,4 +34,4 @@ function LogDebug
     {
         Write-Debug "$args"
     }
-}
+}