[azdatalake][STG84] Append with flush

.github/CODEOWNERS

@@ -28,7 +28,7 @@
 /sdk/data/azappconfig/        @antkmsft @jhendrixMSFT @rickwinter
 
 # PRLabel: %Cosmos
-/sdk/data/azcosmos/           @ealsur @kirankumarkolli
+/sdk/data/azcosmos/           @ealsur @kirankumarkolli @simorenoh @kushagraThapar
 
 # PRLabel: %Tables
 /sdk/data/aztables/           @jhendrixMSFT

eng/common/pipelines/templates/steps/create-tags-and-git-release.yml

@@ -1,6 +1,7 @@
 parameters:
   ArtifactLocation: 'not-specified'
   PackageRepository: 'not-specified'
+  PackageFilter: ''
   ReleaseSha: 'not-specified'
   RepoId: $(Build.Repository.Name)
   WorkingDirectory: ''
@@ -14,10 +15,11 @@ steps:
     arguments: >
       -artifactLocation ${{ parameters.ArtifactLocation }}
       -packageRepository ${{ parameters.PackageRepository }}
+      -packageFilter "${{ parameters.PackageFilter }}"
       -releaseSha ${{ parameters.ReleaseSha }}
       -repoId ${{ parameters.RepoId }}
       -workingDirectory '${{ parameters.WorkingDirectory }}'
     pwsh: true
   timeoutInMinutes: 5
   env:
-    GH_TOKEN: $(azuresdk-github-pat)
+    GH_TOKEN: $(azuresdk-github-pat)

eng/common/scripts/Submit-PullRequest.ps1

@@ -23,12 +23,12 @@ The body message for the pull request.
 The labels added to the PRs. Multple labels seperated by comma, e.g "bug, service"
 .PARAMETER UserReviewers
 User reviewers to request after opening the PR. Users should be a comma-
-separated list with no preceeding `@` symbol (e.g. "user1,usertwo,user3")
+separated list with no preceding `@` symbol (e.g. "user1,usertwo,user3")
 .PARAMETER TeamReviewers
 List of github teams to add as reviewers
 .PARAMETER Assignees
 Users to assign to the PR after opening. Users should be a comma-separated list
-with no preceeding `@` symbol (e.g. "user1,usertwo,user3")
+with no preceding `@` symbol (e.g. "user1,usertwo,user3")
 .PARAMETER CloseAfterOpenForTesting
 Close the PR after opening to save on CI resources and prevent alerts to code
 owners, assignees, requested reviewers, or others.

eng/common/scripts/create-tags-and-git-release.ps1

@@ -7,6 +7,7 @@ param (
   $artifactLocation, # the root of the artifact folder. DevOps $(System.ArtifactsDirectory)
   $workingDirectory, # directory that package artifacts will be extracted into for examination (if necessary)
   $packageRepository, # used to indicate destination against which we will check the existing version.
+  $packageFilter,
   # valid options: PyPI, Nuget, NPM, Maven, C, CPP
   # used by CreateTags
   $releaseSha, # the SHA for the artifacts. DevOps: $(Release.Artifacts.<artifactAlias>.SourceVersion) or $(Build.SourceVersion)
@@ -26,6 +27,11 @@ Write-Host "Using API URL $apiUrl"
 # VERIFY PACKAGES
 $pkgList = VerifyPackages -artifactLocation $artifactLocation -workingDirectory $workingDirectory -apiUrl $apiUrl -releaseSha $releaseSha -continueOnError $continueOnError
 
+if ($packageFilter) {
+    Write-Host "Filtering discovered packages to '$packageFilter'"
+    [array]$pkgList = $pkgList | Where-Object { $_.PackageId -like $packageFilter }
+}
+
 if ($pkgList) {
   Write-Host "Given the visible artifacts, github releases will be created for the following:"
 

eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1

@@ -257,7 +257,7 @@ function DeployStressPackage(
     if ($pkg.Dockerfile -or $pkg.DockerBuildDir) {
         throw "The chart.yaml docker config is deprecated, please use the scenarios matrix instead."
     }
-    
+
 
     foreach ($dockerBuildConfig in $dockerBuildConfigs) {
         $dockerFilePath = $dockerBuildConfig.dockerFilePath
@@ -285,7 +285,7 @@ function DeployStressPackage(
             $dockerBuildCmd += $dockerBuildFolder
 
             Run @dockerBuildCmd
-            
+
             Write-Host "`nContainer image '$imageTag' successfully built. To run commands on the container locally:" -ForegroundColor Blue
             Write-Host "  docker run -it $imageTag" -ForegroundColor DarkBlue
             Write-Host "  docker run -it $imageTag <shell, e.g. 'bash' 'pwsh' 'sh'>" -ForegroundColor DarkBlue
@@ -302,7 +302,7 @@ function DeployStressPackage(
             }
         }
         $generatedHelmValues.scenarios = @( foreach ($scenario in $generatedHelmValues.scenarios) {
-            $dockerPath = if ("image" -notin $scenario) {
+            $dockerPath = if ("image" -notin $scenario.keys) {
                 $dockerFilePath
             } else {
                 Join-Path $pkg.Directory $scenario.image
@@ -476,7 +476,7 @@ function generateRetryTestsHelmValues ($pkg, $releaseName, $generatedHelmValues)
             $failedJobsScenario += $job.split("-$($pkg.ReleaseName)")[0]
         }
     }
-    
+
     $releaseName = "$($pkg.ReleaseName)-$revision-retry"
 
     $retryTestsHelmVal = @{"scenarios"=@()}

eng/common/testproxy/scripts/tag-merge/README.md

@@ -11,6 +11,7 @@ This script merely allows the abstraction of some of this "combination" work.
 
 ### PreReqs
 
+- Must have []`pshell 6+`](https://learn.microsoft.com/powershell/scripting/install/installing-powershell-on-windows)
 - Must have `git` available on your PATH
 - Must have the `test-proxy` available on your PATH
   - `test-proxy` is honored when the proxy is installed as a `dotnet tool`

eng/common/testproxy/scripts/tag-merge/merge-proxy-tags.ps1

@@ -1,3 +1,6 @@
+#Requires -Version 6.0
+#Requires -PSEdition Core
+
 <#
 .SYNOPSIS
 Merge multiple asset tagss worth of content into a single asset tag.
@@ -66,9 +69,6 @@ function Git-Command($CommandString, $WorkingDirectory, $HardExit=$true) {
 }
 
 function Resolve-Proxy {
-    # this script requires the presence of git
-    Test-Exe-In-Path -ExeToLookFor "git" | Out-Null
-
     $testProxyExe = "test-proxy"
     # this script requires the presence of the test-proxy on the PATH
     $proxyToolPresent = Test-Exe-In-Path -ExeToLookFor "test-proxy" -ExitOnError $false
@@ -184,7 +184,13 @@ function Start-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDire
 function Finish-Message($AssetsJson, $TargetTags, $AssetsRepoLocation, $MountDirectory) {
     $len = $TargetTags.Length
 
-    Write-Host "`nSuccessfully combined $len tags. Invoke `"test-proxy push " -NoNewLine
+    if ($TargetTags.GetType().Name -eq "String") {
+        $len = 1
+    }
+
+    $suffix = if ($len -gt 1) { "s" } else { "" }
+
+    Write-Host "`nSuccessfully combined $len tag$suffix. Invoke `"test-proxy push " -NoNewLine
     Write-Host $AssetsJson -ForegroundColor Green -NoNewLine
     Write-Host "`" to push the results as a new tag."
 }
@@ -270,12 +276,22 @@ function Combine-Tags($RemainingTags, $AssetsRepoLocation, $MountDirectory){
     # if we have successfully gotten to the end without any non-zero exit codes...delete the mergeprogress file, we're g2g
     Cleanup-Incomplete-Progress $MountDirectory
 
-    return $pushedTags
+    return @($pushedTags)
 }
 
 $ErrorActionPreference = "Stop"
 
-# resolve the proxy location so that we can invoke it easily
+# this script requires the presence of git
+Test-Exe-In-Path -ExeToLookFor "git" | Out-Null
+
+# this script expects at least powershell 6 (core)
+
+if ($PSVersionTable["PSVersion"].Major -lt 6) {
+    Write-Error "This script requires a version of powershell newer than 6. See http://aka.ms/powershell for resolution."
+    exit 1
+}
+
+# resolve the proxy location so that we can invoke it easily, if not present we exit here.
 $proxyExe = Resolve-Proxy
 
 $AssetsJson = Resolve-Path $AssetsJson

eng/pipelines/templates/jobs/archetype-go-release.yml

@@ -74,10 +74,11 @@ stages:
                 - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml
                   parameters:
                     ArtifactLocation: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}
+                    PackageFilter: sdk/${{ parameters.ServiceDirectory }}
                     ReleaseSha: $(Build.SourceVersion)
                     RepoId: Azure/azure-sdk-for-go
                     WorkingDirectory: $(System.DefaultWorkingDirectory)
-      
+
       - ${{ if not(and(startsWith(parameters.ServiceDirectory, 'resourcemanager'), ne(parameters.ServiceDirectory, 'resourcemanager/internal'))) }}:
         - deployment: UpdatePackageVersion
           displayName: "Update Package Version"

eng/scripts/modtidy.ps1

@@ -0,0 +1,23 @@
+Param(
+    [string]$searchPath = $PWD.Path
+)
+
+$modFiles = Get-ChildItem -Path $searchPath -Include "go.mod" -Recurse
+
+$tidyErrors = $modFiles | ForEach-Object -Parallel {
+    Set-Location $_.Directory
+    Write-Host (Get-Location)
+    $output = go mod tidy 2>&1
+    if ($LASTEXITCODE) {
+        return @{ Directory = $_.Directory; Output = $output }
+    }
+}
+
+if ($tidyErrors) {
+    Write-Error "Encountered the following tidy failures:" -ErrorAction 'Continue'
+    foreach ($err in $tidyErrors) {
+        Write-Host "=== $($err.Directory) ==="
+        $err.Output
+    }
+    exit 1
+}

eng/tools/generator/cmd/v2/common/fileProcessor.go

@@ -5,6 +5,7 @@ package common
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io/fs"
 	"io/ioutil"
@@ -543,11 +544,9 @@ func replaceModuleImport(path, rpName, namespaceName, previousVersion, currentVe
 		if err != nil {
 			return err
 		}
-
 		if info.IsDir() {
 			return nil
 		}
-
 		suffix := false
 		for i := 0; i < len(suffixes) && !suffix; i++ {
 			suffix = strings.HasSuffix(info.Name(), suffixes[i])
@@ -566,11 +565,25 @@ func replaceModuleImport(path, rpName, namespaceName, previousVersion, currentVe
 				}
 			}
 		}
-
 		return nil
 	})
 }
 
+func getModuleVersion(autorestPath string) (*semver.Version, error) {
+	data, err := os.ReadFile(autorestPath)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, line := range strings.Split(string(data), "\n") {
+		if strings.HasPrefix(line, autorest_md_module_version_prefix) {
+			return semver.NewVersion(strings.TrimSpace(line[len(autorest_md_module_version_prefix):]))
+		}
+	}
+
+	return nil, errors.New("module-version does not exist in autorest.md")
+}
+
 func existSuffixFile(path, suffix string) bool {
 
 	existed := false

eng/tools/generator/cmd/v2/common/generation.go

@@ -306,23 +306,28 @@ func (ctx *GenerateContext) GenerateForSingleRPNamespace(generateParam *Generate
 			return nil, err
 		}
 
+		oldModuleVersion, err := getModuleVersion(filepath.Join(packagePath, "autorest.md"))
+		if err != nil {
+			return nil, err
+		}
+
 		log.Printf("Replace version in autorest.md and constants...")
 		if err = ReplaceVersion(packagePath, version.String()); err != nil {
 			return nil, err
 		}
 
-		if _, err := os.Stat(filepath.Join(packagePath, "fake")); !os.IsNotExist(err) && changelog.HasBreakingChanges() {
+		if _, err := os.Stat(filepath.Join(packagePath, "fake")); !os.IsNotExist(err) && oldModuleVersion.Major() != version.Major() {
 			log.Printf("Replace fake module v2+...")
-			if err = replaceModuleImport(packagePath, generateParam.RPName, generateParam.NamespaceName, previousVersion, version.String(),
+			if err = replaceModuleImport(packagePath, generateParam.RPName, generateParam.NamespaceName, oldModuleVersion.String(), version.String(),
 				"fake", ".go"); err != nil {
 				return nil, err
 			}
 		}
 
 		// When sdk has major version bump, the live test needs to update the module referenced in the code.
-		if changelog.HasBreakingChanges() && existSuffixFile(packagePath, "_live_test.go") {
+		if oldModuleVersion.Major() != version.Major() && existSuffixFile(packagePath, "_live_test.go") {
 			log.Printf("Replace live test module v2+...")
-			if err = replaceModuleImport(packagePath, generateParam.RPName, generateParam.NamespaceName, previousVersion, version.String(),
+			if err = replaceModuleImport(packagePath, generateParam.RPName, generateParam.NamespaceName, oldModuleVersion.String(), version.String(),
 				"", "_live_test.go"); err != nil {
 				return nil, err
 			}

sdk/ai/azopenai/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Release History
 
+## 0.4.1 (2024-01-16)
+
+### Bugs Fixed
+
+- `AudioTranscriptionOptions.Filename` and `AudioTranslationOptions.Filename` fields are now properly propagated, allowing 
+  for disambiguating the format of an audio file when OpenAI can't detect it. (PR#22210) 
+
 ## 0.4.0 (2023-12-11)
 
 Support for many of the features mentioned in OpenAI's November Dev Day and Microsoft's 2023 Ignite conference

sdk/ai/azopenai/README.md

@@ -84,7 +84,7 @@ comments.
 
 <!-- LINKS -->
 [azure_openai_access]: https://learn.microsoft.com/azure/cognitive-services/openai/overview#how-do-i-get-access-to-azure-openai
-[azopenai_repo]: https://github.com/Azure/azure-sdk-for-go/tree/main/sdk/cognitiveservices/azopenai
+[azopenai_repo]: https://github.com/Azure/azure-sdk-for-go/tree/main/sdk/ai/azopenai
 [azopenai_pkg_go]: https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/ai/azopenai
 [azure_identity]: https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity
 [azure_sub]: https://azure.microsoft.com/free/
@@ -95,4 +95,4 @@ comments.
 [coc]: https://opensource.microsoft.com/codeofconduct/
 [coc_faq]: https://opensource.microsoft.com/codeofconduct/faq/
 [coc_contact]: mailto:opencode@microsoft.com
-[azure_openai_quickstart]: https://learn.microsoft.com/azure/cognitive-services/openai/quickstart
+[azure_openai_quickstart]: https://learn.microsoft.com/azure/cognitive-services/openai/quickstart

sdk/ai/azopenai/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "go",
   "TagPrefix": "go/ai/azopenai",
-  "Tag": "go/ai/azopenai_b42da78821"
+  "Tag": "go/ai/azopenai_85a01b7ac6"
 }