Merge main into datalake feature branch

.github/workflows/event-processor.yml

@@ -11,8 +11,6 @@ on:
   # pull request merged is the closed event with github.event.pull_request.merged = true
   pull_request_target:
     types: [closed, labeled, opened, reopened, review_requested, synchronize, unlabeled]
-  pull_request_review:
-    types: [submitted]
 
 # This removes all unnecessary permissions, the ones needed will be set below.
 # https://docs.github.com/en/actions/security-guides/automatic-token-authentication#permissions-for-the-github_token
@@ -57,7 +55,7 @@ jobs:
         run: >
           dotnet tool install
           Azure.Sdk.Tools.GitHubEventProcessor
-          --version 1.0.0-dev.20230422.1
+          --version 1.0.0-dev.20230505.2
           --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json
           --global
         shell: bash

.github/workflows/scheduled-event-processor.yml

@@ -34,7 +34,7 @@ jobs:
         run: >
           dotnet tool install
           Azure.Sdk.Tools.GitHubEventProcessor
-          --version 1.0.0-dev.20230422.1
+          --version 1.0.0-dev.20230505.2
           --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json
           --global
         shell: bash

eng/.golangci.yml

@@ -3,3 +3,13 @@ run:
   # default is true. Enables skipping of directories:
   #   vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
   skip-dirs-use-default: true
+  timeout: 10m
+
+linters:
+  enable:
+    - gocritic
+
+linters-settings:
+  gocritic:
+    enabled-checks:
+      - evalOrder

eng/common/pipelines/templates/steps/detect-api-changes.yml

@@ -1,6 +1,7 @@
 parameters:
   ArtifactPath: $(Build.ArtifactStagingDirectory)
   Artifacts: []
+  ArtifactName: 'packages'
 
 steps:
   - pwsh: |
@@ -20,6 +21,7 @@ steps:
         -PullRequestNumber $(System.PullRequest.PullRequestNumber)
         -RepoFullName $(Build.Repository.Name)
         -APIViewUri $(ApiChangeDetectRequestUrl)
+        -ArtifactName ${{ parameters.ArtifactName }}
       pwsh: true
     displayName: Detect API changes
     condition: and(succeededOrFailed(), eq(variables['Build.Reason'],'PullRequest'))

eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml

@@ -19,5 +19,15 @@ steps:
             exit 1
           }
         }
-      displayName: Prevent changes to eng/common outside of azure-sdk-tools repo
+        if ((!"$(System.PullRequest.SourceBranch)".StartsWith("sync-.github/workflows")) -and "$(System.PullRequest.TargetBranch)" -match "^(refs/heads/)?$(DefaultBranch)$")
+        {
+          $filesInCommonDir  = & "eng/common/scripts/get-changedfiles.ps1" -DiffPath '.github/workflows/*'
+          if (($LASTEXITCODE -eq 0) -and ($filesInCommonDir.Count -gt 0))
+          {
+            Write-Host "##vso[task.LogIssue type=error;]Changes to files under '.github/workflows' directory should not be made in this Repo`n${filesInCommonDir}"
+            Write-Host "##vso[task.LogIssue type=error;]Please follow workflow at https://github.com/Azure/azure-sdk-tools/blob/main/doc/workflows/engsys_workflows.md"
+            exit 1
+          }
+        }
+      displayName: Prevent changes to eng/common and .github/workflows outside of azure-sdk-tools repo
       condition: and(succeeded(), ne(variables['Skip.EngCommonWorkflowEnforcer'], 'true'), not(endsWith(variables['Build.Repository.Name'], '-pr')))

eng/common/scripts/Delete-RemoteBranches.ps1

@@ -10,6 +10,7 @@ param(
   $CentralRepoId,
   # We start from the sync PRs, use the branch name to get the PR number of central repo. E.g. sync-eng/common-(<branchName>)-(<PrNumber>). Have group name on PR number.
   # For sync-eng/common work, we use regex as "^sync-eng/common.*-(?<PrNumber>\d+).*$".
+  # For sync-.github/workflows work, we use regex as "^sync-.github/workflows.*-(?<PrNumber>\d+).*$".
   $BranchRegex,
   # Date format: e.g. Tuesday, April 12, 2022 1:36:02 PM. Allow to use other date format.
   [AllowNull()]
@@ -69,7 +70,7 @@ foreach ($res in $responses)
       LogError "No PR number found in the branch name. Please check the branch name [ $branchName ]. Skipping..."
       continue
     }
-      
+
     try {
       $centralPR = Get-GitHubPullRequest -RepoId $CentralRepoId -PullRequestNumber $pullRequestNumber -AuthToken $AuthToken
       LogDebug "Found central PR pull request: $($centralPR.html_url)"
@@ -78,7 +79,7 @@ foreach ($res in $responses)
         continue
       }
     }
-    catch 
+    catch
     {
       # If there is no central PR for the PR number, log error and skip.
       LogError "Get-GitHubPullRequests failed with exception:`n$_"
@@ -107,15 +108,15 @@ foreach ($res in $responses)
         LogDebug "The branch $branch last commit date [ $commitDate ] is newer than the date $LastCommitOlderThan. Skipping."
         continue
       }
-      
+
       LogDebug "Branch [ $branchName ] in repo [ $RepoId ] has a last commit date [ $commitDate ] that is older than $LastCommitOlderThan. "
     }
     catch {
       LogError "Get-GithubReferenceCommitDate failed with exception:`n$_"
       exit 1
     }
-  } 
-  
+  }
+
   try {
     if ($PSCmdlet.ShouldProcess("[ $branchName ] in [ $RepoId ]", "Deleting branches on cleanup script")) {
       Remove-GitHubSourceReferences -RepoId $RepoId -Ref $branch -AuthToken $AuthToken

eng/common/testproxy/target_version.txt

@@ -1 +1 @@
-1.0.0-dev.20230417.1
+1.0.0-dev.20230427.1

eng/common/testproxy/transition-scripts/README.md

@@ -46,7 +46,7 @@ To utilize this methodology, the user must set input argument `TestProxyExe` to
 
 Other requirements:
 
-- [x] Install [docker](https://docs.docker.com/engine/install/) or [podman](https://podman.io/getting-started/installation.html)
+- [x] Install [docker](https://docs.docker.com/engine/install/) or [podman](https://podman.io/)
 - [x] Set the environment variable `GIT_TOKEN` a valid token representing YOUR user
 
 ## Permissions

eng/config.json

@@ -28,6 +28,10 @@
             "Name": "azqueue",
             "CoverageGoal": 0.60
         },
+        {
+            "Name": "azfile",
+            "CoverageGoal": 0.75
+        },
         {
             "Name": "aztemplate",
             "CoverageGoal": 0.50

eng/pipelines/templates/variables/globals.yml

@@ -1,5 +1,5 @@
 variables:
-  GoLintCLIVersion: 'v1.51.1'
+  GoLintCLIVersion: 'v1.52.2'
   Package.EnableSBOMSigning: true
   # Enable go native component governance detection
   # https://docs.opensource.microsoft.com/tools/cg/index.html

eng/tools/generator/cmd/v2/common/fileProcessor.go

@@ -94,9 +94,9 @@ func ReadV2ModuleNameToGetNamespace(path string) (map[string][]PackageInfo, erro
 		return nil, fmt.Errorf("last `track2` section does not properly end")
 	}
 
-	s := strings.ReplaceAll(path, "\\", "/")
-	s1 := strings.Split(s, "/")
-	specName := s1[len(s1)-3]
+	_, after, _ := strings.Cut(strings.ReplaceAll(path, "\\", "/"), "specification")
+	before, _, _ := strings.Cut(after, "resource-manager")
+	specName := strings.Trim(before, "/")
 
 	for i := range start {
 		// get the content of the `track2` section

sdk/azcore/CHANGELOG.md

@@ -1,15 +1,32 @@
 # Release History
 
-## 1.5.1 (Unreleased)
+## 1.6.1 (Unreleased)
 
 ### Features Added
+* Added supporting features to enable distributed tracing.
+  * Added func `runtime.StartSpan()` for use by SDKs to start spans.
+  * Added method `WithContext()` to `runtime.Request` to support shallow cloning with a new context.
+  * Added field `TracingNamespace` to `runtime.PipelineOptions`.
+  * Added field `Tracer` to `runtime.NewPollerOptions` and `runtime.NewPollerFromResumeTokenOptions` types.
+  * Added field `SpanFromContext` to `tracing.TracerOptions`.
+  * Added methods `Enabled()`, `SetAttributes()`, and `SpanFromContext()` to `tracing.Tracer`.
+  * Added supporting pipeline policies to include HTTP spans when creating clients.
+* Added package `fake` to support generated fakes packages in SDKs.
+  * The package contains public surface area exposed by fake servers and supporting APIs intended only for use by the fake server implementations.
+  * Added an internal fake poller implementation.
 
 ### Breaking Changes
 
 ### Bugs Fixed
 
 ### Other Changes
 
+## 1.6.0 (2023-05-04)
+
+### Features Added
+* Added support for ARM cross-tenant authentication. Set the `AuxiliaryTenants` field of `arm.ClientOptions` to enable.
+* Added `TenantID` field to `policy.TokenRequestOptions`.
+
 ## 1.5.0 (2023-04-06)
 
 ### Features Added

sdk/azcore/arm/runtime/pipeline.go

@@ -13,6 +13,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	armpolicy "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm/policy"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/exported"
 	azpolicy "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	azruntime "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
 )
@@ -34,7 +35,7 @@ func NewPipeline(module, version string, cred azcore.TokenCredential, plOpts azr
 	})
 	perRetry := make([]azpolicy.Policy, len(plOpts.PerRetry), len(plOpts.PerRetry)+1)
 	copy(perRetry, plOpts.PerRetry)
-	plOpts.PerRetry = append(perRetry, authPolicy)
+	plOpts.PerRetry = append(perRetry, authPolicy, exported.PolicyFunc(httpTraceNamespacePolicy))
 	if !options.DisableRPRegistration {
 		regRPOpts := armpolicy.RegistrationOptions{ClientOptions: options.ClientOptions}
 		regPolicy, err := NewRPRegistrationPolicy(cred, &regRPOpts)

sdk/azcore/arm/runtime/policy_bearer_token.go

@@ -5,6 +5,7 @@ package runtime
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"net/http"
 	"strings"
@@ -63,11 +64,28 @@ func NewBearerTokenPolicy(cred azcore.TokenCredential, opts *armpolicy.BearerTok
 	p.scopes = make([]string, len(opts.Scopes))
 	copy(p.scopes, opts.Scopes)
 	p.btp = azruntime.NewBearerTokenPolicy(cred, opts.Scopes, &azpolicy.BearerTokenOptions{
-		AuthorizationHandler: azpolicy.AuthorizationHandler{OnRequest: p.onRequest},
+		AuthorizationHandler: azpolicy.AuthorizationHandler{
+			OnChallenge: p.onChallenge,
+			OnRequest:   p.onRequest,
+		},
 	})
 	return p
 }
 
+func (b *BearerTokenPolicy) onChallenge(req *azpolicy.Request, res *http.Response, authNZ func(azpolicy.TokenRequestOptions) error) error {
+	challenge := res.Header.Get(shared.HeaderWWWAuthenticate)
+	claims, err := parseChallenge(challenge)
+	if err != nil {
+		// the challenge contains claims we can't parse
+		return err
+	} else if claims != "" {
+		// request a new token having the specified claims, send the request again
+		return authNZ(azpolicy.TokenRequestOptions{Claims: claims, Scopes: b.scopes})
+	}
+	// auth challenge didn't include claims, so this is a simple authorization failure
+	return azruntime.NewResponseError(res)
+}
+
 // onRequest authorizes requests with one or more bearer tokens
 func (b *BearerTokenPolicy) onRequest(req *azpolicy.Request, authNZ func(azpolicy.TokenRequestOptions) error) error {
 	// authorize the request with a token for the primary tenant
@@ -97,3 +115,31 @@ func (b *BearerTokenPolicy) onRequest(req *azpolicy.Request, authNZ func(azpolic
 func (b *BearerTokenPolicy) Do(req *azpolicy.Request) (*http.Response, error) {
 	return b.btp.Do(req)
 }
+
+// parseChallenge parses claims from an authentication challenge issued by ARM so a client can request a token
+// that will satisfy conditional access policies. It returns a non-nil error when the given value contains
+// claims it can't parse. If the value contains no claims, it returns an empty string and a nil error.
+func parseChallenge(wwwAuthenticate string) (string, error) {
+	claims := ""
+	var err error
+	for _, param := range strings.Split(wwwAuthenticate, ",") {
+		if _, after, found := strings.Cut(param, "claims="); found {
+			if claims != "" {
+				// The header contains multiple challenges, at least two of which specify claims. The specs allow this
+				// but it's unclear what a client should do in this case and there's as yet no concrete example of it.
+				err = fmt.Errorf("found multiple claims challenges in %q", wwwAuthenticate)
+				break
+			}
+			// trim stuff that would get an error from RawURLEncoding; claims may or may not be padded
+			claims = strings.Trim(after, `\"=`)
+			// we don't return this error because it's something unhelpful like "illegal base64 data at input byte 42"
+			if b, decErr := base64.RawURLEncoding.DecodeString(claims); decErr == nil {
+				claims = string(b)
+			} else {
+				err = fmt.Errorf("failed to parse claims from %q", wwwAuthenticate)
+				break
+			}
+		}
+	}
+	return claims, err
+}

sdk/azcore/arm/runtime/policy_bearer_token_test.go

@@ -203,7 +203,6 @@ func TestAuxiliaryTenants(t *testing.T) {
 }
 
 func TestBearerTokenPolicyChallengeParsing(t *testing.T) {
-	t.Skip("unskip this test after adding back CAE support")
 	for _, test := range []struct {
 		challenge, desc, expectedClaims string
 		err                             error
@@ -262,10 +261,9 @@ func TestBearerTokenPolicyChallengeParsing(t *testing.T) {
 			cred := mockCredential{
 				getTokenImpl: func(ctx context.Context, actual azpolicy.TokenRequestOptions) (azcore.AccessToken, error) {
 					calls += 1
-					// TODO: uncomment after restoring TokenRequestOptions.Claims
-					// if calls == 2 && test.expectedClaims != "" {
-					// require.Equal(t, test.expectedClaims, actual.Claims)
-					// }
+					if calls == 2 && test.expectedClaims != "" {
+						require.Equal(t, test.expectedClaims, actual.Claims)
+					}
 					return azcore.AccessToken{Token: "...", ExpiresOn: time.Now().Add(time.Hour).UTC()}, nil
 				},
 			}

sdk/azcore/arm/runtime/policy_trace_namespace.go

@@ -0,0 +1,31 @@
+//go:build go1.18
+// +build go1.18
+
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package runtime
+
+import (
+	"net/http"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm/internal/resource"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/tracing"
+)
+
+// httpTraceNamespacePolicy is a policy that adds the az.namespace attribute to the current Span
+func httpTraceNamespacePolicy(req *policy.Request) (resp *http.Response, err error) {
+	rawTracer := req.Raw().Context().Value(shared.CtxWithTracingTracer{})
+	if tracer, ok := rawTracer.(tracing.Tracer); ok {
+		rt, err := resource.ParseResourceType(req.Raw().URL.Path)
+		if err == nil {
+			// add the namespace attribute to the current span
+			if span, ok := tracer.SpanFromContext(req.Raw().Context()); ok {
+				span.SetAttributes(tracing.Attribute{Key: "az.namespace", Value: rt.Namespace})
+			}
+		}
+	}
+	return req.Next()
+}