Handle STS Credentials region (#8605)

* update API

* explicitly set region in serviceAccountCredentials

* add changelog entry

* relocate changelog entry to proper release

* update test fixture in helm tests

* Adding changelog file to new location

* Deleting changelog file from old location

* Adding changelog file to new location

* Deleting changelog file from old location

* add STS tests

* Adding changelog file to new location

* Deleting changelog file from old location

* Adding changelog file to new location

* Deleting changelog file from old location

* Adding changelog file to new location

* Deleting changelog file from old location

* make region not required

* add e2e tests against default region

* correct typo in aws e2e tests

* bump envoy-gloo version

* remove unnecessary tests from AWS e2e test

---------

Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>
Co-authored-by: changelog-bot <changelog-bot>

Makefile

@@ -81,7 +81,7 @@ else
   endif
 endif
 
-ENVOY_GLOO_IMAGE ?= quay.io/solo-io/envoy-gloo:1.23.12-patch1
+ENVOY_GLOO_IMAGE ?= quay.io/solo-io/envoy-gloo:1.23.12-patch2
 
 # The full SHA of the currently checked out commit
 CHECKED_OUT_SHA := $(shell git rev-parse HEAD)

changelog/v1.13.27/handle-sts-credentials-region.yaml

@@ -0,0 +1,10 @@
+changelog:
+- type: FIX
+  issueLink: https://github.com/solo-io/gloo/issues/8578
+  resolvesIssue: false 
+  description: >
+    Support role chaining using EKS ServiceAccounts outside of us-east-1
+- type: DEPENDENCY_BUMP
+  dependencyOwner: solo-io
+  dependencyRepo: envoy-gloo
+  dependencyTag: v1.23.12-patch2

install/helm/gloo/crds/gloo.solo.io_v1_Settings.yaml

@@ -359,6 +359,8 @@ spec:
                         properties:
                           cluster:
                             type: string
+                          region:
+                            type: string
                           timeout:
                             type: string
                           uri:

install/helm/gloo/templates/18-settings.yaml

@@ -44,8 +44,10 @@ spec:
         cluster: aws_sts_cluster
 {{- if not .Values.settings.aws.stsCredentialsRegion }}
         uri: sts.amazonaws.com
+        region: us-east-1
 {{- else }}
         uri: sts.{{ .Values.settings.aws.stsCredentialsRegion }}.amazonaws.com
+        region: {{ .Values.settings.aws.stsCredentialsRegion }}
 {{- end }}
 {{- else if .Values.settings.aws.enableCredentialsDiscovery }}
       enableCredentialsDiscovey: true

install/test/fixtures/settings/sts_discovery.yaml

@@ -36,6 +36,7 @@ spec:
       serviceAccountCredentials:
         cluster: aws_sts_cluster
         uri: sts.us-east-2.amazonaws.com
+        region: us-east-2
       propagateOriginalRouting: true
   kubernetesArtifactSource: {}
   kubernetesConfigSource: {}

projects/gloo/api/external/envoy/extensions/aws/filter.proto

@@ -116,6 +116,8 @@ message AWSLambdaConfig {
     string uri = 2 [ (validate.rules).string.min_bytes = 1 ];
     // timeout for the request
     google.protobuf.Duration timeout = 3;
+    // Region for the sts endpoint. Defaults to us-east-1
+    string region = 4;
   }
 
   // Send downstream path and method as `x-envoy-original-path` and

test/e2e/aws_test.go

@@ -45,7 +45,8 @@ import (
 
 var _ = Describe("AWS Lambda", func() {
 	const (
-		region               = "us-east-1"
+		defaultRegion        = "us-east-1"
+		secondaryRegion      = "us-east-2"
 		webIdentityTokenFile = "AWS_WEB_IDENTITY_TOKEN_FILE"
 		jwtPrivateKey        = "JWT_PRIVATE_KEY"
 		awsRoleArn           = "AWS_ROLE_ARN"
@@ -133,11 +134,11 @@ var _ = Describe("AWS Lambda", func() {
 		upstream = &gloov1.Upstream{
 			Metadata: &core.Metadata{
 				Namespace: "default",
-				Name:      region,
+				Name:      defaultRegion,
 			},
 			UpstreamType: &gloov1.Upstream_Aws{
 				Aws: &aws_plugin.UpstreamSpec{
-					Region:    region,
+					Region:    defaultRegion,
 					SecretRef: secret.Metadata.Ref(),
 				},
 			},
@@ -223,7 +224,7 @@ var _ = Describe("AWS Lambda", func() {
 			},
 			UpstreamType: &gloov1.Upstream_Aws{
 				Aws: &aws_plugin.UpstreamSpec{
-					Region:    region,
+					Region:    defaultRegion,
 					SecretRef: secret.Metadata.Ref(),
 					// this is a separate account ID from the one that all other lambda
 					// functions tested in this file are in
@@ -649,7 +650,7 @@ var _ = Describe("AWS Lambda", func() {
 			secret = &gloov1.Secret{
 				Metadata: &core.Metadata{
 					Namespace: "default",
-					Name:      region,
+					Name:      defaultRegion,
 				},
 				Kind: &gloov1.Secret_Aws{
 					Aws: &gloov1.AwsSecret{
@@ -702,7 +703,7 @@ var _ = Describe("AWS Lambda", func() {
 
 		addCredentials := func() {
 			localAwsCredentials := credentials.NewSharedCredentials("", "")
-			sess, err := session.NewSession(&aws.Config{Region: aws.String(region), Credentials: localAwsCredentials})
+			sess, err := session.NewSession(&aws.Config{Region: aws.String(defaultRegion), Credentials: localAwsCredentials})
 			if err != nil {
 				Fail("no AWS creds available")
 			}
@@ -714,7 +715,7 @@ var _ = Describe("AWS Lambda", func() {
 			secret = &gloov1.Secret{
 				Metadata: &core.Metadata{
 					Namespace: "default",
-					Name:      region,
+					Name:      defaultRegion,
 				},
 				Kind: &gloov1.Secret_Aws{
 					Aws: &gloov1.AwsSecret{
@@ -810,7 +811,7 @@ var _ = Describe("AWS Lambda", func() {
 			}
 		}
 
-		addUpstreamSts := func() {
+		addUpstreamSts := func(region string) {
 			upstream = &gloov1.Upstream{
 				Metadata: &core.Metadata{
 					Namespace: "default",
@@ -844,10 +845,19 @@ var _ = Describe("AWS Lambda", func() {
 			}))
 		}
 
-		setupEnvoySts := func(justGloo bool) {
+		setupEnvoySts := func(justGloo bool, region string) {
 			ctx, cancel = context.WithCancel(context.Background())
 			defaults.HttpPort = services.NextBindPort()
 			defaults.HttpsPort = services.NextBindPort()
+
+			var uri string
+			if region == "" {
+				region = defaultRegion
+				uri = "sts.amazonaws.com"
+			} else {
+				uri = fmt.Sprintf("sts.%s.amazonaws.com", region)
+			}
+
 			ns := defaults.GlooSystem
 			ro := &services.RunOptions{
 				NsToWrite: ns,
@@ -862,7 +872,8 @@ var _ = Describe("AWS Lambda", func() {
 							CredentialsFetcher: &gloov1.GlooOptions_AWSOptions_ServiceAccountCredentials{
 								ServiceAccountCredentials: &aws2.AWSLambdaConfig_ServiceAccountCredentials{
 									Cluster: "aws_sts_cluster",
-									Uri:     "sts.amazonaws.com",
+									Uri:     uri,
+									Region:  region,
 								},
 							},
 						},
@@ -884,29 +895,49 @@ var _ = Describe("AWS Lambda", func() {
 			os.Unsetenv(webIdentityTokenFile)
 		})
 		Context("No gateway translation ", func() {
-			BeforeEach(func() {
-				setupEnvoySts(true)
-				addCredentialsSts()
-				addUpstreamSts()
+			Context("primary region", func() {
+				BeforeEach(func() {
+					setupEnvoySts(true, defaultRegion)
+					addCredentialsSts()
+					addUpstreamSts(defaultRegion)
+				})
+				/*
+				 * these tests can start failing if certs get rotated underneath us.
+				 * the fix is to update the rotated thumbprint on our fake AWS OIDC per
+				 * https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc_verify-thumbprint.html
+				 */
+				It("should be able to call lambda", testProxy)
+
+				It("should be able to call lambda with response transform", testProxyWithResponseTransform)
+
+				It("should be able to call lambda with request transform", testProxyWithRequestTransform)
+
+				It("should be able to call lambda with request and response transforms", testProxyWithRequestAndResponseTransforms)
+			})
+			Context("secondary region", func() {
+				BeforeEach(func() {
+					setupEnvoySts(true, secondaryRegion)
+					addCredentialsSts()
+					addUpstreamSts(secondaryRegion)
+				})
+
+				It("should be able to call lambda", testProxy)
+			})
+			Context("default region", func() {
+				BeforeEach(func() {
+					setupEnvoySts(true, "")
+					addCredentialsSts()
+					addUpstreamSts(defaultRegion)
+				})
+
+				It("should be able to call lambda", testProxy)
 			})
-			/*
-			 * these tests can start failing if certs get rotated underneath us.
-			 * the fix is to update the rotated thumbprint on our fake AWS OIDC per
-			 * https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc_verify-thumbprint.html
-			 */
-			It("should be able to call lambda", testProxy)
-
-			It("should be able lambda with response transform", testProxyWithResponseTransform)
-
-			It("should be able to call lambda with request transform", testProxyWithRequestTransform)
-
-			It("should be able to call lambda with request and response transforms", testProxyWithRequestAndResponseTransforms)
 		})
 		Context("With gateway translation", func() {
 			BeforeEach(func() {
-				setupEnvoySts(false)
+				setupEnvoySts(false, defaultRegion)
 				addCredentialsSts()
-				addUpstreamSts()
+				addUpstreamSts(defaultRegion)
 			})
 			It("should be able to call lambda via gateway", testLambdaWithVirtualService)