From 30ec4271c26ed2e889b675a6d177c4b429216d00 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Mon, 20 Feb 2023 22:25:37 -0800 Subject: [PATCH 01/27] adding support for azure managed prometheus Signed-off-by: Raghav Gupta --- ...e_managed_prometheus_http_round_tripper.go | 110 ++++++++++++++++++ ...aged_prometheus_http_round_tripper_test.go | 68 +++++++++++ pkg/scalers/prometheus_scaler.go | 49 ++++++-- pkg/scalers/prometheus_scaler_test.go | 46 +++++--- 4 files changed, 245 insertions(+), 28 deletions(-) create mode 100644 pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go create mode 100644 pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go new file mode 100644 index 00000000000..2189453e303 --- /dev/null +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go @@ -0,0 +1,110 @@ +package azure + +import ( + "crypto/tls" + "fmt" + "net" + "net/http" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + kedautil "github.com/kedacore/keda/v2/pkg/util" +) + +const ( + defaultAzureManagedPrometheusResourceURL = "https://prometheus.monitor.azure.com/.default" +) + +var azureManagedPrometheusResourceURLInCloud = map[string]string{ + "AZUREPUBLICCLOUD": "https://prometheus.monitor.azure.com/.default", + "AZUREUSGOVERNMENTCLOUD": "https://prometheus.monitor.usgovcloudapi.net/.default", + "AZURECHINACLOUD": "https://prometheus.monitor.chinacloudapp.cn/.default", +} + +type azureManagedPrometheusHttpRoundTripper struct { + chainedCredential *azidentity.ChainedTokenCredential + next http.RoundTripper + resourceURL string +} + +func TryAndGetAzureManagedPrometheusHttpRoundTripper(podIdentity kedav1alpha1.AuthPodIdentity, triggerMetadata map[string]string) (http.RoundTripper, error) { + + switch podIdentity.Provider { + case kedav1alpha1.PodIdentityProviderAzureWorkload, kedav1alpha1.PodIdentityProviderAzure: + + if triggerMetadata == nil { + return nil, fmt.Errorf("trigger metadata cannot be nil") + } + + tlsConfig := &tls.Config{ + MinVersion: kedautil.GetMinTLSVersion(), + InsecureSkipVerify: false, + } + + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: tlsConfig, + } + + chainedCred, err := NewChainedCredential(podIdentity.IdentityID, podIdentity.Provider) + if err != nil { + return nil, err + } + + resourceUrlBasedOnCloud, err := getResourceUrlBasedOnCloud(triggerMetadata) + if err != nil { + return nil, err + } + + rt := &azureManagedPrometheusHttpRoundTripper{ + next: transport, + chainedCredential: chainedCred, + resourceURL: resourceUrlBasedOnCloud, + } + return rt, nil + } + + // Not azure managed prometheus. Don't do anything. + return nil, nil +} + +func getResourceUrlBasedOnCloud(triggerMetadata map[string]string) (string, error) { + if cloud, ok := triggerMetadata["cloud"]; ok { + if strings.EqualFold(cloud, PrivateCloud) { + if resource, ok := triggerMetadata["azureManagedPrometheusResourceURL"]; ok && resource != "" { + return resource, nil + } + + return "", fmt.Errorf("azureManagedPrometheusResourceURL must be provided for %s cloud type", PrivateCloud) + } + + if resource, ok := azureManagedPrometheusResourceURLInCloud[strings.ToUpper(cloud)]; ok { + return resource, nil + } + return "", fmt.Errorf("there is no cloud environment matching the name %s", cloud) + } + + // return default resourceURL for public cloud + return defaultAzureManagedPrometheusResourceURL, nil +} + +// Sets Auhtorization header for requests +func (rt *azureManagedPrometheusHttpRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + token, err := rt.chainedCredential.GetToken(req.Context(), policy.TokenRequestOptions{Scopes: []string{rt.resourceURL}}) + if err != nil { + return nil, err + } + + bearerAccessToken := "Bearer " + token.Token + req.Header.Set("Authorization", bearerAccessToken) + + return rt.next.RoundTrip(req) +} diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go new file mode 100644 index 00000000000..b85d9b46ead --- /dev/null +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go @@ -0,0 +1,68 @@ +package azure + +import ( + "testing" + + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" +) + +type testAzureManagedPrometheusCloudResourceURLData struct { + testName string + cloud string + azureManagedPrometheusResourceURL string + isError bool + expectedValue string +} + +var testAzureManagedPrometheusCloudResourceURLTestData = []testAzureManagedPrometheusCloudResourceURLData{ + {"test public cloud", "AZUREPUBLICCLOUD", "", false, "https://prometheus.monitor.azure.com/.default"}, + {"test usgov cloud", "AZUREUSGOVERNMENTCLOUD", "", false, "https://prometheus.monitor.usgovcloudapi.net/.default"}, + {"test china cloud", "AZURECHINACLOUD", "", false, "https://prometheus.monitor.chinacloudapp.cn/.default"}, + {"test private cloud success", "PRIVATE", "https://some.private.url", false, "https://some.private.url"}, + {"test private cloud failure", "PRIVATE", "", true, ""}, + {"test non existing cloud", "BOGUS_CLOUD", "", true, ""}, +} + +func TestAzureManagedPrometheusGetResourceUrlBasedOnCloud(t *testing.T) { + for _, testData := range testAzureManagedPrometheusCloudResourceURLTestData { + value, err := getResourceUrlBasedOnCloud(map[string]string{"cloud": testData.cloud, "azureManagedPrometheusResourceURL": testData.azureManagedPrometheusResourceURL}) + if testData.isError { + if err == nil { + t.Errorf("Test: %v; Expected error but got success. testData: %v", testData.testName, testData) + } + } else { + if err == nil { + if testData.expectedValue != value { + t.Errorf("Test: %v; Expected value %v but got %v testData: %v", testData.testName, testData.expectedValue, value, testData) + } + } else { + t.Errorf("Test: %v; Expected success but got error: %v", testData.testName, err) + } + } + } +} + +type testTryAndGetAzureManagedPrometheusHttpRoundTripperData struct { + testName string + podIdentityProvider kedav1alpha1.PodIdentityProvider + isError bool +} + +var testTryAndGetAzureManagedPrometheusHttpRoundTripperTestData = []testTryAndGetAzureManagedPrometheusHttpRoundTripperData{ + {"test azure workload identity trigger metadata absent", kedav1alpha1.PodIdentityProviderAzureWorkload, true}, + {"test azure pod identity trigger metadata absent", kedav1alpha1.PodIdentityProviderAzureWorkload, true}, + {"test not azure identity", kedav1alpha1.PodIdentityProviderNone, false}, +} + +func TestTryAndGetAzureManagedPrometheusHttpRoundTripper(t *testing.T) { + for _, testData := range testTryAndGetAzureManagedPrometheusHttpRoundTripperTestData { + _, err := TryAndGetAzureManagedPrometheusHttpRoundTripper(kedav1alpha1.AuthPodIdentity{Provider: testData.podIdentityProvider}, nil) + if testData.isError { + if err == nil { + t.Errorf("Test: %v; Expected error but got success. testData: %v", testData.testName, testData) + } + } else if err != nil { + t.Errorf("Test: %v; Expected success but got error: %v", testData.testName, err) + } + } +} diff --git a/pkg/scalers/prometheus_scaler.go b/pkg/scalers/prometheus_scaler.go index b96e61e7a81..79c485072d6 100644 --- a/pkg/scalers/prometheus_scaler.go +++ b/pkg/scalers/prometheus_scaler.go @@ -16,6 +16,7 @@ import ( "k8s.io/metrics/pkg/apis/external_metrics" "github.com/kedacore/keda/v2/pkg/scalers/authentication" + "github.com/kedacore/keda/v2/pkg/scalers/azure" kedautil "github.com/kedacore/keda/v2/pkg/util" ) @@ -89,17 +90,32 @@ func NewPrometheusScaler(config *ScalerConfig) (Scaler, error) { httpClient := kedautil.CreateHTTPClient(config.GlobalHTTPTimeout, meta.unsafeSsl) - if meta.prometheusAuth != nil && (meta.prometheusAuth.CA != "" || meta.prometheusAuth.EnableTLS) { - // create http.RoundTripper with auth settings from ScalerConfig - transport, err := authentication.CreateHTTPRoundTripper( - authentication.NetHTTP, - meta.prometheusAuth, - ) + if meta.prometheusAuth != nil { + if meta.prometheusAuth.CA != "" || meta.prometheusAuth.EnableTLS { + // create http.RoundTripper with auth settings from ScalerConfig + transport, err := authentication.CreateHTTPRoundTripper( + authentication.NetHTTP, + meta.prometheusAuth, + ) + if err != nil { + logger.V(1).Error(err, "init Prometheus client http transport") + return nil, err + } + httpClient.Transport = transport + } + } else { + // could be the case of azure managed prometheus. Try and get the roundtripper. + transport, err := azure.TryAndGetAzureManagedPrometheusHttpRoundTripper(config.PodIdentity, config.TriggerMetadata) + if err != nil { - logger.V(1).Error(err, "init Prometheus client http transport") + logger.V(1).Error(err, "error while init Azure Managed Prometheus client http transport") return nil, err } - httpClient.Transport = transport + + // transport should not be nil if its a case of azure managed prometheus + if transport != nil { + httpClient.Transport = transport + } } return &prometheusScaler{ @@ -182,14 +198,27 @@ func parsePrometheusMetadata(config *ScalerConfig) (meta *prometheusMetadata, er meta.scalerIndex = config.ScalerIndex + err = parseAuthConfig(config, meta) + if err != nil { + return nil, err + } + + return meta, nil +} + +func parseAuthConfig(config *ScalerConfig, meta *prometheusMetadata) error { // parse auth configs from ScalerConfig auth, err := authentication.GetAuthConfigs(config.TriggerMetadata, config.AuthParams) if err != nil { - return nil, err + return err + } + + if auth != nil && config.PodIdentity.Provider != "" { + return fmt.Errorf("pod identity cannot be enabled with other auth types") } meta.prometheusAuth = auth - return meta, nil + return nil } func (s *prometheusScaler) Close(context.Context) error { diff --git a/pkg/scalers/prometheus_scaler_test.go b/pkg/scalers/prometheus_scaler_test.go index 674622cac7a..e875e250d65 100644 --- a/pkg/scalers/prometheus_scaler_test.go +++ b/pkg/scalers/prometheus_scaler_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/go-logr/logr" + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" "github.com/stretchr/testify/assert" ) @@ -56,32 +57,39 @@ var prometheusMetricIdentifiers = []prometheusMetricIdentifier{ } type prometheusAuthMetadataTestData struct { - metadata map[string]string - authParams map[string]string - isError bool + metadata map[string]string + authParams map[string]string + podIdentityProvider kedav1alpha1.PodIdentityProvider + isError bool } var testPrometheusAuthMetadata = []prometheusAuthMetadataTestData{ // success TLS - {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "tls"}, map[string]string{"ca": "caaa", "cert": "ceert", "key": "keey"}, false}, + {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "tls"}, map[string]string{"ca": "caaa", "cert": "ceert", "key": "keey"}, "", false}, // TLS, ca is optional - {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "tls"}, map[string]string{"cert": "ceert", "key": "keey"}, false}, + {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "tls"}, map[string]string{"cert": "ceert", "key": "keey"}, "", false}, // fail TLS, key not given - {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "tls"}, map[string]string{"ca": "caaa", "cert": "ceert"}, true}, + {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "tls"}, map[string]string{"ca": "caaa", "cert": "ceert"}, "", true}, // fail TLS, cert not given - {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "tls"}, map[string]string{"ca": "caaa", "key": "keey"}, true}, + {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "tls"}, map[string]string{"ca": "caaa", "key": "keey"}, "", true}, // success bearer default - {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "bearer"}, map[string]string{"bearerToken": "tooooken"}, false}, + {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "bearer"}, map[string]string{"bearerToken": "tooooken"}, "", false}, // fail bearerAuth with no token - {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "bearer"}, map[string]string{}, true}, + {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "bearer"}, map[string]string{}, "", true}, // success basicAuth - {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "basic"}, map[string]string{"username": "user", "password": "pass"}, false}, + {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "basic"}, map[string]string{"username": "user", "password": "pass"}, "", false}, // fail basicAuth with no username - {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "basic"}, map[string]string{}, true}, + {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "basic"}, map[string]string{}, "", true}, - {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "tls, basic"}, map[string]string{"ca": "caaa", "cert": "ceert", "key": "keey", "username": "user", "password": "pass"}, false}, + {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "tls, basic"}, map[string]string{"ca": "caaa", "cert": "ceert", "key": "keey", "username": "user", "password": "pass"}, "", false}, - {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "tls,basic"}, map[string]string{"username": "user", "password": "pass"}, true}, + {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "tls,basic"}, map[string]string{"username": "user", "password": "pass"}, "", true}, + // pod identity and other auth modes enabled together + {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up", "authModes": "basic"}, map[string]string{"username": "user", "password": "pass"}, "azure-workload", true}, + // azure workload identity + {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up"}, nil, "azure-workload", false}, + // azure pod identity + {map[string]string{"serverAddress": "http://localhost:9090", "metricName": "http_requests_total", "threshold": "100", "query": "up"}, nil, "azure", false}, } func TestPrometheusParseMetadata(t *testing.T) { @@ -117,7 +125,7 @@ func TestPrometheusGetMetricSpecForScaling(t *testing.T) { func TestPrometheusScalerAuthParams(t *testing.T) { for _, testData := range testPrometheusAuthMetadata { - meta, err := parsePrometheusMetadata(&ScalerConfig{TriggerMetadata: testData.metadata, AuthParams: testData.authParams}) + meta, err := parsePrometheusMetadata(&ScalerConfig{TriggerMetadata: testData.metadata, AuthParams: testData.authParams, PodIdentity: kedav1alpha1.AuthPodIdentity{Provider: testData.podIdentityProvider}}) if err != nil && !testData.isError { t.Error("Expected success but got error", err) @@ -127,10 +135,12 @@ func TestPrometheusScalerAuthParams(t *testing.T) { } if err == nil { - if (meta.prometheusAuth.EnableBearerAuth && !strings.Contains(testData.metadata["authModes"], "bearer")) || - (meta.prometheusAuth.EnableBasicAuth && !strings.Contains(testData.metadata["authModes"], "basic")) || - (meta.prometheusAuth.EnableTLS && !strings.Contains(testData.metadata["authModes"], "tls")) { - t.Error("wrong auth mode detected") + if meta.prometheusAuth != nil { + if (meta.prometheusAuth.EnableBearerAuth && !strings.Contains(testData.metadata["authModes"], "bearer")) || + (meta.prometheusAuth.EnableBasicAuth && !strings.Contains(testData.metadata["authModes"], "basic")) || + (meta.prometheusAuth.EnableTLS && !strings.Contains(testData.metadata["authModes"], "tls")) { + t.Error("wrong auth mode detected") + } } } } From 21d6f65e3b3048909e7441600e14d644e837a06f Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Wed, 22 Feb 2023 21:01:08 -0800 Subject: [PATCH 02/27] review comments Signed-off-by: Raghav Gupta --- ...e_managed_prometheus_http_round_tripper.go | 72 +++------- ...aged_prometheus_http_round_tripper_test.go | 63 ++------ pkg/scalers/prometheus_scaler.go | 2 +- pkg/scalers/prometheus_scaler_test.go | 3 +- .../autorest/azure/environments.go | 135 +++++++++--------- 5 files changed, 105 insertions(+), 170 deletions(-) diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go index 2189453e303..74af4d1b3a4 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go @@ -1,36 +1,27 @@ package azure import ( - "crypto/tls" "fmt" - "net" "net/http" - "strings" - "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" - kedautil "github.com/kedacore/keda/v2/pkg/util" -) + az "github.com/Azure/go-autorest/autorest/azure" -const ( - defaultAzureManagedPrometheusResourceURL = "https://prometheus.monitor.azure.com/.default" + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/kedacore/keda/v2/pkg/util" ) -var azureManagedPrometheusResourceURLInCloud = map[string]string{ - "AZUREPUBLICCLOUD": "https://prometheus.monitor.azure.com/.default", - "AZUREUSGOVERNMENTCLOUD": "https://prometheus.monitor.usgovcloudapi.net/.default", - "AZURECHINACLOUD": "https://prometheus.monitor.chinacloudapp.cn/.default", -} - type azureManagedPrometheusHttpRoundTripper struct { chainedCredential *azidentity.ChainedTokenCredential next http.RoundTripper resourceURL string } -func TryAndGetAzureManagedPrometheusHttpRoundTripper(podIdentity kedav1alpha1.AuthPodIdentity, triggerMetadata map[string]string) (http.RoundTripper, error) { +// Tries to get a round tripper. +// If the pod identity represents azure auth, it creates a round tripper and returns that. Returns error if fails to create one. +// If its not azure auth, then this becomes a no-op. Neither returns round tripper nor error. +func TryAndGetAzureManagedPrometheusHTTPRoundTripper(podIdentity kedav1alpha1.AuthPodIdentity, triggerMetadata map[string]string) (http.RoundTripper, error) { switch podIdentity.Provider { case kedav1alpha1.PodIdentityProviderAzureWorkload, kedav1alpha1.PodIdentityProviderAzure: @@ -39,63 +30,36 @@ func TryAndGetAzureManagedPrometheusHttpRoundTripper(podIdentity kedav1alpha1.Au return nil, fmt.Errorf("trigger metadata cannot be nil") } - tlsConfig := &tls.Config{ - MinVersion: kedautil.GetMinTLSVersion(), - InsecureSkipVerify: false, - } - - transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).DialContext, - TLSHandshakeTimeout: 10 * time.Second, - TLSClientConfig: tlsConfig, - } - chainedCred, err := NewChainedCredential(podIdentity.IdentityID, podIdentity.Provider) if err != nil { return nil, err } - resourceUrlBasedOnCloud, err := getResourceUrlBasedOnCloud(triggerMetadata) + azureManagedPrometheusResourceURLProvider := func(env az.Environment) (string, error) { + if env.ResourceIdentifiers.AzureManagedPrometheus == az.NotAvailable { + return "", fmt.Errorf("Azure Managed Prometheus is not avaiable in cloud %s", env.Name) + } + return env.ResourceIdentifiers.AzureManagedPrometheus, nil + } + + resourceURLBasedOnCloud, err := ParseEnvironmentProperty(triggerMetadata, "azureManagedPrometheusResourceURL", azureManagedPrometheusResourceURLProvider) if err != nil { return nil, err } + transport := util.CreateHTTPTransport(false) rt := &azureManagedPrometheusHttpRoundTripper{ next: transport, chainedCredential: chainedCred, - resourceURL: resourceUrlBasedOnCloud, + resourceURL: resourceURLBasedOnCloud, } return rt, nil } - // Not azure managed prometheus. Don't do anything. + // Not azure managed prometheus. Don't create a round tripper and don't return error. return nil, nil } -func getResourceUrlBasedOnCloud(triggerMetadata map[string]string) (string, error) { - if cloud, ok := triggerMetadata["cloud"]; ok { - if strings.EqualFold(cloud, PrivateCloud) { - if resource, ok := triggerMetadata["azureManagedPrometheusResourceURL"]; ok && resource != "" { - return resource, nil - } - - return "", fmt.Errorf("azureManagedPrometheusResourceURL must be provided for %s cloud type", PrivateCloud) - } - - if resource, ok := azureManagedPrometheusResourceURLInCloud[strings.ToUpper(cloud)]; ok { - return resource, nil - } - return "", fmt.Errorf("there is no cloud environment matching the name %s", cloud) - } - - // return default resourceURL for public cloud - return defaultAzureManagedPrometheusResourceURL, nil -} - // Sets Auhtorization header for requests func (rt *azureManagedPrometheusHttpRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { token, err := rt.chainedCredential.GetToken(req.Context(), policy.TokenRequestOptions{Scopes: []string{rt.resourceURL}}) diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go index b85d9b46ead..8d60a288d87 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go @@ -6,63 +6,28 @@ import ( kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" ) -type testAzureManagedPrometheusCloudResourceURLData struct { - testName string - cloud string - azureManagedPrometheusResourceURL string - isError bool - expectedValue string -} - -var testAzureManagedPrometheusCloudResourceURLTestData = []testAzureManagedPrometheusCloudResourceURLData{ - {"test public cloud", "AZUREPUBLICCLOUD", "", false, "https://prometheus.monitor.azure.com/.default"}, - {"test usgov cloud", "AZUREUSGOVERNMENTCLOUD", "", false, "https://prometheus.monitor.usgovcloudapi.net/.default"}, - {"test china cloud", "AZURECHINACLOUD", "", false, "https://prometheus.monitor.chinacloudapp.cn/.default"}, - {"test private cloud success", "PRIVATE", "https://some.private.url", false, "https://some.private.url"}, - {"test private cloud failure", "PRIVATE", "", true, ""}, - {"test non existing cloud", "BOGUS_CLOUD", "", true, ""}, -} - -func TestAzureManagedPrometheusGetResourceUrlBasedOnCloud(t *testing.T) { - for _, testData := range testAzureManagedPrometheusCloudResourceURLTestData { - value, err := getResourceUrlBasedOnCloud(map[string]string{"cloud": testData.cloud, "azureManagedPrometheusResourceURL": testData.azureManagedPrometheusResourceURL}) - if testData.isError { - if err == nil { - t.Errorf("Test: %v; Expected error but got success. testData: %v", testData.testName, testData) - } - } else { - if err == nil { - if testData.expectedValue != value { - t.Errorf("Test: %v; Expected value %v but got %v testData: %v", testData.testName, testData.expectedValue, value, testData) - } - } else { - t.Errorf("Test: %v; Expected success but got error: %v", testData.testName, err) - } - } - } -} - -type testTryAndGetAzureManagedPrometheusHttpRoundTripperData struct { +type testTryAndGetAzureManagedPrometheusHTTPRoundTripperData struct { testName string podIdentityProvider kedav1alpha1.PodIdentityProvider isError bool } -var testTryAndGetAzureManagedPrometheusHttpRoundTripperTestData = []testTryAndGetAzureManagedPrometheusHttpRoundTripperData{ +var testTryAndGetAzureManagedPrometheusHTTPRoundTripperTestData = []testTryAndGetAzureManagedPrometheusHTTPRoundTripperData{ {"test azure workload identity trigger metadata absent", kedav1alpha1.PodIdentityProviderAzureWorkload, true}, {"test azure pod identity trigger metadata absent", kedav1alpha1.PodIdentityProviderAzureWorkload, true}, {"test not azure identity", kedav1alpha1.PodIdentityProviderNone, false}, } -func TestTryAndGetAzureManagedPrometheusHttpRoundTripper(t *testing.T) { - for _, testData := range testTryAndGetAzureManagedPrometheusHttpRoundTripperTestData { - _, err := TryAndGetAzureManagedPrometheusHttpRoundTripper(kedav1alpha1.AuthPodIdentity{Provider: testData.podIdentityProvider}, nil) - if testData.isError { - if err == nil { - t.Errorf("Test: %v; Expected error but got success. testData: %v", testData.testName, testData) - } - } else if err != nil { - t.Errorf("Test: %v; Expected success but got error: %v", testData.testName, err) - } - } +func TestTryAndGetAzureManagedPrometheusHTTPRoundTripper(t *testing.T) { + TryAndGetAzureManagedPrometheusHTTPRoundTripper(kedav1alpha1.AuthPodIdentity{Provider: kedav1alpha1.PodIdentityProviderAzureWorkload}, map[string]string{"cloud": "AZUREGERMANCLOUD", "azureManagedPrometheusResourceURL": "asas"}) + // for _, testData := range testTryAndGetAzureManagedPrometheusHTTPRoundTripperTestData { + // _, err := TryAndGetAzureManagedPrometheusHTTPRoundTripper(kedav1alpha1.AuthPodIdentity{Provider: testData.podIdentityProvider}, nil) + // if testData.isError { + // if err == nil { + // t.Errorf("Test: %v; Expected error but got success. testData: %v", testData.testName, testData) + // } + // } else if err != nil { + // t.Errorf("Test: %v; Expected success but got error: %v", testData.testName, err) + // } + // } } diff --git a/pkg/scalers/prometheus_scaler.go b/pkg/scalers/prometheus_scaler.go index 79c485072d6..cd244dfdb77 100644 --- a/pkg/scalers/prometheus_scaler.go +++ b/pkg/scalers/prometheus_scaler.go @@ -105,7 +105,7 @@ func NewPrometheusScaler(config *ScalerConfig) (Scaler, error) { } } else { // could be the case of azure managed prometheus. Try and get the roundtripper. - transport, err := azure.TryAndGetAzureManagedPrometheusHttpRoundTripper(config.PodIdentity, config.TriggerMetadata) + transport, err := azure.TryAndGetAzureManagedPrometheusHTTPRoundTripper(config.PodIdentity, config.TriggerMetadata) if err != nil { logger.V(1).Error(err, "error while init Azure Managed Prometheus client http transport") diff --git a/pkg/scalers/prometheus_scaler_test.go b/pkg/scalers/prometheus_scaler_test.go index e875e250d65..f50e9759cd3 100644 --- a/pkg/scalers/prometheus_scaler_test.go +++ b/pkg/scalers/prometheus_scaler_test.go @@ -8,8 +8,9 @@ import ( "testing" "github.com/go-logr/logr" - kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" "github.com/stretchr/testify/assert" + + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" ) type parsePrometheusMetadataTestData struct { diff --git a/vendor/github.com/Azure/go-autorest/autorest/azure/environments.go b/vendor/github.com/Azure/go-autorest/autorest/azure/environments.go index b0a53769f26..15438bffc1d 100644 --- a/vendor/github.com/Azure/go-autorest/autorest/azure/environments.go +++ b/vendor/github.com/Azure/go-autorest/autorest/azure/environments.go @@ -42,19 +42,20 @@ var environments = map[string]Environment{ // ResourceIdentifier contains a set of Azure resource IDs. type ResourceIdentifier struct { - Graph string `json:"graph"` - KeyVault string `json:"keyVault"` - Datalake string `json:"datalake"` - Batch string `json:"batch"` - OperationalInsights string `json:"operationalInsights"` - OSSRDBMS string `json:"ossRDBMS"` - Storage string `json:"storage"` - Synapse string `json:"synapse"` - ServiceBus string `json:"serviceBus"` - SQLDatabase string `json:"sqlDatabase"` - CosmosDB string `json:"cosmosDB"` - ManagedHSM string `json:"managedHSM"` - MicrosoftGraph string `json:"microsoftGraph"` + Graph string `json:"graph"` + KeyVault string `json:"keyVault"` + Datalake string `json:"datalake"` + Batch string `json:"batch"` + OperationalInsights string `json:"operationalInsights"` + OSSRDBMS string `json:"ossRDBMS"` + Storage string `json:"storage"` + Synapse string `json:"synapse"` + ServiceBus string `json:"serviceBus"` + SQLDatabase string `json:"sqlDatabase"` + CosmosDB string `json:"cosmosDB"` + ManagedHSM string `json:"managedHSM"` + MicrosoftGraph string `json:"microsoftGraph"` + AzureManagedPrometheus string `json:"azureManagedPrometheus"` } // Environment represents a set of endpoints for each of Azure's Clouds. @@ -126,19 +127,20 @@ var ( SynapseEndpointSuffix: "dev.azuresynapse.net", DatalakeSuffix: "azuredatalakestore.net", ResourceIdentifiers: ResourceIdentifier{ - Graph: "https://graph.windows.net/", - KeyVault: "https://vault.azure.net", - Datalake: "https://datalake.azure.net/", - Batch: "https://batch.core.windows.net/", - OperationalInsights: "https://api.loganalytics.io", - OSSRDBMS: "https://ossrdbms-aad.database.windows.net", - Storage: "https://storage.azure.com/", - Synapse: "https://dev.azuresynapse.net", - ServiceBus: "https://servicebus.azure.net/", - SQLDatabase: "https://database.windows.net/", - CosmosDB: "https://cosmos.azure.com", - ManagedHSM: "https://managedhsm.azure.net", - MicrosoftGraph: "https://graph.microsoft.com/", + Graph: "https://graph.windows.net/", + KeyVault: "https://vault.azure.net", + Datalake: "https://datalake.azure.net/", + Batch: "https://batch.core.windows.net/", + OperationalInsights: "https://api.loganalytics.io", + OSSRDBMS: "https://ossrdbms-aad.database.windows.net", + Storage: "https://storage.azure.com/", + Synapse: "https://dev.azuresynapse.net", + ServiceBus: "https://servicebus.azure.net/", + SQLDatabase: "https://database.windows.net/", + CosmosDB: "https://cosmos.azure.com", + ManagedHSM: "https://managedhsm.azure.net", + MicrosoftGraph: "https://graph.microsoft.com/", + AzureManagedPrometheus: "https://prometheus.monitor.azure.com/.default", }, } @@ -175,19 +177,20 @@ var ( SynapseEndpointSuffix: "dev.azuresynapse.usgovcloudapi.net", DatalakeSuffix: NotAvailable, ResourceIdentifiers: ResourceIdentifier{ - Graph: "https://graph.windows.net/", - KeyVault: "https://vault.usgovcloudapi.net", - Datalake: NotAvailable, - Batch: "https://batch.core.usgovcloudapi.net/", - OperationalInsights: "https://api.loganalytics.us", - OSSRDBMS: "https://ossrdbms-aad.database.usgovcloudapi.net", - Storage: "https://storage.azure.com/", - Synapse: "https://dev.azuresynapse.usgovcloudapi.net", - ServiceBus: "https://servicebus.azure.net/", - SQLDatabase: "https://database.usgovcloudapi.net/", - CosmosDB: "https://cosmos.azure.com", - ManagedHSM: NotAvailable, - MicrosoftGraph: "https://graph.microsoft.us/", + Graph: "https://graph.windows.net/", + KeyVault: "https://vault.usgovcloudapi.net", + Datalake: NotAvailable, + Batch: "https://batch.core.usgovcloudapi.net/", + OperationalInsights: "https://api.loganalytics.us", + OSSRDBMS: "https://ossrdbms-aad.database.usgovcloudapi.net", + Storage: "https://storage.azure.com/", + Synapse: "https://dev.azuresynapse.usgovcloudapi.net", + ServiceBus: "https://servicebus.azure.net/", + SQLDatabase: "https://database.usgovcloudapi.net/", + CosmosDB: "https://cosmos.azure.com", + ManagedHSM: NotAvailable, + MicrosoftGraph: "https://graph.microsoft.us/", + AzureManagedPrometheus: "https://prometheus.monitor.usgovcloudapi.net/.default", }, } @@ -224,19 +227,20 @@ var ( SynapseEndpointSuffix: "dev.azuresynapse.azure.cn", DatalakeSuffix: NotAvailable, ResourceIdentifiers: ResourceIdentifier{ - Graph: "https://graph.chinacloudapi.cn/", - KeyVault: "https://vault.azure.cn", - Datalake: NotAvailable, - Batch: "https://batch.chinacloudapi.cn/", - OperationalInsights: NotAvailable, - OSSRDBMS: "https://ossrdbms-aad.database.chinacloudapi.cn", - Storage: "https://storage.azure.com/", - Synapse: "https://dev.azuresynapse.net", - ServiceBus: "https://servicebus.azure.net/", - SQLDatabase: "https://database.chinacloudapi.cn/", - CosmosDB: "https://cosmos.azure.com", - ManagedHSM: NotAvailable, - MicrosoftGraph: "https://microsoftgraph.chinacloudapi.cn", + Graph: "https://graph.chinacloudapi.cn/", + KeyVault: "https://vault.azure.cn", + Datalake: NotAvailable, + Batch: "https://batch.chinacloudapi.cn/", + OperationalInsights: NotAvailable, + OSSRDBMS: "https://ossrdbms-aad.database.chinacloudapi.cn", + Storage: "https://storage.azure.com/", + Synapse: "https://dev.azuresynapse.net", + ServiceBus: "https://servicebus.azure.net/", + SQLDatabase: "https://database.chinacloudapi.cn/", + CosmosDB: "https://cosmos.azure.com", + ManagedHSM: NotAvailable, + MicrosoftGraph: "https://microsoftgraph.chinacloudapi.cn", + AzureManagedPrometheus: "https://prometheus.monitor.chinacloudapp.cn/.default", }, } @@ -273,19 +277,20 @@ var ( SynapseEndpointSuffix: NotAvailable, DatalakeSuffix: NotAvailable, ResourceIdentifiers: ResourceIdentifier{ - Graph: "https://graph.cloudapi.de/", - KeyVault: "https://vault.microsoftazure.de", - Datalake: NotAvailable, - Batch: "https://batch.cloudapi.de/", - OperationalInsights: NotAvailable, - OSSRDBMS: "https://ossrdbms-aad.database.cloudapi.de", - Storage: "https://storage.azure.com/", - Synapse: NotAvailable, - ServiceBus: "https://servicebus.azure.net/", - SQLDatabase: "https://database.cloudapi.de/", - CosmosDB: "https://cosmos.azure.com", - ManagedHSM: NotAvailable, - MicrosoftGraph: NotAvailable, + Graph: "https://graph.cloudapi.de/", + KeyVault: "https://vault.microsoftazure.de", + Datalake: NotAvailable, + Batch: "https://batch.cloudapi.de/", + OperationalInsights: NotAvailable, + OSSRDBMS: "https://ossrdbms-aad.database.cloudapi.de", + Storage: "https://storage.azure.com/", + Synapse: NotAvailable, + ServiceBus: "https://servicebus.azure.net/", + SQLDatabase: "https://database.cloudapi.de/", + CosmosDB: "https://cosmos.azure.com", + ManagedHSM: NotAvailable, + MicrosoftGraph: NotAvailable, + AzureManagedPrometheus: NotAvailable, }, } ) From 7480787a50f205c8cb0fa1e8b5adb34405f442c4 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Thu, 23 Feb 2023 13:00:27 -0800 Subject: [PATCH 03/27] fix statci check failure Signed-off-by: Raghav Gupta --- ...e_managed_prometheus_http_round_tripper.go | 9 ++++---- ...aged_prometheus_http_round_tripper_test.go | 21 +++++++++---------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go index 74af4d1b3a4..cfd5eade447 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go @@ -12,7 +12,7 @@ import ( "github.com/kedacore/keda/v2/pkg/util" ) -type azureManagedPrometheusHttpRoundTripper struct { +type azureManagedPrometheusHTTPRoundTripper struct { chainedCredential *azidentity.ChainedTokenCredential next http.RoundTripper resourceURL string @@ -22,7 +22,6 @@ type azureManagedPrometheusHttpRoundTripper struct { // If the pod identity represents azure auth, it creates a round tripper and returns that. Returns error if fails to create one. // If its not azure auth, then this becomes a no-op. Neither returns round tripper nor error. func TryAndGetAzureManagedPrometheusHTTPRoundTripper(podIdentity kedav1alpha1.AuthPodIdentity, triggerMetadata map[string]string) (http.RoundTripper, error) { - switch podIdentity.Provider { case kedav1alpha1.PodIdentityProviderAzureWorkload, kedav1alpha1.PodIdentityProviderAzure: @@ -37,7 +36,7 @@ func TryAndGetAzureManagedPrometheusHTTPRoundTripper(podIdentity kedav1alpha1.Au azureManagedPrometheusResourceURLProvider := func(env az.Environment) (string, error) { if env.ResourceIdentifiers.AzureManagedPrometheus == az.NotAvailable { - return "", fmt.Errorf("Azure Managed Prometheus is not avaiable in cloud %s", env.Name) + return "", fmt.Errorf("Azure Managed Prometheus is not available in cloud %s", env.Name) } return env.ResourceIdentifiers.AzureManagedPrometheus, nil } @@ -48,7 +47,7 @@ func TryAndGetAzureManagedPrometheusHTTPRoundTripper(podIdentity kedav1alpha1.Au } transport := util.CreateHTTPTransport(false) - rt := &azureManagedPrometheusHttpRoundTripper{ + rt := &azureManagedPrometheusHTTPRoundTripper{ next: transport, chainedCredential: chainedCred, resourceURL: resourceURLBasedOnCloud, @@ -61,7 +60,7 @@ func TryAndGetAzureManagedPrometheusHTTPRoundTripper(podIdentity kedav1alpha1.Au } // Sets Auhtorization header for requests -func (rt *azureManagedPrometheusHttpRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { +func (rt *azureManagedPrometheusHTTPRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { token, err := rt.chainedCredential.GetToken(req.Context(), policy.TokenRequestOptions{Scopes: []string{rt.resourceURL}}) if err != nil { return nil, err diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go index 8d60a288d87..87380cdc3fb 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go @@ -19,15 +19,14 @@ var testTryAndGetAzureManagedPrometheusHTTPRoundTripperTestData = []testTryAndGe } func TestTryAndGetAzureManagedPrometheusHTTPRoundTripper(t *testing.T) { - TryAndGetAzureManagedPrometheusHTTPRoundTripper(kedav1alpha1.AuthPodIdentity{Provider: kedav1alpha1.PodIdentityProviderAzureWorkload}, map[string]string{"cloud": "AZUREGERMANCLOUD", "azureManagedPrometheusResourceURL": "asas"}) - // for _, testData := range testTryAndGetAzureManagedPrometheusHTTPRoundTripperTestData { - // _, err := TryAndGetAzureManagedPrometheusHTTPRoundTripper(kedav1alpha1.AuthPodIdentity{Provider: testData.podIdentityProvider}, nil) - // if testData.isError { - // if err == nil { - // t.Errorf("Test: %v; Expected error but got success. testData: %v", testData.testName, testData) - // } - // } else if err != nil { - // t.Errorf("Test: %v; Expected success but got error: %v", testData.testName, err) - // } - // } + for _, testData := range testTryAndGetAzureManagedPrometheusHTTPRoundTripperTestData { + _, err := TryAndGetAzureManagedPrometheusHTTPRoundTripper(kedav1alpha1.AuthPodIdentity{Provider: testData.podIdentityProvider}, nil) + if testData.isError { + if err == nil { + t.Errorf("Test: %v; Expected error but got success. testData: %v", testData.testName, testData) + } + } else if err != nil { + t.Errorf("Test: %v; Expected success but got error: %v", testData.testName, err) + } + } } From b822e5ba6ef67ded533a915a8a0a1ad9cb561a94 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Thu, 23 Feb 2023 19:59:18 -0800 Subject: [PATCH 04/27] fix static check Signed-off-by: Raghav Gupta --- .../azure/azure_managed_prometheus_http_round_tripper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go index cfd5eade447..72a6b18edc7 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go @@ -36,7 +36,7 @@ func TryAndGetAzureManagedPrometheusHTTPRoundTripper(podIdentity kedav1alpha1.Au azureManagedPrometheusResourceURLProvider := func(env az.Environment) (string, error) { if env.ResourceIdentifiers.AzureManagedPrometheus == az.NotAvailable { - return "", fmt.Errorf("Azure Managed Prometheus is not available in cloud %s", env.Name) + return "", fmt.Errorf("azure managed prometheus is not available in cloud %s", env.Name) } return env.ResourceIdentifiers.AzureManagedPrometheus, nil } From 0a8aad47f6b69ee351fa934b5b2adde3f1d47667 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Mon, 27 Feb 2023 12:33:17 -0800 Subject: [PATCH 05/27] add e2e tests Signed-off-by: Raghav Gupta --- ...e_managed_prometheus_http_round_tripper.go | 13 +- pkg/scalers/prometheus_scaler.go | 1 + tests/.env | 1 + .../azure_managed_prometheus_test.go | 331 ++++++++++++++++++ .../autorest/azure/environments.go | 135 ++++--- 5 files changed, 409 insertions(+), 72 deletions(-) create mode 100644 tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go index 72a6b18edc7..67545d547fb 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go @@ -3,6 +3,7 @@ package azure import ( "fmt" "net/http" + "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" @@ -12,6 +13,12 @@ import ( "github.com/kedacore/keda/v2/pkg/util" ) +var azureManagedPrometheusResourceURLInCloud = map[string]string{ + "AZUREPUBLICCLOUD": "https://prometheus.monitor.azure.com/.default", + "AZUREUSGOVERNMENTCLOUD": "https://prometheus.monitor.usgovcloudapi.net/.default", + "AZURECHINACLOUD": "https://prometheus.monitor.chinacloudapp.cn/.default", +} + type azureManagedPrometheusHTTPRoundTripper struct { chainedCredential *azidentity.ChainedTokenCredential next http.RoundTripper @@ -35,10 +42,11 @@ func TryAndGetAzureManagedPrometheusHTTPRoundTripper(podIdentity kedav1alpha1.Au } azureManagedPrometheusResourceURLProvider := func(env az.Environment) (string, error) { - if env.ResourceIdentifiers.AzureManagedPrometheus == az.NotAvailable { + if resource, ok := azureManagedPrometheusResourceURLInCloud[strings.ToUpper(env.Name)]; ok { + return resource, nil + } else { return "", fmt.Errorf("azure managed prometheus is not available in cloud %s", env.Name) } - return env.ResourceIdentifiers.AzureManagedPrometheus, nil } resourceURLBasedOnCloud, err := ParseEnvironmentProperty(triggerMetadata, "azureManagedPrometheusResourceURL", azureManagedPrometheusResourceURLProvider) @@ -62,6 +70,7 @@ func TryAndGetAzureManagedPrometheusHTTPRoundTripper(podIdentity kedav1alpha1.Au // Sets Auhtorization header for requests func (rt *azureManagedPrometheusHTTPRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { token, err := rt.chainedCredential.GetToken(req.Context(), policy.TokenRequestOptions{Scopes: []string{rt.resourceURL}}) + if err != nil { return nil, err } diff --git a/pkg/scalers/prometheus_scaler.go b/pkg/scalers/prometheus_scaler.go index cd244dfdb77..5a21f83ef42 100644 --- a/pkg/scalers/prometheus_scaler.go +++ b/pkg/scalers/prometheus_scaler.go @@ -105,6 +105,7 @@ func NewPrometheusScaler(config *ScalerConfig) (Scaler, error) { } } else { // could be the case of azure managed prometheus. Try and get the roundtripper. + // If its not the case of azure managed prometheus, we will get both transport and err as nil and proceed assuming no auth. transport, err := azure.TryAndGetAzureManagedPrometheusHTTPRoundTripper(config.PodIdentity, config.TriggerMetadata) if err != nil { diff --git a/tests/.env b/tests/.env index 95bdbe5b268..1b9e5e0e054 100644 --- a/tests/.env +++ b/tests/.env @@ -16,6 +16,7 @@ AZURE_DEVOPS_PROJECT= TF_AZURE_EVENTHBUS_MANAGEMENT_CONNECTION_STRING= TF_AZURE_KEYVAULT_URI= TF_AZURE_LOG_ANALYTICS_WORKSPACE_ID= +TF_AZURE_MANAGED_PROMETHEUS_QUERY_ENDPOINT= TF_AZURE_RESOURCE_GROUP= TF_AZURE_SERVICE_BUS_CONNECTION_STRING= TF_AZURE_SP_APP_ID= diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go new file mode 100644 index 00000000000..0821353b586 --- /dev/null +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go @@ -0,0 +1,331 @@ +//go:build e2e +// +build e2e + +package azure_managed_prometheus_test + +import ( + "fmt" + "os" + "testing" + + "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" + "k8s.io/client-go/kubernetes" + + . "github.com/kedacore/keda/v2/tests/helper" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +const ( + testName = "azure-managed-prometheus-test" +) + +var ( + testNamespace = fmt.Sprintf("%s-ns", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + monitoredAppName = fmt.Sprintf("%s-monitored-app", testName) + publishDeploymentName = fmt.Sprintf("%s-publish", testName) + scaledObjectName = fmt.Sprintf("%s-so", testName) + prometheusQueryEndpoint = os.Getenv("TF_AZURE_MANAGED_PROMETHEUS_QUERY_ENDPOINT") + minReplicaCount = 0 + maxReplicaCount = 2 +) + +type templateData struct { + TestNamespace string + DeploymentName string + MonitoredAppName string + PublishDeploymentName string + ScaledObjectName string + PrometheusQueryEndpoint string + MinReplicaCount int + MaxReplicaCount int +} + +const ( + azureManagedPrometheusConfigMapTemplate = `apiVersion: v1 +kind: ConfigMap +metadata: + name: ama-metrics-prometheus-config + namespace: kube-system +data: + prometheus-config: |- + global: + evaluation_interval: 1m + scrape_interval: 1m + scrape_timeout: 10s + scrape_configs: + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: kubernetes_namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: kubernetes_pod_name +--- +` + + deploymentTemplate = `apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: test-app + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} +spec: + replicas: 0 + selector: + matchLabels: + app: test-app + template: + metadata: + labels: + app: test-app + type: keda-testing + spec: + containers: + - name: prom-test-app + image: ghcr.io/kedacore/tests-prometheus:latest + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault +--- +` + + monitoredAppDeploymentTemplate = `apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: {{.MonitoredAppName}} + name: {{.MonitoredAppName}} + namespace: {{.TestNamespace}} +spec: + replicas: 1 + selector: + matchLabels: + app: {{.MonitoredAppName}} + template: + metadata: + labels: + app: {{.MonitoredAppName}} + type: {{.MonitoredAppName}} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + spec: + containers: + - name: prom-test-app + image: ghcr.io/kedacore/tests-prometheus:latest + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault +--- +` + + monitoredAppServiceTemplate = `apiVersion: v1 +kind: Service +metadata: + labels: + app: {{.MonitoredAppName}} + name: {{.MonitoredAppName}} + namespace: {{.TestNamespace}} + annotations: + prometheus.io/scrape: "true" +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 8080 + selector: + type: {{.MonitoredAppName}} +` + + azureManagedPrometheusWorkloadIdentityAuthTemplate = `apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: azure-managed-prometheus-trigger-auth-wi + namespace: {{.TestNamespace}} +spec: + podIdentity: + provider: azure-workload +` + + scaledObjectTemplate = `apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.ScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + minReplicaCount: {{.MinReplicaCount}} + maxReplicaCount: {{.MaxReplicaCount}} + pollingInterval: 3 + cooldownPeriod: 1 + triggers: + - type: prometheus + metadata: + serverAddress: {{.PrometheusQueryEndpoint}} + metricName: http_requests_total + threshold: '20' + activationThreshold: '20' + query: sum(rate(http_requests_total{app="{{.MonitoredAppName}}"}[2m])) + authenticationRef: + name: azure-managed-prometheus-trigger-auth-wi +` + + generateLowLevelLoadJobTemplate = `apiVersion: batch/v1 +kind: Job +metadata: + name: generate-low-level-requests-job + namespace: {{.TestNamespace}} +spec: + template: + spec: + containers: + - image: quay.io/zroubalik/hey + name: test + command: ["/bin/sh"] + args: ["-c", "for i in $(seq 1 60);do echo $i;/hey -c 5 -n 30 http://{{.MonitoredAppName}}.{{.TestNamespace}}.svc;sleep 1;done"] + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + restartPolicy: Never + activeDeadlineSeconds: 100 + backoffLimit: 2 + ` + + generateLoadJobTemplate = `apiVersion: batch/v1 +kind: Job +metadata: + name: generate-requests-job + namespace: {{.TestNamespace}} +spec: + template: + spec: + containers: + - image: quay.io/zroubalik/hey + name: test + command: ["/bin/sh"] + args: ["-c", "for i in $(seq 1 60);do echo $i;/hey -c 5 -n 80 http://{{.MonitoredAppName}}.{{.TestNamespace}}.svc;sleep 1;done"] + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + restartPolicy: Never + activeDeadlineSeconds: 100 + backoffLimit: 2 +` +) + +// TestPrometheusScaler creates deployments - there are two deployments - both using the same image but one deployment +// is directly tied to the KEDA HPA while the other is isolated that can be used for metrics +// even when the KEDA deployment is at zero - the service points to both deployments +func TestAzureManagedPrometheusScaler(t *testing.T) { + kc := GetKubernetesClient(t) + + // Create kubernetes resources for testing + CreateNamespace(t, kc, testNamespace) + data, templates := getTemplateData() + KubectlApplyMultipleWithTemplate(t, data, templates) + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, monitoredAppName, testNamespace, 1, 60, 3), + "replica count should be %d after 3 minutes", minReplicaCount) + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3), + "replica count should be %d after 3 minutes", minReplicaCount) + + testActivation(t, kc, data) + testScaleOut(t, kc, data) + testScaleIn(t, kc) + + // cleanup + KubectlDeleteMultipleWithTemplate(t, data, templates) + DeleteNamespace(t, kc, testNamespace) +} + +func testActivation(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing activation ---") + KubectlApplyWithTemplate(t, data, "generateLowLevelLoadJobTemplate", generateLowLevelLoadJobTemplate) + + AssertReplicaCountNotChangeDuringTimePeriod(t, kc, deploymentName, testNamespace, minReplicaCount, 60) +} + +func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) { + t.Log("--- testing scale out ---") + KubectlApplyWithTemplate(t, data, "generateLoadJobTemplate", generateLoadJobTemplate) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 120, 5), + "replica count should be %d after 5 minutes", maxReplicaCount) +} + +func testScaleIn(t *testing.T, kc *kubernetes.Clientset) { + t.Log("--- testing scale in ---") + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 120, 5), + "replica count should be %d after 5 minutes", minReplicaCount) +} + +func getTemplateData() (templateData, []Template) { + return templateData{ + TestNamespace: testNamespace, + DeploymentName: deploymentName, + PublishDeploymentName: publishDeploymentName, + ScaledObjectName: scaledObjectName, + MonitoredAppName: monitoredAppName, + PrometheusQueryEndpoint: prometheusQueryEndpoint, + MinReplicaCount: minReplicaCount, + MaxReplicaCount: maxReplicaCount, + }, []Template{ + {Name: "azureManagedPrometheusConfigMapTemplate", Config: azureManagedPrometheusConfigMapTemplate}, + {Name: "monitoredAppDeploymentTemplate", Config: monitoredAppDeploymentTemplate}, + {Name: "deploymentTemplate", Config: deploymentTemplate}, + {Name: "monitoredAppServiceTemplate", Config: monitoredAppServiceTemplate}, + {Name: "azureManagedPrometheusWorkloadIdentityAuthTemplate", Config: azureManagedPrometheusWorkloadIdentityAuthTemplate}, + {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, + } +} diff --git a/vendor/github.com/Azure/go-autorest/autorest/azure/environments.go b/vendor/github.com/Azure/go-autorest/autorest/azure/environments.go index 15438bffc1d..b0a53769f26 100644 --- a/vendor/github.com/Azure/go-autorest/autorest/azure/environments.go +++ b/vendor/github.com/Azure/go-autorest/autorest/azure/environments.go @@ -42,20 +42,19 @@ var environments = map[string]Environment{ // ResourceIdentifier contains a set of Azure resource IDs. type ResourceIdentifier struct { - Graph string `json:"graph"` - KeyVault string `json:"keyVault"` - Datalake string `json:"datalake"` - Batch string `json:"batch"` - OperationalInsights string `json:"operationalInsights"` - OSSRDBMS string `json:"ossRDBMS"` - Storage string `json:"storage"` - Synapse string `json:"synapse"` - ServiceBus string `json:"serviceBus"` - SQLDatabase string `json:"sqlDatabase"` - CosmosDB string `json:"cosmosDB"` - ManagedHSM string `json:"managedHSM"` - MicrosoftGraph string `json:"microsoftGraph"` - AzureManagedPrometheus string `json:"azureManagedPrometheus"` + Graph string `json:"graph"` + KeyVault string `json:"keyVault"` + Datalake string `json:"datalake"` + Batch string `json:"batch"` + OperationalInsights string `json:"operationalInsights"` + OSSRDBMS string `json:"ossRDBMS"` + Storage string `json:"storage"` + Synapse string `json:"synapse"` + ServiceBus string `json:"serviceBus"` + SQLDatabase string `json:"sqlDatabase"` + CosmosDB string `json:"cosmosDB"` + ManagedHSM string `json:"managedHSM"` + MicrosoftGraph string `json:"microsoftGraph"` } // Environment represents a set of endpoints for each of Azure's Clouds. @@ -127,20 +126,19 @@ var ( SynapseEndpointSuffix: "dev.azuresynapse.net", DatalakeSuffix: "azuredatalakestore.net", ResourceIdentifiers: ResourceIdentifier{ - Graph: "https://graph.windows.net/", - KeyVault: "https://vault.azure.net", - Datalake: "https://datalake.azure.net/", - Batch: "https://batch.core.windows.net/", - OperationalInsights: "https://api.loganalytics.io", - OSSRDBMS: "https://ossrdbms-aad.database.windows.net", - Storage: "https://storage.azure.com/", - Synapse: "https://dev.azuresynapse.net", - ServiceBus: "https://servicebus.azure.net/", - SQLDatabase: "https://database.windows.net/", - CosmosDB: "https://cosmos.azure.com", - ManagedHSM: "https://managedhsm.azure.net", - MicrosoftGraph: "https://graph.microsoft.com/", - AzureManagedPrometheus: "https://prometheus.monitor.azure.com/.default", + Graph: "https://graph.windows.net/", + KeyVault: "https://vault.azure.net", + Datalake: "https://datalake.azure.net/", + Batch: "https://batch.core.windows.net/", + OperationalInsights: "https://api.loganalytics.io", + OSSRDBMS: "https://ossrdbms-aad.database.windows.net", + Storage: "https://storage.azure.com/", + Synapse: "https://dev.azuresynapse.net", + ServiceBus: "https://servicebus.azure.net/", + SQLDatabase: "https://database.windows.net/", + CosmosDB: "https://cosmos.azure.com", + ManagedHSM: "https://managedhsm.azure.net", + MicrosoftGraph: "https://graph.microsoft.com/", }, } @@ -177,20 +175,19 @@ var ( SynapseEndpointSuffix: "dev.azuresynapse.usgovcloudapi.net", DatalakeSuffix: NotAvailable, ResourceIdentifiers: ResourceIdentifier{ - Graph: "https://graph.windows.net/", - KeyVault: "https://vault.usgovcloudapi.net", - Datalake: NotAvailable, - Batch: "https://batch.core.usgovcloudapi.net/", - OperationalInsights: "https://api.loganalytics.us", - OSSRDBMS: "https://ossrdbms-aad.database.usgovcloudapi.net", - Storage: "https://storage.azure.com/", - Synapse: "https://dev.azuresynapse.usgovcloudapi.net", - ServiceBus: "https://servicebus.azure.net/", - SQLDatabase: "https://database.usgovcloudapi.net/", - CosmosDB: "https://cosmos.azure.com", - ManagedHSM: NotAvailable, - MicrosoftGraph: "https://graph.microsoft.us/", - AzureManagedPrometheus: "https://prometheus.monitor.usgovcloudapi.net/.default", + Graph: "https://graph.windows.net/", + KeyVault: "https://vault.usgovcloudapi.net", + Datalake: NotAvailable, + Batch: "https://batch.core.usgovcloudapi.net/", + OperationalInsights: "https://api.loganalytics.us", + OSSRDBMS: "https://ossrdbms-aad.database.usgovcloudapi.net", + Storage: "https://storage.azure.com/", + Synapse: "https://dev.azuresynapse.usgovcloudapi.net", + ServiceBus: "https://servicebus.azure.net/", + SQLDatabase: "https://database.usgovcloudapi.net/", + CosmosDB: "https://cosmos.azure.com", + ManagedHSM: NotAvailable, + MicrosoftGraph: "https://graph.microsoft.us/", }, } @@ -227,20 +224,19 @@ var ( SynapseEndpointSuffix: "dev.azuresynapse.azure.cn", DatalakeSuffix: NotAvailable, ResourceIdentifiers: ResourceIdentifier{ - Graph: "https://graph.chinacloudapi.cn/", - KeyVault: "https://vault.azure.cn", - Datalake: NotAvailable, - Batch: "https://batch.chinacloudapi.cn/", - OperationalInsights: NotAvailable, - OSSRDBMS: "https://ossrdbms-aad.database.chinacloudapi.cn", - Storage: "https://storage.azure.com/", - Synapse: "https://dev.azuresynapse.net", - ServiceBus: "https://servicebus.azure.net/", - SQLDatabase: "https://database.chinacloudapi.cn/", - CosmosDB: "https://cosmos.azure.com", - ManagedHSM: NotAvailable, - MicrosoftGraph: "https://microsoftgraph.chinacloudapi.cn", - AzureManagedPrometheus: "https://prometheus.monitor.chinacloudapp.cn/.default", + Graph: "https://graph.chinacloudapi.cn/", + KeyVault: "https://vault.azure.cn", + Datalake: NotAvailable, + Batch: "https://batch.chinacloudapi.cn/", + OperationalInsights: NotAvailable, + OSSRDBMS: "https://ossrdbms-aad.database.chinacloudapi.cn", + Storage: "https://storage.azure.com/", + Synapse: "https://dev.azuresynapse.net", + ServiceBus: "https://servicebus.azure.net/", + SQLDatabase: "https://database.chinacloudapi.cn/", + CosmosDB: "https://cosmos.azure.com", + ManagedHSM: NotAvailable, + MicrosoftGraph: "https://microsoftgraph.chinacloudapi.cn", }, } @@ -277,20 +273,19 @@ var ( SynapseEndpointSuffix: NotAvailable, DatalakeSuffix: NotAvailable, ResourceIdentifiers: ResourceIdentifier{ - Graph: "https://graph.cloudapi.de/", - KeyVault: "https://vault.microsoftazure.de", - Datalake: NotAvailable, - Batch: "https://batch.cloudapi.de/", - OperationalInsights: NotAvailable, - OSSRDBMS: "https://ossrdbms-aad.database.cloudapi.de", - Storage: "https://storage.azure.com/", - Synapse: NotAvailable, - ServiceBus: "https://servicebus.azure.net/", - SQLDatabase: "https://database.cloudapi.de/", - CosmosDB: "https://cosmos.azure.com", - ManagedHSM: NotAvailable, - MicrosoftGraph: NotAvailable, - AzureManagedPrometheus: NotAvailable, + Graph: "https://graph.cloudapi.de/", + KeyVault: "https://vault.microsoftazure.de", + Datalake: NotAvailable, + Batch: "https://batch.cloudapi.de/", + OperationalInsights: NotAvailable, + OSSRDBMS: "https://ossrdbms-aad.database.cloudapi.de", + Storage: "https://storage.azure.com/", + Synapse: NotAvailable, + ServiceBus: "https://servicebus.azure.net/", + SQLDatabase: "https://database.cloudapi.de/", + CosmosDB: "https://cosmos.azure.com", + ManagedHSM: NotAvailable, + MicrosoftGraph: NotAvailable, }, } ) From b24e3ced55911dc9dc950f1caf0c51f9202f9605 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Mon, 27 Feb 2023 13:17:43 -0800 Subject: [PATCH 06/27] add changelog Signed-off-by: Raghav Gupta --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb6ba017095..024c2cbc67d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ Here is an overview of all **stable** additions: - **General**: Introduce new ArangoDB Scaler ([#4000](https://github.com/kedacore/keda/issues/4000)) - **Prometheus Metrics**: Introduce scaler activity in Prometheus metrics ([#4114](https://github.com/kedacore/keda/issues/4114)) - **Prometheus Metrics**: Introduce scaler latency in Prometheus metrics ([#4037](https://github.com/kedacore/keda/issues/4037)) +- **Prometheus Scaler**: Extend Prometheus Scaler to support Azure managed service for Prometheus ([#4153](https://github.com/kedacore/keda/issues/4153)) Here is an overview of all new **experimental** features: From ec2557c2fccf69095935737931546a511c04713f Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Mon, 27 Feb 2023 14:33:10 -0800 Subject: [PATCH 07/27] fixing static check and adding env var check Signed-off-by: Raghav Gupta --- .../azure/azure_managed_prometheus_http_round_tripper.go | 4 ++-- .../azure_managed_prometheus/azure_managed_prometheus_test.go | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go index 67545d547fb..0a0c537979c 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go @@ -44,9 +44,9 @@ func TryAndGetAzureManagedPrometheusHTTPRoundTripper(podIdentity kedav1alpha1.Au azureManagedPrometheusResourceURLProvider := func(env az.Environment) (string, error) { if resource, ok := azureManagedPrometheusResourceURLInCloud[strings.ToUpper(env.Name)]; ok { return resource, nil - } else { - return "", fmt.Errorf("azure managed prometheus is not available in cloud %s", env.Name) } + + return "", fmt.Errorf("azure managed prometheus is not available in cloud %s", env.Name) } resourceURLBasedOnCloud, err := ParseEnvironmentProperty(triggerMetadata, "azureManagedPrometheusResourceURL", azureManagedPrometheusResourceURLProvider) diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go index 0821353b586..def9d44e84c 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go @@ -10,6 +10,7 @@ import ( "github.com/joho/godotenv" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "k8s.io/client-go/kubernetes" . "github.com/kedacore/keda/v2/tests/helper" @@ -269,6 +270,8 @@ spec: // is directly tied to the KEDA HPA while the other is isolated that can be used for metrics // even when the KEDA deployment is at zero - the service points to both deployments func TestAzureManagedPrometheusScaler(t *testing.T) { + require.NotEmpty(t, prometheusQueryEndpoint, "TF_AZURE_MANAGED_PROMETHEUS_QUERY_ENDPOINT env variable is required for azure managed prometheus tests") + kc := GetKubernetesClient(t) // Create kubernetes resources for testing From ad1d61fa21f075463faf9efd8a1254e6aef5bba6 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Mon, 27 Feb 2023 19:47:08 -0800 Subject: [PATCH 08/27] adding e2e for pod identity Signed-off-by: Raghav Gupta --- .../azure_managed_prometheus_test.go | 129 ++++++++++++------ 1 file changed, 88 insertions(+), 41 deletions(-) diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go index def9d44e84c..c96966333bd 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go @@ -5,7 +5,6 @@ package azure_managed_prometheus_test import ( "fmt" - "os" "testing" "github.com/joho/godotenv" @@ -20,16 +19,33 @@ import ( var _ = godotenv.Load("../../.env") const ( - testName = "azure-managed-prometheus-test" + testNameWorkloadIdentity = "azure-managed-prometheus-workload-identity-test" + testNamePodIdentity = "azure-managed-prometheus-pod-identity-test" ) +// Workload Identity test vars var ( - testNamespace = fmt.Sprintf("%s-ns", testName) - deploymentName = fmt.Sprintf("%s-deployment", testName) - monitoredAppName = fmt.Sprintf("%s-monitored-app", testName) - publishDeploymentName = fmt.Sprintf("%s-publish", testName) - scaledObjectName = fmt.Sprintf("%s-so", testName) - prometheusQueryEndpoint = os.Getenv("TF_AZURE_MANAGED_PROMETHEUS_QUERY_ENDPOINT") + testNamespaceWI = fmt.Sprintf("%s-ns", testNameWorkloadIdentity) + deploymentNameWI = fmt.Sprintf("%s-deployment", testNameWorkloadIdentity) + monitoredAppNameWI = fmt.Sprintf("%s-monitored-app", testNameWorkloadIdentity) + publishDeploymentNameWI = fmt.Sprintf("%s-publish", testNameWorkloadIdentity) + scaledObjectNameWI = fmt.Sprintf("%s-so", testNameWorkloadIdentity) + workloadIdentityProvider = "azure-workload" +) + +// Pod Identity test vars +var ( + testNamespacePod = fmt.Sprintf("%s-ns", testNamePodIdentity) + deploymentNamePod = fmt.Sprintf("%s-deployment", testNamePodIdentity) + monitoredAppNamePod = fmt.Sprintf("%s-monitored-app", testNamePodIdentity) + publishDeploymentNamePod = fmt.Sprintf("%s-publish", testNamePodIdentity) + scaledObjectNamePod = fmt.Sprintf("%s-so", testNamePodIdentity) + podIdentityProvider = "azure" +) + +// Common for pod and workload identity tests +var ( + prometheusQueryEndpoint = "https://raggupta-amw-eus-corp-2-1l1j.eastus.prometheus.monitor.azure.com" minReplicaCount = 0 maxReplicaCount = 2 ) @@ -40,6 +56,7 @@ type templateData struct { MonitoredAppName string PublishDeploymentName string ScaledObjectName string + PodIdentityProvider string PrometheusQueryEndpoint string MinReplicaCount int MaxReplicaCount int @@ -179,14 +196,14 @@ spec: type: {{.MonitoredAppName}} ` - azureManagedPrometheusWorkloadIdentityAuthTemplate = `apiVersion: keda.sh/v1alpha1 + azureManagedPrometheusAuthTemplate = `apiVersion: keda.sh/v1alpha1 kind: TriggerAuthentication metadata: - name: azure-managed-prometheus-trigger-auth-wi + name: azure-managed-prometheus-trigger-auth namespace: {{.TestNamespace}} spec: podIdentity: - provider: azure-workload + provider: {{.PodIdentityProvider}} ` scaledObjectTemplate = `apiVersion: keda.sh/v1alpha1 @@ -210,7 +227,7 @@ spec: activationThreshold: '20' query: sum(rate(http_requests_total{app="{{.MonitoredAppName}}"}[2m])) authenticationRef: - name: azure-managed-prometheus-trigger-auth-wi + name: azure-managed-prometheus-trigger-auth ` generateLowLevelLoadJobTemplate = `apiVersion: batch/v1 @@ -266,69 +283,99 @@ spec: ` ) -// TestPrometheusScaler creates deployments - there are two deployments - both using the same image but one deployment +// TestAzureManagedPrometheusScalerWithWorkloadIdentity creates deployments - there are two deployments - both using the same image but one deployment +// is directly tied to the KEDA HPA while the other is isolated that can be used for metrics +// even when the KEDA deployment is at zero - the service points to both deployments +func TestAzureManagedPrometheusScalerWithWorkloadIdentity(t *testing.T) { + testAzureManagedPrometheusScaler(t, getTemplateDataForWorkloadIdentityTest(), getTemplates()) +} + +// TestAzureManagedPrometheusScalerWithWorkloadIdentity creates deployments - there are two deployments - both using the same image but one deployment // is directly tied to the KEDA HPA while the other is isolated that can be used for metrics // even when the KEDA deployment is at zero - the service points to both deployments -func TestAzureManagedPrometheusScaler(t *testing.T) { +func TestAzureManagedPrometheusScalerWithPodIdentity(t *testing.T) { + testAzureManagedPrometheusScaler(t, getTemplateDataForPodIdentityTest(), getTemplates()) +} + +func testAzureManagedPrometheusScaler(t *testing.T, data templateData, templates []Template) { require.NotEmpty(t, prometheusQueryEndpoint, "TF_AZURE_MANAGED_PROMETHEUS_QUERY_ENDPOINT env variable is required for azure managed prometheus tests") kc := GetKubernetesClient(t) // Create kubernetes resources for testing - CreateNamespace(t, kc, testNamespace) - data, templates := getTemplateData() + CreateNamespace(t, kc, data.TestNamespace) + KubectlApplyMultipleWithTemplate(t, data, templates) - assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, monitoredAppName, testNamespace, 1, 60, 3), + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, data.MonitoredAppName, data.TestNamespace, 1, 60, 3), "replica count should be %d after 3 minutes", minReplicaCount) - assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3), + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, minReplicaCount, 60, 3), "replica count should be %d after 3 minutes", minReplicaCount) testActivation(t, kc, data) testScaleOut(t, kc, data) - testScaleIn(t, kc) + testScaleIn(t, kc, data) // cleanup KubectlDeleteMultipleWithTemplate(t, data, templates) - DeleteNamespace(t, kc, testNamespace) + DeleteNamespace(t, kc, data.TestNamespace) } func testActivation(t *testing.T, kc *kubernetes.Clientset, data templateData) { t.Log("--- testing activation ---") KubectlApplyWithTemplate(t, data, "generateLowLevelLoadJobTemplate", generateLowLevelLoadJobTemplate) - AssertReplicaCountNotChangeDuringTimePeriod(t, kc, deploymentName, testNamespace, minReplicaCount, 60) + AssertReplicaCountNotChangeDuringTimePeriod(t, kc, data.DeploymentName, data.TestNamespace, minReplicaCount, 60) } func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) { t.Log("--- testing scale out ---") KubectlApplyWithTemplate(t, data, "generateLoadJobTemplate", generateLoadJobTemplate) - assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 120, 5), + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, maxReplicaCount, 120, 5), "replica count should be %d after 5 minutes", maxReplicaCount) } -func testScaleIn(t *testing.T, kc *kubernetes.Clientset) { +func testScaleIn(t *testing.T, kc *kubernetes.Clientset, data templateData) { t.Log("--- testing scale in ---") - assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 120, 5), + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, minReplicaCount, 120, 5), "replica count should be %d after 5 minutes", minReplicaCount) } -func getTemplateData() (templateData, []Template) { +func getTemplates() []Template { + return []Template{ + {Name: "azureManagedPrometheusConfigMapTemplate", Config: azureManagedPrometheusConfigMapTemplate}, + {Name: "monitoredAppDeploymentTemplate", Config: monitoredAppDeploymentTemplate}, + {Name: "deploymentTemplate", Config: deploymentTemplate}, + {Name: "monitoredAppServiceTemplate", Config: monitoredAppServiceTemplate}, + {Name: "azureManagedPrometheusAuthTemplate", Config: azureManagedPrometheusAuthTemplate}, + {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, + } +} + +func getTemplateDataForPodIdentityTest() templateData { + return templateData{ + TestNamespace: testNamespacePod, + DeploymentName: deploymentNamePod, + PublishDeploymentName: publishDeploymentNamePod, + ScaledObjectName: scaledObjectNamePod, + MonitoredAppName: monitoredAppNamePod, + PodIdentityProvider: podIdentityProvider, + PrometheusQueryEndpoint: prometheusQueryEndpoint, + MinReplicaCount: minReplicaCount, + MaxReplicaCount: maxReplicaCount, + } +} + +func getTemplateDataForWorkloadIdentityTest() templateData { return templateData{ - TestNamespace: testNamespace, - DeploymentName: deploymentName, - PublishDeploymentName: publishDeploymentName, - ScaledObjectName: scaledObjectName, - MonitoredAppName: monitoredAppName, - PrometheusQueryEndpoint: prometheusQueryEndpoint, - MinReplicaCount: minReplicaCount, - MaxReplicaCount: maxReplicaCount, - }, []Template{ - {Name: "azureManagedPrometheusConfigMapTemplate", Config: azureManagedPrometheusConfigMapTemplate}, - {Name: "monitoredAppDeploymentTemplate", Config: monitoredAppDeploymentTemplate}, - {Name: "deploymentTemplate", Config: deploymentTemplate}, - {Name: "monitoredAppServiceTemplate", Config: monitoredAppServiceTemplate}, - {Name: "azureManagedPrometheusWorkloadIdentityAuthTemplate", Config: azureManagedPrometheusWorkloadIdentityAuthTemplate}, - {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, - } + TestNamespace: testNamespaceWI, + DeploymentName: deploymentNameWI, + PublishDeploymentName: publishDeploymentNameWI, + ScaledObjectName: scaledObjectNameWI, + MonitoredAppName: monitoredAppNameWI, + PodIdentityProvider: workloadIdentityProvider, + PrometheusQueryEndpoint: prometheusQueryEndpoint, + MinReplicaCount: minReplicaCount, + MaxReplicaCount: maxReplicaCount, + } } From ac10cd44804436a93af3e0d7e430fff67fecc314 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Mon, 27 Feb 2023 19:49:40 -0800 Subject: [PATCH 09/27] fix query endpoint Signed-off-by: Raghav Gupta --- .../azure_managed_prometheus/azure_managed_prometheus_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go index c96966333bd..ac3fae35b21 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go @@ -5,6 +5,7 @@ package azure_managed_prometheus_test import ( "fmt" + "os" "testing" "github.com/joho/godotenv" @@ -45,7 +46,7 @@ var ( // Common for pod and workload identity tests var ( - prometheusQueryEndpoint = "https://raggupta-amw-eus-corp-2-1l1j.eastus.prometheus.monitor.azure.com" + prometheusQueryEndpoint = os.Getenv("TF_AZURE_MANAGED_PROMETHEUS_QUERY_ENDPOINT") minReplicaCount = 0 maxReplicaCount = 2 ) From 6829008066278a2d67b5370e6168e243979867c7 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Mon, 27 Feb 2023 19:53:43 -0800 Subject: [PATCH 10/27] fix audience Signed-off-by: Raghav Gupta --- .../azure/azure_managed_prometheus_http_round_tripper.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go index 0a0c537979c..b3bd400d798 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper.go @@ -15,8 +15,8 @@ import ( var azureManagedPrometheusResourceURLInCloud = map[string]string{ "AZUREPUBLICCLOUD": "https://prometheus.monitor.azure.com/.default", - "AZUREUSGOVERNMENTCLOUD": "https://prometheus.monitor.usgovcloudapi.net/.default", - "AZURECHINACLOUD": "https://prometheus.monitor.chinacloudapp.cn/.default", + "AZUREUSGOVERNMENTCLOUD": "https://prometheus.monitor.azure.us/.default", + "AZURECHINACLOUD": "https://prometheus.monitor.azure.cn/.default", } type azureManagedPrometheusHTTPRoundTripper struct { From e22c02d29c2d9636f1dc0eb2f4a5188815d8b101 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Wed, 1 Mar 2023 13:10:54 -0800 Subject: [PATCH 11/27] split wi and pod identity tests Signed-off-by: Raghav Gupta --- ...aged_prometheus_http_round_tripper_test.go | 49 ++++++++++++- ...anaged_prometheus_aad_pod_identity_test.go | 51 +++++++++++++ .../azure_managed_prometheus_aad_wi_test.go | 51 +++++++++++++ ...> azure_managed_prometheus_test_helper.go} | 73 +------------------ 4 files changed, 152 insertions(+), 72 deletions(-) create mode 100644 tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_pod_identity_test.go create mode 100644 tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_wi_test.go rename tests/scalers/prometheus/azure_managed_prometheus/{azure_managed_prometheus_test.go => azure_managed_prometheus_test_helper.go} (73%) diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go index 87380cdc3fb..9cf3801607f 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go @@ -12,13 +12,38 @@ type testTryAndGetAzureManagedPrometheusHTTPRoundTripperData struct { isError bool } +type testAzureManagedPrometheusResourceURLData struct { + testName string + podIdentityProvider kedav1alpha1.PodIdentityProvider + metadata map[string]string + resourceURL string + isError bool +} + var testTryAndGetAzureManagedPrometheusHTTPRoundTripperTestData = []testTryAndGetAzureManagedPrometheusHTTPRoundTripperData{ - {"test azure workload identity trigger metadata absent", kedav1alpha1.PodIdentityProviderAzureWorkload, true}, + {"test azure WI trigger metadata absent", kedav1alpha1.PodIdentityProviderAzureWorkload, true}, {"test azure pod identity trigger metadata absent", kedav1alpha1.PodIdentityProviderAzureWorkload, true}, {"test not azure identity", kedav1alpha1.PodIdentityProviderNone, false}, } -func TestTryAndGetAzureManagedPrometheusHTTPRoundTripper(t *testing.T) { +var testAzureManagedPrometheusResourceURLTestData = []testAzureManagedPrometheusResourceURLData{ + // workload identity + + // with default cloud + {"test default azure cloud with WI", kedav1alpha1.PodIdentityProviderAzureWorkload, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up"}, "https://prometheus.monitor.azure.com/.default", false}, + // with public cloud + {"test azure public cloud with WI", kedav1alpha1.PodIdentityProviderAzureWorkload, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "AZUREPUBLICCLOUD"}, "https://prometheus.monitor.azure.com/.default", false}, + // with china cloud + {"test azure china cloud with WI", kedav1alpha1.PodIdentityProviderAzureWorkload, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "AZURECHINACLOUD"}, "https://prometheus.monitor.azure.cn/.default", false}, + // with US GOV cloud + {"test azure US GOV cloud with WI", kedav1alpha1.PodIdentityProviderAzureWorkload, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "AZUREUSGOVERNMENTCLOUD"}, "https://prometheus.monitor.azure.us/.default", false}, + // with private cloud success + {"test azure private cloud with WI", kedav1alpha1.PodIdentityProviderAzureWorkload, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "PRIVATE", "azureManagedPrometheusResourceURL": "blah-blah-resourceURL"}, "https://prometheus.monitor.azure.com/.default", false}, + // with private cloud failure + {"test default azure cloud with WI", kedav1alpha1.PodIdentityProviderAzureWorkload, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "PRIVATE"}, "", true}, +} + +func TestTryAndGetAzureManagedPrometheusHTTPRoundTripperForTriggerMetadataAbsent(t *testing.T) { for _, testData := range testTryAndGetAzureManagedPrometheusHTTPRoundTripperTestData { _, err := TryAndGetAzureManagedPrometheusHTTPRoundTripper(kedav1alpha1.AuthPodIdentity{Provider: testData.podIdentityProvider}, nil) if testData.isError { @@ -30,3 +55,23 @@ func TestTryAndGetAzureManagedPrometheusHTTPRoundTripper(t *testing.T) { } } } + +func TestTryAndGetAzureManagedPrometheusHTTPRoundTripperWithTriggerForResourceURL(t *testing.T) { + for _, testData := range testAzureManagedPrometheusResourceURLTestData { + rt, err := TryAndGetAzureManagedPrometheusHTTPRoundTripper(kedav1alpha1.AuthPodIdentity{Provider: testData.podIdentityProvider}, testData.metadata) + if testData.isError { + if err == nil { + t.Errorf("Test: %v; Expected error but got success. testData: %v", testData.testName, testData) + } + } else { + if err != nil { + t.Errorf("Test: %v; Expected success but got error: %v", testData.testName, err) + + azureRT := rt.(*azureManagedPrometheusHTTPRoundTripper) + if azureRT == nil || azureRT.resourceURL != testData.resourceURL { + t.Errorf("Test: %v; Expected success but got error: %v", testData.testName, err) + } + } + } + } +} diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_pod_identity_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_pod_identity_test.go new file mode 100644 index 00000000000..0a91de9a6bf --- /dev/null +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_pod_identity_test.go @@ -0,0 +1,51 @@ +//go:build e2e +// +build e2e + +package azure_managed_prometheus + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/joho/godotenv" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +const ( + testNamePodIdentity = "azure-managed-prometheus-pod-identity-test" +) + +// Pod Identity test vars +var ( + randomNumberPod = rand.Int() + testNamespacePod = fmt.Sprintf("%s-ns-%d", testNamePodIdentity, randomNumberPod) + deploymentNamePod = fmt.Sprintf("%s-deployment-%d", testNamePodIdentity, randomNumberPod) + monitoredAppNamePod = fmt.Sprintf("%s-monitored-app-%d", testNamePodIdentity, randomNumberPod) + publishDeploymentNamePod = fmt.Sprintf("%s-publish-%d", testNamePodIdentity, randomNumberPod) + scaledObjectNamePod = fmt.Sprintf("%s-so-%d", testNamePodIdentity, randomNumberPod) + podIdentityProvider = "azure" +) + +// TestAzureManagedPrometheusScalerWithWorkloadIdentity creates deployments - there are two deployments - both using the same image but one deployment +// is directly tied to the KEDA HPA while the other is isolated that can be used for metrics +// even when the KEDA deployment is at zero - the service points to both deployments +func TestAzureManagedPrometheusScalerWithPodIdentity(t *testing.T) { + testAzureManagedPrometheusScaler(t, getTemplateDataForPodIdentityTest()) +} + +func getTemplateDataForPodIdentityTest() templateData { + return templateData{ + TestNamespace: testNamespacePod, + DeploymentName: deploymentNamePod, + PublishDeploymentName: publishDeploymentNamePod, + ScaledObjectName: scaledObjectNamePod, + MonitoredAppName: monitoredAppNamePod, + PodIdentityProvider: podIdentityProvider, + PrometheusQueryEndpoint: prometheusQueryEndpoint, + MinReplicaCount: minReplicaCount, + MaxReplicaCount: maxReplicaCount, + } +} diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_wi_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_wi_test.go new file mode 100644 index 00000000000..80137faaf40 --- /dev/null +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_wi_test.go @@ -0,0 +1,51 @@ +//go:build e2e +// +build e2e + +package azure_managed_prometheus + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/joho/godotenv" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +const ( + testNameWorkloadIdentity = "azure-managed-prometheus-workload-identity-test" +) + +// Workload Identity test vars +var ( + randomNumberWI = rand.Int() + testNamespaceWI = fmt.Sprintf("%s-ns-%d", testNameWorkloadIdentity, randomNumberWI) + deploymentNameWI = fmt.Sprintf("%s-deployment-%d", testNameWorkloadIdentity, randomNumberWI) + monitoredAppNameWI = fmt.Sprintf("%s-monitored-app-%d", testNameWorkloadIdentity, randomNumberWI) + publishDeploymentNameWI = fmt.Sprintf("%s-publish-%d", testNameWorkloadIdentity, randomNumberWI) + scaledObjectNameWI = fmt.Sprintf("%s-so-%d", testNameWorkloadIdentity, randomNumberWI) + workloadIdentityProvider = "azure-workload" +) + +// TestAzureManagedPrometheusScalerWithWorkloadIdentity creates deployments - there are two deployments - both using the same image but one deployment +// is directly tied to the KEDA HPA while the other is isolated that can be used for metrics +// even when the KEDA deployment is at zero - the service points to both deployments +func TestAzureManagedPrometheusScalerWithWorkloadIdentity(t *testing.T) { + testAzureManagedPrometheusScaler(t, getTemplateDataForWorkloadIdentityTest()) +} + +func getTemplateDataForWorkloadIdentityTest() templateData { + return templateData{ + TestNamespace: testNamespaceWI, + DeploymentName: deploymentNameWI, + PublishDeploymentName: publishDeploymentNameWI, + ScaledObjectName: scaledObjectNameWI, + MonitoredAppName: monitoredAppNameWI, + PodIdentityProvider: workloadIdentityProvider, + PrometheusQueryEndpoint: prometheusQueryEndpoint, + MinReplicaCount: minReplicaCount, + MaxReplicaCount: maxReplicaCount, + } +} diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_helper.go similarity index 73% rename from tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go rename to tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_helper.go index ac3fae35b21..d13f97820f3 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test.go +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_helper.go @@ -1,10 +1,9 @@ //go:build e2e // +build e2e -package azure_managed_prometheus_test +package azure_managed_prometheus import ( - "fmt" "os" "testing" @@ -19,31 +18,6 @@ import ( // Load environment variables from .env file var _ = godotenv.Load("../../.env") -const ( - testNameWorkloadIdentity = "azure-managed-prometheus-workload-identity-test" - testNamePodIdentity = "azure-managed-prometheus-pod-identity-test" -) - -// Workload Identity test vars -var ( - testNamespaceWI = fmt.Sprintf("%s-ns", testNameWorkloadIdentity) - deploymentNameWI = fmt.Sprintf("%s-deployment", testNameWorkloadIdentity) - monitoredAppNameWI = fmt.Sprintf("%s-monitored-app", testNameWorkloadIdentity) - publishDeploymentNameWI = fmt.Sprintf("%s-publish", testNameWorkloadIdentity) - scaledObjectNameWI = fmt.Sprintf("%s-so", testNameWorkloadIdentity) - workloadIdentityProvider = "azure-workload" -) - -// Pod Identity test vars -var ( - testNamespacePod = fmt.Sprintf("%s-ns", testNamePodIdentity) - deploymentNamePod = fmt.Sprintf("%s-deployment", testNamePodIdentity) - monitoredAppNamePod = fmt.Sprintf("%s-monitored-app", testNamePodIdentity) - publishDeploymentNamePod = fmt.Sprintf("%s-publish", testNamePodIdentity) - scaledObjectNamePod = fmt.Sprintf("%s-so", testNamePodIdentity) - podIdentityProvider = "azure" -) - // Common for pod and workload identity tests var ( prometheusQueryEndpoint = os.Getenv("TF_AZURE_MANAGED_PROMETHEUS_QUERY_ENDPOINT") @@ -284,21 +258,7 @@ spec: ` ) -// TestAzureManagedPrometheusScalerWithWorkloadIdentity creates deployments - there are two deployments - both using the same image but one deployment -// is directly tied to the KEDA HPA while the other is isolated that can be used for metrics -// even when the KEDA deployment is at zero - the service points to both deployments -func TestAzureManagedPrometheusScalerWithWorkloadIdentity(t *testing.T) { - testAzureManagedPrometheusScaler(t, getTemplateDataForWorkloadIdentityTest(), getTemplates()) -} - -// TestAzureManagedPrometheusScalerWithWorkloadIdentity creates deployments - there are two deployments - both using the same image but one deployment -// is directly tied to the KEDA HPA while the other is isolated that can be used for metrics -// even when the KEDA deployment is at zero - the service points to both deployments -func TestAzureManagedPrometheusScalerWithPodIdentity(t *testing.T) { - testAzureManagedPrometheusScaler(t, getTemplateDataForPodIdentityTest(), getTemplates()) -} - -func testAzureManagedPrometheusScaler(t *testing.T, data templateData, templates []Template) { +func testAzureManagedPrometheusScaler(t *testing.T, data templateData) { require.NotEmpty(t, prometheusQueryEndpoint, "TF_AZURE_MANAGED_PROMETHEUS_QUERY_ENDPOINT env variable is required for azure managed prometheus tests") kc := GetKubernetesClient(t) @@ -306,6 +266,7 @@ func testAzureManagedPrometheusScaler(t *testing.T, data templateData, templates // Create kubernetes resources for testing CreateNamespace(t, kc, data.TestNamespace) + templates := getTemplates() KubectlApplyMultipleWithTemplate(t, data, templates) assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, data.MonitoredAppName, data.TestNamespace, 1, 60, 3), "replica count should be %d after 3 minutes", minReplicaCount) @@ -352,31 +313,3 @@ func getTemplates() []Template { {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, } } - -func getTemplateDataForPodIdentityTest() templateData { - return templateData{ - TestNamespace: testNamespacePod, - DeploymentName: deploymentNamePod, - PublishDeploymentName: publishDeploymentNamePod, - ScaledObjectName: scaledObjectNamePod, - MonitoredAppName: monitoredAppNamePod, - PodIdentityProvider: podIdentityProvider, - PrometheusQueryEndpoint: prometheusQueryEndpoint, - MinReplicaCount: minReplicaCount, - MaxReplicaCount: maxReplicaCount, - } -} - -func getTemplateDataForWorkloadIdentityTest() templateData { - return templateData{ - TestNamespace: testNamespaceWI, - DeploymentName: deploymentNameWI, - PublishDeploymentName: publishDeploymentNameWI, - ScaledObjectName: scaledObjectNameWI, - MonitoredAppName: monitoredAppNameWI, - PodIdentityProvider: workloadIdentityProvider, - PrometheusQueryEndpoint: prometheusQueryEndpoint, - MinReplicaCount: minReplicaCount, - MaxReplicaCount: maxReplicaCount, - } -} From 30c9cc85ad6834289edd45a646412730be7654ca Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Wed, 1 Mar 2023 13:14:00 -0800 Subject: [PATCH 12/27] minor test fix Signed-off-by: Raghav Gupta --- .../azure_managed_prometheus_http_round_tripper_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go index 9cf3801607f..874d4928ae5 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go @@ -68,8 +68,11 @@ func TestTryAndGetAzureManagedPrometheusHTTPRoundTripperWithTriggerForResourceUR t.Errorf("Test: %v; Expected success but got error: %v", testData.testName, err) azureRT := rt.(*azureManagedPrometheusHTTPRoundTripper) - if azureRT == nil || azureRT.resourceURL != testData.resourceURL { - t.Errorf("Test: %v; Expected success but got error: %v", testData.testName, err) + if azureRT == nil { + t.Errorf("Test: %v; Expected azure round tripper but got nil", testData.testName) + } + if azureRT.resourceURL != testData.resourceURL { + t.Errorf("Test: %v; Expected resourceURL %v but got %v", testData.testName, testData.resourceURL, azureRT.resourceURL) } } } From 64cd79f53780fcb9cac855d8f70c11c45bc0308e Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Wed, 1 Mar 2023 13:17:26 -0800 Subject: [PATCH 13/27] comment fix Signed-off-by: Raghav Gupta --- .../azure_managed_prometheus_aad_pod_identity_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_pod_identity_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_pod_identity_test.go index 0a91de9a6bf..76f8e481401 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_pod_identity_test.go +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_pod_identity_test.go @@ -29,7 +29,7 @@ var ( podIdentityProvider = "azure" ) -// TestAzureManagedPrometheusScalerWithWorkloadIdentity creates deployments - there are two deployments - both using the same image but one deployment +// TestAzureManagedPrometheusScalerWithPodIdentity creates deployments - there are two deployments - both using the same image but one deployment // is directly tied to the KEDA HPA while the other is isolated that can be used for metrics // even when the KEDA deployment is at zero - the service points to both deployments func TestAzureManagedPrometheusScalerWithPodIdentity(t *testing.T) { From dce24727f325737ce985adc4063e764a7b5d8bb7 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Wed, 1 Mar 2023 13:22:26 -0800 Subject: [PATCH 14/27] more UTs Signed-off-by: Raghav Gupta --- ..._managed_prometheus_http_round_tripper_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go index 874d4928ae5..7d581379827 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go @@ -41,6 +41,21 @@ var testAzureManagedPrometheusResourceURLTestData = []testAzureManagedPrometheus {"test azure private cloud with WI", kedav1alpha1.PodIdentityProviderAzureWorkload, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "PRIVATE", "azureManagedPrometheusResourceURL": "blah-blah-resourceURL"}, "https://prometheus.monitor.azure.com/.default", false}, // with private cloud failure {"test default azure cloud with WI", kedav1alpha1.PodIdentityProviderAzureWorkload, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "PRIVATE"}, "", true}, + + // pod identity + + // with default cloud + {"test default azure cloud with WI", kedav1alpha1.PodIdentityProviderAzure, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up"}, "https://prometheus.monitor.azure.com/.default", false}, + // with public cloud + {"test azure public cloud with WI", kedav1alpha1.PodIdentityProviderAzure, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "AZUREPUBLICCLOUD"}, "https://prometheus.monitor.azure.com/.default", false}, + // with china cloud + {"test azure china cloud with WI", kedav1alpha1.PodIdentityProviderAzure, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "AZURECHINACLOUD"}, "https://prometheus.monitor.azure.cn/.default", false}, + // with US GOV cloud + {"test azure US GOV cloud with WI", kedav1alpha1.PodIdentityProviderAzure, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "AZUREUSGOVERNMENTCLOUD"}, "https://prometheus.monitor.azure.us/.default", false}, + // with private cloud success + {"test azure private cloud with WI", kedav1alpha1.PodIdentityProviderAzure, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "PRIVATE", "azureManagedPrometheusResourceURL": "blah-blah-resourceURL"}, "https://prometheus.monitor.azure.com/.default", false}, + // with private cloud failure + {"test default azure cloud with WI", kedav1alpha1.PodIdentityProviderAzure, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "PRIVATE"}, "", true}, } func TestTryAndGetAzureManagedPrometheusHTTPRoundTripperForTriggerMetadataAbsent(t *testing.T) { From 9fe841e2a7321a8c2e8f31ccca5aacbe69b94b9b Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Wed, 1 Mar 2023 14:43:50 -0800 Subject: [PATCH 15/27] fix static checks Signed-off-by: Raghav Gupta --- .../azure/azure_managed_prometheus_http_round_tripper_test.go | 3 +-- ...theus_test_helper.go => azure_managed_prometheus_helper.go} | 2 +- ... => azure_managed_prometheus_test_aad_pod_identity_test.go} | 2 +- ...wi_test.go => azure_managed_prometheus_test_aad_wi_test.go} | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) rename tests/scalers/prometheus/azure_managed_prometheus/{azure_managed_prometheus_test_helper.go => azure_managed_prometheus_helper.go} (99%) rename tests/scalers/prometheus/azure_managed_prometheus/{azure_managed_prometheus_aad_pod_identity_test.go => azure_managed_prometheus_test_aad_pod_identity_test.go} (97%) rename tests/scalers/prometheus/azure_managed_prometheus/{azure_managed_prometheus_aad_wi_test.go => azure_managed_prometheus_test_aad_wi_test.go} (98%) diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go index 7d581379827..15c552eb48f 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go @@ -85,8 +85,7 @@ func TestTryAndGetAzureManagedPrometheusHTTPRoundTripperWithTriggerForResourceUR azureRT := rt.(*azureManagedPrometheusHTTPRoundTripper) if azureRT == nil { t.Errorf("Test: %v; Expected azure round tripper but got nil", testData.testName) - } - if azureRT.resourceURL != testData.resourceURL { + } else if azureRT.resourceURL != testData.resourceURL { t.Errorf("Test: %v; Expected resourceURL %v but got %v", testData.testName, testData.resourceURL, azureRT.resourceURL) } } diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_helper.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_helper.go similarity index 99% rename from tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_helper.go rename to tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_helper.go index d13f97820f3..87c0712f44e 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_helper.go +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_helper.go @@ -1,7 +1,7 @@ //go:build e2e // +build e2e -package azure_managed_prometheus +package azure_managed_prometheus_test import ( "os" diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_pod_identity_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity_test.go similarity index 97% rename from tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_pod_identity_test.go rename to tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity_test.go index 76f8e481401..eaaf0cbac5f 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_pod_identity_test.go +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity_test.go @@ -1,7 +1,7 @@ //go:build e2e // +build e2e -package azure_managed_prometheus +package azure_managed_prometheus_test import ( "fmt" diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_wi_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi_test.go similarity index 98% rename from tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_wi_test.go rename to tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi_test.go index 80137faaf40..0c35f19e676 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_aad_wi_test.go +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi_test.go @@ -1,7 +1,7 @@ //go:build e2e // +build e2e -package azure_managed_prometheus +package azure_managed_prometheus_test import ( "fmt" From 9f9ebf62896519cab703cd6f7a09471e78552356 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Wed, 1 Mar 2023 14:46:56 -0800 Subject: [PATCH 16/27] fix static check Signed-off-by: Raghav Gupta --- .../azure_managed_prometheus_helper.go | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_helper.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_helper.go index 87c0712f44e..636fb822abe 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_helper.go +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_helper.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" "k8s.io/client-go/kubernetes" - . "github.com/kedacore/keda/v2/tests/helper" + "github.com/kedacore/keda/v2/tests/helper" ) // Load environment variables from .env file @@ -261,16 +261,16 @@ spec: func testAzureManagedPrometheusScaler(t *testing.T, data templateData) { require.NotEmpty(t, prometheusQueryEndpoint, "TF_AZURE_MANAGED_PROMETHEUS_QUERY_ENDPOINT env variable is required for azure managed prometheus tests") - kc := GetKubernetesClient(t) + kc := helper.GetKubernetesClient(t) // Create kubernetes resources for testing - CreateNamespace(t, kc, data.TestNamespace) + helper.CreateNamespace(t, kc, data.TestNamespace) templates := getTemplates() - KubectlApplyMultipleWithTemplate(t, data, templates) - assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, data.MonitoredAppName, data.TestNamespace, 1, 60, 3), + helper.KubectlApplyMultipleWithTemplate(t, data, templates) + assert.True(t, helper.WaitForDeploymentReplicaReadyCount(t, kc, data.MonitoredAppName, data.TestNamespace, 1, 60, 3), "replica count should be %d after 3 minutes", minReplicaCount) - assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, minReplicaCount, 60, 3), + assert.True(t, helper.WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, minReplicaCount, 60, 3), "replica count should be %d after 3 minutes", minReplicaCount) testActivation(t, kc, data) @@ -278,33 +278,33 @@ func testAzureManagedPrometheusScaler(t *testing.T, data templateData) { testScaleIn(t, kc, data) // cleanup - KubectlDeleteMultipleWithTemplate(t, data, templates) - DeleteNamespace(t, kc, data.TestNamespace) + helper.KubectlDeleteMultipleWithTemplate(t, data, templates) + helper.DeleteNamespace(t, kc, data.TestNamespace) } func testActivation(t *testing.T, kc *kubernetes.Clientset, data templateData) { t.Log("--- testing activation ---") - KubectlApplyWithTemplate(t, data, "generateLowLevelLoadJobTemplate", generateLowLevelLoadJobTemplate) + helper.KubectlApplyWithTemplate(t, data, "generateLowLevelLoadJobTemplate", generateLowLevelLoadJobTemplate) - AssertReplicaCountNotChangeDuringTimePeriod(t, kc, data.DeploymentName, data.TestNamespace, minReplicaCount, 60) + helper.AssertReplicaCountNotChangeDuringTimePeriod(t, kc, data.DeploymentName, data.TestNamespace, minReplicaCount, 60) } func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) { t.Log("--- testing scale out ---") - KubectlApplyWithTemplate(t, data, "generateLoadJobTemplate", generateLoadJobTemplate) + helper.KubectlApplyWithTemplate(t, data, "generateLoadJobTemplate", generateLoadJobTemplate) - assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, maxReplicaCount, 120, 5), + assert.True(t, helper.WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, maxReplicaCount, 120, 5), "replica count should be %d after 5 minutes", maxReplicaCount) } func testScaleIn(t *testing.T, kc *kubernetes.Clientset, data templateData) { t.Log("--- testing scale in ---") - assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, minReplicaCount, 120, 5), + assert.True(t, helper.WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, minReplicaCount, 120, 5), "replica count should be %d after 5 minutes", minReplicaCount) } -func getTemplates() []Template { - return []Template{ +func getTemplates() []helper.Template { + return []helper.Template{ {Name: "azureManagedPrometheusConfigMapTemplate", Config: azureManagedPrometheusConfigMapTemplate}, {Name: "monitoredAppDeploymentTemplate", Config: monitoredAppDeploymentTemplate}, {Name: "deploymentTemplate", Config: deploymentTemplate}, From 4fe3b8aa52d65d2e8a46e2d699d58b16f54dcf0d Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Wed, 1 Mar 2023 14:58:02 -0800 Subject: [PATCH 17/27] fix UT Signed-off-by: Raghav Gupta --- ...managed_prometheus_http_round_tripper_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go index 15c552eb48f..39eea8123f3 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go @@ -38,7 +38,7 @@ var testAzureManagedPrometheusResourceURLTestData = []testAzureManagedPrometheus // with US GOV cloud {"test azure US GOV cloud with WI", kedav1alpha1.PodIdentityProviderAzureWorkload, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "AZUREUSGOVERNMENTCLOUD"}, "https://prometheus.monitor.azure.us/.default", false}, // with private cloud success - {"test azure private cloud with WI", kedav1alpha1.PodIdentityProviderAzureWorkload, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "PRIVATE", "azureManagedPrometheusResourceURL": "blah-blah-resourceURL"}, "https://prometheus.monitor.azure.com/.default", false}, + {"test azure private cloud with WI", kedav1alpha1.PodIdentityProviderAzureWorkload, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "PRIVATE", "azureManagedPrometheusResourceURL": "blah-blah-resourceURL"}, "blah-blah-resourceURL", false}, // with private cloud failure {"test default azure cloud with WI", kedav1alpha1.PodIdentityProviderAzureWorkload, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "PRIVATE"}, "", true}, @@ -53,7 +53,7 @@ var testAzureManagedPrometheusResourceURLTestData = []testAzureManagedPrometheus // with US GOV cloud {"test azure US GOV cloud with WI", kedav1alpha1.PodIdentityProviderAzure, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "AZUREUSGOVERNMENTCLOUD"}, "https://prometheus.monitor.azure.us/.default", false}, // with private cloud success - {"test azure private cloud with WI", kedav1alpha1.PodIdentityProviderAzure, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "PRIVATE", "azureManagedPrometheusResourceURL": "blah-blah-resourceURL"}, "https://prometheus.monitor.azure.com/.default", false}, + {"test azure private cloud with WI", kedav1alpha1.PodIdentityProviderAzure, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "PRIVATE", "azureManagedPrometheusResourceURL": "blah-blah-resourceURL"}, "blah-blah-resourceURL", false}, // with private cloud failure {"test default azure cloud with WI", kedav1alpha1.PodIdentityProviderAzure, map[string]string{"serverAddress": "http://dummy-azure-monitor-workspace", "metricName": "http_requests_total", "threshold": "100", "query": "up", "cloud": "PRIVATE"}, "", true}, } @@ -81,13 +81,13 @@ func TestTryAndGetAzureManagedPrometheusHTTPRoundTripperWithTriggerForResourceUR } else { if err != nil { t.Errorf("Test: %v; Expected success but got error: %v", testData.testName, err) + } - azureRT := rt.(*azureManagedPrometheusHTTPRoundTripper) - if azureRT == nil { - t.Errorf("Test: %v; Expected azure round tripper but got nil", testData.testName) - } else if azureRT.resourceURL != testData.resourceURL { - t.Errorf("Test: %v; Expected resourceURL %v but got %v", testData.testName, testData.resourceURL, azureRT.resourceURL) - } + azureRT := rt.(*azureManagedPrometheusHTTPRoundTripper) + if azureRT == nil { + t.Errorf("Test: %v; Expected azure round tripper but got nil", testData.testName) + } else if azureRT.resourceURL != testData.resourceURL { + t.Errorf("Test: %v; Expected resourceURL %v but got %v", testData.testName, testData.resourceURL, azureRT.resourceURL) } } } From f2adf0242d9d421e07c9fde2867ecc56aa0ba351 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Wed, 1 Mar 2023 14:59:23 -0800 Subject: [PATCH 18/27] fix UT Signed-off-by: Raghav Gupta --- ...e_managed_prometheus_http_round_tripper_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go index 39eea8123f3..951deed17b4 100644 --- a/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go +++ b/pkg/scalers/azure/azure_managed_prometheus_http_round_tripper_test.go @@ -81,13 +81,13 @@ func TestTryAndGetAzureManagedPrometheusHTTPRoundTripperWithTriggerForResourceUR } else { if err != nil { t.Errorf("Test: %v; Expected success but got error: %v", testData.testName, err) - } - - azureRT := rt.(*azureManagedPrometheusHTTPRoundTripper) - if azureRT == nil { - t.Errorf("Test: %v; Expected azure round tripper but got nil", testData.testName) - } else if azureRT.resourceURL != testData.resourceURL { - t.Errorf("Test: %v; Expected resourceURL %v but got %v", testData.testName, testData.resourceURL, azureRT.resourceURL) + } else { + azureRT := rt.(*azureManagedPrometheusHTTPRoundTripper) + if azureRT == nil { + t.Errorf("Test: %v; Expected azure round tripper but got nil", testData.testName) + } else if azureRT.resourceURL != testData.resourceURL { + t.Errorf("Test: %v; Expected resourceURL %v but got %v", testData.testName, testData.resourceURL, azureRT.resourceURL) + } } } } From 9153bf48fb68b887cd0c81ec55f6311eb0489609 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Wed, 1 Mar 2023 16:33:20 -0800 Subject: [PATCH 19/27] fix e2e - shorten namespace name Signed-off-by: Raghav Gupta --- .../azure_managed_prometheus_test_aad_pod_identity_test.go | 2 +- .../azure_managed_prometheus_test_aad_wi_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity_test.go index eaaf0cbac5f..eb1a389de97 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity_test.go +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity_test.go @@ -15,7 +15,7 @@ import ( var _ = godotenv.Load("../../.env") const ( - testNamePodIdentity = "azure-managed-prometheus-pod-identity-test" + testNamePodIdentity = "azure-managed-prome-pi-test" ) // Pod Identity test vars diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi_test.go index 0c35f19e676..363223a969c 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi_test.go +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi_test.go @@ -15,7 +15,7 @@ import ( var _ = godotenv.Load("../../.env") const ( - testNameWorkloadIdentity = "azure-managed-prometheus-workload-identity-test" + testNameWorkloadIdentity = "azure-managed-prom-wi-test" ) // Workload Identity test vars From a2aea80f90e4e9b60765ee677a75204022f8beab Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Wed, 1 Mar 2023 16:42:03 -0800 Subject: [PATCH 20/27] fix e2e - shorten namespace name Signed-off-by: Raghav Gupta --- .../azure_managed_prometheus_test_aad_pod_identity_test.go | 2 +- .../azure_managed_prometheus_test_aad_wi_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity_test.go index eb1a389de97..120da1f4a74 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity_test.go +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity_test.go @@ -15,7 +15,7 @@ import ( var _ = godotenv.Load("../../.env") const ( - testNamePodIdentity = "azure-managed-prome-pi-test" + testNamePodIdentity = "amp-pi-test" ) // Pod Identity test vars diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi_test.go index 363223a969c..fd8b7c62140 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi_test.go +++ b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi_test.go @@ -15,7 +15,7 @@ import ( var _ = godotenv.Load("../../.env") const ( - testNameWorkloadIdentity = "azure-managed-prom-wi-test" + testNameWorkloadIdentity = "amp-wi-test" ) // Workload Identity test vars From 562e1ff231a7bfb628fb7350b8d99e772038523f Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Wed, 1 Mar 2023 20:51:50 -0800 Subject: [PATCH 21/27] rename test file Signed-off-by: Raghav Gupta --- ..._test.go => azure_managed_prometheus_test_aad_pod_identity.go} | 0 ...est_aad_wi_test.go => azure_managed_prometheus_test_aad_wi.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/scalers/prometheus/azure_managed_prometheus/{azure_managed_prometheus_test_aad_pod_identity_test.go => azure_managed_prometheus_test_aad_pod_identity.go} (100%) rename tests/scalers/prometheus/azure_managed_prometheus/{azure_managed_prometheus_test_aad_wi_test.go => azure_managed_prometheus_test_aad_wi.go} (100%) diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity.go similarity index 100% rename from tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity_test.go rename to tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity.go diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi_test.go b/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi.go similarity index 100% rename from tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi_test.go rename to tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi.go From 2e6c89de53ec565d788fb1768e0e206c50dcf121 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Thu, 2 Mar 2023 14:05:26 -0800 Subject: [PATCH 22/27] try resolve linking issues for e2e test packages Signed-off-by: Raghav Gupta --- ...naged_prometheus_aad_pod_identity_test.go} | 16 ++++---- ..._prometheus_aad_workload_identity_test.go} | 16 ++++---- .../helper/helper.go} | 40 +++++++++---------- 3 files changed, 38 insertions(+), 34 deletions(-) rename tests/scalers/{prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity.go => azure/azure_managed_prometheus/azure_managed_prometheus_aad_pod_identity/azure_managed_prometheus_aad_pod_identity_test.go} (78%) rename tests/scalers/{prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi.go => azure/azure_managed_prometheus/azure_managed_prometheus_aad_workload_identity/azure_managed_prometheus_aad_workload_identity_test.go} (78%) rename tests/scalers/{prometheus/azure_managed_prometheus/azure_managed_prometheus_helper.go => azure/azure_managed_prometheus/helper/helper.go} (88%) diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity.go b/tests/scalers/azure/azure_managed_prometheus/azure_managed_prometheus_aad_pod_identity/azure_managed_prometheus_aad_pod_identity_test.go similarity index 78% rename from tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity.go rename to tests/scalers/azure/azure_managed_prometheus/azure_managed_prometheus_aad_pod_identity/azure_managed_prometheus_aad_pod_identity_test.go index 120da1f4a74..35672792db4 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_pod_identity.go +++ b/tests/scalers/azure/azure_managed_prometheus/azure_managed_prometheus_aad_pod_identity/azure_managed_prometheus_aad_pod_identity_test.go @@ -1,7 +1,7 @@ //go:build e2e // +build e2e -package azure_managed_prometheus_test +package azure_managed_prometheus_aad_pod_identity_test import ( "fmt" @@ -9,6 +9,8 @@ import ( "testing" "github.com/joho/godotenv" + + . "github.com/kedacore/keda/v2/tests/scalers/azure/azure_managed_prometheus/helper" ) // Load environment variables from .env file @@ -33,19 +35,19 @@ var ( // is directly tied to the KEDA HPA while the other is isolated that can be used for metrics // even when the KEDA deployment is at zero - the service points to both deployments func TestAzureManagedPrometheusScalerWithPodIdentity(t *testing.T) { - testAzureManagedPrometheusScaler(t, getTemplateDataForPodIdentityTest()) + TestAzureManagedPrometheusScaler(t, getTemplateDataForPodIdentityTest()) } -func getTemplateDataForPodIdentityTest() templateData { - return templateData{ +func getTemplateDataForPodIdentityTest() TemplateData { + return TemplateData{ TestNamespace: testNamespacePod, DeploymentName: deploymentNamePod, PublishDeploymentName: publishDeploymentNamePod, ScaledObjectName: scaledObjectNamePod, MonitoredAppName: monitoredAppNamePod, PodIdentityProvider: podIdentityProvider, - PrometheusQueryEndpoint: prometheusQueryEndpoint, - MinReplicaCount: minReplicaCount, - MaxReplicaCount: maxReplicaCount, + PrometheusQueryEndpoint: PrometheusQueryEndpoint, + MinReplicaCount: MinReplicaCount, + MaxReplicaCount: MaxReplicaCount, } } diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi.go b/tests/scalers/azure/azure_managed_prometheus/azure_managed_prometheus_aad_workload_identity/azure_managed_prometheus_aad_workload_identity_test.go similarity index 78% rename from tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi.go rename to tests/scalers/azure/azure_managed_prometheus/azure_managed_prometheus_aad_workload_identity/azure_managed_prometheus_aad_workload_identity_test.go index fd8b7c62140..81a20732208 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_test_aad_wi.go +++ b/tests/scalers/azure/azure_managed_prometheus/azure_managed_prometheus_aad_workload_identity/azure_managed_prometheus_aad_workload_identity_test.go @@ -1,7 +1,7 @@ //go:build e2e // +build e2e -package azure_managed_prometheus_test +package azure_managed_prometheus_aad_workload_identity_test import ( "fmt" @@ -9,6 +9,8 @@ import ( "testing" "github.com/joho/godotenv" + + . "github.com/kedacore/keda/v2/tests/scalers/azure/azure_managed_prometheus/helper" ) // Load environment variables from .env file @@ -33,19 +35,19 @@ var ( // is directly tied to the KEDA HPA while the other is isolated that can be used for metrics // even when the KEDA deployment is at zero - the service points to both deployments func TestAzureManagedPrometheusScalerWithWorkloadIdentity(t *testing.T) { - testAzureManagedPrometheusScaler(t, getTemplateDataForWorkloadIdentityTest()) + TestAzureManagedPrometheusScaler(t, getTemplateDataForWorkloadIdentityTest()) } -func getTemplateDataForWorkloadIdentityTest() templateData { - return templateData{ +func getTemplateDataForWorkloadIdentityTest() TemplateData { + return TemplateData{ TestNamespace: testNamespaceWI, DeploymentName: deploymentNameWI, PublishDeploymentName: publishDeploymentNameWI, ScaledObjectName: scaledObjectNameWI, MonitoredAppName: monitoredAppNameWI, PodIdentityProvider: workloadIdentityProvider, - PrometheusQueryEndpoint: prometheusQueryEndpoint, - MinReplicaCount: minReplicaCount, - MaxReplicaCount: maxReplicaCount, + PrometheusQueryEndpoint: PrometheusQueryEndpoint, + MinReplicaCount: MinReplicaCount, + MaxReplicaCount: MaxReplicaCount, } } diff --git a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_helper.go b/tests/scalers/azure/azure_managed_prometheus/helper/helper.go similarity index 88% rename from tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_helper.go rename to tests/scalers/azure/azure_managed_prometheus/helper/helper.go index 636fb822abe..3e763df4781 100644 --- a/tests/scalers/prometheus/azure_managed_prometheus/azure_managed_prometheus_helper.go +++ b/tests/scalers/azure/azure_managed_prometheus/helper/helper.go @@ -1,7 +1,7 @@ //go:build e2e // +build e2e -package azure_managed_prometheus_test +package helper import ( "os" @@ -20,12 +20,12 @@ var _ = godotenv.Load("../../.env") // Common for pod and workload identity tests var ( - prometheusQueryEndpoint = os.Getenv("TF_AZURE_MANAGED_PROMETHEUS_QUERY_ENDPOINT") - minReplicaCount = 0 - maxReplicaCount = 2 + PrometheusQueryEndpoint = os.Getenv("TF_AZURE_MANAGED_PROMETHEUS_QUERY_ENDPOINT") + MinReplicaCount = 0 + MaxReplicaCount = 2 ) -type templateData struct { +type TemplateData struct { TestNamespace string DeploymentName string MonitoredAppName string @@ -189,8 +189,8 @@ metadata: spec: scaleTargetRef: name: {{.DeploymentName}} - minReplicaCount: {{.MinReplicaCount}} - maxReplicaCount: {{.MaxReplicaCount}} + MinReplicaCount: {{.MinReplicaCount}} + MaxReplicaCount: {{.MaxReplicaCount}} pollingInterval: 3 cooldownPeriod: 1 triggers: @@ -258,8 +258,8 @@ spec: ` ) -func testAzureManagedPrometheusScaler(t *testing.T, data templateData) { - require.NotEmpty(t, prometheusQueryEndpoint, "TF_AZURE_MANAGED_PROMETHEUS_QUERY_ENDPOINT env variable is required for azure managed prometheus tests") +func TestAzureManagedPrometheusScaler(t *testing.T, data TemplateData) { + require.NotEmpty(t, PrometheusQueryEndpoint, "TF_AZURE_MANAGED_PROMETHEUS_QUERY_ENDPOINT env variable is required for azure managed prometheus tests") kc := helper.GetKubernetesClient(t) @@ -269,9 +269,9 @@ func testAzureManagedPrometheusScaler(t *testing.T, data templateData) { templates := getTemplates() helper.KubectlApplyMultipleWithTemplate(t, data, templates) assert.True(t, helper.WaitForDeploymentReplicaReadyCount(t, kc, data.MonitoredAppName, data.TestNamespace, 1, 60, 3), - "replica count should be %d after 3 minutes", minReplicaCount) - assert.True(t, helper.WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, minReplicaCount, 60, 3), - "replica count should be %d after 3 minutes", minReplicaCount) + "replica count should be %d after 3 minutes", MinReplicaCount) + assert.True(t, helper.WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, MinReplicaCount, 60, 3), + "replica count should be %d after 3 minutes", MinReplicaCount) testActivation(t, kc, data) testScaleOut(t, kc, data) @@ -282,25 +282,25 @@ func testAzureManagedPrometheusScaler(t *testing.T, data templateData) { helper.DeleteNamespace(t, kc, data.TestNamespace) } -func testActivation(t *testing.T, kc *kubernetes.Clientset, data templateData) { +func testActivation(t *testing.T, kc *kubernetes.Clientset, data TemplateData) { t.Log("--- testing activation ---") helper.KubectlApplyWithTemplate(t, data, "generateLowLevelLoadJobTemplate", generateLowLevelLoadJobTemplate) - helper.AssertReplicaCountNotChangeDuringTimePeriod(t, kc, data.DeploymentName, data.TestNamespace, minReplicaCount, 60) + helper.AssertReplicaCountNotChangeDuringTimePeriod(t, kc, data.DeploymentName, data.TestNamespace, MinReplicaCount, 60) } -func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data templateData) { +func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data TemplateData) { t.Log("--- testing scale out ---") helper.KubectlApplyWithTemplate(t, data, "generateLoadJobTemplate", generateLoadJobTemplate) - assert.True(t, helper.WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, maxReplicaCount, 120, 5), - "replica count should be %d after 5 minutes", maxReplicaCount) + assert.True(t, helper.WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, MaxReplicaCount, 120, 5), + "replica count should be %d after 5 minutes", MaxReplicaCount) } -func testScaleIn(t *testing.T, kc *kubernetes.Clientset, data templateData) { +func testScaleIn(t *testing.T, kc *kubernetes.Clientset, data TemplateData) { t.Log("--- testing scale in ---") - assert.True(t, helper.WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, minReplicaCount, 120, 5), - "replica count should be %d after 5 minutes", minReplicaCount) + assert.True(t, helper.WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, MinReplicaCount, 120, 5), + "replica count should be %d after 5 minutes", MinReplicaCount) } func getTemplates() []helper.Template { From 7ff04036d1def598c466613b4307afbb49e40db7 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Fri, 3 Mar 2023 13:47:43 -0800 Subject: [PATCH 23/27] e2e fix Signed-off-by: Raghav Gupta --- tests/scalers/azure/azure_managed_prometheus/helper/helper.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scalers/azure/azure_managed_prometheus/helper/helper.go b/tests/scalers/azure/azure_managed_prometheus/helper/helper.go index 3e763df4781..da3585f766c 100644 --- a/tests/scalers/azure/azure_managed_prometheus/helper/helper.go +++ b/tests/scalers/azure/azure_managed_prometheus/helper/helper.go @@ -189,8 +189,8 @@ metadata: spec: scaleTargetRef: name: {{.DeploymentName}} - MinReplicaCount: {{.MinReplicaCount}} - MaxReplicaCount: {{.MaxReplicaCount}} + minReplicaCount: {{.MinReplicaCount}} + maxReplicaCount: {{.MaxReplicaCount}} pollingInterval: 3 cooldownPeriod: 1 triggers: From ff05b1636df67125ad1171c8b42aadec9f334f31 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Sun, 5 Mar 2023 12:03:46 -0800 Subject: [PATCH 24/27] deploying config map as part of one time KEDA setup Signed-off-by: Raghav Gupta --- .../azure_managed_prometheus/helper/helper.go | 46 ----------------- .../azure_managed_prometheus_setup_test.go | 49 +++++++++++++++++++ tests/utils/cleanup_test.go | 4 ++ tests/utils/setup_test.go | 5 ++ 4 files changed, 58 insertions(+), 46 deletions(-) create mode 100644 tests/utils/azure_managed_prometheus_setup_test.go diff --git a/tests/scalers/azure/azure_managed_prometheus/helper/helper.go b/tests/scalers/azure/azure_managed_prometheus/helper/helper.go index da3585f766c..dfc28cf6e85 100644 --- a/tests/scalers/azure/azure_managed_prometheus/helper/helper.go +++ b/tests/scalers/azure/azure_managed_prometheus/helper/helper.go @@ -38,51 +38,6 @@ type TemplateData struct { } const ( - azureManagedPrometheusConfigMapTemplate = `apiVersion: v1 -kind: ConfigMap -metadata: - name: ama-metrics-prometheus-config - namespace: kube-system -data: - prometheus-config: |- - global: - evaluation_interval: 1m - scrape_interval: 1m - scrape_timeout: 10s - scrape_configs: - - job_name: kubernetes-pods - kubernetes_sd_configs: - - role: pod - relabel_configs: - - action: keep - regex: true - source_labels: - - __meta_kubernetes_pod_annotation_prometheus_io_scrape - - action: replace - regex: (.+) - source_labels: - - __meta_kubernetes_pod_annotation_prometheus_io_path - target_label: __metrics_path__ - - action: replace - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:$2 - source_labels: - - __address__ - - __meta_kubernetes_pod_annotation_prometheus_io_port - target_label: __address__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - action: replace - source_labels: - - __meta_kubernetes_namespace - target_label: kubernetes_namespace - - action: replace - source_labels: - - __meta_kubernetes_pod_name - target_label: kubernetes_pod_name ---- -` - deploymentTemplate = `apiVersion: apps/v1 kind: Deployment metadata: @@ -305,7 +260,6 @@ func testScaleIn(t *testing.T, kc *kubernetes.Clientset, data TemplateData) { func getTemplates() []helper.Template { return []helper.Template{ - {Name: "azureManagedPrometheusConfigMapTemplate", Config: azureManagedPrometheusConfigMapTemplate}, {Name: "monitoredAppDeploymentTemplate", Config: monitoredAppDeploymentTemplate}, {Name: "deploymentTemplate", Config: deploymentTemplate}, {Name: "monitoredAppServiceTemplate", Config: monitoredAppServiceTemplate}, diff --git a/tests/utils/azure_managed_prometheus_setup_test.go b/tests/utils/azure_managed_prometheus_setup_test.go new file mode 100644 index 00000000000..7a777f233de --- /dev/null +++ b/tests/utils/azure_managed_prometheus_setup_test.go @@ -0,0 +1,49 @@ +package utils + +type templateData struct{} + +const ( + azureManagedPrometheusConfigMapTemplate = `apiVersion: v1 +kind: ConfigMap +metadata: + name: ama-metrics-prometheus-config + namespace: kube-system +data: + prometheus-config: |- + global: + evaluation_interval: 1m + scrape_interval: 1m + scrape_timeout: 10s + scrape_configs: + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: kubernetes_namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: kubernetes_pod_name +` +) diff --git a/tests/utils/cleanup_test.go b/tests/utils/cleanup_test.go index 219dc2159c9..5e43f08c3ef 100644 --- a/tests/utils/cleanup_test.go +++ b/tests/utils/cleanup_test.go @@ -84,3 +84,7 @@ func TestRemoveCertManager(t *testing.T) { DeleteNamespace(t, KubeClient, CertManagerNamespace) } + +func TestRemoveAzureManagedPrometheusComponents(t *testing.T) { + KubectlDeleteWithTemplate(t, templateData{}, "azureManagedPrometheusConfigMapTemplate", azureManagedPrometheusConfigMapTemplate) +} diff --git a/tests/utils/setup_test.go b/tests/utils/setup_test.go index 1d4da933756..4d9d3ef689a 100644 --- a/tests/utils/setup_test.go +++ b/tests/utils/setup_test.go @@ -249,3 +249,8 @@ func TestSetupAadPodIdentityComponents(t *testing.T) { AzureAdPodIdentityNamespace, AzureADMsiClientID, AzureADMsiID)) require.NoErrorf(t, err, "cannot install aad pod identity webhook - %s", err) } + +func TestSetupAzureManagedPrometheusComponents(t *testing.T) { + // this will install config map in kube-system namespace, as needed by azure manage prometheus collector agent + KubectlApplyWithTemplate(t, templateData{}, "azureManagedPrometheusConfigMapTemplate", azureManagedPrometheusConfigMapTemplate) +} From a95dd4978baf53793f9a1053f6aadf4fb66fe51d Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Sun, 5 Mar 2023 18:37:56 -0800 Subject: [PATCH 25/27] moving config map to helper package to resolve linking issues Signed-off-by: Raghav Gupta --- tests/utils/cleanup_test.go | 3 ++- .../helper.go} | 6 +++--- tests/utils/setup_test.go | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) rename tests/utils/{azure_managed_prometheus_setup_test.go => helper/helper.go} (91%) diff --git a/tests/utils/cleanup_test.go b/tests/utils/cleanup_test.go index 5e43f08c3ef..14766cb886e 100644 --- a/tests/utils/cleanup_test.go +++ b/tests/utils/cleanup_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" . "github.com/kedacore/keda/v2/tests/helper" + "github.com/kedacore/keda/v2/tests/utils/helper" ) func TestRemoveKEDA(t *testing.T) { @@ -86,5 +87,5 @@ func TestRemoveCertManager(t *testing.T) { } func TestRemoveAzureManagedPrometheusComponents(t *testing.T) { - KubectlDeleteWithTemplate(t, templateData{}, "azureManagedPrometheusConfigMapTemplate", azureManagedPrometheusConfigMapTemplate) + KubectlDeleteWithTemplate(t, helper.EmptyTemplateData{}, "azureManagedPrometheusConfigMapTemplate", helper.AzureManagedPrometheusConfigMapTemplate) } diff --git a/tests/utils/azure_managed_prometheus_setup_test.go b/tests/utils/helper/helper.go similarity index 91% rename from tests/utils/azure_managed_prometheus_setup_test.go rename to tests/utils/helper/helper.go index 7a777f233de..29f4498927f 100644 --- a/tests/utils/azure_managed_prometheus_setup_test.go +++ b/tests/utils/helper/helper.go @@ -1,9 +1,9 @@ -package utils +package helper -type templateData struct{} +type EmptyTemplateData struct{} const ( - azureManagedPrometheusConfigMapTemplate = `apiVersion: v1 + AzureManagedPrometheusConfigMapTemplate = `apiVersion: v1 kind: ConfigMap metadata: name: ama-metrics-prometheus-config diff --git a/tests/utils/setup_test.go b/tests/utils/setup_test.go index 4d9d3ef689a..9ce5535e0ce 100644 --- a/tests/utils/setup_test.go +++ b/tests/utils/setup_test.go @@ -15,6 +15,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/kedacore/keda/v2/tests/helper" + "github.com/kedacore/keda/v2/tests/utils/helper" ) func TestVerifyCommands(t *testing.T) { @@ -252,5 +253,5 @@ func TestSetupAadPodIdentityComponents(t *testing.T) { func TestSetupAzureManagedPrometheusComponents(t *testing.T) { // this will install config map in kube-system namespace, as needed by azure manage prometheus collector agent - KubectlApplyWithTemplate(t, templateData{}, "azureManagedPrometheusConfigMapTemplate", azureManagedPrometheusConfigMapTemplate) + KubectlApplyWithTemplate(t, helper.EmptyTemplateData{}, "azureManagedPrometheusConfigMapTemplate", helper.AzureManagedPrometheusConfigMapTemplate) } From 0c513f5ff11493a9befc429b5fa12942a20c7afa Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Mon, 6 Mar 2023 11:40:12 -0800 Subject: [PATCH 26/27] increasing test timeout Signed-off-by: Raghav Gupta --- .../azure/azure_managed_prometheus/helper/helper.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/scalers/azure/azure_managed_prometheus/helper/helper.go b/tests/scalers/azure/azure_managed_prometheus/helper/helper.go index dfc28cf6e85..ec0672f970d 100644 --- a/tests/scalers/azure/azure_managed_prometheus/helper/helper.go +++ b/tests/scalers/azure/azure_managed_prometheus/helper/helper.go @@ -248,14 +248,14 @@ func testScaleOut(t *testing.T, kc *kubernetes.Clientset, data TemplateData) { t.Log("--- testing scale out ---") helper.KubectlApplyWithTemplate(t, data, "generateLoadJobTemplate", generateLoadJobTemplate) - assert.True(t, helper.WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, MaxReplicaCount, 120, 5), - "replica count should be %d after 5 minutes", MaxReplicaCount) + assert.True(t, helper.WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, MaxReplicaCount, 144, 5), + "replica count should be %d after 12 minutes", MaxReplicaCount) } func testScaleIn(t *testing.T, kc *kubernetes.Clientset, data TemplateData) { t.Log("--- testing scale in ---") - assert.True(t, helper.WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, MinReplicaCount, 120, 5), - "replica count should be %d after 5 minutes", MinReplicaCount) + assert.True(t, helper.WaitForDeploymentReplicaReadyCount(t, kc, data.DeploymentName, data.TestNamespace, MinReplicaCount, 144, 5), + "replica count should be %d after 12 minutes", MinReplicaCount) } func getTemplates() []helper.Template { From 8a1dd26e371c5eae157fa0c32ab06cb7460c8299 Mon Sep 17 00:00:00 2001 From: Raghav Gupta Date: Mon, 6 Mar 2023 11:46:40 -0800 Subject: [PATCH 27/27] moving config map setup Signed-off-by: Raghav Gupta --- tests/utils/setup_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/utils/setup_test.go b/tests/utils/setup_test.go index 9ce5535e0ce..8dd0ac42749 100644 --- a/tests/utils/setup_test.go +++ b/tests/utils/setup_test.go @@ -56,6 +56,12 @@ func TestSetupHelm(t *testing.T) { require.NoErrorf(t, err, "cannot get helm version - %s", err) } +// doing early in the sequence of tests so that config map update has time to be effective before the azure tests get executed. +func TestSetupAzureManagedPrometheusComponents(t *testing.T) { + // this will install config map in kube-system namespace, as needed by azure manage prometheus collector agent + KubectlApplyWithTemplate(t, helper.EmptyTemplateData{}, "azureManagedPrometheusConfigMapTemplate", helper.AzureManagedPrometheusConfigMapTemplate) +} + func TestSetupWorkloadIdentityComponents(t *testing.T) { if AzureRunWorkloadIdentityTests == "" || AzureRunWorkloadIdentityTests == StringFalse { t.Skip("skipping as workload identity tests are disabled") @@ -250,8 +256,3 @@ func TestSetupAadPodIdentityComponents(t *testing.T) { AzureAdPodIdentityNamespace, AzureADMsiClientID, AzureADMsiID)) require.NoErrorf(t, err, "cannot install aad pod identity webhook - %s", err) } - -func TestSetupAzureManagedPrometheusComponents(t *testing.T) { - // this will install config map in kube-system namespace, as needed by azure manage prometheus collector agent - KubectlApplyWithTemplate(t, helper.EmptyTemplateData{}, "azureManagedPrometheusConfigMapTemplate", helper.AzureManagedPrometheusConfigMapTemplate) -}