prepare 6.7.12 release (#200)

* environment should still be usable even if the client timed out * bump dependency versions for SDK fixes * update go-server-sdk-dynamodb to get newer AWS SDK * minor clarification about Relay.Close (#234) * Use the Go releaser template (#233) * [ch102248] big segment sync with redis (#235) * fix makefile so it tries building all the test code first before running any of it (#239) * (big segments #1) add basic big segments configuration for SDK clients (#237) * (big segments #2) add more abstraction around big segments implementation for testability (#238) * [102253] bigsegment status / config (#242) * big segments integration test + misc fixes (#240) * use latest URL paths for big segments endpoints * add SDK DynamoDB integration for big segments (#241) * fix broken link in Markdown docs (#246) * make sure newly added credentials for existing environments are accepted in requests (#244) * don't return 503 if SDK initialization has timed out * add in-repo docs about error/503 behavior (#249) * [ch102255] BigSegments DynamoDB (#245) * add init timeout config option + better test coverage + misc refactoring (#250) * fix example build command * use public prerelease tags instead of private dependencies * fix Go installation in CI * update SDK dependencies for JSON number parsing bugfix * update gorilla/mux to 1.8.0 * update OpenCensus packages * add Go 1.16 CI + "latest Go" CI + use latest 1.15 patch for release * cimg images use "current", not "latest" * seems there isn't any cimg/go "latest" or "current" * add daily package build test in CI * job names * bump SDK version for traffic allocation feature * [ch113491] update alpine base image (#258) * use latest prerelease SDK * fix enabling of test tags in CI * add DynamoDB docker image in CI * set a polling base URI in end-to-end tests since big segments logic will use it * fix initialization logic so SDK client creation errors aren't lost when big segments are enabled * fix use of prefix key in DynamoDB + improve tests (#260) * more debug logging, less info logging for big segments logic * make logging of big segments patch version mismatch clearer and use Warn level * fix log parameter * fix DynamoDB updates for big segments metadata * add test to make sure sync time and cursor can be updated independently * only start big seg synchronizer if necessary * use SDK GA releases * change applyPatch to exit early on version mismatch; go back to restarting stream in this case * add unit tests for version mismatch behavior + DRY tests * add log assertion * fix retry logic on big segments stream failure * add more logging for big segments connection status * fix logging assertion * add more big segments integration tests * fix overly-time-sensitive file data tests * fix more flaky tests * run big segments tests with DynamoDB too * Migrate transitive dep (jwt-go) to use modern version without vulnerability. * Edit doc * move Relay release logic to .ldrelease script * suppress SDK big segments status query if we've never synced big segments * dump Relay logs including debug logs if integration test fails * include environment prefix in BigSegmentSynchronizer logging * increase big segment integration test timeout (#274) * generate client-side stream pings if big segments have changed * clear big segments cache as needed + simplify state management * fix tests and simplify component creation * use GA releases of SDK packages * disable CI package-build-test in Go 1.16+ * Migrate Relay release to Releaser v2 and support dry run (#278) * Adding degraded doc blurb for big segments (#280) * respect Redis password & TLS options for big segments; add Redis password integration tests * redact Redis URL password in logs and status resource * update go-server-sdk-redis-redigo to 1.2.1 for Redis URL logging fix * Part 1, add the config and the documentation for the new config * Part 2, Add the configuration validation and test * Part 3, the actual logic to include the headers in the CORS Access-Control-Allow-Headers * Linter * update Alpine version to 3.14.2 to fix openssl CVEs * Fix the global variable modification * Go format * turn off unnecessary metrics integrations in config for Docker smoke test * rename test.env to smoke-test.env to clarify what it's for * fix setting of custom Access-Control-Allow-Origin and add test (#285) * add more explanatory test output and more verbose debugging for big segments integration tests (#287) * update to Go 1.16.10 + Alpine 3.14.3; add some docs about releases (#288) * update go-server-sdk-consul version for Consul API version update * override x/crypto dependency version for CVE-2020-29652 * bump Prometheus dependency to eliminate jwt-go vulnerability * drop support for Go 1.14 & 1.15 * make sure defaults are always applied for base URL properties * rm unused * rm unnecessary linter directive * add separate configuration for server-side/client-side SDK base URLs & update the defaults * remove Whitesource CI job + remove obsolete dependency issue note * don't include any big segment status info in status resource unless that feature is active (#296) * don't include any big segment status info in status resource unless that feature is active * fix Big Segments staleness logic in status resource * documentation * update x/text package for vulnerability GO-2021-0113 * add Trivy security scan to CI (#297) * add daily re-scan with Trivy * use long timeout when awaiting changes related to file mod watching * update Go version to 1.17.6 (#301) * always terminate if auto-config stream fails with a fatal error * pass along tags header when proxying events * comments, rm debugging * fix auth header logic * fix auth header logic some more * comments * add tags header to CORS header whitelist (#304) * update to Alpine 3.14.4 for CVE-2022-0778 fix * force upgrade of openssl in Alpine * also upgrade libretls * fix it in both files * update to Alpine 3.14.5 for CVE-2022-0778/CVE-2018-25032 (#308) * update to Alpine 3.14.5 for CVE-2022-0778 * revert patches that are now included in Alpine 3.14.5 * add scripts for checking and updating Go/Alpine versions (#309) * update to Alpine 3.14.5 for CVE-2022-0778 * add scripts for checking and updating Go/Alpine versions * also make sure the Docker images really exist * update CONTRIBUTING.md * fix file rename * revert patches that are now included in Alpine 3.14.5 * update Alpine to 3.14.6 for CVE-2022-28391 * update SDK packages (includes sc-136333 fix) * don't include "v" prefix in Docker image version * update go-server-sdk-dynamodb for data size error fix & add docs (#316) * update builds to use Go 1.17.9 and fix the update script * update go-server-sdk-consul to latest release * update remote Docker version * update golang.org/x/crypto for CVE-2022-27191 (#321) * update golang.org/x/crypto for CVE-2022-27191 * fix go.sum * update eventsource for SSE output efficiency fix (#322) * Cache the replay event in case we get multiple new client connections (#189) * Cache the replay event in case we get multiple new client connections * Use singleflight to ensure only one replay event is generated at a time Co-authored-by: Moshe Good <moshe@squareup.com> * don't install curl in Docker images * fix makefile logic for lint step * remove indirect curl-based request logic in integration tests * fix linter installation * update Go to 1.17.11, Alpine to 3.16.0 * improve concurrency test to verify that the data is or isn't from a separate query * fix lint warnings and remove unnecessary error return * update libssl & libcrypto versions for CVE-2022-2097 * add security scan of already-published Docker image (#328) * update Alpine version and some Go libraries to address CVEs (#329) * use Alpine 3.16.1 * update golang.org/x/net and golang.org/x/sync patch versions for CVEs * update golang.org/x/sys patch version for CVE * update Prometheus client library for CVE-2022-21698 * ensure that DynamoDB config is consistent between Big Segments and regular data store * comment Co-authored-by: Eli Bishop <eli@launchdarkly.com> Co-authored-by: LaunchDarklyCI <dev@launchdarkly.com> Co-authored-by: Ben Woskow <48036130+bwoskow-ld@users.noreply.github.com> Co-authored-by: hroederld <hroeder@launchdarkly.com> Co-authored-by: LaunchDarklyReleaseBot <launchdarklyreleasebot@launchdarkly.com> Co-authored-by: Dan Richelson <drichelson@launchdarkly.com> Co-authored-by: Dan Richelson <drichelson@users.noreply.github.com> Co-authored-by: Ben Woskow <bwoskow@launchdarkly.com> Co-authored-by: Louis Chan <lchan@launchdarkly.com> Co-authored-by: Louis Chan <91093020+louis-launchdarkly@users.noreply.github.com> Co-authored-by: Moshe Good <moshegood@users.noreply.github.com> Co-authored-by: Moshe Good <moshe@squareup.com>
launchdarkly · Jul 28, 2022 · 256e97c · 256e97c
1 parent 19d2715
commit 256e97c
Show file tree

Hide file tree

Showing 6 changed files with 81 additions and 44 deletions.
diff --git a/internal/core/bigsegments/store.go b/internal/core/bigsegments/store.go
@@ -52,15 +52,13 @@ func DefaultBigSegmentStoreFactory(
 	// Currently, the only supported store type is Redis, and if Redis is enabled then big segments
 	// are enabled.
 	if allConfig.Redis.URL.IsDefined() {
-		bigSegmentRedis, err := newRedisBigSegmentStore(allConfig.Redis, envConfig.Prefix, false, loggers)
+		bigSegmentRedis, err := newRedisBigSegmentStore(allConfig.Redis, envConfig, false, loggers)
 		if err != nil {
 			return nil, err
 		}
 		return bigSegmentRedis, nil
 	} else if allConfig.DynamoDB.Enabled {
-		dbConfig := allConfig.DynamoDB
-		return newDynamoDBBigSegmentStore(
-			dbConfig.URL, aws.Config{}, loggers, dbConfig.TableName, envConfig.Prefix)
+		return newDynamoDBBigSegmentStore(allConfig.DynamoDB, envConfig, aws.Config{}, loggers)
 	}
 	return nil, nil
 }

diff --git a/internal/core/bigsegments/store_dynamodb.go b/internal/core/bigsegments/store_dynamodb.go
@@ -3,7 +3,8 @@ package bigsegments
 import (
 	"strconv"
 
-	ct "github.com/launchdarkly/go-configtypes"
+	"github.com/launchdarkly/ld-relay/v6/config"
+	"github.com/launchdarkly/ld-relay/v6/internal/core/sdks"
 	"gopkg.in/launchdarkly/go-sdk-common.v2/ldlog"
 	"gopkg.in/launchdarkly/go-sdk-common.v2/ldtime"
 
@@ -49,14 +50,15 @@ func dynamoDBPrefixedKey(prefix, key string) string {
 }
 
 func newDynamoDBBigSegmentStore(
-	url ct.OptURLAbsolute,
+	dbConfig config.DynamoDBConfig,
+	envConfig config.EnvConfig,
 	config aws.Config,
 	loggers ldlog.Loggers,
-	table string,
-	prefix string,
 ) (*dynamoDBBigSegmentStore, error) {
-	if url.IsDefined() {
-		config.Endpoint = aws.String(url.String())
+	endpoint, table, prefix := sdks.GetDynamoDBBasicProperties(dbConfig, envConfig)
+
+	if endpoint != nil {
+		config.Endpoint = endpoint
 	}
 
 	sess, err := session.NewSession(&config)

diff --git a/internal/core/bigsegments/store_dynamodb_test.go b/internal/core/bigsegments/store_dynamodb_test.go
@@ -1,3 +1,4 @@
+//go:build big_segment_external_store_tests
 // +build big_segment_external_store_tests
 
 package bigsegments
@@ -9,7 +10,7 @@ import (
 	"testing"
 	"time"
 
-	ct "github.com/launchdarkly/go-configtypes"
+	"github.com/launchdarkly/ld-relay/v6/config"
 	"gopkg.in/launchdarkly/go-sdk-common.v2/ldlog"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -84,9 +85,12 @@ func dynamoDBMakeOperations(store *dynamoDBBigSegmentStore) bigSegmentOperations
 func withDynamoDBStoreGeneric(prefix string) func(*testing.T, func(BigSegmentStore, bigSegmentOperations)) {
 	return func(t *testing.T, action func(BigSegmentStore, bigSegmentOperations)) {
 		require.NoError(t, clearTestData(prefix))
-		url, err := ct.NewOptURLAbsoluteFromString(localEndpoint)
-		require.NoError(t, err)
-		store, err := newDynamoDBBigSegmentStore(url, testDynamoDBConfig, ldlog.NewDisabledLoggers(), testTableName, prefix)
+		store, err := newDynamoDBBigSegmentStore(
+			config.DynamoDBConfig{TableName: testTableName},
+			config.EnvConfig{Prefix: prefix},
+			testDynamoDBConfig,
+			ldlog.NewDisabledLoggers(),
+		)
 		require.NoError(t, err)
 		require.NotNil(t, store)
 		defer store.Close()
@@ -155,21 +159,21 @@ func createTableIfNecessary() error {
 	}
 	createParams := dynamodb.CreateTableInput{
 		AttributeDefinitions: []*dynamodb.AttributeDefinition{
-			&dynamodb.AttributeDefinition{
+			{
 				AttributeName: aws.String(tablePartitionKey),
 				AttributeType: aws.String(dynamodb.ScalarAttributeTypeS),
 			},
-			&dynamodb.AttributeDefinition{
+			{
 				AttributeName: aws.String(tableSortKey),
 				AttributeType: aws.String(dynamodb.ScalarAttributeTypeS),
 			},
 		},
 		KeySchema: []*dynamodb.KeySchemaElement{
-			&dynamodb.KeySchemaElement{
+			{
 				AttributeName: aws.String(tablePartitionKey),
 				KeyType:       aws.String(dynamodb.KeyTypeHash),
 			},
-			&dynamodb.KeySchemaElement{
+			{
 				AttributeName: aws.String(tableSortKey),
 				KeyType:       aws.String(dynamodb.KeyTypeRange),
 			},

diff --git a/internal/core/bigsegments/store_redis.go b/internal/core/bigsegments/store_redis.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/go-redis/redis/v8"
 	"github.com/launchdarkly/ld-relay/v6/config"
+	"github.com/launchdarkly/ld-relay/v6/internal/core/sdks"
 )
 
 func redisLockKey(prefix string) string {
@@ -43,17 +44,19 @@ type redisBigSegmentStore struct {
 // newRedisBigSegmentStore creates an instance of RedisBigSegmentStore.
 func newRedisBigSegmentStore(
 	redisConfig config.RedisConfig,
-	prefix string,
+	envConfig config.EnvConfig,
 	checkOnStartup bool,
 	loggers ldlog.Loggers,
 ) (*redisBigSegmentStore, error) {
+	redisURL, prefix := sdks.GetRedisBasicProperties(redisConfig, envConfig)
+
 	opts := redis.UniversalOptions{}
 
 	// Relay's Redis configuration allows setting the server address either as a URL or as a
 	// host & port, but our config validation logic simplifies this so that it is always a URL.
 	// However, it is still possible to set the Password and TLS options separately from the
 	// URL, so we still need to check for those.
-	parsed, err := redis.ParseURL(redisConfig.URL.String())
+	parsed, err := redis.ParseURL(redisURL)
 	if err != nil {
 		return nil, err
 	}

diff --git a/internal/core/bigsegments/store_redis_test.go b/internal/core/bigsegments/store_redis_test.go
@@ -1,3 +1,4 @@
+//go:build big_segment_external_store_tests
 // +build big_segment_external_store_tests
 
 package bigsegments
@@ -6,10 +7,11 @@ import (
 	"context"
 	"testing"
 
-	"gopkg.in/launchdarkly/go-sdk-common.v2/ldlog"
+	"github.com/launchdarkly/ld-relay/v6/config"
 
 	"github.com/launchdarkly/go-configtypes"
-	"github.com/launchdarkly/ld-relay/v6/config"
+	"gopkg.in/launchdarkly/go-sdk-common.v2/ldlog"
+
 	"github.com/stretchr/testify/require"
 )
 
@@ -25,7 +27,7 @@ func TestRedisGenericAll(t *testing.T) {
 func makeStore(t *testing.T) *redisBigSegmentStore {
 	redisConfig := config.RedisConfig{}
 	redisConfig.URL, _ = configtypes.NewOptURLAbsoluteFromString("redis://127.0.0.1:6379")
-	store, err := newRedisBigSegmentStore(redisConfig, testPrefix, true, ldlog.NewDisabledLoggers())
+	store, err := newRedisBigSegmentStore(redisConfig, config.EnvConfig{Prefix: testPrefix}, true, ldlog.NewDisabledLoggers())
 	require.NoError(t, err)
 	require.NoError(t, store.client.FlushAll(context.Background()).Err())
 	return store

diff --git a/internal/core/sdks/data_stores.go b/internal/core/sdks/data_stores.go
@@ -121,12 +121,14 @@ func ConfigureDataStore(
 	return ldcomponents.InMemoryDataStore(), DataStoreEnvironmentInfo{}, nil
 }
 
-func makeRedisDataStoreBuilder(
-	allConfig config.Config,
+// GetRedisBasicProperties transforms the configuration properties to the standard parameters
+// used for Redis. This function is exported to ensure consistency between the SDK
+// configuration and the internal big segment store for Redis.
+func GetRedisBasicProperties(
+	dbConfig config.RedisConfig,
 	envConfig config.EnvConfig,
-) (builder *ldredis.DataStoreBuilder, url string) {
-	dbConfig := allConfig.Redis
-	redisURL := dbConfig.URL.String()
+) (redisURL, prefix string) {
+	redisURL = dbConfig.URL.String()
 
 	if dbConfig.TLS {
 		if strings.HasPrefix(redisURL, "redis:") {
@@ -135,42 +137,68 @@ func makeRedisDataStoreBuilder(
 		}
 	}
 
+	prefix = envConfig.Prefix
+	if prefix == "" {
+		prefix = ldredis.DefaultPrefix
+	}
+
+	return
+}
+
+func makeRedisDataStoreBuilder(
+	allConfig config.Config,
+	envConfig config.EnvConfig,
+) (builder *ldredis.DataStoreBuilder, url string) {
+	redisURL, prefix := GetRedisBasicProperties(allConfig.Redis, envConfig)
+
 	var dialOptions []redigo.DialOption
-	if dbConfig.Password != "" {
-		dialOptions = append(dialOptions, redigo.DialPassword(dbConfig.Password))
+	if allConfig.Redis.Password != "" {
+		dialOptions = append(dialOptions, redigo.DialPassword(allConfig.Redis.Password))
 	}
 
 	b := ldredis.DataStore().
 		URL(redisURL).
-		Prefix(envConfig.Prefix).
+		Prefix(prefix).
 		DialOptions(dialOptions...)
 	return b, redisURL
 }
 
-func makeDynamoDBDataStoreBuilder(
-	allConfig config.Config,
+// GetDynamoDBBasicProperties transforms the configuration properties to the standard parameters
+// used for DynamoDB. This function is exported to ensure consistency between the SDK
+// configuration and the internal big segment store for DynamoDB.
+func GetDynamoDBBasicProperties(
+	dbConfig config.DynamoDBConfig,
 	envConfig config.EnvConfig,
-) (*lddynamodb.DataStoreBuilder, string, error) {
+) (endpoint *string, tableName, prefix string) {
 	// Note that the global TableName can be omitted if you specify a TableName for each environment
 	// (this is why we need an Enabled property here, since the other properties are all optional).
 	// You can also specify a prefix for each environment, as with the other databases.
-	dbConfig := allConfig.DynamoDB
-	tableName := envConfig.TableName
+	tableName = envConfig.TableName
 	if tableName == "" {
 		tableName = dbConfig.TableName
 	}
+
+	prefix = envConfig.Prefix
+
+	if dbConfig.URL.IsDefined() {
+		endpoint = aws.String(dbConfig.URL.String())
+	}
+
+	return
+}
+
+func makeDynamoDBDataStoreBuilder(
+	allConfig config.Config,
+	envConfig config.EnvConfig,
+) (*lddynamodb.DataStoreBuilder, string, error) {
+	endpoint, tableName, prefix := GetDynamoDBBasicProperties(allConfig.DynamoDB, envConfig)
 	if tableName == "" {
 		return nil, "", errDynamoDBWithNoTableName
 	}
 	builder := lddynamodb.DataStore(tableName).
-		Prefix(envConfig.Prefix)
-	if dbConfig.URL.IsDefined() {
-		awsOptions := session.Options{
-			Config: aws.Config{
-				Endpoint: aws.String(dbConfig.URL.String()),
-			},
-		}
-		builder.SessionOptions(awsOptions)
+		Prefix(prefix)
+	if endpoint != nil {
+		builder.SessionOptions(session.Options{Config: aws.Config{Endpoint: endpoint}})
 	}
 	return builder, tableName, nil
 }