chore: Introduce opt-in flag to enable persistence support (#250)

The SDK test harness has historically been self-sufficient. With the
introduction of persistence-based unit tests, that is no longer the
case. Externally started services are required for each persistence
technology supported.

Since we want to retain the ease of use associated with a our current
`go run .` usage, we are going to disable persistence support by
default. Users will have to provide a `-enable-persistence-tests` if
they want the test harness to attempt those tests.

SDKs under test are still required to provide the appropriate
capabilities. In the event those capabilities are provided, but the test
harness flag is not provided, persistence tests will be skipped.

docs/running.md

@@ -16,6 +16,7 @@ Options besides `-url`:
 * `-junit <FILEPATH>` - writes test results in JUnit XML format to the specified file
 * `-debug` - enables verbose logging of test actions for failed tests
 * `-debug-all` - enables verbose logging of test actions for all tests
+* `-enable-persistence-tests` - enables tests that require external persistence (e.g. a database like redis)
 * `-record-failures` - record failed test IDs to the given file. Recorded tests can be skipped by the next run of 
 the harness via `-skip-from`.
 * `-skip-from` - skips any test IDs recorded in the specified file. May be used in conjunction with `-record-failures`

docs/service_spec.md

@@ -102,11 +102,23 @@ v4 of the event schema originally required a `contextKeys` property on all featu
 
 This means that the SDK supports technology migrations, a feature which allows customers to migrate between data sources using well-defined migration stages.
 
-#### Capability `"persistent-data-store"`
+#### Capability `"persistent-data-store-{integration}"`
 
-This means the SDK is capable of interacting with external persistent data stores. The test harness must further identify which store types are supported through additional capabilities listed below.
+This means the SDK is capable of interacting with external persistent data stores. Each capability defines the specific persistence technology the SDK is capable of supporting. The full list of supported integrations is as follows:
 
-- `persistent-data-store-redis`: This means the SDK is capable of interacting with a Redis data store.
+- `consul`
+- `dynamodb`
+- `redis`
+
+The persistence store tests are unique in that they rely on external services to be available prior to running the test harness. Because of this, the test harness will not run these tests by default, even if the capabilities exist, without also provided the `-enable-persistence-tests` flag.
+
+These services can be easily provided through the use of docker. Example commands are shown below for your convenience.
+
+```shell
+docker run -p 8000:8000 amazon/dynamodb-local
+docker run -p 8500:8500 hashicorp/consul
+docker run -p 6379:6379 redis
+```
 
 #### Capability `"polling-gzip"`
 

framework/harness/harness.go

@@ -108,6 +108,7 @@ func NewTestHarness(
 	testServiceBaseURL string,
 	testHarnessExternalHostname string,
 	testHarnessPort int,
+	testHarnessEnablePersistenceTests bool,
 	statusQueryTimeout time.Duration,
 	debugLogger framework.Logger,
 	startupOutput io.Writer,
@@ -129,6 +130,21 @@ func NewTestHarness(
 	if err != nil {
 		return nil, err
 	}
+
+	// If we aren't running persistence tests, remove the capabilities that would enable it.
+	if !testHarnessEnablePersistenceTests {
+		filteredCapabilities := make([]string, 0, len(testServiceInfo.Capabilities))
+		for _, c := range testServiceInfo.Capabilities {
+			if c == servicedef.CapabilityPersistentDataStoreRedis ||
+				c == servicedef.CapabilityPersistentDataStoreDynamoDB ||
+				c == servicedef.CapabilityPersistentDataStoreConsul {
+				debugLogger.Printf("Disabling capability %q because persistence tests are disabled", c)
+				continue
+			}
+			filteredCapabilities = append(filteredCapabilities, c)
+		}
+		testServiceInfo.Capabilities = filteredCapabilities
+	}
 	h.testServiceInfo = testServiceInfo
 
 	if err := startServer(testHarnessPort, http.HandlerFunc(h.serveHTTP)); err != nil {

main.go

@@ -55,6 +55,7 @@ func run(params commandParams) (*ldtest.Results, error) {
 		params.serviceURL,
 		params.host,
 		params.port,
+		params.enablePersistenceTests,
 		time.Duration(params.queryTimeoutSeconds)*time.Second,
 		mainDebugLogger,
 		os.Stdout,

params.go

@@ -9,17 +9,18 @@ import (
 )
 
 type commandParams struct {
-	serviceURL          string
-	port                int
-	host                string
-	filters             ldtest.RegexFilters
-	stopServiceAtEnd    bool
-	debug               bool
-	debugAll            bool
-	jUnitFile           string
-	recordFailures      string
-	skipFile            string
-	queryTimeoutSeconds int
+	serviceURL             string
+	port                   int
+	host                   string
+	filters                ldtest.RegexFilters
+	stopServiceAtEnd       bool
+	debug                  bool
+	debugAll               bool
+	enablePersistenceTests bool
+	jUnitFile              string
+	recordFailures         string
+	skipFile               string
+	queryTimeoutSeconds    int
 }
 
 func (c *commandParams) Read(args []string) bool {
@@ -33,6 +34,8 @@ func (c *commandParams) Read(args []string) bool {
 	fs.BoolVar(&c.stopServiceAtEnd, "stop-service-at-end", false, "tell test service to exit after the test run")
 	fs.BoolVar(&c.debug, "debug", false, "enable debug logging for failed tests")
 	fs.BoolVar(&c.debugAll, "debug-all", false, "enable debug logging for all tests")
+	fs.BoolVar(&c.enablePersistenceTests, "enable-persistence-tests", false,
+		"enable tests that require external persistence support")
 	fs.StringVar(&c.jUnitFile, "junit", "", "write JUnit XML output to the specified path")
 	fs.StringVar(&c.recordFailures, "record-failures", "", "record failed test IDs to the given file.\n"+
 		"recorded tests can be skipped by the next run of the harness via -skip-from")

sdktests/server_side_persistence_base.go

@@ -41,7 +41,10 @@ const (
 )
 
 func doServerSidePersistentTests(t *ldtest.T) {
+	ranAtLeastOnce := false
+
 	if t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreRedis) {
+		ranAtLeastOnce = true
 		rdb := redis.NewClient(&redis.Options{
 			Addr:     "localhost:6379",
 			Password: "", // no password set
@@ -52,6 +55,7 @@ func doServerSidePersistentTests(t *ldtest.T) {
 	}
 
 	if t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreConsul) {
+		ranAtLeastOnce = true
 		config := consul.DefaultConfig()
 		config.Address = "localhost:8500"
 
@@ -62,6 +66,7 @@ func doServerSidePersistentTests(t *ldtest.T) {
 	}
 
 	if t.Capabilities().Has(servicedef.CapabilityPersistentDataStoreDynamoDB) {
+		ranAtLeastOnce = true
 		mySession := session.Must(session.NewSession(
 			aws.NewConfig().
 				WithRegion("us-east-1").
@@ -81,6 +86,10 @@ func doServerSidePersistentTests(t *ldtest.T) {
 
 		t.Run("dynamodb", newServerSidePersistentTests(t, &store, "").Run)
 	}
+
+	if !ranAtLeastOnce {
+		t.Skip()
+	}
 }
 
 type PersistentStore interface {

sdktests/testsuite_entry_point.go

@@ -92,10 +92,7 @@ func doAllServerSideTests(t *ldtest.T) {
 	t.Run("migrations", doServerSideMigrationTests)
 	t.Run("hooks", doCommonHooksTests)
 	t.Run("wrapper", doServerSideWrapperTests)
-
-	if t.Capabilities().Has(servicedef.CapabilityPersistentDataStore) {
-		t.Run("persistent data store", doServerSidePersistentTests)
-	}
+	t.Run("persistent data store", doServerSidePersistentTests)
 }
 
 func doAllClientSideTests(t *ldtest.T) {

servicedef/service_params.go

@@ -39,7 +39,6 @@ const (
 	CapabilityPollingGzip                 = "polling-gzip"
 	CapabilityEvaluationHooks             = "evaluation-hooks"
 	CapabilityClientPrereqEvents          = "client-prereq-events"
-	CapabilityPersistentDataStore         = "persistent-data-store"
 	CapabilityPersistentDataStoreRedis    = "persistent-data-store-redis"
 	CapabilityPersistentDataStoreConsul   = "persistent-data-store-consul"
 	CapabilityPersistentDataStoreDynamoDB = "persistent-data-store-dynamodb"