(offline mode #2) add configuration for offline mode (#199)

config/config.go

@@ -44,6 +44,8 @@ const (
 	//
 	// For instance, if EnvDataStorePrefix is "LD-$CID", the value of that setting for an environment
 	// whose ID is "12345" would be "LD-12345".
+	//
+	// The same convention is used in OfflineModeConfig.
 	AutoConfigEnvironmentIDPlaceholder = "$CID"
 )
 
@@ -78,6 +80,7 @@ var DefaultLoggers = logging.MakeDefaultLoggers() //nolint:gochecknoglobals
 type Config struct {
 	Main        MainConfig
 	AutoConfig  AutoConfigConfig
+	OfflineMode OfflineModeConfig
 	Events      EventsConfig
 	Redis       RedisConfig
 	Consul      ConsulConfig
@@ -124,6 +127,14 @@ type AutoConfigConfig struct {
 	EnvAllowedOrigin      ct.OptStringList `conf:"ENV_ALLOWED_ORIGIN"`
 }
 
+// OfflineModeConfig contains configuration parameters for the offline/file data source feature.
+type OfflineModeConfig struct {
+	FileDataSource        string           `conf:"FILE_DATA_SOURCE"`
+	EnvDatastorePrefix    string           `conf:"ENV_DATASTORE_PREFIX"`
+	EnvDatastoreTableName string           `conf:"ENV_DATASTORE_TABLE_NAME"`
+	EnvAllowedOrigin      ct.OptStringList `conf:"ENV_ALLOWED_ORIGIN"`
+}
+
 // EventsConfig contains configuration parameters for proxying events.
 //
 // Since configuration options can be set either programmatically, or from a file, or from environment

config/config_from_env.go

@@ -41,6 +41,21 @@ func LoadConfigFromEnvironmentBase(c *Config, loggers ldlog.Loggers) ct.Validati
 
 	reader.ReadStruct(&c.AutoConfig, false)
 
+	reader.ReadStruct(&c.OfflineMode, false)
+
+	// The following properties have the same environment variable names in AutoConfigConfig and in
+	// OfflineModeConfig, because only one of those can be used at a time. We'll blank them out for
+	// whichever section is not being used.
+	if c.AutoConfig.Key != "" {
+		c.OfflineMode.EnvAllowedOrigin = ct.OptStringList{}
+		c.OfflineMode.EnvDatastorePrefix = ""
+		c.OfflineMode.EnvDatastoreTableName = ""
+	} else if c.OfflineMode.FileDataSource != "" {
+		c.AutoConfig.EnvAllowedOrigin = ct.OptStringList{}
+		c.AutoConfig.EnvDatastorePrefix = ""
+		c.AutoConfig.EnvDatastoreTableName = ""
+	}
+
 	reader.ReadStruct(&c.Events, false)
 	rejectObsoleteVariableName("EVENTS_SAMPLING_INTERVAL", "", reader)
 

config/config_validation.go

@@ -10,10 +10,13 @@ import (
 )
 
 var (
-	errTLSEnabledWithoutCertOrKey  = errors.New("TLS cert and key are required if TLS is enabled")
-	errAutoConfPropertiesWithNoKey = errors.New("must specify auto-configuration key if other auto-configuration properties are set")
-	errAutoConfWithEnvironments    = errors.New("cannot configure specific environments if auto-configuration is enabled")
-	errAutoConfWithoutDBDisambig   = errors.New(`when using auto-configuration with database storage, database prefix (or,` +
+	errTLSEnabledWithoutCertOrKey      = errors.New("TLS cert and key are required if TLS is enabled")
+	errAutoConfPropertiesWithNoKey     = errors.New("must specify auto-configuration key if other auto-configuration properties are set")
+	errAutoConfWithEnvironments        = errors.New("cannot configure specific environments if auto-configuration is enabled")
+	errFileDataWithAutoConf            = errors.New("cannot specify both auto-configuration key and file data source")
+	errOfflineModePropertiesWithNoFile = errors.New("must specify offline mode filename if other offline mode properties are set")
+	errOfflineModeWithEnvironments     = errors.New("cannot configure specific environments if offline mode is enabled")
+	errAutoConfWithoutDBDisambig       = errors.New(`when using auto-configuration with database storage, database prefix (or,` +
 		` if using DynamoDB, table name) must be specified and must contain "` + AutoConfigEnvironmentIDPlaceholder + `"`)
 	errRedisURLWithHostAndPort = errors.New("please specify Redis URL or host/port, but not both")
 	errRedisBadHostname        = errors.New("invalid Redis hostname")
@@ -76,6 +79,19 @@ func validateConfigEnvironments(result *ct.ValidationResult, c *Config) {
 	} else if len(c.Environment) != 0 {
 		result.AddError(nil, errAutoConfWithEnvironments)
 	}
+	if c.OfflineMode.FileDataSource == "" {
+		if c.OfflineMode.EnvDatastorePrefix != "" || c.OfflineMode.EnvDatastoreTableName != "" ||
+			len(c.OfflineMode.EnvAllowedOrigin.Values()) != 0 {
+			result.AddError(nil, errOfflineModePropertiesWithNoFile)
+		}
+	} else {
+		if c.AutoConfig.Key != "" {
+			result.AddError(nil, errFileDataWithAutoConf)
+		}
+		if len(c.Environment) != 0 {
+			result.AddError(nil, errOfflineModeWithEnvironments)
+		}
+	}
 
 	for envName, envConfig := range c.Environment {
 		if envConfig.SDKKey == "" {

config/test_data_configs_invalid_test.go

@@ -19,6 +19,11 @@ func makeInvalidConfigs() []testDataInvalidConfig {
 		makeInvalidConfigAutoConfAllowedOriginWithNoKey(),
 		makeInvalidConfigAutoConfPrefixWithNoKey(),
 		makeInvalidConfigAutoConfTableNameWithNoKey(),
+		makeInvalidConfigFileDataWithAutoConfKey(),
+		makeInvalidConfigFileDataWithEnvironments(),
+		makeInvalidConfigOfflineModeAllowedOriginWithNoFile(),
+		makeInvalidConfigOfflineModePrefixWithNoFile(),
+		makeInvalidConfigOfflineModeTableNameWithNoFile(),
 		makeInvalidConfigRedisInvalidHostname(),
 		makeInvalidConfigRedisInvalidDockerPort(),
 		makeInvalidConfigRedisConflictingParams(),
@@ -146,6 +151,70 @@ EnvDatastoreTableName = table
 	return c
 }
 
+func makeInvalidConfigFileDataWithAutoConfKey() testDataInvalidConfig {
+	c := testDataInvalidConfig{name: "file data source with auto-config key"}
+	c.envVarsError = errFileDataWithAutoConf.Error()
+	c.envVars = map[string]string{
+		"FILE_DATA_SOURCE": "my-file-path",
+		"AUTO_CONFIG_KEY":  "autokey",
+	}
+	c.fileContent = `
+[OfflineMode]
+FileDataSource = my-file-path
+
+[AutoConfig]
+Key = autokey
+`
+	return c
+}
+
+func makeInvalidConfigFileDataWithEnvironments() testDataInvalidConfig {
+	c := testDataInvalidConfig{name: "file data source with environments"}
+	c.envVarsError = errOfflineModeWithEnvironments.Error()
+	c.envVars = map[string]string{
+		"FILE_DATA_SOURCE": "my-file-path",
+		"LD_ENV_envname":   "sdk-key",
+	}
+	c.fileContent = `
+[OfflineMode]
+FileDataSource = my-file-path
+
+[Environment "envname"]
+SDKKey = sdk-key
+`
+	return c
+}
+
+func makeInvalidConfigOfflineModeAllowedOriginWithNoFile() testDataInvalidConfig {
+	c := testDataInvalidConfig{name: "offline mode allowed origin with no file"}
+	c.fileError = errOfflineModePropertiesWithNoFile.Error()
+	c.fileContent = `
+[OfflineMode]
+EnvAllowedOrigin = http://origin
+`
+	return c
+}
+
+func makeInvalidConfigOfflineModePrefixWithNoFile() testDataInvalidConfig {
+	c := testDataInvalidConfig{name: "offline mode prefix with no file"}
+	c.fileError = errOfflineModePropertiesWithNoFile.Error()
+	c.fileContent = `
+[OfflineMode]
+EnvDatastorePrefix = prefix
+`
+	return c
+}
+
+func makeInvalidConfigOfflineModeTableNameWithNoFile() testDataInvalidConfig {
+	c := testDataInvalidConfig{name: "offline mode table name with no file"}
+	c.fileError = errOfflineModePropertiesWithNoFile.Error()
+	c.fileContent = `
+[OfflineMode]
+EnvDatastoreTableName = table
+`
+	return c
+}
+
 func makeInvalidConfigRedisInvalidHostname() testDataInvalidConfig {
 	c := testDataInvalidConfig{name: "Redis - invalid hostname"}
 	c.envVarsError = "invalid Redis hostname"

config/test_data_configs_valid_test.go

@@ -51,6 +51,7 @@ func makeValidConfigs() []testDataValidConfig {
 		makeValidConfigAllBaseProperties(),
 		makeValidConfigAutoConfig(),
 		makeValidConfigAutoConfigWithDatabase(),
+		makeValidConfigFileData(),
 		makeValidConfigRedisMinimal(),
 		makeValidConfigRedisAll(),
 		makeValidConfigRedisURL(),
@@ -260,6 +261,21 @@ Enabled = true
 	return c
 }
 
+func makeValidConfigFileData() testDataValidConfig {
+	c := testDataValidConfig{name: "file data properties"}
+	c.makeConfig = func(c *Config) {
+		c.OfflineMode.FileDataSource = "my-file-path"
+	}
+	c.envVars = map[string]string{
+		"FILE_DATA_SOURCE": "my-file-path",
+	}
+	c.fileContent = `
+[OfflineMode]
+FileDataSource = my-file-path
+`
+	return c
+}
+
 func makeValidConfigRedisMinimal() testDataValidConfig {
 	c := testDataValidConfig{name: "Redis - minimal parameters"}
 	c.makeConfig = func(c *Config) {

docs/configuration.md

@@ -86,6 +86,20 @@ Property in file         | Environment var            | Type   | Default | Descr
 _(6)_ When using a database store, if there are multiple environments, it is necessary to have a different prefix for each environment (or, if using DynamoDB, a different table name). The `envDataStorePrefix` and `envDatastoreTableName` properties support this by recognizing the special symbol `$CID` as a placeholder for the environment's client-side ID. For instance, if an environment's ID is `1234567890abcdef` and you set `envDatastorePrefix` to `ld-flags-$CID`, the actual prefix used for that environment will be `ld-flags-1234567890abcdef`.
 
 
+### File section: `[OfflineMode]`
+
+This section can only be used if the [offline mode](https://docs.launchdarkly.com/home/advanced/relay-proxy/offline-mode) feature is enabled for your account.
+
+Property in file         | Environment var            | Type   | Default | Description
+------------------------ | -------------------------- | :----: | :------ | -----------
+`fileDataSource`         | `FILE_DATA_SOURCE`         | String |         | Path to the offline mode data file that you have downloaded from LaunchDarkly.
+`envDatastorePrefix`     | `ENV_DATASTORE_PREFIX`     | String |         | If using a Redis, Consul, or DynamoDB store, this string will be added to all database keys to distinguish them from any other environments that are using the database. _(6)_
+`envDatastoreTableName ` | `ENV_DATASTORE_TABLE_NAME` | String |         | If using a DynamoDB store, this specifies the table name. _(6)_
+`envAllowedOrigin`       | `ENV_ALLOWED_ORIGIN`       | URI    |         | If provided, adds CORS headers to prevent access from other domains. This variable can be provided multiple times per environment (if using the `ENV_ALLOWED_ORIGIN` variable, specify a comma-delimited list).
+
+Note that the last three properties have the same meanings and the same environment variables names as the corresponding properties in the `[AutoConfig]` section described above. It is not possible to use `[OfflineMode]` and `[AutoConfig]` at the same time.
+
+
 ### File section: `[Events]`
 
 To learn more, read [Forwarding events](./events.md)

internal/filedata/archive_manager_test.go

@@ -225,10 +225,10 @@ func TestFileDeletedAndThenRecreatedWithInvalidDataAndThenValidData(t *testing.T
 		p.expectEnvironmentsAdded(testEnv1, testEnv2)
 
 		require.NoError(t, os.Remove(p.filePath))
-		<-time.After(time.Millisecond * 100)
+		<-time.After(time.Millisecond * 200)
 
 		writeMalformedArchive(p.filePath)
-		<-time.After(time.Millisecond * 100)
+		<-time.After(time.Millisecond * 200)
 
 		testEnv1a := testEnv1.withMetadataChange().withSDKDataChange()
 		writeArchive(t, p.filePath, false, nil, testEnv1a, testEnv2)