Merge pull request #163 from launchdarkly/eb/ch82207/status-detail

(v6 - #2) add connection status & data store status to status resource

core/relay_core.go

@@ -21,7 +21,6 @@ import (
 	"github.com/launchdarkly/ld-relay/v6/core/sdks"
 	"github.com/launchdarkly/ld-relay/v6/core/streams"
 	"gopkg.in/launchdarkly/go-sdk-common.v2/ldlog"
-	ld "gopkg.in/launchdarkly/go-server-sdk.v5"
 )
 
 var (
@@ -60,20 +59,6 @@ type RelayCore struct {
 	lock                          sync.RWMutex
 }
 
-// ClientFactoryFromLDClientFactory translates from the client factory type that we expose to host
-// applications, which uses the real LDClient type, to the more general factory type that we use
-// internally which uses the sdks.ClientFactoryFunc abstraction. The latter makes our code a bit
-// cleaner and easier to test, but isn't of any use when hosting Relay in an application.
-func ClientFactoryFromLDClientFactory(fn func(sdkKey config.SDKKey, config ld.Config) (*ld.LDClient, error)) sdks.ClientFactoryFunc {
-	if fn == nil {
-		return nil
-	}
-	return func(sdkKey config.SDKKey, config ld.Config) (sdks.LDClientContext, error) {
-		client, err := fn(sdkKey, config)
-		return client, err
-	}
-}
-
 // NewRelayCore creates and configures an instance of RelayCore, and immediately starts initializing
 // all configured environments.
 func NewRelayCore(

core/relay_core_endpoints_status.go

@@ -5,8 +5,19 @@ import (
 	"net/http"
 	"regexp"
 
+	"gopkg.in/launchdarkly/go-sdk-common.v2/ldtime"
+
 	"github.com/launchdarkly/ld-relay/v6/core/config"
+	"github.com/launchdarkly/ld-relay/v6/core/relayenv"
 	ld "gopkg.in/launchdarkly/go-server-sdk.v5"
+	"gopkg.in/launchdarkly/go-server-sdk.v5/interfaces"
+)
+
+const (
+	statusEnvConnected    = "connected"
+	statusEnvDisconnected = "disconnected"
+	statusRelayHealthy    = "healthy"
+	statusRelayDegraded   = "degraded"
 )
 
 type statusRep struct {
@@ -17,17 +28,47 @@ type statusRep struct {
 }
 
 type environmentStatusRep struct {
-	SDKKey    string `json:"sdkKey"`
-	EnvID     string `json:"envId,omitempty"`
-	MobileKey string `json:"mobileKey,omitempty"`
-	Status    string `json:"status"`
+	SDKKey           string              `json:"sdkKey"`
+	EnvID            string              `json:"envId,omitempty"`
+	EnvKey           string              `json:"envKey,omitempty"`
+	EnvName          string              `json:"envName,omitempty"`
+	ProjKey          string              `json:"projKey,omitempty"`
+	ProjName         string              `json:"projName,omitempty"`
+	MobileKey        string              `json:"mobileKey,omitempty"`
+	ExpiringSDKKey   string              `json:"expiringSdkKey,omitempty"`
+	Status           string              `json:"status"`
+	ConnectionStatus connectionStatusRep `json:"connectionStatus"`
+	DataStoreStatus  *dataStoreStatusRep `json:"dataStoreStatus,omitempty"`
+}
+
+type connectionStatusRep struct {
+	State      interfaces.DataSourceState `json:"state"`
+	StateSince ldtime.UnixMillisecondTime `json:"stateSince"`
+	LastError  *connectionErrorRep        `json:"lastError,omitempty"`
+}
+
+type connectionErrorRep struct {
+	Kind interfaces.DataSourceErrorKind `json:"kind"`
+	Time ldtime.UnixMillisecondTime     `json:"time"`
+}
+
+type dataStoreStatusRep struct {
+	State      string                     `json:"state"`
+	StateSince ldtime.UnixMillisecondTime `json:"stateSince"`
 }
 
-var hexdigit = regexp.MustCompile(`[a-fA-F\d]`) //nolint:gochecknoglobals
+var (
+	hexDigitRegex    = regexp.MustCompile(`[a-fA-F\d]`)        //nolint:gochecknoglobals
+	alphaPrefixRegex = regexp.MustCompile(`^[a-z][a-z][a-z]-`) //nolint:gochecknoglobals
+)
 
-func obscureKey(key string) string {
-	if len(key) > 8 {
-		return key[0:4] + hexdigit.ReplaceAllString(key[4:len(key)-5], "*") + key[len(key)-5:]
+// ObscureKey returns an obfuscated version of an SDK key or mobile key.
+func ObscureKey(key string) string {
+	if alphaPrefixRegex.MatchString(key) {
+		return key[0:4] + ObscureKey(key[4:])
+	}
+	if len(key) > 4 {
+		return hexDigitRegex.ReplaceAllString(key[:len(key)-5], "*") + key[len(key)-5:]
 	}
 	return key
 }
@@ -48,28 +89,61 @@ func statusHandler(core *RelayCore) http.Handler {
 			for _, c := range clientCtx.GetCredentials() {
 				switch c := c.(type) {
 				case config.SDKKey:
-					status.SDKKey = obscureKey(string(c))
+					status.SDKKey = ObscureKey(string(c))
 				case config.MobileKey:
-					status.MobileKey = obscureKey(string(c))
+					status.MobileKey = ObscureKey(string(c))
 				case config.EnvironmentID:
 					status.EnvID = string(c)
 				}
 			}
 
 			client := clientCtx.GetClient()
-			if client == nil || !client.Initialized() {
-				status.Status = "disconnected"
+			if client == nil {
+				status.Status = statusEnvDisconnected
+				status.ConnectionStatus.State = interfaces.DataSourceStateInitializing
+				status.ConnectionStatus.StateSince = ldtime.UnixMillisFromTime(clientCtx.GetCreationTime())
 				healthy = false
 			} else {
-				status.Status = "connected"
+				if client.Initialized() {
+					status.Status = statusEnvConnected
+				} else {
+					status.Status = statusEnvDisconnected
+					healthy = false
+				}
+				sourceStatus := client.GetDataSourceStatus()
+				status.ConnectionStatus = connectionStatusRep{
+					State:      sourceStatus.State,
+					StateSince: ldtime.UnixMillisFromTime(sourceStatus.StateSince),
+				}
+				if sourceStatus.LastError.Kind != "" {
+					status.ConnectionStatus.LastError = &connectionErrorRep{
+						Kind: sourceStatus.LastError.Kind,
+						Time: ldtime.UnixMillisFromTime(sourceStatus.LastError.Time),
+					}
+				}
+				storeStatus := client.GetDataStoreStatus()
+				status.DataStoreStatus = &dataStoreStatusRep{
+					State:      "VALID",
+					StateSince: ldtime.UnixMillisFromTime(storeStatus.LastUpdated),
+				}
+				if !storeStatus.Available {
+					status.DataStoreStatus.State = "INTERRUPTED"
+				}
+			}
+
+			statusKey := clientCtx.GetName()
+			if core.envLogNameMode == relayenv.LogNameIsEnvID {
+				// If we're identifying environments by environment ID in the log (which we do if there's any
+				// chance that the environment name could change) then we should also identify them that way here.
+				statusKey = status.EnvID
 			}
-			resp.Environments[clientCtx.GetName()] = status
+			resp.Environments[statusKey] = status
 		}
 
 		if healthy {
-			resp.Status = "healthy"
+			resp.Status = statusRelayHealthy
 		} else {
-			resp.Status = "degraded"
+			resp.Status = statusRelayDegraded
 		}
 
 		data, _ := json.Marshal(resp)

core/relay_core_endpoints_test.go

@@ -27,6 +27,15 @@ func buildPreRoutedRequest(verb string, body []byte, headers http.Header, vars m
 	return req
 }
 
+func TestObscureKey(t *testing.T) {
+	assert.Equal(t, "********-**-*89abc", ObscureKey("def01234-56-789abc"))
+	assert.Equal(t, "sdk-********-**-*89abc", ObscureKey("sdk-def01234-56-789abc"))
+	assert.Equal(t, "mob-********-**-*89abc", ObscureKey("mob-def01234-56-789abc"))
+	assert.Equal(t, "89abc", ObscureKey("89abc"))
+	assert.Equal(t, "9abc", ObscureKey("9abc"))
+	assert.Equal(t, "sdk-9abc", ObscureKey("sdk-9abc"))
+}
+
 func TestReportFlagEvalFailsallowMethodOptionsHandlerWithUninitializedClientAndStore(t *testing.T) {
 	headers := make(http.Header)
 	headers.Set("Content-Type", "application/json")

core/relay_core_test.go

@@ -19,7 +19,7 @@ import (
 )
 
 func makeBasicCore(config c.Config) (*RelayCore, error) {
-	return NewRelayCore(config, ldlog.NewDefaultLoggers(), testclient.FakeLDClientFactory(true), "", "", false)
+	return NewRelayCore(config, ldlog.NewDisabledLoggers(), testclient.FakeLDClientFactory(true), "", "", false)
 }
 
 func TestNewRelayCoreRejectsConfigWithContradictoryProperties(t *testing.T) {
@@ -144,7 +144,7 @@ func TestRelayCoreWaitForAllEnvironments(t *testing.T) {
 	}
 
 	t.Run("returns nil if all environments initialize successfully", func(t *testing.T) {
-		core, err := NewRelayCore(config, ldlog.NewDefaultLoggers(), testclient.FakeLDClientFactory(true), "", "", false)
+		core, err := NewRelayCore(config, ldlog.NewDisabledLoggers(), testclient.FakeLDClientFactory(true), "", "", false)
 		require.NoError(t, err)
 		defer core.Close()
 
@@ -160,7 +160,7 @@ func TestRelayCoreWaitForAllEnvironments(t *testing.T) {
 			}
 			return testclient.FakeLDClientFactory(true)(sdkKey, config)
 		}
-		core, err := NewRelayCore(config, ldlog.NewDefaultLoggers(), oneEnvFails, "", "", false)
+		core, err := NewRelayCore(config, ldlog.NewDisabledLoggers(), oneEnvFails, "", "", false)
 		require.NoError(t, err)
 		defer core.Close()
 

core/relayenv/env_context.go

@@ -92,6 +92,9 @@ type EnvContext interface {
 
 	// SetSecureMode changes the secure mode setting.
 	SetSecureMode(bool)
+
+	// GetCreationTime returns the time that this EnvContext was created.
+	GetCreationTime() time.Time
 }
 
 // GetEnvironmentID is a helper for extracting the EnvironmentID, if any, from the set of credentials.

core/relayenv/env_context_impl.go

@@ -67,6 +67,7 @@ type envContextImpl struct {
 	globalLoggers    ldlog.Loggers
 	ttl              time.Duration
 	initErr          error
+	creationTime     time.Time
 }
 
 // Implementation of the DataStoreQueries interface that the streams package uses as an abstraction of
@@ -136,6 +137,7 @@ func NewEnvContext(
 		metricsManager:   metricsManager,
 		globalLoggers:    loggers,
 		ttl:              envConfig.TTL.GetOrElse(0),
+		creationTime:     time.Now(),
 	}
 
 	envStreams := streams.NewEnvStreams(
@@ -406,6 +408,10 @@ func (c *envContextImpl) SetSecureMode(secureMode bool) {
 	c.secureMode = secureMode
 }
 
+func (c *envContextImpl) GetCreationTime() time.Time {
+	return c.creationTime
+}
+
 func (c *envContextImpl) Close() error {
 	c.mu.Lock()
 	for _, client := range c.clients {

core/sdks/client_factory.go

@@ -1,11 +1,13 @@
 package sdks
 
 import (
+	"sync"
 	"time"
 
 	"github.com/launchdarkly/ld-relay/v6/core/config"
 	"gopkg.in/launchdarkly/go-sdk-common.v2/lduser"
 	ld "gopkg.in/launchdarkly/go-server-sdk.v5"
+	"gopkg.in/launchdarkly/go-server-sdk.v5/interfaces"
 )
 
 // LDClientContext defines a minimal interface for a LaunchDarkly client.
@@ -16,15 +18,87 @@ import (
 type LDClientContext interface {
 	Initialized() bool
 	SecureModeHash(lduser.User) string
+	GetDataSourceStatus() interfaces.DataSourceStatus
+	GetDataStoreStatus() DataStoreStatusInfo
 	Close() error
 }
 
+type ldClientContextImpl struct {
+	*ld.LDClient
+	storeStatusTime time.Time
+	lock            sync.Mutex
+}
+
+// DataStoreStatusInfo combines the Available property from interfaces.DataStoreStatus with a
+// timestamp.
+type DataStoreStatusInfo struct {
+	// Available is copied from interfaces.DataStoreStatus.
+	Available bool
+
+	// LastUpdated is the time when the status last changed.
+	LastUpdated time.Time
+}
+
 // ClientFactoryFunc is a function that creates the LaunchDarkly client. This is normally
 // DefaultClientFactory, but it can be changed in order to make configuration changes or for testing.
 type ClientFactoryFunc func(sdkKey config.SDKKey, config ld.Config) (LDClientContext, error)
 
 // DefaultClientFactory is the default ClientFactoryFunc implementation, which just passes the
 // specified configuration to the SDK client constructor.
 func DefaultClientFactory(sdkKey config.SDKKey, config ld.Config) (LDClientContext, error) {
-	return ld.MakeCustomClient(string(sdkKey), config, time.Second*10)
+	c, err := ld.MakeCustomClient(string(sdkKey), config, time.Second*10)
+	if err != nil {
+		return nil, err
+	}
+	return wrapLDClient(c), nil
+}
+
+// ClientFactoryFromLDClientFactory translates from the client factory type that we expose to host
+// applications, which uses the real LDClient type, to the more general factory type that we use
+// internally which uses the sdks.ClientFactoryFunc abstraction. The latter makes our code a bit
+// cleaner and easier to test, but isn't of any use when hosting Relay in an application.
+func ClientFactoryFromLDClientFactory(fn func(sdkKey config.SDKKey, config ld.Config) (*ld.LDClient, error)) ClientFactoryFunc {
+	if fn == nil {
+		return nil
+	}
+	return func(sdkKey config.SDKKey, config ld.Config) (LDClientContext, error) {
+		c, err := fn(sdkKey, config)
+		if err != nil {
+			return nil, err
+		}
+		return wrapLDClient(c), nil
+	}
+}
+
+func wrapLDClient(c *ld.LDClient) LDClientContext {
+	ret := &ldClientContextImpl{LDClient: c}
+	ret.storeStatusTime = time.Now()
+	// In Relay's status reporting, we want to be provide a "stateSince" timestamp for the data store status
+	// like we have for the data source status. However, the SDK API does not provide this by default. So to
+	// keep track of the time of the last status change, we add a status listener that just updates the
+	// timestamp whenever it gets a new status.
+	storeStatusCh := c.GetDataStoreStatusProvider().AddStatusListener()
+	go func() {
+		for range storeStatusCh { // if the SDK client is closed, this channel will also be closed
+			ret.lock.Lock()
+			ret.storeStatusTime = time.Now()
+			ret.lock.Unlock()
+		}
+	}()
+	return ret
+}
+
+func (c *ldClientContextImpl) GetDataSourceStatus() interfaces.DataSourceStatus {
+	return c.GetDataSourceStatusProvider().GetStatus()
+}
+
+func (c *ldClientContextImpl) GetDataStoreStatus() DataStoreStatusInfo {
+	status := c.GetDataStoreStatusProvider().GetStatus()
+	c.lock.Lock()
+	statusTime := c.storeStatusTime
+	c.lock.Unlock()
+	return DataStoreStatusInfo{
+		Available:   status.Available,
+		LastUpdated: statusTime,
+	}
 }

core/sharedtest/json_helpers.go

@@ -0,0 +1,30 @@
+package sharedtest
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"gopkg.in/launchdarkly/go-sdk-common.v2/ldvalue"
+)
+
+// AssertJSONPathMatch checks for a value within a nested JSON data structure.
+func AssertJSONPathMatch(t *testing.T, expected interface{}, inValue ldvalue.Value, path ...string) {
+	expectedValue := ldvalue.CopyArbitraryValue(expected)
+	value := inValue
+	for _, p := range path {
+		value = value.GetByKey(p)
+	}
+	if !expectedValue.Equal(value) {
+		assert.Fail(
+			t,
+			"did not find expected JSON value",
+			"at path [%s] in %s\nexpected: %s\nfound: %s",
+			strings.Join(path, "."),
+			inValue,
+			expectedValue,
+			value,
+		)
+	}
+}