diff --git a/pkg/api/add_events/add_events.go b/pkg/api/add_events/add_events.go index 0f13d5e..f7f9e7d 100644 --- a/pkg/api/add_events/add_events.go +++ b/pkg/api/add_events/add_events.go @@ -27,7 +27,7 @@ import ( ) const ( - AttrBundleKey = "bundle_key" + AttrSessionKey = "session_key" AttrServerHost = "serverHost" AttrOrigServerHost = "__origServerHost" AttrLogFile = "logfile" diff --git a/pkg/client/add_events.go b/pkg/client/add_events.go index f04ce66..b2c1a13 100644 --- a/pkg/client/add_events.go +++ b/pkg/client/add_events.go @@ -52,11 +52,51 @@ func NewEventWithMeta( bundle *add_events.EventBundle, groupBy []string, serverHost string, + debug bool, ) EventWithMeta { // initialise key := "" info := make(add_events.SessionInfo) + // adjust server host attribute + adjustServerHost(bundle, serverHost) + + // construct Key + bundleKey := extractKeyAndUpdateInfo(bundle, groupBy, key, info) + + // in debug mode include bundleKey + if debug { + info[add_events.AttrSessionKey] = bundleKey + } + + return EventWithMeta{ + EventBundle: bundle, + Key: bundleKey, + SessionInfo: info, + } +} + +func extractKeyAndUpdateInfo(bundle *add_events.EventBundle, groupBy []string, key string, info add_events.SessionInfo) string { + // construct key and move attributes from attrs to sessionInfo + for _, k := range groupBy { + val, ok := bundle.Event.Attrs[k] + key += k + ":" + if ok { + key += fmt.Sprintf("%s", val) + + // move to session info and remove from attributes + info[k] = val + delete(bundle.Event.Attrs, k) + } + key += "___DELIM___" + } + + // use md5 to shorten the key + hash := md5.Sum([]byte(key)) + return hex.EncodeToString(hash[:]) +} + +func adjustServerHost(bundle *add_events.EventBundle, serverHost string) { // figure out serverHost value usedServerHost := serverHost // if event's ServerHost is set, use it @@ -84,35 +124,6 @@ func NewEventWithMeta( // so that it can be in the next step moved to sessionInfo delete(bundle.Event.Attrs, add_events.AttrOrigServerHost) bundle.Event.Attrs[add_events.AttrServerHost] = usedServerHost - - // construct Key - // move attributes from attrs to sessionInfo - for _, k := range groupBy { - val, ok := bundle.Event.Attrs[k] - key += k + ":" - if ok { - key += fmt.Sprintf("%s", val) - - // move to session info and remove from attributes - info[k] = val - delete(bundle.Event.Attrs, k) - } - key += "___DELIM___" - } - - // use md5 to shorten the key - hash := md5.Sum([]byte(key)) - bundleKey := hex.EncodeToString(hash[:]) - - // during the development we can include the key - // so that we can see this in UI - // info[add_events.AttrBundleKey] = bundleKey - - return EventWithMeta{ - EventBundle: bundle, - Key: bundleKey, - SessionInfo: info, - } } // AddEvents enqueues given events for processing (sending to Dataset). @@ -130,16 +141,7 @@ func (client *DataSetClient) AddEvents(bundles []*add_events.EventBundle) error // store there information about the host bundlesWithMeta := make(map[string][]EventWithMeta) for _, bundle := range bundles { - bef := len(bundle.Event.Attrs) - bWM := NewEventWithMeta(bundle, client.Config.BufferSettings.GroupBy, client.serverHost) - aftB := len(bundle.Event.Attrs) - aftM := len(bWM.EventBundle.Event.Attrs) - client.Logger.Warn("Amount of attributes:", - zap.String("Key", bWM.Key), - zap.Int("AttributesBefore", bef), - zap.Int("AttributesAfter", aftB), - zap.Int("AttributesWithMetaAfter", aftM), - ) + bWM := NewEventWithMeta(bundle, client.Config.BufferSettings.GroupBy, client.serverHost, client.Config.Debug) list, found := bundlesWithMeta[bWM.Key] if !found { diff --git a/pkg/client/add_events_test.go b/pkg/client/add_events_test.go index d270c76..fa9ca44 100644 --- a/pkg/client/add_events_test.go +++ b/pkg/client/add_events_test.go @@ -122,6 +122,7 @@ func TestNewEventWithMeta(t *testing.T) { tests := []struct { name string groupBy []string + debug bool expKey string expAttrs add_events.EventAttrs expSessionInfo add_events.SessionInfo @@ -130,6 +131,7 @@ func TestNewEventWithMeta(t *testing.T) { { name: "empty group by", groupBy: []string{}, + debug: false, expKey: "d41d8cd98f00b204e9800998ecf8427e", expAttrs: add_events.EventAttrs{ k1: v1, k2: v2, k3: v3, @@ -137,10 +139,25 @@ func TestNewEventWithMeta(t *testing.T) { expSessionInfo: add_events.SessionInfo{}, }, + // when no grouping is used, then attributes are kept + { + name: "empty group by", + groupBy: []string{}, + debug: true, + expKey: "d41d8cd98f00b204e9800998ecf8427e", + expAttrs: add_events.EventAttrs{ + k1: v1, k2: v2, k3: v3, + }, + expSessionInfo: add_events.SessionInfo{ + add_events.AttrSessionKey: "d41d8cd98f00b204e9800998ecf8427e", + }, + }, + // group by not specified attribute - 1 { name: "group by unused attribute - 1", groupBy: []string{k4}, + debug: false, expKey: "746ea36093d470e90a9c2fbf07c8ed17", expAttrs: add_events.EventAttrs{ k1: v1, k2: v2, k3: v3, @@ -152,6 +169,7 @@ func TestNewEventWithMeta(t *testing.T) { { name: "group by unused attribute - 2", groupBy: []string{k5}, + debug: false, expKey: "d9b675aac82295ce36dad72278888308", expAttrs: add_events.EventAttrs{ k1: v1, k2: v2, k3: v3, @@ -163,6 +181,7 @@ func TestNewEventWithMeta(t *testing.T) { { name: "group by two attributes - 1", groupBy: []string{k1, k2}, + debug: false, expKey: "4410d57b15f30fb22e92fc3e2338f288", expAttrs: add_events.EventAttrs{ k3: v3, @@ -176,6 +195,7 @@ func TestNewEventWithMeta(t *testing.T) { { name: "group by two attributes - 2", groupBy: []string{k2, k1}, + debug: false, expKey: "ce28b69e77b27f012501095cb343e2ae", expAttrs: add_events.EventAttrs{ k3: v3, @@ -203,6 +223,7 @@ func TestNewEventWithMeta(t *testing.T) { &add_events.EventBundle{Event: event}, tt.groupBy, "serverHost", + tt.debug, ) tt.expAttrs[add_events.AttrServerHost] = "serverHost" @@ -1250,7 +1271,7 @@ func TestAddEventsServerHostLogic(t *testing.T) { extractBundles := func(req add_events.AddEventsRequest) []tBundle { bundles := make([]tBundle, 0) - delete(*req.SessionInfo, add_events.AttrBundleKey) + delete(*req.SessionInfo, add_events.AttrSessionKey) for _, ev := range req.Events { bundles = append(bundles, tBundle{attrs: ev.Attrs, info: *req.SessionInfo}) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 6f7a707..eb49c74 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -25,6 +25,8 @@ import ( "github.com/scalyr/dataset-go/pkg/buffer_config" ) +const DebugModeDefault = false + // DataSetTokens wrap DataSet access tokens type DataSetTokens struct { WriteLog string @@ -49,6 +51,7 @@ type DataSetConfig struct { Tokens DataSetTokens BufferSettings buffer_config.DataSetBufferSettings ServerHostSettings server_host_config.DataSetServerHostSettings + Debug bool } func NewDefaultDataSetConfig() DataSetConfig { @@ -57,6 +60,7 @@ func NewDefaultDataSetConfig() DataSetConfig { Tokens: DataSetTokens{}, BufferSettings: buffer_config.NewDefaultDataSetBufferSettings(), ServerHostSettings: server_host_config.NewDefaultDataSetServerHostSettings(), + Debug: DebugModeDefault, } } @@ -90,6 +94,13 @@ func WithServerHostSettings(serverHostSettings server_host_config.DataSetServerH } } +func WithDebug(debug bool) DataSetConfigOption { + return func(c *DataSetConfig) error { + c.Debug = debug + return nil + } +} + func FromEnv() DataSetConfigOption { return func(c *DataSetConfig) error { if c.Tokens.WriteLog == "" { @@ -109,6 +120,7 @@ func FromEnv() DataSetConfigOption { } c.BufferSettings = buffer_config.NewDefaultDataSetBufferSettings() c.ServerHostSettings = server_host_config.NewDefaultDataSetServerHostSettings() + c.Debug = DebugModeDefault return nil } } @@ -135,11 +147,12 @@ func (cfg *DataSetConfig) WithOptions(opts ...DataSetConfigOption) (*DataSetConf func (cfg *DataSetConfig) String() string { return fmt.Sprintf( - "Endpoint: %s, Tokens: (%s), BufferSettings: (%s), ServerHostSettings: (%s)", + "Endpoint: %s, Tokens: (%s), BufferSettings: (%s), ServerHostSettings: (%s), Debug: (%t)", cfg.Endpoint, cfg.Tokens.String(), cfg.BufferSettings.String(), cfg.ServerHostSettings.String(), + cfg.Debug, ) } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 28feb32..9472a92 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -105,6 +105,7 @@ func TestNewConfigBasedOnExistingWithNewConfigOptions(t *testing.T) { WithTokens(DataSetTokens{WriteLog: "writeLogOpt1"}), WithBufferSettings(*bufCfg), WithServerHostSettings(*hostCfg), + WithDebug(true), ) // AND bufCfg2, _ := buffer_config.New( @@ -141,6 +142,7 @@ func TestNewConfigBasedOnExistingWithNewConfigOptions(t *testing.T) { ServerHost: "AAA", UseHostName: false, }, cfg5.ServerHostSettings) + assert.Equal(t, true, cfg5.Debug) // AND new config is changed assert.Equal(t, "https://fooOpt2", cfg6.Endpoint) @@ -157,11 +159,12 @@ func TestNewConfigBasedOnExistingWithNewConfigOptions(t *testing.T) { ServerHost: "BBB", UseHostName: true, }, cfg6.ServerHostSettings) + assert.Equal(t, true, cfg6.Debug) } func TestNewDefaultDataSetConfigToString(t *testing.T) { cfg := NewDefaultDataSetConfig() - assert.Equal(t, cfg.String(), "Endpoint: https://app.scalyr.com, Tokens: (WriteLog: false, ReadLog: false, WriteConfig: false, ReadConfig: false), BufferSettings: (MaxLifetime: 5s, MaxSize: 6225920, GroupBy: [], RetryRandomizationFactor: 0.500000, RetryMultiplier: 1.500000, RetryInitialInterval: 5s, RetryMaxInterval: 30s, RetryMaxElapsedTime: 5m0s, RetryShutdownTimeout: 30s), ServerHostSettings: (UseHostName: true, ServerHost: )") + assert.Equal(t, cfg.String(), "Endpoint: https://app.scalyr.com, Tokens: (WriteLog: false, ReadLog: false, WriteConfig: false, ReadConfig: false), BufferSettings: (MaxLifetime: 5s, MaxSize: 6225920, GroupBy: [], RetryRandomizationFactor: 0.500000, RetryMultiplier: 1.500000, RetryInitialInterval: 5s, RetryMaxInterval: 30s, RetryMaxElapsedTime: 5m0s, RetryShutdownTimeout: 30s), ServerHostSettings: (UseHostName: true, ServerHost: ), Debug: (false)") } func TestNewDefaultDataSetConfigIsValid(t *testing.T) {