From e9313876ab3377849ee803644de962f8cb9519f8 Mon Sep 17 00:00:00 2001 From: Stuart Carnie Date: Wed, 11 Oct 2017 07:08:31 -0700 Subject: [PATCH] EXPLAIN ANALYZE * Introduces EXPLAIN ANALYZE command, which produces a detailed tree of operations used to execute the query. introduce context.Context to APIs metrics package * create groups of named measurements * safe for concurrent access tracing package EXPLAIN ANALYZE implementation for OSS Serialize EXPLAIN ANALYZE traces from remote nodes use context.Background for tests group with other stdlib packages additional documentation and remove unused API use influxdb/pkg/testing/assert remove testify reference --- Godeps | 1 + LICENSE_OF_DEPENDENCIES.md | 1 + coordinator/shard_mapper.go | 7 +- coordinator/shard_mapper_test.go | 7 +- coordinator/statement_executor.go | 129 ++- coordinator/statement_executor_test.go | 11 +- pkg/metrics/context.go | 20 + pkg/metrics/counter.go | 28 + pkg/metrics/counter_test.go | 14 + pkg/metrics/default_registry.go | 36 + pkg/metrics/descriptors.go | 64 ++ pkg/metrics/descriptors_test.go | 21 + pkg/metrics/doc.go | 6 + pkg/metrics/group.go | 37 + pkg/metrics/group_registry.go | 79 ++ pkg/metrics/registry.go | 87 ++ pkg/metrics/registry_test.go | 63 ++ pkg/metrics/timer.go | 34 + pkg/metrics/timer_test.go | 14 + pkg/testing/assert/assertions.go | 96 ++ pkg/testing/assert/doc.go | 4 + pkg/testing/assert/helper.go | 53 + pkg/tracing/context.go | 32 + pkg/tracing/doc.go | 26 + pkg/tracing/fields/field.go | 117 ++ pkg/tracing/fields/fields.go | 61 ++ pkg/tracing/fields/fields_test.go | 101 ++ pkg/tracing/labels/labels.go | 74 ++ pkg/tracing/labels/labels_test.go | 101 ++ pkg/tracing/rawspan.go | 18 + pkg/tracing/span.go | 84 ++ pkg/tracing/spancontext.go | 27 + pkg/tracing/trace.go | 141 +++ pkg/tracing/trace_encoding.go | 136 +++ pkg/tracing/tree.go | 74 ++ pkg/tracing/util.go | 26 + pkg/tracing/wire/binary.go | 7 + pkg/tracing/wire/binary.pb.go | 1292 +++++++++++++++++++++++ pkg/tracing/wire/binary.proto | 44 + query/explain.go | 5 +- query/internal/internal.pb.go | 111 +- query/internal/internal.proto | 1 + query/iterator.gen.go | 80 +- query/iterator.gen.go.tmpl | 67 +- query/iterator.go | 119 ++- query/iterator_test.go | 9 +- query/point.gen.go | 71 +- query/point.gen.go.tmpl | 15 +- query/select.go | 127 ++- query/select_test.go | 35 +- query/subquery.go | 14 +- tsdb/engine.go | 3 +- tsdb/engine/tsm1/engine.go | 149 ++- tsdb/engine/tsm1/engine_test.go | 15 +- tsdb/engine/tsm1/file_store.gen.go | 70 ++ tsdb/engine/tsm1/file_store.gen.go.tmpl | 14 + tsdb/engine/tsm1/file_store.go | 26 +- tsdb/engine/tsm1/file_store_test.go | 75 +- tsdb/engine/tsm1/iterator.gen.go | 153 +++ tsdb/engine/tsm1/iterator.gen.go.tmpl | 35 + tsdb/engine/tsm1/iterator.go | 34 +- tsdb/shard.go | 12 +- tsdb/shard_test.go | 17 +- tsdb/store_test.go | 5 +- 64 files changed, 3997 insertions(+), 438 deletions(-) create mode 100644 pkg/metrics/context.go create mode 100644 pkg/metrics/counter.go create mode 100644 pkg/metrics/counter_test.go create mode 100644 pkg/metrics/default_registry.go create mode 100644 pkg/metrics/descriptors.go create mode 100644 pkg/metrics/descriptors_test.go create mode 100644 pkg/metrics/doc.go create mode 100644 pkg/metrics/group.go create mode 100644 pkg/metrics/group_registry.go create mode 100644 pkg/metrics/registry.go create mode 100644 pkg/metrics/registry_test.go create mode 100644 pkg/metrics/timer.go create mode 100644 pkg/metrics/timer_test.go create mode 100644 pkg/testing/assert/assertions.go create mode 100644 pkg/testing/assert/doc.go create mode 100644 pkg/testing/assert/helper.go create mode 100644 pkg/tracing/context.go create mode 100644 pkg/tracing/doc.go create mode 100644 pkg/tracing/fields/field.go create mode 100644 pkg/tracing/fields/fields.go create mode 100644 pkg/tracing/fields/fields_test.go create mode 100644 pkg/tracing/labels/labels.go create mode 100644 pkg/tracing/labels/labels_test.go create mode 100644 pkg/tracing/rawspan.go create mode 100644 pkg/tracing/span.go create mode 100644 pkg/tracing/spancontext.go create mode 100644 pkg/tracing/trace.go create mode 100644 pkg/tracing/trace_encoding.go create mode 100644 pkg/tracing/tree.go create mode 100644 pkg/tracing/util.go create mode 100644 pkg/tracing/wire/binary.go create mode 100644 pkg/tracing/wire/binary.pb.go create mode 100644 pkg/tracing/wire/binary.proto diff --git a/Godeps b/Godeps index 84196b0e652..21c3dacbd3d 100644 --- a/Godeps +++ b/Godeps @@ -20,5 +20,6 @@ github.com/spaolacci/murmur3 0d12bf811670bf6a1a63828dfbd003eded177fce github.com/tinylib/msgp ad0ff2e232ad2e37faf67087fb24bf8d04a8ce20 github.com/uber-go/atomic 74ca5ec650841aee9f289dce76e928313a37cbc6 github.com/uber-go/zap fbae0281ffd546fa6d1959fec6075ac5da7fb577 +github.com/xlab/treeprint 06dfc6fa17cdde904617990a0c2d89e3e332dbb3 golang.org/x/crypto 9477e0b78b9ac3d0b03822fd95422e2fe07627cd golang.org/x/sys 062cd7e4e68206d8bab9b18396626e855c992658 diff --git a/LICENSE_OF_DEPENDENCIES.md b/LICENSE_OF_DEPENDENCIES.md index da4ff1d04a8..c24b8d0740f 100644 --- a/LICENSE_OF_DEPENDENCIES.md +++ b/LICENSE_OF_DEPENDENCIES.md @@ -25,3 +25,4 @@ - github.com/uber-go/zap [MIT LICENSE](https://github.com/uber-go/zap/blob/master/LICENSE.txt) - golang.org/x/crypto [BSD LICENSE](https://github.com/golang/crypto/blob/master/LICENSE) - jquery 2.1.4 [MIT LICENSE](https://github.com/jquery/jquery/blob/master/LICENSE.txt) +- github.com/xlab/treeprint [MIT LICENSE](https://github.com/xlab/treeprint/blob/master/LICENSE) diff --git a/coordinator/shard_mapper.go b/coordinator/shard_mapper.go index e8378acf5cd..3afb969d4ea 100644 --- a/coordinator/shard_mapper.go +++ b/coordinator/shard_mapper.go @@ -1,6 +1,7 @@ package coordinator import ( + "context" "io" "time" @@ -160,7 +161,7 @@ func (a *LocalShardMapping) MapType(m *influxql.Measurement, field string) influ return typ } -func (a *LocalShardMapping) CreateIterator(m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { +func (a *LocalShardMapping) CreateIterator(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { source := Source{ Database: m.Database, RetentionPolicy: m.RetentionPolicy, @@ -184,7 +185,7 @@ func (a *LocalShardMapping) CreateIterator(m *influxql.Measurement, opt query.It inputs := make([]query.Iterator, 0, len(measurements)) if err := func() error { for _, measurement := range measurements { - input, err := sg.CreateIterator(measurement, opt) + input, err := sg.CreateIterator(ctx, measurement, opt) if err != nil { return err } @@ -197,7 +198,7 @@ func (a *LocalShardMapping) CreateIterator(m *influxql.Measurement, opt query.It } return query.Iterators(inputs).Merge(opt) } - return sg.CreateIterator(m.Name, opt) + return sg.CreateIterator(ctx, m.Name, opt) } func (a *LocalShardMapping) IteratorCost(m *influxql.Measurement, opt query.IteratorOptions) (query.IteratorCost, error) { diff --git a/coordinator/shard_mapper_test.go b/coordinator/shard_mapper_test.go index 0f9aa687f8d..eb2e04434cd 100644 --- a/coordinator/shard_mapper_test.go +++ b/coordinator/shard_mapper_test.go @@ -1,6 +1,7 @@ package coordinator_test import ( + "context" "reflect" "testing" "time" @@ -40,7 +41,7 @@ func TestLocalShardMapper(t *testing.T) { } var sh MockShard - sh.CreateIteratorFn = func(measurement string, opt query.IteratorOptions) (query.Iterator, error) { + sh.CreateIteratorFn = func(ctx context.Context, measurement string, opt query.IteratorOptions) (query.Iterator, error) { if measurement != "cpu" { t.Errorf("unexpected measurement: %s", measurement) } @@ -74,7 +75,7 @@ func TestLocalShardMapper(t *testing.T) { t.Fatalf("unexpected number of shard mappings: %d", len(m.ShardMap)) } - if _, err := ic.CreateIterator(measurement, query.IteratorOptions{}); err != nil { + if _, err := ic.CreateIterator(context.Background(), measurement, query.IteratorOptions{}); err != nil { t.Fatalf("unexpected error: %s", err) } @@ -97,7 +98,7 @@ func TestLocalShardMapper(t *testing.T) { t.Fatalf("unexpected number of shard mappings: %d", len(m.ShardMap)) } - if _, err := ic.CreateIterator(measurement, query.IteratorOptions{}); err != nil { + if _, err := ic.CreateIterator(context.Background(), measurement, query.IteratorOptions{}); err != nil { t.Fatalf("unexpected error: %s", err) } } diff --git a/coordinator/statement_executor.go b/coordinator/statement_executor.go index 49a9b35150e..8e515d2844b 100644 --- a/coordinator/statement_executor.go +++ b/coordinator/statement_executor.go @@ -2,6 +2,7 @@ package coordinator import ( "bytes" + "context" "errors" "fmt" "io" @@ -14,6 +15,8 @@ import ( "github.com/influxdata/influxdb/influxql" "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/monitor" + "github.com/influxdata/influxdb/pkg/tracing" + "github.com/influxdata/influxdb/pkg/tracing/fields" "github.com/influxdata/influxdb/query" "github.com/influxdata/influxdb/services/meta" "github.com/influxdata/influxdb/tsdb" @@ -56,7 +59,7 @@ type StatementExecutor struct { func (e *StatementExecutor) ExecuteStatement(stmt influxql.Statement, ctx query.ExecutionContext) error { // Select statements are handled separately so that they can be streamed. if stmt, ok := stmt.(*influxql.SelectStatement); ok { - return e.executeSelectStatement(stmt, &ctx) + return e.executeSelectStatement(context.Background(), stmt, &ctx) } var rows models.Rows @@ -136,7 +139,11 @@ func (e *StatementExecutor) ExecuteStatement(stmt influxql.Statement, ctx query. } err = e.executeDropUserStatement(stmt) case *influxql.ExplainStatement: - rows, err = e.executeExplainStatement(stmt, &ctx) + if stmt.Analyze { + rows, err = e.executeExplainAnalyzeStatement(stmt, &ctx) + } else { + rows, err = e.executeExplainStatement(stmt, &ctx) + } case *influxql.GrantStatement: if ctx.ReadOnly { messages = append(messages, query.ReadOnlyWarning(stmt.String())) @@ -401,17 +408,13 @@ func (e *StatementExecutor) executeDropUserStatement(q *influxql.DropUserStateme return e.MetaClient.DropUser(q.Name) } -func (e *StatementExecutor) executeExplainStatement(q *influxql.ExplainStatement, ctx *query.ExecutionContext) (models.Rows, error) { - if q.Analyze { - return nil, errors.New("analyze is currently unimplemented") - } - +func (e *StatementExecutor) executeExplainStatement(q *influxql.ExplainStatement, ectx *query.ExecutionContext) (models.Rows, error) { opt := query.SelectOptions{ - InterruptCh: ctx.InterruptCh, - NodeID: ctx.ExecutionOptions.NodeID, + InterruptCh: ectx.InterruptCh, + NodeID: ectx.ExecutionOptions.NodeID, MaxSeriesN: e.MaxSelectSeriesN, MaxBucketsN: e.MaxSelectBucketsN, - Authorizer: ctx.Authorizer, + Authorizer: ectx.Authorizer, } // Prepare the query for execution, but do not actually execute it. @@ -437,6 +440,74 @@ func (e *StatementExecutor) executeExplainStatement(q *influxql.ExplainStatement return models.Rows{row}, nil } +func (e *StatementExecutor) executeExplainAnalyzeStatement(q *influxql.ExplainStatement, ectx *query.ExecutionContext) (models.Rows, error) { + stmt := q.Statement + t, span := tracing.NewTrace("select") + ctx := tracing.NewContextWithTrace(context.Background(), t) + ctx = tracing.NewContextWithSpan(ctx, span) + start := time.Now() + + itrs, columns, err := e.createIterators(ctx, stmt, ectx) + if err != nil { + return nil, err + } + + iterTime := time.Since(start) + + // Generate a row emitter from the iterator set. + em := query.NewEmitter(itrs, stmt.TimeAscending(), ectx.ChunkSize) + em.Columns = columns + if stmt.Location != nil { + em.Location = stmt.Location + } + em.OmitTime = stmt.OmitTime + em.EmitName = stmt.EmitName + + // Emit rows to the results channel. + var writeN int64 + for { + var row *models.Row + row, _, err = em.Emit() + if err != nil { + goto CLEANUP + } else if row == nil { + // Check if the query was interrupted while emitting. + select { + case <-ectx.InterruptCh: + err = query.ErrQueryInterrupted + goto CLEANUP + default: + } + break + } + + writeN += int64(len(row.Values)) + } + +CLEANUP: + em.Close() + if err != nil { + return nil, err + } + + totalTime := time.Since(start) + span.MergeFields( + fields.Duration("total_time", totalTime), + fields.Duration("planning_time", iterTime), + fields.Duration("execution_time", totalTime-iterTime), + ) + span.Finish() + + row := &models.Row{ + Columns: []string{"EXPLAIN ANALYZE"}, + } + for _, s := range strings.Split(t.Tree().String(), "\n") { + row.Values = append(row.Values, []interface{}{s}) + } + + return models.Rows{row}, nil +} + func (e *StatementExecutor) executeGrantStatement(stmt *influxql.GrantStatement) error { return e.MetaClient.SetPrivilege(stmt.User, stmt.On, stmt.Privilege) } @@ -469,14 +540,14 @@ func (e *StatementExecutor) executeSetPasswordUserStatement(q *influxql.SetPassw return e.MetaClient.UpdateUser(q.Name, q.Password) } -func (e *StatementExecutor) executeSelectStatement(stmt *influxql.SelectStatement, ctx *query.ExecutionContext) error { - itrs, columns, err := e.createIterators(stmt, ctx) +func (e *StatementExecutor) executeSelectStatement(ctx context.Context, stmt *influxql.SelectStatement, ectx *query.ExecutionContext) error { + itrs, columns, err := e.createIterators(ctx, stmt, ectx) if err != nil { return err } // Generate a row emitter from the iterator set. - em := query.NewEmitter(itrs, stmt.TimeAscending(), ctx.ChunkSize) + em := query.NewEmitter(itrs, stmt.TimeAscending(), ectx.ChunkSize) em.Columns = columns if stmt.Location != nil { em.Location = stmt.Location @@ -501,7 +572,7 @@ func (e *StatementExecutor) executeSelectStatement(stmt *influxql.SelectStatemen } else if row == nil { // Check if the query was interrupted while emitting. select { - case <-ctx.InterruptCh: + case <-ectx.InterruptCh: return query.ErrQueryInterrupted default: } @@ -518,13 +589,13 @@ func (e *StatementExecutor) executeSelectStatement(stmt *influxql.SelectStatemen } result := &query.Result{ - StatementID: ctx.StatementID, + StatementID: ectx.StatementID, Series: []*models.Row{row}, Partial: partial, } // Send results or exit if closing. - if err := ctx.Send(result); err != nil { + if err := ectx.Send(result); err != nil { return err } @@ -538,12 +609,12 @@ func (e *StatementExecutor) executeSelectStatement(stmt *influxql.SelectStatemen } var messages []*query.Message - if ctx.ReadOnly { + if ectx.ReadOnly { messages = append(messages, query.ReadOnlyWarning(stmt.String())) } - return ctx.Send(&query.Result{ - StatementID: ctx.StatementID, + return ectx.Send(&query.Result{ + StatementID: ectx.StatementID, Messages: messages, Series: []*models.Row{{ Name: "result", @@ -555,8 +626,8 @@ func (e *StatementExecutor) executeSelectStatement(stmt *influxql.SelectStatemen // Always emit at least one result. if !emitted { - return ctx.Send(&query.Result{ - StatementID: ctx.StatementID, + return ectx.Send(&query.Result{ + StatementID: ectx.StatementID, Series: make([]*models.Row, 0), }) } @@ -564,24 +635,24 @@ func (e *StatementExecutor) executeSelectStatement(stmt *influxql.SelectStatemen return nil } -func (e *StatementExecutor) createIterators(stmt *influxql.SelectStatement, ctx *query.ExecutionContext) ([]query.Iterator, []string, error) { +func (e *StatementExecutor) createIterators(ctx context.Context, stmt *influxql.SelectStatement, ectx *query.ExecutionContext) ([]query.Iterator, []string, error) { opt := query.SelectOptions{ - InterruptCh: ctx.InterruptCh, - NodeID: ctx.ExecutionOptions.NodeID, + InterruptCh: ectx.InterruptCh, + NodeID: ectx.ExecutionOptions.NodeID, MaxSeriesN: e.MaxSelectSeriesN, MaxBucketsN: e.MaxSelectBucketsN, - Authorizer: ctx.Authorizer, + Authorizer: ectx.Authorizer, } // Create a set of iterators from a selection. - itrs, columns, err := query.Select(stmt, e.ShardMapper, opt) + itrs, columns, err := query.Select(ctx, stmt, e.ShardMapper, opt) if err != nil { return nil, nil, err } if e.MaxSelectPointN > 0 { monitor := query.PointLimitMonitor(itrs, query.DefaultStatsInterval, e.MaxSelectPointN) - ctx.Query.Monitor(monitor) + ectx.Query.Monitor(monitor) } return itrs, columns, nil } @@ -1073,8 +1144,8 @@ func (e *StatementExecutor) NormalizeStatement(stmt influxql.Statement, defaultD case *influxql.Measurement: switch stmt.(type) { case *influxql.DropSeriesStatement, *influxql.DeleteSeriesStatement: - // DB and RP not supported by these statements so don't rewrite into invalid - // statements + // DB and RP not supported by these statements so don't rewrite into invalid + // statements default: err = e.normalizeMeasurement(node, defaultDatabase) } diff --git a/coordinator/statement_executor_test.go b/coordinator/statement_executor_test.go index 2c845f3cc85..0da4cdfc555 100644 --- a/coordinator/statement_executor_test.go +++ b/coordinator/statement_executor_test.go @@ -2,6 +2,7 @@ package coordinator_test import ( "bytes" + "context" "errors" "io" "os" @@ -50,7 +51,7 @@ func TestQueryExecutor_ExecuteQuery_SelectStatement(t *testing.T) { } var sh MockShard - sh.CreateIteratorFn = func(m string, opt query.IteratorOptions) (query.Iterator, error) { + sh.CreateIteratorFn = func(ctx context.Context, m string, opt query.IteratorOptions) (query.Iterator, error) { return &FloatIterator{Points: []query.FloatPoint{ {Name: "cpu", Time: int64(0 * time.Second), Aux: []interface{}{float64(100)}}, {Name: "cpu", Time: int64(1 * time.Second), Aux: []interface{}{float64(200)}}, @@ -103,7 +104,7 @@ func TestQueryExecutor_ExecuteQuery_MaxSelectBucketsN(t *testing.T) { } var sh MockShard - sh.CreateIteratorFn = func(m string, opt query.IteratorOptions) (query.Iterator, error) { + sh.CreateIteratorFn = func(ctx context.Context, m string, opt query.IteratorOptions) (query.Iterator, error) { return &FloatIterator{ Points: []query.FloatPoint{{Name: "cpu", Time: int64(0 * time.Second), Aux: []interface{}{float64(100)}}}, }, nil @@ -384,7 +385,7 @@ func (s *TSDBStore) TagValues(_ query.Authorizer, database string, cond influxql type MockShard struct { Measurements []string FieldDimensionsFn func(measurements []string) (fields map[string]influxql.DataType, dimensions map[string]struct{}, err error) - CreateIteratorFn func(m string, opt query.IteratorOptions) (query.Iterator, error) + CreateIteratorFn func(ctx context.Context, m string, opt query.IteratorOptions) (query.Iterator, error) IteratorCostFn func(m string, opt query.IteratorOptions) (query.IteratorCost, error) ExpandSourcesFn func(sources influxql.Sources) (influxql.Sources, error) } @@ -417,8 +418,8 @@ func (sh *MockShard) MapType(measurement, field string) influxql.DataType { return influxql.Unknown } -func (sh *MockShard) CreateIterator(measurement string, opt query.IteratorOptions) (query.Iterator, error) { - return sh.CreateIteratorFn(measurement, opt) +func (sh *MockShard) CreateIterator(ctx context.Context, measurement string, opt query.IteratorOptions) (query.Iterator, error) { + return sh.CreateIteratorFn(ctx, measurement, opt) } func (sh *MockShard) IteratorCost(measurement string, opt query.IteratorOptions) (query.IteratorCost, error) { diff --git a/pkg/metrics/context.go b/pkg/metrics/context.go new file mode 100644 index 00000000000..ee407ac9e5c --- /dev/null +++ b/pkg/metrics/context.go @@ -0,0 +1,20 @@ +package metrics + +import "context" + +type key int + +const ( + groupKey key = iota +) + +// NewContextWithGroup returns a new context with the given Group added. +func NewContextWithGroup(ctx context.Context, c *Group) context.Context { + return context.WithValue(ctx, groupKey, c) +} + +// GroupFromContext returns the Group associated with ctx or nil if no Group has been assigned. +func GroupFromContext(ctx context.Context) *Group { + c, _ := ctx.Value(groupKey).(*Group) + return c +} diff --git a/pkg/metrics/counter.go b/pkg/metrics/counter.go new file mode 100644 index 00000000000..6f2e526cd2f --- /dev/null +++ b/pkg/metrics/counter.go @@ -0,0 +1,28 @@ +package metrics + +import ( + "strconv" + "sync/atomic" +) + +// The Counter type represents a numeric counter that is safe to use from concurrent goroutines. +type Counter struct { + val int64 + desc *desc +} + +// Name identifies the name of the counter. +func (c *Counter) Name() string { return c.desc.Name } + +// Value atomically returns the current value of the counter. +func (c *Counter) Value() int64 { return atomic.LoadInt64(&c.val) } + +// Add atomically adds d to the counter. +func (c *Counter) Add(d int64) { atomic.AddInt64(&c.val, d) } + +// String returns a string representation using the name and value of the counter. +func (c *Counter) String() string { + var buf [16]byte + v := strconv.AppendInt(buf[:0], c.val, 10) + return c.desc.Name + ": " + string(v) +} diff --git a/pkg/metrics/counter_test.go b/pkg/metrics/counter_test.go new file mode 100644 index 00000000000..d444cd80697 --- /dev/null +++ b/pkg/metrics/counter_test.go @@ -0,0 +1,14 @@ +package metrics + +import ( + "testing" +) + +func TestCounter_Add(t *testing.T) { + c := Counter{} + c.Add(5) + c.Add(5) + if exp, got := int64(10), c.Value(); exp != got { + t.Errorf("unexpected value; exp=%d, got=%d", exp, got) + } +} diff --git a/pkg/metrics/default_registry.go b/pkg/metrics/default_registry.go new file mode 100644 index 00000000000..893221ef11e --- /dev/null +++ b/pkg/metrics/default_registry.go @@ -0,0 +1,36 @@ +package metrics + +var defaultRegistry = NewRegistry() + +// MustRegisterGroup registers a new group using the specified name. +// If the group name is not unique, MustRegisterGroup will panic. +// +// MustRegisterGroup is not safe to call from multiple goroutines. +func MustRegisterGroup(name string) GID { + return defaultRegistry.MustRegisterGroup(name) +} + +// MustRegisterCounter registers a new counter metric with the default registry +// using the provided descriptor. +// If the metric name is not unique, MustRegisterCounter will panic. +// +// MustRegisterCounter is not safe to call from multiple goroutines. +func MustRegisterCounter(name string, opts ...descOption) ID { + return defaultRegistry.MustRegisterCounter(name, opts...) +} + +// MustRegisterTimer registers a new timer metric with the default registry +// using the provided descriptor. +// If the metric name is not unique, MustRegisterTimer will panic. +// +// MustRegisterTimer is not safe to call from multiple goroutines. +func MustRegisterTimer(name string, opts ...descOption) ID { + return defaultRegistry.MustRegisterTimer(name, opts...) +} + +// NewGroup returns a new measurement group from the default registry. +// +// NewGroup is safe to call from multiple goroutines. +func NewGroup(gid GID) *Group { + return defaultRegistry.NewGroup(gid) +} diff --git a/pkg/metrics/descriptors.go b/pkg/metrics/descriptors.go new file mode 100644 index 00000000000..57154fe9a0b --- /dev/null +++ b/pkg/metrics/descriptors.go @@ -0,0 +1,64 @@ +package metrics + +type groupDesc struct { + Name string + id GID +} + +type metricType int + +const ( + counterMetricType metricType = iota + timerMetricType +) + +type desc struct { + Name string + mt metricType + gid GID + id ID +} + +type descOption func(*desc) + +// WithGroup assigns the associated measurement to the group identified by gid originally +// returned from MustRegisterGroup. +func WithGroup(gid GID) descOption { + return func(d *desc) { + d.gid = gid + } +} + +func newDesc(name string, opts ...descOption) *desc { + desc := &desc{Name: name} + for _, o := range opts { + o(desc) + } + return desc +} + +const ( + idMask = (1 << 32) - 1 + gidShift = 32 +) + +type ( + GID uint32 + ID uint64 +) + +func newID(id int, gid GID) ID { + return ID(gid)<> gidShift) +} + +func (id *ID) setGID(gid GID) { + *id |= ID(gid) << gidShift +} diff --git a/pkg/metrics/descriptors_test.go b/pkg/metrics/descriptors_test.go new file mode 100644 index 00000000000..0c45da875d6 --- /dev/null +++ b/pkg/metrics/descriptors_test.go @@ -0,0 +1,21 @@ +package metrics + +import ( + "testing" + + "github.com/influxdata/influxdb/pkg/testing/assert" +) + +func TestID_newID(t *testing.T) { + var id = newID(0xff, 0xff0f0fff) + assert.Equal(t, id, ID(0xff0f0fff000000ff)) + assert.Equal(t, id.id(), 0xff) + assert.Equal(t, id.gid(), 0xff0f0fff) +} + +func TestID_setGID(t *testing.T) { + var id = ID(1) + assert.Equal(t, id.gid(), 0) + id.setGID(1) + assert.Equal(t, id.gid(), 1) +} diff --git a/pkg/metrics/doc.go b/pkg/metrics/doc.go new file mode 100644 index 00000000000..cb0feac4ad6 --- /dev/null +++ b/pkg/metrics/doc.go @@ -0,0 +1,6 @@ +/* +Package metrics provides various measurements that are safe for concurrent access. + +Measurements are arranged into groups that are efficient to create and access. +*/ +package metrics diff --git a/pkg/metrics/group.go b/pkg/metrics/group.go new file mode 100644 index 00000000000..0a02bb0c654 --- /dev/null +++ b/pkg/metrics/group.go @@ -0,0 +1,37 @@ +package metrics + +// The Group type represents an instance of a set of measurements that are used for +// instrumenting a specific request. +type Group struct { + g *groupRegistry + counters []Counter + timers []Timer +} + +// Name returns the name of the group. +func (g *Group) Name() string { return g.g.desc.Name } + +// GetCounter returns the counter identified by the id that was returned +// by MustRegisterCounter for the same group. +// Using an id from a different group will result in undefined behavior. +func (g *Group) GetCounter(id ID) *Counter { return &g.counters[id.id()] } + +// GetTimer returns the timer identified by the id that was returned +// by MustRegisterTimer for the same group. +// Using an id from a different group will result in undefined behavior. +func (g *Group) GetTimer(id ID) *Timer { return &g.timers[id.id()] } + +// The Metric type defines a Name +type Metric interface { + Name() string +} + +// ForEach calls fn for all measurements of the group. +func (g *Group) ForEach(fn func(v Metric)) { + for i := range g.counters { + fn(&g.counters[i]) + } + for i := range g.timers { + fn(&g.timers[i]) + } +} diff --git a/pkg/metrics/group_registry.go b/pkg/metrics/group_registry.go new file mode 100644 index 00000000000..f457f8f8f4d --- /dev/null +++ b/pkg/metrics/group_registry.go @@ -0,0 +1,79 @@ +package metrics + +import ( + "fmt" + "sort" +) + +// The groupRegistry type represents a set of metrics that are measured together. +type groupRegistry struct { + desc *groupDesc + descriptors []*desc + group Group +} + +func (g *groupRegistry) register(desc *desc) error { + p := sort.Search(len(g.descriptors), func(i int) bool { + return g.descriptors[i].Name == desc.Name + }) + + if p != len(g.descriptors) { + return fmt.Errorf("metric name '%s' already in use", desc.Name) + } + + g.descriptors = append(g.descriptors, desc) + sort.Slice(g.descriptors, func(i, j int) bool { + return g.descriptors[i].Name < g.descriptors[j].Name + }) + + return nil +} + +func (g *groupRegistry) mustRegister(desc *desc) { + if err := g.register(desc); err != nil { + panic(err.Error()) + } +} + +// MustRegisterCounter registers a new counter metric using the provided descriptor. +// If the metric name is not unique, MustRegisterCounter will panic. +// +// MustRegisterCounter is not safe to call from multiple goroutines. +func (g *groupRegistry) mustRegisterCounter(desc *desc) ID { + desc.mt = counterMetricType + g.mustRegister(desc) + + desc.id = newID(len(g.group.counters), g.desc.id) + g.group.counters = append(g.group.counters, Counter{desc: desc}) + + return desc.id +} + +// MustRegisterTimer registers a new timer metric using the provided descriptor. +// If the metric name is not unique, MustRegisterTimer will panic. +// +// MustRegisterTimer is not safe to call from multiple goroutines. +func (g *groupRegistry) mustRegisterTimer(desc *desc) ID { + desc.mt = timerMetricType + g.mustRegister(desc) + + desc.id = newID(len(g.group.timers), g.desc.id) + g.group.timers = append(g.group.timers, Timer{desc: desc}) + + return desc.id +} + +// newCollector returns a Collector with a copy of all the registered counters. +// +// newCollector is safe to call from multiple goroutines. +func (g *groupRegistry) newGroup() *Group { + c := &Group{ + g: g, + counters: make([]Counter, len(g.group.counters)), + timers: make([]Timer, len(g.group.timers)), + } + copy(c.counters, g.group.counters) + copy(c.timers, g.group.timers) + + return c +} diff --git a/pkg/metrics/registry.go b/pkg/metrics/registry.go new file mode 100644 index 00000000000..b6eea5f19db --- /dev/null +++ b/pkg/metrics/registry.go @@ -0,0 +1,87 @@ +package metrics + +import ( + "fmt" + "sort" +) + +type Registry struct { + descriptors []*groupDesc + groups []groupRegistry +} + +const ( + // DefaultGroup is the identifier for the default group. + DefaultGroup = GID(0) +) + +// NewRegistry creates a new Registry with a single group identified by DefaultGroup. +func NewRegistry() *Registry { + var r Registry + r.MustRegisterGroup("global") + return &r +} + +func (r *Registry) register(gd *groupDesc) error { + p := sort.Search(len(r.descriptors), func(i int) bool { + return r.descriptors[i].Name == gd.Name + }) + + if p != len(r.descriptors) { + return fmt.Errorf("group name '%s' already in use", gd.Name) + } + + r.descriptors = append(r.descriptors, gd) + sort.Slice(r.descriptors, func(i, j int) bool { + return r.descriptors[i].Name < r.descriptors[j].Name + }) + + gd.id = GID(len(r.groups)) + r.groups = append(r.groups, groupRegistry{desc: gd}) + + return nil +} + +func (r *Registry) mustRegister(gd *groupDesc) { + if err := r.register(gd); err != nil { + panic(err.Error()) + } +} + +// MustRegisterGroup registers a new group and panics if a group already exists with the same name. +// +// MustRegisterGroup is not safe to call from concurrent goroutines. +func (r *Registry) MustRegisterGroup(name string) GID { + gd := &groupDesc{Name: name} + r.mustRegister(gd) + return gd.id +} + +func (r *Registry) mustGetGroupRegistry(id GID) *groupRegistry { + if int(id) >= len(r.groups) { + panic(fmt.Sprintf("invalid group ID")) + } + return &r.groups[id] +} + +// MustRegisterCounter registers a new counter metric using the provided descriptor. +// If the metric name is not unique within the group, MustRegisterCounter will panic. +// +// MustRegisterCounter is not safe to call from concurrent goroutines. +func (r *Registry) MustRegisterCounter(name string, opts ...descOption) ID { + desc := newDesc(name, opts...) + return r.mustGetGroupRegistry(desc.gid).mustRegisterCounter(desc) +} + +// MustRegisterTimer registers a new timer metric using the provided descriptor. +// If the metric name is not unique within the group, MustRegisterTimer will panic. +// +// MustRegisterTimer is not safe to call from concurrent goroutines. +func (r *Registry) MustRegisterTimer(name string, opts ...descOption) ID { + desc := newDesc(name, opts...) + return r.mustGetGroupRegistry(desc.gid).mustRegisterTimer(desc) +} + +func (r *Registry) NewGroup(gid GID) *Group { + return r.mustGetGroupRegistry(gid).newGroup() +} diff --git a/pkg/metrics/registry_test.go b/pkg/metrics/registry_test.go new file mode 100644 index 00000000000..78496f37f15 --- /dev/null +++ b/pkg/metrics/registry_test.go @@ -0,0 +1,63 @@ +package metrics + +import ( + "testing" + + "github.com/influxdata/influxdb/pkg/testing/assert" +) + +func TestRegistry_MustRegisterCounter(t *testing.T) { + r := NewRegistry() + id := r.MustRegisterCounter("counter") + assert.Equal(t, id, ID(0), "invalid id") +} + +func TestRegistry_MustRegisterCounter_Panics(t *testing.T) { + r := NewRegistry() + r.MustRegisterCounter("counter") + assert.PanicsWithValue(t, "metric name 'counter' already in use", func() { + r.MustRegisterCounter("counter") + }) +} + +func TestRegistry_NewGroup_CounterIsZero(t *testing.T) { + r := NewRegistry() + id := r.MustRegisterCounter("counter") + + c := r.NewGroup(DefaultGroup).GetCounter(id) + c.Add(1) + assert.Equal(t, int64(1), c.Value()) + + c = r.NewGroup(DefaultGroup).GetCounter(id) + assert.Equal(t, int64(0), c.Value()) +} + +func TestRegistry_MustRegisterTimer(t *testing.T) { + r := NewRegistry() + id := r.MustRegisterTimer("timer") + assert.Equal(t, ID(0), id, "invalid id") +} + +func TestRegistry_MustRegisterTimer_Panics(t *testing.T) { + r := NewRegistry() + r.MustRegisterCounter("timer") + assert.PanicsWithValue(t, "metric name 'timer' already in use", func() { + r.MustRegisterCounter("timer") + }) +} + +func TestRegistry_MustRegisterMultiple(t *testing.T) { + r := NewRegistry() + cnt := r.MustRegisterCounter("counter") + tmr := r.MustRegisterTimer("timer") + assert.Equal(t, ID(0), cnt, "invalid id") + assert.Equal(t, ID(0), tmr, "invalid id") +} + +func TestRegistry_MustRegister_Panics_Across_Measurements(t *testing.T) { + r := NewRegistry() + r.MustRegisterCounter("foo") + assert.PanicsWithValue(t, "metric name 'foo' already in use", func() { + r.MustRegisterCounter("foo") + }) +} diff --git a/pkg/metrics/timer.go b/pkg/metrics/timer.go new file mode 100644 index 00000000000..a0382c56db4 --- /dev/null +++ b/pkg/metrics/timer.go @@ -0,0 +1,34 @@ +package metrics + +import ( + "sync/atomic" + "time" +) + +// The timer type is used to store a duration. +type Timer struct { + val int64 + desc *desc +} + +// Name returns the name of the timer. +func (t *Timer) Name() string { return t.desc.Name } + +// Value atomically returns the value of the timer. +func (t *Timer) Value() time.Duration { return time.Duration(atomic.LoadInt64(&t.val)) } + +// Update sets the timer value to d. +func (t *Timer) Update(d time.Duration) { atomic.StoreInt64(&t.val, int64(d)) } + +// UpdateSince sets the timer value to the difference between since and the current time. +func (t *Timer) UpdateSince(since time.Time) { t.Update(time.Since(since)) } + +// String returns a string representation using the name and value of the timer. +func (t *Timer) String() string { return t.desc.Name + ": " + time.Duration(t.val).String() } + +// Time updates the timer to the duration it takes to call f. +func (t *Timer) Time(f func()) { + s := time.Now() + f() + t.UpdateSince(s) +} diff --git a/pkg/metrics/timer_test.go b/pkg/metrics/timer_test.go new file mode 100644 index 00000000000..aca439b1044 --- /dev/null +++ b/pkg/metrics/timer_test.go @@ -0,0 +1,14 @@ +package metrics + +import ( + "testing" + "time" + + "github.com/influxdata/influxdb/pkg/testing/assert" +) + +func TestTimer_Update(t *testing.T) { + var c Timer + c.Update(100 * time.Millisecond) + assert.Equal(t, c.Value(), 100*time.Millisecond, "unexpected value") +} diff --git a/pkg/testing/assert/assertions.go b/pkg/testing/assert/assertions.go new file mode 100644 index 00000000000..52a452f6221 --- /dev/null +++ b/pkg/testing/assert/assertions.go @@ -0,0 +1,96 @@ +package assert + +import ( + "bytes" + "fmt" + "reflect" +) + +type TestingT interface { + Helper() + Errorf(format string, args ...interface{}) +} + +// Equal asserts that the values are equal and returns +// true if the assertion was successful. +func Equal(t TestingT, got, expected interface{}, msgAndArgs ...interface{}) bool { + if ValuesAreEqual(got, expected) { + return true + } + + t.Helper() + got, expected = formatValues(got, expected) + fail(t, fmt.Sprintf("Not Equal: got=%s, exp=%s", got, expected), msgAndArgs...) + return false +} + +// NotEqual asserts that the values are not equal and returns +// true if the assertion was successful. +func NotEqual(t TestingT, got, expected interface{}, msgAndArgs ...interface{}) bool { + if !ValuesAreEqual(got, expected) { + return true + } + + t.Helper() + _, expected = formatValues(got, expected) + fail(t, fmt.Sprintf("Equal: should not be %s", expected), msgAndArgs...) + return false +} + +// PanicsWithValue asserts that fn panics, and that +// the recovered panic value equals the expected panic value. +// +// Returns true if the assertion was successful. +func PanicsWithValue(t TestingT, expected interface{}, fn PanicTestFunc, msgAndArgs ...interface{}) bool { + t.Helper() + if funcDidPanic, got := didPanic(fn); !funcDidPanic { + return fail(t, fmt.Sprintf("func %#v should panic\n\r\tPanic value:\t%v", fn, got), msgAndArgs...) + } else if got != expected { + return fail(t, fmt.Sprintf("func %#v should panic with value:\t%v\n\r\tPanic value:\t%v", fn, expected, got), msgAndArgs...) + } + + return true +} + +// ValuesAreEqual determines if the values are equal. +func ValuesAreEqual(got, expected interface{}) bool { + if got == nil || expected == nil { + return got == expected + } + + if exp, ok := expected.([]byte); ok { + act, ok := got.([]byte) + if !ok { + return false + } else if exp == nil || act == nil { + return exp == nil && act == nil + } + return bytes.Equal(exp, act) + } + + return reflect.DeepEqual(expected, got) + +} + +// ValuesAreExactlyEqual determines if the values are equal and +// their types are the same. +func ValuesAreExactlyEqual(got, expected interface{}) bool { + if ValuesAreEqual(got, expected) { + return true + } + + actualType := reflect.TypeOf(got) + if actualType == nil { + return false + } + expectedValue := reflect.ValueOf(expected) + if expectedValue.IsValid() && expectedValue.Type().ConvertibleTo(actualType) { + // Attempt comparison after type conversion + return reflect.DeepEqual(expectedValue.Convert(actualType).Interface(), got) + } + + return false +} + +// PanicTestFunc defines a function that is called to determine whether a panic occurs. +type PanicTestFunc func() diff --git a/pkg/testing/assert/doc.go b/pkg/testing/assert/doc.go new file mode 100644 index 00000000000..174facb15f0 --- /dev/null +++ b/pkg/testing/assert/doc.go @@ -0,0 +1,4 @@ +/* +Package assert provides helper functions that can be used with the standard Go testing package. +*/ +package assert diff --git a/pkg/testing/assert/helper.go b/pkg/testing/assert/helper.go new file mode 100644 index 00000000000..ca0b84c6206 --- /dev/null +++ b/pkg/testing/assert/helper.go @@ -0,0 +1,53 @@ +package assert + +import ( + "fmt" + "reflect" +) + +func fail(t TestingT, failureMsg string, msgAndArgs ...interface{}) bool { + t.Helper() + + msg := formatMsgAndArgs(msgAndArgs...) + if msg == "" { + t.Errorf("%s", failureMsg) + } else { + t.Errorf("%s: %s", failureMsg, msg) + } + + return false +} + +func formatValues(got, expected interface{}) (string, string) { + if reflect.TypeOf(got) != reflect.TypeOf(expected) { + return fmt.Sprintf("%T(%#v)", got, got), fmt.Sprintf("%T(%#v)", expected, expected) + } + + return fmt.Sprintf("%#v", got), fmt.Sprintf("%#v", expected) +} + +func formatMsgAndArgs(msgAndArgs ...interface{}) string { + if len(msgAndArgs) == 0 || msgAndArgs == nil { + return "" + } + if len(msgAndArgs) == 1 { + return msgAndArgs[0].(string) + } + if len(msgAndArgs) > 1 { + return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) + } + return "" +} + +// didPanic returns true if fn panics when called. +func didPanic(fn PanicTestFunc) (panicked bool, message interface{}) { + defer func() { + if message = recover(); message != nil { + panicked = true + } + }() + + fn() + + return panicked, message +} diff --git a/pkg/tracing/context.go b/pkg/tracing/context.go new file mode 100644 index 00000000000..0b984688549 --- /dev/null +++ b/pkg/tracing/context.go @@ -0,0 +1,32 @@ +package tracing + +import "context" + +type key int + +const ( + spanKey key = iota + traceKey +) + +// NewContextWithSpan returns a new context with the given Span added. +func NewContextWithSpan(ctx context.Context, c *Span) context.Context { + return context.WithValue(ctx, spanKey, c) +} + +// SpanFromContext returns the Span associated with ctx or nil if no Span has been assigned. +func SpanFromContext(ctx context.Context) *Span { + c, _ := ctx.Value(spanKey).(*Span) + return c +} + +// NewContextWithTrace returns a new context with the given Trace added. +func NewContextWithTrace(ctx context.Context, t *Trace) context.Context { + return context.WithValue(ctx, traceKey, t) +} + +// TraceFromContext returns the Trace associated with ctx or nil if no Trace has been assigned. +func TraceFromContext(ctx context.Context) *Trace { + c, _ := ctx.Value(traceKey).(*Trace) + return c +} diff --git a/pkg/tracing/doc.go b/pkg/tracing/doc.go new file mode 100644 index 00000000000..4e7b582d630 --- /dev/null +++ b/pkg/tracing/doc.go @@ -0,0 +1,26 @@ +/* +Package tracing provides a way for capturing hierarchical traces. + +To start a new trace with a root span named select + + trace, span := tracing.NewTrace("select") + +It is recommended that a span be forwarded to callees using the +context package. Firstly, create a new context with the span associated +as follows + + ctx = tracing.NewContextWithSpan(ctx, span) + +followed by calling the API with the new context + + SomeAPI(ctx, ...) + +Once the trace is complete, it may be converted to a graph with the Tree method. + + tree := t.Tree() + +The tree is intended to be used with the Walk function in order to generate +different presentations. The default Tree#String method returns a tree. + +*/ +package tracing diff --git a/pkg/tracing/fields/field.go b/pkg/tracing/fields/field.go new file mode 100644 index 00000000000..38e49071ed8 --- /dev/null +++ b/pkg/tracing/fields/field.go @@ -0,0 +1,117 @@ +package fields + +import ( + "fmt" + "math" + "time" +) + +type fieldType int + +const ( + stringType fieldType = iota + boolType + int64Type + uint64Type + durationType + float64Type +) + +// Field instances are constructed via Bool, String, and so on. +// +// "heavily influenced by" (i.e., partially stolen from) +// https://github.com/opentracing/opentracing-go/log +type Field struct { + key string + fieldType fieldType + numericVal int64 + stringVal string +} + +// String adds a string-valued key:value pair to a Span.LogFields() record +func String(key, val string) Field { + return Field{ + key: key, + fieldType: stringType, + stringVal: val, + } +} + +// Bool adds a bool-valued key:value pair to a Span.LogFields() record +func Bool(key string, val bool) Field { + var numericVal int64 + if val { + numericVal = 1 + } + return Field{ + key: key, + fieldType: boolType, + numericVal: numericVal, + } +} + +/// Int64 adds an int64-valued key:value pair to a Span.LogFields() record +func Int64(key string, val int64) Field { + return Field{ + key: key, + fieldType: int64Type, + numericVal: val, + } +} + +// Uint64 adds a uint64-valued key:value pair to a Span.LogFields() record +func Uint64(key string, val uint64) Field { + return Field{ + key: key, + fieldType: uint64Type, + numericVal: int64(val), + } +} + +// Uint64 adds a uint64-valued key:value pair to a Span.LogFields() record +func Duration(key string, val time.Duration) Field { + return Field{ + key: key, + fieldType: durationType, + numericVal: int64(val), + } +} + +// Float64 adds a float64-valued key:value pair to a Span.LogFields() record +func Float64(key string, val float64) Field { + return Field{ + key: key, + fieldType: float64Type, + numericVal: int64(math.Float64bits(val)), + } +} + +// Key returns the field's key. +func (lf Field) Key() string { + return lf.key +} + +// Value returns the field's value as interface{}. +func (lf Field) Value() interface{} { + switch lf.fieldType { + case stringType: + return lf.stringVal + case boolType: + return lf.numericVal != 0 + case int64Type: + return int64(lf.numericVal) + case uint64Type: + return uint64(lf.numericVal) + case durationType: + return time.Duration(lf.numericVal) + case float64Type: + return math.Float64frombits(uint64(lf.numericVal)) + default: + return nil + } +} + +// String returns a string representation of the key and value. +func (lf Field) String() string { + return fmt.Sprint(lf.key, ": ", lf.Value()) +} diff --git a/pkg/tracing/fields/fields.go b/pkg/tracing/fields/fields.go new file mode 100644 index 00000000000..825cf255095 --- /dev/null +++ b/pkg/tracing/fields/fields.go @@ -0,0 +1,61 @@ +package fields + +import "sort" + +type Fields []Field + +// Merge merges other with the current set, replacing any matching keys from other. +func (fs *Fields) Merge(other Fields) { + var list []Field + i, j := 0, 0 + for i < len(*fs) && j < len(other) { + if (*fs)[i].key < other[j].key { + list = append(list, (*fs)[i]) + i++ + } else if (*fs)[i].key > other[j].key { + list = append(list, other[j]) + j++ + } else { + // equal, then "other" replaces existing key + list = append(list, other[j]) + i++ + j++ + } + } + + if i < len(*fs) { + list = append(list, (*fs)[i:]...) + } else if j < len(other) { + list = append(list, other[j:]...) + } + + *fs = list +} + +// New creates a new set of fields, sorted by Key. +// Duplicate keys are removed. +func New(args ...Field) Fields { + fields := Fields(args) + sort.Slice(fields, func(i, j int) bool { + return fields[i].key < fields[j].key + }) + + // deduplicate + // loop invariant: fields[:i] has no duplicates + for i := 0; i < len(fields)-1; i++ { + j := i + 1 + // find all duplicate keys + for j < len(fields) && fields[i].key == fields[j].key { + j++ + } + + d := (j - 1) - i // number of duplicate keys + if d > 0 { + // copy over duplicate keys in order to maintain loop invariant + copy(fields[i+1:], fields[j:]) + fields = fields[:len(fields)-d] + } + } + + return fields +} diff --git a/pkg/tracing/fields/fields_test.go b/pkg/tracing/fields/fields_test.go new file mode 100644 index 00000000000..f28bb7ade7c --- /dev/null +++ b/pkg/tracing/fields/fields_test.go @@ -0,0 +1,101 @@ +package fields + +import ( + "testing" + + "github.com/influxdata/influxdb/pkg/testing/assert" +) + +func makeFields(args ...string) Fields { + if len(args)%2 != 0 { + panic("uneven number of arguments") + } + + var f Fields + for i := 0; i+1 < len(args); i += 2 { + f = append(f, String(args[i], args[i+1])) + } + return f +} + +func TestNew(t *testing.T) { + cases := []struct { + n string + l []string + exp Fields + }{ + { + n: "empty", + l: nil, + exp: makeFields(), + }, + { + n: "not duplicates", + l: []string{"k01", "v01", "k03", "v03", "k02", "v02"}, + exp: makeFields("k01", "v01", "k02", "v02", "k03", "v03"), + }, + { + n: "duplicates at end", + l: []string{"k01", "v01", "k02", "v02", "k02", "v02"}, + exp: makeFields("k01", "v01", "k02", "v02"), + }, + { + n: "duplicates at start", + l: []string{"k01", "v01", "k02", "v02", "k01", "v01"}, + exp: makeFields("k01", "v01", "k02", "v02"), + }, + { + n: "duplicates in middle", + l: []string{"k01", "v01", "k02", "v02", "k03", "v03", "k02", "v02", "k02", "v02"}, + exp: makeFields("k01", "v01", "k02", "v02", "k03", "v03"), + }, + } + + for _, tc := range cases { + t.Run(tc.n, func(t *testing.T) { + l := New(makeFields(tc.l...)...) + assert.Equal(t, tc.exp, l) + }) + } +} + +func TestFields_Merge(t *testing.T) { + cases := []struct { + n string + l, r Fields + exp Fields + }{ + { + n: "no matching keys", + l: New(String("k05", "v05"), String("k03", "v03"), String("k01", "v01")), + r: New(String("k02", "v02"), String("k04", "v04"), String("k00", "v00")), + exp: New(String("k05", "v05"), String("k03", "v03"), String("k01", "v01"), String("k02", "v02"), String("k04", "v04"), String("k00", "v00")), + }, + { + n: "multiple matching keys", + l: New(String("k05", "v05"), String("k03", "v03"), String("k01", "v01")), + r: New(String("k02", "v02"), String("k03", "v03a"), String("k05", "v05a")), + exp: New(String("k05", "v05a"), String("k03", "v03a"), String("k01", "v01"), String("k02", "v02")), + }, + { + n: "source empty", + l: New(), + r: New(String("k02", "v02"), String("k04", "v04"), String("k00", "v00")), + exp: New(String("k02", "v02"), String("k04", "v04"), String("k00", "v00")), + }, + { + n: "other empty", + l: New(String("k02", "v02"), String("k04", "v04"), String("k00", "v00")), + r: New(), + exp: New(String("k02", "v02"), String("k04", "v04"), String("k00", "v00")), + }, + } + + for _, tc := range cases { + t.Run(tc.n, func(t *testing.T) { + l := tc.l + l.Merge(tc.r) + assert.Equal(t, tc.exp, l) + }) + } +} diff --git a/pkg/tracing/labels/labels.go b/pkg/tracing/labels/labels.go new file mode 100644 index 00000000000..90afda7dabf --- /dev/null +++ b/pkg/tracing/labels/labels.go @@ -0,0 +1,74 @@ +package labels + +import "sort" + +type Label struct { + Key, Value string +} + +// The Labels type represents a set of labels, sorted by Key. +type Labels []Label + +// Merge merges other with the current set, replacing any matching keys from other. +func (ls *Labels) Merge(other Labels) { + var list []Label + i, j := 0, 0 + for i < len(*ls) && j < len(other) { + if (*ls)[i].Key < other[j].Key { + list = append(list, (*ls)[i]) + i++ + } else if (*ls)[i].Key > other[j].Key { + list = append(list, other[j]) + j++ + } else { + // equal, then "other" replaces existing key + list = append(list, other[j]) + i++ + j++ + } + } + + if i < len(*ls) { + list = append(list, (*ls)[i:]...) + } else if j < len(other) { + list = append(list, other[j:]...) + } + + *ls = list +} + +// New takes an even number of strings representing key-value pairs +// and creates a new slice of Labels. Duplicates are removed, however, +// there is no guarantee which will be removed +func New(args ...string) Labels { + if len(args)%2 != 0 { + panic("uneven number of arguments to label.Labels") + } + var labels Labels + for i := 0; i+1 < len(args); i += 2 { + labels = append(labels, Label{Key: args[i], Value: args[i+1]}) + } + + sort.Slice(labels, func(i, j int) bool { + return labels[i].Key < labels[j].Key + }) + + // deduplicate + // loop invariant: labels[:i] has no duplicates + for i := 0; i < len(labels)-1; i++ { + j := i + 1 + // find all duplicate keys + for j < len(labels) && labels[i].Key == labels[j].Key { + j++ + } + + d := (j - 1) - i // number of duplicate keys + if d > 0 { + // copy over duplicate keys in order to maintain loop invariant + copy(labels[i+1:], labels[j:]) + labels = labels[:len(labels)-d] + } + } + + return labels +} diff --git a/pkg/tracing/labels/labels_test.go b/pkg/tracing/labels/labels_test.go new file mode 100644 index 00000000000..7e8bcc11f4c --- /dev/null +++ b/pkg/tracing/labels/labels_test.go @@ -0,0 +1,101 @@ +package labels + +import ( + "testing" + + "github.com/influxdata/influxdb/pkg/testing/assert" +) + +func makeLabels(args ...string) Labels { + if len(args)%2 != 0 { + panic("uneven number of arguments") + } + + var l Labels + for i := 0; i+1 < len(args); i += 2 { + l = append(l, Label{Key: args[i], Value: args[i+1]}) + } + return l +} + +func TestNew(t *testing.T) { + cases := []struct { + n string + l []string + exp Labels + }{ + { + n: "empty", + l: nil, + exp: makeLabels(), + }, + { + n: "not duplicates", + l: []string{"k01", "v01", "k03", "v03", "k02", "v02"}, + exp: makeLabels("k01", "v01", "k02", "v02", "k03", "v03"), + }, + { + n: "duplicates at end", + l: []string{"k01", "v01", "k02", "v02", "k02", "v02"}, + exp: makeLabels("k01", "v01", "k02", "v02"), + }, + { + n: "duplicates at start", + l: []string{"k01", "v01", "k02", "v02", "k01", "v01"}, + exp: makeLabels("k01", "v01", "k02", "v02"), + }, + { + n: "duplicates in middle", + l: []string{"k01", "v01", "k02", "v02", "k03", "v03", "k02", "v02", "k02", "v02"}, + exp: makeLabels("k01", "v01", "k02", "v02", "k03", "v03"), + }, + } + + for _, tc := range cases { + t.Run(tc.n, func(t *testing.T) { + l := New(tc.l...) + assert.Equal(t, l, tc.exp) + }) + } +} + +func TestLabels_Merge(t *testing.T) { + cases := []struct { + n string + l, r Labels + exp Labels + }{ + { + n: "no matching keys", + l: New("k05", "v05", "k03", "v03", "k01", "v01"), + r: New("k02", "v02", "k04", "v04", "k00", "v00"), + exp: New("k05", "v05", "k03", "v03", "k01", "v01", "k02", "v02", "k04", "v04", "k00", "v00"), + }, + { + n: "multiple matching keys", + l: New("k05", "v05", "k03", "v03", "k01", "v01"), + r: New("k02", "v02", "k03", "v03a", "k05", "v05a"), + exp: New("k05", "v05a", "k03", "v03a", "k01", "v01", "k02", "v02"), + }, + { + n: "source empty", + l: New(), + r: New("k02", "v02", "k04", "v04", "k00", "v00"), + exp: New("k02", "v02", "k04", "v04", "k00", "v00"), + }, + { + n: "other empty", + l: New("k02", "v02", "k04", "v04", "k00", "v00"), + r: New(), + exp: New("k02", "v02", "k04", "v04", "k00", "v00"), + }, + } + + for _, tc := range cases { + t.Run(tc.n, func(t *testing.T) { + l := tc.l + l.Merge(tc.r) + assert.Equal(t, l, tc.exp) + }) + } +} diff --git a/pkg/tracing/rawspan.go b/pkg/tracing/rawspan.go new file mode 100644 index 00000000000..12e37e52a57 --- /dev/null +++ b/pkg/tracing/rawspan.go @@ -0,0 +1,18 @@ +package tracing + +import ( + "time" + + "github.com/influxdata/influxdb/pkg/tracing/fields" + "github.com/influxdata/influxdb/pkg/tracing/labels" +) + +// RawSpan represents the data associated with a span. +type RawSpan struct { + Context SpanContext + ParentSpanID uint64 // ParentSpanID identifies the parent of this span or 0 if this is the root span. + Name string // Name is the operation name given to this span. + Start time.Time // Start identifies the start time of the span. + Labels labels.Labels // Labels contains additional metadata about this span. + Fields fields.Fields // Fields contains typed values associated with this span. +} diff --git a/pkg/tracing/span.go b/pkg/tracing/span.go new file mode 100644 index 00000000000..c8bcfb4cbfb --- /dev/null +++ b/pkg/tracing/span.go @@ -0,0 +1,84 @@ +package tracing + +import ( + "sync" + "time" + + "github.com/influxdata/influxdb/pkg/tracing/fields" + "github.com/influxdata/influxdb/pkg/tracing/labels" +) + +// The Span type denotes a specific operation for a Trace. +// A Span may have one or more children, identifying additional +// details about a trace. +type Span struct { + tracer *Trace + mu sync.Mutex + raw RawSpan +} + +type StartSpanOption interface { + applyStart(*Span) +} + +// The StartTime start span option specifies the start time of +// the new span rather than using now. +type StartTime time.Time + +func (t StartTime) applyStart(s *Span) { + s.raw.Start = time.Time(t) +} + +// StartSpan creates a new child span using time.Now as the start time. +func (s *Span) StartSpan(name string, opt ...StartSpanOption) *Span { + return s.tracer.startSpan(name, s.raw.Context, opt) +} + +// Context returns a SpanContext that can be serialized and passed to a remote node to continue a trace. +func (s *Span) Context() SpanContext { + return s.raw.Context +} + +// SetLabels replaces any existing labels for the Span with args. +func (s *Span) SetLabels(args ...string) { + s.mu.Lock() + s.raw.Labels = labels.New(args...) + s.mu.Unlock() +} + +// MergeLabels merges args with any existing labels defined +// for the Span. +func (s *Span) MergeLabels(args ...string) { + ls := labels.New(args...) + s.mu.Lock() + s.raw.Labels.Merge(ls) + s.mu.Unlock() +} + +// SetFields replaces any existing fields for the Span with args. +func (s *Span) SetFields(set fields.Fields) { + s.mu.Lock() + s.raw.Fields = set + s.mu.Unlock() +} + +// MergeFields merges the provides args with any existing fields defined +// for the Span. +func (s *Span) MergeFields(args ...fields.Field) { + set := fields.New(args...) + s.mu.Lock() + s.raw.Fields.Merge(set) + s.mu.Unlock() +} + +// Finish marks the end of the span and records it to the associated Trace. +// If Finish is not called, the span will not appear in the trace. +func (s *Span) Finish() { + s.mu.Lock() + s.tracer.addRawSpan(s.raw) + s.mu.Unlock() +} + +func (s *Span) Tree() *TreeNode { + return s.tracer.TreeFrom(s.raw.Context.SpanID) +} diff --git a/pkg/tracing/spancontext.go b/pkg/tracing/spancontext.go new file mode 100644 index 00000000000..62cf7af6951 --- /dev/null +++ b/pkg/tracing/spancontext.go @@ -0,0 +1,27 @@ +package tracing + +import ( + "github.com/gogo/protobuf/proto" + "github.com/influxdata/influxdb/pkg/tracing/wire" +) + +// A SpanContext represents the minimal information to identify a span in a trace. +// This is typically serialized to continue a trace on a remote node. +type SpanContext struct { + TraceID uint64 // TraceID is assigned a random number to this trace. + SpanID uint64 // SpanID is assigned a random number to identify this span. +} + +func (s SpanContext) MarshalBinary() ([]byte, error) { + ws := wire.SpanContext(s) + return proto.Marshal(&ws) +} + +func (s *SpanContext) UnmarshalBinary(data []byte) error { + var ws wire.SpanContext + err := proto.Unmarshal(data, &ws) + if err == nil { + *s = SpanContext(ws) + } + return err +} diff --git a/pkg/tracing/trace.go b/pkg/tracing/trace.go new file mode 100644 index 00000000000..0143d1fa065 --- /dev/null +++ b/pkg/tracing/trace.go @@ -0,0 +1,141 @@ +package tracing + +import ( + "sort" + "sync" + "time" +) + +// The Trace type functions as a container for capturing Spans used to +// trace the execution of a request. +type Trace struct { + mu sync.Mutex + spans map[uint64]RawSpan +} + +// NewTrace starts a new trace and returns a root span identified by the provided name. +// +// Additional options may be specified to override the default behavior when creating the span. +func NewTrace(name string, opt ...StartSpanOption) (*Trace, *Span) { + t := &Trace{spans: make(map[uint64]RawSpan)} + s := &Span{tracer: t} + s.raw.Name = name + s.raw.Context.TraceID, s.raw.Context.SpanID = randomID2() + setOptions(s, opt) + + return t, s +} + +// NewTraceFromSpan starts a new trace and returns the associated span, which is a child of the +// parent span context. +func NewTraceFromSpan(name string, parent SpanContext, opt ...StartSpanOption) (*Trace, *Span) { + t := &Trace{spans: make(map[uint64]RawSpan)} + s := &Span{tracer: t} + s.raw.Name = name + s.raw.ParentSpanID = parent.SpanID + s.raw.Context.TraceID = parent.TraceID + s.raw.Context.SpanID = randomID() + setOptions(s, opt) + + return t, s +} + +func (t *Trace) startSpan(name string, sc SpanContext, opt []StartSpanOption) *Span { + s := &Span{tracer: t} + s.raw.Name = name + s.raw.Context.SpanID = randomID() + s.raw.Context.TraceID = sc.TraceID + s.raw.ParentSpanID = sc.SpanID + setOptions(s, opt) + + return s +} + +func setOptions(s *Span, opt []StartSpanOption) { + for _, o := range opt { + o.applyStart(s) + } + + if s.raw.Start.IsZero() { + s.raw.Start = time.Now() + } +} + +func (t *Trace) addRawSpan(raw RawSpan) { + t.mu.Lock() + t.spans[raw.Context.SpanID] = raw + t.mu.Unlock() +} + +// Tree returns a graph of the current trace. +func (t *Trace) Tree() *TreeNode { + t.mu.Lock() + defer t.mu.Unlock() + + for _, s := range t.spans { + if s.ParentSpanID == 0 { + return t.treeFrom(s.Context.SpanID) + } + } + return nil +} + +// Merge combines other with the current trace. This is +// typically necessary when traces are transferred from a remote. +func (t *Trace) Merge(other *Trace) { + for k, s := range other.spans { + t.spans[k] = s + } +} + +func (t *Trace) TreeFrom(root uint64) *TreeNode { + t.mu.Lock() + defer t.mu.Unlock() + return t.treeFrom(root) +} + +func (t *Trace) treeFrom(root uint64) *TreeNode { + c := map[uint64]*TreeNode{} + + for k, s := range t.spans { + c[k] = &TreeNode{Raw: s} + } + + if _, ok := c[root]; !ok { + return nil + } + + for _, n := range c { + if n.Raw.ParentSpanID != 0 { + if pn := c[n.Raw.ParentSpanID]; pn != nil { + pn.Children = append(pn.Children, n) + } + } + } + + // sort nodes + var v treeSortVisitor + Walk(&v, c[root]) + + return c[root] +} + +type treeSortVisitor struct{} + +func (v *treeSortVisitor) Visit(node *TreeNode) Visitor { + sort.Slice(node.Children, func(i, j int) bool { + lt, rt := node.Children[i].Raw.Start.UnixNano(), node.Children[j].Raw.Start.UnixNano() + if lt < rt { + return true + } else if lt > rt { + return false + } + + ln, rn := node.Children[i].Raw.Name, node.Children[j].Raw.Name + if ln < rn { + return true + } + return false + }) + return v +} diff --git a/pkg/tracing/trace_encoding.go b/pkg/tracing/trace_encoding.go new file mode 100644 index 00000000000..31c3b3384a0 --- /dev/null +++ b/pkg/tracing/trace_encoding.go @@ -0,0 +1,136 @@ +package tracing + +import ( + "math" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/influxdata/influxdb/pkg/tracing/fields" + "github.com/influxdata/influxdb/pkg/tracing/labels" + "github.com/influxdata/influxdb/pkg/tracing/wire" +) + +func fieldsToWire(set fields.Fields) []wire.Field { + var r []wire.Field + for _, f := range set { + wf := wire.Field{Key: f.Key()} + switch val := f.Value().(type) { + case string: + wf.FieldType = wire.FieldTypeString + wf.Value = &wire.Field_StringVal{StringVal: val} + + case bool: + var numericVal int64 + if val { + numericVal = 1 + } + wf.FieldType = wire.FieldTypeBool + wf.Value = &wire.Field_NumericVal{NumericVal: numericVal} + + case int64: + wf.FieldType = wire.FieldTypeInt64 + wf.Value = &wire.Field_NumericVal{NumericVal: val} + + case uint64: + wf.FieldType = wire.FieldTypeUint64 + wf.Value = &wire.Field_NumericVal{NumericVal: int64(val)} + + case time.Duration: + wf.FieldType = wire.FieldTypeDuration + wf.Value = &wire.Field_NumericVal{NumericVal: int64(val)} + + case float64: + wf.FieldType = wire.FieldTypeFloat64 + wf.Value = &wire.Field_NumericVal{NumericVal: int64(math.Float64bits(val))} + + default: + continue + } + + r = append(r, wf) + } + return r +} + +func labelsToWire(set labels.Labels) []string { + var r []string + for i := range set { + r = append(r, set[i].Key, set[i].Value) + } + return r +} + +func (t *Trace) MarshalBinary() ([]byte, error) { + wt := wire.Trace{} + for _, sp := range t.spans { + wt.Spans = append(wt.Spans, &wire.Span{ + Context: wire.SpanContext{ + TraceID: sp.Context.TraceID, + SpanID: sp.Context.SpanID, + }, + ParentSpanID: sp.ParentSpanID, + Name: sp.Name, + Start: sp.Start, + Labels: labelsToWire(sp.Labels), + Fields: fieldsToWire(sp.Fields), + }) + } + + return proto.Marshal(&wt) +} + +func wireToFields(wfs []wire.Field) fields.Fields { + var fs []fields.Field + for _, wf := range wfs { + switch wf.FieldType { + case wire.FieldTypeString: + fs = append(fs, fields.String(wf.Key, wf.GetStringVal())) + + case wire.FieldTypeBool: + var boolVal bool + if wf.GetNumericVal() != 0 { + boolVal = true + } + fs = append(fs, fields.Bool(wf.Key, boolVal)) + + case wire.FieldTypeInt64: + fs = append(fs, fields.Int64(wf.Key, wf.GetNumericVal())) + + case wire.FieldTypeUint64: + fs = append(fs, fields.Uint64(wf.Key, uint64(wf.GetNumericVal()))) + + case wire.FieldTypeDuration: + fs = append(fs, fields.Duration(wf.Key, time.Duration(wf.GetNumericVal()))) + + case wire.FieldTypeFloat64: + fs = append(fs, fields.Float64(wf.Key, math.Float64frombits(uint64(wf.GetNumericVal())))) + } + } + + return fields.New(fs...) +} + +func (t *Trace) UnmarshalBinary(data []byte) error { + var wt wire.Trace + if err := proto.Unmarshal(data, &wt); err != nil { + return err + } + + t.spans = make(map[uint64]RawSpan) + + for _, sp := range wt.Spans { + t.spans[sp.Context.SpanID] = RawSpan{ + Context: SpanContext{ + TraceID: sp.Context.TraceID, + SpanID: sp.Context.SpanID, + }, + ParentSpanID: sp.ParentSpanID, + Name: sp.Name, + Start: sp.Start, + Labels: labels.New(sp.Labels...), + Fields: wireToFields(sp.Fields), + } + } + + return nil +} diff --git a/pkg/tracing/tree.go b/pkg/tracing/tree.go new file mode 100644 index 00000000000..0321be64124 --- /dev/null +++ b/pkg/tracing/tree.go @@ -0,0 +1,74 @@ +package tracing + +import ( + "github.com/xlab/treeprint" +) + +// A Visitor's Visit method is invoked for each node encountered by Walk. +// If the result of Visit is not nil, Walk visits each of the children. +type Visitor interface { + Visit(*TreeNode) Visitor +} + +// A TreeNode represents a single node in the graph. +type TreeNode struct { + Raw RawSpan + Children []*TreeNode +} + +// String returns the tree as a string. +func (t *TreeNode) String() string { + if t == nil { + return "" + } + tv := newTreeVisitor() + Walk(tv, t) + return tv.root.String() +} + +// Walk traverses the graph in a depth-first order, calling v.Visit +// for each node until completion or v.Visit returns nil. +func Walk(v Visitor, node *TreeNode) { + if v = v.Visit(node); v == nil { + return + } + + for _, c := range node.Children { + Walk(v, c) + } +} + +type treeVisitor struct { + root treeprint.Tree + trees []treeprint.Tree +} + +func newTreeVisitor() *treeVisitor { + t := treeprint.New() + return &treeVisitor{root: t, trees: []treeprint.Tree{t}} +} + +func (v *treeVisitor) Visit(n *TreeNode) Visitor { + t := v.trees[len(v.trees)-1].AddBranch(n.Raw.Name) + v.trees = append(v.trees, t) + + if labels := n.Raw.Labels; len(labels) > 0 { + l := t.AddBranch("labels") + for _, ll := range n.Raw.Labels { + l.AddNode(ll.Key + ": " + ll.Value) + } + } + + for _, k := range n.Raw.Fields { + t.AddNode(k.String()) + } + + for _, cn := range n.Children { + Walk(v, cn) + } + + v.trees[len(v.trees)-1] = nil + v.trees = v.trees[:len(v.trees)-1] + + return nil +} diff --git a/pkg/tracing/util.go b/pkg/tracing/util.go new file mode 100644 index 00000000000..f98cc776a1f --- /dev/null +++ b/pkg/tracing/util.go @@ -0,0 +1,26 @@ +package tracing + +import ( + "math/rand" + "sync" + "time" +) + +var ( + seededIDGen = rand.New(rand.NewSource(time.Now().UnixNano())) + seededIDLock sync.Mutex +) + +func randomID() (n uint64) { + seededIDLock.Lock() + n = uint64(seededIDGen.Int63()) + seededIDLock.Unlock() + return +} + +func randomID2() (n uint64, m uint64) { + seededIDLock.Lock() + n, m = uint64(seededIDGen.Int63()), uint64(seededIDGen.Int63()) + seededIDLock.Unlock() + return +} diff --git a/pkg/tracing/wire/binary.go b/pkg/tracing/wire/binary.go new file mode 100644 index 00000000000..62bb854ce94 --- /dev/null +++ b/pkg/tracing/wire/binary.go @@ -0,0 +1,7 @@ +/* +Package wire is used to serialize a trace. + +*/ +package wire + +//go:generate protoc -I$GOPATH/src -I. --gogofaster_out=Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types:. binary.proto diff --git a/pkg/tracing/wire/binary.pb.go b/pkg/tracing/wire/binary.pb.go new file mode 100644 index 00000000000..377bea888e3 --- /dev/null +++ b/pkg/tracing/wire/binary.pb.go @@ -0,0 +1,1292 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: binary.proto + +/* + Package wire is a generated protocol buffer package. + + It is generated from these files: + binary.proto + + It has these top-level messages: + SpanContext + Span + Trace + Field +*/ +package wire + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "github.com/gogo/protobuf/gogoproto" +import _ "github.com/gogo/protobuf/types" + +import time "time" + +import github_com_gogo_protobuf_types "github.com/gogo/protobuf/types" + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf +var _ = time.Kitchen + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +type Field_FieldType int32 + +const ( + FieldTypeString Field_FieldType = 0 + FieldTypeBool Field_FieldType = 1 + FieldTypeInt64 Field_FieldType = 2 + FieldTypeUint64 Field_FieldType = 3 + FieldTypeDuration Field_FieldType = 4 + FieldTypeFloat64 Field_FieldType = 6 +) + +var Field_FieldType_name = map[int32]string{ + 0: "STRING", + 1: "BOOL", + 2: "INT_64", + 3: "UINT_64", + 4: "DURATION", + 6: "FLOAT_64", +} +var Field_FieldType_value = map[string]int32{ + "STRING": 0, + "BOOL": 1, + "INT_64": 2, + "UINT_64": 3, + "DURATION": 4, + "FLOAT_64": 6, +} + +func (x Field_FieldType) String() string { + return proto.EnumName(Field_FieldType_name, int32(x)) +} +func (Field_FieldType) EnumDescriptor() ([]byte, []int) { return fileDescriptorBinary, []int{3, 0} } + +type SpanContext struct { + TraceID uint64 `protobuf:"varint,1,opt,name=trace_id,json=traceId,proto3" json:"trace_id,omitempty"` + SpanID uint64 `protobuf:"varint,2,opt,name=span_id,json=spanId,proto3" json:"span_id,omitempty"` +} + +func (m *SpanContext) Reset() { *m = SpanContext{} } +func (m *SpanContext) String() string { return proto.CompactTextString(m) } +func (*SpanContext) ProtoMessage() {} +func (*SpanContext) Descriptor() ([]byte, []int) { return fileDescriptorBinary, []int{0} } + +func (m *SpanContext) GetTraceID() uint64 { + if m != nil { + return m.TraceID + } + return 0 +} + +func (m *SpanContext) GetSpanID() uint64 { + if m != nil { + return m.SpanID + } + return 0 +} + +type Span struct { + Context SpanContext `protobuf:"bytes,1,opt,name=context" json:"context"` + ParentSpanID uint64 `protobuf:"varint,2,opt,name=parent_span_id,json=parentSpanId,proto3" json:"parent_span_id,omitempty"` + Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` + Start time.Time `protobuf:"bytes,4,opt,name=start_time,json=startTime,stdtime" json:"start_time"` + Labels []string `protobuf:"bytes,5,rep,name=labels" json:"labels,omitempty"` + Fields []Field `protobuf:"bytes,6,rep,name=fields" json:"fields"` +} + +func (m *Span) Reset() { *m = Span{} } +func (m *Span) String() string { return proto.CompactTextString(m) } +func (*Span) ProtoMessage() {} +func (*Span) Descriptor() ([]byte, []int) { return fileDescriptorBinary, []int{1} } + +func (m *Span) GetContext() SpanContext { + if m != nil { + return m.Context + } + return SpanContext{} +} + +func (m *Span) GetParentSpanID() uint64 { + if m != nil { + return m.ParentSpanID + } + return 0 +} + +func (m *Span) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *Span) GetStart() time.Time { + if m != nil { + return m.Start + } + return time.Time{} +} + +func (m *Span) GetLabels() []string { + if m != nil { + return m.Labels + } + return nil +} + +func (m *Span) GetFields() []Field { + if m != nil { + return m.Fields + } + return nil +} + +type Trace struct { + Spans []*Span `protobuf:"bytes,1,rep,name=spans" json:"spans,omitempty"` +} + +func (m *Trace) Reset() { *m = Trace{} } +func (m *Trace) String() string { return proto.CompactTextString(m) } +func (*Trace) ProtoMessage() {} +func (*Trace) Descriptor() ([]byte, []int) { return fileDescriptorBinary, []int{2} } + +func (m *Trace) GetSpans() []*Span { + if m != nil { + return m.Spans + } + return nil +} + +type Field struct { + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + FieldType Field_FieldType `protobuf:"varint,2,opt,name=field_type,json=fieldType,proto3,enum=wire.Field_FieldType" json:"field_type,omitempty"` + // Types that are valid to be assigned to Value: + // *Field_NumericVal + // *Field_StringVal + Value isField_Value `protobuf_oneof:"value"` +} + +func (m *Field) Reset() { *m = Field{} } +func (m *Field) String() string { return proto.CompactTextString(m) } +func (*Field) ProtoMessage() {} +func (*Field) Descriptor() ([]byte, []int) { return fileDescriptorBinary, []int{3} } + +type isField_Value interface { + isField_Value() + MarshalTo([]byte) (int, error) + Size() int +} + +type Field_NumericVal struct { + NumericVal int64 `protobuf:"fixed64,3,opt,name=numeric_val,json=numericVal,proto3,oneof"` +} +type Field_StringVal struct { + StringVal string `protobuf:"bytes,4,opt,name=string_val,json=stringVal,proto3,oneof"` +} + +func (*Field_NumericVal) isField_Value() {} +func (*Field_StringVal) isField_Value() {} + +func (m *Field) GetValue() isField_Value { + if m != nil { + return m.Value + } + return nil +} + +func (m *Field) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +func (m *Field) GetFieldType() Field_FieldType { + if m != nil { + return m.FieldType + } + return FieldTypeString +} + +func (m *Field) GetNumericVal() int64 { + if x, ok := m.GetValue().(*Field_NumericVal); ok { + return x.NumericVal + } + return 0 +} + +func (m *Field) GetStringVal() string { + if x, ok := m.GetValue().(*Field_StringVal); ok { + return x.StringVal + } + return "" +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*Field) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _Field_OneofMarshaler, _Field_OneofUnmarshaler, _Field_OneofSizer, []interface{}{ + (*Field_NumericVal)(nil), + (*Field_StringVal)(nil), + } +} + +func _Field_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*Field) + // value + switch x := m.Value.(type) { + case *Field_NumericVal: + _ = b.EncodeVarint(3<<3 | proto.WireFixed64) + _ = b.EncodeFixed64(uint64(x.NumericVal)) + case *Field_StringVal: + _ = b.EncodeVarint(4<<3 | proto.WireBytes) + _ = b.EncodeStringBytes(x.StringVal) + case nil: + default: + return fmt.Errorf("Field.Value has unexpected type %T", x) + } + return nil +} + +func _Field_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*Field) + switch tag { + case 3: // value.numeric_val + if wire != proto.WireFixed64 { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeFixed64() + m.Value = &Field_NumericVal{int64(x)} + return true, err + case 4: // value.string_val + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeStringBytes() + m.Value = &Field_StringVal{x} + return true, err + default: + return false, nil + } +} + +func _Field_OneofSizer(msg proto.Message) (n int) { + m := msg.(*Field) + // value + switch x := m.Value.(type) { + case *Field_NumericVal: + n += proto.SizeVarint(3<<3 | proto.WireFixed64) + n += 8 + case *Field_StringVal: + n += proto.SizeVarint(4<<3 | proto.WireBytes) + n += proto.SizeVarint(uint64(len(x.StringVal))) + n += len(x.StringVal) + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +func init() { + proto.RegisterType((*SpanContext)(nil), "wire.SpanContext") + proto.RegisterType((*Span)(nil), "wire.Span") + proto.RegisterType((*Trace)(nil), "wire.Trace") + proto.RegisterType((*Field)(nil), "wire.Field") + proto.RegisterEnum("wire.Field_FieldType", Field_FieldType_name, Field_FieldType_value) +} +func (m *SpanContext) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpanContext) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.TraceID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintBinary(dAtA, i, uint64(m.TraceID)) + } + if m.SpanID != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintBinary(dAtA, i, uint64(m.SpanID)) + } + return i, nil +} + +func (m *Span) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Span) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0xa + i++ + i = encodeVarintBinary(dAtA, i, uint64(m.Context.Size())) + n1, err := m.Context.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n1 + if m.ParentSpanID != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintBinary(dAtA, i, uint64(m.ParentSpanID)) + } + if len(m.Name) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintBinary(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + dAtA[i] = 0x22 + i++ + i = encodeVarintBinary(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(m.Start))) + n2, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.Start, dAtA[i:]) + if err != nil { + return 0, err + } + i += n2 + if len(m.Labels) > 0 { + for _, s := range m.Labels { + dAtA[i] = 0x2a + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if len(m.Fields) > 0 { + for _, msg := range m.Fields { + dAtA[i] = 0x32 + i++ + i = encodeVarintBinary(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *Trace) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Trace) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Spans) > 0 { + for _, msg := range m.Spans { + dAtA[i] = 0xa + i++ + i = encodeVarintBinary(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *Field) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Field) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Key) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintBinary(dAtA, i, uint64(len(m.Key))) + i += copy(dAtA[i:], m.Key) + } + if m.FieldType != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintBinary(dAtA, i, uint64(m.FieldType)) + } + if m.Value != nil { + nn3, err := m.Value.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += nn3 + } + return i, nil +} + +func (m *Field_NumericVal) MarshalTo(dAtA []byte) (int, error) { + i := 0 + dAtA[i] = 0x19 + i++ + i = encodeFixed64Binary(dAtA, i, uint64(m.NumericVal)) + return i, nil +} +func (m *Field_StringVal) MarshalTo(dAtA []byte) (int, error) { + i := 0 + dAtA[i] = 0x22 + i++ + i = encodeVarintBinary(dAtA, i, uint64(len(m.StringVal))) + i += copy(dAtA[i:], m.StringVal) + return i, nil +} +func encodeFixed64Binary(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Binary(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} +func encodeVarintBinary(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *SpanContext) Size() (n int) { + var l int + _ = l + if m.TraceID != 0 { + n += 1 + sovBinary(uint64(m.TraceID)) + } + if m.SpanID != 0 { + n += 1 + sovBinary(uint64(m.SpanID)) + } + return n +} + +func (m *Span) Size() (n int) { + var l int + _ = l + l = m.Context.Size() + n += 1 + l + sovBinary(uint64(l)) + if m.ParentSpanID != 0 { + n += 1 + sovBinary(uint64(m.ParentSpanID)) + } + l = len(m.Name) + if l > 0 { + n += 1 + l + sovBinary(uint64(l)) + } + l = github_com_gogo_protobuf_types.SizeOfStdTime(m.Start) + n += 1 + l + sovBinary(uint64(l)) + if len(m.Labels) > 0 { + for _, s := range m.Labels { + l = len(s) + n += 1 + l + sovBinary(uint64(l)) + } + } + if len(m.Fields) > 0 { + for _, e := range m.Fields { + l = e.Size() + n += 1 + l + sovBinary(uint64(l)) + } + } + return n +} + +func (m *Trace) Size() (n int) { + var l int + _ = l + if len(m.Spans) > 0 { + for _, e := range m.Spans { + l = e.Size() + n += 1 + l + sovBinary(uint64(l)) + } + } + return n +} + +func (m *Field) Size() (n int) { + var l int + _ = l + l = len(m.Key) + if l > 0 { + n += 1 + l + sovBinary(uint64(l)) + } + if m.FieldType != 0 { + n += 1 + sovBinary(uint64(m.FieldType)) + } + if m.Value != nil { + n += m.Value.Size() + } + return n +} + +func (m *Field_NumericVal) Size() (n int) { + var l int + _ = l + n += 9 + return n +} +func (m *Field_StringVal) Size() (n int) { + var l int + _ = l + l = len(m.StringVal) + n += 1 + l + sovBinary(uint64(l)) + return n +} + +func sovBinary(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozBinary(x uint64) (n int) { + return sovBinary(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *SpanContext) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpanContext: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpanContext: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TraceID", wireType) + } + m.TraceID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.TraceID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SpanID", wireType) + } + m.SpanID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SpanID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipBinary(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBinary + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Span) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Span: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Span: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Context", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthBinary + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Context.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ParentSpanID", wireType) + } + m.ParentSpanID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ParentSpanID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthBinary + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Start", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthBinary + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(&m.Start, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthBinary + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Labels = append(m.Labels, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthBinary + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Fields = append(m.Fields, Field{}) + if err := m.Fields[len(m.Fields)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBinary(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBinary + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Trace) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Trace: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Trace: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Spans", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthBinary + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Spans = append(m.Spans, &Span{}) + if err := m.Spans[len(m.Spans)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBinary(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBinary + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Field) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Field: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Field: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthBinary + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Key = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldType", wireType) + } + m.FieldType = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FieldType |= (Field_FieldType(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 1 { + return fmt.Errorf("proto: wrong wireType = %d for field NumericVal", wireType) + } + var v int64 + if (iNdEx + 8) > l { + return io.ErrUnexpectedEOF + } + iNdEx += 8 + v = int64(dAtA[iNdEx-8]) + v |= int64(dAtA[iNdEx-7]) << 8 + v |= int64(dAtA[iNdEx-6]) << 16 + v |= int64(dAtA[iNdEx-5]) << 24 + v |= int64(dAtA[iNdEx-4]) << 32 + v |= int64(dAtA[iNdEx-3]) << 40 + v |= int64(dAtA[iNdEx-2]) << 48 + v |= int64(dAtA[iNdEx-1]) << 56 + m.Value = &Field_NumericVal{v} + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StringVal", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowBinary + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthBinary + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Value = &Field_StringVal{string(dAtA[iNdEx:postIndex])} + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipBinary(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthBinary + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipBinary(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBinary + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBinary + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBinary + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthBinary + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowBinary + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipBinary(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthBinary = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowBinary = fmt.Errorf("proto: integer overflow") +) + +func init() { proto.RegisterFile("binary.proto", fileDescriptorBinary) } + +var fileDescriptorBinary = []byte{ + // 624 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x52, 0x41, 0x6f, 0xda, 0x4c, + 0x10, 0xc5, 0xc1, 0x98, 0x78, 0x48, 0xf8, 0xcc, 0x7e, 0x4d, 0x85, 0x5c, 0x09, 0x5b, 0x44, 0xaa, + 0xc8, 0xa1, 0x8e, 0x92, 0x46, 0xdc, 0xe3, 0xa0, 0xb4, 0x96, 0x22, 0xa8, 0x0c, 0xe9, 0xa1, 0x17, + 0xb4, 0xc0, 0x42, 0xad, 0x1a, 0xaf, 0x65, 0x2f, 0x69, 0xf9, 0x07, 0x15, 0xa7, 0x9c, 0x7a, 0xe3, + 0xd4, 0x43, 0xff, 0x4a, 0x8e, 0x3d, 0xf7, 0xe0, 0x56, 0xee, 0x1f, 0xa9, 0x76, 0x0d, 0x26, 0xed, + 0xc5, 0x9a, 0x99, 0xf7, 0xe6, 0xbd, 0x9d, 0x27, 0xc3, 0xc1, 0xc8, 0x0b, 0x70, 0xb4, 0xb4, 0xc2, + 0x88, 0x32, 0x8a, 0xe4, 0x8f, 0x5e, 0x44, 0xf4, 0x17, 0x33, 0x8f, 0xbd, 0x5f, 0x8c, 0xac, 0x31, + 0x9d, 0x9f, 0xce, 0xe8, 0x8c, 0x9e, 0x0a, 0x70, 0xb4, 0x98, 0x8a, 0x4e, 0x34, 0xa2, 0xca, 0x96, + 0x74, 0x63, 0x46, 0xe9, 0xcc, 0x27, 0x3b, 0x16, 0xf3, 0xe6, 0x24, 0x66, 0x78, 0x1e, 0x66, 0x84, + 0xe6, 0x3b, 0xa8, 0xf4, 0x43, 0x1c, 0x5c, 0xd1, 0x80, 0x91, 0x4f, 0x0c, 0x3d, 0x87, 0x7d, 0x16, + 0xe1, 0x31, 0x19, 0x7a, 0x93, 0xba, 0x64, 0x4a, 0x2d, 0xd9, 0xae, 0xa4, 0x89, 0x51, 0x1e, 0xf0, + 0x99, 0xd3, 0x71, 0xcb, 0x02, 0x74, 0x26, 0xe8, 0x18, 0xca, 0x71, 0x88, 0x03, 0x4e, 0xdb, 0x13, + 0x34, 0x48, 0x13, 0x43, 0xe1, 0x4a, 0x4e, 0xc7, 0x55, 0x38, 0xe4, 0x4c, 0x9a, 0x5f, 0xf6, 0x40, + 0xe6, 0x23, 0x74, 0x06, 0xe5, 0x71, 0x66, 0x20, 0x44, 0x2b, 0xe7, 0x35, 0x8b, 0x1f, 0x63, 0x3d, + 0x72, 0xb6, 0xe5, 0x87, 0xc4, 0x28, 0xb8, 0x5b, 0x1e, 0x6a, 0x43, 0x35, 0xc4, 0x11, 0x09, 0xd8, + 0xf0, 0x6f, 0x1f, 0x2d, 0x4d, 0x8c, 0x83, 0x37, 0x02, 0xd9, 0xb8, 0x1d, 0x84, 0xbb, 0x6e, 0x82, + 0x10, 0xc8, 0x01, 0x9e, 0x93, 0x7a, 0xd1, 0x94, 0x5a, 0xaa, 0x2b, 0x6a, 0x74, 0x03, 0x10, 0x33, + 0x1c, 0xb1, 0x21, 0x3f, 0xbe, 0x2e, 0x8b, 0x17, 0xe8, 0x56, 0x96, 0x8c, 0xb5, 0x4d, 0xc6, 0x1a, + 0x6c, 0x93, 0xb1, 0x6b, 0xfc, 0x29, 0x69, 0x62, 0x94, 0xfa, 0x7c, 0xeb, 0xfe, 0xa7, 0x21, 0xb9, + 0xaa, 0x10, 0xe0, 0x14, 0xf4, 0x14, 0x14, 0x1f, 0x8f, 0x88, 0x1f, 0xd7, 0x4b, 0x66, 0xb1, 0xa5, + 0xba, 0x9b, 0x0e, 0x9d, 0x80, 0x32, 0xf5, 0x88, 0x3f, 0x89, 0xeb, 0x8a, 0x59, 0x6c, 0x55, 0xce, + 0x2b, 0xd9, 0x8d, 0xd7, 0x7c, 0xb6, 0xb9, 0x6e, 0x43, 0x68, 0x9e, 0x40, 0x49, 0x24, 0x8a, 0x4c, + 0x28, 0xf1, 0xf3, 0xe2, 0xba, 0x24, 0x56, 0x60, 0x17, 0x8b, 0x9b, 0x01, 0xcd, 0x6f, 0x45, 0x28, + 0x09, 0x09, 0xa4, 0x41, 0xf1, 0x03, 0x59, 0x8a, 0x00, 0x55, 0x97, 0x97, 0xe8, 0x0a, 0x40, 0x08, + 0x0e, 0xd9, 0x32, 0x24, 0x22, 0x9f, 0xea, 0xf9, 0xd1, 0x23, 0xd7, 0xec, 0x3b, 0x58, 0x86, 0xc4, + 0x3e, 0x4c, 0x13, 0x43, 0xcd, 0x5b, 0x57, 0x9d, 0x6e, 0x4b, 0x74, 0x06, 0x95, 0x60, 0x31, 0x27, + 0x91, 0x37, 0x1e, 0xde, 0x61, 0x5f, 0xe4, 0xa6, 0xd9, 0xd5, 0x34, 0x31, 0xa0, 0x9b, 0x8d, 0xdf, + 0x62, 0xff, 0x75, 0xc1, 0x85, 0x20, 0xef, 0x90, 0xc5, 0xf3, 0x8c, 0xbc, 0x60, 0x26, 0x36, 0x78, + 0x9e, 0x6a, 0x66, 0xd0, 0x17, 0xd3, 0x6c, 0x41, 0x8d, 0xb7, 0x4d, 0xf3, 0x87, 0x04, 0x3b, 0x6f, + 0x64, 0x80, 0xd2, 0x1f, 0xb8, 0x4e, 0xf7, 0x95, 0x56, 0xd0, 0xff, 0x5f, 0xad, 0xcd, 0xff, 0x72, + 0x28, 0x5b, 0x47, 0xcf, 0x40, 0xb6, 0x7b, 0xbd, 0x1b, 0x4d, 0xd2, 0x6b, 0xab, 0xb5, 0x79, 0xb8, + 0x3b, 0x82, 0x52, 0x1f, 0x35, 0x40, 0x71, 0xba, 0x83, 0x61, 0xfb, 0x42, 0xdb, 0xd3, 0xd1, 0x6a, + 0x6d, 0x56, 0x73, 0xd8, 0x09, 0x58, 0xfb, 0x02, 0x99, 0x50, 0xbe, 0xdd, 0x10, 0x8a, 0xff, 0xc8, + 0xdf, 0x7a, 0x82, 0x71, 0x0c, 0xfb, 0x9d, 0x5b, 0xf7, 0x72, 0xe0, 0xf4, 0xba, 0x9a, 0xac, 0x1f, + 0xad, 0xd6, 0x66, 0x2d, 0xa7, 0x74, 0x16, 0x11, 0x66, 0x1e, 0x0d, 0x50, 0x13, 0xf6, 0xaf, 0x6f, + 0x7a, 0x97, 0x42, 0x47, 0xd1, 0x9f, 0xac, 0xd6, 0xa6, 0x96, 0x93, 0xae, 0x7d, 0x8a, 0x59, 0xfb, + 0x42, 0x97, 0x3f, 0x7f, 0x6d, 0x14, 0xec, 0x32, 0x94, 0xee, 0xb0, 0xbf, 0x20, 0xb6, 0xf6, 0x90, + 0x36, 0xa4, 0xef, 0x69, 0x43, 0xfa, 0x95, 0x36, 0xa4, 0xfb, 0xdf, 0x8d, 0xc2, 0x48, 0x11, 0xff, + 0xd6, 0xcb, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x10, 0xad, 0x27, 0x39, 0xc8, 0x03, 0x00, 0x00, +} diff --git a/pkg/tracing/wire/binary.proto b/pkg/tracing/wire/binary.proto new file mode 100644 index 00000000000..d0bda520740 --- /dev/null +++ b/pkg/tracing/wire/binary.proto @@ -0,0 +1,44 @@ +syntax = "proto3"; +package wire; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; +import "google/protobuf/timestamp.proto"; + +message SpanContext { + uint64 trace_id = 1 [(gogoproto.customname) = "TraceID"]; + uint64 span_id = 2 [(gogoproto.customname) = "SpanID"]; +} + +message Span { + SpanContext context = 1 [(gogoproto.nullable) = false]; + uint64 parent_span_id = 2 [(gogoproto.customname) = "ParentSpanID"]; + string name = 3; + google.protobuf.Timestamp start_time = 4 [(gogoproto.customname) = "Start", (gogoproto.stdtime) = true, (gogoproto.nullable) = false]; + repeated string labels = 5; + repeated Field fields = 6 [(gogoproto.nullable) = false]; +} + +message Trace { + repeated Span spans = 1; +} + +message Field { + enum FieldType { + option (gogoproto.goproto_enum_prefix) = false; + + STRING = 0 [(gogoproto.enumvalue_customname) = "FieldTypeString"]; + BOOL = 1 [(gogoproto.enumvalue_customname) = "FieldTypeBool"]; + INT_64 = 2 [(gogoproto.enumvalue_customname) = "FieldTypeInt64"]; + UINT_64 = 3 [(gogoproto.enumvalue_customname) = "FieldTypeUint64"]; + DURATION = 4 [(gogoproto.enumvalue_customname) = "FieldTypeDuration"]; + FLOAT_64 = 6 [(gogoproto.enumvalue_customname) = "FieldTypeFloat64"]; + } + + string key = 1; + FieldType field_type = 2 [(gogoproto.customname) = "FieldType"]; + + oneof value { + sfixed64 numeric_val = 3 [(gogoproto.customname) = "NumericVal"]; + string string_val = 4 [(gogoproto.customname) = "StringVal"]; + } +} diff --git a/query/explain.go b/query/explain.go index ee0fb8bb0f4..d40d7a7630a 100644 --- a/query/explain.go +++ b/query/explain.go @@ -2,6 +2,7 @@ package query import ( "bytes" + "context" "fmt" "io" "strings" @@ -13,7 +14,7 @@ func (p *preparedStatement) Explain() (string, error) { // Determine the cost of all iterators created as part of this plan. ic := &explainIteratorCreator{ic: p.ic} p.ic = ic - itrs, _, err := p.Select() + itrs, _, err := p.Select(context.Background()) p.ic = ic.ic if err != nil { @@ -63,7 +64,7 @@ type explainIteratorCreator struct { nodes []planNode } -func (e *explainIteratorCreator) CreateIterator(m *influxql.Measurement, opt IteratorOptions) (Iterator, error) { +func (e *explainIteratorCreator) CreateIterator(ctx context.Context, m *influxql.Measurement, opt IteratorOptions) (Iterator, error) { cost, err := e.ic.IteratorCost(m, opt) if err != nil { return nil, err diff --git a/query/internal/internal.pb.go b/query/internal/internal.pb.go index cbf9b82bb65..9b9ae168cb9 100644 --- a/query/internal/internal.pb.go +++ b/query/internal/internal.pb.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-gogo. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: internal/internal.proto -// DO NOT EDIT! /* Package query is a generated protocol buffer package. @@ -48,6 +47,7 @@ type Point struct { BooleanValue *bool `protobuf:"varint,10,opt,name=BooleanValue" json:"BooleanValue,omitempty"` UnsignedValue *uint64 `protobuf:"varint,12,opt,name=UnsignedValue" json:"UnsignedValue,omitempty"` Stats *IteratorStats `protobuf:"bytes,11,opt,name=Stats" json:"Stats,omitempty"` + Trace []byte `protobuf:"bytes,13,opt,name=Trace" json:"Trace,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -140,6 +140,13 @@ func (m *Point) GetStats() *IteratorStats { return nil } +func (m *Point) GetTrace() []byte { + if m != nil { + return m.Trace + } + return nil +} + type Aux struct { DataType *int32 `protobuf:"varint,1,req,name=DataType" json:"DataType,omitempty"` FloatValue *float64 `protobuf:"fixed64,2,opt,name=FloatValue" json:"FloatValue,omitempty"` @@ -537,54 +544,54 @@ func init() { func init() { proto.RegisterFile("internal/internal.proto", fileDescriptorInternal) } var fileDescriptorInternal = []byte{ - // 769 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x55, 0xe1, 0x8e, 0xe3, 0x34, - 0x10, 0x56, 0x92, 0x4d, 0xb7, 0x71, 0xb7, 0xf4, 0x30, 0x65, 0xb1, 0xd0, 0x09, 0x45, 0x11, 0x48, - 0x11, 0xa0, 0x22, 0xed, 0x2f, 0x7e, 0x21, 0xf5, 0xd8, 0x5b, 0x54, 0xe9, 0xae, 0x7b, 0x72, 0x97, - 0xfd, 0x6f, 0x9a, 0xd9, 0xc8, 0x52, 0xea, 0x14, 0xc7, 0x41, 0xed, 0x03, 0xf0, 0x10, 0x3c, 0x16, - 0x4f, 0xc2, 0x2b, 0x20, 0x8f, 0x9d, 0x34, 0x5d, 0x81, 0xf6, 0x7e, 0x75, 0xbe, 0x6f, 0xa6, 0x63, - 0xcf, 0xcc, 0x37, 0x0e, 0xf9, 0x42, 0x2a, 0x03, 0x5a, 0x89, 0xea, 0x87, 0xce, 0x58, 0xec, 0x75, - 0x6d, 0x6a, 0x1a, 0xff, 0xde, 0x82, 0x3e, 0x66, 0xff, 0x84, 0x24, 0xfe, 0x50, 0x4b, 0x65, 0x28, - 0x25, 0x17, 0x6b, 0xb1, 0x03, 0x16, 0xa4, 0x61, 0x9e, 0x70, 0xb4, 0x2d, 0xf7, 0x20, 0xca, 0x86, - 0x85, 0x8e, 0xb3, 0x36, 0x72, 0x72, 0x07, 0x2c, 0x4a, 0xc3, 0x3c, 0xe2, 0x68, 0xd3, 0x57, 0x24, - 0x5a, 0xcb, 0x8a, 0x5d, 0xa4, 0x61, 0x3e, 0xe6, 0xd6, 0xa4, 0xaf, 0x49, 0xb4, 0x6c, 0x0f, 0x2c, - 0x4e, 0xa3, 0x7c, 0x72, 0x43, 0x16, 0x78, 0xd8, 0x62, 0xd9, 0x1e, 0xb8, 0xa5, 0xe9, 0x57, 0x84, - 0x2c, 0xcb, 0x52, 0x43, 0x29, 0x0c, 0x14, 0x6c, 0x94, 0x06, 0xf9, 0x94, 0x0f, 0x18, 0xeb, 0xbf, - 0xab, 0x6a, 0x61, 0x1e, 0x45, 0xd5, 0x02, 0xbb, 0x4c, 0x83, 0x3c, 0xe0, 0x03, 0x86, 0x66, 0xe4, - 0x6a, 0xa5, 0x0c, 0x94, 0xa0, 0x5d, 0xc4, 0x38, 0x0d, 0xf2, 0x88, 0x9f, 0x71, 0x34, 0x25, 0x93, - 0x8d, 0xd1, 0x52, 0x95, 0x2e, 0x24, 0x49, 0x83, 0x3c, 0xe1, 0x43, 0xca, 0x66, 0x79, 0x53, 0xd7, - 0x15, 0x08, 0xe5, 0x42, 0x48, 0x1a, 0xe4, 0x63, 0x7e, 0xc6, 0xd1, 0xaf, 0xc9, 0xf4, 0x57, 0xd5, - 0xc8, 0x52, 0x41, 0xe1, 0x82, 0xae, 0xd2, 0x20, 0xbf, 0xe0, 0xe7, 0x24, 0xfd, 0x96, 0xc4, 0x1b, - 0x23, 0x4c, 0xc3, 0x26, 0x69, 0x90, 0x4f, 0x6e, 0xe6, 0xbe, 0xde, 0x95, 0x01, 0x2d, 0x4c, 0xad, - 0xd1, 0xc7, 0x5d, 0x48, 0xf6, 0x77, 0x80, 0xad, 0xa1, 0x5f, 0x92, 0xf1, 0xad, 0x30, 0xe2, 0xe1, - 0xb8, 0x77, 0x3d, 0x8f, 0x79, 0x8f, 0x9f, 0xd5, 0x1f, 0xbe, 0x58, 0x7f, 0xf4, 0x72, 0xfd, 0x17, - 0x2f, 0xd7, 0x1f, 0x7f, 0x4c, 0xfd, 0xa3, 0xff, 0xa8, 0x3f, 0xfb, 0x33, 0x26, 0xb3, 0xae, 0xd8, - 0xfb, 0xbd, 0x91, 0xb5, 0x42, 0x9d, 0xbc, 0x3d, 0xec, 0x35, 0x0b, 0xf0, 0x60, 0xb4, 0xad, 0x4e, - 0xac, 0x2a, 0xc2, 0x34, 0xca, 0x13, 0xa7, 0x84, 0x6f, 0xc8, 0xe8, 0x4e, 0x42, 0x55, 0x34, 0xec, - 0x53, 0x94, 0xca, 0xd4, 0xb7, 0xee, 0x51, 0x68, 0x0e, 0x4f, 0xdc, 0x3b, 0xe9, 0xf7, 0xe4, 0x72, - 0x53, 0xb7, 0x7a, 0x0b, 0x0d, 0x8b, 0x30, 0x8e, 0xfa, 0xb8, 0xf7, 0x20, 0x9a, 0x56, 0xc3, 0x0e, - 0x94, 0xe1, 0x5d, 0x08, 0xfd, 0x8e, 0x8c, 0x6d, 0x2b, 0xf4, 0x1f, 0xa2, 0xc2, 0xba, 0x27, 0x37, - 0xb3, 0x6e, 0x22, 0x9e, 0xe6, 0x7d, 0x80, 0xed, 0xf5, 0xad, 0xdc, 0x81, 0x6a, 0xec, 0xad, 0x51, - 0xb0, 0x09, 0x1f, 0x30, 0x94, 0x91, 0xcb, 0x5f, 0x74, 0xdd, 0xee, 0xdf, 0x1c, 0xd9, 0x67, 0xe8, - 0xec, 0xa0, 0xad, 0xf0, 0x4e, 0x56, 0x15, 0xb6, 0x24, 0xe6, 0x68, 0xd3, 0xd7, 0x24, 0xb1, 0xbf, - 0x43, 0xe1, 0x9e, 0x08, 0xeb, 0xfd, 0xb9, 0x56, 0x85, 0xb4, 0x1d, 0x42, 0xd1, 0x26, 0xfc, 0x44, - 0x58, 0xef, 0xc6, 0x08, 0x6d, 0x70, 0xbd, 0x12, 0x1c, 0xe9, 0x89, 0xb0, 0xf7, 0x78, 0xab, 0x0a, - 0xf4, 0x11, 0xf4, 0x75, 0xd0, 0x2a, 0xe9, 0x5d, 0xbd, 0x15, 0x98, 0xf4, 0x73, 0x4c, 0xda, 0x63, - 0x9b, 0x73, 0xd9, 0x6c, 0x41, 0x15, 0x52, 0x95, 0xa8, 0xce, 0x31, 0x3f, 0x11, 0x74, 0x4e, 0xe2, - 0x77, 0x72, 0x27, 0x0d, 0xaa, 0x3a, 0xe2, 0x0e, 0xd0, 0x6b, 0x32, 0xba, 0x7f, 0x7a, 0x6a, 0xc0, - 0xb0, 0x29, 0xd2, 0x1e, 0x59, 0x7e, 0xe3, 0xc2, 0x3f, 0x71, 0xbc, 0x43, 0xf6, 0x66, 0x1b, 0xff, - 0x87, 0x99, 0xbb, 0x99, 0x87, 0xae, 0x22, 0x2d, 0xf7, 0xf8, 0xb0, 0x5c, 0xbb, 0xd3, 0x7b, 0xc2, - 0xe6, 0xbb, 0x85, 0xa2, 0xdd, 0x03, 0x7b, 0x85, 0x2e, 0x8f, 0xec, 0x44, 0xde, 0x8b, 0xc3, 0x06, - 0xb4, 0x84, 0x66, 0xcd, 0x28, 0xa6, 0x1c, 0x30, 0xf6, 0xbc, 0x7b, 0x5d, 0x80, 0x86, 0x82, 0xcd, - 0xf1, 0x8f, 0x1d, 0xcc, 0x7e, 0x24, 0x57, 0x03, 0x41, 0x34, 0x34, 0x27, 0xf1, 0xca, 0xc0, 0xae, - 0x61, 0xc1, 0xff, 0x8a, 0xc6, 0x05, 0x64, 0x7f, 0x05, 0x64, 0x32, 0xa0, 0xbb, 0xed, 0xfc, 0x4d, - 0x34, 0xe0, 0x15, 0xdc, 0x63, 0x9a, 0x93, 0x19, 0x07, 0x03, 0xca, 0x36, 0xf8, 0x43, 0x5d, 0xc9, - 0xed, 0x11, 0x57, 0x34, 0xe1, 0xcf, 0xe9, 0xfe, 0x4d, 0x8d, 0xdc, 0x0e, 0x60, 0xd5, 0x73, 0x12, - 0x73, 0x28, 0xe1, 0xe0, 0x37, 0xd2, 0x01, 0x7b, 0xde, 0xaa, 0x79, 0x10, 0xba, 0x04, 0xe3, 0xf7, - 0xb0, 0xc7, 0xd9, 0x4f, 0x27, 0x39, 0xe3, 0xbd, 0x5a, 0xed, 0x66, 0x1d, 0x60, 0x67, 0x7a, 0x3c, - 0x98, 0x5b, 0x38, 0x9c, 0x5b, 0xb6, 0x24, 0xd3, 0xb3, 0x97, 0x08, 0x07, 0xe6, 0xbb, 0x1b, 0xf8, - 0x81, 0xf9, 0xd6, 0x5e, 0x93, 0x11, 0x7e, 0x0d, 0xd6, 0x5d, 0x0a, 0x87, 0xb2, 0x05, 0x19, 0xb9, - 0x8d, 0xb4, 0x2b, 0xfc, 0x28, 0x2a, 0xff, 0x95, 0xb0, 0x26, 0x7e, 0x10, 0xec, 0x23, 0x16, 0xba, - 0x35, 0xb0, 0xf6, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x55, 0x55, 0xec, 0xe3, 0x77, 0x06, 0x00, - 0x00, + // 780 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x55, 0xdb, 0x6e, 0xe3, 0x36, + 0x10, 0x05, 0xa5, 0xc8, 0xb1, 0xe8, 0xb8, 0xd9, 0xb2, 0x69, 0x4a, 0x14, 0x8b, 0x42, 0x10, 0x5a, + 0x40, 0x68, 0x8b, 0x14, 0xc8, 0x53, 0x9f, 0x0a, 0x78, 0x9b, 0x4d, 0x11, 0x60, 0x37, 0x59, 0xd0, + 0x6e, 0xde, 0x59, 0x6b, 0x22, 0x10, 0x90, 0x29, 0x97, 0xa2, 0x0a, 0xfb, 0x03, 0xf6, 0x23, 0xfa, + 0x59, 0xfd, 0xa3, 0x82, 0x43, 0x4a, 0x96, 0x83, 0x16, 0xd9, 0x27, 0xcf, 0x39, 0x33, 0xe6, 0xe5, + 0xcc, 0x19, 0x8a, 0x7e, 0xa5, 0xb4, 0x05, 0xa3, 0x65, 0xfd, 0x53, 0x1f, 0x5c, 0x6d, 0x4d, 0x63, + 0x1b, 0x96, 0xfc, 0xd9, 0x81, 0xd9, 0xe7, 0x1f, 0x63, 0x9a, 0x7c, 0x68, 0x94, 0xb6, 0x8c, 0xd1, + 0x93, 0x7b, 0xb9, 0x01, 0x4e, 0xb2, 0xa8, 0x48, 0x05, 0xc6, 0x8e, 0x5b, 0xc9, 0xaa, 0xe5, 0x91, + 0xe7, 0x5c, 0x8c, 0x9c, 0xda, 0x00, 0x8f, 0xb3, 0xa8, 0x88, 0x05, 0xc6, 0xec, 0x15, 0x8d, 0xef, + 0x55, 0xcd, 0x4f, 0xb2, 0xa8, 0x98, 0x0a, 0x17, 0xb2, 0xd7, 0x34, 0x5e, 0x74, 0x3b, 0x9e, 0x64, + 0x71, 0x31, 0xbb, 0xa6, 0x57, 0xb8, 0xd9, 0xd5, 0xa2, 0xdb, 0x09, 0x47, 0xb3, 0x6f, 0x28, 0x5d, + 0x54, 0x95, 0x81, 0x4a, 0x5a, 0x28, 0xf9, 0x24, 0x23, 0xc5, 0x5c, 0x8c, 0x18, 0x97, 0xbf, 0xad, + 0x1b, 0x69, 0x1f, 0x65, 0xdd, 0x01, 0x3f, 0xcd, 0x48, 0x41, 0xc4, 0x88, 0x61, 0x39, 0x3d, 0xbb, + 0xd3, 0x16, 0x2a, 0x30, 0xbe, 0x62, 0x9a, 0x91, 0x22, 0x16, 0x47, 0x1c, 0xcb, 0xe8, 0x6c, 0x69, + 0x8d, 0xd2, 0x95, 0x2f, 0x49, 0x33, 0x52, 0xa4, 0x62, 0x4c, 0xb9, 0x55, 0xde, 0x34, 0x4d, 0x0d, + 0x52, 0xfb, 0x12, 0x9a, 0x91, 0x62, 0x2a, 0x8e, 0x38, 0xf6, 0x2d, 0x9d, 0xff, 0xae, 0x5b, 0x55, + 0x69, 0x28, 0x7d, 0xd1, 0x59, 0x46, 0x8a, 0x13, 0x71, 0x4c, 0xb2, 0xef, 0x69, 0xb2, 0xb4, 0xd2, + 0xb6, 0x7c, 0x96, 0x91, 0x62, 0x76, 0x7d, 0x11, 0xee, 0x7b, 0x67, 0xc1, 0x48, 0xdb, 0x18, 0xcc, + 0x09, 0x5f, 0xc2, 0x2e, 0x68, 0xb2, 0x32, 0x72, 0x0d, 0x7c, 0x9e, 0x91, 0xe2, 0x4c, 0x78, 0x90, + 0xff, 0x43, 0x50, 0x30, 0xf6, 0x35, 0x9d, 0xde, 0x48, 0x2b, 0x57, 0xfb, 0xad, 0xef, 0x44, 0x22, + 0x06, 0xfc, 0x4c, 0x95, 0xe8, 0x45, 0x55, 0xe2, 0x97, 0x55, 0x39, 0x79, 0x59, 0x95, 0xe4, 0x53, + 0x54, 0x99, 0xfc, 0x87, 0x2a, 0xf9, 0xc7, 0x84, 0x9e, 0xf7, 0x12, 0x3c, 0x6c, 0xad, 0x6a, 0x34, + 0xba, 0xe7, 0xed, 0x6e, 0x6b, 0x38, 0xc1, 0x8d, 0x31, 0x76, 0xee, 0x71, 0x5e, 0x89, 0xb2, 0xb8, + 0x48, 0xbd, 0x3f, 0xbe, 0xa3, 0x93, 0x5b, 0x05, 0x75, 0xd9, 0xf2, 0xcf, 0xd1, 0x40, 0xf3, 0x20, + 0xe8, 0xa3, 0x34, 0x02, 0x9e, 0x44, 0x48, 0xb2, 0x1f, 0xe9, 0xe9, 0xb2, 0xe9, 0xcc, 0x1a, 0x5a, + 0x1e, 0x63, 0x1d, 0x0b, 0x75, 0xef, 0x41, 0xb6, 0x9d, 0x81, 0x0d, 0x68, 0x2b, 0xfa, 0x12, 0xf6, + 0x03, 0x9d, 0x3a, 0x29, 0xcc, 0x5f, 0xb2, 0xc6, 0x7b, 0xcf, 0xae, 0xcf, 0xfb, 0x3e, 0x05, 0x5a, + 0x0c, 0x05, 0x4e, 0xeb, 0x1b, 0xb5, 0x01, 0xdd, 0xba, 0x53, 0xa3, 0x8d, 0x53, 0x31, 0x62, 0x18, + 0xa7, 0xa7, 0xbf, 0x99, 0xa6, 0xdb, 0xbe, 0xd9, 0xf3, 0x2f, 0x30, 0xd9, 0x43, 0x77, 0xc3, 0x5b, + 0x55, 0xd7, 0x28, 0x49, 0x22, 0x30, 0x66, 0xaf, 0x69, 0xea, 0x7e, 0xc7, 0x76, 0x3e, 0x10, 0x2e, + 0xfb, 0x6b, 0xa3, 0x4b, 0xe5, 0x14, 0x42, 0x2b, 0xa7, 0xe2, 0x40, 0xb8, 0xec, 0xd2, 0x4a, 0x63, + 0x71, 0xe8, 0x52, 0x6c, 0xe9, 0x81, 0x70, 0xe7, 0x78, 0xab, 0x4b, 0xcc, 0x51, 0xcc, 0xf5, 0xd0, + 0x39, 0xe9, 0x5d, 0xb3, 0x96, 0xb8, 0xe8, 0x97, 0xb8, 0xe8, 0x80, 0xdd, 0x9a, 0x8b, 0x76, 0x0d, + 0xba, 0x54, 0xba, 0x42, 0xcf, 0x4e, 0xc5, 0x81, 0x70, 0x0e, 0x7d, 0xa7, 0x36, 0xca, 0xa2, 0xd7, + 0x63, 0xe1, 0x01, 0xbb, 0xa4, 0x93, 0x87, 0xa7, 0xa7, 0x16, 0x2c, 0x1a, 0x37, 0x16, 0x01, 0x39, + 0x7e, 0xe9, 0xcb, 0x3f, 0xf3, 0xbc, 0x47, 0xee, 0x64, 0xcb, 0xf0, 0x87, 0x73, 0x7f, 0xb2, 0x00, + 0xfd, 0x8d, 0x8c, 0xda, 0xe2, 0x73, 0x73, 0xe9, 0x77, 0x1f, 0x08, 0xb7, 0xde, 0x0d, 0x94, 0xdd, + 0x16, 0xf8, 0x2b, 0x4c, 0x05, 0xe4, 0x3a, 0xf2, 0x5e, 0xee, 0x96, 0x60, 0x14, 0xb4, 0xf7, 0x9c, + 0xe1, 0x92, 0x23, 0xc6, 0xed, 0xf7, 0x60, 0x4a, 0x30, 0x50, 0xf2, 0x0b, 0xfc, 0x63, 0x0f, 0xf3, + 0x9f, 0xe9, 0xd9, 0xc8, 0x10, 0x2d, 0x2b, 0x68, 0x72, 0x67, 0x61, 0xd3, 0x72, 0xf2, 0xbf, 0xa6, + 0xf1, 0x05, 0xf9, 0xdf, 0x84, 0xce, 0x46, 0x74, 0x3f, 0x9d, 0x7f, 0xc8, 0x16, 0x82, 0x83, 0x07, + 0xcc, 0x0a, 0x7a, 0x2e, 0xc0, 0x82, 0x76, 0x02, 0x7f, 0x68, 0x6a, 0xb5, 0xde, 0xe3, 0x88, 0xa6, + 0xe2, 0x39, 0x3d, 0xbc, 0xb4, 0xb1, 0x9f, 0x01, 0xbc, 0xf5, 0x05, 0x4d, 0x04, 0x54, 0xb0, 0x0b, + 0x13, 0xe9, 0x81, 0xdb, 0xef, 0xae, 0x5d, 0x49, 0x53, 0x81, 0x0d, 0x73, 0x38, 0xe0, 0xfc, 0x97, + 0x83, 0x9d, 0xf1, 0x5c, 0x9d, 0xf1, 0xbd, 0x26, 0xa8, 0xcc, 0x80, 0x47, 0x7d, 0x8b, 0xc6, 0x7d, + 0xcb, 0x17, 0x74, 0x7e, 0xf4, 0x3e, 0x61, 0xc3, 0x82, 0xba, 0x24, 0x34, 0x2c, 0x48, 0x7b, 0x49, + 0x27, 0xf8, 0x8d, 0xb8, 0xef, 0x97, 0xf0, 0x28, 0xbf, 0xa2, 0x13, 0x3f, 0x91, 0x6e, 0x84, 0x1f, + 0x65, 0x1d, 0xbe, 0x1d, 0x2e, 0xc4, 0xcf, 0x84, 0x7b, 0xc4, 0x22, 0x3f, 0x06, 0x2e, 0xfe, 0x37, + 0x00, 0x00, 0xff, 0xff, 0x3d, 0xd1, 0x25, 0x7b, 0x8d, 0x06, 0x00, 0x00, } diff --git a/query/internal/internal.proto b/query/internal/internal.proto index 2bb2889e53f..8ebb0e5d232 100644 --- a/query/internal/internal.proto +++ b/query/internal/internal.proto @@ -16,6 +16,7 @@ message Point { optional uint64 UnsignedValue = 12; optional IteratorStats Stats = 11; + optional bytes Trace = 13; } message Aux { diff --git a/query/iterator.gen.go b/query/iterator.gen.go index f1963ecaf1c..159e6da2b25 100644 --- a/query/iterator.gen.go +++ b/query/iterator.gen.go @@ -8,8 +8,7 @@ package query import ( "container/heap" - "encoding/binary" - "fmt" + "context" "io" "sort" "sync" @@ -17,7 +16,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/influxdata/influxdb/influxql" - internal "github.com/influxdata/influxdb/query/internal" ) // DefaultStatsInterval is the default value for IteratorEncoder.StatsInterval. @@ -3384,8 +3382,8 @@ type floatReaderIterator struct { } // newFloatReaderIterator returns a new instance of floatReaderIterator. -func newFloatReaderIterator(r io.Reader, stats IteratorStats) *floatReaderIterator { - dec := NewFloatPointDecoder(r) +func newFloatReaderIterator(ctx context.Context, r io.Reader, stats IteratorStats) *floatReaderIterator { + dec := NewFloatPointDecoder(ctx, r) dec.stats = stats return &floatReaderIterator{ @@ -6777,8 +6775,8 @@ type integerReaderIterator struct { } // newIntegerReaderIterator returns a new instance of integerReaderIterator. -func newIntegerReaderIterator(r io.Reader, stats IteratorStats) *integerReaderIterator { - dec := NewIntegerPointDecoder(r) +func newIntegerReaderIterator(ctx context.Context, r io.Reader, stats IteratorStats) *integerReaderIterator { + dec := NewIntegerPointDecoder(ctx, r) dec.stats = stats return &integerReaderIterator{ @@ -10170,8 +10168,8 @@ type unsignedReaderIterator struct { } // newUnsignedReaderIterator returns a new instance of unsignedReaderIterator. -func newUnsignedReaderIterator(r io.Reader, stats IteratorStats) *unsignedReaderIterator { - dec := NewUnsignedPointDecoder(r) +func newUnsignedReaderIterator(ctx context.Context, r io.Reader, stats IteratorStats) *unsignedReaderIterator { + dec := NewUnsignedPointDecoder(ctx, r) dec.stats = stats return &unsignedReaderIterator{ @@ -13549,8 +13547,8 @@ type stringReaderIterator struct { } // newStringReaderIterator returns a new instance of stringReaderIterator. -func newStringReaderIterator(r io.Reader, stats IteratorStats) *stringReaderIterator { - dec := NewStringPointDecoder(r) +func newStringReaderIterator(ctx context.Context, r io.Reader, stats IteratorStats) *stringReaderIterator { + dec := NewStringPointDecoder(ctx, r) dec.stats = stats return &stringReaderIterator{ @@ -16928,8 +16926,8 @@ type booleanReaderIterator struct { } // newBooleanReaderIterator returns a new instance of booleanReaderIterator. -func newBooleanReaderIterator(r io.Reader, stats IteratorStats) *booleanReaderIterator { - dec := NewBooleanPointDecoder(r) +func newBooleanReaderIterator(ctx context.Context, r io.Reader, stats IteratorStats) *booleanReaderIterator { + dec := NewBooleanPointDecoder(ctx, r) dec.stats = stats return &booleanReaderIterator{ @@ -16963,39 +16961,6 @@ func (itr *booleanReaderIterator) Next() (*BooleanPoint, error) { return p, nil } -// IteratorEncoder is an encoder for encoding an iterator's points to w. -type IteratorEncoder struct { - w io.Writer - - // Frequency with which stats are emitted. - StatsInterval time.Duration -} - -// NewIteratorEncoder encodes an iterator's points to w. -func NewIteratorEncoder(w io.Writer) *IteratorEncoder { - return &IteratorEncoder{ - w: w, - - StatsInterval: DefaultStatsInterval, - } -} - -// EncodeIterator encodes and writes all of itr's points to the underlying writer. -func (enc *IteratorEncoder) EncodeIterator(itr Iterator) error { - switch itr := itr.(type) { - case FloatIterator: - return enc.encodeFloatIterator(itr) - case IntegerIterator: - return enc.encodeIntegerIterator(itr) - case StringIterator: - return enc.encodeStringIterator(itr) - case BooleanIterator: - return enc.encodeBooleanIterator(itr) - default: - panic(fmt.Sprintf("unsupported iterator for encoder: %T", itr)) - } -} - // encodeFloatIterator encodes all points from itr to the underlying writer. func (enc *IteratorEncoder) encodeFloatIterator(itr FloatIterator) error { ticker := time.NewTicker(enc.StatsInterval) @@ -17210,26 +17175,3 @@ func (enc *IteratorEncoder) encodeBooleanIterator(itr BooleanIterator) error { } return nil } - -// encode a stats object in the point stream. -func (enc *IteratorEncoder) encodeStats(stats IteratorStats) error { - buf, err := proto.Marshal(&internal.Point{ - Name: proto.String(""), - Tags: proto.String(""), - Time: proto.Int64(0), - Nil: proto.Bool(false), - - Stats: encodeIteratorStats(&stats), - }) - if err != nil { - return err - } - - if err := binary.Write(enc.w, binary.BigEndian, uint32(len(buf))); err != nil { - return err - } - if _, err := enc.w.Write(buf); err != nil { - return err - } - return nil -} diff --git a/query/iterator.gen.go.tmpl b/query/iterator.gen.go.tmpl index 3f54b3f43c5..feaf4fdc1ab 100644 --- a/query/iterator.gen.go.tmpl +++ b/query/iterator.gen.go.tmpl @@ -1,9 +1,8 @@ package query import ( + "context" "container/heap" - "encoding/binary" - "fmt" "io" "sort" "sync" @@ -11,7 +10,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/influxdata/influxdb/influxql" - internal "github.com/influxdata/influxdb/query/internal" ) // DefaultStatsInterval is the default value for IteratorEncoder.StatsInterval. @@ -1723,13 +1721,13 @@ type {{$k.name}}ReaderIterator struct { } // new{{$k.Name}}ReaderIterator returns a new instance of {{$k.name}}ReaderIterator. -func new{{$k.Name}}ReaderIterator(r io.Reader, stats IteratorStats) *{{$k.name}}ReaderIterator { - dec := New{{$k.Name}}PointDecoder(r) +func new{{$k.Name}}ReaderIterator(ctx context.Context, r io.Reader, stats IteratorStats) *{{$k.name}}ReaderIterator { + dec := New{{$k.Name}}PointDecoder(ctx, r) dec.stats = stats return &{{$k.name}}ReaderIterator{ r: r, - dec: dec, + dec: dec, } } @@ -1759,40 +1757,6 @@ func (itr *{{$k.name}}ReaderIterator) Next() (*{{$k.Name}}Point, error) { } {{end}} - -// IteratorEncoder is an encoder for encoding an iterator's points to w. -type IteratorEncoder struct { - w io.Writer - - // Frequency with which stats are emitted. - StatsInterval time.Duration -} - -// NewIteratorEncoder encodes an iterator's points to w. -func NewIteratorEncoder(w io.Writer) *IteratorEncoder { - return &IteratorEncoder{ - w: w, - - StatsInterval: DefaultStatsInterval, - } -} - -// EncodeIterator encodes and writes all of itr's points to the underlying writer. -func (enc *IteratorEncoder) EncodeIterator(itr Iterator) error { - switch itr := itr.(type) { - case FloatIterator: - return enc.encodeFloatIterator(itr) - case IntegerIterator: - return enc.encodeIntegerIterator(itr) - case StringIterator: - return enc.encodeStringIterator(itr) - case BooleanIterator: - return enc.encodeBooleanIterator(itr) - default: - panic(fmt.Sprintf("unsupported iterator for encoder: %T", itr)) - } -} - {{range .}} // encode{{.Name}}Iterator encodes all points from itr to the underlying writer. func (enc *IteratorEncoder) encode{{.Name}}Iterator(itr {{.Name}}Iterator) error { @@ -1839,27 +1803,4 @@ func (enc *IteratorEncoder) encode{{.Name}}Iterator(itr {{.Name}}Iterator) error {{end}} -// encode a stats object in the point stream. -func (enc *IteratorEncoder) encodeStats(stats IteratorStats) error { - buf, err := proto.Marshal(&internal.Point{ - Name: proto.String(""), - Tags: proto.String(""), - Time: proto.Int64(0), - Nil: proto.Bool(false), - - Stats: encodeIteratorStats(&stats), - }) - if err != nil { - return err - } - - if err := binary.Write(enc.w, binary.BigEndian, uint32(len(buf))); err != nil { - return err - } - if _, err := enc.w.Write(buf); err != nil { - return err - } - return nil -} - {{end}} diff --git a/query/iterator.go b/query/iterator.go index abe1993ef34..49756df3a25 100644 --- a/query/iterator.go +++ b/query/iterator.go @@ -1,16 +1,18 @@ package query import ( + "context" + "encoding/binary" "errors" "fmt" "io" + "regexp" "sync" "time" - "regexp" - "github.com/gogo/protobuf/proto" "github.com/influxdata/influxdb/influxql" + "github.com/influxdata/influxdb/pkg/tracing" internal "github.com/influxdata/influxdb/query/internal" ) @@ -640,18 +642,18 @@ func DrainIterators(itrs []Iterator) { } // NewReaderIterator returns an iterator that streams from a reader. -func NewReaderIterator(r io.Reader, typ influxql.DataType, stats IteratorStats) Iterator { +func NewReaderIterator(ctx context.Context, r io.Reader, typ influxql.DataType, stats IteratorStats) Iterator { switch typ { case influxql.Float: - return newFloatReaderIterator(r, stats) + return newFloatReaderIterator(ctx, r, stats) case influxql.Integer: - return newIntegerReaderIterator(r, stats) + return newIntegerReaderIterator(ctx, r, stats) case influxql.Unsigned: - return newUnsignedReaderIterator(r, stats) + return newUnsignedReaderIterator(ctx, r, stats) case influxql.String: - return newStringReaderIterator(r, stats) + return newStringReaderIterator(ctx, r, stats) case influxql.Boolean: - return newBooleanReaderIterator(r, stats) + return newBooleanReaderIterator(ctx, r, stats) default: return &nilFloatReaderIterator{r: r} } @@ -660,7 +662,7 @@ func NewReaderIterator(r io.Reader, typ influxql.DataType, stats IteratorStats) // IteratorCreator is an interface to create Iterators. type IteratorCreator interface { // Creates a simple iterator for use in an InfluxQL query. - CreateIterator(source *influxql.Measurement, opt IteratorOptions) (Iterator, error) + CreateIterator(ctx context.Context, source *influxql.Measurement, opt IteratorOptions) (Iterator, error) // Determines the potential cost for creating an iterator. IteratorCost(source *influxql.Measurement, opt IteratorOptions) (IteratorCost, error) @@ -1426,6 +1428,22 @@ func decodeIteratorStats(pb *internal.IteratorStats) IteratorStats { } } +func decodeIteratorTrace(ctx context.Context, data []byte) error { + pt := tracing.TraceFromContext(ctx) + if pt == nil { + return nil + } + + var ct tracing.Trace + if err := ct.UnmarshalBinary(data); err != nil { + return err + } + + pt.Merge(&ct) + + return nil +} + // IteratorCost contains statistics retrieved for explaining what potential // cost may be incurred by instantiating an iterator. type IteratorCost struct { @@ -1530,3 +1548,86 @@ func abs(v int64) int64 { } return v } + +// IteratorEncoder is an encoder for encoding an iterator's points to w. +type IteratorEncoder struct { + w io.Writer + + // Frequency with which stats are emitted. + StatsInterval time.Duration +} + +// NewIteratorEncoder encodes an iterator's points to w. +func NewIteratorEncoder(w io.Writer) *IteratorEncoder { + return &IteratorEncoder{ + w: w, + + StatsInterval: DefaultStatsInterval, + } +} + +// EncodeIterator encodes and writes all of itr's points to the underlying writer. +func (enc *IteratorEncoder) EncodeIterator(itr Iterator) error { + switch itr := itr.(type) { + case FloatIterator: + return enc.encodeFloatIterator(itr) + case IntegerIterator: + return enc.encodeIntegerIterator(itr) + case StringIterator: + return enc.encodeStringIterator(itr) + case BooleanIterator: + return enc.encodeBooleanIterator(itr) + default: + panic(fmt.Sprintf("unsupported iterator for encoder: %T", itr)) + } +} + +func (enc *IteratorEncoder) EncodeTrace(trace *tracing.Trace) error { + data, err := trace.MarshalBinary() + if err != nil { + return err + } + + buf, err := proto.Marshal(&internal.Point{ + Name: proto.String(""), + Tags: proto.String(""), + Time: proto.Int64(0), + Nil: proto.Bool(false), + + Trace: data, + }) + if err != nil { + return err + } + + if err = binary.Write(enc.w, binary.BigEndian, uint32(len(buf))); err != nil { + return err + } + if _, err = enc.w.Write(buf); err != nil { + return err + } + return nil +} + +// encode a stats object in the point stream. +func (enc *IteratorEncoder) encodeStats(stats IteratorStats) error { + buf, err := proto.Marshal(&internal.Point{ + Name: proto.String(""), + Tags: proto.String(""), + Time: proto.Int64(0), + Nil: proto.Bool(false), + + Stats: encodeIteratorStats(&stats), + }) + if err != nil { + return err + } + + if err = binary.Write(enc.w, binary.BigEndian, uint32(len(buf))); err != nil { + return err + } + if _, err = enc.w.Write(buf); err != nil { + return err + } + return nil +} diff --git a/query/iterator_test.go b/query/iterator_test.go index 16bfcda1e49..e4f07233189 100644 --- a/query/iterator_test.go +++ b/query/iterator_test.go @@ -2,6 +2,7 @@ package query_test import ( "bytes" + "context" "fmt" "math" "reflect" @@ -1525,7 +1526,7 @@ func TestIterator_EncodeDecode(t *testing.T) { } // Decode from the buffer. - dec := query.NewReaderIterator(&buf, influxql.Float, itr.Stats()) + dec := query.NewReaderIterator(context.Background(), &buf, influxql.Float, itr.Stats()) // Initial stats should exist immediately. fdec := dec.(query.FloatIterator) @@ -1553,12 +1554,12 @@ func TestIterator_EncodeDecode(t *testing.T) { // IteratorCreator is a mockable implementation of SelectStatementExecutor.IteratorCreator. type IteratorCreator struct { - CreateIteratorFn func(m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) + CreateIteratorFn func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) FieldDimensionsFn func(m *influxql.Measurement) (fields map[string]influxql.DataType, dimensions map[string]struct{}, err error) } -func (ic *IteratorCreator) CreateIterator(m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { - return ic.CreateIteratorFn(m, opt) +func (ic *IteratorCreator) CreateIterator(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { + return ic.CreateIteratorFn(ctx, m, opt) } func (ic *IteratorCreator) FieldDimensions(m *influxql.Measurement) (fields map[string]influxql.DataType, dimensions map[string]struct{}, err error) { diff --git a/query/point.gen.go b/query/point.gen.go index 2e2cf648ca6..457ec9da086 100644 --- a/query/point.gen.go +++ b/query/point.gen.go @@ -7,6 +7,7 @@ package query import ( + "context" "encoding/binary" "io" @@ -181,11 +182,12 @@ func (enc *FloatPointEncoder) EncodeFloatPoint(p *FloatPoint) error { type FloatPointDecoder struct { r io.Reader stats IteratorStats + ctx context.Context } // NewFloatPointDecoder returns a new instance of FloatPointDecoder that reads from r. -func NewFloatPointDecoder(r io.Reader) *FloatPointDecoder { - return &FloatPointDecoder{r: r} +func NewFloatPointDecoder(ctx context.Context, r io.Reader) *FloatPointDecoder { + return &FloatPointDecoder{r: r, ctx: ctx} } // Stats returns iterator stats embedded within the stream. @@ -218,6 +220,15 @@ func (dec *FloatPointDecoder) DecodeFloatPoint(p *FloatPoint) error { continue } + if len(pb.Trace) > 0 { + var err error + err = decodeIteratorTrace(dec.ctx, pb.Trace) + if err != nil { + return err + } + continue + } + // Decode into point object. *p = *decodeFloatPoint(&pb) @@ -392,11 +403,12 @@ func (enc *IntegerPointEncoder) EncodeIntegerPoint(p *IntegerPoint) error { type IntegerPointDecoder struct { r io.Reader stats IteratorStats + ctx context.Context } // NewIntegerPointDecoder returns a new instance of IntegerPointDecoder that reads from r. -func NewIntegerPointDecoder(r io.Reader) *IntegerPointDecoder { - return &IntegerPointDecoder{r: r} +func NewIntegerPointDecoder(ctx context.Context, r io.Reader) *IntegerPointDecoder { + return &IntegerPointDecoder{r: r, ctx: ctx} } // Stats returns iterator stats embedded within the stream. @@ -429,6 +441,15 @@ func (dec *IntegerPointDecoder) DecodeIntegerPoint(p *IntegerPoint) error { continue } + if len(pb.Trace) > 0 { + var err error + err = decodeIteratorTrace(dec.ctx, pb.Trace) + if err != nil { + return err + } + continue + } + // Decode into point object. *p = *decodeIntegerPoint(&pb) @@ -601,11 +622,12 @@ func (enc *UnsignedPointEncoder) EncodeUnsignedPoint(p *UnsignedPoint) error { type UnsignedPointDecoder struct { r io.Reader stats IteratorStats + ctx context.Context } // NewUnsignedPointDecoder returns a new instance of UnsignedPointDecoder that reads from r. -func NewUnsignedPointDecoder(r io.Reader) *UnsignedPointDecoder { - return &UnsignedPointDecoder{r: r} +func NewUnsignedPointDecoder(ctx context.Context, r io.Reader) *UnsignedPointDecoder { + return &UnsignedPointDecoder{r: r, ctx: ctx} } // Stats returns iterator stats embedded within the stream. @@ -638,6 +660,15 @@ func (dec *UnsignedPointDecoder) DecodeUnsignedPoint(p *UnsignedPoint) error { continue } + if len(pb.Trace) > 0 { + var err error + err = decodeIteratorTrace(dec.ctx, pb.Trace) + if err != nil { + return err + } + continue + } + // Decode into point object. *p = *decodeUnsignedPoint(&pb) @@ -812,11 +843,12 @@ func (enc *StringPointEncoder) EncodeStringPoint(p *StringPoint) error { type StringPointDecoder struct { r io.Reader stats IteratorStats + ctx context.Context } // NewStringPointDecoder returns a new instance of StringPointDecoder that reads from r. -func NewStringPointDecoder(r io.Reader) *StringPointDecoder { - return &StringPointDecoder{r: r} +func NewStringPointDecoder(ctx context.Context, r io.Reader) *StringPointDecoder { + return &StringPointDecoder{r: r, ctx: ctx} } // Stats returns iterator stats embedded within the stream. @@ -849,6 +881,15 @@ func (dec *StringPointDecoder) DecodeStringPoint(p *StringPoint) error { continue } + if len(pb.Trace) > 0 { + var err error + err = decodeIteratorTrace(dec.ctx, pb.Trace) + if err != nil { + return err + } + continue + } + // Decode into point object. *p = *decodeStringPoint(&pb) @@ -1023,11 +1064,12 @@ func (enc *BooleanPointEncoder) EncodeBooleanPoint(p *BooleanPoint) error { type BooleanPointDecoder struct { r io.Reader stats IteratorStats + ctx context.Context } // NewBooleanPointDecoder returns a new instance of BooleanPointDecoder that reads from r. -func NewBooleanPointDecoder(r io.Reader) *BooleanPointDecoder { - return &BooleanPointDecoder{r: r} +func NewBooleanPointDecoder(ctx context.Context, r io.Reader) *BooleanPointDecoder { + return &BooleanPointDecoder{r: r, ctx: ctx} } // Stats returns iterator stats embedded within the stream. @@ -1060,6 +1102,15 @@ func (dec *BooleanPointDecoder) DecodeBooleanPoint(p *BooleanPoint) error { continue } + if len(pb.Trace) > 0 { + var err error + err = decodeIteratorTrace(dec.ctx, pb.Trace) + if err != nil { + return err + } + continue + } + // Decode into point object. *p = *decodeBooleanPoint(&pb) diff --git a/query/point.gen.go.tmpl b/query/point.gen.go.tmpl index f869918bb1b..c2c4ea3cf11 100644 --- a/query/point.gen.go.tmpl +++ b/query/point.gen.go.tmpl @@ -1,6 +1,7 @@ package query import ( + "context" "encoding/binary" "io" @@ -188,11 +189,12 @@ func (enc *{{.Name}}PointEncoder) Encode{{.Name}}Point(p *{{.Name}}Point) error type {{.Name}}PointDecoder struct { r io.Reader stats IteratorStats + ctx context.Context } // New{{.Name}}PointDecoder returns a new instance of {{.Name}}PointDecoder that reads from r. -func New{{.Name}}PointDecoder(r io.Reader) *{{.Name}}PointDecoder { - return &{{.Name}}PointDecoder{r: r} +func New{{.Name}}PointDecoder(ctx context.Context, r io.Reader) *{{.Name}}PointDecoder { + return &{{.Name}}PointDecoder{r: r, ctx: ctx} } // Stats returns iterator stats embedded within the stream. @@ -225,6 +227,15 @@ func (dec *{{.Name}}PointDecoder) Decode{{.Name}}Point(p *{{.Name}}Point) error continue } + if len(pb.Trace) > 0 { + var err error + err = decodeIteratorTrace(dec.ctx, pb.Trace) + if err != nil { + return err + } + continue + } + // Decode into point object. *p = *decode{{.Name}}Point(&pb) diff --git a/query/select.go b/query/select.go index e7a82ee7633..afbee74e4a8 100644 --- a/query/select.go +++ b/query/select.go @@ -1,6 +1,7 @@ package query import ( + "context" "errors" "fmt" "io" @@ -8,6 +9,7 @@ import ( "sort" "github.com/influxdata/influxdb/influxql" + "github.com/influxdata/influxdb/pkg/tracing" ) // SelectOptions are options that customize the select call. @@ -54,7 +56,7 @@ type ShardGroup interface { // Select is a prepared statement that is ready to be executed. type PreparedStatement interface { // Select creates the Iterators that will be used to read the query. - Select() ([]Iterator, []string, error) + Select(ctx context.Context) ([]Iterator, []string, error) // Explain outputs the explain plan for this statement. Explain() (string, error) @@ -77,14 +79,14 @@ func Prepare(stmt *influxql.SelectStatement, shardMapper ShardMapper, opt Select // Select compiles, prepares, and then initiates execution of the query using the // default compile options. -func Select(stmt *influxql.SelectStatement, shardMapper ShardMapper, opt SelectOptions) ([]Iterator, []string, error) { +func Select(ctx context.Context, stmt *influxql.SelectStatement, shardMapper ShardMapper, opt SelectOptions) ([]Iterator, []string, error) { s, err := Prepare(stmt, shardMapper, opt) if err != nil { return nil, nil, err } // Must be deferred so it runs after Select. defer s.Close() - return s.Select() + return s.Select(ctx) } type preparedStatement struct { @@ -97,8 +99,8 @@ type preparedStatement struct { columns []string } -func (p *preparedStatement) Select() ([]Iterator, []string, error) { - itrs, err := buildIterators(p.stmt, p.ic, p.opt) +func (p *preparedStatement) Select(ctx context.Context) ([]Iterator, []string, error) { + itrs, err := buildIterators(ctx, p.stmt, p.ic, p.opt) if err != nil { return nil, nil, err } @@ -109,7 +111,8 @@ func (p *preparedStatement) Close() error { return p.ic.Close() } -func buildIterators(stmt *influxql.SelectStatement, ic IteratorCreator, opt IteratorOptions) ([]Iterator, error) { +func buildIterators(ctx context.Context, stmt *influxql.SelectStatement, ic IteratorCreator, opt IteratorOptions) ([]Iterator, error) { + span := tracing.SpanFromContext(ctx) // Retrieve refs for each call and var ref. info := newSelectInfo(stmt) if len(info.calls) > 1 && len(info.refs) > 0 { @@ -125,7 +128,22 @@ func buildIterators(stmt *influxql.SelectStatement, ic IteratorCreator, opt Iter // If there are multiple auxilary fields and no calls then construct an aux iterator. if len(info.calls) == 0 && len(info.refs) > 0 { - return buildAuxIterators(stmt.Fields, ic, stmt.Sources, opt) + if span != nil { + span = span.StartSpan("auxiliary_iterators") + defer span.Finish() + + span.SetLabels("statement", stmt.String()) + ctx = tracing.NewContextWithSpan(ctx, span) + } + return buildAuxIterators(ctx, stmt.Fields, ic, stmt.Sources, opt) + } + + if span != nil { + span = span.StartSpan("field_iterators") + defer span.Finish() + + span.SetLabels("statement", stmt.String()) + ctx = tracing.NewContextWithSpan(ctx, span) } // Include auxiliary fields from top() and bottom() when not writing the results. @@ -167,18 +185,18 @@ func buildIterators(stmt *influxql.SelectStatement, ic IteratorCreator, opt Iter } } - return buildFieldIterators(fields, ic, stmt.Sources, opt, selector, stmt.Target != nil) + return buildFieldIterators(ctx, fields, ic, stmt.Sources, opt, selector, stmt.Target != nil) } // buildAuxIterators creates a set of iterators from a single combined auxiliary iterator. -func buildAuxIterators(fields influxql.Fields, ic IteratorCreator, sources influxql.Sources, opt IteratorOptions) ([]Iterator, error) { +func buildAuxIterators(ctx context.Context, fields influxql.Fields, ic IteratorCreator, sources influxql.Sources, opt IteratorOptions) ([]Iterator, error) { // Create the auxiliary iterators for each source. inputs := make([]Iterator, 0, len(sources)) if err := func() error { for _, source := range sources { switch source := source.(type) { case *influxql.Measurement: - input, err := ic.CreateIterator(source, opt) + input, err := ic.CreateIterator(ctx, source, opt) if err != nil { return err } @@ -189,7 +207,7 @@ func buildAuxIterators(fields influxql.Fields, ic IteratorCreator, sources influ stmt: source.Statement, } - input, err := b.buildAuxIterator(opt) + input, err := b.buildAuxIterator(ctx, opt) if err != nil { return err } @@ -304,9 +322,10 @@ func buildAuxIterator(expr influxql.Expr, aitr AuxIterator, opt IteratorOptions) } // buildFieldIterators creates an iterator for each field expression. -func buildFieldIterators(fields influxql.Fields, ic IteratorCreator, sources influxql.Sources, opt IteratorOptions, selector, writeMode bool) ([]Iterator, error) { +func buildFieldIterators(ctx context.Context, fields influxql.Fields, ic IteratorCreator, sources influxql.Sources, opt IteratorOptions, selector, writeMode bool) ([]Iterator, error) { // Create iterators from fields against the iterator creator. itrs := make([]Iterator, len(fields)) + span := tracing.SpanFromContext(ctx) if err := func() error { hasAuxFields := false @@ -321,8 +340,22 @@ func buildFieldIterators(fields influxql.Fields, ic IteratorCreator, sources inf continue } + var localSpan *tracing.Span + localContext := ctx + + if span != nil { + localSpan = span.StartSpan("expression") + localSpan.SetLabels("expr", f.Expr.String()) + localContext = tracing.NewContextWithSpan(ctx, localSpan) + } + expr := influxql.Reduce(f.Expr, nil) - itr, err := buildExprIterator(expr, ic, sources, opt, selector, writeMode) + itr, err := buildExprIterator(localContext, expr, ic, sources, opt, selector, writeMode) + + if localSpan != nil { + localSpan.Finish() + } + if err != nil { return err } else if itr == nil { @@ -371,7 +404,7 @@ func buildFieldIterators(fields influxql.Fields, ic IteratorCreator, sources inf } // buildExprIterator creates an iterator for an expression. -func buildExprIterator(expr influxql.Expr, ic IteratorCreator, sources influxql.Sources, opt IteratorOptions, selector, writeMode bool) (Iterator, error) { +func buildExprIterator(ctx context.Context, expr influxql.Expr, ic IteratorCreator, sources influxql.Sources, opt IteratorOptions, selector, writeMode bool) (Iterator, error) { opt.Expr = expr b := exprIteratorBuilder{ ic: ic, @@ -383,13 +416,13 @@ func buildExprIterator(expr influxql.Expr, ic IteratorCreator, sources influxql. switch expr := expr.(type) { case *influxql.VarRef: - return b.buildVarRefIterator(expr) + return b.buildVarRefIterator(ctx, expr) case *influxql.Call: - return b.buildCallIterator(expr) + return b.buildCallIterator(ctx, expr) case *influxql.BinaryExpr: - return b.buildBinaryExprIterator(expr) + return b.buildBinaryExprIterator(ctx, expr) case *influxql.ParenExpr: - return buildExprIterator(expr.Expr, ic, sources, opt, selector, writeMode) + return buildExprIterator(ctx, expr.Expr, ic, sources, opt, selector, writeMode) case *influxql.NilLiteral: return &nilFloatIterator{}, nil default: @@ -405,13 +438,13 @@ type exprIteratorBuilder struct { writeMode bool } -func (b *exprIteratorBuilder) buildVarRefIterator(expr *influxql.VarRef) (Iterator, error) { +func (b *exprIteratorBuilder) buildVarRefIterator(ctx context.Context, expr *influxql.VarRef) (Iterator, error) { inputs := make([]Iterator, 0, len(b.sources)) if err := func() error { for _, source := range b.sources { switch source := source.(type) { case *influxql.Measurement: - input, err := b.ic.CreateIterator(source, b.opt) + input, err := b.ic.CreateIterator(ctx, source, b.opt) if err != nil { return err } @@ -422,7 +455,7 @@ func (b *exprIteratorBuilder) buildVarRefIterator(expr *influxql.VarRef) (Iterat stmt: source.Statement, } - input, err := subquery.buildVarRefIterator(expr, b.opt) + input, err := subquery.buildVarRefIterator(ctx, expr, b.opt) if err != nil { return err } @@ -448,7 +481,7 @@ func (b *exprIteratorBuilder) buildVarRefIterator(expr *influxql.VarRef) (Iterat return itr, nil } -func (b *exprIteratorBuilder) buildCallIterator(expr *influxql.Call) (Iterator, error) { +func (b *exprIteratorBuilder) buildCallIterator(ctx context.Context, expr *influxql.Call) (Iterator, error) { // TODO(jsternberg): Refactor this. This section needs to die in a fire. opt := b.opt // Eliminate limits and offsets if they were previously set. These are handled by the caller. @@ -456,7 +489,7 @@ func (b *exprIteratorBuilder) buildCallIterator(expr *influxql.Call) (Iterator, switch expr.Name { case "distinct": opt.Ordered = true - input, err := buildExprIterator(expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, b.selector, false) + input, err := buildExprIterator(ctx, expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, b.selector, false) if err != nil { return nil, err } @@ -467,7 +500,7 @@ func (b *exprIteratorBuilder) buildCallIterator(expr *influxql.Call) (Iterator, return NewIntervalIterator(input, opt), nil case "sample": opt.Ordered = true - input, err := buildExprIterator(expr.Args[0], b.ic, b.sources, opt, b.selector, false) + input, err := buildExprIterator(ctx, expr.Args[0], b.ic, b.sources, opt, b.selector, false) if err != nil { return nil, err } @@ -476,7 +509,7 @@ func (b *exprIteratorBuilder) buildCallIterator(expr *influxql.Call) (Iterator, return newSampleIterator(input, opt, int(size.Val)) case "holt_winters", "holt_winters_with_fit": opt.Ordered = true - input, err := buildExprIterator(expr.Args[0], b.ic, b.sources, opt, b.selector, false) + input, err := buildExprIterator(ctx, expr.Args[0], b.ic, b.sources, opt, b.selector, false) if err != nil { return nil, err } @@ -502,7 +535,7 @@ func (b *exprIteratorBuilder) buildCallIterator(expr *influxql.Call) (Iterator, } opt.Ordered = true - input, err := buildExprIterator(expr.Args[0], b.ic, b.sources, opt, b.selector, false) + input, err := buildExprIterator(ctx, expr.Args[0], b.ic, b.sources, opt, b.selector, false) if err != nil { return nil, err } @@ -532,14 +565,14 @@ func (b *exprIteratorBuilder) buildCallIterator(expr *influxql.Call) (Iterator, panic(fmt.Sprintf("invalid series aggregate function: %s", expr.Name)) case "cumulative_sum": opt.Ordered = true - input, err := buildExprIterator(expr.Args[0], b.ic, b.sources, opt, b.selector, false) + input, err := buildExprIterator(ctx, expr.Args[0], b.ic, b.sources, opt, b.selector, false) if err != nil { return nil, err } return newCumulativeSumIterator(input, opt) case "integral": opt.Ordered = true - input, err := buildExprIterator(expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false) + input, err := buildExprIterator(ctx, expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false) if err != nil { return nil, err } @@ -576,7 +609,7 @@ func (b *exprIteratorBuilder) buildCallIterator(expr *influxql.Call) (Iterator, builder.selector = true builder.writeMode = false - i, err := builder.callIterator(call, callOpt) + i, err := builder.callIterator(ctx, call, callOpt) if err != nil { return nil, err } @@ -589,7 +622,7 @@ func (b *exprIteratorBuilder) buildCallIterator(expr *influxql.Call) (Iterator, builder.writeMode = false ref := expr.Args[0].(*influxql.VarRef) - i, err := builder.buildVarRefIterator(ref) + i, err := builder.buildVarRefIterator(ctx, ref) if err != nil { return nil, err } @@ -629,7 +662,7 @@ func (b *exprIteratorBuilder) buildCallIterator(expr *influxql.Call) (Iterator, builder.selector = true builder.writeMode = false - i, err := builder.callIterator(call, callOpt) + i, err := builder.callIterator(ctx, call, callOpt) if err != nil { return nil, err } @@ -642,7 +675,7 @@ func (b *exprIteratorBuilder) buildCallIterator(expr *influxql.Call) (Iterator, builder.writeMode = false ref := expr.Args[0].(*influxql.VarRef) - i, err := builder.buildVarRefIterator(ref) + i, err := builder.buildVarRefIterator(ctx, ref) if err != nil { return nil, err } @@ -659,7 +692,7 @@ func (b *exprIteratorBuilder) buildCallIterator(expr *influxql.Call) (Iterator, switch arg0 := expr.Args[0].(type) { case *influxql.Call: if arg0.Name == "distinct" { - input, err := buildExprIterator(arg0, b.ic, b.sources, opt, b.selector, false) + input, err := buildExprIterator(ctx, arg0, b.ic, b.sources, opt, b.selector, false) if err != nil { return nil, err } @@ -668,36 +701,36 @@ func (b *exprIteratorBuilder) buildCallIterator(expr *influxql.Call) (Iterator, } fallthrough case "min", "max", "sum", "first", "last", "mean": - return b.callIterator(expr, opt) + return b.callIterator(ctx, expr, opt) case "median": opt.Ordered = true - input, err := buildExprIterator(expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false) + input, err := buildExprIterator(ctx, expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false) if err != nil { return nil, err } return newMedianIterator(input, opt) case "mode": - input, err := buildExprIterator(expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false) + input, err := buildExprIterator(ctx, expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false) if err != nil { return nil, err } return NewModeIterator(input, opt) case "stddev": - input, err := buildExprIterator(expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false) + input, err := buildExprIterator(ctx, expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false) if err != nil { return nil, err } return newStddevIterator(input, opt) case "spread": // OPTIMIZE(benbjohnson): convert to map/reduce - input, err := buildExprIterator(expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false) + input, err := buildExprIterator(ctx, expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false) if err != nil { return nil, err } return newSpreadIterator(input, opt) case "percentile": opt.Ordered = true - input, err := buildExprIterator(expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false) + input, err := buildExprIterator(ctx, expr.Args[0].(*influxql.VarRef), b.ic, b.sources, opt, false, false) if err != nil { return nil, err } @@ -730,7 +763,7 @@ func (b *exprIteratorBuilder) buildCallIterator(expr *influxql.Call) (Iterator, return itr, nil } -func (b *exprIteratorBuilder) buildBinaryExprIterator(expr *influxql.BinaryExpr) (Iterator, error) { +func (b *exprIteratorBuilder) buildBinaryExprIterator(ctx context.Context, expr *influxql.BinaryExpr) (Iterator, error) { if rhs, ok := expr.RHS.(influxql.Literal); ok { // The right hand side is a literal. It is more common to have the RHS be a literal, // so we check that one first and have this be the happy path. @@ -739,24 +772,24 @@ func (b *exprIteratorBuilder) buildBinaryExprIterator(expr *influxql.BinaryExpr) return nil, fmt.Errorf("unable to construct an iterator from two literals: LHS: %T, RHS: %T", lhs, rhs) } - lhs, err := buildExprIterator(expr.LHS, b.ic, b.sources, b.opt, b.selector, false) + lhs, err := buildExprIterator(ctx, expr.LHS, b.ic, b.sources, b.opt, b.selector, false) if err != nil { return nil, err } return buildRHSTransformIterator(lhs, rhs, expr.Op, b.opt) } else if lhs, ok := expr.LHS.(influxql.Literal); ok { - rhs, err := buildExprIterator(expr.RHS, b.ic, b.sources, b.opt, b.selector, false) + rhs, err := buildExprIterator(ctx, expr.RHS, b.ic, b.sources, b.opt, b.selector, false) if err != nil { return nil, err } return buildLHSTransformIterator(lhs, rhs, expr.Op, b.opt) } else { // We have two iterators. Combine them into a single iterator. - lhs, err := buildExprIterator(expr.LHS, b.ic, b.sources, b.opt, false, false) + lhs, err := buildExprIterator(ctx, expr.LHS, b.ic, b.sources, b.opt, false, false) if err != nil { return nil, err } - rhs, err := buildExprIterator(expr.RHS, b.ic, b.sources, b.opt, false, false) + rhs, err := buildExprIterator(ctx, expr.RHS, b.ic, b.sources, b.opt, false, false) if err != nil { return nil, err } @@ -764,13 +797,13 @@ func (b *exprIteratorBuilder) buildBinaryExprIterator(expr *influxql.BinaryExpr) } } -func (b *exprIteratorBuilder) callIterator(expr *influxql.Call, opt IteratorOptions) (Iterator, error) { +func (b *exprIteratorBuilder) callIterator(ctx context.Context, expr *influxql.Call, opt IteratorOptions) (Iterator, error) { inputs := make([]Iterator, 0, len(b.sources)) if err := func() error { for _, source := range b.sources { switch source := source.(type) { case *influxql.Measurement: - input, err := b.ic.CreateIterator(source, opt) + input, err := b.ic.CreateIterator(ctx, source, opt) if err != nil { return err } @@ -779,7 +812,7 @@ func (b *exprIteratorBuilder) callIterator(expr *influxql.Call, opt IteratorOpti // Identify the name of the field we are using. arg0 := expr.Args[0].(*influxql.VarRef) - input, err := buildExprIterator(arg0, b.ic, []influxql.Source{source}, opt, b.selector, false) + input, err := buildExprIterator(ctx, arg0, b.ic, []influxql.Source{source}, opt, b.selector, false) if err != nil { return err } diff --git a/query/select_test.go b/query/select_test.go index 614fe8b297a..fa58e991c02 100644 --- a/query/select_test.go +++ b/query/select_test.go @@ -1,6 +1,7 @@ package query_test import ( + "context" "fmt" "math/rand" "reflect" @@ -2765,7 +2766,7 @@ func TestSelect(t *testing.T) { "value": tt.typ, }, Dimensions: []string{"host", "region"}, - CreateIteratorFn: func(m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { + CreateIteratorFn: func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { if m.Name != "cpu" { t.Fatalf("unexpected source: %s", m.Name) } @@ -2789,7 +2790,7 @@ func TestSelect(t *testing.T) { }, } - itrs, _, err := query.Select(MustParseSelectStatement(tt.q), &shardMapper, query.SelectOptions{}) + itrs, _, err := query.Select(context.Background(), MustParseSelectStatement(tt.q), &shardMapper, query.SelectOptions{}) if err != nil { if tt.err == "" { t.Fatal(err) @@ -2819,7 +2820,7 @@ func TestSelect_Raw(t *testing.T) { "s": influxql.String, "b": influxql.Boolean, }, - CreateIteratorFn: func(m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { + CreateIteratorFn: func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { if m.Name != "cpu" { t.Fatalf("unexpected source: %s", m.Name) } @@ -2846,7 +2847,7 @@ func TestSelect_Raw(t *testing.T) { } stmt := MustParseSelectStatement(`SELECT f, i, u, s, b FROM cpu`) - itrs, _, err := query.Select(stmt, &shardMapper, query.SelectOptions{}) + itrs, _, err := query.Select(context.Background(), stmt, &shardMapper, query.SelectOptions{}) if err != nil { t.Errorf("parse error: %s", err) } else if a, err := Iterators(itrs).ReadAll(); err != nil { @@ -2888,7 +2889,7 @@ func TestSelect_BinaryExpr(t *testing.T) { "i": influxql.Integer, "u": influxql.Unsigned, }, - CreateIteratorFn: func(m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { + CreateIteratorFn: func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { if m.Name != "cpu" { t.Fatalf("unexpected source: %s", m.Name) } @@ -3753,7 +3754,7 @@ func TestSelect_BinaryExpr(t *testing.T) { } { t.Run(test.Name, func(t *testing.T) { stmt := MustParseSelectStatement(test.Statement) - itrs, _, err := query.Select(stmt, &shardMapper, query.SelectOptions{}) + itrs, _, err := query.Select(context.Background(), stmt, &shardMapper, query.SelectOptions{}) if err != nil { if have, want := err.Error(), test.Err; want != "" { if have != want { @@ -3782,7 +3783,7 @@ func TestSelect_BinaryExpr_Boolean(t *testing.T) { "one": influxql.Boolean, "two": influxql.Boolean, }, - CreateIteratorFn: func(m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { + CreateIteratorFn: func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { if m.Name != "cpu" { t.Fatalf("unexpected source: %s", m.Name) } @@ -3838,7 +3839,7 @@ func TestSelect_BinaryExpr_Boolean(t *testing.T) { } { t.Run(test.Name, func(t *testing.T) { stmt := MustParseSelectStatement(test.Statement) - itrs, _, err := query.Select(stmt, &shardMapper, query.SelectOptions{}) + itrs, _, err := query.Select(context.Background(), stmt, &shardMapper, query.SelectOptions{}) if err != nil { t.Errorf("%s: parse error: %s", test.Name, err) } else if a, err := Iterators(itrs).ReadAll(); err != nil { @@ -3861,7 +3862,7 @@ func TestSelect_BinaryExpr_NilValues(t *testing.T) { "total": influxql.Float, "value": influxql.Float, }, - CreateIteratorFn: func(m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { + CreateIteratorFn: func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { if m.Name != "cpu" { t.Fatalf("unexpected source: %s", m.Name) } @@ -3919,7 +3920,7 @@ func TestSelect_BinaryExpr_NilValues(t *testing.T) { } { t.Run(test.Name, func(t *testing.T) { stmt := MustParseSelectStatement(test.Statement) - itrs, _, err := query.Select(stmt, &shardMapper, query.SelectOptions{}) + itrs, _, err := query.Select(context.Background(), stmt, &shardMapper, query.SelectOptions{}) if err != nil { t.Errorf("%s: parse error: %s", test.Name, err) } else if a, err := Iterators(itrs).ReadAll(); err != nil { @@ -3941,13 +3942,13 @@ func (m *ShardMapper) MapShards(sources influxql.Sources, t influxql.TimeRange, } type ShardGroup struct { - CreateIteratorFn func(m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) + CreateIteratorFn func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) Fields map[string]influxql.DataType Dimensions []string } -func (sh *ShardGroup) CreateIterator(m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { - return sh.CreateIteratorFn(m, opt) +func (sh *ShardGroup) CreateIterator(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { + return sh.CreateIteratorFn(ctx, m, opt) } func (sh *ShardGroup) IteratorCost(m *influxql.Measurement, opt query.IteratorOptions) (query.IteratorCost, error) { @@ -3994,7 +3995,7 @@ func benchmarkSelect(b *testing.B, stmt *influxql.SelectStatement, shardMapper q b.ReportAllocs() for i := 0; i < b.N; i++ { - itrs, _, err := query.Select(stmt, shardMapper, query.SelectOptions{}) + itrs, _, err := query.Select(context.Background(), stmt, shardMapper, query.SelectOptions{}) if err != nil { b.Fatal(err) } @@ -4010,7 +4011,7 @@ func NewRawBenchmarkIteratorCreator(pointN int) query.ShardMapper { Fields: map[string]influxql.DataType{ "fval": influxql.Float, }, - CreateIteratorFn: func(m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { + CreateIteratorFn: func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { if opt.Expr != nil { panic("unexpected expression") } @@ -4049,7 +4050,7 @@ func benchmarkSelectDedupe(b *testing.B, seriesN, pointsPerSeries int) { Fields: map[string]influxql.DataType{ "sval": influxql.String, }, - CreateIteratorFn: func(m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { + CreateIteratorFn: func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { if opt.Expr != nil { panic("unexpected expression") } @@ -4083,7 +4084,7 @@ func benchmarkSelectTop(b *testing.B, seriesN, pointsPerSeries int) { Fields: map[string]influxql.DataType{ "sval": influxql.Float, }, - CreateIteratorFn: func(m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { + CreateIteratorFn: func(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions) (query.Iterator, error) { if m.Name != "cpu" { b.Fatalf("unexpected source: %s", m.Name) } diff --git a/query/subquery.go b/query/subquery.go index 834ac21ad3e..275532f2c4e 100644 --- a/query/subquery.go +++ b/query/subquery.go @@ -1,6 +1,10 @@ package query -import "github.com/influxdata/influxdb/influxql" +import ( + "context" + + "github.com/influxdata/influxdb/influxql" +) type subqueryBuilder struct { ic IteratorCreator @@ -8,7 +12,7 @@ type subqueryBuilder struct { } // buildAuxIterator constructs an auxiliary Iterator from a subquery. -func (b *subqueryBuilder) buildAuxIterator(opt IteratorOptions) (Iterator, error) { +func (b *subqueryBuilder) buildAuxIterator(ctx context.Context, opt IteratorOptions) (Iterator, error) { // Retrieve a list of fields needed for conditions. auxFields := opt.Aux conds := influxql.ExprNames(opt.Condition) @@ -26,7 +30,7 @@ func (b *subqueryBuilder) buildAuxIterator(opt IteratorOptions) (Iterator, error } subOpt.Aux = auxFields - itrs, err := buildIterators(b.stmt, b.ic, subOpt) + itrs, err := buildIterators(ctx, b.stmt, b.ic, subOpt) if err != nil { return nil, err } @@ -85,7 +89,7 @@ func (b *subqueryBuilder) mapAuxField(name *influxql.VarRef) IteratorMap { return nil } -func (b *subqueryBuilder) buildVarRefIterator(expr *influxql.VarRef, opt IteratorOptions) (Iterator, error) { +func (b *subqueryBuilder) buildVarRefIterator(ctx context.Context, expr *influxql.VarRef, opt IteratorOptions) (Iterator, error) { // Look for the field or tag that is driving this query. driver := b.mapAuxField(expr) if driver == nil { @@ -116,7 +120,7 @@ func (b *subqueryBuilder) buildVarRefIterator(expr *influxql.VarRef, opt Iterato } subOpt.Aux = auxFields - itrs, err := buildIterators(b.stmt, b.ic, subOpt) + itrs, err := buildIterators(ctx, b.stmt, b.ic, subOpt) if err != nil { return nil, err } diff --git a/tsdb/engine.go b/tsdb/engine.go index 2c8a1a25e5b..3abe3201970 100644 --- a/tsdb/engine.go +++ b/tsdb/engine.go @@ -1,6 +1,7 @@ package tsdb import ( + "context" "errors" "fmt" "io" @@ -43,7 +44,7 @@ type Engine interface { Restore(r io.Reader, basePath string) error Import(r io.Reader, basePath string) error - CreateIterator(measurement string, opt query.IteratorOptions) (query.Iterator, error) + CreateIterator(ctx context.Context, measurement string, opt query.IteratorOptions) (query.Iterator, error) IteratorCost(measurement string, opt query.IteratorOptions) (query.IteratorCost, error) WritePoints(points []models.Point) error diff --git a/tsdb/engine/tsm1/engine.go b/tsdb/engine/tsm1/engine.go index 6e6ecc1fedc..49fd1f03ca4 100644 --- a/tsdb/engine/tsm1/engine.go +++ b/tsdb/engine/tsm1/engine.go @@ -4,6 +4,7 @@ package tsm1 // import "github.com/influxdata/influxdb/tsdb/engine/tsm1" import ( "archive/tar" "bytes" + "context" "fmt" "io" "io/ioutil" @@ -12,6 +13,7 @@ import ( "path/filepath" "regexp" "runtime" + "strconv" "strings" "sync" "sync/atomic" @@ -22,6 +24,8 @@ import ( "github.com/influxdata/influxdb/pkg/bytesutil" "github.com/influxdata/influxdb/pkg/estimator" "github.com/influxdata/influxdb/pkg/limiter" + "github.com/influxdata/influxdb/pkg/metrics" + "github.com/influxdata/influxdb/pkg/tracing" "github.com/influxdata/influxdb/query" "github.com/influxdata/influxdb/tsdb" _ "github.com/influxdata/influxdb/tsdb/index" @@ -47,6 +51,14 @@ var ( keyFieldSeparatorBytes = []byte(keyFieldSeparator) ) +var ( + tsmGroup = metrics.MustRegisterGroup("tsm1") + numberOfRefCursorsCounter = metrics.MustRegisterCounter("cursors_ref", metrics.WithGroup(tsmGroup)) + numberOfAuxCursorsCounter = metrics.MustRegisterCounter("cursors_aux", metrics.WithGroup(tsmGroup)) + numberOfCondCursorsCounter = metrics.MustRegisterCounter("cursors_cond", metrics.WithGroup(tsmGroup)) + planningTimer = metrics.MustRegisterTimer("planning_time", metrics.WithGroup(tsmGroup)) +) + const ( // keyFieldSeparator separates the series key from the field name in the composite key // that identifies a specific field in series @@ -1670,12 +1682,29 @@ func (e *Engine) cleanupTempTSMFiles() error { } // KeyCursor returns a KeyCursor for the given key starting at time t. -func (e *Engine) KeyCursor(key []byte, t int64, ascending bool) *KeyCursor { - return e.FileStore.KeyCursor(key, t, ascending) +func (e *Engine) KeyCursor(ctx context.Context, key []byte, t int64, ascending bool) *KeyCursor { + return e.FileStore.KeyCursor(ctx, key, t, ascending) } // CreateIterator returns an iterator for the measurement based on opt. -func (e *Engine) CreateIterator(measurement string, opt query.IteratorOptions) (query.Iterator, error) { +func (e *Engine) CreateIterator(ctx context.Context, measurement string, opt query.IteratorOptions) (query.Iterator, error) { + if span := tracing.SpanFromContext(ctx); span != nil { + labels := []string{"shard_id", strconv.Itoa(int(e.id)), "measurement", measurement} + if opt.Condition != nil { + labels = append(labels, "cond", opt.Condition.String()) + } + + span = span.StartSpan("create_iterator") + span.SetLabels(labels...) + ctx = tracing.NewContextWithSpan(ctx, span) + + group := metrics.NewGroup(tsmGroup) + ctx = metrics.NewContextWithGroup(ctx, group) + start := time.Now() + + defer group.GetTimer(planningTimer).UpdateSince(start) + } + if call, ok := opt.Expr.(*influxql.Call); ok { if opt.Interval.IsZero() { if call.Name == "first" || call.Name == "last" { @@ -1685,31 +1714,31 @@ func (e *Engine) CreateIterator(measurement string, opt query.IteratorOptions) ( refOpt.Ordered = true refOpt.Expr = call.Args[0] - itrs, err := e.createVarRefIterator(measurement, refOpt) + itrs, err := e.createVarRefIterator(ctx, measurement, refOpt) if err != nil { return nil, err } - return newMergeFinalizerIterator(itrs, opt, e.logger) + return newMergeFinalizerIterator(ctx, itrs, opt, e.logger) } } - inputs, err := e.createCallIterator(measurement, call, opt) + inputs, err := e.createCallIterator(ctx, measurement, call, opt) if err != nil { return nil, err } else if len(inputs) == 0 { return nil, nil } - return newMergeFinalizerIterator(inputs, opt, e.logger) + return newMergeFinalizerIterator(ctx, inputs, opt, e.logger) } - itrs, err := e.createVarRefIterator(measurement, opt) + itrs, err := e.createVarRefIterator(ctx, measurement, opt) if err != nil { return nil, err } - return newMergeFinalizerIterator(itrs, opt, e.logger) + return newMergeFinalizerIterator(ctx, itrs, opt, e.logger) } -func (e *Engine) createCallIterator(measurement string, call *influxql.Call, opt query.IteratorOptions) ([]query.Iterator, error) { +func (e *Engine) createCallIterator(ctx context.Context, measurement string, call *influxql.Call, opt query.IteratorOptions) ([]query.Iterator, error) { ref, _ := call.Args[0].(*influxql.VarRef) if exists, err := e.index.MeasurementExists([]byte(measurement)); err != nil { @@ -1745,7 +1774,7 @@ func (e *Engine) createCallIterator(measurement string, call *influxql.Call, opt default: } - inputs, err := e.createTagSetIterators(ref, measurement, t, opt) + inputs, err := e.createTagSetIterators(ctx, ref, measurement, t, opt) if err != nil { return err } else if len(inputs) == 0 { @@ -1779,7 +1808,7 @@ func (e *Engine) createCallIterator(measurement string, call *influxql.Call, opt } // createVarRefIterator creates an iterator for a variable reference. -func (e *Engine) createVarRefIterator(measurement string, opt query.IteratorOptions) ([]query.Iterator, error) { +func (e *Engine) createVarRefIterator(ctx context.Context, measurement string, opt query.IteratorOptions) ([]query.Iterator, error) { ref, _ := opt.Expr.(*influxql.VarRef) if exists, err := e.index.MeasurementExists([]byte(measurement)); err != nil { @@ -1807,7 +1836,7 @@ func (e *Engine) createVarRefIterator(measurement string, opt query.IteratorOpti itrs := make([]query.Iterator, 0, len(tagSets)) if err := func() error { for _, t := range tagSets { - inputs, err := e.createTagSetIterators(ref, measurement, t, opt) + inputs, err := e.createTagSetIterators(ctx, ref, measurement, t, opt) if err != nil { return err } else if len(inputs) == 0 { @@ -1857,7 +1886,7 @@ func (e *Engine) createVarRefIterator(measurement string, opt query.IteratorOpti } // createTagSetIterators creates a set of iterators for a tagset. -func (e *Engine) createTagSetIterators(ref *influxql.VarRef, name string, t *query.TagSet, opt query.IteratorOptions) ([]query.Iterator, error) { +func (e *Engine) createTagSetIterators(ctx context.Context, ref *influxql.VarRef, name string, t *query.TagSet, opt query.IteratorOptions) ([]query.Iterator, error) { // Set parallelism by number of logical cpus. parallelism := runtime.GOMAXPROCS(0) if parallelism > len(t.SeriesKeys) { @@ -1892,7 +1921,7 @@ func (e *Engine) createTagSetIterators(ref *influxql.VarRef, name string, t *que wg.Add(1) go func(i int) { defer wg.Done() - groups[i].itrs, groups[i].err = e.createTagSetGroupIterators(ref, name, groups[i].keys, t, groups[i].filters, opt) + groups[i].itrs, groups[i].err = e.createTagSetGroupIterators(ctx, ref, name, groups[i].keys, t, groups[i].filters, opt) }(i) } wg.Wait() @@ -1923,7 +1952,7 @@ func (e *Engine) createTagSetIterators(ref *influxql.VarRef, name string, t *que } // createTagSetGroupIterators creates a set of iterators for a subset of a tagset's series. -func (e *Engine) createTagSetGroupIterators(ref *influxql.VarRef, name string, seriesKeys []string, t *query.TagSet, filters []influxql.Expr, opt query.IteratorOptions) ([]query.Iterator, error) { +func (e *Engine) createTagSetGroupIterators(ctx context.Context, ref *influxql.VarRef, name string, seriesKeys []string, t *query.TagSet, filters []influxql.Expr, opt query.IteratorOptions) ([]query.Iterator, error) { itrs := make([]query.Iterator, 0, len(seriesKeys)) for i, seriesKey := range seriesKeys { var conditionFields []influxql.VarRef @@ -1932,7 +1961,7 @@ func (e *Engine) createTagSetGroupIterators(ref *influxql.VarRef, name string, s conditionFields = influxql.ExprNames(filters[i]) } - itr, err := e.createVarRefSeriesIterator(ref, name, seriesKey, t, filters[i], conditionFields, opt) + itr, err := e.createVarRefSeriesIterator(ctx, ref, name, seriesKey, t, filters[i], conditionFields, opt) if err != nil { return itrs, err } else if itr == nil { @@ -1959,7 +1988,7 @@ func (e *Engine) createTagSetGroupIterators(ref *influxql.VarRef, name string, s } // createVarRefSeriesIterator creates an iterator for a variable reference for a series. -func (e *Engine) createVarRefSeriesIterator(ref *influxql.VarRef, name string, seriesKey string, t *query.TagSet, filter influxql.Expr, conditionFields []influxql.VarRef, opt query.IteratorOptions) (query.Iterator, error) { +func (e *Engine) createVarRefSeriesIterator(ctx context.Context, ref *influxql.VarRef, name string, seriesKey string, t *query.TagSet, filter influxql.Expr, conditionFields []influxql.VarRef, opt query.IteratorOptions) (query.Iterator, error) { _, tfs := models.ParseKey([]byte(seriesKey)) tags := query.NewTags(tfs.Map()) @@ -1967,6 +1996,26 @@ func (e *Engine) createVarRefSeriesIterator(ref *influxql.VarRef, name string, s itrOpt := opt itrOpt.Condition = filter + var curCounter, auxCounter, condCounter *metrics.Counter + if col := metrics.GroupFromContext(ctx); col != nil { + curCounter = col.GetCounter(numberOfRefCursorsCounter) + auxCounter = col.GetCounter(numberOfAuxCursorsCounter) + condCounter = col.GetCounter(numberOfCondCursorsCounter) + } + + // Build main cursor. + var cur cursor + if ref != nil { + cur = e.buildCursor(ctx, name, seriesKey, tfs, ref, opt) + // If the field doesn't exist then don't build an iterator. + if cur == nil { + return nil, nil + } + if curCounter != nil { + curCounter.Add(1) + } + } + // Build auxilary cursors. // Tag values should be returned if the field doesn't exist. var aux []cursorAt @@ -1975,8 +2024,11 @@ func (e *Engine) createVarRefSeriesIterator(ref *influxql.VarRef, name string, s for i, ref := range opt.Aux { // Create cursor from field if a tag wasn't requested. if ref.Type != influxql.Tag { - cur := e.buildCursor(name, seriesKey, tfs, &ref, opt) + cur := e.buildCursor(ctx, name, seriesKey, tfs, &ref, opt) if cur != nil { + if auxCounter != nil { + auxCounter.Add(1) + } aux[i] = newBufCursor(cur, opt.Ascending) continue } @@ -2039,8 +2091,11 @@ func (e *Engine) createVarRefSeriesIterator(ref *influxql.VarRef, name string, s for i, ref := range conditionFields { // Create cursor from field if a tag wasn't requested. if ref.Type != influxql.Tag { - cur := e.buildCursor(name, seriesKey, tfs, &ref, opt) + cur := e.buildCursor(ctx, name, seriesKey, tfs, &ref, opt) if cur != nil { + if condCounter != nil { + condCounter.Add(1) + } conds[i] = newBufCursor(cur, opt.Ascending) continue } @@ -2088,16 +2143,6 @@ func (e *Engine) createVarRefSeriesIterator(ref *influxql.VarRef, name string, s return newFloatIterator(name, tags, itrOpt, nil, aux, conds, condNames), nil } - // Build main cursor. - cur := e.buildCursor(name, seriesKey, tfs, ref, opt) - - // If the field doesn't exist then don't build an iterator. - if cur == nil { - cursorsAt(aux).close() - cursorsAt(conds).close() - return nil, nil - } - // Remove name if requested. if opt.StripName { name = "" @@ -2120,7 +2165,7 @@ func (e *Engine) createVarRefSeriesIterator(ref *influxql.VarRef, name string, s } // buildCursor creates an untyped cursor for a field. -func (e *Engine) buildCursor(measurement, seriesKey string, tags models.Tags, ref *influxql.VarRef, opt query.IteratorOptions) cursor { +func (e *Engine) buildCursor(ctx context.Context, measurement, seriesKey string, tags models.Tags, ref *influxql.VarRef, opt query.IteratorOptions) cursor { // Check if this is a system field cursor. switch ref.Val { case "_name": @@ -2157,28 +2202,28 @@ func (e *Engine) buildCursor(measurement, seriesKey string, tags models.Tags, re case influxql.Float: switch f.Type { case influxql.Integer: - cur := e.buildIntegerCursor(measurement, seriesKey, ref.Val, opt) + cur := e.buildIntegerCursor(ctx, measurement, seriesKey, ref.Val, opt) return &floatCastIntegerCursor{cursor: cur} case influxql.Unsigned: - cur := e.buildUnsignedCursor(measurement, seriesKey, ref.Val, opt) + cur := e.buildUnsignedCursor(ctx, measurement, seriesKey, ref.Val, opt) return &floatCastUnsignedCursor{cursor: cur} } case influxql.Integer: switch f.Type { case influxql.Float: - cur := e.buildFloatCursor(measurement, seriesKey, ref.Val, opt) + cur := e.buildFloatCursor(ctx, measurement, seriesKey, ref.Val, opt) return &integerCastFloatCursor{cursor: cur} case influxql.Unsigned: - cur := e.buildUnsignedCursor(measurement, seriesKey, ref.Val, opt) + cur := e.buildUnsignedCursor(ctx, measurement, seriesKey, ref.Val, opt) return &integerCastUnsignedCursor{cursor: cur} } case influxql.Unsigned: switch f.Type { case influxql.Float: - cur := e.buildFloatCursor(measurement, seriesKey, ref.Val, opt) + cur := e.buildFloatCursor(ctx, measurement, seriesKey, ref.Val, opt) return &unsignedCastFloatCursor{cursor: cur} case influxql.Integer: - cur := e.buildIntegerCursor(measurement, seriesKey, ref.Val, opt) + cur := e.buildIntegerCursor(ctx, measurement, seriesKey, ref.Val, opt) return &unsignedCastIntegerCursor{cursor: cur} } } @@ -2188,15 +2233,15 @@ func (e *Engine) buildCursor(measurement, seriesKey string, tags models.Tags, re // Return appropriate cursor based on type. switch f.Type { case influxql.Float: - return e.buildFloatCursor(measurement, seriesKey, ref.Val, opt) + return e.buildFloatCursor(ctx, measurement, seriesKey, ref.Val, opt) case influxql.Integer: - return e.buildIntegerCursor(measurement, seriesKey, ref.Val, opt) + return e.buildIntegerCursor(ctx, measurement, seriesKey, ref.Val, opt) case influxql.Unsigned: - return e.buildUnsignedCursor(measurement, seriesKey, ref.Val, opt) + return e.buildUnsignedCursor(ctx, measurement, seriesKey, ref.Val, opt) case influxql.String: - return e.buildStringCursor(measurement, seriesKey, ref.Val, opt) + return e.buildStringCursor(ctx, measurement, seriesKey, ref.Val, opt) case influxql.Boolean: - return e.buildBooleanCursor(measurement, seriesKey, ref.Val, opt) + return e.buildBooleanCursor(ctx, measurement, seriesKey, ref.Val, opt) default: panic("unreachable") } @@ -2225,42 +2270,42 @@ func matchTagValues(tags models.Tags, condition influxql.Expr) []string { } // buildFloatCursor creates a cursor for a float field. -func (e *Engine) buildFloatCursor(measurement, seriesKey, field string, opt query.IteratorOptions) floatCursor { +func (e *Engine) buildFloatCursor(ctx context.Context, measurement, seriesKey, field string, opt query.IteratorOptions) floatCursor { key := SeriesFieldKeyBytes(seriesKey, field) cacheValues := e.Cache.Values(key) - keyCursor := e.KeyCursor(key, opt.SeekTime(), opt.Ascending) + keyCursor := e.KeyCursor(ctx, key, opt.SeekTime(), opt.Ascending) return newFloatCursor(opt.SeekTime(), opt.Ascending, cacheValues, keyCursor) } // buildIntegerCursor creates a cursor for an integer field. -func (e *Engine) buildIntegerCursor(measurement, seriesKey, field string, opt query.IteratorOptions) integerCursor { +func (e *Engine) buildIntegerCursor(ctx context.Context, measurement, seriesKey, field string, opt query.IteratorOptions) integerCursor { key := SeriesFieldKeyBytes(seriesKey, field) cacheValues := e.Cache.Values(key) - keyCursor := e.KeyCursor(key, opt.SeekTime(), opt.Ascending) + keyCursor := e.KeyCursor(ctx, key, opt.SeekTime(), opt.Ascending) return newIntegerCursor(opt.SeekTime(), opt.Ascending, cacheValues, keyCursor) } // buildUnsignedCursor creates a cursor for an unsigned field. -func (e *Engine) buildUnsignedCursor(measurement, seriesKey, field string, opt query.IteratorOptions) unsignedCursor { +func (e *Engine) buildUnsignedCursor(ctx context.Context, measurement, seriesKey, field string, opt query.IteratorOptions) unsignedCursor { key := SeriesFieldKeyBytes(seriesKey, field) cacheValues := e.Cache.Values(key) - keyCursor := e.KeyCursor(key, opt.SeekTime(), opt.Ascending) + keyCursor := e.KeyCursor(ctx, key, opt.SeekTime(), opt.Ascending) return newUnsignedCursor(opt.SeekTime(), opt.Ascending, cacheValues, keyCursor) } // buildStringCursor creates a cursor for a string field. -func (e *Engine) buildStringCursor(measurement, seriesKey, field string, opt query.IteratorOptions) stringCursor { +func (e *Engine) buildStringCursor(ctx context.Context, measurement, seriesKey, field string, opt query.IteratorOptions) stringCursor { key := SeriesFieldKeyBytes(seriesKey, field) cacheValues := e.Cache.Values(key) - keyCursor := e.KeyCursor(key, opt.SeekTime(), opt.Ascending) + keyCursor := e.KeyCursor(ctx, key, opt.SeekTime(), opt.Ascending) return newStringCursor(opt.SeekTime(), opt.Ascending, cacheValues, keyCursor) } // buildBooleanCursor creates a cursor for a boolean field. -func (e *Engine) buildBooleanCursor(measurement, seriesKey, field string, opt query.IteratorOptions) booleanCursor { +func (e *Engine) buildBooleanCursor(ctx context.Context, measurement, seriesKey, field string, opt query.IteratorOptions) booleanCursor { key := SeriesFieldKeyBytes(seriesKey, field) cacheValues := e.Cache.Values(key) - keyCursor := e.KeyCursor(key, opt.SeekTime(), opt.Ascending) + keyCursor := e.KeyCursor(ctx, key, opt.SeekTime(), opt.Ascending) return newBooleanCursor(opt.SeekTime(), opt.Ascending, cacheValues, keyCursor) } diff --git a/tsdb/engine/tsm1/engine_test.go b/tsdb/engine/tsm1/engine_test.go index 97df758cd7e..54f91511c72 100644 --- a/tsdb/engine/tsm1/engine_test.go +++ b/tsdb/engine/tsm1/engine_test.go @@ -3,6 +3,7 @@ package tsm1_test import ( "archive/tar" "bytes" + "context" "fmt" "io" "io/ioutil" @@ -269,7 +270,7 @@ func TestEngine_CreateIterator_Cache_Ascending(t *testing.T) { t.Fatalf("failed to write points: %s", err.Error()) } - itr, err := e.CreateIterator("cpu", query.IteratorOptions{ + itr, err := e.CreateIterator(context.Background(), "cpu", query.IteratorOptions{ Expr: influxql.MustParseExpr(`value`), Dimensions: []string{"host"}, StartTime: influxql.MinTime, @@ -321,7 +322,7 @@ func TestEngine_CreateIterator_Cache_Descending(t *testing.T) { t.Fatalf("failed to write points: %s", err.Error()) } - itr, err := e.CreateIterator("cpu", query.IteratorOptions{ + itr, err := e.CreateIterator(context.Background(), "cpu", query.IteratorOptions{ Expr: influxql.MustParseExpr(`value`), Dimensions: []string{"host"}, StartTime: influxql.MinTime, @@ -374,7 +375,7 @@ func TestEngine_CreateIterator_TSM_Ascending(t *testing.T) { } e.MustWriteSnapshot() - itr, err := e.CreateIterator("cpu", query.IteratorOptions{ + itr, err := e.CreateIterator(context.Background(), "cpu", query.IteratorOptions{ Expr: influxql.MustParseExpr(`value`), Dimensions: []string{"host"}, StartTime: influxql.MinTime, @@ -427,7 +428,7 @@ func TestEngine_CreateIterator_TSM_Descending(t *testing.T) { } e.MustWriteSnapshot() - itr, err := e.CreateIterator("cpu", query.IteratorOptions{ + itr, err := e.CreateIterator(context.Background(), "cpu", query.IteratorOptions{ Expr: influxql.MustParseExpr(`value`), Dimensions: []string{"host"}, StartTime: influxql.MinTime, @@ -482,7 +483,7 @@ func TestEngine_CreateIterator_Aux(t *testing.T) { t.Fatalf("failed to write points: %s", err.Error()) } - itr, err := e.CreateIterator("cpu", query.IteratorOptions{ + itr, err := e.CreateIterator(context.Background(), "cpu", query.IteratorOptions{ Expr: influxql.MustParseExpr(`value`), Aux: []influxql.VarRef{{Val: "F"}}, Dimensions: []string{"host"}, @@ -545,7 +546,7 @@ func TestEngine_CreateIterator_Condition(t *testing.T) { t.Fatalf("failed to write points: %s", err.Error()) } - itr, err := e.CreateIterator("cpu", query.IteratorOptions{ + itr, err := e.CreateIterator(context.Background(), "cpu", query.IteratorOptions{ Expr: influxql.MustParseExpr(`value`), Dimensions: []string{"host"}, Condition: influxql.MustParseExpr(`X = 10 OR Y > 150`), @@ -874,7 +875,7 @@ func benchmarkIterator(b *testing.B, opt query.IteratorOptions, pointN int) { b.ReportAllocs() for i := 0; i < b.N; i++ { - itr, err := e.CreateIterator("cpu", opt) + itr, err := e.CreateIterator(context.Background(), "cpu", opt) if err != nil { b.Fatal(err) } diff --git a/tsdb/engine/tsm1/file_store.gen.go b/tsdb/engine/tsm1/file_store.gen.go index 75985b899a6..bc8f3529bb1 100644 --- a/tsdb/engine/tsm1/file_store.gen.go +++ b/tsdb/engine/tsm1/file_store.gen.go @@ -20,6 +20,10 @@ func (c *KeyCursor) ReadFloatBlock(buf *[]FloatValue) ([]FloatValue, error) { if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter(floatBlocksDecodedCounter).Add(1) + c.col.GetCounter(floatBlocksSizeCounter).Add(int64(first.entry.Size)) + } // Remove values we already read values = FloatValues(values).Exclude(first.readMin, first.readMax) @@ -88,6 +92,11 @@ func (c *KeyCursor) ReadFloatBlock(buf *[]FloatValue) ([]FloatValue, error) { if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter(floatBlocksDecodedCounter).Add(1) + c.col.GetCounter(floatBlocksSizeCounter).Add(int64(cur.entry.Size)) + } + // Remove any tombstoned values v = c.filterFloatValues(tombstones, v) @@ -147,6 +156,11 @@ func (c *KeyCursor) ReadFloatBlock(buf *[]FloatValue) ([]FloatValue, error) { if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter(floatBlocksDecodedCounter).Add(1) + c.col.GetCounter(floatBlocksSizeCounter).Add(int64(cur.entry.Size)) + } + // Remove any tombstoned values v = c.filterFloatValues(tombstones, v) @@ -183,6 +197,10 @@ func (c *KeyCursor) ReadIntegerBlock(buf *[]IntegerValue) ([]IntegerValue, error if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter(integerBlocksDecodedCounter).Add(1) + c.col.GetCounter(integerBlocksSizeCounter).Add(int64(first.entry.Size)) + } // Remove values we already read values = IntegerValues(values).Exclude(first.readMin, first.readMax) @@ -251,6 +269,11 @@ func (c *KeyCursor) ReadIntegerBlock(buf *[]IntegerValue) ([]IntegerValue, error if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter(integerBlocksDecodedCounter).Add(1) + c.col.GetCounter(integerBlocksSizeCounter).Add(int64(cur.entry.Size)) + } + // Remove any tombstoned values v = c.filterIntegerValues(tombstones, v) @@ -310,6 +333,11 @@ func (c *KeyCursor) ReadIntegerBlock(buf *[]IntegerValue) ([]IntegerValue, error if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter(integerBlocksDecodedCounter).Add(1) + c.col.GetCounter(integerBlocksSizeCounter).Add(int64(cur.entry.Size)) + } + // Remove any tombstoned values v = c.filterIntegerValues(tombstones, v) @@ -346,6 +374,10 @@ func (c *KeyCursor) ReadUnsignedBlock(buf *[]UnsignedValue) ([]UnsignedValue, er if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter(unsignedBlocksDecodedCounter).Add(1) + c.col.GetCounter(unsignedBlocksSizeCounter).Add(int64(first.entry.Size)) + } // Remove values we already read values = UnsignedValues(values).Exclude(first.readMin, first.readMax) @@ -414,6 +446,11 @@ func (c *KeyCursor) ReadUnsignedBlock(buf *[]UnsignedValue) ([]UnsignedValue, er if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter(unsignedBlocksDecodedCounter).Add(1) + c.col.GetCounter(unsignedBlocksSizeCounter).Add(int64(cur.entry.Size)) + } + // Remove any tombstoned values v = c.filterUnsignedValues(tombstones, v) @@ -473,6 +510,11 @@ func (c *KeyCursor) ReadUnsignedBlock(buf *[]UnsignedValue) ([]UnsignedValue, er if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter(unsignedBlocksDecodedCounter).Add(1) + c.col.GetCounter(unsignedBlocksSizeCounter).Add(int64(cur.entry.Size)) + } + // Remove any tombstoned values v = c.filterUnsignedValues(tombstones, v) @@ -509,6 +551,10 @@ func (c *KeyCursor) ReadStringBlock(buf *[]StringValue) ([]StringValue, error) { if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter(stringBlocksDecodedCounter).Add(1) + c.col.GetCounter(stringBlocksSizeCounter).Add(int64(first.entry.Size)) + } // Remove values we already read values = StringValues(values).Exclude(first.readMin, first.readMax) @@ -577,6 +623,11 @@ func (c *KeyCursor) ReadStringBlock(buf *[]StringValue) ([]StringValue, error) { if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter(stringBlocksDecodedCounter).Add(1) + c.col.GetCounter(stringBlocksSizeCounter).Add(int64(cur.entry.Size)) + } + // Remove any tombstoned values v = c.filterStringValues(tombstones, v) @@ -636,6 +687,11 @@ func (c *KeyCursor) ReadStringBlock(buf *[]StringValue) ([]StringValue, error) { if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter(stringBlocksDecodedCounter).Add(1) + c.col.GetCounter(stringBlocksSizeCounter).Add(int64(cur.entry.Size)) + } + // Remove any tombstoned values v = c.filterStringValues(tombstones, v) @@ -672,6 +728,10 @@ func (c *KeyCursor) ReadBooleanBlock(buf *[]BooleanValue) ([]BooleanValue, error if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter(booleanBlocksDecodedCounter).Add(1) + c.col.GetCounter(booleanBlocksSizeCounter).Add(int64(first.entry.Size)) + } // Remove values we already read values = BooleanValues(values).Exclude(first.readMin, first.readMax) @@ -740,6 +800,11 @@ func (c *KeyCursor) ReadBooleanBlock(buf *[]BooleanValue) ([]BooleanValue, error if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter(booleanBlocksDecodedCounter).Add(1) + c.col.GetCounter(booleanBlocksSizeCounter).Add(int64(cur.entry.Size)) + } + // Remove any tombstoned values v = c.filterBooleanValues(tombstones, v) @@ -799,6 +864,11 @@ func (c *KeyCursor) ReadBooleanBlock(buf *[]BooleanValue) ([]BooleanValue, error if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter(booleanBlocksDecodedCounter).Add(1) + c.col.GetCounter(booleanBlocksSizeCounter).Add(int64(cur.entry.Size)) + } + // Remove any tombstoned values v = c.filterBooleanValues(tombstones, v) diff --git a/tsdb/engine/tsm1/file_store.gen.go.tmpl b/tsdb/engine/tsm1/file_store.gen.go.tmpl index f7a0aca904b..734a325a804 100644 --- a/tsdb/engine/tsm1/file_store.gen.go.tmpl +++ b/tsdb/engine/tsm1/file_store.gen.go.tmpl @@ -16,6 +16,10 @@ func (c *KeyCursor) Read{{.Name}}Block(buf *[]{{.Name}}Value) ([]{{.Name}}Value, if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter({{.name}}BlocksDecodedCounter).Add(1) + c.col.GetCounter({{.name}}BlocksSizeCounter).Add(int64(first.entry.Size)) + } // Remove values we already read values = {{.Name}}Values(values).Exclude(first.readMin, first.readMax) @@ -84,6 +88,11 @@ func (c *KeyCursor) Read{{.Name}}Block(buf *[]{{.Name}}Value) ([]{{.Name}}Value, if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter({{.name}}BlocksDecodedCounter).Add(1) + c.col.GetCounter({{.name}}BlocksSizeCounter).Add(int64(cur.entry.Size)) + } + // Remove any tombstoned values v = c.filter{{.Name}}Values(tombstones, v) @@ -143,6 +152,11 @@ func (c *KeyCursor) Read{{.Name}}Block(buf *[]{{.Name}}Value) ([]{{.Name}}Value, if err != nil { return nil, err } + if c.col != nil { + c.col.GetCounter({{.name}}BlocksDecodedCounter).Add(1) + c.col.GetCounter({{.name}}BlocksSizeCounter).Add(int64(cur.entry.Size)) + } + // Remove any tombstoned values v = c.filter{{.Name}}Values(tombstones, v) diff --git a/tsdb/engine/tsm1/file_store.go b/tsdb/engine/tsm1/file_store.go index aee0bbc4752..d2dedcd0da6 100644 --- a/tsdb/engine/tsm1/file_store.go +++ b/tsdb/engine/tsm1/file_store.go @@ -2,6 +2,7 @@ package tsm1 import ( "bytes" + "context" "fmt" "io/ioutil" "math" @@ -15,6 +16,7 @@ import ( "time" "github.com/influxdata/influxdb/models" + "github.com/influxdata/influxdb/pkg/metrics" "github.com/influxdata/influxdb/query" "github.com/uber-go/zap" ) @@ -120,6 +122,19 @@ const ( statFileStoreCount = "numFiles" ) +var ( + floatBlocksDecodedCounter = metrics.MustRegisterCounter("float_blocks_decoded", metrics.WithGroup(tsmGroup)) + floatBlocksSizeCounter = metrics.MustRegisterCounter("float_blocks_size_bytes", metrics.WithGroup(tsmGroup)) + integerBlocksDecodedCounter = metrics.MustRegisterCounter("integer_blocks_decoded", metrics.WithGroup(tsmGroup)) + integerBlocksSizeCounter = metrics.MustRegisterCounter("integer_blocks_size_bytes", metrics.WithGroup(tsmGroup)) + unsignedBlocksDecodedCounter = metrics.MustRegisterCounter("unsigned_blocks_decoded", metrics.WithGroup(tsmGroup)) + unsignedBlocksSizeCounter = metrics.MustRegisterCounter("unsigned_blocks_size_bytes", metrics.WithGroup(tsmGroup)) + stringBlocksDecodedCounter = metrics.MustRegisterCounter("string_blocks_decoded", metrics.WithGroup(tsmGroup)) + stringBlocksSizeCounter = metrics.MustRegisterCounter("string_blocks_size_bytes", metrics.WithGroup(tsmGroup)) + booleanBlocksDecodedCounter = metrics.MustRegisterCounter("boolean_blocks_decoded", metrics.WithGroup(tsmGroup)) + booleanBlocksSizeCounter = metrics.MustRegisterCounter("boolean_blocks_size_bytes", metrics.WithGroup(tsmGroup)) +) + // FileStore is an abstraction around multiple TSM files. type FileStore struct { mu sync.RWMutex @@ -509,10 +524,10 @@ func (f *FileStore) TSMReader(path string) *TSMReader { } // KeyCursor returns a KeyCursor for key and t across the files in the FileStore. -func (f *FileStore) KeyCursor(key []byte, t int64, ascending bool) *KeyCursor { +func (f *FileStore) KeyCursor(ctx context.Context, key []byte, t int64, ascending bool) *KeyCursor { f.mu.RLock() defer f.mu.RUnlock() - return newKeyCursor(f, key, t, ascending) + return newKeyCursor(ctx, f, key, t, ascending) } // Stats returns the stats of the underlying files, preferring the cached version if it is still valid. @@ -952,6 +967,9 @@ type KeyCursor struct { current []*location buf []Value + ctx context.Context + col *metrics.Group + // pos is the index within seeks. Based on ascending, it will increment or // decrement through the size of seeks slice. pos int @@ -1011,10 +1029,12 @@ func (a ascLocations) Less(i, j int) bool { // newKeyCursor returns a new instance of KeyCursor. // This function assumes the read-lock has been taken. -func newKeyCursor(fs *FileStore, key []byte, t int64, ascending bool) *KeyCursor { +func newKeyCursor(ctx context.Context, fs *FileStore, key []byte, t int64, ascending bool) *KeyCursor { c := &KeyCursor{ key: key, seeks: fs.locations(key, t, ascending), + ctx: ctx, + col: metrics.GroupFromContext(ctx), ascending: ascending, } diff --git a/tsdb/engine/tsm1/file_store_test.go b/tsdb/engine/tsm1/file_store_test.go index e989a704ab3..db856b0827c 100644 --- a/tsdb/engine/tsm1/file_store_test.go +++ b/tsdb/engine/tsm1/file_store_test.go @@ -1,6 +1,7 @@ package tsm1_test import ( + "context" "fmt" "io/ioutil" "os" @@ -71,7 +72,7 @@ func TestFileStore_SeekToAsc_FromStart(t *testing.T) { fs.Replace(nil, files) buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) // Search for an entry that exists in the second file values, err := c.ReadFloatBlock(&buf) if err != nil { @@ -111,7 +112,7 @@ func TestFileStore_SeekToAsc_Duplicate(t *testing.T) { fs.Replace(nil, files) buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) // Search for an entry that exists in the second file values, err := c.ReadFloatBlock(&buf) if err != nil { @@ -184,7 +185,7 @@ func TestFileStore_SeekToAsc_BeforeStart(t *testing.T) { // Search for an entry that exists in the second file buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) values, err := c.ReadFloatBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -226,7 +227,7 @@ func TestFileStore_SeekToAsc_BeforeStart_OverlapFloat(t *testing.T) { // Search for an entry that exists in the second file buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) values, err := c.ReadFloatBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -293,7 +294,7 @@ func TestFileStore_SeekToAsc_BeforeStart_OverlapInteger(t *testing.T) { // Search for an entry that exists in the second file buf := make([]tsm1.IntegerValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) values, err := c.ReadIntegerBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -359,7 +360,7 @@ func TestFileStore_SeekToAsc_BeforeStart_OverlapUnsigned(t *testing.T) { // Search for an entry that exists in the second file buf := make([]tsm1.UnsignedValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) values, err := c.ReadUnsignedBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -425,7 +426,7 @@ func TestFileStore_SeekToAsc_BeforeStart_OverlapBoolean(t *testing.T) { // Search for an entry that exists in the second file buf := make([]tsm1.BooleanValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) values, err := c.ReadBooleanBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -491,7 +492,7 @@ func TestFileStore_SeekToAsc_BeforeStart_OverlapString(t *testing.T) { // Search for an entry that exists in the second file buf := make([]tsm1.StringValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) values, err := c.ReadStringBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -556,7 +557,7 @@ func TestFileStore_SeekToAsc_OverlapMinFloat(t *testing.T) { fs.Replace(nil, files) buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) // Search for an entry that exists in the second file values, err := c.ReadFloatBlock(&buf) if err != nil { @@ -636,7 +637,7 @@ func TestFileStore_SeekToAsc_OverlapMinInteger(t *testing.T) { fs.Replace(nil, files) buf := make([]tsm1.IntegerValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) // Search for an entry that exists in the second file values, err := c.ReadIntegerBlock(&buf) if err != nil { @@ -715,7 +716,7 @@ func TestFileStore_SeekToAsc_OverlapMinUnsigned(t *testing.T) { fs.Replace(nil, files) buf := make([]tsm1.UnsignedValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) // Search for an entry that exists in the second file values, err := c.ReadUnsignedBlock(&buf) if err != nil { @@ -794,7 +795,7 @@ func TestFileStore_SeekToAsc_OverlapMinBoolean(t *testing.T) { fs.Replace(nil, files) buf := make([]tsm1.BooleanValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) // Search for an entry that exists in the second file values, err := c.ReadBooleanBlock(&buf) if err != nil { @@ -873,7 +874,7 @@ func TestFileStore_SeekToAsc_OverlapMinString(t *testing.T) { fs.Replace(nil, files) buf := make([]tsm1.StringValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) // Search for an entry that exists in the second file values, err := c.ReadStringBlock(&buf) if err != nil { @@ -951,7 +952,7 @@ func TestFileStore_SeekToAsc_Middle(t *testing.T) { // Search for an entry that exists in the second file buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 3, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 3, true) values, err := c.ReadFloatBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -1007,7 +1008,7 @@ func TestFileStore_SeekToAsc_End(t *testing.T) { fs.Replace(nil, files) buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 2, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 2, true) values, err := c.ReadFloatBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -1046,7 +1047,7 @@ func TestFileStore_SeekToDesc_FromStart(t *testing.T) { // Search for an entry that exists in the second file buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, false) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, false) values, err := c.ReadFloatBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -1085,7 +1086,7 @@ func TestFileStore_SeekToDesc_Duplicate(t *testing.T) { // Search for an entry that exists in the second file buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 2, false) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 2, false) values, err := c.ReadFloatBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -1144,7 +1145,7 @@ func TestFileStore_SeekToDesc_OverlapMaxFloat(t *testing.T) { // Search for an entry that exists in the second file buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 5, false) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 5, false) values, err := c.ReadFloatBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -1209,7 +1210,7 @@ func TestFileStore_SeekToDesc_OverlapMaxInteger(t *testing.T) { // Search for an entry that exists in the second file buf := make([]tsm1.IntegerValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 5, false) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 5, false) values, err := c.ReadIntegerBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -1271,7 +1272,7 @@ func TestFileStore_SeekToDesc_OverlapMaxUnsigned(t *testing.T) { // Search for an entry that exists in the second file buf := make([]tsm1.UnsignedValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 5, false) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 5, false) values, err := c.ReadUnsignedBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -1334,7 +1335,7 @@ func TestFileStore_SeekToDesc_OverlapMaxBoolean(t *testing.T) { // Search for an entry that exists in the second file buf := make([]tsm1.BooleanValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 5, false) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 5, false) values, err := c.ReadBooleanBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -1397,7 +1398,7 @@ func TestFileStore_SeekToDesc_OverlapMaxString(t *testing.T) { // Search for an entry that exists in the second file buf := make([]tsm1.StringValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 5, false) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 5, false) values, err := c.ReadStringBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -1458,7 +1459,7 @@ func TestFileStore_SeekToDesc_AfterEnd(t *testing.T) { fs.Replace(nil, files) buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 4, false) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 4, false) values, err := c.ReadFloatBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -1497,7 +1498,7 @@ func TestFileStore_SeekToDesc_AfterEnd_OverlapFloat(t *testing.T) { fs.Replace(nil, files) buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 10, false) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 10, false) values, err := c.ReadFloatBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -1594,7 +1595,7 @@ func TestFileStore_SeekToDesc_AfterEnd_OverlapInteger(t *testing.T) { fs.Replace(nil, files) buf := make([]tsm1.IntegerValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 11, false) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 11, false) values, err := c.ReadIntegerBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -1671,7 +1672,7 @@ func TestFileStore_SeekToDesc_AfterEnd_OverlapUnsigned(t *testing.T) { fs.Replace(nil, files) buf := make([]tsm1.UnsignedValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 11, false) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 11, false) values, err := c.ReadUnsignedBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -1748,7 +1749,7 @@ func TestFileStore_SeekToDesc_AfterEnd_OverlapBoolean(t *testing.T) { fs.Replace(nil, files) buf := make([]tsm1.BooleanValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 11, false) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 11, false) values, err := c.ReadBooleanBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -1845,7 +1846,7 @@ func TestFileStore_SeekToDesc_AfterEnd_OverlapString(t *testing.T) { fs.Replace(nil, files) buf := make([]tsm1.StringValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 11, false) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 11, false) values, err := c.ReadStringBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -1945,7 +1946,7 @@ func TestFileStore_SeekToDesc_Middle(t *testing.T) { // Search for an entry that exists in the second file buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 3, false) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 3, false) values, err := c.ReadFloatBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -2018,7 +2019,7 @@ func TestFileStore_SeekToDesc_End(t *testing.T) { fs.Replace(nil, files) buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 2, false) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 2, false) values, err := c.ReadFloatBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -2060,7 +2061,7 @@ func TestKeyCursor_TombstoneRange(t *testing.T) { } buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) expValues := []int{0, 2} for _, v := range expValues { values, err := c.ReadFloatBlock(&buf) @@ -2105,7 +2106,7 @@ func TestKeyCursor_TombstoneRange_PartialFloat(t *testing.T) { } buf := make([]tsm1.FloatValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) values, err := c.ReadFloatBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -2149,7 +2150,7 @@ func TestKeyCursor_TombstoneRange_PartialInteger(t *testing.T) { } buf := make([]tsm1.IntegerValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) values, err := c.ReadIntegerBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -2193,7 +2194,7 @@ func TestKeyCursor_TombstoneRange_PartialUnsigned(t *testing.T) { } buf := make([]tsm1.UnsignedValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) values, err := c.ReadUnsignedBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -2237,7 +2238,7 @@ func TestKeyCursor_TombstoneRange_PartialString(t *testing.T) { } buf := make([]tsm1.StringValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) values, err := c.ReadStringBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -2281,7 +2282,7 @@ func TestKeyCursor_TombstoneRange_PartialBoolean(t *testing.T) { } buf := make([]tsm1.BooleanValue, 1000) - c := fs.KeyCursor([]byte("cpu"), 0, true) + c := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) values, err := c.ReadBooleanBlock(&buf) if err != nil { t.Fatalf("unexpected error reading values: %v", err) @@ -2403,7 +2404,7 @@ func TestFileStore_Replace(t *testing.T) { } // Should record references to the two existing TSM files - cur := fs.KeyCursor([]byte("cpu"), 0, true) + cur := fs.KeyCursor(context.Background(), []byte("cpu"), 0, true) // Should move the existing files out of the way, but allow query to complete if err := fs.Replace(files[:2], []string{replacement}); err != nil { diff --git a/tsdb/engine/tsm1/iterator.gen.go b/tsdb/engine/tsm1/iterator.gen.go index baacd08e817..87ec43ca3a3 100644 --- a/tsdb/engine/tsm1/iterator.gen.go +++ b/tsdb/engine/tsm1/iterator.gen.go @@ -13,6 +13,9 @@ import ( "sync" "github.com/influxdata/influxdb/influxql" + "github.com/influxdata/influxdb/pkg/metrics" + "github.com/influxdata/influxdb/pkg/tracing" + "github.com/influxdata/influxdb/pkg/tracing/fields" "github.com/influxdata/influxdb/query" "github.com/influxdata/influxdb/tsdb" "github.com/uber-go/zap" @@ -142,6 +145,36 @@ func (itr *floatFinalizerIterator) Close() error { return itr.FloatIterator.Close() } +type floatInstrumentedIterator struct { + query.FloatIterator + span *tracing.Span + group *metrics.Group +} + +func newFloatInstrumentedIterator(inner query.FloatIterator, span *tracing.Span, group *metrics.Group) *floatInstrumentedIterator { + return &floatInstrumentedIterator{FloatIterator: inner, span: span, group: group} +} + +func (itr *floatInstrumentedIterator) Close() error { + var f []fields.Field + itr.group.ForEach(func(v metrics.Metric) { + switch m := v.(type) { + case *metrics.Counter: + f = append(f, fields.Int64(m.Name(), m.Value())) + + case *metrics.Timer: + f = append(f, fields.Duration(m.Name(), m.Value())) + + default: + panic("unexpected metrics") + } + }) + itr.span.SetFields(fields.New(f...)) + itr.span.Finish() + + return itr.FloatIterator.Close() +} + type floatIterator struct { cur floatCursor aux []cursorAt @@ -575,6 +608,36 @@ func (itr *integerFinalizerIterator) Close() error { return itr.IntegerIterator.Close() } +type integerInstrumentedIterator struct { + query.IntegerIterator + span *tracing.Span + group *metrics.Group +} + +func newIntegerInstrumentedIterator(inner query.IntegerIterator, span *tracing.Span, group *metrics.Group) *integerInstrumentedIterator { + return &integerInstrumentedIterator{IntegerIterator: inner, span: span, group: group} +} + +func (itr *integerInstrumentedIterator) Close() error { + var f []fields.Field + itr.group.ForEach(func(v metrics.Metric) { + switch m := v.(type) { + case *metrics.Counter: + f = append(f, fields.Int64(m.Name(), m.Value())) + + case *metrics.Timer: + f = append(f, fields.Duration(m.Name(), m.Value())) + + default: + panic("unexpected metrics") + } + }) + itr.span.SetFields(fields.New(f...)) + itr.span.Finish() + + return itr.IntegerIterator.Close() +} + type integerIterator struct { cur integerCursor aux []cursorAt @@ -1008,6 +1071,36 @@ func (itr *unsignedFinalizerIterator) Close() error { return itr.UnsignedIterator.Close() } +type unsignedInstrumentedIterator struct { + query.UnsignedIterator + span *tracing.Span + group *metrics.Group +} + +func newUnsignedInstrumentedIterator(inner query.UnsignedIterator, span *tracing.Span, group *metrics.Group) *unsignedInstrumentedIterator { + return &unsignedInstrumentedIterator{UnsignedIterator: inner, span: span, group: group} +} + +func (itr *unsignedInstrumentedIterator) Close() error { + var f []fields.Field + itr.group.ForEach(func(v metrics.Metric) { + switch m := v.(type) { + case *metrics.Counter: + f = append(f, fields.Int64(m.Name(), m.Value())) + + case *metrics.Timer: + f = append(f, fields.Duration(m.Name(), m.Value())) + + default: + panic("unexpected metrics") + } + }) + itr.span.SetFields(fields.New(f...)) + itr.span.Finish() + + return itr.UnsignedIterator.Close() +} + type unsignedIterator struct { cur unsignedCursor aux []cursorAt @@ -1441,6 +1534,36 @@ func (itr *stringFinalizerIterator) Close() error { return itr.StringIterator.Close() } +type stringInstrumentedIterator struct { + query.StringIterator + span *tracing.Span + group *metrics.Group +} + +func newStringInstrumentedIterator(inner query.StringIterator, span *tracing.Span, group *metrics.Group) *stringInstrumentedIterator { + return &stringInstrumentedIterator{StringIterator: inner, span: span, group: group} +} + +func (itr *stringInstrumentedIterator) Close() error { + var f []fields.Field + itr.group.ForEach(func(v metrics.Metric) { + switch m := v.(type) { + case *metrics.Counter: + f = append(f, fields.Int64(m.Name(), m.Value())) + + case *metrics.Timer: + f = append(f, fields.Duration(m.Name(), m.Value())) + + default: + panic("unexpected metrics") + } + }) + itr.span.SetFields(fields.New(f...)) + itr.span.Finish() + + return itr.StringIterator.Close() +} + type stringIterator struct { cur stringCursor aux []cursorAt @@ -1874,6 +1997,36 @@ func (itr *booleanFinalizerIterator) Close() error { return itr.BooleanIterator.Close() } +type booleanInstrumentedIterator struct { + query.BooleanIterator + span *tracing.Span + group *metrics.Group +} + +func newBooleanInstrumentedIterator(inner query.BooleanIterator, span *tracing.Span, group *metrics.Group) *booleanInstrumentedIterator { + return &booleanInstrumentedIterator{BooleanIterator: inner, span: span, group: group} +} + +func (itr *booleanInstrumentedIterator) Close() error { + var f []fields.Field + itr.group.ForEach(func(v metrics.Metric) { + switch m := v.(type) { + case *metrics.Counter: + f = append(f, fields.Int64(m.Name(), m.Value())) + + case *metrics.Timer: + f = append(f, fields.Duration(m.Name(), m.Value())) + + default: + panic("unexpected metrics") + } + }) + itr.span.SetFields(fields.New(f...)) + itr.span.Finish() + + return itr.BooleanIterator.Close() +} + type booleanIterator struct { cur booleanCursor aux []cursorAt diff --git a/tsdb/engine/tsm1/iterator.gen.go.tmpl b/tsdb/engine/tsm1/iterator.gen.go.tmpl index 3ac81cfe3c6..5805cc400cd 100644 --- a/tsdb/engine/tsm1/iterator.gen.go.tmpl +++ b/tsdb/engine/tsm1/iterator.gen.go.tmpl @@ -7,6 +7,9 @@ import ( "sync" "github.com/influxdata/influxdb/influxql" + "github.com/influxdata/influxdb/pkg/metrics" + "github.com/influxdata/influxdb/pkg/tracing" + "github.com/influxdata/influxdb/pkg/tracing/field" "github.com/influxdata/influxdb/query" "github.com/influxdata/influxdb/tsdb" "github.com/uber-go/zap" @@ -138,6 +141,38 @@ func (itr *{{.name}}FinalizerIterator) Close() error { return itr.{{.Name}}Iterator.Close() } + +type {{.name}}InstrumentedIterator struct { + query.{{.Name}}Iterator + span *tracing.Span + group *metrics.Group +} + +func new{{.Name}}InstrumentedIterator(inner query.{{.Name}}Iterator, span *tracing.Span, group *metrics.Group) *{{.name}}InstrumentedIterator { + return &{{.name}}InstrumentedIterator{ {{.Name}}Iterator: inner, span: span, group: group} +} + +func (itr *{{.name}}InstrumentedIterator) Close() error { + var f []field.Field + itr.group.ForEach(func(v metrics.Metric) { + switch m := v.(type) { + case *metrics.Counter: + f = append(f, field.Int64(m.Name(), m.Value())) + + case *metrics.Timer: + f = append(f, field.Duration(m.Name(), m.Value())) + + default: + panic("unexpected metrics") + } + }) + itr.span.SetFieldSet(field.Fields(f...)) + itr.span.Finish() + + return itr.{{.Name}}Iterator.Close() +} + + type {{.name}}Iterator struct { cur {{.name}}Cursor aux []cursorAt diff --git a/tsdb/engine/tsm1/iterator.go b/tsdb/engine/tsm1/iterator.go index e2375215c99..e0fce2d384f 100644 --- a/tsdb/engine/tsm1/iterator.go +++ b/tsdb/engine/tsm1/iterator.go @@ -1,8 +1,11 @@ package tsm1 import ( + "context" "fmt" + "github.com/influxdata/influxdb/pkg/metrics" + "github.com/influxdata/influxdb/pkg/tracing" "github.com/influxdata/influxdb/query" "github.com/influxdata/influxdb/tsdb" "github.com/uber-go/zap" @@ -153,13 +156,13 @@ func (c cursorsAt) close() { // newMergeFinalizerIterator creates a new Merge iterator from the inputs. If the call to Merge succeeds, // the resulting Iterator will be wrapped in a finalizer iterator. // If Merge returns an error, the inputs will be closed. -func newMergeFinalizerIterator(inputs []query.Iterator, opt query.IteratorOptions, log zap.Logger) (query.Iterator, error) { +func newMergeFinalizerIterator(ctx context.Context, inputs []query.Iterator, opt query.IteratorOptions, log zap.Logger) (query.Iterator, error) { itr, err := query.Iterators(inputs).Merge(opt) if err != nil { query.Iterators(inputs).Close() return nil, err } - return newFinalizerIterator(itr, log), nil + return newInstrumentedIterator(ctx, newFinalizerIterator(itr, log)), nil } // newFinalizerIterator creates a new iterator that installs a runtime finalizer @@ -186,3 +189,30 @@ func newFinalizerIterator(itr query.Iterator, log zap.Logger) query.Iterator { panic(fmt.Sprintf("unsupported finalizer iterator type: %T", itr)) } } + +func newInstrumentedIterator(ctx context.Context, itr query.Iterator) query.Iterator { + if itr == nil { + return nil + } + + span := tracing.SpanFromContext(ctx) + grp := metrics.GroupFromContext(ctx) + if span == nil || grp == nil { + return itr + } + + switch inner := itr.(type) { + case query.FloatIterator: + return newFloatInstrumentedIterator(inner, span, grp) + case query.IntegerIterator: + return newIntegerInstrumentedIterator(inner, span, grp) + case query.UnsignedIterator: + return newUnsignedInstrumentedIterator(inner, span, grp) + case query.StringIterator: + return newStringInstrumentedIterator(inner, span, grp) + case query.BooleanIterator: + return newBooleanInstrumentedIterator(inner, span, grp) + default: + panic(fmt.Sprintf("unsupported instrumented iterator type: %T", itr)) + } +} diff --git a/tsdb/shard.go b/tsdb/shard.go index 35ec90d55d1..e723938af36 100644 --- a/tsdb/shard.go +++ b/tsdb/shard.go @@ -2,6 +2,7 @@ package tsdb import ( "bytes" + "context" "errors" "fmt" "io" @@ -817,19 +818,18 @@ func (s *Shard) WriteTo(w io.Writer) (int64, error) { } // CreateIterator returns an iterator for the data in the shard. -func (s *Shard) CreateIterator(measurement string, opt query.IteratorOptions) (query.Iterator, error) { +func (s *Shard) CreateIterator(ctx context.Context, measurement string, opt query.IteratorOptions) (query.Iterator, error) { engine, err := s.engine() if err != nil { return nil, err } - if strings.HasPrefix(measurement, "_") { if itr, ok, err := s.createSystemIterator(engine, measurement, opt); ok { return itr, err } // Unknown system source so pass this to the engine. } - return engine.CreateIterator(measurement, opt) + return engine.CreateIterator(ctx, measurement, opt) } // createSystemIterator returns an iterator for a field of system source. @@ -1164,7 +1164,7 @@ type ShardGroup interface { MeasurementsByRegex(re *regexp.Regexp) []string FieldDimensions(measurements []string) (fields map[string]influxql.DataType, dimensions map[string]struct{}, err error) MapType(measurement, field string) influxql.DataType - CreateIterator(measurement string, opt query.IteratorOptions) (query.Iterator, error) + CreateIterator(ctx context.Context, measurement string, opt query.IteratorOptions) (query.Iterator, error) IteratorCost(measurement string, opt query.IteratorOptions) (query.IteratorCost, error) ExpandSources(sources influxql.Sources) (influxql.Sources, error) } @@ -1245,10 +1245,10 @@ func (a Shards) MapType(measurement, field string) influxql.DataType { return typ } -func (a Shards) CreateIterator(measurement string, opt query.IteratorOptions) (query.Iterator, error) { +func (a Shards) CreateIterator(ctx context.Context, measurement string, opt query.IteratorOptions) (query.Iterator, error) { itrs := make([]query.Iterator, 0, len(a)) for _, sh := range a { - itr, err := sh.CreateIterator(measurement, opt) + itr, err := sh.CreateIterator(ctx, measurement, opt) if err != nil { query.Iterators(itrs).Close() return nil, err diff --git a/tsdb/shard_test.go b/tsdb/shard_test.go index 399fa88d154..939a48145a6 100644 --- a/tsdb/shard_test.go +++ b/tsdb/shard_test.go @@ -1,6 +1,7 @@ package tsdb_test import ( + "context" "fmt" "io/ioutil" "os" @@ -464,7 +465,7 @@ func TestShard_WritePoints_FieldConflictConcurrentQuery(t *testing.T) { sh.WritePoints(points) - iter, err := sh.CreateIterator("cpu", query.IteratorOptions{ + iter, err := sh.CreateIterator(context.Background(), "cpu", query.IteratorOptions{ Expr: influxql.MustParseExpr(`value`), Aux: []influxql.VarRef{{Val: "value"}}, Dimensions: []string{}, @@ -525,7 +526,7 @@ func TestShard_WritePoints_FieldConflictConcurrentQuery(t *testing.T) { sh.WritePoints(points) - iter, err := sh.CreateIterator("cpu", query.IteratorOptions{ + iter, err := sh.CreateIterator(context.Background(), "cpu", query.IteratorOptions{ Expr: influxql.MustParseExpr(`value`), Aux: []influxql.VarRef{{Val: "value"}}, Dimensions: []string{}, @@ -609,7 +610,7 @@ func TestShard_CreateIterator_Ascending(t *testing.T) { // Calling CreateIterator when the engine is not open will return // ErrEngineClosed. - _, got := sh.CreateIterator("cpu", query.IteratorOptions{}) + _, got := sh.CreateIterator(context.Background(), "cpu", query.IteratorOptions{}) if exp := tsdb.ErrEngineClosed; got != exp { t.Fatalf("got %v, expected %v", got, exp) } @@ -626,7 +627,7 @@ cpu,host=serverB,region=uswest value=25 0 // Create iterator. var err error - itr, err = sh.CreateIterator("cpu", query.IteratorOptions{ + itr, err = sh.CreateIterator(context.Background(), "cpu", query.IteratorOptions{ Expr: influxql.MustParseExpr(`value`), Aux: []influxql.VarRef{{Val: "val2"}}, Dimensions: []string{"host"}, @@ -694,7 +695,7 @@ func TestShard_CreateIterator_Descending(t *testing.T) { // Calling CreateIterator when the engine is not open will return // ErrEngineClosed. - _, got := sh.CreateIterator("cpu", query.IteratorOptions{}) + _, got := sh.CreateIterator(context.Background(), "cpu", query.IteratorOptions{}) if exp := tsdb.ErrEngineClosed; got != exp { t.Fatalf("got %v, expected %v", got, exp) } @@ -711,7 +712,7 @@ cpu,host=serverB,region=uswest value=25 0 // Create iterator. var err error - itr, err = sh.CreateIterator("cpu", query.IteratorOptions{ + itr, err = sh.CreateIterator(context.Background(), "cpu", query.IteratorOptions{ Expr: influxql.MustParseExpr(`value`), Aux: []influxql.VarRef{{Val: "val2"}}, Dimensions: []string{"host"}, @@ -795,7 +796,7 @@ func TestShard_Disabled_WriteQuery(t *testing.T) { t.Fatalf(err.Error()) } - _, got := sh.CreateIterator("cpu", query.IteratorOptions{}) + _, got := sh.CreateIterator(context.Background(), "cpu", query.IteratorOptions{}) if err == nil { t.Fatalf("expected shard disabled error") } @@ -810,7 +811,7 @@ func TestShard_Disabled_WriteQuery(t *testing.T) { t.Fatalf("unexpected error: %v", err) } - if _, err = sh.CreateIterator("cpu", query.IteratorOptions{}); err != nil { + if _, err = sh.CreateIterator(context.Background(), "cpu", query.IteratorOptions{}); err != nil { t.Fatalf("unexpected error: %v", got) } } diff --git a/tsdb/store_test.go b/tsdb/store_test.go index eb6f395521f..b780f058531 100644 --- a/tsdb/store_test.go +++ b/tsdb/store_test.go @@ -2,6 +2,7 @@ package tsdb_test import ( "bytes" + "context" "fmt" "io/ioutil" "math" @@ -353,7 +354,7 @@ func TestShards_CreateIterator(t *testing.T) { shards := s.ShardGroup([]uint64{0, 1}) // Create iterator. - itr, err := shards.CreateIterator("cpu", query.IteratorOptions{ + itr, err := shards.CreateIterator(context.Background(), "cpu", query.IteratorOptions{ Expr: influxql.MustParseExpr(`value`), Dimensions: []string{"host"}, Ascending: true, @@ -443,7 +444,7 @@ func TestStore_BackupRestoreShard(t *testing.T) { } // Read data from - itr, err := s0.Shard(100).CreateIterator("cpu", query.IteratorOptions{ + itr, err := s0.Shard(100).CreateIterator(context.Background(), "cpu", query.IteratorOptions{ Expr: influxql.MustParseExpr(`value`), Ascending: true, StartTime: influxql.MinTime,