Skip to content

Commit

Permalink
Pool otlploghttp transform maps (#5378)
Browse files Browse the repository at this point in the history
Part of #5196 

### Benchmarks

```console
goos: linux
goarch: amd64
pkg: go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp/internal/transform
cpu: Intel(R) Core(TM) i7-8550U CPU @ 1.80GHz
               │   old.txt   │               new.txt               │
               │   sec/op    │   sec/op     vs base                │
ResourceLogs-8   6.033µ ± 4%   5.249µ ± 8%  -13.00% (p=0.000 n=10)

               │    old.txt    │               new.txt                │
               │     B/op      │     B/op      vs base                │
ResourceLogs-8   10.602Ki ± 0%   8.299Ki ± 0%  -21.72% (p=0.000 n=10)

               │  old.txt   │              new.txt              │
               │ allocs/op  │ allocs/op   vs base               │
ResourceLogs-8   188.0 ± 0%   178.0 ± 0%  -5.32% (p=0.000 n=10)
```
  • Loading branch information
MrAlias authored May 22, 2024
1 parent 0a7aae7 commit e800298
Show file tree
Hide file tree
Showing 2 changed files with 44 additions and 12 deletions.
45 changes: 33 additions & 12 deletions exporters/otlp/otlplog/otlploghttp/internal/transform/log.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package transform // import "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp/internal/transform"

import (
"sync"
"time"

cpb "go.opentelemetry.io/proto/otlp/common/v1"
Expand All @@ -24,19 +25,30 @@ func ResourceLogs(records []log.Record) []*lpb.ResourceLogs {
return nil
}

resMap := resourceLogsMap(records)
resMap := resourceLogsMapPool.Get().(map[attribute.Distinct]*lpb.ResourceLogs)
defer func() {
clear(resMap)
resourceLogsMapPool.Put(resMap)
}()
resourceLogsMap(&resMap, records)

out := make([]*lpb.ResourceLogs, 0, len(resMap))
for _, rl := range resMap {
out = append(out, rl)
}
return out
}

func resourceLogsMap(records []log.Record) map[attribute.Distinct]*lpb.ResourceLogs {
out := make(map[attribute.Distinct]*lpb.ResourceLogs)
var resourceLogsMapPool = sync.Pool{
New: func() any {
return make(map[attribute.Distinct]*lpb.ResourceLogs)
},
}

func resourceLogsMap(dst *map[attribute.Distinct]*lpb.ResourceLogs, records []log.Record) {
for _, r := range records {
res := r.Resource()
rl, ok := out[res.Equivalent()]
rl, ok := (*dst)[res.Equivalent()]
if !ok {
rl = new(lpb.ResourceLogs)
if res.Len() > 0 {
Expand All @@ -45,28 +57,38 @@ func resourceLogsMap(records []log.Record) map[attribute.Distinct]*lpb.ResourceL
}
}
rl.SchemaUrl = res.SchemaURL()
out[res.Equivalent()] = rl
(*dst)[res.Equivalent()] = rl
}
rl.ScopeLogs = ScopeLogs(records)
}
return out
}

// ScopeLogs returns a slice of OTLP ScopeLogs generated from recoreds.
func ScopeLogs(records []log.Record) []*lpb.ScopeLogs {
scopeMap := scopeLogsMap(records)
scopeMap := scopeLogsMapPool.Get().(map[instrumentation.Scope]*lpb.ScopeLogs)
defer func() {
clear(scopeMap)
scopeLogsMapPool.Put(scopeMap)
}()
scopeLogsMap(&scopeMap, records)

out := make([]*lpb.ScopeLogs, 0, len(scopeMap))
for _, sl := range scopeMap {
out = append(out, sl)
}
return out
}

func scopeLogsMap(records []log.Record) map[instrumentation.Scope]*lpb.ScopeLogs {
out := make(map[instrumentation.Scope]*lpb.ScopeLogs)
var scopeLogsMapPool = sync.Pool{
New: func() any {
return make(map[instrumentation.Scope]*lpb.ScopeLogs)
},
}

func scopeLogsMap(dst *map[instrumentation.Scope]*lpb.ScopeLogs, records []log.Record) {
for _, r := range records {
scope := r.InstrumentationScope()
sl, ok := out[scope]
sl, ok := (*dst)[scope]
if !ok {
sl = new(lpb.ScopeLogs)
var emptyScope instrumentation.Scope
Expand All @@ -77,11 +99,10 @@ func scopeLogsMap(records []log.Record) map[instrumentation.Scope]*lpb.ScopeLogs
}
sl.SchemaUrl = scope.SchemaURL
}
out[scope] = sl
(*dst)[scope] = sl
}
sl.LogRecords = append(sl.LogRecords, LogRecord(r))
}
return out
}

// LogRecord returns an OTLP LogRecord generated from record.
Expand Down
11 changes: 11 additions & 0 deletions exporters/otlp/otlplog/otlploghttp/internal/transform/log_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,3 +230,14 @@ func TestSeverityNumber(t *testing.T) {
assert.Equal(t, want, SeverityNumber(api.Severity(i)))
}
}

func BenchmarkResourceLogs(b *testing.B) {
b.ReportAllocs()
b.RunParallel(func(pb *testing.PB) {
var out []*lpb.ResourceLogs
for pb.Next() {
out = ResourceLogs(records)
}
_ = out
})
}

0 comments on commit e800298

Please sign in to comment.