Skip to content

Commit

Permalink
[chore] translator/loki: remove deprecated code (#23827)
Browse files Browse the repository at this point in the history 
**Description:** <Describe what has changed.>
When did the massive cleaning of the deprecated code in loki translator,
forgot to remove LogsToLoki method

**Link to tracking Issue:** 

#15653
  • Loading branch information
@mar4uk
mar4uk authored Jun 29, 2023
1 parent 0e12ba7 commit 3c934d6
Show file tree
Hide file tree
Showing 2 changed files with 0 additions and 316 deletions.
85 changes: 0 additions & 85 deletions pkg/translator/loki/logs_to_loki.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,91 +200,6 @@ type pushRequestGroup struct {
report *PushReport
}

// LogsToLoki converts a Logs pipeline data into a Loki PushRequest.
// Labels for each record are inferred based on the hints "loki.attribute.labels"
// and "loki.resource.labels". Each hint might contain a comma-separated list of
// attributes (resource or record) that should be promoted to a Loki label. Those
// attributes are removed from the body as a result, otherwise they would be shown
// in duplicity in Loki.
// PushStreams are created based on the labels: all records containing the same
// set of labels are part of the same stream. All streams are then packed within
// the resulting PushRequest.
// When this function isn't able to marshal a log record, the log record is dropped
// and processing continues, so that the caller can decide to either skip the entire
// batch or send only the data that could be parsed. The caller can use the PushReport
// to make this decision, as it includes all of the errors that were encountered,
// as well as the number of items dropped and submitted.
// Deprecated: [v0.62.0] will be removed after v0.63.0. Use LogsToLokiRequests instead.
func LogsToLoki(ld plog.Logs) (*push.PushRequest, *PushReport) {
report := &PushReport{}

streams := make(map[string]*push.Stream)
rls := ld.ResourceLogs()
for i := 0; i < rls.Len(); i++ {
ills := rls.At(i).ScopeLogs()

for j := 0; j < ills.Len(); j++ {
scope := ills.At(j).Scope()
logs := ills.At(j).LogRecords()
for k := 0; k < logs.Len(); k++ {

// similarly, we may remove attributes, so change only our version
log := plog.NewLogRecord()
logs.At(k).CopyTo(log)

// we may remove attributes, so we make a copy and change our version
resource := pcommon.NewResource()
rls.At(i).Resource().CopyTo(resource)

// adds level attribute from log.severityNumber
addLogLevelAttributeAndHint(log)

format := getFormatFromFormatHint(log.Attributes(), resource.Attributes())

mergedLabels := convertAttributesAndMerge(log.Attributes(), resource.Attributes())
// remove the attributes that were promoted to labels
removeAttributes(log.Attributes(), mergedLabels)
removeAttributes(resource.Attributes(), mergedLabels)

// create the stream name based on the labels
labels := mergedLabels.String()

entry, err := convertLogToLokiEntry(log, resource, format, scope)
if err != nil {
// Couldn't convert so dropping log.
report.Errors = append(report.Errors, fmt.Errorf("failed to convert, dropping log: %w", err))
report.NumDropped++
continue
}

report.NumSubmitted++

if stream, ok := streams[labels]; ok {
stream.Entries = append(stream.Entries, *entry)
continue
}

streams[labels] = &push.Stream{
Labels: labels,
Entries: []push.Entry{*entry},
}
}
}
}

pr := &push.PushRequest{
Streams: make([]push.Stream, len(streams)),
}

i := 0
for _, stream := range streams {
pr.Streams[i] = *stream
i++
}

return pr, report
}

func addLogLevelAttributeAndHint(log plog.LogRecord) {
if log.SeverityNumber() == plog.SeverityNumberUnspecified {
return
Expand Down
231 changes: 0 additions & 231 deletions pkg/translator/loki/logs_to_loki_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,237 +389,6 @@ func TestLogsToLokiRequestWithoutTenant(t *testing.T) {
}
}

func TestLogsToLoki(t *testing.T) {
testCases := []struct {
desc string
hints map[string]interface{}
attrs map[string]interface{}
res map[string]interface{}
severity plog.SeverityNumber
instrumentationScope *instrumentationScope
levelAttribute string
expectedLabel string
expectedLines []string
}{
{
desc: "with attribute to label and regular attribute",
attrs: map[string]interface{}{
"host.name": "guarana",
"http.status": 200,
},
hints: map[string]interface{}{
hintAttributes: "host.name",
},
expectedLabel: `{exporter="OTLP", host.name="guarana"}`,
expectedLines: []string{
`{"traceid":"01020304000000000000000000000000","attributes":{"http.status":200}}`,
`{"traceid":"01020304050000000000000000000000","attributes":{"http.status":200}}`,
`{"traceid":"01020304050600000000000000000000","attributes":{"http.status":200}}`,
},
},
{
desc: "with resource to label and regular resource",
res: map[string]interface{}{
"host.name": "guarana",
"region.az": "eu-west-1a",
},
hints: map[string]interface{}{
hintResources: "host.name",
},
expectedLabel: `{exporter="OTLP", host.name="guarana"}`,
expectedLines: []string{
`{"traceid":"01020304000000000000000000000000","resources":{"region.az":"eu-west-1a"}}`,
`{"traceid":"01020304050000000000000000000000","resources":{"region.az":"eu-west-1a"}}`,
`{"traceid":"01020304050600000000000000000000","resources":{"region.az":"eu-west-1a"}}`,
},
},
{
desc: "with logfmt format",
res: map[string]interface{}{
"host.name": "guarana",
"region.az": "eu-west-1a",
},
hints: map[string]interface{}{
hintResources: "host.name",
hintFormat: formatLogfmt,
},
expectedLabel: `{exporter="OTLP", host.name="guarana"}`,
expectedLines: []string{
`traceID=01020304000000000000000000000000 resource_region.az=eu-west-1a`,
`traceID=01020304050000000000000000000000 resource_region.az=eu-west-1a`,
`traceID=01020304050600000000000000000000 resource_region.az=eu-west-1a`,
},
},
{
desc: "with severity to label",
severity: plog.SeverityNumberDebug4,
expectedLabel: `{exporter="OTLP", level="DEBUG4"}`,
expectedLines: []string{
`{"traceid":"01020304000000000000000000000000"}`,
`{"traceid":"01020304050000000000000000000000"}`,
`{"traceid":"01020304050600000000000000000000"}`,
},
},
{
desc: "with severity, already existing level",
severity: plog.SeverityNumberDebug4,
levelAttribute: "dummy",
expectedLabel: `{exporter="OTLP", level="dummy"}`,
expectedLines: []string{
`{"traceid":"01020304000000000000000000000000"}`,
`{"traceid":"01020304050000000000000000000000"}`,
`{"traceid":"01020304050600000000000000000000"}`,
},
},
{
desc: "with instrumentation_scope contains name",
instrumentationScope: &instrumentationScope{
Name: "example-name",
},
expectedLabel: `{exporter="OTLP"}`,
expectedLines: []string{
`{"traceid":"01020304000000000000000000000000","instrumentation_scope":{"name":"example-name"}}`,
`{"traceid":"01020304050000000000000000000000","instrumentation_scope":{"name":"example-name"}}`,
`{"traceid":"01020304050600000000000000000000","instrumentation_scope":{"name":"example-name"}}`,
},
},
{
desc: "with instrumentation_scope contains name and version",
instrumentationScope: &instrumentationScope{
Name: "example-name",
Version: "v1",
},
expectedLabel: `{exporter="OTLP"}`,
expectedLines: []string{
`{"traceid":"01020304000000000000000000000000","instrumentation_scope":{"name":"example-name","version":"v1"}}`,
`{"traceid":"01020304050000000000000000000000","instrumentation_scope":{"name":"example-name","version":"v1"}}`,
`{"traceid":"01020304050600000000000000000000","instrumentation_scope":{"name":"example-name","version":"v1"}}`,
},
},
{
desc: "with instrumentation_scope contains only version",
instrumentationScope: &instrumentationScope{
Version: "v1",
},
expectedLabel: `{exporter="OTLP"}`,
expectedLines: []string{
`{"traceid":"01020304000000000000000000000000"}`,
`{"traceid":"01020304050000000000000000000000"}`,
`{"traceid":"01020304050600000000000000000000"}`,
},
},
{
desc: "with instrumentation_scope contains name and with logfmt format",
instrumentationScope: &instrumentationScope{
Name: "example-name",
},
hints: map[string]interface{}{
hintFormat: formatLogfmt,
},
expectedLabel: `{exporter="OTLP"}`,
expectedLines: []string{
`traceID=01020304000000000000000000000000 instrumentation_scope_name=example-name`,
`traceID=01020304050000000000000000000000 instrumentation_scope_name=example-name`,
`traceID=01020304050600000000000000000000 instrumentation_scope_name=example-name`,
},
},
{
desc: "with instrumentation_scope contains name and version with logfmt format",
instrumentationScope: &instrumentationScope{
Name: "example-name",
Version: "v1",
},
hints: map[string]interface{}{
hintFormat: formatLogfmt,
},
expectedLabel: `{exporter="OTLP"}`,
expectedLines: []string{
`traceID=01020304000000000000000000000000 instrumentation_scope_name=example-name instrumentation_scope_version=v1`,
`traceID=01020304050000000000000000000000 instrumentation_scope_name=example-name instrumentation_scope_version=v1`,
`traceID=01020304050600000000000000000000 instrumentation_scope_name=example-name instrumentation_scope_version=v1`,
},
},
{
desc: "with instrumentation_scope contains only version with logfmt format",
instrumentationScope: &instrumentationScope{
Version: "v1",
},
hints: map[string]interface{}{
hintFormat: formatLogfmt,
},
expectedLabel: `{exporter="OTLP"}`,
expectedLines: []string{
`traceID=01020304000000000000000000000000`,
`traceID=01020304050000000000000000000000`,
`traceID=01020304050600000000000000000000`,
},
},
}
for _, tC := range testCases {
t.Run(tC.desc, func(t *testing.T) {
// prepare
ld := plog.NewLogs()
ld.ResourceLogs().AppendEmpty()
ld.ResourceLogs().At(0).ScopeLogs().AppendEmpty()
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().AppendEmpty()
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().AppendEmpty()
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().AppendEmpty()
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).SetSeverityNumber(tC.severity)
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).SetTraceID(pcommon.TraceID([16]byte{1, 2, 3, 4}))
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).SetSeverityNumber(tC.severity)
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).SetTraceID(pcommon.TraceID([16]byte{1, 2, 3, 4, 5}))
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(2).SetSeverityNumber(tC.severity)
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(2).SetTraceID(pcommon.TraceID([16]byte{1, 2, 3, 4, 5, 6}))

// copy the attributes from the test case to the log entry
if len(tC.attrs) > 0 {
assert.NoError(t, ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().FromRaw(tC.attrs))
assert.NoError(t, ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().FromRaw(tC.attrs))
assert.NoError(t, ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(2).Attributes().FromRaw(tC.attrs))
}
if len(tC.res) > 0 {
assert.NoError(t, ld.ResourceLogs().At(0).Resource().Attributes().FromRaw(tC.res))
}
if len(tC.levelAttribute) > 0 {
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutStr(levelAttributeName, tC.levelAttribute)
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().PutStr(levelAttributeName, tC.levelAttribute)
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(2).Attributes().PutStr(levelAttributeName, tC.levelAttribute)
}

if tC.instrumentationScope != nil {
ld.ResourceLogs().At(0).ScopeLogs().At(0).Scope().SetName(tC.instrumentationScope.Name)
ld.ResourceLogs().At(0).ScopeLogs().At(0).Scope().SetVersion(tC.instrumentationScope.Version)
}

// we can't use copy here, as the value (Value) will be used as string lookup later, so, we need to convert it to string now
for k, v := range tC.hints {
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutStr(k, fmt.Sprintf("%v", v))
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().PutStr(k, fmt.Sprintf("%v", v))
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(2).Attributes().PutStr(k, fmt.Sprintf("%v", v))
}

// test
pushRequest, report := LogsToLoki(ld)
entries := pushRequest.Streams[0].Entries

var entriesLines []string
for i := 0; i < len(entries); i++ {
entriesLines = append(entriesLines, entries[i].Line)
}

// actualPushRequest is populated within the test http server, we check it here as assertions are better done at the
// end of the test function
assert.Empty(t, report.Errors)
assert.Equal(t, 0, report.NumDropped)
assert.Equal(t, ld.LogRecordCount(), report.NumSubmitted)
assert.Len(t, pushRequest.Streams, 1)
assert.Equal(t, tC.expectedLabel, pushRequest.Streams[0].Labels)
assert.Len(t, entries, ld.LogRecordCount())
assert.ElementsMatch(t, tC.expectedLines, entriesLines)
})
}
}

func TestLogToLokiEntry(t *testing.T) {
testCases := []struct {
name string
Expand Down

0 comments on commit 3c934d6

Please sign in to comment.