Merge pull request #9 from jum/extract-gcp

Add additional field extractions for log/slog with slogdriver.

README.md

@@ -135,6 +135,7 @@ The following [log-opts](https://docs.docker.com/config/containers/logging/confi
 | local-logging        | false   | Enables logging to a local file, so logs can be viewed with the `docker logs` command. If false, the command will show no output                                                                                                                                            |
 | extract-severity     | true    | Extracts the `severity` from JSON logs to set them for the log that will be sent to GCP. It will be removed from the jsonPayload section, since it is set at the root level. Currently the supported severity field names to extract are the following: `severity`, `level` |
 | extract-msg          | true    | Extracts the `msg` field from JSON logs to set the `message` field GCP expects. It will be removed from the jsonPayload section, since it is set at the root level. Fields named msg are produced for example by the golang log/slog package.                               |
+| extract-gcp          | false   | Extract trace, labels and source location fields if present and formatted for Google cloud logging. This is produced for example by the golang log/slog package with the slogdriver handler |
 | exclude-timestamp    | false   | Excludes timestamp fields from the final jsonPayload, since docker sends its own nanosecond precision timestamp for each log. Currently it can remove fields with the following names: `timestamp`, `time`, `ts`                                                            |
 | sleep-interval       | 500     | Milliseconds to sleep when there are no logs to send before checking again. The higher the value, the lower the CPU usage will be                                                                                                                                           |
 | credentials-file     |         | Absolute path to the GCP credentials JSON file to use when authenticating (only necessary when running the plugin outside of GCP)                                                                                                                                           |

ngcplogger.go

@@ -5,6 +5,9 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log/slog"
+	"reflect"
+	"runtime"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -15,6 +18,7 @@ import (
 
 	"cloud.google.com/go/compute/metadata"
 	"cloud.google.com/go/logging"
+	"cloud.google.com/go/logging/apiv2/loggingpb"
 	"github.com/containerd/log"
 	mrpb "google.golang.org/genproto/googleapis/api/monitoredres"
 )
@@ -79,6 +83,7 @@ type nGCPLogger struct {
 	extractSeverity    bool
 	excludeTimestamp   bool
 	extractMsg         bool
+	extractGcp         bool
 }
 
 type dockerLogEntry struct {
@@ -204,6 +209,7 @@ func New(info logger.Info) (logger.Logger, error) {
 		extractSeverity:    true,
 		excludeTimestamp:   false,
 		extractMsg:         true,
+		extractGcp:         false,
 	}
 
 	if info.Config[logCmdKey] == "true" {
@@ -222,6 +228,9 @@ func New(info logger.Info) (logger.Logger, error) {
 	if info.Config["extract-msg"] == "false" {
 		l.extractMsg = false
 	}
+	if info.Config["extract-gcp"] == "true" {
+		l.extractGcp = true
+	}
 
 	if instanceResource != nil {
 		l.instance = instanceResource
@@ -264,20 +273,25 @@ func (l *nGCPLogger) Log(lMsg *logger.Message) error {
 
 	if len(logLine) > 0 {
 		var payload any
-		severity := logging.Default
+		entry := logging.Entry{
+			Labels:    map[string]string{},
+			Timestamp: ts,
+			Severity:  logging.Default,
+		}
 
 		if l.extractJsonMessage && logLine[0] == '{' && logLine[len(logLine)-1] == '}' {
 			var m map[string]any
 			err := json.Unmarshal(logLine, &m)
 			if err != nil {
 				payload = fmt.Sprintf("Error parsing JSON: %s", string(logLine))
-				severity = logging.Critical
+				entry.Severity = logging.Critical
 			} else {
-				severity = l.extractSeverityFromPayload(m)
+				entry.Severity = l.extractSeverityFromPayload(m)
 				l.excludeTimestampFromPayload(m)
 				l.extractMsgFromPayload(m)
 				m["instance"] = l.instance
 				m["container"] = l.container
+				l.extractGcpFromPayload(m, &entry)
 				payload = m
 			}
 		} else {
@@ -288,12 +302,8 @@ func (l *nGCPLogger) Log(lMsg *logger.Message) error {
 			}
 		}
 
-		l.logger.Log(logging.Entry{
-			Labels:    map[string]string{},
-			Timestamp: ts,
-			Severity:  severity,
-			Payload:   payload,
-		})
+		entry.Payload = payload
+		l.logger.Log(entry)
 	}
 	return nil
 }
@@ -345,6 +355,54 @@ func (l *nGCPLogger) extractMsgFromPayload(m map[string]any) {
 	}
 }
 
+func assertOrLog[T any](val any) T {
+	var v T
+	var ok bool
+	v, ok = val.(T)
+	if !ok {
+		_, file, line, ok := runtime.Caller(1)
+		if !ok {
+			file = "unknown"
+		}
+		slog.Error("unexpected type", "want", reflect.TypeOf(v).String(), "got", reflect.TypeOf(val).String(), "file", file, "line", line)
+	}
+	return v
+}
+
+func (l *nGCPLogger) extractGcpFromPayload(m map[string]any, entry *logging.Entry) {
+
+	if l.extractGcp {
+		if val, exists := m["logging.googleapis.com/sourceLocation"]; exists {
+			v := assertOrLog[map[string]any](val)
+			entry.SourceLocation = &loggingpb.LogEntrySourceLocation{
+				File:     assertOrLog[string](v["file"]),
+				Line:     int64(assertOrLog[float64](v["line"])),
+				Function: assertOrLog[string](v["function"]),
+			}
+			delete(m, "logging.googleapis.com/sourceLocation")
+		}
+		if val, exists := m["logging.googleapis.com/trace"]; exists {
+			entry.Trace = assertOrLog[string](val)
+			delete(m, "logging.googleapis.com/trace")
+		}
+		if val, exists := m["logging.googleapis.com/spanId"]; exists {
+			entry.SpanID = assertOrLog[string](val)
+			delete(m, "logging.googleapis.com/spanId")
+		}
+		if val, exists := m["logging.googleapis.com/trace_sampled"]; exists {
+			entry.TraceSampled = assertOrLog[bool](val)
+			delete(m, "logging.googleapis.com/trace_sampled")
+		}
+		if val, exists := m["logging.googleapis.com/labels"]; exists {
+			v := assertOrLog[map[string]any](val)
+			for k, v := range v {
+				entry.Labels[k] = assertOrLog[string](v)
+			}
+			delete(m, "logging.googleapis.com/labels")
+		}
+	}
+}
+
 func (l *nGCPLogger) Close() error {
 	err := l.logger.Flush()
 	if err != nil {