From 5df3c191bdc444a266f0cdc62124c510a9d14fe4 Mon Sep 17 00:00:00 2001 From: Andres Smith Date: Sun, 28 Apr 2024 18:59:43 +0200 Subject: [PATCH 1/5] Possible solution for nanoandrew4/ngcplogs#2 --- driver.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/driver.go b/driver.go index ebcc235..7798f1e 100644 --- a/driver.go +++ b/driver.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "fmt" "github.com/containerd/fifo" + "github.com/docker/docker/api/types/backend" "github.com/docker/docker/api/types/plugins/logdriver" "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/logger/jsonfilelog" @@ -139,9 +140,11 @@ func createMessageFromBuffer(buf *logdriver.LogEntry) *logger.Message { msg.Line = buf.Line msg.Source = buf.Source if buf.PartialLogMetadata != nil { - msg.PLogMetaData.ID = buf.PartialLogMetadata.Id - msg.PLogMetaData.Last = buf.PartialLogMetadata.Last - msg.PLogMetaData.Ordinal = int(buf.PartialLogMetadata.Ordinal) + msg.PLogMetaData = &backend.PartialLogMetaData{ + ID: buf.PartialLogMetadata.Id, + Last: buf.PartialLogMetadata.Last, + Ordinal: int(buf.PartialLogMetadata.Ordinal), + } } msg.Timestamp = time.Unix(0, buf.TimeNano) return &msg From 1a1c2c6d1ff7436b7eeec61ba494df423e669a33 Mon Sep 17 00:00:00 2001 From: Jens-Uwe Mager Date: Fri, 26 Apr 2024 19:08:27 +0000 Subject: [PATCH 2/5] Add an extractMsg option to convert msg into message. The short form msg is not recognized by stackdriver logging but used in many go programs, for example any that use log/slog. --- ngcplogger.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ngcplogger.go b/ngcplogger.go index 15eab8d..182d42d 100644 --- a/ngcplogger.go +++ b/ngcplogger.go @@ -77,6 +77,7 @@ type nGCPLogger struct { extractJsonMessage bool extractSeverity bool excludeTimestamp bool + extractMsg bool } type dockerLogEntry struct { @@ -201,6 +202,7 @@ func New(info logger.Info) (logger.Logger, error) { extractJsonMessage: true, extractSeverity: true, excludeTimestamp: false, + extractMsg: true, } if info.Config[logCmdKey] == "true" { @@ -216,6 +218,9 @@ func New(info logger.Info) (logger.Logger, error) { if info.Config["exclude-timestamp"] == "true" { l.excludeTimestamp = true } + if info.Config["extract-msg"] == "false" { + l.extractMsg = false + } if instanceResource != nil { l.instance = instanceResource @@ -269,6 +274,7 @@ func (l *nGCPLogger) Log(lMsg *logger.Message) error { } else { severity = l.extractSeverityFromPayload(m) l.excludeTimestampFromPayload(m) + l.extractMsgFromPayload(m) m["instance"] = l.instance m["container"] = l.container payload = m @@ -328,6 +334,16 @@ func (l *nGCPLogger) excludeTimestampFromPayload(m map[string]any) { } } +func (l *nGCPLogger) extractMsgFromPayload(m map[string]any) { + + if l.extractMsg { + if msg, exists := m["msg"]; exists { + m["message"] = msg + delete(m, "msg") + } + } +} + func (l *nGCPLogger) Close() error { err := l.logger.Flush() if err != nil { From 2dfa0bd061adb60c09ec724a5a8e48fb751c241f Mon Sep 17 00:00:00 2001 From: Jens-Uwe Mager Date: Sat, 27 Apr 2024 23:59:31 +0200 Subject: [PATCH 3/5] Forgot to fmt. --- ngcplogger.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ngcplogger.go b/ngcplogger.go index 182d42d..bfcb395 100644 --- a/ngcplogger.go +++ b/ngcplogger.go @@ -5,11 +5,12 @@ import ( "encoding/json" "errors" "fmt" - "google.golang.org/api/option" "sync" "sync/atomic" "time" + "google.golang.org/api/option" + "github.com/docker/docker/daemon/logger" "cloud.google.com/go/compute/metadata" @@ -77,7 +78,7 @@ type nGCPLogger struct { extractJsonMessage bool extractSeverity bool excludeTimestamp bool - extractMsg bool + extractMsg bool } type dockerLogEntry struct { @@ -202,7 +203,7 @@ func New(info logger.Info) (logger.Logger, error) { extractJsonMessage: true, extractSeverity: true, excludeTimestamp: false, - extractMsg: true, + extractMsg: true, } if info.Config[logCmdKey] == "true" { From b9e8499641dad7d30e814740867c46bf26892f43 Mon Sep 17 00:00:00 2001 From: Jens-Uwe Mager Date: Sun, 28 Apr 2024 18:14:02 +0200 Subject: [PATCH 4/5] Document the extract-msg option. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index dfd6466..ee092a3 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,7 @@ The following [log-opts](https://docs.docker.com/config/containers/logging/confi | extract-json-message | true | Enables unmarshalling JSON messages and sending the jsonPayload as the unmarshalled map. Kind of the whole point of this plugin, but you can disable it so it behaves just like the `gcplogs` plugin if you wish | | local-logging | false | Enables logging to a local file, so logs can be viewed with the `docker logs` command. If false, the command will show no output | | extract-severity | true | Extracts the severity from JSON logs to set them for the log that will be sent to GCP. It will be removed from the jsonPayload section, since it is set at the root level. Currently the supported severity field names to extract are the following: `severity`, `level` | +| extract-msg | true | Extracts the msg field from JSON logs to set the message field GCP expects. It will be removed from the jsonPayload section, since it is set at the root level. Fields named msg are producted for example by the golang log/slog package. | | exclude-timestamp | false | Excludes timestamp fields from the final jsonPayload, since docker sends its own nanosecond precision timestamp for each log. Currently it can remove fields with the following names: `timestamp`, `time`, `ts` | | sleep-interval | 500 | Milliseconds to sleep when there are no logs to send before checking again. The higher the value, the lower the CPU usage will be | | credentials-file | | Absolute path to the GCP credentials JSON file to use when authenticating (only necessary when running the plugin outside of GCP) | @@ -142,4 +143,4 @@ If you want to build the plugin yourself, use the makefile with the following co make all ``` -See the Makefile for other build targets \ No newline at end of file +See the Makefile for other build targets From dc9d1aa47c23f1fd0cef47e08ab87e93b5307241 Mon Sep 17 00:00:00 2001 From: Jens-Uwe Mager Date: Sun, 28 Apr 2024 19:06:31 +0200 Subject: [PATCH 5/5] Fix typo. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ee092a3..9515ac8 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ The following [log-opts](https://docs.docker.com/config/containers/logging/confi | extract-json-message | true | Enables unmarshalling JSON messages and sending the jsonPayload as the unmarshalled map. Kind of the whole point of this plugin, but you can disable it so it behaves just like the `gcplogs` plugin if you wish | | local-logging | false | Enables logging to a local file, so logs can be viewed with the `docker logs` command. If false, the command will show no output | | extract-severity | true | Extracts the severity from JSON logs to set them for the log that will be sent to GCP. It will be removed from the jsonPayload section, since it is set at the root level. Currently the supported severity field names to extract are the following: `severity`, `level` | -| extract-msg | true | Extracts the msg field from JSON logs to set the message field GCP expects. It will be removed from the jsonPayload section, since it is set at the root level. Fields named msg are producted for example by the golang log/slog package. | +| extract-msg | true | Extracts the msg field from JSON logs to set the message field GCP expects. It will be removed from the jsonPayload section, since it is set at the root level. Fields named msg are produced for example by the golang log/slog package. | | exclude-timestamp | false | Excludes timestamp fields from the final jsonPayload, since docker sends its own nanosecond precision timestamp for each log. Currently it can remove fields with the following names: `timestamp`, `time`, `ts` | | sleep-interval | 500 | Milliseconds to sleep when there are no logs to send before checking again. The higher the value, the lower the CPU usage will be | | credentials-file | | Absolute path to the GCP credentials JSON file to use when authenticating (only necessary when running the plugin outside of GCP) |