From 12122de1171fad61ac57e3c845d6ab8c5f01dc6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Thu, 18 Jan 2024 14:24:13 -0500 Subject: [PATCH 1/7] api: loki_config_instance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- doc/api-extensions.md | 6 ++++++ internal/version/api.go | 1 + 2 files changed, 7 insertions(+) diff --git a/doc/api-extensions.md b/doc/api-extensions.md index b8b9b4c8cbf..78cbdf90130 100644 --- a/doc/api-extensions.md +++ b/doc/api-extensions.md @@ -2305,3 +2305,9 @@ Adds a `description` field to certificate. Adds a new `virtio-blk` value for `io.bus` on `disk` devices which allows for the attached disk to be connected to the `virtio-blk` bus. + +## `loki_config_instance` + +Adds a new `loki.instance` server configuration key to customize the `instance` field in Loki events. +This can be used to expose the name of the cluster rather than the individual system name sending +the event as that's usually already covered by the `location` field. diff --git a/internal/version/api.go b/internal/version/api.go index 495098388be..b27a595f931 100644 --- a/internal/version/api.go +++ b/internal/version/api.go @@ -389,6 +389,7 @@ var APIExtensions = []string{ "ovn_ssl_config", "certificate_description", "disk_io_bus_virtio_blk", + "loki_config_instance", } // APIExtensionsCount returns the number of available API extensions. From 709579863dca24b53eef990ae0309abbb6630c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Thu, 18 Jan 2024 14:30:41 -0500 Subject: [PATCH 2/7] incusd/config: Add loki.instance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/cluster/config/config.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/internal/server/cluster/config/config.go b/internal/server/cluster/config/config.go index 740e879fbcb..3bbedd0ab8a 100644 --- a/internal/server/cluster/config/config.go +++ b/internal/server/cluster/config/config.go @@ -183,7 +183,7 @@ func (c *Config) InstancesPlacementScriptlet() string { } // LokiServer returns all the Loki settings needed to connect to a server. -func (c *Config) LokiServer() (string, string, string, string, []string, string, []string) { +func (c *Config) LokiServer() (string, string, string, string, string, string, []string, []string) { var types []string var labels []string @@ -195,7 +195,7 @@ func (c *Config) LokiServer() (string, string, string, string, []string, string, labels = strings.Split(c.m.GetString("loki.labels"), ",") } - return c.m.GetString("loki.api.url"), c.m.GetString("loki.auth.username"), c.m.GetString("loki.auth.password"), c.m.GetString("loki.api.ca_cert"), labels, c.m.GetString("loki.loglevel"), types + return c.m.GetString("loki.api.url"), c.m.GetString("loki.auth.username"), c.m.GetString("loki.auth.password"), c.m.GetString("loki.api.ca_cert"), c.m.GetString("loki.instance"), c.m.GetString("loki.loglevel"), labels, types } // ACME returns all ACME settings needed for certificate renewal. @@ -593,6 +593,15 @@ var ConfigSchema = config.Schema{ // shortdesc: URL to the Loki server "loki.api.url": {}, + // gendoc:generate(entity=server, group=loki, key=loki.instance) + // This allows replacing the default instance value (server host name) by a more relevant value like a cluster identifier. + // --- + // type: string + // scope: global + // defaultdesc: Local server host name or cluster member name + // shortdesc: Name to use as the instance field in Loki events. + "loki.instance": {}, + // gendoc:generate(entity=server, group=loki, key=loki.labels) // Specify a comma-separated list of values that should be used as labels for a Loki log entry. // --- From 291882e0de6ca7bb760009ce6c5caedda3c4d538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Thu, 18 Jan 2024 14:56:58 -0500 Subject: [PATCH 3/7] incusd/loki: Add support for overriding instance name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/loki/loki.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/internal/server/loki/loki.go b/internal/server/loki/loki.go index 3d4c51dd740..c13a843c0f3 100644 --- a/internal/server/loki/loki.go +++ b/internal/server/loki/loki.go @@ -9,7 +9,6 @@ import ( "io" "net/http" "net/url" - "os" "reflect" "sort" "strconv" @@ -38,6 +37,7 @@ type config struct { username string password string labels []string + instance string logLevel string timeout time.Duration types []string @@ -61,7 +61,7 @@ type Client struct { } // NewClient returns a Client. -func NewClient(ctx context.Context, url *url.URL, username string, password string, caCert string, labels []string, logLevel string, types []string) *Client { +func NewClient(ctx context.Context, url *url.URL, username string, password string, caCert string, instance string, logLevel string, labels []string, types []string) *Client { client := Client{ cfg: config{ batchSize: 10 * 1024, @@ -69,6 +69,7 @@ func NewClient(ctx context.Context, url *url.URL, username string, password stri caCert: caCert, username: username, password: password, + instance: instance, labels: labels, logLevel: logLevel, timeout: 10 * time.Second, @@ -227,17 +228,12 @@ func (c *Client) HandleEvent(event api.Event) { return } - hostname, err := os.Hostname() - if err != nil { - hostname = "none" - } - entry := entry{ labels: LabelSet{ "app": "incus", "type": event.Type, "location": event.Location, - "instance": hostname, + "instance": c.cfg.instance, }, Entry: Entry{ Timestamp: event.Timestamp, From 71b7717334a5338ba5553d9cc8e2281e98db5da8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Thu, 18 Jan 2024 14:57:20 -0500 Subject: [PATCH 4/7] incusd: Add support for loki.instance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- cmd/incusd/api_1.0.go | 6 ++++-- cmd/incusd/daemon.go | 18 ++++++++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/cmd/incusd/api_1.0.go b/cmd/incusd/api_1.0.go index 404e09aee1c..d0aa6ad5c39 100644 --- a/cmd/incusd/api_1.0.go +++ b/cmd/incusd/api_1.0.go @@ -868,6 +868,8 @@ func doApi10UpdateTriggers(d *Daemon, nodeChanged, clusterChanged map[string]str fallthrough case "loki.api.ca_cert": fallthrough + case "loki.instance": + fallthrough case "loki.labels": fallthrough case "loki.loglevel": @@ -984,12 +986,12 @@ func doApi10UpdateTriggers(d *Daemon, nodeChanged, clusterChanged map[string]str } if lokiChanged { - lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiLabels, lokiLoglevel, lokiTypes := clusterConfig.LokiServer() + lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiInstance, lokiLoglevel, lokiLabels, lokiTypes := clusterConfig.LokiServer() if lokiURL == "" || lokiLoglevel == "" || len(lokiTypes) == 0 { d.internalListener.RemoveHandler("loki") } else { - err := d.setupLoki(lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiLabels, lokiLoglevel, lokiTypes) + err := d.setupLoki(lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiInstance, lokiLoglevel, lokiLabels, lokiTypes) if err != nil { return err } diff --git a/cmd/incusd/daemon.go b/cmd/incusd/daemon.go index 958933fd310..7b9d109d7be 100644 --- a/cmd/incusd/daemon.go +++ b/cmd/incusd/daemon.go @@ -762,22 +762,32 @@ func (d *Daemon) Init() error { return nil } -func (d *Daemon) setupLoki(URL string, cert string, key string, caCert string, labels []string, logLevel string, types []string) error { +func (d *Daemon) setupLoki(URL string, cert string, key string, caCert string, instanceName string, logLevel string, labels []string, types []string) error { + // Stop any existing loki client. if d.lokiClient != nil { d.lokiClient.Stop() } + // Check basic requirements for starting a new client. if URL == "" || logLevel == "" || len(types) == 0 { return nil } + // Validate the URL. u, err := url.Parse(URL) if err != nil { return err } - d.lokiClient = loki.NewClient(d.shutdownCtx, u, cert, key, caCert, labels, logLevel, types) + // Figure out the instance name. + if instanceName == "" { + instanceName = d.serverName + } + + // Start a new client. + d.lokiClient = loki.NewClient(d.shutdownCtx, u, cert, key, caCert, instanceName, logLevel, labels, types) + // Attach the new client to the log handler. d.internalListener.AddHandler("loki", d.lokiClient.HandleEvent) return nil @@ -1337,7 +1347,7 @@ func (d *Daemon) init() error { d.proxy = proxy.FromConfig(d.globalConfig.ProxyHTTPS(), d.globalConfig.ProxyHTTP(), d.globalConfig.ProxyIgnoreHosts()) d.gateway.HeartbeatOfflineThreshold = d.globalConfig.OfflineThreshold() - lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiLabels, lokiLoglevel, lokiTypes := d.globalConfig.LokiServer() + lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiInstance, lokiLoglevel, lokiLabels, lokiTypes := d.globalConfig.LokiServer() oidcIssuer, oidcClientID, oidcAudience := d.globalConfig.OIDCServer() syslogSocketEnabled := d.localConfig.SyslogSocket() openfgaAPIURL, openfgaAPIToken, openfgaStoreID, openFGAAuthorizationModelID := d.globalConfig.OpenFGA() @@ -1348,7 +1358,7 @@ func (d *Daemon) init() error { // Setup Loki logger. if lokiURL != "" { - err = d.setupLoki(lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiLabels, lokiLoglevel, lokiTypes) + err = d.setupLoki(lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiInstance, lokiLoglevel, lokiLabels, lokiTypes) if err != nil { return err } From dff47f51e98249cd56350f372ae36dcfc95481e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Thu, 18 Jan 2024 14:56:22 -0500 Subject: [PATCH 5/7] doc: Update configs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- doc/config_options.txt | 8 ++++++++ internal/server/metadata/configuration.json | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/doc/config_options.txt b/doc/config_options.txt index 502039a41a5..67da63a8595 100644 --- a/doc/config_options.txt +++ b/doc/config_options.txt @@ -1592,6 +1592,14 @@ Specify the protocol, name or IP and port. For example `https://loki.example.com ``` +```{config:option} loki.instance server-loki +:defaultdesc: "Local server host name or cluster member name" +:scope: "global" +:shortdesc: "Name to use as the instance field in Loki events." +:type: "string" +This allows replacing the default instance value (server host name) by a more relevant value like a cluster identifier. +``` + ```{config:option} loki.labels server-loki :scope: "global" :shortdesc: "Labels for a Loki log entry" diff --git a/internal/server/metadata/configuration.json b/internal/server/metadata/configuration.json index 7af6273e5ef..7dcfe0615a1 100644 --- a/internal/server/metadata/configuration.json +++ b/internal/server/metadata/configuration.json @@ -1745,6 +1745,15 @@ "type": "string" } }, + { + "loki.instance": { + "defaultdesc": "Local server host name or cluster member name", + "longdesc": "This allows replacing the default instance value (server host name) by a more relevant value like a cluster identifier.", + "scope": "global", + "shortdesc": "Name to use as the instance field in Loki events.", + "type": "string" + } + }, { "loki.labels": { "longdesc": "Specify a comma-separated list of values that should be used as labels for a Loki log entry.", From e6e92e7df13f82ff3405d1a7f17538420b25986a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Thu, 18 Jan 2024 14:52:55 -0500 Subject: [PATCH 6/7] grafana: Add instance filters for Loki MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- grafana/incus.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grafana/incus.json b/grafana/incus.json index 5236563d896..8d8df86475d 100644 --- a/grafana/incus.json +++ b/grafana/incus.json @@ -3694,7 +3694,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "builder", - "expr": "{app=\"incus\", type=\"lifecycle\"}", + "expr": "{app=\"incus\", type=\"lifecycle\", instance=\"$job\"}", "queryType": "range", "refId": "A" } @@ -3731,7 +3731,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "builder", - "expr": "{app=\"incus\", type=\"logging\"}", + "expr": "{app=\"incus\", type=\"logging\", instance=\"$job\"}", "queryType": "range", "refId": "A" } From 410085c65965afd7a1cc8d64d79be80356b9b1ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Thu, 18 Jan 2024 15:03:41 -0500 Subject: [PATCH 7/7] incusd/loki: Fix variable shadowing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/loki/loki.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/server/loki/loki.go b/internal/server/loki/loki.go index c13a843c0f3..bfa02c55f2d 100644 --- a/internal/server/loki/loki.go +++ b/internal/server/loki/loki.go @@ -61,7 +61,7 @@ type Client struct { } // NewClient returns a Client. -func NewClient(ctx context.Context, url *url.URL, username string, password string, caCert string, instance string, logLevel string, labels []string, types []string) *Client { +func NewClient(ctx context.Context, u *url.URL, username string, password string, caCert string, instance string, logLevel string, labels []string, types []string) *Client { client := Client{ cfg: config{ batchSize: 10 * 1024, @@ -74,7 +74,7 @@ func NewClient(ctx context.Context, url *url.URL, username string, password stri logLevel: logLevel, timeout: 10 * time.Second, types: types, - url: url, + url: u, }, client: &http.Client{}, ctx: ctx,