Add support for loki.instance

cmd/incusd/api_1.0.go

@@ -868,6 +868,8 @@ func doApi10UpdateTriggers(d *Daemon, nodeChanged, clusterChanged map[string]str
 			fallthrough
 		case "loki.api.ca_cert":
 			fallthrough
+		case "loki.instance":
+			fallthrough
 		case "loki.labels":
 			fallthrough
 		case "loki.loglevel":
@@ -984,12 +986,12 @@ func doApi10UpdateTriggers(d *Daemon, nodeChanged, clusterChanged map[string]str
 	}
 
 	if lokiChanged {
-		lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiLabels, lokiLoglevel, lokiTypes := clusterConfig.LokiServer()
+		lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiInstance, lokiLoglevel, lokiLabels, lokiTypes := clusterConfig.LokiServer()
 
 		if lokiURL == "" || lokiLoglevel == "" || len(lokiTypes) == 0 {
 			d.internalListener.RemoveHandler("loki")
 		} else {
-			err := d.setupLoki(lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiLabels, lokiLoglevel, lokiTypes)
+			err := d.setupLoki(lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiInstance, lokiLoglevel, lokiLabels, lokiTypes)
 			if err != nil {
 				return err
 			}

cmd/incusd/daemon.go

@@ -762,22 +762,32 @@ func (d *Daemon) Init() error {
 	return nil
 }
 
-func (d *Daemon) setupLoki(URL string, cert string, key string, caCert string, labels []string, logLevel string, types []string) error {
+func (d *Daemon) setupLoki(URL string, cert string, key string, caCert string, instanceName string, logLevel string, labels []string, types []string) error {
+	// Stop any existing loki client.
 	if d.lokiClient != nil {
 		d.lokiClient.Stop()
 	}
 
+	// Check basic requirements for starting a new client.
 	if URL == "" || logLevel == "" || len(types) == 0 {
 		return nil
 	}
 
+	// Validate the URL.
 	u, err := url.Parse(URL)
 	if err != nil {
 		return err
 	}
 
-	d.lokiClient = loki.NewClient(d.shutdownCtx, u, cert, key, caCert, labels, logLevel, types)
+	// Figure out the instance name.
+	if instanceName == "" {
+		instanceName = d.serverName
+	}
+
+	// Start a new client.
+	d.lokiClient = loki.NewClient(d.shutdownCtx, u, cert, key, caCert, instanceName, logLevel, labels, types)
 
+	// Attach the new client to the log handler.
 	d.internalListener.AddHandler("loki", d.lokiClient.HandleEvent)
 
 	return nil
@@ -1337,7 +1347,7 @@ func (d *Daemon) init() error {
 	d.proxy = proxy.FromConfig(d.globalConfig.ProxyHTTPS(), d.globalConfig.ProxyHTTP(), d.globalConfig.ProxyIgnoreHosts())
 
 	d.gateway.HeartbeatOfflineThreshold = d.globalConfig.OfflineThreshold()
-	lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiLabels, lokiLoglevel, lokiTypes := d.globalConfig.LokiServer()
+	lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiInstance, lokiLoglevel, lokiLabels, lokiTypes := d.globalConfig.LokiServer()
 	oidcIssuer, oidcClientID, oidcAudience := d.globalConfig.OIDCServer()
 	syslogSocketEnabled := d.localConfig.SyslogSocket()
 	openfgaAPIURL, openfgaAPIToken, openfgaStoreID, openFGAAuthorizationModelID := d.globalConfig.OpenFGA()
@@ -1348,7 +1358,7 @@ func (d *Daemon) init() error {
 
 	// Setup Loki logger.
 	if lokiURL != "" {
-		err = d.setupLoki(lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiLabels, lokiLoglevel, lokiTypes)
+		err = d.setupLoki(lokiURL, lokiUsername, lokiPassword, lokiCACert, lokiInstance, lokiLoglevel, lokiLabels, lokiTypes)
 		if err != nil {
 			return err
 		}

doc/api-extensions.md

@@ -2305,3 +2305,9 @@ Adds a `description` field to certificate.
 
 Adds a new `virtio-blk` value for `io.bus` on `disk` devices which allows
 for the attached disk to be connected to the `virtio-blk` bus.
+
+## `loki_config_instance`
+
+Adds a new `loki.instance` server configuration key to customize the `instance` field in Loki events.
+This can be used to expose the name of the cluster rather than the individual system name sending
+the event as that's usually already covered by the `location` field.

doc/config_options.txt

@@ -1592,6 +1592,14 @@ Specify the protocol, name or IP and port. For example `https://loki.example.com
 
 ```
 
+```{config:option} loki.instance server-loki
+:defaultdesc: "Local server host name or cluster member name"
+:scope: "global"
+:shortdesc: "Name to use as the instance field in Loki events."
+:type: "string"
+This allows replacing the default instance value (server host name) by a more relevant value like a cluster identifier.
+```
+
 ```{config:option} loki.labels server-loki
 :scope: "global"
 :shortdesc: "Labels for a Loki log entry"

grafana/incus.json

@@ -3694,7 +3694,7 @@
             "uid": "${DS_LOKI}"
           },
           "editorMode": "builder",
-          "expr": "{app=\"incus\", type=\"lifecycle\"}",
+          "expr": "{app=\"incus\", type=\"lifecycle\", instance=\"$job\"}",
           "queryType": "range",
           "refId": "A"
         }
@@ -3731,7 +3731,7 @@
             "uid": "${DS_LOKI}"
           },
           "editorMode": "builder",
-          "expr": "{app=\"incus\", type=\"logging\"}",
+          "expr": "{app=\"incus\", type=\"logging\", instance=\"$job\"}",
           "queryType": "range",
           "refId": "A"
         }

internal/server/cluster/config/config.go

@@ -183,7 +183,7 @@ func (c *Config) InstancesPlacementScriptlet() string {
 }
 
 // LokiServer returns all the Loki settings needed to connect to a server.
-func (c *Config) LokiServer() (string, string, string, string, []string, string, []string) {
+func (c *Config) LokiServer() (string, string, string, string, string, string, []string, []string) {
 	var types []string
 	var labels []string
 
@@ -195,7 +195,7 @@ func (c *Config) LokiServer() (string, string, string, string, []string, string,
 		labels = strings.Split(c.m.GetString("loki.labels"), ",")
 	}
 
-	return c.m.GetString("loki.api.url"), c.m.GetString("loki.auth.username"), c.m.GetString("loki.auth.password"), c.m.GetString("loki.api.ca_cert"), labels, c.m.GetString("loki.loglevel"), types
+	return c.m.GetString("loki.api.url"), c.m.GetString("loki.auth.username"), c.m.GetString("loki.auth.password"), c.m.GetString("loki.api.ca_cert"), c.m.GetString("loki.instance"), c.m.GetString("loki.loglevel"), labels, types
 }
 
 // ACME returns all ACME settings needed for certificate renewal.
@@ -593,6 +593,15 @@ var ConfigSchema = config.Schema{
 	//  shortdesc: URL to the Loki server
 	"loki.api.url": {},
 
+	// gendoc:generate(entity=server, group=loki, key=loki.instance)
+	// This allows replacing the default instance value (server host name) by a more relevant value like a cluster identifier.
+	// ---
+	//  type: string
+	//  scope: global
+	//  defaultdesc: Local server host name or cluster member name
+	//  shortdesc: Name to use as the instance field in Loki events.
+	"loki.instance": {},
+
 	// gendoc:generate(entity=server, group=loki, key=loki.labels)
 	// Specify a comma-separated list of values that should be used as labels for a Loki log entry.
 	// ---

internal/server/loki/loki.go

@@ -9,7 +9,6 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"os"
 	"reflect"
 	"sort"
 	"strconv"
@@ -38,6 +37,7 @@ type config struct {
 	username  string
 	password  string
 	labels    []string
+	instance  string
 	logLevel  string
 	timeout   time.Duration
 	types     []string
@@ -61,19 +61,20 @@ type Client struct {
 }
 
 // NewClient returns a Client.
-func NewClient(ctx context.Context, url *url.URL, username string, password string, caCert string, labels []string, logLevel string, types []string) *Client {
+func NewClient(ctx context.Context, u *url.URL, username string, password string, caCert string, instance string, logLevel string, labels []string, types []string) *Client {
 	client := Client{
 		cfg: config{
 			batchSize: 10 * 1024,
 			batchWait: 1 * time.Second,
 			caCert:    caCert,
 			username:  username,
 			password:  password,
+			instance:  instance,
 			labels:    labels,
 			logLevel:  logLevel,
 			timeout:   10 * time.Second,
 			types:     types,
-			url:       url,
+			url:       u,
 		},
 		client:  &http.Client{},
 		ctx:     ctx,
@@ -227,17 +228,12 @@ func (c *Client) HandleEvent(event api.Event) {
 		return
 	}
 
-	hostname, err := os.Hostname()
-	if err != nil {
-		hostname = "none"
-	}
-
 	entry := entry{
 		labels: LabelSet{
 			"app":      "incus",
 			"type":     event.Type,
 			"location": event.Location,
-			"instance": hostname,
+			"instance": c.cfg.instance,
 		},
 		Entry: Entry{
 			Timestamp: event.Timestamp,

internal/server/metadata/configuration.json

@@ -1745,6 +1745,15 @@
 							"type": "string"
 						}
 					},
+					{
+						"loki.instance": {
+							"defaultdesc": "Local server host name or cluster member name",
+							"longdesc": "This allows replacing the default instance value (server host name) by a more relevant value like a cluster identifier.",
+							"scope": "global",
+							"shortdesc": "Name to use as the instance field in Loki events.",
+							"type": "string"
+						}
+					},
 					{
 						"loki.labels": {
 							"longdesc": "Specify a comma-separated list of values that should be used as labels for a Loki log entry.",

internal/version/api.go

@@ -389,6 +389,7 @@ var APIExtensions = []string{
 	"ovn_ssl_config",
 	"certificate_description",
 	"disk_io_bus_virtio_blk",
+	"loki_config_instance",
 }
 
 // APIExtensionsCount returns the number of available API extensions.