Support RFC3164 syslog entries (#1556)

* Upgrade deps

* Port changes

* Add docs and unique changes

* Update docs/sources/reference/components/loki/loki.source.syslog.md

Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com>

* Add k206 ref

* Add local strings

* Add unit tests

* Fix existing tests

* Remove unused check

* Use shared constants

* Fix promtailconvert test

* Revert "Use shared constants"

This reverts commit fb83e9a.

* Do not panic during conversion

* Update unit test

* Update converters

* Fix typo in docs

---------

Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com>
Co-authored-by: Paulin Todev <paulin.todev@gmail.com>

CHANGELOG.md

@@ -13,6 +13,7 @@ Main (unreleased)
 ### Features
 
 - Add the function `path_join` to the stdlib. (@wildum)
+- Add support to `loki.source.syslog` for the RFC3164 format ("BSD syslog"). (@sushain97)
 
 ### Bugfixes
 

docs/sources/reference/components/loki/loki.source.syslog.md

@@ -8,9 +8,10 @@ title: loki.source.syslog
 
 # loki.source.syslog
 
-`loki.source.syslog` listens for syslog messages over TCP or UDP connections
-and forwards them to other `loki.*` components. The messages must be compliant
-with the [RFC5424](https://www.rfc-editor.org/rfc/rfc5424) format.
+`loki.source.syslog` listens for syslog messages over TCP or UDP connections and
+forwards them to other `loki.*` components. The messages must be compliant with
+the [RFC5424](https://www.rfc-editor.org/rfc/rfc5424) syslog protocol or the
+[RFC3164](https://datatracker.ietf.org/doc/html/rfc3164) BSD syslog protocol.
 
 The component starts a new syslog listener for each of the given `config`
 blocks and fans out incoming entries to the list of receivers in `forward_to`.
@@ -71,16 +72,17 @@ The following arguments can be used to configure a `listener`. Only the
 `address` field is required and any omitted fields take their default
 values.
 
-Name                     | Type          | Description                                                                   | Default  | Required
--------------------------|---------------|-------------------------------------------------------------------------------|----------|---------
-`address`                | `string`      | The `<host:port>` address to listen to for syslog messages.                   |          | yes
-`protocol`               | `string`      | The protocol to listen to for syslog messages. Must be either `tcp` or `udp`. | `tcp`    | no
-`idle_timeout`           | `duration`    | The idle timeout for tcp connections.                                         | `"120s"` | no
-`label_structured_data`  | `bool`        | Whether to translate syslog structured data to loki labels.                   | `false`  | no
-`labels`                 | `map(string)` | The labels to associate with each received syslog record.                     | `{}`     | no
-`use_incoming_timestamp` | `bool`        | Whether to set the timestamp to the incoming syslog record timestamp.         | `false`  | no
-`use_rfc5424_message`    | `bool`        | Whether to forward the full RFC5424-formatted syslog message.                 | `false`  | no
-`max_message_length`     | `int`         | The maximum limit to the length of syslog messages.                           | `8192`   | no
+Name                     | Type          | Description                                                                   | Default   | Required
+-------------------------|---------------|-------------------------------------------------------------------------------|-----------|---------
+`address`                | `string`      | The `<host:port>` address to listen to for syslog messages.                   |           | yes
+`protocol`               | `string`      | The protocol to listen to for syslog messages. Must be either `tcp` or `udp`. | `tcp`     | no
+`idle_timeout`           | `duration`    | The idle timeout for tcp connections.                                         | `"120s" ` | no
+`label_structured_data`  | `bool`        | Whether to translate syslog structured data to loki labels.                   | `false`   | no
+`labels`                 | `map(string)` | The labels to associate with each received syslog record.                     | `{}`      | no
+`use_incoming_timestamp` | `bool`        | Whether to set the timestamp to the incoming syslog record timestamp.         | `false`   | no
+`use_rfc5424_message`    | `bool`        | Whether to forward the full RFC5424-formatted syslog message.                 | `false`   | no
+`max_message_length`     | `int`         | The maximum limit to the length of syslog messages.                           | `8192`    | no
+`syslog_format`          | `string`      | The format for incoming messages. Must be either `rfc5424` or `rfc3164`.      | `rfc5424` | no
 
 By default, the component assigns the log entry timestamp as the time it was processed.
 

go.mod

@@ -64,8 +64,8 @@ require (
 	github.com/grafana/jfr-parser/pprof v0.0.0-20240126072739-986e71dc0361
 	github.com/grafana/jsonparser v0.0.0-20240209175146-098958973a2d
 	github.com/grafana/kafka_exporter v0.0.0-20240409084445-5e3488ad9f9a
-	github.com/grafana/loki/pkg/push v0.0.0-20240514112848-a1b1eeb09583 // k201 branch
-	github.com/grafana/loki/v3 v3.0.0-20240513110952-8622293f23b1 // k201 branch
+	github.com/grafana/loki/pkg/push v0.0.0-20240617182007-6c33561108ad // k206 branch
+	github.com/grafana/loki/v3 v3.0.0-20240617182007-6c33561108ad // k206 branch
 	github.com/grafana/pyroscope-go/godeltaprof v0.1.8
 	github.com/grafana/pyroscope/api v0.4.0
 	github.com/grafana/pyroscope/ebpf v0.4.8
@@ -87,12 +87,12 @@ require (
 	github.com/hashicorp/vault/api/auth/userpass v0.6.0
 	github.com/heroku/x v0.0.61
 	github.com/iamseth/oracledb_exporter v0.0.0-20230918193147-95e16f21ceee
-	github.com/influxdata/go-syslog/v3 v3.0.1-0.20230911200830-875f5bc594a4
 	github.com/jaegertracing/jaeger v1.60.0
 	github.com/jmespath/go-jmespath v0.4.0
 	github.com/jonboulle/clockwork v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12
 	github.com/klauspost/compress v1.17.9
+	github.com/leodido/go-syslog/v4 v4.1.0
 	github.com/lib/pq v1.10.9
 	github.com/magefile/mage v1.15.0 // indirect
 	github.com/miekg/dns v1.1.61
@@ -701,7 +701,7 @@ require (
 	github.com/relvacode/iso8601 v1.4.0 // indirect
 	github.com/remeh/sizedwaitgroup v1.0.0 // indirect
 	github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03 // indirect
-	github.com/rivo/uniseg v0.4.2 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/safchain/ethtool v0.3.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect

go.sum

@@ -1219,10 +1219,10 @@ github.com/grafana/jsonparser v0.0.0-20240209175146-098958973a2d h1:YwbJJ/PrVWVd
 github.com/grafana/jsonparser v0.0.0-20240209175146-098958973a2d/go.mod h1:796sq+UcONnSlzA3RtlBZ+b/hrerkZXiEmO8oMjyRwY=
 github.com/grafana/kafka_exporter v0.0.0-20240409084445-5e3488ad9f9a h1:jqM4NNdx8LSquKo8bPx+XWn91S2b+sgNvEcFfSJQtHY=
 github.com/grafana/kafka_exporter v0.0.0-20240409084445-5e3488ad9f9a/go.mod h1:ZXGGyeTUMenf/H1CDBK9lv3azjswfa0nVzLoQAYmnDc=
-github.com/grafana/loki/pkg/push v0.0.0-20240514112848-a1b1eeb09583 h1:dN3eF1S5fvVu2l9WoqYSvmNmPK8Uh2vjE4yUsBq80l4=
-github.com/grafana/loki/pkg/push v0.0.0-20240514112848-a1b1eeb09583/go.mod h1:lJEF/Wh5MYlmBem6tOYAFObkLsuikfrEf8Iy9AdMPiQ=
-github.com/grafana/loki/v3 v3.0.0-20240513110952-8622293f23b1 h1:agFDevQdNVgJYIn5P3+u5oWG+CUnEAZHLZ9jABEXM+0=
-github.com/grafana/loki/v3 v3.0.0-20240513110952-8622293f23b1/go.mod h1:Fv4JImqGS4r+v36vJ4CP9yNM/Krj824Ull/nSuNdI9o=
+github.com/grafana/loki/pkg/push v0.0.0-20240617182007-6c33561108ad h1:4XAhmVmKbGoxmEPaeXWVib4M+SvCaTk2L+W612KRQ3s=
+github.com/grafana/loki/pkg/push v0.0.0-20240617182007-6c33561108ad/go.mod h1:lJEF/Wh5MYlmBem6tOYAFObkLsuikfrEf8Iy9AdMPiQ=
+github.com/grafana/loki/v3 v3.0.0-20240617182007-6c33561108ad h1:eaHz6gH7OM3LWGp1h2QZ+/wkeyjNspk3QwjD4YUWR5g=
+github.com/grafana/loki/v3 v3.0.0-20240617182007-6c33561108ad/go.mod h1:fKBY3k60xjlGcIdbOaJiy3aWopF65FpQvQWczCp2SZ0=
 github.com/grafana/mysqld_exporter v0.12.2-0.20231005125903-364b9c41e595 h1:I9sRknI5ajd8whPOX0nBDXy5B6xUfhItClMy+6R4oqE=
 github.com/grafana/mysqld_exporter v0.12.2-0.20231005125903-364b9c41e595/go.mod h1:U8ifHC5pT2WuVTO7ki4KZmWLjfEKfktQiU3bh0J8scw=
 github.com/grafana/node_exporter v0.18.1-grafana-r01.0.20231004161416-702318429731 h1:vyyIYY2sLpmgFIckJ1vSO/oYkvB0thDF6UiFYp5PThM=
@@ -1473,8 +1473,6 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf
 github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37 h1:Lm6kyC3JBiJQvJrus66He0E4viqDc/m5BdiFNSkIFfU=
 github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37/go.mod h1:+OaHNKQvQ9oOCr+DgkF95PkiDx20fLHpzMp8SmRPQTg=
 github.com/influxdata/go-syslog/v2 v2.0.1/go.mod h1:hjvie1UTaD5E1fTnDmxaCw8RRDrT4Ve+XHr5O2dKSCo=
-github.com/influxdata/go-syslog/v3 v3.0.1-0.20230911200830-875f5bc594a4 h1:2r2WiFeAwiJ/uyx1qIKnV1L4C9w/2V8ehlbJY4gjFaM=
-github.com/influxdata/go-syslog/v3 v3.0.1-0.20230911200830-875f5bc594a4/go.mod h1:1yEQhaLb/cETXCqQmdh7lDjupNAReO7c83AHyK2dJ48=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
 github.com/influxdata/tail v1.0.1-0.20200707181643-03a791b270e4/go.mod h1:VeiWgI3qaGdJWust2fP27a6J+koITo/1c/UhxeOxgaM=
 github.com/influxdata/tdigest v0.0.2-0.20210216194612-fc98d27c9e8b h1:i44CesU68ZBRvtCjBi3QSosCIKrjmMbYlQMFAwVLds4=
@@ -2213,8 +2211,8 @@ github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8d
 github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 h1:Qp27Idfgi6ACvFQat5+VJvlYToylpM/hcyLBI3WaKPA=
 github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052/go.mod h1:uvX/8buq8uVeiZiFht+0lqSLBHF+uGV8BrTv8W/SIwk=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8=
-github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=

internal/component/loki/source/syslog/internal/syslogtarget/syslogtarget.go

@@ -1,6 +1,6 @@
 package syslogtarget
 
-// This code is copied from Promtail v2.8.4 (4a564456861c0ab7441b815aa49a7c0f22a05f4c)
+// This code is copied from Promtail v3.1.0 (935aee77ed389c825d36b8d6a85c0d83895a24d1)
 // The syslogtarget package is used to configure and run the targets that can
 // read syslog entries and forward them to other loki components.
 
@@ -15,8 +15,9 @@ import (
 	"github.com/grafana/loki/v3/clients/pkg/promtail/scrapeconfig"
 	"github.com/grafana/loki/v3/clients/pkg/promtail/targets/target"
 	"github.com/grafana/loki/v3/pkg/logproto"
-	"github.com/influxdata/go-syslog/v3"
-	"github.com/influxdata/go-syslog/v3/rfc5424"
+	"github.com/leodido/go-syslog/v4"
+	"github.com/leodido/go-syslog/v4/rfc3164"
+	"github.com/leodido/go-syslog/v4/rfc5424"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/relabel"
@@ -60,7 +61,6 @@ func NewSyslogTarget(
 	relabel []*relabel.Config,
 	config *scrapeconfig.SyslogTargetConfig,
 ) (*SyslogTarget, error) {
-
 	t := &SyslogTarget{
 		metrics:       metrics,
 		logger:        logger,
@@ -109,7 +109,7 @@ func (t *SyslogTarget) handleMessageError(err error) {
 	t.metrics.syslogParsingErrors.Inc()
 }
 
-func (t *SyslogTarget) handleMessage(connLabels labels.Labels, msg syslog.Message) {
+func (t *SyslogTarget) handleMessageRFC5424(connLabels labels.Labels, msg syslog.Message) {
 	rfc5424Msg := msg.(*rfc5424.SyslogMessage)
 
 	if rfc5424Msg.Message == nil {
@@ -176,6 +176,64 @@ func (t *SyslogTarget) handleMessage(connLabels labels.Labels, msg syslog.Messag
 	t.messages <- message{filtered, m, timestamp}
 }
 
+func (t *SyslogTarget) handleMessageRFC3164(connLabels labels.Labels, msg syslog.Message) {
+	rfc3164Msg := msg.(*rfc3164.SyslogMessage)
+
+	if rfc3164Msg.Message == nil {
+		t.metrics.syslogEmptyMessages.Inc()
+		return
+	}
+
+	lb := labels.NewBuilder(connLabels)
+	if v := rfc3164Msg.SeverityLevel(); v != nil {
+		lb.Set("__syslog_message_severity", *v)
+	}
+	if v := rfc3164Msg.FacilityLevel(); v != nil {
+		lb.Set("__syslog_message_facility", *v)
+	}
+	if v := rfc3164Msg.Hostname; v != nil {
+		lb.Set("__syslog_message_hostname", *v)
+	}
+	if v := rfc3164Msg.Appname; v != nil {
+		lb.Set("__syslog_message_app_name", *v)
+	}
+	if v := rfc3164Msg.ProcID; v != nil {
+		lb.Set("__syslog_message_proc_id", *v)
+	}
+	if v := rfc3164Msg.MsgID; v != nil {
+		lb.Set("__syslog_message_msg_id", *v)
+	}
+
+	processed, _ := relabel.Process(lb.Labels(), t.relabelConfig...)
+
+	filtered := make(model.LabelSet)
+	for _, lbl := range processed {
+		if strings.HasPrefix(lbl.Name, "__") {
+			continue
+		}
+		filtered[model.LabelName(lbl.Name)] = model.LabelValue(lbl.Value)
+	}
+
+	var timestamp time.Time
+	if t.config.UseIncomingTimestamp && rfc3164Msg.Timestamp != nil {
+		timestamp = *rfc3164Msg.Timestamp
+	} else {
+		timestamp = time.Now()
+	}
+
+	m := *rfc3164Msg.Message
+
+	t.messages <- message{filtered, m, timestamp}
+}
+
+func (t *SyslogTarget) handleMessage(connLabels labels.Labels, msg syslog.Message) {
+	if t.config.IsRFC3164Message() {
+		t.handleMessageRFC3164(connLabels, msg)
+	} else {
+		t.handleMessageRFC5424(connLabels, msg)
+	}
+}
+
 func (t *SyslogTarget) messageSender(entries chan<- loki.Entry) {
 	for msg := range t.messages {
 		entries <- loki.Entry{

internal/component/loki/source/syslog/internal/syslogtarget/syslogtarget_test.go

@@ -20,7 +20,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/grafana/loki/v3/clients/pkg/promtail/scrapeconfig"
 	"github.com/grafana/loki/v3/clients/pkg/promtail/targets/syslog/syslogparser"
-	"github.com/influxdata/go-syslog/v3"
+	"github.com/leodido/go-syslog/v4"
 	promconfig "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/relabel"
@@ -919,7 +919,7 @@ func TestParseStream_WithAsyncPipe(t *testing.T) {
 		results = append(results, res)
 	}
 
-	err := syslogparser.ParseStream(pipe, cb, DefaultMaxMessageLength)
+	err := syslogparser.ParseStream(false, pipe, cb, DefaultMaxMessageLength)
 	require.NoError(t, err)
 	require.Equal(t, 3, len(results))
 }

internal/component/loki/source/syslog/internal/syslogtarget/transport.go

@@ -21,7 +21,7 @@ import (
 	"github.com/grafana/dskit/backoff"
 	"github.com/grafana/loki/v3/clients/pkg/promtail/scrapeconfig"
 	"github.com/grafana/loki/v3/clients/pkg/promtail/targets/syslog/syslogparser"
-	"github.com/influxdata/go-syslog/v3"
+	"github.com/leodido/go-syslog/v4"
 	"github.com/mwitkow/go-conntrack"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/prometheus/model/labels"
@@ -321,7 +321,7 @@ func (t *TCPTransport) handleConnection(cn net.Conn) {
 
 	lbs := t.connectionLabels(ipFromConn(c).String())
 
-	err := syslogparser.ParseStream(c, func(result *syslog.Result) {
+	err := syslogparser.ParseStream(t.config.IsRFC3164Message(), c, func(result *syslog.Result) {
 		if err := result.Error; err != nil {
 			t.handleMessageError(err)
 			return
@@ -429,7 +429,8 @@ func (t *UDPTransport) acceptPackets() {
 func (t *UDPTransport) handleRcv(c *ConnPipe) {
 	defer t.openConnections.Done()
 
-	lbs := t.connectionLabels(c.addr.String())
+	udpAddr, _ := net.ResolveUDPAddr("udp", c.addr.String())
+	lbs := t.connectionLabels(udpAddr.IP.String())
 
 	for {
 		datagram := make([]byte, t.maxMessageLength())
@@ -445,7 +446,7 @@ func (t *UDPTransport) handleRcv(c *ConnPipe) {
 
 		r := bytes.NewReader(datagram[:n])
 
-		err = syslogparser.ParseStream(r, func(result *syslog.Result) {
+		err = syslogparser.ParseStream(t.config.IsRFC3164Message(), r, func(result *syslog.Result) {
 			if err := result.Error; err != nil {
 				t.handleMessageError(err)
 			} else {

internal/component/loki/source/syslog/syslog.go

@@ -116,7 +116,13 @@ func (c *Component) Update(args component.Arguments) error {
 		entryHandler := loki.NewEntryHandler(c.handler.Chan(), func() {})
 
 		for _, cfg := range newArgs.SyslogListeners {
-			t, err := st.NewSyslogTarget(c.metrics, c.opts.Logger, entryHandler, rcs, cfg.Convert())
+			promtailCfg, cfgErr := cfg.Convert()
+			if cfgErr != nil {
+				level.Error(c.opts.Logger).Log("msg", "failed to convert syslog listener config", "err", cfgErr)
+				continue
+			}
+
+			t, err := st.NewSyslogTarget(c.metrics, c.opts.Logger, entryHandler, rcs, promtailCfg)
 			if err != nil {
 				level.Error(c.opts.Logger).Log("msg", "failed to create syslog listener with provided config", "err", err)
 				continue

internal/component/loki/source/syslog/syslog_test.go

@@ -30,18 +30,17 @@ func Test(t *testing.T) {
 	args := Arguments{}
 	tcpListenerAddr, udpListenerAddr := getFreeAddr(t), getFreeAddr(t)
 
-	args.SyslogListeners = []ListenerConfig{
-		{
-			ListenAddress:  tcpListenerAddr,
-			ListenProtocol: "tcp",
-			Labels:         map[string]string{"protocol": "tcp"},
-		},
-		{
-			ListenAddress:  udpListenerAddr,
-			ListenProtocol: "udp",
-			Labels:         map[string]string{"protocol": "udp"},
-		},
-	}
+	l1 := DefaultListenerConfig
+	l1.ListenAddress = tcpListenerAddr
+	l1.ListenProtocol = "tcp"
+	l1.Labels = map[string]string{"protocol": "tcp"}
+
+	l2 := DefaultListenerConfig
+	l2.ListenAddress = udpListenerAddr
+	l2.ListenProtocol = "udp"
+	l2.Labels = map[string]string{"protocol": "udp"}
+
+	args.SyslogListeners = []ListenerConfig{l1, l2}
 	args.ForwardTo = []loki.LogsReceiver{ch1, ch2}
 
 	// Create and run the component.
@@ -112,12 +111,11 @@ func TestWithRelabelRules(t *testing.T) {
 	args := Arguments{}
 	tcpListenerAddr := getFreeAddr(t)
 
-	args.SyslogListeners = []ListenerConfig{
-		{
-			ListenAddress: tcpListenerAddr,
-			Labels:        map[string]string{"protocol": "tcp"},
-		},
-	}
+	l := DefaultListenerConfig
+	l.ListenAddress = tcpListenerAddr
+	l.Labels = map[string]string{"protocol": "tcp"}
+
+	args.SyslogListeners = []ListenerConfig{l}
 	args.ForwardTo = []loki.LogsReceiver{ch1}
 
 	// Create a handler which will be used to retrieve relabeling rules.