Initial Loki/Promtail support

.github/workflows/containerize.yml

@@ -8,6 +8,9 @@ jobs:
     name: Build Container
     runs-on: ubuntu-latest
     steps:
+    - name: Get build dependencies
+      run: sudo apt-get update && sudo apt-get install libsystemd-dev
+
     - name: Login to Docker Hub
       run: docker login -u '${{ secrets.DOCKER_USER }}' -p '${{ secrets.DOCKER_PASS }}'
 

.github/workflows/release.yml

@@ -8,6 +8,9 @@ jobs:
     name: Release
     runs-on: ubuntu-latest
     steps:
+    - name: Get build dependencies
+      run: sudo apt-get update && sudo apt-get install libsystemd-dev
+
     - name: Login to Docker Hub
       run: docker login -u '${{ secrets.DOCKER_USER }}' -p '${{ secrets.DOCKER_PASS }}'
 

.github/workflows/test.yml

@@ -5,6 +5,9 @@ jobs:
     name: Lint
     runs-on: ubuntu-latest
     steps:
+    - name: Get build dependencies
+      run: sudo apt-get update && sudo apt-get install libsystemd-dev
+
     - name: Set up Go 1.14
       uses: actions/setup-go@v1
       with:
@@ -25,6 +28,10 @@ jobs:
         platform: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
     steps:
+    - name: Get build dependencies
+      run: sudo apt-get update && sudo apt-get install libsystemd-dev
+      if: matrix.platform == 'ubuntu-latest'
+
     - name: Set up Go 1.14
       uses: actions/setup-go@v1
       with:
@@ -42,6 +49,10 @@ jobs:
         platform: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
     steps:
+    - name: Get build dependencies
+      run: sudo apt-get update && sudo apt-get install libsystemd-dev
+      if: matrix.platform == 'ubuntu-latest'
+
     - name: Set up Go 1.14
       uses: actions/setup-go@v1
       with:

Makefile

@@ -105,20 +105,8 @@ all: protos agent agentctl
 agent: cmd/agent/agent
 agentctl: cmd/agentctl/agentctl
 
-# some platforms need CGO_ENABLED=1 for node_exporter
-need_cgo = 0
-ifeq ($(shell go env GOOS),darwin)
-need_cgo = 1
-else ifeq ($(shell go env GOOS),freebsd)
-need_cgo = 1
-endif
-
 cmd/agent/agent: cmd/agent/main.go
-ifeq ($(need_cgo),1)
 	CGO_ENABLED=1 go build $(CGO_FLAGS) -o $@ ./$(@D)
-else
-	CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
-endif
 	$(NETGO_CHECK)
 
 cmd/agentctl/agentctl: cmd/agentctl/main.go
@@ -144,11 +132,7 @@ push-agentctl-image:
 	docker push $(IMAGE_PREFIX)/agentctl:$(IMAGE_TAG)
 
 install:
-ifeq ($(need_cgo),1)
 	CGO_ENABLED=1 go install $(CGO_FLAGS) ./cmd/agent
-else
-	CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/agent
-endif
 	CGO_ENABLED=0 go install $(GO_FLAGS) ./cmd/agentctl
 
 #######################

README.md

@@ -52,7 +52,7 @@ the remote system.
 - [x] Prometheus metrics
 - [x] A second clustering mode to solve sharding monitoring availability problems.
 - [x] Support for integrations (embedded exporters/automatic scrape configs)
-- [ ] Promtail for Loki logs
+- [x] Promtail for Loki logs
 - [ ] `carbon-relay-ng` for Graphite metrics.
 
 ## Getting Started

cmd/agent/Dockerfile

@@ -3,10 +3,14 @@ COPY . /src/agent
 WORKDIR /src/agent
 ARG RELEASE_BUILD=true
 ARG IMAGE_TAG
+RUN apt-get update && apt-get install -qy libsystemd-dev
 RUN make clean && make IMAGE_TAG=${IMAGE_TAG} RELEASE_BUILD=${RELEASE_BUILD} BUILD_IN_CONTAINER=false agent
 
-FROM alpine:3.9
-RUN apk add --update --no-cache ca-certificates
+FROM debian:stretch-slim
+RUN apt-get update && \
+  apt-get install -qy \
+  tzdata ca-certificates libsystemd-dev && \
+  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 COPY --from=build /src/agent/cmd/agent/agent /bin/agent
 COPY cmd/agent/agent-local-config.yaml /etc/agent/agent.yaml
 

cmd/agent/main.go

@@ -10,6 +10,7 @@ import (
 	// Adds version information
 	_ "github.com/grafana/agent/pkg/build"
 	"github.com/grafana/agent/pkg/integrations"
+	"github.com/grafana/agent/pkg/loki"
 
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/go-kit/kit/log/level"
@@ -40,6 +41,11 @@ func main() {
 		os.Exit(1)
 	}
 
+	lokiLogs, err := loki.New(cfg.Loki, util.Logger)
+	if err != nil {
+		level.Error(util.Logger).Log("msg", "failed to create loki log collection instance", "err", err)
+	}
+
 	srv, err := server.New(cfg.Server)
 	if err != nil {
 		level.Error(util.Logger).Log("msg", "failed to create server", "err", err)
@@ -76,6 +82,7 @@ func main() {
 	}
 
 	manager.Stop()
+	lokiLogs.Stop()
 	promMetrics.Stop()
 	level.Info(util.Logger).Log("msg", "agent exiting")
 }

docs/configuration-reference.md

@@ -6,6 +6,8 @@ Prometheus instances.
 
 * [server_config](#server_config)
 * [prometheus_config](#prometheus_config)
+* [loki_config](#loki_config)
+* [integrations_config](#integrations_config)
 
 ## File Format
 
@@ -37,6 +39,9 @@ Support contents and default values of `agent.yaml`:
 # Configures Prometheus instances.
 [prometheus: <prometheus_config>]
 
+# Configures Loki log collection.
+[loki: <loki_config>]
+
 # Configures integrations for the Agent.
 [integrations: <integrations_config>]
 ```
@@ -1414,6 +1419,27 @@ metadata_config:
   [ send_interval: <duration> | default = 1m ]
 ```
 
+### loki_config
+
+The `loki_config` block configures how the Agent collects logs and sends them to a Loki push API endpoint. `loki_config` is identical to how Promtail is configured, except deprecated
+fields have been removed and the server_config is not supported.
+
+Please refer to the
+[Promtail documentation](https://github.com/grafana/loki/tree/master/docs/sources/clients/promtail#client_config)
+for the supported values for these fields.
+
+```yaml
+clients:
+  - [<promtail.client_config>]
+
+[positions: <promtail.position_config>]
+
+scrape_configs:
+  - [<promtail.scrape_config>]
+
+[target_config: <promtail.target_config>]
+```
+
 ### integrations_config
 
 The `integrations_config` block configures how the Agent runs integrations that

example/docker-compose/agent/config/agent.yaml

@@ -48,3 +48,17 @@ prometheus:
 
       remote_write:
         - url: http://cortex:9009/api/prom/push
+
+loki:
+  positions:
+    filename: /tmp/positions.yaml
+  clients:
+    - url: http://loki:3100/loki/api/v1/push
+  scrape_configs:
+  - job_name: system
+    static_configs:
+      - targets:
+        - localhost
+        labels:
+          job: varlogs
+          __path__: /var/log/*log

example/docker-compose/docker-compose.no-agent.yaml

@@ -0,0 +1,26 @@
+version: "2"
+services:
+  cortex:
+    image: cortexproject/cortex:v0.6.1
+    volumes:
+      - /tmp/cortex:/tmp/cortex
+      - ./cortex/config:/etc/cortex-config
+    entrypoint:
+      - /bin/cortex
+      - -config.file=/etc/cortex-config/cortex.yaml
+    ports:
+      - "9009:9009"
+
+  grafana:
+    image: grafana/grafana:6.6.1
+    entrypoint:
+      - /usr/share/grafana/bin/grafana-server
+      - --homepath=/usr/share/grafana
+      - --config=/etc/grafana-config/grafana.ini
+    volumes:
+      - ./grafana/config:/etc/grafana-config
+      - ./grafana/datasources:/etc/grafana/provisioning/datasources
+      - ./grafana/dashboards-provisioning:/etc/grafana/provisioning/dashboards
+      - ./grafana/dashboards:/var/lib/grafana/dashboards
+    ports:
+      - "3000:3000"

example/docker-compose/docker-compose.yaml

@@ -5,13 +5,20 @@ services:
     volumes:
       - /tmp/agent:/tmp/agent
       - ./agent/config:/etc/agent-config
+      - /var/log:/var/log
     entrypoint:
       - /bin/agent
       - -config.file=/etc/agent-config/agent.yaml
       - -prometheus.wal-directory=/tmp/agent/wal
     ports:
       - "12345:12345"
 
+  loki:
+    image: grafana/loki:1.5.0
+    ports: 
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+
   cortex:
     image: cortexproject/cortex:v0.6.1
     volumes:

example/docker-compose/grafana/datasources/datasource.yml

@@ -13,3 +13,13 @@ datasources:
   isDefault: false
   version: 1
   editable: false
+- name: Loki
+  type: loki
+  access: proxy 
+  orgId: 1
+  url: http://loki:3100
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: false
+

go.mod

@@ -3,45 +3,56 @@ module github.com/grafana/agent
 go 1.12
 
 require (
-	github.com/cortexproject/cortex v1.0.1-0.20200409122148-163437e76cad
-	github.com/go-kit/kit v0.10.0
-	github.com/gogo/protobuf v1.3.1
-	github.com/golang/protobuf v1.4.1
-	github.com/gorilla/mux v1.7.3
-	github.com/oklog/run v1.1.0
-	github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02
-	github.com/opentracing/opentracing-go v1.1.1-0.20200124165624-2876d2018785
-	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.6.0
-	github.com/prometheus/common v0.10.0
-	github.com/prometheus/node_exporter v1.0.0
-	github.com/prometheus/procfs v0.0.11
-	github.com/prometheus/prometheus v1.8.2-0.20200213233353-b90be6f32a33
-	github.com/spf13/cobra v0.0.3
-	github.com/stretchr/testify v1.5.1
-	github.com/weaveworks/common v0.0.0-20200310113808-2708ba4e60a4
-	go.uber.org/atomic v1.6.0
-	google.golang.org/grpc v1.29.0
-	gopkg.in/alecthomas/kingpin.v2 v2.2.6
-	gopkg.in/yaml.v2 v2.3.0
+  github.com/cortexproject/cortex v1.2.1-0.20200803161316-7014ff11ed70
+  github.com/go-kit/kit v0.10.0
+  github.com/gogo/protobuf v1.3.1
+  github.com/golang/protobuf v1.4.2
+  github.com/gorilla/mux v1.7.3
+  github.com/grafana/loki v1.5.1-0.20200810203821-80693cae6f60
+  github.com/oklog/run v1.1.0
+  github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02
+  github.com/opentracing/opentracing-go v1.2.0
+  github.com/pkg/errors v0.9.1
+  github.com/prometheus/client_golang v1.7.1
+  github.com/prometheus/common v0.10.0
+  github.com/prometheus/node_exporter v1.0.0
+  github.com/prometheus/procfs v0.1.3
+  github.com/prometheus/prometheus v1.8.2-0.20200727090838-6f296594a852
+  github.com/spf13/cobra v0.0.3
+  github.com/stretchr/testify v1.5.1
+  github.com/weaveworks/common v0.0.0-20200625145055-4b1847531bc9
+  go.uber.org/atomic v1.6.0
+  google.golang.org/grpc v1.30.0
+  gopkg.in/alecthomas/kingpin.v2 v2.2.6
+  gopkg.in/yaml.v2 v2.3.0
 )
 
 // Needed for Cortex's dependencies to work properly.
 replace (
-	google.golang.org/api => google.golang.org/api v0.14.0
-	google.golang.org/grpc => google.golang.org/grpc v1.25.1
+  github.com/sercand/kuberesolver => github.com/sercand/kuberesolver v2.1.0+incompatible
+  go.etcd.io/etcd => go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738
+  google.golang.org/api => google.golang.org/api v0.14.0
+  google.golang.org/grpc => google.golang.org/grpc v1.25.1
+  k8s.io/client-go => k8s.io/client-go v0.17.5
 )
 
 // Replace directives from Prometheus
 replace k8s.io/klog => github.com/simonpasquier/klog-gokit v0.1.0
 
 // Replace directives from Cortex
 replace (
-	git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999
-	github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v36.2.0+incompatible
-	github.com/Azure/go-autorest => github.com/Azure/go-autorest v13.3.0+incompatible
+  git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999
+  github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v36.2.0+incompatible
+  github.com/Azure/go-autorest => github.com/Azure/go-autorest v13.3.0+incompatible
+  github.com/gocql/gocql => github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85
+  github.com/hpcloud/tail => github.com/grafana/tail v0.0.0-20191024143944-0b54ddf21fe7
+  github.com/satori/go.uuid => github.com/satori/go.uuid v1.2.0
 )
 
 replace github.com/prometheus/prometheus => github.com/grafana/prometheus v1.8.2-0.20200518163447-007aa83a0a1f
 
 replace gopkg.in/yaml.v2 => github.com/rfratto/go-yaml v0.0.0-20200521142311-984fc90c8a04
+
+// Needed to keep FreeBSD builds working; newer versions of procfs use
+// syscall.ENODATA which doesn't work with FreeBSD.
+replace github.com/prometheus/procfs => github.com/prometheus/procfs v0.0.11