diff --git a/.gitignore b/.gitignore index fca0ab6..90b199a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ pkg-build build *.deb +.idea +/cgroup-exporter diff --git a/README.md b/README.md index 7f755bc..6503ee3 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ container_cpu_user_seconds_total{id="/system.slice/wpa_supplicant.service"} 524. container_cpu_user_seconds_total{id="/system.slice/ssh.service"} 1.30 container_cpu_user_seconds_total{id="/system.slice/docker.service"} 2219.16 container_cpu_user_seconds_total{id="/system.slice/NetworkManager.service"} 4283.36 +container_cpu_user_seconds_total{id="/docker/grafana"} 0.70 : # HELP container_memory_usage_bytes Current memory usage in bytes, including all memory regardless of when it was accessed @@ -27,6 +28,7 @@ container_memory_usage_bytes{id="/system.slice/wpa_supplicant.service"} 1871872 container_memory_usage_bytes{id="/system.slice/ssh.service"} 61440 container_memory_usage_bytes{id="/system.slice/docker.service"} 37171200 container_memory_usage_bytes{id="/system.slice/NetworkManager.service"} 18305024 +container_memory_usage_bytes{id="/docker/grafana"} 61407232 : # HELP container_memory_rss Size of RSS in bytes. @@ -35,5 +37,27 @@ container_memory_rss{id="/system.slice/wpa_supplicant.service"} 331776 container_memory_rss{id="/system.slice/ssh.service"} 110592 container_memory_rss{id="/system.slice/docker.service"} 24072192 container_memory_rss{id="/system.slice/NetworkManager.service"} 5066752 +container_memory_rss{id="/docker/grafana"} 16224256 : ``` + +## options + +| arg | description | +| --- | --- | +| `--metrics.docker` | enable docker container metrics | + + +## customize systemd + +You can customize systemd setting with options. + +`/etc/systemd/system/prometheus-cgroup-exporter.service.d/local.conf`: + +``` +[Service] +ExecStart= +ExecStart=/usr/local/bin/cgroup-exporter --metrics.docker +``` + +see [systemd.unit / Example 2. Overriding vendor settings](https://www.freedesktop.org/software/systemd/man/systemd.unit.html#id-1.14.3). diff --git a/deb.json b/deb.json index 1673cdf..f9362f7 100644 --- a/deb.json +++ b/deb.json @@ -1,6 +1,6 @@ { "name": "prometheus-cgroup-exporter", - "version": "0.1.2", + "version": "0.1.3", "maintainer": "GROOVE X Development Team ", "description": "prometheus cgroup exporter", "changelog-cmd": "git log --pretty='format:%cd %h %s %d [%an]' --date=iso --merges", diff --git a/go.mod b/go.mod index 60d800b..b3d53d9 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,17 @@ module github.com/groove-x/cgroup-exporter go 1.11 require ( + github.com/Microsoft/go-winio v0.4.14 // indirect github.com/containerd/cgroups v0.0.0-20181001140508-d2400726cfa7 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d // indirect + github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/docker v1.13.1 + github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.3.4-0.20181018102642-03db2b60b820 // indirect github.com/godbus/dbus v4.1.1-0.20180905195443-5f1bd775722e+incompatible // indirect github.com/gogo/protobuf v1.2.0 // indirect + github.com/moby/moby v1.13.1 + github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/runtime-spec v1.0.2-0.20180913141938-5806c3563733 // indirect - github.com/pkg/errors v0.8.1-0.20181014145847-6ed0a2e59ebe // indirect - golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06 // indirect + golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect ) diff --git a/go.sum b/go.sum index ec1f21a..90aa68e 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,47 @@ +github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/containerd/cgroups v0.0.0-20181001140508-d2400726cfa7 h1:HpTs4G3F32Jue3Hu3Tc4hD4d9AhALxYE4+vCO0SkSvw= github.com/containerd/cgroups v0.0.0-20181001140508-d2400726cfa7/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d h1:t5Wuyh53qYyg9eqn4BbnlIT+vmhyww0TatL+zT3uWgI= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= +github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.4-0.20181018102642-03db2b60b820 h1:EQSiSOsNbDAIw+kdd4NPThBl+510eNrB58SRYcKwUK4= github.com/docker/go-units v0.3.4-0.20181018102642-03db2b60b820/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/godbus/dbus v4.1.1-0.20180905195443-5f1bd775722e+incompatible h1:1EAOqHEDzmRZ+SWNSMZ3nnczBDhMT1jIfE3QZ2iHZ9s= github.com/godbus/dbus v4.1.1-0.20180905195443-5f1bd775722e+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/moby/moby v1.13.1 h1:mC5WwQwCXt/dYxZ1cIrRsnJAWw7VdtcTZUIGr4tXzOM= +github.com/moby/moby v1.13.1/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/runtime-spec v1.0.2-0.20180913141938-5806c3563733 h1:M9SVV7xezQ8PC/0NPAek6TUr1IdwVx5wGkdojnXwalM= github.com/opencontainers/runtime-spec v1.0.2-0.20180913141938-5806c3563733/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/pkg/errors v0.8.1-0.20181014145847-6ed0a2e59ebe h1:iN/Ye31z0uoRJNk0Cd8EtLFjKmkl59NtvEw2zt6nbBA= -github.com/pkg/errors v0.8.1-0.20181014145847-6ed0a2e59ebe/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06 h1:0oC8rFnE+74kEmuHZ46F6KHsMr5Gx2gUQPuNz28iQZM= -golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/main.go b/main.go index 2cd19b9..ef6528f 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "flag" "fmt" "log" @@ -14,6 +15,8 @@ import ( "time" "github.com/containerd/cgroups" + "github.com/docker/docker/api/types" + "github.com/moby/moby/client" ) var ( @@ -21,7 +24,8 @@ var ( version string git string - address = flag.String("address", ":48900", "address") + address = flag.String("address", ":48900", "address") + enableDocker = flag.Bool("metrics.docker", false, "docker container metrics") ) func main() { @@ -39,7 +43,16 @@ func main() { log.Fatalf("cgroups load: %s", err) } - http.HandleFunc("/metrics", exportMetrics(system)) + var dockerClient *client.Client + if *enableDocker { + dockerClient, err = client.NewEnvClient() + if err != nil { + log.Fatalf("%v", err) + } + defer dockerClient.Close() + } + + http.HandleFunc("/metrics", exportMetrics(system, dockerClient)) server := &http.Server{ Addr: *address, @@ -69,35 +82,89 @@ func subsystem() ([]cgroups.Subsystem, error) { return s, nil } -func exportMetrics(system cgroups.Cgroup) func(w http.ResponseWriter, r *http.Request) { +func statsCgroups(ctx context.Context, system cgroups.Cgroup) (map[string]*cgroups.Metrics, error) { + processes, err := system.Processes(cgroups.Devices, true) + if err != nil { + return nil, fmt.Errorf("cgroups load: %s", err) + } + + groups := make(map[string]*cgroups.Metrics, len(processes)) + for _, p := range processes { + name := strings.TrimPrefix(p.Path, "/sys/fs/cgroup/devices") + name = strings.TrimSuffix(name, "/") + if _, ok := groups[name]; ok { + continue + } + + control, err := cgroups.Load(subsystem, func(subsystem cgroups.Name) (string, error) { + return name, nil + }) + if err != nil { + log.Printf("cgroups load: %s", err) + continue + } + stats, err := control.Stat(cgroups.IgnoreNotExist) + if err != nil { + log.Printf("control stat: %s", err) + continue + } + groups[name] = stats + } + return groups, nil +} + +type dockerStats struct { + CPU types.CPUStats `json:"cpu_stats,omitempty"` + PreCPU types.CPUStats `json:"precpu_stats,omitempty"` // "Pre"="Previous" + Memory types.MemoryStats `json:"memory_stats,omitempty"` +} + +func statsDockerContainers(ctx context.Context, dockerClient *client.Client) (map[string]dockerStats, error) { + if dockerClient == nil { + return nil, nil + } + containers, err := dockerClient.ContainerList(ctx, types.ContainerListOptions{ + All: true, + Limit: 0, + }) + if err != nil { + return nil, fmt.Errorf("list docker containers: %s", err) + } + + dockerContainers := make(map[string]dockerStats, len(containers)) + for _, container := range containers { + res, err := dockerClient.ContainerStats(ctx, container.ID, false) + if err != nil { + log.Printf("failed to stats docker container %s: %s", container.ID, err) + continue + } + var stats dockerStats + if err := json.NewDecoder(res.Body).Decode(&stats); err != nil { + res.Body.Close() + return nil, fmt.Errorf("failed to decode stats json: %s", err) + } + res.Body.Close() + name := fmt.Sprintf("/docker%s", strings.Join(container.Names, "/")) + dockerContainers[name] = stats + } + + return dockerContainers, nil +} + +func exportMetrics(system cgroups.Cgroup, dockerClient *client.Client) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { - processes, err := system.Processes(cgroups.Devices, true) + ctx := r.Context() + + groups, err := statsCgroups(ctx, system) if err != nil { - msg := fmt.Sprintf("cgroups load: %s", err) - http.Error(w, msg, http.StatusInternalServerError) + http.Error(w, err.Error(), http.StatusInternalServerError) + return } - groups := make(map[string]*cgroups.Metrics, len(processes)) - for _, p := range processes { - name := strings.TrimPrefix(p.Path, "/sys/fs/cgroup/devices") - name = strings.TrimSuffix(name, "/") - if _, ok := groups[name]; ok { - continue - } - - control, err := cgroups.Load(subsystem, func(subsystem cgroups.Name) (string, error) { - return name, nil - }) - if err != nil { - log.Printf("cgroups load: %s", err) - continue - } - stats, err := control.Stat(cgroups.IgnoreNotExist) - if err != nil { - log.Printf("control stat: %s", err) - continue - } - groups[name] = stats + dockerContainers, err := statsDockerContainers(ctx, dockerClient) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return } fmt.Fprintln(w, `# HELP container_cpu_user_seconds_total Cumulative user cpu time consumed in seconds. @@ -106,6 +173,10 @@ func exportMetrics(system cgroups.Cgroup) func(w http.ResponseWriter, r *http.Re fmt.Fprintf(w, `container_cpu_user_seconds_total{id=%s} %.2f`, strconv.Quote(name), float64(stats.CPU.Usage.User)/1000000000.0) fmt.Fprintln(w) } + for name, stats := range dockerContainers { + fmt.Fprintf(w, `container_cpu_user_seconds_total{id=%s} %.2f`, strconv.Quote(name), float64(stats.CPU.CPUUsage.UsageInUsermode)/1000000000.0) + fmt.Fprintln(w) + } fmt.Fprintln(w, `# HELP container_memory_usage_bytes Current memory usage in bytes, including all memory regardless of when it was accessed # TYPE container_memory_usage_bytes gauge`) @@ -113,6 +184,10 @@ func exportMetrics(system cgroups.Cgroup) func(w http.ResponseWriter, r *http.Re fmt.Fprintf(w, `container_memory_usage_bytes{id=%s} %d`, strconv.Quote(name), stats.Memory.Usage.Usage) fmt.Fprintln(w) } + for name, stats := range dockerContainers { + fmt.Fprintf(w, `container_memory_usage_bytes{id=%s} %d`, strconv.Quote(name), stats.Memory.Usage) + fmt.Fprintln(w) + } fmt.Fprintln(w, `# HELP container_memory_rss Size of RSS in bytes. # TYPE container_memory_rss gauge`) @@ -120,6 +195,10 @@ func exportMetrics(system cgroups.Cgroup) func(w http.ResponseWriter, r *http.Re fmt.Fprintf(w, `container_memory_rss{id=%s} %d`, strconv.Quote(name), stats.Memory.RSS) fmt.Fprintln(w) } + for name, stats := range dockerContainers { + fmt.Fprintf(w, `container_memory_rss{id=%s} %d`, strconv.Quote(name), stats.Memory.Stats["rss"]) + fmt.Fprintln(w) + } return }