From 6e0decbe03bfc819c018d435e8d908170e32ad29 Mon Sep 17 00:00:00 2001 From: Matt Heon Date: Tue, 6 Feb 2024 08:24:28 -0500 Subject: [PATCH] Send container stats over API on a per-interface basis This mirrors how the Docker API handles things, allowing us to be more compatible with Docker and more verbose on the Libpod API. Stats are given as per network interface in the container, but still aggregated for `podman stats` and `podman pod stats` display (so the CLI does not change, only the Libpod and Compat APIs). Signed-off-by: Matt Heon --- cmd/podman/containers/stats.go | 10 +++- docs/source/markdown/podman-stats.1.md.in | 3 +- libpod/define/containerstate.go | 26 +++++++--- libpod/networking_freebsd.go | 51 +++++-------------- libpod/networking_linux.go | 33 +++++++----- libpod/stats_common.go | 6 +++ libpod/stats_freebsd.go | 14 ----- libpod/stats_linux.go | 12 ----- .../handlers/compat/containers_stats_linux.go | 29 +++++------ pkg/domain/infra/abi/pods_stats.go | 9 +++- test/apiv2/19-stats.at | 18 +++++++ 11 files changed, 107 insertions(+), 104 deletions(-) diff --git a/cmd/podman/containers/stats.go b/cmd/podman/containers/stats.go index ed462a7af662..a0467957b900 100644 --- a/cmd/podman/containers/stats.go +++ b/cmd/podman/containers/stats.go @@ -215,7 +215,15 @@ func (s *containerStats) MemPerc() string { } func (s *containerStats) NetIO() string { - return combineHumanValues(s.NetInput, s.NetOutput) + var netInput uint64 + var netOutput uint64 + + for _, net := range s.Network { + netInput += net.RxBytes + netOutput += net.TxBytes + } + + return combineHumanValues(netInput, netOutput) } func (s *containerStats) BlockIO() string { diff --git a/docs/source/markdown/podman-stats.1.md.in b/docs/source/markdown/podman-stats.1.md.in index 5e96224da1f4..f44a7f8ef51c 100644 --- a/docs/source/markdown/podman-stats.1.md.in +++ b/docs/source/markdown/podman-stats.1.md.in @@ -50,9 +50,8 @@ Valid placeholders for the Go template are listed below: | .MemUsage | Memory usage | | .MemUsageBytes | Memory usage (IEC) | | .Name | Container Name | -| .NetInput | Network Input | | .NetIO | Network IO | -| .NetOutput | Network Output | +| .Network | Network I/O, separated by network interface | | .PerCPU | CPU time consumed by all tasks [1] | | .PIDs | Number of PIDs | | .PIDS | Number of PIDs (yes, we know this is a dup) | diff --git a/libpod/define/containerstate.go b/libpod/define/containerstate.go index 4520dc41db17..e9679888617a 100644 --- a/libpod/define/containerstate.go +++ b/libpod/define/containerstate.go @@ -141,11 +141,23 @@ type ContainerStats struct { MemUsage uint64 MemLimit uint64 MemPerc float64 - NetInput uint64 - NetOutput uint64 - BlockInput uint64 - BlockOutput uint64 - PIDs uint64 - UpTime time.Duration - Duration uint64 + // Map of interface name to network statistics for that interface. + Network map[string]ContainerNetworkStats + BlockInput uint64 + BlockOutput uint64 + PIDs uint64 + UpTime time.Duration + Duration uint64 +} + +// Statistics for an individual container network interface +type ContainerNetworkStats struct { + RxBytes uint64 + RxDropped uint64 + RxErrors uint64 + RxPackets uint64 + TxBytes uint64 + TxDropped uint64 + TxErrors uint64 + TxPackets uint64 } diff --git a/libpod/networking_freebsd.go b/libpod/networking_freebsd.go index ff894803bbe8..c5f4667ba10e 100644 --- a/libpod/networking_freebsd.go +++ b/libpod/networking_freebsd.go @@ -13,6 +13,7 @@ import ( "github.com/containers/buildah/pkg/jail" "github.com/containers/common/libnetwork/types" + "github.com/containers/podman/v4/libpod/define" "github.com/containers/storage/pkg/lockfile" "github.com/sirupsen/logrus" ) @@ -45,33 +46,6 @@ type NetstatAddress struct { Collisions uint64 `json:"collisions"` } -// copied from github.com/vishvanada/netlink which does not build on freebsd -type LinkStatistics64 struct { - RxPackets uint64 - TxPackets uint64 - RxBytes uint64 - TxBytes uint64 - RxErrors uint64 - TxErrors uint64 - RxDropped uint64 - TxDropped uint64 - Multicast uint64 - Collisions uint64 - RxLengthErrors uint64 - RxOverErrors uint64 - RxCrcErrors uint64 - RxFrameErrors uint64 - RxFifoErrors uint64 - RxMissedErrors uint64 - TxAbortedErrors uint64 - TxCarrierErrors uint64 - TxFifoErrors uint64 - TxHeartbeatErrors uint64 - TxWindowErrors uint64 - RxCompressed uint64 - TxCompressed uint64 -} - type RootlessNetNS struct { dir string Lock *lockfile.LockFile @@ -223,7 +197,7 @@ func (r *Runtime) teardownNetNS(ctr *Container) error { // TODO (5.0): return the statistics per network interface // This would allow better compat with docker. -func getContainerNetIO(ctr *Container) (*LinkStatistics64, error) { +func getContainerNetIO(ctr *Container) (map[string]define.ContainerNetworkStats, error) { if ctr.state.NetNS == "" { // If NetNS is nil, it was set as none, and no netNS // was set up this is a valid state and thus return no @@ -249,8 +223,9 @@ func getContainerNetIO(ctr *Container) (*LinkStatistics64, error) { return nil, err } + res := make(map[string]define.ContainerNetworkStats) + // Sum all the interface stats - in practice only Tx/TxBytes are needed - res := &LinkStatistics64{} for _, ifaddr := range stats.Statistics.Interface { // Each interface has two records, one for link-layer which has // an MTU field and one for IP which doesn't. We only want the @@ -260,14 +235,16 @@ func getContainerNetIO(ctr *Container) (*LinkStatistics64, error) { // if we move to per-interface stats in future, this can be // reported separately. if ifaddr.Mtu > 0 { - res.RxPackets += ifaddr.ReceivedPackets - res.TxPackets += ifaddr.SentPackets - res.RxBytes += ifaddr.ReceivedBytes - res.TxBytes += ifaddr.SentBytes - res.RxErrors += ifaddr.ReceivedErrors - res.TxErrors += ifaddr.SentErrors - res.RxDropped += ifaddr.DroppedPackets - res.Collisions += ifaddr.Collisions + linkStats := define.ContainerNetworkStats{ + RxPackets: ifaddr.ReceivedPackets, + TxPackets: ifaddr.SentPackets, + RxBytes: ifaddr.ReceivedBytes, + TxBytes: ifaddr.SentBytes, + RxErrors: ifaddr.ReceivedErrors, + TxErrors: ifaddr.SentErrors, + RxDropped: ifaddr.DroppedPackets, + } + res[ifaddr.Name] = linkStats } } diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go index 977fdf108a82..c19631a445fa 100644 --- a/libpod/networking_linux.go +++ b/libpod/networking_linux.go @@ -13,6 +13,7 @@ import ( "github.com/containers/common/libnetwork/types" netUtil "github.com/containers/common/libnetwork/util" "github.com/containers/common/pkg/netns" + "github.com/containers/podman/v4/libpod/define" "github.com/containers/podman/v4/pkg/rootless" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" @@ -186,10 +187,9 @@ func getContainerNetNS(ctr *Container) (string, *Container, error) { return "", nil, nil } -// TODO (5.0): return the statistics per network interface -// This would allow better compat with docker. -func getContainerNetIO(ctr *Container) (*netlink.LinkStatistics, error) { - var netStats *netlink.LinkStatistics +// Returns a map of interface name to statistics for that interface. +func getContainerNetIO(ctr *Container) (map[string]define.ContainerNetworkStats, error) { + perNetworkStats := make(map[string]define.ContainerNetworkStats) netNSPath, otherCtr, netPathErr := getContainerNetNS(ctr) if netPathErr != nil { @@ -222,21 +222,26 @@ func getContainerNetIO(ctr *Container) (*netlink.LinkStatistics, error) { if err != nil { return err } - if netStats == nil { - netStats = link.Attrs().Statistics - continue - } - // Currently only Tx/RxBytes are used. - // In the future we should return all stats per interface so that - // api users have a better options. stats := link.Attrs().Statistics - netStats.TxBytes += stats.TxBytes - netStats.RxBytes += stats.RxBytes + if stats != nil { + newStats := define.ContainerNetworkStats{ + RxBytes: stats.RxBytes, + RxDropped: stats.RxDropped, + RxErrors: stats.RxErrors, + RxPackets: stats.RxPackets, + TxBytes: stats.TxBytes, + TxDropped: stats.TxDropped, + TxErrors: stats.TxErrors, + TxPackets: stats.TxPackets, + } + + perNetworkStats[dev] = newStats + } } } return nil }) - return netStats, err + return perNetworkStats, err } // joinedNetworkNSPath returns netns path and bool if netns was set diff --git a/libpod/stats_common.go b/libpod/stats_common.go index 338f67a4ea8d..015192dc72a0 100644 --- a/libpod/stats_common.go +++ b/libpod/stats_common.go @@ -41,6 +41,12 @@ func (c *Container) GetContainerStats(previousStats *define.ContainerStats) (*de } } + netStats, err := getContainerNetIO(c) + if err != nil { + return nil, err + } + stats.Network = netStats + if err := c.getPlatformContainerStats(stats, previousStats); err != nil { return nil, err } diff --git a/libpod/stats_freebsd.go b/libpod/stats_freebsd.go index 3945e977ee6d..30bb5acad0cc 100644 --- a/libpod/stats_freebsd.go +++ b/libpod/stats_freebsd.go @@ -80,20 +80,6 @@ func (c *Container) getPlatformContainerStats(stats *define.ContainerStats, prev stats.MemLimit = c.getMemLimit() stats.SystemNano = now - netStats, err := getContainerNetIO(c) - if err != nil { - return err - } - - // Handle case where the container is not in a network namespace - if netStats != nil { - stats.NetInput = netStats.RxBytes - stats.NetOutput = netStats.TxBytes - } else { - stats.NetInput = 0 - stats.NetOutput = 0 - } - return nil } diff --git a/libpod/stats_linux.go b/libpod/stats_linux.go index ad8dfc45f4e8..839c04fc0951 100644 --- a/libpod/stats_linux.go +++ b/libpod/stats_linux.go @@ -39,10 +39,6 @@ func (c *Container) getPlatformContainerStats(stats *define.ContainerStats, prev return fmt.Errorf("unable to obtain cgroup stats: %w", err) } conState := c.state.State - netStats, err := getContainerNetIO(c) - if err != nil { - return err - } // If the current total usage in the cgroup is less than what was previously // recorded then it means the container was restarted and runs in a new cgroup @@ -69,14 +65,6 @@ func (c *Container) getPlatformContainerStats(stats *define.ContainerStats, prev stats.CPUSystemNano = cgroupStats.CpuStats.CpuUsage.UsageInKernelmode stats.SystemNano = now stats.PerCPU = cgroupStats.CpuStats.CpuUsage.PercpuUsage - // Handle case where the container is not in a network namespace - if netStats != nil { - stats.NetInput = netStats.RxBytes - stats.NetOutput = netStats.TxBytes - } else { - stats.NetInput = 0 - stats.NetOutput = 0 - } return nil } diff --git a/pkg/api/handlers/compat/containers_stats_linux.go b/pkg/api/handlers/compat/containers_stats_linux.go index f94fe65f1f50..79f818661431 100644 --- a/pkg/api/handlers/compat/containers_stats_linux.go +++ b/pkg/api/handlers/compat/containers_stats_linux.go @@ -119,23 +119,20 @@ streamLabel: // A label to flatten the scope return } - // FIXME: network inspection does not yet work entirely net := make(map[string]docker.NetworkStats) - networkName := inspect.NetworkSettings.EndpointID - if networkName == "" { - networkName = "network" - } - net[networkName] = docker.NetworkStats{ - RxBytes: stats.NetInput, - RxPackets: 0, - RxErrors: 0, - RxDropped: 0, - TxBytes: stats.NetOutput, - TxPackets: 0, - TxErrors: 0, - TxDropped: 0, - EndpointID: inspect.NetworkSettings.EndpointID, - InstanceID: "", + for netName, netStats := range stats.Network { + net[netName] = docker.NetworkStats{ + RxBytes: netStats.RxBytes, + RxPackets: netStats.RxPackets, + RxErrors: netStats.RxErrors, + RxDropped: netStats.RxDropped, + TxBytes: netStats.TxBytes, + TxPackets: netStats.TxPackets, + TxErrors: netStats.TxErrors, + TxDropped: netStats.TxDropped, + EndpointID: inspect.NetworkSettings.EndpointID, + InstanceID: "", + } } resources := ctnr.LinuxResources() diff --git a/pkg/domain/infra/abi/pods_stats.go b/pkg/domain/infra/abi/pods_stats.go index 447ceab9de7b..1a18c18e6fac 100644 --- a/pkg/domain/infra/abi/pods_stats.go +++ b/pkg/domain/infra/abi/pods_stats.go @@ -44,12 +44,19 @@ func (ic *ContainerEngine) podsToStatsReport(pods []*libpod.Pod) ([]*entities.Po } podID := pods[i].ID()[:12] for j := range podStats { + var podNetInput uint64 + var podNetOutput uint64 + for _, stats := range podStats[j].Network { + podNetInput += stats.RxBytes + podNetOutput += stats.TxBytes + } + r := entities.PodStatsReport{ CPU: floatToPercentString(podStats[j].CPU), MemUsage: combineHumanValues(podStats[j].MemUsage, podStats[j].MemLimit), MemUsageBytes: combineBytesValues(podStats[j].MemUsage, podStats[j].MemLimit), Mem: floatToPercentString(podStats[j].MemPerc), - NetIO: combineHumanValues(podStats[j].NetInput, podStats[j].NetOutput), + NetIO: combineHumanValues(podNetInput, podNetOutput), BlockIO: combineHumanValues(podStats[j].BlockInput, podStats[j].BlockOutput), PIDS: pidsToString(podStats[j].PIDs), CID: podStats[j].ContainerID[:12], diff --git a/test/apiv2/19-stats.at b/test/apiv2/19-stats.at index 8d8a9ef5906a..2ca4b02dafdc 100644 --- a/test/apiv2/19-stats.at +++ b/test/apiv2/19-stats.at @@ -9,3 +9,21 @@ if root; then # regression for https://github.com/containers/podman/issues/15754 t GET libpod/containers/container1/stats?stream=false 200 .cpu_stats.online_cpus=1 fi + +podman run -dt --name testctr1 $IMAGE top &>/dev/null + +t GET libpod/containers/testctr1/stats?stream=false 200 '.networks | length'=1 + +podman rm -f testctr1 + +podman network create testnet1 +podman network create testnet2 + +podman run -dt --name testctr2 --net testnet1,testnet2 $IMAGE top &>/dev/null + +t GET libpod/containers/testctr2/stats?stream=false 200 '.networks | length'=2 + +podman rm -f testctr2 + +podman network rm testnet1 +podman network rm testnet2