Skip to content

Commit

Permalink
Merge pull request kubernetes#60328 from Random-Liu/fix-pod-stats
Browse files Browse the repository at this point in the history 
Automatic merge from submit-queue (batch tested with PRs 60011, 59256, 59293, 60328, 60367). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add CPU/Memory pod stats for CRI stats.

For kubernetes/enhancements#286.

Add CPU and memory stats for pod.

@kubernetes/sig-node-pr-reviews 
/cc @dashpole @yujuhong @abhi @yguo0905 
Signed-off-by: Lantao Liu <lantaol@google.com>



**What this PR does / why we need it**:

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:

**Release note**:

```release-note
Summary API will include pod CPU and Memory stats for CRI container runtime.
```
  • Loading branch information
Kubernetes Submit Queue committed Feb 27, 2018
2 parents d615319 + 3688650 commit 3039a77
Show file tree
Hide file tree
Showing 4 changed files with 55 additions and 22 deletions.
1 change: 1 addition & 0 deletions pkg/kubelet/stats/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ go_test(
"//pkg/kubelet/apis/cri/testing:go_default_library",
"//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
"//pkg/kubelet/cadvisor/testing:go_default_library",
"//pkg/kubelet/cm:go_default_library",
"//pkg/kubelet/container:go_default_library",
"//pkg/kubelet/container/testing:go_default_library",
"//pkg/kubelet/kuberuntime:go_default_library",
Expand Down
6 changes: 3 additions & 3 deletions pkg/kubelet/stats/cadvisor_stats_provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
}
podStats.EphemeralStorage = calcEphemeralStorage(podStats.Containers, ephemeralStats, &rootFsInfo)
// Lookup the pod-level cgroup's CPU and memory stats
podInfo := getcadvisorPodInfoFromPodUID(podUID, allInfos)
podInfo := getCadvisorPodInfoFromPodUID(podUID, allInfos)
if podInfo != nil {
cpu, memory := cadvisorInfoToCPUandMemoryStats(podInfo)
podStats.CPU = cpu
Expand Down Expand Up @@ -251,8 +251,8 @@ func isPodManagedContainer(cinfo *cadvisorapiv2.ContainerInfo) bool {
return managed
}

// getcadvisorPodInfoFromPodUID returns a pod cgroup information by matching the podUID with its CgroupName identifier base name
func getcadvisorPodInfoFromPodUID(podUID types.UID, infos map[string]cadvisorapiv2.ContainerInfo) *cadvisorapiv2.ContainerInfo {
// getCadvisorPodInfoFromPodUID returns a pod cgroup information by matching the podUID with its CgroupName identifier base name
func getCadvisorPodInfoFromPodUID(podUID types.UID, infos map[string]cadvisorapiv2.ContainerInfo) *cadvisorapiv2.ContainerInfo {
for key, info := range infos {
if cm.IsSystemdStyleName(key) {
key = cm.RevertFromSystemdToCgroupStyleName(key)
Expand Down
48 changes: 29 additions & 19 deletions pkg/kubelet/stats/cri_stats_provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,23 +119,22 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
containerMap[c.Id] = c
}

caInfos, err := getCRICadvisorStats(p.cadvisor)
allInfos, err := getCadvisorContainerInfo(p.cadvisor)
if err != nil {
return nil, fmt.Errorf("failed to get container info from cadvisor: %v", err)
return nil, fmt.Errorf("failed to fetch cadvisor stats: %v", err)
}
caInfos := getCRICadvisorStats(allInfos)

for _, stats := range resp {
containerID := stats.Attributes.Id
container, found := containerMap[containerID]
if !found {
glog.Errorf("Unable to find container id %q in container stats list", containerID)
continue
}

podSandboxID := container.PodSandboxId
podSandbox, found := podSandboxMap[podSandboxID]
if !found {
glog.Errorf("Unable to find pod sandbox id %q in pod stats list", podSandboxID)
continue
}

Expand All @@ -145,12 +144,8 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
if !found {
ps = buildPodStats(podSandbox)
// Fill stats from cadvisor is available for full set of required pod stats
caPodSandbox, found := caInfos[podSandboxID]
if !found {
glog.V(4).Infof("Unable to find cadvisor stats for sandbox %q", podSandboxID)
} else {
p.addCadvisorPodStats(ps, &caPodSandbox)
}
p.addCadvisorPodNetworkStats(ps, podSandboxID, caInfos)
p.addCadvisorPodCPUMemoryStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos)
sandboxIDToPodStats[podSandboxID] = ps
}
cs := p.makeContainerStats(stats, container, &rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata().GetUid())
Expand Down Expand Up @@ -269,11 +264,30 @@ func (p *criStatsProvider) makePodStorageStats(s *statsapi.PodStats, rootFsInfo
return s
}

func (p *criStatsProvider) addCadvisorPodStats(
func (p *criStatsProvider) addCadvisorPodNetworkStats(
ps *statsapi.PodStats,
caPodSandbox *cadvisorapiv2.ContainerInfo,
podSandboxID string,
caInfos map[string]cadvisorapiv2.ContainerInfo,
) {
ps.Network = cadvisorInfoToNetworkStats(ps.PodRef.Name, caPodSandbox)
caPodSandbox, found := caInfos[podSandboxID]
if found {
ps.Network = cadvisorInfoToNetworkStats(ps.PodRef.Name, &caPodSandbox)
} else {
glog.V(4).Infof("Unable to find cadvisor stats for sandbox %q", podSandboxID)
}
}

func (p *criStatsProvider) addCadvisorPodCPUMemoryStats(
ps *statsapi.PodStats,
podUID types.UID,
allInfos map[string]cadvisorapiv2.ContainerInfo,
) {
podCgroupInfo := getCadvisorPodInfoFromPodUID(podUID, allInfos)
if podCgroupInfo != nil {
cpu, memory := cadvisorInfoToCPUandMemoryStats(podCgroupInfo)
ps.CPU = cpu
ps.Memory = memory
}
}

func (p *criStatsProvider) makeContainerStats(
Expand Down Expand Up @@ -395,12 +409,8 @@ func (p *criStatsProvider) addCadvisorContainerStats(
}
}

func getCRICadvisorStats(ca cadvisor.Interface) (map[string]cadvisorapiv2.ContainerInfo, error) {
func getCRICadvisorStats(infos map[string]cadvisorapiv2.ContainerInfo) map[string]cadvisorapiv2.ContainerInfo {
stats := make(map[string]cadvisorapiv2.ContainerInfo)
infos, err := getCadvisorContainerInfo(ca)
if err != nil {
return nil, fmt.Errorf("failed to fetch cadvisor stats: %v", err)
}
infos = removeTerminatedContainerInfo(infos)
for key, info := range infos {
// On systemd using devicemapper each mount into the container has an
Expand All @@ -416,7 +426,7 @@ func getCRICadvisorStats(ca cadvisor.Interface) (map[string]cadvisorapiv2.Contai
}
stats[path.Base(key)] = info
}
return stats, nil
return stats
}

// TODO Cache the metrics in container log manager
Expand Down
22 changes: 22 additions & 0 deletions pkg/kubelet/stats/cri_stats_provider_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
critest "k8s.io/kubernetes/pkg/kubelet/apis/cri/testing"
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
"k8s.io/kubernetes/pkg/kubelet/cm"
kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
"k8s.io/kubernetes/pkg/kubelet/leaky"
Expand Down Expand Up @@ -78,6 +79,7 @@ func TestCRIListPodStats(t *testing.T) {
rootFsInfo = getTestFsInfo(1000)

sandbox0 = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns")
sandbox0Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox0.PodSandboxStatus.Metadata.Uid))
container0 = makeFakeContainer(sandbox0, cName0, 0, false)
containerStats0 = makeFakeContainerStats(container0, imageFsMountpoint)
containerLogStats0 = makeFakeLogStats(1000)
Expand All @@ -86,11 +88,13 @@ func TestCRIListPodStats(t *testing.T) {
containerLogStats1 = makeFakeLogStats(2000)

sandbox1 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns")
sandbox1Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox1.PodSandboxStatus.Metadata.Uid))
container2 = makeFakeContainer(sandbox1, cName2, 0, false)
containerStats2 = makeFakeContainerStats(container2, imageFsMountpoint)
containerLogStats2 = makeFakeLogStats(3000)

sandbox2 = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns")
sandbox2Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox2.PodSandboxStatus.Metadata.Uid))
container3 = makeFakeContainer(sandbox2, cName3, 0, true)
containerStats3 = makeFakeContainerStats(container3, imageFsMountpoint)
container4 = makeFakeContainer(sandbox2, cName3, 1, false)
Expand All @@ -112,11 +116,14 @@ func TestCRIListPodStats(t *testing.T) {
"/kubelet": getTestContainerInfo(seedKubelet, "", "", ""),
"/system": getTestContainerInfo(seedMisc, "", "", ""),
sandbox0.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
sandbox0Cgroup: getTestContainerInfo(seedSandbox0, "", "", ""),
container0.ContainerStatus.Id: getTestContainerInfo(seedContainer0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName0),
container1.ContainerStatus.Id: getTestContainerInfo(seedContainer1, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName1),
sandbox1.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox1, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
sandbox1Cgroup: getTestContainerInfo(seedSandbox1, "", "", ""),
container2.ContainerStatus.Id: getTestContainerInfo(seedContainer2, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, cName2),
sandbox2.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox2, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, leaky.PodInfraContainerName),
sandbox2Cgroup: getTestContainerInfo(seedSandbox2, "", "", ""),
container4.ContainerStatus.Id: getTestContainerInfo(seedContainer3, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, cName3),
}

Expand Down Expand Up @@ -199,6 +206,7 @@ func TestCRIListPodStats(t *testing.T) {
checkCRIRootfsStats(assert, c1, containerStats1, nil)
checkCRILogsStats(assert, c1, &rootFsInfo, containerLogStats1)
checkCRINetworkStats(assert, p0.Network, infos[sandbox0.PodSandboxStatus.Id].Stats[0].Network)
checkCRIPodCPUAndMemoryStats(assert, p0, infos[sandbox0Cgroup].Stats[0])

p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}]
assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
Expand All @@ -212,6 +220,7 @@ func TestCRIListPodStats(t *testing.T) {
checkCRIRootfsStats(assert, c2, containerStats2, &imageFsInfo)
checkCRILogsStats(assert, c2, &rootFsInfo, containerLogStats2)
checkCRINetworkStats(assert, p1.Network, infos[sandbox1.PodSandboxStatus.Id].Stats[0].Network)
checkCRIPodCPUAndMemoryStats(assert, p1, infos[sandbox1Cgroup].Stats[0])

p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}]
assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano())
Expand All @@ -227,6 +236,7 @@ func TestCRIListPodStats(t *testing.T) {

checkCRILogsStats(assert, c3, &rootFsInfo, containerLogStats4)
checkCRINetworkStats(assert, p2.Network, infos[sandbox2.PodSandboxStatus.Id].Stats[0].Network)
checkCRIPodCPUAndMemoryStats(assert, p2, infos[sandbox2Cgroup].Stats[0])

mockCadvisor.AssertExpectations(t)
}
Expand Down Expand Up @@ -453,6 +463,18 @@ func checkCRINetworkStats(assert *assert.Assertions, actual *statsapi.NetworkSta
assert.Equal(expected.Interfaces[0].TxErrors, *actual.TxErrors)
}

func checkCRIPodCPUAndMemoryStats(assert *assert.Assertions, actual statsapi.PodStats, cs *cadvisorapiv2.ContainerStats) {
assert.Equal(cs.Timestamp.UnixNano(), actual.CPU.Time.UnixNano())
assert.Equal(cs.Cpu.Usage.Total, *actual.CPU.UsageCoreNanoSeconds)
assert.Equal(cs.CpuInst.Usage.Total, *actual.CPU.UsageNanoCores)

assert.Equal(cs.Memory.Usage, *actual.Memory.UsageBytes)
assert.Equal(cs.Memory.WorkingSet, *actual.Memory.WorkingSetBytes)
assert.Equal(cs.Memory.RSS, *actual.Memory.RSSBytes)
assert.Equal(cs.Memory.ContainerData.Pgfault, *actual.Memory.PageFaults)
assert.Equal(cs.Memory.ContainerData.Pgmajfault, *actual.Memory.MajorPageFaults)
}

func makeFakeLogStats(seed int) *volume.Metrics {
m := &volume.Metrics{}
m.Used = resource.NewQuantity(int64(seed+offsetUsage), resource.BinarySI)
Expand Down

0 comments on commit 3039a77

Please sign in to comment.