Skip to content

Commit

Permalink
Revert "fix: get container ID from kube rather than docker (aws#371)"
Browse files Browse the repository at this point in the history
This reverts commit 14de538.
  • Loading branch information
drakedevel committed Nov 11, 2019
1 parent 7eab704 commit de8e10f
Show file tree
Hide file tree
Showing 8 changed files with 228 additions and 9 deletions.
5 changes: 5 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ require (
github.com/coreos/go-iptables v0.4.0
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/deckarep/golang-set v1.7.1
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v1.13.1
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/gogo/protobuf v1.1.1 // indirect
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
Expand All @@ -27,6 +31,7 @@ require (
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
github.com/onsi/ginkgo v1.8.0 // indirect
github.com/onsi/gomega v1.5.0 // indirect
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
github.com/operator-framework/operator-sdk v0.0.7
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pkg/errors v0.8.0
Expand Down
10 changes: 10 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,14 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ=
github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
Expand Down Expand Up @@ -79,6 +87,8 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/operator-framework/operator-sdk v0.0.7 h1:feujqHLhibLBbDVrSAFswpSzTVS5mEuarvywJ079mYE=
github.com/operator-framework/operator-sdk v0.0.7/go.mod h1:iVyukRkam5JZa8AnjYf+/G3rk7JI1+M6GsU0sq0B9NA=
github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
Expand Down
24 changes: 24 additions & 0 deletions ipamd/ipamd.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (

"github.com/aws/amazon-vpc-cni-k8s/ipamd/datastore"
"github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils"
"github.com/aws/amazon-vpc-cni-k8s/pkg/docker"
"github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig"
"github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi"
"github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils"
Expand Down Expand Up @@ -165,6 +166,7 @@ type IPAMContext struct {
k8sClient k8sapi.K8SAPIs
useCustomNetworking bool
eniConfig eniconfig.ENIConfig
dockerClient docker.APIs
networkClient networkutils.NetworkAPIs
maxIPsPerENI int
maxENI int
Expand Down Expand Up @@ -237,6 +239,7 @@ func New(k8sapiClient k8sapi.K8SAPIs, eniConfig *eniconfig.ENIConfigController)

c.k8sClient = k8sapiClient
c.networkClient = networkutils.New()
c.dockerClient = docker.New()
c.eniConfig = eniConfig

client, err := awsutils.New()
Expand Down Expand Up @@ -420,6 +423,27 @@ func (c *IPAMContext) getLocalPodsWithRetry() ([]*k8sapi.K8SPodInfo, error) {
return nil, nil
}

var containers map[string]*docker.ContainerInfo
for retry := 1; retry <= maxK8SRetries; retry++ {
containers, err = c.dockerClient.GetRunningContainers()
if err == nil {
break
}
log.Infof("Not able to get local containers yet (attempt %d/%d): %v", retry, maxK8SRetries, err)
time.Sleep(retryK8SInterval)
}

// TODO consider using map
for _, pod := range pods {
// needs to find the container ID
for _, container := range containers {
if container.K8SUID == pod.UID {
log.Debugf("Found pod(%v)'s container ID: %v ", container.Name, container.ID)
pod.Container = container.ID
break
}
}
}
return pods, nil
}

Expand Down
27 changes: 19 additions & 8 deletions ipamd/ipamd_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ import (
"github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1"
"github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils"
mock_awsutils "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils/mocks"
"github.com/aws/amazon-vpc-cni-k8s/pkg/docker"
mock_docker "github.com/aws/amazon-vpc-cni-k8s/pkg/docker/mocks"
mock_eniconfig "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig/mocks"
"github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi"
mock_k8sapi "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi/mocks"
Expand Down Expand Up @@ -57,18 +59,20 @@ const (
func setup(t *testing.T) (*gomock.Controller,
*mock_awsutils.MockAPIs,
*mock_k8sapi.MockK8SAPIs,
*mock_docker.MockAPIs,
*mock_networkutils.MockNetworkAPIs,
*mock_eniconfig.MockENIConfig) {
ctrl := gomock.NewController(t)
return ctrl,
mock_awsutils.NewMockAPIs(ctrl),
mock_k8sapi.NewMockK8SAPIs(ctrl),
mock_docker.NewMockAPIs(ctrl),
mock_networkutils.NewMockNetworkAPIs(ctrl),
mock_eniconfig.NewMockENIConfig(ctrl)
}

func TestNodeInit(t *testing.T) {
ctrl, mockAWS, mockK8S, mockNetwork, _ := setup(t)
ctrl, mockAWS, mockK8S, mockDocker, mockNetwork, _ := setup(t)
defer ctrl.Finish()

mockContext := &IPAMContext{
Expand All @@ -80,6 +84,7 @@ func TestNodeInit(t *testing.T) {
warmIPTarget: 3,
primaryIP: make(map[string]string),
terminating: int32(0),
dockerClient: mockDocker,
networkClient: mockNetwork}

eni1 := awsutils.ENIMetadata{
Expand Down Expand Up @@ -139,8 +144,14 @@ func TestNodeInit(t *testing.T) {
mockNetwork.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet)

mockAWS.EXPECT().GetLocalIPv4().Return(ipaddr01)
k8sName := "/k8s_POD_" + "pod1" + "_" + "default" + "_" + "pod-uid" + "_0"
mockK8S.EXPECT().K8SGetLocalPodIPs().Return([]*k8sapi.K8SPodInfo{{Name: "pod1",
Namespace: "default", UID: "pod-uid", Container: "container-uid", IP: ipaddr02}}, nil)
Namespace: "default", UID: "pod-uid", IP: ipaddr02}}, nil)

var dockerList = make(map[string]*docker.ContainerInfo, 0)
dockerList["pod-uid"] = &docker.ContainerInfo{ID: "docker-id",
Name: k8sName, K8SUID: "pod-uid"}
mockDocker.EXPECT().GetRunningContainers().Return(dockerList, nil)

var rules []netlink.Rule
mockNetwork.EXPECT().GetRuleList().Return(rules, nil)
Expand All @@ -167,7 +178,7 @@ func TestIncreaseIPPoolCustomENI(t *testing.T) {
}

func testIncreaseIPPool(t *testing.T, useENIConfig bool) {
ctrl, mockAWS, mockK8S, mockNetwork, mockENIConfig := setup(t)
ctrl, mockAWS, mockK8S, _, mockNetwork, mockENIConfig := setup(t)
defer ctrl.Finish()

mockContext := &IPAMContext{
Expand Down Expand Up @@ -244,7 +255,7 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool) {

func TestTryAddIPToENI(t *testing.T) {
_ = os.Unsetenv(envCustomNetworkCfg)
ctrl, mockAWS, mockK8S, mockNetwork, mockENIConfig := setup(t)
ctrl, mockAWS, mockK8S, _, mockNetwork, mockENIConfig := setup(t)
defer ctrl.Finish()

warmIpTarget := 3
Expand Down Expand Up @@ -309,7 +320,7 @@ func TestTryAddIPToENI(t *testing.T) {
}

func TestNodeIPPoolReconcile(t *testing.T) {
ctrl, mockAWS, mockK8S, mockNetwork, _ := setup(t)
ctrl, mockAWS, mockK8S, _, mockNetwork, _ := setup(t)
defer ctrl.Finish()

mockContext := &IPAMContext{
Expand Down Expand Up @@ -380,7 +391,7 @@ func TestNodeIPPoolReconcile(t *testing.T) {
}

func TestGetWarmENITarget(t *testing.T) {
ctrl, _, _, _, _ := setup(t)
ctrl, _, _, _, _, _ := setup(t)
defer ctrl.Finish()

_ = os.Setenv("WARM_IP_TARGET", "5")
Expand All @@ -397,7 +408,7 @@ func TestGetWarmENITarget(t *testing.T) {
}

func TestGetWarmIPTargetState(t *testing.T) {
ctrl, mockAWS, mockK8S, mockNetwork, _ := setup(t)
ctrl, mockAWS, mockK8S, _, mockNetwork, _ := setup(t)
defer ctrl.Finish()

mockContext := &IPAMContext{
Expand Down Expand Up @@ -441,7 +452,7 @@ func TestGetWarmIPTargetState(t *testing.T) {
}

func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) {
ctrl, mockAWS, mockK8S, mockNetwork, mockENIConfig := setup(t)
ctrl, mockAWS, mockK8S, _, mockNetwork, mockENIConfig := setup(t)
defer ctrl.Finish()

type fields struct {
Expand Down
3 changes: 2 additions & 1 deletion ipamd/rpc_handler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
)

func TestServer_AddNetwork(t *testing.T) {
ctrl, mockAWS, mockK8S, mockNetwork, _ := setup(t)
ctrl, mockAWS, mockK8S, mockDocker, mockNetwork, _ := setup(t)
defer ctrl.Finish()

mockContext := &IPAMContext{
Expand All @@ -36,6 +36,7 @@ func TestServer_AddNetwork(t *testing.T) {
maxENI: 4,
warmENITarget: 1,
warmIPTarget: 3,
dockerClient: mockDocker,
networkClient: mockNetwork,
dataStore: datastore.NewDataStore(),
}
Expand Down
91 changes: 91 additions & 0 deletions pkg/docker/docker.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package docker

import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
"golang.org/x/net/context"

"github.com/pkg/errors"

log "github.com/cihub/seelog"
)

// ContainerInfo provides container information
type ContainerInfo struct {
ID string
Name string
K8SUID string
}

// APIs provides Docker API
type APIs interface {
GetRunningContainers() (map[string]*ContainerInfo, error)
}

type Client struct{}

func New() *Client {
return &Client{}
}

func (c *Client) GetRunningContainers() (map[string]*ContainerInfo, error) {
var containerInfos = make(map[string]*ContainerInfo)

cli, err := client.NewEnvClient()
if err != nil {
return nil, err
}
defer cli.Close()

containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{})
if err != nil {
return nil, err
}

for _, container := range containers {
log.Infof("GetRunningContainers: Discovered running docker: %s %s %s State: %s Status: %s ",
container.ID, container.Names[0], container.Labels["io.kubernetes.pod.uid"], container.State, container.Status)

containerType, ok := container.Labels["io.kubernetes.docker.type"]
if !ok {
log.Infof("GetRunningContainers: skip non pause container")
continue
}

if containerType != "podsandbox" {
log.Infof("GetRunningContainers: skip container type: %s", containerType)
continue
}

log.Debugf("GetRunningContainers: containerType %s", containerType)

if container.State != "running" {
log.Infof("GetRunningContainers: skip container who is not running")
continue
}

uid := container.Labels["io.kubernetes.pod.uid"]
_, ok = containerInfos[uid]
if !ok {
containerInfos[uid] = &ContainerInfo{
ID: container.ID,
Name: container.Names[0],
K8SUID: uid}
continue
}

if container.Names[0] != containerInfos[uid].Name {
log.Infof("GetRunningContainers: same uid matched by container:%s, %s container id %s",
containerInfos[uid].Name, container.Names[0], container.ID)
continue
}

if container.Names[0] == containerInfos[uid].Name {
log.Errorf("GetRunningContainers: Conflict container id %s for container %s",
container.ID, containerInfos[uid].Name)
return nil, errors.New("conflict docker runtime info")
}
}

return containerInfos, nil
}
16 changes: 16 additions & 0 deletions pkg/docker/generate_mocks.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
// http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package docker

//go:generate go run ../../scripts/mockgen.go github.com/aws/amazon-vpc-cni-k8s/pkg/docker APIs mocks/docker_mocks.go
61 changes: 61 additions & 0 deletions pkg/docker/mocks/docker_mocks.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit de8e10f

Please sign in to comment.