diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b9938799db..0bc54b5b6d4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,28 +5,46 @@ orbs: aws-cli: circleci/aws-cli@0.1.19 k8s: circleci/kubernetes@0.11.0 -jobs: - build: +references: + build_job: &build docker: - image: circleci/golang:1.13-stretch working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}} - environment: + environment: &env GO111MODULE: "on" steps: - checkout - run: go get -u golang.org/x/lint/golint - run: go get -u golang.org/x/tools/cmd/goimports - run: make check-format - - run: make build-linux - - run: make lint + - run: make build-linux "ARCH=$ARCH" + # TODO: Make lint a hard-check once baselined + - run: make lint LINT_FLAGS= - run: make vet - - run: make unit-test + # TODO: Enable architecture specific unit-test when Circle CI supports it! + - run: make unit-test ARCH=amd64 + +jobs: + build_x86_64: + <<: *build + environment: + <<: *env + # Target the GOARCH for x86_64 + ARCH: amd64 + + build_aarch64: + <<: *build + environment: + <<: *env + # Target the GOARCH for aarch64 + ARCH: arm64 + integration_test: docker: - image: circleci/golang:1.13-stretch working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}} environment: - GO111MODULE: "on" + <<: *env RUN_CONFORMANCE: "true" steps: - checkout @@ -47,11 +65,13 @@ workflows: version: 2 check: jobs: - - build + - build_x86_64 + - build_aarch64 - hold: type: approval requires: - - build + - build_x86_64 + - build_aarch64 - integration_test: requires: - hold diff --git a/.dockerignore b/.dockerignore index 1c3ff64b5e5..7e2cc191ac4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,6 @@ aws-cni aws-k8s-agent +cni-metrics-helper +grpc-health-probe portmap -cni-metrics-helper/cni-metrics-helper +routed-eni-cni-plugin diff --git a/.gitignore b/.gitignore index 7932a705dfd..8ccf2dc3c38 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,6 @@ verify-network .idea/ *.iml .DS_Store -cni-metrics-helper/cni-metrics-helper portmap grpc-health-probe +cni-metrics-helper diff --git a/Makefile b/Makefile index e51ecbb6a08..e0081f58f30 100644 --- a/Makefile +++ b/Makefile @@ -12,134 +12,224 @@ # language governing permissions and limitations under the License. # -.PHONY: all build-linux clean format check-format docker docker-build lint unit-test vet download-portmap build-docker-test build-metrics docker-metrics metrics-unit-test docker-metrics-test docker-vet - -IMAGE ?= amazon/amazon-k8s-cni -VERSION ?= $(shell git describe --tags --always --dirty) -LDFLAGS ?= -X main.version=$(VERSION) -DOCKER_ARGS ?= -ALLPKGS := $(shell go list ./...) - -ARCH ?= $(shell uname -m) +.PHONY: all dist check clean \ + lint format check-format vet docker-vet \ + build-linux docker \ + unit-test unit-test-race build-docker-test docker-func-test \ + build-metrics docker-metrics \ + metrics-unit-test docker-metrics-test + +# VERSION is the source revision that executables and images are built from. +VERSION = $(shell git describe --tags --always --dirty || echo "unknown") + +# DESTDIR is where distribution output (container images) is placed. +DESTDIR = . + +# IMAGE is the primary AWS VPC CNI plugin container image. +IMAGE = amazon/amazon-k8s-cni +IMAGE_NAME = $(IMAGE)$(IMAGE_ARCH_SUFFIX):$(VERSION) +IMAGE_DIST = $(DESTDIR)/$(subst /,_,$(IMAGE_NAME)).tar.gz +# METRICS_IMAGE is the CNI metrics publisher sidecar container image. +METRICS_IMAGE = amazon/cni-metrics-helper +METRICS_IMAGE_NAME = $(METRICS_IMAGE)$(IMAGE_ARCH_SUFFIX):$(VERSION) +METRICS_IMAGE_DIST = $(DESTDIR)/$(subst /,_,$(METRICS_IMAGE_NAME)).tar.gz +# TEST_IMAGE is the testing environment container image. +TEST_IMAGE = amazon-k8s-cni-test +TEST_IMAGE_NAME = $(TEST_IMAGE)$(IMAGE_ARCH_SUFFIX):$(VERSION) +# These values derive ARCH and DOCKER_ARCH which are needed by dependencies in +# image build defaulting to system's architecture when not specified. +# +# UNAME_ARCH is the runtime architecture of the building host. +UNAME_ARCH = $(shell uname -m) +# ARCH is the target architecture which is being built for. +# +# These are pairs of input_arch to derived_arch separated by colons: +ARCH = $(lastword $(subst :, ,$(filter $(UNAME_ARCH):%,x86_64:amd64 aarch64:arm64))) +# DOCKER_ARCH is the docker specific architecture specifier used for building on +# multiarch container images. +DOCKER_ARCH = $(lastword $(subst :, ,$(filter $(ARCH):%,amd64:amd64 arm64:arm64v8))) +# IMAGE_ARCH_SUFFIX is the `-arch` suffix included in the container image name. +# +# This is only applied to the arm64 container image by default. Override to +# provide an alternate suffix or to omit. +IMAGE_ARCH_SUFFIX = $(addprefix -,$(filter $(ARCH),arm64)) +# GOLANG_IMAGE is the building golang container image used. +GOLANG_IMAGE = golang:1.13-stretch +# For the requseted build, these are the set of Go specific build environment +# variables. +export GOARCH ?= $(ARCH) +export GOOS = linux +export CGO_ENABLED = 0 +# NOTE: Provided for local toolchains that require explicit module feature flag. +export GO111MODULE = on +export GOPROXY = direct + +# LDFLAGS is the set of flags used when building golang executables. +LDFLAGS = -X main.version=$(VERSION) +# ALLPKGS is the set of packages provided in source. +ALLPKGS = $(shell go list ./...) +# BINS is the set of built command executables. +BINS = aws-k8s-agent aws-cni grpc-health-probe cni-metrics-helper +# DOCKER_ARGS is extra arguments passed during container image build. +DOCKER_ARGS = +# DOCKER_RUN_FLAGS is set the flags passed during runs of containers. +DOCKER_RUN_FLAGS = --rm -ti $(DOCKER_ARGS) +# DOCKER_BUILD_FLAGS is the set of flags passed during container image builds +# based on the requested build. +DOCKER_BUILD_FLAGS = --build-arg GOARCH="$(ARCH)" \ + --build-arg docker_arch="$(DOCKER_ARCH)" \ + --build-arg golang_image="$(GOLANG_IMAGE)" \ + --network=host \ + $(DOCKER_ARGS) + +# Default to building an executable using the host's Go toolchain. +.DEFAULT_GOAL = build-linux + +# Build both CNI and metrics helper container images. +all: docker docker-metrics -ifeq ($(ARCH),aarch64) - ARCH = arm64 -else -endif -ifeq ($(ARCH),x86_64) - ARCH = amd64 -endif +dist: all + mkdir -p $(DESTDIR) + docker save $(IMAGE_NAME) | gzip > $(IMAGE_DIST) + docker save $(METRICS_IMAGE_NAME) | gzip > $(METRICS_IMAGE_DIST) -# Default to build the Linux binary +# Build the VPC CNI plugin agent using the host's Go toolchain. +build-linux: BUILD_FLAGS = -ldflags '-s -w $(LDFLAGS)' build-linux: - GOOS=linux GOARCH=$(ARCH) CGO_ENABLED=0 go build -o aws-k8s-agent -ldflags "-s -w $(LDFLAGS)" ./cmd/aws-k8s-agent/ - GOOS=linux GOARCH=$(ARCH) CGO_ENABLED=0 go build -o aws-cni -ldflags " -s -w $(LDFLAGS)" ./cmd/routed-eni-cni-plugin/ - GOOS=linux GOARCH=$(ARCH) CGO_ENABLED=0 go build -o grpc-health-probe -ldflags "-s -w $(LDFLAGS)" ./cmd/grpc-health-probe/ - -# Download portmap plugin -download-portmap: - mkdir -p tmp/downloads - mkdir -p tmp/plugins - curl -L -o tmp/downloads/cni-plugins-$(ARCH).tgz https://github.com/containernetworking/plugins/releases/download/v0.7.5/cni-plugins-$(ARCH)-v0.7.5.tgz - tar -vxf tmp/downloads/cni-plugins-$(ARCH).tgz -C tmp/plugins - cp tmp/plugins/portmap . - rm -rf tmp - -# Build CNI Docker image + go build $(BUILD_FLAGS) -o aws-k8s-agent ./cmd/aws-k8s-agent + go build $(BUILD_FLAGS) -o aws-cni ./cmd/routed-eni-cni-plugin + go build $(BUILD_FLAGS) -o grpc-health-probe ./cmd/grpc-health-probe + +# Build VPC CNI plugin & agent container image. docker: - @docker build $(DOCKER_ARGS) --build-arg arch="$(ARCH)" -f scripts/dockerfiles/Dockerfile.release -t "$(IMAGE):$(VERSION)" . - @echo "Built Docker image \"$(IMAGE):$(VERSION)\"" + docker build $(DOCKER_BUILD_FLAGS) \ + -f scripts/dockerfiles/Dockerfile.release \ + -t "$(IMAGE_NAME)" \ + . + @echo "Built Docker image \"$(IMAGE_NAME)\"" +# Run the built cni container image to use in functional testing docker-func-test: docker - docker run $(DOCKER_ARGS) -it "$(IMAGE):$(VERSION)" + docker run $(DOCKER_RUN_FLAGS) \ + "$(IMAGE_NAME)" -# unit-test +# Run unit tests unit-test: - GOOS=linux CGO_ENABLED=1 go test -v -cover $(ALLPKGS) + go test -v -cover $(ALLPKGS) -# unit-test-race +# Run unit tests with race detection (can only be run natively) +unit-test-race: CGO_ENABLED=1 +unit-test-race: GOARCH= unit-test-race: - GOOS=linux CGO_ENABLED=1 go test -v -cover -race -timeout 10s ./cmd/*/... - GOOS=linux CGO_ENABLED=1 go test -v -cover -race -timeout 150s ./pkg/awsutils/... - GOOS=linux CGO_ENABLED=1 go test -v -cover -race -timeout 10s ./pkg/k8sapi/... - GOOS=linux CGO_ENABLED=1 go test -v -cover -race -timeout 10s ./pkg/networkutils/... - GOOS=linux CGO_ENABLED=1 go test -v -cover -race -timeout 10s ./pkg/utils/... - GOOS=linux CGO_ENABLED=1 go test -v -cover -race -timeout 10s ./pkg/eniconfig/... - GOOS=linux CGO_ENABLED=1 go test -v -cover -race -timeout 10s ./pkg/ipamd/... - + go test -v -cover -race -timeout 10s ./cmd/... + go test -v -cover -race -timeout 150s ./pkg/awsutils/... + go test -v -cover -race -timeout 10s ./pkg/k8sapi/... + go test -v -cover -race -timeout 10s ./pkg/networkutils/... + go test -v -cover -race -timeout 10s ./pkg/utils/... + go test -v -cover -race -timeout 10s ./pkg/eniconfig/... + go test -v -cover -race -timeout 10s ./pkg/ipamd/... + +# Build the unit test driver container image. build-docker-test: - @docker build -f scripts/dockerfiles/Dockerfile.test -t amazon-k8s-cni-test:latest . + docker build $(DOCKER_BUILD_FLAGS) \ + -f scripts/dockerfiles/Dockerfile.test \ + -t $(TEST_IMAGE_NAME) \ + . +# Run unit tests inside of the testing container image. docker-unit-test: build-docker-test - docker run -e GO111MODULE=on \ - amazon-k8s-cni-test:latest make unit-test + docker run $(DOCKER_RUN_ARGS) \ + $(TEST_IMAGE_NAME) \ + make unit-test -# Build metrics +# Build metrics helper agent. build-metrics: - GOOS=linux GOARCH=$(ARCH) CGO_ENABLED=0 go build -ldflags="-s -w" -o cni-metrics-helper ./cmd/cni-metrics-helper/ + go build -ldflags="-s -w" -o cni-metrics-helper ./cmd/cni-metrics-helper -# Build metrics Docker image +# Build metrics helper agent Docker image. docker-metrics: - @docker build --build-arg arch="$(ARCH)" -f scripts/dockerfiles/Dockerfile.metrics -t "amazon/cni-metrics-helper:$(VERSION)" . + docker build $(DOCKER_BUILD_FLAGS) \ + -f scripts/dockerfiles/Dockerfile.metrics \ + -t "$(METRICS_IMAGE_NAME)" \ + . @echo "Built Docker image \"amazon/cni-metrics-helper:$(VERSION)\"" +# Run metrics helper unit test suite (must be run natively). +metrics-unit-test: CGO_ENABLED=1 +metrics-unit-test: GOARCH= metrics-unit-test: - GOOS=linux CGO_ENABLED=1 go test -v -cover -race -timeout 10s ./cmd/cni-metrics-helper/metrics/... + go test -v -cover -race -timeout 10s \ + ./cmd/cni-metrics-helper/metrics/... +# Run metrics helper unit test suite in a container. docker-metrics-test: - docker run -v $(shell pwd):/usr/src/app/src/github.com/aws/amazon-vpc-cni-k8s \ - --workdir=/usr/src/app/src/github.com/aws/amazon-vpc-cni-k8s \ - --env GOPATH=/usr/src/app \ - golang:1.10 make metrics-unit-test - -# Build both CNI and metrics helper -all: docker docker-metrics + docker run $(DOCKER_RUN_FLAGS) \ + -v $(shell pwd -P):/src --workdir=/src \ + -e GOARCH -e GOOS -e GO111MODULE \ + $(GOLANG_IMAGE) \ + make metrics-unit-test generate: go generate -x ./... $(MAKE) format +# Generate descriptors for supported ENI configurations. generate-limits: go run pkg/awsutils/gen_vpc_ip_limits.go -# golint -# To install: go get -u golang.org/x/lint/golint +# Fetch portmap the port-forwarding management CNI plugin +portmap: FETCH_VERSION=0.7.5 +portmap: FETCH_URL=https://github.com/containernetworking/plugins/releases/download/v$(FETCH_VERSION)/cni-plugins-$(GOARCH)-v$(FETCH_VERSION).tgz +portmap: VISIT_URL=https://github.com/containernetworking/plugins/tree/v$(FETCH_VERSION)/plugins/meta/portmap +portmap: + @echo "Fetching portmap CNI plugin v$(FETCH_VERSION) from upstream release" + @echo + @echo "Visit upstream project for portmap plugin details:" + @echo "$(VISIT_URL)" + @echo + curl -L $(FETCH_URL) | tar -z -x ./portmap + +# Run all source code checks. +check: check-format lint vet + +# Run golint on source code. +# +# To install: +# +# go get -u golang.org/x/lint/golint +# +lint: LINT_FLAGS = -set_exit_status lint: - golint pkg/awsutils/*.go - golint cmd/routed-eni-cni-plugin/*.go - golint cmd/routed-eni-cni-plugin/driver/*.go - golint pkg/k8sapi/*.go - golint pkg/networkutils/*.go - golint pkg/ipamd/*.go - golint pkg/ipamd/*/*.go - -# go vet + @command -v golint >/dev/null || { echo "ERROR: golint not installed"; exit 1; } + find . \ + -type f -name '*.go' \ + -not -name 'mock_*' -not -name 'mocks_*' \ + -print0 | sort -z | xargs -0 -L1 -- golint $(LINT_FLAGS) 2>/dev/null + +# Run go vet on source code. vet: - GOOS=linux go vet ./... + go vet ./... +# Run go vet inside of a container. docker-vet: build-docker-test - docker run -e GO111MODULE=on \ - amazon-k8s-cni-test:latest make vet - -clean: - rm -f aws-k8s-agent - rm -f aws-cni - rm -f grpc-health-probe - rm -f cni-metrics-helper - rm -f portmap - -files := $(shell find . -not -name 'mock_publisher.go' -not -name 'rpc.pb.go' -not -name 'integration_test.go' -name '*.go' -print) -unformatted = $(shell goimports -l $(files)) + docker run $(DOCKER_RUN_FLAGS) \ + $(TEST_IMAGE_NAME) make vet +# Format all Go source code files. format: - @echo "== format" - @goimports -w $(files) - @sync - -check-format: - @echo "== check formatting" -ifneq "$(unformatted)" "" - @echo "needs formatting: $(unformatted)" - @echo "run 'make format'" - @exit 1 -endif + @command -v goimports >/dev/null || { echo "ERROR: goimports not installed"; exit 1; } + find ./* \ + -type f \ + -not -name 'mock_publisher.go' \ + -not -name 'rpc.pb.go' \ + -name '*.go' \ + -print0 | sort -z | xargs -0 -- goimports $(or $(FORMAT_FLAGS),-w) + +# Check formatting of source code files without modification. +check-format: FORMAT_FLAGS = -l +check-format: format + +# Clean temporary files and build artifacts from the project. +clean: + @rm -f -- $(BINS) + @rm -f -- portmap diff --git a/README.md b/README.md index 4ff0248c7ac..7bac826ca95 100644 --- a/README.md +++ b/README.md @@ -299,7 +299,19 @@ Default: Unset Valid Values: `stdout` or a file path -Specifies where to write the logging output. Either to stdout or to override the default file. +Specifies where to write the logging output of L-IPamD. Either to stdout or to override the default file. + +--- + +`AWS_VPC_K8S_PLUGIN_LOG_FILE` + +Type: String + +Default: Unset + +Valid Values: `stdout` or a file path + +Specifies where to write the logging output for CNI plugin. Either to stdout or to override the default file. --- diff --git a/cmd/aws-k8s-agent/main.go b/cmd/aws-k8s-agent/main.go index d39fc3aa8e2..d697cea6b43 100644 --- a/cmd/aws-k8s-agent/main.go +++ b/cmd/aws-k8s-agent/main.go @@ -1,4 +1,4 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// Copyright 2017-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -16,29 +16,27 @@ package main import ( "os" - log "github.com/cihub/seelog" - "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig" "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd" "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" ) -const ( - defaultLogFilePath = "/host/var/log/aws-routed-eni/ipamd.log" -) +const binaryName = "L-IPamD" + +var version string -var ( - version string -) func main() { os.Exit(_main()) } func _main() int { - defer log.Flush() - logger.SetupLogger(logger.GetLogFileLocation(defaultLogFilePath)) + //Do not add anything before initializing logger + logConfig := logger.Configuration{ + BinaryName: binaryName, + } + log := logger.New(&logConfig) log.Infof("Starting L-IPAMD %s ...", version) diff --git a/cmd/cni-metrics-helper/main.go b/cmd/cni-metrics-helper/main.go index 7644e7a8101..23b84feebec 100644 --- a/cmd/cni-metrics-helper/main.go +++ b/cmd/cni-metrics-helper/main.go @@ -1,4 +1,4 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -21,7 +21,7 @@ import ( "strings" "time" - "github.com/golang/glog" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "github.com/spf13/pflag" "github.com/aws/amazon-vpc-cni-k8s/cmd/cni-metrics-helper/metrics" @@ -29,6 +29,8 @@ import ( "github.com/aws/amazon-vpc-cni-k8s/pkg/publisher" ) +var log = logger.DefaultLogger() + type options struct { kubeconfig string pullInterval int @@ -40,11 +42,8 @@ type options struct { func main() { options := &options{} flags := pflag.NewFlagSet("", pflag.ExitOnError) - // Add glog flags flags.AddGoFlagSet(flag.CommandLine) - _ = flags.Lookup("logtostderr").Value.Set("true") - flags.Lookup("logtostderr").DefValue = "true" - flags.Lookup("logtostderr").NoOptDefVal = "true" + flags.BoolVar(&options.submitCW, "cloudwatch", true, "a bool") flags.Usage = func() { @@ -54,12 +53,12 @@ func main() { err := flags.Parse(os.Args) if err != nil { - glog.Fatalf("Error on parsing parameters: %s", err) + log.Fatalf("Error on parsing parameters: %s", err) } err = flag.CommandLine.Parse([]string{}) if err != nil { - glog.Fatalf("Error on parsing commandline: %s", err) + log.Fatalf("Error on parsing commandline: %s", err) } if options.help { @@ -77,12 +76,11 @@ func main() { } } - glog.Infof("Starting CNIMetricsHelper. Sending metrics to CloudWatch: %v", options.submitCW) + log.Infof("Starting CNIMetricsHelper. Sending metrics to CloudWatch: %v", options.submitCW) kubeClient, err := k8sapi.CreateKubeClient() if err != nil { - glog.Errorf("Failed to create client: %v", err) - os.Exit(1) + log.Fatalf("Failed to create client: %v", err) } discoverController := k8sapi.NewController(kubeClient) @@ -96,8 +94,7 @@ func main() { cw, err = publisher.New(ctx) if err != nil { - glog.Errorf("Failed to create publisher: %v", err) - os.Exit(1) + log.Fatalf("Failed to create publisher: %v", err) } go cw.Start() defer cw.Stop() @@ -109,7 +106,7 @@ func main() { // metric loop var pullInterval = 30 // seconds for range time.Tick(time.Duration(pullInterval) * time.Second) { - glog.Info("Collecting metrics ...") + log.Info("Collecting metrics ...") metrics.Handler(cniMetric) } } diff --git a/cmd/cni-metrics-helper/metrics/cni_metrics.go b/cmd/cni-metrics-helper/metrics/cni_metrics.go index b9debdaae10..ee74f7d0a93 100644 --- a/cmd/cni-metrics-helper/metrics/cni_metrics.go +++ b/cmd/cni-metrics-helper/metrics/cni_metrics.go @@ -15,7 +15,6 @@ package metrics import ( - "github.com/golang/glog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" @@ -152,14 +151,13 @@ func CNIMetricsNew(c clientset.Interface, cw publisher.Publisher, d *k8sapi.Cont } func (t *CNIMetricsTarget) grabMetricsFromTarget(cniPod string) ([]byte, error) { - glog.Infof("Grabbing metrics from CNI: %s", cniPod) output, err := getMetricsFromPod(t.kubeClient, cniPod, metav1.NamespaceSystem, metricsPort) if err != nil { - glog.Errorf("grabMetricsFromTarget: Failed to grab CNI endpoint: %v", err) + log.Errorf("grabMetricsFromTarget: Failed to grab CNI endpoint: %v", err) return nil, err } - glog.V(5).Infof("cni-metrics text output: %s", string(output)) + log.Infof("cni-metrics text output: %s", string(output)) return output, nil } diff --git a/cmd/cni-metrics-helper/metrics/metrics.go b/cmd/cni-metrics-helper/metrics/metrics.go index 797697dc4c2..30e87edb9bd 100644 --- a/cmd/cni-metrics-helper/metrics/metrics.go +++ b/cmd/cni-metrics-helper/metrics/metrics.go @@ -1,4 +1,4 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -21,14 +21,17 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudwatch" - "github.com/golang/glog" + dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" clientset "k8s.io/client-go/kubernetes" "github.com/aws/amazon-vpc-cni-k8s/pkg/publisher" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" ) +var log = logger.DefaultLogger() + type metricMatcher func(metric *dto.Metric) bool type actionFuncType func(aggregatedValue *float64, sampleValue float64) @@ -97,27 +100,16 @@ func getMetricsFromPod(client clientset.Interface, podName string, namespace str } func processGauge(metric *dto.Metric, act *metricsAction) { - if act.logToFile { - glog.Infof("Label: %v, Value: %v ", metric.GetLabel(), metric.GetGauge().GetValue()) - } else { - glog.V(10).Info("processing GAUGE: ", metric.GetGauge().GetValue()) - } act.actionFunc(&act.data.curSingleDataPoint, metric.GetGauge().GetValue()) } func processCounter(metric *dto.Metric, act *metricsAction) { - if act.logToFile { - glog.Infof("Label: %v, Value: %v ", metric.GetLabel(), metric.GetCounter().GetValue()) - } else { - glog.V(10).Info("processing COUNTER: ", metric.GetCounter().GetValue()) - } act.actionFunc(&act.data.curSingleDataPoint, metric.GetCounter().GetValue()) } func processPercentile(metric *dto.Metric, act *metricsAction) { var p99 float64 - glog.V(10).Info("processing PERCENTILE: ", p99) summary := metric.GetSummary() quantiles := summary.GetQuantile() @@ -130,33 +122,27 @@ func processPercentile(metric *dto.Metric, act *metricsAction) { } func processHistogram(metric *dto.Metric, act *metricsAction) { - glog.V(5).Info("processing HISTOGRAM:", metric.GetLabel()) - glog.V(5).Info(metric.GetHistogram()) histogram := metric.GetHistogram() for _, bucket := range histogram.GetBucket() { - glog.V(10).Info("processing bucket:", bucket) existingBucket := false for _, bucketInAct := range act.bucket.curBucket { if bucket.GetUpperBound() == *bucketInAct.UpperBound { // found the matching bucket - glog.V(10).Infof("Found the matching bucket with UpperBound: %f", *bucketInAct.UpperBound) act.actionFunc(bucketInAct.CumulativeCount, float64(bucket.GetCumulativeCount())) - glog.V(10).Infof("Found: *bucketInAct.CumulativeCount:%f, bucket.GetCumulativeCount():%f", - *bucketInAct.CumulativeCount, float64(bucket.GetCumulativeCount())) existingBucket = true break } } if !existingBucket { - glog.V(10).Infof("Create a new bucket with upperBound:%f", bucket.GetUpperBound()) upperBound := new(float64) *upperBound = float64(bucket.GetUpperBound()) cumulativeCount := new(float64) *cumulativeCount = float64(bucket.GetCumulativeCount()) newBucket := &bucketPoint{UpperBound: upperBound, CumulativeCount: cumulativeCount} act.bucket.curBucket = append(act.bucket.curBucket, newBucket) + log.Infof("Created a new bucket with upperBound:%f", bucket.GetUpperBound()) } } } @@ -186,7 +172,7 @@ func postProcessingCounter(convert metricsConvert) bool { } if resetDetected || (noPreviousDataPoint && !noCurrentDataPoint) { - glog.Infof("Reset detected resetDetected: %v, noPreviousDataPoint: %v, noCurrentDataPoint: %v", + log.Infof("Reset detected resetDetected: %v, noPreviousDataPoint: %v, noCurrentDataPoint: %v", resetDetected, noPreviousDataPoint, noCurrentDataPoint) } return resetDetected || (noPreviousDataPoint && !noCurrentDataPoint) @@ -199,28 +185,28 @@ func postProcessingHistogram(convert metricsConvert) bool { for _, action := range convert.actions { numOfBuckets := len(action.bucket.curBucket) if numOfBuckets == 0 { - glog.Info("Post Histogram Processing: no bucket found") + log.Info("Post Histogram Processing: no bucket found") continue } for i := 1; i < numOfBuckets; i++ { - glog.V(10).Infof("Found numOfBuckets-i:=%d, *action.bucket.curBucket[numOfBuckets-i].CumulativeCount=%f", + log.Infof("Found numOfBuckets-i:=%d, *action.bucket.curBucket[numOfBuckets-i].CumulativeCount=%f", numOfBuckets-i, *action.bucket.curBucket[numOfBuckets-i].CumulativeCount) // Delta against the previous bucket value // e.g. diff between bucket LE250000 and previous bucket LE125000 *action.bucket.curBucket[numOfBuckets-i].CumulativeCount -= *action.bucket.curBucket[numOfBuckets-i-1].CumulativeCount - glog.V(10).Infof("Found numOfBuckets-i:=%d, *action.bucket.curBucket[numOfBuckets-i].CumulativeCount=%f, *action.bucket.curBucket[numOfBuckets-i-1].CumulativeCount=%f", + log.Infof("Found numOfBuckets-i:=%d, *action.bucket.curBucket[numOfBuckets-i].CumulativeCount=%f, *action.bucket.curBucket[numOfBuckets-i-1].CumulativeCount=%f", numOfBuckets-i, *action.bucket.curBucket[numOfBuckets-i].CumulativeCount, *action.bucket.curBucket[numOfBuckets-i-1].CumulativeCount) // Delta against the previous value if action.bucket.lastBucket != nil { - glog.V(10).Infof("Found *action.bucket.lastBucket[numOfBuckets-i].CumulativeCount=%f", + log.Infof("Found *action.bucket.lastBucket[numOfBuckets-i].CumulativeCount=%f", *action.bucket.lastBucket[numOfBuckets-i].CumulativeCount) currentTotal := *action.bucket.curBucket[numOfBuckets-i].CumulativeCount // Only do delta if there is no restart for metric target if *action.bucket.curBucket[numOfBuckets-i].CumulativeCount >= *action.bucket.lastBucket[numOfBuckets-i].CumulativeCount { *action.bucket.curBucket[numOfBuckets-i].CumulativeCount -= *action.bucket.lastBucket[numOfBuckets-i].CumulativeCount - glog.V(10).Infof("Found *action.bucket.lastBucket[numOfBuckets-i].CumulativeCount=%f, *action.bucket.lastBucket[numOfBuckets-i].CumulativeCount=%f", + log.Infof("Found *action.bucket.lastBucket[numOfBuckets-i].CumulativeCount=%f, *action.bucket.lastBucket[numOfBuckets-i].CumulativeCount=%f", *action.bucket.curBucket[numOfBuckets-i].CumulativeCount, *action.bucket.lastBucket[numOfBuckets-i].CumulativeCount) } else { resetDetected = true @@ -250,7 +236,6 @@ func postProcessingHistogram(convert metricsConvert) bool { } func processMetric(family *dto.MetricFamily, convert metricsConvert) (bool, error) { - glog.Info("Processing metric: ", family.GetName()) resetDetected := false mType := family.GetType() @@ -301,8 +286,6 @@ func produceHistogram(act metricsAction, cw publisher.Publisher) { prevUpperBound = *bucket.UpperBound if *bucket.CumulativeCount != 0 { - glog.Infof("Produce HISTOGRAM metrics: %s, max:%f, min:%f, count: %f, sum: %f", - act.cwMetricName, mid, mid, *bucket.CumulativeCount, mid*float64(*bucket.CumulativeCount)) dataPoint := &cloudwatch.MetricDatum{ MetricName: aws.String(act.cwMetricName), StatisticValues: &cloudwatch.StatisticSet{ @@ -337,7 +320,7 @@ func produceCloudWatchMetrics(t metricsTarget, families map[string]*dto.MetricFa for _, action := range convertMetrics.actions { switch mType { case dto.MetricType_COUNTER: - glog.Infof("Produce COUNTER metrics: %s, value: %f", action.cwMetricName, action.data.curSingleDataPoint) + log.Infof("Produce COUNTER metrics: %s, value: %f", action.cwMetricName, action.data.curSingleDataPoint) if t.submitCloudWatch() { dataPoint := &cloudwatch.MetricDatum{ MetricName: aws.String(action.cwMetricName), @@ -347,7 +330,6 @@ func produceCloudWatchMetrics(t metricsTarget, families map[string]*dto.MetricFa cw.Publish(dataPoint) } case dto.MetricType_GAUGE: - glog.Infof("Produce GAUGE metrics: %s, value: %f", action.cwMetricName, action.data.curSingleDataPoint) if t.submitCloudWatch() { dataPoint := &cloudwatch.MetricDatum{ MetricName: aws.String(action.cwMetricName), @@ -357,7 +339,6 @@ func produceCloudWatchMetrics(t metricsTarget, families map[string]*dto.MetricFa cw.Publish(dataPoint) } case dto.MetricType_SUMMARY: - glog.Infof("Produce PERCENTILE metrics: %s, value: %f", action.cwMetricName, action.data.curSingleDataPoint) if t.submitCloudWatch() { dataPoint := &cloudwatch.MetricDatum{ MetricName: aws.String(action.cwMetricName), @@ -397,13 +378,9 @@ func metricsListGrabAggregateConvert(t metricsTarget) (map[string]*dto.MetricFam resetMetrics(interestingMetrics) targetList := t.getTargetList() - glog.Info("targetList: ", targetList) - glog.Info("len(targetList)", len(targetList)) for _, target := range targetList { - glog.Infof("Grab/Aggregate metrics from %v", target) rawOutput, err := t.grabMetricsFromTarget(target) if err != nil { - glog.Errorf("Failed to getMetricsFromTarget: %v", err) // it may take times to remove some metric targets continue } @@ -412,13 +389,11 @@ func metricsListGrabAggregateConvert(t metricsTarget) (map[string]*dto.MetricFam origFamilies, err := parser.TextToMetricFamilies(bytes.NewReader(rawOutput)) if err != nil { - glog.Warning("Failed to parse metrics:", err) return nil, nil, true, err } families, err = filterMetrics(origFamilies, interestingMetrics) if err != nil { - glog.Warning("Failed to filter metrics:", err) return nil, nil, true, err } @@ -447,7 +422,7 @@ func Handler(t metricsTarget) { families, interestingMetrics, resetDetected, err := metricsListGrabAggregateConvert(t) if err != nil || resetDetected { - glog.Info("Skipping 1st poll after reset, error:", err) + log.Infof("Skipping 1st poll after reset, error: %v", err) } cw := t.getCWMetricsPublisher() diff --git a/cmd/grpc-health-probe/main.go b/cmd/grpc-health-probe/main.go index 37b2c978553..99f6b4ab4fb 100644 --- a/cmd/grpc-health-probe/main.go +++ b/cmd/grpc-health-probe/main.go @@ -3,11 +3,12 @@ package main import ( "context" "flag" - "log" "os" "os/signal" "time" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" + "google.golang.org/grpc" "google.golang.org/grpc/codes" healthpb "google.golang.org/grpc/health/grpc_health_v1" @@ -23,6 +24,8 @@ var ( verbose bool ) +var log = logger.DefaultLogger() + const ( // StatusInvalidArguments indicates specified invalid arguments. StatusInvalidArguments = 1 @@ -35,7 +38,6 @@ const ( ) func init() { - log.SetFlags(0) flag.StringVar(&remoteURL, "addr", "", "(required) tcp host:port to connect") flag.StringVar(&serviceName, "service", "", "service name to check (default: \"\")") flag.StringVar(&userAgent, "user-agent", "grpc-health-probe", "user-agent header value of health check requests") @@ -48,7 +50,7 @@ func init() { flag.Parse() argError := func(s string, v ...interface{}) { - log.Printf("error: "+s, v...) + log.Infof("error: "+s, v...) os.Exit(StatusInvalidArguments) } @@ -63,8 +65,8 @@ func init() { argError("--rpc-timeout must be greater than zero (specified: %v)", rpcTimeoutDur) } if verbose { - log.Printf("parsed options:") - log.Printf("> remoteUrl=%s conn-timeout=%v rpc-timeout=%v", remoteURL, connTimeoutDur, rpcTimeoutDur) + log.Info("parsed options:") + log.Infof("> remoteUrl=%s conn-timeout=%v rpc-timeout=%v", remoteURL, connTimeoutDur, rpcTimeoutDur) } } @@ -76,7 +78,7 @@ func main() { go func() { sig := <-c if sig == os.Interrupt { - log.Printf("cancellation received") + log.Infof("cancellation received") cancel() return } @@ -89,7 +91,7 @@ func main() { opts = append(opts, grpc.WithInsecure()) if verbose { - log.Print("establishing connection") + log.Info("establishing connection") } connStart := time.Now() dialCtx, cancel2 := context.WithTimeout(ctx, connTimeoutDur) @@ -97,41 +99,40 @@ func main() { conn, err := grpc.DialContext(dialCtx, remoteURL, opts...) if err != nil { if err == context.DeadlineExceeded { - log.Printf("timeout: failed to connect service %q within %v", remoteURL, connTimeoutDur) + log.Infof("timeout: failed to connect service %q within %v", remoteURL, connTimeoutDur) } else { - log.Printf("error: failed to connect service at %q: %+v", remoteURL, err) + log.Infof("error: failed to connect service at %q: %+v", remoteURL, err) } os.Exit(StatusConnectionFailure) } connDuration := time.Since(connStart) defer conn.Close() if verbose { - log.Printf("connection establisted (took %v)", connDuration) + log.Infof("connection established (took %v)", connDuration) } rpcStart := time.Now() rpcCtx, rpcCancel := context.WithTimeout(ctx, rpcTimeoutDur) defer rpcCancel() resp, err := healthpb.NewHealthClient(conn).Check(rpcCtx, &healthpb.HealthCheckRequest{Service: serviceName}) - log.Print(resp) if err != nil { if stat, ok := status.FromError(err); ok && stat.Code() == codes.Unimplemented { - log.Printf("error: this server does not implement the grpc health protocol (grpc.health.v1.Health)") + log.Infof("error: this server does not implement the grpc health protocol (grpc.health.v1.Health)") } else if stat, ok := status.FromError(err); ok && stat.Code() == codes.DeadlineExceeded { - log.Printf("timeout: health rpc did not complete within %v", rpcTimeoutDur) + log.Infof("timeout: health rpc did not complete within %v", rpcTimeoutDur) } else { - log.Printf("error: health rpc failed: %+v", err) + log.Infof("error: health rpc failed: %+v", err) } os.Exit(StatusRPCFailure) } rpcDuration := time.Since(rpcStart) if resp.GetStatus() != healthpb.HealthCheckResponse_SERVING { - log.Printf("service unhealthy (responded with %q)", resp.GetStatus().String()) + log.Infof("service unhealthy (responded with %q)", resp.GetStatus().String()) os.Exit(StatusUnhealthy) } if verbose { - log.Printf("time elapsed: connect=%v rpc=%v", connDuration, rpcDuration) + log.Infof("time elapsed: connect=%v rpc=%v", connDuration, rpcDuration) } - log.Printf("status: %v", resp.GetStatus().String()) + log.Infof("status: %v", resp.GetStatus().String()) } diff --git a/cmd/routed-eni-cni-plugin/cni.go b/cmd/routed-eni-cni-plugin/cni.go index 5b44712fbe2..3b0819e36f0 100644 --- a/cmd/routed-eni-cni-plugin/cni.go +++ b/cmd/routed-eni-cni-plugin/cni.go @@ -1,4 +1,4 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// Copyright 2017-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -23,7 +23,6 @@ import ( "runtime" "strings" - log "github.com/cihub/seelog" "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" "github.com/containernetworking/cni/pkg/types/current" @@ -44,12 +43,9 @@ import ( const ( ipamDAddress = "127.0.0.1:50051" - defaultLogFilePath = "/var/log/aws-routed-eni/plugin.log" ) -var ( - version string -) +var version string // NetConf stores the common network config for the CNI plugin type NetConf struct { @@ -69,6 +65,10 @@ type NetConf struct { // MTU for eth0 MTU string `json:"mtu"` + + PluginLogFile string `json:"pluginLogFile"` + + PluginLogLevel string `json:"pluginLogLevel"` } // K8sArgs is the valid CNI_ARGS used for Kubernetes @@ -94,44 +94,49 @@ func init() { runtime.LockOSThread() } +// LoadNetConf converts inputs (i.e. stdin) to NetConf +func LoadNetConf(bytes []byte) (*NetConf, logger.Logger, error) { + conf := &NetConf{} + if err := json.Unmarshal(bytes, conf); err != nil { + return nil, nil, errors.Wrap(err, "add cmd: error loading config from args") + } + + if len(conf.VethPrefix) > 4 { + return nil, nil, errors.New("conf.VethPrefix can be at most 4 characters long") + } + + logConfig := logger.Configuration{ + BinaryName: conf.Name, + LogLevel: conf.PluginLogLevel, + LogLocation: conf.PluginLogFile, + } + log := logger.New(&logConfig) + return conf, log, nil +} + func cmdAdd(args *skel.CmdArgs) error { return add(args, typeswrapper.New(), grpcwrapper.New(), rpcwrapper.New(), driver.New()) } func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrapper.GRPC, rpcClient rpcwrapper.RPC, driverClient driver.NetworkAPIs) error { - log.Infof("Received CNI add request: ContainerID(%s) Netns(%s) IfName(%s) Args(%s) Path(%s) argsStdinData(%s)", - args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData) - conf := NetConf{} - if err := json.Unmarshal(args.StdinData, &conf); err != nil { - log.Errorf("Error loading config from args: %v", err) + conf, log, err := LoadNetConf(args.StdinData) + if err != nil { return errors.Wrap(err, "add cmd: error loading config from args") } + log.Infof("Received CNI add request: ContainerID(%s) Netns(%s) IfName(%s) Args(%s) Path(%s) argsStdinData(%s)", + args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData) + k8sArgs := K8sArgs{} if err := cniTypes.LoadArgs(args.Args, &k8sArgs); err != nil { log.Errorf("Failed to load k8s config from arg: %v", err) return errors.Wrap(err, "add cmd: failed to load k8s config from arg") } - // Default the host-side veth prefix to 'eni'. - if conf.VethPrefix == "" { - conf.VethPrefix = "eni" - } - if len(conf.VethPrefix) > 4 { - return errors.New("conf.VethPrefix can be at most 4 characters long") - } - - // MTU - if conf.MTU == "" { - log.Debug("MTU not set, defaulting to 9001") - conf.MTU = "9001" - } mtu := networkutils.GetEthernetMTU(conf.MTU) - cniVersion := conf.CNIVersion - // Set up a connection to the ipamD server. conn, err := grpcClient.Dial(ipamDAddress, grpc.WithInsecure()) if err != nil { @@ -184,7 +189,7 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap // Note: the maximum length for linux interface name is 15 hostVethName := generateHostVethName(conf.VethPrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) - err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, addr, int(r.DeviceNumber), r.VPCcidrs, r.UseExternalSNAT, mtu) + err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, addr, int(r.DeviceNumber), r.VPCcidrs, r.UseExternalSNAT, mtu, log) if err != nil { log.Errorf("Failed SetupPodNetwork for pod %s namespace %s sandbox %s: %v", @@ -222,7 +227,7 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap IPs: ips, } - return cniTypes.PrintResult(result, cniVersion) + return cniTypes.PrintResult(result, conf.CNIVersion) } // generateHostVethName returns a name to be used on the host-side veth device. @@ -239,15 +244,14 @@ func cmdDel(args *skel.CmdArgs) error { func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrapper.GRPC, rpcClient rpcwrapper.RPC, driverClient driver.NetworkAPIs) error { + _, log, err := LoadNetConf(args.StdinData) + if err != nil { + return errors.Wrap(err, "add cmd: error loading config from args") + } + log.Infof("Received CNI del request: ContainerID(%s) Netns(%s) IfName(%s) Args(%s) Path(%s) argsStdinData(%s)", args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData) - conf := NetConf{} - if err := json.Unmarshal(args.StdinData, &conf); err != nil { - log.Errorf("Failed to load netconf from args %v", err) - return errors.Wrap(err, "del cmd: failed to load netconf from args") - } - k8sArgs := K8sArgs{} if err := cniTypes.LoadArgs(args.Args, &k8sArgs); err != nil { log.Errorf("Failed to load k8s config from args: %v", err) @@ -298,13 +302,13 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap return errors.New("del cmd: failed to process delete request") } - deletedPodIp := net.ParseIP(r.IPv4Addr) - if deletedPodIp != nil { + deletedPodIP := net.ParseIP(r.IPv4Addr) + if deletedPodIP != nil { addr := &net.IPNet{ - IP: deletedPodIp, + IP: deletedPodIP, Mask: net.IPv4Mask(255, 255, 255, 255), } - err = driverClient.TeardownNS(addr, int(r.DeviceNumber)) + err = driverClient.TeardownNS(addr, int(r.DeviceNumber), log) if err != nil { log.Errorf("Failed on TeardownPodNetwork for pod %s namespace %s sandbox %s: %v", string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), err) @@ -318,19 +322,14 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap } func main() { - logger.SetupLogger(logger.GetLogFileLocation(defaultLogFilePath)) - - log.Infof("Starting CNI Plugin %s ...", version) + log := logger.Get() exitCode := 0 if e := skel.PluginMainWithError(cmdAdd, cmdDel, cniSpecVersion.All); e != nil { - exitCode = 1 - log.Error("Failed CNI request: ", e) if err := e.Print(); err != nil { - log.Errorf("Error writing error JSON to stdout: %v", err) + log.Errorf("Failed to write error to stdout: %v", err) } + exitCode = 1 } - - log.Flush() os.Exit(exitCode) } diff --git a/cmd/routed-eni-cni-plugin/cni_test.go b/cmd/routed-eni-cni-plugin/cni_test.go index 95bc685c86f..a2d34b13f1d 100644 --- a/cmd/routed-eni-cni-plugin/cni_test.go +++ b/cmd/routed-eni-cni-plugin/cni_test.go @@ -38,11 +38,20 @@ const ( ifName = "eth0" cniVersion = "1.0" cniName = "aws-cni" + pluginLogLevel = "Debug" + pluginLogFile = "/var/log/aws-routed-eni/plugin.log" cniType = "aws-cni" ipAddr = "10.0.1.15" devNum = 4 ) +var netConf = &NetConf{CNIVersion: cniVersion, + Name: cniName, + Type: cniType, + PluginLogLevel: pluginLogLevel, + PluginLogFile: pluginLogFile, +} + func setup(t *testing.T) (*gomock.Controller, *mock_typeswrapper.MockCNITYPES, *mock_grpcwrapper.MockGRPC, @@ -66,10 +75,7 @@ func TestCmdAdd(t *testing.T) { ctrl, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork := setup(t) defer ctrl.Finish() - netconf := &NetConf{CNIVersion: cniVersion, - Name: cniName, - Type: cniType} - stdinData, _ := json.Marshal(netconf) + stdinData, _ := json.Marshal(netConf) cmdArgs := &skel.CmdArgs{ContainerID: containerID, Netns: netNS, @@ -93,7 +99,7 @@ func TestCmdAdd(t *testing.T) { } mocksNetwork.EXPECT().SetupNS(gomock.Any(), cmdArgs.IfName, cmdArgs.Netns, - addr, int(addNetworkReply.DeviceNumber), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + addr, int(addNetworkReply.DeviceNumber), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mocksTypes.EXPECT().PrintResult(gomock.Any(), gomock.Any()).Return(nil) @@ -105,10 +111,7 @@ func TestCmdAddNetworkErr(t *testing.T) { ctrl, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork := setup(t) defer ctrl.Finish() - netconf := &NetConf{CNIVersion: cniVersion, - Name: cniName, - Type: cniType} - stdinData, _ := json.Marshal(netconf) + stdinData, _ := json.Marshal(netConf) cmdArgs := &skel.CmdArgs{ContainerID: containerID, Netns: netNS, @@ -135,10 +138,7 @@ func TestCmdAddErrSetupPodNetwork(t *testing.T) { ctrl, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork := setup(t) defer ctrl.Finish() - netconf := &NetConf{CNIVersion: cniVersion, - Name: cniName, - Type: cniType} - stdinData, _ := json.Marshal(netconf) + stdinData, _ := json.Marshal(netConf) cmdArgs := &skel.CmdArgs{ContainerID: containerID, Netns: netNS, @@ -162,7 +162,7 @@ func TestCmdAddErrSetupPodNetwork(t *testing.T) { } mocksNetwork.EXPECT().SetupNS(gomock.Any(), cmdArgs.IfName, cmdArgs.Netns, - addr, int(addNetworkReply.DeviceNumber), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("error on SetupPodNetwork")) + addr, int(addNetworkReply.DeviceNumber), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New("error on SetupPodNetwork")) // when SetupPodNetwork fails, expect to return IP back to datastore delNetworkReply := &rpc.DelNetworkReply{Success: true, IPv4Addr: ipAddr, DeviceNumber: devNum} @@ -177,10 +177,7 @@ func TestCmdDel(t *testing.T) { ctrl, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork := setup(t) defer ctrl.Finish() - netconf := &NetConf{CNIVersion: cniVersion, - Name: cniName, - Type: cniType} - stdinData, _ := json.Marshal(netconf) + stdinData, _ := json.Marshal(netConf) cmdArgs := &skel.CmdArgs{ContainerID: containerID, Netns: netNS, @@ -204,7 +201,7 @@ func TestCmdDel(t *testing.T) { Mask: net.IPv4Mask(255, 255, 255, 255), } - mocksNetwork.EXPECT().TeardownNS(addr, int(delNetworkReply.DeviceNumber)).Return(nil) + mocksNetwork.EXPECT().TeardownNS(addr, int(delNetworkReply.DeviceNumber), gomock.Any()).Return(nil) err := del(cmdArgs, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork) assert.Nil(t, err) @@ -214,10 +211,7 @@ func TestCmdDelErrDelNetwork(t *testing.T) { ctrl, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork := setup(t) defer ctrl.Finish() - netconf := &NetConf{CNIVersion: cniVersion, - Name: cniName, - Type: cniType} - stdinData, _ := json.Marshal(netconf) + stdinData, _ := json.Marshal(netConf) cmdArgs := &skel.CmdArgs{ContainerID: containerID, Netns: netNS, @@ -244,10 +238,7 @@ func TestCmdDelErrTeardown(t *testing.T) { ctrl, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork := setup(t) defer ctrl.Finish() - netconf := &NetConf{CNIVersion: cniVersion, - Name: cniName, - Type: cniType} - stdinData, _ := json.Marshal(netconf) + stdinData, _ := json.Marshal(netConf) cmdArgs := &skel.CmdArgs{ContainerID: containerID, Netns: netNS, @@ -271,7 +262,7 @@ func TestCmdDelErrTeardown(t *testing.T) { Mask: net.IPv4Mask(255, 255, 255, 255), } - mocksNetwork.EXPECT().TeardownNS(addr, int(delNetworkReply.DeviceNumber)).Return(errors.New("error on teardown")) + mocksNetwork.EXPECT().TeardownNS(addr, int(delNetworkReply.DeviceNumber), gomock.Any()).Return(errors.New("error on teardown")) err := del(cmdArgs, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork) assert.Error(t, err) diff --git a/cmd/routed-eni-cni-plugin/driver/driver.go b/cmd/routed-eni-cni-plugin/driver/driver.go index fba6f25fec9..b433aec2453 100644 --- a/cmd/routed-eni-cni-plugin/driver/driver.go +++ b/cmd/routed-eni-cni-plugin/driver/driver.go @@ -1,4 +1,4 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// Copyright 2017-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -23,12 +23,11 @@ import ( "github.com/containernetworking/cni/pkg/ns" "github.com/vishvananda/netlink" - log "github.com/cihub/seelog" - "github.com/aws/amazon-vpc-cni-k8s/pkg/ipwrapper" "github.com/aws/amazon-vpc-cni-k8s/pkg/netlinkwrapper" "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils" "github.com/aws/amazon-vpc-cni-k8s/pkg/nswrapper" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" ) const ( @@ -42,8 +41,8 @@ const ( // NetworkAPIs defines network API calls type NetworkAPIs interface { - SetupNS(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet, table int, vpcCIDRs []string, useExternalSNAT bool, mtu int) error - TeardownNS(addr *net.IPNet, table int) error + SetupNS(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet, table int, vpcCIDRs []string, useExternalSNAT bool, mtu int, log logger.Logger) error + TeardownNS(addr *net.IPNet, table int, log logger.Logger) error } type linuxNetwork struct { @@ -164,13 +163,13 @@ func (createVethContext *createVethPairContext) run(hostNS ns.NetNS) error { } // SetupNS wires up linux networking for a pod's network -func (os *linuxNetwork) SetupNS(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet, table int, vpcCIDRs []string, useExternalSNAT bool, mtu int) error { +func (os *linuxNetwork) SetupNS(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet, table int, vpcCIDRs []string, useExternalSNAT bool, mtu int, log logger.Logger) error { log.Debugf("SetupNS: hostVethName=%s, contVethName=%s, netnsPath=%s, table=%d, mtu=%d", hostVethName, contVethName, netnsPath, table, mtu) - return setupNS(hostVethName, contVethName, netnsPath, addr, table, vpcCIDRs, useExternalSNAT, os.netLink, os.ns, mtu) + return setupNS(hostVethName, contVethName, netnsPath, addr, table, vpcCIDRs, useExternalSNAT, os.netLink, os.ns, mtu, log) } func setupNS(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet, table int, vpcCIDRs []string, useExternalSNAT bool, - netLink netlinkwrapper.NetLink, ns nswrapper.NS, mtu int) error { + netLink netlinkwrapper.NetLink, ns nswrapper.NS, mtu int, log logger.Logger) error { // Clean up if hostVeth exists. if oldHostVeth, err := netLink.LinkByName(hostVethName); err == nil { if err = netLink.LinkDel(oldHostVeth); err != nil { @@ -243,7 +242,7 @@ func setupNS(hostVethName string, contVethName string, netnsPath string, addr *n err = netLink.RuleAdd(podRule) if isRuleExistsError(err) { - log.Warn("Rule already exists [%v]", podRule) + log.Warnf("Rule already exists [%v]", podRule) } else { if err != nil { log.Errorf("Failed to add pod IP rule [%v]: %v", podRule, err) @@ -291,12 +290,12 @@ func addContainerRule(netLink netlinkwrapper.NetLink, isToContainer bool, addr * } // TeardownPodNetwork cleanup ip rules -func (os *linuxNetwork) TeardownNS(addr *net.IPNet, table int) error { +func (os *linuxNetwork) TeardownNS(addr *net.IPNet, table int, log logger.Logger) error { log.Debugf("TeardownNS: addr %s, table %d", addr.String(), table) - return tearDownNS(addr, table, os.netLink) + return tearDownNS(addr, table, os.netLink, log) } -func tearDownNS(addr *net.IPNet, table int, netLink netlinkwrapper.NetLink) error { +func tearDownNS(addr *net.IPNet, table int, netLink netlinkwrapper.NetLink, log logger.Logger) error { if addr == nil { return errors.New("can't tear down network namespace with no IP address") } diff --git a/cmd/routed-eni-cni-plugin/driver/driver_test.go b/cmd/routed-eni-cni-plugin/driver/driver_test.go index 4be2a852b41..9ec590fac5a 100644 --- a/cmd/routed-eni-cni-plugin/driver/driver_test.go +++ b/cmd/routed-eni-cni-plugin/driver/driver_test.go @@ -15,6 +15,7 @@ package driver import ( "errors" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "net" "testing" @@ -32,20 +33,23 @@ import ( const ( testMAC = "01:23:45:67:89:ab" - testMAC1 = "01:23:45:67:89:a0" - testMAC2 = "01:23:45:67:89:a1" testIP = "10.0.10.10" testContVethName = "eth0" testHostVethName = "aws-eth0" testFD = 10 testnetnsPath = "/proc/1234/netns" testTable = 10 - testeniIP = "10.10.10.20" - testeniMAC = "01:23:45:67:89:ab" - testeniSubnet = "10.10.0.0/16" mtu = 9001 ) +var logConfig = logger.Configuration{ + BinaryName: "aws-cni", + LogLevel: "Debug", + LogLocation: "/var/log/test.log", +} + +var log = logger.New(&logConfig) + func setup(t *testing.T) (*gomock.Controller, *mock_netlinkwrapper.MockNetLink, *mocks_ip.MockIP, @@ -530,7 +534,7 @@ func TestSetupPodNetwork(t *testing.T) { Mask: net.IPv4Mask(255, 255, 255, 255), } var cidrs []string - err = setupNS(testHostVethName, testContVethName, testnetnsPath, addr, testTable, cidrs, true, mockNetLink, mockNS, mtu) + err = setupNS(testHostVethName, testContVethName, testnetnsPath, addr, testTable, cidrs, true, mockNetLink, mockNS, mtu, log) assert.NoError(t, err) } @@ -549,7 +553,7 @@ func TestSetupPodNetworkErrLinkByName(t *testing.T) { Mask: net.IPv4Mask(255, 255, 255, 255), } var cidrs []string - err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, testTable, cidrs, false, mockNetLink, mockNS, mtu) + err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, testTable, cidrs, false, mockNetLink, mockNS, mtu, log) assert.Error(t, err) } @@ -571,7 +575,7 @@ func TestSetupPodNetworkErrLinkSetup(t *testing.T) { Mask: net.IPv4Mask(255, 255, 255, 255), } var cidrs []string - err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, testTable, cidrs, false, mockNetLink, mockNS, mtu) + err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, testTable, cidrs, false, mockNetLink, mockNS, mtu, log) assert.Error(t, err) } @@ -604,7 +608,7 @@ func TestSetupPodNetworkErrRouteReplace(t *testing.T) { Mask: net.IPv4Mask(255, 255, 255, 255), } var cidrs []string - err = setupNS(testHostVethName, testContVethName, testnetnsPath, addr, testTable, cidrs, false, mockNetLink, mockNS, mtu) + err = setupNS(testHostVethName, testContVethName, testnetnsPath, addr, testTable, cidrs, false, mockNetLink, mockNS, mtu, log) assert.Error(t, err) } @@ -652,7 +656,7 @@ func TestSetupPodNetworkPrimaryIntf(t *testing.T) { } var cidrs []string - err = setupNS(testHostVethName, testContVethName, testnetnsPath, addr, 0, cidrs, false, mockNetLink, mockNS, mtu) + err = setupNS(testHostVethName, testContVethName, testnetnsPath, addr, 0, cidrs, false, mockNetLink, mockNS, mtu, log) assert.NoError(t, err) } @@ -683,7 +687,7 @@ func TestTearDownPodNetwork(t *testing.T) { IP: net.ParseIP(testIP), Mask: net.IPv4Mask(255, 255, 255, 255), } - err := tearDownNS(addr, 0, mockNetLink) + err := tearDownNS(addr, 0, mockNetLink, log) assert.NoError(t, err) } @@ -712,6 +716,6 @@ func TestTearDownPodNetworkMain(t *testing.T) { IP: net.ParseIP(testIP), Mask: net.IPv4Mask(255, 255, 255, 255), } - err := tearDownNS(addr, 0, mockNetLink) + err := tearDownNS(addr, 0, mockNetLink, log) assert.NoError(t, err) } diff --git a/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go b/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go index f17f64184e9..7a459e88a64 100644 --- a/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go +++ b/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go @@ -1,4 +1,4 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -10,15 +10,15 @@ // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either // express or implied. See the License for the specific language governing // permissions and limitations under the License. -// // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/aws/amazon-vpc-cni-k8s/cmd/routed-eni-cni-plugin/driver (interfaces: NetworkAPIs) +// Source: github.com/aws/amazon-vpc-cni-k8s/plugins/routed-eni/driver (interfaces: NetworkAPIs) // Package mock_driver is a generated GoMock package. package mock_driver import ( + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" net "net" reflect "reflect" @@ -49,29 +49,25 @@ func (m *MockNetworkAPIs) EXPECT() *MockNetworkAPIsMockRecorder { } // SetupNS mocks base method -func (m *MockNetworkAPIs) SetupNS(arg0, arg1, arg2 string, arg3 *net.IPNet, arg4 int, arg5 []string, arg6 bool, arg7 int) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetupNS", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) +func (m *MockNetworkAPIs) SetupNS(arg0, arg1, arg2 string, arg3 *net.IPNet, arg4 int, arg5 []string, arg6 bool, arg7 int, arg8 logger.Logger) error { + ret := m.ctrl.Call(m, "SetupNS", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) ret0, _ := ret[0].(error) return ret0 } // SetupNS indicates an expected call of SetupNS -func (mr *MockNetworkAPIsMockRecorder) SetupNS(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupNS", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupNS), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) +func (mr *MockNetworkAPIsMockRecorder) SetupNS(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupNS", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupNS), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) } // TeardownNS mocks base method -func (m *MockNetworkAPIs) TeardownNS(arg0 *net.IPNet, arg1 int) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "TeardownNS", arg0, arg1) +func (m *MockNetworkAPIs) TeardownNS(arg0 *net.IPNet, arg1 int, arg2 logger.Logger) error { + ret := m.ctrl.Call(m, "TeardownNS", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // TeardownNS indicates an expected call of TeardownNS -func (mr *MockNetworkAPIsMockRecorder) TeardownNS(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TeardownNS", reflect.TypeOf((*MockNetworkAPIs)(nil).TeardownNS), arg0, arg1) +func (mr *MockNetworkAPIsMockRecorder) TeardownNS(arg0, arg1, arg2 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TeardownNS", reflect.TypeOf((*MockNetworkAPIs)(nil).TeardownNS), arg0, arg1, arg2) } diff --git a/github.com/aws/amazon-vpc-cni-k8s/pkg/cri/mocks/cri_mocks.go b/github.com/aws/amazon-vpc-cni-k8s/pkg/cri/mocks/cri_mocks.go new file mode 100644 index 00000000000..bdb90e6abdc --- /dev/null +++ b/github.com/aws/amazon-vpc-cni-k8s/pkg/cri/mocks/cri_mocks.go @@ -0,0 +1,64 @@ +// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/aws/amazon-vpc-cni-k8s/pkg/cri (interfaces: APIs) + +// Package mock_cri is a generated GoMock package. +package mock_cri + +import ( + reflect "reflect" + + cri "github.com/aws/amazon-vpc-cni-k8s/pkg/cri" + logger "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" + gomock "github.com/golang/mock/gomock" +) + +// MockAPIs is a mock of APIs interface +type MockAPIs struct { + ctrl *gomock.Controller + recorder *MockAPIsMockRecorder +} + +// MockAPIsMockRecorder is the mock recorder for MockAPIs +type MockAPIsMockRecorder struct { + mock *MockAPIs +} + +// NewMockAPIs creates a new mock instance +func NewMockAPIs(ctrl *gomock.Controller) *MockAPIs { + mock := &MockAPIs{ctrl: ctrl} + mock.recorder = &MockAPIsMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockAPIs) EXPECT() *MockAPIsMockRecorder { + return m.recorder +} + +// GetRunningPodSandboxes mocks base method +func (m *MockAPIs) GetRunningPodSandboxes(arg0 logger.Logger) (map[string]*cri.SandboxInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRunningPodSandboxes", arg0) + ret0, _ := ret[0].(map[string]*cri.SandboxInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRunningPodSandboxes indicates an expected call of GetRunningPodSandboxes +func (mr *MockAPIsMockRecorder) GetRunningPodSandboxes(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRunningPodSandboxes", reflect.TypeOf((*MockAPIs)(nil).GetRunningPodSandboxes), arg0) +} diff --git a/go.mod b/go.mod index 96a85aeaae6..b75613cf00c 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/aws/amazon-vpc-cni-k8s go 1.13 require ( + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect github.com/aws/aws-sdk-go v1.26.8 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect github.com/cihub/seelog v0.0.0-20151216151435-d2c6e5aa9fbf @@ -33,14 +35,17 @@ require ( github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 // indirect github.com/sirupsen/logrus v1.4.1 // indirect github.com/spf13/pflag v1.0.2 - github.com/stretchr/testify v1.3.0 + github.com/stretchr/testify v1.4.0 github.com/vishvananda/netlink v1.1.0 + go.uber.org/zap v1.13.0 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect google.golang.org/grpc v1.23.1 + gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.0.0 k8s.io/api v0.0.0-20180712090710-2d6f90ab1293 k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d k8s.io/client-go v0.0.0-20180806134042-1f13a808da65 diff --git a/go.sum b/go.sum index d868e038389..98786bde6ad 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,10 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/aws/aws-sdk-go v1.26.8 h1:W+MPuCFLSO/itZkZ5GFOui0YC1j3lZ507/m5DFPtzE4= github.com/aws/aws-sdk-go v1.26.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= @@ -49,6 +53,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= @@ -69,6 +74,9 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -102,6 +110,7 @@ github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e h1:n/3MEhJQjQxrO github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 h1:agujYaXJSxSo18YNX3jzl+4G6Bstwt+kqv47GS12uL0= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -114,18 +123,30 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.3.0 h1:sFPn2GLc3poCkfrpIXGhBD2X0CMIo4Q/zSULXrj/+uc= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -135,6 +156,7 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -170,6 +192,11 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= @@ -179,18 +206,27 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= k8s.io/api v0.0.0-20180712090710-2d6f90ab1293 h1:hROmpFC7JMobXFXMmD7ZKZLhDKvr1IKfFJoYS/45G/8= k8s.io/api v0.0.0-20180712090710-2d6f90ab1293/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d h1:MZjlsu9igBoVPZkXpIGoxI6EonqNsXXZU7hhvfQLkd4= diff --git a/misc/10-aws.conflist b/misc/10-aws.conflist index a7cb87791c3..fe26c8346f7 100644 --- a/misc/10-aws.conflist +++ b/misc/10-aws.conflist @@ -6,7 +6,9 @@ "name": "aws-cni", "type": "aws-cni", "vethPrefix": "__VETHPREFIX__", - "mtu": "__MTU__" + "mtu": "__MTU__", + "pluginLogFile": "__PLUGINLOGFILE__", + "pluginLogLevel": "__PLUGINLOGLEVEL__" }, { "type": "portmap", @@ -14,4 +16,4 @@ "snat": true } ] -} +} \ No newline at end of file diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go index 5c602f61bca..f60c3bff8a2 100644 --- a/pkg/awsutils/awsutils.go +++ b/pkg/awsutils/awsutils.go @@ -25,7 +25,7 @@ import ( "github.com/pkg/errors" - log "github.com/cihub/seelog" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "github.com/prometheus/client_golang/prometheus" "github.com/aws/amazon-vpc-cni-k8s/pkg/ec2metadata" @@ -76,6 +76,8 @@ const ( // ErrENINotFound is an error when ENI is not found. var ErrENINotFound = errors.New("ENI is not found") +var log = logger.Get() + var ( awsAPILatency = prometheus.NewSummaryVec( prometheus.SummaryOpts{ @@ -738,7 +740,8 @@ func (cache *EC2InstanceMetadataCache) tagENI(eniID string, maxBackoffDelay time awsAPILatency.WithLabelValues("CreateTags", fmt.Sprint(err != nil)).Observe(msSince(start)) if err != nil { awsAPIErrInc("CreateTags", err) - return log.Warnf("Failed to tag the newly created ENI %s: %v", eniID, err) + log.Warnf("Failed to tag the newly created ENI %s:", eniID) + return err } log.Debugf("Successfully tagged ENI: %s", eniID) return nil diff --git a/pkg/awsutils/gen_vpc_ip_limits.go b/pkg/awsutils/gen_vpc_ip_limits.go index 4adbffda0fd..8bb2f746ee8 100644 --- a/pkg/awsutils/gen_vpc_ip_limits.go +++ b/pkg/awsutils/gen_vpc_ip_limits.go @@ -18,7 +18,7 @@ package main import ( - "log" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "os" "sort" "text/template" @@ -40,13 +40,14 @@ type ENILimit struct { // Helper function to call the EC2 DescribeInstanceTypes API and generate the IP limit file. func main() { + log := logger.Get() // Get session sess := session.Must(session.NewSessionWithOptions(session.Options{ SharedConfigState: session.SharedConfigEnable, })) _, err := sess.Config.Credentials.Get() if err != nil { - log.Fatal(err) + log.Fatalf(err) } svc := ec2.New(sess) describeInstanceTypesInput := &ec2.DescribeInstanceTypesInput{} @@ -55,7 +56,7 @@ func main() { for { output, err := svc.DescribeInstanceTypes(describeInstanceTypesInput) if err != nil { - log.Fatal(err) + log.Fatalf(err) } // We just want the type name, ENI and IP limits for _, info := range output.InstanceTypes { @@ -93,7 +94,7 @@ func main() { // Generate the file f, err := os.Create(ipLimitFileName) if err != nil { - log.Fatal(err) + log.Fatalf(err) } limitsTemplate.Execute(f, struct { Timestamp string diff --git a/pkg/cri/cri.go b/pkg/cri/cri.go index 78546358980..23c740b1bce 100644 --- a/pkg/cri/cri.go +++ b/pkg/cri/cri.go @@ -5,10 +5,9 @@ import ( "errors" "os" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "google.golang.org/grpc" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" - - log "github.com/cihub/seelog" ) const ( @@ -25,7 +24,7 @@ type SandboxInfo struct { } type APIs interface { - GetRunningPodSandboxes() (map[string]*SandboxInfo, error) + GetRunningPodSandboxes(log logger.Logger) (map[string]*SandboxInfo, error) } type Client struct{} @@ -34,7 +33,8 @@ func New() *Client { return &Client{} } -func (c *Client) GetRunningPodSandboxes() (map[string]*SandboxInfo, error) { +//GetRunningPodSandboxes get running sandboxIDs +func (c *Client) GetRunningPodSandboxes(log logger.Logger) (map[string]*SandboxInfo, error) { socketPath := dockerSocketPath if info, err := os.Stat("/var/run/cri.sock"); err == nil && !info.IsDir() { socketPath = criSocketPath @@ -48,7 +48,7 @@ func (c *Client) GetRunningPodSandboxes() (map[string]*SandboxInfo, error) { client := runtimeapi.NewRuntimeServiceClient(conn) - // List all ready sandboxes from the CRI + // List all ready sandboxes froms the CRI filter := &runtimeapi.PodSandboxFilter{ State: &runtimeapi.PodSandboxStateValue{ State: runtimeapi.PodSandboxState_SANDBOX_READY, diff --git a/pkg/cri/mocks/cri_mocks.go b/pkg/cri/mocks/cri_mocks.go index dbab45380fa..bdb90e6abdc 100644 --- a/pkg/cri/mocks/cri_mocks.go +++ b/pkg/cri/mocks/cri_mocks.go @@ -1,4 +1,4 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -10,7 +10,6 @@ // on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either // express or implied. See the License for the specific language governing // permissions and limitations under the License. -// // Code generated by MockGen. DO NOT EDIT. // Source: github.com/aws/amazon-vpc-cni-k8s/pkg/cri (interfaces: APIs) @@ -22,6 +21,7 @@ import ( reflect "reflect" cri "github.com/aws/amazon-vpc-cni-k8s/pkg/cri" + logger "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" gomock "github.com/golang/mock/gomock" ) @@ -49,16 +49,16 @@ func (m *MockAPIs) EXPECT() *MockAPIsMockRecorder { } // GetRunningPodSandboxes mocks base method -func (m *MockAPIs) GetRunningPodSandboxes() (map[string]*cri.SandboxInfo, error) { +func (m *MockAPIs) GetRunningPodSandboxes(arg0 logger.Logger) (map[string]*cri.SandboxInfo, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetRunningPodSandboxes") + ret := m.ctrl.Call(m, "GetRunningPodSandboxes", arg0) ret0, _ := ret[0].(map[string]*cri.SandboxInfo) ret1, _ := ret[1].(error) return ret0, ret1 } // GetRunningPodSandboxes indicates an expected call of GetRunningPodSandboxes -func (mr *MockAPIsMockRecorder) GetRunningPodSandboxes() *gomock.Call { +func (mr *MockAPIsMockRecorder) GetRunningPodSandboxes(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRunningPodSandboxes", reflect.TypeOf((*MockAPIs)(nil).GetRunningPodSandboxes)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRunningPodSandboxes", reflect.TypeOf((*MockAPIs)(nil).GetRunningPodSandboxes), arg0) } diff --git a/pkg/ec2wrapper/ec2wrapper.go b/pkg/ec2wrapper/ec2wrapper.go index 684cbfd62fd..34f0268ea20 100644 --- a/pkg/ec2wrapper/ec2wrapper.go +++ b/pkg/ec2wrapper/ec2wrapper.go @@ -1,15 +1,14 @@ -// package ec2wrapper is used to wrap around the ec2 service APIs +//Package ec2wrapper is used to wrap around the ec2 service APIs package ec2wrapper import ( "github.com/aws/amazon-vpc-cni-k8s/pkg/ec2metadatawrapper" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" - "github.com/golang/glog" - "github.com/pkg/errors" ) @@ -20,6 +19,8 @@ const ( clusterIDTag = "CLUSTER_ID" ) +var log = logger.Get() + // EC2Wrapper is used to wrap around EC2 service APIs to obtain ClusterID from // the ec2 instance tags type EC2Wrapper struct { @@ -27,7 +28,7 @@ type EC2Wrapper struct { instanceIdentityDocument ec2metadata.EC2InstanceIdentityDocument } -// New returns an instance of the EC2 wrapper +//NewMetricsClient returns an instance of the EC2 wrapper func NewMetricsClient() (*EC2Wrapper, error) { metricsSession := session.Must(session.NewSession()) ec2MetadataClient := ec2metadatawrapper.New(nil) @@ -63,7 +64,7 @@ func (e *EC2Wrapper) GetClusterTag(tagKey string) (string, error) { }, } - glog.Info("Calling DescribeTags with key ", tagKey) + log.Infof("Calling DescribeTags with key %s", tagKey) results, err := e.ec2ServiceClient.DescribeTags(&input) if err != nil { return "", errors.Wrap(err, "GetClusterTag: Unable to obtain EC2 instance tags") diff --git a/pkg/eniconfig/eniconfig.go b/pkg/eniconfig/eniconfig.go index 44d3556551b..eb0757b54da 100644 --- a/pkg/eniconfig/eniconfig.go +++ b/pkg/eniconfig/eniconfig.go @@ -25,8 +25,7 @@ import ( "github.com/pkg/errors" "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" - - log "github.com/cihub/seelog" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" sdkVersion "github.com/operator-framework/operator-sdk/version" corev1 "k8s.io/api/core/v1" ) @@ -54,6 +53,8 @@ type ENIConfig interface { var ErrNoENIConfig = errors.New("eniconfig: eniconfig is not available") +var log = logger.Get() + // ENIConfigController defines global context for ENIConfig controller type ENIConfigController struct { eni map[string]*v1alpha1.ENIConfigSpec diff --git a/pkg/ipamd/datastore/data_store.go b/pkg/ipamd/datastore/data_store.go index 9b6681f5d22..7105e94e4a5 100644 --- a/pkg/ipamd/datastore/data_store.go +++ b/pkg/ipamd/datastore/data_store.go @@ -18,11 +18,10 @@ import ( "sync" "time" - log "github.com/cihub/seelog" + "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - - "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi" ) const ( @@ -139,6 +138,7 @@ type DataStore struct { eniIPPools map[string]*ENIIPPool podsIP map[PodKey]PodIPInfo lock sync.RWMutex + log logger.Logger } // PodInfos contains pods IP information which uses key name_namespace_sandbox @@ -166,11 +166,12 @@ func prometheusRegister() { } // NewDataStore returns DataStore structure -func NewDataStore() *DataStore { +func NewDataStore(log logger.Logger) *DataStore { prometheusRegister() return &DataStore{ eniIPPools: make(map[string]*ENIIPPool), podsIP: make(map[PodKey]PodIPInfo), + log: log, } } @@ -179,7 +180,7 @@ func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary bool) erro ds.lock.Lock() defer ds.lock.Unlock() - log.Debug("DataStore Add an ENI ", eniID) + ds.log.Debugf("DataStore Add an ENI %s", eniID) _, ok := ds.eniIPPools[eniID] if ok { @@ -200,8 +201,8 @@ func (ds *DataStore) AddIPv4AddressToStore(eniID string, ipv4 string) error { ds.lock.Lock() defer ds.lock.Unlock() - log.Tracef("Adding ENI(%s)'s IPv4 address %s to datastore", eniID, ipv4) - log.Tracef("IP Address Pool stats: total: %d, assigned: %d", ds.total, ds.assigned) + ds.log.Debugf("Adding ENI(%s)'s IPv4 address %s to datastore", eniID, ipv4) + ds.log.Debugf("IP Address Pool stats: total: %d, assigned: %d", ds.total, ds.assigned) curENI, ok := ds.eniIPPools[eniID] if !ok { @@ -219,7 +220,7 @@ func (ds *DataStore) AddIPv4AddressToStore(eniID string, ipv4 string) error { totalIPs.Set(float64(ds.total)) curENI.IPv4Addresses[ipv4] = &AddressInfo{Address: ipv4, Assigned: false} - log.Infof("Added ENI(%s)'s IP %s to datastore", eniID, ipv4) + ds.log.Infof("Added ENI(%s)'s IP %s to datastore", eniID, ipv4) return nil } @@ -227,8 +228,8 @@ func (ds *DataStore) AddIPv4AddressToStore(eniID string, ipv4 string) error { func (ds *DataStore) DelIPv4AddressFromStore(eniID string, ipv4 string, force bool) error { ds.lock.Lock() defer ds.lock.Unlock() - log.Debugf("Deleting ENI(%s)'s IPv4 address %s from datastore", eniID, ipv4) - log.Debugf("IP Address Pool stats: total: %d, assigned: %d", ds.total, ds.assigned) + ds.log.Debugf("Deleting ENI(%s)'s IPv4 address %s from datastore", eniID, ipv4) + ds.log.Debugf("IP Address Pool stats: total: %d, assigned: %d", ds.total, ds.assigned) curENI, ok := ds.eniIPPools[eniID] if !ok { @@ -244,7 +245,7 @@ func (ds *DataStore) DelIPv4AddressFromStore(eniID string, ipv4 string, force bo if !force { return errors.New(IPInUseError) } - log.Warnf("Force deleting assigned ip %s on eni %s", ipv4, eniID) + ds.log.Warnf("Force deleting assigned ip %s on eni %s", ipv4, eniID) forceRemovedIPs.Inc() decrementAssignedCount(ds, curENI, ipAddr) for key, info := range ds.podsIP { @@ -261,7 +262,7 @@ func (ds *DataStore) DelIPv4AddressFromStore(eniID string, ipv4 string, force bo delete(curENI.IPv4Addresses, ipv4) - log.Infof("Deleted ENI(%s)'s IP %s from datastore", eniID, ipv4) + ds.log.Infof("Deleted ENI(%s)'s IP %s from datastore", eniID, ipv4) return nil } @@ -271,7 +272,7 @@ func (ds *DataStore) AssignPodIPv4Address(k8sPod *k8sapi.K8SPodInfo) (ip string, ds.lock.Lock() defer ds.lock.Unlock() - log.Debugf("AssignIPv4Address: IP address pool stats: total: %d, assigned %d", ds.total, ds.assigned) + ds.log.Debugf("AssignIPv4Address: IP address pool stats: total: %d, assigned %d", ds.total, ds.assigned) podKey := PodKey{ name: k8sPod.Name, namespace: k8sPod.Namespace, @@ -281,11 +282,11 @@ func (ds *DataStore) AssignPodIPv4Address(k8sPod *k8sapi.K8SPodInfo) (ip string, if ok { if ipAddr.IP == k8sPod.IP && k8sPod.IP != "" { // The caller invoke multiple times to assign(PodName/NameSpace --> same IPAddress). It is not a error, but not very efficient. - log.Infof("AssignPodIPv4Address: duplicate pod assign for IP %s, name %s, namespace %s, sandbox %s", + ds.log.Infof("AssignPodIPv4Address: duplicate pod assign for IP %s, name %s, namespace %s, sandbox %s", k8sPod.IP, k8sPod.Name, k8sPod.Namespace, k8sPod.Sandbox) return ipAddr.IP, ipAddr.DeviceNumber, nil } - log.Errorf("AssignPodIPv4Address: current IP %s is changed to IP %s for pod(name %s, namespace %s, sandbox %s)", + ds.log.Errorf("AssignPodIPv4Address: current IP %s is changed to IP %s for pod(name %s, namespace %s, sandbox %s)", ipAddr, k8sPod.IP, k8sPod.Name, k8sPod.Namespace, k8sPod.Sandbox) return "", 0, errors.New("AssignPodIPv4Address: invalid pod with multiple IP addresses") } @@ -297,7 +298,7 @@ func (ds *DataStore) assignPodIPv4AddressUnsafe(podKey PodKey, k8sPod *k8sapi.K8 for _, eni := range ds.eniIPPools { if (k8sPod.IP == "") && (len(eni.IPv4Addresses) == eni.AssignedIPv4Addresses) { // Skip this ENI, since it has no available IP addresses - log.Debugf("AssignPodIPv4Address: Skip ENI %s that does not have available addresses", eni.ID) + ds.log.Debugf("AssignPodIPv4Address: Skip ENI %s that does not have available addresses", eni.ID) continue } for _, addr := range eni.IPv4Addresses { @@ -306,7 +307,7 @@ func (ds *DataStore) assignPodIPv4AddressUnsafe(podKey PodKey, k8sPod *k8sapi.K8 if !addr.Assigned { incrementAssignedCount(ds, eni, addr) } - log.Infof("AssignPodIPv4Address: Reassign IP %v to pod (name %s, namespace %s)", + ds.log.Infof("AssignPodIPv4Address: Reassign IP %v to pod (name %s, namespace %s)", addr.Address, k8sPod.Name, k8sPod.Namespace) ds.podsIP[podKey] = PodIPInfo{IP: addr.Address, DeviceNumber: eni.DeviceNumber} return addr.Address, eni.DeviceNumber, nil @@ -314,14 +315,14 @@ func (ds *DataStore) assignPodIPv4AddressUnsafe(podKey PodKey, k8sPod *k8sapi.K8 if !addr.Assigned && k8sPod.IP == "" && !addr.inCoolingPeriod() { // This is triggered by a pod's Add Network command from CNI plugin incrementAssignedCount(ds, eni, addr) - log.Infof("AssignPodIPv4Address: Assign IP %v to pod (name %s, namespace %s sandbox %s)", + ds.log.Infof("AssignPodIPv4Address: Assign IP %v to pod (name %s, namespace %s sandbox %s)", addr.Address, k8sPod.Name, k8sPod.Namespace, k8sPod.Sandbox) ds.podsIP[podKey] = PodIPInfo{IP: addr.Address, DeviceNumber: eni.DeviceNumber} return addr.Address, eni.DeviceNumber, nil } } } - log.Errorf("DataStore has no available IP addresses") + ds.log.Errorf("DataStore has no available IP addresses") return "", 0, errors.New("assignPodIPv4AddressUnsafe: no available IP addresses") } @@ -376,36 +377,36 @@ func (ds *DataStore) isRequiredForMinimumIPTarget(minimumIPTarget int, eni *ENII func (ds *DataStore) getDeletableENI(warmIPTarget int, minimumIPTarget int) *ENIIPPool { for _, eni := range ds.eniIPPools { if eni.IsPrimary { - log.Debugf("ENI %s cannot be deleted because it is primary", eni.ID) + ds.log.Debugf("ENI %s cannot be deleted because it is primary", eni.ID) continue } if eni.isTooYoung() { - log.Debugf("ENI %s cannot be deleted because it is too young", eni.ID) + ds.log.Debugf("ENI %s cannot be deleted because it is too young", eni.ID) continue } if eni.hasIPInCooling() { - log.Debugf("ENI %s cannot be deleted because has IPs in cooling", eni.ID) + ds.log.Debugf("ENI %s cannot be deleted because has IPs in cooling", eni.ID) continue } if eni.hasPods() { - log.Debugf("ENI %s cannot be deleted because it has pods assigned", eni.ID) + ds.log.Debugf("ENI %s cannot be deleted because it has pods assigned", eni.ID) continue } if warmIPTarget != 0 && ds.isRequiredForWarmIPTarget(warmIPTarget, eni) { - log.Debugf("ENI %s cannot be deleted because it is required for WARM_IP_TARGET: %d", eni.ID, warmIPTarget) + ds.log.Debugf("ENI %s cannot be deleted because it is required for WARM_IP_TARGET: %d", eni.ID, warmIPTarget) continue } if minimumIPTarget != 0 && ds.isRequiredForMinimumIPTarget(minimumIPTarget, eni) { - log.Debugf("ENI %s cannot be deleted because it is required for MINIMUM_IP_TARGET: %d", eni.ID, minimumIPTarget) + ds.log.Debugf("ENI %s cannot be deleted because it is required for MINIMUM_IP_TARGET: %d", eni.ID, minimumIPTarget) continue } - log.Debugf("getDeletableENI: found a deletable ENI %s", eni.ID) + ds.log.Debugf("getDeletableENI: found a deletable ENI %s", eni.ID) return eni } return nil @@ -435,7 +436,7 @@ func (ds *DataStore) GetENINeedsIP(maxIPperENI int, skipPrimary bool) *ENIIPPool eniIDs := make([]string, 0) for eniID, eni := range ds.eniIPPools { if skipPrimary && eni.IsPrimary { - log.Debugf("Skip the primary ENI for need IP check") + ds.log.Debugf("Skip the primary ENI for need IP check") continue } eniIDs = append(eniIDs, eniID) @@ -444,7 +445,7 @@ func (ds *DataStore) GetENINeedsIP(maxIPperENI int, skipPrimary bool) *ENIIPPool for _, eniID := range eniIDs { eni := ds.eniIPPools[eniID] if len(eni.IPv4Addresses) < maxIPperENI { - log.Debugf("Found ENI %s that has less than the maximum number of IP addresses allocated: cur=%d, max=%d", + ds.log.Debugf("Found ENI %s that has less than the maximum number of IP addresses allocated: cur=%d, max=%d", eni.ID, len(eni.IPv4Addresses), maxIPperENI) return eni } @@ -467,7 +468,7 @@ func (ds *DataStore) RemoveUnusedENIFromStore(warmIPTarget int, minimumIPTarget removableENI := deletableENI.ID eniIPCount := len(ds.eniIPPools[removableENI].IPv4Addresses) ds.total -= eniIPCount - log.Infof("RemoveUnusedENIFromStore %s: IP address pool stats: free %d addresses, total: %d, assigned: %d", + ds.log.Infof("RemoveUnusedENIFromStore %s: IP address pool stats: free %d addresses, total: %d, assigned: %d", removableENI, eniIPCount, ds.total, ds.assigned) delete(ds.eniIPPools, removableENI) @@ -494,7 +495,7 @@ func (ds *DataStore) RemoveENIFromDataStore(eni string, force bool) error { // This scenario can occur if the reconciliation process discovered this eni was detached // from the EC2 instance outside of the control of ipamd. If this happens, there's nothing // we can do other than force all pods to be unassigned from the IPs on this eni. - log.Warnf("Force removing eni %s with %d assigned pods", eni, eniIPPool.AssignedIPv4Addresses) + ds.log.Warnf("Force removing eni %s with %d assigned pods", eni, eniIPPool.AssignedIPv4Addresses) forceRemovedENIs.Inc() forceRemovedIPs.Add(float64(eniIPPool.AssignedIPv4Addresses)) for _, addr := range eniIPPool.IPv4Addresses { @@ -510,7 +511,7 @@ func (ds *DataStore) RemoveENIFromDataStore(eni string, force bool) error { } ds.total -= len(eniIPPool.IPv4Addresses) - log.Infof("RemoveENIFromDataStore %s: IP address pool stats: free %d addresses, total: %d, assigned: %d", + ds.log.Infof("RemoveENIFromDataStore %s: IP address pool stats: free %d addresses, total: %d, assigned: %d", eni, len(eniIPPool.IPv4Addresses), ds.total, ds.assigned) delete(ds.eniIPPools, eni) @@ -524,7 +525,7 @@ func (ds *DataStore) RemoveENIFromDataStore(eni string, force bool) error { func (ds *DataStore) UnassignPodIPv4Address(k8sPod *k8sapi.K8SPodInfo) (ip string, deviceNumber int, err error) { ds.lock.Lock() defer ds.lock.Unlock() - log.Debugf("UnassignPodIPv4Address: IP address pool stats: total:%d, assigned %d, pod(Name: %s, Namespace: %s, Sandbox %s)", + ds.log.Debugf("UnassignPodIPv4Address: IP address pool stats: total:%d, assigned %d, pod(Name: %s, Namespace: %s, Sandbox %s)", ds.total, ds.assigned, k8sPod.Name, k8sPod.Namespace, k8sPod.Sandbox) podKey := PodKey{ @@ -534,7 +535,7 @@ func (ds *DataStore) UnassignPodIPv4Address(k8sPod *k8sapi.K8SPodInfo) (ip strin } ipAddr, ok := ds.podsIP[podKey] if !ok { - log.Warnf("UnassignPodIPv4Address: Failed to find pod %s namespace %q, sandbox %q", + ds.log.Warnf("UnassignPodIPv4Address: Failed to find pod %s namespace %q, sandbox %q", k8sPod.Name, k8sPod.Namespace, k8sPod.Sandbox) return "", 0, ErrUnknownPod } @@ -543,14 +544,14 @@ func (ds *DataStore) UnassignPodIPv4Address(k8sPod *k8sapi.K8SPodInfo) (ip strin ip, ok := eni.IPv4Addresses[ipAddr.IP] if ok && ip.Assigned { decrementAssignedCount(ds, eni, ip) - log.Infof("UnassignPodIPv4Address: pod (Name: %s, NameSpace %s Sandbox %s)'s ipAddr %s, DeviceNumber%d", + ds.log.Infof("UnassignPodIPv4Address: pod (Name: %s, NameSpace %s Sandbox %s)'s ipAddr %s, DeviceNumber%d", k8sPod.Name, k8sPod.Namespace, k8sPod.Sandbox, ip.Address, eni.DeviceNumber) delete(ds.podsIP, podKey) return ip.Address, eni.DeviceNumber, nil } } - log.Warnf("UnassignPodIPv4Address: Failed to find pod %s namespace %s sandbox %s using IP %s", + ds.log.Warnf("UnassignPodIPv4Address: Failed to find pod %s namespace %s sandbox %s using IP %s", k8sPod.Name, k8sPod.Namespace, k8sPod.Sandbox, ipAddr.IP) return "", 0, ErrUnknownPodIP } @@ -565,10 +566,10 @@ func (ds *DataStore) GetPodInfos() *map[string]PodIPInfo { for podKey, podInfo := range ds.podsIP { key := podKey.name + "_" + podKey.namespace + "_" + podKey.sandbox podInfos[key] = podInfo - log.Debugf("GetPodInfos: key %s", key) + ds.log.Debugf("GetPodInfos: key %s", key) } - log.Debugf("GetPodInfos: len %d", len(ds.podsIP)) + ds.log.Debugf("GetPodInfos: len %d", len(ds.podsIP)) return &podInfos } diff --git a/pkg/ipamd/datastore/data_store_test.go b/pkg/ipamd/datastore/data_store_test.go index 2a5515be745..c331a5ff4dd 100644 --- a/pkg/ipamd/datastore/data_store_test.go +++ b/pkg/ipamd/datastore/data_store_test.go @@ -14,6 +14,7 @@ package datastore import ( + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "testing" "time" @@ -21,8 +22,16 @@ import ( "github.com/stretchr/testify/assert" ) +var logConfig = logger.Configuration{ + BinaryName: "aws-cni", + LogLevel: "Debug", + LogLocation: "/var/log/test.log", +} + +var log = logger.New(&logConfig) + func TestAddENI(t *testing.T) { - ds := NewDataStore() + ds := NewDataStore(log) err := ds.AddENI("eni-1", 1, true) assert.NoError(t, err) @@ -40,7 +49,7 @@ func TestAddENI(t *testing.T) { } func TestDeleteENI(t *testing.T) { - ds := NewDataStore() + ds := NewDataStore(log) err := ds.AddENI("eni-1", 1, true) assert.NoError(t, err) @@ -88,7 +97,7 @@ func TestDeleteENI(t *testing.T) { } func TestAddENIIPv4Address(t *testing.T) { - ds := NewDataStore() + ds := NewDataStore(log) err := ds.AddENI("eni-1", 1, true) assert.NoError(t, err) @@ -126,7 +135,7 @@ func TestAddENIIPv4Address(t *testing.T) { } func TestGetENIIPPools(t *testing.T) { - ds := NewDataStore() + ds := NewDataStore(log) err := ds.AddENI("eni-1", 1, true) assert.NoError(t, err) @@ -159,7 +168,7 @@ func TestGetENIIPPools(t *testing.T) { } func TestDelENIIPv4Address(t *testing.T) { - ds := NewDataStore() + ds := NewDataStore(log) err := ds.AddENI("eni-1", 1, true) assert.NoError(t, err) @@ -214,7 +223,7 @@ func TestDelENIIPv4Address(t *testing.T) { } func TestPodIPv4Address(t *testing.T) { - ds := NewDataStore() + ds := NewDataStore(log) ds.AddENI("eni-1", 1, true) @@ -347,7 +356,7 @@ func TestPodIPv4Address(t *testing.T) { } func TestWarmENIInteractions(t *testing.T) { - ds := NewDataStore() + ds := NewDataStore(log) ds.AddENI("eni-1", 1, true) ds.AddENI("eni-2", 2, false) diff --git a/pkg/ipamd/introspect.go b/pkg/ipamd/introspect.go index 7421191fe46..dacef6f40f1 100644 --- a/pkg/ipamd/introspect.go +++ b/pkg/ipamd/introspect.go @@ -20,11 +20,8 @@ import ( "os" "strconv" "strings" - "sync" "time" - log "github.com/cihub/seelog" - "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/retry" ) @@ -63,7 +60,6 @@ func (c *IPAMContext) ServeIntrospection() { server := c.setupIntrospectionServer() for { - once := sync.Once{} _ = retry.RetryWithBackoff(retry.NewSimpleBackoff(time.Second, time.Minute, 0.2, 2), func() error { var ln net.Listener var err error @@ -79,9 +75,6 @@ func (c *IPAMContext) ServeIntrospection() { err = server.Serve(ln) } - once.Do(func() { - log.Error("Error running http API: ", err) - }) return err }) } @@ -125,7 +118,7 @@ func (c *IPAMContext) setupIntrospectionServer() *http.Server { addr = defaultIntrospectionBindAddress } - log.Info("Serving introspection endpoints on ", addr) + log.Infof("Serving introspection endpoints on %s", addr) server := &http.Server{ Addr: addr, diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index e8938979122..8a766f1675c 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -23,9 +23,9 @@ import ( "sync/atomic" "time" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - log "github.com/cihub/seelog" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "k8s.io/apimachinery/pkg/util/sets" @@ -111,6 +111,8 @@ const ( eniNoManageTagKey = "node.k8s.amazonaws.com/no_manage" ) +var log = logger.Get() + var ( ipamdErr = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -185,7 +187,7 @@ type IPAMContext struct { terminating int32 // Flag to warn that the pod is about to shut down. } -// Keep track of recently freed IPs to avoid reading stale EC2 metadata +//ReconcileCooldownCache keep track of recently freed IPs to avoid reading stale EC2 metadata type ReconcileCooldownCache struct { cache map[string]time.Time lock sync.RWMutex @@ -247,7 +249,6 @@ func New(k8sapiClient k8sapi.K8SAPIs, eniConfig *eniconfig.ENIConfigController) client, err := awsutils.New() if err != nil { - log.Errorf("Failed to initialize awsutil interface %v", err) return nil, errors.Wrap(err, "ipamd: can not initialize with AWS SDK interface") } c.awsClient = client @@ -275,7 +276,6 @@ func (c *IPAMContext) nodeInit() error { allENIs, err := c.awsClient.GetAttachedENIs() if err != nil { - log.Error("Failed to retrieve ENI info") return errors.New("ipamd init: failed to retrieve attached ENIs info") } enis, numUnmanaged := filterUnmanagedENIs(allENIs) @@ -288,7 +288,6 @@ func (c *IPAMContext) nodeInit() error { c.unmanagedENI = numUnmanaged c.maxIPsPerENI, err = c.awsClient.GetENIipLimit() if err != nil { - log.Error("Failed to get IPs per ENI limit") return err } c.updateIPStats(numUnmanaged) @@ -302,18 +301,16 @@ func (c *IPAMContext) nodeInit() error { _, vpcCIDR, err := net.ParseCIDR(c.awsClient.GetVPCIPv4CIDR()) if err != nil { - log.Error("Failed to parse VPC IPv4 CIDR", err.Error()) return errors.Wrap(err, "ipamd init: failed to retrieve VPC CIDR") } primaryIP := net.ParseIP(c.awsClient.GetLocalIPv4()) err = c.networkClient.SetupHostNetwork(vpcCIDR, vpcCIDRs, c.awsClient.GetPrimaryENImac(), &primaryIP) if err != nil { - log.Error("Failed to set up host network", err) return errors.Wrap(err, "ipamd init: failed to set up host network") } - c.dataStore = datastore.NewDataStore() + c.dataStore = datastore.NewDataStore(log) for _, eni := range enis { log.Debugf("Discovered ENI %s, trying to set it up", eni.ENIID) // Retry ENI sync @@ -326,7 +323,7 @@ func (c *IPAMContext) nodeInit() error { } if retry > maxRetryCheckENI { - log.Errorf("Unable to discover attached IPs for ENI from metadata service") + log.Warn("Unable to discover attached IPs for ENI from metadata service") ipamdErrInc("waitENIAttachedMaxRetryExceeded") break } @@ -334,7 +331,7 @@ func (c *IPAMContext) nodeInit() error { log.Warnf("Error trying to set up ENI %s: %v", eni.ENIID, err) if strings.Contains(err.Error(), "setupENINetwork: failed to find the link which uses MAC address") { // If we can't find the matching link for this MAC address, there is no point in retrying for this ENI. - log.Errorf("Unable to match link for this ENI, going to the next one.") + log.Debug("Unable to match link for this ENI, going to the next one.") break } log.Debugf("Unable to discover IPs for this ENI yet (attempt %d/%d)", retry, maxRetryCheckENI) @@ -430,7 +427,7 @@ func (c *IPAMContext) getLocalPodsWithRetry() ([]*k8sapi.K8SPodInfo, error) { // could leak IPs or unassign an IP from a still-running pod. var sandboxes map[string]*cri.SandboxInfo for retry := 1; retry <= maxK8SRetries; retry++ { - sandboxes, err = c.criClient.GetRunningPodSandboxes() + sandboxes, err = c.criClient.GetRunningPodSandboxes(log) if err == nil { break } @@ -755,7 +752,6 @@ func (c *IPAMContext) setupENI(eni string, eniMetadata awsutils.ENIMetadata) err eniPrimaryIP := eniMetadata.PrimaryIPv4Address() err = c.networkClient.SetupENINetwork(eniPrimaryIP, eniMetadata.MAC, eniMetadata.DeviceNumber, eniMetadata.SubnetIPv4CIDR) if err != nil { - log.Errorf("Failed to set up networking for ENI %s", eni) return errors.Wrapf(err, "failed to set up ENI %s network", eni) } } @@ -882,9 +878,9 @@ func (c *IPAMContext) nodeIPPoolTooLow() bool { available := total - used poolTooLow := available < c.maxIPsPerENI*c.warmENITarget || (c.warmENITarget == 0 && available == 0) if poolTooLow { - log.Tracef("IP pool is too low: available (%d) < ENI target (%d) * addrsPerENI (%d)", available, c.warmENITarget, c.maxIPsPerENI) + log.Debugf("IP pool is too low: available (%d) < ENI target (%d) * addrsPerENI (%d)", available, c.warmENITarget, c.maxIPsPerENI) } else { - log.Tracef("IP pool is NOT too low: available (%d) >= ENI target (%d) * addrsPerENI (%d)", available, c.warmENITarget, c.maxIPsPerENI) + log.Debugf("IP pool is NOT too low: available (%d) >= ENI target (%d) * addrsPerENI (%d)", available, c.warmENITarget, c.maxIPsPerENI) } return poolTooLow } @@ -915,9 +911,9 @@ func (c *IPAMContext) shouldRemoveExtraENIs() bool { // We need the +1 to make sure we are not going below the WARM_ENI_TARGET. shouldRemoveExtra := available >= (c.warmENITarget+1)*c.maxIPsPerENI if shouldRemoveExtra { - log.Tracef("It might be possible to remove extra ENIs because available (%d) >= (ENI target (%d) + 1) * addrsPerENI (%d): ", available, c.warmENITarget, c.maxIPsPerENI) + log.Debugf("It might be possible to remove extra ENIs because available (%d) >= (ENI target (%d) + 1) * addrsPerENI (%d): ", available, c.warmENITarget, c.maxIPsPerENI) } else { - log.Tracef("Its NOT possible to remove extra ENIs because available (%d) < (ENI target (%d) + 1) * addrsPerENI (%d): ", available, c.warmENITarget, c.maxIPsPerENI) + log.Debugf("Its NOT possible to remove extra ENIs because available (%d) < (ENI target (%d) + 1) * addrsPerENI (%d): ", available, c.warmENITarget, c.maxIPsPerENI) } return shouldRemoveExtra } @@ -1074,7 +1070,7 @@ func UseCustomNetworkCfg() bool { if err == nil { return parsedValue } - log.Error("Failed to parse "+envCustomNetworkCfg+"; using default: false", err.Error()) + log.Errorf("Failed to parse %s; using default: false, err: %v", envCustomNetworkCfg, err) } return false } @@ -1148,7 +1144,7 @@ func (c *IPAMContext) ipTargetState() (short int, over int, enabled bool) { // over is less than the warm IP target alone if it would imply reducing total IPs below the minimum target over = max(min(over, total-c.minimumIPTarget), 0) - log.Tracef("Current warm IP stats: target: %d, total: %d, assigned: %d, available: %d, short: %d, over %d", c.warmIPTarget, total, assigned, available, short, over) + log.Infof("Current warm IP stats: target: %d, total: %d, assigned: %d, available: %d, short: %d, over %d", c.warmIPTarget, total, assigned, available, short, over) return short, over, true } diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index 2c33f78ec54..4df550faf41 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -152,7 +152,7 @@ func TestNodeInit(t *testing.T) { var criList = make(map[string]*cri.SandboxInfo, 0) criList["pod-uid"] = &cri.SandboxInfo{ID: "sandbox-id", Name: k8sName, K8SUID: "pod-uid"} - mockCRI.EXPECT().GetRunningPodSandboxes().Return(criList, nil) + mockCRI.EXPECT().GetRunningPodSandboxes(gomock.Any()).Return(criList, nil) var rules []netlink.Rule mockNetwork.EXPECT().GetRuleList().Return(rules, nil) @@ -193,7 +193,7 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool) { terminating: int32(0), } - mockContext.dataStore = datastore.NewDataStore() + mockContext.dataStore = datastore.NewDataStore(log) primary := true notPrimary := false @@ -286,7 +286,7 @@ func TestTryAddIPToENI(t *testing.T) { terminating: int32(0), } - mockContext.dataStore = datastore.NewDataStore() + mockContext.dataStore = datastore.NewDataStore(log) podENIConfig := &v1alpha1.ENIConfigSpec{ SecurityGroups: []string{"sg1-id", "sg2-id"}, @@ -348,7 +348,7 @@ func TestNodeIPPoolReconcile(t *testing.T) { terminating: int32(0), } - mockContext.dataStore = datastore.NewDataStore() + mockContext.dataStore = datastore.NewDataStore(log) primary := true notPrimary := false @@ -440,7 +440,7 @@ func TestGetWarmIPTargetState(t *testing.T) { terminating: int32(0), } - mockContext.dataStore = datastore.NewDataStore() + mockContext.dataStore = datastore.NewDataStore(log) _, _, warmIPTargetDefined := mockContext.ipTargetState() assert.False(t, warmIPTargetDefined) @@ -488,9 +488,9 @@ func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) { fields fields want bool }{ - {"Test new ds, all defaults", fields{14, 1, 0, datastore.NewDataStore()}, true}, - {"Test new ds, 0 ENIs", fields{14, 0, 0, datastore.NewDataStore()}, true}, - {"Test new ds, 3 warm IPs", fields{14, 0, 3, datastore.NewDataStore()}, true}, + {"Test new ds, all defaults", fields{14, 1, 0, datastore.NewDataStore(log)}, true}, + {"Test new ds, 0 ENIs", fields{14, 0, 0, datastore.NewDataStore(log)}, true}, + {"Test new ds, 3 warm IPs", fields{14, 0, 3, datastore.NewDataStore(log)}, true}, {"Test 3 unused IPs, 1 warm", fields{3, 1, 1, datastoreWith3FreeIPs()}, false}, {"Test 1 used, 1 warm ENI", fields{3, 1, 0, datastoreWith1Pod1()}, true}, {"Test 1 used, 0 warm ENI", fields{3, 0, 0, datastoreWith1Pod1()}, false}, @@ -519,7 +519,7 @@ func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) { } func datastoreWith3FreeIPs() *datastore.DataStore { - datastoreWith3FreeIPs := datastore.NewDataStore() + datastoreWith3FreeIPs := datastore.NewDataStore(log) _ = datastoreWith3FreeIPs.AddENI(primaryENIid, 1, true) _ = datastoreWith3FreeIPs.AddIPv4AddressToStore(primaryENIid, ipaddr01) _ = datastoreWith3FreeIPs.AddIPv4AddressToStore(primaryENIid, ipaddr02) diff --git a/pkg/ipamd/metrics.go b/pkg/ipamd/metrics.go index 273ee9994fe..310bbafd317 100644 --- a/pkg/ipamd/metrics.go +++ b/pkg/ipamd/metrics.go @@ -20,7 +20,6 @@ import ( "time" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/retry" - log "github.com/cihub/seelog" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -39,14 +38,14 @@ func (c *IPAMContext) ServeMetrics() { return } - log.Info("Serving metrics on port ", metricsPort) + log.Infof("Serving metrics on port %d", metricsPort) server := c.setupMetricsServer() for { once := sync.Once{} _ = retry.RetryWithBackoff(retry.NewSimpleBackoff(time.Second, time.Minute, 0.2, 2), func() error { err := server.ListenAndServe() once.Do(func() { - log.Error("Error running http API: ", err) + log.Errorf("Error running http API: %v", err) }) return err }) diff --git a/pkg/ipamd/rpc_handler.go b/pkg/ipamd/rpc_handler.go index e5c16503b04..3ee6dfca9a1 100644 --- a/pkg/ipamd/rpc_handler.go +++ b/pkg/ipamd/rpc_handler.go @@ -19,7 +19,6 @@ import ( "os/signal" "syscall" - log "github.com/cihub/seelog" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "golang.org/x/net/context" @@ -104,7 +103,7 @@ func (s *server) DelNetwork(ctx context.Context, in *rpc.DelNetworkRequest) (*rp // RunRPCHandler handles request from gRPC func (c *IPAMContext) RunRPCHandler() error { - log.Info("Serving RPC Handler on ", ipamdgRPCaddress) + log.Infof("Serving RPC Handler on ", ipamdgRPCaddress) listener, err := net.Listen("tcp", ipamdgRPCaddress) if err != nil { log.Errorf("Failed to listen gRPC port: %v", err) diff --git a/pkg/ipamd/rpc_handler_test.go b/pkg/ipamd/rpc_handler_test.go index 949e5a234a9..8e9057d8470 100644 --- a/pkg/ipamd/rpc_handler_test.go +++ b/pkg/ipamd/rpc_handler_test.go @@ -38,7 +38,7 @@ func TestServer_AddNetwork(t *testing.T) { warmIPTarget: 3, criClient: mockCRI, networkClient: mockNetwork, - dataStore: datastore.NewDataStore(), + dataStore: datastore.NewDataStore(log), } rpcServer := server{ipamContext: mockContext} diff --git a/pkg/k8sapi/discovery.go b/pkg/k8sapi/discovery.go index 9b19a5b5680..92b7a5bf7b0 100644 --- a/pkg/k8sapi/discovery.go +++ b/pkg/k8sapi/discovery.go @@ -10,7 +10,7 @@ import ( "github.com/pkg/errors" - log "github.com/cihub/seelog" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" clientset "k8s.io/client-go/kubernetes" @@ -54,6 +54,8 @@ type K8SPodInfo struct { UID string } +var log = logger.Get() + // ErrInformerNotSynced indicates that it has not synced with API server yet var ErrInformerNotSynced = errors.New("discovery: informer not synced") @@ -88,14 +90,10 @@ func CreateKubeClient() (clientset.Interface, error) { log.Infof("Testing communication with server") v, err := kubeClient.Discovery().ServerVersion() if err != nil { - errMsg := "Failed to communicate with K8S Server. Please check instance security groups or http proxy setting" - log.Infof(errMsg) - fmt.Printf(errMsg) return nil, fmt.Errorf("error communicating with apiserver: %v", err) } - log.Infof("Running with Kubernetes cluster version: v%s.%s. git version: %s. git tree state: %s. commit: %s. platform: %s", + log.Infof("Successful communication with the Cluster! Cluster Version is: v%s.%s. git version: %s. git tree state: %s. commit: %s. platform: %s", v.Major, v.Minor, v.GitVersion, v.GitTreeState, v.GitCommit, v.Platform) - log.Info("Communication with server successful") return kubeClient, nil } @@ -113,11 +111,11 @@ func (d *Controller) GetCNIPods() []string { cniPods = append(cniPods, k) } - log.Info("GetCNIPods discovered", cniPods) + log.Infof("GetCNIPods discovered %v", cniPods) return cniPods } -// DiscoverLocalK8SPods discovers CNI pods, aws-node, running in the cluster +// DiscoverCNIK8SPods discovers CNI pods, aws-node, running in the cluster func (d *Controller) DiscoverCNIK8SPods() { // create the pod watcher d.DiscoverK8SPods(cache.NewListWatchFromClient(d.kubeClient.CoreV1().RESTClient(), "pods", metav1.NamespaceSystem, fields.Everything())) @@ -256,7 +254,7 @@ func (d *Controller) handlePodUpdate(key string) error { d.workerPodsLock.Lock() defer d.workerPodsLock.Unlock() - log.Tracef("Update for pod %s: %+v, %+v", podName, pod.Status, pod.Spec) + log.Debugf("Update for pod %s: %+v, %+v", podName, pod.Status, pod.Spec) // Save pod info d.workerPods[key] = &K8SPodInfo{ diff --git a/pkg/networkutils/network.go b/pkg/networkutils/network.go index b53f2da4a3d..dbbaa5380a2 100644 --- a/pkg/networkutils/network.go +++ b/pkg/networkutils/network.go @@ -32,7 +32,7 @@ import ( "github.com/pkg/errors" "golang.org/x/sys/unix" - log "github.com/cihub/seelog" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "github.com/coreos/go-iptables/iptables" "github.com/vishvananda/netlink" @@ -107,6 +107,8 @@ const ( retryLinkByMacInterval = 3 * time.Second ) +var log = logger.Get() + // NetworkAPIs defines the host level and the eni level network related operations type NetworkAPIs interface { // SetupNodeNetwork performs node level network configuration @@ -613,7 +615,7 @@ func getBoolEnvVar(name string, defaultValue bool) bool { if strValue := os.Getenv(name); strValue != "" { parsedValue, err := strconv.ParseBool(strValue) if err != nil { - log.Error("Failed to parse "+name+"; using default: "+fmt.Sprint(defaultValue), err.Error()) + log.Errorf("Failed to parse "+name+"; using default: "+fmt.Sprint(defaultValue), err.Error()) return defaultValue } return parsedValue @@ -625,11 +627,11 @@ func getConnmark() uint32 { if connmark := os.Getenv(envConnmark); connmark != "" { mark, err := strconv.ParseInt(connmark, 0, 64) if err != nil { - log.Error("Failed to parse "+envConnmark+"; will use ", defaultConnmark, err.Error()) + log.Errorf("Failed to parse %s; will use %d, error: %v", envConnmark, defaultConnmark, err) return defaultConnmark } if mark > math.MaxUint32 || mark <= 0 { - log.Error(""+envConnmark+" out of range; will use ", defaultConnmark) + log.Errorf("%s out of range; will use %s", envConnmark, defaultConnmark) return defaultConnmark } return uint32(mark) diff --git a/pkg/publisher/publisher.go b/pkg/publisher/publisher.go index 9fae78a1367..aadbbe0ed0c 100644 --- a/pkg/publisher/publisher.go +++ b/pkg/publisher/publisher.go @@ -1,4 +1,4 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -26,7 +26,7 @@ import ( "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface" - "github.com/golang/glog" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "github.com/pkg/errors" ) @@ -48,19 +48,9 @@ const ( // maxDataPoints is the maximum number of data points per PutMetricData API request maxDataPoints = 20 - - // Default cluster id if unable to detect something more suitable - defaultClusterID = "k8s-cluster" ) -var ( - // List of EC2 tags (in priority order) to use as the CLUSTER_ID metric dimension - clusterIDTags = []string{ - "eks:cluster-name", - "CLUSTER_ID", - "Name", - } -) +var log = logger.Get() // Publisher defines the interface to publish one or more data points type Publisher interface { @@ -96,7 +86,16 @@ func New(ctx context.Context) (Publisher, error) { if err != nil { return nil, errors.Wrap(err, "publisher: unable to obtain EC2 service client") } - clusterID := getClusterID(ec2Client) + clusterID, err := ec2Client.GetClusterTag("CLUSTER_ID") + if err != nil || clusterID == "" { + log.Infof("Failed to obtain cluster-id, fetching name. %v", err) + clusterID, err = ec2Client.GetClusterTag("Name") + if err != nil || clusterID == "" { + log.Errorf("Failed to obtain cluster-id or name, defaulting to 'k8s-cluster'. %v", err) + clusterID = "k8s-cluster" + } + } + log.Infof("Using cluster ID %s", clusterID) // Get CloudWatch client ec2MetadataClient := ec2metadatawrapper.New(nil) @@ -121,20 +120,20 @@ func New(ctx context.Context) (Publisher, error) { // Start is used to setup the monitor loop func (p *cloudWatchPublisher) Start() { - glog.Info("Starting monitor loop for CloudWatch publisher") + log.Info("Starting monitor loop for CloudWatch publisher") p.monitor(defaultInterval) } // Stop is used to cancel the monitor loop func (p *cloudWatchPublisher) Stop() { - glog.Info("Stopping monitor loop for CloudWatch publisher") + log.Info("Stopping monitor loop for CloudWatch publisher") p.cancel() } // Publish is a variadic function to publish one or more metric data points func (p *cloudWatchPublisher) Publish(metricDataPoints ...*cloudwatch.MetricDatum) { // Fetch dimensions for override - glog.V(2).Info("Fetching CloudWatch dimensions") + log.Info("Fetching CloudWatch dimensions") dimensions := p.getCloudWatchMetricDatumDimensions() // Grab lock @@ -158,7 +157,7 @@ func (p *cloudWatchPublisher) pushLocal() { func (p *cloudWatchPublisher) push(metricData []*cloudwatch.MetricDatum) { if len(metricData) == 0 { - glog.Warning("Missing data for publishing CloudWatch metrics") + log.Info("Missing data for publishing CloudWatch metrics") return } @@ -173,7 +172,7 @@ func (p *cloudWatchPublisher) push(metricData []*cloudwatch.MetricDatum) { // Publish data err := p.send(input) if err != nil { - glog.Errorf("Unable to publish CloudWatch metrics: %v", err) + log.Warnf("Unable to publish CloudWatch metrics: %v", err) } // Mutate slice @@ -187,7 +186,7 @@ func (p *cloudWatchPublisher) push(metricData []*cloudwatch.MetricDatum) { } func (p *cloudWatchPublisher) send(input cloudwatch.PutMetricDataInput) error { - glog.Info("Sending data to CloudWatch metrics") + log.Info("Sending data to CloudWatch metrics") _, err := p.cloudwatchClient.PutMetricData(&input) return err } @@ -210,22 +209,6 @@ func (p *cloudWatchPublisher) getCloudWatchMetricNamespace() *string { return aws.String(cloudwatchMetricNamespace) } -func getClusterID(ec2Client *ec2wrapper.EC2Wrapper) string { - var clusterID string - var err error - for _, tag := range clusterIDTags { - clusterID, err = ec2Client.GetClusterTag(tag) - if err == nil && clusterID != "" { - break - } - } - if clusterID == "" { - clusterID = defaultClusterID - } - glog.Info("Using cluster ID ", clusterID) - return clusterID -} - func (p *cloudWatchPublisher) getCloudWatchMetricDatumDimensions() []*cloudwatch.Dimension { return []*cloudwatch.Dimension{ { diff --git a/pkg/utils/logger/config.go b/pkg/utils/logger/config.go new file mode 100644 index 00000000000..a3be9cdc9db --- /dev/null +++ b/pkg/utils/logger/config.go @@ -0,0 +1,46 @@ +package logger + +import ( + "os" +) + +const ( + defaultLogFilePath = "/host/var/log/aws-routed-eni/ipamd.log" + defaultLogLevel = "Debug" + binaryName = "L-IPamD" +) + +// Configuration stores the config for the logger +type Configuration struct { + BinaryName string + LogLevel string + LogLocation string +} + +func LoadLogConfig() *Configuration { + return &Configuration{ + LogLevel: getLogLevel(), + LogLocation: getLogFileLocation(), + BinaryName: binaryName, + } +} + +// GetLogFileLocation returns the log file path +func getLogFileLocation() string { + logFilePath := os.Getenv(envLogFilePath) + if logFilePath == "" { + logFilePath = defaultLogFilePath + } + return logFilePath +} + +func getLogLevel() string { + logLevel := os.Getenv(envLogLevel) + switch logLevel { + case "": + logLevel = defaultLogLevel + return logLevel + default: + return logLevel + } +} diff --git a/pkg/utils/logger/logger.go b/pkg/utils/logger/logger.go index fef39e29121..87f73d46057 100644 --- a/pkg/utils/logger/logger.go +++ b/pkg/utils/logger/logger.go @@ -1,4 +1,4 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// Copyright 2017-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -14,68 +14,72 @@ package logger import ( - "fmt" - "os" - "strings" - - log "github.com/cihub/seelog" + "sync" ) const ( - envLogLevel = "AWS_VPC_K8S_CNI_LOGLEVEL" - envLogFilePath = "AWS_VPC_K8S_CNI_LOG_FILE" - // logConfigFormat defines the seelog format, with a rolling file - // writer. We cannot do this in code and have to resort to using - // LoggerFromConfigAsString as seelog doesn't have a usable public - // implementation of NewRollingFileWriterTime - logConfigFormat = ` - - - %s - - - - - -` + envLogLevel = "AWS_VPC_K8S_CNI_LOGLEVEL" + envLogFilePath = "AWS_VPC_K8S_CNI_LOG_FILE" + pluginBinaryName = "aws-cni" ) -// GetLogFileLocation returns the log file path -func GetLogFileLocation(defaultLogFilePath string) string { - logFilePath := os.Getenv(envLogFilePath) - if logFilePath == "" { - logFilePath = defaultLogFilePath - } - return logFilePath +var once sync.Once + +//Log is global variable so that log functions can be directly accessed +var log Logger + +//Fields Type to pass when we want to call WithFields for structured logging +type Fields map[string]interface{} + +//Logger is our contract for the logger +type Logger interface { + Debugf(format string, args ...interface{}) + + Debug(format string) + + Infof(format string, args ...interface{}) + + Info(format string) + + Warnf(format string, args ...interface{}) + + Warn(format string) + + Errorf(format string, args ...interface{}) + + Error(format string) + + Fatalf(format string, args ...interface{}) + + Panicf(format string, args ...interface{}) + + WithFields(keyValues Fields) Logger } -// SetupLogger sets up a file logger -func SetupLogger(logFilePath string) { - logger, err := log.LoggerFromConfigAsString(fmt.Sprintf(logConfigFormat, getLogLevel(), getLogOutput(logFilePath))) - if err != nil { - fmt.Println("Error setting up logger: ", err) - return - } - err = log.ReplaceLogger(logger) - if err != nil { - fmt.Println("Error replacing logger: ", err) - return +// Get returns an default instance of the zap logger +func Get() Logger { + var logf = &structuredLogger{} + if logf.isEmpty() { + logConfig := LoadLogConfig() + log = New(logConfig) + return log } + log = logf + return log } -func getLogLevel() string { - seelogLevel, ok := log.LogLevelFromString(strings.ToLower(os.Getenv(envLogLevel))) - if !ok { - seelogLevel = log.DebugLvl - } - return seelogLevel.String() +func (logf *structuredLogger) isEmpty() bool { + return logf.zapLogger == nil } -func getLogOutput(logFilePath string) string { - switch logFilePath { - case "stdout": - return `` - default: - return fmt.Sprintf(``, logFilePath) +//New logger initializes logger +func New(inputLogConfig *Configuration) Logger { + if inputLogConfig.BinaryName != pluginBinaryName { + logConfig := LoadLogConfig() + log = logConfig.newZapLogger() + return log } + + log = inputLogConfig.newZapLogger() + return log } diff --git a/pkg/utils/logger/logger_test.go b/pkg/utils/logger/logger_test.go index fefd1174cac..458435a3be2 100644 --- a/pkg/utils/logger/logger_test.go +++ b/pkg/utils/logger/logger_test.go @@ -1,4 +1,4 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// Copyright 2017-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -17,57 +17,46 @@ import ( "os" "testing" - log "github.com/cihub/seelog" "github.com/stretchr/testify/assert" + "go.uber.org/zap/zapcore" ) -func TestGetLogFileLocationReturnsOverriddenPath(t *testing.T) { - path := "/tmp/foo" +func TestEnvLogFilePath(t *testing.T) { + path := "/var/log/test.log" _ = os.Setenv(envLogFilePath, path) defer os.Unsetenv(envLogFilePath) - assert.Equal(t, path, GetLogFileLocation("/tmp/bar")) + assert.Equal(t, path, getLogFileLocation()) } func TestGetLogFileLocationReturnsDefaultPath(t *testing.T) { - path := "/tmp/foo" - assert.Equal(t, path, GetLogFileLocation(path)) + defaultPath := "/host/var/log/aws-routed-eni/ipamd.log" + assert.Equal(t, defaultPath, getLogFileLocation()) } func TestLogLevelReturnsOverriddenLevel(t *testing.T) { _ = os.Setenv(envLogLevel, "INFO") defer os.Unsetenv(envLogLevel) - var expectedLogLevel log.LogLevel - expectedLogLevel = log.InfoLvl - assert.Equal(t, expectedLogLevel.String(), getLogLevel()) + var expectedLogLevel zapcore.Level + expectedLogLevel = zapcore.InfoLevel + inputLogLevel := getLogLevel() + assert.Equal(t, expectedLogLevel, getZapLevel(inputLogLevel)) } func TestLogLevelReturnsDefaultLevelWhenEnvNotSet(t *testing.T) { - var expectedLogLevel log.LogLevel - expectedLogLevel = log.DebugLvl - assert.Equal(t, expectedLogLevel.String(), getLogLevel()) + var expectedLogLevel zapcore.Level + expectedLogLevel = zapcore.DebugLevel + inputLogLevel := getLogLevel() + assert.Equal(t, expectedLogLevel, getZapLevel(inputLogLevel)) } func TestLogLevelReturnsDefaultLevelWhenEnvSetToInvalidValue(t *testing.T) { _ = os.Setenv(envLogLevel, "EVERYTHING") defer os.Unsetenv(envLogLevel) - var expectedLogLevel log.LogLevel - expectedLogLevel = log.DebugLvl - assert.Equal(t, expectedLogLevel.String(), getLogLevel()) -} - -func TestLogOutputReturnsFileWhenValueNotStdout(t *testing.T) { - path := "/tmp/foo" - - var expectedOutput = `` - assert.Equal(t, expectedOutput, getLogOutput(path)) -} - -func TestLogOutputReturnsConsole(t *testing.T) { - path := "stdout" - - var expectedOutput = `` - assert.Equal(t, expectedOutput, getLogOutput(path)) + var expectedLogLevel zapcore.Level + inputLogLevel := getLogLevel() + expectedLogLevel = zapcore.DebugLevel + assert.Equal(t, expectedLogLevel, getZapLevel(inputLogLevel)) } diff --git a/pkg/utils/logger/zaplogger.go b/pkg/utils/logger/zaplogger.go new file mode 100644 index 00000000000..b3aea66c96e --- /dev/null +++ b/pkg/utils/logger/zaplogger.go @@ -0,0 +1,163 @@ +// Copyright 2017-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package logger + +import ( + "os" + "runtime" + "strings" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + lumberjack "gopkg.in/natefinch/lumberjack.v2" +) + +type structuredLogger struct { + zapLogger *zap.SugaredLogger +} + +// getZapLevel converts log level string to zapcore.Level +func getZapLevel(inputLogLevel string) zapcore.Level { + lvl := strings.ToLower(inputLogLevel) + + switch lvl { + case "debug": + return zapcore.DebugLevel + case "info": + return zapcore.InfoLevel + case "warn": + return zapcore.WarnLevel + case "error": + return zapcore.ErrorLevel + case "fatal": + return zapcore.FatalLevel + default: + return zapcore.DebugLevel + } +} + +func (logf *structuredLogger) Debugf(format string, args ...interface{}) { + logf.zapLogger.Debugf(format, args...) +} + +func (logf *structuredLogger) Debug(format string) { + logf.zapLogger.Desugar().Debug(format) +} + +func (logf *structuredLogger) Infof(format string, args ...interface{}) { + logf.zapLogger.Infof(format, args...) +} + +func (logf *structuredLogger) Info(format string) { + logf.zapLogger.Desugar().Info(format) +} + +func (logf *structuredLogger) Warnf(format string, args ...interface{}) { + logf.zapLogger.Warnf(format, args...) +} + +func (logf *structuredLogger) Warn(format string) { + logf.zapLogger.Desugar().Warn(format) +} + +func (logf *structuredLogger) Error(format string) { + logf.zapLogger.Desugar().Error(format) +} + +func (logf *structuredLogger) Errorf(format string, args ...interface{}) { + logf.zapLogger.Errorf(format, args...) +} + +func (logf *structuredLogger) Fatalf(format string, args ...interface{}) { + logf.zapLogger.Fatalf(format, args...) +} + +func (logf *structuredLogger) Panicf(format string, args ...interface{}) { + logf.zapLogger.Fatalf(format, args...) +} + +func (logf *structuredLogger) WithFields(fields Fields) Logger { + var f = make([]interface{}, 0) + for k, v := range fields { + f = append(f, k) + f = append(f, v) + } + newLogger := logf.zapLogger.With(f...) + return &structuredLogger{newLogger} +} + +func getEncoder() zapcore.Encoder { + encoderConfig := zap.NewProductionEncoderConfig() + encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + return zapcore.NewJSONEncoder(encoderConfig) +} + +func (logConfig *Configuration) newZapLogger() *structuredLogger { + var cores []zapcore.Core + + logLevel := getZapLevel(logConfig.LogLevel) + + logFilePath := logConfig.LogLocation + + var writer zapcore.WriteSyncer + + if logFilePath != "stdout" { + writer = getLogWriter(logFilePath) + } else { + writer = zapcore.Lock(os.Stdout) + } + + cores = append(cores, zapcore.NewCore(getEncoder(), writer, logLevel)) + + combinedCore := zapcore.NewTee(cores...) + + logger := zap.New(combinedCore, + zap.AddCaller(), + zap.AddCallerSkip(2), + ) + defer logger.Sync() + sugar := logger.Sugar() + + return &structuredLogger{ + zapLogger: sugar, + } +} + +//getLogWriter is for lumberjack +func getLogWriter(logFilePath string) zapcore.WriteSyncer { + lumberJackLogger := &lumberjack.Logger{ + Filename: logFilePath, + MaxSize: 100, + MaxBackups: 5, + MaxAge: 30, + Compress: true, + } + return zapcore.AddSync(lumberJackLogger) +} + +// DefaultLogger creates and returns a new default logger. +func DefaultLogger() Logger { + productionConfig := zap.NewProductionConfig() + productionConfig.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + productionConfig.EncoderConfig.EncodeCaller = func(caller zapcore.EntryCaller, enc zapcore.PrimitiveArrayEncoder) { + _, caller.File, caller.Line, _ = runtime.Caller(8) + enc.AppendString(caller.FullPath()) + } + logger, _ := productionConfig.Build() + defer logger.Sync() + sugar := logger.Sugar() + return &structuredLogger{ + zapLogger: sugar, + } +} diff --git a/scripts/dockerfiles/Dockerfile.metrics b/scripts/dockerfiles/Dockerfile.metrics index 0086f00a565..00c2dfc3524 100644 --- a/scripts/dockerfiles/Dockerfile.metrics +++ b/scripts/dockerfiles/Dockerfile.metrics @@ -1,33 +1,31 @@ -FROM golang:1.13-stretch as builder -WORKDIR /go/src/github.com/aws/amazon-vpc-cni-k8s - -ARG arch -ENV ARCH=$arch +ARG docker_arch +ARG golang_image=golang:1.13-stretch -# Force the go compiler to use modules. +FROM $golang_image as builder +WORKDIR /go/src/github.com/aws/amazon-vpc-cni-k8s +ARG GOARCH= +# Configure build with Go modules ENV GO111MODULE=on ENV GOPROXY=direct -# go.mod and go.sum go into their own layers. -COPY go.mod . -COPY go.sum . - -# This ensures `go mod download` happens only when go.mod and go.sum change. +# Copy modules in before the rest of the source to only expire cache on module +# changes: +COPY go.mod go.sum ./ RUN go mod download -COPY . . +COPY . ./ RUN make build-metrics -FROM amazonlinux:2 +FROM $docker_arch/amazonlinux:2 RUN yum update -y && \ - yum clean all + yum clean all + +# Copy our bundled certs to the first place go will check: see +# https://golang.org/src/pkg/crypto/x509/root_unix.go +COPY ./misc/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt WORKDIR /app COPY --from=builder /go/src/github.com/aws/amazon-vpc-cni-k8s/cni-metrics-helper /app -# Copy our bundled certs to the first place go will check: see -# https://golang.org/src/pkg/crypto/x509/root_unix.go -COPY --from=builder /go/src/github.com/aws/amazon-vpc-cni-k8s/misc/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt - ENTRYPOINT /app/cni-metrics-helper --cloudwatch=false diff --git a/scripts/dockerfiles/Dockerfile.release b/scripts/dockerfiles/Dockerfile.release index 31e88b4f5f1..06910a35d76 100644 --- a/scripts/dockerfiles/Dockerfile.release +++ b/scripts/dockerfiles/Dockerfile.release @@ -1,30 +1,28 @@ -FROM golang:1.13-stretch as builder -WORKDIR /go/src/github.com/aws/amazon-vpc-cni-k8s - -ARG arch -ENV ARCH=$arch +ARG docker_arch +ARG golang_image=golang:1.13-stretch -# Force the go compiler to use modules. +FROM $golang_image as builder +WORKDIR /go/src/github.com/aws/amazon-vpc-cni-k8s +ARG GOARCH= +# Configure build with Go modules ENV GO111MODULE=on ENV GOPROXY=direct -# Add goimports -RUN go get -u golang.org/x/tools/cmd/goimports - -# go.mod and go.sum go into their own layers. -COPY go.mod . -COPY go.sum . - -# This ensures `go mod download` happens only when go.mod and go.sum change. +# Copy modules in before the rest of the source to only expire cache on module +# changes: +COPY go.mod go.sum ./ RUN go mod download -COPY . . -RUN make build-linux && make download-portmap +COPY Makefile ./ +RUN make portmap + +COPY . ./ +RUN make build-linux -FROM amazonlinux:2 +# Build the architecture specific container image: +FROM $docker_arch/amazonlinux:2 RUN yum update -y && \ - yum install -y iptables && \ - yum install -y iproute && \ + yum install -y iptables iproute && \ yum clean all WORKDIR /app diff --git a/scripts/dockerfiles/Dockerfile.test b/scripts/dockerfiles/Dockerfile.test index 5cbd529098b..842f68a4db6 100644 --- a/scripts/dockerfiles/Dockerfile.test +++ b/scripts/dockerfiles/Dockerfile.test @@ -1,4 +1,6 @@ -FROM golang:1.13-stretch +ARG docker_arch +ARG golang_image=golang:1.13-stretch +FROM $golang_image WORKDIR /go/src/github.com/aws/amazon-vpc-cni-k8s ARG arch diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index b5f38678ffb..a6566a9977b 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -36,9 +36,13 @@ if [ ! -f grpc-health-probe ]; then exit 1 fi -AGENT_LOG_PATH=${AGENT_LOG_PATH:-aws-k8s-agent.log} -HOST_CNI_BIN_PATH=${HOST_CNI_BIN_PATH:-/host/opt/cni/bin} -HOST_CNI_CONFDIR_PATH=${HOST_CNI_CONFDIR_PATH:-/host/etc/cni/net.d} +AGENT_LOG_PATH=${AGENT_LOG_PATH:-"aws-k8s-agent.log"} +HOST_CNI_BIN_PATH=${HOST_CNI_BIN_PATH:-"/host/opt/cni/bin"} +HOST_CNI_CONFDIR_PATH=${HOST_CNI_CONFDIR_PATH:-"/host/etc/cni/net.d"} +AWS_VPC_K8S_CNI_VETHPREFIX=${AWS_VPC_K8S_CNI_VETHPREFIX:-"eni"} +AWS_VPC_ENI_MTU=${AWS_VPC_ENI_MTU:-"9001"} +AWS_VPC_K8S_PLUGIN_LOG_FILE=${AWS_VPC_K8S_PLUGIN_LOG_FILE:-"/var/log/aws-routed-eni/plugin.log"} +AWS_VPC_K8s_PLUGIN_LOG_LEVEL=${AWS_VPC_K8S_PLUGIN_LOG_LEVEL:-"Debug"} # Checks for IPAM connectivity on localhost port 50051, retrying connectivity # check with a timeout of 36 seconds @@ -75,8 +79,10 @@ cp portmap "$HOST_CNI_BIN_PATH" cp aws-cni "$HOST_CNI_BIN_PATH" cp aws-cni-support.sh "$HOST_CNI_BIN_PATH" -sed -i s/__VETHPREFIX__/"${AWS_VPC_K8S_CNI_VETHPREFIX:-"eni"}"/g 10-aws.conflist -sed -i s/__MTU__/"${AWS_VPC_ENI_MTU:-"9001"}"/g 10-aws.conflist +sed -i s~__VETHPREFIX__~"${AWS_VPC_K8S_CNI_VETHPREFIX}"~g 10-aws.conflist +sed -i s~__MTU__~"${AWS_VPC_ENI_MTU}"~g 10-aws.conflist +sed -i s~__PLUGINLOGFILE__~"${AWS_VPC_K8S_PLUGIN_LOG_FILE}"~g 10-aws.conflist +sed -i s~__PLUGINLOGLEVEL__~"${AWS_VPC_K8s_PLUGIN_LOG_LEVEL}"~g 10-aws.conflist cp 10-aws.conflist "$HOST_CNI_CONFDIR_PATH" echo " ok."