Add init container

.dockerignore

@@ -3,4 +3,5 @@ aws-k8s-agent
 cni-metrics-helper
 grpc-health-probe
 portmap
+loopback
 routed-eni-cni-plugin

Makefile

@@ -14,7 +14,7 @@
 
 .PHONY: all dist check clean \
 		lint format check-format vet docker-vet \
-		build-linux docker \
+		build-linux docker init \
 		unit-test unit-test-race build-docker-test docker-func-test \
 		build-metrics docker-metrics \
 		metrics-unit-test docker-metrics-test
@@ -29,6 +29,10 @@ DESTDIR = .
 IMAGE = amazon/amazon-k8s-cni
 IMAGE_NAME = $(IMAGE)$(IMAGE_ARCH_SUFFIX):$(VERSION)
 IMAGE_DIST = $(DESTDIR)/$(subst /,_,$(IMAGE_NAME)).tar.gz
+# INIT_IMAGE is the init container for AWS VPC CNI.
+INIT_IMAGE = amazon/amazon-k8s-cni-init
+INIT_IMAGE_NAME = $(INIT_IMAGE)$(IMAGE_ARCH_SUFFIX):$(VERSION)
+INIT_IMAGE_DIST = $(DESTDIR)/$(subst /,_,$(INIT_IMAGE_NAME)).tar.gz
 # METRICS_IMAGE is the CNI metrics publisher sidecar container image.
 METRICS_IMAGE = amazon/cni-metrics-helper
 METRICS_IMAGE_NAME = $(METRICS_IMAGE)$(IMAGE_ARCH_SUFFIX):$(VERSION)
@@ -69,6 +73,11 @@ LDFLAGS = -X main.version=$(VERSION)
 ALLPKGS = $(shell go list ./...)
 # BINS is the set of built command executables.
 BINS = aws-k8s-agent aws-cni grpc-health-probe cni-metrics-helper
+# Plugin binaries
+# Not copied: bandwidth bridge dhcp firewall flannel host-device host-local ipvlan macvlan ptp sbr static tuning vlan
+# For gnu tar, the full path in the tar file is required
+PLUGIN_BINS = ./loopback ./portmap
+
 # DOCKER_ARGS is extra arguments passed during container image build.
 DOCKER_ARGS =
 # DOCKER_RUN_FLAGS is set the flags passed during runs of containers.
@@ -85,11 +94,12 @@ DOCKER_BUILD_FLAGS = --build-arg GOARCH="$(ARCH)" \
 .DEFAULT_GOAL = build-linux
 
 # Build both CNI and metrics helper container images.
-all: docker docker-metrics
+all: docker init docker-metrics
 
 dist: all
 	mkdir -p $(DESTDIR)
 	docker save $(IMAGE_NAME) | gzip > $(IMAGE_DIST)
+	docker save $(INIT_IMAGE_NAME) | gzip > $(INIT_IMAGE_DIST)
 	docker save $(METRICS_IMAGE_NAME) | gzip > $(METRICS_IMAGE_DIST)
 
 # Build the VPC CNI plugin agent using the host's Go toolchain.
@@ -108,7 +118,14 @@ docker:
 		.
 	@echo "Built Docker image \"$(IMAGE_NAME)\""
 
-# Run the built cni container image to use in functional testing
+init:
+	docker build $(DOCKER_BUILD_FLAGS) \
+		-f scripts/dockerfiles/Dockerfile.init \
+		-t "$(INIT_IMAGE_NAME)" \
+		.
+	@echo "Built Docker image \"$(INIT_IMAGE_NAME)\""
+
+# Run the built CNI container image to use in functional testing
 docker-func-test: docker
 	docker run $(DOCKER_RUN_FLAGS) \
 		"$(IMAGE_NAME)"
@@ -179,17 +196,17 @@ generate-limits: GOOS=
 generate-limits:
 	go run pkg/awsutils/gen_vpc_ip_limits.go
 
-# Fetch portmap the port-forwarding management CNI plugin
-portmap: FETCH_VERSION=0.8.6
-portmap: FETCH_URL=https://github.com/containernetworking/plugins/releases/download/v$(FETCH_VERSION)/cni-plugins-$(GOOS)-$(GOARCH)-v$(FETCH_VERSION).tgz
-portmap: VISIT_URL=https://github.com/containernetworking/plugins/tree/v$(FETCH_VERSION)/plugins/meta/portmap
-portmap:
-	@echo "Fetching portmap CNI plugin v$(FETCH_VERSION) from upstream release"
+# Fetch the CNI plugins
+plugins: FETCH_VERSION=0.8.6
+plugins: FETCH_URL=https://github.com/containernetworking/plugins/releases/download/v$(FETCH_VERSION)/cni-plugins-$(GOOS)-$(GOARCH)-v$(FETCH_VERSION).tgz
+plugins: VISIT_URL=https://github.com/containernetworking/plugins/tree/v$(FETCH_VERSION)/plugins/
+plugins:
+	@echo "Fetching Container networking plugins v$(FETCH_VERSION) from upstream release"
 	@echo
-	@echo "Visit upstream project for portmap plugin details:"
+	@echo "Visit upstream project for plugin details:"
 	@echo "$(VISIT_URL)"
 	@echo
-	curl -L $(FETCH_URL) | tar -z -x ./portmap
+	curl -L $(FETCH_URL) | tar -zx $(PLUGIN_BINS)
 
 debug-script: FETCH_URL=https://raw.githubusercontent.com/awslabs/amazon-eks-ami/master/log-collector-script/linux/eks-log-collector.sh
 debug-script: VISIT_URL=https://github.com/awslabs/amazon-eks-ami/tree/master/log-collector-script/linux
@@ -245,5 +262,5 @@ check-format: format
 # Clean temporary files and build artifacts from the project.
 clean:
 	@rm -f -- $(BINS)
-	@rm -f -- portmap
+	@rm -f -- $(PLUGIN_BINS)
 	@rm -f -- coverage.txt

config/master/aws-k8s-cni.yaml

@@ -105,6 +105,8 @@
                 - "fargate"
       "containers":
       - "env":
+        - name: "AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER"
+          value: "false"
         - "name": "AWS_VPC_ENI_MTU"
           "value": "9001"
         - "name": "AWS_VPC_K8S_CNI_LOGLEVEL"
@@ -137,7 +139,8 @@
           "requests":
             "cpu": "10m"
         "securityContext":
-          "privileged": true
+          capabilities:
+            add: ["NET_ADMIN"]
         "volumeMounts":
         - "mountPath": "/host/opt/cni/bin"
           "name": "cni-bin-dir"
@@ -170,6 +173,15 @@
       - "hostPath":
           "path": "/var/run/dockershim.sock"
         "name": "dockershim"
+      initContainers:
+        - name: "aws-vpc-cni-init"
+          image: "973117571331.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:latest"
+          imagePullPolicy: "Always"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: "/host/opt/cni/bin"
+              name: "cni-bin-dir"
   "updateStrategy":
     "rollingUpdate":
       "maxUnavailable": "10%"

scripts/dockerfiles/Dockerfile.init

@@ -0,0 +1,30 @@
+ARG docker_arch
+ARG golang_image=golang:1.13-stretch
+
+FROM $golang_image as builder
+WORKDIR /go/src/github.com/aws/amazon-vpc-cni-k8s
+ARG GOARCH
+# Configure build with Go modules
+ENV GO111MODULE=on
+ENV GOPROXY=direct
+
+COPY Makefile ./
+RUN make plugins && make debug-script
+
+COPY . ./
+
+# Build the architecture specific container image:
+FROM $docker_arch/amazonlinux:2
+RUN yum update -y && \
+    yum install -y iproute procps-ng && \
+    yum clean all
+
+WORKDIR /init
+
+COPY --from=builder \
+    /go/src/github.com/aws/amazon-vpc-cni-k8s/loopback \
+    /go/src/github.com/aws/amazon-vpc-cni-k8s/portmap \
+    /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-cni-support.sh \
+    /go/src/github.com/aws/amazon-vpc-cni-k8s/scripts/init.sh /init/
+
+ENTRYPOINT /init/init.sh

scripts/dockerfiles/Dockerfile.release

@@ -8,13 +8,12 @@ ARG GOARCH
 ENV GO111MODULE=on
 ENV GOPROXY=direct
 
-# Copy modules in before the rest of the source to only expire cache on module
-# changes:
+# Copy modules in before the rest of the source to only expire cache on module changes:
 COPY go.mod go.sum ./
 RUN go mod download
 
 COPY Makefile ./
-RUN make portmap && make debug-script
+RUN make plugins && make debug-script
 
 COPY . ./
 RUN make build-linux

scripts/entrypoint.sh

@@ -44,27 +44,35 @@ AWS_VPC_ENI_MTU=${AWS_VPC_ENI_MTU:-"9001"}
 AWS_VPC_K8S_PLUGIN_LOG_FILE=${AWS_VPC_K8S_PLUGIN_LOG_FILE:-"/var/log/aws-routed-eni/plugin.log"}
 AWS_VPC_K8S_PLUGIN_LOG_LEVEL=${AWS_VPC_K8S_PLUGIN_LOG_LEVEL:-"Debug"}
 
+AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER=${AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER:-"true"}
+
 # Checks for IPAM connectivity on localhost port 50051, retrying connectivity
 # check with a timeout of 36 seconds
 wait_for_ipam() {
     local __sleep_time=0
 
     until [ $__sleep_time -eq 8 ]; do
-        sleep $(( __sleep_time++ ))
+        sleep $((__sleep_time++))
         if ./grpc-health-probe -addr 127.0.0.1:50051 >/dev/null 2>&1; then
             return 0
         fi
     done
     return 1
 }
 
-echo -n "Copying portmap binary ... "
-
-HOST_PORTMAP="$HOST_CNI_BIN_PATH/portmap"
-if [[ -f "$HOST_PORTMAP" ]]; then
-    rm "$HOST_PORTMAP"
+# If there is no init container, copy the required files
+if [[ "$AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER" != "false" ]]; then
+    # Copy files
+    echo "Copying CNI plugin binaries ... "
+    PLUGIN_BINS="portmap aws-cni-support.sh"
+    for b in $PLUGIN_BINS; do
+        # If the file exist, delete it first
+        if [[ -f "$HOST_CNI_BIN_PATH/$b" ]]; then
+            rm "$HOST_CNI_BIN_PATH/$b"
+        fi
+        cp "$b" "$HOST_CNI_BIN_PATH"
+    done
 fi
-cp portmap "$HOST_CNI_BIN_PATH"
 
 echo -n "Starting IPAM daemon in the background ... "
 ./aws-k8s-agent | tee -i "$AGENT_LOG_PATH" 2>&1 &
@@ -81,10 +89,9 @@ fi
 
 echo "ok."
 
-echo -n "Copying additional CNI plugin binaries and config files ... "
+echo -n "Copying CNI plugin binary and config file ... "
 
 cp aws-cni "$HOST_CNI_BIN_PATH"
-cp aws-cni-support.sh "$HOST_CNI_BIN_PATH"
 
 sed -i s~__VETHPREFIX__~"${AWS_VPC_K8S_CNI_VETHPREFIX}"~g 10-aws.conflist
 sed -i s~__MTU__~"${AWS_VPC_ENI_MTU}"~g 10-aws.conflist

scripts/init.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+PLUGIN_BINS="loopback portmap aws-cni-support.sh"
+
+for b in $PLUGIN_BINS; do
+    if [ ! -f "$b" ]; then
+        echo "Required $b executable not found."
+        exit 1
+    fi
+done
+
+HOST_CNI_BIN_PATH=${HOST_CNI_BIN_PATH:-"/host/opt/cni/bin"}
+
+# Copy files
+echo "Copying CNI plugin binaries ... "
+
+for b in $PLUGIN_BINS; do
+    # If the file exist, delete it first
+    if [[ -f "$HOST_CNI_BIN_PATH/$b" ]]; then
+        rm "$HOST_CNI_BIN_PATH/$b"
+    fi
+    cp "$b" "$HOST_CNI_BIN_PATH"
+done
+
+# Configure rp_filter
+echo "Configure rp_filter loose... "
+TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60")
+HOST_IP=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/local-ipv4)
+PRIMARY_IF=$(ip -4 -o a | grep "$HOST_IP" | awk '{print $2}')
+sysctl -w "net.ipv4.conf.$PRIMARY_IF.rp_filter=2"
+
+cat "/proc/sys/net/ipv4/conf/$PRIMARY_IF/rp_filter"
+
+echo "CNI init container done"