From f14798c0898e51a3d2a664d72e22474aaa466ebb Mon Sep 17 00:00:00 2001 From: Jay Pipes Date: Tue, 26 Nov 2019 12:56:44 -0500 Subject: [PATCH] copy CNI plugin and config in entrypoint not agent Replaces the hard-coded copying of the CNI plugin binary and configuration file from the main.go of the aws-k8s-agent (the IPAMd daemon) into a new entrypoint.sh script that backgrounds the IPAM daemon, waits for it to listen on port 50051 and then proceeds to copy the CNI plugin binaries and configuration files into the well-known directory for Kubelet to see them as ready. --- Makefile | 3 + config/v1.5/aws-k8s-cni.yaml | 4 ++ main.go | 52 --------------- main_test.go | 48 -------------- scripts/dockerfiles/Dockerfile.release | 4 +- scripts/entrypoint.sh | 91 ++++++++++++++++++++++++++ 6 files changed, 100 insertions(+), 102 deletions(-) delete mode 100644 main_test.go create mode 100755 scripts/entrypoint.sh diff --git a/Makefile b/Makefile index ffb866a65d..03426f66ca 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,9 @@ docker: @docker build --build-arg arch="$(ARCH)" -f scripts/dockerfiles/Dockerfile.release -t "$(IMAGE):$(VERSION)" . @echo "Built Docker image \"$(IMAGE):$(VERSION)\"" +docker-func-test: docker + docker run -it "$(IMAGE):$(VERSION)" + # unit-test unit-test: GOOS=linux CGO_ENABLED=1 go test -v -cover $(ALLPKGS) diff --git a/config/v1.5/aws-k8s-cni.yaml b/config/v1.5/aws-k8s-cni.yaml index 6a8bc3ac7e..a988fcc727 100644 --- a/config/v1.5/aws-k8s-cni.yaml +++ b/config/v1.5/aws-k8s-cni.yaml @@ -98,6 +98,10 @@ spec: env: - name: AWS_VPC_K8S_CNI_LOGLEVEL value: DEBUG + - name: AWS_VPC_K8S_CNI_VETHPREFIX + value: eni + - name: AWS_VPC_K8S_CNI_MTU + value: 9001 - name: MY_NODE_NAME valueFrom: fieldRef: diff --git a/main.go b/main.go index 0c5cb2989e..5bb44878da 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,6 @@ package main import ( - "io" "os" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" @@ -74,21 +73,6 @@ func _main() int { // CNI introspection endpoints go ipamContext.ServeIntrospection() - // Copy the CNI plugin and config. This will mark the node as Ready. - log.Info("Copying /app/aws-cni to /host/opt/cni/bin/aws-cni") - err = copyFileContents("/app/aws-cni", "/host/opt/cni/bin/aws-cni") - if err != nil { - log.Errorf("Failed to copy aws-cni: %v", err) - return 1 - } - - log.Info("Copying /app/10-aws.conflist to /host/etc/cni/net.d/10-aws.conflist") - err = copyFileContents("/app/10-aws.conflist", "/host/etc/cni/net.d/10-aws.conflist") - if err != nil { - log.Errorf("Failed to copy 10-aws.conflist: %v", err) - return 1 - } - // Start the RPC listener err = ipamContext.RunRPCHandler() if err != nil { @@ -97,39 +81,3 @@ func _main() int { } return 0 } - -// copyFileContents copies a file -func copyFileContents(src, dst string) error { - in, err := os.Open(src) - if err != nil { - return err - } - defer in.Close() - out, err := os.Create(dst) - if err != nil { - return err - } - defer func() { - e := out.Close() - if err == nil { - err = e - } - }() - if _, err = io.Copy(out, in); err != nil { - return err - } - err = out.Sync() - if err != nil { - return err - } - si, err := os.Stat(src) - if err != nil { - return err - } - err = os.Chmod(dst, si.Mode()) - if err != nil { - return err - } - log.Debugf("Copied file from %q to %q", src, dst) - return err -} diff --git a/main_test.go b/main_test.go deleted file mode 100644 index b82297bf16..0000000000 --- a/main_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package main - -import ( - "os" - "testing" -) - -func Test_copyFileContents(t *testing.T) { - type args struct { - src string - dst string - } - tests := []struct { - name string - args args - wantErr bool - }{ - {"Missing file", args{"missing", "out.txt"}, true}, - {"Copy executable", args{"./testdata/executable", "executable"}, false}, - {"Copy regular-file", args{"./testdata/regular-file.txt", "regular-file.txt"}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - dst := os.TempDir() + "/" + tt.args.dst - - err := copyFileContents(tt.args.src, dst) - if err != nil { - if !tt.wantErr { - t.Errorf("copyFileContents() error = %v, wantErr %v", err, tt.wantErr) - } - } else if !tt.wantErr { - defer func() { - e := os.Remove(dst) - if e != nil { - t.Errorf("Failed to clean up: %s", dst) - } - }() - si, ei := os.Stat(tt.args.src) - so, eo := os.Stat(dst) - if ei != nil || eo != nil { - t.Errorf("src error: %v, dst error: %v", ei, eo) - } else if si.Mode() != so.Mode() { - t.Errorf("File mode doesn't match! src: %v, dst: %v", si.Mode(), so.Mode()) - } - } - }) - } -} diff --git a/scripts/dockerfiles/Dockerfile.release b/scripts/dockerfiles/Dockerfile.release index 5ec1a13f60..43ee6a419f 100644 --- a/scripts/dockerfiles/Dockerfile.release +++ b/scripts/dockerfiles/Dockerfile.release @@ -31,6 +31,6 @@ COPY --from=builder /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-cni \ /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-k8s-agent \ /go/src/github.com/aws/amazon-vpc-cni-k8s/grpc_health_probe \ /go/src/github.com/aws/amazon-vpc-cni-k8s/scripts/aws-cni-support.sh \ - /go/src/github.com/aws/amazon-vpc-cni-k8s/scripts/install-aws.sh /app/ + /go/src/github.com/aws/amazon-vpc-cni-k8s/scripts/entrypoint.sh /app/ -ENTRYPOINT /app/install-aws.sh +ENTRYPOINT /app/entrypoint.sh diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh new file mode 100755 index 0000000000..a4fa2d272e --- /dev/null +++ b/scripts/entrypoint.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +# NOTE(jaypipes): Normally, we would prefer *not* to have an entrypoint script +# and instead just start the agent daemon as the container's CMD. However, the +# design of CNI is such that Kubelet looks for the presence of binaries and CNI +# configuration files in specific directories, and the presence of those files +# is the trigger to Kubelet that that particular CNI plugin is "ready". +# +# In the case of the AWS VPC CNI plugin, we have two components to the plugin. +# The first component is the actual CNI binary that is execve'd from Kubelet +# when a container is started or destroyed. The second component is the +# aws-k8s-agent daemon which houses the IPAM controller. +# +# As mentioned above, Kubelet considers a CNI plugin "ready" when it sees the +# binary and configuration file for the plugin in a well-known directory. For +# the AWS VPC CNI plugin binary, we only want to copy the CNI plugin binary +# into that well-known directory AFTER we have succeessfully started the IPAM +# daemon and know that it can connect to Kubernetes and the local EC2 metadata +# service. This is why this entrypoint script exists; we start the IPAM daemon +# and wait until we know it is up and running successfully before copying the +# CNI plugin binary and its configuration file to the well-known directory that +# Kubelet looks in. + +# turn on bash's job control +set -m + +# Check for all the required binaries before we go forward +if [ ! -x $(command -v aws-k8s-agent) ]; then + echo "Required aws-k8s-agent executable not found." + exit 1 +fi +if [ ! -x $(command -v grp_health_probe) ]; then + echo "Required grp_health_probe executable not found." + exit 1 +fi + +AGENT_LOG_PATH=${AGENT_LOG_PATH:-aws-k8s-agent.log} +HOST_CNI_BIN_PATH=${HOST_CNI_BIN_PATH:-/host/opt/cni/bin} +HOST_CNI_CONFDIR_PATH=${HOST_CNI_CONFDIR_PATH:-/host/opt/cni/net.d} + +# Checks for IPAM connectivity on localhost port 50051, retrying connectivity +# check with a timeout of 36 seconds +wait_for_ipam() { + local __sleep_time=0 + + until [ $__sleep_time -eq 8 ]; do + sleep $(( __sleep_time++ )) + if $(./grpc_health_probe -addr 127.0.0.1:50051 >/dev/null 2>&1); then + return 0 + fi + done + return 1 +} + +echo -n "starting IPAM daemon in background ... " +./aws-k8s-agent > $AGENT_LOG_PATH 2>&1 & +echo "ok." + +echo -n "checking for IPAM connectivity ... " + +if ! wait_for_ipam; then + echo " failed." + echo "timed out waiting for IPAM daemon to start." + exit 1 +fi + +echo "ok." + +echo -n "copying CNI plugin binaries and config files ... " + +cp portmap $HOST_CNI_BIN_PATH +cp aws-cni $HOST_CNI_BIN_PATH$ +cp aws-cni-support.sh $HOST_CNI_BIN_PATH + +sed -i s/__VETHPREFIX__/"${AWS_VPC_K8S_CNI_VETHPREFIX:-"eni"}"/g 10-aws.conflist +sed -i s/__MTU__/"${AWS_VPC_ENI_MTU:-"9001"}"/g 10-aws.conflist +cp 10-aws.conflist $HOST_CNI_CONFDIR_PATH + +echo " ok." + +if [[ -f "$HOST_CNI_CONFDIR_PATH/aws.conf" ]]; then + rm "$HOST_CNI_CONFDIR_PATH/aws.conf" +fi + +# bring the aws-k8s-agent process back into the foreground +echo "foregrounding IPAM daemon ... " +fg %1 >/dev/null 2>&1 || $(echo "failed (process terminated)" && cat $AGENT_LOG_PATH && exit 1) + +# Best practice states we should send the container's CMD output to stdout, so +# let's tee back up the log file into stdout +cat $AGENT_LOG_PATH