From ce22cd48dafad2c40de0935c7b92956321cb42a4 Mon Sep 17 00:00:00 2001 From: Angus Lees Date: Wed, 20 May 2020 17:16:42 +1000 Subject: [PATCH] Limit scope of logs writable by ipamd container Reduce the logs exposed to ipamd container to just `/var/log/aws-routed-eni/` rather than all of `/var/log` Also correct documented log file defaults: - `AWS_VPC_K8S_PLUGIN_LOG_FILE` defaults to `/var/log/aws-routed-eni/plugin.log` in scripts/entrypoint.sh#L44 - `AWS_VPC_K8S_CNI_LOG_FILE` defaults to `/host/var/log/aws-routed-eni/ipamd.log` in utils/logger/config.go#L47 --- README.md | 8 ++++---- config/master/aws-k8s-cni-cn.yaml | 7 +++++-- config/master/aws-k8s-cni-us-gov-east-1.yaml | 7 +++++-- config/master/aws-k8s-cni-us-gov-west-1.yaml | 7 +++++-- config/master/aws-k8s-cni.yaml | 7 +++++-- config/master/manifests.jsonnet | 10 ++++++++-- go.mod | 3 ++- go.sum | 13 +++++++++++++ 8 files changed, 47 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 7d7b33d2b18..7c02673765c 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ configuration, ipamd always try to keep one extra ENI. When number of pods running on the node exceeds the number of addresses on a single ENI, the CNI backend start allocating a new ENI and start using following allocation scheme: -For example, a m4.4xlarge node can have up to 8 ENIs, and each ENI can have up to 30 IP addresses. See +For example, a m4.4xlarge node can have up to 8 ENIs, and each ENI can have up to 30 IP addresses. See [Elastic Network Interfaces documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html) for details. * If the number of current running Pods is between 0 and 29, ipamd will allocate one more eni. And Warm-Pool size is 2 eni * (30 -1) = 58 @@ -245,7 +245,7 @@ until `WARM_IP_TARGET` free IP addresses are available. EC2 API and that might cause throttling of the requests. It is strongly suggested to set `MINIMUM_IP_TARGET` when using `WARM_IP_TARGET`. If both `WARM_IP_TARGET` and `MINIMUM_IP_TARGET` are set, `ipamd` will attempt to meet both constraints. -This environment variable overrides `WARM_ENI_TARGET` behavior. For a detailed explanation, see +This environment variable overrides `WARM_ENI_TARGET` behavior. For a detailed explanation, see [`WARM_ENI_TARGET`, `WARM_IP_TARGET` and `MINIMUM_IP_TARGET`](https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/eni-and-ip-target.md). @@ -301,7 +301,7 @@ Specifies the loglevel for `ipamd`. Type: String -Default: Unset +Default: `/host/var/log/aws-routed-eni/ipamd.log` Valid Values: `stdout` or a file path @@ -313,7 +313,7 @@ Specifies where to write the logging output of `ipamd`. Either to stdout or to o Type: String -Default: Unset +Default: `/var/log/aws-routed-eni/plugin.log` Valid Values: `stdout` or a file path diff --git a/config/master/aws-k8s-cni-cn.yaml b/config/master/aws-k8s-cni-cn.yaml index 26a118020a2..3042b236105 100644 --- a/config/master/aws-k8s-cni-cn.yaml +++ b/config/master/aws-k8s-cni-cn.yaml @@ -111,6 +111,8 @@ "value": "false" - "name": "AWS_VPC_K8S_CNI_LOGLEVEL" "value": "DEBUG" + - "name": "AWS_VPC_K8S_CNI_LOG_FILE" + "value": "/host/var/log/aws-routed-eni/ipamd.log" - "name": "AWS_VPC_K8S_CNI_VETHPREFIX" "value": "eni" - "name": "MY_NODE_NAME" @@ -147,7 +149,7 @@ "name": "cni-bin-dir" - "mountPath": "/host/etc/cni/net.d" "name": "cni-net-dir" - - "mountPath": "/host/var/log" + - "mountPath": "/host/var/log/aws-routed-eni" "name": "log-dir" - "mountPath": "/var/run/docker.sock" "name": "dockersock" @@ -175,7 +177,8 @@ "path": "/etc/cni/net.d" "name": "cni-net-dir" - "hostPath": - "path": "/var/log" + "path": "/var/log/aws-routed-eni" + "type": "DirectoryOrCreate" "name": "log-dir" - "hostPath": "path": "/var/run/docker.sock" diff --git a/config/master/aws-k8s-cni-us-gov-east-1.yaml b/config/master/aws-k8s-cni-us-gov-east-1.yaml index 230889ecbd6..8b947661c95 100644 --- a/config/master/aws-k8s-cni-us-gov-east-1.yaml +++ b/config/master/aws-k8s-cni-us-gov-east-1.yaml @@ -111,6 +111,8 @@ "value": "false" - "name": "AWS_VPC_K8S_CNI_LOGLEVEL" "value": "DEBUG" + - "name": "AWS_VPC_K8S_CNI_LOG_FILE" + "value": "/host/var/log/aws-routed-eni/ipamd.log" - "name": "AWS_VPC_K8S_CNI_VETHPREFIX" "value": "eni" - "name": "MY_NODE_NAME" @@ -147,7 +149,7 @@ "name": "cni-bin-dir" - "mountPath": "/host/etc/cni/net.d" "name": "cni-net-dir" - - "mountPath": "/host/var/log" + - "mountPath": "/host/var/log/aws-routed-eni" "name": "log-dir" - "mountPath": "/var/run/docker.sock" "name": "dockersock" @@ -175,7 +177,8 @@ "path": "/etc/cni/net.d" "name": "cni-net-dir" - "hostPath": - "path": "/var/log" + "path": "/var/log/aws-routed-eni" + "type": "DirectoryOrCreate" "name": "log-dir" - "hostPath": "path": "/var/run/docker.sock" diff --git a/config/master/aws-k8s-cni-us-gov-west-1.yaml b/config/master/aws-k8s-cni-us-gov-west-1.yaml index 9747a660392..93b81c704b0 100644 --- a/config/master/aws-k8s-cni-us-gov-west-1.yaml +++ b/config/master/aws-k8s-cni-us-gov-west-1.yaml @@ -111,6 +111,8 @@ "value": "false" - "name": "AWS_VPC_K8S_CNI_LOGLEVEL" "value": "DEBUG" + - "name": "AWS_VPC_K8S_CNI_LOG_FILE" + "value": "/host/var/log/aws-routed-eni/ipamd.log" - "name": "AWS_VPC_K8S_CNI_VETHPREFIX" "value": "eni" - "name": "MY_NODE_NAME" @@ -147,7 +149,7 @@ "name": "cni-bin-dir" - "mountPath": "/host/etc/cni/net.d" "name": "cni-net-dir" - - "mountPath": "/host/var/log" + - "mountPath": "/host/var/log/aws-routed-eni" "name": "log-dir" - "mountPath": "/var/run/docker.sock" "name": "dockersock" @@ -175,7 +177,8 @@ "path": "/etc/cni/net.d" "name": "cni-net-dir" - "hostPath": - "path": "/var/log" + "path": "/var/log/aws-routed-eni" + "type": "DirectoryOrCreate" "name": "log-dir" - "hostPath": "path": "/var/run/docker.sock" diff --git a/config/master/aws-k8s-cni.yaml b/config/master/aws-k8s-cni.yaml index d2bf70ef612..53ac36e7c48 100644 --- a/config/master/aws-k8s-cni.yaml +++ b/config/master/aws-k8s-cni.yaml @@ -111,6 +111,8 @@ "value": "false" - "name": "AWS_VPC_K8S_CNI_LOGLEVEL" "value": "DEBUG" + - "name": "AWS_VPC_K8S_CNI_LOG_FILE" + "value": "/host/var/log/aws-routed-eni/ipamd.log" - "name": "AWS_VPC_K8S_CNI_VETHPREFIX" "value": "eni" - "name": "MY_NODE_NAME" @@ -147,7 +149,7 @@ "name": "cni-bin-dir" - "mountPath": "/host/etc/cni/net.d" "name": "cni-net-dir" - - "mountPath": "/host/var/log" + - "mountPath": "/host/var/log/aws-routed-eni" "name": "log-dir" - "mountPath": "/var/run/docker.sock" "name": "dockersock" @@ -175,7 +177,8 @@ "path": "/etc/cni/net.d" "name": "cni-net-dir" - "hostPath": - "path": "/var/log" + "path": "/var/log/aws-routed-eni" + "type": "DirectoryOrCreate" "name": "log-dir" - "hostPath": "path": "/var/run/docker.sock" diff --git a/config/master/manifests.jsonnet b/config/master/manifests.jsonnet index 22619f62eaa..39155cb7474 100644 --- a/config/master/manifests.jsonnet +++ b/config/master/manifests.jsonnet @@ -155,6 +155,7 @@ local awsnode = { AWS_VPC_ENI_MTU: "9001", AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER: "false", AWS_VPC_K8S_CNI_LOGLEVEL: "DEBUG", + AWS_VPC_K8S_CNI_LOG_FILE: "/host/var/log/aws-routed-eni/ipamd.log", AWS_VPC_K8S_CNI_VETHPREFIX: "eni", MY_NODE_NAME: { valueFrom: { @@ -175,7 +176,7 @@ local awsnode = { volumeMounts: [ {mountPath: "/host/opt/cni/bin", name: "cni-bin-dir"}, {mountPath: "/host/etc/cni/net.d", name: "cni-net-dir"}, - {mountPath: "/host/var/log", name: "log-dir"}, + {mountPath: "/host/var/log/aws-routed-eni", name: "log-dir"}, {mountPath: "/var/run/docker.sock", name: "dockersock"}, {mountPath: "/var/run/dockershim.sock", name: "dockershim"}, ], @@ -185,7 +186,12 @@ local awsnode = { volumes: [ {name: "cni-bin-dir", hostPath: {path: "/opt/cni/bin"}}, {name: "cni-net-dir", hostPath: {path: "/etc/cni/net.d"}}, - {name: "log-dir", hostPath: {path: "/var/log"}}, + {name: "log-dir", + hostPath: { + path: "/var/log/aws-routed-eni", + type: "DirectoryOrCreate", + }, + }, {name: "dockersock", hostPath: {path: "/var/run/docker.sock"}}, {name: "dockershim", hostPath: {path: "/var/run/dockershim.sock"}}, ], diff --git a/go.mod b/go.mod index f08eba0b1ae..208ea8a912c 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/golang/mock v1.4.1 github.com/golang/protobuf v1.3.5 github.com/google/btree v1.0.0 // indirect + github.com/google/go-jsonnet v0.16.0 // indirect github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect github.com/googleapis/gnostic v0.2.0 // indirect github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect @@ -37,7 +38,7 @@ require ( go.uber.org/zap v1.15.0 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect golang.org/x/net v0.0.0-20200202094626-16171245cfb2 - golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 + golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect google.golang.org/grpc v1.29.0 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect diff --git a/go.sum b/go.sum index 6efeddbff83..30825c83ae7 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -65,6 +67,8 @@ github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-jsonnet v0.16.0 h1:Nb4EEOp+rdeGGyB1rQ5eisgSAqrTnhf9ip+X6lzZbY0= +github.com/google/go-jsonnet v0.16.0/go.mod h1:sOcuej3UW1vpPTZOr8L7RQimqai1a57bt5j22LzGZCw= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= @@ -96,6 +100,11 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -135,6 +144,7 @@ github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 h1:2c1EFnZHIPCW8qKWgHMH/fX2PkSabFc5mrVzfUNdg5U= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -198,11 +208,14 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=