Merge pull request #446 from aws/release-1.4.1

Release 1.4.1

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## v1.4.1
+
+* Feature - [Add flag to disable metrics and introspection](https://github.com/aws/amazon-vpc-cni-k8s/pull/436) (#436, @mogren)
+* Bug fix - [Adding additional CRD for Calico that was missing](https://github.com/aws/amazon-vpc-cni-k8s/pull/410) (#410, @wmorgan6796)
+* Improvement - [Update CNI metrics](https://github.com/aws/amazon-vpc-cni-k8s/pull/413) (#413, @mogren)
+
 ## v1.4.0
 
 * Feature - [Add an environment variable to limit the number of ENIs](https://github.com/aws/amazon-vpc-cni-k8s/pull/251) (#251, @pdbogen)

README.md

@@ -147,6 +147,17 @@ Default: Unset
 Valid Values: stdout or a file path
 Specifies where to write the logging output. Either to stdout or to override the default file.
 
+`DISABLE_INTROSPECTION`
+Type: Boolean
+Default: `false`
+Specifies whether introspection endpoints are disabled on a worker node. Setting this to `true` will reduce the debugging 
+information we can get from the node when running the `aws-cni-support.sh` script.
+
+`DISABLE_METRICS`
+Type: Boolean
+Default: `false`
+Specifies whether prometeus metrics endpoints are enabled on a worker node.
+
 ### Notes
 
 `L-IPAMD`(aws-node daemonSet) running on every worker node requires access to kubernetes API server.  If it can **not** reach kubernetes API server, ipamD will exit and CNI will not be able to get any IP address for Pods.  Here is a way to confirm if `L-IPAMD` has access to the kubernetes API server.

config/v1.3/aws-k8s-cni.yaml

@@ -69,7 +69,7 @@ spec:
       tolerations:
       - operator: Exists
       containers:
-      - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.3.3
+      - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.3.4
         imagePullPolicy: Always
         ports:
         - containerPort: 61678
@@ -126,5 +126,3 @@ spec:
     plural: eniconfigs
     singular: eniconfig
     kind: ENIConfig
-
-

config/v1.4/aws-k8s-cni-1.10.yaml

@@ -69,7 +69,7 @@ spec:
       tolerations:
       - operator: Exists
       containers:
-      - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.4.0
+      - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.4.1
         imagePullPolicy: Always
         ports:
         - containerPort: 61678

config/v1.4/aws-k8s-cni.yaml

@@ -69,7 +69,7 @@ spec:
       tolerations:
       - operator: Exists
       containers:
-      - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.4.0
+      - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.4.1
         imagePullPolicy: Always
         ports:
         - containerPort: 61678

misc/cni_metrics_helper.yaml → config/v1.4/cni-metrics-helper.yaml

@@ -5,39 +5,39 @@ kind: ClusterRole
 metadata:
   name: cni-metrics-helper
 rules:
-- apiGroups: [""]
-  resources:
-  - nodes
-  - pods
-  - pods/proxy
-  - services
-  - resourcequotas
-  - replicationcontrollers
-  - limitranges
-  - persistentvolumeclaims
-  - persistentvolumes
-  - namespaces
-  - endpoints
-  verbs: ["list", "watch", "get"]
-- apiGroups: ["extensions"]
-  resources:
-  - daemonsets
-  - deployments
-  - replicasets
-  verbs: ["list", "watch"]
-- apiGroups: ["apps"]
-  resources:
-  - statefulsets
-  verbs: ["list", "watch"]
-- apiGroups: ["batch"]
-  resources:
-  - cronjobs
-  - jobs
-  verbs: ["list", "watch"]
-- apiGroups: ["autoscaling"]
-  resources:
-  - horizontalpodautoscalers
-  verbs: ["list", "watch"]
+  - apiGroups: [""]
+    resources:
+      - nodes
+      - pods
+      - pods/proxy
+      - services
+      - resourcequotas
+      - replicationcontrollers
+      - limitranges
+      - persistentvolumeclaims
+      - persistentvolumes
+      - namespaces
+      - endpoints
+    verbs: ["list", "watch", "get"]
+  - apiGroups: ["extensions"]
+    resources:
+      - daemonsets
+      - deployments
+      - replicasets
+    verbs: ["list", "watch"]
+  - apiGroups: ["apps"]
+    resources:
+      - statefulsets
+    verbs: ["list", "watch"]
+  - apiGroups: ["batch"]
+    resources:
+      - cronjobs
+      - jobs
+    verbs: ["list", "watch"]
+  - apiGroups: ["autoscaling"]
+    resources:
+      - horizontalpodautoscalers
+    verbs: ["list", "watch"]
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -55,9 +55,9 @@ roleRef:
   kind: ClusterRole
   name: cni-metrics-helper
 subjects:
-- kind: ServiceAccount
-  name: cni-metrics-helper
-  namespace: kube-system
+  - kind: ServiceAccount
+    name: cni-metrics-helper
+    namespace: kube-system
 ---
 kind: Deployment
 apiVersion: extensions/v1beta1
@@ -77,9 +77,9 @@ spec:
     spec:
       serviceAccountName: cni-metrics-helper
       containers:
-      - image: 694065802095.dkr.ecr.us-west-2.amazonaws.com/cni-metrics-helper:0.1.1
+      - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/cni-metrics-helper:v1.4.1
         imagePullPolicy: Always
         name: cni-metrics-helper
         env:
           - name: USE_CLOUDWATCH
-            value: "no"
+            value: "yes"

ipamd/introspect.go

@@ -16,64 +16,66 @@ package ipamd
 import (
 	"encoding/json"
 	"net/http"
+	"os"
 	"strconv"
 	"sync"
 	"time"
 
-	log "github.com/cihub/seelog"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/utils"
+	log "github.com/cihub/seelog"
 )
 
 const (
-	// IntrospectionPort is the port for ipamd introspection
-	IntrospectionPort = 61678
+	// introspectionAddress is listening on localhost 61679 for ipamd introspection
+	introspectionAddress = "127.0.0.1:61679"
+
+	// Environment variable to disable the introspection endpoints
+	envDisableIntrospection = "DISABLE_INTROSPECTION"
 )
 
 type rootResponse struct {
 	AvailableCommands []string
 }
 
 // LoggingHandler is a object for handling http request
-type LoggingHandler struct{ h http.Handler }
-
-// NewLoggingHandler creates a new LoggingHandler object.
-func NewLoggingHandler(handler http.Handler) LoggingHandler {
-	return LoggingHandler{h: handler}
+type LoggingHandler struct {
+	h http.Handler
 }
 
 func (lh LoggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	log.Info("Handling http request", "method", r.Method, "from", r.RemoteAddr, "uri", r.RequestURI)
+	log.Info("Handling http request: ", ", method: ", r.Method, ", from: ", r.RemoteAddr, ", URI: ", r.RequestURI)
 	lh.h.ServeHTTP(w, r)
 }
 
-// SetupHTTP sets up ipamd introspection service endpoint
-func (c *IPAMContext) SetupHTTP() {
-	server := c.setupServer()
+// ServeIntrospection sets up ipamd introspection endpoints
+func (c *IPAMContext) ServeIntrospection() {
+	if disableIntrospection() {
+		log.Info("Introspection endpoints disabled")
+		return
+	}
 
+	log.Info("Serving introspection endpoints on ", introspectionAddress)
+	server := c.setupIntrospectionServer()
 	for {
 		once := sync.Once{}
-		utils.RetryWithBackoff(utils.NewSimpleBackoff(time.Second, time.Minute, 0.2, 2), func() error {
-			// TODO, make this cancellable and use the passed in context; for
-			// now, not critical if this gets interrupted
+		_ = utils.RetryWithBackoff(utils.NewSimpleBackoff(time.Second, time.Minute, 0.2, 2), func() error {
 			err := server.ListenAndServe()
 			once.Do(func() {
-				log.Error("Error running http api", "err", err)
+				log.Error("Error running http API: ", err)
 			})
 			return err
 		})
 	}
 }
 
-func (c *IPAMContext) setupServer() *http.Server {
+func (c *IPAMContext) setupIntrospectionServer() *http.Server {
 	serverFunctions := map[string]func(w http.ResponseWriter, r *http.Request){
 		"/v1/enis":                      eniV1RequestHandler(c),
-		"/v1/pods":                      podV1RequestHandler(c),
-		"/v1/networkutils-env-settings": networkEnvV1RequestHandler(c),
-		"/v1/ipamd-env-settings":        ipamdEnvV1RequestHandler(c),
 		"/v1/eni-configs":               eniConfigRequestHandler(c),
+		"/v1/pods":                      podV1RequestHandler(c),
+		"/v1/networkutils-env-settings": networkEnvV1RequestHandler(),
+		"/v1/ipamd-env-settings":        ipamdEnvV1RequestHandler(),
 	}
 	paths := make([]string, 0, len(serverFunctions))
 	for path := range serverFunctions {
@@ -84,96 +86,110 @@ func (c *IPAMContext) setupServer() *http.Server {
 	availableCommandResponse, err := json.Marshal(&availableCommands)
 
 	if err != nil {
-		log.Error("Failed to Marshal: %v", err)
+		log.Errorf("Failed to marshal: %v", err)
 	}
 
 	defaultHandler := func(w http.ResponseWriter, r *http.Request) {
-		w.Write(availableCommandResponse)
+		logErr(w.Write(availableCommandResponse))
 	}
 
 	serveMux := http.NewServeMux()
 	serveMux.HandleFunc("/", defaultHandler)
 	for key, fn := range serverFunctions {
 		serveMux.HandleFunc(key, fn)
 	}
-	serveMux.Handle("/metrics", promhttp.Handler())
 
 	// Log all requests and then pass through to serveMux
 	loggingServeMux := http.NewServeMux()
 	loggingServeMux.Handle("/", LoggingHandler{serveMux})
 
 	server := &http.Server{
-		Addr:         ":" + strconv.Itoa(IntrospectionPort),
+		Addr:         introspectionAddress,
 		Handler:      loggingServeMux,
 		ReadTimeout:  5 * time.Second,
 		WriteTimeout: 5 * time.Second,
 	}
-
 	return server
 }
 
 func eniV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		responseJSON, err := json.Marshal(ipam.dataStore.GetENIInfos())
 		if err != nil {
-			log.Error("Failed to marshal ENI data: %v", err)
+			log.Errorf("Failed to marshal ENI data: %v", err)
 			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
 			return
 		}
-		w.Write(responseJSON)
+		logErr(w.Write(responseJSON))
 	}
 }
 
 func podV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		responseJSON, err := json.Marshal(ipam.dataStore.GetPodInfos())
 		if err != nil {
-			log.Error("Failed to marshal pod data: %v", err)
+			log.Errorf("Failed to marshal pod data: %v", err)
 			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
 			return
 		}
-		w.Write(responseJSON)
+		logErr(w.Write(responseJSON))
 	}
 }
 
 func eniConfigRequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		responseJSON, err := json.Marshal(ipam.eniConfig.Getter())
 		if err != nil {
-			log.Error("Failed to marshal pod data: %v", err)
+			log.Errorf("Failed to marshal ENI config: %v", err)
 			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
 			return
 		}
-		w.Write(responseJSON)
+		logErr(w.Write(responseJSON))
 	}
 }
 
-func networkEnvV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) {
+func networkEnvV1RequestHandler() func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		responseJSON, err := json.Marshal(networkutils.GetConfigForDebug())
 		if err != nil {
-			log.Error("Failed to marshal env var data: %v", err)
+			log.Errorf("Failed to marshal network env var data: %v", err)
 			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
 			return
 		}
-		w.Write(responseJSON)
+		logErr(w.Write(responseJSON))
 	}
 }
 
-func ipamdEnvV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) {
+func ipamdEnvV1RequestHandler() func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		responseJSON, err := json.Marshal(GetConfigForDebug())
 		if err != nil {
-			log.Error("Failed to marshal env var data: %v", err)
+			log.Errorf("Failed to marshal ipamd env var data: %v", err)
 			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
 			return
 		}
-		w.Write(responseJSON)
+		logErr(w.Write(responseJSON))
 	}
 }
 
-func metricsHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) {
-	return func(w http.ResponseWriter, r *http.Request) {
-		promhttp.Handler()
+func logErr(_ int, err error) {
+	if err != nil {
+		log.Errorf("Write failed: %v", err)
+	}
+}
+
+// disableIntrospection returns true if we should disable the introspection
+func disableIntrospection() bool {
+	return getEnvBoolWithDefault(envDisableIntrospection, false)
+}
+
+func getEnvBoolWithDefault(envName string, def bool) bool {
+	if strValue := os.Getenv(envName); strValue != "" {
+		parsedValue, err := strconv.ParseBool(strValue)
+		if err == nil {
+			return parsedValue
+		}
+		log.Errorf("Failed to parse %s, using default `%t`: %v", envName, def, err.Error())
 	}
+	return def
 }