Add flag to disable metrics and introspection

ipamd/introspect.go

@@ -16,20 +16,22 @@ package ipamd
 import (
 	"encoding/json"
 	"net/http"
+	"os"
 	"strconv"
 	"sync"
 	"time"
 
-	log "github.com/cihub/seelog"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/utils"
+	log "github.com/cihub/seelog"
 )
 
 const (
-	// IntrospectionPort is the port for ipamd introspection
-	IntrospectionPort = 61678
+	// introspectionAddress is listening on localhost 61679 for ipamd introspection
+	introspectionAddress = "127.0.0.1:61679"
+
+	// Environment variable to disable the introspection endpoints
+	envDisableIntrospection = "DISABLE_INTROSPECTION"
 )
 
 type rootResponse struct {
@@ -42,29 +44,33 @@ type LoggingHandler struct {
 }
 
 func (lh LoggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	log.Info("Handling http request", "method", r.Method, "from", r.RemoteAddr, "uri", r.RequestURI)
+	log.Info("Handling http request: ", ", method: ", r.Method, ", from: ", r.RemoteAddr, ", URI: ", r.RequestURI)
 	lh.h.ServeHTTP(w, r)
 }
 
-// SetupHTTP sets up ipamd introspection service endpoint
-func (c *IPAMContext) SetupHTTP() {
-	server := c.setupServer()
+// ServeIntrospection sets up ipamd introspection endpoints
+func (c *IPAMContext) ServeIntrospection() {
+	if disableIntrospection() {
+		log.Info("Introspection endpoints disabled")
+		return
+	}
 
+	log.Info("Serving introspection endpoints on ", introspectionAddress)
+	server := c.setupIntrospectionServer()
 	for {
 		once := sync.Once{}
-		utils.RetryWithBackoff(utils.NewSimpleBackoff(time.Second, time.Minute, 0.2, 2), func() error {
-			// TODO, make this cancellable and use the passed in context; for
-			// now, not critical if this gets interrupted
+		_ = utils.RetryWithBackoff(utils.NewSimpleBackoff(time.Second, time.Minute, 0.2, 2), func() error {
 			err := server.ListenAndServe()
 			once.Do(func() {
-				log.Error("Error running http api", "err", err)
+				log.Error("Error running http API: ", err)
 			})
 			return err
 		})
 	}
 }
 
-func (c *IPAMContext) setupServer() *http.Server {
+func (c *IPAMContext) setupIntrospectionServer() *http.Server {
+	// If enabled, add introspection endpoints
 	serverFunctions := map[string]func(w http.ResponseWriter, r *http.Request){
 		"/v1/enis":                      eniV1RequestHandler(c),
 		"/v1/eni-configs":               eniConfigRequestHandler(c),
@@ -81,26 +87,24 @@ func (c *IPAMContext) setupServer() *http.Server {
 	availableCommandResponse, err := json.Marshal(&availableCommands)
 
 	if err != nil {
-		log.Error("Failed to Marshal: %v", err)
+		log.Error("Failed to marshal: %v", err)
 	}
 
 	defaultHandler := func(w http.ResponseWriter, r *http.Request) {
 		logErr(w.Write(availableCommandResponse))
 	}
-
 	serveMux := http.NewServeMux()
 	serveMux.HandleFunc("/", defaultHandler)
 	for key, fn := range serverFunctions {
 		serveMux.HandleFunc(key, fn)
 	}
-	serveMux.Handle("/metrics", promhttp.Handler())
 
 	// Log all requests and then pass through to serveMux
 	loggingServeMux := http.NewServeMux()
 	loggingServeMux.Handle("/", LoggingHandler{serveMux})
 
 	server := &http.Server{
-		Addr:         ":" + strconv.Itoa(IntrospectionPort),
+		Addr:         introspectionAddress,
 		Handler:      loggingServeMux,
 		ReadTimeout:  5 * time.Second,
 		WriteTimeout: 5 * time.Second,
@@ -112,7 +116,7 @@ func eniV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Requ
 	return func(w http.ResponseWriter, r *http.Request) {
 		responseJSON, err := json.Marshal(ipam.dataStore.GetENIInfos())
 		if err != nil {
-			log.Error("Failed to marshal ENI data: %v", err)
+			log.Errorf("Failed to marshal ENI data: %v", err)
 			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
 			return
 		}
@@ -124,7 +128,7 @@ func podV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Requ
 	return func(w http.ResponseWriter, r *http.Request) {
 		responseJSON, err := json.Marshal(ipam.dataStore.GetPodInfos())
 		if err != nil {
-			log.Error("Failed to marshal pod data: %v", err)
+			log.Errorf("Failed to marshal pod data: %v", err)
 			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
 			return
 		}
@@ -136,7 +140,7 @@ func eniConfigRequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.
 	return func(w http.ResponseWriter, r *http.Request) {
 		responseJSON, err := json.Marshal(ipam.eniConfig.Getter())
 		if err != nil {
-			log.Error("Failed to marshal ENI config: %v", err)
+			log.Errorf("Failed to marshal ENI config: %v", err)
 			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
 			return
 		}
@@ -148,7 +152,7 @@ func networkEnvV1RequestHandler() func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		responseJSON, err := json.Marshal(networkutils.GetConfigForDebug())
 		if err != nil {
-			log.Error("Failed to marshal network env var data: %v", err)
+			log.Errorf("Failed to marshal network env var data: %v", err)
 			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
 			return
 		}
@@ -160,7 +164,7 @@ func ipamdEnvV1RequestHandler() func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		responseJSON, err := json.Marshal(GetConfigForDebug())
 		if err != nil {
-			log.Error("Failed to marshal ipamd env var data: %v", err)
+			log.Errorf("Failed to marshal ipamd env var data: %v", err)
 			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
 			return
 		}
@@ -173,3 +177,19 @@ func logErr(_ int, err error) {
 		log.Errorf("Write failed: %v", err)
 	}
 }
+
+// disableIntrospection returns true if we should disable the introspection
+func disableIntrospection() bool {
+	return getEnvBoolWithDefault(envDisableIntrospection, false)
+}
+
+func getEnvBoolWithDefault(envName string, def bool) bool {
+	if strValue := os.Getenv(envName); strValue != "" {
+		parsedValue, err := strconv.ParseBool(strValue)
+		if err == nil {
+			return parsedValue
+		}
+		log.Errorf("Failed to parse %s, using default `%t`: %v", envName, def, err.Error())
+	}
+	return def
+}

ipamd/ipamd_test.go

@@ -42,7 +42,6 @@ const (
 	primaryENIid     = "eni-00000000"
 	secENIid         = "eni-00000001"
 	testAttachmentID = "eni-00000000-attach"
-	eniID            = "eni-5731da78"
 	primaryMAC       = "12:ef:2a:98:e5:5a"
 	secMAC           = "12:ef:2a:98:e5:5b"
 	primaryDevice    = 0
@@ -51,10 +50,8 @@ const (
 	secSubnet        = "10.10.20.0/24"
 	ipaddr01         = "10.10.10.11"
 	ipaddr02         = "10.10.10.12"
-	ipaddr03         = "10.10.10.13"
 	ipaddr11         = "10.10.20.11"
 	ipaddr12         = "10.10.20.12"
-	ipaddr13         = "10.10.20.13"
 	vpcCIDR          = "10.10.0.0/16"
 )
 
@@ -163,12 +160,12 @@ func TestNodeInit(t *testing.T) {
 }
 
 func TestIncreaseIPPoolDefault(t *testing.T) {
-	os.Unsetenv(envCustomNetworkCfg)
+	_ = os.Unsetenv(envCustomNetworkCfg)
 	testIncreaseIPPool(t, false)
 }
 
 func TestIncreaseIPPoolCustomENI(t *testing.T) {
-	os.Setenv(envCustomNetworkCfg, "true")
+	_ = os.Setenv(envCustomNetworkCfg, "true")
 	testIncreaseIPPool(t, true)
 }
 
@@ -337,15 +334,15 @@ func TestGetWarmENITarget(t *testing.T) {
 	ctrl, _, _, _, _, _ := setup(t)
 	defer ctrl.Finish()
 
-	os.Setenv("WARM_IP_TARGET", "5")
+	_ = os.Setenv("WARM_IP_TARGET", "5")
 	warmIPTarget := getWarmIPTarget()
 	assert.Equal(t, warmIPTarget, 5)
 
-	os.Unsetenv("WARM_IP_TARGET")
+	_ = os.Unsetenv("WARM_IP_TARGET")
 	warmIPTarget = getWarmIPTarget()
 	assert.Equal(t, warmIPTarget, noWarmIPTarget)
 
-	os.Setenv("WARM_IP_TARGET", "non-integer-string")
+	_ = os.Setenv("WARM_IP_TARGET", "non-integer-string")
 	warmIPTarget = getWarmIPTarget()
 	assert.Equal(t, warmIPTarget, noWarmIPTarget)
 }
@@ -355,32 +352,32 @@ func TestGetMaxENI(t *testing.T) {
 	defer ctrl.Finish()
 
 	// MaxENI 5 is less than upper bound of 10, so 5
-	os.Setenv("MAX_ENI", "5")
+	_ = os.Setenv("MAX_ENI", "5")
 	maxENI := getMaxENI(10)
 	assert.Equal(t, maxENI, 5)
 
 	// MaxENI 5 is greater than upper bound of 4, so 4
-	os.Setenv("MAX_ENI", "5")
+	_ = os.Setenv("MAX_ENI", "5")
 	maxENI = getMaxENI(4)
 	assert.Equal(t, maxENI, 4)
 
 	// MaxENI 0 is 0, which means disabled; so use upper bound
-	os.Setenv("MAX_ENI", "0")
+	_ = os.Setenv("MAX_ENI", "0")
 	maxENI = getMaxENI(4)
 	assert.Equal(t, maxENI, 4)
 
 	// MaxENI 1 is less than upper bound of 4, so 1.
-	os.Setenv("MAX_ENI", "1")
+	_ = os.Setenv("MAX_ENI", "1")
 	maxENI = getMaxENI(4)
 	assert.Equal(t, maxENI, 1)
 
 	// Empty MaxENI means disabled, so use upper bound
-	os.Unsetenv("MAX_ENI")
+	_ = os.Unsetenv("MAX_ENI")
 	maxENI = getMaxENI(10)
 	assert.Equal(t, maxENI, 10)
 
 	// Invalid MaxENI means disabled, so use upper bound
-	os.Setenv("MAX_ENI", "non-integer-string")
+	_ = os.Setenv("MAX_ENI", "non-integer-string")
 	maxENI = getMaxENI(10)
 	assert.Equal(t, maxENI, 10)
 }
@@ -398,30 +395,30 @@ func TestGetWarmIPTargetState(t *testing.T) {
 
 	mockContext.dataStore = datastore.NewDataStore()
 
-	os.Unsetenv("WARM_IP_TARGET")
+	_ = os.Unsetenv("WARM_IP_TARGET")
 	_, _, warmIPTargetDefined := mockContext.ipTargetState()
 	assert.False(t, warmIPTargetDefined)
 
-	os.Setenv("WARM_IP_TARGET", "5")
+	_ = os.Setenv("WARM_IP_TARGET", "5")
 	short, over, warmIPTargetDefined := mockContext.ipTargetState()
 	assert.True(t, warmIPTargetDefined)
 	assert.Equal(t, 5, short)
 	assert.Equal(t, 0, over)
 
 	// add 2 addresses to datastore
-	mockContext.dataStore.AddENI("eni-1", 1, true)
-	mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.1")
-	mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.2")
+	_ = mockContext.dataStore.AddENI("eni-1", 1, true)
+	_ = mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.1")
+	_ = mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.2")
 
 	short, over, warmIPTargetDefined = mockContext.ipTargetState()
 	assert.True(t, warmIPTargetDefined)
 	assert.Equal(t, 3, short)
 	assert.Equal(t, 0, over)
 
 	// add 3 more addresses to datastore
-	mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.3")
-	mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.4")
-	mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.5")
+	_ = mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.3")
+	_ = mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.4")
+	_ = mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.5")
 
 	short, over, warmIPTargetDefined = mockContext.ipTargetState()
 	assert.True(t, warmIPTargetDefined)

ipamd/metrics.go

@@ -0,0 +1,73 @@
+// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//      http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package ipamd
+
+import (
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+
+	log "github.com/cihub/seelog"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"github.com/aws/amazon-vpc-cni-k8s/pkg/utils"
+)
+
+const (
+	// metricsPort is the port for prometheus metrics
+	metricsPort = 61678
+
+	// Environment variable to disable the metrics endpoint on 61678
+	envDisableMetrics = "DISABLE_METRICS"
+)
+
+// ServeMetrics sets up ipamd metrics and introspection endpoints
+func (c *IPAMContext) ServeMetrics() {
+	if disableMetrics() {
+		log.Info("Metrics endpoint disabled")
+		return
+	}
+
+	log.Info("Serving metrics on port ", metricsPort)
+	server := c.setupMetricsServer()
+	for {
+		once := sync.Once{}
+		_ = utils.RetryWithBackoff(utils.NewSimpleBackoff(time.Second, time.Minute, 0.2, 2), func() error {
+			err := server.ListenAndServe()
+			once.Do(func() {
+				log.Error("Error running http API: ", err)
+			})
+			return err
+		})
+	}
+}
+
+func (c *IPAMContext) setupMetricsServer() *http.Server {
+	// Always add the metrics endpoint
+	serveMux := http.NewServeMux()
+	serveMux.Handle("/metrics", promhttp.Handler())
+	server := &http.Server{
+		Addr:         ":" + strconv.Itoa(metricsPort),
+		Handler:      serveMux,
+		ReadTimeout:  5 * time.Second,
+		WriteTimeout: 5 * time.Second,
+	}
+	return server
+}
+
+// disableMetrics returns true if we should disable metrics
+func disableMetrics() bool {
+	return getEnvBoolWithDefault(envDisableMetrics, false)
+}

ipamd/rpc_handler.go

@@ -31,7 +31,7 @@ import (
 )
 
 const (
-	port = "127.0.0.1:50051"
+	ipamdgRPCaddress = "127.0.0.1:50051"
 )
 
 type server struct {
@@ -91,7 +91,7 @@ func (s *server) DelNetwork(ctx context.Context, in *pb.DelNetworkRequest) (*pb.
 
 // RunRPCHandler handles request from gRPC
 func (c *IPAMContext) RunRPCHandler() error {
-	lis, err := net.Listen("tcp", port)
+	lis, err := net.Listen("tcp", ipamdgRPCaddress)
 	if err != nil {
 		log.Errorf("Failed to listen gRPC port: %v", err)
 		return errors.Wrap(err, "ipamd: failed to listen to gRPC port")