Skip to content

Commit

Permalink
Add flag to disable metrics and introspection
Browse files Browse the repository at this point in the history 
Adds the environment variables DISABLE_INTROSPECTION and DISABLE_METRICS
to make it possible to turn off metrics and debugging.
  • Loading branch information
Claes Mogren committed May 1, 2019
1 parent 0d3b925 commit 2b08772
Show file tree
Hide file tree
Showing 8 changed files with 272 additions and 122 deletions.
173 changes: 111 additions & 62 deletions README.md
View file

Large diffs are not rendered by default.

66 changes: 43 additions & 23 deletions ipamd/introspect.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,22 @@ package ipamd
import (
"encoding/json"
"net/http"
"os"
"strconv"
"sync"
"time"

log "github.com/cihub/seelog"
"github.com/prometheus/client_golang/prometheus/promhttp"

"github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils"
"github.com/aws/amazon-vpc-cni-k8s/pkg/utils"
log "github.com/cihub/seelog"
)

const (
// IntrospectionPort is the port for ipamd introspection
IntrospectionPort = 61678
// introspectionAddress is listening on localhost 61679 for ipamd introspection
introspectionAddress = "127.0.0.1:61679"

// Environment variable to disable the introspection endpoints
envDisableIntrospection = "DISABLE_INTROSPECTION"
)

type rootResponse struct {
Expand All @@ -42,29 +44,33 @@ type LoggingHandler struct {
}

func (lh LoggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
log.Info("Handling http request", "method", r.Method, "from", r.RemoteAddr, "uri", r.RequestURI)
log.Info("Handling http request: ", ", method: ", r.Method, ", from: ", r.RemoteAddr, ", URI: ", r.RequestURI)
lh.h.ServeHTTP(w, r)
}

// SetupHTTP sets up ipamd introspection service endpoint
func (c *IPAMContext) SetupHTTP() {
server := c.setupServer()
// ServeIntrospection sets up ipamd introspection endpoints
func (c *IPAMContext) ServeIntrospection() {
if disableIntrospection() {
log.Info("Introspection endpoints disabled")
return
}

log.Info("Serving introspection endpoints on ", introspectionAddress)
server := c.setupIntrospectionServer()
for {
once := sync.Once{}
utils.RetryWithBackoff(utils.NewSimpleBackoff(time.Second, time.Minute, 0.2, 2), func() error {
// TODO, make this cancellable and use the passed in context; for
// now, not critical if this gets interrupted
_ = utils.RetryWithBackoff(utils.NewSimpleBackoff(time.Second, time.Minute, 0.2, 2), func() error {
err := server.ListenAndServe()
once.Do(func() {
log.Error("Error running http api", "err", err)
log.Error("Error running http API: ", err)
})
return err
})
}
}

func (c *IPAMContext) setupServer() *http.Server {
func (c *IPAMContext) setupIntrospectionServer() *http.Server {
// If enabled, add introspection endpoints
serverFunctions := map[string]func(w http.ResponseWriter, r *http.Request){
"/v1/enis": eniV1RequestHandler(c),
"/v1/eni-configs": eniConfigRequestHandler(c),
Expand All @@ -81,26 +87,24 @@ func (c *IPAMContext) setupServer() *http.Server {
availableCommandResponse, err := json.Marshal(&availableCommands)

if err != nil {
log.Error("Failed to Marshal: %v", err)
log.Error("Failed to marshal: %v", err)
}

defaultHandler := func(w http.ResponseWriter, r *http.Request) {
logErr(w.Write(availableCommandResponse))
}

serveMux := http.NewServeMux()
serveMux.HandleFunc("/", defaultHandler)
for key, fn := range serverFunctions {
serveMux.HandleFunc(key, fn)
}
serveMux.Handle("/metrics", promhttp.Handler())

// Log all requests and then pass through to serveMux
loggingServeMux := http.NewServeMux()
loggingServeMux.Handle("/", LoggingHandler{serveMux})

server := &http.Server{
Addr: ":" + strconv.Itoa(IntrospectionPort),
Addr: introspectionAddress,
Handler: loggingServeMux,
ReadTimeout: 5 * time.Second,
WriteTimeout: 5 * time.Second,
Expand All @@ -112,7 +116,7 @@ func eniV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Requ
return func(w http.ResponseWriter, r *http.Request) {
responseJSON, err := json.Marshal(ipam.dataStore.GetENIInfos())
if err != nil {
log.Error("Failed to marshal ENI data: %v", err)
log.Errorf("Failed to marshal ENI data: %v", err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
Expand All @@ -124,7 +128,7 @@ func podV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Requ
return func(w http.ResponseWriter, r *http.Request) {
responseJSON, err := json.Marshal(ipam.dataStore.GetPodInfos())
if err != nil {
log.Error("Failed to marshal pod data: %v", err)
log.Errorf("Failed to marshal pod data: %v", err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
Expand All @@ -136,7 +140,7 @@ func eniConfigRequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.
return func(w http.ResponseWriter, r *http.Request) {
responseJSON, err := json.Marshal(ipam.eniConfig.Getter())
if err != nil {
log.Error("Failed to marshal ENI config: %v", err)
log.Errorf("Failed to marshal ENI config: %v", err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
Expand All @@ -148,7 +152,7 @@ func networkEnvV1RequestHandler() func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
responseJSON, err := json.Marshal(networkutils.GetConfigForDebug())
if err != nil {
log.Error("Failed to marshal network env var data: %v", err)
log.Errorf("Failed to marshal network env var data: %v", err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
Expand All @@ -160,7 +164,7 @@ func ipamdEnvV1RequestHandler() func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
responseJSON, err := json.Marshal(GetConfigForDebug())
if err != nil {
log.Error("Failed to marshal ipamd env var data: %v", err)
log.Errorf("Failed to marshal ipamd env var data: %v", err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
Expand All @@ -173,3 +177,19 @@ func logErr(_ int, err error) {
log.Errorf("Write failed: %v", err)
}
}

// disableIntrospection returns true if we should disable the introspection
func disableIntrospection() bool {
return getEnvBoolWithDefault(envDisableIntrospection, false)
}

func getEnvBoolWithDefault(envName string, def bool) bool {
if strValue := os.Getenv(envName); strValue != "" {
parsedValue, err := strconv.ParseBool(strValue)
if err == nil {
return parsedValue
}
log.Errorf("Failed to parse %s, using default `%t`: %v", envName, def, err.Error())
}
return def
}
41 changes: 19 additions & 22 deletions ipamd/ipamd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ const (
primaryENIid = "eni-00000000"
secENIid = "eni-00000001"
testAttachmentID = "eni-00000000-attach"
eniID = "eni-5731da78"
primaryMAC = "12:ef:2a:98:e5:5a"
secMAC = "12:ef:2a:98:e5:5b"
primaryDevice = 0
Expand All @@ -51,10 +50,8 @@ const (
secSubnet = "10.10.20.0/24"
ipaddr01 = "10.10.10.11"
ipaddr02 = "10.10.10.12"
ipaddr03 = "10.10.10.13"
ipaddr11 = "10.10.20.11"
ipaddr12 = "10.10.20.12"
ipaddr13 = "10.10.20.13"
vpcCIDR = "10.10.0.0/16"
)

Expand Down Expand Up @@ -163,12 +160,12 @@ func TestNodeInit(t *testing.T) {
}

func TestIncreaseIPPoolDefault(t *testing.T) {
os.Unsetenv(envCustomNetworkCfg)
_ = os.Unsetenv(envCustomNetworkCfg)
testIncreaseIPPool(t, false)
}

func TestIncreaseIPPoolCustomENI(t *testing.T) {
os.Setenv(envCustomNetworkCfg, "true")
_ = os.Setenv(envCustomNetworkCfg, "true")
testIncreaseIPPool(t, true)
}

Expand Down Expand Up @@ -337,15 +334,15 @@ func TestGetWarmENITarget(t *testing.T) {
ctrl, _, _, _, _, _ := setup(t)
defer ctrl.Finish()

os.Setenv("WARM_IP_TARGET", "5")
_ = os.Setenv("WARM_IP_TARGET", "5")
warmIPTarget := getWarmIPTarget()
assert.Equal(t, warmIPTarget, 5)

os.Unsetenv("WARM_IP_TARGET")
_ = os.Unsetenv("WARM_IP_TARGET")
warmIPTarget = getWarmIPTarget()
assert.Equal(t, warmIPTarget, noWarmIPTarget)

os.Setenv("WARM_IP_TARGET", "non-integer-string")
_ = os.Setenv("WARM_IP_TARGET", "non-integer-string")
warmIPTarget = getWarmIPTarget()
assert.Equal(t, warmIPTarget, noWarmIPTarget)
}
Expand All @@ -355,32 +352,32 @@ func TestGetMaxENI(t *testing.T) {
defer ctrl.Finish()

// MaxENI 5 is less than upper bound of 10, so 5
os.Setenv("MAX_ENI", "5")
_ = os.Setenv("MAX_ENI", "5")
maxENI := getMaxENI(10)
assert.Equal(t, maxENI, 5)

// MaxENI 5 is greater than upper bound of 4, so 4
os.Setenv("MAX_ENI", "5")
_ = os.Setenv("MAX_ENI", "5")
maxENI = getMaxENI(4)
assert.Equal(t, maxENI, 4)

// MaxENI 0 is 0, which means disabled; so use upper bound
os.Setenv("MAX_ENI", "0")
_ = os.Setenv("MAX_ENI", "0")
maxENI = getMaxENI(4)
assert.Equal(t, maxENI, 4)

// MaxENI 1 is less than upper bound of 4, so 1.
os.Setenv("MAX_ENI", "1")
_ = os.Setenv("MAX_ENI", "1")
maxENI = getMaxENI(4)
assert.Equal(t, maxENI, 1)

// Empty MaxENI means disabled, so use upper bound
os.Unsetenv("MAX_ENI")
_ = os.Unsetenv("MAX_ENI")
maxENI = getMaxENI(10)
assert.Equal(t, maxENI, 10)

// Invalid MaxENI means disabled, so use upper bound
os.Setenv("MAX_ENI", "non-integer-string")
_ = os.Setenv("MAX_ENI", "non-integer-string")
maxENI = getMaxENI(10)
assert.Equal(t, maxENI, 10)
}
Expand All @@ -398,30 +395,30 @@ func TestGetWarmIPTargetState(t *testing.T) {

mockContext.dataStore = datastore.NewDataStore()

os.Unsetenv("WARM_IP_TARGET")
_ = os.Unsetenv("WARM_IP_TARGET")
_, _, warmIPTargetDefined := mockContext.ipTargetState()
assert.False(t, warmIPTargetDefined)

os.Setenv("WARM_IP_TARGET", "5")
_ = os.Setenv("WARM_IP_TARGET", "5")
short, over, warmIPTargetDefined := mockContext.ipTargetState()
assert.True(t, warmIPTargetDefined)
assert.Equal(t, 5, short)
assert.Equal(t, 0, over)

// add 2 addresses to datastore
mockContext.dataStore.AddENI("eni-1", 1, true)
mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.1")
mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.2")
_ = mockContext.dataStore.AddENI("eni-1", 1, true)
_ = mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.1")
_ = mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.2")

short, over, warmIPTargetDefined = mockContext.ipTargetState()
assert.True(t, warmIPTargetDefined)
assert.Equal(t, 3, short)
assert.Equal(t, 0, over)

// add 3 more addresses to datastore
mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.3")
mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.4")
mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.5")
_ = mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.3")
_ = mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.4")
_ = mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.5")

short, over, warmIPTargetDefined = mockContext.ipTargetState()
assert.True(t, warmIPTargetDefined)
Expand Down
73 changes: 73 additions & 0 deletions ipamd/metrics.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
// http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package ipamd

import (
"net/http"
"strconv"
"sync"
"time"

log "github.com/cihub/seelog"
"github.com/prometheus/client_golang/prometheus/promhttp"

"github.com/aws/amazon-vpc-cni-k8s/pkg/utils"
)

const (
// metricsPort is the port for prometheus metrics
metricsPort = 61678

// Environment variable to disable the metrics endpoint on 61678
envDisableMetrics = "DISABLE_METRICS"
)

// ServeMetrics sets up ipamd metrics and introspection endpoints
func (c *IPAMContext) ServeMetrics() {
if disableMetrics() {
log.Info("Metrics endpoint disabled")
return
}

log.Info("Serving metrics on port ", metricsPort)
server := c.setupMetricsServer()
for {
once := sync.Once{}
_ = utils.RetryWithBackoff(utils.NewSimpleBackoff(time.Second, time.Minute, 0.2, 2), func() error {
err := server.ListenAndServe()
once.Do(func() {
log.Error("Error running http API: ", err)
})
return err
})
}
}

func (c *IPAMContext) setupMetricsServer() *http.Server {
// Always add the metrics endpoint
serveMux := http.NewServeMux()
serveMux.Handle("/metrics", promhttp.Handler())
server := &http.Server{
Addr: ":" + strconv.Itoa(metricsPort),
Handler: serveMux,
ReadTimeout: 5 * time.Second,
WriteTimeout: 5 * time.Second,
}
return server
}

// disableMetrics returns true if we should disable metrics
func disableMetrics() bool {
return getEnvBoolWithDefault(envDisableMetrics, false)
}
4 changes: 2 additions & 2 deletions ipamd/rpc_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import (
)

const (
port = "127.0.0.1:50051"
ipamdgRPCaddress = "127.0.0.1:50051"
)

type server struct {
Expand Down Expand Up @@ -91,7 +91,7 @@ func (s *server) DelNetwork(ctx context.Context, in *pb.DelNetworkRequest) (*pb.

// RunRPCHandler handles request from gRPC
func (c *IPAMContext) RunRPCHandler() error {
lis, err := net.Listen("tcp", port)
lis, err := net.Listen("tcp", ipamdgRPCaddress)
if err != nil {
log.Errorf("Failed to listen gRPC port: %v", err)
return errors.Wrap(err, "ipamd: failed to listen to gRPC port")
Expand Down
Loading

0 comments on commit 2b08772

Please sign in to comment.