Add debug pprof server to Felix/Typha

api/pkg/apis/projectcalico/v3/felixconfig.go

@@ -408,6 +408,12 @@ type FelixConfigurationSpec struct {
 	// +kubebuilder:validation:Type=string
 	// +kubebuilder:validation:Pattern=`^([0-9]+(\\.[0-9]+)?(ms|s|m|h))*$`
 	DebugSimulateDataplaneHangAfter *metav1.Duration `json:"debugSimulateDataplaneHangAfter,omitempty" configv1timescale:"seconds"`
+	// DebugHost is the host IP or hostname to bind the debug port to.  Only used
+	// if DebugPort is set. [Default:localhost]
+	DebugHost *string `json:"debugHost,omitempty"`
+	// DebugPort if set, enables Felix's debug HTTP port, which allows memory and CPU profiles
+	// to be retrieved.  The debug port is not secure, it should not be exposed to the internet.
+	DebugPort *int `json:"debugPort,omitempty" validate:"omitempty,gte=0,lte=65535"`
 
 	IptablesNATOutgoingInterfaceFilter string `json:"iptablesNATOutgoingInterfaceFilter,omitempty" validate:"omitempty,ifaceFilter"`
 

felix/config/config_params.go

@@ -370,6 +370,10 @@ type Config struct {
 	DebugSimulateDataplaneHangAfter time.Duration `config:"seconds;0"`
 	DebugPanicAfter                 time.Duration `config:"seconds;0"`
 	DebugSimulateDataRace           bool          `config:"bool;false"`
+	// DebugHost is the host to bind the debug server port to.  Only used if DebugPort is non-zero.
+	DebugHost string `config:"host-address;localhost"`
+	// DebugPort is the port to bind the pprof debug server to or 0 to disable the debug port.
+	DebugPort int `config:"int(0,65535);"`
 
 	// Configure where Felix gets its routing information.
 	// - workloadIPs: use workload endpoints to construct routes.

felix/daemon/daemon.go

@@ -28,46 +28,44 @@ import (
 	"syscall"
 	"time"
 
+	apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 	"k8s.io/client-go/kubernetes"
 
-	"github.com/projectcalico/calico/libcalico-go/lib/backend/syncersv1/dedupebuffer"
-	"github.com/projectcalico/calico/libcalico-go/lib/winutils"
-
-	"github.com/projectcalico/calico/libcalico-go/lib/seedrng"
-
-	apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3"
-
+	"github.com/projectcalico/calico/felix/buildinfo"
+	"github.com/projectcalico/calico/felix/calc"
+	"github.com/projectcalico/calico/felix/config"
+	dp "github.com/projectcalico/calico/felix/dataplane"
+	"github.com/projectcalico/calico/felix/jitter"
+	"github.com/projectcalico/calico/felix/logutils"
+	"github.com/projectcalico/calico/felix/policysync"
+	"github.com/projectcalico/calico/felix/proto"
+	"github.com/projectcalico/calico/felix/statusrep"
+	"github.com/projectcalico/calico/felix/usagerep"
 	"github.com/projectcalico/calico/libcalico-go/lib/apiconfig"
 	libapiv3 "github.com/projectcalico/calico/libcalico-go/lib/apis/v3"
 	"github.com/projectcalico/calico/libcalico-go/lib/backend"
 	bapi "github.com/projectcalico/calico/libcalico-go/lib/backend/api"
 	"github.com/projectcalico/calico/libcalico-go/lib/backend/k8s"
 	"github.com/projectcalico/calico/libcalico-go/lib/backend/model"
+	"github.com/projectcalico/calico/libcalico-go/lib/backend/syncersv1/dedupebuffer"
 	"github.com/projectcalico/calico/libcalico-go/lib/backend/syncersv1/felixsyncer"
 	"github.com/projectcalico/calico/libcalico-go/lib/backend/syncersv1/updateprocessors"
 	"github.com/projectcalico/calico/libcalico-go/lib/backend/watchersyncer"
 	client "github.com/projectcalico/calico/libcalico-go/lib/clientv3"
+	"github.com/projectcalico/calico/libcalico-go/lib/debugserver"
 	cerrors "github.com/projectcalico/calico/libcalico-go/lib/errors"
 	"github.com/projectcalico/calico/libcalico-go/lib/health"
 	lclogutils "github.com/projectcalico/calico/libcalico-go/lib/logutils"
+	"github.com/projectcalico/calico/libcalico-go/lib/metricsserver"
 	"github.com/projectcalico/calico/libcalico-go/lib/options"
+	"github.com/projectcalico/calico/libcalico-go/lib/seedrng"
 	"github.com/projectcalico/calico/libcalico-go/lib/set"
+	"github.com/projectcalico/calico/libcalico-go/lib/winutils"
 	"github.com/projectcalico/calico/pod2daemon/binder"
 	"github.com/projectcalico/calico/typha/pkg/discovery"
 	"github.com/projectcalico/calico/typha/pkg/syncclient"
-
-	"github.com/projectcalico/calico/felix/buildinfo"
-	"github.com/projectcalico/calico/felix/calc"
-	"github.com/projectcalico/calico/felix/config"
-	dp "github.com/projectcalico/calico/felix/dataplane"
-	"github.com/projectcalico/calico/felix/jitter"
-	"github.com/projectcalico/calico/felix/logutils"
-	"github.com/projectcalico/calico/felix/policysync"
-	"github.com/projectcalico/calico/felix/proto"
-	"github.com/projectcalico/calico/felix/statusrep"
-	"github.com/projectcalico/calico/felix/usagerep"
 )
 
 const (
@@ -408,6 +406,10 @@ configRetry:
 		simulateDataRace()
 	}
 
+	if configParams.DebugPort != 0 {
+		debugserver.StartDebugPprofServer(configParams.DebugHost, configParams.DebugPort)
+	}
+
 	// Start up the dataplane driver.  This may be the internal go-based driver or an external
 	// one.
 	var dpDriver dp.DataplaneDriver
@@ -681,7 +683,11 @@ configRetry:
 		})
 		gaugeHost.Set(1)
 		prometheus.MustRegister(gaugeHost)
-		go dp.ServePrometheusMetrics(configParams)
+		dp.ConfigurePrometheusMetrics(configParams)
+		go metricsserver.ServePrometheusMetricsForever(
+			configParams.PrometheusMetricsHost,
+			configParams.PrometheusMetricsPort,
+		)
 	}
 
 	// Register signal handlers to dump memory/CPU profiles.

felix/dataplane/driver.go

@@ -20,25 +20,19 @@ import (
 	"context"
 	"math/bits"
 	"net"
-	"net/http"
 	"os/exec"
 	"runtime/debug"
-	"strconv"
 	"strings"
-	"time"
-
-	coreV1 "k8s.io/api/core/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/utils/clock"
 
+	apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
-
-	apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3"
+	coreV1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/utils/clock"
 
 	"github.com/projectcalico/calico/felix/aws"
 	"github.com/projectcalico/calico/felix/bpf"
@@ -419,11 +413,7 @@ func SupportsBPF() error {
 	return bpf.SupportsBPFDataplane()
 }
 
-func ServePrometheusMetrics(configParams *config.Config) {
-	log.WithFields(log.Fields{
-		"host": configParams.PrometheusMetricsHost,
-		"port": configParams.PrometheusMetricsPort,
-	}).Info("Starting prometheus metrics endpoint")
+func ConfigurePrometheusMetrics(configParams *config.Config) {
 	if configParams.PrometheusGoMetricsEnabled && configParams.PrometheusProcessMetricsEnabled && configParams.PrometheusWireGuardMetricsEnabled {
 		log.Info("Including Golang, Process and WireGuard metrics")
 	} else {
@@ -440,12 +430,4 @@ func ServePrometheusMetrics(configParams *config.Config) {
 			prometheus.Unregister(wireguard.MustNewWireguardMetrics())
 		}
 	}
-	http.Handle("/metrics", promhttp.Handler())
-	addr := net.JoinHostPort(configParams.PrometheusMetricsHost, strconv.Itoa(configParams.PrometheusMetricsPort))
-	for {
-		err := http.ListenAndServe(addr, nil)
-		log.WithError(err).Error(
-			"Prometheus metrics endpoint failed, trying to restart it...")
-		time.Sleep(1 * time.Second)
-	}
 }

felix/dataplane/driver_windows.go

@@ -16,18 +16,12 @@ package dataplane
 
 import (
 	"fmt"
-	"net"
-	"net/http"
 	"os/exec"
-	"strconv"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"k8s.io/client-go/kubernetes"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
+	log "github.com/sirupsen/logrus"
+	"k8s.io/client-go/kubernetes"
 
 	"github.com/projectcalico/calico/felix/config"
 	windataplane "github.com/projectcalico/calico/felix/dataplane/windows"
@@ -62,11 +56,7 @@ func SupportsBPF() error {
 	return fmt.Errorf("BPF dataplane is not supported on Windows")
 }
 
-func ServePrometheusMetrics(configParams *config.Config) {
-	log.WithFields(log.Fields{
-		"host": configParams.PrometheusMetricsHost,
-		"port": configParams.PrometheusMetricsPort,
-	}).Info("Starting prometheus metrics endpoint")
+func ConfigurePrometheusMetrics(configParams *config.Config) {
 	if configParams.PrometheusGoMetricsEnabled && configParams.PrometheusProcessMetricsEnabled {
 		log.Info("Including Golang, and Process metrics")
 	} else {
@@ -79,12 +69,4 @@ func ServePrometheusMetrics(configParams *config.Config) {
 			prometheus.Unregister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
 		}
 	}
-	http.Handle("/metrics", promhttp.Handler())
-	addr := net.JoinHostPort(configParams.PrometheusMetricsHost, strconv.Itoa(configParams.PrometheusMetricsPort))
-	for {
-		err := http.ListenAndServe(addr, nil)
-		log.WithError(err).Error(
-			"Prometheus metrics endpoint failed, trying to restart it...")
-		time.Sleep(1 * time.Second)
-	}
 }

felix/fv/debug_port_test.go

@@ -0,0 +1,91 @@
+// Copyright (c) 2023 Tigera, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build fvtests
+
+package fv_test
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/projectcalico/calico/felix/fv/infrastructure"
+	"github.com/projectcalico/calico/libcalico-go/lib/apiconfig"
+)
+
+var _ = infrastructure.DatastoreDescribe("_BPF-SAFE_ Debug port tests", []apiconfig.DatastoreType{apiconfig.EtcdV3, apiconfig.Kubernetes}, func(getInfra infrastructure.InfraFactory) {
+	const nodeCount = 1
+
+	var (
+		infra           infrastructure.DatastoreInfra
+		topologyOptions infrastructure.TopologyOptions
+		tc              infrastructure.TopologyContainers
+	)
+
+	BeforeEach(func() {
+		infra = getInfra()
+		topologyOptions = infrastructure.DefaultTopologyOptions()
+	})
+
+	JustBeforeEach(func() {
+		tc, _ = infrastructure.StartNNodeTopology(nodeCount, topologyOptions, infra)
+	})
+
+	get := func(path string) error {
+		debugServer := fmt.Sprintf("http://%s:%d/", tc.Felixes[0].IP, 6061)
+		httpClient := http.Client{
+			Timeout: 2 * time.Second,
+		}
+		resp, err := httpClient.Get(debugServer + path)
+		if err != nil {
+			return err
+		}
+		defer func() { _ = resp.Body.Close() }()
+		if resp.StatusCode != 200 {
+			return fmt.Errorf("bad status code: %d", resp.StatusCode)
+		}
+		return nil
+	}
+
+	getFn := func(path string) func() error {
+		return func() error {
+			return get(path)
+		}
+	}
+
+	Describe("with debug port enabled on 0.0.0.0", func() {
+		BeforeEach(func() {
+			topologyOptions.ExtraEnvVars["FELIX_DebugPort"] = "6061"
+			topologyOptions.ExtraEnvVars["FELIX_DebugHost"] = "0.0.0.0"
+		})
+
+		It("should serve expected URLs", func() {
+			Eventually(getFn("debug/pprof/profile?seconds=1")).ShouldNot(HaveOccurred())
+			Expect(get("debug/pprof/heap")).NotTo(HaveOccurred())
+			Expect(get("metrics")).To(HaveOccurred(), "Metrics on the debug port?")
+		})
+
+		AfterEach(func() {
+			tc.Stop()
+			if CurrentGinkgoTestDescription().Failed {
+				infra.DumpErrorData()
+			}
+			infra.Stop()
+		})
+	})
+})

felix/fv/infrastructure/felix.go

@@ -30,8 +30,8 @@ import (
 
 	"github.com/projectcalico/calico/felix/bpf/jump"
 	"github.com/projectcalico/calico/felix/bpf/polprog"
-
 	"github.com/projectcalico/calico/felix/fv/containers"
+	"github.com/projectcalico/calico/felix/fv/metrics"
 	"github.com/projectcalico/calico/felix/fv/tcpdump"
 	"github.com/projectcalico/calico/felix/fv/utils"
 )
@@ -483,3 +483,35 @@ func (f *Felix) IPTablesChains(table string) map[string][]string {
 	}
 	return out
 }
+
+func (f *Felix) PromMetric(name string) PrometheusMetric {
+	return PrometheusMetric{
+		f:    f,
+		Name: name,
+	}
+}
+
+type PrometheusMetric struct {
+	f    *Felix
+	Name string
+}
+
+func (p PrometheusMetric) Raw() (string, error) {
+	return metrics.GetFelixMetric(p.f.IP, p.Name)
+}
+
+func (p PrometheusMetric) Int() (int, error) {
+	raw, err := p.Raw()
+	if err != nil {
+		return 0, err
+	}
+	return strconv.Atoi(raw)
+}
+
+func (p PrometheusMetric) Float() (float64, error) {
+	raw, err := p.Raw()
+	if err != nil {
+		return 0, err
+	}
+	return strconv.ParseFloat(raw, 64)
+}