Merge branch 'controller-prometheus' of https://github.com/Kong/kuber…

…netes-ingress-controller into controller-prometheus

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/lithammer/dedent v1.1.0
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.1
+	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/common v0.26.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.2.1

internal/ctrlutils/utils.go

@@ -12,6 +12,12 @@ import (
 	"github.com/kong/kubernetes-ingress-controller/internal/annotations"
 )
 
+const (
+	// classSpec indicates the fieldName for objects which support indicating their Ingress Class by spec
+	classSpec  = "IngressClassName"
+	PROMTHPORT = 10257
+)
+
 // HasAnnotation is a helper function to determine whether an object has a given annotation, and whether it's
 // to the value provided.
 func HasAnnotation(obj client.Object, key, expectedValue string) bool {

internal/manager/prometheus.go

@@ -0,0 +1,103 @@
+package manager
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/pprof"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/go-logr/logr"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"github.com/kong/kubernetes-ingress-controller/internal/ctrlutils"
+	"github.com/kong/kubernetes-ingress-controller/internal/mgrutils"
+)
+
+func RunHTTP(log logr.Logger) {
+	stopCh := make(chan struct{})
+	var wg sync.WaitGroup
+	mux := http.NewServeMux()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		ServeHTTP(true,
+			ctrlutils.PROMTHPORT, mux, stopCh,
+			log,
+			&wg)
+	}()
+}
+
+// ServeHTTP enable HTTP Server
+// - prometheus
+func ServeHTTP(enableProfiling bool,
+	port int,
+	mux *http.ServeMux,
+	stop <-chan struct{},
+	logger logr.Logger,
+	wg *sync.WaitGroup) {
+	defer wg.Done()
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	})
+
+	mux.Handle("/metrics", promhttp.Handler())
+
+	mux.HandleFunc("/build", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		b, _ := json.Marshal(mgrutils.Version("UNKNOWN", "UNKNOWN", mgrutils.KICREPO))
+		if _, err := w.Write(b); err != nil {
+			logger.Error(err, " endpoint /build failed to write response.")
+		}
+	})
+
+	mux.HandleFunc("/stop", func(w http.ResponseWriter, r *http.Request) {
+		err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
+		if err != nil {
+			logger.Error(err, "failed to send SIGTERM to self.")
+		}
+	})
+
+	if enableProfiling {
+		mux.HandleFunc("/debug/pprof/", pprof.Index)
+		mux.HandleFunc("/debug/pprof/heap", pprof.Index)
+		mux.HandleFunc("/debug/pprof/mutex", pprof.Index)
+		mux.HandleFunc("/debug/pprof/goroutine", pprof.Index)
+		mux.HandleFunc("/debug/pprof/threadcreate", pprof.Index)
+		mux.HandleFunc("/debug/pprof/block", pprof.Index)
+		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+	}
+
+	server := &http.Server{
+		Addr:              fmt.Sprintf(":%v", port),
+		Handler:           mux,
+		ReadTimeout:       10 * time.Second,
+		ReadHeaderTimeout: 10 * time.Second,
+		WriteTimeout:      300 * time.Second,
+		IdleTimeout:       120 * time.Second,
+	}
+	serveDone := make(chan struct{})
+	var wglocal sync.WaitGroup
+	wglocal.Add(1)
+	go func() {
+		defer wglocal.Done()
+		select {
+		case <-stop:
+			if err := server.Shutdown(context.Background()); err != nil {
+				logger.Error(err, "failed to shut down server.")
+			}
+		case <-serveDone:
+		}
+	}()
+	if err := server.ListenAndServe(); err != http.ErrServerClosed {
+		logger.Error(err, "server stopped with err.")
+		close(serveDone)
+	}
+	wg.Wait()
+}

internal/manager/run.go

@@ -104,6 +104,8 @@ func Run(ctx context.Context, c *Config, diagnostic util.ConfigDumpDiagnostic) e
 		setupLog.Info("WARNING: status updates were disabled, resources like Ingress objects will not receive updates to their statuses.")
 	}
 
+	go RunHTTP(setupLog)
+
 	setupLog.Info("starting manager")
 	return mgr.Start(ctx)
 }

internal/mgrutils/reports.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"runtime"
 
 	"github.com/hashicorp/go-uuid"
 	"k8s.io/client-go/kubernetes"
@@ -13,6 +14,10 @@ import (
 	"github.com/kong/kubernetes-ingress-controller/internal/util"
 )
 
+const (
+	KICREPO = "github.com/kong/kubernetes-ingress-controller"
+)
+
 // RunReport runs the anonymous data report and reports any errors that have occurred.
 func RunReport(ctx context.Context, kubeCfg *rest.Config, kongCfg sendconfig.Kong, kicVersion string) error {
 	// if anonymous reports are enabled this helps provide Kong with insights about usage of the ingress controller
@@ -77,3 +82,16 @@ func RunReport(ctx context.Context, kubeCfg *rest.Config, kongCfg sendconfig.Kon
 
 	return nil
 }
+
+// Version returns information about the release.
+// long term cli/ingress-controller/version.go
+func Version(RELEASE, COMMIT, REPO interface{}) string {
+	return fmt.Sprintf(`-------------------------------------------------------------------------------
+Kong Ingress controller
+  Release:    %v
+  Build:      %v
+  Repository: %v
+  Go:         %v
+-------------------------------------------------------------------------------
+`, RELEASE, COMMIT, REPO, runtime.Version())
+}

internal/prometheus/clusterRole.yaml

@@ -0,0 +1,33 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: prometheus
+rules:
+- apiGroups: [""]
+  resources:
+  - nodes
+  - nodes/proxy
+  - services
+  - endpoints
+  - pods
+  verbs: ["get", "list", "watch"]
+- apiGroups:
+  - extensions
+  resources:
+  - ingresses
+  verbs: ["get", "list", "watch"]
+- nonResourceURLs: ["/metrics"]
+  verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: monitoring

internal/prometheus/config-map.yaml

@@ -0,0 +1,152 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-server-conf
+  labels:
+    name: prometheus-server-conf
+  namespace: monitoring
+data:
+  prometheus.rules: |-
+    groups:
+    - name: devopscube demo alert
+      rules:
+      - alert: High Pod Memory
+        expr: sum(container_memory_usage_bytes) > 1
+        for: 1m
+        labels:
+          severity: slack
+        annotations:
+          summary: High Memory Usage
+  prometheus.yml: |-
+    global:
+      scrape_interval: 5s
+      evaluation_interval: 5s
+    rule_files:
+      - /etc/prometheus/prometheus.rules
+    alerting:
+      alertmanagers:
+      - scheme: http
+        static_configs:
+        - targets:
+          - "alertmanager.monitoring.svc:9093"
+
+    scrape_configs:
+      - job_name: 'kubernetes-apiservers'
+
+        kubernetes_sd_configs:
+        - role: endpoints
+        scheme: https
+
+        tls_config:
+          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+
+        relabel_configs:
+        - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
+          action: keep
+          regex: default;kubernetes;https
+
+      - job_name: 'kubernetes-nodes'
+
+        scheme: https
+
+        tls_config:
+          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+
+        kubernetes_sd_configs:
+        - role: node
+
+        relabel_configs:
+        - action: labelmap
+          regex: __meta_kubernetes_node_label_(.+)
+        - target_label: __address__
+          replacement: kubernetes.default.svc:443
+        - source_labels: [__meta_kubernetes_node_name]
+          regex: (.+)
+          target_label: __metrics_path__
+          replacement: /api/v1/nodes/${1}/proxy/metrics
+
+      
+      - job_name: 'kubernetes-pods'
+
+        kubernetes_sd_configs:
+        - role: pod
+
+        relabel_configs:
+        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+          action: keep
+          regex: true
+        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+          action: replace
+          target_label: __metrics_path__
+          regex: (.+)
+        - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+          action: replace
+          regex: ([^:]+)(?::\d+)?;(\d+)
+          replacement: $1:$2
+          target_label: __address__
+        - action: labelmap
+          regex: __meta_kubernetes_pod_label_(.+)
+        - source_labels: [__meta_kubernetes_namespace]
+          action: replace
+          target_label: kubernetes_namespace
+        - source_labels: [__meta_kubernetes_pod_name]
+          action: replace
+          target_label: kubernetes_pod_name
+      
+      - job_name: 'kube-state-metrics'
+        static_configs:
+          - targets: ['kube-state-metrics.kube-system.svc.cluster.local:8080']
+
+      - job_name: 'kubernetes-cadvisor'
+
+        scheme: https
+
+        tls_config:
+          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+
+        kubernetes_sd_configs:
+        - role: node
+
+        relabel_configs:
+        - action: labelmap
+          regex: __meta_kubernetes_node_label_(.+)
+        - target_label: __address__
+          replacement: kubernetes.default.svc:443
+        - source_labels: [__meta_kubernetes_node_name]
+          regex: (.+)
+          target_label: __metrics_path__
+          replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
+      
+      - job_name: 'kubernetes-service-endpoints'
+
+        kubernetes_sd_configs:
+        - role: endpoints
+
+        relabel_configs:
+        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+          action: keep
+          regex: true
+        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
+          action: replace
+          target_label: __scheme__
+          regex: (https?)
+        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
+          action: replace
+          target_label: __metrics_path__
+          regex: (.+)
+        - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
+          action: replace
+          target_label: __address__
+          regex: ([^:]+)(?::\d+)?;(\d+)
+          replacement: $1:$2
+        - action: labelmap
+          regex: __meta_kubernetes_service_label_(.+)
+        - source_labels: [__meta_kubernetes_namespace]
+          action: replace
+          target_label: kubernetes_namespace
+        - source_labels: [__meta_kubernetes_service_name]
+          action: replace
+          target_label: kubernetes_name

internal/prometheus/prometheus-deployment.yaml

@@ -0,0 +1,38 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus-deployment
+  namespace: monitoring
+  labels:
+    app: prometheus-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: prometheus-server
+  template:
+    metadata:
+      labels:
+        app: prometheus-server
+    spec:
+      containers:
+        - name: prometheus
+          image: prom/prometheus
+          args:
+            - "--config.file=/etc/prometheus/prometheus.yml"
+            - "--storage.tsdb.path=/prometheus/"
+          ports:
+            - containerPort: 9090
+          volumeMounts:
+            - name: prometheus-config-volume
+              mountPath: /etc/prometheus/
+            - name: prometheus-storage-volume
+              mountPath: /prometheus/
+      volumes:
+        - name: prometheus-config-volume
+          configMap:
+            defaultMode: 420
+            name: prometheus-server-conf
+
+        - name: prometheus-storage-volume
+          emptyDir: {}