From c344db029eb061767b0cf82dd3339dc741bbb3f2 Mon Sep 17 00:00:00 2001 From: ccfishk Date: Thu, 8 Jul 2021 14:30:53 -0700 Subject: [PATCH 1/2] prometheus setup for controller --- internal/prometheus/clusterRole.yaml | 33 ++++ internal/prometheus/config-map.yaml | 152 ++++++++++++++++++ .../prometheus/prometheus-deployment.yaml | 38 +++++ internal/prometheus/prometheus-gateway.yaml | 35 ++++ internal/prometheus/prometheus-service.yaml | 16 ++ 5 files changed, 274 insertions(+) create mode 100644 internal/prometheus/clusterRole.yaml create mode 100644 internal/prometheus/config-map.yaml create mode 100644 internal/prometheus/prometheus-deployment.yaml create mode 100644 internal/prometheus/prometheus-gateway.yaml create mode 100644 internal/prometheus/prometheus-service.yaml diff --git a/internal/prometheus/clusterRole.yaml b/internal/prometheus/clusterRole.yaml new file mode 100644 index 0000000000..96a7a8d90e --- /dev/null +++ b/internal/prometheus/clusterRole.yaml @@ -0,0 +1,33 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - extensions + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: default + namespace: monitoring \ No newline at end of file diff --git a/internal/prometheus/config-map.yaml b/internal/prometheus/config-map.yaml new file mode 100644 index 0000000000..746f6c8140 --- /dev/null +++ b/internal/prometheus/config-map.yaml @@ -0,0 +1,152 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-server-conf + labels: + name: prometheus-server-conf + namespace: monitoring +data: + prometheus.rules: |- + groups: + - name: devopscube demo alert + rules: + - alert: High Pod Memory + expr: sum(container_memory_usage_bytes) > 1 + for: 1m + labels: + severity: slack + annotations: + summary: High Memory Usage + prometheus.yml: |- + global: + scrape_interval: 5s + evaluation_interval: 5s + rule_files: + - /etc/prometheus/prometheus.rules + alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: + - "alertmanager.monitoring.svc:9093" + + scrape_configs: + - job_name: 'kubernetes-apiservers' + + kubernetes_sd_configs: + - role: endpoints + scheme: https + + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + relabel_configs: + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: default;kubernetes;https + + - job_name: 'kubernetes-nodes' + + scheme: https + + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + kubernetes_sd_configs: + - role: node + + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics + + + - job_name: 'kubernetes-pods' + + kubernetes_sd_configs: + - role: pod + + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + + - job_name: 'kube-state-metrics' + static_configs: + - targets: ['kube-state-metrics.kube-system.svc.cluster.local:8080'] + + - job_name: 'kubernetes-cadvisor' + + scheme: https + + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + kubernetes_sd_configs: + - role: node + + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + + - job_name: 'kubernetes-service-endpoints' + + kubernetes_sd_configs: + - role: endpoints + + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name \ No newline at end of file diff --git a/internal/prometheus/prometheus-deployment.yaml b/internal/prometheus/prometheus-deployment.yaml new file mode 100644 index 0000000000..2ccb852098 --- /dev/null +++ b/internal/prometheus/prometheus-deployment.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-deployment + namespace: monitoring + labels: + app: prometheus-server +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus-server + template: + metadata: + labels: + app: prometheus-server + spec: + containers: + - name: prometheus + image: prom/prometheus + args: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus/" + ports: + - containerPort: 9090 + volumeMounts: + - name: prometheus-config-volume + mountPath: /etc/prometheus/ + - name: prometheus-storage-volume + mountPath: /prometheus/ + volumes: + - name: prometheus-config-volume + configMap: + defaultMode: 420 + name: prometheus-server-conf + + - name: prometheus-storage-volume + emptyDir: {} diff --git a/internal/prometheus/prometheus-gateway.yaml b/internal/prometheus/prometheus-gateway.yaml new file mode 100644 index 0000000000..eaafbde305 --- /dev/null +++ b/internal/prometheus/prometheus-gateway.yaml @@ -0,0 +1,35 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: prometheus-gateway + namespace: monitoring +spec: + selector: + istio: ingressgateway # use istio default controller + servers: + - port: + number: 80 + name: http + protocol: http + hosts: + - "*" +--- +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: prometheus + namespace: monitoring +spec: + hosts: + - "*" + gateways: + - prometheus-gateway + http: + - match: + - uri: + prefix: / + route: + - destination: + host: prometheus-service + port: + number: 9090 diff --git a/internal/prometheus/prometheus-service.yaml b/internal/prometheus/prometheus-service.yaml new file mode 100644 index 0000000000..36771f9fbb --- /dev/null +++ b/internal/prometheus/prometheus-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus-service + namespace: monitoring + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '9090' + +spec: + selector: + app: prometheus-server + ports: + - port: 9090 + protocol: TCP + targetPort: 9090 From d424017c6f1221c2e69212735dbc79ac1bc33a69 Mon Sep 17 00:00:00 2001 From: ccfishk Date: Wed, 14 Jul 2021 11:51:56 -0700 Subject: [PATCH 2/2] Add prometheus library and remove yaml for less dependency --- go.mod | 1 + internal/ctrlutils/utils.go | 6 ++ internal/manager/prometheus.go | 103 +++++++++++++++++++++++++++++++++ internal/manager/run.go | 2 + internal/mgrutils/reports.go | 18 ++++++ 5 files changed, 130 insertions(+) create mode 100644 internal/manager/prometheus.go diff --git a/go.mod b/go.mod index 4ae0040efa..e9390f186f 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/lithammer/dedent v1.1.0 github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.4.1 + github.com/prometheus/client_golang v1.11.0 github.com/prometheus/common v0.26.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.2.1 diff --git a/internal/ctrlutils/utils.go b/internal/ctrlutils/utils.go index 4c1ed19a89..5923081e6c 100644 --- a/internal/ctrlutils/utils.go +++ b/internal/ctrlutils/utils.go @@ -12,6 +12,12 @@ import ( "github.com/kong/kubernetes-ingress-controller/internal/annotations" ) +const ( + // classSpec indicates the fieldName for objects which support indicating their Ingress Class by spec + classSpec = "IngressClassName" + PROMTHPORT = 10257 +) + // HasAnnotation is a helper function to determine whether an object has a given annotation, and whether it's // to the value provided. func HasAnnotation(obj client.Object, key, expectedValue string) bool { diff --git a/internal/manager/prometheus.go b/internal/manager/prometheus.go new file mode 100644 index 0000000000..f99c6a4cf1 --- /dev/null +++ b/internal/manager/prometheus.go @@ -0,0 +1,103 @@ +package manager + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/pprof" + "sync" + "syscall" + "time" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/kong/kubernetes-ingress-controller/internal/ctrlutils" + "github.com/kong/kubernetes-ingress-controller/internal/mgrutils" +) + +func RunHTTP(log logr.Logger) { + stopCh := make(chan struct{}) + var wg sync.WaitGroup + mux := http.NewServeMux() + wg.Add(1) + go func() { + defer wg.Done() + ServeHTTP(true, + ctrlutils.PROMTHPORT, mux, stopCh, + log, + &wg) + }() +} + +// ServeHTTP enable HTTP Server +// - prometheus +func ServeHTTP(enableProfiling bool, + port int, + mux *http.ServeMux, + stop <-chan struct{}, + logger logr.Logger, + wg *sync.WaitGroup) { + defer wg.Done() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + mux.Handle("/metrics", promhttp.Handler()) + + mux.HandleFunc("/build", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + b, _ := json.Marshal(mgrutils.Version("UNKNOWN", "UNKNOWN", mgrutils.KICREPO)) + if _, err := w.Write(b); err != nil { + logger.Error(err, " endpoint /build failed to write response.") + } + }) + + mux.HandleFunc("/stop", func(w http.ResponseWriter, r *http.Request) { + err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM) + if err != nil { + logger.Error(err, "failed to send SIGTERM to self.") + } + }) + + if enableProfiling { + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/heap", pprof.Index) + mux.HandleFunc("/debug/pprof/mutex", pprof.Index) + mux.HandleFunc("/debug/pprof/goroutine", pprof.Index) + mux.HandleFunc("/debug/pprof/threadcreate", pprof.Index) + mux.HandleFunc("/debug/pprof/block", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + } + + server := &http.Server{ + Addr: fmt.Sprintf(":%v", port), + Handler: mux, + ReadTimeout: 10 * time.Second, + ReadHeaderTimeout: 10 * time.Second, + WriteTimeout: 300 * time.Second, + IdleTimeout: 120 * time.Second, + } + serveDone := make(chan struct{}) + var wglocal sync.WaitGroup + wglocal.Add(1) + go func() { + defer wglocal.Done() + select { + case <-stop: + if err := server.Shutdown(context.Background()); err != nil { + logger.Error(err, "failed to shut down server.") + } + case <-serveDone: + } + }() + if err := server.ListenAndServe(); err != http.ErrServerClosed { + logger.Error(err, "server stopped with err.") + close(serveDone) + } + wg.Wait() +} diff --git a/internal/manager/run.go b/internal/manager/run.go index 375f301283..2ca00fb627 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -104,6 +104,8 @@ func Run(ctx context.Context, c *Config, diagnostic util.ConfigDumpDiagnostic) e setupLog.Info("WARNING: status updates were disabled, resources like Ingress objects will not receive updates to their statuses.") } + go RunHTTP(setupLog) + setupLog.Info("starting manager") return mgr.Start(ctx) } diff --git a/internal/mgrutils/reports.go b/internal/mgrutils/reports.go index dc2df89c41..be90ad0faa 100644 --- a/internal/mgrutils/reports.go +++ b/internal/mgrutils/reports.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "runtime" "github.com/hashicorp/go-uuid" "k8s.io/client-go/kubernetes" @@ -13,6 +14,10 @@ import ( "github.com/kong/kubernetes-ingress-controller/internal/util" ) +const ( + KICREPO = "github.com/kong/kubernetes-ingress-controller" +) + // RunReport runs the anonymous data report and reports any errors that have occurred. func RunReport(ctx context.Context, kubeCfg *rest.Config, kongCfg sendconfig.Kong, kicVersion string) error { // if anonymous reports are enabled this helps provide Kong with insights about usage of the ingress controller @@ -77,3 +82,16 @@ func RunReport(ctx context.Context, kubeCfg *rest.Config, kongCfg sendconfig.Kon return nil } + +// Version returns information about the release. +// long term cli/ingress-controller/version.go +func Version(RELEASE, COMMIT, REPO interface{}) string { + return fmt.Sprintf(`------------------------------------------------------------------------------- +Kong Ingress controller + Release: %v + Build: %v + Repository: %v + Go: %v +------------------------------------------------------------------------------- +`, RELEASE, COMMIT, REPO, runtime.Version()) +}