From cd37ef2c1b978e7640eee07ed614978af3ba3208 Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Fri, 28 Jul 2017 15:23:40 -0700 Subject: [PATCH 1/3] *: expose etcdhttp.Health, define proxy health handler Signed-off-by: Gyu-Ho Lee --- etcdserver/api/etcdhttp/metrics.go | 19 ++++++------- proxy/grpcproxy/health.go | 43 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 proxy/grpcproxy/health.go diff --git a/etcdserver/api/etcdhttp/metrics.go b/etcdserver/api/etcdhttp/metrics.go index 03ae616422d..81b63605618 100644 --- a/etcdserver/api/etcdhttp/metrics.go +++ b/etcdserver/api/etcdhttp/metrics.go @@ -29,13 +29,13 @@ import ( const ( pathMetrics = "/metrics" - pathHealth = "/health" + PathHealth = "/health" ) // HandleMetricsHealth registers metrics and health handlers. func HandleMetricsHealth(mux *http.ServeMux, srv *etcdserver.EtcdServer) { mux.Handle(pathMetrics, prometheus.Handler()) - mux.Handle(pathHealth, newHealthHandler(srv)) + mux.Handle(PathHealth, NewHealthHandler(func() Health { return checkHealth(srv) })) } // HandlePrometheus registers prometheus handler on '/metrics'. @@ -45,18 +45,18 @@ func HandlePrometheus(mux *http.ServeMux) { // HandleHealth registers health handler on '/health'. func HandleHealth(mux *http.ServeMux, srv *etcdserver.EtcdServer) { - mux.Handle(pathHealth, newHealthHandler(srv)) + mux.Handle(PathHealth, NewHealthHandler(func() Health { return checkHealth(srv) })) } -// newHealthHandler handles '/health' requests. -func newHealthHandler(srv *etcdserver.EtcdServer) http.HandlerFunc { +// NewHealthHandler handles '/health' requests. +func NewHealthHandler(hfunc func() Health) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { w.Header().Set("Allow", http.MethodGet) http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) return } - h := checkHealth(srv) + h := hfunc() d, _ := json.Marshal(h) if !h.Health { http.Error(w, string(d), http.StatusServiceUnavailable) @@ -67,14 +67,15 @@ func newHealthHandler(srv *etcdserver.EtcdServer) http.HandlerFunc { } } +// Health defines etcd server health status. // TODO: remove manual parsing in etcdctl cluster-health -type health struct { +type Health struct { Health bool `json:"health"` Errors []string `json:"errors,omitempty"` } -func checkHealth(srv *etcdserver.EtcdServer) health { - h := health{Health: false} +func checkHealth(srv *etcdserver.EtcdServer) Health { + h := Health{Health: false} as := srv.Alarms() if len(as) > 0 { diff --git a/proxy/grpcproxy/health.go b/proxy/grpcproxy/health.go new file mode 100644 index 00000000000..e660800e907 --- /dev/null +++ b/proxy/grpcproxy/health.go @@ -0,0 +1,43 @@ +// Copyright 2017 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpcproxy + +import ( + "net/http" + "time" + + "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/etcdserver/api/etcdhttp" + "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" + + "golang.org/x/net/context" +) + +// HandleHealth registers health handler on '/health'. +func HandleHealth(mux *http.ServeMux, c *clientv3.Client) { + mux.Handle(etcdhttp.PathHealth, etcdhttp.NewHealthHandler(func() etcdhttp.Health { return checkHealth(c) })) +} + +func checkHealth(c *clientv3.Client) etcdhttp.Health { + h := etcdhttp.Health{Health: false} + ctx, cancel := context.WithTimeout(c.Ctx(), time.Second) + _, err := c.Get(ctx, "a") + cancel() + h.Health = err == nil || err == rpctypes.ErrPermissionDenied + if !h.Health { + h.Errors = append(h.Errors, err.Error()) + } + return h +} From b8fd5c3dba14ab840e1db9eaf77ac82b4e5da785 Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Fri, 28 Jul 2017 15:24:17 -0700 Subject: [PATCH 2/3] etcdmain: add '/health' endpoint to grpc-proxy Signed-off-by: Gyu-Ho Lee --- etcdmain/grpc_proxy.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/etcdmain/grpc_proxy.go b/etcdmain/grpc_proxy.go index bee6755ff64..969dd83ed51 100644 --- a/etcdmain/grpc_proxy.go +++ b/etcdmain/grpc_proxy.go @@ -154,7 +154,7 @@ func startGRPCProxy(cmd *cobra.Command, args []string) { client := mustNewClient() - srvhttp, httpl := mustHTTPListener(m, tlsinfo) + srvhttp, httpl := mustHTTPListener(m, tlsinfo, client) errc := make(chan error) go func() { errc <- newGRPCProxyServer(client).Serve(grpcl) }() go func() { errc <- srvhttp.Serve(httpl) }() @@ -164,6 +164,7 @@ func startGRPCProxy(cmd *cobra.Command, args []string) { go func() { mux := http.NewServeMux() etcdhttp.HandlePrometheus(mux) + grpcproxy.HandleHealth(mux, client) plog.Fatal(http.Serve(mhttpl, mux)) }() } @@ -310,10 +311,11 @@ func newGRPCProxyServer(client *clientv3.Client) *grpc.Server { return server } -func mustHTTPListener(m cmux.CMux, tlsinfo *transport.TLSInfo) (*http.Server, net.Listener) { +func mustHTTPListener(m cmux.CMux, tlsinfo *transport.TLSInfo, c *clientv3.Client) (*http.Server, net.Listener) { httpmux := http.NewServeMux() httpmux.HandleFunc("/", http.NotFound) etcdhttp.HandlePrometheus(httpmux) + grpcproxy.HandleHealth(httpmux, c) if grpcProxyEnablePprof { for p, h := range debugutil.PProfHandlers() { httpmux.Handle(p, h) From 661da1e609b449b713631d041b6884c63776ed29 Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Fri, 28 Jul 2017 15:24:28 -0700 Subject: [PATCH 3/3] e2e: test /metrics, /health endpoint in grpc-proxy Signed-off-by: Gyu-Ho Lee --- e2e/metrics_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/e2e/metrics_test.go b/e2e/metrics_test.go index 48d5edec2fc..d5f13c6804d 100644 --- a/e2e/metrics_test.go +++ b/e2e/metrics_test.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build !cluster_proxy - package e2e import (