From b84eb71291f88e848cda4533173873b06ee2dc0a Mon Sep 17 00:00:00 2001 From: Lynn Date: Fri, 6 Mar 2020 17:09:58 +0800 Subject: [PATCH] server: if status address already in use, return an error --- server/http_status.go | 62 ++++++++++++++++++++++--------------------- server/server.go | 5 ++++ server/tidb_test.go | 22 +++++++++++++++ 3 files changed, 59 insertions(+), 30 deletions(-) diff --git a/server/http_status.go b/server/http_status.go index 30022922251cd..deaf1c5303174 100644 --- a/server/http_status.go +++ b/server/http_status.go @@ -69,6 +69,32 @@ func sleepWithCtx(ctx context.Context, d time.Duration) { } } +func (s *Server) listenStatusHTTPServer() error { + s.statusAddr = fmt.Sprintf("%s:%d", s.cfg.Status.StatusHost, s.cfg.Status.StatusPort) + if s.cfg.Status.StatusPort == 0 { + s.statusAddr = fmt.Sprintf("%s:%d", s.cfg.Status.StatusHost, defaultStatusPort) + } + + logutil.BgLogger().Info("for status and metrics report", zap.String("listening on addr", s.statusAddr)) + tlsConfig, err := s.cfg.Security.ToTLSConfig() + if err != nil { + logutil.BgLogger().Error("invalid TLS config", zap.Error(err)) + return errors.Trace(err) + } + + if tlsConfig != nil { + // we need to manage TLS here for cmux to distinguish between HTTP and gRPC. + s.statusListener, err = tls.Listen("tcp", s.statusAddr, tlsConfig) + } else { + s.statusListener, err = net.Listen("tcp", s.statusAddr) + } + if err != nil { + logutil.BgLogger().Info("listen failed", zap.Error(err)) + return errors.Trace(err) + } + return nil +} + func (s *Server) startHTTPServer() { router := mux.NewRouter() @@ -121,13 +147,8 @@ func (s *Server) startHTTPServer() { router.Handle("/mvcc/hex/{hexKey}", mvccTxnHandler{tikvHandlerTool, opMvccGetByHex}) router.Handle("/mvcc/index/{db}/{table}/{index}/{handle}", mvccTxnHandler{tikvHandlerTool, opMvccGetByIdx}) - addr := fmt.Sprintf("%s:%d", s.cfg.Status.StatusHost, s.cfg.Status.StatusPort) - if s.cfg.Status.StatusPort == 0 { - addr = fmt.Sprintf("%s:%d", s.cfg.Status.StatusHost, defaultStatusPort) - } - // HTTP path for web UI. - if host, port, err := net.SplitHostPort(addr); err == nil { + if host, port, err := net.SplitHostPort(s.statusAddr); err == nil { if host == "" { host = "localhost" } @@ -262,36 +283,17 @@ func (s *Server) startHTTPServer() { logutil.BgLogger().Error("write HTTP index page failed", zap.Error(err)) } }) - - logutil.BgLogger().Info("for status and metrics report", zap.String("listening on addr", addr)) - s.setupStatusServerAndRPCServer(addr, serverMux) + s.startStatusServerAndRPCServer(serverMux) } -func (s *Server) setupStatusServerAndRPCServer(addr string, serverMux *http.ServeMux) { - tlsConfig, err := s.cfg.Security.ToTLSConfig() - if err != nil { - logutil.BgLogger().Error("invalid TLS config", zap.Error(err)) - return - } - - var l net.Listener - if tlsConfig != nil { - // we need to manage TLS here for cmux to distinguish between HTTP and gRPC. - l, err = tls.Listen("tcp", addr, tlsConfig) - } else { - l, err = net.Listen("tcp", addr) - } - if err != nil { - logutil.BgLogger().Info("listen failed", zap.Error(err)) - return - } - m := cmux.New(l) +func (s *Server) startStatusServerAndRPCServer(serverMux *http.ServeMux) { + m := cmux.New(s.statusListener) // Match connections in order: // First HTTP, and otherwise grpc. httpL := m.Match(cmux.HTTP1Fast()) grpcL := m.Match(cmux.Any()) - s.statusServer = &http.Server{Addr: addr, Handler: CorsHandler{handler: serverMux, cfg: s.cfg}} + s.statusServer = &http.Server{Addr: s.statusAddr, Handler: CorsHandler{handler: serverMux, cfg: s.cfg}} s.grpcServer = NewRPCServer(s.cfg, s.dom, s) go util.WithRecovery(func() { @@ -304,7 +306,7 @@ func (s *Server) setupStatusServerAndRPCServer(addr string, serverMux *http.Serv logutil.BgLogger().Error("http server error", zap.Error(err)) }, nil) - err = m.Serve() + err := m.Serve() if err != nil { logutil.BgLogger().Error("start status/rpc server error", zap.Error(err)) } diff --git a/server/server.go b/server/server.go index 69d3c161ba460..9052307268a11 100644 --- a/server/server.go +++ b/server/server.go @@ -117,6 +117,8 @@ type Server struct { // a supervisor automatically restart it, then new client connection will be created, but we can't server it. // So we just stop the listener and store to force clients to chose other TiDB servers. stopListenerCh chan struct{} + statusAddr string + statusListener net.Listener statusServer *http.Server grpcServer *grpc.Server } @@ -255,6 +257,9 @@ func NewServer(cfg *config.Config, driver IDriver) (*Server, error) { s.listener = pplistener } + if s.cfg.Status.ReportStatus && err == nil { + err = s.listenStatusHTTPServer() + } if err != nil { return nil, errors.Trace(err) } diff --git a/server/tidb_test.go b/server/tidb_test.go index 86a2d4c28f3f8..4c7540d6d2c58 100644 --- a/server/tidb_test.go +++ b/server/tidb_test.go @@ -22,6 +22,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "fmt" "io/ioutil" "math/big" "os" @@ -178,6 +179,27 @@ func (ts *tidbTestSuite) TestStatusAPI(c *C) { ts.runTestStatusAPI(c) } +func (ts *tidbTestSuite) TestStatusPort(c *C) { + var err error + ts.store, err = mockstore.NewMockTikvStore() + session.DisableStats4Test() + c.Assert(err, IsNil) + ts.domain, err = session.BootstrapSession(ts.store) + c.Assert(err, IsNil) + ts.tidbdrv = NewTiDBDriver(ts.store) + cfg := config.NewConfig() + cfg.Port = genPort() + cfg.Status.ReportStatus = true + cfg.Status.StatusPort = ts.statusPort + cfg.Performance.TCPKeepAlive = true + + server, err := NewServer(cfg, ts.tidbdrv) + c.Assert(err, NotNil) + c.Assert(err.Error(), Equals, + fmt.Sprintf("listen tcp 0.0.0.0:%d: bind: address already in use", ts.statusPort)) + c.Assert(server, IsNil) +} + func (ts *tidbTestSuite) TestStatusAPIWithTLS(c *C) { caCert, caKey, err := generateCert(0, "TiDB CA 2", nil, nil, "/tmp/ca-key-2.pem", "/tmp/ca-cert-2.pem") c.Assert(err, IsNil)