stability: add fault-trigger server (#312)

* test: add fault-trigger server

tests/cmd/fault-trigger/main.go

@@ -0,0 +1,55 @@
+// Copyright 2019 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	_ "net/http/pprof"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/pingcap/tidb-operator/tests/pkg/fault-trigger/api"
+	"github.com/pingcap/tidb-operator/tests/pkg/fault-trigger/manager"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/util/logs"
+)
+
+var (
+	port      int
+	pprofPort int
+)
+
+func init() {
+	flag.IntVar(&port, "port", 23332, "The port that the fault trigger's http service runs on (default 23332)")
+	flag.IntVar(&pprofPort, "pprof-port", 6060, "The port that the pprof's http service runs on (default 6060)")
+
+	flag.Parse()
+}
+
+func main() {
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	mgr := manager.NewManager()
+
+	server := api.NewServer(mgr, port)
+
+	go wait.Forever(func() {
+		server.StartServer()
+	}, 5*time.Second)
+
+	glog.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", pprofPort), nil))
+}

tests/pkg/fault-trigger/api/response.go

@@ -0,0 +1,43 @@
+// copyright 2019 pingcap, inc.
+//
+// licensed under the apache license, version 2.0 (the "license");
+// you may not use this file except in compliance with the license.
+// you may obtain a copy of the license at
+//
+//     http://www.apache.org/licenses/license-2.0
+//
+// unless required by applicable law or agreed to in writing, software
+// distributed under the license is distributed on an "as is" basis,
+// see the license for the specific language governing permissions and
+// limitations under the license.
+
+package api
+
+import "net/http"
+
+// Response defines a new response struct for http
+type Response struct {
+	Action     string      `json:"action"`
+	StatusCode int         `json:"status_code"`
+	Message    string      `json:"message,omitempty"`
+	Payload    interface{} `json:"payload,omitempty"`
+}
+
+func newResponse(action string) *Response {
+	return &Response{Action: action, StatusCode: http.StatusOK}
+}
+
+func (r *Response) statusCode(code int) *Response {
+	r.StatusCode = code
+	return r
+}
+
+func (r *Response) message(msg string) *Response {
+	r.Message = msg
+	return r
+}
+
+func (r *Response) payload(payload interface{}) *Response {
+	r.Payload = payload
+	return r
+}

tests/pkg/fault-trigger/api/router.go

@@ -0,0 +1,40 @@
+// Copyright 2019 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import restful "github.com/emicklei/go-restful"
+
+const (
+	apiPrefix = "/pingcap.com/api/v1"
+)
+
+func (s *Server) newService() *restful.WebService {
+	ws := new(restful.WebService)
+	ws.
+		Path(apiPrefix).
+		Consumes(restful.MIME_JSON).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/vms").To(s.listVMs))
+	ws.Route(ws.GET("/vm/{name}/start").To(s.startVM))
+	ws.Route(ws.GET("/vm/{name}/stop").To(s.stopVM))
+
+	ws.Route(ws.GET("/etcd/start").To(s.startETCD))
+	ws.Route(ws.GET("/etcd/stop").To(s.stopETCD))
+
+	ws.Route(ws.GET("/kubelet/start").To(s.startKubelet))
+	ws.Route(ws.GET("/kubelet/stop").To(s.stopKubelet))
+
+	return ws
+}

tests/pkg/fault-trigger/api/server.go

@@ -0,0 +1,195 @@
+// Copyright 2019 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"fmt"
+	"net/http"
+
+	restful "github.com/emicklei/go-restful"
+	"github.com/golang/glog"
+	"github.com/pingcap/tidb-operator/tests/pkg/fault-trigger/manager"
+)
+
+// Server is a web service to control fault trigger
+type Server struct {
+	mgr *manager.Manager
+
+	port int
+}
+
+// NewServer returns a api server
+func NewServer(mgr *manager.Manager, port int) *Server {
+	return &Server{
+		mgr:  mgr,
+		port: port,
+	}
+}
+
+// StartServer starts a fault-trigger server
+func (s *Server) StartServer() {
+	ws := s.newService()
+
+	restful.Add(ws)
+
+	glog.Infof("starting fault-trigger server, listening on 0.0.0.0:%d", s.port)
+	glog.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", s.port), nil))
+}
+
+func (s *Server) listVMs(req *restful.Request, resp *restful.Response) {
+	res := newResponse("listVMs")
+	vms, err := s.mgr.ListVMs()
+	if err != nil {
+		res.message(err.Error()).statusCode(http.StatusInternalServerError)
+		if err = resp.WriteEntity(res); err != nil {
+			glog.Errorf("failed to response, methods: listVMs, error: %v", err)
+		}
+		return
+	}
+
+	res.payload(vms).statusCode(http.StatusOK)
+
+	if err = resp.WriteEntity(res); err != nil {
+		glog.Errorf("failed to response, method: listVMs, error: %v", err)
+	}
+}
+
+func (s *Server) startVM(req *restful.Request, resp *restful.Response) {
+	res := newResponse("startVM")
+	name := req.PathParameter("name")
+
+	targetVM, err := s.getVM(name)
+	if err != nil {
+		res.message(fmt.Sprintf("failed to get vm %s, error: %v", name, err)).
+			statusCode(http.StatusInternalServerError)
+		if err = resp.WriteEntity(res); err != nil {
+			glog.Errorf("failed to response, methods: startVM, error: %v", err)
+		}
+		return
+	}
+
+	if targetVM == nil {
+		res.message(fmt.Sprintf("vm %s not found", name)).statusCode(http.StatusNotFound)
+		if err = resp.WriteEntity(res); err != nil {
+			glog.Errorf("failed to response, methods: startVM, error: %v", err)
+		}
+		return
+	}
+
+	s.vmAction(req, resp, res, targetVM, s.mgr.StartVM, "startVM")
+}
+
+func (s *Server) stopVM(req *restful.Request, resp *restful.Response) {
+	res := newResponse("stopVM")
+	name := req.PathParameter("name")
+
+	targetVM, err := s.getVM(name)
+	if err != nil {
+		res.message(fmt.Sprintf("failed to get vm %s, error: %v", name, err)).
+			statusCode(http.StatusInternalServerError)
+		if err = resp.WriteEntity(res); err != nil {
+			glog.Errorf("failed to response, methods: stopVM, error: %v", err)
+		}
+		return
+	}
+
+	if targetVM == nil {
+		res.message(fmt.Sprintf("vm %s not found", name)).statusCode(http.StatusNotFound)
+		if err = resp.WriteEntity(res); err != nil {
+			glog.Errorf("failed to response, methods: stopVM, error: %v", err)
+		}
+		return
+	}
+
+	s.vmAction(req, resp, res, targetVM, s.mgr.StopVM, "stopVM")
+}
+
+func (s *Server) startETCD(req *restful.Request, resp *restful.Response) {
+	s.action(req, resp, s.mgr.StartETCD, "startETCD")
+}
+
+func (s *Server) stopETCD(req *restful.Request, resp *restful.Response) {
+	s.action(req, resp, s.mgr.StopETCD, "stopETCD")
+}
+
+func (s *Server) startKubelet(req *restful.Request, resp *restful.Response) {
+	s.action(req, resp, s.mgr.StartKubelet, "startKubelet")
+}
+
+func (s *Server) stopKubelet(req *restful.Request, resp *restful.Response) {
+	s.action(req, resp, s.mgr.StopKubelet, "stopKubelet")
+}
+
+func (s *Server) action(
+	req *restful.Request,
+	resp *restful.Response,
+	fn func() error,
+	method string,
+) {
+	res := newResponse(method)
+
+	if err := fn(); err != nil {
+		res.message(fmt.Sprintf("failed to %s, error: %v", method, err)).
+			statusCode(http.StatusInternalServerError)
+		if err = resp.WriteEntity(res); err != nil {
+			glog.Errorf("failed to response, methods: %s, error: %v", method, err)
+		}
+		return
+	}
+
+	res.message("OK").statusCode(http.StatusOK)
+
+	if err := resp.WriteEntity(res); err != nil {
+		glog.Errorf("failed to response, method: %s, error: %v", method, err)
+	}
+}
+
+func (s *Server) vmAction(
+	req *restful.Request,
+	resp *restful.Response,
+	res *Response,
+	targetVM *manager.VM,
+	fn func(vm *manager.VM) error,
+	method string,
+) {
+	if err := fn(targetVM); err != nil {
+		res.message(fmt.Sprintf("failed to %s vm: %s, error: %v", method, targetVM.Name, err)).
+			statusCode(http.StatusInternalServerError)
+		if err = resp.WriteEntity(res); err != nil {
+			glog.Errorf("failed to response, methods: %s, error: %v", method, err)
+		}
+		return
+	}
+
+	res.message("OK").statusCode(http.StatusOK)
+
+	if err := resp.WriteEntity(res); err != nil {
+		glog.Errorf("failed to response, method: %s, error: %v", method, err)
+	}
+}
+
+func (s *Server) getVM(name string) (*manager.VM, error) {
+	vms, err := s.mgr.ListVMs()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, vm := range vms {
+		if name == vm.Name || name == vm.IP {
+			return vm, nil
+		}
+	}
+
+	return nil, nil
+}

tests/pkg/fault-trigger/manager/etcd.go

@@ -0,0 +1,44 @@
+// Copyright 2019 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package manager
+
+import (
+	"os/exec"
+
+	"github.com/golang/glog"
+)
+
+// StartETCD starts etcd
+func (m *Manager) StartETCD() error {
+	shell := "systemctl start etcd"
+	cmd := exec.Command("/bin/sh", "-c", shell)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		glog.Errorf("exec: [%s] failed, output: %s, error: %v", shell, string(output), err)
+		return err
+	}
+	return nil
+}
+
+// StopETCD stops etcd
+func (m *Manager) StopETCD() error {
+	shell := "systemctl stop etcd"
+	cmd := exec.Command("/bin/sh", "-c", shell)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		glog.Errorf("exec: [%s] failed, output: %s, error: %v", shell, string(output), err)
+		return err
+	}
+	return nil
+}