diff --git a/tests/cmd/fault-trigger/main.go b/tests/cmd/fault-trigger/main.go new file mode 100644 index 0000000000..8474c676a5 --- /dev/null +++ b/tests/cmd/fault-trigger/main.go @@ -0,0 +1,55 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "net/http" + _ "net/http/pprof" + "time" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/tests/pkg/fault-trigger/api" + "github.com/pingcap/tidb-operator/tests/pkg/fault-trigger/manager" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/util/logs" +) + +var ( + port int + pprofPort int +) + +func init() { + flag.IntVar(&port, "port", 23332, "The port that the fault trigger's http service runs on (default 23332)") + flag.IntVar(&pprofPort, "pprof-port", 6060, "The port that the pprof's http service runs on (default 6060)") + + flag.Parse() +} + +func main() { + logs.InitLogs() + defer logs.FlushLogs() + + mgr := manager.NewManager() + + server := api.NewServer(mgr, port) + + go wait.Forever(func() { + server.StartServer() + }, 5*time.Second) + + glog.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", pprofPort), nil)) +} diff --git a/tests/pkg/fault-trigger/api/response.go b/tests/pkg/fault-trigger/api/response.go new file mode 100644 index 0000000000..9280cf66f7 --- /dev/null +++ b/tests/pkg/fault-trigger/api/response.go @@ -0,0 +1,43 @@ +// copyright 2019 pingcap, inc. +// +// licensed under the apache license, version 2.0 (the "license"); +// you may not use this file except in compliance with the license. +// you may obtain a copy of the license at +// +// http://www.apache.org/licenses/license-2.0 +// +// unless required by applicable law or agreed to in writing, software +// distributed under the license is distributed on an "as is" basis, +// see the license for the specific language governing permissions and +// limitations under the license. + +package api + +import "net/http" + +// Response defines a new response struct for http +type Response struct { + Action string `json:"action"` + StatusCode int `json:"status_code"` + Message string `json:"message,omitempty"` + Payload interface{} `json:"payload,omitempty"` +} + +func newResponse(action string) *Response { + return &Response{Action: action, StatusCode: http.StatusOK} +} + +func (r *Response) statusCode(code int) *Response { + r.StatusCode = code + return r +} + +func (r *Response) message(msg string) *Response { + r.Message = msg + return r +} + +func (r *Response) payload(payload interface{}) *Response { + r.Payload = payload + return r +} diff --git a/tests/pkg/fault-trigger/api/router.go b/tests/pkg/fault-trigger/api/router.go new file mode 100644 index 0000000000..bb58405bab --- /dev/null +++ b/tests/pkg/fault-trigger/api/router.go @@ -0,0 +1,40 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import restful "github.com/emicklei/go-restful" + +const ( + apiPrefix = "/pingcap.com/api/v1" +) + +func (s *Server) newService() *restful.WebService { + ws := new(restful.WebService) + ws. + Path(apiPrefix). + Consumes(restful.MIME_JSON). + Produces(restful.MIME_JSON) + + ws.Route(ws.GET("/vms").To(s.listVMs)) + ws.Route(ws.GET("/vm/{name}/start").To(s.startVM)) + ws.Route(ws.GET("/vm/{name}/stop").To(s.stopVM)) + + ws.Route(ws.GET("/etcd/start").To(s.startETCD)) + ws.Route(ws.GET("/etcd/stop").To(s.stopETCD)) + + ws.Route(ws.GET("/kubelet/start").To(s.startKubelet)) + ws.Route(ws.GET("/kubelet/stop").To(s.stopKubelet)) + + return ws +} diff --git a/tests/pkg/fault-trigger/api/server.go b/tests/pkg/fault-trigger/api/server.go new file mode 100644 index 0000000000..6d0fce8edc --- /dev/null +++ b/tests/pkg/fault-trigger/api/server.go @@ -0,0 +1,195 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "fmt" + "net/http" + + restful "github.com/emicklei/go-restful" + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/tests/pkg/fault-trigger/manager" +) + +// Server is a web service to control fault trigger +type Server struct { + mgr *manager.Manager + + port int +} + +// NewServer returns a api server +func NewServer(mgr *manager.Manager, port int) *Server { + return &Server{ + mgr: mgr, + port: port, + } +} + +// StartServer starts a fault-trigger server +func (s *Server) StartServer() { + ws := s.newService() + + restful.Add(ws) + + glog.Infof("starting fault-trigger server, listening on 0.0.0.0:%d", s.port) + glog.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", s.port), nil)) +} + +func (s *Server) listVMs(req *restful.Request, resp *restful.Response) { + res := newResponse("listVMs") + vms, err := s.mgr.ListVMs() + if err != nil { + res.message(err.Error()).statusCode(http.StatusInternalServerError) + if err = resp.WriteEntity(res); err != nil { + glog.Errorf("failed to response, methods: listVMs, error: %v", err) + } + return + } + + res.payload(vms).statusCode(http.StatusOK) + + if err = resp.WriteEntity(res); err != nil { + glog.Errorf("failed to response, method: listVMs, error: %v", err) + } +} + +func (s *Server) startVM(req *restful.Request, resp *restful.Response) { + res := newResponse("startVM") + name := req.PathParameter("name") + + targetVM, err := s.getVM(name) + if err != nil { + res.message(fmt.Sprintf("failed to get vm %s, error: %v", name, err)). + statusCode(http.StatusInternalServerError) + if err = resp.WriteEntity(res); err != nil { + glog.Errorf("failed to response, methods: startVM, error: %v", err) + } + return + } + + if targetVM == nil { + res.message(fmt.Sprintf("vm %s not found", name)).statusCode(http.StatusNotFound) + if err = resp.WriteEntity(res); err != nil { + glog.Errorf("failed to response, methods: startVM, error: %v", err) + } + return + } + + s.vmAction(req, resp, res, targetVM, s.mgr.StartVM, "startVM") +} + +func (s *Server) stopVM(req *restful.Request, resp *restful.Response) { + res := newResponse("stopVM") + name := req.PathParameter("name") + + targetVM, err := s.getVM(name) + if err != nil { + res.message(fmt.Sprintf("failed to get vm %s, error: %v", name, err)). + statusCode(http.StatusInternalServerError) + if err = resp.WriteEntity(res); err != nil { + glog.Errorf("failed to response, methods: stopVM, error: %v", err) + } + return + } + + if targetVM == nil { + res.message(fmt.Sprintf("vm %s not found", name)).statusCode(http.StatusNotFound) + if err = resp.WriteEntity(res); err != nil { + glog.Errorf("failed to response, methods: stopVM, error: %v", err) + } + return + } + + s.vmAction(req, resp, res, targetVM, s.mgr.StopVM, "stopVM") +} + +func (s *Server) startETCD(req *restful.Request, resp *restful.Response) { + s.action(req, resp, s.mgr.StartETCD, "startETCD") +} + +func (s *Server) stopETCD(req *restful.Request, resp *restful.Response) { + s.action(req, resp, s.mgr.StopETCD, "stopETCD") +} + +func (s *Server) startKubelet(req *restful.Request, resp *restful.Response) { + s.action(req, resp, s.mgr.StartKubelet, "startKubelet") +} + +func (s *Server) stopKubelet(req *restful.Request, resp *restful.Response) { + s.action(req, resp, s.mgr.StopKubelet, "stopKubelet") +} + +func (s *Server) action( + req *restful.Request, + resp *restful.Response, + fn func() error, + method string, +) { + res := newResponse(method) + + if err := fn(); err != nil { + res.message(fmt.Sprintf("failed to %s, error: %v", method, err)). + statusCode(http.StatusInternalServerError) + if err = resp.WriteEntity(res); err != nil { + glog.Errorf("failed to response, methods: %s, error: %v", method, err) + } + return + } + + res.message("OK").statusCode(http.StatusOK) + + if err := resp.WriteEntity(res); err != nil { + glog.Errorf("failed to response, method: %s, error: %v", method, err) + } +} + +func (s *Server) vmAction( + req *restful.Request, + resp *restful.Response, + res *Response, + targetVM *manager.VM, + fn func(vm *manager.VM) error, + method string, +) { + if err := fn(targetVM); err != nil { + res.message(fmt.Sprintf("failed to %s vm: %s, error: %v", method, targetVM.Name, err)). + statusCode(http.StatusInternalServerError) + if err = resp.WriteEntity(res); err != nil { + glog.Errorf("failed to response, methods: %s, error: %v", method, err) + } + return + } + + res.message("OK").statusCode(http.StatusOK) + + if err := resp.WriteEntity(res); err != nil { + glog.Errorf("failed to response, method: %s, error: %v", method, err) + } +} + +func (s *Server) getVM(name string) (*manager.VM, error) { + vms, err := s.mgr.ListVMs() + if err != nil { + return nil, err + } + + for _, vm := range vms { + if name == vm.Name || name == vm.IP { + return vm, nil + } + } + + return nil, nil +} diff --git a/tests/pkg/fault-trigger/manager/etcd.go b/tests/pkg/fault-trigger/manager/etcd.go new file mode 100644 index 0000000000..13c0df2f4c --- /dev/null +++ b/tests/pkg/fault-trigger/manager/etcd.go @@ -0,0 +1,44 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +import ( + "os/exec" + + "github.com/golang/glog" +) + +// StartETCD starts etcd +func (m *Manager) StartETCD() error { + shell := "systemctl start etcd" + cmd := exec.Command("/bin/sh", "-c", shell) + output, err := cmd.CombinedOutput() + if err != nil { + glog.Errorf("exec: [%s] failed, output: %s, error: %v", shell, string(output), err) + return err + } + return nil +} + +// StopETCD stops etcd +func (m *Manager) StopETCD() error { + shell := "systemctl stop etcd" + cmd := exec.Command("/bin/sh", "-c", shell) + output, err := cmd.CombinedOutput() + if err != nil { + glog.Errorf("exec: [%s] failed, output: %s, error: %v", shell, string(output), err) + return err + } + return nil +} diff --git a/tests/pkg/fault-trigger/manager/kubelet.go b/tests/pkg/fault-trigger/manager/kubelet.go new file mode 100644 index 0000000000..6cdbf11227 --- /dev/null +++ b/tests/pkg/fault-trigger/manager/kubelet.go @@ -0,0 +1,44 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +import ( + "os/exec" + + "github.com/golang/glog" +) + +// StartKubelet starts kubelet +func (m *Manager) StartKubelet() error { + shell := "systemctl start kubelet" + cmd := exec.Command("/bin/sh", "-c", shell) + output, err := cmd.CombinedOutput() + if err != nil { + glog.Errorf("exec: [%s] failed, output: %s, error: %v", shell, string(output), err) + return err + } + return nil +} + +// StopKubelet stops kubelet +func (m *Manager) StopKubelet() error { + shell := "systemctl stop kubelet" + cmd := exec.Command("/bin/sh", "-c", shell) + output, err := cmd.CombinedOutput() + if err != nil { + glog.Errorf("exec: [%s] failed, output: %s, error: %v", shell, string(output), err) + return err + } + return nil +} diff --git a/tests/pkg/fault-trigger/manager/manager.go b/tests/pkg/fault-trigger/manager/manager.go new file mode 100644 index 0000000000..55f6ee77d0 --- /dev/null +++ b/tests/pkg/fault-trigger/manager/manager.go @@ -0,0 +1,22 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +// Manager to manager fault trigger +type Manager struct{} + +// NewManager returns a manager instance +func NewManager() *Manager { + return &Manager{} +} diff --git a/tests/pkg/fault-trigger/manager/types.go b/tests/pkg/fault-trigger/manager/types.go new file mode 100644 index 0000000000..fa2fa099ec --- /dev/null +++ b/tests/pkg/fault-trigger/manager/types.go @@ -0,0 +1,23 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +// VM defines the descriptive information of a virtual machine +type VM struct { + Host string `json:"host"` + Port int64 `json:"port"` + Name string `json:"name"` + IP string `json:"ip"` + Role []string `json:"role"` +} diff --git a/tests/pkg/fault-trigger/manager/vm.go b/tests/pkg/fault-trigger/manager/vm.go new file mode 100644 index 0000000000..e83df64b00 --- /dev/null +++ b/tests/pkg/fault-trigger/manager/vm.go @@ -0,0 +1,216 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +import ( + "errors" + "fmt" + "os/exec" + "strings" + + "github.com/golang/glog" +) + +// ListVMs lists vms +func (m *Manager) ListVMs() ([]*VM, error) { + shell := fmt.Sprintf("virsh list --all") + cmd := exec.Command("/bin/sh", "-c", shell) + output, err := cmd.CombinedOutput() + if err != nil { + glog.Errorf("exec: [%s] failed, output: %s, error: %v", shell, string(output), err) + return nil, err + } + vms := parserVMs(string(output)) + //"virsh domifaddr vm1 --interface eth0 --source agent" + for _, vm := range vms { + vmIP, err := getVMIP(vm.Name) + if err != nil { + glog.Errorf("can not get vm %s ip", vm.Name) + continue + } + vm.IP = vmIP + } + return vms, nil +} + +// StopVM stops vm +func (m *Manager) StopVM(v *VM) error { + shell := fmt.Sprintf("virsh destroy %s", v.Name) + cmd := exec.Command("/bin/sh", "-c", shell) + output, err := cmd.CombinedOutput() + if err != nil { + glog.Errorf("exec: [%s] failed, output: %s, error: %v", shell, string(output), err) + return err + } + return nil +} + +// StartVM starts vm +func (m *Manager) StartVM(v *VM) error { + shell := fmt.Sprintf("virsh start %s", v.Name) + cmd := exec.Command("/bin/sh", "-c", shell) + output, err := cmd.CombinedOutput() + if err != nil { + glog.Errorf("exec: [%s] failed, output: %s, error: %v", shell, string(output), err) + return err + } + return nil +} + +func getVMIP(name string) (string, error) { + shell := fmt.Sprintf("virsh domifaddr %s --interface eth0 --source agent", name) + cmd := exec.Command("/bin/sh", "-c", shell) + output, err := cmd.CombinedOutput() + if err != nil { + glog.Warningf("exec: [%s] failed, output: %s, error: %v", shell, string(output), err) + mac, err := getVMMac(name) + if err != nil { + return "", err + } + + ipNeighShell := fmt.Sprintf("ip neigh | grep -i %s", mac) + cmd = exec.Command("/bin/sh", "-c", ipNeighShell) + ipNeighOutput, err := cmd.CombinedOutput() + if err != nil { + glog.Errorf("exec: [%s] failed, output: %s, error: %v", ipNeighShell, string(ipNeighOutput), err) + return "", err + } + + return parserIPFromIPNeign(string(ipNeighOutput)) + } + + return parserIP(string(output)) +} + +func getVMMac(name string) (string, error) { + shell := fmt.Sprintf("virsh domiflist %s", name) + cmd := exec.Command("/bin/sh", "-c", shell) + output, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + + return parserMac(string(output)) +} + +// example input : +// Interface Type Source Model MAC +// ------------------------------------------------------- +// vnet1 bridge br0 virtio 52:54:00:d4:9e:bb +// output: 52:54:00:d4:9e:bb, nil +func parserMac(data string) (string, error) { + data = stripEmpty(data) + lines := strings.Split(data, "\n") + + for _, line := range lines { + if !strings.Contains(line, "bridge") { + continue + } + + fields := strings.Split(line, " ") + if len(fields) < 5 { + continue + } + + return fields[4], nil + } + + return "", errors.New("mac not found") +} + +// example input: +// Name MAC address Protocol Address +// ------------------------------------------------------------------------------- +// eth0 52:54:00:4c:5b:c0 ipv4 172.16.30.216/24 +// - - ipv6 fe80::5054:ff:fe4c:5bc0/64 +// output: 172.16.30.216, nil +func parserIP(data string) (string, error) { + data = stripEmpty(data) + lines := strings.Split(data, "\n") + for _, line := range lines { + if !strings.HasPrefix(line, "eth0") { + continue + } + + fields := strings.Split(line, " ") + if len(fields) < 4 { + continue + } + + ip := strings.Split(fields[3], "/")[0] + return ip, nil + } + + return "", errors.New("ip not found") +} + +// example input: +// 172.16.30.216 dev br0 lladdr 52:54:00:4c:5b:c0 STALE +// output: 172.16.30.216, nil +func parserIPFromIPNeign(data string) (string, error) { + fields := strings.Split(strings.Trim(data, "\n"), " ") + if len(fields) != 6 { + return "", errors.New("ip not found") + } + + return fields[0], nil +} + +// example input: +// Id Name State +// ---------------------------------------------------- +// 6 vm2 running +// 11 vm3 running +// 12 vm1 running +// - vm-template shut off +func parserVMs(data string) []*VM { + data = stripEmpty(data) + lines := strings.Split(data, "\n") + var vms []*VM + for _, line := range lines { + fields := strings.Split(line, " ") + if len(fields) < 3 { + continue + } + if !strings.HasPrefix(fields[1], "vm") { + continue + } + + if strings.HasPrefix(fields[1], "vm-template") { + continue + } + vm := &VM{ + Name: fields[1], + } + vms = append(vms, vm) + } + return vms +} + +func stripEmpty(data string) string { + stripLines := []string{} + lines := strings.Split(data, "\n") + for _, line := range lines { + stripFields := []string{} + fields := strings.Split(line, " ") + for _, field := range fields { + if len(field) > 0 { + stripFields = append(stripFields, field) + } + } + stripLine := strings.Join(stripFields, " ") + stripLines = append(stripLines, stripLine) + } + return strings.Join(stripLines, "\n") +} diff --git a/tests/pkg/fault-trigger/manager/vm_test.go b/tests/pkg/fault-trigger/manager/vm_test.go new file mode 100644 index 0000000000..66d8f772aa --- /dev/null +++ b/tests/pkg/fault-trigger/manager/vm_test.go @@ -0,0 +1,71 @@ +package manager + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestParseMac(t *testing.T) { + g := NewGomegaWithT(t) + + data := ` +Interface Type Source Model MAC +------------------------------------------------------- +vnet1 bridge br0 virtio 52:54:00:d4:9e:bb +` + output, err := parserMac(data) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("52:54:00:d4:9e:bb")) +} + +func TestParseIP(t *testing.T) { + g := NewGomegaWithT(t) + + data := ` +Name MAC address Protocol Address +------------------------------------------------------------------------------- +eth0 52:54:00:4c:5b:c0 ipv4 172.16.30.216/24 +- - ipv6 fe80::5054:ff:fe4c:5bc0/64 +` + output, err := parserIP(data) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("172.16.30.216")) +} + +func TestParseIPFromIPNeign(t *testing.T) { + g := NewGomegaWithT(t) + + data := ` +172.16.30.216 dev br0 lladdr 52:54:00:4c:5b:c0 STALE +` + output, err := parserIPFromIPNeign(data) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("172.16.30.216")) +} + +func TestParseVMs(t *testing.T) { + g := NewGomegaWithT(t) + + data := ` + Id Name State +---------------------------------------------------- + 6 vm2 running + 11 vm3 running + 12 vm1 running + - vm-template shut off +` + vms := parserVMs(data) + + var expectedVMs []*VM + expectedVMs = append(expectedVMs, &VM{ + Name: "vm2", + }) + expectedVMs = append(expectedVMs, &VM{ + Name: "vm3", + }) + expectedVMs = append(expectedVMs, &VM{ + Name: "vm1", + }) + g.Expect(vms).To(Equal(expectedVMs)) +}