Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add slack notify #439

Merged
merged 9 commits into from
May 5, 2019
Merged

add slack notify #439

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
5d6f8b1
add slack notify
xiaojingchen Apr 30, 2019
f4e294e
Merge branch 'master' into add-slack-notification
xiaojingchen Apr 30, 2019
c87cbff
remove unused code
xiaojingchen Apr 30, 2019
852625f
fix some code
xiaojingchen Apr 30, 2019
4743c3b
add notify when call os.Exit
xiaojingchen Apr 30, 2019
c63e1d4
fix
xiaojingchen Apr 30, 2019
6cee63b
fix
xiaojingchen May 5, 2019
887deb4
remove blank line
xiaojingchen May 5, 2019
31a5c8a
Merge branch 'master' into add-slack-notification
xiaojingchen May 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 21 additions & 15 deletions tests/actions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,6 @@ import (
"strings"
"time"

_ "github.com/go-sql-driver/mysql"
"github.com/golang/glog"
pingcapErrors "github.com/pingcap/errors"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/tidb-operator/tests/pkg/apimachinery"
admissionV1beta1 "k8s.io/api/admissionregistration/v1beta1"
"k8s.io/api/apps/v1beta1"
batchv1 "k8s.io/api/batch/v1"
Expand All @@ -44,14 +39,20 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"

_ "github.com/go-sql-driver/mysql"
"github.com/golang/glog"
pingcapErrors "github.com/pingcap/errors"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1"
"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
"github.com/pingcap/tidb-operator/pkg/controller"
"github.com/pingcap/tidb-operator/pkg/label"
"github.com/pingcap/tidb-operator/tests/pkg/apimachinery"
"github.com/pingcap/tidb-operator/tests/pkg/blockwriter"
"github.com/pingcap/tidb-operator/tests/pkg/metrics"
"github.com/pingcap/tidb-operator/tests/pkg/util"
"github.com/pingcap/tidb-operator/tests/pkg/webhook"
"github.com/pingcap/tidb-operator/tests/slack"
)

const (
Expand Down Expand Up @@ -306,7 +307,7 @@ func (oa *operatorActions) DeployOperator(info *OperatorConfig) error {

func (oa *operatorActions) DeployOperatorOrDie(info *OperatorConfig) {
if err := oa.DeployOperator(info); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand All @@ -329,7 +330,7 @@ func (oa *operatorActions) CleanOperator(info *OperatorConfig) error {

func (oa *operatorActions) CleanOperatorOrDie(info *OperatorConfig) {
if err := oa.CleanOperator(info); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand Down Expand Up @@ -385,7 +386,7 @@ func (oa *operatorActions) DeployTidbCluster(info *TidbClusterConfig) error {

func (oa *operatorActions) DeployTidbClusterOrDie(info *TidbClusterConfig) {
if err := oa.DeployTidbCluster(info); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand Down Expand Up @@ -461,7 +462,7 @@ func (oa *operatorActions) CleanTidbCluster(info *TidbClusterConfig) error {

func (oa *operatorActions) CleanTidbClusterOrDie(info *TidbClusterConfig) {
if err := oa.CleanTidbCluster(info); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand Down Expand Up @@ -529,7 +530,7 @@ func (oa *operatorActions) CheckTidbClusterStatus(info *TidbClusterConfig) error

func (oa *operatorActions) CheckTidbClusterStatusOrDie(info *TidbClusterConfig) {
if err := oa.CheckTidbClusterStatus(info); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand All @@ -553,7 +554,7 @@ func (oa *operatorActions) BeginInsertDataTo(info *TidbClusterConfig) error {
func (oa *operatorActions) BeginInsertDataToOrDie(info *TidbClusterConfig) {
err := oa.BeginInsertDataTo(info)
if err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand Down Expand Up @@ -594,7 +595,7 @@ func (oa *operatorActions) ScaleTidbCluster(info *TidbClusterConfig) error {

func (oa *operatorActions) ScaleTidbClusterOrDie(info *TidbClusterConfig) {
if err := oa.ScaleTidbCluster(info); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand Down Expand Up @@ -675,7 +676,7 @@ func (oa *operatorActions) UpgradeTidbCluster(info *TidbClusterConfig) error {

func (oa *operatorActions) UpgradeTidbClusterOrDie(info *TidbClusterConfig) {
if err := oa.UpgradeTidbCluster(info); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand Down Expand Up @@ -1964,7 +1965,7 @@ func strPtr(s string) *string { return &s }

func (oa *operatorActions) RegisterWebHookAndServiceOrDie(info *OperatorConfig) {
if err := oa.RegisterWebHookAndService(info); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand Down Expand Up @@ -2109,7 +2110,12 @@ func (oa *operatorActions) StartValidatingAdmissionWebhookServerOrDie(info *Oper
}
err = server.ListenAndServeTLS("", "")
if err != nil {
glog.Errorf("fail to start webhook server err %v", err)
err = fmt.Errorf("fail to start webhook server err %v", err)
glog.Error(err)
sendErr := slack.SendErrMsg(err.Error())
if sendErr != nil {
glog.Error(sendErr)
}
os.Exit(4)
}
}
Expand Down
4 changes: 3 additions & 1 deletion tests/backup/backupcase.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import (
"fmt"
"time"

"github.com/pingcap/tidb-operator/tests/slack"

"github.com/golang/glog"
"github.com/pingcap/tidb-operator/tests"
"k8s.io/apimachinery/pkg/util/wait"
Expand Down Expand Up @@ -116,7 +118,7 @@ func (bc *BackupCase) Run() error {

func (bc *BackupCase) RunOrDie() {
if err := bc.Run(); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand Down
5 changes: 4 additions & 1 deletion tests/cmd/stability/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ import (
_ "net/http/pprof"
"time"

"github.com/pingcap/tidb-operator/tests/slack"

"github.com/golang/glog"
"github.com/jinzhu/copier"
"github.com/pingcap/tidb-operator/tests"
Expand Down Expand Up @@ -237,5 +239,6 @@ func main() {
if err != nil {
glog.Errorf("failed to clean temp dirs, this error can be ignored.")
}
glog.Infof("\nFinished.")

slack.NotifyAndCompleted("\nFinished.")
}
11 changes: 7 additions & 4 deletions tests/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"os"
"strings"

"github.com/pingcap/tidb-operator/tests/slack"

"github.com/pingcap/tidb-operator/tests/pkg/blockwriter"

"github.com/golang/glog"
Expand Down Expand Up @@ -67,6 +69,7 @@ func NewConfig() (*Config, error) {
flag.StringVar(&cfg.OperatorTag, "operator-tag", "master", "operator tag used to choose charts")
flag.StringVar(&cfg.OperatorImage, "operator-image", "pingcap/tidb-operator:latest", "operator image")
flag.StringVar(&cfg.OperatorRepoDir, "operator-repo-dir", "/tidb-operator", "local directory to which tidb-operator cloned")
flag.StringVar(&slack.WebhookUrl, "slack-webhook-url", "", "slack webhook url")
flag.Parse()

operatorRepo, err := ioutil.TempDir("", "tidb-operator")
Expand All @@ -86,10 +89,10 @@ func NewConfig() (*Config, error) {
func ParseConfigOrDie() *Config {
cfg, err := NewConfig()
if err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
if err := cfg.Parse(); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}

glog.Infof("using config: %+v", cfg)
Expand Down Expand Up @@ -138,7 +141,7 @@ func (c *Config) GetTiDBVersion() (string, error) {
func (c *Config) GetTiDBVersionOrDie() string {
v, err := c.GetTiDBVersion()
if err != nil {
panic(err)
slack.NotifyAndPanic(err)
}

return v
Expand All @@ -153,7 +156,7 @@ func (c *Config) GetUpgradeTidbVersions() []string {
func (c *Config) GetUpgradeTidbVersionsOrDie() []string {
versions := c.GetUpgradeTidbVersions()
if len(versions) < 1 {
panic("upgrade tidb verions is empty")
slack.NotifyAndPanic(fmt.Errorf("upgrade tidb verions is empty"))
}

return versions
Expand Down
16 changes: 9 additions & 7 deletions tests/failover.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"strings"
"time"

"github.com/pingcap/tidb-operator/tests/slack"

_ "github.com/go-sql-driver/mysql"
"github.com/golang/glog"
"github.com/pingcap/errors"
Expand Down Expand Up @@ -123,7 +125,7 @@ func (oa *operatorActions) TruncateSSTFileThenCheckFailover(info *TidbClusterCon

func (oa *operatorActions) TruncateSSTFileThenCheckFailoverOrDie(info *TidbClusterConfig, tikvFailoverPeriod time.Duration) {
if err := oa.TruncateSSTFileThenCheckFailover(info, tikvFailoverPeriod); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand Down Expand Up @@ -197,7 +199,7 @@ func (oa *operatorActions) CheckFailoverPendingOrDie(clusters []*TidbClusterConf
}
return true, nil
}); err != nil {
panic("failed to check failover pending")
slack.NotifyAndPanic(fmt.Errorf("failed to check failover pending"))
}
}

Expand Down Expand Up @@ -278,7 +280,7 @@ func (oa *operatorActions) CheckFailoverOrDie(clusters []*TidbClusterConfig, fau
}
return true, nil
}); err != nil {
panic("failed to check failover")
slack.NotifyAndPanic(fmt.Errorf("failed to check failover"))
}
}

Expand Down Expand Up @@ -318,7 +320,7 @@ func (oa *operatorActions) CheckRecoverOrDie(clusters []*TidbClusterConfig) {
}
return true, nil
}); err != nil {
panic("failed to check recover")
slack.NotifyAndPanic(fmt.Errorf("failed to check recover"))
}
}

Expand Down Expand Up @@ -475,7 +477,7 @@ func (oa *operatorActions) CheckOneApiserverDownOrDie(operatorConfig *OperatorCo
affectedPods := map[string]*corev1.Pod{}
apiserverPod, err := GetApiserverPod(oa.kubeCli, faultNode)
if err != nil {
panic(fmt.Errorf("can't find apiserver in node:%s", faultNode))
slack.NotifyAndPanic(fmt.Errorf("can't find apiserver in node:%s", faultNode))
}
if apiserverPod != nil {
affectedPods[apiserverPod.GetName()] = apiserverPod
Expand All @@ -496,7 +498,7 @@ func (oa *operatorActions) CheckOneApiserverDownOrDie(operatorConfig *OperatorCo
}
dnsPod, err := GetDnsPod(oa.kubeCli, faultNode)
if err != nil {
panic(fmt.Errorf("can't find controller-manager in node:%s", faultNode))
slack.NotifyAndPanic(fmt.Errorf("can't find controller-manager in node:%s", faultNode))
}
if dnsPod != nil {
affectedPods[dnsPod.GetName()] = dnsPod
Expand All @@ -523,7 +525,7 @@ func (oa *operatorActions) CheckOneApiserverDownOrDie(operatorConfig *OperatorCo

func (oa *operatorActions) CheckK8sAvailableOrDie(excludeNodes map[string]string, excludePods map[string]*corev1.Pod) {
if err := oa.CheckK8sAvailable(excludeNodes, excludePods); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand Down
14 changes: 8 additions & 6 deletions tests/fault.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"os"
"time"

"github.com/pingcap/tidb-operator/tests/slack"

"github.com/golang/glog"
"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
"github.com/pingcap/tidb-operator/pkg/controller"
Expand Down Expand Up @@ -154,7 +156,7 @@ func (fa *faultTriggerActions) StopNodeOrDie() (string, string, time.Time) {
var err error
var now time.Time
if pn, n, now, err = fa.StopNode(); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
return pn, n, now
}
Expand Down Expand Up @@ -189,7 +191,7 @@ func (fa *faultTriggerActions) StartNode(physicalNode string, node string) error

func (fa *faultTriggerActions) StartNodeOrDie(physicalNode string, node string) {
if err := fa.StartNode(physicalNode, node); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand All @@ -213,7 +215,7 @@ func (fa *faultTriggerActions) StopETCD(nodes ...string) error {

func (fa *faultTriggerActions) StopETCDOrDie(nodes ...string) {
if err := fa.StopETCD(nodes...); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand All @@ -237,7 +239,7 @@ func (fa *faultTriggerActions) StartETCD(nodes ...string) error {

func (fa *faultTriggerActions) StartETCDOrDie(nodes ...string) {
if err := fa.StartETCD(nodes...); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand Down Expand Up @@ -288,7 +290,7 @@ func (fa *faultTriggerActions) StopKubeAPIServer(node string) error {

func (fa *faultTriggerActions) StopKubeAPIServerOrDie(node string) {
if err := fa.StopKubeAPIServer(node); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand All @@ -299,7 +301,7 @@ func (fa *faultTriggerActions) StartKubeAPIServer(node string) error {

func (fa *faultTriggerActions) StartKubeAPIServerOrDie(node string) {
if err := fa.StartKubeAPIServer(node); err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
}

Expand Down
1 change: 1 addition & 0 deletions tests/manifests/stability/stability.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ spec:
command:
- /usr/local/bin/stability-test
- --config=/etc/tidb-operator-stability/config.yaml
- --slack-webhook-url=""
volumeMounts:
- mountPath: /logDir
name: logdir
Expand Down
10 changes: 6 additions & 4 deletions tests/pkg/client/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"os"
"time"

"github.com/pingcap/tidb-operator/tests/slack"

"github.com/juju/errors"
"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned/typed/pingcap.com/v1alpha1"
Expand All @@ -30,7 +32,7 @@ func init() {
func NewCliOrDie() (versioned.Interface, kubernetes.Interface) {
cfg, err := GetConfig()
if err != nil {
panic(err)
slack.NotifyAndPanic(err)
}

return buildClientsOrDie(cfg)
Expand Down Expand Up @@ -65,7 +67,7 @@ func Union(kube kubernetes.Interface, tidb versioned.Interface) Client {
func NewOrDie() Client {
cfg, err := clientcmd.BuildConfigFromFlags(masterUrl, kubeconfigPath)
if err != nil {
panic(err)
slack.NotifyAndPanic(err)
}
return Union(kubernetes.NewForConfigOrDie(cfg), versioned.NewForConfigOrDie(cfg))
}
Expand Down Expand Up @@ -96,12 +98,12 @@ func buildClientsOrDie(cfg *rest.Config) (versioned.Interface, kubernetes.Interf
cfg.Timeout = 30 * time.Second
cli, err := versioned.NewForConfig(cfg)
if err != nil {
panic(err)
slack.NotifyAndPanic(err)
}

kubeCli, err := kubernetes.NewForConfig(cfg)
if err != nil {
panic(err)
slack.NotifyAndPanic(err)
}

return cli, kubeCli
Expand Down
Loading