add an AdvancedStatefulSet job to verify all functions (#1543) (#1576)

* add a AdvancedStatefulSet job

Signed-off-by: Yecheng Fu <fuyecheng@pingcap.com>

* use sts interface getter

Signed-off-by: Yecheng Fu <fuyecheng@pingcap.com>

* upgrade helm sts

Signed-off-by: Yecheng Fu <fuyecheng@pingcap.com>

* fix features

Signed-off-by: Yecheng Fu <fuyecheng@pingcap.com>

* fix admission webhook

Signed-off-by: Yecheng Fu <fuyecheng@pingcap.com>

Co-authored-by: Yecheng Fu <cofyc.jackson@gmail.com>

charts/tidb-operator/templates/admission/admission-webhook-deployment.yaml

@@ -34,6 +34,9 @@ spec:
             - --tls-cert-file=/var/serving-cert/cert.pem
             - --tls-private-key-file=/var/serving-cert/key.pem
             {{- end }}
+            {{- if .Values.features }}
+            - --features={{ join "," .Values.features }}
+            {{- end }}
           livenessProbe:
             failureThreshold: 5
             httpGet:

charts/tidb-operator/templates/admission/admission-webhook-registration.yaml

@@ -126,7 +126,7 @@ webhooks:
         resources: ["statefulsets"]
       - operations: [ "UPDATE" ]
         apiGroups: [ "apps.pingcap.com"]
-        apiVersions: ["v1alpha1"]
+        apiVersions: ["v1alpha1", "v1"]
         resources: ["statefulsets"]
 {{- end }}
 ---

charts/tidb-operator/values.yaml

@@ -150,7 +150,7 @@ apiserver:
 #   kubectl apply -f manifests/advanced-statefulset-crd.v1.yaml      # k8s version >= 1.16.0
 advancedStatefulset:
   create: false
-  image: pingcap/advanced-statefulset:v0.2.4
+  image: pingcap/advanced-statefulset:v0.3.1
   imagePullPolicy: IfNotPresent
   serviceAccount: advanced-statefulset-controller
   logLevel: 2

ci/pingcap_tidb_operator_build_kind.groovy

@@ -193,6 +193,9 @@ def call(BUILD_BRANCH, CREDENTIALS_ID, CODECOV_CREDENTIALS_ID) {
 		builds["E2E v1.16.4"] = {
 			build("${MIRRORS} IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=8 KUBE_VERSION=v1.16.4 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.16.4_ ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]'", artifacts)
 		}
+		builds["E2E v1.12.10 AdvancedStatefulSet"] = {
+			build("${MIRRORS} IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=8 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_advanced_statefulset ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]' --operator-features AdvancedStatefulSet=true", artifacts)
+		}
 		builds["E2E v1.17.0"] = {
 			build("${MIRRORS} IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=8 KUBE_VERSION=v1.17.0 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.17.0_ ./hack/e2e.sh -- -preload-images --ginkgo.skip='\\[Serial\\]'", artifacts)
 		}

go.mod

@@ -47,7 +47,7 @@ require (
 	github.com/openshift/generic-admission-server v1.14.0
 	github.com/opentracing/opentracing-go v1.1.0 // indirect
 	github.com/pierrec/lz4 v2.0.5+incompatible // indirect
-	github.com/pingcap/advanced-statefulset v0.2.4
+	github.com/pingcap/advanced-statefulset v0.3.1
 	github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 // indirect
 	github.com/pingcap/errors v0.11.0
 	github.com/pingcap/kvproto v0.0.0-20191217072959-393e6c0fd4b7

go.sum

@@ -644,8 +644,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
-github.com/pingcap/advanced-statefulset v0.2.4 h1:0PRkKLEiwlWgdFWNtbBOa+Dio7MZLqZ3escd0R3s8No=
-github.com/pingcap/advanced-statefulset v0.2.4/go.mod h1:rg2p1v6AGsKhvEZi6Sm0YNYJCmdXdZZhQ6Sviei7Ivs=
+github.com/pingcap/advanced-statefulset v0.3.1 h1:LxfAdpY2MV/b0MUlASYWjcPfUR161Xly1rA7oaIi684=
+github.com/pingcap/advanced-statefulset v0.3.1/go.mod h1:rg2p1v6AGsKhvEZi6Sm0YNYJCmdXdZZhQ6Sviei7Ivs=
 github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg=
 github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
 github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4=

pkg/features/features.go

@@ -16,12 +16,13 @@ package features
 import (
 	"flag"
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
 
 	"k8s.io/apimachinery/pkg/util/sets"
-	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/klog"
 )
 
 var (
@@ -54,13 +55,15 @@ type FeatureGate interface {
 	SetFromMap(m map[string]bool)
 }
 
+var _ flag.Value = &featureGate{}
+
 type featureGate struct {
 	lock            sync.Mutex
 	enabledFeatures map[string]bool
 }
 
 func (f *featureGate) AddFlag(flagset *flag.FlagSet) {
-	flag.Var(cliflag.NewMapStringBool(&f.enabledFeatures), "features", fmt.Sprintf("A set of key={true,false} pairs to enable/disable features, available features: %s", strings.Join(allFeatures.List(), ",")))
+	flag.Var(f, "features", fmt.Sprintf("A set of key={true,false} pairs to enable/disable features, available features: %s", strings.Join(allFeatures.List(), ",")))
 }
 
 func (f *featureGate) Enabled(key string) bool {
@@ -70,6 +73,16 @@ func (f *featureGate) Enabled(key string) bool {
 	return false
 }
 
+// String returns a string containing all enabled feature gates, formatted as "key1=value1,key2=value2,...".
+func (f *featureGate) String() string {
+	pairs := []string{}
+	for k, v := range f.enabledFeatures {
+		pairs = append(pairs, fmt.Sprintf("%s=%t", k, v))
+	}
+	sort.Strings(pairs)
+	return strings.Join(pairs, ",")
+}
+
 func (f *featureGate) Set(value string) error {
 	m := make(map[string]bool)
 	for _, s := range strings.Split(value, ",") {
@@ -99,6 +112,8 @@ func (f *featureGate) SetFromMap(m map[string]bool) {
 	for k, v := range m {
 		f.enabledFeatures[k] = v
 	}
+
+	klog.V(1).Infof("feature gates: %v", f.enabledFeatures)
 }
 
 func NewFeatureGate() FeatureGate {

pkg/scheduler/scheduler.go

@@ -101,13 +101,15 @@ func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.E
 
 	component, ok := pod.Labels[label.ComponentLabelKey]
 	if !ok {
+		glog.Warningf("can't find component label in pod labels: %s/%s", ns, podName)
 		return &schedulerapiv1.ExtenderFilterResult{
 			Nodes: args.Nodes,
 		}, nil
 	}
 
 	predicatesByComponent, ok := s.predicates[component]
 	if !ok {
+		glog.Warningf("no predicate for component %q, ignored", component)
 		return &schedulerapiv1.ExtenderFilterResult{
 			Nodes: args.Nodes,
 		}, nil

pkg/webhook/admission_hooks.go

@@ -30,7 +30,7 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/klog"
 
-	asappsv1alpha1 "github.com/pingcap/advanced-statefulset/pkg/apis/apps/v1alpha1"
+	asappsv1 "github.com/pingcap/advanced-statefulset/pkg/apis/apps/v1"
 	asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned"
 	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
 	"github.com/pingcap/tidb-operator/pkg/webhook/statefulset"
@@ -75,7 +75,7 @@ func (a *AdmissionHook) Validate(ar *admission.AdmissionRequest) *admission.Admi
 	case "StatefulSet":
 		expectedGroup := "apps"
 		if features.DefaultFeatureGate.Enabled(features.AdvancedStatefulSet) {
-			expectedGroup = asappsv1alpha1.GroupName
+			expectedGroup = asappsv1.GroupName
 		}
 		if expectedGroup != ar.Kind.Group {
 			return a.unknownAdmissionRequest(ar)
@@ -142,6 +142,6 @@ func (a *AdmissionHook) Initialize(cfg *rest.Config, stopCh <-chan struct{}) err
 }
 
 func (a *AdmissionHook) unknownAdmissionRequest(ar *admission.AdmissionRequest) *admission.AdmissionResponse {
-	klog.Infof("success to %v %s[%s/%s]", ar.Operation, ar.Kind.Kind, ar.Name, ar.Namespace)
+	klog.Infof("success to %v %s[%s/%s]", ar.Operation, ar.Kind.Kind, ar.Namespace, ar.Name)
 	return util.ARSuccess()
 }

tests/actions.go

@@ -3315,7 +3315,7 @@ func (oa *operatorActions) checkManualPauseComponent(info *TidbClusterConfig, co
 		}
 	}
 
-	// wait for the tidb statefulset is upgrade to the protect one
+	// wait for the tidb or tikv statefulset is upgraded to the protected one
 	if err = wait.Poll(DefaultPollInterval, 30*time.Minute, fn); err != nil {
 		return fmt.Errorf("fail to upgrade to annotation %s pod, err: %v", component, err)
 	}

tests/config.go

@@ -40,17 +40,18 @@ const (
 type Config struct {
 	configFile string
 
-	TidbVersions         string  `yaml:"tidb_versions" json:"tidb_versions"`
-	InstallOperator      bool    `yaml:"install_opeartor" json:"install_opeartor"`
-	OperatorTag          string  `yaml:"operator_tag" json:"operator_tag"`
-	OperatorImage        string  `yaml:"operator_image" json:"operator_image"`
-	UpgradeOperatorTag   string  `yaml:"upgrade_operator_tag" json:"upgrade_operator_tag"`
-	UpgradeOperatorImage string  `yaml:"upgrade_operator_image" json:"upgrade_operator_image"`
-	LogDir               string  `yaml:"log_dir" json:"log_dir"`
-	FaultTriggerPort     int     `yaml:"fault_trigger_port" json:"fault_trigger_port"`
-	Nodes                []Nodes `yaml:"nodes" json:"nodes"`
-	ETCDs                []Nodes `yaml:"etcds" json:"etcds"`
-	APIServers           []Nodes `yaml:"apiservers" json:"apiservers"`
+	TidbVersions         string          `yaml:"tidb_versions" json:"tidb_versions"`
+	InstallOperator      bool            `yaml:"install_opeartor" json:"install_opeartor"`
+	OperatorTag          string          `yaml:"operator_tag" json:"operator_tag"`
+	OperatorImage        string          `yaml:"operator_image" json:"operator_image"`
+	OperatorFeatures     map[string]bool `yaml:"operator_features" json:"operator_features"`
+	UpgradeOperatorTag   string          `yaml:"upgrade_operator_tag" json:"upgrade_operator_tag"`
+	UpgradeOperatorImage string          `yaml:"upgrade_operator_image" json:"upgrade_operator_image"`
+	LogDir               string          `yaml:"log_dir" json:"log_dir"`
+	FaultTriggerPort     int             `yaml:"fault_trigger_port" json:"fault_trigger_port"`
+	Nodes                []Nodes         `yaml:"nodes" json:"nodes"`
+	ETCDs                []Nodes         `yaml:"etcds" json:"etcds"`
+	APIServers           []Nodes         `yaml:"apiservers" json:"apiservers"`
 	CertFile             string
 	KeyFile              string
 

tests/e2e/config/config.go

@@ -15,12 +15,14 @@ package config
 
 import (
 	"flag"
+	"fmt"
 	"io/ioutil"
 
 	"github.com/pingcap/tidb-operator/tests"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
@@ -36,6 +38,7 @@ func RegisterTiDBOperatorFlags(flags *flag.FlagSet) {
 	flags.BoolVar(&TestConfig.InstallOperator, "install-operator", true, "install a default operator")
 	flags.StringVar(&TestConfig.OperatorTag, "operator-tag", "master", "operator tag used to choose charts")
 	flags.StringVar(&TestConfig.OperatorImage, "operator-image", "pingcap/tidb-operator:latest", "operator image")
+	flags.Var(cliflag.NewMapStringBool(&TestConfig.OperatorFeatures), "operator-features", "a set of key=value pairs that describe feature gates for operator")
 	flags.StringVar(&TestConfig.UpgradeOperatorTag, "upgrade-operator-tag", "", "upgrade operator tag used to choose charts")
 	flags.StringVar(&TestConfig.UpgradeOperatorImage, "upgrade-operator-image", "", "upgrade operator image")
 	flags.StringVar(&TestConfig.OperatorRepoDir, "operator-repo-dir", "/tidb-operator", "local directory to which tidb-operator cloned")
@@ -74,6 +77,14 @@ func AfterReadingAllFlags() error {
 
 // NewDefaultOperatorConfig creates default operator configuration.
 func NewDefaultOperatorConfig(cfg *tests.Config) *tests.OperatorConfig {
+	features := []string{}
+	for k, v := range cfg.OperatorFeatures {
+		t := "false"
+		if v {
+			t = "true"
+		}
+		features = append(features, fmt.Sprintf("%s=%s", k, t))
+	}
 	return &tests.OperatorConfig{
 		Namespace:                 "pingcap",
 		ReleaseName:               "operator",
@@ -82,19 +93,17 @@ func NewDefaultOperatorConfig(cfg *tests.Config) *tests.OperatorConfig {
 		ControllerManagerReplicas: tests.IntPtr(2),
 		SchedulerImage:            "k8s.gcr.io/kube-scheduler",
 		SchedulerReplicas:         tests.IntPtr(2),
-		Features: []string{
-			"StableScheduling=true",
-		},
-		LogLevel:           "4",
-		WebhookServiceName: "webhook-service",
-		WebhookSecretName:  "webhook-secret",
-		WebhookConfigName:  "webhook-config",
-		ImagePullPolicy:    v1.PullIfNotPresent,
-		TestMode:           true,
-		WebhookEnabled:     true,
-		StsWebhookEnabled:  true,
-		PodWebhookEnabled:  false,
-		Cabundle:           "",
+		Features:                  features,
+		LogLevel:                  "4",
+		WebhookServiceName:        "webhook-service",
+		WebhookSecretName:         "webhook-secret",
+		WebhookConfigName:         "webhook-config",
+		ImagePullPolicy:           v1.PullIfNotPresent,
+		TestMode:                  true,
+		WebhookEnabled:            true,
+		StsWebhookEnabled:         true,
+		PodWebhookEnabled:         false,
+		Cabundle:                  "",
 	}
 }
 

tests/e2e/tidbcluster/tidbcluster.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/gomega"
+	"github.com/pingcap/advanced-statefulset/pkg/apis/apps/v1/helper"
 	asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned"
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
@@ -51,6 +52,7 @@ import (
 	utilversion "k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
+	typedappsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/klog"
 	aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
@@ -78,6 +80,10 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 	var genericCli client.Client
 	var fwCancel context.CancelFunc
 	var fw portforward.PortForward
+	/**
+	 * StatefulSet or AdvancedStatefulSet interface.
+	 */
+	var stsGetter func(namespace string) typedappsv1.StatefulSetInterface
 
 	ginkgo.BeforeEach(func() {
 		ns = f.Namespace.Name
@@ -103,6 +109,11 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 		fwCancel = cancel
 		cfg = e2econfig.TestConfig
 		ocfg = e2econfig.NewDefaultOperatorConfig(cfg)
+		if ocfg.Enabled(features.AdvancedStatefulSet) {
+			stsGetter = helper.NewHijackClient(c, asCli).AppsV1().StatefulSets
+		} else {
+			stsGetter = c.AppsV1().StatefulSets
+		}
 		oa = tests.NewOperatorActions(cli, c, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f)
 	})
 
@@ -448,7 +459,17 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 			}),
 		}
 
-		oldPumpSet, err := c.AppsV1().StatefulSets(tc.Namespace).Get(controller.PumpMemberName(tc.Name), metav1.GetOptions{})
+		// If using advanced statefulset, we must upgrade all Kubernetes statefulsets to advanced statefulsets first.
+		if ocfg.Enabled(features.AdvancedStatefulSet) {
+			stsList, err := c.AppsV1().StatefulSets(ns).List(metav1.ListOptions{})
+			framework.ExpectNoError(err)
+			for _, sts := range stsList.Items {
+				_, err = helper.Upgrade(c, asCli, &sts)
+				framework.ExpectNoError(err)
+			}
+		}
+
+		oldPumpSet, err := stsGetter(tc.Namespace).Get(controller.PumpMemberName(tc.Name), metav1.GetOptions{})
 		framework.ExpectNoError(err, "Expected get pump statefulset")
 
 		oldRev := oldPumpSet.Status.CurrentRevision
@@ -458,7 +479,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 		framework.ExpectNoError(err, "Expected update tc")
 
 		err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) {
-			pumpSet, err := c.AppsV1().StatefulSets(tc.Namespace).Get(controller.PumpMemberName(tc.Name), metav1.GetOptions{})
+			pumpSet, err := stsGetter(tc.Namespace).Get(controller.PumpMemberName(tc.Name), metav1.GetOptions{})
 			if errors.IsNotFound(err) {
 				return false, err
 			}
@@ -554,7 +575,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 		}
 
 		for setName := range setNameToRevision {
-			oldSet, err := c.AppsV1().StatefulSets(tc.Namespace).Get(setName, metav1.GetOptions{})
+			oldSet, err := stsGetter(tc.Namespace).Get(setName, metav1.GetOptions{})
 			framework.ExpectNoError(err, "Expected get statefulset %s", setName)
 
 			oldRev := oldSet.Status.CurrentRevision
@@ -565,13 +586,15 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 
 		tc, err = cli.PingcapV1alpha1().TidbClusters(cluster.Namespace).Get(cluster.ClusterName, metav1.GetOptions{})
 		framework.ExpectNoError(err, "Expected get tidbcluster")
-		tc.Spec.TiDB.Config = &v1alpha1.TiDBConfig{}
-		tc.Spec.TiDB.ConfigUpdateStrategy = &updateStrategy
-		tc.Spec.TiKV.Config = &v1alpha1.TiKVConfig{}
-		tc.Spec.TiKV.ConfigUpdateStrategy = &updateStrategy
-		tc.Spec.PD.Config = &v1alpha1.PDConfig{}
-		tc.Spec.PD.ConfigUpdateStrategy = &updateStrategy
-		_, err = cli.PingcapV1alpha1().TidbClusters(tc.Namespace).Update(tc)
+		err = controller.GuaranteedUpdate(genericCli, tc, func() error {
+			tc.Spec.TiDB.Config = &v1alpha1.TiDBConfig{}
+			tc.Spec.TiDB.ConfigUpdateStrategy = &updateStrategy
+			tc.Spec.TiKV.Config = &v1alpha1.TiKVConfig{}
+			tc.Spec.TiKV.ConfigUpdateStrategy = &updateStrategy
+			tc.Spec.PD.Config = &v1alpha1.PDConfig{}
+			tc.Spec.PD.ConfigUpdateStrategy = &updateStrategy
+			return nil
+		})
 		framework.ExpectNoError(err, "Expected update tidbcluster")
 
 		// check for 2 minutes to ensure the tidb statefulset do not get rolling-update
@@ -583,7 +606,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 			framework.ExpectEqual(tc.Status.TiDB.Phase, v1alpha1.NormalPhase, "TiDB should not be updated")
 
 			for setName, oldRev := range setNameToRevision {
-				newSet, err := c.AppsV1().StatefulSets(tc.Namespace).Get(setName, metav1.GetOptions{})
+				newSet, err := stsGetter(tc.Namespace).Get(setName, metav1.GetOptions{})
 				framework.ExpectNoError(err, "Expected get tidb statefulset")
 				framework.ExpectEqual(newSet.Status.CurrentRevision, oldRev, "Expected no rolling-update of %s when manage config in-place", setName)
 				framework.ExpectEqual(newSet.Status.UpdateRevision, oldRev, "Expected no rolling-update of %s when manage config in-place", setName)
@@ -597,7 +620,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 
 		err = wait.PollImmediate(5*time.Second, 3*time.Minute, func() (bool, error) {
 			for setName := range setNameToRevision {
-				newSet, err := c.AppsV1().StatefulSets(tc.Namespace).Get(setName, metav1.GetOptions{})
+				newSet, err := stsGetter(tc.Namespace).Get(setName, metav1.GetOptions{})
 				if err != nil {
 					return false, err
 				}