e2e: reduce ephemeral storage usage and request more (#2262)

* add tidb v4 image into preload list

* set tidb version explicitly

* remove v2 tidb image

* reduce storage usage and request more in CI job

* fix

* fix some issues

* more QPS allowed in e2e

* more debug information

ci/pingcap_tidb_operator_build_kind.groovy

@@ -41,13 +41,13 @@ spec:
       requests:
         cpu: <%= resources.requests.cpu %>
         memory: <%= resources.requests.memory %>
-        ephemeral-storage: 60Gi
+        ephemeral-storage: 70Gi
     <% } %>
     <% if (resources.limits) { %>
       limits:
         cpu: <%= resources.limits.cpu %>
         memory: <%= resources.limits.memory %>
-        ephemeral-storage: 60Gi
+        ephemeral-storage: 70Gi
     <% } %>
 <% } %>
     # kind needs /lib/modules and cgroups from the host
@@ -253,6 +253,7 @@ def call(BUILD_BRANCH, CREDENTIALS_ID, CODECOV_CREDENTIALS_ID) {
 							string(credentialsId: "${CODECOV_CREDENTIALS_ID}", variable: 'CODECOV_TOKEN')
 						]) {
 							sh """#!/bin/bash
+							set -eu
 							echo "info: building"
 							make build e2e-build
 							if [ "${BUILD_BRANCH}" == "master" ]; then
@@ -267,6 +268,7 @@ def call(BUILD_BRANCH, CREDENTIALS_ID, CODECOV_CREDENTIALS_ID) {
 					stage("Prepare for e2e") {
 						withCredentials([usernamePassword(credentialsId: 'TIDB_OPERATOR_HUB_AUTH', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
 							sh """#!/bin/bash
+							set -eu
 							echo "info: logging into hub.pingcap.net"
 							docker login -u \$USERNAME --password-stdin hub.pingcap.net <<< \$PASSWORD
 							echo "info: build and push images for e2e"

hack/run-e2e.sh

@@ -196,7 +196,7 @@ function e2e::image_load() {
         $TIDB_BACKUP_MANAGER_IMAGE
         $E2E_IMAGE
     )
-    echo "info: pull if images does not exist"
+    echo "info: pull if images do not exist"
     for image in ${images[@]}; do
         if ! docker inspect -f '{{.Id}}' $image &>/dev/null; then
             echo "info: pulling $image"
@@ -330,8 +330,6 @@ e2e_args=(
     --operator-image="${TIDB_OPERATOR_IMAGE}"
     --backup-image="${TIDB_BACKUP_MANAGER_IMAGE}"
     --e2e-image="${E2E_IMAGE}"
-    # two tidb versions can be configuraed: <defaultVersion>,<upgradeToVersion>
-    --tidb-versions=v3.0.7,v3.0.8
     --chart-dir=/charts
     -v=4
 )

tests/e2e/config/config.go

@@ -34,7 +34,6 @@ var TestConfig *tests.Config = tests.NewDefaultConfig()
 func RegisterTiDBOperatorFlags(flags *flag.FlagSet) {
 	flags.StringVar(&TestConfig.LogDir, "log-dir", "/logDir", "log directory")
 	flags.IntVar(&TestConfig.FaultTriggerPort, "fault-trigger-port", 23332, "the http port of fault trigger service")
-	flags.StringVar(&TestConfig.TidbVersions, "tidb-versions", "v3.0.6,v3.0.7,v3.0.8", "tidb versions")
 	flags.StringVar(&TestConfig.E2EImage, "e2e-image", "", "e2e image")
 	flags.BoolVar(&TestConfig.InstallOperator, "install-operator", true, "install a default operator")
 	flags.StringVar(&TestConfig.OperatorTag, "operator-tag", "master", "operator tag used to choose charts")

tests/e2e/e2e.go

@@ -30,6 +30,7 @@ import (
 	"github.com/onsi/gomega"
 	asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned"
 	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
+	"github.com/pingcap/tidb-operator/pkg/label"
 	"github.com/pingcap/tidb-operator/pkg/version"
 	"github.com/pingcap/tidb-operator/tests"
 	e2econfig "github.com/pingcap/tidb-operator/tests/e2e/config"
@@ -39,6 +40,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	runtimeutils "k8s.io/apimachinery/pkg/util/runtime"
@@ -145,8 +147,9 @@ func setupSuite() {
 	// which be used and we must clean them later.
 	// We set local-storage class as default for simplicity.
 	// The default storage class of kind is local-path-provisioner which
-	// consumes local storage like local-volume-provisioner.
-	if framework.TestContext.Provider == "gke" || framework.TestContext.Provider == "aws" {
+	// consumes local storage like local-volume-provisioner. However, it's not
+	// stable in our e2e testing.
+	if framework.TestContext.Provider == "gke" || framework.TestContext.Provider == "aws" || framework.TestContext.Provider == "kind" {
 		defaultSCName := "local-storage"
 		list, err := c.StorageV1().StorageClasses().List(metav1.ListOptions{})
 		framework.ExpectNoError(err)
@@ -204,6 +207,11 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 	if err := exec.Command("sh", "-c", clearValidatingWebhookConfigurationsCmd).Run(); err != nil {
 		framework.Failf("failed to clear validatingwebhookconfigurations (cmd: %q, error: %v", clearValidatingWebhookConfigurationsCmd, err)
 	}
+	ginkgo.By("Clear tidb-operator mutatingwebhookconfigurations")
+	clearMutatingWebhookConfigurationsCmd := "kubectl delete mutatingwebhookconfiguration -l app.kubernetes.io/name=tidb-operator"
+	if err := exec.Command("sh", "-c", clearMutatingWebhookConfigurationsCmd).Run(); err != nil {
+		framework.Failf("failed to clear mutatingwebhookconfigurations (cmd: %q, error: %v", clearMutatingWebhookConfigurationsCmd, err)
+	}
 	setupSuite()
 	// override with hard-coded value
 	e2econfig.TestConfig.ManifestDir = "/manifests"
@@ -218,6 +226,8 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 	}
 	// Get clients
 	config, err := framework.LoadConfig()
+	config.QPS = 20
+	config.Burst = 50
 	framework.ExpectNoError(err, "failed to load config")
 	cli, err := versioned.NewForConfig(config)
 	framework.ExpectNoError(err, "failed to create clientset")
@@ -261,6 +271,59 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 		return true, nil
 	})
 	framework.ExpectNoError(err, "failed to wait for all PVs to be available")
+
+	// tidb-operator will set persistentVolumeReclaimPolicy to Retain if users
+	// reqeust this. To reduce storage usage, we set
+	// persistentVolumeReclaimPolicy to Delete if the PVC namespace is gone.
+	go wait.Forever(func() {
+		pvList, err := kubeCli.CoreV1().PersistentVolumes().List(metav1.ListOptions{})
+		if err != nil {
+			framework.Logf("failed to list pvs: %v", err)
+			return
+		}
+		var (
+			total          int = len(pvList.Items)
+			retainReleased int
+			skipped        int
+			failed         int
+			succeeded      int
+		)
+		defer func() {
+			framework.Logf("recycling orphan PVs (total: %d, retainReleased: %d, skipped: %d, failed: %d, succeeded: %d)", total, retainReleased, skipped, failed, succeeded)
+		}()
+		for _, pv := range pvList.Items {
+			if pv.Spec.PersistentVolumeReclaimPolicy != v1.PersistentVolumeReclaimRetain || pv.Status.Phase != v1.VolumeReleased {
+				continue
+			}
+			retainReleased++
+			pvcNamespaceName, ok := pv.Labels[label.NamespaceLabelKey]
+			if !ok {
+				framework.Logf("label %q does not exist in PV %q", label.NamespaceLabelKey, pv.Name)
+				failed++
+				continue
+			}
+			_, err := kubeCli.CoreV1().Namespaces().Get(pvcNamespaceName, metav1.GetOptions{})
+			if err != nil && !apierrors.IsNotFound(err) {
+				framework.Logf("failed to get namespace %q: %v", pvcNamespaceName, err)
+				failed++
+				continue
+			}
+			if apierrors.IsNotFound(err) {
+				skipped++
+				continue
+			}
+			pv.Spec.PersistentVolumeReclaimPolicy = v1.PersistentVolumeReclaimDelete
+			_, err = kubeCli.CoreV1().PersistentVolumes().Update(&pv)
+			if err != nil {
+				failed++
+				framework.Logf("failed to set PersistentVolumeReclaimPolicy of PV %q to Delete: %v", pv.Name, err)
+			} else {
+				succeeded++
+				framework.Logf("successfully set PersistentVolumeReclaimPolicy of PV %q to Delete", pv.Name)
+			}
+		}
+	}, time.Second*10)
+
 	ginkgo.By("Labeling nodes")
 	oa := tests.NewOperatorActions(cli, kubeCli, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, nil, e2econfig.TestConfig, nil, nil, nil)
 	oa.LabelNodesOrDie()

tests/e2e/tidbcluster/serial.go

@@ -438,16 +438,15 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
 		ginkgo.It("[PodAdmissionWebhook] able to upgrade TiDB Cluster with pod admission webhook", func() {
 			klog.Info("start to upgrade tidbcluster with pod admission webhook")
 			// deploy new cluster and test upgrade and scale-in/out with pod admission webhook
-			cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "admission", "", "")
+			cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "admission", "", utilimage.TiDBV3Version)
 			cluster.Resources["pd.replicas"] = "3"
 			cluster.Resources["tikv.replicas"] = "3"
 			cluster.Resources["tidb.replicas"] = "2"
 
 			oa.DeployTidbClusterOrDie(&cluster)
 			oa.CheckTidbClusterStatusOrDie(&cluster)
-			upgradeVersions := cfg.GetUpgradeTidbVersionsOrDie()
-			ginkgo.By(fmt.Sprintf("Upgrading tidb cluster from %s to %s", cluster.ClusterVersion, upgradeVersions[0]))
-			cluster.UpgradeAll(upgradeVersions[0])
+			ginkgo.By(fmt.Sprintf("Upgrading tidb cluster from %s to %s", cluster.ClusterVersion, utilimage.TiDBV3UpgradeVersion))
+			cluster.UpgradeAll(utilimage.TiDBV3UpgradeVersion)
 			oa.UpgradeTidbClusterOrDie(&cluster)
 			// TODO: find a more graceful way to check tidbcluster during upgrading
 			oa.CheckTidbClusterStatusOrDie(&cluster)
@@ -986,7 +985,7 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
 
 		ginkgo.It("Deploy TidbCluster and Upgrade Operator", func() {
 			tcName := "tidbcluster"
-			cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, tcName, "", "")
+			cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, tcName, "", utilimage.TiDBV3Version)
 			cluster.Resources["pd.replicas"] = "3"
 			cluster.Resources["tikv.replicas"] = "3"
 			cluster.Resources["tidb.replicas"] = "2"
@@ -1115,7 +1114,7 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
 
 		ginkgo.It("basic deployment", func() {
 			tcName := "basic"
-			cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, tcName, "", "")
+			cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, tcName, "", utilimage.TiDBV3Version)
 			cluster.Resources["pd.replicas"] = "1"
 			cluster.Resources["tikv.replicas"] = "1"
 			cluster.Resources["tidb.replicas"] = "1"

tests/e2e/tidbcluster/tidbcluster.go

@@ -143,13 +143,8 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 				Name:    "basic-v3",
 			},
 			{
-				Version: utilimage.TiDBV2Version,
-				Name:    "basic-v2",
-				Values: map[string]string{
-					// verify v2.1.x configuration compatibility
-					// https://github.com/pingcap/tidb-operator/pull/950
-					"tikv.resources.limits.storage": "1G",
-				},
+				Version: utilimage.TiDBV4Version,
+				Name:    "basic-v4",
 			},
 		}
 
@@ -213,7 +208,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 			ginkgo.By("Testing HostNetwork feature with Advanced StatefulSet")
 		}
 
-		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "host-network", "", "")
+		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "host-network", "", utilimage.TiDBV3Version)
 		cluster.Resources["pd.replicas"] = "1"
 		cluster.Resources["tidb.replicas"] = "1"
 		cluster.Resources["tikv.replicas"] = "1"
@@ -231,7 +226,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 	})
 
 	ginkgo.It("Upgrading TiDB Cluster", func() {
-		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "cluster", "admin", "")
+		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "cluster", "admin", utilimage.TiDBV3Version)
 		cluster.Resources["pd.replicas"] = "3"
 
 		ginkgo.By("Creating webhook certs and self signing it")
@@ -250,10 +245,9 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 		oa.CheckTidbClusterStatusOrDie(&cluster)
 		oa.CheckDisasterToleranceOrDie(&cluster)
 
-		upgradeVersions := cfg.GetUpgradeTidbVersionsOrDie()
-		ginkgo.By(fmt.Sprintf("Upgrading tidb cluster from %s to %s", cluster.ClusterVersion, upgradeVersions[0]))
+		ginkgo.By(fmt.Sprintf("Upgrading tidb cluster from %s to %s", cluster.ClusterVersion, utilimage.TiDBV3UpgradeVersion))
 		ctx, cancel := context.WithCancel(context.Background())
-		cluster.UpgradeAll(upgradeVersions[0])
+		cluster.UpgradeAll(utilimage.TiDBV3UpgradeVersion)
 		oa.UpgradeTidbClusterOrDie(&cluster)
 		oa.CheckUpgradeOrDie(ctx, &cluster)
 		oa.CheckTidbClusterStatusOrDie(&cluster)
@@ -273,8 +267,8 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 	})
 
 	ginkgo.It("Backup and restore TiDB Cluster", func() {
-		clusterA := newTidbClusterConfig(e2econfig.TestConfig, ns, "cluster3", "admin", "")
-		clusterB := newTidbClusterConfig(e2econfig.TestConfig, ns, "cluster4", "admin", "")
+		clusterA := newTidbClusterConfig(e2econfig.TestConfig, ns, "cluster3", "admin", utilimage.TiDBV3Version)
+		clusterB := newTidbClusterConfig(e2econfig.TestConfig, ns, "cluster4", "admin", utilimage.TiDBV3Version)
 		oa.DeployTidbClusterOrDie(&clusterA)
 		oa.DeployTidbClusterOrDie(&clusterB)
 		oa.CheckTidbClusterStatusOrDie(&clusterA)
@@ -311,7 +305,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 		framework.ExpectNoError(err)
 		err = oa.WaitForTidbClusterReady(tcFrom, 30*time.Minute, 15*time.Second)
 		framework.ExpectNoError(err)
-		clusterFrom := newTidbClusterConfig(e2econfig.TestConfig, ns, tcNameFrom, "", "")
+		clusterFrom := newTidbClusterConfig(e2econfig.TestConfig, ns, tcNameFrom, "", utilimage.TiDBV3Version)
 
 		// create restore cluster
 		tcTo := fixture.GetTidbCluster(ns, tcNameTo, utilimage.TiDBV4Version)
@@ -322,7 +316,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 		framework.ExpectNoError(err)
 		err = oa.WaitForTidbClusterReady(tcTo, 30*time.Minute, 15*time.Second)
 		framework.ExpectNoError(err)
-		clusterTo := newTidbClusterConfig(e2econfig.TestConfig, ns, tcNameTo, "", "")
+		clusterTo := newTidbClusterConfig(e2econfig.TestConfig, ns, tcNameTo, "", utilimage.TiDBV3Version)
 
 		// import some data to sql with blockwriter
 		ginkgo.By(fmt.Sprintf("Begin inserting data into cluster %q", clusterFrom.ClusterName))
@@ -447,7 +441,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 	})
 
 	ginkgo.It("Service: Sync TiDB service", func() {
-		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "service-it", "admin", "")
+		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "service-it", "admin", utilimage.TiDBV3Version)
 		cluster.Resources["pd.replicas"] = "1"
 		cluster.Resources["tidb.replicas"] = "1"
 		cluster.Resources["tikv.replicas"] = "1"
@@ -540,7 +534,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 	// Basic IT for managed in TidbCluster CR
 	// TODO: deploy pump through CR in backup and restore IT
 	ginkgo.It("Pump: Test managing Pump in TidbCluster CRD", func() {
-		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "pump-it", "admin", "")
+		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "pump-it", "admin", utilimage.TiDBV3Version)
 		cluster.Resources["pd.replicas"] = "1"
 		cluster.Resources["tikv.replicas"] = "1"
 		cluster.Resources["tidb.replicas"] = "1"
@@ -675,7 +669,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 	})
 
 	ginkgo.It("API: Migrate from helm to CRD", func() {
-		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "helm-migration", "admin", "")
+		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "helm-migration", "admin", utilimage.TiDBV3Version)
 		cluster.Resources["pd.replicas"] = "1"
 		cluster.Resources["tikv.replicas"] = "1"
 		cluster.Resources["tidb.replicas"] = "1"
@@ -787,7 +781,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 	})
 
 	ginkgo.It("Restarter: Testing restarting by annotations", func() {
-		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "restarter", "admin", "")
+		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "restarter", "admin", utilimage.TiDBV3Version)
 		cluster.Resources["pd.replicas"] = "1"
 		cluster.Resources["tikv.replicas"] = "1"
 		cluster.Resources["tidb.replicas"] = "1"
@@ -885,7 +879,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 	})
 
 	ginkgo.It("TidbMonitor: Deploying and checking monitor", func() {
-		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "monitor-test", "admin", "")
+		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "monitor-test", "admin", utilimage.TiDBV3Version)
 		cluster.Resources["pd.replicas"] = "1"
 		cluster.Resources["tikv.replicas"] = "1"
 		cluster.Resources["tidb.replicas"] = "1"
@@ -1088,7 +1082,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 	})
 
 	ginkgo.It("tidb-scale: clear TiDB failureMembers when scale TiDB to zero", func() {
-		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "tidb-scale", "admin", "")
+		cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "tidb-scale", "admin", utilimage.TiDBV3Version)
 		cluster.Resources["pd.replicas"] = "3"
 		cluster.Resources["tikv.replicas"] = "1"
 		cluster.Resources["tidb.replicas"] = "1"
@@ -1203,7 +1197,7 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 
 			ginkgo.By("Upgrading tidb cluster")
 			err = controller.GuaranteedUpdate(genericCli, tc, func() error {
-				tc.Spec.Version = "nightly"
+				tc.Spec.Version = utilimage.TiDBV4UpgradeVersion
 				return nil
 			})
 			framework.ExpectNoError(err)
@@ -1214,10 +1208,6 @@ var _ = ginkgo.Describe("[tidb-operator] TiDBCluster", func() {
 })
 
 func newTidbClusterConfig(cfg *tests.Config, ns, clusterName, password, tidbVersion string) tests.TidbClusterConfig {
-	if tidbVersion == "" {
-		tidbVersion = cfg.GetTiDBVersionOrDie()
-	}
-	topologyKey := "rack"
 	return tests.TidbClusterConfig{
 		Namespace:        ns,
 		ClusterName:      clusterName,
@@ -1257,7 +1247,7 @@ func newTidbClusterConfig(cfg *tests.Config, ns, clusterName, password, tidbVers
 			BatchSize:   1,
 			RawSize:     1,
 		},
-		TopologyKey:            topologyKey,
+		TopologyKey:            "rack",
 		EnableConfigMapRollout: true,
 		ClusterVersion:         tidbVersion,
 	}