diff --git a/ci/e2e_gke.groovy b/ci/e2e_gke.groovy new file mode 100644 index 0000000000..12fb0d429c --- /dev/null +++ b/ci/e2e_gke.groovy @@ -0,0 +1,136 @@ +// +// Jenkins pipeline for GKE e2e job. +// +// This script is written in declarative syntax. Refer to +// https://jenkins.io/doc/book/pipeline/syntax/ for more details. +// +// Note that parameters of the job is configured in this script. +// + +import groovy.transform.Field + +@Field +def podYAML = ''' +apiVersion: v1 +kind: Pod +spec: + containers: + - name: main + image: gcr.io/k8s-testimages/kubekins-e2e:v20191108-9467d02-master + command: + - runner.sh + - sleep + - 1d + # we need privileged mode in order to do docker in docker + securityContext: + privileged: true + env: + - name: DOCKER_IN_DOCKER_ENABLED + value: "true" + resources: + requests: + memory: "4000Mi" + cpu: 2000m + volumeMounts: + # dind expects /var/lib/docker to be volume + - name: docker-root + mountPath: /var/lib/docker + volumes: + - name: docker-root + emptyDir: {} +''' + +pipeline { + agent { + kubernetes { + yaml podYAML + defaultContainer "main" + customWorkspace "/home/jenkins/agent/workspace/go/src/github.com/pingcap/tidb-operator" + } + } + + parameters { + string(name: 'GIT_URL', defaultValue: 'git@github.com:pingcap/tidb-operator.git', description: 'git repo url') + string(name: 'GIT_REF', defaultValue: 'master', description: 'git ref spec to checkout, e.g. master, release-1.1') + string(name: 'PR_ID', defaultValue: '', description: 'pull request ID, this will override GIT_REF if set, e.g. 1889') + string(name: 'CLUSTER', defaultValue: 'jenkins-tidb-operator-e2e', description: 'the name of the cluster') + string(name: 'GCP_PROJECT', defaultValue: 'smooth-tendril-207212', description: 'the GCP project ID') + string(name: 'GCP_ZONE', defaultValue: 'us-central1-b', description: 'the GCP zone') + string(name: 'GINKGO_NODES', defaultValue: '8', description: 'the number of ginkgo nodes') + } + + environment { + GIT_REF = '' + ARTIFACTS = "${env.WORKSPACE}/artifacts" + } + + stages { + stage("Prepare") { + steps { + // The declarative model for Jenkins Pipelines has a restricted + // subset of syntax that it allows in the stage blocks. We use + // script step to bypass the restriction. + // https://jenkins.io/doc/book/pipeline/syntax/#script + script { + GIT_REF = params.GIT_REF + if (params.PR_ID != "") { + GIT_REF = "refs/remotes/origin/pr/${params.PR_ID}/head" + } + } + echo "env.NODE_NAME: ${env.NODE_NAME}" + echo "env.WORKSPACE: ${env.WORKSPACE}" + echo "GIT_REF: ${GIT_REF}" + echo "ARTIFACTS: ${ARTIFACTS}" + } + } + + stage("Checkout") { + steps { + checkout scm: [ + $class: 'GitSCM', + branches: [[name: GIT_REF]], + userRemoteConfigs: [[ + credentialsId: 'github-sre-bot-ssh', + refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*', + url: "${params.GIT_URL}", + ]] + ] + } + } + + stage("Run") { + steps { + withCredentials([ + file(credentialsId: 'TIDB_OPERATOR_GCP_CREDENTIALS', variable: 'GCP_CREDENTIALS'), + file(credentialsId: 'TIDB_OPERATOR_GCP_SSH_PRIVATE_KEY', variable: 'GCP_SSH_PRIVATE_KEY'), + file(credentialsId: 'TIDB_OPERATOR_GCP_SSH_PUBLIC_KEY', variable: 'GCP_SSH_PUBLIC_KEY'), + ]) { + sh """ + #!/bin/bash + export PROVIDER=gke + export CLUSTER=${params.CLUSTER} + export GCP_ZONE=${params.GCP_ZONE} + export GCP_PROJECT=${params.GCP_PROJECT} + export GINKGO_NODES=${params.GINKGO_NODES} + export REPORT_DIR=${ARTIFACTS} + echo "info: try to clean the cluster created previously" + SKIP_BUILD=y SKIP_IMAGE_BUILD=y SKIP_UP=y SKIP_TEST=y ./hack/e2e.sh + echo "info: begin to run e2e" + ./hack/e2e.sh -- --ginkgo.skip='\\[Serial\\]' --ginkgo.focus='\\[tidb-operator\\]' + """ + } + } + } + } + + post { + always { + dir(ARTIFACTS) { + archiveArtifacts artifacts: "**", allowEmptyArchive: true + junit testResults: "*.xml", allowEmptyResults: true + } + } + } +} + +// vim: et sw=4 ts=4 diff --git a/hack/e2e.sh b/hack/e2e.sh index 4bb01c4fdc..3e29c60268 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -64,11 +64,12 @@ Environments: QUAY_IO_MIRROR configure mirror for quay.io KIND_DATA_HOSTPATH (kind only) the host path of data directory for kind cluster, defaults: none GCP_PROJECT (gke only) the GCP project to run in - GCP_SERVICE_ACCOUNT (gke only) the GCP service account to use + GCP_CREDENTIALS (gke only) the GCP service account to use GCP_REGION (gke only) the GCP region, if specified a regional cluster is creaetd GCP_ZONE (gke only) the GCP zone, if specified a zonal cluster is created GCP_SSH_PRIVATE_KEY (gke only) the path to the private ssh key GCP_SSH_PUBLIC_KEY (gke only) the path to the public ssh key + GCP_MACHINE_TYPE (gke only) the machine type of instance, defaults: n1-standard-4 AWS_ACCESS_KEY_ID (eks only) the aws access key id AWS_SECRET_ACCESS_KEY (eks only) the aws secret access key AWS_REGION (eks only) the aws region @@ -115,6 +116,7 @@ Examples: - Kubernetes Engine Admin - Service Account User - Storage Admin + - Compute Instance Admin (v1) You can create ssh keypair with ssh-keygen at ~/.ssh/google_compute_engine or specifc existing ssh keypair with following environments: @@ -125,7 +127,7 @@ Examples: Then run with following additional GCP-specific environments: export GCP_PROJECT= - export GCP_SERVICE_ACCOUNT= + export GCP_CREDENTIALS= export GCP_ZONE=us-central1-b PROVIDER=gke ./hack/e2e.sh -- @@ -186,11 +188,12 @@ SKIP_TEST=${SKIP_TEST:-} REUSE_CLUSTER=${REUSE_CLUSTER:-} KIND_DATA_HOSTPATH=${KIND_DATA_HOSTPATH:-none} GCP_PROJECT=${GCP_PROJECT:-} -GCP_SERVICE_ACCOUNT=${GCP_SERVICE_ACCOUNT:-} +GCP_CREDENTIALS=${GCP_CREDENTIALS:-} GCP_REGION=${GCP_REGION:-} GCP_ZONE=${GCP_ZONE:-} GCP_SSH_PRIVATE_KEY=${GCP_SSH_PRIVATE_KEY:-} GCP_SSH_PUBLIC_KEY=${GCP_SSH_PUBLIC_KEY:-} +GCP_MACHINE_TYPE=${GCP_MACHINE_TYPE:-n1-standard-4} AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} AWS_REGION=${AWS_REGION:-} @@ -213,7 +216,7 @@ echo "SKIP_UP: $SKIP_UP" echo "SKIP_DOWN: $SKIP_DOWN" echo "KIND_DATA_HOSTPATH: $KIND_DATA_HOSTPATH" echo "GCP_PROJECT: $GCP_PROJECT" -echo "GCP_SERVICE_ACCOUNT: $GCP_SERVICE_ACCOUNT" +echo "GCP_CREDENTIALS: $GCP_CREDENTIALS" echo "GCP_REGION: $GCP_REGION" echo "GCP_ZONE: $GCP_ZONE" echo "AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID" @@ -415,8 +418,8 @@ elif [ "$PROVIDER" == "gke" ]; then echo "error: GCP_PROJECT is required" exit 1 fi - if [ -z "$GCP_SERVICE_ACCOUNT" ]; then - echo "error: GCP_SERVICE_ACCOUNT is required" + if [ -z "$GCP_CREDENTIALS" ]; then + echo "error: GCP_CREDENTIALS is required" exit 1 fi if [ -z "$GCP_REGION" -a -z "$GCP_ZONE" ]; then @@ -430,21 +433,25 @@ elif [ "$PROVIDER" == "gke" ]; then if [ ! -d ~/.ssh ]; then mkdir ~/.ssh fi - if [ ! -e ~/.ssh/google_compute_engine -o -n "$GCP_SSH_PRIVATE_KEY" ]; then + if [ ! -e ~/.ssh/google_compute_engine -a -n "$GCP_SSH_PRIVATE_KEY" ]; then echo "Copying $GCP_SSH_PRIVATE_KEY to ~/.ssh/google_compute_engine" >&2 cp $GCP_SSH_PRIVATE_KEY ~/.ssh/google_compute_engine chmod 0600 ~/.ssh/google_compute_engine fi - if [ ! -e ~/.ssh/google_compute_engine.pub -o -n "$GCP_SSH_PUBLIC_KEY" ]; then + if [ ! -e ~/.ssh/google_compute_engine.pub -a -n "$GCP_SSH_PUBLIC_KEY" ]; then echo "Copying $GCP_SSH_PUBLIC_KEY to ~/.ssh/google_compute_engine.pub" >&2 cp $GCP_SSH_PUBLIC_KEY ~/.ssh/google_compute_engine.pub chmod 0600 ~/.ssh/google_compute_engine.pub fi + ! read -r -d '' nodePoolsJSON </dev/null) || true + if [ -z "$KUBECONTEXT" ]; then + echo "error: KUBECONTEXT cannot be detected" + exit 1 + fi fi if [ -z "$SKIP_IMAGE_LOAD" ]; then @@ -298,7 +320,7 @@ e2e_args=( ${ginkgo_args[@]:-} /usr/local/bin/e2e.test -- - --provider=skeleton + --provider=${PROVIDER} --clean-start=true --delete-namespace-on-failure=false --repo-root=$ROOT @@ -339,6 +361,16 @@ if [ "$PROVIDER" == "eks" ]; then docker_args+=( -v $HOME/.aws:/root/.aws ) +elif [ "$PROVIDER" == "gke" ]; then + e2e_args+=( + --gce-project ${GCP_PROJECT} + --gce-region ${GCP_REGION} + --gce-zone ${GCP_ZONE} + ) + docker_args+=( + -v ${GCP_CREDENTIALS}:${GCP_CREDENTIALS} + --env GOOGLE_APPLICATION_CREDENTIALS=${GCP_CREDENTIALS} + ) fi if [ -n "$REPORT_DIR" ]; then diff --git a/tests/e2e/e2e.go b/tests/e2e/e2e.go index 0e838d254c..0c7f927d77 100644 --- a/tests/e2e/e2e.go +++ b/tests/e2e/e2e.go @@ -35,6 +35,7 @@ import ( e2econfig "github.com/pingcap/tidb-operator/tests/e2e/config" utilimage "github.com/pingcap/tidb-operator/tests/e2e/util/image" v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtimeutils "k8s.io/apimachinery/pkg/util/runtime" @@ -43,6 +44,7 @@ import ( "k8s.io/component-base/logs" "k8s.io/klog" aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + storageutil "k8s.io/kubernetes/pkg/apis/storage/v1/util" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -120,6 +122,39 @@ func setupSuite() { e2elog.Logf("WARNING: Waiting for all daemonsets to be ready failed: %v", err) } + // By using default storage class in GKE/EKS, network attached storage + // which be used and we must clean them later. + // We set local-storage class as default for simplicity. + // The default storage class of kind is local-path-provisioner which + // consumes local storage like local-volume-provisioner. + if framework.TestContext.Provider == "gke" || framework.TestContext.Provider == "eks" { + defaultSCName := "local-storage" + list, err := c.StorageV1().StorageClasses().List(metav1.ListOptions{}) + framework.ExpectNoError(err) + // only one storage class can be marked default + // https://kubernetes.io/docs/tasks/administer-cluster/change-default-storage-class/#changing-the-default-storageclass + var localStorageSC *storagev1.StorageClass + for i, sc := range list.Items { + if sc.Name == defaultSCName { + localStorageSC = &list.Items[i] + } else if storageutil.IsDefaultAnnotation(sc.ObjectMeta) { + delete(sc.ObjectMeta.Annotations, storageutil.IsDefaultStorageClassAnnotation) + _, err = c.StorageV1().StorageClasses().Update(&sc) + framework.ExpectNoError(err) + } + } + if localStorageSC == nil { + e2elog.Fail("local-storage storage class not found") + } + if localStorageSC.Annotations == nil { + localStorageSC.Annotations = map[string]string{} + } + localStorageSC.Annotations[storageutil.IsDefaultStorageClassAnnotation] = "true" + e2elog.Logf("Setting %q as the default storage class", localStorageSC.Name) + _, err = c.StorageV1().StorageClasses().Update(localStorageSC) + framework.ExpectNoError(err) + } + // Log the version of the server and this client. e2elog.Logf("e2e test version: %s", version.Get().GitVersion) diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index 15f2546fe1..5a49655b5a 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -42,6 +42,12 @@ func handleFlags() { flag.Parse() } +func init() { + framework.RegisterProvider("kind", func() (framework.ProviderInterface, error) { + return framework.NullProvider{}, nil + }) +} + func TestMain(m *testing.M) { // Register test flags, then parse flags. handleFlags() diff --git a/tests/e2e/tidbcluster/tidbcluster.go b/tests/e2e/tidbcluster/tidbcluster.go index cf10d24507..b450875b73 100644 --- a/tests/e2e/tidbcluster/tidbcluster.go +++ b/tests/e2e/tidbcluster/tidbcluster.go @@ -897,16 +897,16 @@ func newTidbClusterConfig(cfg *tests.Config, ns, clusterName, password, tidbVers Resources: map[string]string{ "pd.resources.limits.cpu": "1000m", "pd.resources.limits.memory": "2Gi", - "pd.resources.requests.cpu": "200m", - "pd.resources.requests.memory": "200Mi", + "pd.resources.requests.cpu": "20m", + "pd.resources.requests.memory": "20Mi", "tikv.resources.limits.cpu": "2000m", "tikv.resources.limits.memory": "4Gi", - "tikv.resources.requests.cpu": "200m", - "tikv.resources.requests.memory": "200Mi", + "tikv.resources.requests.cpu": "20m", + "tikv.resources.requests.memory": "20Mi", "tidb.resources.limits.cpu": "2000m", "tidb.resources.limits.memory": "4Gi", - "tidb.resources.requests.cpu": "200m", - "tidb.resources.requests.memory": "200Mi", + "tidb.resources.requests.cpu": "20m", + "tidb.resources.requests.memory": "20Mi", "tidb.initSql": strconv.Quote("create database e2e;"), "discovery.image": cfg.OperatorImage, }, diff --git a/tests/pkg/fixture/fixture.go b/tests/pkg/fixture/fixture.go index a8295fae81..85f37707cf 100644 --- a/tests/pkg/fixture/fixture.go +++ b/tests/pkg/fixture/fixture.go @@ -25,8 +25,8 @@ var ( BestEffort = corev1.ResourceRequirements{} BurstbleSmall = corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("200m"), - corev1.ResourceMemory: resource.MustParse("200Mi"), + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1000m"), @@ -35,8 +35,8 @@ var ( } BurstbleMedium = corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("200m"), - corev1.ResourceMemory: resource.MustParse("200Mi"), + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("2000m"),