Skip to content

Commit

Permalink
run e2e tests in gke (#1889) (#1901)
Browse files Browse the repository at this point in the history
  • Loading branch information
sre-bot authored Mar 10, 2020
1 parent 8f0bf1a commit 0676858
Show file tree
Hide file tree
Showing 8 changed files with 244 additions and 25 deletions.
136 changes: 136 additions & 0 deletions ci/e2e_gke.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
//
// Jenkins pipeline for GKE e2e job.
//
// This script is written in declarative syntax. Refer to
// https://jenkins.io/doc/book/pipeline/syntax/ for more details.
//
// Note that parameters of the job is configured in this script.
//

import groovy.transform.Field

@Field
def podYAML = '''
apiVersion: v1
kind: Pod
spec:
containers:
- name: main
image: gcr.io/k8s-testimages/kubekins-e2e:v20191108-9467d02-master
command:
- runner.sh
- sleep
- 1d
# we need privileged mode in order to do docker in docker
securityContext:
privileged: true
env:
- name: DOCKER_IN_DOCKER_ENABLED
value: "true"
resources:
requests:
memory: "4000Mi"
cpu: 2000m
volumeMounts:
# dind expects /var/lib/docker to be volume
- name: docker-root
mountPath: /var/lib/docker
volumes:
- name: docker-root
emptyDir: {}
'''

pipeline {
agent {
kubernetes {
yaml podYAML
defaultContainer "main"
customWorkspace "/home/jenkins/agent/workspace/go/src/github.com/pingcap/tidb-operator"
}
}

parameters {
string(name: 'GIT_URL', defaultValue: 'git@github.com:pingcap/tidb-operator.git', description: 'git repo url')
string(name: 'GIT_REF', defaultValue: 'master', description: 'git ref spec to checkout, e.g. master, release-1.1')
string(name: 'PR_ID', defaultValue: '', description: 'pull request ID, this will override GIT_REF if set, e.g. 1889')
string(name: 'CLUSTER', defaultValue: 'jenkins-tidb-operator-e2e', description: 'the name of the cluster')
string(name: 'GCP_PROJECT', defaultValue: 'smooth-tendril-207212', description: 'the GCP project ID')
string(name: 'GCP_ZONE', defaultValue: 'us-central1-b', description: 'the GCP zone')
string(name: 'GINKGO_NODES', defaultValue: '8', description: 'the number of ginkgo nodes')
}

environment {
GIT_REF = ''
ARTIFACTS = "${env.WORKSPACE}/artifacts"
}

stages {
stage("Prepare") {
steps {
// The declarative model for Jenkins Pipelines has a restricted
// subset of syntax that it allows in the stage blocks. We use
// script step to bypass the restriction.
// https://jenkins.io/doc/book/pipeline/syntax/#script
script {
GIT_REF = params.GIT_REF
if (params.PR_ID != "") {
GIT_REF = "refs/remotes/origin/pr/${params.PR_ID}/head"
}
}
echo "env.NODE_NAME: ${env.NODE_NAME}"
echo "env.WORKSPACE: ${env.WORKSPACE}"
echo "GIT_REF: ${GIT_REF}"
echo "ARTIFACTS: ${ARTIFACTS}"
}
}

stage("Checkout") {
steps {
checkout scm: [
$class: 'GitSCM',
branches: [[name: GIT_REF]],
userRemoteConfigs: [[
credentialsId: 'github-sre-bot-ssh',
refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*',
url: "${params.GIT_URL}",
]]
]
}
}

stage("Run") {
steps {
withCredentials([
file(credentialsId: 'TIDB_OPERATOR_GCP_CREDENTIALS', variable: 'GCP_CREDENTIALS'),
file(credentialsId: 'TIDB_OPERATOR_GCP_SSH_PRIVATE_KEY', variable: 'GCP_SSH_PRIVATE_KEY'),
file(credentialsId: 'TIDB_OPERATOR_GCP_SSH_PUBLIC_KEY', variable: 'GCP_SSH_PUBLIC_KEY'),
]) {
sh """
#!/bin/bash
export PROVIDER=gke
export CLUSTER=${params.CLUSTER}
export GCP_ZONE=${params.GCP_ZONE}
export GCP_PROJECT=${params.GCP_PROJECT}
export GINKGO_NODES=${params.GINKGO_NODES}
export REPORT_DIR=${ARTIFACTS}
echo "info: try to clean the cluster created previously"
SKIP_BUILD=y SKIP_IMAGE_BUILD=y SKIP_UP=y SKIP_TEST=y ./hack/e2e.sh
echo "info: begin to run e2e"
./hack/e2e.sh -- --ginkgo.skip='\\[Serial\\]' --ginkgo.focus='\\[tidb-operator\\]'
"""
}
}
}
}

post {
always {
dir(ARTIFACTS) {
archiveArtifacts artifacts: "**", allowEmptyArchive: true
junit testResults: "*.xml", allowEmptyResults: true
}
}
}
}

// vim: et sw=4 ts=4
30 changes: 20 additions & 10 deletions hack/e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,12 @@ Environments:
QUAY_IO_MIRROR configure mirror for quay.io
KIND_DATA_HOSTPATH (kind only) the host path of data directory for kind cluster, defaults: none
GCP_PROJECT (gke only) the GCP project to run in
GCP_SERVICE_ACCOUNT (gke only) the GCP service account to use
GCP_CREDENTIALS (gke only) the GCP service account to use
GCP_REGION (gke only) the GCP region, if specified a regional cluster is creaetd
GCP_ZONE (gke only) the GCP zone, if specified a zonal cluster is created
GCP_SSH_PRIVATE_KEY (gke only) the path to the private ssh key
GCP_SSH_PUBLIC_KEY (gke only) the path to the public ssh key
GCP_MACHINE_TYPE (gke only) the machine type of instance, defaults: n1-standard-4
AWS_ACCESS_KEY_ID (eks only) the aws access key id
AWS_SECRET_ACCESS_KEY (eks only) the aws secret access key
AWS_REGION (eks only) the aws region
Expand Down Expand Up @@ -113,6 +114,7 @@ Examples:
- Kubernetes Engine Admin
- Service Account User
- Storage Admin
- Compute Instance Admin (v1)
You can create ssh keypair with ssh-keygen at ~/.ssh/google_compute_engine
or specifc existing ssh keypair with following environments:
Expand All @@ -123,7 +125,7 @@ Examples:
Then run with following additional GCP-specific environments:
export GCP_PROJECT=<project>
export GCP_SERVICE_ACCOUNT=<path-to-gcp-service-account>
export GCP_CREDENTIALS=<path-to-gcp-service-account>
export GCP_ZONE=us-central1-b
PROVIDER=gke ./hack/e2e.sh -- <e2e args>
Expand Down Expand Up @@ -183,11 +185,12 @@ SKIP_TEST=${SKIP_TEST:-}
REUSE_CLUSTER=${REUSE_CLUSTER:-}
KIND_DATA_HOSTPATH=${KIND_DATA_HOSTPATH:-none}
GCP_PROJECT=${GCP_PROJECT:-}
GCP_SERVICE_ACCOUNT=${GCP_SERVICE_ACCOUNT:-}
GCP_CREDENTIALS=${GCP_CREDENTIALS:-}
GCP_REGION=${GCP_REGION:-}
GCP_ZONE=${GCP_ZONE:-}
GCP_SSH_PRIVATE_KEY=${GCP_SSH_PRIVATE_KEY:-}
GCP_SSH_PUBLIC_KEY=${GCP_SSH_PUBLIC_KEY:-}
GCP_MACHINE_TYPE=${GCP_MACHINE_TYPE:-n1-standard-4}
AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-}
AWS_REGION=${AWS_REGION:-}
Expand All @@ -209,7 +212,7 @@ echo "SKIP_UP: $SKIP_UP"
echo "SKIP_DOWN: $SKIP_DOWN"
echo "KIND_DATA_HOSTPATH: $KIND_DATA_HOSTPATH"
echo "GCP_PROJECT: $GCP_PROJECT"
echo "GCP_SERVICE_ACCOUNT: $GCP_SERVICE_ACCOUNT"
echo "GCP_CREDENTIALS: $GCP_CREDENTIALS"
echo "GCP_REGION: $GCP_REGION"
echo "GCP_ZONE: $GCP_ZONE"
echo "AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID"
Expand Down Expand Up @@ -411,8 +414,8 @@ elif [ "$PROVIDER" == "gke" ]; then
echo "error: GCP_PROJECT is required"
exit 1
fi
if [ -z "$GCP_SERVICE_ACCOUNT" ]; then
echo "error: GCP_SERVICE_ACCOUNT is required"
if [ -z "$GCP_CREDENTIALS" ]; then
echo "error: GCP_CREDENTIALS is required"
exit 1
fi
if [ -z "$GCP_REGION" -a -z "$GCP_ZONE" ]; then
Expand All @@ -426,21 +429,25 @@ elif [ "$PROVIDER" == "gke" ]; then
if [ ! -d ~/.ssh ]; then
mkdir ~/.ssh
fi
if [ ! -e ~/.ssh/google_compute_engine -o -n "$GCP_SSH_PRIVATE_KEY" ]; then
if [ ! -e ~/.ssh/google_compute_engine -a -n "$GCP_SSH_PRIVATE_KEY" ]; then
echo "Copying $GCP_SSH_PRIVATE_KEY to ~/.ssh/google_compute_engine" >&2
cp $GCP_SSH_PRIVATE_KEY ~/.ssh/google_compute_engine
chmod 0600 ~/.ssh/google_compute_engine
fi
if [ ! -e ~/.ssh/google_compute_engine.pub -o -n "$GCP_SSH_PUBLIC_KEY" ]; then
if [ ! -e ~/.ssh/google_compute_engine.pub -a -n "$GCP_SSH_PUBLIC_KEY" ]; then
echo "Copying $GCP_SSH_PUBLIC_KEY to ~/.ssh/google_compute_engine.pub" >&2
cp $GCP_SSH_PUBLIC_KEY ~/.ssh/google_compute_engine.pub
chmod 0600 ~/.ssh/google_compute_engine.pub
fi
! read -r -d '' nodePoolsJSON <<EOF
{"default":{"Nodes":${KUBE_WORKERS},"MachineType":"${GCP_MACHINE_TYPE}"}}
EOF
kubetest2_args+=(
--cluster-name "$CLUSTER"
--project "$GCP_PROJECT"
--gcp-service-account "$GCP_SERVICE_ACCOUNT"
--gcp-service-account "$GCP_CREDENTIALS"
--environment prod
--node-pools "$nodePoolsJSON"
)
if [ -n "$GCP_REGION" ]; then
kubetest2_args+=(
Expand Down Expand Up @@ -486,10 +493,13 @@ export PROVIDER
export CLUSTER
export KUBECONFIG
export GCP_PROJECT
export GCP_REGION
export GCP_ZONE
export GCP_CREDENTIALS
export IMAGE_TAG
export TIDB_OPERATOR_IMAGE=$DOCKER_REGISTRY/pingcap/tidb-operator:${IMAGE_TAG}
export E2E_IMAGE=$DOCKER_REGISTRY/pingcap/tidb-operator-e2e:${IMAGE_TAG}
export PATH=$PATH:$OUTPUT_BIN
export PATH=$OUTPUT_BIN:$PATH

# Environments for kubetest2
if [ -n "${REPORT_DIR:-}" ]; then
Expand Down
2 changes: 1 addition & 1 deletion hack/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ HELM_BIN=$OUTPUT_BIN/helm
HELM_VERSION=${HELM_VERSION:-2.9.1}
KIND_VERSION=${KIND_VERSION:-0.7.0}
KIND_BIN=$OUTPUT_BIN/kind
KUBETEST2_VERSION=v0.0.7
KUBETEST2_VERSION=v0.0.8
KUBETSTS2_BIN=$OUTPUT_BIN/kubetest2
AWS_K8S_TESTER_VERSION=v0.6.2
AWS_K8S_TESTER_BIN=$OUTPUT_BIN/aws-k8s-tester
Expand Down
40 changes: 36 additions & 4 deletions hack/run-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ source $ROOT/hack/lib.sh
PROVIDER=${PROVIDER:-}
CLUSTER=${CLUSTER:-}
GCP_PROJECT=${GCP_PROJECT:-}
GCP_REGION=${GCP_REGION:-}
GCP_ZONE=${GCP_ZONE:-}
GCP_CREDENTIALS=${GCP_CREDENTIALS:-}
IMAGE_TAG=${IMAGE_TAG:-}
TIDB_OPERATOR_IMAGE=${TIDB_OPERATOR_IMAGE:-localhost:5000/pingcap/tidb-operator:latest}
E2E_IMAGE=${E2E_IMAGE:-localhost:5000/pingcap/tidb-operator-e2e:latest}
Expand Down Expand Up @@ -117,8 +120,8 @@ EOF
elif [ "$PROVIDER" == "gke" ]; then
# disks are created under /mnt/stateful_partition directory
# https://cloud.google.com/container-optimized-os/docs/concepts/disks-and-filesystem
for n in $($KUBECTL_BIN --context $KUBECONTEXT get nodes -ojsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
gcloud compute ssh $n --command 'sudo bash -c '"'"'
for n in $($KUBECTL_BIN --context "$KUBECONTEXT" get nodes -ojsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
gcloud compute ssh e2e@$n --command 'sudo bash -c '"'"'
test -d /mnt/stateful_partition/disks || mkdir -p /mnt/stateful_partition/disks
df -h /mnt/stateful_partition/disks
test -d /mnt/disks || mkdir -p /mnt/disks
Expand Down Expand Up @@ -256,9 +259,28 @@ function e2e::image_load() {
hack::ensure_kubectl
hack::ensure_helm

if [ "$PROVIDER" == "gke" ]; then
if [ -n "$GCP_CREDENTIALS" ]; then
gcloud auth activate-service-account --key-file "$GCP_CREDENTIALS"
fi
if [ -n "$GCP_REGION" ]; then
gcloud config set compute/region "$GCP_REGION"
fi
if [ -n "$GCP_ZONE" ]; then
gcloud config set compute/zone "$GCP_ZONE"
fi
gcloud container clusters get-credentials "$CLUSTER"
elif [ "$PROVIDER" == "eks" ]; then
:
fi

if [ -z "$KUBECONTEXT" ]; then
KUBECONTEXT=$(kubectl config current-context)
echo "info: KUBECONTEXT is not set, current context $KUBECONTEXT is used"
KUBECONTEXT=$(kubectl config current-context 2>/dev/null) || true
if [ -z "$KUBECONTEXT" ]; then
echo "error: KUBECONTEXT cannot be detected"
exit 1
fi
fi

e2e::image_load
Expand Down Expand Up @@ -288,7 +310,7 @@ e2e_args=(
${ginkgo_args[@]:-}
/usr/local/bin/e2e.test
--
--provider=skeleton
--provider=${PROVIDER}
--clean-start=true
--delete-namespace-on-failure=false
--repo-root=$ROOT
Expand Down Expand Up @@ -329,6 +351,16 @@ if [ "$PROVIDER" == "eks" ]; then
docker_args+=(
-v $HOME/.aws:/root/.aws
)
elif [ "$PROVIDER" == "gke" ]; then
e2e_args+=(
--gce-project ${GCP_PROJECT}
--gce-region ${GCP_REGION}
--gce-zone ${GCP_ZONE}
)
docker_args+=(
-v ${GCP_CREDENTIALS}:${GCP_CREDENTIALS}
--env GOOGLE_APPLICATION_CREDENTIALS=${GCP_CREDENTIALS}
)
fi

if [ -n "$REPORT_DIR" ]; then
Expand Down
35 changes: 35 additions & 0 deletions tests/e2e/e2e.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
e2econfig "github.com/pingcap/tidb-operator/tests/e2e/config"
utilimage "github.com/pingcap/tidb-operator/tests/e2e/util/image"
v1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
runtimeutils "k8s.io/apimachinery/pkg/util/runtime"
Expand All @@ -43,6 +44,7 @@ import (
"k8s.io/component-base/logs"
"k8s.io/klog"
aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
storageutil "k8s.io/kubernetes/pkg/apis/storage/v1/util"
"k8s.io/kubernetes/test/e2e/framework"
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
Expand Down Expand Up @@ -120,6 +122,39 @@ func setupSuite() {
e2elog.Logf("WARNING: Waiting for all daemonsets to be ready failed: %v", err)
}

// By using default storage class in GKE/EKS, network attached storage
// which be used and we must clean them later.
// We set local-storage class as default for simplicity.
// The default storage class of kind is local-path-provisioner which
// consumes local storage like local-volume-provisioner.
if framework.TestContext.Provider == "gke" || framework.TestContext.Provider == "eks" {
defaultSCName := "local-storage"
list, err := c.StorageV1().StorageClasses().List(metav1.ListOptions{})
framework.ExpectNoError(err)
// only one storage class can be marked default
// https://kubernetes.io/docs/tasks/administer-cluster/change-default-storage-class/#changing-the-default-storageclass
var localStorageSC *storagev1.StorageClass
for i, sc := range list.Items {
if sc.Name == defaultSCName {
localStorageSC = &list.Items[i]
} else if storageutil.IsDefaultAnnotation(sc.ObjectMeta) {
delete(sc.ObjectMeta.Annotations, storageutil.IsDefaultStorageClassAnnotation)
_, err = c.StorageV1().StorageClasses().Update(&sc)
framework.ExpectNoError(err)
}
}
if localStorageSC == nil {
e2elog.Fail("local-storage storage class not found")
}
if localStorageSC.Annotations == nil {
localStorageSC.Annotations = map[string]string{}
}
localStorageSC.Annotations[storageutil.IsDefaultStorageClassAnnotation] = "true"
e2elog.Logf("Setting %q as the default storage class", localStorageSC.Name)
_, err = c.StorageV1().StorageClasses().Update(localStorageSC)
framework.ExpectNoError(err)
}

// Log the version of the server and this client.
e2elog.Logf("e2e test version: %s", version.Get().GitVersion)

Expand Down
Loading

0 comments on commit 0676858

Please sign in to comment.