-
Notifications
You must be signed in to change notification settings - Fork 498
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
- Loading branch information
Showing
9 changed files
with
287 additions
and
38 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Copyright 2020 PingCAP, Inc. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| # | ||
| # aws-k8s-tester cannot clean all resources created when some error happened. | ||
| # This script is used to clean resources created by aws-k8s-tester in our CI. | ||
| # | ||
| # DO NOT USE THIS SCRIPT FOR OTHER USES! | ||
| # | ||
|
|
||
| function get_stacks() { | ||
| aws cloudformation list-stacks --stack-status-filter CREATE_COMPLETE DELETE_FAILED --query 'StackSummaries[*].StackName' --output text | ||
| } | ||
|
|
||
| function fix_eks_mng_deletion_issues() { | ||
| local cluster="$1" | ||
| local mng="$2" | ||
| while IFS=$'\n' read -r line; do | ||
| read -r code resourceIds <<< $line | ||
| if [ "$code" == "Ec2SecurityGroupDeletionFailure" ]; then | ||
| echo "info: clear security group '$resourceIds'" | ||
| for eni in $(aws ec2 describe-network-interfaces --filters "Name=group-id,Values=$resourceIds" --query 'NetworkInterfaces[*].NetworkInterfaceId' --output text); do | ||
| echo "info: clear leaked network interfaces '$eni'" | ||
| aws ec2 delete-network-interface --network-interface-id "$eni" | ||
| done | ||
| aws ec2 delete-security-group --group-id $resourceIds | ||
| fi | ||
| done <<< $(aws eks describe-nodegroup --cluster-name "$cluster" --nodegroup-name "$mng" --query 'nodegroup.health.issues' --output json | jq -r '.[].resourceIds |= join(",") | .[] | "\(.code)\t\(.resourceIds)"') | ||
| } | ||
|
|
||
| function clean_eks() { | ||
| local CLUSTER="$1" | ||
| echo "info: deleting mng stack" | ||
| local regex='^'$CLUSTER'-mng-[0-9]+$' | ||
| local mngStack= | ||
| for stackName in $(get_stacks); do | ||
| if [[ ! "$stackName" =~ $regex ]]; then | ||
| continue | ||
| fi | ||
| mngStack=$stackName | ||
| break | ||
| done | ||
| if [ -n "$mngStack" ]; then | ||
| echo "info: mng stack found '$mngStack', deleting it" | ||
| aws cloudformation delete-stack --stack-name $mngStack | ||
| aws cloudformation wait stack-delete-complete --stack-name $mngStack | ||
| if [ $? -ne 0 ]; then | ||
| echo "error: failed to delete mng stack '$mngStack', delete related resource first" | ||
| for mngName in $(aws eks list-nodegroups --cluster-name jenkins-tidb-operator-e2e2 --query 'nodegroups[*]' --output text); do | ||
| fix_eks_mng_deletion_issues "$CLUSTER" $mngName | ||
| done | ||
| aws cloudformation delete-stack --stack-name $mngStack | ||
| aws cloudformation wait stack-delete-complete --stack-name $mngStack | ||
| fi | ||
| else | ||
| echo "info: mng stack not found, skipped" | ||
| fi | ||
|
|
||
| echo "info: deleting cluster/cluster-role/mng-role/vpc stacks" | ||
| local stacks=( | ||
| $CLUSTER-cluster | ||
| $CLUSTER-role-cluster | ||
| $CLUSTER-role-mng | ||
| $CLUSTER-vpc | ||
| ) | ||
| for stack in ${stacks[@]}; do | ||
| echo "info: deleting stack $stack" | ||
| aws cloudformation delete-stack --stack-name $stack | ||
| aws cloudformation wait stack-delete-complete --stack-name $stack | ||
| done | ||
| } | ||
|
|
||
| # https://github.com/aws/aws-cli#other-configurable-variables | ||
| if [ -n "${AWS_REGION}" ]; then | ||
| export AWS_DEFAULT_REGION=${AWS_REGION:-} | ||
| fi | ||
|
|
||
| aws sts get-caller-identity | ||
| if [ $? -ne 0 ]; then | ||
| echo "error: failed to get caller identity" | ||
| exit 1 | ||
| fi | ||
|
|
||
| for CLUSTER in $@; do | ||
| echo "info: start to clean eks test cluster '$CLUSTER'" | ||
| clean_eks "$CLUSTER" | ||
| if [ $? -eq 0 ]; then | ||
| echo "info: succesfully cleaned the eks test cluster '$CLUSTER'" | ||
| else | ||
| echo "fatal: failed to clean the eks test cluster '$CLUSTER'" | ||
| exit 1 | ||
| fi | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| // | ||
| // Jenkins pipeline for EKS e2e job. | ||
| // | ||
| // This script is written in declarative syntax. Refer to | ||
| // https://jenkins.io/doc/book/pipeline/syntax/ for more details. | ||
| // | ||
| // Note that parameters of the job is configured in this script. | ||
| // | ||
|
|
||
| import groovy.transform.Field | ||
|
|
||
| @Field | ||
| def podYAML = ''' | ||
| apiVersion: v1 | ||
| kind: Pod | ||
| spec: | ||
| containers: | ||
| - name: main | ||
| image: gcr.io/k8s-testimages/kubekins-e2e:v20200311-1e25827-master | ||
| command: | ||
| - runner.sh | ||
| - sleep | ||
| - 1d | ||
| # we need privileged mode in order to do docker in docker | ||
| securityContext: | ||
| privileged: true | ||
| env: | ||
| - name: DOCKER_IN_DOCKER_ENABLED | ||
| value: "true" | ||
| resources: | ||
| requests: | ||
| memory: "4000Mi" | ||
| cpu: 2000m | ||
| volumeMounts: | ||
| # dind expects /var/lib/docker to be volume | ||
| - name: docker-root | ||
| mountPath: /var/lib/docker | ||
| volumes: | ||
| - name: docker-root | ||
| emptyDir: {} | ||
| ''' | ||
|
|
||
| pipeline { | ||
| agent { | ||
| kubernetes { | ||
| yaml podYAML | ||
| defaultContainer "main" | ||
| customWorkspace "/home/jenkins/agent/workspace/go/src/github.com/pingcap/tidb-operator" | ||
| } | ||
| } | ||
|
|
||
| options { | ||
| timeout(time: 3, unit: 'HOURS') | ||
| } | ||
|
|
||
| parameters { | ||
| string(name: 'GIT_URL', defaultValue: 'git@github.com:pingcap/tidb-operator.git', description: 'git repo url') | ||
| string(name: 'GIT_REF', defaultValue: 'master', description: 'git ref spec to checkout, e.g. master, release-1.1') | ||
| string(name: 'PR_ID', defaultValue: '', description: 'pull request ID, this will override GIT_REF if set, e.g. 1889') | ||
| string(name: 'CLUSTER', defaultValue: 'jenkins-tidb-operator-e2e', description: 'the name of the cluster') | ||
| string(name: 'AWS_REGION', defaultValue: 'us-west-2', description: 'the AWS region') | ||
| string(name: 'GINKGO_NODES', defaultValue: '8', description: 'the number of ginkgo nodes') | ||
| } | ||
|
|
||
| environment { | ||
| GIT_REF = '' | ||
| ARTIFACTS = "${env.WORKSPACE}/artifacts" | ||
| } | ||
|
|
||
| stages { | ||
| stage("Prepare") { | ||
| steps { | ||
| // The declarative model for Jenkins Pipelines has a restricted | ||
| // subset of syntax that it allows in the stage blocks. We use | ||
| // script step to bypass the restriction. | ||
| // https://jenkins.io/doc/book/pipeline/syntax/#script | ||
| script { | ||
| GIT_REF = params.GIT_REF | ||
| if (params.PR_ID != "") { | ||
| GIT_REF = "refs/remotes/origin/pr/${params.PR_ID}/head" | ||
| } | ||
| } | ||
| echo "env.NODE_NAME: ${env.NODE_NAME}" | ||
| echo "env.WORKSPACE: ${env.WORKSPACE}" | ||
| echo "GIT_REF: ${GIT_REF}" | ||
| echo "ARTIFACTS: ${ARTIFACTS}" | ||
| } | ||
| } | ||
|
|
||
| stage("Checkout") { | ||
| steps { | ||
| checkout scm: [ | ||
| $class: 'GitSCM', | ||
| branches: [[name: GIT_REF]], | ||
| userRemoteConfigs: [[ | ||
| credentialsId: 'github-sre-bot-ssh', | ||
| refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*', | ||
| url: "${params.GIT_URL}", | ||
| ]] | ||
| ] | ||
| } | ||
| } | ||
|
|
||
| stage("Run") { | ||
| steps { | ||
| withCredentials([ | ||
| string(credentialsId: 'TIDB_OPERATOR_AWS_ACCESS_KEY_ID', variable: 'AWS_ACCESS_KEY_ID'), | ||
| string(credentialsId: 'TIDB_OPERATOR_AWS_SECRET_ACCESS_KEY', variable: 'AWS_SECRET_ACCESS_KEY'), | ||
| ]) { | ||
| sh """ | ||
| #!/bin/bash | ||
| export PROVIDER=eks | ||
| export CLUSTER=${params.CLUSTER} | ||
| export AWS_REGION=${params.AWS_REGION} | ||
| export GINKGO_NODES=${params.GINKGO_NODES} | ||
| export REPORT_DIR=${ARTIFACTS} | ||
| echo "info: try to clean the cluster created previously" | ||
| ./ci/aws-clean-eks.sh \$CLUSTER | ||
| echo "info: begin to run e2e" | ||
| ./hack/e2e.sh -- --ginkgo.skip='\\[Serial\\]' --ginkgo.focus='\\[tidb-operator\\]' | ||
| """ | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| post { | ||
| always { | ||
| dir(ARTIFACTS) { | ||
| archiveArtifacts artifacts: "**", allowEmptyArchive: true | ||
| junit testResults: "*.xml", allowEmptyResults: true | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // vim: et sw=4 ts=4 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.