From da683864aef544ea6e6fa921c727d52c3d1bcfcd Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Fri, 10 Apr 2020 21:17:40 +0800 Subject: [PATCH] scripts to run e2e against OpenShift 4 (#2141) --- .../tidb-cluster/templates/monitor-rbac.yaml | 10 ++ .../admission-webhook-deployment.yaml | 17 +- .../admission/admission-webhook-service.yaml | 2 +- .../templates/controller-manager-rbac.yaml | 23 +-- ci/run-in-vm.sh | 159 ++++++++++++++++++ ci/vm.groovy | 144 ++++++++++++++++ hack/e2e-openshift.sh | 130 ++++++++++++++ hack/run-e2e.sh | 22 +++ hack/run-in-container.sh | 9 +- pkg/monitor/monitor/monitor_manager.go | 57 +++++-- pkg/monitor/monitor/util.go | 24 +-- tests/actions.go | 12 +- tests/e2e/e2e.go | 44 +++-- tests/e2e/e2e_test.go | 3 + 14 files changed, 587 insertions(+), 69 deletions(-) create mode 100755 ci/run-in-vm.sh create mode 100644 ci/vm.groovy create mode 100755 hack/e2e-openshift.sh diff --git a/charts/tidb-cluster/templates/monitor-rbac.yaml b/charts/tidb-cluster/templates/monitor-rbac.yaml index 1bb7aca7b2..628100b7c1 100644 --- a/charts/tidb-cluster/templates/monitor-rbac.yaml +++ b/charts/tidb-cluster/templates/monitor-rbac.yaml @@ -23,6 +23,16 @@ rules: resources: - pods verbs: ["get", "list", "watch"] + {{- if .Capabilities.APIVersions.Has "security.openshift.io/v1" }} +- apiGroups: + - security.openshift.io + resourceNames: + - anyuid + resources: + - securitycontextconstraints + verbs: + - use + {{- end }} {{- if .Values.rbac.crossNamespace }} - nonResourceURLs: ["/metrics"] verbs: ["get"] diff --git a/charts/tidb-operator/templates/admission/admission-webhook-deployment.yaml b/charts/tidb-operator/templates/admission/admission-webhook-deployment.yaml index c5d98d2b4d..1efd651728 100644 --- a/charts/tidb-operator/templates/admission/admission-webhook-deployment.yaml +++ b/charts/tidb-operator/templates/admission/admission-webhook-deployment.yaml @@ -30,6 +30,8 @@ spec: imagePullPolicy: {{ .Values.imagePullPolicy | default "IfNotPresent" }} command: - /usr/local/bin/tidb-admission-webhook + # use > 1024 port, then we can run it as non-root user + - --secure-port=6443 {{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }} - --tls-cert-file=/var/serving-cert/tls.crt - --tls-private-key-file=/var/serving-cert/tls.key @@ -41,7 +43,7 @@ spec: failureThreshold: 5 httpGet: path: /healthz - port: 443 + port: 6443 scheme: HTTPS initialDelaySeconds: 5 timeoutSeconds: 5 @@ -49,7 +51,7 @@ spec: failureThreshold: 5 httpGet: path: /healthz - port: 443 + port: 6443 scheme: HTTPS initialDelaySeconds: 5 timeoutSeconds: 5 @@ -58,16 +60,23 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - {{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }} volumeMounts: + {{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }} - mountPath: /var/serving-cert name: serving-cert + {{- else }} + - mountPath: /apiserver.local.config + name: apiserver-local-config {{- end }} - {{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }} volumes: + {{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }} - name: serving-cert secret: defaultMode: 420 secretName: {{ .Values.admissionWebhook.apiservice.tlsSecret }} + {{- else }} + # rootfs maybe read-only, we need to an empty dir volume to store self-signed certifiates, etc. + - name: apiserver-local-config + emptyDir: {} {{- end }} {{- end }} diff --git a/charts/tidb-operator/templates/admission/admission-webhook-service.yaml b/charts/tidb-operator/templates/admission/admission-webhook-service.yaml index c4fa485745..fa64e37f29 100644 --- a/charts/tidb-operator/templates/admission/admission-webhook-service.yaml +++ b/charts/tidb-operator/templates/admission/admission-webhook-service.yaml @@ -13,7 +13,7 @@ spec: ports: - name: https-webhook # optional port: 443 - targetPort: 443 + targetPort: 6443 selector: app.kubernetes.io/name: {{ template "chart.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/tidb-operator/templates/controller-manager-rbac.yaml b/charts/tidb-operator/templates/controller-manager-rbac.yaml index 626e8cc020..bf67c2ea19 100644 --- a/charts/tidb-operator/templates/controller-manager-rbac.yaml +++ b/charts/tidb-operator/templates/controller-manager-rbac.yaml @@ -1,3 +1,6 @@ +{{/* +Delete permission is required in OpenShift because we can't own resources we created if we can't delete them. +*/}} {{- if .Values.rbac.create }} kind: ServiceAccount apiVersion: v1 @@ -29,16 +32,16 @@ rules: verbs: ["*"] - apiGroups: [""] resources: ["endpoints","configmaps"] - verbs: ["create", "get", "list", "watch", "update"] + verbs: ["create", "get", "list", "watch", "update","delete"] - apiGroups: [""] resources: ["serviceaccounts"] - verbs: ["create","get","update"] + verbs: ["create","get","update","delete"] - apiGroups: ["batch"] resources: ["jobs"] verbs: ["get", "list", "watch", "create", "update", "delete"] - apiGroups: [""] resources: ["secrets"] - verbs: ["create", "update", "get", "list", "watch"] + verbs: ["create", "update", "get", "list", "watch","delete"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "create", "update", "delete"] @@ -83,10 +86,10 @@ Ref: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#privilege-esc */}} - apiGroups: ["rbac.authorization.k8s.io"] resources: [clusterroles,roles] - verbs: ["escalate","create","get","update"] + verbs: ["escalate","create","get","update", "delete"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["rolebindings","clusterrolebindings"] - verbs: ["create","get","update"] + verbs: ["create","get","update", "delete"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 @@ -126,16 +129,16 @@ rules: verbs: ["*"] - apiGroups: [""] resources: ["endpoints","configmaps"] - verbs: ["create", "get", "list", "watch", "update"] + verbs: ["create", "get", "list", "watch", "update", "delete"] - apiGroups: [""] resources: ["serviceaccounts"] - verbs: ["create","get","update"] + verbs: ["create","get","update","delete"] - apiGroups: ["batch"] resources: ["jobs"] verbs: ["get", "list", "watch", "create", "update", "delete"] - apiGroups: [""] resources: ["secrets"] - verbs: ["create", "update", "get", "list", "watch"] + verbs: ["create", "update", "get", "list", "watch", "delete"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "create", "update", "delete"] @@ -153,10 +156,10 @@ rules: verbs: ["*"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles"] - verbs: ["escalate","create","get","update"] + verbs: ["escalate","create","get","update", "delete"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["rolebindings"] - verbs: ["create","get","update"] + verbs: ["create","get","update", "delete"] {{- if .Values.features | has "AdvancedStatefulSet=true" }} - apiGroups: - apps.pingcap.com diff --git a/ci/run-in-vm.sh b/ci/run-in-vm.sh new file mode 100755 index 0000000000..0edd0e401b --- /dev/null +++ b/ci/run-in-vm.sh @@ -0,0 +1,159 @@ +#!/bin/bash + +# Copyright 2020 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This is a helper script to start a VM and run command in it. +# +# TODO create an isolated network + +set -o errexit +set -o nounset +set -o pipefail + +ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd) +cd $ROOT + +source "${ROOT}/hack/lib.sh" + +GCP_CREDENTIALS=${GCP_CREDENTIALS:-} +GCP_PROJECT=${GCP_PROJECT:-} +GCP_ZONE=${GCP_ZONE:-} +GCP_SSH_PRIVATE_KEY=${GCP_SSH_PRIVATE_KEY:-} +GCP_SSH_PUBLIC_KEY=${GCP_SSH_PUBLIC_KEY:-} +NAME=${NAME:-tidb-operator-e2e} +GIT_URL=${GIT_URL:-https://github.com/pingcap/tidb-operator} +GIT_REF=${GIT_REF:-origin/master} +SYNC_FILES=${SYNC_FILES:-} + +echo "GCP_CREDENTIALS: $GCP_CREDENTIALS" +echo "GCP_PROJECT: $GCP_PROJECT" +echo "GCP_ZONE: $GCP_ZONE" +echo "GCP_SSH_PRIVATE_KEY: $GCP_SSH_PRIVATE_KEY" +echo "GCP_SSH_PUBLIC_KEY: $GCP_SSH_PUBLIC_KEY" +echo "NAME: $NAME" +echo "GIT_URL: $GIT_URL" +echo "GIT_REF: $GIT_REF" +echo "SYNC_FILES: $SYNC_FILES" + +# Pre-created nested virtualization enabled image with following commands: +# +# gcloud compute disks create disk1 --image-project centos-cloud --image-family centos-8 --zone us-central1-b +# gcloud compute images create centos-8-nested-vm \ +# --source-disk disk1 --source-disk-zone us-central1-b \ +# --licenses "https://compute.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" +# gcloud compute disks delete disk1 +# +# Refer to +# https://cloud.google.com/compute/docs/instances/enable-nested-virtualization-vm-instances +# for more details. +IMAGE=centos-8-nested-vm + +echo "info: configure gcloud" +if [ -z "$GCP_PROJECT" ]; then + echo "error: GCP_PROJECT is required" + exit 1 +fi +if [ -z "$GCP_CREDENTIALS" ]; then + echo "error: GCP_CREDENTIALS is required" + exit 1 +fi +if [ -z "$GCP_ZONE" ]; then + echo "error: GCP_ZONE is required" + exit 1 +fi +gcloud auth activate-service-account --key-file "$GCP_CREDENTIALS" +gcloud config set core/project $GCP_PROJECT +gcloud config set compute/zone $GCP_ZONE + +echo "info: preparing ssh keypairs for GCP" +if [ ! -d ~/.ssh ]; then + mkdir ~/.ssh +fi +if [ ! -e ~/.ssh/google_compute_engine -a -n "$GCP_SSH_PRIVATE_KEY" ]; then + echo "Copying $GCP_SSH_PRIVATE_KEY to ~/.ssh/google_compute_engine" >&2 + cp $GCP_SSH_PRIVATE_KEY ~/.ssh/google_compute_engine + chmod 0600 ~/.ssh/google_compute_engine +fi +if [ ! -e ~/.ssh/google_compute_engine.pub -a -n "$GCP_SSH_PUBLIC_KEY" ]; then + echo "Copying $GCP_SSH_PUBLIC_KEY to ~/.ssh/google_compute_engine.pub" >&2 + cp $GCP_SSH_PUBLIC_KEY ~/.ssh/google_compute_engine.pub + chmod 0600 ~/.ssh/google_compute_engine.pub +fi + +function gcloud_resource_exists() { + local args=($(tr -s '_' ' ' <<<"$1")) + unset args[$[${#args[@]}-1]] + local name="$2" + x=$(${args[@]} list --filter="name='$name'" --format='table[no-heading](name)' | wc -l) + [ "$x" -ge 1 ] +} + +function gcloud_compute_instances_exists() { + gcloud_resource_exists ${FUNCNAME[0]} $@ +} + +function e2e::down() { + echo "info: tearing down" + if ! gcloud_compute_instances_exists $NAME; then + echo "info: instance '$NAME' does not exist, skipped" + return 0 + fi + echo "info: deleting instance '$NAME'" + gcloud compute instances delete $NAME -q +} + +function e2e::up() { + echo "info: setting up" + echo "info: creating instance '$NAME'" + gcloud compute instances create $NAME \ + --machine-type n1-standard-8 \ + --min-cpu-platform "Intel Haswell" \ + --image $IMAGE \ + --boot-disk-size 30GB \ + --local-ssd interface=scsi +} + +function e2e::test() { + echo "info: testing" + echo "info: syncing files $SYNC_FILES" + while IFS=$',' read -r line; do + IFS=':' read -r src dst <<< "$line" + if [ -z "$dst" ]; then + dst="$src" + fi + gcloud compute scp $src vagrant@$NAME:$dst + done <<< "$SYNC_FILES" + local tmpfile=$(mktemp) + trap "rm -f $tmpfile" RETURN + cat < $tmpfile +sudo yum install -y git +cd \$HOME +sudo rm -rf tidb-operator +git init tidb-operator +cd tidb-operator +git fetch --tags --progress ${GIT_URL} +refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/* +GIT_COMMIT=\$(git rev-parse ${GIT_REF}^{commit}) +git checkout -f \${GIT_COMMIT} +$@ +EOF + cat $tmpfile + gcloud compute scp $tmpfile vagrant@$NAME:/tmp/e2e.sh + gcloud compute ssh vagrant@$NAME --command "bash /tmp/e2e.sh" +} + +e2e::down +trap 'e2e::down' EXIT +e2e::up +e2e::test "$@" diff --git a/ci/vm.groovy b/ci/vm.groovy new file mode 100644 index 0000000000..64acda4de4 --- /dev/null +++ b/ci/vm.groovy @@ -0,0 +1,144 @@ +// +// Jenkins pipeline for VM jobs. +// +// This script is written in declarative syntax. Refer to +// https://jenkins.io/doc/book/pipeline/syntax/ for more details. +// +// Note that parameters of the job is configured in this script. +// + +import groovy.transform.Field + +@Field +def podYAML = ''' +apiVersion: v1 +kind: Pod +spec: + containers: + - name: main + image: gcr.io/k8s-testimages/kubekins-e2e:v20200311-1e25827-master + command: + - runner.sh + - sleep + - 1d + # we need privileged mode in order to do docker in docker + securityContext: + privileged: true + env: + - name: DOCKER_IN_DOCKER_ENABLED + value: "true" + resources: + requests: + memory: "4000Mi" + cpu: 2000m + volumeMounts: + # dind expects /var/lib/docker to be volume + - name: docker-root + mountPath: /var/lib/docker + volumes: + - name: docker-root + emptyDir: {} +''' + +// Able to override default values in Jenkins job via environment variables. + +if (!env.DEFAULT_GIT_URL) { + env.DEFAULT_GIT_URL = "https://github.com/pingcap/tidb-operator" +} + +if (!env.DEFAULT_GIT_REF) { + env.DEFAULT_GIT_REF = "master" +} + +if (!env.DEFAULT_GCP_PROJECT) { + env.DEFAULT_GCP_PROJECT = "" +} + +if (!env.DEFAULT_GCP_ZONE) { + env.DEFAULT_GCP_ZONE = "us-central1-b" +} + +if (!env.DEFAULT_NAME) { + env.DEFAULT_NAME = "tidb-operator-e2e" +} + +pipeline { + agent { + kubernetes { + yaml podYAML + defaultContainer "main" + customWorkspace "/home/jenkins/agent/workspace/go/src/github.com/pingcap/tidb-operator" + } + } + + options { + timeout(time: 3, unit: 'HOURS') + } + + parameters { + string(name: 'GIT_URL', defaultValue: env.DEFAULT_GIT_URL, description: 'git repo url') + string(name: 'GIT_REF', defaultValue: env.DEFAULT_GIT_REF, description: 'git ref spec to checkout, e.g. master, release-1.1') + string(name: 'PR_ID', defaultValue: '', description: 'pull request ID, this will override GIT_REF if set, e.g. 1889') + string(name: 'GCP_PROJECT', defaultValue: env.DEFAULT_GCP_PROJECT, description: 'the GCP project ID') + string(name: 'GCP_ZONE', defaultValue: env.DEFAULT_GCP_ZONE, description: 'the GCP zone') + string(name: 'NAME', defaultValue: env.DEFAULT_NAME, description: 'the name of VM instance') + } + + environment { + GIT_REF = '' + } + + stages { + stage("Prepare") { + steps { + // The declarative model for Jenkins Pipelines has a restricted + // subset of syntax that it allows in the stage blocks. We use + // script step to bypass the restriction. + // https://jenkins.io/doc/book/pipeline/syntax/#script + script { + GIT_REF = params.GIT_REF + if (params.PR_ID != "") { + GIT_REF = "refs/remotes/origin/pr/${params.PR_ID}/head" + } + } + echo "env.NODE_NAME: ${env.NODE_NAME}" + echo "env.WORKSPACE: ${env.WORKSPACE}" + echo "GIT_REF: ${GIT_REF}" + } + } + + stage("Checkout") { + steps { + checkout scm: [ + $class: 'GitSCM', + branches: [[name: GIT_REF]], + userRemoteConfigs: [[ + refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*', + url: "${params.GIT_URL}", + ]] + ] + } + } + + stage("Run") { + steps { + withCredentials([ + file(credentialsId: 'TIDB_OPERATOR_GCP_CREDENTIALS', variable: 'GCP_CREDENTIALS'), + file(credentialsId: 'TIDB_OPERATOR_GCP_SSH_PRIVATE_KEY', variable: 'GCP_SSH_PRIVATE_KEY'), + file(credentialsId: 'TIDB_OPERATOR_GCP_SSH_PUBLIC_KEY', variable: 'GCP_SSH_PUBLIC_KEY'), + file(credentialsId: 'TIDB_OPERATOR_REDHAT_PULL_SECRET', variable: 'REDHAT_PULL_SECRET'), + ]) { + sh """ + #!/bin/bash + export GIT_REF=${GIT_REF} + export SYNC_FILES=\$REDHAT_PULL_SECRET:/tmp/pull-secret.txt + # TODO make the command configurable + ./ci/run-in-vm.sh PULL_SECRET_FILE=/tmp/pull-secret.txt ./hack/e2e-openshift.sh + """ + } + } + } + } +} + +// vim: et sw=4 ts=4 diff --git a/hack/e2e-openshift.sh b/hack/e2e-openshift.sh new file mode 100755 index 0000000000..65757b9849 --- /dev/null +++ b/hack/e2e-openshift.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash + +# Copyright 2020 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# E2E entrypoint script for OpenShift. +# + +set -o errexit +set -o nounset +set -o pipefail + +ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd) +cd $ROOT + +PULL_SECRET_FILE=${PULL_SECRET_FILE:-} + +if [ ! -e "$PULL_SECRET_FILE" ]; then + echo "error: pull secret file '$PULL_SECRET_FILE' does not exist" + exit 1 +fi + +vmx_cnt=$(grep -cw vmx /proc/cpuinfo) +if [ "$vmx_cnt" -gt 0 ]; then + echo "info: nested virtualization enabled (vmx cnt: $vmx_cnt)" +else + echo "error: nested virtualization not enabled, please refer to https://cloud.google.com/compute/docs/instances/enable-nested-virtualization-vm-instances" + exit 1 +fi + +echo "info: install required software packages" +sudo yum install -y jq git make golang +sudo yum install -y yum-utils +sudo yum-config-manager \ + --add-repo https://download.docker.com/linux/centos/docker-ce.repo +sudo yum install -y --nobest docker-ce docker-ce-cli containerd.io +if ! systemctl is-active --quiet docker; then + sudo systemctl start docker +fi +echo "info: printing docker information" +sudo docker info +sudo chmod o+rw /var/run/docker.sock + +CRC_HOME=$HOME/.crc +echo "info: mouting disk onto $CRC_HOME" +if ! mountpoint $CRC_HOME &>/dev/null; then + sudo mkfs.ext4 -F /dev/disk/by-id/google-local-ssd-0 + sudo rm -rf $CRC_HOME + mkdir $CRC_HOME + sudo mount /dev/disk/by-id/google-local-ssd-0 $CRC_HOME + sudo chown -R $(id -u):$(id -g) $CRC_HOME +fi + +echo "info: downloading latest crc" +cd $HOME +CRC_VERSION=$(curl --retry 10 -L -s 'https://mirror.openshift.com/pub/openshift-v4/clients/crc/latest/release-info.json' | jq -r '.version.crcVersion') +if ! test -e crc-linux-amd64.tar.xz; then + curl --retryn 10 -LO https://mirror.openshift.com/pub/openshift-v4/clients/crc/$CRC_VERSION/crc-linux-amd64.tar.xz + tar -xvf crc-linux-amd64.tar.xz +fi +export PATH=$HOME/crc-linux-$CRC_VERSION-amd64:$PATH + +crc version + +echo "info: starting the OpenShift clsuter" +crcStatus=$(crc status | awk '/CRC VM:/ {print $3}') +if [[ "$crcStatus" == "Running" ]]; then + crc status +else + crc setup + crc config set cpus 6 + crc config set memory 24576 + crc start --pull-secret-file $PULL_SECRET_FILE +fi + +echo "info: login" +eval $(crc oc-env) +KUBEADMIN_PASSWORD=$(cat $HOME/.crc/cache/crc_libvirt_*/kubeadmin-password) +oc login -u kubeadmin -p "$KUBEADMIN_PASSWORD" https://api.crc.testing:6443 --insecure-skip-tls-verify + +echo "info: building images" +cd $HOME/tidb-operator +./hack/run-in-container.sh bash -c 'make docker e2e-docker +images=( + tidb-operator:latest + tidb-backup-manager:latest + tidb-operator-e2e:latest +) +for image in ${images[@]}; do + docker save localhost:5000/pingcap/$image -o $image.tar.gz +done +' + +echo "info: pusing images" +OC_PROJECT=openshift +oc extract secret/router-ca --keys=tls.crt -n openshift-ingress-operator +sudo mkdir /etc/docker/certs.d/default-route-openshift-image-registry.apps-crc.testing/ -p +sudo mv tls.crt /etc/docker/certs.d/default-route-openshift-image-registry.apps-crc.testing/ +docker login -u kubeadmin --password-stdin default-route-openshift-image-registry.apps-crc.testing <<< "$(oc whoami -t)" + +images=( + tidb-operator:latest + tidb-backup-manager:latest + tidb-operator-e2e:latest +) +for image in ${images[@]}; do + sudo chown -R $(id -u):$(id -g) $image.tar.gz + docker load -i $image.tar.gz + docker tag localhost:5000/pingcap/$image image-registry.openshift-image-registry.svc:5000/$OC_PROJECT/$image + docker tag localhost:5000/pingcap/$image default-route-openshift-image-registry.apps-crc.testing/$OC_PROJECT/$image + docker push default-route-openshift-image-registry.apps-crc.testing/$OC_PROJECT/$image +done + +export PROVIDER=openshift +export TIDB_OPERATOR_IMAGE=image-registry.openshift-image-registry.svc:5000/$OC_PROJECT/tidb-operator:latest +export TIDB_BACKUP_MANAGER_IMAGE=image-registry.openshift-image-registry.svc:5000/$OC_PROJECT/tidb-backup-manager:latest +export E2E_IMAGE=image-registry.openshift-image-registry.svc:5000/$OC_PROJECT/tidb-operator-e2e:latest +# 'Restarter' test starts 1 replica of pd and tikv and can pass in single-node OpenShift cluster. +./hack/run-e2e.sh --ginkgo.focus 'Restarter' diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh index 7542cba0ef..97961d7a58 100755 --- a/hack/run-e2e.sh +++ b/hack/run-e2e.sh @@ -128,6 +128,28 @@ EOF echo "info: provider is $PROVIDER, skipped" elif [ "$PROVIDER" == "eks" ]; then echo "info: provider is $PROVIDER, skipped" + elif [ "$PROVIDER" == "openshift" ]; then + CRC_IP=$(crc ip) + ssh -i ~/.crc/machines/crc/id_rsa -o StrictHostKeyChecking=no core@$CRC_IP <<'EOF' +sudo bash -c ' +test -d /mnt/disks || mkdir -p /mnt/disks +df -h /mnt/disks +if mountpoint /mnt/disks &>/dev/null; then + echo "info: /mnt/disks is a mountpoint" +else + echo "info: /mnt/disks is not a mountpoint, creating local volumes on the rootfs" +fi +cd /mnt/disks +for ((i = 1; i <= 32; i++)) { + if [ ! -d vol$i ]; then + mkdir vol$i + fi + if ! mountpoint vol$i &>/dev/null; then + mount --bind vol$i vol$i + fi +} +' +EOF fi echo "info: installing local-volume-provisioner" $KUBECTL_BIN --context $KUBECONTEXT apply -f ${ROOT}/manifests/local-dind/local-volume-provisioner.yaml diff --git a/hack/run-in-container.sh b/hack/run-in-container.sh index f821724200..317751f8e3 100755 --- a/hack/run-in-container.sh +++ b/hack/run-in-container.sh @@ -78,15 +78,20 @@ fi args=(bash) if [ $# -gt 0 ]; then - args=($@) + args=("$@") fi docker_args=( - -it --rm + --rm -h $NAME --name $NAME ) +if [ -t 1 ]; then + # Allocate a pseudo-TTY when the STDIN is a terminal + docker_args+=(-it) +fi + # required by dind docker_args+=( --privileged diff --git a/pkg/monitor/monitor/monitor_manager.go b/pkg/monitor/monitor/monitor_manager.go index 065d1c8d1c..7fafa365a5 100644 --- a/pkg/monitor/monitor/monitor_manager.go +++ b/pkg/monitor/monitor/monitor_manager.go @@ -20,7 +20,11 @@ import ( informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" v1alpha1listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" + utildiscovery "github.com/pingcap/tidb-operator/pkg/util/discovery" corev1 "k8s.io/api/core/v1" + rbac "k8s.io/api/rbac/v1" + "k8s.io/client-go/discovery" + discoverycachedmemory "k8s.io/client-go/discovery/cached/memory" kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" appslisters "k8s.io/client-go/listers/apps/v1" @@ -30,12 +34,13 @@ import ( ) type MonitorManager struct { - typedControl controller.TypedControlInterface - deploymentLister appslisters.DeploymentLister - tcLister v1alpha1listers.TidbClusterLister - pvLister corelisters.PersistentVolumeLister - pvControl controller.PVControlInterface - recorder record.EventRecorder + discoveryInterface discovery.CachedDiscoveryInterface + typedControl controller.TypedControlInterface + deploymentLister appslisters.DeploymentLister + tcLister v1alpha1listers.TidbClusterLister + pvLister corelisters.PersistentVolumeLister + pvControl controller.PVControlInterface + recorder record.EventRecorder } const ( @@ -52,12 +57,13 @@ func NewMonitorManager( pvcLister := kubeInformerFactory.Core().V1().PersistentVolumeClaims().Lister() pvLister := kubeInformerFactory.Core().V1().PersistentVolumes().Lister() return &MonitorManager{ - typedControl: typedControl, - deploymentLister: kubeInformerFactory.Apps().V1().Deployments().Lister(), - tcLister: informerFactory.Pingcap().V1alpha1().TidbClusters().Lister(), - pvControl: controller.NewRealPVControl(kubeCli, pvcLister, pvLister, recorder), - pvLister: pvLister, - recorder: recorder, + discoveryInterface: discoverycachedmemory.NewMemCacheClient(kubeCli.Discovery()), + typedControl: typedControl, + deploymentLister: kubeInformerFactory.Apps().V1().Deployments().Lister(), + tcLister: informerFactory.Pingcap().V1alpha1().TidbClusters().Lister(), + pvControl: controller.NewRealPVControl(kubeCli, pvcLister, pvLister, recorder), + pvLister: pvLister, + recorder: recorder, } } @@ -211,8 +217,31 @@ func (mm *MonitorManager) syncTidbMonitorRbac(monitor *v1alpha1.TidbMonitor) (*c klog.Errorf("tm[%s/%s]'s serviceaccount failed to sync,err: %v", monitor.Namespace, monitor.Name, err) return nil, err } + policyRules := []rbac.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + } + if supported, err := utildiscovery.IsAPIGroupVersionSupported(mm.discoveryInterface, "security.openshift.io/v1"); err != nil { + return nil, err + } else if supported { + // We must use 'anyuid' SecurityContextConstraint to run our container as root. + // https://docs.openshift.com/container-platform/4.3/authentication/managing-security-context-constraints.html + policyRules = append(policyRules, rbac.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + ResourceNames: []string{"anyuid"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + }) + } if controller.ClusterScoped { - cr := getMonitorClusterRole(monitor) + policyRules = append(policyRules, rbac.PolicyRule{ + NonResourceURLs: []string{"/metrics"}, + Verbs: []string{"get"}, + }) + cr := getMonitorClusterRole(monitor, policyRules) cr, err = mm.typedControl.CreateOrUpdateClusterRole(monitor, cr) if err != nil { klog.Errorf("tm[%s/%s]'s clusterrole failed to sync,err: %v", monitor.Namespace, monitor.Name, err) @@ -227,7 +256,7 @@ func (mm *MonitorManager) syncTidbMonitorRbac(monitor *v1alpha1.TidbMonitor) (*c return sa, nil } - role := getMonitorRole(monitor) + role := getMonitorRole(monitor, policyRules) role, err = mm.typedControl.CreateOrUpdateRole(monitor, role) if err != nil { klog.Errorf("tm[%s/%s]'s role failed to sync,err: %v", monitor.Namespace, monitor.Name, err) diff --git a/pkg/monitor/monitor/util.go b/pkg/monitor/monitor/util.go index 61e74e196e..8e5b18ff68 100644 --- a/pkg/monitor/monitor/util.go +++ b/pkg/monitor/monitor/util.go @@ -118,7 +118,7 @@ func getMonitorServiceAccount(monitor *v1alpha1.TidbMonitor) *core.ServiceAccoun return sa } -func getMonitorClusterRole(monitor *v1alpha1.TidbMonitor) *rbac.ClusterRole { +func getMonitorClusterRole(monitor *v1alpha1.TidbMonitor, policyRules []rbac.PolicyRule) *rbac.ClusterRole { return &rbac.ClusterRole{ ObjectMeta: meta.ObjectMeta{ Name: GetMonitorObjectName(monitor), @@ -126,21 +126,11 @@ func getMonitorClusterRole(monitor *v1alpha1.TidbMonitor) *rbac.ClusterRole { Labels: buildTidbMonitorLabel(monitor.Name), OwnerReferences: []meta.OwnerReference{controller.GetTiDBMonitorOwnerRef(monitor)}, }, - Rules: []rbac.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - NonResourceURLs: []string{"/metrics"}, - Verbs: []string{"get"}, - }, - }, + Rules: policyRules, } } -func getMonitorRole(monitor *v1alpha1.TidbMonitor) *rbac.Role { +func getMonitorRole(monitor *v1alpha1.TidbMonitor, policyRules []rbac.PolicyRule) *rbac.Role { return &rbac.Role{ ObjectMeta: meta.ObjectMeta{ Name: GetMonitorObjectName(monitor), @@ -148,13 +138,7 @@ func getMonitorRole(monitor *v1alpha1.TidbMonitor) *rbac.Role { Labels: buildTidbMonitorLabel(monitor.Name), OwnerReferences: []meta.OwnerReference{controller.GetTiDBMonitorOwnerRef(monitor)}, }, - Rules: []rbac.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - }, + Rules: policyRules, } } diff --git a/tests/actions.go b/tests/actions.go index 1b33a81572..e67fe51c4f 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -475,7 +475,17 @@ func (oa *operatorActions) runKubectlOrDie(args ...string) string { } func (oa *operatorActions) CleanCRDOrDie() { - oa.runKubectlOrDie("delete", "crds", "--all") + crdList, err := oa.apiExtCli.ApiextensionsV1beta1().CustomResourceDefinitions().List(metav1.ListOptions{}) + framework.ExpectNoError(err) + for _, crd := range crdList.Items { + if !strings.HasSuffix(crd.Name, ".pingcap.com") { + framework.Logf("CRD %q ignored", crd.Name) + continue + } + framework.Logf("Deleting CRD %q", crd.Name) + err = oa.apiExtCli.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(crd.Name, &metav1.DeleteOptions{}) + framework.ExpectNoError(err) + } } // InstallCRDOrDie install CRDs and wait for them to be established in Kubernetes. diff --git a/tests/e2e/e2e.go b/tests/e2e/e2e.go index 9991599cdc..202d13c0d5 100644 --- a/tests/e2e/e2e.go +++ b/tests/e2e/e2e.go @@ -81,16 +81,20 @@ func setupSuite() { // Delete any namespaces except those created by the system. This ensures no // lingering resources are left over from a previous test run. if framework.TestContext.CleanStart { - deleted, err := framework.DeleteNamespaces(c, nil, /* deleteFilter */ - []string{ - metav1.NamespaceSystem, - metav1.NamespaceDefault, - metav1.NamespacePublic, - v1.NamespaceNodeLease, - // kind local path provisioner namespace since 0.7.0 - // https://github.com/kubernetes-sigs/kind/blob/v0.7.0/pkg/build/node/storage.go#L35 - "local-path-storage", - }) + reservedNamespaces := []string{ + metav1.NamespaceSystem, + metav1.NamespaceDefault, + metav1.NamespacePublic, + v1.NamespaceNodeLease, + } + if framework.TestContext.Provider == "kind" { + // kind local path provisioner namespace since 0.7.0 + // https://github.com/kubernetes-sigs/kind/blob/v0.7.0/pkg/build/node/storage.go#L35 + reservedNamespaces = append(reservedNamespaces, "local-path-storage") + } else if framework.TestContext.Provider == "openshift" { + reservedNamespaces = append(reservedNamespaces, "openshift") + } + deleted, err := framework.DeleteNamespaces(c, nil, reservedNamespaces) if err != nil { e2elog.Failf("Error deleting orphaned namespaces: %v", err) } @@ -190,13 +194,13 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { if err := exec.Command("sh", "-c", helmClearCmd).Run(); err != nil { framework.Failf("failed to clear helm releases (cmd: %q, error: %v", helmClearCmd, err) } - ginkgo.By("Clear non-kubernetes apiservices") - clearNonK8SAPIServicesCmd := "kubectl delete apiservices -l kube-aggregator.kubernetes.io/automanaged!=onstart" - if err := exec.Command("sh", "-c", clearNonK8SAPIServicesCmd).Run(); err != nil { - framework.Failf("failed to clear non-kubernetes apiservices (cmd: %q, error: %v", clearNonK8SAPIServicesCmd, err) + ginkgo.By("Clear tidb-operator apiservices") + clearAPIServicesCmd := "kubectl delete apiservices -l app.kubernetes.io/name=tidb-operator" + if err := exec.Command("sh", "-c", clearAPIServicesCmd).Run(); err != nil { + framework.Failf("failed to clear non-kubernetes apiservices (cmd: %q, error: %v", clearAPIServicesCmd, err) } - ginkgo.By("Clear validatingwebhookconfigurations") - clearValidatingWebhookConfigurationsCmd := "kubectl delete validatingwebhookconfiguration --all" + ginkgo.By("Clear tidb-operator validatingwebhookconfigurations") + clearValidatingWebhookConfigurationsCmd := "kubectl delete validatingwebhookconfiguration -l app.kubernetes.io/name=tidb-operator" if err := exec.Command("sh", "-c", clearValidatingWebhookConfigurationsCmd).Run(); err != nil { framework.Failf("failed to clear validatingwebhookconfigurations (cmd: %q, error: %v", clearValidatingWebhookConfigurationsCmd, err) } @@ -229,6 +233,9 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { pvList, err := kubeCli.CoreV1().PersistentVolumes().List(metav1.ListOptions{}) framework.ExpectNoError(err, "failed to list pvList") for _, pv := range pvList.Items { + if pv.Spec.StorageClassName != "local-storage" { + continue + } if pv.Spec.PersistentVolumeReclaimPolicy == v1.PersistentVolumeReclaimDelete { continue } @@ -237,13 +244,16 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { _, err = kubeCli.CoreV1().PersistentVolumes().Update(&pv) framework.ExpectNoError(err, fmt.Sprintf("failed to update pv %s", pv.Name)) } - ginkgo.By("Wait for all PVs to be available") + ginkgo.By("Wait for all local PVs to be available") err = wait.Poll(time.Second, time.Minute, func() (bool, error) { pvList, err := kubeCli.CoreV1().PersistentVolumes().List(metav1.ListOptions{}) if err != nil { return false, err } for _, pv := range pvList.Items { + if pv.Spec.StorageClassName != "local-storage" { + continue + } if pv.Status.Phase != v1.VolumeAvailable { return false, nil } diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index 3552bf2bb6..41f4c52b9a 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -47,6 +47,9 @@ func init() { framework.RegisterProvider("kind", func() (framework.ProviderInterface, error) { return framework.NullProvider{}, nil }) + framework.RegisterProvider("openshift", func() (framework.ProviderInterface, error) { + return framework.NullProvider{}, nil + }) } func TestMain(m *testing.M) {