Skip to content

Commit

Permalink
scripts to run e2e against OpenShift 4 (#2141)
Browse files Browse the repository at this point in the history
  • Loading branch information
cofyc authored Apr 10, 2020
1 parent 6fece6a commit da68386
Show file tree
Hide file tree
Showing 14 changed files with 587 additions and 69 deletions.
10 changes: 10 additions & 0 deletions charts/tidb-cluster/templates/monitor-rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,16 @@ rules:
resources:
- pods
verbs: ["get", "list", "watch"]
{{- if .Capabilities.APIVersions.Has "security.openshift.io/v1" }}
- apiGroups:
- security.openshift.io
resourceNames:
- anyuid
resources:
- securitycontextconstraints
verbs:
- use
{{- end }}
{{- if .Values.rbac.crossNamespace }}
- nonResourceURLs: ["/metrics"]
verbs: ["get"]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ spec:
imagePullPolicy: {{ .Values.imagePullPolicy | default "IfNotPresent" }}
command:
- /usr/local/bin/tidb-admission-webhook
# use > 1024 port, then we can run it as non-root user
- --secure-port=6443
{{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }}
- --tls-cert-file=/var/serving-cert/tls.crt
- --tls-private-key-file=/var/serving-cert/tls.key
Expand All @@ -41,15 +43,15 @@ spec:
failureThreshold: 5
httpGet:
path: /healthz
port: 443
port: 6443
scheme: HTTPS
initialDelaySeconds: 5
timeoutSeconds: 5
readinessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: 443
port: 6443
scheme: HTTPS
initialDelaySeconds: 5
timeoutSeconds: 5
Expand All @@ -58,16 +60,23 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
{{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }}
volumeMounts:
{{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }}
- mountPath: /var/serving-cert
name: serving-cert
{{- else }}
- mountPath: /apiserver.local.config
name: apiserver-local-config
{{- end }}
{{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }}
volumes:
{{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }}
- name: serving-cert
secret:
defaultMode: 420
secretName: {{ .Values.admissionWebhook.apiservice.tlsSecret }}
{{- else }}
# rootfs maybe read-only, we need to an empty dir volume to store self-signed certifiates, etc.
- name: apiserver-local-config
emptyDir: {}
{{- end }}
{{- end }}
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ spec:
ports:
- name: https-webhook # optional
port: 443
targetPort: 443
targetPort: 6443
selector:
app.kubernetes.io/name: {{ template "chart.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
Expand Down
23 changes: 13 additions & 10 deletions charts/tidb-operator/templates/controller-manager-rbac.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
{{/*
Delete permission is required in OpenShift because we can't own resources we created if we can't delete them.
*/}}
{{- if .Values.rbac.create }}
kind: ServiceAccount
apiVersion: v1
Expand Down Expand Up @@ -29,16 +32,16 @@ rules:
verbs: ["*"]
- apiGroups: [""]
resources: ["endpoints","configmaps"]
verbs: ["create", "get", "list", "watch", "update"]
verbs: ["create", "get", "list", "watch", "update","delete"]
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["create","get","update"]
verbs: ["create","get","update","delete"]
- apiGroups: ["batch"]
resources: ["jobs"]
verbs: ["get", "list", "watch", "create", "update", "delete"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["create", "update", "get", "list", "watch"]
verbs: ["create", "update", "get", "list", "watch","delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "create", "update", "delete"]
Expand Down Expand Up @@ -83,10 +86,10 @@ Ref: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#privilege-esc
*/}}
- apiGroups: ["rbac.authorization.k8s.io"]
resources: [clusterroles,roles]
verbs: ["escalate","create","get","update"]
verbs: ["escalate","create","get","update", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["rolebindings","clusterrolebindings"]
verbs: ["create","get","update"]
verbs: ["create","get","update", "delete"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
Expand Down Expand Up @@ -126,16 +129,16 @@ rules:
verbs: ["*"]
- apiGroups: [""]
resources: ["endpoints","configmaps"]
verbs: ["create", "get", "list", "watch", "update"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["create","get","update"]
verbs: ["create","get","update","delete"]
- apiGroups: ["batch"]
resources: ["jobs"]
verbs: ["get", "list", "watch", "create", "update", "delete"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["create", "update", "get", "list", "watch"]
verbs: ["create", "update", "get", "list", "watch", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "create", "update", "delete"]
Expand All @@ -153,10 +156,10 @@ rules:
verbs: ["*"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["roles"]
verbs: ["escalate","create","get","update"]
verbs: ["escalate","create","get","update", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["rolebindings"]
verbs: ["create","get","update"]
verbs: ["create","get","update", "delete"]
{{- if .Values.features | has "AdvancedStatefulSet=true" }}
- apiGroups:
- apps.pingcap.com
Expand Down
159 changes: 159 additions & 0 deletions ci/run-in-vm.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
#!/bin/bash

# Copyright 2020 PingCAP, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.

#
# This is a helper script to start a VM and run command in it.
#
# TODO create an isolated network

set -o errexit
set -o nounset
set -o pipefail

ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
cd $ROOT

source "${ROOT}/hack/lib.sh"

GCP_CREDENTIALS=${GCP_CREDENTIALS:-}
GCP_PROJECT=${GCP_PROJECT:-}
GCP_ZONE=${GCP_ZONE:-}
GCP_SSH_PRIVATE_KEY=${GCP_SSH_PRIVATE_KEY:-}
GCP_SSH_PUBLIC_KEY=${GCP_SSH_PUBLIC_KEY:-}
NAME=${NAME:-tidb-operator-e2e}
GIT_URL=${GIT_URL:-https://github.com/pingcap/tidb-operator}
GIT_REF=${GIT_REF:-origin/master}
SYNC_FILES=${SYNC_FILES:-}

echo "GCP_CREDENTIALS: $GCP_CREDENTIALS"
echo "GCP_PROJECT: $GCP_PROJECT"
echo "GCP_ZONE: $GCP_ZONE"
echo "GCP_SSH_PRIVATE_KEY: $GCP_SSH_PRIVATE_KEY"
echo "GCP_SSH_PUBLIC_KEY: $GCP_SSH_PUBLIC_KEY"
echo "NAME: $NAME"
echo "GIT_URL: $GIT_URL"
echo "GIT_REF: $GIT_REF"
echo "SYNC_FILES: $SYNC_FILES"

# Pre-created nested virtualization enabled image with following commands:
#
# gcloud compute disks create disk1 --image-project centos-cloud --image-family centos-8 --zone us-central1-b
# gcloud compute images create centos-8-nested-vm \
# --source-disk disk1 --source-disk-zone us-central1-b \
# --licenses "https://compute.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx"
# gcloud compute disks delete disk1
#
# Refer to
# https://cloud.google.com/compute/docs/instances/enable-nested-virtualization-vm-instances
# for more details.
IMAGE=centos-8-nested-vm

echo "info: configure gcloud"
if [ -z "$GCP_PROJECT" ]; then
echo "error: GCP_PROJECT is required"
exit 1
fi
if [ -z "$GCP_CREDENTIALS" ]; then
echo "error: GCP_CREDENTIALS is required"
exit 1
fi
if [ -z "$GCP_ZONE" ]; then
echo "error: GCP_ZONE is required"
exit 1
fi
gcloud auth activate-service-account --key-file "$GCP_CREDENTIALS"
gcloud config set core/project $GCP_PROJECT
gcloud config set compute/zone $GCP_ZONE

echo "info: preparing ssh keypairs for GCP"
if [ ! -d ~/.ssh ]; then
mkdir ~/.ssh
fi
if [ ! -e ~/.ssh/google_compute_engine -a -n "$GCP_SSH_PRIVATE_KEY" ]; then
echo "Copying $GCP_SSH_PRIVATE_KEY to ~/.ssh/google_compute_engine" >&2
cp $GCP_SSH_PRIVATE_KEY ~/.ssh/google_compute_engine
chmod 0600 ~/.ssh/google_compute_engine
fi
if [ ! -e ~/.ssh/google_compute_engine.pub -a -n "$GCP_SSH_PUBLIC_KEY" ]; then
echo "Copying $GCP_SSH_PUBLIC_KEY to ~/.ssh/google_compute_engine.pub" >&2
cp $GCP_SSH_PUBLIC_KEY ~/.ssh/google_compute_engine.pub
chmod 0600 ~/.ssh/google_compute_engine.pub
fi

function gcloud_resource_exists() {
local args=($(tr -s '_' ' ' <<<"$1"))
unset args[$[${#args[@]}-1]]
local name="$2"
x=$(${args[@]} list --filter="name='$name'" --format='table[no-heading](name)' | wc -l)
[ "$x" -ge 1 ]
}

function gcloud_compute_instances_exists() {
gcloud_resource_exists ${FUNCNAME[0]} $@
}

function e2e::down() {
echo "info: tearing down"
if ! gcloud_compute_instances_exists $NAME; then
echo "info: instance '$NAME' does not exist, skipped"
return 0
fi
echo "info: deleting instance '$NAME'"
gcloud compute instances delete $NAME -q
}

function e2e::up() {
echo "info: setting up"
echo "info: creating instance '$NAME'"
gcloud compute instances create $NAME \
--machine-type n1-standard-8 \
--min-cpu-platform "Intel Haswell" \
--image $IMAGE \
--boot-disk-size 30GB \
--local-ssd interface=scsi
}

function e2e::test() {
echo "info: testing"
echo "info: syncing files $SYNC_FILES"
while IFS=$',' read -r line; do
IFS=':' read -r src dst <<< "$line"
if [ -z "$dst" ]; then
dst="$src"
fi
gcloud compute scp $src vagrant@$NAME:$dst
done <<< "$SYNC_FILES"
local tmpfile=$(mktemp)
trap "rm -f $tmpfile" RETURN
cat <<EOF > $tmpfile
sudo yum install -y git
cd \$HOME
sudo rm -rf tidb-operator
git init tidb-operator
cd tidb-operator
git fetch --tags --progress ${GIT_URL} +refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*
GIT_COMMIT=\$(git rev-parse ${GIT_REF}^{commit})
git checkout -f \${GIT_COMMIT}
$@
EOF
cat $tmpfile
gcloud compute scp $tmpfile vagrant@$NAME:/tmp/e2e.sh
gcloud compute ssh vagrant@$NAME --command "bash /tmp/e2e.sh"
}

e2e::down
trap 'e2e::down' EXIT
e2e::up
e2e::test "$@"
Loading

0 comments on commit da68386

Please sign in to comment.