Skip to content

Commit

Permalink
scripts to run e2e against OpenShift
Browse files Browse the repository at this point in the history
  • Loading branch information
cofyc committed Apr 9, 2020
1 parent f93fcc6 commit 9d8c898
Show file tree
Hide file tree
Showing 12 changed files with 499 additions and 38 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ spec:
imagePullPolicy: {{ .Values.imagePullPolicy | default "IfNotPresent" }}
command:
- /usr/local/bin/tidb-admission-webhook
# use > 1024 port, then we can run it as non-root user
- --secure-port=6443
{{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }}
- --tls-cert-file=/var/serving-cert/tls.crt
- --tls-private-key-file=/var/serving-cert/tls.key
Expand All @@ -41,15 +43,15 @@ spec:
failureThreshold: 5
httpGet:
path: /healthz
port: 443
port: 6443
scheme: HTTPS
initialDelaySeconds: 5
timeoutSeconds: 5
readinessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: 443
port: 6443
scheme: HTTPS
initialDelaySeconds: 5
timeoutSeconds: 5
Expand All @@ -58,16 +60,23 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
{{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }}
volumeMounts:
{{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }}
- mountPath: /var/serving-cert
name: serving-cert
{{- else }}
- mountPath: /apiserver.local.config
name: apiserver-local-config
{{- end }}
{{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }}
volumes:
{{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }}
- name: serving-cert
secret:
defaultMode: 420
secretName: {{ .Values.admissionWebhook.apiservice.tlsSecret }}
{{- else }}
# rootfs maybe read-only, we need to an empty dir volume to store self-signed certifiates, etc.
- name: apiserver-local-config
emptyDir: {}
{{- end }}
{{- end }}
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ spec:
ports:
- name: https-webhook # optional
port: 443
targetPort: 443
targetPort: 6443
selector:
app.kubernetes.io/name: {{ template "chart.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
Expand Down
23 changes: 13 additions & 10 deletions charts/tidb-operator/templates/controller-manager-rbac.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
{{/*
Delete permission is required in OpenShift because we can't own resources we created if we can't delete them.
*/}}
{{- if .Values.rbac.create }}
kind: ServiceAccount
apiVersion: v1
Expand Down Expand Up @@ -29,16 +32,16 @@ rules:
verbs: ["*"]
- apiGroups: [""]
resources: ["endpoints","configmaps"]
verbs: ["create", "get", "list", "watch", "update"]
verbs: ["create", "get", "list", "watch", "update","delete"]
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["create","get","update"]
verbs: ["create","get","update","delete"]
- apiGroups: ["batch"]
resources: ["jobs"]
verbs: ["get", "list", "watch", "create", "update", "delete"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["create", "update", "get", "list", "watch"]
verbs: ["create", "update", "get", "list", "watch","delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "create", "update", "delete"]
Expand Down Expand Up @@ -83,10 +86,10 @@ Ref: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#privilege-esc
*/}}
- apiGroups: ["rbac.authorization.k8s.io"]
resources: [clusterroles,roles]
verbs: ["escalate","create","get","update"]
verbs: ["escalate","create","get","update", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["rolebindings","clusterrolebindings"]
verbs: ["create","get","update"]
verbs: ["create","get","update", "delete"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
Expand Down Expand Up @@ -126,16 +129,16 @@ rules:
verbs: ["*"]
- apiGroups: [""]
resources: ["endpoints","configmaps"]
verbs: ["create", "get", "list", "watch", "update"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["create","get","update"]
verbs: ["create","get","update","delete"]
- apiGroups: ["batch"]
resources: ["jobs"]
verbs: ["get", "list", "watch", "create", "update", "delete"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["create", "update", "get", "list", "watch"]
verbs: ["create", "update", "get", "list", "watch", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "create", "update", "delete"]
Expand All @@ -153,10 +156,10 @@ rules:
verbs: ["*"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["roles"]
verbs: ["escalate","create","get","update"]
verbs: ["escalate","create","get","update", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["rolebindings"]
verbs: ["create","get","update"]
verbs: ["create","get","update", "delete"]
{{- if .Values.features | has "AdvancedStatefulSet=true" }}
- apiGroups:
- apps.pingcap.com
Expand Down
143 changes: 143 additions & 0 deletions ci/run-in-vm.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
#!/bin/bash

# Copyright 2020 PingCAP, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.

#
# This is a helper script to start a VM and run command in it.
#
# TODO create an isolated network

set -o errexit
set -o nounset
set -o pipefail

ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
cd $ROOT

source "${ROOT}/hack/lib.sh"

GCP_CREDENTIALS=${GCP_CREDENTIALS:-}
GCP_PROJECT=${GCP_PROJECT:-}
GCP_ZONE=${GCP_ZONE:-}
GCP_SSH_PRIVATE_KEY=${GCP_SSH_PRIVATE_KEY:-}
GCP_SSH_PUBLIC_KEY=${GCP_SSH_PUBLIC_KEY:-}
NAME=${NAME:-tidb-operator-e2e}
GIT_URL=${GIT_URL:-https://github.com/pingcap/tidb-operator}
GIT_REF=${GIT_REF:-origin/master}
SYNC_FILES=${SYNC_FILES:-}

# Pre-created nested virtualization enabled image with following commands:
#
# gcloud compute disks create disk1 --image-project centos-cloud --image-family centos-8 --zone us-central1-b
# gcloud compute images create centos-8-nested-vm \
# --source-disk disk1 --source-disk-zone us-central1-b \
# --licenses "https://compute.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx"
# gcloud compute disks delete disk1
#
# Refer to
# https://cloud.google.com/compute/docs/instances/enable-nested-virtualization-vm-instances
# for more details.
IMAGE=centos-8-nested-vm

echo "info: configure gcloud"
if [ -z "$GCP_PROJECT" ]; then
echo "error: GCP_PROJECT is required"
exit 1
fi
if [ -z "$GCP_CREDENTIALS" ]; then
echo "error: GCP_CREDENTIALS is required"
exit 1
fi
if [ -z "$GCP_ZONE" ]; then
echo "error: GCP_ZONE is required"
exit 1
fi
gcloud auth activate-service-account --key-file "$GCP_CREDENTIALS"
gcloud config set core/project $GCP_PROJECT
gcloud config set compute/zone $GCP_ZONE

echo "info: preparing ssh keypairs for GCP"
if [ ! -d ~/.ssh ]; then
mkdir ~/.ssh
fi
if [ ! -e ~/.ssh/google_compute_engine -a -n "$GCP_SSH_PRIVATE_KEY" ]; then
echo "Copying $GCP_SSH_PRIVATE_KEY to ~/.ssh/google_compute_engine" >&2
cp $GCP_SSH_PRIVATE_KEY ~/.ssh/google_compute_engine
chmod 0600 ~/.ssh/google_compute_engine
fi
if [ ! -e ~/.ssh/google_compute_engine.pub -a -n "$GCP_SSH_PUBLIC_KEY" ]; then
echo "Copying $GCP_SSH_PUBLIC_KEY to ~/.ssh/google_compute_engine.pub" >&2
cp $GCP_SSH_PUBLIC_KEY ~/.ssh/google_compute_engine.pub
chmod 0600 ~/.ssh/google_compute_engine.pub
fi

function gcloud_resource_exists() {
local args=($(tr -s '_' ' ' <<<"$1"))
unset args[$[${#args[@]}-1]]
local name="$2"
x=$(${args[@]} list --filter="name='$name'" --format='table[no-heading](name)' | wc -l)
[ "$x" -ge 1 ]
}

function gcloud_compute_instances_exists() {
gcloud_resource_exists ${FUNCNAME[0]} $@
}

function e2e::down() {
echo "info: tearing down"
if ! gcloud_compute_instances_exists $NAME; then
echo "info: instance '$NAME' does not exist, skipped"
return 0
fi
echo "info: deleting instance '$NAME'"
gcloud compute instances delete $NAME -q
}

function e2e::up() {
echo "info: setting up"
echo "info: creating instance '$NAME'"
gcloud compute instances create $NAME \
--machine-type n1-standard-8 \
--min-cpu-platform "Intel Haswell" \
--image $IMAGE \
--boot-disk-size 30GB \
--local-ssd interface=scsi
}

function e2e::test() {
echo "info: testing"
echo "info: syncing files $SYNC_FILES"
while IFS=$',' read -r line; do
IFS=':' read -r src dst <<< "$line"
if [ -z "$dst" ]; then
dst="$src"
fi
gcloud compute scp $src vagrant@$NAME:$dst
done <<< "$SYNC_FILES"
gcloud compute ssh vagrant@$NAME --command "
sudo yum install -y git
rm -rf tidb-operator
git init tidb-operator
cd tidb-operator
git fetch --tags --progress ${GIT_URL} +refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*
GIT_COMMIT=\$(git rev-parse ${GIT_REF}^{commit})
git checkout -f \${GIT_COMMIT}
echo $@
"
}

e2e::down
trap 'e2e::down' EXIT
e2e::up
e2e::test "$@"
Loading

0 comments on commit 9d8c898

Please sign in to comment.