-
Notifications
You must be signed in to change notification settings - Fork 4.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
cluster up: add persistent volumes on startup
- Loading branch information
Showing
14 changed files
with
243 additions
and
61 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,125 @@ | ||
| package openshift | ||
|
|
||
| import ( | ||
| "fmt" | ||
|
|
||
| kapi "k8s.io/kubernetes/pkg/api" | ||
| kbatch "k8s.io/kubernetes/pkg/apis/batch" | ||
| kclientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" | ||
|
|
||
| "github.com/openshift/origin/pkg/bootstrap/docker/errors" | ||
| "github.com/openshift/origin/pkg/client" | ||
| ) | ||
|
|
||
| const pvCount = 1000 | ||
| const createPVScript = `#/bin/bash | ||
| set -e | ||
| function create_pv() { | ||
| BASEDIR=${1} | ||
| NAME=${2} | ||
| cat <<EOF | ||
| apiVersion: v1 | ||
| kind: PersistentVolume | ||
| metadata: | ||
| name: ${NAME} | ||
| labels: | ||
| volume: ${NAME} | ||
| spec: | ||
| capacity: | ||
| storage: 100Gi | ||
| accessModes: | ||
| - ReadWriteOnce | ||
| hostPath: | ||
| path: ${BASEDIR}/${NAME} | ||
| persistentVolumeReclaimPolicy: Retain | ||
| EOF | ||
| } | ||
| mkdir -p /var/lib/origin/openshift.local.pv/registry /var/lib/origin/openshift.local.pv/pv{1..%[1]d} | ||
| if ! chcon -t svirt_sandbox_file_t -R /var/lib/origin/openshift.local.pv &> /dev/null; then | ||
| echo "Not applying SELinux labels" | ||
| fi | ||
| chmod 770 -R /var/lib/origin/openshift.local.pv | ||
| for i in {1..%[1]d}; do | ||
| create_pv "%[2]s" pv${i} | oc create -f - | ||
| # Pause every 10 reps to rate-limit creation | ||
| export index=${i} | ||
| if [[ "$(expr ${index} %% 10)" == "0" ]]; then | ||
| sleep 1 | ||
| fi | ||
| done | ||
| ` | ||
|
|
||
| func (h *Helper) SetupPersistentStorage(osclient client.Interface, kclient kclientset.Interface, dir string) error { | ||
|
|
||
| // Check whether a PV installer SA exists. If it does, PVs have already | ||
| // been provisioned. | ||
| sa, err := kclient.Core().ServiceAccounts("default").Get("pvinstaller") | ||
| if err == nil { | ||
| return nil | ||
| } | ||
|
|
||
| // Create installer SA | ||
| sa = &kapi.ServiceAccount{} | ||
| sa.Name = "pvinstaller" | ||
| _, err = kclient.Core().ServiceAccounts("default").Create(sa) | ||
| if err != nil { | ||
| return errors.NewError("cannot create pvinstaller service account").WithCause(err).WithDetails(h.OriginLog()) | ||
| } | ||
| err = AddSCCToServiceAccount(kclient, "privileged", "pvinstaller", "default") | ||
| if err != nil { | ||
| return errors.NewError("cannot add privileged SCC to pvinstaller service account").WithCause(err).WithDetails(h.OriginLog()) | ||
| } | ||
| err = AddClusterRole(osclient, "cluster-admin", "system:serviceaccount:default:pvinstaller") | ||
| if err != nil { | ||
| return errors.NewError("cannot add cluster role to pvinstaller service account").WithCause(err).WithDetails(h.OriginLog()) | ||
| } | ||
|
|
||
| setupJob := persistentStorageSetupJob(dir, h.image) | ||
| _, err = kclient.Batch().Jobs("default").Create(setupJob) | ||
| if err != nil { | ||
| return errors.NewError("cannot create job to setup persistent volumes").WithCause(err).WithDetails(h.OriginLog()) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func persistentStorageSetupJob(dir, image string) *kbatch.Job { | ||
| // Job volume | ||
| volume := kapi.Volume{} | ||
| volume.Name = "pvdir" | ||
| volume.HostPath = &kapi.HostPathVolumeSource{Path: dir} | ||
|
|
||
| // Volume mount | ||
| mount := kapi.VolumeMount{} | ||
| mount.Name = "pvdir" | ||
| mount.MountPath = "/var/lib/origin/openshift.local.pv" | ||
|
|
||
| // Job container | ||
| container := kapi.Container{} | ||
| container.Name = "storage-setup-job" | ||
| container.Image = image | ||
| container.Command = []string{"/bin/bash", "-c", fmt.Sprintf(createPVScript, pvCount, dir)} | ||
| privileged := true | ||
| container.SecurityContext = &kapi.SecurityContext{ | ||
| Privileged: &privileged, | ||
| } | ||
| container.VolumeMounts = []kapi.VolumeMount{mount} | ||
|
|
||
| // Job | ||
| completions := int32(1) | ||
| deadline := int64(60 * 20) | ||
| job := &kbatch.Job{} | ||
| job.GenerateName = "persistent-storage-setup-" | ||
| job.Spec.Completions = &completions | ||
| job.Spec.ActiveDeadlineSeconds = &deadline | ||
| job.Spec.Template.Spec.Volumes = []kapi.Volume{volume} | ||
| job.Spec.Template.Spec.RestartPolicy = kapi.RestartPolicyNever | ||
| job.Spec.Template.Spec.ServiceAccountName = "pvinstaller" | ||
| job.Spec.Template.Spec.Containers = []kapi.Container{container} | ||
| return job | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.