-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Add local 3.x testing setup * Adapt access mode change for 3.x Now client/API user should proivde access mode in decimal converted octal. For example, (0)777 (octal) should be sent as 511 (decimal) to the API service for create volume call. * Use /home for client mount Many cloud environments has restrictions on root (/). Therefore, use /home for client mount as the default mount in our examples. * Document storage class immutability ... and deletion side affects of Storage Class
- Loading branch information
Showing
8 changed files
with
280 additions
and
15 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
104 changes: 104 additions & 0 deletions
104
kind-cluster/test-configs/local_cluster_3.x/k8s_client.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| apiVersion: apps/v1 | ||
| kind: DaemonSet | ||
| metadata: | ||
| name: client | ||
| namespace: kube-system | ||
| spec: | ||
| selector: | ||
| matchLabels: | ||
| role: client | ||
| template: | ||
| metadata: | ||
| annotations: | ||
| prometheus.io/scrape: 'true' | ||
| prometheus.io/path: '/prometheus' | ||
| prometheus.io/port: '55000' | ||
| labels: | ||
| role: client | ||
| version: "2" | ||
| spec: | ||
| containers: | ||
| - name: quobyte-client | ||
| image: quay.io/quobyte/quobyte-client:3 | ||
| imagePullPolicy: Always | ||
| env: | ||
| - name: QUOBYTE_CLIENT_LOG_LEVEL | ||
| value: INFO | ||
| - name: QUOBYTE_REGISTRY | ||
| # Your Quobyte registry endpoint | ||
| value: venkat.corp.quobyte.com:2985 # Example: hydrogen.quobyte.com:12354 | ||
| - name: QUOBYTE_MOUNT_POINT | ||
| # Corresponding volume mount must be one directory below this path | ||
| # Example volumeMount is : /home/quobyte and clientMountPoint is /home/quobyte/mounts | ||
| value: /home/quobyte/mounts | ||
| # Enabling access keys requires Quobyte version 3.0 or later | ||
| - name: ENABLE_ACCESS_KEY_MOUNTS | ||
| value: "false" # to enable, set it to true | ||
| - name: NODENAME | ||
| valueFrom: | ||
| fieldRef: | ||
| fieldPath: spec.nodeName | ||
| ports: | ||
| - name: http-port | ||
| containerPort: 55000 | ||
| hostPort: 55000 | ||
| protocol: TCP | ||
| readinessProbe: | ||
| timeoutSeconds: 5 | ||
| httpGet: | ||
| port: 55000 | ||
| path: / | ||
| livenessProbe: | ||
| initialDelaySeconds: 30 | ||
| timeoutSeconds: 5 | ||
| httpGet: | ||
| port: 55000 | ||
| path: / | ||
| command: | ||
| - /bin/bash | ||
| - -xec | ||
| - | | ||
| ENABLE_ACCESS_CONTEXTS="" | ||
| if [[ ${ENABLE_ACCESS_KEY_MOUNTS} = true ]]; then | ||
| ENABLE_ACCESS_CONTEXTS="--enable-access-contexts" | ||
| fi | ||
| if cut -d" " -f2 /proc/self/mounts | grep -q ${QUOBYTE_MOUNT_POINT}; then | ||
| umount -l ${QUOBYTE_MOUNT_POINT} | ||
| fi | ||
| mkdir -p /root/.quobyte ${QUOBYTE_MOUNT_POINT} | ||
| if find "$QUOBYTE_MOUNT_POINT" -mindepth 1 -print -quit 2>/dev/null | grep -q .; then | ||
| echo "POLLUTED MOUNT POINT DETECTED! Cannot use $QUOBYTE_MOUNT_POINT as a mount point." | ||
| echo "Please remove all files and directories from $QUOBYTE_MOUNT_POINT and " | ||
| echo "run 'chattr +i $QUOBYTE_MOUNT_POINT' to prevent future mount point pollution." | ||
| else | ||
| # set the mount point immutable. As long as mount.quobyte does not run, | ||
| # other processes cannot write data to this dir. | ||
| chattr +i ${QUOBYTE_MOUNT_POINT} || \ | ||
| echo "WARNING: The local filesystem does not support IMMUTABLE flag. Mount point pollution is possible." | ||
| /usr/bin/mount.quobyte --hostname ${NODENAME} \ | ||
| --http-port 55000 -f \ | ||
| -d ${QUOBYTE_CLIENT_LOG_LEVEL} -l /dev/stdout ${OPTS} \ | ||
| --minidump-path /tmp/minidumps --allow-minidump-upload \ | ||
| ${QUOBYTE_REGISTRY}/ ${QUOBYTE_MOUNT_POINT} ${ENABLE_ACCESS_CONTEXTS} | ||
| fi | ||
| securityContext: | ||
| privileged: true | ||
| volumeMounts: | ||
| - name: quobyte-mount | ||
| mountPath: /home/quobyte | ||
| mountPropagation: Bidirectional | ||
| - name: minidumps-dir | ||
| mountPath: /tmp/minidumps | ||
| lifecycle: | ||
| preStop: | ||
| exec: | ||
| command: ["/bin/bash", "-xc", "umount -l ${QUOBYTE_MOUNT_POINT}"] | ||
| nodeSelector: | ||
| quobyte_client: "true" | ||
| volumes: | ||
| - name: quobyte-mount | ||
| hostPath: | ||
| path: /home/quobyte | ||
| - name: minidumps-dir | ||
| hostPath: | ||
| path: /var/lib/quobyte/.minidumps |
16 changes: 16 additions & 0 deletions
16
kind-cluster/test-configs/local_cluster_3.x/k8s_quobyte_secret.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| apiVersion: v1 | ||
| kind: Secret | ||
| metadata: | ||
| name: quobyte-admin-credentials | ||
| type: "kubernetes.io/quobyte" | ||
| data: | ||
| # IMPORTANT: All the values should be base64 encoded. | ||
| # IMPORTANT: All the keys are case sensitive and should be used AS IS. | ||
| # echo -n "value" | base64 gives the base64 encoded data for 'value' | ||
|
|
||
| # Base64 encoded Quobyte API user name. Key must be 'user'. | ||
| # Update data (YWRtaW4=) with base64 encoded API user name for your Quobyte deployment. | ||
| user: YWRtaW4= | ||
| # Base64 encoded Quobyte API password. Key must be 'password'. | ||
| # Update data (cXVvYnl0ZQ==) with base64 encoded API password for your Quobyte deployment. | ||
| password: cXVvYnl0ZQ== |
46 changes: 46 additions & 0 deletions
46
kind-cluster/test-configs/local_cluster_3.x/k8s_storage_class.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| apiVersion: storage.k8s.io/v1 | ||
| kind: StorageClass | ||
| metadata: | ||
| name: quobyte-csi | ||
| # provisioner must be csi.quobyte.com | ||
| provisioner: csi.quobyte.com | ||
| # Should be set to 'true' to expand volume size. | ||
| # https://kubernetes.io/docs/concepts/storage/persistent-volumes/#csi-volume-expansion | ||
| # Volume shrinking is not possible and expansion only possible for dynamically provisioned volumes. | ||
| # By default, Quobyte volumes are not size restricted, | ||
| # but can be size restricted by creating a Quobyte quota | ||
| # for the volume (by setting parameters.createQuota: true). | ||
| # Volume expansion only makes sense if volume is created with a quota. | ||
| allowVolumeExpansion: true | ||
| parameters: | ||
| # quobyteTenant is used as tenant if CSI driver is deployed with | ||
| # --use_k8s_namespace_as_tenant=false. | ||
| # If CSI driver is deployed with --use_k8s_namespace_as_tenant=true, | ||
| # quobyteTenant is ignored and K8S namespace in which PVC is created | ||
| # becomes Quobyte tenant | ||
| # i.e; K8S namespace "tenant_x" maps to "tenant_x" in Quobyte storage. | ||
| quobyteTenant: "My Tenant" | ||
| # secret is used for dynamic volume provisioning. | ||
| # The user credentials provided in this secret must have | ||
| # volume create/delete permissions for `quobyteTenant` above. | ||
| csi.storage.k8s.io/provisioner-secret-name: "quobyte-admin-credentials" | ||
| csi.storage.k8s.io/provisioner-secret-namespace: "quobyte" | ||
| # Resize volume requires secrets to communicate with Quobyte API | ||
| csi.storage.k8s.io/controller-expand-secret-name: "quobyte-admin-credentials" | ||
| csi.storage.k8s.io/controller-expand-secret-namespace: "quobyte" | ||
| # Mount secrets | ||
| csi.storage.k8s.io/node-publish-secret-name: "quobyte-admin-credentials" | ||
| csi.storage.k8s.io/node-publish-secret-namespace: "quobyte" | ||
| # Quobyte volume configuration | ||
| quobyteConfig: "BASE" | ||
| # creates quota for the volume if set to true. The size of the Quota | ||
| # is the storage requested in PVC. If false, creates volume without size limit. | ||
| createQuota: "true" | ||
| user: root | ||
| group: root | ||
| accessMode: "777" | ||
|
|
||
| # spaces are not allowed and requires Quobyte 3.x | ||
| #labels: "encrypted:yes,mediatype:hdd" | ||
| # Set reclaimPolicy: Retain to keep the volume even after PV deletion | ||
| reclaimPolicy: Delete |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| # Used to generate CSI deployment files specific to given k8s version | ||
| k8sVersion: "1.25.0" | ||
| quobyte: | ||
| # Quobyte cluster version (valid values - 2 for Quobyte 2.x/3 for Quobyte 3.x) | ||
| version: 3 | ||
| # apiURL should be of the form http(s)://<ip or resolvable host>:<port> | ||
| # Example Quobyte API: http://hydrogen.quobyte.com:26801 | ||
| apiURL: "http://venkat.corp.quobyte.com:2978" | ||
|
|
||
| # Replication for csi controller pod, must be at least one | ||
| # Note that node driver which is responsible for mounting volume cannot be replicated | ||
| csiControllerReplicas: 1 | ||
|
|
||
| # Maps /etc/ssl/certs/ from host into Quobyte CSI containers. | ||
| # Must set to true, if you have https:// API URL. Otherwise, | ||
| # can be set to false. | ||
| # If you are using private CA certificate, add the CA to | ||
| # all k8s hosts before deploying driver. | ||
| mapHostCertsIntoContainer: true | ||
|
|
||
| # Quobyte client should be deployed with <clientMountPoint>/mounts | ||
| # For example, if you set clientMountPoint: /mnt/quobyte then quobyte | ||
| # client should be deployed with /mnt/quobyte/mounts as the mount point | ||
| clientMountPoint: /home/quobyte | ||
|
|
||
| # Should be a valid DNS name. Do not change this between upgrades, otherwise | ||
| # requires manual delete of Pods, PVCs, PVs and backing volumes. | ||
| # StorageClass.provisioner must match the value configured here. | ||
| csiProvisionerName: csi.quobyte.com | ||
|
|
||
| # When set to true, uses PVC.namespace as Quobyte tenant. | ||
| # This does not create tenants automatically, your storage system must | ||
| # have tenants that match the namespace. | ||
| useK8SNamespaceAsTenant: false | ||
|
|
||
| # Set this to true to mount Quobyte volumes using Quobyte file system access keys. | ||
| # Then, add access key information to your K8S secret and use this secret in StorageClass as | ||
| # mount secret. Quobyte-CSI and Client uses access key information | ||
| # from the mount secret to securely mount volumes into k8s. | ||
| # When mount access keys are enabled in CSI driver, Quobyte client must also be | ||
| # enabled with access key contexts. To enable access key contexts on native client, | ||
| # add "enable-access-contexts" in /etc/quobyte/client-service.cfg. | ||
| # For container based client configuration, see client.yaml definition. | ||
| # Requires Quobyte 3.x to enable mount access keys | ||
| enableAccessKeyMounts: false | ||
|
|
||
| # Enabling this feature requires additional driver setup (see README.md) | ||
| # Quobyte 3.x is recommended for snapshots | ||
| enableSnapshots: false | ||
|
|
||
| # Set to true to deploy csi driver with pod security policies. | ||
| # Change required ONLY if you have pod security policies enabled K8S environment. | ||
| podSecurityPolicies: false | ||
|
|
||
| # Set to true to schedule erase volume task immediately (supported by Quobyte 3.x) | ||
| immediateErase: false | ||
|
|
||
| podKiller: | ||
| # To disable pod killer, uninstall current CSI driver (helm uninstall <chart-name>) | ||
| # set enable: false and install CSI driver again | ||
| enable: true | ||
| # should be a valid golang time.Duration | ||
| monitoringInterval: 5s | ||
|
|
||
| # The dev configuration is intended for Quobyte Developers and internal use. | ||
| # Please do NOT change the dev: configuration unless otherwise advised to change. | ||
| dev: | ||
| # CSI Release version | ||
| csiProvisionerVersion: "v1.8.3" | ||
| # Release container | ||
| # github.com/quobyte/quobyte-csi | ||
| csiImage: "quay.io/quobyte/csi:v1.8.3" | ||
| # github.com/quobyte/pod-killer | ||
| podKillerImage: quay.io/quobyte/pod-killer:v0.1.3 | ||
| # k8s sidecar containers (https://github.com/kubernetes-csi/) | ||
| # Updating k8s...Image might require RBAC files update | ||
| # https://github.com/quobyte/quobyte-csi/tree/master/quobyte-csi-driver/templates/pods/rbac | ||
| k8sProvisionerImage: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0 | ||
| k8sResizerImage: k8s.gcr.io/sig-storage/csi-resizer:v1.5.0 | ||
| k8sNodeRegistrarImage: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.1 | ||
| k8sAttacherImage: k8s.gcr.io/sig-storage/csi-attacher:v3.5.0 | ||
| # when updating image for snapshotter, update snaptshotter setup CRD with | ||
| # instructions in README (CRD should be pulled from matched release). | ||
| # Additionally, ./quobyte-csi-driver/k8s-snapshot-controller.yaml | ||
| # (see this file for source link) should be updated with appropriate version | ||
| # files (Do NOT forget updating namespace to kube-system) | ||
| k8sSnapshotterImage: k8s.gcr.io/sig-storage/csi-snapshotter:v6.0.1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters