From 0ba0122e6f0c810a932beeb57eb8fc65b4bf6dc0 Mon Sep 17 00:00:00 2001 From: Denys Kondratenko Date: Fri, 10 Feb 2023 20:06:52 +0100 Subject: [PATCH 1/2] update CSI driver, enable multi-node --- .../deploy/csi-hostpath-attacher.yaml.tmpl | 16 +- .../deploy/csi-hostpath-driverinfo.yaml.tmpl | 6 + .../deploy/csi-hostpath-plugin.yaml.tmpl | 111 ++++++++------ .../deploy/csi-hostpath-provisioner.yaml.tmpl | 81 ----------- .../deploy/csi-hostpath-resizer.yaml.tmpl | 16 +- .../deploy/csi-hostpath-snapshotter.yaml.tmpl | 81 ----------- ...ac-external-health-monitor-agent.yaml.tmpl | 66 --------- ...ternal-health-monitor-controller.yaml.tmpl | 2 +- .../rbac/rbac-external-provisioner.yaml.tmpl | 22 ++- .../rbac/rbac-external-resizer.yaml.tmpl | 2 +- .../rbac/rbac-external-snapshotter.yaml.tmpl | 59 ++++---- .../rbac/rbac-hostpath.yaml.tmpl | 137 ++++++++++++++++++ .../rbac-volume-snapshot-controller.yaml.tmpl | 15 +- ...e-snapshot-controller-deployment.yaml.tmpl | 10 ++ pkg/minikube/assets/addons.go | 52 +++---- 15 files changed, 307 insertions(+), 369 deletions(-) delete mode 100644 deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-provisioner.yaml.tmpl delete mode 100644 deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-snapshotter.yaml.tmpl delete mode 100644 deploy/addons/csi-hostpath-driver/rbac/rbac-external-health-monitor-agent.yaml.tmpl create mode 100644 deploy/addons/csi-hostpath-driver/rbac/rbac-hostpath.yaml.tmpl diff --git a/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-attacher.yaml.tmpl b/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-attacher.yaml.tmpl index cf5a80154002..e49d56209610 100644 --- a/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-attacher.yaml.tmpl +++ b/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-attacher.yaml.tmpl @@ -38,25 +38,15 @@ spec: replicas: 1 selector: matchLabels: - app: csi-hostpath-attacher + app.kubernetes.io/name: csi-hostpath-attacher template: metadata: labels: - app: csi-hostpath-attacher + app.kubernetes.io/name: csi-hostpath-attacher addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/minikube-addons: csi-hostpath-driver spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - csi-hostpathplugin - topologyKey: kubernetes.io/hostname - serviceAccountName: csi-attacher + serviceAccount: csi-attacher containers: - name: csi-attacher image: {{.CustomRegistries.Attacher | default .ImageRepository | default .Registries.Attacher }}{{.Images.Attacher}} diff --git a/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-driverinfo.yaml.tmpl b/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-driverinfo.yaml.tmpl index 4b2be01185d4..e509d234c20d 100644 --- a/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-driverinfo.yaml.tmpl +++ b/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-driverinfo.yaml.tmpl @@ -27,3 +27,9 @@ spec: # To determine at runtime which mode a volume uses, pod info and its # "csi.storage.k8s.io/ephemeral" entry are needed. podInfoOnMount: true + # No attacher needed. + attachRequired: false + storageCapacity: true + # Kubernetes may use fsGroup to change permissions and ownership + # of the volume to match user requested fsGroup in the pod's SecurityPolicy + fsGroupPolicy: File diff --git a/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-plugin.yaml.tmpl b/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-plugin.yaml.tmpl index b737e916ed1c..467521d35d9e 100644 --- a/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-plugin.yaml.tmpl +++ b/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-plugin.yaml.tmpl @@ -16,64 +16,37 @@ # are needed only because of condition explained in # https://github.com/kubernetes/kubernetes/issues/69608 -kind: Service -apiVersion: v1 -metadata: - name: csi-hostpathplugin - namespace: kube-system - labels: - app: csi-hostpathplugin - addonmanager.kubernetes.io/mode: Reconcile -spec: - selector: - app: csi-hostpathplugin - ports: - - name: dummy - port: 12345 ---- -kind: StatefulSet +kind: DaemonSet apiVersion: apps/v1 metadata: name: csi-hostpathplugin namespace: kube-system labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: plugin addonmanager.kubernetes.io/mode: Reconcile spec: - serviceName: "csi-hostpathplugin" - # One replica only: - # Host path driver only works when everything runs - # on a single node. We achieve that by starting it once and then - # co-locate all other pods via inter-pod affinity - replicas: 1 selector: matchLabels: - app: csi-hostpathplugin + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: plugin addonmanager.kubernetes.io/mode: Reconcile template: metadata: labels: - app: csi-hostpathplugin + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: plugin addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/minikube-addons: csi-hostpath-driver spec: - serviceAccount: csi-external-health-monitor-controller + serviceAccount: csi-hostpathplugin-sa containers: - - name: csi-external-health-monitor-agent - image: {{.CustomRegistries.HostMonitorAgent | default .ImageRepository | default .Registries.HostMonitorAgent }}{{.Images.HostMonitorAgent}} - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: ADDRESS - value: /csi/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - name: csi-external-health-monitor-controller image: {{.CustomRegistries.HostMonitorController | default .ImageRepository | default .Registries.HostMonitorController }}{{.Images.HostMonitorController}} args: @@ -163,6 +136,62 @@ spec: - --csi-address=/csi/csi.sock - --health-port=9898 + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0 + args: + - -v=5 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --enable-capacity + - --capacity-ownerref-level=0 # pod is owner + - --node-deployment=true + - --strict-topology=true + - --immediate-topology=false + - --worker-threads=5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + + - name: csi-snapshotter + image: {{.CustomRegistries.Snapshotter | default .ImageRepository | default .Registries.Snapshotter }}{{.Images.Snapshotter}} + args: + - -v=5 + - --csi-address=/csi/csi.sock + - --node-deployment + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + securityContext: + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: - hostPath: path: /var/lib/kubelet/plugins/csi-hostpath diff --git a/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-provisioner.yaml.tmpl b/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-provisioner.yaml.tmpl deleted file mode 100644 index b8c1acccc7df..000000000000 --- a/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-provisioner.yaml.tmpl +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2018 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: Service -apiVersion: v1 -metadata: - name: csi-hostpath-provisioner - namespace: kube-system - labels: - app: csi-hostpath-provisioner - addonmanager.kubernetes.io/mode: Reconcile -spec: - selector: - app: csi-hostpath-provisioner - ports: - - name: dummy - port: 12345 - ---- -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: csi-hostpath-provisioner - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: Reconcile -spec: - serviceName: "csi-hostpath-provisioner" - replicas: 1 - selector: - matchLabels: - app: csi-hostpath-provisioner - template: - metadata: - labels: - app: csi-hostpath-provisioner - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/minikube-addons: csi-hostpath-driver - spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - csi-hostpathplugin - topologyKey: kubernetes.io/hostname - serviceAccountName: csi-provisioner - containers: - - name: csi-provisioner - image: {{.CustomRegistries.Provisioner | default .ImageRepository | default .Registries.Provisioner }}{{.Images.Provisioner}} - args: - - -v=5 - - --csi-address=/csi/csi.sock - - --feature-gates=Topology=true - securityContext: - # This is necessary only for systems with SELinux, where - # non-privileged sidecar containers cannot access unix domain socket - # created by privileged CSI driver container. - privileged: true - volumeMounts: - - mountPath: /csi - name: socket-dir - volumes: - - hostPath: - path: /var/lib/kubelet/plugins/csi-hostpath - type: DirectoryOrCreate - name: socket-dir diff --git a/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-resizer.yaml.tmpl b/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-resizer.yaml.tmpl index 7995ce80067f..e1270b6a709f 100644 --- a/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-resizer.yaml.tmpl +++ b/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-resizer.yaml.tmpl @@ -40,25 +40,15 @@ spec: replicas: 1 selector: matchLabels: - app: csi-hostpath-resizer + app.kubernetes.io/name: csi-hostpath-resizer template: metadata: labels: - app: csi-hostpath-resizer + app.kubernetes.io/name: csi-hostpath-resizer addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/minikube-addons: csi-hostpath-driver spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - csi-hostpathplugin - topologyKey: kubernetes.io/hostname - serviceAccountName: csi-resizer + serviceAccount: csi-resizer containers: - name: csi-resizer image: {{.CustomRegistries.Resizer | default .ImageRepository | default .Registries.Resizer }}{{.Images.Resizer}} diff --git a/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-snapshotter.yaml.tmpl b/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-snapshotter.yaml.tmpl deleted file mode 100644 index 708dd6119408..000000000000 --- a/deploy/addons/csi-hostpath-driver/deploy/csi-hostpath-snapshotter.yaml.tmpl +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2018 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: Service -apiVersion: v1 -metadata: - name: csi-hostpath-snapshotter - namespace: kube-system - labels: - app: csi-hostpath-snapshotter - addonmanager.kubernetes.io/mode: Reconcile -spec: - selector: - app: csi-hostpath-snapshotter - ports: - - name: dummy - port: 12345 - ---- -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: csi-hostpath-snapshotter - namespace: kube-system - labels: - addonmanager.kubernetes.io/mode: Reconcile -spec: - serviceName: "csi-hostpath-snapshotter" - replicas: 1 - selector: - matchLabels: - app: csi-hostpath-snapshotter - addonmanager.kubernetes.io/mode: Reconcile - template: - metadata: - labels: - app: csi-hostpath-snapshotter - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/minikube-addons: csi-hostpath-driver - spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - csi-hostpathplugin - topologyKey: kubernetes.io/hostname - serviceAccount: csi-snapshotter - containers: - - name: csi-snapshotter - image: {{.CustomRegistries.Snapshotter | default .ImageRepository | default .Registries.Snapshotter }}{{.Images.Snapshotter}} - args: - - -v=5 - - --csi-address=/csi/csi.sock - securityContext: - # This is necessary only for systems with SELinux, where - # non-privileged sidecar containers cannot access unix domain socket - # created by privileged CSI driver container. - privileged: true - volumeMounts: - - mountPath: /csi - name: socket-dir - volumes: - - hostPath: - path: /var/lib/kubelet/plugins/csi-hostpath - type: DirectoryOrCreate - name: socket-dir diff --git a/deploy/addons/csi-hostpath-driver/rbac/rbac-external-health-monitor-agent.yaml.tmpl b/deploy/addons/csi-hostpath-driver/rbac/rbac-external-health-monitor-agent.yaml.tmpl deleted file mode 100644 index e2cb6fc82d3c..000000000000 --- a/deploy/addons/csi-hostpath-driver/rbac/rbac-external-health-monitor-agent.yaml.tmpl +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2018 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This YAML file contains all RBAC objects that are necessary to run external -# CSI health monitor agent. -# -# In production, each CSI driver deployment has to be customized: -# - to avoid conflicts, use non-default namespace and different names -# for non-namespaced entities like the ClusterRole -# - decide whether the deployment replicates the external CSI -# health monitor agent, in which case leadership election must be enabled; -# this influences the RBAC setup, see below - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-external-health-monitor-agent - namespace: kube-system - ---- -# Health monitor agent must be able to work with PVs, PVCs, Nodes and Pods -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: external-health-monitor-agent-runner -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "patch"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-external-health-monitor-agent-role -subjects: - - kind: ServiceAccount - name: csi-external-health-monitor-agent - namespace: kube-system -roleRef: - kind: ClusterRole - name: external-health-monitor-agent-runner - apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/deploy/addons/csi-hostpath-driver/rbac/rbac-external-health-monitor-controller.yaml.tmpl b/deploy/addons/csi-hostpath-driver/rbac/rbac-external-health-monitor-controller.yaml.tmpl index 0cc7a064557c..19c8eda4ae8b 100644 --- a/deploy/addons/csi-hostpath-driver/rbac/rbac-external-health-monitor-controller.yaml.tmpl +++ b/deploy/addons/csi-hostpath-driver/rbac/rbac-external-health-monitor-controller.yaml.tmpl @@ -91,4 +91,4 @@ subjects: roleRef: kind: Role name: external-health-monitor-controller-cfg - apiGroup: rbac.authorization.k8s.io \ No newline at end of file + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/addons/csi-hostpath-driver/rbac/rbac-external-provisioner.yaml.tmpl b/deploy/addons/csi-hostpath-driver/rbac/rbac-external-provisioner.yaml.tmpl index 70f14c2dc70c..649176901248 100644 --- a/deploy/addons/csi-hostpath-driver/rbac/rbac-external-provisioner.yaml.tmpl +++ b/deploy/addons/csi-hostpath-driver/rbac/rbac-external-provisioner.yaml.tmpl @@ -63,6 +63,10 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] + # Access to volumeattachments is only needed when the CSI driver + # has the PUBLISH_UNPUBLISH_VOLUME controller capability. + # In that case, external-provisioner will watch volumeattachments + # to determine when it is safe to delete a volume. - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch"] @@ -92,12 +96,24 @@ metadata: rules: # Only one of the following rules for endpoints or leases is required based on # what is set for `--leader-election-type`. Endpoints are deprecated in favor of Leases. -- apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] +# Permissions for CSIStorageCapacity are only needed enabling the publishing +# of storage capacity information. +- apiGroups: ["storage.k8s.io"] + resources: ["csistoragecapacities"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +# The GET permissions below are needed for walking up the ownership chain +# for CSIStorageCapacity. They are sufficient for deployment via +# StatefulSet (only needs to get Pod) and Deployment (needs to get +# Pod and then ReplicaSet to find the Deployment). +- apiGroups: [""] + resources: ["pods"] + verbs: ["get"] +- apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get"] --- kind: RoleBinding diff --git a/deploy/addons/csi-hostpath-driver/rbac/rbac-external-resizer.yaml.tmpl b/deploy/addons/csi-hostpath-driver/rbac/rbac-external-resizer.yaml.tmpl index bf0e1ebfd63e..44cc536ebaf0 100644 --- a/deploy/addons/csi-hostpath-driver/rbac/rbac-external-resizer.yaml.tmpl +++ b/deploy/addons/csi-hostpath-driver/rbac/rbac-external-resizer.yaml.tmpl @@ -71,7 +71,7 @@ roleRef: apiGroup: rbac.authorization.k8s.io --- -# Resizer must be able to work with end point in current namespace +# Resizer must be able to work with `leases` in current namespace # if (and only if) leadership election is enabled kind: Role apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/addons/csi-hostpath-driver/rbac/rbac-external-snapshotter.yaml.tmpl b/deploy/addons/csi-hostpath-driver/rbac/rbac-external-snapshotter.yaml.tmpl index 74880728341a..5d9507cef5a3 100644 --- a/deploy/addons/csi-hostpath-driver/rbac/rbac-external-snapshotter.yaml.tmpl +++ b/deploy/addons/csi-hostpath-driver/rbac/rbac-external-snapshotter.yaml.tmpl @@ -12,12 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -# RBAC file for the snapshot controller. + +# Together with the RBAC file for external-provisioner, this YAML file +# contains all RBAC objects that are necessary to run external CSI +# snapshotter. # -# The snapshot controller implements the control loop for CSI snapshot functionality. -# It should be installed as part of the base Kubernetes distribution in an appropriate -# namespace for components implementing base system functionality. For installing with -# Vanilla Kubernetes, kube-system makes sense for the namespace. +# In production, each CSI driver deployment has to be customized: +# - to avoid conflicts, use non-default namespace and different names +# for non-namespaced entities like the ClusterRole +# - optionally rename the non-namespaced ClusterRole if there +# are conflicts with other deployments apiVersion: v1 kind: ServiceAccount @@ -30,35 +34,27 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: # rename if there are conflicts - name: csi-snapshotter-runner + name: external-snapshotter-runner rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] + # Secret permission is optional. + # Enable it if your driver needs secret. + # For example, `csi.storage.k8s.io/snapshotter-secret-name` is set in VolumeSnapshotClass. + # See https://kubernetes-csi.github.io/docs/secrets-and-credentials.html for more details. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses"] verbs: ["get", "list", "watch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] + verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] + verbs: ["update", "patch"] --- kind: ClusterRoleBinding @@ -68,11 +64,12 @@ metadata: subjects: - kind: ServiceAccount name: csi-snapshotter + # replace with non-default namespace name namespace: kube-system roleRef: kind: ClusterRole # change the name also here if the ClusterRole gets renamed - name: csi-snapshotter-runner + name: external-snapshotter-runner apiGroup: rbac.authorization.k8s.io --- @@ -80,17 +77,17 @@ kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: namespace: kube-system - name: csi-snapshotter-leaderelection + name: external-snapshotter-leaderelection rules: - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: csi-snapshotter-leaderelection + name: external-snapshotter-leaderelection namespace: kube-system subjects: - kind: ServiceAccount @@ -98,5 +95,5 @@ subjects: namespace: kube-system roleRef: kind: Role - name: csi-snapshotter-leaderelection - apiGroup: rbac.authorization.k8s.io \ No newline at end of file + name: external-snapshotter-leaderelection + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/addons/csi-hostpath-driver/rbac/rbac-hostpath.yaml.tmpl b/deploy/addons/csi-hostpath-driver/rbac/rbac-hostpath.yaml.tmpl new file mode 100644 index 000000000000..7c93d9bfa5b0 --- /dev/null +++ b/deploy/addons/csi-hostpath-driver/rbac/rbac-hostpath.yaml.tmpl @@ -0,0 +1,137 @@ +# Copyright 2023 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# All of the individual sidecar RBAC roles get bound +# to this account. +kind: ServiceAccount +apiVersion: v1 +metadata: + name: csi-hostpathplugin-sa + namespace: kube-system + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: serviceaccount +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: provisioner-cluster-role + name: csi-hostpathplugin-provisioner-cluster-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-provisioner-runner +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: snapshotter-cluster-role + name: csi-hostpathplugin-snapshotter-cluster-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-snapshotter-runner +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: health-monitor-cluster-role + name: csi-hostpathplugin-health-monitor-cluster-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-health-monitor-controller-runner +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: provisioner-role + name: csi-hostpathplugin-provisioner-role + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: snapshotter-role + name: csi-hostpathplugin-snapshotter-role + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: external-snapshotter-leaderelection +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/instance: hostpath.csi.k8s.io + app.kubernetes.io/part-of: csi-driver-host-path + app.kubernetes.io/name: csi-hostpathplugin + app.kubernetes.io/component: snapshotter-role + name: csi-hostpathplugin-health-monitor-role + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: external-health-monitor-controller-cfg +subjects: +- kind: ServiceAccount + name: csi-hostpathplugin-sa + namespace: kube-system diff --git a/deploy/addons/volumesnapshots/rbac-volume-snapshot-controller.yaml.tmpl b/deploy/addons/volumesnapshots/rbac-volume-snapshot-controller.yaml.tmpl index f8064c7f8c9f..ddc4e5dcc5ce 100644 --- a/deploy/addons/volumesnapshots/rbac-volume-snapshot-controller.yaml.tmpl +++ b/deploy/addons/volumesnapshots/rbac-volume-snapshot-controller.yaml.tmpl @@ -44,9 +44,6 @@ rules: - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] @@ -55,13 +52,19 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] + verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots/status"] - verbs: ["update"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding diff --git a/deploy/addons/volumesnapshots/volume-snapshot-controller-deployment.yaml.tmpl b/deploy/addons/volumesnapshots/volume-snapshot-controller-deployment.yaml.tmpl index 6c0e3ac8fbd9..2061d393ae1a 100644 --- a/deploy/addons/volumesnapshots/volume-snapshot-controller-deployment.yaml.tmpl +++ b/deploy/addons/volumesnapshots/volume-snapshot-controller-deployment.yaml.tmpl @@ -18,6 +18,15 @@ spec: selector: matchLabels: app: snapshot-controller + # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable + # in #504 the snapshot-controller will exit after around 7.5 seconds if it + # can't find the v1 CRDs so this value should be greater than that + minReadySeconds: 15 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: @@ -30,4 +39,5 @@ spec: args: - "--v=5" - "--leader-election=true" + - "--enable-distributed-snapshotting" imagePullPolicy: IfNotPresent diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index b2aab884de8a..2da8ec94f8f6 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -617,9 +617,9 @@ var Addons = map[string]*Addon{ "volume-snapshot-controller-deployment.yaml", "0640"), }, false, "volumesnapshots", "Kubernetes", "", "https://minikube.sigs.k8s.io/docs/tutorials/volume_snapshots_and_csi/", map[string]string{ - "SnapshotController": "sig-storage/snapshot-controller:v4.0.0@sha256:00fcc441ea9f72899c25eed61d602272a2a58c5f0014332bdcb5ac24acef08e4", + "SnapshotController": "sig-storage/snapshot-controller:v6.1.0@sha256:823c75d0c45d1427f6d850070956d9ca657140a7bbf828381541d1d808475280", }, map[string]string{ - "SnapshotController": "k8s.gcr.io", + "SnapshotController": "registry.k8s.io", }), "csi-hostpath-driver": NewAddon([]*BinAsset{ MustBinAsset(addons.CsiHostpathDriverAssets, @@ -628,9 +628,9 @@ var Addons = map[string]*Addon{ "rbac-external-attacher.yaml", "0640"), MustBinAsset(addons.CsiHostpathDriverAssets, - "csi-hostpath-driver/rbac/rbac-external-health-monitor-agent.yaml.tmpl", + "csi-hostpath-driver/rbac/rbac-hostpath.yaml.tmpl", vmpath.GuestAddonsDir, - "rbac-external-health-monitor-agent.yaml", + "rbac-hostpath.yaml", "0640"), MustBinAsset(addons.CsiHostpathDriverAssets, "csi-hostpath-driver/rbac/rbac-external-health-monitor-controller.yaml.tmpl", @@ -667,46 +667,34 @@ var Addons = map[string]*Addon{ vmpath.GuestAddonsDir, "csi-hostpath-plugin.yaml", "0640"), - MustBinAsset(addons.CsiHostpathDriverAssets, - "csi-hostpath-driver/deploy/csi-hostpath-provisioner.yaml.tmpl", - vmpath.GuestAddonsDir, - "csi-hostpath-provisioner.yaml", - "0640"), MustBinAsset(addons.CsiHostpathDriverAssets, "csi-hostpath-driver/deploy/csi-hostpath-resizer.yaml.tmpl", vmpath.GuestAddonsDir, "csi-hostpath-resizer.yaml", "0640"), - MustBinAsset(addons.CsiHostpathDriverAssets, - "csi-hostpath-driver/deploy/csi-hostpath-snapshotter.yaml.tmpl", - vmpath.GuestAddonsDir, - "csi-hostpath-snapshotter.yaml", - "0640"), MustBinAsset(addons.CsiHostpathDriverAssets, "csi-hostpath-driver/deploy/csi-hostpath-storageclass.yaml.tmpl", vmpath.GuestAddonsDir, "csi-hostpath-storageclass.yaml", "0640"), }, false, "csi-hostpath-driver", "Kubernetes", "", "https://minikube.sigs.k8s.io/docs/tutorials/volume_snapshots_and_csi/", map[string]string{ - "Attacher": "sig-storage/csi-attacher:v3.1.0@sha256:50c3cfd458fc8e0bf3c8c521eac39172009382fc66dc5044a330d137c6ed0b09", - "HostMonitorAgent": "sig-storage/csi-external-health-monitor-agent:v0.2.0@sha256:c20d4a4772599e68944452edfcecc944a1df28c19e94b942d526ca25a522ea02", - "HostMonitorController": "sig-storage/csi-external-health-monitor-controller:v0.2.0@sha256:14988b598a180cc0282f3f4bc982371baf9a9c9b80878fb385f8ae8bd04ecf16", - "NodeDriverRegistrar": "sig-storage/csi-node-driver-registrar:v2.0.1@sha256:e07f914c32f0505e4c470a62a40ee43f84cbf8dc46ff861f31b14457ccbad108", - "HostPathPlugin": "sig-storage/hostpathplugin:v1.6.0@sha256:b526bd29630261eceecf2d38c84d4f340a424d57e1e2661111e2649a4663b659", - "LivenessProbe": "sig-storage/livenessprobe:v2.2.0@sha256:48da0e4ed7238ad461ea05f68c25921783c37b315f21a5c5a2780157a6460994", - "Resizer": "sig-storage/csi-resizer:v1.1.0@sha256:7a5ba58a44e0d749e0767e4e37315bcf6a61f33ce3185c1991848af4db0fb70a", - "Snapshotter": "sig-storage/csi-snapshotter:v4.0.0@sha256:51f2dfde5bccac7854b3704689506aeecfb793328427b91115ba253a93e60782", - "Provisioner": "sig-storage/csi-provisioner:v2.1.0@sha256:20c828075d1e36f679d6a91e905b0927141eef5e15be0c9a1ca4a6a0ed9313d2", + "Attacher": "sig-storage/csi-attacher:v4.0.0@sha256:9a685020911e2725ad019dbce6e4a5ab93d51e3d4557f115e64343345e05781b", + "HostMonitorController": "sig-storage/csi-external-health-monitor-controller:v0.7.0@sha256:80b9ba94aa2afe24553d69bd165a6a51552d1582d68618ec00d3b804a7d9193c", + "NodeDriverRegistrar": "sig-storage/csi-node-driver-registrar:v2.6.0@sha256:f1c25991bac2fbb7f5fcf91ed9438df31e30edee6bed5a780464238aa09ad24c", + "HostPathPlugin": "sig-storage/hostpathplugin:v1.9.0@sha256:92257881c1d6493cf18299a24af42330f891166560047902b8d431fb66b01af5", + "LivenessProbe": "sig-storage/livenessprobe:v2.8.0@sha256:cacee2b5c36dd59d4c7e8469c05c9e4ef53ecb2df9025fa8c10cdaf61bce62f0", + "Resizer": "sig-storage/csi-resizer:v1.6.0@sha256:425d8f1b769398127767b06ed97ce62578a3179bcb99809ce93a1649e025ffe7", + "Snapshotter": "sig-storage/csi-snapshotter:v6.1.0@sha256:291334908ddf71a4661fd7f6d9d97274de8a5378a2b6fdfeb2ce73414a34f82f", + "Provisioner": "sig-storage/csi-provisioner:v3.3.0@sha256:ee3b525d5b89db99da3b8eb521d9cd90cb6e9ef0fbb651e98bb37be78d36b5b8", }, map[string]string{ - "Attacher": "k8s.gcr.io", - "HostMonitorAgent": "k8s.gcr.io", - "HostMonitorController": "k8s.gcr.io", - "NodeDriverRegistrar": "k8s.gcr.io", - "HostPathPlugin": "k8s.gcr.io", - "LivenessProbe": "k8s.gcr.io", - "Resizer": "k8s.gcr.io", - "Snapshotter": "k8s.gcr.io", - "Provisioner": "k8s.gcr.io", + "Attacher": "registry.k8s.io", + "HostMonitorController": "registry.k8s.io", + "NodeDriverRegistrar": "registry.k8s.io", + "HostPathPlugin": "registry.k8s.io", + "LivenessProbe": "registry.k8s.io", + "Resizer": "registry.k8s.io", + "Snapshotter": "registry.k8s.io", + "Provisioner": "registry.k8s.io", }), "portainer": NewAddon([]*BinAsset{ MustBinAsset(addons.PortainerAssets, From 874de40ca7bf4b1309aaece2202746677388c903 Mon Sep 17 00:00:00 2001 From: Denys Kondratenko Date: Mon, 13 Feb 2023 22:26:03 +0100 Subject: [PATCH 2/2] add multi-node cluster doc for the csi hostpath driver --- site/content/en/docs/handbook/persistent_volumes.md | 2 ++ site/content/en/docs/tutorials/multi_node.md | 4 ++++ .../en/docs/tutorials/volume_snapshots_and_csi.md | 11 +++++++++++ 3 files changed, 17 insertions(+) diff --git a/site/content/en/docs/handbook/persistent_volumes.md b/site/content/en/docs/handbook/persistent_volumes.md index 76185b80f63c..9b04360614c3 100644 --- a/site/content/en/docs/handbook/persistent_volumes.md +++ b/site/content/en/docs/handbook/persistent_volumes.md @@ -51,3 +51,5 @@ In addition, minikube implements a very simple, canonical implementation of dyna The default [Storage Provisioner Controller](https://github.com/kubernetes/minikube/blob/master/pkg/storage/storage_provisioner.go) is managed internally, in the minikube codebase, demonstrating how easy it is to plug a custom storage controller into kubernetes as a storage component of the system, and provides pods with dynamically, to test your pod's behaviour when persistent storage is mapped to it. Note that this is not a CSI based storage provider, rather, it simply declares a PersistentVolume object of type hostpath dynamically when the controller see's that there is an outstanding storage request. + +There is also [CSI Hostpath Driver]({{< ref "/docs/tutorials/volume_snapshots_and_csi" >}}) addon that enables dynamic provisioning and supports multi-node clusters as well as snapshots. diff --git a/site/content/en/docs/tutorials/multi_node.md b/site/content/en/docs/tutorials/multi_node.md index 2a737f6748ec..2aefde937b05 100644 --- a/site/content/en/docs/tutorials/multi_node.md +++ b/site/content/en/docs/tutorials/multi_node.md @@ -14,6 +14,10 @@ date: 2019-11-24 - minikube 1.10.1 or higher - kubectl +## Caveat + +Default [host-path volume provisioner]({{< ref "/docs/handbook/persistent_volumes" >}}) doesn't support multi-node clusters ([#12360](https://github.com/kubernetes/minikube/issues/12360)). To be able to provision or claim volumes in multi-node clusters, you could use [CSI Hostpath Driver]({{< ref "/docs/tutorials/volume_snapshots_and_csi" >}}) addon. + ## Tutorial - Start a cluster with 2 nodes in the driver of your choice: diff --git a/site/content/en/docs/tutorials/volume_snapshots_and_csi.md b/site/content/en/docs/tutorials/volume_snapshots_and_csi.md index 33024b2dcddf..21ce3b5620c4 100644 --- a/site/content/en/docs/tutorials/volume_snapshots_and_csi.md +++ b/site/content/en/docs/tutorials/volume_snapshots_and_csi.md @@ -46,6 +46,10 @@ Once both addons are enabled, you can create persistent volumes and snapshots us volume snapshots, you can find some example yaml files along with a step-by-step [here](https://kubernetes-csi.github.io/docs/snapshot-restore-feature.html)). The driver stores all persistent volumes in the `/var/lib/csi-hostpath-data/` directory of minikube's host. +## Multi-Node Clusters + +`csi-hostpath-driver` addon supports [Multi-Node Clusters]({{< ref "/docs/tutorials/multi_node" >}}) volume provisioning. It deploys `DaemonSet` that runs `hostpath` on each node to provision and claim volumes (See [#12360](https://github.com/kubernetes/minikube/issues/12360) for more details). + ## Tutorial In this tutorial, you use `volumesnapshots` addon(1) and `csi-hostpath-driver` addon(2a). @@ -65,6 +69,13 @@ minikube addons enable volumesnapshots minikube addons enable csi-hostpath-driver ``` +Optionally you could use it as a default storage class for the dynamic volume claims: +```shell +minikube addons disable storage-provisioner +minikube addons disable default-storageclass +kubectl patch storageclass csi-hostpath-sc -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' +``` +

3Check volume snapshot class

When you create the volume snapshot, you have to register [Volume Snapshot Classes](https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes/) to your cluster.