From ead16456cd623c15c357f66a3727d047d6e919e0 Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Thu, 17 Jun 2021 15:50:53 -0700 Subject: [PATCH 01/15] Use chart-releaser github action --- .github/cr.yaml | 1 + .github/workflows/helm-chart-release.yaml | 27 +++++++++++++++++++++++ .github/workflows/release.yaml | 15 +------------ 3 files changed, 29 insertions(+), 14 deletions(-) create mode 100644 .github/cr.yaml create mode 100644 .github/workflows/helm-chart-release.yaml diff --git a/.github/cr.yaml b/.github/cr.yaml new file mode 100644 index 00000000..6e2fe6c3 --- /dev/null +++ b/.github/cr.yaml @@ -0,0 +1 @@ +release-name-template: "helm-chart-{{ .Name }}-{{ .Version }}" diff --git a/.github/workflows/helm-chart-release.yaml b/.github/workflows/helm-chart-release.yaml new file mode 100644 index 00000000..bfa48726 --- /dev/null +++ b/.github/workflows/helm-chart-release.yaml @@ -0,0 +1,27 @@ +name: Release Helm Charts + +on: + push: + branches: + - master + paths: + - "charts/**" + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.2.1 + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + with: + config: .github/cr.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f278084b..71917fee 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -3,7 +3,7 @@ on: push: # Sequence of patterns matched against refs/tags tags: - - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 + - "v*" # Push events to matching v*, i.e. v1.0, v20.15.10 jobs: build: name: Release @@ -11,9 +11,6 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v1 - - name: Create Helm chart - run: | - tar cvzf helm-chart.tgz helm - name: Create Release id: create-release uses: actions/create-release@v1 @@ -28,13 +25,3 @@ jobs: See [CHANGELOG](https://github.com/kubernetes-sigs/aws-fsx-csi-driver/blob/master/CHANGELOG-0.x.md) for full list of changes draft: false prerelease: false - - name: Upload Release Asset - id: upload-release-asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create-release.outputs.upload_url }} - asset_path: ./helm-chart.tgz - asset_name: helm-chart.tgz - asset_content_type: application/gzip From 745ca5ac29bed080f32996af91911e524714829b Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Thu, 15 Jul 2021 15:16:16 -0700 Subject: [PATCH 02/15] Helm chart 1.0: shuffle values to better match ebs/efs charts --- charts/aws-fsx-csi-driver/Chart.yaml | 4 +- .../aws-fsx-csi-driver/templates/_helpers.tpl | 33 +++--- .../templates/controller-deployment.yaml | 105 ++++++++++++++++++ ...nt.yaml => controller-serviceaccount.yaml} | 16 +-- .../templates/controller.yaml | 74 ------------ .../templates/csidriver.yaml | 2 +- .../{node.yaml => node-daemonset.yaml} | 64 ++++++----- .../templates/node-serviceaccount.yaml | 13 +++ charts/aws-fsx-csi-driver/values.yaml | 77 ++++++------- 9 files changed, 214 insertions(+), 174 deletions(-) create mode 100644 charts/aws-fsx-csi-driver/templates/controller-deployment.yaml rename charts/aws-fsx-csi-driver/templates/{serviceaccount.yaml => controller-serviceaccount.yaml} (76%) delete mode 100644 charts/aws-fsx-csi-driver/templates/controller.yaml rename charts/aws-fsx-csi-driver/templates/{node.yaml => node-daemonset.yaml} (61%) create mode 100644 charts/aws-fsx-csi-driver/templates/node-serviceaccount.yaml diff --git a/charts/aws-fsx-csi-driver/Chart.yaml b/charts/aws-fsx-csi-driver/Chart.yaml index 7e210dd8..247a476e 100644 --- a/charts/aws-fsx-csi-driver/Chart.yaml +++ b/charts/aws-fsx-csi-driver/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 appVersion: "0.4.0" name: aws-fsx-csi-driver description: A Helm chart for AWS FSx for Lustre CSI Driver -version: 0.3.0 -kubeVersion: ">=1.14.0-0" +version: 1.0.0 +kubeVersion: ">=1.17.0-0" home: https://github.com/kubernetes-sigs/aws-fsx-csi-driver sources: - https://github.com/kubernetes-sigs/aws-fsx-csi-driver diff --git a/charts/aws-fsx-csi-driver/templates/_helpers.tpl b/charts/aws-fsx-csi-driver/templates/_helpers.tpl index 5e2ca3be..9ccef356 100644 --- a/charts/aws-fsx-csi-driver/templates/_helpers.tpl +++ b/charts/aws-fsx-csi-driver/templates/_helpers.tpl @@ -2,7 +2,7 @@ {{/* Expand the name of the chart. */}} -{{- define "helm.name" -}} +{{- define "aws-fsx-csi-driver.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} @@ -11,7 +11,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "helm.fullname" -}} +{{- define "aws-fsx-csi-driver.fullname" -}} {{- if .Values.fullnameOverride -}} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- else -}} @@ -27,37 +27,30 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "helm.chart" -}} +{{- define "aws-fsx-csi-driver.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* Common labels */}} -{{- define "helm.labels" -}} -helm.sh/chart: {{ include "helm.chart" . }} -{{ include "helm.selectorLabels" . }} +{{- define "aws-fsx-csi-driver.labels" -}} +{{ include "aws-fsx-csi-driver.selectorLabels" . }} +{{- if ne .Release.Name "kustomize" }} +helm.sh/chart: {{ include "aws-fsx-csi-driver.chart" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} {{- end -}} {{/* -Selector labels +Common selector labels */}} -{{- define "helm.selectorLabels" -}} -app.kubernetes.io/name: {{ include "helm.name" . }} +{{- define "aws-fsx-csi-driver.selectorLabels" -}} +app.kubernetes.io/name: {{ include "aws-fsx-csi-driver.name" . }} +{{- if ne .Release.Name "kustomize" }} app.kubernetes.io/instance: {{ .Release.Name }} -{{- end -}} - -{{/* -Create the name of the service account to use -*/}} -{{- define "helm.serviceAccountName" -}} -{{- if .Values.serviceAccount.create -}} - {{ default (include "helm.fullname" .) .Values.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.serviceAccount.name }} -{{- end -}} +{{- end }} {{- end -}} diff --git a/charts/aws-fsx-csi-driver/templates/controller-deployment.yaml b/charts/aws-fsx-csi-driver/templates/controller-deployment.yaml new file mode 100644 index 00000000..0c09505a --- /dev/null +++ b/charts/aws-fsx-csi-driver/templates/controller-deployment.yaml @@ -0,0 +1,105 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fsx-csi-controller + labels: + {{- include "aws-fsx-csi-driver.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.controller.replicaCount }} + selector: + matchLabels: + app: fsx-csi-controller + {{- include "aws-fsx-csi-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: fsx-csi-controller + {{- include "aws-fsx-csi-driver.labels" . | nindent 8 }} + spec: + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.controller.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + hostNetwork: true + serviceAccountName: {{ .Values.controller.serviceAccount.name }} + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - name: fsx-plugin + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --endpoint=$(CSI_ENDPOINT) + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-secret + key: key_id + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-secret + key: access_key + optional: true + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + ports: + - name: healthz + containerPort: 9910 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + failureThreshold: 5 + {{- with .Values.controller.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: csi-provisioner + image: {{ printf "%s:%s" .Values.sidecars.provisioner.image.repository .Values.sidecars.provisioner.image.tag }} + args: + - --csi-address=$(ADDRESS) + - --timeout=5m + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with default .Values.controller.resources .Values.sidecars.provisioner.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + args: + - --csi-address=/csi/csi.sock + - --health-port=9910 + volumeMounts: + - name: socket-dir + mountPath: /csi + {{- with default .Values.controller.resources .Values.sidecars.livenessProbe.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: socket-dir + emptyDir: {} diff --git a/charts/aws-fsx-csi-driver/templates/serviceaccount.yaml b/charts/aws-fsx-csi-driver/templates/controller-serviceaccount.yaml similarity index 76% rename from charts/aws-fsx-csi-driver/templates/serviceaccount.yaml rename to charts/aws-fsx-csi-driver/templates/controller-serviceaccount.yaml index 7e4807d3..0556a73d 100644 --- a/charts/aws-fsx-csi-driver/templates/serviceaccount.yaml +++ b/charts/aws-fsx-csi-driver/templates/controller-serviceaccount.yaml @@ -1,14 +1,15 @@ -{{- if .Values.serviceAccount.create -}} +{{- if .Values.controller.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "helm.serviceAccountName" . }} + name: {{ .Values.controller.serviceAccount.name }} labels: - {{- include "helm.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} + {{- include "aws-fsx-csi-driver.labels" . | nindent 4 }} + {{- with .Values.controller.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} +{{- end }} --- kind: ClusterRole @@ -16,7 +17,7 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: fsx-csi-external-provisioner-role labels: - {{- include "helm.labels" . | nindent 4 }} + {{- include "aws-fsx-csi-driver.labels" . | nindent 4 }} rules: - apiGroups: [""] resources: ["persistentvolumes"] @@ -46,13 +47,12 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: fsx-csi-external-provisioner-binding labels: - {{- include "helm.labels" . | nindent 4 }} + {{- include "aws-fsx-csi-driver.labels" . | nindent 4 }} subjects: - kind: ServiceAccount - name: {{ include "helm.serviceAccountName" . }} + name: {{ .Values.controller.serviceAccount.name }} namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole name: fsx-csi-external-provisioner-role apiGroup: rbac.authorization.k8s.io -{{- end -}} \ No newline at end of file diff --git a/charts/aws-fsx-csi-driver/templates/controller.yaml b/charts/aws-fsx-csi-driver/templates/controller.yaml deleted file mode 100644 index 0a6b754a..00000000 --- a/charts/aws-fsx-csi-driver/templates/controller.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "helm.fullname" . }}-controller - labels: - {{- include "helm.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.controllerService.replicaCount }} - selector: - matchLabels: - {{- include "helm.selectorLabels" . | nindent 6 }}-controller - template: - metadata: - labels: - {{- include "helm.selectorLabels" . | nindent 8 }}-controller - spec: - serviceAccountName: {{ include "helm.serviceAccountName" . }} - priorityClassName: system-cluster-critical - tolerations: - - key: CriticalAddonsOnly - operator: Exists - {{- if .Values.imagePullSecrets }} - imagePullSecrets: - {{- range .Values.imagePullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} - containers: - - name: fsx-plugin - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - - --endpoint=$(CSI_ENDPOINT) - env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: aws-secret - key: key_id - optional: true - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: aws-secret - key: access_key - optional: true - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - {{- toYaml .Values.controllerService.fsxPlugin.resources | nindent 12 }} - - name: csi-provisioner - image: "{{ .Values.controllerService.csiProvisioner.image.repository }}:{{ .Values.controllerService.csiProvisioner.image.tag }}" - args: - - --csi-address=$(ADDRESS) - - --timeout=5m - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: - {{- toYaml .Values.controllerService.csiProvisioner.resources | nindent 12 }} - - volumes: - - name: socket-dir - emptyDir: {} - {{- with .Values.controllerService.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/charts/aws-fsx-csi-driver/templates/csidriver.yaml b/charts/aws-fsx-csi-driver/templates/csidriver.yaml index f8479e86..ca7cc57a 100644 --- a/charts/aws-fsx-csi-driver/templates/csidriver.yaml +++ b/charts/aws-fsx-csi-driver/templates/csidriver.yaml @@ -1,4 +1,4 @@ -apiVersion: storage.k8s.io/v1beta1 +apiVersion: {{ ternary "storage.k8s.io/v1" "storage.k8s.io/v1beta1" (semverCompare ">=1.18.0-0" .Capabilities.KubeVersion.Version) }} kind: CSIDriver metadata: name: fsx.csi.aws.com diff --git a/charts/aws-fsx-csi-driver/templates/node.yaml b/charts/aws-fsx-csi-driver/templates/node-daemonset.yaml similarity index 61% rename from charts/aws-fsx-csi-driver/templates/node.yaml rename to charts/aws-fsx-csi-driver/templates/node-daemonset.yaml index 66c9f23a..5e4f0fe3 100644 --- a/charts/aws-fsx-csi-driver/templates/node.yaml +++ b/charts/aws-fsx-csi-driver/templates/node-daemonset.yaml @@ -1,32 +1,38 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: {{ include "helm.fullname" . }}-daemonset + name: fsx-csi-node labels: - {{- include "helm.labels" . | nindent 4 }} + {{- include "aws-fsx-csi-driver.labels" . | nindent 4 }} spec: selector: matchLabels: - {{- include "helm.selectorLabels" . | nindent 6 }}-daemonset + app: fsx-csi-node + {{- include "aws-fsx-csi-driver.selectorLabels" . | nindent 6 }} template: metadata: labels: - {{- include "helm.selectorLabels" . | nindent 8 }}-daemonset + app: fsx-csi-node + {{- include "aws-fsx-csi-driver.labels" . | nindent 8 }} spec: - hostNetwork: true - {{- if .Values.nodeService.dnsPolicy }} - dnsPolicy: "{{ .Values.nodeService.dnsPolicy }}" - {{- end }} - {{- with .Values.nodeService.dnsConfig }} - dnsConfig: - {{- toYaml . | nindent 8 }} - {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: - {{- range .Values.imagePullSecrets }} + {{- range .Values.imagePullSecrets }} - name: {{ . }} + {{- end }} + {{- end }} + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} {{- end }} + hostNetwork: true + dnsPolicy: {{ .Values.node.dnsPolicy }} + {{- with .Values.node.dnsConfig }} + dnsConfig: {{- toYaml . | nindent 8 }} {{- end }} + serviceAccountName: {{ .Values.node.serviceAccount.name }} + priorityClassName: system-node-critical containers: - name: fsx-plugin securityContext: @@ -45,22 +51,24 @@ spec: - name: plugin-dir mountPath: /csi ports: - - containerPort: 9810 - name: healthz + - name: healthz + containerPort: 9810 protocol: TCP livenessProbe: - failureThreshold: 5 httpGet: path: /healthz port: healthz initialDelaySeconds: 10 timeoutSeconds: 3 periodSeconds: 2 + failureThreshold: 5 + {{- with .Values.node.resources }} resources: - {{- toYaml .Values.nodeService.fsxPlugin.resources | nindent 12 }} - - name: csi-driver-registrar - image: "{{ .Values.nodeService.csiDriverRegistrar.image.repository }}:{{ .Values.nodeService.csiDriverRegistrar.image.tag }}" - imagePullPolicy: {{ .Values.nodeService.csiDriverRegistrar.image.pullPolicy }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: node-driver-registrar + image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} + imagePullPolicy: {{ .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) @@ -78,19 +86,23 @@ spec: mountPath: /csi - name: registration-dir mountPath: /registration + {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} resources: - {{- toYaml .Values.nodeService.csiDriverRegistrar.resources | nindent 12 }} + {{- toYaml . | nindent 12 }} + {{- end }} - name: liveness-probe - imagePullPolicy: {{ .Values.nodeService.livenessProbe.image.pullPolicy }} - image: "{{ .Values.nodeService.livenessProbe.image.repository }}:{{ .Values.nodeService.livenessProbe.image.tag }}" + image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ .Values.sidecars.livenessProbe.image.pullPolicy }} args: - --csi-address=/csi/csi.sock - --health-port=9810 volumeMounts: - mountPath: /csi name: plugin-dir + {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} resources: - {{- toYaml .Values.nodeService.livenessProbe.resources | nindent 12 }} + {{- toYaml . | nindent 12 }} + {{- end }} volumes: - name: kubelet-dir hostPath: @@ -104,7 +116,3 @@ spec: hostPath: path: /var/lib/kubelet/plugins/fsx.csi.aws.com/ type: DirectoryOrCreate - {{- with .Values.nodeService.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/charts/aws-fsx-csi-driver/templates/node-serviceaccount.yaml b/charts/aws-fsx-csi-driver/templates/node-serviceaccount.yaml new file mode 100644 index 00000000..369a3366 --- /dev/null +++ b/charts/aws-fsx-csi-driver/templates/node-serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.node.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.node.serviceAccount.name }} + labels: + {{- include "aws-fsx-csi-driver.labels" . | nindent 4 }} + {{- with .Values.node.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} +--- diff --git a/charts/aws-fsx-csi-driver/values.yaml b/charts/aws-fsx-csi-driver/values.yaml index 95a69050..011b9217 100644 --- a/charts/aws-fsx-csi-driver/values.yaml +++ b/charts/aws-fsx-csi-driver/values.yaml @@ -7,63 +7,58 @@ image: tag: v0.4.0 pullPolicy: IfNotPresent -controllerService: - replicaCount: 2 - - nodeSelector: - kubernetes.io/os: linux - - fsxPlugin: - resources: {} - - csiProvisioner: +sidecars: + livenessProbe: image: - repository: k8s.gcr.io/sig-storage/csi-provisioner - tag: v2.1.1 + repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe + tag: v2.2.0-eks-1-18-2 pullPolicy: IfNotPresent - - resources: {} - -nodeService: - nodeSelector: - kubernetes.io/os: linux - - fsxPlugin: resources: {} - - csiDriverRegistrar: + nodeDriverRegistrar: image: - repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar - tag: v2.1.0 + repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar + tag: v2.1.0-eks-1-18-2 pullPolicy: IfNotPresent - resources: {} - - livenessProbe: + provisioner: image: - repository: k8s.gcr.io/sig-storage/livenessprobe - tag: v2.2.0 - pullPolicy: Always - + repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner + tag: v2.1.1-eks-1-18-2 + pullPolicy: IfNotPresent resources: {} - dnsPolicy: "" - dnsConfig: {} +controller: + nodeSelector: {} + replicaCount: 2 + resources: {} + serviceAccount: + # Specifies whether a service account should be created + create: true + ## Enable if EKS IAM for SA is used + # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/fsx-csi-role + name: fsx-csi-controller-sa + annotations: {} + +node: + nodeSelector: {} + resources: {} + dnsPolicy: ClusterFirst + dnsConfig: + {} # Example config which uses the AWS nameservers # dnsPolicy: "None" # dnsConfig: # nameservers: # - 169.254.169.253 + serviceAccount: + # Specifies whether a service account should be created + create: true + ## Enable if EKS IAM for SA is used + # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/fsx-csi-role + name: fsx-csi-node-sa + annotations: {} nameOverride: "" fullnameOverride: "" imagePullSecrets: [] - -serviceAccount: - # Specifies whether a service account should be created - create: true - annotations: {} - ## Enable if EKS IAM for SA is used - # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/fsx-csi-role - name: fsx-csi-controller-sa From c99a8ccc465926e2806f6a5b1aaa6eb509525b0d Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Fri, 16 Jul 2021 15:16:23 -0700 Subject: [PATCH 03/15] Update README, helm CHANGELOG --- charts/aws-fsx-csi-driver/CHANGELOG.md | 13 +++++++++++++ docs/README.md | 5 +++-- 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 charts/aws-fsx-csi-driver/CHANGELOG.md diff --git a/charts/aws-fsx-csi-driver/CHANGELOG.md b/charts/aws-fsx-csi-driver/CHANGELOG.md new file mode 100644 index 00000000..25c73028 --- /dev/null +++ b/charts/aws-fsx-csi-driver/CHANGELOG.md @@ -0,0 +1,13 @@ +# Helm chart + +# v1.0.0 +* Remove support for Helm 2 +* Reorganize values to be more consistent with EFS and EBS helm charts + * controllerService -> controller + * nodeService -> node +* Add node.serviceAccount +* Add dnsPolicy and dnsConfig +* Add imagePullSecrets +* Remove extraArgs, securityContext, podSecurityContext +* Bump sidecar images to support kubernetes >=1.20 +* Require kubernetes >=1.17 diff --git a/docs/README.md b/docs/README.md index b769d654..5b80efdd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -105,13 +105,14 @@ kubectl apply -f secret.yaml #### Deploy driver ```sh -kubectl apply -k "github.com/kubernetes-sigs/aws-fsx-csi-driver/deploy/kubernetes/overlays/stable/?ref=master" +kubectl apply -k "github.com/kubernetes-sigs/aws-fsx-csi-driver/deploy/kubernetes/overlays/stable/?ref=release-0.4" ``` Alternatively, you could also install the driver using helm: ```sh helm repo add aws-fsx-csi-driver https://kubernetes-sigs.github.io/aws-fsx-csi-driver/ -helm install aws-fsx-csi-driver aws-fsx-csi-driver/aws-fsx-csi-driver +helm repo update +helm upgrade --install aws-fsx-csi-driver --namespace kube-system aws-fsx-csi-driver/aws-fsx-csi-driver ``` ### Examples Before the example, you need to: From a4674b2e8e4129189031734050f106c0a363dc16 Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Fri, 16 Jul 2021 15:24:23 -0700 Subject: [PATCH 04/15] Add make generate-kustomize --- Makefile | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 462cfc1f..d7e18982 100644 --- a/Makefile +++ b/Makefile @@ -14,13 +14,14 @@ PKG=github.com/kubernetes-sigs/aws-fsx-csi-driver IMAGE?=amazon/aws-fsx-csi-driver -VERSION=v0.4.0-dirty +VERSION=v0.4.0 GIT_COMMIT?=$(shell git rev-parse HEAD) BUILD_DATE?=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ") LDFLAGS?="-X ${PKG}/pkg/driver.driverVersion=${VERSION} -X ${PKG}/pkg/driver.gitCommit=${GIT_COMMIT} -X ${PKG}/pkg/driver.buildDate=${BUILD_DATE}" GO111MODULE=on GOPROXY=direct GOPATH=$(shell go env GOPATH) +GOOS=$(shell go env GOOS) .EXPORT_ALL_VARIABLES: @@ -29,6 +30,14 @@ aws-fsx-csi-driver: mkdir -p bin CGO_ENABLED=0 GOOS=linux go build -ldflags ${LDFLAGS} -o bin/aws-fsx-csi-driver ./cmd/ +bin /tmp/helm: + @mkdir -p $@ + +bin/helm: | /tmp/helm bin + @curl -o /tmp/helm/helm.tar.gz -sSL https://get.helm.sh/helm-v3.5.3-${GOOS}-amd64.tar.gz + @tar -zxf /tmp/helm/helm.tar.gz -C bin --strip-components=1 + @rm -rf /tmp/helm/* + .PHONY: verify verify: ./hack/verify-all @@ -66,3 +75,10 @@ image-release: .PHONY: push-release push-release: docker push $(IMAGE):$(VERSION) + +generate-kustomize: bin/helm + cd charts/aws-fsx-csi-driver && ../../bin/helm template kustomize . -s templates/csidriver.yaml > ../../deploy/kubernetes/base/csidriver.yaml + cd charts/aws-fsx-csi-driver && ../../bin/helm template kustomize . -s templates/node-daemonset.yaml > ../../deploy/kubernetes/base/node-daemonset.yaml + cd charts/aws-fsx-csi-driver && ../../bin/helm template kustomize . -s templates/node-serviceaccount.yaml > ../../deploy/kubernetes/base/node-serviceaccount.yaml + cd charts/aws-fsx-csi-driver && ../../bin/helm template kustomize . -s templates/controller-deployment.yaml > ../../deploy/kubernetes/base/controller-deployment.yaml + cd charts/aws-fsx-csi-driver && ../../bin/helm template kustomize . -s templates/controller-serviceaccount.yaml > ../../deploy/kubernetes/base/controller-serviceaccount.yaml From 9f7686311f4d7e1124b16109e1de6e88bb511644 Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Fri, 16 Jul 2021 15:24:37 -0700 Subject: [PATCH 05/15] Run make generate-kustomize and delete old yamls --- ...roller.yaml => controller-deployment.yaml} | 51 +++++++++++++------ ...ac.yaml => controller-serviceaccount.yaml} | 23 ++++----- deploy/kubernetes/base/csidriver.yaml | 4 +- deploy/kubernetes/base/kustomization.yaml | 7 +-- .../base/{node.yaml => node-daemonset.yaml} | 34 +++++++------ .../kubernetes/base/node-serviceaccount.yaml | 8 +++ .../overlays/dev/kustomization.yaml | 8 --- 7 files changed, 78 insertions(+), 57 deletions(-) rename deploy/kubernetes/base/{controller.yaml => controller-deployment.yaml} (55%) rename deploy/kubernetes/base/{rbac.yaml => controller-serviceaccount.yaml} (75%) rename deploy/kubernetes/base/{node.yaml => node-daemonset.yaml} (73%) create mode 100644 deploy/kubernetes/base/node-serviceaccount.yaml delete mode 100644 deploy/kubernetes/overlays/dev/kustomization.yaml diff --git a/deploy/kubernetes/base/controller.yaml b/deploy/kubernetes/base/controller-deployment.yaml similarity index 55% rename from deploy/kubernetes/base/controller.yaml rename to deploy/kubernetes/base/controller-deployment.yaml index c3c5a920..ed79792d 100644 --- a/deploy/kubernetes/base/controller.yaml +++ b/deploy/kubernetes/base/controller-deployment.yaml @@ -1,34 +1,37 @@ --- -kind: Deployment +# Source: aws-fsx-csi-driver/templates/controller-deployment.yaml apiVersion: apps/v1 +kind: Deployment metadata: name: fsx-csi-controller - namespace: kube-system + labels: + app.kubernetes.io/name: aws-fsx-csi-driver spec: replicas: 2 selector: matchLabels: app: fsx-csi-controller + app.kubernetes.io/name: aws-fsx-csi-driver template: metadata: labels: app: fsx-csi-controller + app.kubernetes.io/name: aws-fsx-csi-driver spec: nodeSelector: kubernetes.io/os: linux - kubernetes.io/arch: amd64 - serviceAccount: fsx-csi-controller-sa + hostNetwork: true + serviceAccountName: fsx-csi-controller-sa priorityClassName: system-cluster-critical tolerations: - - key: CriticalAddonsOnly - operator: Exists + - key: CriticalAddonsOnly + operator: Exists containers: - name: fsx-plugin - image: amazon/aws-fsx-csi-driver:latest - args : + image: "amazon/aws-fsx-csi-driver:v0.4.0" + imagePullPolicy: IfNotPresent + args: - --endpoint=$(CSI_ENDPOINT) - - --logtostderr - - --v=5 env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock @@ -47,21 +50,37 @@ spec: volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + ports: + - name: healthz + containerPort: 9910 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + failureThreshold: 5 - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:v1.3.0 + image: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner:v2.1.1-eks-1-18-2 args: - - --timeout=5m - --csi-address=$(ADDRESS) - - --v=5 - - --enable-leader-election - - --leader-election-type=leases + - --timeout=5m env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: liveness-probe + image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.2.0-eks-1-18-2 + args: + - --csi-address=/csi/csi.sock + - --health-port=9910 + volumeMounts: + - name: socket-dir + mountPath: /csi volumes: - name: socket-dir emptyDir: {} - diff --git a/deploy/kubernetes/base/rbac.yaml b/deploy/kubernetes/base/controller-serviceaccount.yaml similarity index 75% rename from deploy/kubernetes/base/rbac.yaml rename to deploy/kubernetes/base/controller-serviceaccount.yaml index 67161054..1b15c515 100644 --- a/deploy/kubernetes/base/rbac.yaml +++ b/deploy/kubernetes/base/controller-serviceaccount.yaml @@ -1,20 +1,19 @@ --- - +# Source: aws-fsx-csi-driver/templates/controller-serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: name: fsx-csi-controller-sa - namespace: kube-system - #Enable if EKS IAM for SA is used - #annotations: - # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/fsx-csi-role - + labels: + app.kubernetes.io/name: aws-fsx-csi-driver --- - +# Source: aws-fsx-csi-driver/templates/controller-serviceaccount.yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: fsx-csi-external-provisioner-role + labels: + app.kubernetes.io/name: aws-fsx-csi-driver rules: - apiGroups: [""] resources: ["persistentvolumes"] @@ -37,21 +36,19 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] - --- - +# Source: aws-fsx-csi-driver/templates/controller-serviceaccount.yaml kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: fsx-csi-external-provisioner-binding + labels: + app.kubernetes.io/name: aws-fsx-csi-driver subjects: - kind: ServiceAccount name: fsx-csi-controller-sa - namespace: kube-system + namespace: default roleRef: kind: ClusterRole name: fsx-csi-external-provisioner-role apiGroup: rbac.authorization.k8s.io - ---- - diff --git a/deploy/kubernetes/base/csidriver.yaml b/deploy/kubernetes/base/csidriver.yaml index c9c42e3e..beb5bcef 100644 --- a/deploy/kubernetes/base/csidriver.yaml +++ b/deploy/kubernetes/base/csidriver.yaml @@ -1,6 +1,6 @@ --- - -apiVersion: storage.k8s.io/v1beta1 +# Source: aws-fsx-csi-driver/templates/csidriver.yaml +apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: fsx.csi.aws.com diff --git a/deploy/kubernetes/base/kustomization.yaml b/deploy/kubernetes/base/kustomization.yaml index b87bfea9..f5f7ae54 100644 --- a/deploy/kubernetes/base/kustomization.yaml +++ b/deploy/kubernetes/base/kustomization.yaml @@ -2,7 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: kube-system resources: - - controller.yaml - - node.yaml - - rbac.yaml - csidriver.yaml + - node-daemonset.yaml + - node-serviceaccount.yaml + - controller-deployment.yaml + - controller-serviceaccount.yaml diff --git a/deploy/kubernetes/base/node.yaml b/deploy/kubernetes/base/node-daemonset.yaml similarity index 73% rename from deploy/kubernetes/base/node.yaml rename to deploy/kubernetes/base/node-daemonset.yaml index a072c62a..ce5d80d9 100644 --- a/deploy/kubernetes/base/node.yaml +++ b/deploy/kubernetes/base/node-daemonset.yaml @@ -1,32 +1,36 @@ --- - -kind: DaemonSet +# Source: aws-fsx-csi-driver/templates/node-daemonset.yaml apiVersion: apps/v1 +kind: DaemonSet metadata: name: fsx-csi-node - namespace: kube-system + labels: + app.kubernetes.io/name: aws-fsx-csi-driver spec: selector: matchLabels: app: fsx-csi-node + app.kubernetes.io/name: aws-fsx-csi-driver template: metadata: labels: app: fsx-csi-node + app.kubernetes.io/name: aws-fsx-csi-driver spec: nodeSelector: kubernetes.io/os: linux - kubernetes.io/arch: amd64 hostNetwork: true + dnsPolicy: ClusterFirst + serviceAccountName: fsx-csi-node-sa + priorityClassName: system-node-critical containers: - name: fsx-plugin securityContext: privileged: true - image: amazon/aws-fsx-csi-driver:latest + image: "amazon/aws-fsx-csi-driver:v0.4.0" + imagePullPolicy: IfNotPresent args: - --endpoint=$(CSI_ENDPOINT) - - --logtostderr - - --v=5 env: - name: CSI_ENDPOINT value: unix:/csi/csi.sock @@ -37,23 +41,23 @@ spec: - name: plugin-dir mountPath: /csi ports: - - containerPort: 9810 - name: healthz + - name: healthz + containerPort: 9810 protocol: TCP livenessProbe: - failureThreshold: 5 httpGet: path: /healthz port: healthz initialDelaySeconds: 10 timeoutSeconds: 3 periodSeconds: 2 - - name: csi-driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 + failureThreshold: 5 + - name: node-driver-registrar + image: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar:v2.1.0-eks-1-18-2 + imagePullPolicy: IfNotPresent args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v=5 env: - name: ADDRESS value: /csi/csi.sock @@ -69,8 +73,8 @@ spec: - name: registration-dir mountPath: /registration - name: liveness-probe - imagePullPolicy: Always - image: quay.io/k8scsi/livenessprobe:v1.1.0 + image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.2.0-eks-1-18-2 + imagePullPolicy: IfNotPresent args: - --csi-address=/csi/csi.sock - --health-port=9810 diff --git a/deploy/kubernetes/base/node-serviceaccount.yaml b/deploy/kubernetes/base/node-serviceaccount.yaml new file mode 100644 index 00000000..5f44f558 --- /dev/null +++ b/deploy/kubernetes/base/node-serviceaccount.yaml @@ -0,0 +1,8 @@ +--- +# Source: aws-fsx-csi-driver/templates/node-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fsx-csi-node-sa + labels: + app.kubernetes.io/name: aws-fsx-csi-driver diff --git a/deploy/kubernetes/overlays/dev/kustomization.yaml b/deploy/kubernetes/overlays/dev/kustomization.yaml deleted file mode 100644 index 0667d20f..00000000 --- a/deploy/kubernetes/overlays/dev/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -bases: -- ../../base -images: -- name: amazon/aws-fsx-csi-driver - newName: chengpan/aws-fsx-csi-driver - newTag: latest From 8355ae37c6f306d92f0848fd28e66e9cb12ccf87 Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Fri, 16 Jul 2021 16:04:49 -0700 Subject: [PATCH 06/15] Set replicaCount=1 since NODE_COUNT=1, else controller never becomes ready --- hack/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hack/values.yaml b/hack/values.yaml index e69de29b..6c939de5 100644 --- a/hack/values.yaml +++ b/hack/values.yaml @@ -0,0 +1,2 @@ +controller: + replicaCount: 1 From de19088bda26ea8f3df7081a0dd7a2796f8ee074 Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Tue, 3 Aug 2021 13:35:41 -0700 Subject: [PATCH 07/15] Add Values.controller.tolerations --- .../templates/controller-deployment.yaml | 10 ++++++++-- charts/aws-fsx-csi-driver/values.yaml | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/charts/aws-fsx-csi-driver/templates/controller-deployment.yaml b/charts/aws-fsx-csi-driver/templates/controller-deployment.yaml index 0c09505a..244332f3 100644 --- a/charts/aws-fsx-csi-driver/templates/controller-deployment.yaml +++ b/charts/aws-fsx-csi-driver/templates/controller-deployment.yaml @@ -31,8 +31,14 @@ spec: serviceAccountName: {{ .Values.controller.serviceAccount.name }} priorityClassName: system-cluster-critical tolerations: - - key: CriticalAddonsOnly - operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + effect: NoExecute + tolerationSeconds: 300 + {{- with .Values.controller.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: fsx-plugin image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" diff --git a/charts/aws-fsx-csi-driver/values.yaml b/charts/aws-fsx-csi-driver/values.yaml index 011b9217..510c3ee6 100644 --- a/charts/aws-fsx-csi-driver/values.yaml +++ b/charts/aws-fsx-csi-driver/values.yaml @@ -38,6 +38,7 @@ controller: # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/fsx-csi-role name: fsx-csi-controller-sa annotations: {} + tolerations: [] node: nodeSelector: {} From e63dda393f94030bfeb54c0857ad2e99e8d0acb0 Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Tue, 3 Aug 2021 13:37:44 -0700 Subject: [PATCH 08/15] Default driver image tag to .Chart.AppVersion --- charts/aws-fsx-csi-driver/templates/controller-deployment.yaml | 2 +- charts/aws-fsx-csi-driver/templates/node-daemonset.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/aws-fsx-csi-driver/templates/controller-deployment.yaml b/charts/aws-fsx-csi-driver/templates/controller-deployment.yaml index 244332f3..e42c52c5 100644 --- a/charts/aws-fsx-csi-driver/templates/controller-deployment.yaml +++ b/charts/aws-fsx-csi-driver/templates/controller-deployment.yaml @@ -41,7 +41,7 @@ spec: {{- end }} containers: - name: fsx-plugin - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: {{ printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: - --endpoint=$(CSI_ENDPOINT) diff --git a/charts/aws-fsx-csi-driver/templates/node-daemonset.yaml b/charts/aws-fsx-csi-driver/templates/node-daemonset.yaml index 5e4f0fe3..3752efa1 100644 --- a/charts/aws-fsx-csi-driver/templates/node-daemonset.yaml +++ b/charts/aws-fsx-csi-driver/templates/node-daemonset.yaml @@ -37,7 +37,7 @@ spec: - name: fsx-plugin securityContext: privileged: true - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: {{ printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: - --endpoint=$(CSI_ENDPOINT) From 69de92ce08d68002637416b1bcb21e673c5153b3 Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Tue, 3 Aug 2021 13:46:47 -0700 Subject: [PATCH 09/15] Add Values.node.tolerations and Values.node.tolerateAllTaints --- .../templates/node-daemonset.yaml | 13 +++++++++++++ charts/aws-fsx-csi-driver/values.yaml | 2 ++ 2 files changed, 15 insertions(+) diff --git a/charts/aws-fsx-csi-driver/templates/node-daemonset.yaml b/charts/aws-fsx-csi-driver/templates/node-daemonset.yaml index 3752efa1..41b6923c 100644 --- a/charts/aws-fsx-csi-driver/templates/node-daemonset.yaml +++ b/charts/aws-fsx-csi-driver/templates/node-daemonset.yaml @@ -33,6 +33,19 @@ spec: {{- end }} serviceAccountName: {{ .Values.node.serviceAccount.name }} priorityClassName: system-node-critical + tolerations: + {{- if .Values.node.tolerateAllTaints }} + - operator: Exists + {{- else }} + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + effect: NoExecute + tolerationSeconds: 300 + {{- end }} + {{- with .Values.node.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: fsx-plugin securityContext: diff --git a/charts/aws-fsx-csi-driver/values.yaml b/charts/aws-fsx-csi-driver/values.yaml index 510c3ee6..4dbab31c 100644 --- a/charts/aws-fsx-csi-driver/values.yaml +++ b/charts/aws-fsx-csi-driver/values.yaml @@ -58,6 +58,8 @@ node: # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/fsx-csi-role name: fsx-csi-node-sa annotations: {} + tolerateAllTaints: false + tolerations: [] nameOverride: "" fullnameOverride: "" From f9c1b50c1464ac91c445f67f8f1d289bcec1a6f2 Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Tue, 3 Aug 2021 13:53:33 -0700 Subject: [PATCH 10/15] Add extra-create-metadata and leader-election flags to csi-provisioner --- charts/aws-fsx-csi-driver/templates/controller-deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/aws-fsx-csi-driver/templates/controller-deployment.yaml b/charts/aws-fsx-csi-driver/templates/controller-deployment.yaml index e42c52c5..77edaf34 100644 --- a/charts/aws-fsx-csi-driver/templates/controller-deployment.yaml +++ b/charts/aws-fsx-csi-driver/templates/controller-deployment.yaml @@ -84,6 +84,8 @@ spec: args: - --csi-address=$(ADDRESS) - --timeout=5m + - --extra-create-metadata + - --leader-election=true env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock From 1702134243c3b9a6d0b3df287750d8a76971bff5 Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Tue, 3 Aug 2021 13:54:25 -0700 Subject: [PATCH 11/15] Run make generate-kustomize --- deploy/kubernetes/base/controller-deployment.yaml | 11 ++++++++--- deploy/kubernetes/base/node-daemonset.yaml | 8 +++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/deploy/kubernetes/base/controller-deployment.yaml b/deploy/kubernetes/base/controller-deployment.yaml index ed79792d..266ee24c 100644 --- a/deploy/kubernetes/base/controller-deployment.yaml +++ b/deploy/kubernetes/base/controller-deployment.yaml @@ -24,11 +24,14 @@ spec: serviceAccountName: fsx-csi-controller-sa priorityClassName: system-cluster-critical tolerations: - - key: CriticalAddonsOnly - operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + effect: NoExecute + tolerationSeconds: 300 containers: - name: fsx-plugin - image: "amazon/aws-fsx-csi-driver:v0.4.0" + image: amazon/aws-fsx-csi-driver:v0.4.0 imagePullPolicy: IfNotPresent args: - --endpoint=$(CSI_ENDPOINT) @@ -67,6 +70,8 @@ spec: args: - --csi-address=$(ADDRESS) - --timeout=5m + - --extra-create-metadata + - --leader-election=true env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock diff --git a/deploy/kubernetes/base/node-daemonset.yaml b/deploy/kubernetes/base/node-daemonset.yaml index ce5d80d9..68e6c438 100644 --- a/deploy/kubernetes/base/node-daemonset.yaml +++ b/deploy/kubernetes/base/node-daemonset.yaml @@ -23,11 +23,17 @@ spec: dnsPolicy: ClusterFirst serviceAccountName: fsx-csi-node-sa priorityClassName: system-node-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + effect: NoExecute + tolerationSeconds: 300 containers: - name: fsx-plugin securityContext: privileged: true - image: "amazon/aws-fsx-csi-driver:v0.4.0" + image: amazon/aws-fsx-csi-driver:v0.4.0 imagePullPolicy: IfNotPresent args: - --endpoint=$(CSI_ENDPOINT) From 3fd96fc0ea9624a0c6e2345e977239d4527d918c Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Tue, 3 Aug 2021 14:00:53 -0700 Subject: [PATCH 12/15] Update helm CHANGELOG --- charts/aws-fsx-csi-driver/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/aws-fsx-csi-driver/CHANGELOG.md b/charts/aws-fsx-csi-driver/CHANGELOG.md index 25c73028..147db9d2 100644 --- a/charts/aws-fsx-csi-driver/CHANGELOG.md +++ b/charts/aws-fsx-csi-driver/CHANGELOG.md @@ -8,6 +8,7 @@ * Add node.serviceAccount * Add dnsPolicy and dnsConfig * Add imagePullSecrets +* Add controller.tolerations, node.tolerations, and node.tolerateAllTaints * Remove extraArgs, securityContext, podSecurityContext * Bump sidecar images to support kubernetes >=1.20 * Require kubernetes >=1.17 From 0200261baa2095b782e1269518d47431a9fd6498 Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Tue, 3 Aug 2021 14:14:18 -0700 Subject: [PATCH 13/15] Wipe 0.x.0 helm chart README --- charts/aws-fsx-csi-driver/README.md | 76 ----------------------------- 1 file changed, 76 deletions(-) delete mode 100644 charts/aws-fsx-csi-driver/README.md diff --git a/charts/aws-fsx-csi-driver/README.md b/charts/aws-fsx-csi-driver/README.md deleted file mode 100644 index 7e6e7b2b..00000000 --- a/charts/aws-fsx-csi-driver/README.md +++ /dev/null @@ -1,76 +0,0 @@ - -## Amazon FSx for Lustre CSI Driver Helm Chart -For more info view [aws-fsx-csi-driver](https://github.com/kubernetes-sigs/aws-fsx-csi-driver) - -## Prerequisites -- Helm 3+ -- Kubernetes > 1.17.x, can be deployed to any namespace. -- Kubernetes < 1.17.x, namespace **must** be `kube-system`, as `system-cluster-critical` hard coded to this namespace. -## Install chart -```shell script -helm install . --name aws-fsx-csi-driver -``` - -## Upgrade release -```shell script -helm upgrade aws-fsx-csi-driver \ - --install . \ - --version 0.1.0 \ - --namespace kube-system \ - -f values.yaml -``` -## Uninstalling the Chart -```shell script -helm delete aws-fsx-csi-driver --namespace [NAMESPACE] -``` -## Parameters - -The following table lists the configurable parameters of the aws-fsx-csi-driver chart and their default values. - -| Parameter | Description | Default | -| ------------------------------------------------------| ------------------------------------------------------------- | ------------------------------------------ | -| `controllerService.replicaCount` | Num of replicas for controller | `2` | -| `controllerService.nodeSelector` | Controllers node selector | `kubernetes.io/os: linux` | -| `controllerService.podSecurityContext` | Security context for controller pods | `{}` | -| | | | -| `controllerService.fsxPlugin.image.repository` | aws-fsx-csi-driver image name | `amazon/aws-fsx-csi-driver` | -| `controllerService.fsxPlugin.image.tag` | aws-fsx-csi-driver image tag | `latest` | -| `controllerService.fsxPlugin.image.pullPolicy` | aws-fsx-csi-driver image pull policy | `IfNotPresent` | -| `controllerService.fsxPlugin.extraArgs` | Extra arguments to be passed to aws-fsx-csi-driver fsxPlugin | `--logtostderr --v=5` | -| `controllerService.fsxPlugin.securityContext` | Security context for the container | `{}` | -| `controllerService.fsxPlugin.resources` | CPU/Memory resource requests/limits | `{}` | -| | | | -| `controllerService.csiProvisioner.image.repository` | csi-provisioner image name | `quay.io/k8scsi/csi-provisioner` | -| `controllerService.csiProvisioner.image.tag` | csi-provisioner image tag | `v1.3.0` | -| `controllerService.csiProvisioner.image.pullPolicy` | csi-provisioner image pull policy | `IfNotPresent` | -| `controllerService.csiProvisioner.extraArgs` | Extra arguments to be passed to csi-provisioner | `--timeout=5m --v=5 --enable-leader-election --leader-election-type=leases`| -| `controllerService.csiProvisioner.securityContext` | Security context for the container | `{}` | -| `controllerService.csiProvisioner.resources` | CPU/Memory resource requests/limits | `{}` | -| | | | -| `controllerService.nodeSelector` | Controllers node selector | `kubernetes.io/os: linux` | -| `nodeService.podSecurityContext` | Security context for controller pods | `{}` | -| | | | -| `nodeService.fsxPlugin.image.repository` | aws-fsx-csi-driver image name | `amazon/aws-fsx-csi-driver` | -| `nodeService.fsxPlugin.image.tag` | aws-fsx-csi-driver image tag | `latest` | -| `nodeService.fsxPlugin.image.pullPolicy` | aws-fsx-csi-driver image pull policy | `IfNotPresent` | -| `nodeService.fsxPlugin.extraArgs` | Extra arguments to be passed to aws-fsx-csi-driver fsxPlugin | `--logtostderr --v=5` | -| `nodeService.fsxPlugin.securityContext` | Security context for the container | `privileged: true` | -| `nodeService.fsxPlugin.resources` | CPU/Memory resource requests/limits | `{}` | -| | | | -| `nodeService.csiDriverRegistrar.image.repository` | csi-node-driver-registrar image name | `quay.io/k8scsi/csi-node-driver-registrar` | -| `nodeService.csiDriverRegistrar.image.tag` | csi-node-driver-registrar image tag | `v1.1.0` | -| `nodeService.csiDriverRegistrar.image.pullPolicy` | csi-node-driver-registrar image pull policy | `IfNotPresent` | -| `nodeService.csiDriverRegistrar.extraArgs` | Extra arguments to be passed to aws-fsx-csi-driver fsxPlugin | `--v=5` | -| `nodeService.csiDriverRegistrar.securityContext` | Security context for the container | `{}` | -| `nodeService.csiDriverRegistrar.resources` | CPU/Memory resource requests/limits | `{}` | -| | | | -| `nodeService.livenessProbe.image.repository` | livenessprobe image name | `quay.io/k8scsi/livenessprobe` | -| `nodeService.livenessProbe.image.tag` | livenessprobe image tag | `v1.1.0` | -| `nodeService.livenessProbe.image.pullPolicy` | livenessprobe image pull policy | `Always` | -| `nodeService.livenessProbe.resources` | CPU/Memory resource requests/limits | `{}` | -| | | | -| `nameOverride` | String to partially override aws-fsx-csi-driver.fullname | `""` | -| `fullnameOverride` | String to fully override aws-fsx-csi-driver.fullname | `""` | -| `serviceAccount.create` | Specifies whether a service account should be created | `true` | -| `serviceAccount.annotations` | Additional Service Account annotations | `{}` | -| `serviceAccount.name` | Service Account name | `fsx-csi-controller-sa` | From a6e7303a4901498d4af8a259b7b59f5f48f6ca52 Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Wed, 4 Aug 2021 13:01:01 -0700 Subject: [PATCH 14/15] Fail if *.serviceAccount.create is false and user hasn't created their own serviceAccount --- .../templates/controller-serviceaccount.yaml | 5 +++++ charts/aws-fsx-csi-driver/templates/node-serviceaccount.yaml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/charts/aws-fsx-csi-driver/templates/controller-serviceaccount.yaml b/charts/aws-fsx-csi-driver/templates/controller-serviceaccount.yaml index 0556a73d..d4b2b070 100644 --- a/charts/aws-fsx-csi-driver/templates/controller-serviceaccount.yaml +++ b/charts/aws-fsx-csi-driver/templates/controller-serviceaccount.yaml @@ -9,6 +9,11 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} +{{- else }} + {{- $exists := (lookup "v1" "ServiceAccount" .Release.Namespace .Values.controller.serviceAccount.name) }} + {{- if not $exists }} + {{- fail (printf "create serviceaccount %s/%s or set .controller.serviceaccount.create true" .Release.Namespace .Values.controller.serviceAccount.name) }} + {{- end }} {{- end }} --- diff --git a/charts/aws-fsx-csi-driver/templates/node-serviceaccount.yaml b/charts/aws-fsx-csi-driver/templates/node-serviceaccount.yaml index 369a3366..7e63f125 100644 --- a/charts/aws-fsx-csi-driver/templates/node-serviceaccount.yaml +++ b/charts/aws-fsx-csi-driver/templates/node-serviceaccount.yaml @@ -9,5 +9,10 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} +{{- else }} + {{- $exists := (lookup "v1" "ServiceAccount" .Release.Namespace .Values.node.serviceAccount.name) }} + {{- if not $exists }} + {{- fail (printf "create serviceaccount %s/%s or set .node.serviceaccount.create true" .Release.Namespace .Values.node.serviceAccount.name) }} + {{- end }} {{- end }} --- From 34ec8f5e325be97a4d2e3710769bcf36b7469ad6 Mon Sep 17 00:00:00 2001 From: Matthew Wong Date: Wed, 4 Aug 2021 13:12:45 -0700 Subject: [PATCH 15/15] Update README to mention breaking changes in kustomize configuration and helm chart --- docs/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/README.md b/docs/README.md index 5b80efdd..ea8d1f30 100644 --- a/docs/README.md +++ b/docs/README.md @@ -109,11 +109,24 @@ kubectl apply -k "github.com/kubernetes-sigs/aws-fsx-csi-driver/deploy/kubernete ``` Alternatively, you could also install the driver using helm: + ```sh helm repo add aws-fsx-csi-driver https://kubernetes-sigs.github.io/aws-fsx-csi-driver/ helm repo update helm upgrade --install aws-fsx-csi-driver --namespace kube-system aws-fsx-csi-driver/aws-fsx-csi-driver ``` + +###### Upgrading from version release-0.4 to master of the kustomize configuration + +In the master branch and the next release there are breaking changes that require you to `--force` to `kubectl apply`: +```sh +kubectl apply -k "github.com/kubernetes-sigs/aws-fsx-csi-driver/deploy/kubernetes/overlays/stable/?ref=master" --force +``` + +##### Upgrading from version 0.x to 1.x of the helm chart + +Version 1.0.0 removed and renamed almost all values to be more consistent with the EBS and EFS CSI driver helm charts. For details, see the [CHANGELOG](./charts/aws-fsx-csi-driver/CHANGELOG.md). + ### Examples Before the example, you need to: * Get yourself familiar with how to setup Kubernetes on AWS and [create FSx for Lustre filesystem](https://docs.aws.amazon.com/fsx/latest/LustreGuide/getting-started.html#getting-started-step1) if you are using static provisioning.