diff --git a/apps/csi-driver/csi-driver.yaml b/apps/csi-driver/csi-driver.yaml new file mode 100644 index 000000000000..6c35a9a8b9a2 --- /dev/null +++ b/apps/csi-driver/csi-driver.yaml @@ -0,0 +1,113 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: "csi.networkservicemesh.io" +spec: + # Only ephemeral, inline volumes are supported. There is no need for a + # controller to provision and attach volumes. + attachRequired: false + + # Request the pod information which the CSI driver uses to verify that an + # ephemeral mount was requested. + podInfoOnMount: true + + # Don't change ownership on the contents of the mount since the + # NS registration Unix Domain Socket is typically open to all (i.e. 0777). + fsGroupPolicy: None + + # Declare support for ephemeral volumes only. + volumeLifecycleModes: + - Ephemeral + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nsm-csi + labels: + app: nsm-csi +spec: + selector: + matchLabels: + app: nsm-csi + template: + metadata: + labels: + app: nsm-csi + spec: + containers: + # This is the container which runs the NSM CSI driver. + - name: nsm-csi-driver + image: ghcr.io/networkservicemesh/ci/cmd-csi-driver:1 + imagePullPolicy: IfNotPresent + env: + # The CSI driver needs a unique node ID. The node name can be + # used for this purpose. + - name: NSM_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NSM_SOCKET_DIR + value: "/nsm-socket" + - name: NSM_CSI_SOCKET_PATH + value: "/nsm-csi/csi.sock" + - name: NSM_VERSION + value: "cmd-csi-driver:1" + volumeMounts: + # The volume containing the Network Service API socket. + # The NSM CSI driver will mount this directory into containers. + - mountPath: /nsm-socket + name: nsm-socket + readOnly: true + # The volume that will contain the CSI driver socket shared + # with the kubelet and the driver registrar. + - mountPath: /nsm-csi + name: nsm-csi-socket-dir + # The volume containing mount points for containers. + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: mountpoint-dir + securityContext: + privileged: true + # This container runs the CSI Node Driver Registrar which takes care + # of all the little details required to register a CSI driver with + # the kubelet. + - name: node-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + imagePullPolicy: IfNotPresent + args: [ + "-csi-address", "/nsm-csi/csi.sock", + "-kubelet-registration-path", "/var/lib/kubelet/plugins/csi.networkservicemesh.io/csi.sock", + ] + volumeMounts: + # The registrar needs access to the NSM CSI driver socket + - mountPath: /nsm-csi + name: nsm-csi-socket-dir + # The registrar needs access to the Kubelet plugin registration + # directory + - name: kubelet-plugin-registration-dir + mountPath: /registration + volumes: + - name: nsm-socket + hostPath: + path: /var/lib/networkservicemesh + type: DirectoryOrCreate + # This volume is where the socket for kubelet->driver communication lives + - name: nsm-csi-socket-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.networkservicemesh.io + type: DirectoryOrCreate + # This volume is where the NSM CSI driver mounts volumes + - name: mountpoint-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + # This volume is where the node-driver-registrar registers the plugin + # with kubelet + - name: kubelet-plugin-registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + - name: exclude-prefixes-volume + emptyDir: {} diff --git a/apps/csi-driver/kustomization.yaml b/apps/csi-driver/kustomization.yaml new file mode 100644 index 000000000000..8a48ccca1955 --- /dev/null +++ b/apps/csi-driver/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- csi-driver.yaml diff --git a/examples/features/webhook/README.md b/examples/features/webhook/README.md index b2a61716b711..5a2e47af89e1 100644 --- a/examples/features/webhook/README.md +++ b/examples/features/webhook/README.md @@ -1,6 +1,6 @@ -# Client requests for postgresql service +# Client requests for nginx service -This example demonstrates how Postgres-client can get connectivity to Postgres-server deployment via NSM. +This example demonstrates how the client can get connectivity to the nginx-server via NSM. Client pod and server deployment located on different nodes. diff --git a/examples/pss/README.md b/examples/pss/README.md new file mode 100644 index 000000000000..721c5000cbeb --- /dev/null +++ b/examples/pss/README.md @@ -0,0 +1,45 @@ +# Pod Security Standard (PSS) examples + +Contain basic setup for NSM that includes `nsm-admission-webhook` `nsmgr`, `forwarder-vpp`, `registry-k8s` and [NSM CSI driver](https://github.com/networkservicemesh/cmd-csi-driver). +CSI driver allows us to avoid using `hostPath` volumes in workloads. + +Based on the [PSS profile](https://kubernetes.io/docs/concepts/security/pod-security-standards/), the admission-webhook adds the required security settings to the NSM sidecar containers. + +**_Please note_** that the webhook only knows about the profile from **_the namespace labels_**. + +## Requires + +- [spire_csi](../spire/single_cluster_csi) + +## Includes + +- [Nginx service](use-cases/nginx) + +## Run + +Apply NSM resources: + +```bash +kubectl apply -k https://github.com/networkservicemesh/deployments-k8s/examples/pss/nsm-system?ref=8a2d2e4576c907697958881d8eba454a6432798b +``` + +Wait for admission-webhook-k8s: + +```bash +WH=$(kubectl get pods -l app=admission-webhook-k8s -n nsm-system --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}') +kubectl wait --for=condition=ready --timeout=1m pod ${WH} -n nsm-system +``` + +## Cleanup + +Due to CSI driver limitations, we first need to remove pods that contain a volume mounted by the driver: +```bash +kubectl delete ds/forwarder-vpp -n nsm-system +``` + +To free resources follow the next commands: +```bash +WH=$(kubectl get pods -l app=admission-webhook-k8s -n nsm-system --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}') +kubectl delete mutatingwebhookconfiguration ${WH} +kubectl delete ns nsm-system +``` diff --git a/examples/pss/nsm-system/kustomization.yaml b/examples/pss/nsm-system/kustomization.yaml new file mode 100644 index 000000000000..803159b43749 --- /dev/null +++ b/examples/pss/nsm-system/kustomization.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: nsm-system + +resources: + - nsm-system-namespace.yaml + - ../../../apps/csi-driver + - ../../../apps/nsmgr + - ../../../apps/forwarder-vpp + - ../../../apps/registry-k8s + - ../../../apps/admission-webhook-k8s + +patches: + - path: patch-nsmgr.yaml + - path: patch-forwarder-vpp.yaml + - path: patch-registry-k8s.yaml + - path: patch-admission-webhook-k8s.yaml diff --git a/examples/pss/nsm-system/nsm-system-namespace.yaml b/examples/pss/nsm-system/nsm-system-namespace.yaml new file mode 100644 index 000000000000..e952c71a0b30 --- /dev/null +++ b/examples/pss/nsm-system/nsm-system-namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: nsm-system diff --git a/examples/pss/nsm-system/patch-admission-webhook-k8s.yaml b/examples/pss/nsm-system/patch-admission-webhook-k8s.yaml new file mode 100644 index 000000000000..68671eb2f702 --- /dev/null +++ b/examples/pss/nsm-system/patch-admission-webhook-k8s.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: admission-webhook-k8s +spec: + template: + spec: + containers: + - name: admission-webhook-k8s + env: + - name: NSM_ENVS + value: NSM_LOG_LEVEL=TRACE,NSM_LOCALDNSSERVERENABLED=False diff --git a/examples/pss/nsm-system/patch-forwarder-vpp.yaml b/examples/pss/nsm-system/patch-forwarder-vpp.yaml new file mode 100644 index 000000000000..80847ca355a0 --- /dev/null +++ b/examples/pss/nsm-system/patch-forwarder-vpp.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: forwarder-vpp +spec: + template: + spec: + volumes: + - name: spire-agent-socket + hostPath: null + csi: + driver: "csi.spiffe.io" + readOnly: true + - name: nsm-socket + hostPath: null + csi: + driver: "csi.networkservicemesh.io" + readOnly: true diff --git a/examples/pss/nsm-system/patch-nsmgr.yaml b/examples/pss/nsm-system/patch-nsmgr.yaml new file mode 100644 index 000000000000..f42d0e17c0d2 --- /dev/null +++ b/examples/pss/nsm-system/patch-nsmgr.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nsmgr +spec: + template: + spec: + volumes: + - name: spire-agent-socket + hostPath: null + csi: + driver: "csi.spiffe.io" + readOnly: true diff --git a/examples/pss/nsm-system/patch-registry-k8s.yaml b/examples/pss/nsm-system/patch-registry-k8s.yaml new file mode 100644 index 000000000000..21e22535e183 --- /dev/null +++ b/examples/pss/nsm-system/patch-registry-k8s.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: registry-k8s +spec: + template: + spec: + volumes: + - name: spire-agent-socket + hostPath: null + csi: + driver: "csi.spiffe.io" + readOnly: true diff --git a/examples/pss/use-cases/nginx/README.md b/examples/pss/use-cases/nginx/README.md new file mode 100644 index 000000000000..c3ad3ebfdc3b --- /dev/null +++ b/examples/pss/use-cases/nginx/README.md @@ -0,0 +1,38 @@ +# Nginx service + +This example uses the `restricted` PSS policy for the namespace. +The `restricted` policy requires additional `securityContext` settings as well as not using `hostPath`. + +We can see how the client can get connectivity to nginx-server via NSM. +Client pod and server deployment located on different nodes. + +## Requires + +Make sure that you have completed steps from [PSS](../..). + +## Run + +Deploy client and nginx-nse +```bash +kubectl apply -k https://github.com/networkservicemesh/deployments-k8s/examples/pss/use-cases/nginx?ref=8a2d2e4576c907697958881d8eba454a6432798b +``` + +Wait for applications ready: +```bash +kubectl wait --for=condition=ready --timeout=5m pod -l app=nse-kernel -n ns-nginx +``` +```bash +kubectl wait --for=condition=ready --timeout=1m pod -l app=nettools -n ns-nginx +``` + +Try to connect from client to nginx service: +```bash +kubectl exec pods/nettools -n ns-nginx -- curl 172.16.1.100:8080 | grep -o "Welcome to nginx" +``` + +## Cleanup + +Delete ns: +```bash +kubectl delete ns ns-nginx +``` diff --git a/examples/pss/use-cases/nginx/client.yaml b/examples/pss/use-cases/nginx/client.yaml new file mode 100644 index 000000000000..7fb5e46cd253 --- /dev/null +++ b/examples/pss/use-cases/nginx/client.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: nettools + labels: + app: nettools + annotations: + networkservicemesh.io: kernel://nginx/nsm-1 +spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: nettools + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + image: travelping/nettools:1.10.1 + imagePullPolicy: IfNotPresent + # simple `sleep` command would work + # but we need `trap` to be able to delete pods quckly + command: ["/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"] + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - nse-kernel diff --git a/examples/pss/use-cases/nginx/kustomization.yaml b/examples/pss/use-cases/nginx/kustomization.yaml new file mode 100644 index 000000000000..3d69580bc95a --- /dev/null +++ b/examples/pss/use-cases/nginx/kustomization.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: ns-nginx + +resources: +- ns-nginx.yaml +- client.yaml +- netsvc.yaml +- ../../../../apps/nse-kernel + +patches: + - path: patch-nse.yaml diff --git a/examples/pss/use-cases/nginx/netsvc.yaml b/examples/pss/use-cases/nginx/netsvc.yaml new file mode 100644 index 000000000000..527ae50f519a --- /dev/null +++ b/examples/pss/use-cases/nginx/netsvc.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: networkservicemesh.io/v1 +kind: NetworkService +metadata: + name: nginx +spec: + payload: ETHERNET diff --git a/examples/pss/use-cases/nginx/ns-nginx.yaml b/examples/pss/use-cases/nginx/ns-nginx.yaml new file mode 100644 index 000000000000..eb70adddac38 --- /dev/null +++ b/examples/pss/use-cases/nginx/ns-nginx.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ns-nginx + labels: + pod-security.kubernetes.io/enforce: restricted diff --git a/examples/pss/use-cases/nginx/patch-nse.yaml b/examples/pss/use-cases/nginx/patch-nse.yaml new file mode 100644 index 000000000000..5f7e5b992646 --- /dev/null +++ b/examples/pss/use-cases/nginx/patch-nse.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nse-kernel +spec: + template: + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: nginx + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + # restricted k8s policy requires unprivileged containers + image: nginxinc/nginx-unprivileged + ports: + - containerPort: 8080 + - name: nse + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + env: + - name: NSM_SERVICE_NAMES + value: "nginx" + - name: NSM_REGISTER_SERVICE + value: "false" + - name: NSM_CIDR_PREFIX + value: 172.16.1.100/31 + volumes: + - name: spire-agent-socket + hostPath: null + csi: + driver: "csi.spiffe.io" + readOnly: true + - name: nsm-socket + hostPath: null + csi: + driver: "csi.networkservicemesh.io" + readOnly: true diff --git a/examples/spire/single_cluster_csi/README.md b/examples/spire/single_cluster_csi/README.md new file mode 100644 index 000000000000..242d368db43c --- /dev/null +++ b/examples/spire/single_cluster_csi/README.md @@ -0,0 +1,33 @@ +# Spire CSI + +This setup deploys SPIRE along with [SPIFFE CSI driver](https://github.com/spiffe/spiffe-csi) + +## Run + +To apply spire deployments following the next command: +```bash +kubectl apply -k https://github.com/networkservicemesh/deployments-k8s/examples/spire/single_cluster_csi?ref=8a2d2e4576c907697958881d8eba454a6432798b +``` + +Wait for PODs status ready: +```bash +kubectl wait -n spire --timeout=1m --for=condition=ready pod -l app=spire-server +``` +```bash +kubectl wait -n spire --timeout=1m --for=condition=ready pod -l app=spire-agent +``` + +Apply the ClusterSPIFFEID CR for the cluster: +```bash +kubectl apply -f https://raw.githubusercontent.com/networkservicemesh/deployments-k8s/8a2d2e4576c907697958881d8eba454a6432798b/examples/spire/single_cluster/clusterspiffeid-template.yaml +``` + +## Cleanup + +Delete ns: +```bash +kubectl delete crd clusterspiffeids.spire.spiffe.io +kubectl delete crd clusterfederatedtrustdomains.spire.spiffe.io +kubectl delete validatingwebhookconfiguration.admissionregistration.k8s.io/spire-controller-manager-webhook +kubectl delete ns spire +``` diff --git a/examples/spire/single_cluster_csi/kustomization.yaml b/examples/spire/single_cluster_csi/kustomization.yaml new file mode 100644 index 000000000000..b77978022493 --- /dev/null +++ b/examples/spire/single_cluster_csi/kustomization.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: spire + +resources: + - spiffe-csi-driver.yaml + - ../single_cluster + +patches: + - path: patch-agent-daemonset.yaml diff --git a/examples/spire/single_cluster_csi/patch-agent-daemonset.yaml b/examples/spire/single_cluster_csi/patch-agent-daemonset.yaml new file mode 100644 index 000000000000..e804c9a09fc7 --- /dev/null +++ b/examples/spire/single_cluster_csi/patch-agent-daemonset.yaml @@ -0,0 +1,82 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: spire-agent + namespace: spire +spec: + template: + spec: + containers: + - name: spiffe-csi-driver + image: ghcr.io/spiffe/spiffe-csi-driver:0.2.3 + imagePullPolicy: IfNotPresent + args: [ + "-workload-api-socket-dir", "/spire-agent-socket", + "-csi-socket-path", "/spiffe-csi/csi.sock", + ] + env: + # The CSI driver needs a unique node ID. The node name can be + # used for this purpose. + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + # The volume containing the SPIRE agent socket. The SPIFFE CSI + # driver will mount this directory into containers. + - mountPath: /spire-agent-socket + name: spire-agent-socket + readOnly: true + # The volume that will contain the CSI driver socket shared + # with the kubelet and the driver registrar. + - mountPath: /spiffe-csi + name: spiffe-csi-socket-dir + # The volume containing mount points for containers. + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: mountpoint-dir + securityContext: + privileged: true + # This container runs the CSI Node Driver Registrar which takes care + # of all the little details required to register a CSI driver with + # the kubelet. + - name: node-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + imagePullPolicy: IfNotPresent + args: [ + "-csi-address", "/spiffe-csi/csi.sock", + "-kubelet-registration-path", "/var/lib/kubelet/plugins/csi.spiffe.io/csi.sock", + ] + volumeMounts: + # The registrar needs access to the SPIFFE CSI driver socket + - mountPath: /spiffe-csi + name: spiffe-csi-socket-dir + # The registrar needs access to the Kubelet plugin registration + # directory + - name: kubelet-plugin-registration-dir + mountPath: /registration + volumes: + # This volume is used to share the Workload API socket between the CSI + # driver and SPIRE agent. Note, an emptyDir volume could also be used, + # however, this can lead to broken bind mounts in the workload + # containers if the agent pod is restarted (since the emptyDir + # directory on the node that was mounted into workload containers by + # the CSI driver belongs to the old pod instance and is no longer + # valid). + # This volume is where the socket for kubelet->driver communication lives + - name: spiffe-csi-socket-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.spiffe.io + type: DirectoryOrCreate + # This volume is where the SPIFFE CSI driver mounts volumes + - name: mountpoint-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + # This volume is where the node-driver-registrar registers the plugin + # with kubelet + - name: kubelet-plugin-registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory diff --git a/examples/spire/single_cluster_csi/spiffe-csi-driver.yaml b/examples/spire/single_cluster_csi/spiffe-csi-driver.yaml new file mode 100644 index 000000000000..35095b7119c4 --- /dev/null +++ b/examples/spire/single_cluster_csi/spiffe-csi-driver.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: "csi.spiffe.io" +spec: + # Only ephemeral, inline volumes are supported. There is no need for a + # controller to provision and attach volumes. + attachRequired: false + + # Request the pod information which the CSI driver uses to verify that an + # ephemeral mount was requested. + podInfoOnMount: true + + # Don't change ownership on the contents of the mount since the Workload API + # Unix Domain Socket is typically open to all (i.e. 0777). + fsGroupPolicy: None + + # Declare support for ephemeral volumes only. + volumeLifecycleModes: + - Ephemeral diff --git a/external-images.yaml b/external-images.yaml index 731b4f91321c..c3408273bec5 100644 --- a/external-images.yaml +++ b/external-images.yaml @@ -4,11 +4,13 @@ images: - docker.io/library/postgres:latest - docker.io/library/nginx:latest + - docker.io/nginxinc/nginx-unprivileged:latest - docker.io/coredns/coredns:1.8.3 - ghcr.io/spiffe/spire-agent:1.6.1 - ghcr.io/spiffe/spire-server:1.6.1 - gcr.io/spiffe-io/wait-for-it:latest - ghcr.io/spiffe/spire-controller-manager:0.2.2 + - ghcr.io/spiffe/spiffe-csi-driver:0.2.3 - quay.io/metallb/speaker:v0.12.1 - quay.io/metallb/controller:v0.12.1 - docker.io/library/alpine:3.15.0