diff --git a/.gitignore b/.gitignore index 66fd13c903ca..f2077f743c65 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ # Dependency directories (remove the comment below to include it) # vendor/ + +# Goland files +.idea/ \ No newline at end of file diff --git a/README.md b/README.md index 57a9c777bbb3..acc478962775 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,4 @@ This repository provides kubernetes yaml deployments and markdown examples for N - [Applications](./apps) - [Basic examples](./examples/basic) - - - +- [SR-IOV examples](./examples/sriov) diff --git a/apps/sriov-forwarder/kustomization.yaml b/apps/sriov-forwarder/kustomization.yaml new file mode 100644 index 000000000000..8b0d2cb9c35e --- /dev/null +++ b/apps/sriov-forwarder/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- sriov-forwarder.yaml diff --git a/apps/sriov-forwarder/sriov-forwarder.yaml b/apps/sriov-forwarder/sriov-forwarder.yaml new file mode 100644 index 000000000000..fa0aa1ad62c2 --- /dev/null +++ b/apps/sriov-forwarder/sriov-forwarder.yaml @@ -0,0 +1,67 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: sriov-forwarder + labels: + app: sriov-forwarder +spec: + selector: + matchLabels: + app: sriov-forwarder + template: + metadata: + labels: + app: sriov-forwarder + spec: + hostNetwork: true + containers: + - image: networkservicemeshci/cmd-forwarder-sriov:e6dad1c9 + imagePullPolicy: IfNotPresent + name: sriov-forwarder + securityContext: + privileged: true + env: + - name: NSM_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: SPIFFE_ENDPOINT_SOCKET + value: unix:///run/spire/sockets/agent.sock + - name: NSM_CONNECT_TO + value: unix:///var/lib/networkservicemesh/nsm.io.sock + - name: NSM_SRIOV_CONFIG_FILE + value: /var/lib/networkservicemesh/sriov.config + volumeMounts: + - name: spire-agent-socket + mountPath: /run/spire/sockets + readOnly: true + - name: nsm + mountPath: /var/lib/networkservicemesh + - name: kubelet-socket + mountPath: /var/lib/kubelet + - name: cgroup + mountPath: /host/sys/fs/cgroup + - name: vfio + mountPath: /host/dev/vfio + volumes: + - name: spire-agent-socket + hostPath: + path: /run/spire/sockets + type: Directory + - name: nsm + hostPath: + path: /var/lib/networkservicemesh + type: Directory + - name: kubelet-socket + hostPath: + path: /var/lib/kubelet + type: Directory + - name: cgroup + hostPath: + path: /sys/fs/cgroup + type: Directory + - name: vfio + hostPath: + path: /dev/vfio + type: DirectoryOrCreate diff --git a/apps/vfio-nsc/kustomization.yaml b/apps/vfio-nsc/kustomization.yaml new file mode 100644 index 000000000000..1b27eb262a0c --- /dev/null +++ b/apps/vfio-nsc/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- nsc.yaml diff --git a/apps/vfio-nsc/nsc.yaml b/apps/vfio-nsc/nsc.yaml new file mode 100644 index 000000000000..17221b30368e --- /dev/null +++ b/apps/vfio-nsc/nsc.yaml @@ -0,0 +1,66 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nsc + labels: + app: nsc +spec: + selector: + matchLabels: + app: nsc + template: + metadata: + labels: + app: nsc + spec: + containers: + - name: pinger + # https://github.com/Bolodya1997/docker-dpdk + image: rrandom312/dpdk-pingpong:latest + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-c", "sleep 60m"] + volumeMounts: + - name: vfio + mountPath: /dev/vfio + + - name: sidecar + image: networkservicemeshci/cmd-nsc:5bea421a + imagePullPolicy: IfNotPresent + env: + - name: SPIFFE_ENDPOINT_SOCKET + value: unix:///run/spire/sockets/agent.sock + - name: NSM_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NSM_CONNECT_TO + value: unix:///var/lib/networkservicemesh/nsm.io.sock + - name: NSM_NETWORK_SERVICES + value: vfio://pingpong?sriovToken=worker.domain/10G + volumeMounts: + - name: spire-agent-socket + mountPath: /run/spire/sockets + readOnly: true + - name: nsm + mountPath: /var/lib/networkservicemesh + - name: vfio + mountPath: /dev/vfio + resources: + limits: + # We expect SR-IOV forwarders on the nodes to be configured: + # master - provides PCI functions targeted to the worker.domain/10G + # worker - provides PCI functions targeted to the master.domain/10G + worker.domain/10G: 1 + + volumes: + - name: vfio + emptyDir: {} + - name: spire-agent-socket + hostPath: + path: /run/spire/sockets + type: Directory + - name: nsm + hostPath: + path: /var/lib/networkservicemesh + type: Directory diff --git a/apps/vfio-nse/kustomization.yaml b/apps/vfio-nse/kustomization.yaml new file mode 100644 index 000000000000..853870d4cb61 --- /dev/null +++ b/apps/vfio-nse/kustomization.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- nse.yaml +- nse-configmap.yaml diff --git a/apps/vfio-nse/nse-configmap.yaml b/apps/vfio-nse/nse-configmap.yaml new file mode 100644 index 000000000000..09b2faa700af --- /dev/null +++ b/apps/vfio-nse/nse-configmap.yaml @@ -0,0 +1,63 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nse +data: + pong.sh: | + #!/bin/bash + # shellcheck disable=SC2064 + + function softlink_target() { + softlink="$1" + + raw_target="$(stat -c %N "${softlink}")" + test $? -eq 0 || return 1 + + target=$(echo "${raw_target}" | sed -E "s/(.*\/)(.*)'/\2/g") + test $? -eq 0 || return 2 + + echo "${target}" + return 0 + } + + function bind_driver() { + pci_addr="$1" + driver="$2" + + echo "${pci_addr}" > "/sys/bus/pci/devices/${pci_addr}/driver/unbind" + echo "${pci_addr}" > "/sys/bus/pci/drivers/${driver}/bind" + test $? -eq 0 || return 1 + + return 0 + } + + ## + ## Begin + ## + + pf_link="$1" + vf_num="$2" + + device="/sys/class/net/${pf_link}/device/virtfn${vf_num}" + + pci_addr=$(softlink_target "${device}") + test $? -eq 0 || exit 1 + + if driver=$(softlink_target "${device}/driver"); then + # Don't forget to set driver back on exit + trap "bind_driver '${pci_addr}' '${driver}'" err exit + fi + + # Bind VFIO driver + bind_driver "${pci_addr}" "vfio-pci" + test $? -eq 0 || exit 3 + + # Run dpdk-pingpong (server) + /root/dpdk-pingpong/build/app/pingpong \ + --no-huge \ + --pci-whitelist="${pci_addr}" \ + -- \ + -n 50 \ + -s \ + -S 0a:55:44:33:22:11 diff --git a/apps/vfio-nse/nse.yaml b/apps/vfio-nse/nse.yaml new file mode 100644 index 000000000000..2b63c8f2437d --- /dev/null +++ b/apps/vfio-nse/nse.yaml @@ -0,0 +1,77 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nse + labels: + app: nse +spec: + selector: + matchLabels: + app: nse + template: + metadata: + labels: + app: nse + spec: + hostNetwork: true + containers: + - name: ponger + # https://github.com/Bolodya1997/docker-dpdk + image: rrandom312/dpdk-pingpong:latest + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "/root/scripts/pong.sh", "eno4", "31"] + securityContext: + privileged: true + volumeMounts: + - name: scripts + mountPath: /root/scripts + readOnly: true + - name: vfio + mountPath: /dev/vfio + + - name: sidecar + image: networkservicemeshci/cmd-nse-vfio:b00ed50b + imagePullPolicy: Always + env: + - name: SPIFFE_ENDPOINT_SOCKET + value: unix:///run/spire/sockets/agent.sock + - name: NSE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NSE_CONNECT_TO + value: unix:///var/lib/networkservicemesh/nsm.io.sock + - name: NSE_SERVICES + value: "pingpong@worker.domain: { addr: 0a:55:44:33:22:11 }" + volumeMounts: + - name: spire-agent-socket + mountPath: /run/spire/sockets + readOnly: true + - name: nsm + mountPath: /var/lib/networkservicemesh + # We don't actually need resources for NSE, but we need to be sure that it is going to be deployed on the + # correct node. + resources: + limits: + # We expect SR-IOV forwarders on the nodes to be configured: + # master - provides PCI functions targeted to the worker.domain/10G + # worker - provides PCI functions targeted to the master.domain/10G + master.domain/10G: 1 + + volumes: + - name: scripts + configMap: + name: nse + - name: spire-agent-socket + hostPath: + path: /run/spire/sockets + type: Directory + - name: nsm + hostPath: + path: /var/lib/networkservicemesh + type: Directory + - name: vfio + hostPath: + path: /dev/vfio + type: DirectoryOrCreate diff --git a/examples/VFIOConnection/README.md b/examples/VFIOConnection/README.md new file mode 100644 index 000000000000..c4a98cebc5c9 --- /dev/null +++ b/examples/VFIOConnection/README.md @@ -0,0 +1,104 @@ +# Test VFIO connection + +This example shows that NSC and NSE can work with each other over the VFIO connection. + +## Run + +Create test namespace: +```bash +NAMESPACE=($(kubectl create -f namespace.yaml)[0]) +NAMESPACE=${NAMESPACE:10} +``` + +Register namespace in `spire` server: +```bash +kubectl exec -n spire spire-server-0 -- \ +/opt/spire/bin/spire-server entry create \ +-spiffeID spiffe://example.org/ns/${NAMESPACE}/sa/default \ +-parentID spiffe://example.org/ns/spire/sa/spire-agent \ +-selector k8s:ns:${NAMESPACE} \ +-selector k8s:sa:default +``` + +Create customization file: +```bash +cat > kustomization.yaml <&1) || (echo "${PING_RESULTS}" 1>&2 && false) +``` +```bash +PONG_PACKETS="$(echo "${PING_RESULTS}" | + grep "rx .* pong packets" | + sed -E 's/rx ([0-9]*) pong packets/\1/g')" \ + || (echo "${PING_RESULTS}" 1>&2 && false) +``` +```bash +test "${PONG_PACKETS}" -ne 0 \ + || (echo "${PING_RESULTS}" 1>&2 && false) +``` + +## Cleanup + +Stop ponger: +```bash +NSE_POD=$(kubectl -n ${NAMESPACE} get pods -l app=nse | + grep -v "NAME" | + sed -E "s/([.]*) .*/\1/g") +``` +```bash +kubectl -n ${NAMESPACE} exec ${NSE_POD} --container ponger -- /bin/bash -c ' \ + sleep 10 && kill $(ps -A | grep "pingpong" | sed -E "s/ *([0-9]*).*/\1/g") 1>/dev/null 2>&1 & \ +' +``` + +Delete ns: +```bash +kubectl delete ns ${NAMESPACE} +``` \ No newline at end of file diff --git a/examples/VFIOConnection/namespace.yaml b/examples/VFIOConnection/namespace.yaml new file mode 100644 index 000000000000..7919ccab9bec --- /dev/null +++ b/examples/VFIOConnection/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + generateName: ns- diff --git a/examples/spire/README.md b/examples/spire/README.md index ce6e264aec8a..83f56934377e 100644 --- a/examples/spire/README.md +++ b/examples/spire/README.md @@ -3,17 +3,14 @@ ## Run To apply spire deployments following the next command: - ```bash kubectl apply -k . ``` Wait for PODs status ready: - ```bash kubectl wait -n spire --timeout=1m --for=condition=ready pod -l app=spire-agent ``` - ```bash kubectl wait -n spire --timeout=1m --for=condition=ready pod -l app=spire-server ``` @@ -31,6 +28,7 @@ kubectl exec -n spire spire-server-0 -- \ ## Cleanup +Delete ns: ```bash kubectl delete ns spire ``` diff --git a/examples/spire/server-statefulset.yaml b/examples/spire/server-statefulset.yaml index d9d30f7fef40..99c8bdfe0c6b 100644 --- a/examples/spire/server-statefulset.yaml +++ b/examples/spire/server-statefulset.yaml @@ -31,9 +31,6 @@ spec: - name: spire-config mountPath: /run/spire/config readOnly: true - - name: spire-data - mountPath: /run/spire/data - readOnly: false livenessProbe: exec: command: @@ -47,13 +44,3 @@ spec: - name: spire-config configMap: name: spire-server - volumeClaimTemplates: - - metadata: - name: spire-data - namespace: spire - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi diff --git a/examples/sriov/README.md b/examples/sriov/README.md new file mode 100644 index 000000000000..f328cdb445be --- /dev/null +++ b/examples/sriov/README.md @@ -0,0 +1,36 @@ +## Requires + +- [spire](../spire) + +## Includes + +- [VFIO Connection](../VFIOConnection) + +## Run + +Create ns for deployments: +```bash +kubectl create ns nsm-system +``` + +Register `nsm-system` namespace in spire: +```bash +kubectl exec -n spire spire-server-0 -- \ +/opt/spire/bin/spire-server entry create \ +-spiffeID spiffe://example.org/ns/nsm-system/sa/default \ +-parentID spiffe://example.org/ns/spire/sa/spire-agent \ +-selector k8s:ns:nsm-system \ +-selector k8s:sa:default +``` + +Apply NSM resources for basic tests: +```bash +kubectl apply -k . +``` + +## Cleanup + +Delete ns: +```bash +kubectl delete ns nsm-system +``` \ No newline at end of file diff --git a/examples/sriov/kustomization.yaml b/examples/sriov/kustomization.yaml new file mode 100644 index 000000000000..a8a1a2be33f2 --- /dev/null +++ b/examples/sriov/kustomization.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: nsm-system + +bases: +- ../../apps/nsmgr +- ../../apps/sriov-forwarder +- ../../apps/registry-memory