Merge pull request #7 from Bolodya1997/sriov

Add SR-IOV examples

.gitignore

@@ -13,3 +13,6 @@
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
+
+# Goland files
+.idea/

README.md

@@ -6,6 +6,4 @@ This repository provides kubernetes yaml deployments and markdown examples for N
 
 - [Applications](./apps)
 - [Basic examples](./examples/basic)
-
-
-
+- [SR-IOV examples](./examples/sriov)

apps/sriov-forwarder/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- sriov-forwarder.yaml

apps/sriov-forwarder/sriov-forwarder.yaml

@@ -0,0 +1,67 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: sriov-forwarder
+  labels:
+    app: sriov-forwarder
+spec:
+  selector:
+    matchLabels:
+      app: sriov-forwarder
+  template:
+    metadata:
+      labels:
+        app: sriov-forwarder
+    spec:
+      hostNetwork: true
+      containers:
+        - image: networkservicemeshci/cmd-forwarder-sriov:e6dad1c9
+          imagePullPolicy: IfNotPresent
+          name: sriov-forwarder
+          securityContext:
+            privileged: true
+          env:
+            - name: NSM_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: SPIFFE_ENDPOINT_SOCKET
+              value: unix:///run/spire/sockets/agent.sock
+            - name: NSM_CONNECT_TO
+              value: unix:///var/lib/networkservicemesh/nsm.io.sock
+            - name: NSM_SRIOV_CONFIG_FILE
+              value: /var/lib/networkservicemesh/sriov.config
+          volumeMounts:
+            - name: spire-agent-socket
+              mountPath: /run/spire/sockets
+              readOnly: true
+            - name: nsm
+              mountPath: /var/lib/networkservicemesh
+            - name: kubelet-socket
+              mountPath: /var/lib/kubelet
+            - name: cgroup
+              mountPath: /host/sys/fs/cgroup
+            - name: vfio
+              mountPath: /host/dev/vfio
+      volumes:
+        - name: spire-agent-socket
+          hostPath:
+            path: /run/spire/sockets
+            type: Directory
+        - name: nsm
+          hostPath:
+            path: /var/lib/networkservicemesh
+            type: Directory
+        - name: kubelet-socket
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: cgroup
+          hostPath:
+            path: /sys/fs/cgroup
+            type: Directory
+        - name: vfio
+          hostPath:
+            path: /dev/vfio
+            type: DirectoryOrCreate

apps/vfio-nsc/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- nsc.yaml

apps/vfio-nsc/nsc.yaml

@@ -0,0 +1,66 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nsc
+  labels:
+    app: nsc
+spec:
+  selector:
+    matchLabels:
+      app: nsc
+  template:
+    metadata:
+      labels:
+        app: nsc
+    spec:
+      containers:
+        - name: pinger
+          # https://github.com/Bolodya1997/docker-dpdk
+          image: rrandom312/dpdk-pingpong:latest
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/bash", "-c", "sleep 60m"]
+          volumeMounts:
+            - name: vfio
+              mountPath: /dev/vfio
+
+        - name: sidecar
+          image: networkservicemeshci/cmd-nsc:5bea421a
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: SPIFFE_ENDPOINT_SOCKET
+              value: unix:///run/spire/sockets/agent.sock
+            - name: NSM_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NSM_CONNECT_TO
+              value: unix:///var/lib/networkservicemesh/nsm.io.sock
+            - name: NSM_NETWORK_SERVICES
+              value: vfio://pingpong?sriovToken=worker.domain/10G
+          volumeMounts:
+            - name: spire-agent-socket
+              mountPath: /run/spire/sockets
+              readOnly: true
+            - name: nsm
+              mountPath: /var/lib/networkservicemesh
+            - name: vfio
+              mountPath: /dev/vfio
+          resources:
+            limits:
+              # We expect SR-IOV forwarders on the nodes to be configured:
+              # master - provides PCI functions targeted to the worker.domain/10G
+              # worker - provides PCI functions targeted to the master.domain/10G
+              worker.domain/10G: 1
+
+      volumes:
+        - name: vfio
+          emptyDir: {}
+        - name: spire-agent-socket
+          hostPath:
+            path: /run/spire/sockets
+            type: Directory
+        - name: nsm
+          hostPath:
+            path: /var/lib/networkservicemesh
+            type: Directory

apps/vfio-nse/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- nse.yaml
+- nse-configmap.yaml

apps/vfio-nse/nse-configmap.yaml

@@ -0,0 +1,63 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nse
+data:
+  pong.sh: |
+    #!/bin/bash
+    # shellcheck disable=SC2064
+
+    function softlink_target() {
+      softlink="$1"
+
+      raw_target="$(stat -c %N "${softlink}")"
+      test $? -eq 0 || return 1
+
+      target=$(echo "${raw_target}" | sed -E "s/(.*\/)(.*)'/\2/g")
+      test $? -eq 0 || return 2
+
+      echo "${target}"
+      return 0
+    }
+
+    function bind_driver() {
+      pci_addr="$1"
+      driver="$2"
+
+      echo "${pci_addr}" > "/sys/bus/pci/devices/${pci_addr}/driver/unbind"
+      echo "${pci_addr}" > "/sys/bus/pci/drivers/${driver}/bind"
+      test $? -eq 0 || return 1
+
+      return 0
+    }
+
+    ##
+    ## Begin
+    ##
+
+    pf_link="$1"
+    vf_num="$2"
+
+    device="/sys/class/net/${pf_link}/device/virtfn${vf_num}"
+
+    pci_addr=$(softlink_target "${device}")
+    test $? -eq 0 || exit 1
+
+    if driver=$(softlink_target "${device}/driver"); then
+      # Don't forget to set driver back on exit
+      trap "bind_driver '${pci_addr}' '${driver}'" err exit
+    fi
+
+    # Bind VFIO driver
+    bind_driver "${pci_addr}" "vfio-pci"
+    test $? -eq 0 || exit 3
+
+    # Run dpdk-pingpong (server)
+    /root/dpdk-pingpong/build/app/pingpong \
+      --no-huge \
+      --pci-whitelist="${pci_addr}" \
+      -- \
+      -n 50 \
+      -s \
+      -S 0a:55:44:33:22:11

apps/vfio-nse/nse.yaml

@@ -0,0 +1,77 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nse
+  labels:
+    app: nse
+spec:
+  selector:
+    matchLabels:
+      app: nse
+  template:
+    metadata:
+      labels:
+        app: nse
+    spec:
+      hostNetwork: true
+      containers:
+        - name: ponger
+          # https://github.com/Bolodya1997/docker-dpdk
+          image: rrandom312/dpdk-pingpong:latest
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/bash", "/root/scripts/pong.sh", "eno4", "31"]
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: scripts
+              mountPath: /root/scripts
+              readOnly: true
+            - name: vfio
+              mountPath: /dev/vfio
+
+        - name: sidecar
+          image: networkservicemeshci/cmd-nse-vfio:b00ed50b
+          imagePullPolicy: Always
+          env:
+            - name: SPIFFE_ENDPOINT_SOCKET
+              value: unix:///run/spire/sockets/agent.sock
+            - name: NSE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NSE_CONNECT_TO
+              value: unix:///var/lib/networkservicemesh/nsm.io.sock
+            - name: NSE_SERVICES
+              value: "pingpong@worker.domain: { addr: 0a:55:44:33:22:11 }"
+          volumeMounts:
+            - name: spire-agent-socket
+              mountPath: /run/spire/sockets
+              readOnly: true
+            - name: nsm
+              mountPath: /var/lib/networkservicemesh
+          # We don't actually need resources for NSE, but we need to be sure that it is going to be deployed on the
+          # correct node.
+          resources:
+            limits:
+              # We expect SR-IOV forwarders on the nodes to be configured:
+              # master - provides PCI functions targeted to the worker.domain/10G
+              # worker - provides PCI functions targeted to the master.domain/10G
+              master.domain/10G: 1
+
+      volumes:
+        - name: scripts
+          configMap:
+            name: nse
+        - name: spire-agent-socket
+          hostPath:
+            path: /run/spire/sockets
+            type: Directory
+        - name: nsm
+          hostPath:
+            path: /var/lib/networkservicemesh
+            type: Directory
+        - name: vfio
+          hostPath:
+            path: /dev/vfio
+            type: DirectoryOrCreate