From da3e689ff7c53314b42a3151db987eab7729a9df Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Thu, 29 Feb 2024 12:53:42 +0530 Subject: [PATCH] ci: test: add framework for ovs test cases * Add ovs test suite and the initial ping test cases. * This also adds the framework to control a remote board (currently assumed to be an Octeon) which is connected back to back with the EP Octeon. * Update the dpdk dependency commit. Change-Id: I804f443f88566c4b38848d003583ee18de33e5c6 Signed-off-by: Ashwin Sekhar T K Signed-off-by: Amit Prakash Shukla Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/dataplane/dpu-offload/+/133522 Reviewed-by: Rahul Bhansali Reviewed-by: Harman Kalra Tested-by: sa_ip-toolkits-Jenkins --- ci/build/build-deps.sh | 1 - ci/build/env/deps/dpdk.env | 2 +- ci/test/common/exe_wrapper.sh | 6 +- ci/test/common/test_list_helper_funcs.sh | 21 +- ci/test/dao-test/common/ep_common_ops.sh | 218 ++++++++++ ci/test/dao-test/common/ep_device_utils.sh | 224 +++++++--- ci/test/dao-test/common/ep_host_utils.sh | 95 ++--- ci/test/dao-test/common/oxk-devbind-basic.sh | 61 --- ci/test/dao-test/common/testpmd.sh | 8 +- ci/test/dao-test/common/utils.sh | 115 ++++- ci/test/dao-test/meson.build | 1 + ci/test/dao-test/ovs/meson.build | 25 ++ ci/test/dao-test/ovs/ovs_ping.sh | 118 ++++++ ci/test/dao-test/ovs/ovs_utils.sh | 399 ++++++++++++++++++ .../virtio/extbuf/virtio_extbuf_1c.sh | 5 +- .../virtio/extbuf/virtio_extbuf_utils.sh | 10 +- .../dao-test/virtio/l2fwd/virtio_l2fwd_1c.sh | 26 +- .../virtio/l2fwd/virtio_l2fwd_utils.sh | 13 +- ci/test/env/cn10k-ovs.env | 12 + ci/test/env/cn10k-unit-tests.env | 1 - ci/test/env/cn10k.env | 57 +-- ci/test/ep/ep_test_run.sh | 127 +----- ci/test/ep/ovs_setup.sh | 162 +++++++ ci/test/ep/sync.sh | 64 +++ ci/test/ep/unit_test_setup.sh | 24 ++ ci/test/ep/virtio_setup.sh | 47 +++ 26 files changed, 1439 insertions(+), 403 deletions(-) create mode 100755 ci/test/dao-test/common/ep_common_ops.sh delete mode 100755 ci/test/dao-test/common/oxk-devbind-basic.sh create mode 100644 ci/test/dao-test/ovs/meson.build create mode 100755 ci/test/dao-test/ovs/ovs_ping.sh create mode 100755 ci/test/dao-test/ovs/ovs_utils.sh create mode 100644 ci/test/env/cn10k-ovs.env create mode 100755 ci/test/ep/ovs_setup.sh create mode 100755 ci/test/ep/sync.sh create mode 100755 ci/test/ep/unit_test_setup.sh create mode 100755 ci/test/ep/virtio_setup.sh diff --git a/ci/build/build-deps.sh b/ci/build/build-deps.sh index 71b44f6..bccd61d 100755 --- a/ci/build/build-deps.sh +++ b/ci/build/build-deps.sh @@ -117,7 +117,6 @@ function compile_libnl() { fetch_dep https://github.com/thom311/libnl/releases/download/libnl3_7_0/$LIBNL_TARBALL.tar.gz fi tar xvf $LIBNL_TARBALL.tar.gz --strip-components=1 - set -x ./configure --host=aarch64-marvell-linux-gnu --prefix=$LIBNL_PREFIX_DIR make; make install; diff --git a/ci/build/env/deps/dpdk.env b/ci/build/env/deps/dpdk.env index f0093cd..438a344 100644 --- a/ci/build/env/deps/dpdk.env +++ b/ci/build/env/deps/dpdk.env @@ -4,4 +4,4 @@ DATAPLANE_REPO=sj1git1.cavium.com:29418/IP/SW/dataplane DPDK_REPO=$DATAPLANE_REPO/dpdk DPDK_BRANCH=dpdk-23.11-devel -DPDK_COMMIT=74ff5554a08ab307dd2f2969f1d1befa1ad5d419 +DPDK_COMMIT=9bea882577ef965e7625b158448bb31f791dfce2 diff --git a/ci/test/common/exe_wrapper.sh b/ci/test/common/exe_wrapper.sh index 61559be..e6b356d 100755 --- a/ci/test/common/exe_wrapper.sh +++ b/ci/test/common/exe_wrapper.sh @@ -27,7 +27,11 @@ TEST_ARGS=$@ source $TEST_ENV_CONF TEST_ENV_VARS="DAO_TEST=$DAO_TEST " -TEST_ENV_VARS="$TEST_ENV_VARS EP_DEVICE=$EP_DEVICE EP_HOST=$EP_HOST" +TEST_ENV_VARS+=" EP_DEVICE=$EP_DEVICE EP_HOST=$EP_HOST EP_REMOTE=$EP_REMOTE" +TEST_ENV_VARS+=" EP_SSH_CMD='$EP_SSH_CMD' EP_DIR=$EP_DIR" +TEST_ENV_VARS+=" EP_REMOTE_SUDO=$EP_REMOTE_SUDO EP_HOST_SUDO=$EP_HOST_SUDO" +TEST_ENV_VARS+=" EP_HOST_MODULE_DIR=${EP_HOST_MODULE_DIR:-}" +TEST_ENV_VARS+=" EP_DEVICE_OVS_PATH=${EP_DEVICE_OVS_PATH:-}" add_test "$DAO_TEST" "$TEST_BINARY" "$TEST_DIR" "$TEST_ARGS" "$TEST_ENV_VARS" diff --git a/ci/test/common/test_list_helper_funcs.sh b/ci/test/common/test_list_helper_funcs.sh index 1cb9b0c..ee103d6 100644 --- a/ci/test/common/test_list_helper_funcs.sh +++ b/ci/test/common/test_list_helper_funcs.sh @@ -5,6 +5,7 @@ # Functions required to manipulate the test.list file. TEST_LIST=$RUN_DIR/test.list +TEST_ENV_VARS_DYNAMIC="" function clean_test_list() { @@ -26,6 +27,7 @@ function get_test_name() local test_num=$1 local num=1 local info="LIST_END" + touch $TEST_LIST while read -r testinfo; do if [[ $num == $test_num ]]; then info=$testinfo @@ -41,6 +43,7 @@ function get_test_info() local test_name=$1 local name local info="LIST_END" + touch $TEST_LIST while read -r testinfo; do name=$(echo $testinfo | awk -F'#' '{print $1}') if [[ $name == $test_name ]]; then @@ -55,14 +58,14 @@ function get_test_exec_bin() { local val val=$(get_test_info $1 | awk -F'#' '{print $2}') - echo $val | sed "s#$BUILD_DIR#$EP_DEVICE_RUN_DIR#g" + echo $val | sed "s#$BUILD_DIR#$EP_DIR#g" } function get_test_dir() { local val val=$(get_test_info $1 | awk -F'#' '{print $3}') - echo $val | sed "s#$BUILD_DIR#$EP_DEVICE_RUN_DIR#g" + echo $val | sed "s#$BUILD_DIR#$EP_DIR#g" } function get_test_args() @@ -161,7 +164,7 @@ function test_info_print() extra_args=$(get_test_extra_args $name) echo "Test Binary/script -> $exec_bin" echo "Test Timeout -> $tmo" - echo "Test Environment -> $envs" + echo "Test Environment -> $envs $TEST_ENV_VARS_DYNAMIC" echo "Test Directory -> $test_dir" # Remove unnecessary arguments from command line @@ -179,6 +182,11 @@ function test_info_print() echo "Test Command -> $cmd" } +function add_test_env() +{ + TEST_ENV_VARS_DYNAMIC+=" $@" +} + function get_test_command() { local name=$1 @@ -207,11 +215,6 @@ function get_test_command() esac done cmd="cd $test_dir && - $EP_DEVICE_SUDO DEPS_PREFIX=$EP_DEVICE_DIR/deps-prefix \ - EP_HOST_DIR=$EP_HOST_DIR \ - EP_DEVICE_DIR=$EP_DEVICE_DIR \ - EP_SSH_CMD='$EP_SSH_CMD' \ - EXTRA_EP_HOST_ENV='$EXTRA_EP_HOST_ENV' $envs $EXTRA_EP_DEVICE_ENV \ - $exec_bin $args $extra_args" + $EP_DEVICE_SUDO $TEST_ENV_VARS_DYNAMIC $envs $exec_bin $args $extra_args" echo "$cmd" } diff --git a/ci/test/dao-test/common/ep_common_ops.sh b/ci/test/dao-test/common/ep_common_ops.sh new file mode 100755 index 0000000..b97b835 --- /dev/null +++ b/ci/test/dao-test/common/ep_common_ops.sh @@ -0,0 +1,218 @@ +#!/bin/bash +# SPDX-License-Identifier: Marvell-MIT +# Copyright (c) 2024 Marvell. + +function ep_common_testpmd_launch() +{ + local pfx=$1 + local args=${@:2} + local eal_args + local app_args="" + + for a in $args; do + if [[ $a == "--" ]]; then + eal_args=$app_args + app_args="" + continue + fi + app_args+=" $a" + done + + echo "Launching testpmd pfx=$pfx" + testpmd_launch $pfx "$eal_args" "$app_args" + echo "Launched testpmd pfx=$pfx" +} + +function ep_common_testpmd_stop() +{ + local pfx=$1 + + echo "Stopping testpmd pfx=$pfx" + testpmd_quit $pfx + testpmd_cleanup $pfx + echo "Stopped testpmd pfx=$pfx" +} + +function ep_common_hugepage_setup() +{ + local hp_sz=$1 + local hp_num=$2 + local hp_pool_sz=$3 + + # Check for hugepages + if mount | grep hugetlbfs | grep none; then + echo "Hugepages already mounted" + else + echo "Mounting Hugepages" + mkdir -p /dev/huge + mount -t hugetlbfs none /dev/huge + fi + echo $hp_num > /proc/sys/vm/nr_hugepages + echo $hp_pool_sz >/sys/kernel/mm/hugepages/hugepages-${hp_sz}kB/nr_hugepages +} + +function ep_common_pcie_addr_get() +{ + local devid=$1 + local num=${2:-} + + if [[ -z $num ]]; then + num=1 + elif [[ $num == "all" ]]; then + num=100 + fi + + echo $(lspci -Dd :$devid | awk '{print $1}' | head -n$num) +} + +function ep_common_if_name_get() +{ + local pcie_addr=$1 + + set +e + grep PCI_SLOT_NAME /sys/class/net/*/device/uevent | grep $pcie_addr | \ + awk -F '/' '{print $5}' + set -e +} + +function ep_common_if_configure() +{ + local ip_addr + local opts + local iface_name + local pcie_addr= + local down= + local vxlan_remote_ip= + local vxlan_local_ip= + local vxlan_vni= + local vlan_id= + + if ! opts=$(getopt \ + -l "ip:,pcie-addr:,down,vxlan-remote-ip:,vxlan-local-ip:,vxlan-vni:,vlan-id:" \ + -- configure_sdp_interface $@); then + echo "Failed to parse arguments" + exit 1 + fi + + eval set -- "$opts" + while [[ $# -gt 1 ]]; do + case $1 in + --ip) shift; ip_addr=$1;; + --pcie-addr) shift; pcie_addr=$1;; + --vxlan-vni) shift; vxlan_vni=$1;; + --vxlan-remote-ip) shift; vxlan_remote_ip=$1;; + --vxlan-local-ip) shift; vxlan_local_ip=$1;; + --vlan-id) shift; vlan_id=$1;; + --down) down=1;; + *) echo "Invalid argument $1"; exit 1;; + esac + shift + done + + iface_name=$(ep_common_if_name_get $pcie_addr) + if [[ -z $iface_name ]]; then + echo "Failed to get interface name for $pcie_addr" + exit + fi + + ep_common_cleanup_interfaces $iface_name + + if [[ -z $down ]]; then + if [[ -n $vlan_id ]]; then + nmcli dev set $iface_name managed no &> /dev/null || true + ifconfig $iface_name up + ifconfig $iface_name 0 + ip link add link $iface_name name $iface_name.v$vlan_id \ + type vlan id $vlan_id + nmcli dev set $iface_name.v$vlan_id managed no &> /dev/null || true + ip link set dev $iface_name.v$vlan_id up + ip addr add $ip_addr/24 dev $iface_name.v$vlan_id + elif [[ -n $vxlan_vni ]]; then + nmcli dev set $iface_name managed no &> /dev/null || true + ifconfig $iface_name up + ifconfig $iface_name $vxlan_local_ip/24 + ip link add $iface_name.vx$vxlan_vni \ + type vxlan id $vxlan_vni \ + remote $vxlan_remote_ip \ + local $vxlan_local_ip \ + dev $iface_name \ + dstport 4789 + nmcli dev set $iface_name.vx$vxlan_vni managed no &> /dev/null || true + ip link set dev $iface_name.vx$vxlan_vni up + ifconfig $iface_name.vx$vxlan_vni $ip_addr/24 + else + nmcli dev set $iface_name managed no &> /dev/null || true + ifconfig $iface_name up + ifconfig $iface_name $ip_addr/24 + fi + fi +} + +function ep_common_ip_forwarding() +{ + local op=$1 + + echo $op > /proc/sys/net/ipv4/ip_forward +} + +function ep_common_ping() +{ + local src=$1 + local dst=$2 + local count=${3:-32} + local ping_out + + ping_out=$(ping -c $count -i 0.2 -I $src $dst || true) + if [[ -n $(echo $ping_out | grep ", 0% packet loss,") ]]; then + echo "SUCCESS" + else + echo "FAILURE" + fi +} + +ep_common_cleanup_interfaces() +{ + local prefix=$1 + local ifcs=$(ifconfig | grep flags | grep "${prefix}.*:" | awk -F ':' '{print $1}') + + for ifc in $ifcs; do + ifconfig $ifc down + ip link del $ifc 2>/dev/null || true + done +} + +function ep_common_set_numvfs() +{ + local dev=$1 + local numvfs=$2 + + echo 0 > /sys/bus/pci/devices/$dev/sriov_numvfs + sleep 1 + echo $numvfs > /sys/bus/pci/devices/$dev/sriov_numvfs + sleep 1 +} + +function ep_common_unbind_driver() +{ + local s=$1 + local dev=$2 + + if [[ -e /sys/bus/$s/devices/$dev/driver/unbind ]]; then + echo $dev > /sys/bus/$s/devices/$dev/driver/unbind + sleep 1 + echo > /sys/bus/$s/devices/$dev/driver_override + sleep 1 + fi +} + +function ep_common_bind_driver() +{ + local s=$1 + local dev=$2 + local driver=$3 + + ep_common_unbind_driver $s $dev + echo $driver > /sys/bus/$s/devices/$dev/driver_override + echo $dev > /sys/bus/$s/drivers/$driver/bind + echo $dev > /sys/bus/$s/drivers_probe +} diff --git a/ci/test/dao-test/common/ep_device_utils.sh b/ci/test/dao-test/common/ep_device_utils.sh index 5a07fc2..653a803 100755 --- a/ci/test/dao-test/common/ep_device_utils.sh +++ b/ci/test/dao-test/common/ep_device_utils.sh @@ -2,10 +2,9 @@ # SPDX-License-Identifier: Marvell-MIT # Copyright (c) 2024 Marvell. -EP_UTILS_SCRIPT_PATH=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -source $EP_UTILS_SCRIPT_PATH/utils.sh - -find_executable "oxk-devbind-basic.sh" DEVBIND "$EP_UTILS_SCRIPT_PATH" +DEVICE_UTILS_SCRIPT_PATH=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) +source $DEVICE_UTILS_SCRIPT_PATH/utils.sh +source "$DEVICE_UTILS_SCRIPT_PATH/ep_common_ops.sh" PCI_VENDOR_ID_CAVIUM="0x177d" @@ -19,6 +18,11 @@ PCI_DEVID_CNXK_RVU_NPA_VF="0xa0fc" PCI_DEVID_CNXK_RVU_AF_VF="0xa0f8" PCI_DEVID_CN10K_RVU_CPT_PF="0xa0f2" PCI_DEVID_CN10K_RVU_CPT_VF="0xa0f3" +PCI_DEVID_CN10K_RVU_SDP_VF="0xa0f7" +PCI_DEVID_CN10K_RVU_DPI_PF="0xa080" +PCI_DEVID_CN10K_RVU_DPI_VF="0xa081" +PCI_DEVID_CN10K_RVU_ESW_PF="0xa0e0" + RVU_DEV_IDS=" $PCI_DEVID_CNXK_RVU_PF $PCI_DEVID_CNXK_RVU_VF @@ -33,38 +37,117 @@ $PCI_DEVID_CN10K_RVU_CPT_VF " CPUPARTNUM_106XX=0xd49 +PCI_DEV_PART_105XX=0xba +PCI_DEV_PART_106XX=0xb9 -function ep_device_vfio_bind() +VFIO_TOKEN="9d75f7af-606e-47ff-8ae4-f459fce4a422" + +function ep_device_eth_interfaces_get() { - $DEVBIND -b vfio-pci $@ + local ssh_ip=$1 + local req_ifcs=${2:-} + local num_ifcs=0 + local eth_ifcs_filtered="" + # Get the SSH IP and the eth interface it is connected to + local ssh_ifc_name=$(ip -f inet addr show | grep $ssh_ip -B 1 | head -n1 | \ + awk -F '[ :]' '{print $3}') + local ssh_ifc=$(cat /sys/class/net/$ssh_ifc_name/device/uevent | grep PCI_SLOT_NAME | \ + awk -F '=' '{print $2}') + local eth_ifcs=$(ep_common_pcie_addr_get $PCI_DEVID_CNXK_RVU_PF all) + + # Filter out the eth ports that are not connected to the SSH interface + for e in $eth_ifcs; do + if [[ "$e" != "$ssh_ifc" ]]; then + eth_ifcs_filtered="$eth_ifcs_filtered $e" + num_ifcs=$((num_ifcs + 1)) + if [[ -n $req_ifcs ]] && [[ $num_ifcs -eq $req_ifcs ]]; then + break + fi + fi + done + if [[ -n $req_ifcs ]] && [[ $num_ifcs -lt $req_ifcs ]]; then + echo "Not enough eth interfaces available" + exit 1 + fi + echo $eth_ifcs_filtered } -function ep_device_vfio_unbind() +function ep_device_sdp_setup() { - $DEVBIND -u vfio-pci $@ + local eth_ifcs="" + local sdp_pcie_pf + # This is the number of bridges/eth ports to set up. + # This is the number of SDP interfaces to bind per eth. + local num_sdp_ifcs_per_eth=2 + local cur_sdp_idx + local cur_eth_idx + local opts + + if ! opts=$(getopt \ + -l "num-sdp-ifcs-per-eth:,eth-ifc:" \ + -- sdp_setup ${@}); then + echo "Failed to parse arguments" + exit 1 + fi + + eval set -- "$opts" + while [[ $# -gt 1 ]]; do + case $1 in + --eth-ifc) shift; eth_ifcs="$eth_ifcs $1";; + --num-sdp-ifcs-per-eth) shift; num_sdp_ifcs_per_eth=$1;; + *) echo "Unknown argument $1"; exit 1;; + esac + shift + done + + # Setting up SDP device + sdp_pcie_pf=$(ep_common_pcie_addr_get $PCI_DEVID_CN10K_RVU_SDP_VF) + + cur_sdp_idx=1 + cur_eth_idx=1 + for eth_pf in $eth_ifcs; do + # Bind to vfio and then create VFs + ep_common_bind_driver pci $eth_pf vfio-pci + ep_common_set_numvfs $eth_pf $num_sdp_ifcs_per_eth + + for j in $(seq 1 $num_sdp_ifcs_per_eth); do + local eth_pcie_addr=$(get_vf_pcie_addr ${eth_pf} $j) + local sdp_pcie_addr=$(get_vf_pcie_addr ${sdp_pcie_pf} $cur_sdp_idx) + + ep_common_bind_driver pci $sdp_pcie_addr vfio-pci + echo "${sdp_pcie_addr},${eth_pcie_addr} " + sleep 1 + cur_sdp_idx=$((cur_sdp_idx + 1)) + done + done } -function ep_device_hugepage_setup() +function ep_device_esw_setup() { - echo "Setting up hugepages" - # Check for hugepages - if mount | grep hugetlbfs | grep none; then - echo "Hugepages already mounted" - else - echo "Mounting Hugepages" - mkdir -p /dev/huge - mount -t hugetlbfs none /dev/huge - fi - echo 24 > /proc/sys/vm/nr_hugepages - echo 6 >/sys/kernel/mm/hugepages/hugepages-524288kB/nr_hugepages - echo "Done setting up hugepages" + local num_esw_vfs=$1 + local esw_pf_pcie + local cur_esw_idx + local esw_vfs="" + + # Setting up ESW device + esw_pf_pcie=$(ep_common_pcie_addr_get $PCI_DEVID_CN10K_RVU_ESW_PF) + ep_common_bind_driver pci $esw_pf_pcie vfio-pci + ep_common_set_numvfs $esw_pf_pcie $num_esw_vfs + + cur_esw_idx=1 + for j in $(seq 1 $num_esw_vfs); do + local esw_pcie_addr=$(get_vf_pcie_addr ${esw_pf_pcie} $j) + esw_vfs="$esw_vfs $esw_pcie_addr" + cur_esw_idx=$((cur_esw_idx + 1)) + done + echo $esw_vfs } function ep_device_dpi_setup() { # Bind DPI devices - local dpi_pf=$(lspci -d :a080 | awk -e '{print $1}') - local dpi_vf + local dpi_pf=$(ep_common_pcie_addr_get $PCI_DEVID_CN10K_RVU_DPI_PF) + local dpi_vfs echo "Binding DPI devices" # Bind required DMA devices to vfio-pci @@ -73,19 +156,14 @@ function ep_device_dpi_setup() echo 512 > /sys/bus/pci/drivers/octeontx2-dpi/module/parameters/mrrs echo 256 > /sys/bus/pci/drivers/octeontx2-dpi/module/parameters/mps - echo $dpi_pf > /sys/bus/pci/devices/$dpi_pf/driver/unbind || true - echo octeontx2-dpi > /sys/bus/pci/devices/$dpi_pf/driver_override - echo $dpi_pf > /sys/bus/pci/drivers_probe + ep_common_bind_driver pci $dpi_pf octeontx2-dpi + ep_common_set_numvfs $dpi_pf 32 - echo 32 >/sys/bus/pci/devices/$dpi_pf/sriov_numvfs - dpi_vf=$(lspci -d :a081 | awk -e '{print $1}' | head -22) - ep_device_vfio_bind $dpi_vf + dpi_vfs=$(ep_common_pcie_addr_get $PCI_DEVID_CN10K_RVU_DPI_VF 22) + for v in $dpi_vfs; do + ep_common_bind_driver pci $v vfio-pci + done echo "Done Binding DPI devices" - - # Bind required RPM VF's to vfio-pci - echo "Binding RPM devices" - ep_device_vfio_bind 0002:01:00.2 0002:01:00.1 - echo "Done Binding RPM devices" } function ep_device_pem_setup() @@ -103,44 +181,13 @@ function ep_device_pem_setup() local dev_name=$(basename "$dev_path") # Bind the device to vfio-platform driver - echo "vfio-platform" | tee "$dev_path/driver_override" || true - echo "$dev_name" | tee "/sys/bus/platform/drivers/vfio-platform/bind" || true - + ep_common_bind_driver platform $dev_name vfio-platform echo "Device $dev_name configured." fi done echo "Done binding PEM/SDP regs devices" } -function ep_device_fw_cleanup() -{ - set +e - ps -ef | grep dao-virtio-l2fwd | grep fw_launch | awk '{print $2}' | head -n1 | xargs -n1 kill -9 - set -e -} - -function ep_device_fw_launch() -{ - local dpi_vf - local cmd - - ep_device_fw_cleanup - - dpi_vf=$(lspci -d :a081 | awk -e '{print $1}' | head -22) - - # Launch EP firmware application - echo "Launching EP Firwmare App" - local virtio_l2fwd - find_executable "dao-virtio-l2fwd" virtio_l2fwd "$EP_UTILS_SCRIPT_PATH/../../../../app" - local dpi_allow="" - for d in $dpi_vf; do - dpi_allow="$dpi_allow -a $d" - done - cmd="$virtio_l2fwd --file-prefix fw_launch -l 4-6 -a 0002:01:00.2 $dpi_allow -- -p 0x1 -v 0x1 -P" - echo $cmd - $cmd -} - function ep_device_get_cpu_partnum() { local partnum=$(grep -m 1 'CPU part' /proc/cpuinfo | awk -F': ' '{print $2}') @@ -223,9 +270,44 @@ function ep_device_get_sclk() echo $sclk } +function ep_device_agent_cleanup() +{ + set +e + pkill -9 octep_cp_agent + rmmod pcie_marvell_cnxk_ep + set -e +} + +function ep_device_agent_init() +{ + local mod=$(lsmod | grep pcie_marvell_cnxk_ep) + local ep_agent=$(pidof octep_cp_agent) + local ep_bin_dir=$EP_DIR/ep_files + local part=$(ep_device_get_part) + local agent_conf + + if [[ 0x${part} == ${PCI_DEV_PART_106XX} ]]; then + agent_conf=$ep_bin_dir/cn106xx.cfg + elif [[ 0x${part} == 0x${PCI_DEV_PART_105XX} ]]; then + agent_conf=$ep_bin_dir/cnf105xx.cfg + else + echo "Unknown part $part" + exit 1 + fi + + if [[ $mod == "" ]]; then + insmod $ep_bin_dir/pcie-marvell-cnxk-ep.ko + fi + + if [[ $ep_agent == "" ]]; then + $ep_bin_dir/octep_cp_agent \ + $agent_conf 2>&1 > $ep_bin_dir/octep_cp_agent.log & + fi +} + function ep_device_get_inactive_if() { - for i in $(lspci -n -d :$PCI_DEVID_CNXK_RVU_PF | awk -e '{print $1}'); do + for i in $(ep_common_pcie_addr_get $PCI_DEVID_CNXK_RVU_PF all); do local ethname= local active= if [ -d "/sys/bus/pci/devices/$i/net" ]; then @@ -243,6 +325,12 @@ function ep_device_get_inactive_if() done } +function ep_device_get_num_cores() +{ + local num_cores=$(lscpu | grep "On-line CPU(s) list" | awk -F '-' '{print $3}') + echo $(($num_cores + 1)) +} + # If this script is directly invoked from the shell execute the # op specified if [[ ${BASH_SOURCE[0]} == ${0} ]]; then @@ -250,6 +338,8 @@ if [[ ${BASH_SOURCE[0]} == ${0} ]]; then ARGS=${@:2} if [[ $(type -t ep_device_$OP) == function ]]; then ep_device_$OP $ARGS + elif [[ $(type -t ep_common_$OP) == function ]]; then + ep_common_$OP $ARGS else $OP $ARGS fi diff --git a/ci/test/dao-test/common/ep_host_utils.sh b/ci/test/dao-test/common/ep_host_utils.sh index 574d8cf..30ada09 100755 --- a/ci/test/dao-test/common/ep_host_utils.sh +++ b/ci/test/dao-test/common/ep_host_utils.sh @@ -3,20 +3,18 @@ # Copyright (c) 2024 Marvell. HOST_UTILS_SCRIPT_PATH=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) -source "$HOST_UTILS_SCRIPT_PATH/testpmd.sh" +source "$HOST_UTILS_SCRIPT_PATH/ep_common_ops.sh" -function ep_host_hugepage_setup() +function ep_host_sdp_setup() { - echo "Setting up hugepages on host" - # Check for hugepages - if mount | grep hugetlbfs | grep none; then - echo "Hugepages already setup" + set +e # Module may be already loaded + if [[ -n ${EP_HOST_MODULE_DIR:-} ]]; then + insmod $EP_HOST_MODULE_DIR/octeon_ep.ko else - mkdir /dev/huge - mount -t hugetlbfs none /dev/huge + insmod $EP_DIR/ep_files/octeon_ep.ko fi - echo 24 > /proc/sys/vm/nr_hugepages - echo 2048 >/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages + set -e + sleep 5 } function ep_host_vdpa_setup() @@ -32,18 +30,20 @@ function ep_host_vdpa_setup() modprobe vhost-vdpa set +e # Module may be already loaded rmmod octep_vdpa - insmod $EP_HOST_DIR/kmod/vdpa/octeon_ep/octep_vdpa.ko + if [[ -n ${EP_HOST_MODULE_DIR:-} ]]; then + insmod $EP_HOST_MODULE_DIR/octep_vdpa.ko + else + insmod $EP_DIR/kmod/vdpa/octeon_ep/octep_vdpa.ko + fi set -e - host_pf=$(lspci -Dn -d :${part}00 | head -1 | cut -f 1 -d " ") + host_pf=$(ep_common_pcie_addr_get ${part}00) vf_cnt=1 vf_cnt_max=$(cat /sys/bus/pci/devices/$host_pf/sriov_totalvfs) vf_cnt=$((vf_cnt >vf_cnt_max ? vf_cnt_max : vf_cnt)) - echo $host_pf > /sys/bus/pci/devices/$host_pf/driver/unbind || true - echo octep_vdpa > /sys/bus/pci/devices/$host_pf/driver_override - echo $host_pf > /sys/bus/pci/drivers_probe - echo $vf_cnt > /sys/bus/pci/devices/$host_pf/sriov_numvfs + ep_common_bind_driver pci $host_pf octep_vdpa + ep_common_set_numvfs $host_pf $vf_cnt sleep 1 # Get the list of management devices @@ -55,7 +55,7 @@ function ep_host_vdpa_setup() done set +x - sdp_vfs=$(lspci -Dn -d :${part}03 | cut -f 1 -d " ") + sdp_vfs=$(ep_common_pcie_addr_get ${part}03) for dev in $sdp_vfs; do set +e # Grep can return non-zero status vdev=$(ls /sys/bus/pci/devices/$dev | grep vdpa) @@ -66,12 +66,8 @@ function ep_host_vdpa_setup() done set -e - set +e # virtio vdpa driver may not be present on host - echo $vdev > /sys/bus/vdpa/drivers/virtio_vdpa/unbind - set -e - echo "Binding $vdev to vhost_vdpa" - echo $vdev > /sys/bus/vdpa/drivers/vhost_vdpa/bind || true + ep_common_bind_driver vdpa $vdev vhost_vdpa done } @@ -85,40 +81,9 @@ function ep_host_vdpa_cleanup() set -e } -function ep_host_testpmd_launch() -{ - local pfx=$1 - local args=${@:2} - local eal_args - local app_args="" - - for a in $args; do - if [[ $a == "--" ]]; then - eal_args=$app_args - app_args="" - continue - fi - app_args+=" $a" - done - - echo "Launching testpmd on Host" - testpmd_launch $pfx "$eal_args" "$app_args" - echo "Launched testpmd on Host" -} - -function ep_host_testpmd_stop() -{ - local pfx=$1 - - echo "Stopping testpmd on Host" - testpmd_quit $pfx - testpmd_cleanup $pfx - echo "Stopped testpmd on Host" -} - EP_SCP_CMD=${EP_SCP_CMD:-"scp -o LogLevel=ERROR -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"} EP_GUEST_DIR="/root/hostshare" -EP_GUEST_SHARE_DIR=$EP_HOST_DIR/guest +EP_GUEST_SHARE_DIR=$EP_DIR/guest function ep_host_launch_guest() { @@ -127,8 +92,8 @@ function ep_host_launch_guest() local in=guest.in.$pfx local out=guest.out.$pfx - $EP_SCP_CMD ci@10.28.34.13:/home/ci/dao_host/qemu-system-x86_64 $EP_HOST_DIR/ - $EP_SCP_CMD ci@10.28.34.13:/home/ci/dao_host/noble-server-cloudimg-amd64.img $EP_HOST_DIR/ + $EP_SCP_CMD ci@10.28.34.13:/home/ci/dao_host/qemu-system-x86_64 $EP_DIR/ + $EP_SCP_CMD ci@10.28.34.13:/home/ci/dao_host/noble-server-cloudimg-amd64.img $EP_DIR/ $EP_SCP_CMD ci@10.28.34.13:/home/ci/dao_host/bios-256k.bin /usr/share/qemu $EP_SCP_CMD ci@10.28.34.13:/home/ci/dao_host/vgabios-stdvga.bin /usr/share/qemu $EP_SCP_CMD ci@10.28.34.13:/home/ci/dao_host/efi-virtio.rom /usr/share/qemu @@ -137,19 +102,19 @@ function ep_host_launch_guest() rm -rf $EP_GUEST_SHARE_DIR mkdir $EP_GUEST_SHARE_DIR - cp $EP_HOST_DIR/ci/test/dao-test/common/utils.sh $EP_GUEST_SHARE_DIR - cp $EP_HOST_DIR/ci/test/dao-test/common/testpmd.sh $EP_GUEST_SHARE_DIR - cp $EP_HOST_DIR/ci/test/dao-test/common/ep_guest_utils.sh $EP_GUEST_SHARE_DIR + cp $EP_DIR/ci/test/dao-test/common/utils.sh $EP_GUEST_SHARE_DIR + cp $EP_DIR/ci/test/dao-test/common/testpmd.sh $EP_GUEST_SHARE_DIR + cp $EP_DIR/ci/test/dao-test/common/ep_guest_utils.sh $EP_GUEST_SHARE_DIR - if [[ -f $EP_HOST_DIR/qemu-system-x86_64 ]]; then - QEMU_BIN=$EP_HOST_DIR/qemu-system-x86_64 + if [[ -f $EP_DIR/qemu-system-x86_64 ]]; then + QEMU_BIN=$EP_DIR/qemu-system-x86_64 else echo "qemu-system-x86_64 not found !!" return 1 fi - if [[ -f $EP_HOST_DIR/noble-server-cloudimg-amd64.img ]]; then - VM_IMAGE=$EP_HOST_DIR/noble-server-cloudimg-amd64.img + if [[ -f $EP_DIR/noble-server-cloudimg-amd64.img ]]; then + VM_IMAGE=$EP_DIR/noble-server-cloudimg-amd64.img else echo "x86 QEMU cloud image not found !!" return 1 @@ -289,10 +254,14 @@ function ep_host_shutdown_guest() # If this script is directly invoked from the shell execute the # op specified if [[ ${BASH_SOURCE[0]} == ${0} ]]; then + source "$HOST_UTILS_SCRIPT_PATH/testpmd.sh" + OP=$1 ARGS=${@:2} if [[ $(type -t ep_host_$OP) == function ]]; then ep_host_$OP $ARGS + elif [[ $(type -t ep_common_$OP) == function ]]; then + ep_common_$OP $ARGS else $OP $ARGS fi diff --git a/ci/test/dao-test/common/oxk-devbind-basic.sh b/ci/test/dao-test/common/oxk-devbind-basic.sh deleted file mode 100755 index 4bc3b59..0000000 --- a/ci/test/dao-test/common/oxk-devbind-basic.sh +++ /dev/null @@ -1,61 +0,0 @@ -#! /bin/sh -# SPDX-License-Identifier: Marvell-MIT -# Copyright (c) 2024 Marvell. - -HELP="usage: oxk-devbind-basic.sh [OPTIONS] devices.. -Script for binding/unbinding devices from Linux kernel drivers. -NOTE: Options -b and -u are exclusive. - OPTIONS: - -b driver - Bind given devices to a given driver - -u - Unbind devices from their driver - devices: Space separated List of DBDF addresses (i.e. 0001:02:00.1)" - -OPTS=$(getopt -u -n $0 -o "b:uh" -- $@) - -driver="" -unbind=0 - -eval set -- "$OPTS" - -while true; do - case "$1" in - -h) echo $HELP; exit 0 ;; - -b) driver=$2; shift 2 ;; - -u) unbind=1; shift ;; - --) shift; break ;; - *) echo "Unknown arguments"; echo $HELP; exit 1 ;; - esac -done - -if [ "x$driver" != "x" -a $unbind -eq 1 ]; then - echo "Cannot have -b and -u" - echo "$HELP" - exit 1 -fi - -if [ "x$driver" = "x" -a $unbind -eq 0 ]; then - echo "Please specify either -b or -u" - echo "$HELP" - exit 1 -fi - -for dbdf in $@; do - ddir="/sys/bus/pci/devices/$dbdf" - cur_drv="$(readlink -n $ddir/driver)" - if [ ! -z "$cur_drv" ]; then - cur_drv="$(basename $cur_drv)" - fi - # If user wants to bind and same driver is bound, skip the device - if [ $unbind -eq 0 -a "x$driver" = "x$cur_drv" ]; then - continue - fi - # Either user wanted to unbind or we have to unbind for re-binding - if [ -e $ddir/driver/unbind ]; then - echo $dbdf > "$ddir/driver/unbind" - fi - # If user specified -b then do try to bind - if [ "x$driver" != "x" ]; then - echo $driver > "$ddir/driver_override" - echo $dbdf > /sys/bus/pci/drivers_probe - fi -done diff --git a/ci/test/dao-test/common/testpmd.sh b/ci/test/dao-test/common/testpmd.sh index 02a0c9b..d2e93bb 100644 --- a/ci/test/dao-test/common/testpmd.sh +++ b/ci/test/dao-test/common/testpmd.sh @@ -5,8 +5,6 @@ TESTPMD_SCRIPT_PATH=$(dirname $(readlink -f "${BASH_SOURCE[0]}")) source $TESTPMD_SCRIPT_PATH/utils.sh -find_executable "dpdk-testpmd" TESTPMD "${DEPS_PREFIX:-}/bin" - function testpmd_cleanup() { local pfx=$1 @@ -65,13 +63,17 @@ function testpmd_launch() local out=testpmd.out.$pfx local in=testpmd.in.$pfx local unbuffer="stdbuf -o0" + local testpmd testpmd_cleanup $pfx rm -f $out rm -f $in touch $in + + find_executable "dpdk-testpmd" testpmd "${EP_DIR:-}/deps-prefix/bin" + tail -f $in | \ - ($unbuffer $TESTPMD $eal_args --file-prefix $pfx -- \ + ($unbuffer $testpmd $eal_args --file-prefix $pfx -- \ $testpmd_args -i &>$out) & # Wait till out file is created while [[ ! -f $out ]]; do diff --git a/ci/test/dao-test/common/utils.sh b/ci/test/dao-test/common/utils.sh index 9324f97..1646112 100644 --- a/ci/test/dao-test/common/utils.sh +++ b/ci/test/dao-test/common/utils.sh @@ -23,10 +23,10 @@ function find_executable() set -e if [[ $bin != "" ]]; then eval "$execvar"="$bin" - return + return 0 fi echo "Cannot find $execname" - exit 1 + return 1 } function ep_ssh_cmd() @@ -56,12 +56,21 @@ function ep_host_ssh_cmd() ep_ssh_cmd EP_HOST false 0 "$cmd" } +function ep_host_ssh_cmd_bg() +{ + local wait=$1 + local cmd=$2 + + ep_ssh_cmd EP_HOST true $wait "$cmd 2>&1 &" +} + function ep_host_op() { local op=$1 local args=${@:2} + local env="EP_HOST_MODULE_DIR=$EP_HOST_MODULE_DIR $EP_HOST=$EP_HOST EP_DIR=$EP_DIR" - ep_host_ssh_cmd "sudo EP_HOST_DIR=$EP_HOST_DIR $EP_HOST_DIR/ci/test/dao-test/common/ep_host_utils.sh $op $args" + ep_host_ssh_cmd "$EP_HOST_SUDO $env $EP_DIR/ci/test/dao-test/common/ep_host_utils.sh $op $args" } function ep_host_op_bg() @@ -69,8 +78,41 @@ function ep_host_op_bg() local wait=$1 local op=$2 local args=${@:3} + local env="EP_HOST_MODULE_DIR=$EP_HOST_MODULE_DIR $EP_HOST=$EP_HOST EP_DIR=$EP_DIR" + + ep_ssh_cmd EP_HOST true $wait "$EP_HOST_SUDO $env nohup $EP_DIR/ci/test/dao-test/common/ep_host_utils.sh $op $args 2>&1 &" +} + +function ep_remote_ssh_cmd() +{ + local cmd=$1 + + ep_ssh_cmd EP_REMOTE false 0 "$cmd" +} + +function ep_remote_ssh_cmd_bg() +{ + local wait=$1 + local cmd=$2 - ep_ssh_cmd EP_HOST true $wait "sudo EP_HOST_DIR=$EP_HOST_DIR nohup $EP_HOST_DIR/ci/test/dao-test/common/ep_host_utils.sh $op $args 2>&1 &" + ep_ssh_cmd EP_REMOTE true $wait "$cmd 2>&1 &" +} + +function ep_remote_op() +{ + local op=$1 + local args=${@:2} + + ep_remote_ssh_cmd "$EP_REMOTE_SUDO EP_DEVICE=$EP_REMOTE EP_DIR=$EP_DIR $EP_DIR/ci/test/dao-test/common/ep_device_utils.sh $op $args" +} + +function ep_remote_op_bg() +{ + local wait=$1 + local op=$2 + local args=${@:3} + + ep_ssh_cmd EP_REMOTE true $wait "$EP_REMOTE_SUDO EP_DEVICE=$EP_REMOTE EP_DIR=$EP_DIR nohup $EP_DIR/ci/test/dao-test/common/ep_device_utils.sh $op $args 2>&1 &" } function ep_device_ssh_cmd() @@ -80,12 +122,20 @@ function ep_device_ssh_cmd() ep_ssh_cmd EP_DEVICE false 0 "$cmd" } +function ep_device_ssh_cmd_bg() +{ + local wait=$1 + local cmd=$2 + + ep_ssh_cmd EP_DEVICE true $wait "$cmd 2>&1 &" +} + function ep_device_op() { local op=$1 local args=${@:2} - ep_device_ssh_cmd "sudo EP_DEVICE_DIR=$EP_DEVICE_DIR $EP_DEVICE_DIR/ci/test/dao-test/common/ep_device_utils.sh $op $args" + ep_device_ssh_cmd "$EP_DEVICE_SUDO EP_DEVICE=$EP_DEVICE EP_DIR=$EP_DIR $EP_DIR/ci/test/dao-test/common/ep_device_utils.sh $op $args" } function ep_device_op_bg() @@ -94,7 +144,7 @@ function ep_device_op_bg() local op=$2 local args=${@:3} - ep_ssh_cmd EP_DEVICE true $wait "sudo EP_DEVICE_DIR=$EP_DEVICE_DIR nohup $EP_DEVICE_DIR/ci/test/dao-test/common/ep_device_utils.sh $op $args 2>&1 &" + ep_ssh_cmd EP_DEVICE true $wait "$EP_DEVICE_SUDO EP_DEVICE=$EP_DEVICE EP_DIR=$EP_DIR nohup $EP_DIR/ci/test/dao-test/common/ep_device_utils.sh $op $args 2>&1 &" } function test_run() @@ -151,6 +201,8 @@ function safe_kill() local pattern=$@ local killpids local ptree=$(get_process_tree) + local maxwait=5 + local elapsed=0 # Safely kill all processes which has the pattern in 'ps -ef' but # make sure that the current process tree is not affected. @@ -158,10 +210,17 @@ function safe_kill() killpids=$(ps -ef | grep "$pattern" | awk '{print $2}') for p in $killpids; do if ! $(echo $ptree | grep -qw $p); then - kill -9 $p > /dev/null 2>&1 + elapsed=0 + while $(kill -s TERM $p > /dev/null 2>&1); do + sleep 1 + elapsed=$((elapsed + 1)) + if [[ $elapsed == $maxwait ]]; then + kill -9 $p > /dev/null 2>&1 + break + fi + done fi done - set -e } function file_offset() @@ -180,3 +239,43 @@ function file_search_pattern() tail -c +$skip_bytes $logfile | grep -q "$pattern" } + +function get_vf_pcie_addr() +{ + local pcie_addr=$1 + local vf_idx=$2 + local base + local device + local func + + base=$(echo $pcie_addr | awk -F ':' '{print $1":"$2":"}') + device=$(printf "%02x" $((vf_idx / 8))) + func=$((vf_idx % 8)) + echo "${base}${device}.${func}" +} + +function form_split_args() +{ + local param=$1 + local values=${@:2} + local args="" + + for v in $values; do + args+="${param} $v " + done + + echo $args +} + +function pkill_with_wait() +{ + local proc=$1 + + set +e + pkill -9 $proc + set -e + + while (ps -ef | grep -v grep | grep $proc &> /dev/null); do + sleep 1 + done +} diff --git a/ci/test/dao-test/meson.build b/ci/test/dao-test/meson.build index 83f3228..b3742af 100644 --- a/ci/test/dao-test/meson.build +++ b/ci/test/dao-test/meson.build @@ -3,3 +3,4 @@ subdir('common') subdir('virtio') +subdir('ovs') diff --git a/ci/test/dao-test/ovs/meson.build b/ci/test/dao-test/ovs/meson.build new file mode 100644 index 0000000..0a2cdec --- /dev/null +++ b/ci/test/dao-test/ovs/meson.build @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Marvell-MIT +# Copyright (c) 2024 Marvell. + +# Test script +tests = [ + # Name, Script, Args + ['ovs_plain_ping', 'ovs_ping.sh', ''], + ['ovs_vlan_ping', 'ovs_ping.sh', ''], + ['ovs_vxlan_ping', 'ovs_ping.sh', ''], + ['ovs_plain_ping_hw_offload', 'ovs_ping.sh', ''], + ['ovs_vlan_ping_hw_offload', 'ovs_ping.sh', ''], + ['ovs_vxlan_ping_hw_offload', 'ovs_ping.sh', ''], +] + +test_dir = meson.current_build_dir() +foreach t : tests + test_name = t[0] + test_script = t[1] + test_args = t[2] + test(test_name, + dao_test_script_wrapper, + env : [['DAO_TEST=' + test_name], ['TEST_DIR=' + test_dir]], + args : [[test_script], [test_args]], + is_parallel : false, suite : 'dao-ovs') +endforeach diff --git a/ci/test/dao-test/ovs/ovs_ping.sh b/ci/test/dao-test/ovs/ovs_ping.sh new file mode 100755 index 0000000..1abc5ed --- /dev/null +++ b/ci/test/dao-test/ovs/ovs_ping.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# SPDX-License-Identifier: Marvell-MIT +# Copyright (c) 2024 Marvell. + +set -euo pipefail + +OVS_PING_SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +source $OVS_PING_SCRIPT_PATH/ovs_utils.sh + +function ovs_ping() +{ + local test_type=$1 + local hw_offload=$2 + local eth_pf_ifcs + local sdp_eth_vf_pairs + local esw_vf_ifcs + local num_eth_ifcs=1 + local num_sdp_ifcs_per_eth=2 + local num_esw_ifcs=1 + local ovs_debug=1 + local ssh_ip=$(echo $EP_DEVICE | awk -F '\@' '{print $2}' 2>/dev/null) + local host_ip="20.0.0.10" + local remote_ip="20.0.0.20" + local extra_args_interface_setup= + local extra_args_remote_ifconfig= + local vxlan_local_ip="30.0.0.2" + local vxlan_remote_ip="30.0.0.254" + local vxlan_subnet="30.0.0.0" + local vxlan_vni=5001 + local vlan_id=100 + + if [[ $test_type == "vlan" ]]; then + extra_args_interface_setup="--vlan-id $vlan_id" + extra_args_remote_ifconfig="--vlan-id $vlan_id" + elif [[ $test_type == "vxlan" ]]; then + extra_args_interface_setup="--vxlan-vni $vxlan_vni --vxlan-subnet $vxlan_subnet" + extra_args_remote_ifconfig="--vxlan-vni $vxlan_vni \ + --vxlan-remote-ip $vxlan_remote_ip --vxlan-local-ip $vxlan_local_ip" + fi + + # Register signal handler + ovs_register_sig_handler + + # Get eth interfaces on device + eth_pf_ifcs=$(ep_device_eth_interfaces_get $ssh_ip $num_eth_ifcs) + + echo "Setting up SDP" + sdp_eth_vf_pairs=$(ep_device_sdp_setup \ + $(form_split_args "--eth-ifc" "$eth_pf_ifcs") \ + --num-sdp-ifcs-per-eth $num_sdp_ifcs_per_eth) + + echo "Setting up ESW" + esw_vf_ifcs=$(ep_device_esw_setup $num_esw_ifcs) + + echo "Launching OVS" + ovs_launch \ + $(form_split_args "--eth-ifc" $eth_pf_ifcs) \ + --hw-offload $hw_offload \ + --debug $ovs_debug + + echo "Setting up OVS interfaces" + ovs_interface_setup \ + $(form_split_args "--eth-ifc" "$eth_pf_ifcs") \ + --num-sdp-ifcs-per-eth $num_sdp_ifcs_per_eth \ + $extra_args_interface_setup + + echo "Running OVS offload" + ovs_offload_launch \ + $(form_split_args "--esw-vf-ifc" "$esw_vf_ifcs") \ + $(form_split_args "--sdp-eth-vf-pair" "$sdp_eth_vf_pairs") + + echo "Configure SDP interface on host" + ep_host_op if_configure --pcie-addr $EP_HOST_SDP_IFACE --ip $host_ip + + echo "Configure remote interface" + ep_remote_op bind_driver pci $EP_REMOTE_IFACE rvu_nicpf + ep_remote_op if_configure --pcie-addr $EP_REMOTE_IFACE \ + --ip $remote_ip $extra_args_remote_ifconfig + + if [[ $(ep_host_op ping $host_ip $remote_ip) != "SUCCESS" ]]; then + echo "Ping Failed" + exit 1 + else + echo "Ping Passed" + fi +} + +function ovs_plain_ping() +{ + ovs_ping plain false +} + +function ovs_vlan_ping() +{ + ovs_ping vlan false +} + +function ovs_vxlan_ping() +{ + ovs_ping vxlan false +} + +function ovs_plain_ping_hw_offload() +{ + ovs_ping plain true +} + +function ovs_vlan_ping_hw_offload() +{ + ovs_ping vlan true +} + +function ovs_vxlan_ping_hw_offload() +{ + ovs_ping vxlan true +} + +test_run ${DAO_TEST} 2 diff --git a/ci/test/dao-test/ovs/ovs_utils.sh b/ci/test/dao-test/ovs/ovs_utils.sh new file mode 100755 index 0000000..e92d5d5 --- /dev/null +++ b/ci/test/dao-test/ovs/ovs_utils.sh @@ -0,0 +1,399 @@ +#!/bin/bash +# SPDX-License-Identifier: Marvell-MIT +# Copyright (c) 2024 Marvell. +set -e + +OVS_UTILS_SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +source $OVS_UTILS_SCRIPT_PATH/../common/utils.sh +source $OVS_UTILS_SCRIPT_PATH/../common/ep_host_utils.sh +source $OVS_UTILS_SCRIPT_PATH/../common/ep_device_utils.sh + +PATH=$PATH:$EP_DEVICE_OVS_PATH/sbin:$EP_DEVICE_OVS_PATH/bin +export PATH=$PATH:$EP_DEVICE_OVS_PATH/share/openvswitch/scripts + +function ovs_cleanup() { + local log=$EP_DEVICE_OVS_PATH/var/log/ovs-vswitchd.log + + set +e + ovs_bridge_del_all + ovs-ctl stop + sleep 2 + + pkill_with_wait ovsdb-server + pkill_with_wait ovs-vswitchd + set -e + + if [[ -e $log ]]; then + echo "=========== OVS LOG =============" + cat $log + fi +} + +function ovs_launch() +{ + local hw_offload=false + local debug + local allow="" + local sock="$EP_DEVICE_OVS_PATH/var/run/openvswitch/db.sock" + local conf="$EP_DEVICE_OVS_PATH/etc/openvswitch/conf.db" + local schema="$EP_DEVICE_OVS_PATH/share/openvswitch/vswitch.ovsschema" + local log="$EP_DEVICE_OVS_PATH/var/log/ovs-vswitchd.log" + + if ! opts=$(getopt \ + -l "hw-offload:,eth-ifc:,debug:" \ + -- ovs_launch $@); then + echo "Failed to parse arguments" + exit 1 + fi + + eval set -- "$opts" + while [[ $# -gt 1 ]]; do + case $1 in + --hw-offload) shift; hw_offload=$1;; + --eth-ifc) shift; allow="$allow --allow=$1";; + --debug) shift; debug=1;; + *) echo "Unknown argument $1"; exit 1;; + esac + shift + done + + mkdir -p $EP_DEVICE_OVS_PATH/var/run/openvswitch + mkdir -p $EP_DEVICE_OVS_PATH/var/log/openvswitch + mkdir -p $EP_DEVICE_OVS_PATH/share/openvswitch + mkdir -p $EP_DEVICE_OVS_PATH/etc/openvswitch + + # Init OVS + ovsdb-tool create $conf $schema + + ovsdb-server \ + --remote=punix:"$sock" \ + --remote=db:Open_vSwitch,Open_vSwitch,manager_options \ + --pidfile \ + --detach + + echo "Starting OVS" + ovs-ctl start \ + --db-sock="$sock" \ + --db-file="$conf" \ + --db-schema="$schema" \ + --no-ovs-vswitchd + + ovs-vsctl --no-wait init + + ovs-vsctl \ + --no-wait \ + set Open_vSwitch . other_config:dpdk-init=true \ + other_config:dpdk-socket-mem="1024" other_config:hw-offload=$hw_offload \ + other_config:dpdk-extra="--vfio-vf-token=\"$VFIO_TOKEN\" $allow" + + ovs-vswitchd \ + unix:"${sock}" \ + --pidfile \ + --detach \ + --log-file=$log + + # Raise log level + if [[ $debug -eq "1" ]]; then + ovs-appctl vlog/set netdev_dpdk:file:dbg + ovs-appctl vlog/set netdev_offload_dpdk:file:dbg + ovs-appctl vlog/set netdev_dpdk:console:info + fi + + ovs-vsctl show +} + +function ovs_interface_setup() +{ + local eth_ifcs="" + local eth_ifc_idx + local eth_ifc_max + local esw_pf_pcie + local opts + local vxlan_vni= + local vxlan_subnet= + local net + local ipaddr + local vlan_id= + + if ! opts=$(getopt \ + -l "eth-ifc:,num-sdp-ifcs-per-eth:,vxlan-vni:,vxlan-subnet:,vlan-id:" \ + -- interface_setup $@); then + echo "Failed to parse arguments" + exit 1 + fi + + eval set -- "$opts" + while [[ $# -gt 1 ]]; do + case $1 in + --eth-ifc) shift; eth_ifcs="$eth_ifcs $1";; + --vxlan-vni) shift; vxlan_vni=$1;; + --vxlan-subnet) shift; vxlan_subnet=$1;; + --vlan-id) shift; vlan_id=$1;; + --num-sdp-ifcs-per-eth) shift; num_sdp_ifcs_per_eth=$1;; + *) echo "Unknown argument $1"; exit 1;; + esac + shift + done + + esw_pf_pcie=$(ep_common_pcie_addr_get $PCI_DEVID_CN10K_RVU_ESW_PF) + eth_ifc_idx=0 + for eth_ifc in $eth_ifcs; do + ovs_bridge_add br${eth_ifc_idx} "netdev" + sleep 1 + + for sdp_idx in $(seq 0 $((num_sdp_ifcs_per_eth - 1))); do + local pfid=$(echo $eth_ifc | awk -F ":" '{print $2}') + + pfid=$((pfid - 1)) + ovs_port_add \ + "br${eth_ifc_idx}" \ + "e${eth_ifc_idx}_vf_rep${sdp_idx}" \ + $esw_pf_pcie \ + "representor=pf${pfid}vf${sdp_idx}" + done + + if [[ -n $vxlan_vni ]]; then + local vni=$((vxlan_vni + eth_ifc_idx)) + net=$((eth_ifc_idx + 2)) + ipaddr=${vxlan_subnet%.*}.$net + ovs_vxlan_port_add "br${eth_ifc_idx}" "vxlan${eth_ifc_idx}" "$ipaddr" $vni + else + ovs_port_add "br${eth_ifc_idx}" "e${eth_ifc_idx}_pf" $eth_ifc + fi + + sleep 1 + + # Bring the bridge up + ovs_iface_link_set "br${eth_ifc_idx}" "up" + eth_ifc_idx=$((eth_ifc_idx + 1)) + done + + eth_ifc_max=$eth_ifc_idx + if [[ -n $vxlan_vni ]]; then + net=254 + eth_ifc_idx=0 + for eth_ifc in $eth_ifcs; do + local k=$((eth_ifc_max + eth_ifc_idx)) + local khex=$(printf "%x" $k) + + br_args="-- br-set-external-id br${k} bridge-id br${k} \ + -- set bridge br${k} fail-mode=standalone \ + other_config:hwaddr=00:00:00:aa:bb:$khex" + + ovs_bridge_add br${k} "netdev" "$br_args" + ovs_port_add "br${k}" "e${eth_ifc_idx}_pf" $eth_ifc + + # Bring the bridge up + ovs_iface_link_set "br${k}" "up" + + ipaddr=${vxlan_subnet%.*}.$net + ip addr add $ipaddr/24 dev "br${k}" + ovs-appctl ovs/route/add $ipaddr/24 "br${k}" + net=$((net - 1)) + eth_ifc_idx=$((eth_ifc_idx + 1)) + done + fi + + echo "List of bridges" + ovs-vsctl list-br + ovs-vsctl show + if [[ -n $vlan_id ]]; then + ovs-vsctl set port e0_vf_rep0 tag=$vlan_id + fi +} + +function ovs_bridge_add() +{ + local br_name=$1 + local dp_type=$2 + local br_args=${3:-} + + ovs-vsctl add-br $br_name -- set bridge $br_name datapath_type=$dp_type $br_args +} + +function ovs_bridge_del() +{ + local br_name=$1 + + timeout 3 ovs-vsctl del-br $br_name +} + +function ovs_bridge_del_all() +{ + local bridges=$(ovs-vsctl list-br) + + if [ $? -eq 0 ]; then + for br in $bridges; do + ovs_bridge_del $br + done + fi +} + +function ovs_port_add() +{ + local br_name=$1 + local port=$2 + local pcie_addr=$3 + local rep_args=${4:-} + + if [[ -n $rep_args ]]; then + rep_args=",$rep_args" + fi + ovs-vsctl add-port $br_name $port -- set Interface $port type=dpdk \ + options:dpdk-devargs="$pcie_addr"$rep_args +} + +function ovs_vxlan_port_add() +{ + local br_name=$1 + local port=$2 + local raddr=$3 + local vni=$4 + + ovs-vsctl add-port $br_name $port -- set Interface $port type=vxlan \ + options:remote_ip=$raddr options:key=$vni +} + +function ovs_port_del() +{ + local br_name=$1 + local port=$2 + + ovs-vsctl del-port --if-exists $br_name $port +} + +function ovs_vlan_tag_add() +{ + local iface=$1 + local vtag=$2 + + ovs-vsctl set port $iface tag=$vtag +} + +function ovs_iface_link_set() +{ + local iface=$1 + local action=$2 + + ip link set dev $iface $action +} + +function ovs_offload_cleanup() +{ + local log=$EP_DEVICE_OVS_PATH/var/log/dao-ovs-offload.log + + pkill_with_wait dao-ovs-offload + + if [[ -e $log ]]; then + echo "OVS OFFLOAD Logs" + echo "================" + cat $log + fi +} + +function ovs_offload_launch() +{ + local num_cores=$(ep_device_get_num_cores) + local num_ports=0 + local opts + local allowlist="" + local portmap="" + local sdp_vf + local eth_vf + local dao_offload + local coremask=0 + local portconf="" + local tmp + + if ! opts=$(getopt \ + -l "sdp-eth-vf-pair:,esw-vf-ifc:" \ + -- get_allowlist $@); then + echo "Failed to parse arguments" + exit 1 + fi + + eval set -- "$opts" + while [[ $# -gt 1 ]]; do + case $1 in + --sdp-eth-vf-pair) shift; + # One additional core required for control thread + if [[ $num_cores -le 2 ]]; then + echo "Error: Number of cores: $num_cores not sufficient" + exit 1 + fi + sdp_vf=$(echo $1 | awk -F ',' '{print $1}'); + eth_vf=$(echo $1 | awk -F ',' '{print $2}'); + allowlist="$allowlist -a $sdp_vf -a $eth_vf"; + portmap="${portmap}(${1}),"; + tmp="($num_ports,0,$((num_cores - 1)))," + tmp+="($((num_ports + 1)),0,$((num_cores - 2))),"; + portconf="${portconf}${tmp}"; + num_ports=$((num_ports + 2)); + num_cores=$((num_cores - 2)); + coremask=$((coremask | 3 << num_cores));; + --esw-vf-ifc) shift; + allowlist="$allowlist -a $1";; + *) echo "Unknown argument $1"; exit 1;; + esac + shift + done + + portmap=${portmap::-1} + portconf=${portconf::-1} + + # 1 extra core for control thread + num_cores=$((num_cores - 1)) + coremask=$((coremask | 1 << num_cores)) + # Convert the coremask to hex + coremask=$(printf "%x" $coremask) + coremask="0x$coremask" + + find_executable "dao-ovs-offload" dao_offload "$OVS_UTILS_SCRIPT_PATH/../../../../app" + + $dao_offload \ + -c $coremask \ + $allowlist \ + --vfio-vf-token="$VFIO_TOKEN" \ + --file-prefix=ep \ + -- \ + -p 0xff \ + --portmap="$portmap" \ + --config="$portconf" &> $EP_DEVICE_OVS_PATH/var/log/dao-ovs-offload.log & + + sleep 10 +} + +function ovs_sig_handler() +{ + local status=$? + local sig=$1 + local vlan_id=${2:-} + + set +e + trap - ERR + trap - INT + trap - TERM + trap - QUIT + trap - EXIT + + if [[ $status -ne 0 ]]; then + echo "$sig Handler" + fi + + ovs_offload_cleanup + ovs_cleanup + # Cleanup any leftover bridges + ep_common_cleanup_interfaces "br" + ep_host_op if_configure --pcie-addr $EP_HOST_SDP_IFACE --down + ep_remote_op if_configure --pcie-addr $EP_REMOTE_IFACE --down +} + +function ovs_register_sig_handler() +{ + local vlan_id=${1:-} + # Register the traps + trap "ovs_sig_handler ERR $vlan_id" ERR + trap "ovs_sig_handler INT $vlan_id" INT + trap "ovs_sig_handler QUIT $vlan_id" QUIT + trap "ovs_sig_handler EXIT $vlan_id" EXIT +} diff --git a/ci/test/dao-test/virtio/extbuf/virtio_extbuf_1c.sh b/ci/test/dao-test/virtio/extbuf/virtio_extbuf_1c.sh index 6e3c13d..41f3b9f 100755 --- a/ci/test/dao-test/virtio/extbuf/virtio_extbuf_1c.sh +++ b/ci/test/dao-test/virtio/extbuf/virtio_extbuf_1c.sh @@ -16,7 +16,7 @@ function virtio_extbuf_1c() extbuf_register_sig_handler ${DAO_TEST} $host_testpmd_pfx $extbuf_out - ep_device_vfio_bind $if0 + ep_common_bind_driver pci $if0 vfio-pci # Launch virtio extbuf if ! extbuf_app_launch $if0 $extbuf_pfx $extbuf_out "4-5" "-p 0x1 -v 0x1 -P -l"; then @@ -27,8 +27,7 @@ function virtio_extbuf_1c() return 1 fi - device_part=$(ep_device_op get_part) - ep_host_op vdpa_setup $device_part + ep_host_op vdpa_setup $(ep_device_get_part) # Start traffic extbuf_host_start_traffic $host_testpmd_pfx diff --git a/ci/test/dao-test/virtio/extbuf/virtio_extbuf_utils.sh b/ci/test/dao-test/virtio/extbuf/virtio_extbuf_utils.sh index 2908e33..21443d2 100755 --- a/ci/test/dao-test/virtio/extbuf/virtio_extbuf_utils.sh +++ b/ci/test/dao-test/virtio/extbuf/virtio_extbuf_utils.sh @@ -109,14 +109,8 @@ function extbuf_app_launch() local extbuf_out=$3 local cores="$4" local app_args="$5" - local eal_args=" - -a 0000:06:00.1 -a 0000:06:00.2 -a 0000:06:00.3 -a 0000:06:00.4 - -a 0000:06:00.5 -a 0000:06:00.6 -a 0000:06:00.7 -a 0000:06:01.0 - -a 0000:06:01.1 -a 0000:06:01.2 -a 0000:06:01.3 -a 0000:06:01.4 - -a 0000:06:01.5 -a 0000:06:01.6 -a 0000:06:01.7 -a 0000:06:02.0 - -a 0000:06:02.1 -a 0000:06:02.2 -a 0000:06:02.3 -a 0000:06:02.4 - -a 0000:06:02.5 -a 0000:06:02.6 - " + local dpi_vfs=$(ep_common_pcie_addr_get $PCI_DEVID_CN10K_RVU_DPI_VF 22) + local eal_args=$(form_split_args "-a" $dpi_vfs) local args="-l $cores -a $interface $eal_args -- $app_args" local unbuffer diff --git a/ci/test/dao-test/virtio/l2fwd/virtio_l2fwd_1c.sh b/ci/test/dao-test/virtio/l2fwd/virtio_l2fwd_1c.sh index 8fda5ac..175b6bf 100755 --- a/ci/test/dao-test/virtio/l2fwd/virtio_l2fwd_1c.sh +++ b/ci/test/dao-test/virtio/l2fwd/virtio_l2fwd_1c.sh @@ -28,7 +28,7 @@ function virtio_l2fwd_1c() l2fwd_register_sig_handler ${DAO_TEST} $host_testpmd_pfx $l2fwd_out - ep_device_vfio_bind $if0 + ep_common_bind_driver pci $if0 vfio-pci # Launch virtio l2fwd if ! l2fwd_app_launch $if0 $l2fwd_pfx $l2fwd_out "4-7" "-p 0x1 -v 0x1 -P -l"; then @@ -39,8 +39,7 @@ function virtio_l2fwd_1c() return 1 fi - device_part=$(ep_device_op get_part) - ep_host_op vdpa_setup $device_part + ep_host_op vdpa_setup $(ep_device_get_part) # Start traffic l2fwd_host_start_traffic $host_testpmd_pfx @@ -65,11 +64,10 @@ function virtio_l2fwd_offload_run() local ff=$3 local tx_spcap=$4 local tx_mpcap=$5 - local cmp_script=$EP_HOST_DIR/ci/test/dao-test/common/scapy/validate_pcap.py + local cmp_script=$EP_DIR/ci/test/dao-test/common/scapy/validate_pcap.py local rpcap=/tmp/rx_multiseg.pcap local tpcap local itr=0 - local device_part local max_offloads=${#virtio_offloads[@]} ((--max_offloads)) @@ -119,16 +117,15 @@ function virtio_l2fwd_multiseg() local l2fwd_pfx=${DAO_TEST} local host_testpmd_pfx=${DAO_TEST}_testpmd_host local l2fwd_out=virtio_l2fwd.${l2fwd_pfx}.out - local tx_mpcap=$EP_HOST_DIR/ci/test/dao-test/virtio/l2fwd/pcap/tx_mseg.pcap - local tx_spcap=$EP_HOST_DIR/ci/test/dao-test/virtio/l2fwd/pcap/tx.pcap + local tx_mpcap=$EP_DIR/ci/test/dao-test/virtio/l2fwd/pcap/tx_mseg.pcap + local tx_spcap=$EP_DIR/ci/test/dao-test/virtio/l2fwd/pcap/tx.pcap local if0=$(ep_device_get_inactive_if) - local device_part local k=0 failed_tests="" l2fwd_register_sig_handler ${DAO_TEST} $host_testpmd_pfx $l2fwd_out - ep_device_vfio_bind $if0 + ep_common_bind_driver pci $if0 vfio-pci # Launch virtio l2fwd if ! l2fwd_app_launch $if0 $l2fwd_pfx $l2fwd_out "4-7" "-p 0x1 -v 0x1 -P -l --max-pkt-len=9200"; then @@ -139,8 +136,7 @@ function virtio_l2fwd_multiseg() return 1 fi - device_part=$(ep_device_op get_part) - ep_host_op vdpa_setup $device_part + ep_host_op vdpa_setup $(ep_device_get_part) virtio_l2fwd_offload_run $host_testpmd_pfx $l2fwd_out "" $tx_spcap $tx_mpcap @@ -155,7 +151,7 @@ function virtio_l2fwd_multiseg() if ! l2fwd_app_launch $if0 $l2fwd_pfx $l2fwd_out "4-7" "-p 0x1 -v 0x1 -P -l --max-pkt-len=9200 -f"; then echo "Failed to launch virtio l2fwd with No fastfree" else - ep_host_op vdpa_setup $device_part + ep_host_op vdpa_setup $(ep_device_get_part) virtio_l2fwd_offload_run $host_testpmd_pfx $l2fwd_out "no_ff" $tx_spcap $tx_mpcap ep_host_op vdpa_cleanup fi @@ -178,12 +174,11 @@ function virtio_l2fwd_guest_1c() local host_pfx=${DAO_TEST}_guest local l2fwd_out=virtio_l2fwd.${l2fwd_pfx}.out local if0=$(ep_device_get_inactive_if) - local device_part local args l2fwd_register_sig_handler ${DAO_TEST} $host_pfx $l2fwd_out - ep_device_vfio_bind $if0 + ep_common_bind_driver pci $if0 vfio-pci # Launch virtio l2fwd if ! l2fwd_app_launch $if0 $l2fwd_pfx $l2fwd_out "4-7" "-p 0x1 -v 0x1 -P -l"; then @@ -193,8 +188,7 @@ function virtio_l2fwd_guest_1c() return 1 fi - device_part=$(ep_device_op get_part) - ep_host_op vdpa_setup $device_part + ep_host_op vdpa_setup $(ep_device_get_part) ep_host_op_bg 220 launch_guest $host_pfx local k=$? if [[ "$k" != "0" ]]; then diff --git a/ci/test/dao-test/virtio/l2fwd/virtio_l2fwd_utils.sh b/ci/test/dao-test/virtio/l2fwd/virtio_l2fwd_utils.sh index b5e555d..0425bc6 100755 --- a/ci/test/dao-test/virtio/l2fwd/virtio_l2fwd_utils.sh +++ b/ci/test/dao-test/virtio/l2fwd/virtio_l2fwd_utils.sh @@ -208,14 +208,8 @@ function l2fwd_app_launch() local l2fwd_out=$3 local cores="$4" local app_args="$5" - local eal_args=" - -a 0000:06:00.1 -a 0000:06:00.2 -a 0000:06:00.3 -a 0000:06:00.4 - -a 0000:06:00.5 -a 0000:06:00.6 -a 0000:06:00.7 -a 0000:06:01.0 - -a 0000:06:01.1 -a 0000:06:01.2 -a 0000:06:01.3 -a 0000:06:01.4 - -a 0000:06:01.5 -a 0000:06:01.6 -a 0000:06:01.7 -a 0000:06:02.0 - -a 0000:06:02.1 -a 0000:06:02.2 -a 0000:06:02.3 -a 0000:06:02.4 - -a 0000:06:02.5 -a 0000:06:02.6 - " + local dpi_vfs=$(ep_common_pcie_addr_get $PCI_DEVID_CN10K_RVU_DPI_VF 22) + local eal_args=$(form_split_args "-a" $dpi_vfs) local args="-l $cores -a $interface $eal_args -- $app_args" local unbuffer @@ -228,7 +222,8 @@ function l2fwd_app_launch() # Wait for virtio_l2fwd to be up local itr=0 - while ! (tail -n20 $l2fwd_out | grep -q "VIRTIO_L2FWD: Entering graph main loop"); do + + while ! (tail -n20 $l2fwd_out | grep -q "VIRTIO_L2FWD: Entering .* main loop"); do sleep 1 itr=$((itr + 1)) if [[ itr -eq 10 ]]; then diff --git a/ci/test/env/cn10k-ovs.env b/ci/test/env/cn10k-ovs.env new file mode 100644 index 0000000..bcb9f57 --- /dev/null +++ b/ci/test/env/cn10k-ovs.env @@ -0,0 +1,12 @@ +#!/bin/bash +# SPDX-License-Identifier: Marvell-MIT +# Copyright (c) 2024 Marvell. + +source $PROJECT_ROOT/ci/test/env/cn10k.env + +DAO_SUITE=dao-ovs + +EP_DEVICE_OVS_PATH=/tmp/ovs + +# Export the path to this conf so that other scripts can source this conf. +export TEST_ENV_CONF=$PROJECT_ROOT/ci/test/env/cn10k-ovs.env diff --git a/ci/test/env/cn10k-unit-tests.env b/ci/test/env/cn10k-unit-tests.env index 2646c9c..ddb12c2 100644 --- a/ci/test/env/cn10k-unit-tests.env +++ b/ci/test/env/cn10k-unit-tests.env @@ -5,7 +5,6 @@ source $PROJECT_ROOT/ci/test/env/cn10k.env DAO_SUITE=dao-unit-tests -NO_HOST=1 # Export the path to this conf so that other scripts can source this conf. export TEST_ENV_CONF=$PROJECT_ROOT/ci/test/env/cn10k-unit-tests.env diff --git a/ci/test/env/cn10k.env b/ci/test/env/cn10k.env index f62215c..9e7b8aa 100644 --- a/ci/test/env/cn10k.env +++ b/ci/test/env/cn10k.env @@ -8,41 +8,33 @@ RUN_DIR=${RUN_DIR:-$BUILD_DIR} # Test run command TEST_RUN_CMD=$PROJECT_ROOT/ci/test/ep/ep_test_run.sh -# Skip syncing build directory to device/host -SKIP_SYNC=${SKIP_SYNC:-} -SKIP_EP_DEVICE_SYNC=${SKIP_EP_DEVICE_SYNC:-} -SKIP_EP_HOST_SYNC=${SKIP_EP_HOST_SYNC:-} +# Dont delete directory while syncing build directory to device/host SYNC_WITH_NO_CLEANUP=${SYNC_WITH_NO_CLEANUP:-} # Skip setting up EP device/host. Useful when repeatedly running tests. SKIP_SETUP=${SKIP_SETUP:-} -SKIP_EP_DEVICE_SETUP=${SKIP_EP_DEVICE_SETUP:-} -SKIP_EP_HOST_SETUP=${SKIP_EP_HOST_SETUP:-} - -# Reboot the target if tests fail -REBOOT_ON_FAIL=${REBOOT_ON_FAIL:-} - -# Platform -PLAT=${PLAT:-"cn10k"} # EP device user@IP. The user is expected to have passwordless ssh. -EP_DEVICE=${EP_DEVICE:-root@127.0.0.1} +EP_DEVICE=${EP_DEVICE:-} # EP host user@IP. The user is expected to have passwordless ssh. -EP_HOST=${EP_HOST:-root@127.0.0.1} +EP_HOST=${EP_HOST:-} -# EP device directory where the build is to synced -EP_DEVICE_DIR=${EP_DEVICE_DIR:-/tmp/dao} +# EP Remote user@IP. The user is expected to have passwordless ssh. +EP_REMOTE=${EP_REMOTE:-} -# EP host directory where the build is to synced -EP_HOST_DIR=${EP_HOST_DIR:-/tmp/dao_host} +# EP directory where the build is to synced +EP_DIR=${EP_DIR:-/tmp/dao} -# Directory from where the tests will eventually run. -EP_DEVICE_RUN_DIR=${EP_DEVICE_RUN_DIR:-$EP_DEVICE_DIR} +# Path on host to pick modules from. This is useful if user wants to run +# tests on a host which has a different kernel version than the one which +# CI uses. +EP_HOST_MODULE_DIR=${EP_HOST_MODULE_DIR:-} # Sudo command used when running the tests -EP_DEVICE_SUDO=sudo -EP_HOST_SUDO=sudo +EP_DEVICE_SUDO=${EP_DEVICE_SUDO:-sudo} +EP_HOST_SUDO=${EP_HOST_SUDO:-sudo} +EP_REMOTE_SUDO=${EP_REMOTE_SUDO:-sudo} # SSH command used to access EP EP_SSH_CMD=${EP_SSH_CMD:-"ssh -o LogLevel=ERROR -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"} @@ -50,10 +42,6 @@ EP_SSH_CMD=${EP_SSH_CMD:-"ssh -o LogLevel=ERROR -o ServerAliveInterval=30 -o Str # SCP command used to copy files on EP EP_SCP_CMD=${EP_SCP_CMD:-"scp -o LogLevel=ERROR -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"} -# Extra env to be set when running the tests on the device/host -EXTRA_EP_DEVICE_ENV=${EXTRA_EP_DEVICE_ENV:-} -EXTRA_EP_HOST_ENV=${EXTRA_EP_HOST_ENV:-} - # Default timeout to be applied to the test commands DEFAULT_CMD_TIMEOUT=${DEFAULT_CMD_TIMEOUT:-30m} @@ -67,18 +55,6 @@ CMD_EXTRA_ARGS="" # List of tests to be run. If list is empty all tests are run except those in SKIP_TESTS. RUN_TESTS=${RUN_TESTS:-} -# Flag to enable target setup needed for perf stage. -PERF_STAGE=${PERF_STAGE:-} - -# Continue testing regardless of failure -CONTINUE_ON_FAILURE=${CONTINUE_ON_FAILURE:-} - -# File to save status into -STATUS_OUTFILE=${STATUS_OUTFILE:-} - -# Flag to enable target setup needed for TM tests -TM_SETUP=${TM_SETUP:-} - FIXME_SKIP_TESTS="" DEFAULT_SKIP_TESTS=" @@ -98,8 +74,9 @@ source $PROJECT_ROOT/ci/test/common/test_list_helper_funcs.sh # Suite of tests to run DAO_SUITE= -# Host not required for the tests -NO_HOST= +# Host and path for picking up the prebuilt binaries +EP_PREBUILT_BINARIES_SERVER=ci@10.28.34.13 +EP_PREBUILT_BINARIES_PATH=/home/ci/board/ep_files # Export the path to this conf so that other scripts can source this conf. export TEST_ENV_CONF=$PROJECT_ROOT/ci/test/env/cn10k.env diff --git a/ci/test/ep/ep_test_run.sh b/ci/test/ep/ep_test_run.sh index 9257323..cc697a5 100755 --- a/ci/test/ep/ep_test_run.sh +++ b/ci/test/ep/ep_test_run.sh @@ -10,6 +10,14 @@ EP_TEST_RUN_SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $EP_TEST_RUN_SCRIPT_PATH/../dao-test/common/ep_device_utils.sh source $EP_TEST_RUN_SCRIPT_PATH/../dao-test/common/ep_host_utils.sh +declare -A DAO_SUITE_SETUP +declare -A DAO_SUITE_CLEANUP + +source $EP_TEST_RUN_SCRIPT_PATH/sync.sh +source $EP_TEST_RUN_SCRIPT_PATH/ovs_setup.sh +source $EP_TEST_RUN_SCRIPT_PATH/virtio_setup.sh +source $EP_TEST_RUN_SCRIPT_PATH/unit_test_setup.sh + SKIP_SYNC=${SKIP_SYNC:-} SKIP_EP_DEVICE_SETUP=${SKIP_EP_DEVICE_SETUP:-} SKIP_EP_HOST_SETUP=${SKIP_EP_HOST_SETUP:-} @@ -17,10 +25,7 @@ EP_HOST=${EP_HOST:-?} EP_DEVICE=${EP_DEVICE:-?} EP_SSH_CMD=${EP_SSH_CMD:-"ssh"} EP_SCP_CMD=${EP_SCP_CMD:-"scp"} -REMOTE_HOST="$EP_SSH_CMD $EP_HOST -n" -REMOTE_DEVICE="$EP_SSH_CMD $EP_DEVICE -n" -EP_DEVICE_DIR=${EP_DEVICE_DIR:-/tmp/dao} -EP_HOST_DIR=${EP_HOST_DIR:-/tmp/dao_host} +EP_DIR=${EP_DIR:-/tmp/dao} PROJECT_ROOT=${PROJECT_ROOT:-$PWD} BUILD_DIR=${BUILD_DIR:-$PWD/build} BUILD_HOST_DIR=${BUILD_HOST_DIR:-$PWD/build_host} @@ -42,92 +47,6 @@ function save_log() fi } -function host_sync() -{ - local sync="rsync -azzh --delete" - if [[ -n $SKIP_EP_HOST_SYNC || -n $SKIP_SYNC || -n $NO_HOST ]]; then - echo "Skip syncing EP host files" - return - fi - - if [[ -z $SYNC_WITH_NO_CLEANUP ]]; then - echo "Cleanup EP host files" - $REMOTE_HOST "rm -rf $EP_HOST_DIR" - fi - - echo "Syncing EP host files" - $REMOTE_HOST "mkdir -p $EP_HOST_DIR" - $sync -e "$EP_SSH_CMD" -r $BUILD_HOST_DIR/* $EP_HOST:$EP_HOST_DIR - $sync -e "$EP_SSH_CMD" -r --exclude "ci/test/dao-tests/*" \ - $PROJECT_ROOT/ci $EP_HOST:$EP_HOST_DIR -} - -function device_sync() -{ - local sync="rsync -azzh --delete" - if [[ -n $SKIP_EP_DEVICE_SYNC || -n $SKIP_SYNC ]]; then - echo "Skip syncing EP device files" - return - fi - - if [[ -z $SYNC_WITH_NO_CLEANUP ]]; then - echo "Cleanup EP device files" - $REMOTE_DEVICE "rm -rf $EP_DEVICE_DIR" - fi - - echo "Syncing EP device files" - $REMOTE_DEVICE "mkdir -p $EP_DEVICE_DIR" - $sync -e "$EP_SSH_CMD" -r $BUILD_DIR/* $EP_DEVICE:$EP_DEVICE_DIR - $sync -e "$EP_SSH_CMD" -r --exclude "ci/test/dao-tests/*" \ - $PROJECT_ROOT/ci $EP_DEVICE:$EP_DEVICE_DIR - $REMOTE_DEVICE "mkdir -p $EP_DEVICE_DIR/deps-prefix" - $sync -e "$EP_SSH_CMD" -r $DEPS_PREFIX/* $EP_DEVICE:$EP_DEVICE_DIR/deps-prefix -} - -function device_setup() -{ - if [[ -n $SKIP_EP_DEVICE_SETUP || -n $SKIP_SETUP ]]; then - echo "Skip EP device setup" - return - fi - - echo "Setting up EP device for suite $DAO_SUITE" - - if [[ $DAO_SUITE == "dao-virtio" ]]; then - ep_device_op fw_cleanup - ep_device_op hugepage_setup - ep_device_op dpi_setup - ep_device_op pem_setup - elif [[ $DAO_SUITE == "dao-unit-tests" ]]; then - ep_device_op hugepage_setup - else - echo "Unknown Suite : $DAO_SUITE" - exit 1 - fi -} - -function host_setup() -{ - local device_part - if [[ -n $SKIP_EP_HOST_SETUP || -n $SKIP_SETUP || -n $NO_HOST ]]; then - echo "Skip EP host setup" - return - fi - - echo "Setting up EP Host for suite $DAO_SUITE" - if [[ $DAO_SUITE == "dao-virtio" ]]; then - device_part=$(ep_device_op get_part) - ep_device_op_bg 4 fw_launch - ep_host_op hugepage_setup - ep_host_op vdpa_setup $device_part - sleep 5 - ep_device_op fw_cleanup - else - echo "Unknown Suite : $DAO_SUITE" - exit 1 - fi -} - function run_test() { local name=$1 @@ -150,7 +69,8 @@ function run_test() cmd=$(get_test_command $name) curtime=$SECONDS - timeout --foreground -v -k 30 -s 3 $tmo $REMOTE_DEVICE "$cmd" + # Can't use ep_ssh_device_cmd here as we need to run the command under timeout + timeout --foreground -v -k 30 -s 3 $tmo $EP_SSH_CMD $EP_DEVICE -n "$cmd" res=$? echo -en "\n$name completed in $((SECONDS - curtime)) seconds ... " if [[ $res -eq 0 ]]; then @@ -207,26 +127,9 @@ function test_exit() trap - TERM trap - ERR trap - QUIT + trap - EXIT - if [[ -z $NO_HOST ]]; then - ep_host_op safe_kill $EP_HOST_DIR - fi - ep_device_op safe_kill $EP_DEVICE_DIR - - if [[ -n $SKIP_EP_HOST_SETUP || -n $SKIP_SETUP || -n $NO_HOST ]]; then - echo "Skip EP host cleanup" - else - local device_part=$(ep_device_op get_part) - echo "Cleaning up EP Host" - ep_host_op vdpa_cleanup $device_part - fi - - if [[ -z $NO_HOST ]]; then - ep_host_ssh_cmd 'sudo dmesg' > host_dmesg.log - save_log host_dmesg.log - fi - ep_device_ssh_cmd 'sudo dmesg' > device_dmesg.log - save_log device_dmesg.log + ${DAO_SUITE_CLEANUP["$DAO_SUITE"]} echo "###########################################################" echo "Run time: $((SECONDS / 60)) mins $((SECONDS % 60)) secs" @@ -258,8 +161,8 @@ trap "sig_handler QUIT NONE" QUIT host_sync device_sync -device_setup -host_setup +remote_sync +${DAO_SUITE_SETUP["$DAO_SUITE"]} run_all_tests test_exit 0 "SUCCESS: Tests Completed" diff --git a/ci/test/ep/ovs_setup.sh b/ci/test/ep/ovs_setup.sh new file mode 100755 index 0000000..26bc2bc --- /dev/null +++ b/ci/test/ep/ovs_setup.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# SPDX-License-Identifier: Marvell-MIT +# Copyright (c) 2024 Marvell. + +set -euo pipefail + +DAO_SUITE_SETUP["dao-ovs"]=dao_ovs_setup +DAO_SUITE_CLEANUP["dao-ovs"]=dao_ovs_cleanup + +function dao_ovs_cleanup() +{ + if [[ -n $SKIP_SETUP ]]; then + echo "Skip agent cleanup" + else + ep_device_op agent_cleanup + fi + + ep_host_op safe_kill $EP_DIR + ep_device_op safe_kill $EP_DIR + ep_device_op safe_kill ovs + ep_host_ssh_cmd "$EP_HOST_SUDO dmesg" > host_dmesg.log + save_log host_dmesg.log + ep_device_ssh_cmd "$EP_DEVICE_SUDO dmesg" > device_dmesg.log + save_log device_dmesg.log +} + +function verify_setup() +{ + local ssh_ip=$(echo $EP_DEVICE | awk -F '\@' '{print $2}' 2>/dev/null) + local remote_ssh_ip=$(echo $EP_REMOTE | awk -F '\@' '{print $2}' 2>/dev/null) + local eth_ifaces=$(ep_device_op eth_interfaces_get $ssh_ip) + local remote_eth_ifaces=$(ep_remote_op eth_interfaces_get $remote_ssh_ip) + local device_ip_start=11 + local remote_ip_start=51 + local remote_ip=20.0.0.51 + local device_ip=20.0.0.52 + local device_sdp_ip=30.0.0.53 + local host_ip=30.0.0.54 + local device_sdp_iface + local device_part + local ping_status + local remote_iface + local host_sdp_iface + local ext_iface + + echo "Device eth interfaces: $eth_ifaces" + echo "Remote eth interfaces: $remote_eth_ifaces" + + for e in $eth_ifaces; do + ep_device_op unbind_driver pci $e + done + for re in $remote_eth_ifaces; do + ep_remote_op unbind_driver pci $re + done + + # Configure the interfaces one by one and check which ones are pinging + for e in $eth_ifaces; do + ep_device_op bind_driver pci $e rvu_nicpf + ep_device_op if_configure --pcie-addr $e --ip $device_ip + for re in $remote_eth_ifaces; do + ep_remote_op bind_driver pci $re rvu_nicpf + ep_remote_op if_configure --pcie-addr $re --ip $remote_ip + echo "Checking $e (Device) <-> $re (Remote)" + if [[ $(ep_device_op ping $device_ip $remote_ip 2) == "SUCCESS" ]]; then + ext_iface=$e + remote_iface=$re + break + fi + ep_remote_op if_configure --pcie-addr $re --down + ep_remote_op unbind_driver pci $re + done + if [[ -n $ext_iface ]]; then + break + fi + ep_device_op if_configure --pcie-addr $e --down + ep_device_op unbind_driver pci $e + done + + if [[ -z $ext_iface ]] || [[ -z $remote_iface ]]; then + echo "Failed to find a valid interface pair" + exit 1 + fi + + add_test_env EP_REMOTE_IFACE=$remote_iface + + device_part=$(ep_device_op get_part) + host_sdp_iface=$(ep_host_op pcie_addr_get ${device_part}00) + device_sdp_iface=$(ep_device_op pcie_addr_get $PCI_DEVID_CN10K_RVU_SDP_VF) + + add_test_env EP_HOST_SDP_IFACE=$host_sdp_iface + + echo "Device External Interface: $ext_iface" + echo "Remote Interface: $remote_iface" + echo "Device SDP VF Interface: $device_sdp_iface" + echo "Host SDP Interface: $host_sdp_iface" + + # Configure host interface + ep_host_op if_configure --pcie-addr $host_sdp_iface --ip $host_ip + + # Configure device SDP VF + ep_device_op bind_driver pci $device_sdp_iface rvu_nicvf + ep_device_op if_configure --pcie-addr $device_sdp_iface --ip $device_sdp_ip + + # Enable IP forwarding on device + ep_device_op ip_forwarding 1 + + # Add route on remote + ep_remote_ssh_cmd "$EP_REMOTE_SUDO ip route add $host_ip via $device_ip" + + # Add route on host + ep_host_ssh_cmd "$EP_HOST_SUDO ip route add $remote_ip via $device_sdp_ip" + + # Ping remote from host + ping_status=$(ep_host_op ping $host_ip $remote_ip 2) + + # Undo all the configurations + ep_host_ssh_cmd "$EP_HOST_SUDO ip route del $remote_ip" + ep_remote_ssh_cmd "$EP_REMOTE_SUDO ip route del $host_ip" + ep_device_op ip_forwarding 0 + ep_device_op if_configure --pcie-addr $ext_iface --down + ep_remote_op if_configure --pcie-addr $remote_iface --down + ep_device_op if_configure --pcie-addr $device_sdp_iface --down + ep_host_op if_configure --pcie-addr $host_sdp_iface --down + + # Unbind the interfaces + ep_device_op unbind_driver pci $device_sdp_iface + ep_remote_op unbind_driver pci $remote_iface + ep_device_op unbind_driver pci $ext_iface + + # Check output of ping + if [[ "$ping_status" == "SUCCESS" ]]; then + echo "Setup verified" + else + echo "Cannot ping remote from host" + exit 1 + fi + +} + +function dao_ovs_setup() +{ + # Delete the existing ovs directory and copy the new one + ep_device_ssh_cmd "$EP_DEVICE_SUDO rm -rf $EP_DEVICE_OVS_PATH" + ep_device_ssh_cmd "mkdir -p $EP_DEVICE_OVS_PATH" + ep_device_ssh_cmd "rsync -a $EP_DIR/ep_files/ovs/* $EP_DEVICE_OVS_PATH" + + if [[ -n $SKIP_SETUP ]]; then + echo "Skip EP device setup" + return + fi + + echo "Setting up EP device for ovs tests" + ep_device_op hugepage_setup 524288 24 6 + + echo "Setting up EP Host for ovs" + ep_host_op sdp_setup + + ep_device_op_bg 10 agent_init + + echo "Verifying setup" + verify_setup +} diff --git a/ci/test/ep/sync.sh b/ci/test/ep/sync.sh new file mode 100755 index 0000000..fb32ee0 --- /dev/null +++ b/ci/test/ep/sync.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# SPDX-License-Identifier: Marvell-MIT +# Copyright (c) 2024 Marvell. + +set -euo pipefail + +function host_sync() +{ + local sync="rsync -azzh --delete" + + if [[ -z $SYNC_WITH_NO_CLEANUP ]]; then + echo "Cleanup EP host files" + ep_host_ssh_cmd "$EP_HOST_SUDO rm -rf $EP_DIR" + fi + + echo "Syncing EP host files" + ep_host_ssh_cmd "mkdir -p $EP_DIR" + ep_host_ssh_cmd "mkdir -p $EP_DIR/ep_files" + $sync -e "$EP_SSH_CMD" -r $BUILD_HOST_DIR/* $EP_HOST:$EP_DIR + $sync -e "$EP_SSH_CMD" -r $PROJECT_ROOT/ci $EP_HOST:$EP_DIR + $sync -e "$EP_SSH_CMD" -r $EP_PREBUILT_BINARIES_SERVER:$EP_PREBUILT_BINARIES_PATH/* \ + /tmp/ep_files + $sync -e "$EP_SSH_CMD" -r /tmp/ep_files/* $EP_HOST:$EP_DIR/ep_files +} + +function device_sync() +{ + local sync="rsync -azzh --delete" + + if [[ -z $SYNC_WITH_NO_CLEANUP ]]; then + echo "Cleanup EP device files" + ep_device_ssh_cmd "$EP_DEVICE_SUDO rm -rf $EP_DIR" + fi + + echo "Syncing EP device files" + ep_device_ssh_cmd "mkdir -p $EP_DIR" + $sync -e "$EP_SSH_CMD" -r $BUILD_DIR/* $EP_DEVICE:$EP_DIR + $sync -e "$EP_SSH_CMD" -r $PROJECT_ROOT/ci $EP_DEVICE:$EP_DIR + ep_device_ssh_cmd "mkdir -p $EP_DIR/deps-prefix" + $sync -e "$EP_SSH_CMD" -r $DEPS_PREFIX/* $EP_DEVICE:$EP_DIR/deps-prefix + $sync -e "$EP_SSH_CMD" -r $EP_PREBUILT_BINARIES_SERVER:$EP_PREBUILT_BINARIES_PATH/* \ + /tmp/ep_files + $sync -e "$EP_SSH_CMD" -r /tmp/ep_files/* $EP_DEVICE:$EP_DIR/ep_files + ep_device_ssh_cmd "$EP_DEVICE_SUDO cp $EP_DIR/ep_files/hostname /usr/bin" +} + +function remote_sync() +{ + local sync="rsync -azzh --delete" + + if [[ -z ${EP_REMOTE:-} ]]; then + echo "EP_REMOTE is not set, skipping remote sync" + return + fi + + if [[ -z $SYNC_WITH_NO_CLEANUP ]]; then + echo "Cleanup EP remote files" + ep_remote_ssh_cmd "$EP_REMOTE_SUDO rm -rf $EP_DIR" + fi + + echo "Syncing EP remote files" + ep_remote_ssh_cmd "mkdir -p $EP_DIR" + $sync -e "$EP_SSH_CMD" -r $PROJECT_ROOT/ci $EP_REMOTE:$EP_DIR +} diff --git a/ci/test/ep/unit_test_setup.sh b/ci/test/ep/unit_test_setup.sh new file mode 100755 index 0000000..74e4ea7 --- /dev/null +++ b/ci/test/ep/unit_test_setup.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# SPDX-License-Identifier: Marvell-MIT +# Copyright (c) 2024 Marvell. + +set -euo pipefail + +DAO_SUITE_SETUP["dao-unit-tests"]=dao_unit_test_setup +DAO_SUITE_CLEANUP["dao-unit-tests"]=dao_unit_test_cleanup + +function dao_unit_test_cleanup() +{ + ep_device_op safe_kill $EP_DIR + ep_device_ssh_cmd "$EP_DEVICE_SUDO dmesg" > device_dmesg.log + save_log device_dmesg.log +} + +function dao_unit_test_setup() +{ + if [[ -n $SKIP_SETUP ]]; then + echo "Skipping setup" + fi + + ep_device_op hugepage_setup 524288 24 6 +} diff --git a/ci/test/ep/virtio_setup.sh b/ci/test/ep/virtio_setup.sh new file mode 100755 index 0000000..ceffd17 --- /dev/null +++ b/ci/test/ep/virtio_setup.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# SPDX-License-Identifier: Marvell-MIT +# Copyright (c) 2024 Marvell. + +set -euo pipefail + +DAO_SUITE_SETUP["dao-virtio"]=dao_virtio_setup +DAO_SUITE_CLEANUP["dao-virtio"]=dao_virtio_cleanup + +function dao_virtio_cleanup() +{ + local device_part + + if [[ -n $SKIP_SETUP ]]; then + echo "Skip EP VDPA cleanup on host" + else + device_part=$(ep_device_op get_part) + echo "Cleaning up VDPA on host" + ep_host_op vdpa_cleanup $device_part + fi + + ep_host_op safe_kill $EP_DIR + ep_device_op safe_kill $EP_DIR + ep_host_ssh_cmd "$EP_HOST_SUDO dmesg" > host_dmesg.log + save_log host_dmesg.log + ep_device_ssh_cmd "$EP_DEVICE_SUDO dmesg" > device_dmesg.log + save_log device_dmesg.log +} + +function dao_virtio_setup() +{ + local device_part + + if [[ -n $SKIP_SETUP ]]; then + echo "Skip setup" + return + fi + + echo "Setting up EP device for virtio tests" + ep_device_op hugepage_setup 524288 24 6 + ep_device_op dpi_setup + ep_device_op pem_setup + + echo "Setting up EP Host for virtio tests" + device_part=$(ep_device_op get_part) + ep_host_op hugepage_setup 2048 24 2048 +}