Skip to content

Commit

Permalink
Merge pull request #576 from glazychev-art/fix_netns
Browse files Browse the repository at this point in the history 
Fix for working with network namespaces
  • Loading branch information
@denis-tingaikin
denis-tingaikin authored Mar 17, 2023
2 parents 25961f7 + 35f8cdd commit 40f70dd
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 73 deletions.
72 changes: 34 additions & 38 deletions pkg/kernel/link.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
package kernel

import (
"io/fs"
"io/ioutil"
"os"
"path/filepath"
Expand Down Expand Up @@ -172,18 +173,6 @@ func (l *link) GetLink() netlink.Link {
// address and/or target interface name.
func FindHostDevice(pciAddress, name string, namespaces ...netns.NsHandle) (Link, error) {
// TODO: add support for shared l interfaces (like Mellanox NICs)

current, err := nshandle.Current()
if err != nil {
return nil, err
}
defer func() {
if err := netns.Set(current); err != nil {
panic(errors.Wrapf(err, "failed to switch back to the current net NS: %v", current).Error())
}
_ = current.Close()
}()

attempts := []func(netns.NsHandle, string, string) (netlink.Link, error){
searchByPCIAddress,
searchByName,
Expand All @@ -210,37 +199,44 @@ func FindHostDevice(pciAddress, name string, namespaces ...netns.NsHandle) (Link

func searchByPCIAddress(ns netns.NsHandle, name, pciAddress string) (netlink.Link, error) {
// execute in context of the pod's namespace
err := netns.Set(ns)
currentNs, err := nshandle.Current()
if err != nil {
return nil, errors.Errorf("failed to enter namespace: %s", err)
return nil, err
}
defer func() { _ = currentNs.Close() }()

pciDevicePath := filepath.Join("/sys/bus/pci/devices", pciAddress)
netDir, err := findNetDir(pciDevicePath)
if err != nil {
return nil, errors.Errorf("no net directory under pci device %s: %q", pciAddress, err)
}
var link netlink.Link
err = nshandle.RunIn(currentNs, ns, func() error {
var netDir string
pciDevicePath := filepath.Join("/sys/bus/pci/devices", pciAddress)
netDir, err = findNetDir(pciDevicePath)
if err != nil {
return errors.Errorf("no net directory under pci device %s: %q", pciAddress, err)
}

fInfos, err := ioutil.ReadDir(netDir)
if err != nil {
return nil, errors.Errorf("failed to read net directory %s: %q", netDir, err)
}
var fInfos []fs.FileInfo
fInfos, err = ioutil.ReadDir(netDir)
if err != nil {
return errors.Errorf("failed to read net directory %s: %q", netDir, err)
}

names := make([]string, 0)
for _, f := range fInfos {
names = append(names, f.Name())
}
names := make([]string, 0)
for _, f := range fInfos {
names = append(names, f.Name())
}

if len(names) == 0 {
return nil, errors.Errorf("no links with PCI address %s found", pciAddress)
}
if len(names) == 0 {
return errors.Errorf("no links with PCI address %s found", pciAddress)
}

link, err := netlink.LinkByName(names[0])
if err != nil {
return nil, errors.Errorf("error getting host device with PCI address %s", pciAddress)
}
link, err = netlink.LinkByName(names[0])
if err != nil {
return errors.Errorf("error getting host device with PCI address %s", pciAddress)
}
return nil
})

return link, nil
return link, err
}

func findNetDir(basePath string) (string, error) {
Expand Down Expand Up @@ -272,13 +268,13 @@ func findNetDir(basePath string) (string, error) {

func searchByName(ns netns.NsHandle, name, pciAddress string) (netlink.Link, error) {
// execute in context of the pod's namespace
err := netns.Set(ns)
handle, err := netlink.NewHandleAt(ns)
if err != nil {
return nil, errors.Errorf("failed to switch to namespace: %s", err)
return nil, errors.Errorf("failed to create netlink handler: %s", err)
}

// get link
link, err := netlink.LinkByName(name)
link, err := handle.LinkByName(name)
if err != nil {
return nil, errors.Errorf("failed to get link with name %s", name)
}
Expand Down
95 changes: 60 additions & 35 deletions pkg/kernel/networkservice/inject/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,42 +37,59 @@ import (
"github.com/networkservicemesh/sdk-kernel/pkg/kernel/tools/nshandle"
)

func moveInterfaceToAnotherNamespace(ifName string, curNetNS, fromNetNS, toNetNS netns.NsHandle) error {
return nshandle.RunIn(curNetNS, fromNetNS, func() error {
link, err := netlink.LinkByName(ifName)
if err != nil {
return errors.Wrapf(err, "failed to get net interface: %v", ifName)
}
func moveInterfaceToAnotherNamespace(ifName string, fromNetNS, toNetNS netns.NsHandle) error {
handle, err := netlink.NewHandleAt(fromNetNS)
if err != nil {
return errors.Wrap(err, "failed to create netlink fromNetNS handle")
}

if err := netlink.LinkSetNsFd(link, int(toNetNS)); err != nil {
return errors.Wrapf(err, "failed to move net interface to net NS: %v %v", ifName, toNetNS)
}
link, err := handle.LinkByName(ifName)
if err != nil {
return errors.Wrapf(err, "failed to get net interface: %v", ifName)
}

return nil
})
if err := handle.LinkSetNsFd(link, int(toNetNS)); err != nil {
return errors.Wrapf(err, "failed to move net interface to net NS: %v %v", ifName, toNetNS)
}
return nil
}

func renameInterface(origIfName, desiredIfName string, curNetNS, targetNetNS netns.NsHandle) error {
return nshandle.RunIn(curNetNS, targetNetNS, func() error {
link, err := netlink.LinkByName(origIfName)
if err != nil {
return errors.Wrapf(err, "failed to get net interface: %v", origIfName)
}
func renameInterface(origIfName, desiredIfName string, targetNetNS netns.NsHandle) error {
handle, err := netlink.NewHandleAt(targetNetNS)
if err != nil {
return errors.Wrap(err, "failed to create netlink targetNetNS handle")
}

if err = netlink.LinkSetDown(link); err != nil {
return errors.Wrapf(err, "failed to rename net interface: %v -> %v", origIfName, desiredIfName)
}
link, err := handle.LinkByName(origIfName)
if err != nil {
return errors.Wrapf(err, "failed to get net interface: %v", origIfName)
}

if err = netlink.LinkSetName(link, desiredIfName); err != nil {
return errors.Wrapf(err, "failed to rename net interface: %v -> %v", origIfName, desiredIfName)
}
if err = handle.LinkSetDown(link); err != nil {
return errors.Wrapf(err, "failed to down net interface: %v -> %v", origIfName, desiredIfName)
}

if err = netlink.LinkSetUp(link); err != nil {
return errors.Wrapf(err, "failed to rename net interface: %v -> %v", origIfName, desiredIfName)
}
if err = handle.LinkSetName(link, desiredIfName); err != nil {
return errors.Wrapf(err, "failed to rename net interface: %v -> %v", origIfName, desiredIfName)
}
return nil
}

return nil
})
func upInterface(ifName string, targetNetNS netns.NsHandle) error {
handle, err := netlink.NewHandleAt(targetNetNS)
if err != nil {
return errors.Wrap(err, "failed to create netlink NS handle")
}

link, err := handle.LinkByName(ifName)
if err != nil {
return errors.Wrapf(err, "failed to get net interface: %v", ifName)
}

if err = handle.LinkSetUp(link); err != nil {
return errors.Wrapf(err, "failed to up net interface: %v", ifName)
}
return nil
}

func move(ctx context.Context, conn *networkservice.Connection, vfRefCountMap map[string]int, vfRefCountMutex sync.Locker, isClient, isMoveBack bool) error {
Expand Down Expand Up @@ -119,7 +136,12 @@ func move(ctx context.Context, conn *networkservice.Connection, vfRefCountMap ma
ifName := mech.GetInterfaceName()
if !isMoveBack {
err = moveToContNetNS(vfConfig, vfRefCountMap, vfRefKey, ifName, hostNetNS, contNetNS)
vfConfig.ContNetNS = contNetNS
if err != nil {
// If we got an error, try to move back the vf to the host namespace
_ = moveToHostNetNS(vfConfig, vfRefCountMap, vfRefKey, ifName, hostNetNS, contNetNS)
} else {
vfConfig.ContNetNS = contNetNS
}
} else {
err = moveToHostNetNS(vfConfig, vfRefCountMap, vfRefKey, ifName, hostNetNS, contNetNS)
}
Expand All @@ -146,12 +168,15 @@ func moveToContNetNS(vfConfig *vfconfig.VFConfig, vfRefCountMap map[string]int,
return
}
if vfConfig != nil && vfConfig.VFInterfaceName != ifName {
err = moveInterfaceToAnotherNamespace(vfConfig.VFInterfaceName, hostNetNS, hostNetNS, contNetNS)
err = moveInterfaceToAnotherNamespace(vfConfig.VFInterfaceName, hostNetNS, contNetNS)
if err == nil {
err = renameInterface(vfConfig.VFInterfaceName, ifName, hostNetNS, contNetNS)
err = renameInterface(vfConfig.VFInterfaceName, ifName, contNetNS)
if err == nil {
err = upInterface(ifName, contNetNS)
}
}
} else {
err = moveInterfaceToAnotherNamespace(ifName, hostNetNS, hostNetNS, contNetNS)
err = moveInterfaceToAnotherNamespace(ifName, hostNetNS, contNetNS)
}
return err
}
Expand All @@ -178,17 +203,17 @@ func moveToHostNetNS(vfConfig *vfconfig.VFConfig, vfRefCountMap map[string]int,
}
return nil
}
err := renameInterface(ifName, vfConfig.VFInterfaceName, hostNetNS, contNetNS)
err := renameInterface(ifName, vfConfig.VFInterfaceName, contNetNS)
if err == nil {
err = moveInterfaceToAnotherNamespace(vfConfig.VFInterfaceName, hostNetNS, contNetNS, hostNetNS)
err = moveInterfaceToAnotherNamespace(vfConfig.VFInterfaceName, contNetNS, hostNetNS)
}
return err
}
link, _ := kernellink.FindHostDevice("", ifName, hostNetNS)
if link != nil {
return nil
}
return moveInterfaceToAnotherNamespace(ifName, hostNetNS, contNetNS, hostNetNS)
return moveInterfaceToAnotherNamespace(ifName, contNetNS, hostNetNS)
}
return nil
}

0 comments on commit 40f70dd

Please sign in to comment.