diff --git a/go.mod b/go.mod index 75c8cbc8..a576f1ce 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/networkservicemesh/api v1.0.1-0.20211110183123-3038992da61a github.com/networkservicemesh/sdk v0.5.1-0.20211110200317-2272de7cade7 github.com/networkservicemesh/sdk-kernel v0.0.0-20211110200529-c70dbe94eb13 + github.com/networkservicemesh/sdk-sriov v0.0.0-20211014093500-f12ea1fa1fb9 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.7.0 github.com/thanhpk/randstr v1.0.4 diff --git a/pkg/networkservice/chains/forwarder/server.go b/pkg/networkservice/chains/forwarder/server.go index 4f3a1d1d..ad15b10d 100644 --- a/pkg/networkservice/chains/forwarder/server.go +++ b/pkg/networkservice/chains/forwarder/server.go @@ -24,6 +24,7 @@ import ( "context" "net" "net/url" + "sync" "time" "git.fd.io/govpp.git/api" @@ -50,6 +51,8 @@ import ( "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/connectioncontext/mtu" "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/kernel" "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/memif" + "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/vlan" + "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/vlan/pciaddresspool" "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/vxlan" "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/wireguard" "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/pinhole" @@ -70,13 +73,14 @@ type xconnectNSServer struct { } // NewServer - returns an implementation of the xconnectns network service -func NewServer(ctx context.Context, name string, authzServer networkservice.NetworkServiceServer, tokenGenerator token.GeneratorFunc, clientURL *url.URL, vppConn Connection, tunnelIP net.IP, tunnelPort uint16, dialTimeout time.Duration, clientDialOptions ...grpc.DialOption) endpoint.Endpoint { +func NewServer(ctx context.Context, name string, authzServer networkservice.NetworkServiceServer, tokenGenerator token.GeneratorFunc, clientURL *url.URL, vppConn Connection, tunnelIP net.IP, tunnelPort uint16, dialTimeout time.Duration, resourcePool pciaddresspool.PCIAddressPool, clientDialOptions ...grpc.DialOption) endpoint.Endpoint { nseClient := registryclient.NewNetworkServiceEndpointRegistryClient(ctx, clientURL, registryclient.WithNSEAdditionalFunctionality(registryrecvfd.NewNetworkServiceEndpointRegistryClient()), registryclient.WithDialOptions(clientDialOptions...), ) nsClient := registryclient.NewNetworkServiceRegistryClient(ctx, clientURL, registryclient.WithDialOptions(clientDialOptions...)) + resourceLock := &sync.Mutex{} rv := &xconnectNSServer{} additionalFunctionality := []networkservice.NetworkServiceServer{ recvfd.NewServer(), @@ -114,6 +118,7 @@ func NewServer(ctx context.Context, name string, authzServer networkservice.Netw kernel.NewClient(vppConn), vxlan.NewClient(vppConn, tunnelIP, vxlan.WithVniPort(tunnelPort)), wireguard.NewClient(vppConn, tunnelIP), + vlan.NewClient(vppConn, resourceLock, resourcePool), filtermechanisms.NewClient(), pinhole.NewClient(vppConn), recvfd.NewClient(), diff --git a/pkg/networkservice/mechanisms/vlan/client.go b/pkg/networkservice/mechanisms/vlan/client.go new file mode 100644 index 00000000..be765207 --- /dev/null +++ b/pkg/networkservice/mechanisms/vlan/client.go @@ -0,0 +1,189 @@ +// Copyright (c) 2021 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vlan + +import ( + "context" + "io" + "sync" + "time" + + "git.fd.io/govpp.git/api" + interfaces "github.com/edwarnicke/govpp/binapi/interface" + "github.com/golang/protobuf/ptypes/empty" + "github.com/pkg/errors" + "google.golang.org/grpc" + + "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/cls" + vlanmech "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vlan" + "github.com/networkservicemesh/api/pkg/api/networkservice/payload" + kernellink "github.com/networkservicemesh/sdk-kernel/pkg/kernel" + "github.com/networkservicemesh/sdk-kernel/pkg/kernel/tools/nshandle" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/chain" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" + "github.com/networkservicemesh/sdk/pkg/tools/log" + "github.com/networkservicemesh/sdk/pkg/tools/postpone" + + "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/vlan/l2vtr" + "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/vlan/linkinit" + "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/vlan/pciaddresspool" + "github.com/networkservicemesh/sdk-vpp/pkg/tools/ifindex" +) + +type vlanClient struct { + vppConn api.Connection +} + +// NewClient returns a VLAN client chain element +func NewClient(vppConn api.Connection, resourceLock sync.Locker, resourcePool pciaddresspool.PCIAddressPool) networkservice.NetworkServiceClient { + return chain.NewNetworkServiceClient( + l2vtr.NewClient(vppConn), + &vlanClient{vppConn: vppConn}, + linkinit.NewClient(vppConn), + pciaddresspool.NewClient(resourceLock, resourcePool), + ) +} + +func (v *vlanClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) { + if request.GetConnection().GetPayload() != payload.Ethernet { + return next.Client(ctx).Request(ctx, request, opts...) + } + + mechanism := &networkservice.Mechanism{ + Cls: cls.REMOTE, + Type: vlanmech.MECHANISM, + Parameters: make(map[string]string), + } + request.MechanismPreferences = append(request.MechanismPreferences, mechanism) + + postponeCtxFunc := postpone.ContextWithValues(ctx) + + conn, err := next.Client(ctx).Request(ctx, request, opts...) + if err != nil { + return nil, err + } + + if err := addSubIf(ctx, conn, v.vppConn); err != nil { + closeCtx, cancelClose := postponeCtxFunc() + defer cancelClose() + + if _, closeErr := v.Close(closeCtx, conn, opts...); closeErr != nil { + err = errors.Wrapf(err, "connection closed with error: %s", closeErr.Error()) + } + + return nil, err + } + + return conn, nil +} + +func (v *vlanClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) { + if conn.GetPayload() != payload.Ethernet { + return next.Client(ctx).Close(ctx, conn, opts...) + } + _ = delSubIf(ctx, conn, v.vppConn) + return next.Client(ctx).Close(ctx, conn, opts...) +} + +func addSubIf(ctx context.Context, conn *networkservice.Connection, vppConn api.Connection) error { + if mechanism := vlanmech.ToMechanism(conn.GetMechanism()); mechanism != nil { + _, ok := ifindex.Load(ctx, true) + if ok { + return nil + } + now := time.Now() + pciAddressInfo, ok := pciaddresspool.Load(ctx) + if !ok { + return errors.Errorf("no link found") + } + hostNetNS, err := nshandle.Current() + if err != nil { + return err + } + defer func() { _ = hostNetNS.Close() }() + kernelLink, err := kernellink.FindHostDevice(pciAddressInfo.GetPCIAddress(), "", hostNetNS) + if err != nil { + return err + } + hostIFName := kernelLink.GetName() + + client, err := interfaces.NewServiceClient(vppConn).SwInterfaceDump(ctx, &interfaces.SwInterfaceDump{ + NameFilterValid: true, + NameFilter: hostIFName, + }) + if err != nil { + return errors.Wrapf(err, "error attempting to get interface dump client to set vlan subinterface on %q", hostIFName) + } + log.FromContext(ctx). + WithField("duration", time.Since(now)). + WithField("HostInterfaceName", hostIFName). + WithField("vppapi", "SwInterfaceDump").Debug("completed") + + for { + details, err := client.Recv() + if err == io.EOF { + break + } + if err != nil { + return errors.Wrapf(err, "error attempting to get interface details to set vlan subinterface on %q", hostIFName) + } + now = time.Now() + swIfIndex := details.SwIfIndex + vlanID := mechanism.GetVlanID() + vlanSubif := &interfaces.CreateVlanSubif{ + SwIfIndex: swIfIndex, + VlanID: vlanID, + } + + rsp, err := interfaces.NewServiceClient(vppConn).CreateVlanSubif(ctx, vlanSubif) + if err != nil { + return errors.WithStack(err) + } + log.FromContext(ctx). + WithField("duration", time.Since(now)). + WithField("HostInterfaceIndex", swIfIndex). + WithField("VlanID", vlanID). + WithField("vppapi", "CreateVlanSubIf").Debug("completed") + + ifindex.Store(ctx, true, rsp.SwIfIndex) + } + } + return nil +} +func delSubIf(ctx context.Context, conn *networkservice.Connection, vppConn api.Connection) error { + if mechanism := vlanmech.ToMechanism(conn.GetMechanism()); mechanism != nil { + swIfIndex, ok := ifindex.Load(ctx, true) + if !ok { + return nil + } + now := time.Now() + vlanSubif := &interfaces.DeleteSubif{ + SwIfIndex: swIfIndex, + } + _, err := interfaces.NewServiceClient(vppConn).DeleteSubif(ctx, vlanSubif) + if err != nil { + return errors.WithStack(err) + } + log.FromContext(ctx). + WithField("duration", time.Since(now)). + WithField("HostInterfaceIndex", swIfIndex). + WithField("vppapi", "DeleteSubif").Debug("completed") + ifindex.Delete(ctx, true) + } + return nil +} diff --git a/pkg/networkservice/mechanisms/vlan/doc.go b/pkg/networkservice/mechanisms/vlan/doc.go new file mode 100644 index 00000000..4a101d38 --- /dev/null +++ b/pkg/networkservice/mechanisms/vlan/doc.go @@ -0,0 +1,18 @@ +// Copyright (c) 2021 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package vlan provides chain elements for implementing the vlan mechanism +package vlan diff --git a/pkg/networkservice/mechanisms/vlan/l2vtr/client.go b/pkg/networkservice/mechanisms/vlan/l2vtr/client.go new file mode 100644 index 00000000..c36664a4 --- /dev/null +++ b/pkg/networkservice/mechanisms/vlan/l2vtr/client.go @@ -0,0 +1,121 @@ +// Copyright (c) 2021 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package l2vtr + +import ( + "context" + "time" + + "git.fd.io/govpp.git/api" + "github.com/edwarnicke/govpp/binapi/l2" + "github.com/golang/protobuf/ptypes/empty" + "github.com/pkg/errors" + "google.golang.org/grpc" + + "github.com/networkservicemesh/api/pkg/api/networkservice" + vlanmech "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vlan" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" + "github.com/networkservicemesh/sdk/pkg/tools/log" + "github.com/networkservicemesh/sdk/pkg/tools/postpone" + + "github.com/networkservicemesh/sdk-vpp/pkg/tools/ifindex" +) + +type l2vtrClient struct { + vppConn api.Connection +} + +func NewClient(vppConn api.Connection) networkservice.NetworkServiceClient { + return &l2vtrClient{vppConn: vppConn} +} + +func (v *l2vtrClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) { + postponeCtxFunc := postpone.ContextWithValues(ctx) + + conn, err := next.Client(ctx).Request(ctx, request, opts...) + if err != nil { + return nil, err + } + + if err = enableVtr(ctx, conn, v.vppConn); err != nil { + closeCtx, cancelClose := postponeCtxFunc() + defer cancelClose() + + if _, closeErr := v.Close(closeCtx, conn, opts...); closeErr != nil { + err = errors.Wrapf(err, "connection closed with error: %s", closeErr.Error()) + } + + return nil, err + } + + return conn, nil +} + +func (v *l2vtrClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) { + _ = disableVtr(ctx, conn, v.vppConn) + return next.Client(ctx).Close(ctx, conn, opts...) +} + +func enableVtr(ctx context.Context, conn *networkservice.Connection, vppConn api.Connection) error { + if mechanism := vlanmech.ToMechanism(conn.GetMechanism()); mechanism != nil { + swIfIndex, ok := ifindex.Load(ctx, true) + if !ok { + return nil + } + now := time.Now() + if _, err := l2.NewServiceClient(vppConn).L2InterfaceVlanTagRewrite(ctx, &l2.L2InterfaceVlanTagRewrite{ + SwIfIndex: swIfIndex, + VtrOp: L2_VTR_POP_1, + PushDot1q: 0, + Tag1: 0, + Tag2: 0, + }); err != nil { + return errors.WithStack(err) + } + log.FromContext(ctx). + WithField("duration", time.Since(now)). + WithField("SwIfIndex", swIfIndex). + WithField("operation", "POP 1"). + WithField("vppapi", "L2InterfaceVlanTagRewrite").Debug("completed") + + } + return nil +} + +func disableVtr(ctx context.Context, conn *networkservice.Connection, vppConn api.Connection) error { + if mechanism := vlanmech.ToMechanism(conn.GetMechanism()); mechanism != nil { + swIfIndex, ok := ifindex.Load(ctx, true) + if !ok { + return nil + } + now := time.Now() + + if _, err := l2.NewServiceClient(vppConn).L2InterfaceVlanTagRewrite(ctx, &l2.L2InterfaceVlanTagRewrite{ + SwIfIndex: swIfIndex, + VtrOp: L2_VTR_DISABLED, + }); err != nil { + return errors.WithStack(err) + } + log.FromContext(ctx). + WithField("duration", time.Since(now)). + WithField("SwIfIndex", swIfIndex). + WithField("operation", "DISABLE"). + WithField("vppapi", "L2InterfaceVlanTagRewrite").Debug("completed") + + } + return nil +} diff --git a/pkg/networkservice/mechanisms/vlan/l2vtr/constants.go b/pkg/networkservice/mechanisms/vlan/l2vtr/constants.go new file mode 100644 index 00000000..6304f35d --- /dev/null +++ b/pkg/networkservice/mechanisms/vlan/l2vtr/constants.go @@ -0,0 +1,24 @@ +// Copyright (c) 2021 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package l2vtr + +// defines enum 'vtr_op'. +const ( + L2_VTR_DISABLED uint32 = 0 + L2_VTR_POP_1 uint32 = 3 + L2_VTR_PUSH_DOT1Q uint32 = 1 +) diff --git a/pkg/networkservice/mechanisms/vlan/l2vtr/doc.go b/pkg/networkservice/mechanisms/vlan/l2vtr/doc.go new file mode 100644 index 00000000..9a01ead8 --- /dev/null +++ b/pkg/networkservice/mechanisms/vlan/l2vtr/doc.go @@ -0,0 +1,18 @@ +// Copyright (c) 2021 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package l2vtr provides chain elements for setting Vlan Tag Rewrite on subinterfaces +package l2vtr diff --git a/pkg/networkservice/mechanisms/vlan/linkinit/client.go b/pkg/networkservice/mechanisms/vlan/linkinit/client.go new file mode 100644 index 00000000..fb89a016 --- /dev/null +++ b/pkg/networkservice/mechanisms/vlan/linkinit/client.go @@ -0,0 +1,188 @@ +// Copyright (c) 2021 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linkinit + +import ( + "context" + "io" + "time" + + "git.fd.io/govpp.git/api" + interfaces "github.com/edwarnicke/govpp/binapi/interface" + "github.com/edwarnicke/govpp/binapi/interface_types" + "github.com/golang/protobuf/ptypes/empty" + "github.com/pkg/errors" + "github.com/vishvananda/netlink" + "google.golang.org/grpc" + + "github.com/networkservicemesh/api/pkg/api/networkservice" + vlanmech "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vlan" + kernellink "github.com/networkservicemesh/sdk-kernel/pkg/kernel" + "github.com/networkservicemesh/sdk-kernel/pkg/kernel/tools/nshandle" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" + "github.com/networkservicemesh/sdk/pkg/tools/log" + "github.com/networkservicemesh/sdk/pkg/tools/postpone" + + "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/vlan/pciaddresspool" + "github.com/networkservicemesh/sdk-vpp/pkg/tools/initvpp" +) + +type linkinitClient struct { + vppConn api.Connection +} + +// NewClient - creates links for vlan remote mechanism +func NewClient(vppConn api.Connection) networkservice.NetworkServiceClient { + return &linkinitClient{vppConn: vppConn} +} + +func (l *linkinitClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) { + postponeCtxFunc := postpone.ContextWithValues(ctx) + + conn, err := next.Client(ctx).Request(ctx, request, opts...) + if err != nil { + return nil, err + } + + if err = initLink(ctx, conn, l.vppConn); err != nil { + closeCtx, cancelClose := postponeCtxFunc() + defer cancelClose() + + if _, closeErr := l.Close(closeCtx, conn, opts...); closeErr != nil { + err = errors.Wrapf(err, "connection closed with error: %s", closeErr.Error()) + } + + return nil, err + } + + return conn, nil +} + +func (l *linkinitClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) { + _ = unsetLink(ctx, conn) + return next.Client(ctx).Close(ctx, conn, opts...) +} + +func initLink(ctx context.Context, conn *networkservice.Connection, vppConn api.Connection) error { + if mechanism := vlanmech.ToMechanism(conn.GetMechanism()); mechanism != nil { + kernelLink, err := getKernelLink(ctx) + if err != nil { + return err + } + hostIFName := kernelLink.GetName() + link := kernelLink.GetLink() + log.FromContext(ctx).WithField("linkinit", "Request").Debugf("initLink", hostIFName) + client, err := interfaces.NewServiceClient(vppConn).SwInterfaceDump(ctx, &interfaces.SwInterfaceDump{ + NameFilterValid: true, + NameFilter: hostIFName, + }) + if err == nil { + _, err = client.Recv() + /* No interface found, it can be created */ + if err == io.EOF { + err = createInterface(ctx, vppConn, link) + if err != nil { + return errors.Wrapf(err, "error attempting to create AF_PACKET inteface") + } + } + } else { + return errors.Wrapf(err, "error attempting to get interface dump client create AF_PACKET inteface") + } + + // Set promiscuous mode + now := time.Now() + + err = netlink.SetPromiscOn(link) + if err != nil { + log.FromContext(ctx). + WithField("duration", time.Since(now)). + WithField("HostInterfaceName", hostIFName). + WithField("netlink", "SetPromiscOn"). + Warn("Promiscuous mode not set!") + } else { + log.FromContext(ctx). + WithField("duration", time.Since(now)). + WithField("HostInterfaceName", hostIFName). + WithField("netlink", "SetPromiscOn").Debug("completed") + } + } + return nil +} + +func unsetLink(ctx context.Context, conn *networkservice.Connection) error { + if mechanism := vlanmech.ToMechanism(conn.GetMechanism()); mechanism != nil { + now := time.Now() + kernelLink, err := getKernelLink(ctx) + if err != nil { + return err + } + link := kernelLink.GetLink() + + // Set promiscuous mode + err = netlink.SetPromiscOff(link) + if err != nil { + log.FromContext(ctx).Warn("Promiscuous mode not set OFF!") + } + log.FromContext(ctx). + WithField("duration", time.Since(now)). + WithField("HostInterfaceName", link.Attrs().Name). + WithField("netlink", "SetPromiscOff").Debug("completed") + } + return nil +} + +func getKernelLink(ctx context.Context) (kernellink.Link, error) { + pciAddressInfo, ok := pciaddresspool.Load(ctx) + if !ok { + return nil, errors.Errorf("no link found") + } + hostNetNS, err := nshandle.Current() + if err != nil { + return nil, err + } + defer func() { _ = hostNetNS.Close() }() + kernelLink, err := kernellink.FindHostDevice(pciAddressInfo.GetPCIAddress(), "", hostNetNS) + if err != nil { + return nil, err + } + return kernelLink, nil +} + +func createInterface(ctx context.Context, vppConn api.Connection, link netlink.Link) error { + swIfIndex, err := initvpp.CreateAfPacket(ctx, vppConn, link) + if err != nil { + return err + } + + if aclErr := initvpp.DenyAllACLToInterface(ctx, vppConn, swIfIndex); aclErr != nil { + return aclErr + } + + now := time.Now() + _, err = interfaces.NewServiceClient(vppConn).SwInterfaceSetFlags(ctx, &interfaces.SwInterfaceSetFlags{ + SwIfIndex: swIfIndex, + Flags: interface_types.IF_STATUS_API_FLAG_ADMIN_UP, + }) + if err != nil { + return err + } + log.FromContext(ctx). + WithField("swIfIndex", swIfIndex). + WithField("duration", time.Since(now)). + WithField("vppapi", "SwInterfaceSetFlags").Debug("completed") + return nil +} diff --git a/pkg/networkservice/mechanisms/vlan/linkinit/doc.go b/pkg/networkservice/mechanisms/vlan/linkinit/doc.go new file mode 100644 index 00000000..ec770b2c --- /dev/null +++ b/pkg/networkservice/mechanisms/vlan/linkinit/doc.go @@ -0,0 +1,18 @@ +// Copyright (c) 2021 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package linkinit provides chain elements for setting link to AF_PACKET interface if it is not already set +package linkinit diff --git a/pkg/networkservice/mechanisms/vlan/pciaddresspool/client.go b/pkg/networkservice/mechanisms/vlan/pciaddresspool/client.go new file mode 100644 index 00000000..f7426748 --- /dev/null +++ b/pkg/networkservice/mechanisms/vlan/pciaddresspool/client.go @@ -0,0 +1,116 @@ +// Copyright (c) 2021 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pciaddresspool provides chain elements to select and free PCI Addresses from pool +package pciaddresspool + +import ( + "context" + "sync" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/pkg/errors" + "google.golang.org/grpc" + + "github.com/networkservicemesh/api/pkg/api/networkservice" + vlanmech "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vlan" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" + "github.com/networkservicemesh/sdk/pkg/tools/log" + "github.com/networkservicemesh/sdk/pkg/tools/postpone" +) + +const ( + serviceDomainLabel = "serviceDomain" +) + +// PCIAddressPool is an interface of domain.Pool +type PCIAddressPool interface { + Select(tokenName string) (string, error) + Free(tokenName string) +} + +type pciAddressPoolClient struct { + resourceLock sync.Locker + pciAddressPool PCIAddressPool +} + +// NewClient returns a new resource pool client chain element +func NewClient( + resourceLock sync.Locker, + pciAddressPool PCIAddressPool) networkservice.NetworkServiceClient { + return &pciAddressPoolClient{ + resourceLock: resourceLock, + pciAddressPool: pciAddressPool, + } +} + +func (r *pciAddressPoolClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) { + log.FromContext(ctx).WithField("pciAddressPoolClient", "Request").Debug("called") + postponeCtxFunc := postpone.ContextWithValues(ctx) + + conn, err := next.Client(ctx).Request(ctx, request, opts...) + if err != nil { + return nil, err + } + + if err = r.setPCIAddress(ctx, conn); err != nil { + closeCtx, cancelClose := postponeCtxFunc() + defer cancelClose() + + if _, closeErr := r.Close(closeCtx, conn, opts...); closeErr != nil { + err = errors.Wrapf(err, "connection closed with error: %s", closeErr.Error()) + } + return nil, err + } + + return conn, nil +} + +func (r *pciAddressPoolClient) setPCIAddress(ctx context.Context, conn *networkservice.Connection) error { + logger := log.FromContext(ctx).WithField("pciAddressPoolClient", "Request") + if mechanism := vlanmech.ToMechanism(conn.GetMechanism()); mechanism != nil { + r.resourceLock.Lock() + defer r.resourceLock.Unlock() + + tokenName := conn.GetLabels()[serviceDomainLabel] + pfPCIAddr, err := r.pciAddressPool.Select(tokenName) + if err != nil { + return errors.Wrapf(err, "failed to select PF for: %s", tokenName) + } + logger.Debugf("Got PCIAddress %v for tokenName %s", pfPCIAddr, tokenName) + + store(ctx, PCIAddressInfo{ + pciAddress: pfPCIAddr, + tokenName: tokenName, + }) + } + return nil +} + +func (r *pciAddressPoolClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) { + r.close(ctx) + return next.Client(ctx).Close(ctx, conn, opts...) +} + +func (r *pciAddressPoolClient) close(ctx context.Context) { + pciAddressInfo, ok := loadAndDelete(ctx) + if !ok { + return + } + r.resourceLock.Lock() + defer r.resourceLock.Unlock() + r.pciAddressPool.Free(pciAddressInfo.tokenName) +} diff --git a/pkg/networkservice/mechanisms/vlan/pciaddresspool/metadata.go b/pkg/networkservice/mechanisms/vlan/pciaddresspool/metadata.go new file mode 100644 index 00000000..eadb4f38 --- /dev/null +++ b/pkg/networkservice/mechanisms/vlan/pciaddresspool/metadata.go @@ -0,0 +1,61 @@ +// Copyright (c) 2021 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pciaddresspool + +import ( + "context" + + "github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata" +) + +type key struct{} + +// PCIAddressInfo store the PCI Address assigned to sarvice domain +type PCIAddressInfo struct { + pciAddress string + tokenName string +} + +func store(ctx context.Context, pciAddressInfo PCIAddressInfo) { + metadata.Map(ctx, true).Store(key{}, pciAddressInfo) +} + +// Load returns the base PCI address stored in per Connection.Id metadata, or nil if no +// value is present. +// The ok result indicates whether value was found in the per Connection.Id metadata. +func Load(ctx context.Context) (value PCIAddressInfo, ok bool) { + rawValue, ok := metadata.Map(ctx, true).Load(key{}) + if !ok { + return + } + value, ok = rawValue.(PCIAddressInfo) + return value, ok +} + +func loadAndDelete(ctx context.Context) (value PCIAddressInfo, ok bool) { + rawValue, ok := metadata.Map(ctx, true).LoadAndDelete(key{}) + if !ok { + return + } + value, ok = rawValue.(PCIAddressInfo) + return value, ok +} + +// GetPCIAddress returns the pci address +func (pai *PCIAddressInfo) GetPCIAddress() string { + return pai.pciAddress +} diff --git a/pkg/tools/domain/config.yml b/pkg/tools/domain/config.yml new file mode 100644 index 00000000..8d5783d5 --- /dev/null +++ b/pkg/tools/domain/config.yml @@ -0,0 +1,16 @@ +physicalFunctions: + 0000:00:04.0: + pfKernelDriver: pf-driver + vfKernelDriver: vf-driver + capabilities: + - default + serviceDomains: + - service.domain.1 + 0000:00:05.0: + pfKernelDriver: pf-driver + vfKernelDriver: vf-driver + capabilities: + - default + serviceDomains: + - service.domain.1 + - service.domain.2 diff --git a/pkg/tools/domain/pool.go b/pkg/tools/domain/pool.go new file mode 100644 index 00000000..62750b4c --- /dev/null +++ b/pkg/tools/domain/pool.go @@ -0,0 +1,125 @@ +// Copyright (c) 2021 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package domain provides a pool for service domain to PCI address mappings +package domain + +import ( + "sort" + "strings" + + "github.com/pkg/errors" + + "github.com/networkservicemesh/sdk-sriov/pkg/sriov/config" +) + +// Pool manages the service domain to PCI address mapping +// WARNING: it is thread unsafe - if you want to use it concurrently, use some synchronization outside +type Pool struct { + pciInfos map[string]*pciInfo + pciToSDomain map[string]*serviceDomain +} + +type pciInfo struct { + pfPCIAddr string + freeDomainCount int +} + +type serviceDomain struct { + pcis []*pciInfo + selectedPfPCIAddr string +} + +// NewPool returns a new Pool +func NewPool(cfg *config.Config) *Pool { + p := &Pool{ + pciInfos: map[string]*pciInfo{}, + pciToSDomain: map[string]*serviceDomain{}, + } + for pfPCIAddr, pFun := range cfg.PhysicalFunctions { + pci := &pciInfo{ + pfPCIAddr: pfPCIAddr, + freeDomainCount: len(pFun.ServiceDomains), + } + p.pciInfos[pfPCIAddr] = pci + for _, sDomain := range pFun.ServiceDomains { + sd := p.pciToSDomain[sDomain] + if sd == nil { + sd = &serviceDomain{ + pcis: []*pciInfo{}, + selectedPfPCIAddr: "", + } + p.pciToSDomain[sDomain] = sd + } + sd.pcis = append(sd.pcis, pci) + } + } + return p +} + +// Select selects a physical function for the given driver type +func (p *Pool) Select(tokenName string) (string, error) { + if _, ok := p.pciToSDomain[tokenName]; !ok { + return "", errors.Errorf("no domain found: %s", tokenName) + } + + selected := p.pciToSDomain[tokenName].selectedPfPCIAddr + if selected != "" { + return selected, nil + } + + toselect := filterAndSort(p.pciToSDomain[tokenName].pcis) + if len(toselect) > 0 { + selected = toselect[0].pfPCIAddr + p.pciToSDomain[tokenName].selectedPfPCIAddr = selected + p.pciInfos[selected].freeDomainCount-- + return selected, nil + } + + // should not happen + return "", errors.Errorf("no free PF find for domain: %s", tokenName) +} + +func filterAndSort(pfs []*pciInfo) []*pciInfo { + toselect := []*pciInfo{} + for j := range pfs { + if pfs[j].freeDomainCount != 0 { + toselect = append(toselect, pfs[j]) + } + } + sort.Slice(toselect, func(i, k int) bool { + switch { + case toselect[i].freeDomainCount < toselect[k].freeDomainCount: + return true + case toselect[i].freeDomainCount > toselect[k].freeDomainCount: + return false + default: + return strings.Compare(toselect[i].pfPCIAddr, toselect[k].pfPCIAddr) < 0 + } + }) + return toselect +} + +// Free re-add the domain to the pool and free the assigned PCI address +func (p *Pool) Free(tokenName string) { + if _, ok := p.pciToSDomain[tokenName]; ok { + selected := p.pciToSDomain[tokenName].selectedPfPCIAddr + if selected != "" { + p.pciInfos[selected].freeDomainCount++ + p.pciToSDomain[tokenName].selectedPfPCIAddr = "" + } + } +} diff --git a/pkg/tools/domain/pool_test.go b/pkg/tools/domain/pool_test.go new file mode 100644 index 00000000..a496d90e --- /dev/null +++ b/pkg/tools/domain/pool_test.go @@ -0,0 +1,119 @@ +// Copyright (c) 2021 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package domain_test + +import ( + "context" + "testing" + + "github.com/networkservicemesh/sdk-sriov/pkg/sriov/config" + "github.com/stretchr/testify/require" + + "github.com/networkservicemesh/sdk-vpp/pkg/tools/domain" +) + +const ( + configFileName = "config.yml" + serviceDomain1 = "service.domain.1" + serviceDomain2 = "service.domain.2" + capability = "default" + pf1PciAddr = "0000:00:04.0" + pf2PciAddr = "0000:00:05.0" +) + +func TestPool_Select_Selected(t *testing.T) { + cfg, err := config.ReadConfig(context.TODO(), configFileName) + require.NoError(t, err) + + p := domain.NewPool(cfg) + + // Should be the same PF for the same domain. + + pfPCIAddr, err := p.Select(serviceDomain1) + require.NoError(t, err) + require.Equal(t, pf1PciAddr, pfPCIAddr) // <-- initial + + pfPCIAddr, err = p.Select(serviceDomain1) + require.NoError(t, err) + require.Equal(t, pf1PciAddr, pfPCIAddr) // <-- same +} + +func TestPool_Select_Other(t *testing.T) { + cfg, err := config.ReadConfig(context.TODO(), configFileName) + require.NoError(t, err) + + p := domain.NewPool(cfg) + + // Should be the same PF for the same domain. + + pfPCIAddr, err := p.Select(serviceDomain1) + require.NoError(t, err) + require.Equal(t, pf1PciAddr, pfPCIAddr) // <-- initial + + pfPCIAddr, err = p.Select(serviceDomain2) + require.NoError(t, err) + require.Equal(t, pf2PciAddr, pfPCIAddr) // <-- different +} + +func TestPool_Select_OtherDifferentOrder(t *testing.T) { + cfg, err := config.ReadConfig(context.TODO(), configFileName) + require.NoError(t, err) + + p := domain.NewPool(cfg) + + // Should be the same PF for the same domain. + + pfPCIAddr, err := p.Select(serviceDomain2) + require.NoError(t, err) + require.Equal(t, pf2PciAddr, pfPCIAddr) // <-- initial + + pfPCIAddr, err = p.Select(serviceDomain1) + require.NoError(t, err) + require.Equal(t, pf1PciAddr, pfPCIAddr) // <-- different +} + +func TestPool_Select_Unknown(t *testing.T) { + cfg, err := config.ReadConfig(context.TODO(), configFileName) + require.NoError(t, err) + + p := domain.NewPool(cfg) + + // Should be the same PF for the same domain. + + pfPCIAddr, err := p.Select(capability) + require.NotNil(t, err, "no domain found: default") + require.Equal(t, "", pfPCIAddr) +} + +func TestPool_Free(t *testing.T) { + cfg, err := config.ReadConfig(context.TODO(), configFileName) + require.NoError(t, err) + + p := domain.NewPool(cfg) + + // Should be the same PF for the same domain. + + pfPCIAddr, err := p.Select(serviceDomain1) + require.NoError(t, err) + require.Equal(t, pf1PciAddr, pfPCIAddr) // <-- initial + + p.Free(serviceDomain1) + + pfPCIAddr, err = p.Select(serviceDomain1) + require.NoError(t, err) + require.Equal(t, pf1PciAddr, pfPCIAddr) // <-- same +}