Add a new chain element to check IP addresses are set on kernel interfaces

go.mod

@@ -11,8 +11,8 @@ require (
 	github.com/networkservicemesh/sdk v0.5.1-0.20240820090035-6fad31a9f0aa
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.8.4
-	github.com/vishvananda/netlink v1.2.1-beta.2.0.20220630165224-c591ada0fb2b
-	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74
+	github.com/vishvananda/netlink v1.3.0
+	github.com/vishvananda/netns v0.0.4
 	go.uber.org/goleak v1.3.0
 	golang.org/x/sys v0.18.0
 	google.golang.org/grpc v1.60.1

go.sum

@@ -63,11 +63,10 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/vishvananda/netlink v1.2.1-beta.2.0.20220630165224-c591ada0fb2b h1:CyMWBGvc1ZOvUBxW51DVTSIIAeJWWJJs+Ko3ouM/AVI=
-github.com/vishvananda/netlink v1.2.1-beta.2.0.20220630165224-c591ada0fb2b/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
-github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
-github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
-github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk=
+github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs=
+github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8=
+github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
 go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc=
 go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.43.0 h1:tFUz2BE6ucxU9PuPCwzbfDeQjMznIySJ4/73a3FSPUs=
@@ -101,10 +100,10 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

pkg/kernel/networkservice/connectioncontextkernel/client.go

@@ -2,6 +2,8 @@
 //
 // Copyright (c) 2021-2023 Nordix Foundation.
 //
+// Copyright (c) 2024 Cisco and/or its affiliates.
+//
 // SPDX-License-Identifier: Apache-2.0
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -31,6 +33,7 @@ import (
 	"github.com/networkservicemesh/sdk/pkg/networkservice/core/chain"
 
 	"github.com/networkservicemesh/sdk-kernel/pkg/kernel/networkservice/connectioncontextkernel/ipcontext/ipaddress"
+	"github.com/networkservicemesh/sdk-kernel/pkg/kernel/networkservice/connectioncontextkernel/ipcontext/ipaddresscheck"
 	"github.com/networkservicemesh/sdk-kernel/pkg/kernel/networkservice/connectioncontextkernel/ipcontext/ipneighbors"
 	"github.com/networkservicemesh/sdk-kernel/pkg/kernel/networkservice/connectioncontextkernel/ipcontext/routes"
 	"github.com/networkservicemesh/sdk-kernel/pkg/kernel/networkservice/connectioncontextkernel/pinggrouprange"
@@ -64,6 +67,7 @@ func NewClient() networkservice.NetworkServiceClient {
 		mtu.NewClient(),
 		ipneighbors.NewClient(),
 		routes.NewClient(),
+		ipaddresscheck.NewClient(),
 		ipaddress.NewClient(),
 		routelocalnet.NewClient(),
 		iptables4nattemplate.NewClient(),

pkg/kernel/networkservice/connectioncontextkernel/ipcontext/ipaddress/common.go

@@ -1,6 +1,6 @@
 // Copyright (c) 2021-2022 Nordix Foundation.
 //
-// Copyright (c) 2020-2023 Cisco and/or its affiliates.
+// Copyright (c) 2020-2024 Cisco and/or its affiliates.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -24,8 +24,8 @@ package ipaddress
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"net"
+	"os"
 	"time"
 
 	"github.com/networkservicemesh/api/pkg/api/networkservice"
@@ -85,29 +85,11 @@ func create(ctx context.Context, conn *networkservice.Connection, isClient bool)
 
 		disableIPv6Filename := fmt.Sprintf("/proc/sys/net/ipv6/conf/%s/disable_ipv6", l.Attrs().Name)
 		if err = nshandle.RunIn(forwarderNetNS, targetNetNS, func() error {
-			return ioutil.WriteFile(disableIPv6Filename, []byte("0"), 0o600)
+			return os.WriteFile(disableIPv6Filename, []byte("0"), 0o600)
 		}); err != nil {
 			return errors.Wrapf(err, "failed to set %s = 0", disableIPv6Filename)
 		}
 
-		ch := make(chan netlink.AddrUpdate)
-		done := make(chan struct{})
-
-		if err = netlink.AddrSubscribeWithOptions(ch, done, netlink.AddrSubscribeOptions{
-			Namespace:      &targetNetNS,
-			ReceiveTimeout: &unix.Timeval{Sec: 1},
-		}); err != nil {
-			return errors.Wrapf(err, "failed to subscribe for interface address updates")
-		}
-
-		defer func() {
-			close(done)
-			// `ch` should be fully read after the `done` close to prevent goroutine leak in `netlink.AddrSubscribeWithOptions`
-			// nolint: revive
-			for range ch {
-			}
-		}()
-
 		// Get IP addresses to add and to remove
 		toAdd, toRemove, err := getIPAddrDifferences(netlinkHandle, l, ipNets)
 		if err != nil {
@@ -121,10 +103,7 @@ func create(ctx context.Context, conn *networkservice.Connection, isClient bool)
 		}
 
 		// Add new IP addresses
-		if err := addNewIPAddrs(ctx, netlinkHandle, l, toAdd); err != nil {
-			return err
-		}
-		return waitForIPNets(ctx, ch, l, toAdd)
+		return addNewIPAddrs(ctx, netlinkHandle, l, toAdd)
 	}
 	return nil
 }
@@ -200,37 +179,3 @@ func getIPAddrDifferences(netlinkHandle *netlink.Handle, l netlink.Link, newIPs
 	}
 	return toAdd, toRemove, nil
 }
-
-func waitForIPNets(ctx context.Context, ch chan netlink.AddrUpdate, l netlink.Link, ipNets []*net.IPNet) error {
-	now := time.Now()
-	for {
-		if len(ipNets) == 0 {
-			return nil
-		}
-		j := -1
-		select {
-		case <-ctx.Done():
-			return errors.Wrapf(ctx.Err(), "timeout waiting for update to add ip addresses %s to %s (type: %s)", ipNets, l.Attrs().Name, l.Type())
-		case update, ok := <-ch:
-			if !ok {
-				return errors.Errorf("failed to receive update to add ip addresses %s to %s (type: %s)", ipNets, l.Attrs().Name, l.Type())
-			}
-			if update.LinkIndex == l.Attrs().Index {
-				for i := range ipNets {
-					if update.LinkAddress.IP.Equal(ipNets[i].IP) && update.Flags&unix.IFA_F_TENTATIVE == 0 {
-						j = i
-						log.FromContext(ctx).
-							WithField("AddrUpdate.LinkAddress", update.LinkAddress).
-							WithField("link.Name", l.Attrs().Name).
-							WithField("duration", time.Since(now)).
-							Debug("complete")
-						break
-					}
-				}
-			}
-		}
-		if j != -1 {
-			ipNets = append(ipNets[:j], ipNets[j+1:]...)
-		}
-	}
-}

pkg/kernel/networkservice/connectioncontextkernel/ipcontext/ipaddresscheck/client.go

@@ -0,0 +1,86 @@
+// Copyright (c) 2024 Cisco and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package ipaddresscheck
+
+import (
+	"context"
+
+	"github.com/golang/protobuf/ptypes/empty"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+
+	"github.com/networkservicemesh/api/pkg/api/networkservice"
+	"github.com/networkservicemesh/sdk/pkg/networkservice/core/next"
+	"github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata"
+	"github.com/networkservicemesh/sdk/pkg/tools/postpone"
+)
+
+type ipaddressCheckClient struct{}
+
+// NewClient provides a NetworkServiceClient that checks the IP is added to a kernel interface
+// on endpoint's side.
+//
+//	           Client
+//	+---------------------------+
+//	|                           |
+//	|                           |
+//	|                           |
+//	|                           |
+//	|                           |
+//	|                           |
+//	|                           |
+//	|                           +-------------------+
+//	|                           |          ipaddresscheck.NewClient()
+//	|                           |
+//	|                           |
+//	|                           |
+//	|                           |
+//	|                           |
+//	|                           |
+//	+---------------------------+
+func NewClient() networkservice.NetworkServiceClient {
+	return &ipaddressCheckClient{}
+}
+
+func (i *ipaddressCheckClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) {
+	postponeCtxFunc := postpone.ContextWithValues(ctx)
+
+	conn, err := next.Client(ctx).Request(ctx, request, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := create(ctx, conn, metadata.IsClient(i)); err != nil {
+		closeCtx, cancelClose := postponeCtxFunc()
+		defer cancelClose()
+
+		if _, closeErr := i.Close(closeCtx, conn, opts...); closeErr != nil {
+			err = errors.Wrapf(err, "connection closed with error: %s", closeErr.Error())
+		}
+
+		return nil, err
+	}
+
+	return conn, nil
+}
+
+func (i *ipaddressCheckClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) {
+	return next.Client(ctx).Close(ctx, conn, opts...)
+}

pkg/kernel/networkservice/connectioncontextkernel/ipcontext/ipaddresscheck/common.go

@@ -0,0 +1,100 @@
+// Copyright (c) 2024 Cisco and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package ipaddresscheck
+
+import (
+	"context"
+	"net"
+	"time"
+
+	"github.com/networkservicemesh/api/pkg/api/networkservice"
+	"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/kernel"
+	"github.com/networkservicemesh/sdk/pkg/tools/log"
+	"github.com/pkg/errors"
+	"github.com/vishvananda/netlink"
+
+	link "github.com/networkservicemesh/sdk-kernel/pkg/kernel"
+)
+
+func create(ctx context.Context, conn *networkservice.Connection, isClient bool) error {
+	if mechanism := kernel.ToMechanism(conn.GetMechanism()); mechanism != nil && mechanism.GetVLAN() == 0 {
+		ipNets := conn.GetContext().GetIpContext().GetSrcIPNets()
+		if isClient {
+			ipNets = conn.GetContext().GetIpContext().GetDstIPNets()
+		}
+		if ipNets == nil {
+			return nil
+		}
+
+		toCheck := make([]*net.IPNet, len(ipNets))
+		copy(toCheck, ipNets)
+
+		netlinkHandle, err := link.GetNetlinkHandle(mechanism.GetNetNSURL())
+		if err != nil {
+			return err
+		}
+		defer netlinkHandle.Close()
+
+		ifName := mechanism.GetInterfaceName()
+		l, err := netlinkHandle.LinkByName(ifName)
+		if err != nil {
+			return errors.Wrapf(err, "failed to find link %s", ifName)
+		}
+
+		return checkIPNets(ctx, netlinkHandle, l, toCheck)
+	}
+	return nil
+}
+
+func checkIPNets(ctx context.Context, netlinkHandle *netlink.Handle, l netlink.Link, ipNets []*net.IPNet) error {
+	now := time.Now()
+
+	current := make(map[string]struct{})
+	for _, ipNet := range ipNets {
+		current[ipNet.String()] = struct{}{}
+	}
+
+	for {
+		time.Sleep(time.Millisecond * 500)
+		if len(current) == 0 {
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return errors.Wrapf(ctx.Err(), "timeout waiting for update to add ip addresses %s to %s (type: %s)", ipNets, l.Attrs().Name, l.Type())
+		default:
+			addrs, err := netlinkHandle.AddrList(l, netlink.FAMILY_ALL)
+			if err != nil {
+				return errors.Wrapf(err, "failed to get ip addresses for %s", l.Attrs().Name)
+			}
+			for _, addr := range addrs {
+				addrString := addr.IPNet.String()
+				if _, ok := current[addrString]; ok {
+					delete(current, addrString)
+					log.FromContext(ctx).
+						WithField("LinkAddress", addr).
+						WithField("link.Name", l.Attrs().Name).
+						WithField("duration", time.Since(now)).
+						Debug("complete")
+				}
+			}
+		}
+	}
+}

pkg/kernel/networkservice/connectioncontextkernel/ipcontext/ipaddresscheck/doc.go

@@ -0,0 +1,18 @@
+// Copyright (c) 2024 Cisco and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package ipaddresscheck provides networkservice chain element that checks if addresses were added to kernel interfaces
+package ipaddresscheck