network: ensure we do not clean ipam leases cache during an update of…

… networkd

pkg/app/boot.go

@@ -0,0 +1,27 @@
+package app
+
+import (
+	"os"
+	"path/filepath"
+)
+
+const bootedPath = "/var/run/modules"
+
+// MarkBooted creates a file in a memory
+// this file then can be used to check if "something" has been restared
+// if its the first time it starts
+func MarkBooted(name string) error {
+	if err := os.MkdirAll(bootedPath, 0770); err != nil {
+		return err
+	}
+	path := filepath.Join(bootedPath, name)
+	_, err := os.OpenFile(path, os.O_RDONLY|os.O_CREATE|os.O_TRUNC|os.O_EXCL, 0666)
+	return err
+}
+
+// IsFirstBoot checks if the a file has been created by MarkBooted function
+func IsFirtBoot(name string) bool {
+	path := filepath.Join(bootedPath, name)
+	_, err := os.Stat(path)
+	return !(err == nil)
+}

pkg/network/ndmz/ipam.go

@@ -12,7 +12,7 @@ import (
 // in the network with netID, and NetResource.
 func allocateIPv4(networkID string) (*net.IPNet, error) {
 	// FIXME: path to the cache disk shouldn't be hardcoded here
-	store, err := disk.New("dmz", "/var/cache/modules/networkd/lease")
+	store, err := disk.New("dmz", ipamPath)
 	if err != nil {
 		return nil, err
 	}

pkg/network/ndmz/ndmz.go

@@ -5,10 +5,12 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/cenkalti/backoff/v3"
 	"github.com/threefoldtech/zos/pkg"
+	"github.com/threefoldtech/zos/pkg/app"
 	"github.com/threefoldtech/zos/pkg/network/ifaceutil"
 	"github.com/threefoldtech/zos/pkg/network/types"
 
@@ -42,10 +44,30 @@ const (
 	PublicIfaceName = "public"
 )
 
+var ipamPath = "/var/cache/modules/networkd/lease"
+
 //Create create the NDMZ network namespace and configure its default routes and addresses
 func Create(nodeID pkg.Identifier) error {
+	path := filepath.Join(ipamPath, "dmz")
+
+	if app.IsFirtBoot("networkd-dmz") {
+		log.Info().Msg("first boot, empty reservation cache")
+
+		if err := os.RemoveAll(path); err != nil {
+			return err
+		}
+
+		if err := os.MkdirAll(path, 0770); err != nil {
+			return err
+		}
+
+		if err := app.MarkBooted("networkd-dmz"); err != nil {
+			return errors.Wrap(err, "fail to mark provisiond as booted")
+		}
 
-	os.RemoveAll("/var/cache/modules/networkd/lease/dmz/")
+	} else {
+		log.Info().Msg("restart detected, keep IPAM lease cache intact")
+	}
 
 	netNS, err := namespace.GetByName(NetNSNDMZ)
 	if err != nil {

pkg/network/ndmz/ndmz_test.go

@@ -1,10 +1,14 @@
 package ndmz
 
 import (
+	"fmt"
+	"io/ioutil"
 	"net"
+	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestIpv6(t *testing.T) {
@@ -32,3 +36,66 @@ func TestIpv6(t *testing.T) {
 	}
 
 }
+
+func TestIPv4Allocate(t *testing.T) {
+	var (
+		err    error
+		origin = ipamPath
+	)
+
+	ipamPath, err = ioutil.TempDir("", "")
+	require.NoError(t, err)
+	defer func() { ipamPath = origin }()
+
+	addr, err := allocateIPv4("network1")
+	require.NoError(t, err)
+
+	addr2, err := allocateIPv4("network1")
+	require.NoError(t, err)
+
+	assert.Equal(t, addr.String(), addr2.String())
+
+	addr3, err := allocateIPv4("network2")
+	require.NoError(t, err)
+	assert.NotEqual(t, addr.String(), addr3.String())
+}
+
+func TestIPv4AllocateConcurent(t *testing.T) {
+	var (
+		err    error
+		origin = ipamPath
+	)
+
+	ipamPath, err = ioutil.TempDir("", "")
+	require.NoError(t, err)
+	defer func() { ipamPath = origin }()
+
+	wg := sync.WaitGroup{}
+	wg.Add(10)
+
+	c := make(chan *net.IPNet)
+
+	for i := 0; i < 10; i++ {
+		go func(c chan *net.IPNet, i int) {
+			defer wg.Done()
+			for y := 0; y < 10; y++ {
+				nw := fmt.Sprintf("network%d%d", i, y)
+				addr, err := allocateIPv4(nw)
+				require.NoError(t, err)
+				c <- addr
+			}
+		}(c, i)
+	}
+
+	go func() {
+		addrs := map[*net.IPNet]struct{}{}
+		for addr := range c {
+			_, exists := addrs[addr]
+			require.False(t, exists)
+			addrs[addr] = struct{}{}
+		}
+	}()
+
+	wg.Wait()
+	close(c)
+}

pkg/provision/local_store.go

@@ -12,6 +12,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zos/pkg"
+	"github.com/threefoldtech/zos/pkg/app"
 	"github.com/threefoldtech/zos/pkg/versioned"
 )
 
@@ -79,25 +80,19 @@ type FSStore struct {
 
 // NewFSStore creates a in memory reservation store
 func NewFSStore(root string) (*FSStore, error) {
-	if err := os.MkdirAll("/var/run/modules", 0770); err != nil {
-		return nil, err
-	}
-
-	_, err := os.OpenFile("/var/run/modules/provisiond", os.O_RDONLY|os.O_CREATE|os.O_TRUNC|os.O_EXCL, 0666)
-	// if we managed to create the file, this means the node
-	// has just booted and this is the first time provisiond starts since boot
-	// so we empty the reservation cache
-
-	// if the file is already present, this mean this is just a restart/update of provisiond
-	// so we need to keep the reservation cache as it is
-	if err == nil {
+	if app.IsFirtBoot("provisiond") {
 		log.Info().Msg("first boot, empty reservation cache")
 		if err := os.RemoveAll(root); err != nil {
 			return nil, err
 		}
 		if err := os.MkdirAll(root, 0770); err != nil {
 			return nil, err
 		}
+
+		if err := app.MarkBooted("provisiond"); err != nil {
+			return nil, errors.Wrap(err, "fail to mark provisiond as booted")
+		}
+
 	} else {
 		log.Info().Msg("restart detected, keep reservation cache intact")
 	}