network: ensure we do not clean ipam leases cache during an update of…

… networkd #520

go.mod

@@ -57,6 +57,7 @@ require (
 	github.com/stretchr/testify v1.4.0
 	github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect
 	github.com/tcnksm/go-input v0.0.0-20180404061846-548a7d7a8ee8
+	github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae
 	github.com/threefoldtech/zbus v0.1.2
 	github.com/urfave/cli v1.22.1
 	github.com/vishvananda/netlink v1.0.0

go.sum

@@ -324,6 +324,8 @@ github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/tcnksm/go-input v0.0.0-20180404061846-548a7d7a8ee8 h1:RB0v+/pc8oMzPsN97aZYEwNuJ6ouRJ2uhjxemJ9zvrY=
 github.com/tcnksm/go-input v0.0.0-20180404061846-548a7d7a8ee8/go.mod h1:IlWNj9v/13q7xFbaK4mbyzMNwrZLaWSHx/aibKIZuIg=
+github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae h1:vgGSvdW5Lqg+I1aZOlG32uyE6xHpLdKhZzcTEktz5wM=
+github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae/go.mod h1:quDq6Se6jlGwiIKia/itDZxqC5rj6/8OdFyMMAwTxCs=
 github.com/threefoldtech/zbus v0.1.2 h1:fBuRy7FaI7UVdZNnJQUqumejm14IHtzEqYYtSxTsjSI=
 github.com/threefoldtech/zbus v0.1.2/go.mod h1:ZtiRpcqzEBJetVQDsEbw0p48h/AF3O1kf0tvd30I0BU=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=

pkg/network/ndmz/ipam.go

@@ -12,7 +12,7 @@ import (
 // in the network with netID, and NetResource.
 func allocateIPv4(networkID string) (*net.IPNet, error) {
 	// FIXME: path to the cache disk shouldn't be hardcoded here
-	store, err := disk.New("dmz", "/var/cache/modules/networkd/lease")
+	store, err := disk.New("ndmz", ipamPath)
 	if err != nil {
 		return nil, err
 	}

pkg/network/ndmz/ndmz.go

@@ -5,10 +5,13 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/cenkalti/backoff/v3"
+	"github.com/termie/go-shutil"
 	"github.com/threefoldtech/zos/pkg"
+	"github.com/threefoldtech/zos/pkg/app"
 	"github.com/threefoldtech/zos/pkg/network/ifaceutil"
 	"github.com/threefoldtech/zos/pkg/network/types"
 
@@ -42,10 +45,46 @@ const (
 	PublicIfaceName = "public"
 )
 
+var ipamPath = "/var/cache/modules/networkd/lease"
+
 //Create create the NDMZ network namespace and configure its default routes and addresses
 func Create(nodeID pkg.Identifier) error {
+	path := filepath.Join(ipamPath, "ndmz")
+
+	if app.IsFirstBoot("networkd-dmz") {
+		log.Info().Msg("first boot, empty reservation cache")
+
+		if err := os.RemoveAll(path); err != nil {
+			return err
+		}
+
+		// TODO @zaibon: remove once all the network has applies this fix
+
+		// This check ensure we do not delete current lease from the node
+		// running a previous version of this code.
+		// if the old leases directory exists, we copy the content
+		// into the new location and delete the old directory
+		var err error
+		oldFolder := filepath.Join(ipamPath, "dmz")
+		if _, err = os.Stat(oldFolder); err == nil {
+			if err := shutil.CopyTree(oldFolder, path, nil); err != nil {
+				return err
+			}
+			err = os.RemoveAll(oldFolder)
+		} else {
+			err = os.MkdirAll(path, 0770)
+		}
+		if err != nil {
+			return err
+		}
 
-	os.RemoveAll("/var/cache/modules/networkd/lease/dmz/")
+		if err := app.MarkBooted("networkd-dmz"); err != nil {
+			return errors.Wrap(err, "fail to mark provisiond as booted")
+		}
+
+	} else {
+		log.Info().Msg("restart detected, keep IPAM lease cache intact")
+	}
 
 	netNS, err := namespace.GetByName(NetNSNDMZ)
 	if err != nil {

pkg/network/ndmz/ndmz_test.go

@@ -1,10 +1,14 @@
 package ndmz
 
 import (
+	"fmt"
+	"io/ioutil"
 	"net"
+	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestIpv6(t *testing.T) {
@@ -32,3 +36,66 @@ func TestIpv6(t *testing.T) {
 	}
 
 }
+
+func TestIPv4Allocate(t *testing.T) {
+	var (
+		err    error
+		origin = ipamPath
+	)
+
+	ipamPath, err = ioutil.TempDir("", "")
+	require.NoError(t, err)
+	defer func() { ipamPath = origin }()
+
+	addr, err := allocateIPv4("network1")
+	require.NoError(t, err)
+
+	addr2, err := allocateIPv4("network1")
+	require.NoError(t, err)
+
+	assert.Equal(t, addr.String(), addr2.String())
+
+	addr3, err := allocateIPv4("network2")
+	require.NoError(t, err)
+	assert.NotEqual(t, addr.String(), addr3.String())
+}
+
+func TestIPv4AllocateConcurent(t *testing.T) {
+	var (
+		err    error
+		origin = ipamPath
+	)
+
+	ipamPath, err = ioutil.TempDir("", "")
+	require.NoError(t, err)
+	defer func() { ipamPath = origin }()
+
+	wg := sync.WaitGroup{}
+	wg.Add(10)
+
+	c := make(chan *net.IPNet)
+
+	for i := 0; i < 10; i++ {
+		go func(c chan *net.IPNet, i int) {
+			defer wg.Done()
+			for y := 0; y < 10; y++ {
+				nw := fmt.Sprintf("network%d%d", i, y)
+				addr, err := allocateIPv4(nw)
+				require.NoError(t, err)
+				c <- addr
+			}
+		}(c, i)
+	}
+
+	go func() {
+		addrs := map[*net.IPNet]struct{}{}
+		for addr := range c {
+			_, exists := addrs[addr]
+			require.False(t, exists)
+			addrs[addr] = struct{}{}
+		}
+	}()
+
+	wg.Wait()
+	close(c)
+}

pkg/provision/local_store.go

@@ -12,6 +12,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zos/pkg"
+	"github.com/threefoldtech/zos/pkg/app"
 	"github.com/threefoldtech/zos/pkg/versioned"
 )
 
@@ -79,25 +80,19 @@ type FSStore struct {
 
 // NewFSStore creates a in memory reservation store
 func NewFSStore(root string) (*FSStore, error) {
-	if err := os.MkdirAll("/var/run/modules", 0770); err != nil {
-		return nil, err
-	}
-
-	_, err := os.OpenFile("/var/run/modules/provisiond", os.O_RDONLY|os.O_CREATE|os.O_TRUNC|os.O_EXCL, 0666)
-	// if we managed to create the file, this means the node
-	// has just booted and this is the first time provisiond starts since boot
-	// so we empty the reservation cache
-
-	// if the file is already present, this mean this is just a restart/update of provisiond
-	// so we need to keep the reservation cache as it is
-	if err == nil {
+	if app.IsFirstBoot("provisiond") {
 		log.Info().Msg("first boot, empty reservation cache")
 		if err := os.RemoveAll(root); err != nil {
 			return nil, err
 		}
 		if err := os.MkdirAll(root, 0770); err != nil {
 			return nil, err
 		}
+
+		if err := app.MarkBooted("provisiond"); err != nil {
+			return nil, errors.Wrap(err, "fail to mark provisiond as booted")
+		}
+
 	} else {
 		log.Info().Msg("restart detected, keep reservation cache intact")
 	}