diff --git a/go.mod b/go.mod index 3ae428d7e..ee22ac262 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,7 @@ require ( github.com/stretchr/testify v1.4.0 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect github.com/tcnksm/go-input v0.0.0-20180404061846-548a7d7a8ee8 + github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae github.com/threefoldtech/zbus v0.1.2 github.com/urfave/cli v1.22.1 github.com/vishvananda/netlink v1.0.0 diff --git a/go.sum b/go.sum index 8df96da81..7afc6da0d 100644 --- a/go.sum +++ b/go.sum @@ -324,6 +324,8 @@ github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tcnksm/go-input v0.0.0-20180404061846-548a7d7a8ee8 h1:RB0v+/pc8oMzPsN97aZYEwNuJ6ouRJ2uhjxemJ9zvrY= github.com/tcnksm/go-input v0.0.0-20180404061846-548a7d7a8ee8/go.mod h1:IlWNj9v/13q7xFbaK4mbyzMNwrZLaWSHx/aibKIZuIg= +github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae h1:vgGSvdW5Lqg+I1aZOlG32uyE6xHpLdKhZzcTEktz5wM= +github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae/go.mod h1:quDq6Se6jlGwiIKia/itDZxqC5rj6/8OdFyMMAwTxCs= github.com/threefoldtech/zbus v0.1.2 h1:fBuRy7FaI7UVdZNnJQUqumejm14IHtzEqYYtSxTsjSI= github.com/threefoldtech/zbus v0.1.2/go.mod h1:ZtiRpcqzEBJetVQDsEbw0p48h/AF3O1kf0tvd30I0BU= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= diff --git a/pkg/network/ndmz/ipam.go b/pkg/network/ndmz/ipam.go index ba991b325..cce933a4d 100644 --- a/pkg/network/ndmz/ipam.go +++ b/pkg/network/ndmz/ipam.go @@ -12,7 +12,7 @@ import ( // in the network with netID, and NetResource. func allocateIPv4(networkID string) (*net.IPNet, error) { // FIXME: path to the cache disk shouldn't be hardcoded here - store, err := disk.New("dmz", "/var/cache/modules/networkd/lease") + store, err := disk.New("ndmz", ipamPath) if err != nil { return nil, err } diff --git a/pkg/network/ndmz/ndmz.go b/pkg/network/ndmz/ndmz.go index 03c4378fd..ad62145c0 100644 --- a/pkg/network/ndmz/ndmz.go +++ b/pkg/network/ndmz/ndmz.go @@ -5,10 +5,13 @@ import ( "fmt" "net" "os" + "path/filepath" "time" "github.com/cenkalti/backoff/v3" + "github.com/termie/go-shutil" "github.com/threefoldtech/zos/pkg" + "github.com/threefoldtech/zos/pkg/app" "github.com/threefoldtech/zos/pkg/network/ifaceutil" "github.com/threefoldtech/zos/pkg/network/types" @@ -42,10 +45,46 @@ const ( PublicIfaceName = "public" ) +var ipamPath = "/var/cache/modules/networkd/lease" + //Create create the NDMZ network namespace and configure its default routes and addresses func Create(nodeID pkg.Identifier) error { + path := filepath.Join(ipamPath, "ndmz") + + if app.IsFirstBoot("networkd-dmz") { + log.Info().Msg("first boot, empty reservation cache") + + if err := os.RemoveAll(path); err != nil { + return err + } + + // TODO @zaibon: remove once all the network has applies this fix + + // This check ensure we do not delete current lease from the node + // running a previous version of this code. + // if the old leases directory exists, we copy the content + // into the new location and delete the old directory + var err error + oldFolder := filepath.Join(ipamPath, "dmz") + if _, err = os.Stat(oldFolder); err == nil { + if err := shutil.CopyTree(oldFolder, path, nil); err != nil { + return err + } + err = os.RemoveAll(oldFolder) + } else { + err = os.MkdirAll(path, 0770) + } + if err != nil { + return err + } - os.RemoveAll("/var/cache/modules/networkd/lease/dmz/") + if err := app.MarkBooted("networkd-dmz"); err != nil { + return errors.Wrap(err, "fail to mark provisiond as booted") + } + + } else { + log.Info().Msg("restart detected, keep IPAM lease cache intact") + } netNS, err := namespace.GetByName(NetNSNDMZ) if err != nil { diff --git a/pkg/network/ndmz/ndmz_test.go b/pkg/network/ndmz/ndmz_test.go index bc82c8c3b..376f5e20f 100644 --- a/pkg/network/ndmz/ndmz_test.go +++ b/pkg/network/ndmz/ndmz_test.go @@ -1,10 +1,14 @@ package ndmz import ( + "fmt" + "io/ioutil" "net" + "sync" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestIpv6(t *testing.T) { @@ -32,3 +36,66 @@ func TestIpv6(t *testing.T) { } } + +func TestIPv4Allocate(t *testing.T) { + var ( + err error + origin = ipamPath + ) + + ipamPath, err = ioutil.TempDir("", "") + require.NoError(t, err) + defer func() { ipamPath = origin }() + + addr, err := allocateIPv4("network1") + require.NoError(t, err) + + addr2, err := allocateIPv4("network1") + require.NoError(t, err) + + assert.Equal(t, addr.String(), addr2.String()) + + addr3, err := allocateIPv4("network2") + require.NoError(t, err) + assert.NotEqual(t, addr.String(), addr3.String()) +} + +func TestIPv4AllocateConcurent(t *testing.T) { + var ( + err error + origin = ipamPath + ) + + ipamPath, err = ioutil.TempDir("", "") + require.NoError(t, err) + defer func() { ipamPath = origin }() + + wg := sync.WaitGroup{} + wg.Add(10) + + c := make(chan *net.IPNet) + + for i := 0; i < 10; i++ { + go func(c chan *net.IPNet, i int) { + defer wg.Done() + for y := 0; y < 10; y++ { + nw := fmt.Sprintf("network%d%d", i, y) + addr, err := allocateIPv4(nw) + require.NoError(t, err) + c <- addr + } + }(c, i) + } + + go func() { + addrs := map[*net.IPNet]struct{}{} + for addr := range c { + _, exists := addrs[addr] + require.False(t, exists) + addrs[addr] = struct{}{} + } + }() + + wg.Wait() + close(c) +} diff --git a/pkg/provision/local_store.go b/pkg/provision/local_store.go index 04aa2815c..b2cb98390 100644 --- a/pkg/provision/local_store.go +++ b/pkg/provision/local_store.go @@ -12,6 +12,7 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/threefoldtech/zos/pkg" + "github.com/threefoldtech/zos/pkg/app" "github.com/threefoldtech/zos/pkg/versioned" ) @@ -79,18 +80,7 @@ type FSStore struct { // NewFSStore creates a in memory reservation store func NewFSStore(root string) (*FSStore, error) { - if err := os.MkdirAll("/var/run/modules", 0770); err != nil { - return nil, err - } - - _, err := os.OpenFile("/var/run/modules/provisiond", os.O_RDONLY|os.O_CREATE|os.O_TRUNC|os.O_EXCL, 0666) - // if we managed to create the file, this means the node - // has just booted and this is the first time provisiond starts since boot - // so we empty the reservation cache - - // if the file is already present, this mean this is just a restart/update of provisiond - // so we need to keep the reservation cache as it is - if err == nil { + if app.IsFirstBoot("provisiond") { log.Info().Msg("first boot, empty reservation cache") if err := os.RemoveAll(root); err != nil { return nil, err @@ -98,6 +88,11 @@ func NewFSStore(root string) (*FSStore, error) { if err := os.MkdirAll(root, 0770); err != nil { return nil, err } + + if err := app.MarkBooted("provisiond"); err != nil { + return nil, errors.Wrap(err, "fail to mark provisiond as booted") + } + } else { log.Info().Msg("restart detected, keep reservation cache intact") }