diff --git a/.github/workflows/publish-development.yaml b/.github/workflows/publish-development.yaml index 16a15f8f9..2168621d6 100644 --- a/.github/workflows/publish-development.yaml +++ b/.github/workflows/publish-development.yaml @@ -11,10 +11,10 @@ jobs: name: Build and upload runs-on: ubuntu-latest steps: - - name: Set up Go 1.16 + - name: Set up Go 1.17 uses: actions/setup-go@v1 with: - go-version: 1.16 + go-version: 1.17 id: go - name: Checkout code into the Go module directory diff --git a/.github/workflows/publish-pre-release.yaml b/.github/workflows/publish-pre-release.yaml index 39bb52a96..e2b19a83c 100644 --- a/.github/workflows/publish-pre-release.yaml +++ b/.github/workflows/publish-pre-release.yaml @@ -19,10 +19,10 @@ jobs: name: Build and upload runs-on: ubuntu-latest steps: - - name: Set up Go 1.16 + - name: Set up Go 1.17 uses: actions/setup-go@v1 with: - go-version: 1.16 + go-version: 1.17 id: go - name: Checkout code into the Go module directory diff --git a/.github/workflows/publish-release.yaml b/.github/workflows/publish-release.yaml index 4a3ad820a..d14beda4b 100644 --- a/.github/workflows/publish-release.yaml +++ b/.github/workflows/publish-release.yaml @@ -20,10 +20,10 @@ jobs: name: Build and upload runs-on: ubuntu-latest steps: - - name: Set up Go 1.16 + - name: Set up Go 1.17 uses: actions/setup-go@v1 with: - go-version: 1.16 + go-version: 1.17 id: go - name: Checkout code into the Go module directory diff --git a/.github/workflows/test-tools.yaml b/.github/workflows/test-tools.yaml index cf22fab12..91fc2a385 100644 --- a/.github/workflows/test-tools.yaml +++ b/.github/workflows/test-tools.yaml @@ -10,10 +10,10 @@ jobs: name: Running Tools Tests runs-on: ubuntu-latest steps: - - name: Set up Go 1.16 + - name: Set up Go 1.17 uses: actions/setup-go@v1 with: - go-version: 1.16 + go-version: 1.17 id: go - name: Checkout code into the Go module directory diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2537cff90..2ee25aeca 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,10 +6,10 @@ jobs: name: Running Daemon Tests runs-on: ubuntu-latest steps: - - name: Set up Go 1.16 + - name: Set up Go 1.17 uses: actions/setup-go@v1 with: - go-version: 1.16 + go-version: 1.17 id: go - name: Prepare dependencies diff --git a/client/node.go b/client/node.go index a5e3d88f6..abbef86c0 100644 --- a/client/node.go +++ b/client/node.go @@ -199,6 +199,18 @@ func (n *NodeClient) NetworkGetPublicConfig(ctx context.Context) (cfg pkg.Public return } +// NetworkGetPublicConfig returns the current public node network configuration. A node with a +// public config can be used as an access node for wireguard. +func (n *NodeClient) NetworkSetPublicConfig(ctx context.Context, cfg pkg.PublicConfig) error { + const cmd = "zos.network.public_config_set" + + if err := n.bus.Call(ctx, n.nodeTwin, cmd, cfg, nil); err != nil { + return err + } + + return nil +} + func (n *NodeClient) SystemDMI(ctx context.Context) (result dmi.DMI, err error) { const cmd = "zos.system.dmi" diff --git a/cmds/modules/capacityd/main.go b/cmds/modules/capacityd/main.go index 1cee99abc..09eedc674 100644 --- a/cmds/modules/capacityd/main.go +++ b/cmds/modules/capacityd/main.go @@ -117,6 +117,8 @@ func action(cli *cli.Context) error { return errors.Wrap(err, "failed during node registration") } + // TODO: monitor change to yggdrasil Ip and update the twin according + log.Info().Uint32("node", node).Uint32("twin", twin).Msg("node registered") // uptime update diff --git a/cmds/modules/capacityd/register.go b/cmds/modules/capacityd/register.go index d67aa48e9..64e002d28 100644 --- a/cmds/modules/capacityd/register.go +++ b/cmds/modules/capacityd/register.go @@ -14,6 +14,7 @@ import ( "github.com/rs/zerolog/log" "github.com/shirou/gopsutil/host" "github.com/threefoldtech/zbus" + "github.com/threefoldtech/zos/pkg" "github.com/threefoldtech/zos/pkg/environment" "github.com/threefoldtech/zos/pkg/geoip" "github.com/threefoldtech/zos/pkg/gridtypes" @@ -27,16 +28,42 @@ const ( ) func registration(ctx context.Context, cl zbus.Client, cap gridtypes.Capacity) (nodeID, twinID uint32, err error) { + var ( + netMgr = stubs.NewNetworkerStub(cl) + ) + env, err := environment.Get() if err != nil { return 0, 0, errors.Wrap(err, "failed to get runtime environment for zos") } + // we need to collect all node information here + // - we already have capacity + // - we get the location (will not change after initial registration) loc, err := geoip.Fetch() if err != nil { log.Fatal().Err(err).Msg("fetch location") } + // - node public config + + var pub *pkg.PublicConfig + if pubCfg, err := netMgr.GetPublicConfig(ctx); err == nil { + pub = &pubCfg + } + + // - yggdrasil + // node always register with ndmz address + var ygg net.IP + if ips, err := netMgr.Addrs(ctx, yggdrasil.YggNSInf, "ndmz"); err == nil { + if len(ips) == 0 { + return 0, 0, errors.Wrap(err, "failed to get yggdrasil ip") + } + if len(ips) == 1 { + ygg = net.IP(ips[0]) + } + } + log.Debug(). Uint64("cru", cap.CRU). Uint64("mru", uint64(cap.MRU)). @@ -53,7 +80,7 @@ func registration(ctx context.Context, cl zbus.Client, cap gridtypes.Capacity) ( exp.MaxInterval = 2 * time.Minute bo := backoff.WithContext(exp, ctx) err = backoff.RetryNotify(func() error { - nodeID, twinID, err = registerNode(ctx, env, cl, sub, cap, loc) + nodeID, twinID, err = registerNode(ctx, env, cl, sub, cap, loc, pub, ygg) return err }, bo, retryNotify) @@ -61,9 +88,92 @@ func registration(ctx context.Context, cl zbus.Client, cap gridtypes.Capacity) ( return 0, 0, errors.Wrap(err, "failed to register node") } + // well the node is registed. but now we need to monitor changes to networking + // to update the node + go func() { + for { + err := watch(ctx, env, cl, sub, cap, loc, pub, ygg) + if errors.Is(err, context.Canceled) { + return + } else if err != nil { + log.Error().Err(err).Msg("watching network changes failed") + <-time.After(3 * time.Second) + } + } + }() + return nodeID, twinID, nil } +func watch( + ctx context.Context, + env environment.Environment, + cl zbus.Client, + sub *substrate.Substrate, + cap gridtypes.Capacity, + loc geoip.Location, + pub *pkg.PublicConfig, + ygg net.IP, +) error { + var ( + netMgr = stubs.NewNetworkerStub(cl) + ) + + pubCh, err := netMgr.PublicAddresses(ctx) + if err != nil { + return errors.Wrap(err, "failed to register on public config changes") + } + + yggCh, err := netMgr.YggAddresses(ctx) + if err != nil { + return errors.Wrap(err, "failed to register on ygg ips changes") + } + + log.Info().Msg("start watching node network changes") + for { + update := false + select { + case <-ctx.Done(): + return ctx.Err() + case pubInput := <-pubCh: + var pubNew *pkg.PublicConfig + if pubInput.HasPublicConfig { + pubNew = &pubInput.PublicConfig + } + if !reflect.DeepEqual(pub, pubNew) { + pub = pubNew + update = true + } + case yggInput := <-yggCh: + var yggNew net.IP + if len(yggInput) > 0 { + yggNew = yggInput[0].IP + } + if !yggNew.Equal(ygg) { + ygg = yggNew + update = true + } + } + + if !update { + continue + } + // some of the node config has changed. we need to try register it again + log.Debug().Msg("node setup seems to have been changed. re-register") + exp := backoff.NewExponentialBackOff() + exp.MaxInterval = 2 * time.Minute + bo := backoff.WithContext(exp, ctx) + err = backoff.RetryNotify(func() error { + _, _, err := registerNode(ctx, env, cl, sub, cap, loc, pub, ygg) + return err + }, bo, retryNotify) + + if err != nil { + return errors.Wrap(err, "failed to register node") + } + } +} + func retryNotify(err error, d time.Duration) { log.Warn().Err(err).Str("sleep", d.String()).Msg("registration failed") } @@ -75,14 +185,15 @@ func registerNode( sub *substrate.Substrate, cap gridtypes.Capacity, loc geoip.Location, + pub *pkg.PublicConfig, + ygg net.IP, ) (nodeID, twinID uint32, err error) { var ( - mgr = stubs.NewIdentityManagerStub(cl) - netMgr = stubs.NewNetworkerStub(cl) + mgr = stubs.NewIdentityManagerStub(cl) ) var pubCfg substrate.OptionPublicConfig - if pub, err := netMgr.GetPublicConfig(ctx); err == nil { + if pub != nil { pubCfg.HasValue = true pubCfg.AsValue = substrate.PublicConfig{ IPv4: pub.IPv4.String(), @@ -116,14 +227,7 @@ func registerNode( return 0, 0, errors.Wrap(err, "failed to ensure account") } - // make sure the node twin exists - cfg := yggdrasil.GenerateConfig(sk) - address, err := cfg.Address() - if err != nil { - return 0, 0, errors.Wrap(err, "failed to get yggdrasil address") - } - - twinID, err = ensureTwin(sub, sk, address) + twinID, err = ensureTwin(sub, sk, ygg) if err != nil { return 0, 0, errors.Wrap(err, "failed to ensure twin") } diff --git a/cmds/modules/gateway/main.go b/cmds/modules/gateway/main.go new file mode 100644 index 000000000..4f4b434e1 --- /dev/null +++ b/cmds/modules/gateway/main.go @@ -0,0 +1,83 @@ +package gateway + +import ( + "context" + + "github.com/pkg/errors" + "github.com/threefoldtech/zos/pkg/gateway" + "github.com/threefoldtech/zos/pkg/utils" + "github.com/urfave/cli/v2" + + "github.com/rs/zerolog/log" + + "github.com/threefoldtech/zbus" +) + +const ( + module = "gateway" +) + +// Module is entry point for module +var Module cli.Command = cli.Command{ + Name: "gateway", + Usage: "manage web gateway proxy", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "root", + Usage: "`ROOT` working directory of the module", + Value: "/var/cache/modules/gateway", + }, + &cli.StringFlag{ + Name: "broker", + Usage: "connection string to the message `BROKER`", + Value: "unix:///var/run/redis.sock", + }, + &cli.UintFlag{ + Name: "workers", + Usage: "number of workers `N`", + Value: 1, + }, + }, + Action: action, +} + +func action(cli *cli.Context) error { + var ( + moduleRoot string = cli.String("root") + msgBrokerCon string = cli.String("broker") + workerNr uint = cli.Uint("workers") + ) + + server, err := zbus.NewRedisServer(module, msgBrokerCon, workerNr) + if err != nil { + return errors.Wrap(err, "fail to connect to message broker server") + } + + client, err := zbus.NewRedisClient(msgBrokerCon) + if err != nil { + return errors.Wrap(err, "failed to connect to zbus broker") + } + + mod, err := gateway.New(cli.Context, client, moduleRoot) + if err != nil { + return errors.Wrap(err, "failed to construct gateway object") + } + server.Register(zbus.ObjectID{Name: "manager", Version: "0.0.1"}, mod) + + ctx, _ := utils.WithSignal(context.Background()) + + log.Info(). + Str("broker", msgBrokerCon). + Uint("worker nr", workerNr). + Msg("starting gateway module") + + utils.OnDone(ctx, func(_ error) { + log.Info().Msg("shutting down") + }) + + if err := server.Run(ctx); err != nil && err != context.Canceled { + return errors.Wrap(err, "unexpected error") + } + + return nil +} diff --git a/cmds/modules/networkd/main.go b/cmds/modules/networkd/main.go index f0d9a575e..3dd99fdff 100644 --- a/cmds/modules/networkd/main.go +++ b/cmds/modules/networkd/main.go @@ -2,7 +2,6 @@ package networkd import ( "context" - "crypto/ed25519" "fmt" "os" "os/exec" @@ -10,9 +9,7 @@ import ( "time" "github.com/pkg/errors" - "github.com/threefoldtech/zos/pkg/network/latency" "github.com/threefoldtech/zos/pkg/network/public" - "github.com/threefoldtech/zos/pkg/zinit" "github.com/urfave/cli/v2" "github.com/cenkalti/backoff/v3" @@ -81,7 +78,8 @@ func action(cli *cli.Context) error { }) publicCfgPath := filepath.Join(root, publicConfigFile) - pub, err := public.LoadPublicConfig(publicCfgPath) + public.SetPersistence(publicCfgPath) + pub, err := public.LoadPublicConfig() log.Debug().Err(err).Msgf("public interface configred: %+v", pub) if err != nil && err != public.ErrNoPublicConfig { return errors.Wrap(err, "failed to get node public_config") @@ -102,18 +100,43 @@ func action(cli *cli.Context) error { return errors.Wrap(err, "failed to host firewall rules") } log.Debug().Msg("starting yggdrasil") - ygg, err := startYggdrasil(ctx, identity.PrivateKey(cli.Context), dmz) + yggNamespace := dmz.Namespace() + if public.HasPublicSetup() { + yggNamespace = public.PublicNamespace + } + + yggNs, err := yggdrasil.NewYggdrasilNamespace(yggNamespace) if err != nil { - return errors.Wrap(err, "fail to start yggdrasil") + return errors.Wrap(err, "failed to create yggdrasil namespace") } - gw, err := ygg.Gateway() + ygg, err := yggdrasil.EnsureYggdrasil(ctx, identity.PrivateKey(cli.Context), yggNs) if err != nil { - return errors.Wrap(err, "fail read yggdrasil subnet") + return errors.Wrap(err, "fail to start yggdrasil") } - if err := dmz.SetIP(gw); err != nil { - return errors.Wrap(err, "fail to configure yggdrasil subnet gateway IP") + if public.HasPublicSetup() { + // if yggdrasil is living inside public namespace + // we still need to setup ndmz to also have yggdrasil but we set the yggdrasil interface + // a different Ip that lives inside the yggdrasil range. + dmzYgg, err := yggdrasil.NewYggdrasilNamespace(dmz.Namespace()) + if err != nil { + return errors.Wrap(err, "failed to setup ygg for dmz namespace") + } + + ip, err := ygg.SubnetFor([]byte(fmt.Sprintf("ygg:%s", dmz.Namespace()))) + if err != nil { + return errors.Wrap(err, "failed to calculate ip for ygg inside dmz") + } + + gw, err := ygg.Gateway() + if err != nil { + return err + } + + if err := dmzYgg.SetYggIP(ip, gw.IP); err != nil { + return errors.Wrap(err, "failed to set yggdrasil ip for dmz") + } } log.Info().Msg("start zbus server") @@ -121,7 +144,7 @@ func action(cli *cli.Context) error { return errors.Wrap(err, "fail to create module root") } - networker, err := network.NewNetworker(identity, publicCfgPath, dmz, ygg) + networker, err := network.NewNetworker(identity, dmz, ygg) if err != nil { return errors.Wrap(err, "error creating network manager") } @@ -165,109 +188,3 @@ func waitYggdrasilBin() { log.Warn().Err(err).Msgf("yggdrasil binary not found, retying in %s", d.String()) }) } - -func fetchPeerList() yggdrasil.Peers { - // Try to fetch public peer - // If we failed to do so, use the fallback hardcoded peer list - var pl yggdrasil.Peers - - // Do not retry more than 4 times - bo := backoff.WithMaxRetries(backoff.NewConstantBackOff(time.Second), 4) - - fetchPeerList := func() error { - p, err := yggdrasil.FetchPeerList() - if err != nil { - log.Debug().Err(err).Msg("failed to fetch yggdrasil peers") - return err - } - pl = p - return nil - } - - err := backoff.Retry(fetchPeerList, bo) - if err != nil { - log.Error().Err(err).Msg("failed to read yggdrasil public peer list online, using fallback") - pl = yggdrasil.PeerListFallback - } - - return pl -} - -func startYggdrasil(ctx context.Context, privateKey ed25519.PrivateKey, dmz ndmz.DMZ) (*network.YggServer, error) { - pl := fetchPeerList() - peersUp := pl.Ups() - endpoints := make([]string, len(peersUp)) - for i, p := range peersUp { - endpoints[i] = p.Endpoint - } - - // filter out the possible yggdrasil public node - var filter latency.IPFilter - ipv4Only, err := dmz.IsIPv4Only() - if err != nil { - return nil, errors.Wrap(err, "failed to check ipv6 support for dmz") - } - - if ipv4Only { - // if we are a hidden node,only keep ipv4 public nodes - filter = latency.IPV4Only - } else { - // if we are a dual stack node, filter out all the nodes from the same - // segment so we do not just connect locally - ips, err := dmz.GetIP(ndmz.FamilyV6) - if err != nil { - return nil, errors.Wrap(err, "failed to get ndmz public ipv6") - } - - for _, ip := range ips { - if ip.IP.IsGlobalUnicast() { - filter = latency.ExcludePrefix(ip.IP[:8]) - break - } - } - } - - ls := latency.NewSorter(endpoints, 5, filter) - results := ls.Run(ctx) - if len(results) == 0 { - return nil, fmt.Errorf("cannot find public yggdrasil peer to connect to") - } - - // select the best 3 public peers - peers := make([]string, 3) - for i := 0; i < 3; i++ { - if len(results) > i { - peers[i] = results[i].Endpoint - log.Info().Str("endpoint", results[i].Endpoint).Msg("yggdrasill public peer selected") - } - } - - z, err := zinit.New("") - if err != nil { - return nil, err - } - - cfg := yggdrasil.GenerateConfig(privateKey) - cfg.Peers = peers - - server := network.NewYggServer(z, &cfg) - - go func() { - select { - case <-ctx.Done(): - if err := server.Stop(); err != nil { - log.Error().Err(err).Msg("error while stopping yggdrasil") - } - if err := z.Close(); err != nil { - log.Error().Err(err).Msg("error while closing zinit client") - } - log.Info().Msg("yggdrasil stopped") - } - }() - - if err := server.Start(); err != nil { - return nil, err - } - - return server, nil -} diff --git a/cmds/modules/zui/net.go b/cmds/modules/zui/net.go index bdfaaefc6..a60b200f2 100644 --- a/cmds/modules/zui/net.go +++ b/cmds/modules/zui/net.go @@ -3,6 +3,7 @@ package zui import ( "context" _ "fmt" + "net" "strings" ui "github.com/gizak/termui/v3" @@ -49,6 +50,10 @@ func addressRender(ctx context.Context, table *widgets.Table, client zbus.Client toString := func(al pkg.NetlinkAddresses) string { var buf strings.Builder for _, a := range al { + if a.IP == nil || len(a.IP) == 0 { + continue + } + if buf.Len() > 0 { buf.WriteString(", ") } @@ -70,7 +75,11 @@ func addressRender(ctx context.Context, table *widgets.Table, client zbus.Client case a := <-ygg: table.Rows[2][1] = toString(a) case a := <-pub: - table.Rows[3][1] = toString(a) + str := "no public config" + if a.HasPublicConfig { + str = toString([]net.IPNet{a.IPv4.IPNet, a.IPv6.IPNet}) + } + table.Rows[3][1] = str } render.Signal() diff --git a/cmds/zos/main.go b/cmds/zos/main.go index e48ab1e73..c5098c919 100644 --- a/cmds/zos/main.go +++ b/cmds/zos/main.go @@ -9,6 +9,7 @@ import ( "github.com/threefoldtech/zos/cmds/modules/capacityd" "github.com/threefoldtech/zos/cmds/modules/contd" "github.com/threefoldtech/zos/cmds/modules/flistd" + "github.com/threefoldtech/zos/cmds/modules/gateway" "github.com/threefoldtech/zos/cmds/modules/networkd" "github.com/threefoldtech/zos/cmds/modules/provisiond" "github.com/threefoldtech/zos/cmds/modules/storaged" @@ -43,6 +44,7 @@ func main() { &networkd.Module, &provisiond.Module, &zbusdebug.Module, + &gateway.Module, }, Action: func(c *cli.Context) error { if !c.Bool("list") { diff --git a/etc/zinit/gateway.yaml b/etc/zinit/gateway.yaml new file mode 100644 index 000000000..58989d515 --- /dev/null +++ b/etc/zinit/gateway.yaml @@ -0,0 +1,3 @@ +exec: gateway --broker unix://var/run/redis.sock --root /var/cache/modules/gateway +after: + - boot diff --git a/go.mod b/go.mod index 02bbdcbe1..b02056382 100644 --- a/go.mod +++ b/go.mod @@ -58,7 +58,7 @@ require ( github.com/stretchr/objx v0.3.0 // indirect github.com/stretchr/testify v1.7.0 github.com/threefoldtech/0-fs v1.3.1-0.20201203163303-d963de9adea7 - github.com/threefoldtech/zbus v0.1.4 + github.com/threefoldtech/zbus v0.1.5 github.com/tyler-smith/go-bip39 v1.1.0 github.com/urfave/cli v1.22.5 github.com/urfave/cli/v2 v2.3.0 diff --git a/go.sum b/go.sum index 1a564b04a..1814ea04a 100644 --- a/go.sum +++ b/go.sum @@ -1058,6 +1058,8 @@ github.com/threefoldtech/0-fs v1.3.1-0.20201203163303-d963de9adea7 h1:64QIPSO1Ac github.com/threefoldtech/0-fs v1.3.1-0.20201203163303-d963de9adea7/go.mod h1:OPPZiE/GthPR2IepjKSc8wa+t/7wl3dtHQyEdUcftZI= github.com/threefoldtech/zbus v0.1.4 h1:c8lF4H9HMDJGknhcEK2p+zJWWM09fByGe3uL9m/iKgM= github.com/threefoldtech/zbus v0.1.4/go.mod h1:ZtiRpcqzEBJetVQDsEbw0p48h/AF3O1kf0tvd30I0BU= +github.com/threefoldtech/zbus v0.1.5 h1:S9kXbjejoRRnJw1yKHEXFGF2vqL+Drae2n4vpj0pGHo= +github.com/threefoldtech/zbus v0.1.5/go.mod h1:ZtiRpcqzEBJetVQDsEbw0p48h/AF3O1kf0tvd30I0BU= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= diff --git a/pkg/gateway.go b/pkg/gateway.go new file mode 100644 index 000000000..23a8dcc1c --- /dev/null +++ b/pkg/gateway.go @@ -0,0 +1,10 @@ +package pkg + +//go:generate mkdir -p stubs + +//go:generate zbusc -module gateway -version 0.0.1 -name manager -package stubs github.com/threefoldtech/zos/pkg+Gateway stubs/gateway_stub.go + +type Gateway interface { + SetNamedProxy(wlID string, prefix string, backends []string) (string, error) + DeleteNamedProxy(wlID string) error +} diff --git a/pkg/gateway/flist.go b/pkg/gateway/flist.go new file mode 100644 index 000000000..ed6f6d936 --- /dev/null +++ b/pkg/gateway/flist.go @@ -0,0 +1,30 @@ +package gateway + +import ( + "context" + "path/filepath" + + "github.com/pkg/errors" + "github.com/threefoldtech/zbus" + "github.com/threefoldtech/zos/pkg" + "github.com/threefoldtech/zos/pkg/stubs" +) + +const ( + flist = "https://hub.grid.tf/azmy.3bot/traefik.flist" +) + +// ensureTraefikBin makes sure traefik flist is mounted. +// TODO: we need to "update" traefik and restart the service +// if new version is available! +func ensureTraefikBin(ctx context.Context, cl zbus.Client) (string, error) { + const bin = "traefik" + flistd := stubs.NewFlisterStub(cl) + + mnt, err := flistd.Mount(ctx, bin, flist, pkg.ReadOnlyMountOptions) + if err != nil { + return "", errors.Wrap(err, "failed to mount traefik flist") + } + + return filepath.Join(mnt, bin), nil +} diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go new file mode 100644 index 000000000..cc1c1a464 --- /dev/null +++ b/pkg/gateway/gateway.go @@ -0,0 +1,217 @@ +package gateway + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + "github.com/threefoldtech/zbus" + "github.com/threefoldtech/zos/pkg" + "github.com/threefoldtech/zos/pkg/stubs" + "github.com/threefoldtech/zos/pkg/zinit" + "gopkg.in/yaml.v2" +) + +const ( + traefikService = "traefik" +) + +type gatewayModule struct { + cl zbus.Client + + proxyConfigPath string + staticConfigPath string + binPath string +} + +type ProxyConfig struct { + Http HTTPConfig +} + +type HTTPConfig struct { + Routers map[string]Router + Services map[string]Service +} + +type Router struct { + Rule string + Service string +} + +type Service struct { + LoadBalancer LoadBalancer +} + +type LoadBalancer struct { + Servers []Server +} + +type Server struct { + Url string +} + +func New(ctx context.Context, cl zbus.Client, root string) (pkg.Gateway, error) { + configPath := filepath.Join(root, "proxy") + // where should service-restart/node-reboot recovery be handled? + err := os.MkdirAll(configPath, 0644) + if err != nil { + return nil, errors.Wrap(err, "couldn't make gateway config dir") + } + + bin, err := ensureTraefikBin(ctx, cl) + if err != nil { + return nil, errors.Wrap(err, "failed to ensure traefik binary") + } + + staticCfgPath := filepath.Join(root, "traefik.yaml") + if err := staticConfig(staticCfgPath, root); err != nil { + return nil, errors.Wrap(err, "failed to create static config") + } + + gw := &gatewayModule{ + cl: cl, + proxyConfigPath: configPath, + staticConfigPath: staticCfgPath, + binPath: bin, + } + // in case there are already active configurations we should always try to ensure running traefik + if _, err := gw.ensureGateway(ctx); err != nil { + log.Error().Err(err).Msg("gateway is not supported") + // this is not a failure because supporting of the gateway can happen + // later if the farmer set the correct network configuration! + } + + return gw, nil +} + +func (g *gatewayModule) isTraefikStarted(z *zinit.Client) (bool, error) { + traefikStatus, err := z.Status(traefikService) + if errors.Is(err, zinit.ErrUnknownService) { + return false, nil + } else if err != nil { + return false, errors.Wrap(err, "failed to check traefik status") + } + + return traefikStatus.State.Is(zinit.ServiceStateRunning), nil +} + +// ensureGateway makes sure that gateway infrastructure is in place and +// that it is supported. +func (g *gatewayModule) ensureGateway(ctx context.Context) (string, error) { + var ( + networker = stubs.NewNetworkerStub(g.cl) + ) + cfg, err := networker.GetPublicConfig(ctx) + if err != nil { + return "", errors.Wrap(err, "gateway is not supported on this node") + } + + if cfg.Domain == "" { + return "", errors.Errorf("gateway is not supported. missing domain configuration") + } + + z, err := zinit.Default() + if err != nil { + return "", errors.Wrap(err, "failed to connect to zinit") + } + defer z.Close() + running, err := g.isTraefikStarted(z) + if err != nil { + return "", errors.Wrap(err, "failed to check traefik status") + } + + if running { + return cfg.Domain, nil + } + + //other wise we start traefik + return cfg.Domain, g.startTraefik(z) +} + +func (g *gatewayModule) startTraefik(z *zinit.Client) error { + cmd := fmt.Sprintf( + "ip netns exec public %s --configfile %s --log.level=DEBUG", + g.binPath, + g.staticConfigPath, + ) + + if err := zinit.AddService(traefikService, zinit.InitService{ + Exec: cmd, + }); err != nil { + return errors.Wrap(err, "failed to add traefik to zinit") + } + + if err := z.Monitor(traefikService); err != nil { + return errors.Wrap(err, "couldn't monitor traefik service") + } + + if err := z.StartWait(time.Second*20, traefikService); err != nil { + return errors.Wrap(err, "waiting for trafik start timed out") + } + + return nil +} + +func (g *gatewayModule) configPath(name string) string { + return filepath.Join(g.proxyConfigPath, fmt.Sprintf("%s.yaml", name)) +} + +func (g *gatewayModule) SetNamedProxy(wlID string, prefix string, backends []string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + domain, err := g.ensureGateway(ctx) + if err != nil { + return "", err + } + + fqdn := fmt.Sprintf("%s.%s", prefix, domain) + + rule := fmt.Sprintf("Host(`%s`)", fqdn) + servers := make([]Server, len(backends)) + for idx, backend := range backends { + servers[idx] = Server{ + Url: backend, + } + } + + config := ProxyConfig{ + Http: HTTPConfig{ + Routers: map[string]Router{ + wlID: { + Rule: rule, + Service: wlID, + }, + }, + Services: map[string]Service{ + wlID: { + LoadBalancer: LoadBalancer{ + Servers: servers, + }, + }, + }, + }, + } + + yamlString, err := yaml.Marshal(&config) + if err != nil { + return "", errors.Wrap(err, "failed to convert config to yaml") + } + log.Debug().Str("yaml-config", string(yamlString)).Msg("configuration file") + if err = os.WriteFile(g.configPath(wlID), yamlString, 0644); err != nil { + return "", errors.Wrap(err, "couldn't open config file for writing") + } + + return fqdn, nil +} + +func (g *gatewayModule) DeleteNamedProxy(wlID string) error { + if err := os.Remove(g.configPath(wlID)); err != nil { + return errors.Wrap(err, "couldn't remove config file") + } + return nil +} diff --git a/pkg/gateway/static.go b/pkg/gateway/static.go new file mode 100644 index 000000000..d44c674ab --- /dev/null +++ b/pkg/gateway/static.go @@ -0,0 +1,16 @@ +package gateway + +import ( + _ "embed" + "fmt" + "os" +) + +//go:embed static/config.yaml +var config string + +// staticConfig write static config to file +func staticConfig(p, root string) error { + config := fmt.Sprintf(config, root) + return os.WriteFile(p, []byte(config), 0644) +} diff --git a/pkg/gateway/static/config.yaml b/pkg/gateway/static/config.yaml new file mode 100644 index 000000000..0e215a969 --- /dev/null +++ b/pkg/gateway/static/config.yaml @@ -0,0 +1,18 @@ +entryPoints: + web: + address: ":80" + + web-secure: + address: ":443" +providers: + file: + directory: "%s/proxy" + watch: true +certificatesResolvers: + resolver: + acme: + # email: your-email@example.com + storage: acme.json + httpChallenge: + # used during the challenge + entryPoint: web diff --git a/pkg/gridtypes/zos/gw.go b/pkg/gridtypes/zos/gw.go new file mode 100644 index 000000000..081100702 --- /dev/null +++ b/pkg/gridtypes/zos/gw.go @@ -0,0 +1,81 @@ +package zos + +import ( + "fmt" + "io" + "net/url" + "regexp" + + "github.com/pkg/errors" + "github.com/threefoldtech/zos/pkg/gridtypes" +) + +var ( + gwNameRegex = regexp.MustCompile(`^\w+$`) +) + +type Backend string + +// check if valid http://x.x.x.x:port or [::]:port +func (b Backend) Valid() error { + u, err := url.Parse(string(b)) + if err != nil { + return errors.Wrap(err, "failed to parse backend") + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("invalid scheme expected http, or https") + } + + return nil +} + +// GatewayNameProxy definition. this will proxy name. to backends +type GatewayNameProxy struct { + // Name of the domain prefix. this must be a valid dns name (with no dots) + Name string `json:"name"` + + // Backends are list of backend ips + Backends []Backend `json:"backends"` +} + +func (g GatewayNameProxy) Valid(getter gridtypes.WorkloadGetter) error { + if !gwNameRegex.MatchString(g.Name) { + return fmt.Errorf("invalid name") + } + if len(g.Backends) == 0 { + return fmt.Errorf("backends list can not be empty") + } + for _, backend := range g.Backends { + if err := backend.Valid(); err != nil { + return errors.Wrapf(err, "failed to validate backend '%s'", backend) + } + } + + return nil +} + +func (g GatewayNameProxy) Challenge(w io.Writer) error { + if _, err := fmt.Fprintf(w, "%s", g.Name); err != nil { + return err + } + + for _, backend := range g.Backends { + if _, err := fmt.Fprintf(w, "%s", string(backend)); err != nil { + return err + } + } + + return nil +} + +func (g GatewayNameProxy) Capacity() (gridtypes.Capacity, error) { + // this has to be calculated per bytes served over the gw. so + // a special handler in reporting that need to calculate and report + // this. + return gridtypes.Capacity{}, nil +} + +// GatewayProxyResult results +type GatewayProxyResult struct { + FQDN string `json:"fqdn"` +} diff --git a/pkg/gridtypes/zos/types.go b/pkg/gridtypes/zos/types.go index 63a70f175..d2b91e25a 100644 --- a/pkg/gridtypes/zos/types.go +++ b/pkg/gridtypes/zos/types.go @@ -15,9 +15,10 @@ const ( ZDBType gridtypes.WorkloadType = "zdb" // ZMachineType type ZMachineType gridtypes.WorkloadType = "zmachine" - - //PublicIPType reservation + //PublicIPType type PublicIPType gridtypes.WorkloadType = "ipv4" + // GatewayNameProxyType type + GatewayNameProxyType gridtypes.WorkloadType = "gateway-proxy" ) func init() { @@ -26,6 +27,7 @@ func init() { gridtypes.RegisterType(ZDBType, ZDB{}) gridtypes.RegisterType(ZMachineType, ZMachine{}) gridtypes.RegisterType(PublicIPType, PublicIP{}) + gridtypes.RegisterType(GatewayNameProxyType, GatewayNameProxy{}) } // DeviceType is the actual type of hardware that the storage device runs on, diff --git a/pkg/network.go b/pkg/network.go index 23ae43963..0508175c8 100644 --- a/pkg/network.go +++ b/pkg/network.go @@ -149,7 +149,7 @@ type Networker interface { // YggAddresses monitoring streams for yggdrasil interface YggAddresses(ctx context.Context) <-chan NetlinkAddresses - PublicAddresses(ctx context.Context) <-chan NetlinkAddresses + PublicAddresses(ctx context.Context) <-chan OptionPublicConfig } // Network type @@ -186,5 +186,12 @@ type PublicConfig struct { GW4 net.IP `json:"gw4"` GW6 net.IP `json:"gw6"` - // Version int `json:"version"` + // Domain is the node domain name like gent01.devnet.grid.tf + // or similar + Domain string `json:"domain"` +} + +type OptionPublicConfig struct { + PublicConfig + HasPublicConfig bool } diff --git a/pkg/network/latency/latency_test.go b/pkg/network/latency/latency_test.go index e30d3de60..757c936d2 100644 --- a/pkg/network/latency/latency_test.go +++ b/pkg/network/latency/latency_test.go @@ -1,4 +1,4 @@ -package latency +package latency_test import ( "context" @@ -8,17 +8,18 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/threefoldtech/zos/pkg/network/latency" "github.com/threefoldtech/zos/pkg/network/yggdrasil" ) func TestLatency(t *testing.T) { - l, err := Latency("explorer.grid.tf:80") + l, err := latency.Latency("explorer.grid.tf:80") require.NoError(t, err) t.Log(l) } func TestLatencySorter(t *testing.T) { - ls := NewSorter([]string{ + ls := latency.NewSorter([]string{ "explorer.grid.tf:80", "google.com:80", }, 2) @@ -30,14 +31,14 @@ func TestLatencySorter(t *testing.T) { for _, r := range results { fmt.Printf("%s %v\n", r.Endpoint, r.Latency) } - assert.Equal(t, len(ls.endpoints), len(results)) + assert.Equal(t, 2, len(results)) } func TestLatencySorterIPV4Only(t *testing.T) { - ls := NewSorter([]string{ + ls := latency.NewSorter([]string{ "tcp://[2a00:1450:400e:806::200e]:443", "tcp://172.217.17.78:443", - }, 1, IPV4Only) + }, 1, latency.IPV4Only) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -56,7 +57,7 @@ func TestYggPeering(t *testing.T) { endpoints[i] = p.Endpoint } - ls := NewSorter(endpoints, 2) + ls := latency.NewSorter(endpoints, 2) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -82,7 +83,7 @@ func TestIPV4Only(t *testing.T) { }, } { t.Run(tc.ip.String(), func(t *testing.T) { - assert.Equal(t, tc.ipv4, IPV4Only(tc.ip)) + assert.Equal(t, tc.ipv4, latency.IPV4Only(tc.ip)) }) } } @@ -105,7 +106,7 @@ func TestExcludePrefix(t *testing.T) { }, } { t.Run(tc.ip.String(), func(t *testing.T) { - assert.Equal(t, tc.expect, ExcludePrefix(tc.prefix[:8])(tc.ip)) + assert.Equal(t, tc.expect, latency.ExcludePrefix(tc.prefix[:8])(tc.ip)) }) } diff --git a/pkg/network/namespace/namespace.go b/pkg/network/namespace/namespace.go index 6c29e4a5c..1a49a790e 100644 --- a/pkg/network/namespace/namespace.go +++ b/pkg/network/namespace/namespace.go @@ -17,6 +17,10 @@ import ( "golang.org/x/sys/unix" ) +var ( + ErrNotNamedNamespace = fmt.Errorf("name space is not named") +) + const ( netNSPath = "/var/run/netns" ) @@ -147,6 +151,19 @@ func Delete(ns ns.NetNS) error { return nil } +// Name gets the name of the namespace if it was created with Create +// otherwise return ErrNotNamedNamespace +func Name(ns ns.NetNS) (string, error) { + path := ns.Path() + + dir, name := filepath.Split(path) + if dir != netNSPath { + return "", ErrNotNamedNamespace + } + + return name, nil +} + // Exists checks if a network namespace exists or not func Exists(name string) bool { nsPath := filepath.Join(netNSPath, name) diff --git a/pkg/network/ndmz/dualstack.go b/pkg/network/ndmz/dualstack.go index e32e5c20a..9358359b1 100644 --- a/pkg/network/ndmz/dualstack.go +++ b/pkg/network/ndmz/dualstack.go @@ -36,8 +36,7 @@ import ( const ( //ndmzBridge is the name of the ipv4 routing bridge in the ndmz namespace - ndmzBridge = "br-ndmz" - ndmzYggBridge = types.YggBridge + ndmzBridge = "br-ndmz" //dmzNamespace name of the dmz namespace dmzNamespace = "ndmz" @@ -50,8 +49,6 @@ const ( // dmzPub6 ipv6 public interface dmzPub6 = "npub6" - dmzYgg = "nygg6" - //nrPubIface is the name of the public interface in a network resource nrPubIface = "public" @@ -72,6 +69,10 @@ func New(nodeID string, public *netlink.Bridge) DMZ { } } +func (d *dmzImpl) Namespace() string { + return dmzNamespace +} + // Create create the NDMZ network namespace and configure its default routes and addresses func (d *dmzImpl) Create(ctx context.Context) error { // There are 2 options for the master: @@ -99,10 +100,6 @@ func (d *dmzImpl) Create(ctx context.Context) error { return errors.Wrapf(err, "ndmz: createRoutingBridge error") } - if err := createYggBridge(ndmzYggBridge, netNS); err != nil { - return errors.Wrapf(err, "ndmz: createYggBridge error") - } - if err := createPubIface6(dmzPub6, d.public, d.nodeID, netNS); err != nil { return errors.Wrapf(err, "ndmz: could not node create pub iface 6") } @@ -231,40 +228,6 @@ func (d *dmzImpl) AttachNR(networkID string, nr *nr.NetResource, ipamLeaseDir st }) } -// IsIPv4Only means dmz only supports ipv4 addresses -func (d *dmzImpl) IsIPv4Only() (bool, error) { - // this is true if DMZPub6 only has local not routable ipv6 addresses - //DMZPub6 - netNS, err := namespace.GetByName(dmzNamespace) - if err != nil { - return false, errors.Wrap(err, "failed to get ndmz namespace") - } - defer netNS.Close() - - var ipv4Only bool - err = netNS.Do(func(_ ns.NetNS) error { - link, err := netlink.LinkByName(dmzPub6) - if err != nil { - return errors.Wrapf(err, "failed to get interface '%s'", dmzPub6) - } - ips, err := netlink.AddrList(link, netlink.FAMILY_V6) - if err != nil { - return errors.Wrapf(err, "failed to list '%s' ips", dmzPub6) - } - - for _, ip := range ips { - if ip.IP.IsGlobalUnicast() && !ifaceutil.IsULA(ip.IP) { - return nil - } - } - - ipv4Only = true - return nil - }) - - return ipv4Only, err -} - func (d *dmzImpl) GetIPFor(inf string) ([]net.IPNet, error) { netns, err := namespace.GetByName(dmzNamespace) @@ -340,46 +303,6 @@ func (d *dmzImpl) GetIP(family int) ([]net.IPNet, error) { return results, err } -// SetIP sets an ip inside dmz -func (d *dmzImpl) SetIP(subnet net.IPNet) error { - netns, err := namespace.GetByName(dmzNamespace) - if err != nil { - return err - } - defer netns.Close() - - err = netns.Do(func(_ ns.NetNS) error { - inf := dmzPub4 - if ip6 := subnet.IP.To16(); ip6 != nil { - // this still can be an ygg address - // so .. - _, ygg, err := net.ParseCIDR("200::/7") - if err != nil { - panic(err) - } - - if ygg.Contains(ip6) { - inf = dmzYgg - } else { - inf = dmzPub6 - } - } - - link, err := netlink.LinkByName(inf) - if err != nil { - return err - } - - if err := netlink.AddrAdd(link, &netlink.Addr{ - IPNet: &subnet, - }); err != nil && !os.IsExist(err) { - return err - } - return nil - }) - return err -} - // SupportsPubIPv4 implements DMZ interface func (d *dmzImpl) SupportsPubIPv4() bool { return true @@ -541,29 +464,6 @@ func createPubIface4(name, nodeID string, netNS ns.NetNS) error { }) } -func createYggBridge(name string, netNS ns.NetNS) error { - if !bridge.Exists(name) { - if _, err := bridge.New(name); err != nil { - return errors.Wrapf(err, "couldn't create bridge %s", name) - } - } - - if !ifaceutil.Exists(dmzYgg, netNS) { - if _, err := macvlan.Create(dmzYgg, name, netNS); err != nil { - return errors.Wrapf(err, "ndmz: couldn't create %s", dmzYgg) - } - } - - return netNS.Do(func(_ ns.NetNS) error { - link, err := netlink.LinkByName(dmzYgg) - if err != nil { - return err - } - - return netlink.LinkSetUp(link) - }) -} - func createRoutingBridge(name string, netNS ns.NetNS) error { if !bridge.Exists(name) { if _, err := bridge.New(name); err != nil { diff --git a/pkg/network/ndmz/ndmz.go b/pkg/network/ndmz/ndmz.go index 7b79e2dc1..49eb7d9e8 100644 --- a/pkg/network/ndmz/ndmz.go +++ b/pkg/network/ndmz/ndmz.go @@ -20,14 +20,13 @@ const ( // DMZ is an interface used to create an DMZ network namespace type DMZ interface { + Namespace() string // create the ndmz network namespace and all requires network interfaces Create(ctx context.Context) error // delete the ndmz network namespace and clean up all network interfaces Delete() error // link a network resource from a user network to ndmz AttachNR(networkID string, nr *nr.NetResource, ipamLeaseDir string) error - // configure an address on the public IPv6 interface - SetIP(net.IPNet) error // GetIP gets ndmz public ips from ndmz GetIP(family int) ([]net.IPNet, error) @@ -37,9 +36,6 @@ type DMZ interface { // workloads SupportsPubIPv4() bool - //IsIPv4Only checks if dmz is ipv4 only (no ipv6 support) - IsIPv4Only() (bool, error) - //Interfaces information about dmz interfaces Interfaces() ([]types.IfaceInfo, error) } diff --git a/pkg/network/networker.go b/pkg/network/networker.go index 68d6283ea..f9c6fadfe 100644 --- a/pkg/network/networker.go +++ b/pkg/network/networker.go @@ -2,20 +2,19 @@ package network import ( "context" + "crypto/ed25519" "encoding/json" "fmt" "net" "os" "os/exec" "path/filepath" - "reflect" "strings" "time" "github.com/blang/semver" "github.com/threefoldtech/zos/pkg/cache" - "github.com/threefoldtech/zos/pkg/environment" "github.com/threefoldtech/zos/pkg/network/macvtap" "github.com/threefoldtech/zos/pkg/network/ndmz" "github.com/threefoldtech/zos/pkg/network/public" @@ -23,7 +22,6 @@ import ( "github.com/threefoldtech/zos/pkg/network/wireguard" "github.com/threefoldtech/zos/pkg/network/yggdrasil" "github.com/threefoldtech/zos/pkg/stubs" - "github.com/threefoldtech/zos/pkg/substrate" "github.com/vishvananda/netlink" @@ -71,15 +69,14 @@ type networker struct { ipamLeaseDir string portSet *set.UIntSet - publicConfig string - ndmz ndmz.DMZ - ygg *YggServer + ndmz ndmz.DMZ + ygg *yggdrasil.YggServer } var _ pkg.Networker = (*networker)(nil) // NewNetworker create a new pkg.Networker that can be used over zbus -func NewNetworker(identity *stubs.IdentityManagerStub, publicCfgPath string, ndmz ndmz.DMZ, ygg *YggServer) (pkg.Networker, error) { +func NewNetworker(identity *stubs.IdentityManagerStub, ndmz ndmz.DMZ, ygg *yggdrasil.YggServer) (pkg.Networker, error) { vd, err := cache.VolatileDir("networkd", 50*mib) if err != nil && !os.IsExist(err) { return nil, fmt.Errorf("failed to create networkd cache directory: %w", err) @@ -100,9 +97,8 @@ func NewNetworker(identity *stubs.IdentityManagerStub, publicCfgPath string, ndm ipamLeaseDir: ipamLease, portSet: set.NewInt(), - publicConfig: publicCfgPath, - ygg: ygg, - ndmz: ndmz, + ygg: ygg, + ndmz: ndmz, } // always add the reserved yggdrasil port to the port set so we make sure they are never @@ -161,10 +157,7 @@ func (n networker) ZDBPrepare(id string) (string, error) { } ips := []*net.IPNet{ - { - IP: ip, - Mask: net.CIDRMask(64, 128), - }, + &ip, } gw, err := n.ygg.Gateway() @@ -338,10 +331,7 @@ func (n *networker) SetupYggTap(name string) (tap pkg.YggdrasilTap, err error) { return tap, err } - tap.IP = net.IPNet{ - IP: ip, - Mask: net.CIDRMask(64, 128), - } + tap.IP = ip gw, err := n.ygg.Gateway() if err != nil { @@ -737,60 +727,45 @@ func (n *networker) SetPublicConfig(cfg pkg.PublicConfig) error { return errors.Wrap(err, "failed to apply public config") } - if err := public.SavePublicConfig(n.publicConfig, &cfg); err != nil { + if err := public.SavePublicConfig(cfg); err != nil { return errors.Wrap(err, "failed to store public config") } - // this is kinda dirty, but we need to update the node public config - // on the block-chain. so ... - env := environment.MustGet() - sub, err := env.GetSubstrate() + // when public setup is updated. it can take a while but the capacityd + // will detect this change and take necessary actions to update the node + ctx := context.Background() + sk := ed25519.PrivateKey(n.identity.PrivateKey(ctx)) + ns, err := yggdrasil.NewYggdrasilNamespace(public.PublicNamespace) if err != nil { - return errors.Wrap(err, "failed to connect to substrate") + return errors.Wrap(err, "failed to setup public namespace for yggdrasil") } - sk := n.identity.PrivateKey(context.Background()) - identity, err := substrate.IdentityFromSecureKey(sk) + ygg, err := yggdrasil.EnsureYggdrasil(context.Background(), sk, ns) if err != nil { return err } - twin, err := sub.GetTwinByPubKey(identity.PublicKey) - if err != nil { - return errors.Wrap(err, "failed to get node twin by public key") - } - - nodeID, err := sub.GetNodeByTwinID(twin) + // if yggdrasil is living inside public namespace + // we still need to setup ndmz to also have yggdrasil but we set the yggdrasil interface + // a different Ip that lives inside the yggdrasil range. + dmzYgg, err := yggdrasil.NewYggdrasilNamespace(n.ndmz.Namespace()) if err != nil { - return errors.Wrap(err, "failed to get node by twin id") + return errors.Wrap(err, "failed to setup ygg for dmz namespace") } - node, err := sub.GetNode(nodeID) + ip, err := ygg.SubnetFor([]byte(fmt.Sprintf("ygg:%s", n.ndmz.Namespace()))) if err != nil { - return errors.Wrapf(err, "failed to get node with id: %d", nodeID) + return errors.Wrap(err, "failed to calculate ip for ygg inside dmz") } - cfg, err = public.GetPublicSetup() + gw, err := ygg.Gateway() if err != nil { - return errors.Wrap(err, "failed to get public setup") + return err } - subCfg := substrate.OptionPublicConfig{ - HasValue: true, - AsValue: substrate.PublicConfig{ - IPv4: cfg.IPv4.String(), - IPv6: cfg.IPv6.String(), - GWv4: cfg.GW4.String(), - GWv6: cfg.GW6.String(), - }, + if err := dmzYgg.SetYggIP(ip, gw.IP); err != nil { + return errors.Wrap(err, "failed to set yggdrasil ip for dmz") } - if reflect.DeepEqual(node.PublicConfig, subCfg) { - //nothing to do - return nil - } - // update the node - node.PublicConfig = subCfg - _, err = sub.UpdateNode(&identity, *node) - return err + return nil } // Get node public namespace config @@ -886,11 +861,17 @@ func (n *networker) YggAddresses(ctx context.Context) <-chan pkg.NetlinkAddresse case <-ctx.Done(): return case <-time.After(30 * time.Second): - ips, err := n.ndmz.GetIPFor(yggdrasil.YggIface) + ips, err := n.ndmz.GetIPFor(yggdrasil.YggNSInf) if err != nil { log.Error().Err(err).Str("inf", yggdrasil.YggIface).Msg("failed to get public IPs") } - ch <- ips + filtered := ips[:0] + for _, ip := range ips { + if yggdrasil.YggRange.Contains(ip.IP) { + filtered = append(filtered, ip) + } + } + ch <- filtered } } }() @@ -898,19 +879,19 @@ func (n *networker) YggAddresses(ctx context.Context) <-chan pkg.NetlinkAddresse return ch } -func (n *networker) PublicAddresses(ctx context.Context) <-chan pkg.NetlinkAddresses { - ch := make(chan pkg.NetlinkAddresses) +func (n *networker) PublicAddresses(ctx context.Context) <-chan pkg.OptionPublicConfig { + ch := make(chan pkg.OptionPublicConfig) go func() { for { select { case <-ctx.Done(): return case <-time.After(30 * time.Second): - ips, err := public.IPs() - if err != nil { - log.Error().Err(err).Msg("failed to get public IPs") + cfg, err := n.GetPublicConfig() + ch <- pkg.OptionPublicConfig{ + PublicConfig: cfg, + HasPublicConfig: err == nil, } - ch <- ips } } }() diff --git a/pkg/network/public/persist.go b/pkg/network/public/persist.go index f61ce5e85..34a1678ed 100644 --- a/pkg/network/public/persist.go +++ b/pkg/network/public/persist.go @@ -8,13 +8,30 @@ import ( "github.com/threefoldtech/zos/pkg" ) +var ( + // persistencePath is path to config file. + persistencePath = "" +) + +func SetPersistence(path string) { + persistencePath = path +} + +func getPersistencePath() string { + if persistencePath == "" { + panic("public config persistence path is not set") + } + return persistencePath +} + // ErrNoPublicConfig is the error returns by ReadPubIface when no public // interface is configured -var ErrNoPublicConfig = errors.New("no public interface configured for this node") +var ErrNoPublicConfig = errors.New("no public configuration") // LoadPublicConfig loads public config from file -func LoadPublicConfig(path string) (*pkg.PublicConfig, error) { - file, err := os.Open(path) +func LoadPublicConfig() (*pkg.PublicConfig, error) { + + file, err := os.Open(getPersistencePath()) if os.IsNotExist(err) { // it's not an error to not have config // but we return a nil config @@ -33,14 +50,8 @@ func LoadPublicConfig(path string) (*pkg.PublicConfig, error) { } // SavePublicConfig stores public config in a file -func SavePublicConfig(path string, cfg *pkg.PublicConfig) error { - if cfg == nil { - if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) { - return errors.Wrap(err, "couldn't delete config file") - } - } - - file, err := os.Create(path) +func SavePublicConfig(cfg pkg.PublicConfig) error { + file, err := os.Create(getPersistencePath()) if err != nil { return errors.Wrap(err, "failed to create configuration file") } diff --git a/pkg/network/public/public.go b/pkg/network/public/public.go index 6a31aee31..7f18cc563 100644 --- a/pkg/network/public/public.go +++ b/pkg/network/public/public.go @@ -24,7 +24,8 @@ const ( publicNsMACDerivationSuffix = "-public" // PublicBridge public bridge name, exists only after a call to EnsurePublicSetup - PublicBridge = types.PublicBridge + PublicBridge = types.PublicBridge + PublicNamespace = types.PublicNamespace ) // EnsurePublicBridge makes sure that the public bridge exists @@ -51,7 +52,7 @@ func ensurePublicBridge() (*netlink.Bridge, error) { //not setup or does not exist. the caller must be able to handle //this case func getPublicNamespace() ns.NetNS { - ns, _ := namespace.GetByName(types.PublicNamespace) + ns, _ := namespace.GetByName(PublicNamespace) return ns } @@ -110,18 +111,29 @@ func setupPublicBridge(br *netlink.Bridge) error { return nil } +func HasPublicSetup() bool { + return namespace.Exists(PublicNamespace) +} + // GetPublicSetup gets the public setup from reality // or error if node has no public setup func GetPublicSetup() (pkg.PublicConfig, error) { - if !namespace.Exists(types.PublicNamespace) { - return pkg.PublicConfig{}, fmt.Errorf("no public config") + if !namespace.Exists(PublicNamespace) { + return pkg.PublicConfig{}, ErrNoPublicConfig } - namespace, err := namespace.GetByName(types.PublicNamespace) + namespace, err := namespace.GetByName(PublicNamespace) if err != nil { return pkg.PublicConfig{}, err } var cfg pkg.PublicConfig + if set, err := LoadPublicConfig(); err != nil { + return pkg.PublicConfig{}, errors.Wrap(err, "failed to load configuration") + } else { + // we only need the domain name from the config + cfg.Domain = set.Domain + } + // everything else is loaded from the actual state of the node. err = namespace.Do(func(_ ns.NetNS) error { link, err := netlink.LinkByName(types.PublicIface) if err != nil { @@ -239,12 +251,12 @@ func findPossibleExit() (string, error) { } func ensureNamespace() (ns.NetNS, error) { - if !namespace.Exists(types.PublicNamespace) { - log.Info().Str("namespace", types.PublicNamespace).Msg("Create network namespace") - return namespace.Create(types.PublicNamespace) + if !namespace.Exists(PublicNamespace) { + log.Info().Str("namespace", PublicNamespace).Msg("Create network namespace") + return namespace.Create(PublicNamespace) } - return namespace.GetByName(types.PublicNamespace) + return namespace.GetByName(PublicNamespace) } func ensurePublicMacvlan(iface *pkg.PublicConfig, pubNS ns.NetNS) (*netlink.Macvlan, error) { @@ -256,12 +268,13 @@ func ensurePublicMacvlan(iface *pkg.PublicConfig, pubNS ns.NetNS) (*netlink.Macv if !ifaceutil.Exists(types.PublicIface, pubNS) { switch iface.Type { + case "": + fallthrough case pkg.MacVlanIface: pubIface, err = macvlan.Create(types.PublicIface, types.PublicBridge, pubNS) if err != nil { return nil, errors.Wrap(err, "failed to create public mac vlan interface") } - default: return nil, fmt.Errorf("unsupported public interface type %s", iface.Type) } @@ -341,11 +354,15 @@ func setupPublicNS(nodeID pkg.Identifier, iface *pkg.PublicConfig) error { err = pubNS.Do(func(_ ns.NetNS) error { if err := options.SetIPv6AcceptRA(options.RAAcceptIfForwardingIsEnabled); err != nil { - return errors.Wrapf(err, "failed to accept_ra=2 in public namespace") + return errors.Wrap(err, "failed to accept_ra=2 in public namespace") } if err := options.SetIPv6LearnDefaultRouteInRA(true); err != nil { - return errors.Wrapf(err, "failed to enable enable_defrtr=1 in public namespace") + return errors.Wrap(err, "failed to enable enable_defrtr=1 in public namespace") + } + + if err := options.SetIPv6Forwarding(true); err != nil { + return errors.Wrap(err, "failed to enable ipv6 forwarding in public namespace") } return nil diff --git a/pkg/network/yggdrasil/namespace.go b/pkg/network/yggdrasil/namespace.go new file mode 100644 index 000000000..69d462061 --- /dev/null +++ b/pkg/network/yggdrasil/namespace.go @@ -0,0 +1,227 @@ +package yggdrasil + +import ( + "fmt" + "net" + "os" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/pkg/errors" + "github.com/threefoldtech/zos/pkg/network/bridge" + "github.com/threefoldtech/zos/pkg/network/ifaceutil" + "github.com/threefoldtech/zos/pkg/network/macvlan" + "github.com/threefoldtech/zos/pkg/network/namespace" + "github.com/threefoldtech/zos/pkg/network/types" + "github.com/vishvananda/netlink" +) + +const ( + // YggNSInf inside the namespace + YggNSInf = "nygg6" + yggBridge = types.YggBridge +) + +var ( + YggRange = net.IPNet{ + IP: net.ParseIP("200::"), + Mask: net.CIDRMask(7, 128), + } +) + +type YggdrasilNamespace interface { + Name() string + // IsIPv4Only checks if namespace has NO public ipv6 on any of its interfaces + IsIPv4Only() (bool, error) + // GetIPs return a list of all IPv6 inside this namespace. + GetIPs() ([]net.IPNet, error) + // SetYggIP sets the ygg ipv6 on the nygg6 iterface. + SetYggIP(ip net.IPNet, gw net.IP) error +} + +// ensureYggPlumbing this ensures that the yggdrasil plumbing is in place inside this namespace +func ensureYggPlumbing(netNS ns.NetNS) error { + if !bridge.Exists(yggBridge) { + if _, err := bridge.New(yggBridge); err != nil { + return errors.Wrapf(err, "couldn't create bridge %s", yggBridge) + } + } + + if !ifaceutil.Exists(YggNSInf, netNS) { + if _, err := macvlan.Create(YggNSInf, yggBridge, netNS); err != nil { + return errors.Wrapf(err, "couldn't create %s inside", YggNSInf) + } + } + + return netNS.Do(func(_ ns.NetNS) error { + link, err := netlink.LinkByName(YggNSInf) + if err != nil { + return err + } + + return netlink.LinkSetUp(link) + }) +} + +func NewYggdrasilNamespace(ns string) (YggdrasilNamespace, error) { + yggNs, err := namespace.GetByName(ns) + if err != nil { + return nil, errors.Wrapf(err, "namespace '%s' not found", ns) + } + if err := ensureYggPlumbing(yggNs); err != nil { + return nil, errors.Wrapf(err, "failed to prepare namespace '%s' for yggdrasil", ns) + } + + return &yggNS{ns}, nil +} + +type yggNS struct { + ns string +} + +func (d *yggNS) Name() string { + return d.ns +} + +func (d *yggNS) setGw(gw net.IP) error { + ipv6routes, err := netlink.RouteList(nil, netlink.FAMILY_V6) + if err != nil { + return err + } + + for _, route := range ipv6routes { + if route.Dst == nil { + //default route! + continue + } + if route.Dst.String() == YggRange.String() { + // we found a match + if err := netlink.RouteDel(&route); err != nil { + return err + } + } + } + + // now add route + return netlink.RouteAdd(&netlink.Route{ + Dst: &YggRange, + Gw: gw, + }) +} +func (d *yggNS) SetYggIP(subnet net.IPNet, gw net.IP) error { + netns, err := namespace.GetByName(d.ns) + if err != nil { + return err + } + defer netns.Close() + + if ip6 := subnet.IP.To16(); ip6 == nil { + return fmt.Errorf("expecting ipv6 for ygg interface") + } + + err = netns.Do(func(_ ns.NetNS) error { + link, err := netlink.LinkByName(YggNSInf) + if err != nil { + return err + } + + ips, err := netlink.AddrList(link, netlink.FAMILY_V6) + if err != nil { + return err + } + + for _, ip := range ips { + if YggRange.Contains(ip.IP) { + _ = netlink.AddrDel(link, &ip) + } + } + + if err := netlink.AddrAdd(link, &netlink.Addr{ + IPNet: &subnet, + }); err != nil && !os.IsExist(err) { + return err + } + + if gw == nil { + return nil + } + // set gw for entire ygg range + + return d.setGw(gw) + }) + return err +} + +func (n *yggNS) GetIPs() ([]net.IPNet, error) { + + netns, err := namespace.GetByName(n.ns) + if err != nil { + return nil, err + } + + defer netns.Close() + + var results []net.IPNet + err = netns.Do(func(_ ns.NetNS) error { + links, err := netlink.LinkList() + if err != nil { + return errors.Wrap(err, "failed to list interfaces") + } + + for _, link := range links { + ips, err := netlink.AddrList(link, netlink.FAMILY_V6) + if err != nil { + return err + } + + for _, ip := range ips { + results = append(results, *ip.IPNet) + } + } + + return nil + }) + + return results, err +} + +func (n *yggNS) IsIPv4Only() (bool, error) { + // this is true if DMZPub6 only has local not routable ipv6 addresses + //DMZPub6 + netNS, err := namespace.GetByName(n.ns) + if err != nil { + return false, errors.Wrap(err, "failed to get ndmz namespace") + } + defer netNS.Close() + + var ipv4Only bool + err = netNS.Do(func(_ ns.NetNS) error { + links, err := netlink.LinkList() + if err != nil { + return errors.Wrap(err, "failed to list interfaces") + } + + for _, link := range links { + ips, err := netlink.AddrList(link, netlink.FAMILY_V6) + if err != nil { + return errors.Wrapf(err, "failed to list '%s' ips", link.Attrs().Name) + } + + for _, ip := range ips { + if YggRange.Contains(ip.IP) { + continue + } + + if ip.IP.IsGlobalUnicast() && !ip.IP.IsPrivate() { + // we found a public IPv6 so we are not ipv4 so ygg can peer + // with other ipv6 peers + return nil + } + } + } + + ipv4Only = true + return nil + }) + + return ipv4Only, err +} diff --git a/pkg/network/yggdrasil/utils.go b/pkg/network/yggdrasil/utils.go new file mode 100644 index 000000000..179d3b66a --- /dev/null +++ b/pkg/network/yggdrasil/utils.go @@ -0,0 +1,115 @@ +package yggdrasil + +import ( + "context" + "crypto/ed25519" + "fmt" + "time" + + "github.com/cenkalti/backoff" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + "github.com/threefoldtech/zos/pkg/network/latency" + "github.com/threefoldtech/zos/pkg/zinit" +) + +func fetchPeerList() Peers { + // Try to fetch public peer + // If we failed to do so, use the fallback hardcoded peer list + var pl Peers + + // Do not retry more than 4 times + bo := backoff.WithMaxRetries(backoff.NewConstantBackOff(time.Second), 4) + + fetchPeerList := func() error { + p, err := FetchPeerList() + if err != nil { + log.Debug().Err(err).Msg("failed to fetch yggdrasil peers") + return err + } + pl = p + return nil + } + + err := backoff.Retry(fetchPeerList, bo) + if err != nil { + log.Error().Err(err).Msg("failed to read yggdrasil public peer list online, using fallback") + pl = PeerListFallback + } + + return pl +} + +func EnsureYggdrasil(ctx context.Context, privateKey ed25519.PrivateKey, ns YggdrasilNamespace) (*YggServer, error) { + pl := fetchPeerList() + peersUp := pl.Ups() + endpoints := make([]string, len(peersUp)) + for i, p := range peersUp { + endpoints[i] = p.Endpoint + } + + // filter out the possible yggdrasil public node + var filter latency.IPFilter + ipv4Only, err := ns.IsIPv4Only() + if err != nil { + return nil, errors.Wrap(err, "failed to check ipv6 support for dmz") + } + + if ipv4Only { + // if we are a hidden node,only keep ipv4 public nodes + filter = latency.IPV4Only + } else { + // if we are a dual stack node, filter out all the nodes from the same + // segment so we do not just connect locally + ips, err := ns.GetIPs() + if err != nil { + return nil, errors.Wrap(err, "failed to get ndmz public ipv6") + } + + for _, ip := range ips { + if ip.IP.IsGlobalUnicast() { + filter = latency.ExcludePrefix(ip.IP[:8]) + break + } + } + } + + ls := latency.NewSorter(endpoints, 5, filter) + results := ls.Run(ctx) + if len(results) == 0 { + return nil, fmt.Errorf("cannot find public yggdrasil peer to connect to") + } + + // select the best 3 public peers + peers := make([]string, 3) + for i := 0; i < 3; i++ { + if len(results) > i { + peers[i] = results[i].Endpoint + log.Info().Str("endpoint", results[i].Endpoint).Msg("yggdrasill public peer selected") + } + } + + z, err := zinit.New("") + if err != nil { + return nil, err + } + + cfg := GenerateConfig(privateKey) + cfg.Peers = peers + + server := NewYggServer(&cfg) + if err := server.Ensure(z, ns.Name()); err != nil { + return nil, err + } + + gw, err := server.Gateway() + if err != nil { + return nil, errors.Wrap(err, "fail read yggdrasil subnet") + } + + if err := ns.SetYggIP(gw, nil); err != nil { + return nil, errors.Wrap(err, "fail to configure yggdrasil subnet gateway IP") + } + + return server, nil +} diff --git a/pkg/network/yygdrasil.go b/pkg/network/yggdrasil/yygdrasil.go similarity index 55% rename from pkg/network/yygdrasil.go rename to pkg/network/yggdrasil/yygdrasil.go index 15b568b15..502680736 100644 --- a/pkg/network/yygdrasil.go +++ b/pkg/network/yggdrasil/yygdrasil.go @@ -1,4 +1,4 @@ -package network +package yggdrasil import ( "crypto/ed25519" @@ -10,9 +10,12 @@ import ( "os" "os/exec" "path/filepath" + "strconv" + "strings" "time" - "github.com/threefoldtech/zos/pkg/network/yggdrasil" + "github.com/pkg/errors" + "github.com/threefoldtech/zos/pkg/network/namespace" "github.com/threefoldtech/zos/pkg/zinit" "github.com/yggdrasil-network/yggdrasil-go/src/address" ) @@ -24,25 +27,71 @@ const ( // YggServer represent a yggdrasil server type YggServer struct { - zinit *zinit.Client - cfg *yggdrasil.NodeConfig + cfg *NodeConfig } // NewYggServer create a new yggdrasil Server -func NewYggServer(zinit *zinit.Client, cfg *yggdrasil.NodeConfig) *YggServer { +func NewYggServer(cfg *NodeConfig) *YggServer { return &YggServer{ - zinit: zinit, - cfg: cfg, + cfg: cfg, } } +func (s *YggServer) pidsOf(ns string) ([]uint32, error) { + output, err := exec.Command("ip", "netns", "pids", ns).CombinedOutput() + if err != nil { + return nil, errors.Wrapf(err, "failed to list namespace '%s' pids", ns) + } + parts := strings.Fields(string(output)) + results := make([]uint32, 0, len(parts)) + for _, str := range parts { + pid, err := strconv.ParseUint(str, 10, 32) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse pid '%s'", str) + } + results = append(results, uint32(pid)) + } + return results, nil +} + // Start creates an yggdrasil zinit service and starts it -func (s *YggServer) Start() error { - status, err := s.zinit.Status(zinitService) - if err == nil && status.State.Is(zinit.ServiceStateRunning) { - return nil +func (s *YggServer) Ensure(z *zinit.Client, ns string) error { + if !namespace.Exists(ns) { + return fmt.Errorf("invalid namespace '%s'", ns) } + status, err := z.Status(zinitService) + + if err == nil && status.State.Is(zinit.ServiceStateRunning) { + pids, err := s.pidsOf(ns) + if err != nil { + return errors.Wrap(err, "failed to check if yggdrasil is running in the correct namespace") + } + + in := func(pid uint32) bool { + for _, p := range pids { + if p == pid { + return true + } + } + + return false + } + + if in(uint32(status.Pid)) { + return nil + } + + // not here we need to stop it + if err := z.StopWait(5*time.Second, zinitService); err != nil { + return errors.Wrap(err, "failed to stop yggdrasil service") + } + if err := z.Forget(zinitService); err != nil { + return errors.Wrap(err, "failed to forget yggdrasil service") + } + } + + //TODO: what if it runs in the correct namespace but wrong config ? if err := writeConfig(confPath, s.cfg); err != nil { return err } @@ -53,7 +102,7 @@ func (s *YggServer) Start() error { } err = zinit.AddService(zinitService, zinit.InitService{ - Exec: fmt.Sprintf("ip netns exec ndmz %s -useconffile %s -loglevel trace", bin, confPath), + Exec: fmt.Sprintf("ip netns exec %s %s -useconffile %s -loglevel trace", ns, bin, confPath), After: []string{ "node-ready", "networkd", @@ -64,26 +113,26 @@ func (s *YggServer) Start() error { return err } - if err := s.zinit.Monitor(zinitService); err != nil { + if err := z.Monitor(zinitService); err != nil { return err } - return s.zinit.StartWait(time.Second*20, zinitService) + return z.StartWait(time.Second*20, zinitService) } -// Stop stop the yggdrasil zinit service -func (s *YggServer) Stop() error { - status, err := s.zinit.Status(zinitService) - if err != nil { - return err - } +// // Stop stop the yggdrasil zinit service +// func (s *YggServer) Stop(z *zinit.Client) error { +// status, err := z.Status(zinitService) +// if err != nil { +// return err +// } - if !status.State.Is(zinit.ServiceStateRunning) { - return nil - } +// if !status.State.Is(zinit.ServiceStateRunning) { +// return nil +// } - return s.zinit.StopWait(time.Second*5, zinitService) -} +// return z.StopWait(time.Second*5, zinitService) +// } // NodeID returns the yggdrasil node ID of s func (s *YggServer) NodeID() (ed25519.PublicKey, error) { @@ -142,16 +191,24 @@ func (s *YggServer) Tun() string { // SubnetFor return an IP address out of the node allocated subnet by hasing b and using it // to generate the last 64 bits of the IPV6 address -func (s *YggServer) SubnetFor(b []byte) (net.IP, error) { +func (s *YggServer) SubnetFor(b []byte) (net.IPNet, error) { subnet, err := s.Subnet() if err != nil { - return nil, err + return net.IPNet{}, err } ip := make([]byte, net.IPv6len) copy(ip, subnet.IP) - return subnetFor(ip, b) + subIP, err := subnetFor(ip, b) + if err != nil { + return net.IPNet{}, err + } + + return net.IPNet{ + IP: subIP, + Mask: net.CIDRMask(64, 128), + }, nil } func subnetFor(prefix net.IP, b []byte) (net.IP, error) { @@ -164,7 +221,7 @@ func subnetFor(prefix net.IP, b []byte) (net.IP, error) { return prefix, nil } -func writeConfig(path string, cfg *yggdrasil.NodeConfig) error { +func writeConfig(path string, cfg *NodeConfig) error { if err := os.MkdirAll(filepath.Dir(path), 0770); err != nil { return err } diff --git a/pkg/network/yygdrasil_test.go b/pkg/network/yggdrasil/yygdrasil_test.go similarity index 89% rename from pkg/network/yygdrasil_test.go rename to pkg/network/yggdrasil/yygdrasil_test.go index acd971abe..55beb606b 100644 --- a/pkg/network/yygdrasil_test.go +++ b/pkg/network/yggdrasil/yygdrasil_test.go @@ -1,4 +1,4 @@ -package network +package yggdrasil import ( "net" @@ -6,7 +6,6 @@ import ( "github.com/stretchr/testify/require" "github.com/threefoldtech/zos/pkg/identity" - "github.com/threefoldtech/zos/pkg/network/yggdrasil" "gotest.tools/assert" ) @@ -14,8 +13,8 @@ func TestAddresses(t *testing.T) { kp, err := identity.FromSeed([]byte("00000000000000000000000000000000")) require.NoError(t, err) - cfg := yggdrasil.GenerateConfig(kp.PrivateKey) - s := NewYggServer(nil, &cfg) + cfg := GenerateConfig(kp.PrivateKey) + s := NewYggServer(&cfg) ip, err := s.Address() require.NoError(t, err) diff --git a/pkg/primitives/gateway.go b/pkg/primitives/gateway.go new file mode 100644 index 000000000..5949e20a5 --- /dev/null +++ b/pkg/primitives/gateway.go @@ -0,0 +1,62 @@ +package primitives + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + "github.com/threefoldtech/zos/pkg/gridtypes" + "github.com/threefoldtech/zos/pkg/gridtypes/zos" + "github.com/threefoldtech/zos/pkg/provision" + "github.com/threefoldtech/zos/pkg/stubs" +) + +func validateNameContract(twinID uint32, name string) error { + // TODO: validate against substrate? + return nil +} + +func (p *Primitives) gwProvision(ctx context.Context, wl *gridtypes.WorkloadWithID) (interface{}, error) { + result := zos.GatewayProxyResult{} + var proxy zos.GatewayNameProxy + if err := json.Unmarshal(wl.Data, &proxy); err != nil { + return nil, fmt.Errorf("failed to unmarshal gateway proxy from reservation: %w", err) + } + backends := make([]string, len(proxy.Backends)) + for idx, backend := range proxy.Backends { + backends[idx] = string(backend) + } + // what we need to do: + // - does this node support gateways ? + // this can be validated by checking if we have a "public" namespace + deployment := provision.GetDeployment(ctx) + twinID := deployment.TwinID + if err := validateNameContract(twinID, proxy.Name); err != nil { + return nil, errors.Wrap(err, "failed to validate name contract") + } + // - Validation of ownership of the name (later) + // this must be done against substrate. Make sure that same user (twin) owns the + // name int he workload config + + // - make necessary calls to gateway daemon. + // gateway := stubs.NewGatewayStub(p.zbus) + // gateway.SetNamedProxy(ctx context.Context, arg0 string, arg1 []string) + gateway := stubs.NewGatewayStub(p.zbus) + fqdn, err := gateway.SetNamedProxy(ctx, wl.ID.String(), proxy.Name, backends) + if err != nil { + return nil, errors.Wrap(err, "failed to setup name proxy") + } + result.FQDN = fqdn + log.Debug().Str("domain", fqdn).Msg("domain reserved") + return result, nil +} + +func (p *Primitives) wgDecommission(ctx context.Context, wl *gridtypes.WorkloadWithID) error { + gateway := stubs.NewGatewayStub(p.zbus) + if err := gateway.DeleteNamedProxy(ctx, wl.ID.String()); err != nil { + return errors.Wrap(err, "failed to delete name proxy") + } + return nil +} diff --git a/pkg/primitives/provisioner.go b/pkg/primitives/provisioner.go index e4570d8f1..aadaa6ead 100644 --- a/pkg/primitives/provisioner.go +++ b/pkg/primitives/provisioner.go @@ -23,18 +23,20 @@ func NewPrimitivesProvisioner(zbus zbus.Client) *Primitives { } provisioners := map[gridtypes.WorkloadType]provision.DeployFunction{ - zos.ZMountType: p.zMountProvision, - zos.NetworkType: p.networkProvision, - zos.ZDBType: p.zdbProvision, - zos.ZMachineType: p.virtualMachineProvision, - zos.PublicIPType: p.publicIPProvision, + zos.ZMountType: p.zMountProvision, + zos.NetworkType: p.networkProvision, + zos.ZDBType: p.zdbProvision, + zos.ZMachineType: p.virtualMachineProvision, + zos.PublicIPType: p.publicIPProvision, + zos.GatewayNameProxyType: p.gwProvision, } decommissioners := map[gridtypes.WorkloadType]provision.RemoveFunction{ - zos.ZMountType: p.zMountDecommission, - zos.NetworkType: p.networkDecommission, - zos.ZDBType: p.zdbDecommission, - zos.ZMachineType: p.vmDecomission, - zos.PublicIPType: p.publicIPDecomission, + zos.ZMountType: p.zMountDecommission, + zos.NetworkType: p.networkDecommission, + zos.ZDBType: p.zdbDecommission, + zos.ZMachineType: p.vmDecomission, + zos.PublicIPType: p.publicIPDecomission, + zos.GatewayNameProxyType: p.wgDecommission, } // only network support update atm diff --git a/pkg/provision/mbus/network.go b/pkg/provision/mbus/network.go index 11c775b1e..50c674911 100644 --- a/pkg/provision/mbus/network.go +++ b/pkg/provision/mbus/network.go @@ -104,16 +104,17 @@ func (n *Network) listInterfaces(ctx context.Context) (interface{}, mw.Response) mgr := stubs.NewNetworkerStub(n.cl) results := make(map[string][]net.IP) type q struct { - inf string - ns string + inf string + ns string + rename string } - for _, i := range []q{{"zos", ""}, {"ygg0", "ndmz"}} { + for _, i := range []q{{"zos", "", "zos"}, {"nygg6", "ndmz", "ygg"}} { ips, err := mgr.Addrs(ctx, i.inf, i.ns) if err != nil { return nil, mw.Error(errors.Wrapf(err, "failed to get ips for '%s' interface", i)) } - results[i.inf] = func() []net.IP { + results[i.rename] = func() []net.IP { list := make([]net.IP, 0, len(ips)) for _, item := range ips { ip := net.IP(item) diff --git a/pkg/stubs/gateway_stub.go b/pkg/stubs/gateway_stub.go new file mode 100644 index 000000000..6daf7bb26 --- /dev/null +++ b/pkg/stubs/gateway_stub.go @@ -0,0 +1,52 @@ +package stubs + +import ( + "context" + zbus "github.com/threefoldtech/zbus" +) + +type GatewayStub struct { + client zbus.Client + module string + object zbus.ObjectID +} + +func NewGatewayStub(client zbus.Client) *GatewayStub { + return &GatewayStub{ + client: client, + module: "gateway", + object: zbus.ObjectID{ + Name: "manager", + Version: "0.0.1", + }, + } +} + +func (s *GatewayStub) DeleteNamedProxy(ctx context.Context, arg0 string) (ret0 error) { + args := []interface{}{arg0} + result, err := s.client.RequestContext(ctx, s.module, s.object, "DeleteNamedProxy", args...) + if err != nil { + panic(err) + } + ret0 = new(zbus.RemoteError) + if err := result.Unmarshal(0, &ret0); err != nil { + panic(err) + } + return +} + +func (s *GatewayStub) SetNamedProxy(ctx context.Context, arg0 string, arg1 string, arg2 []string) (ret0 string, ret1 error) { + args := []interface{}{arg0, arg1, arg2} + result, err := s.client.RequestContext(ctx, s.module, s.object, "SetNamedProxy", args...) + if err != nil { + panic(err) + } + if err := result.Unmarshal(0, &ret0); err != nil { + panic(err) + } + ret1 = new(zbus.RemoteError) + if err := result.Unmarshal(1, &ret1); err != nil { + panic(err) + } + return +} diff --git a/pkg/stubs/host_monitor_stub.go b/pkg/stubs/host_monitor_stub.go index a2509811e..f920ffb3b 100644 --- a/pkg/stubs/host_monitor_stub.go +++ b/pkg/stubs/host_monitor_stub.go @@ -36,7 +36,12 @@ func (s *HostMonitorStub) Uptime(ctx context.Context) (<-chan time.Duration, err if err := event.Unmarshal(&obj); err != nil { panic(err) } - ch <- obj + select { + case <-ctx.Done(): + return + case ch <- obj: + default: + } } }() return ch, nil diff --git a/pkg/stubs/network_stub.go b/pkg/stubs/network_stub.go index e8a0779ed..dcca7e3c0 100644 --- a/pkg/stubs/network_stub.go +++ b/pkg/stubs/network_stub.go @@ -70,7 +70,12 @@ func (s *NetworkerStub) DMZAddresses(ctx context.Context) (<-chan pkg.NetlinkAdd if err := event.Unmarshal(&obj); err != nil { panic(err) } - ch <- obj + select { + case <-ctx.Done(): + return + case ch <- obj: + default: + } } }() return ch, nil @@ -229,8 +234,8 @@ func (s *NetworkerStub) PubTapExists(ctx context.Context, arg0 string) (ret0 boo return } -func (s *NetworkerStub) PublicAddresses(ctx context.Context) (<-chan pkg.NetlinkAddresses, error) { - ch := make(chan pkg.NetlinkAddresses) +func (s *NetworkerStub) PublicAddresses(ctx context.Context) (<-chan pkg.OptionPublicConfig, error) { + ch := make(chan pkg.OptionPublicConfig) recv, err := s.client.Stream(ctx, s.module, s.object, "PublicAddresses") if err != nil { return nil, err @@ -238,11 +243,16 @@ func (s *NetworkerStub) PublicAddresses(ctx context.Context) (<-chan pkg.Netlink go func() { defer close(ch) for event := range recv { - var obj pkg.NetlinkAddresses + var obj pkg.OptionPublicConfig if err := event.Unmarshal(&obj); err != nil { panic(err) } - ch <- obj + select { + case <-ctx.Done(): + return + case ch <- obj: + default: + } } }() return ch, nil @@ -431,7 +441,12 @@ func (s *NetworkerStub) YggAddresses(ctx context.Context) (<-chan pkg.NetlinkAdd if err := event.Unmarshal(&obj); err != nil { panic(err) } - ch <- obj + select { + case <-ctx.Done(): + return + case ch <- obj: + default: + } } }() return ch, nil @@ -479,7 +494,12 @@ func (s *NetworkerStub) ZOSAddresses(ctx context.Context) (<-chan pkg.NetlinkAdd if err := event.Unmarshal(&obj); err != nil { panic(err) } - ch <- obj + select { + case <-ctx.Done(): + return + case ch <- obj: + default: + } } }() return ch, nil diff --git a/pkg/stubs/provision_stub.go b/pkg/stubs/provision_stub.go index fe720ceff..644f370c2 100644 --- a/pkg/stubs/provision_stub.go +++ b/pkg/stubs/provision_stub.go @@ -36,7 +36,12 @@ func (s *ProvisionStub) Counters(ctx context.Context) (<-chan pkg.ProvisionCount if err := event.Unmarshal(&obj); err != nil { panic(err) } - ch <- obj + select { + case <-ctx.Done(): + return + case ch <- obj: + default: + } } }() return ch, nil diff --git a/pkg/stubs/storage_stub.go b/pkg/stubs/storage_stub.go index 417f76355..249bfd9c3 100644 --- a/pkg/stubs/storage_stub.go +++ b/pkg/stubs/storage_stub.go @@ -225,7 +225,12 @@ func (s *StorageModuleStub) Monitor(ctx context.Context) (<-chan pkg.PoolsStats, if err := event.Unmarshal(&obj); err != nil { panic(err) } - ch <- obj + select { + case <-ctx.Done(): + return + case ch <- obj: + default: + } } }() return ch, nil diff --git a/pkg/stubs/system_monitor_stub.go b/pkg/stubs/system_monitor_stub.go index ab2dc2ab6..b8a9111cc 100644 --- a/pkg/stubs/system_monitor_stub.go +++ b/pkg/stubs/system_monitor_stub.go @@ -36,7 +36,12 @@ func (s *SystemMonitorStub) CPU(ctx context.Context) (<-chan pkg.TimesStat, erro if err := event.Unmarshal(&obj); err != nil { panic(err) } - ch <- obj + select { + case <-ctx.Done(): + return + case ch <- obj: + default: + } } }() return ch, nil @@ -55,7 +60,12 @@ func (s *SystemMonitorStub) Disks(ctx context.Context) (<-chan pkg.DisksIOCounte if err := event.Unmarshal(&obj); err != nil { panic(err) } - ch <- obj + select { + case <-ctx.Done(): + return + case ch <- obj: + default: + } } }() return ch, nil @@ -74,7 +84,12 @@ func (s *SystemMonitorStub) Memory(ctx context.Context) (<-chan pkg.VirtualMemor if err := event.Unmarshal(&obj); err != nil { panic(err) } - ch <- obj + select { + case <-ctx.Done(): + return + case ch <- obj: + default: + } } }() return ch, nil @@ -93,7 +108,12 @@ func (s *SystemMonitorStub) Nics(ctx context.Context) (<-chan pkg.NicsIOCounterS if err := event.Unmarshal(&obj); err != nil { panic(err) } - ch <- obj + select { + case <-ctx.Done(): + return + case ch <- obj: + default: + } } }() return ch, nil diff --git a/pkg/stubs/version_monitor_stub.go b/pkg/stubs/version_monitor_stub.go index e3a139e61..5629553d8 100644 --- a/pkg/stubs/version_monitor_stub.go +++ b/pkg/stubs/version_monitor_stub.go @@ -36,7 +36,12 @@ func (s *VersionMonitorStub) Version(ctx context.Context) (<-chan semver.Version if err := event.Unmarshal(&obj); err != nil { panic(err) } - ch <- obj + select { + case <-ctx.Done(): + return + case ch <- obj: + default: + } } }() return ch, nil diff --git a/pkg/zinit/zinit.go b/pkg/zinit/zinit.go index 0234a39f9..a874eb612 100644 --- a/pkg/zinit/zinit.go +++ b/pkg/zinit/zinit.go @@ -35,6 +35,10 @@ func New(socket string) (*Client, error) { return &Client{conn: conn, scan: scan}, nil } +func Default() (*Client, error) { + return New(defaultSocketPath) +} + // Close closes the socket connection func (c *Client) Close() error { if c.conn != nil { diff --git a/qemu/overlay/bin/gateway b/qemu/overlay/bin/gateway new file mode 120000 index 000000000..25eae5c70 --- /dev/null +++ b/qemu/overlay/bin/gateway @@ -0,0 +1 @@ +../../../bin/zos \ No newline at end of file diff --git a/qemu/overlay/etc/zinit/gateway.yaml b/qemu/overlay/etc/zinit/gateway.yaml new file mode 120000 index 000000000..3885a023f --- /dev/null +++ b/qemu/overlay/etc/zinit/gateway.yaml @@ -0,0 +1 @@ +../../../../etc/zinit/gateway.yaml \ No newline at end of file