Zos3/wip (#1252)

* Working public-ip reservation

* fix public ip listing

* fix ci

cmds/modules/provisiond/main.go

@@ -198,6 +198,7 @@ func action(cli *cli.Context) error {
 			zos.NetworkType,
 			zos.PublicIPType,
 		),
+		provision.WithRerunAll(app.IsFirstBoot(module)),
 	)
 
 	if err != nil {
@@ -223,6 +224,10 @@ func action(cli *cli.Context) error {
 		}
 	}()
 
+	if err := app.MarkBooted(module); err != nil {
+		log.Error().Err(err).Msg("failed to mark module as booted")
+	}
+
 	reporter, err := NewReported(store, identity, queues)
 	if err != nil {
 		return errors.Wrap(err, "failed to setup capacity reporter")

cmds/modules/provisiond/swagger/zos-api.yml

@@ -445,8 +445,9 @@ components:
         size:
           type: integer
           format: uint
-        network_id:
+        network:
           type: string
+          description: "network name, can be from another deployment"
         ip:
           type: string
           format: ip

pkg/gridtypes/deployment.go

@@ -265,8 +265,8 @@ func (d *Deployment) Get(name string) (*WorkloadWithID, error) {
 
 	for i := range d.Workloads {
 		wl := &d.Workloads[i]
-		id, _ := NewWorkloadID(d.TwinID, d.DeploymentID, name)
 		if wl.Name == name {
+			id, _ := NewWorkloadID(d.TwinID, d.DeploymentID, name)
 			return &WorkloadWithID{
 				Workload: wl,
 				ID:       id,

pkg/gridtypes/zos/kubernetes.go

@@ -107,9 +107,9 @@ type Kubernetes struct {
 	// Size of the vm, this defines the amount of vCpu, memory, and the disk size
 	// Docs: docs/kubernetes/sizes.md
 	Size uint8 `json:"size"`
-	// NetworkID of the network namepsace in which to run the VM. The network
+	// Network of the network namepsace in which to run the VM. The network
 	// must be provisioned previously.
-	NetworkID NetID `json:"network_id"`
+	Network string `json:"network"`
 	// IP of the VM. The IP must be part of the subnet available in the network
 	// resource defined by the networkID on this node
 	IP net.IP `json:"ip"`
@@ -152,7 +152,7 @@ func (k Kubernetes) Challenge(b io.Writer) error {
 	if _, err := fmt.Fprintf(b, "%s", k.ClusterSecret); err != nil {
 		return err
 	}
-	if _, err := fmt.Fprintf(b, "%s", k.NetworkID); err != nil {
+	if _, err := fmt.Fprintf(b, "%s", k.Network); err != nil {
 		return err
 	}
 	if _, err := fmt.Fprintf(b, "%s", k.IP.String()); err != nil {

pkg/network.go

@@ -83,6 +83,14 @@ type Networker interface {
 	// RemovePubTap removes the public tap device from the host namespace
 	RemovePubTap(PubIPReservationID string) error
 
+	// SetupPubIPFilter sets up filter for this public ip
+	SetupPubIPFilter(filterName string, iface string, ip string, ipv6 string, mac string) error
+
+	// RemovePubIPFilter removes the filter setted up by SetupPubIPFilter
+	RemovePubIPFilter(filterName string) error
+
+	// PubIPFilterExists checks if there is a filter installed with that name
+	PubIPFilterExists(filterName string) bool
 	// DisconnectPubTap disconnects the public tap from the network. The interface
 	// itself is not removed and will need to be cleaned up later
 	DisconnectPubTap(PubIPReservationID string) error

pkg/network/networker.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
@@ -454,6 +455,80 @@ func (n *networker) RemovePubTap(pubIPReservationID string) error {
 	return ifaceutil.Delete(tapIface, nil)
 }
 
+// SetupPubIPFilter sets up filter for this public ip
+func (n *networker) SetupPubIPFilter(filterName string, iface string, ip string, ipv6 string, mac string) error {
+	if n.PubIPFilterExists(filterName) {
+		return nil
+	}
+
+	//TODO: use nft.Apply
+	cmd := exec.Command("/bin/sh", "-c",
+		fmt.Sprintf(`# add vm
+# add a chain for the vm public interface in arp and bridge
+nft 'add chain arp filter %[1]s'
+nft 'add chain bridge filter %[1]s'
+
+# make nft jump to vm chain
+nft 'add rule arp filter input iifname "%[2]s" jump %[1]s'
+nft 'add rule bridge filter forward iifname "%[2]s" jump %[1]s'
+
+# arp rule for vm
+nft 'add rule arp filter %[1]s arp operation reply arp saddr ip . arp saddr ether != { %[3]s . %[4]s } drop'
+
+# filter on L2 fowarding of non-matching ip/mac, drop RA,dhcpv6,dhcp
+nft 'add rule bridge filter %[1]s ip saddr . ether saddr != { %[3]s . %[4]s } counter drop'
+nft 'add rule bridge filter %[1]s ip6 saddr . ether saddr != { %[5]s . %[4]s } counter drop'
+nft 'add rule bridge filter %[1]s icmpv6 type nd-router-advert drop'
+nft 'add rule bridge filter %[1]s ip6 version 6 udp sport 547 drop'
+nft 'add rule bridge filter %[1]s ip version 4 udp sport 67 drop'`, filterName, iface, ip, mac, ipv6))
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return errors.Wrapf(err, "could not setup firewall rules for public ip\n%s", string(output))
+	}
+
+	return nil
+}
+
+// PubIPFilterExists checks if pub ip filter
+func (n *networker) PubIPFilterExists(filterName string) bool {
+	cmd := exec.Command(
+		"/bin/sh",
+		"-c",
+		fmt.Sprintf(`nft list table bridge filter | grep "chain %s"`, filterName),
+	)
+	err := cmd.Run()
+	return err == nil
+}
+
+// RemovePubIPFilter removes the filter setted up by SetupPubIPFilter
+func (n *networker) RemovePubIPFilter(filterName string) error {
+	cmd := exec.Command("/bin/sh", "-c",
+		fmt.Sprintf(`# in bridge table
+nft 'flush chain bridge filter %[1]s'
+# jump to chain rule
+a=$( nft -a list table bridge filter | awk '/jump %[1]s/{ print $NF}' )
+nft 'delete rule bridge filter forward handle '${a}
+# chain itself
+a=$( nft -a list table bridge filter | awk '/chain %[1]s/{ print $NF}' )
+nft 'delete chain bridge filter handle '${a}
+
+# in arp table
+nft 'flush chain arp filter %[1]s'
+# jump to chain rule
+a=$( nft -a list table arp filter | awk '/jump %[1]s/{ print $NF}' )
+nft 'delete rule arp filter input handle '${a}
+# chain itself
+a=$( nft -a list table arp filter | awk '/chain %[1]s/{ print $NF}' )
+nft 'delete chain arp filter handle '${a}`, filterName))
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return errors.Wrapf(err, "could not tear down firewall rules for public ip\n%s", string(output))
+	}
+	return nil
+}
+
 // DisconnectPubTap disconnects the public tap from the network. The interface
 // itself is not removed and will need to be cleaned up later
 func (n *networker) DisconnectPubTap(pubIPReservationID string) error {

pkg/primitives/container.go

@@ -55,11 +55,7 @@ func (p *Primitives) containerProvisionImpl(ctx context.Context, wl *gridtypes.W
 	// check if workload is already deployed
 	_, err := containerClient.Inspect(tenantNS, pkg.ContainerID(containerID))
 	if err == nil {
-		log.Info().Stringer("id", containerID).Msg("container already deployed")
-		return ContainerResult{
-			ID:   string(containerID),
-			IPv4: config.Network.IPs[0].String(),
-		}, nil
+		return ContainerResult{}, provision.ErrDidNotChange
 	}
 
 	if err := validateContainerConfig(config); err != nil {

pkg/primitives/kubernetes.go

@@ -53,13 +53,17 @@ func (p *Primitives) kubernetesProvisionImpl(ctx context.Context, wl *gridtypes.
 		needsInstall = true
 	)
 
+	if vm.Exists(wl.ID.String()) {
+		return result, provision.ErrDidNotChange
+	}
+
 	if err := json.Unmarshal(wl.Data, &config); err != nil {
 		return result, errors.Wrap(err, "failed to decode reservation schema")
 	}
 
 	deployment := provision.GetDeployment(ctx)
 
-	netID := zos.NetworkID(deployment.TwinID, string(config.NetworkID))
+	netID := zos.NetworkID(deployment.TwinID, string(config.Network))
 
 	// check if the network tap already exists
 	// if it does, it's most likely that a vm with the same network id and node id already exists
@@ -132,14 +136,19 @@ func (p *Primitives) kubernetesProvisionImpl(ctx context.Context, wl *gridtypes.
 
 	var pubIface string
 	if len(config.PublicIP) > -0 {
-		pubIface, err = network.SetupPubTap(config.PublicIP)
+		ipWl, err := deployment.Get(config.PublicIP)
+		if err != nil {
+			return zos.KubernetesResult{}, err
+		}
+		name := ipWl.ID.String()
+		pubIface, err = network.SetupPubTap(name)
 		if err != nil {
 			return result, errors.Wrap(err, "could not set up tap device for public network")
 		}
 
 		defer func() {
 			if err != nil {
-				_ = network.RemovePubTap(config.PublicIP)
+				_ = network.RemovePubTap(name)
 			}
 		}()
 	}
@@ -291,7 +300,7 @@ func (p *Primitives) kubernetesDecomission(ctx context.Context, wl *gridtypes.Wo
 
 	deployment := provision.GetDeployment(ctx)
 
-	netID := zos.NetworkID(deployment.TwinID, string(cfg.NetworkID))
+	netID := zos.NetworkID(deployment.TwinID, string(cfg.Network))
 	if err := network.RemoveTap(netID); err != nil {
 		return errors.Wrap(err, "could not clean up tap device")
 	}
@@ -312,7 +321,7 @@ func (p *Primitives) kubernetesDecomission(ctx context.Context, wl *gridtypes.Wo
 func (p *Primitives) buildNetworkInfo(ctx context.Context, deployment gridtypes.Deployment, iface string, pubIface string, cfg Kubernetes) (pkg.VMNetworkInfo, error) {
 	network := stubs.NewNetworkerStub(p.zbus)
 
-	netID := zos.NetworkID(deployment.TwinID, string(cfg.NetworkID))
+	netID := zos.NetworkID(deployment.TwinID, string(cfg.Network))
 	subnet, err := network.GetSubnet(netID)
 	if err != nil {
 		return pkg.VMNetworkInfo{}, errors.Wrapf(err, "could not get network resource subnet")
@@ -398,14 +407,20 @@ func (p *Primitives) getPubIPConfig(wl *gridtypes.WorkloadWithID, name string) (
 	// we used to get this from the explorer, but now we need another
 	// way to do this. for now the only option is to get it from the
 	// reservation itself. hence we added the gatway fields to ip data
+	if wl.Type != zos.PublicIPType {
+		return ip, gw, fmt.Errorf("workload for public IP is of wrong type")
+	}
 
+	if wl.Result.State != gridtypes.StateOk {
+		return ip, gw, fmt.Errorf("public ip workload is not okay")
+	}
 	ipData, err := wl.WorkloadData()
 	if err != nil {
 		return
 	}
-	data, ok := ipData.(zos.PublicIP)
+	data, ok := ipData.(*zos.PublicIP)
 	if !ok {
-		return ip, gw, fmt.Errorf("invalid ip data in deployment")
+		return ip, gw, fmt.Errorf("invalid ip data in deployment got '%T'", ipData)
 	}
 
 	return data.IP.IPNet, data.Gateway, nil

pkg/primitives/public_ip.go

@@ -6,14 +6,14 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
-	"os/exec"
 	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zos/pkg/gridtypes"
 	"github.com/threefoldtech/zos/pkg/gridtypes/zos"
 	"github.com/threefoldtech/zos/pkg/network/ifaceutil"
+	"github.com/threefoldtech/zos/pkg/provision"
 	"github.com/threefoldtech/zos/pkg/stubs"
 )
 
@@ -25,6 +25,11 @@ func (p *Primitives) publicIPProvisionImpl(ctx context.Context, wl *gridtypes.Wo
 	config := zos.PublicIP{}
 
 	network := stubs.NewNetworkerStub(p.zbus)
+	fName := filterName(wl.ID.String())
+
+	if network.PubIPFilterExists(fName) {
+		return result, provision.ErrDidNotChange
+	}
 
 	if err := json.Unmarshal(wl.Data, &config); err != nil {
 		return zos.PublicIPResult{}, errors.Wrap(err, "failed to decode reservation schema")
@@ -36,7 +41,6 @@ func (p *Primitives) publicIPProvisionImpl(ctx context.Context, wl *gridtypes.Wo
 	}
 
 	tapName := fmt.Sprintf("p-%s", wl.ID.String()) // TODO: clean this up, needs to come form networkd
-	fName := filterName(wl.ID.String())
 	mac := ifaceutil.HardwareAddrFromInputBytes([]byte(wl.ID.String()))
 
 	predictedIPv6, err := predictedSlaac(pubIP6Base.IP, mac.String())
@@ -45,15 +49,15 @@ func (p *Primitives) publicIPProvisionImpl(ctx context.Context, wl *gridtypes.Wo
 	}
 
 	result.IP = config.IP
-	err = setupFilters(ctx, fName, tapName, config.IP.IP.To4().String(), predictedIPv6, mac.String())
+	err = network.SetupPubIPFilter(fName, tapName, config.IP.IP.To4().String(), predictedIPv6, mac.String())
 	return
 }
 
 func (p *Primitives) publicIPDecomission(ctx context.Context, wl *gridtypes.WorkloadWithID) error {
 	// Disconnect the public interface from the network if one exists
 	network := stubs.NewNetworkerStub(p.zbus)
 	fName := filterName(wl.ID.String())
-	if err := teardownFilters(ctx, fName); err != nil {
+	if err := network.RemovePubIPFilter(fName); err != nil {
 		log.Error().Err(err).Msg("could not remove filter rules")
 	}
 	return network.DisconnectPubTap(wl.ID.String())
@@ -63,59 +67,6 @@ func filterName(reservationID string) string {
 	return fmt.Sprintf("r-%s", reservationID)
 }
 
-func setupFilters(ctx context.Context, fName string, iface string, ip string, ipv6 string, mac string) error {
-	cmd := exec.CommandContext(ctx, "/bin/sh", "-c",
-		fmt.Sprintf(`# add vm
-# add a chain for the vm public interface in arp and bridge
-nft 'add chain arp filter %[1]s'
-nft 'add chain bridge filter %[1]s'
-
-# make nft jump to vm chain
-nft 'add rule arp filter input iifname "%[2]s" jump %[1]s'
-nft 'add rule bridge filter forward iifname "%[2]s" jump %[1]s'
-
-# arp rule for vm
-nft 'add rule arp filter %[1]s arp operation reply arp saddr ip . arp saddr ether != { %[3]s . %[4]s } drop'
-
-# filter on L2 fowarding of non-matching ip/mac, drop RA,dhcpv6,dhcp
-nft 'add rule bridge filter %[1]s ip saddr . ether saddr != { %[3]s . %[4]s } counter drop'
-nft 'add rule bridge filter %[1]s ip6 saddr . ether saddr != { %[5]s . %[4]s } counter drop'
-nft 'add rule bridge filter %[1]s icmpv6 type nd-router-advert drop'
-nft 'add rule bridge filter %[1]s ip6 version 6 udp sport 547 drop'
-nft 'add rule bridge filter %[1]s ip version 4 udp sport 67 drop'`, fName, iface, ip, mac, ipv6))
-
-	if err := cmd.Run(); err != nil {
-		return errors.Wrap(err, "could not setup firewall rules for public ip")
-	}
-	return nil
-}
-
-func teardownFilters(ctx context.Context, fName string) error {
-	cmd := exec.CommandContext(ctx, "/bin/sh", "-c",
-		fmt.Sprintf(`# in bridge table
-nft 'flush chain bridge filter %[1]s'
-# jump to chain rule
-a=$( nft -a list table bridge filter | awk '/jump %[1]s/{ print $NF}' )
-nft 'delete rule bridge filter forward handle '${a}
-# chain itself
-a=$( nft -a list table bridge filter | awk '/chain %[1]s/{ print $NF}' )
-nft 'delete chain bridge filter handle '${a}
-
-# in arp table
-nft 'flush chain arp filter %[1]s'
-# jump to chain rule
-a=$( nft -a list table arp filter | awk '/jump %[1]s/{ print $NF}' )
-nft 'delete rule arp filter input handle '${a}
-# chain itself
-a=$( nft -a list table arp filter | awk '/chain %[1]s/{ print $NF}' )
-nft 'delete chain arp filter handle '${a}`, fName))
-
-	if err := cmd.Run(); err != nil {
-		return errors.Wrap(err, "could not setup firewall rules for public ip")
-	}
-	return nil
-}
-
 // modified version of: https://github.com/MalteJ/docker/blob/f09b7897d2a54f35a0b26f7cbe750b3c9383a553/daemon/networkdriver/bridge/driver.go#L585
 func predictedSlaac(base net.IP, mac string) (string, error) {
 	// TODO: get pub ipv6 prefix

pkg/provision/api/network.go

@@ -71,7 +71,7 @@ func (n *Network) listPublicIps(request *http.Request) (interface{}, mw.Response
 	if err != nil {
 		return nil, mw.Error(errors.Wrap(err, "failed to list twins"))
 	}
-	var ips []string
+	ips := make([]string, 0)
 	for _, twin := range twins {
 		deploymentsIDs, err := storage.ByTwin(twin)
 		if err != nil {
@@ -94,7 +94,7 @@ func (n *Network) listPublicIps(request *http.Request) (interface{}, mw.Response
 					return nil, mw.Error(err)
 				}
 
-				ip, _ := data.(zos.PublicIP)
+				ip, _ := data.(*zos.PublicIP)
 				if ip.IP.IP != nil {
 					ips = append(ips, ip.IP.String())
 				}