azure private networking

CHANGELOG.md

@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - `Metro` to Equinix Metal options ([#281](https://github.com/flatcar-linux/mantle/pull/281))
 - `update-offer` ore subcommand for AWS marketplace publishing ([#282](https://github.com/flatcar-linux/mantle/pull/282))
 - kola test `cl.swap_activation` for swap activation with CLC ([#284](https://github.com/flatcar-linux/mantle/pull/284))
+- Azure: support for running Kola within an existing vnet and with private addressing ([#295](https://github.com/flatcar-linux/mantle/pull/295))
 
 ### Changed
 - removed `packet` occurrences in favor of `equinixmetal` ([#277](https://github.com/flatcar-linux/mantle/pull/277))

cmd/kola/options.go

@@ -107,7 +107,9 @@ func init() {
 	sv(&kola.AzureOptions.Location, "azure-location", "westus", "Azure location (default \"westus\"")
 	sv(&kola.AzureOptions.Size, "azure-size", "Standard_DS2_v2", "Azure machine size (default \"Standard_DS2_v2\")")
 	sv(&kola.AzureOptions.HyperVGeneration, "azure-hyper-v-generation", "V1", "Azure Hyper-V Generation (\"V1\" or \"V2\")")
+	sv(&kola.AzureOptions.VnetSubnetName, "azure-vnet-subnet-name", "", "Use a pre-existing virtual network for created instances. Specify as vnet-name/subnet-name. If subnet name is omitted then \"default\" is assumed")
 	bv(&kola.AzureOptions.UseGallery, "azure-use-gallery", false, "Use gallery image instead of managed image")
+	bv(&kola.AzureOptions.UsePrivateIPs, "azure-use-private-ips", false, "Assume nodes are reachable using private IP addresses")
 
 	// do-specific options
 	sv(&kola.DOOptions.ConfigPath, "do-config-file", "", "DigitalOcean config file (default \"~/"+auth.DOConfigPath+"\")")

platform/api/azure/api.go

@@ -53,6 +53,10 @@ type API struct {
 	opts       *Options
 }
 
+type Network struct {
+	subnet network.Subnet
+}
+
 // New creates a new Azure client. If no publish settings file is provided or
 // can't be parsed, an anonymous client is created.
 func New(opts *Options) (*API, error) {

platform/api/azure/instance.go

@@ -59,6 +59,7 @@ func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI string,
 		osProfile.CustomData = &ud
 	}
 	var imgRef *compute.ImageReference
+	var plan *compute.Plan
 	if a.opts.DiskURI != "" {
 		imgRef = &compute.ImageReference{
 			ID: &a.opts.DiskURI,
@@ -70,13 +71,19 @@ func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI string,
 			Sku:       &a.opts.Sku,
 			Version:   &a.opts.Version,
 		}
+		plan = &compute.Plan{
+			Publisher: imgRef.Publisher,
+			Product:   imgRef.Offer,
+			Name:      imgRef.Sku,
+		}
 	}
 	return compute.VirtualMachine{
 		Name:     &name,
 		Location: &a.opts.Location,
 		Tags: map[string]*string{
 			"createdBy": util.StrToPtr("mantle"),
 		},
+		Plan: plan,
 		VirtualMachineProperties: &compute.VirtualMachineProperties{
 			HardwareProfile: &compute.HardwareProfile{
 				VMSize: compute.VirtualMachineSizeTypes(a.opts.Size),
@@ -110,11 +117,8 @@ func (a *API) getVMParameters(name, userdata, sshkey, storageAccountURI string,
 	}
 }
 
-func (a *API) CreateInstance(name, userdata, sshkey, resourceGroup, storageAccount string) (*Machine, error) {
-	subnet, err := a.getSubnet(resourceGroup)
-	if err != nil {
-		return nil, fmt.Errorf("preparing network resources: %v", err)
-	}
+func (a *API) CreateInstance(name, userdata, sshkey, resourceGroup, storageAccount string, network Network) (*Machine, error) {
+	subnet := network.subnet
 
 	ip, err := a.createPublicIP(resourceGroup)
 	if err != nil {
@@ -177,8 +181,12 @@ func (a *API) CreateInstance(name, userdata, sshkey, resourceGroup, storageAccou
 	if vm.Name == nil {
 		return nil, fmt.Errorf("couldn't get VM ID")
 	}
-
-	publicaddr, privaddr, err := a.GetIPAddresses(*nic.Name, *ip.Name, resourceGroup)
+	ipName := *ip.Name
+	if a.opts.UsePrivateIPs {
+		// empty IP name means instance is accessible via private IP address
+		ipName = ""
+	}
+	publicaddr, privaddr, err := a.GetIPAddresses(*nic.Name, ipName, resourceGroup)
 	if err != nil {
 		return nil, err
 	}
@@ -188,7 +196,7 @@ func (a *API) CreateInstance(name, userdata, sshkey, resourceGroup, storageAccou
 		PublicIPAddress:  publicaddr,
 		PrivateIPAddress: privaddr,
 		InterfaceName:    *nic.Name,
-		PublicIPName:     *ip.Name,
+		PublicIPName:     ipName,
 	}, nil
 }
 

platform/api/azure/network.go

@@ -17,6 +17,7 @@ package azure
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-02-01/network"
 
@@ -26,19 +27,64 @@ import (
 var (
 	virtualNetworkPrefix = []string{"10.0.0.0/16"}
 	subnetPrefix         = "10.0.0.0/24"
+	kolaSubnet           = "kola-subnet"
+	kolaVnet             = "kola-vn"
 )
 
-func (a *API) PrepareNetworkResources(resourceGroup string) (network.Subnet, error) {
+func (a *API) PrepareNetworkResources(resourceGroup string) (Network, error) {
+	if a.opts.VnetSubnetName != "" {
+		parts := strings.SplitN(a.opts.VnetSubnetName, "/", 2)
+		vnetName := parts[0]
+		subnetName := "default"
+		if len(parts) > 1 {
+			subnetName = parts[1]
+		}
+		result, err := a.netClient.ListAllComplete(context.TODO())
+		if err != nil {
+			return Network{}, fmt.Errorf("failed to query vnets: %w", err)
+		}
+		var net network.VirtualNetwork
+		found := false
+		for result.NotDone() {
+			net = result.Value()
+			if net.Name != nil && *net.Name == vnetName {
+				found = true
+				break
+			}
+			err = result.Next()
+			if err != nil {
+				return Network{}, fmt.Errorf("failed to iterate vnets: %w", err)
+			}
+		}
+		if !found {
+			return Network{}, fmt.Errorf("failed to find vnet %s", vnetName)
+		}
+		subnets := net.VirtualNetworkPropertiesFormat.Subnets
+		if subnets == nil {
+			return Network{}, fmt.Errorf("failed to find subnet %s in vnet %s", subnetName, vnetName)
+		}
+		for _, subnet := range *subnets {
+			if subnet.Name != nil && *subnet.Name == subnetName {
+				return Network{subnet}, nil
+			}
+		}
+		return Network{}, fmt.Errorf("failed to find subnet %s in vnet %s", subnetName, vnetName)
+	}
+
 	if err := a.createVirtualNetwork(resourceGroup); err != nil {
-		return network.Subnet{}, err
+		return Network{}, err
 	}
 
-	return a.createSubnet(resourceGroup)
+	subnet, err := a.createSubnet(resourceGroup)
+	if err != nil {
+		return Network{}, err
+	}
+	return Network{subnet}, nil
 }
 
 func (a *API) createVirtualNetwork(resourceGroup string) error {
-	plog.Infof("Creating VirtualNetwork %s", "kola-vn")
-	future, err := a.netClient.CreateOrUpdate(context.TODO(), resourceGroup, "kola-vn", network.VirtualNetwork{
+	plog.Infof("Creating VirtualNetwork %s", kolaVnet)
+	future, err := a.netClient.CreateOrUpdate(context.TODO(), resourceGroup, kolaVnet, network.VirtualNetwork{
 		Location: &a.opts.Location,
 		VirtualNetworkPropertiesFormat: &network.VirtualNetworkPropertiesFormat{
 			AddressSpace: &network.AddressSpace{
@@ -58,8 +104,8 @@ func (a *API) createVirtualNetwork(resourceGroup string) error {
 }
 
 func (a *API) createSubnet(resourceGroup string) (network.Subnet, error) {
-	plog.Infof("Creating Subnet %s", "kola-subnet")
-	future, err := a.subClient.CreateOrUpdate(context.TODO(), resourceGroup, "kola-vn", "kola-subnet", network.Subnet{
+	plog.Infof("Creating Subnet %s", kolaSubnet)
+	future, err := a.subClient.CreateOrUpdate(context.TODO(), resourceGroup, kolaVnet, kolaSubnet, network.Subnet{
 		SubnetPropertiesFormat: &network.SubnetPropertiesFormat{
 			AddressPrefix: &subnetPrefix,
 		},
@@ -74,8 +120,8 @@ func (a *API) createSubnet(resourceGroup string) (network.Subnet, error) {
 	return future.Result(a.subClient)
 }
 
-func (a *API) getSubnet(resourceGroup string) (network.Subnet, error) {
-	return a.subClient.Get(context.TODO(), resourceGroup, "kola-vn", "kola-subnet", "")
+func (a *API) getSubnet(resourceGroup, vnet, subnet string) (network.Subnet, error) {
+	return a.subClient.Get(context.TODO(), resourceGroup, vnet, subnet, "")
 }
 
 func (a *API) createPublicIP(resourceGroup string) (*network.PublicIPAddress, error) {
@@ -105,7 +151,7 @@ func (a *API) createPublicIP(resourceGroup string) (*network.PublicIPAddress, er
 	return &ip, nil
 }
 
-func (a *API) GetPublicIP(name, resourceGroup string) (string, error) {
+func (a *API) getPublicIP(name, resourceGroup string) (string, error) {
 	ip, err := a.ipClient.Get(context.TODO(), resourceGroup, name, "")
 	if err != nil {
 		return "", err
@@ -120,24 +166,31 @@ func (a *API) GetPublicIP(name, resourceGroup string) (string, error) {
 
 // returns PublicIP, PrivateIP, error
 func (a *API) GetIPAddresses(name, publicIPName, resourceGroup string) (string, string, error) {
-	publicIP, err := a.GetPublicIP(publicIPName, resourceGroup)
-	if err != nil {
-		return "", "", err
-	}
-
 	nic, err := a.intClient.Get(context.TODO(), resourceGroup, name, "")
 	if err != nil {
 		return "", "", err
 	}
-
 	configs := *nic.InterfacePropertiesFormat.IPConfigurations
+	var privateIP *string
 	for _, conf := range configs {
 		if conf.PrivateIPAddress == nil {
 			return "", "", fmt.Errorf("PrivateIPAddress is nil")
 		}
-		return publicIP, *conf.PrivateIPAddress, nil
+		privateIP = conf.PrivateIPAddress
+		break
+	}
+	if privateIP == nil {
+		return "", "", fmt.Errorf("no ip configurations found")
+	}
+	if publicIPName == "" {
+		return *privateIP, *privateIP, nil
+	}
+
+	publicIP, err := a.getPublicIP(publicIPName, resourceGroup)
+	if err != nil {
+		return "", "", err
 	}
-	return "", "", fmt.Errorf("no ip configurations found")
+	return publicIP, *privateIP, nil
 }
 
 func (a *API) GetPrivateIP(name, resourceGroup string) (string, error) {

platform/api/azure/options.go

@@ -35,7 +35,9 @@ type Options struct {
 	Size             string
 	Location         string
 	HyperVGeneration string
+	VnetSubnetName   string
 	UseGallery       bool
+	UsePrivateIPs    bool
 
 	SubscriptionName string
 	SubscriptionID   string

platform/machine/azure/cluster.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 
 	"github.com/flatcar-linux/mantle/platform"
+	"github.com/flatcar-linux/mantle/platform/api/azure"
 	"github.com/flatcar-linux/mantle/platform/conf"
 )
 
@@ -30,6 +31,7 @@ type cluster struct {
 	sshKey         string
 	ResourceGroup  string
 	StorageAccount string
+	Network        azure.Network
 }
 
 func (ac *cluster) vmname() string {
@@ -46,7 +48,7 @@ func (ac *cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error)
 		return nil, err
 	}
 
-	instance, err := ac.flight.api.CreateInstance(ac.vmname(), conf.String(), ac.sshKey, ac.ResourceGroup, ac.StorageAccount)
+	instance, err := ac.flight.api.CreateInstance(ac.vmname(), conf.String(), ac.sshKey, ac.ResourceGroup, ac.StorageAccount, ac.Network)
 	if err != nil {
 		return nil, err
 	}

platform/machine/azure/flight.go

@@ -40,6 +40,7 @@ type flight struct {
 	FakeSSHKey          string
 	ImageResourceGroup  string
 	ImageStorageAccount string
+	Network             azure.Network
 }
 
 // NewFlight creates an instance of a Flight suitable for spawning
@@ -93,8 +94,9 @@ func NewFlight(opts *azure.Options) (platform.Flight, error) {
 			return nil, err
 		}
 
-		_, err = af.api.PrepareNetworkResources(af.ImageResourceGroup)
+		af.Network, err = af.api.PrepareNetworkResources(af.ImageResourceGroup)
 		if err != nil {
+			af.Destroy()
 			return nil, err
 		}
 
@@ -173,7 +175,7 @@ func (af *flight) NewCluster(rconf *platform.RuntimeConfig) (platform.Cluster, e
 	if af.ImageResourceGroup != "" && af.ImageStorageAccount != "" {
 		ac.ResourceGroup = af.ImageResourceGroup
 		ac.StorageAccount = af.ImageStorageAccount
-
+		ac.Network = af.Network
 	} else {
 		ac.ResourceGroup, err = af.api.CreateResourceGroup("kola-cluster")
 		if err != nil {
@@ -185,8 +187,9 @@ func (af *flight) NewCluster(rconf *platform.RuntimeConfig) (platform.Cluster, e
 			return nil, err
 		}
 
-		_, err = af.api.PrepareNetworkResources(ac.ResourceGroup)
+		ac.Network, err = af.api.PrepareNetworkResources(ac.ResourceGroup)
 		if err != nil {
+			ac.Destroy()
 			return nil, err
 		}
 	}