Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor: Organize network module #391

Merged
merged 2 commits into from
Dec 20, 2022
Merged

refactor: Organize network module #391

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5616b42
Organize network module
helayoty Dec 14, 2022
13d74fe
move util to separate pkg
helayoty Dec 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ require (
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.6.0
github.com/stretchr/testify v1.7.0
github.com/thoas/go-funk v0.9.1
github.com/virtual-kubelet/node-cli v0.8.0
github.com/virtual-kubelet/virtual-kubelet v1.6.0
Expand Down Expand Up @@ -70,6 +71,7 @@ require (
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.11.1 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.26.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -454,6 +454,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
Expand Down Expand Up @@ -752,6 +753,7 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
Expand Down
138 changes: 57 additions & 81 deletions pkg/provider/aci_network.go → pkg/network/aci_network.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
Copyright (c) Microsoft Corporation.
Licensed under the Apache 2.0 license.
*/
package provider
package network

import (
"context"
Expand All @@ -12,6 +12,7 @@ import (
"strings"

"github.com/pkg/errors"
"github.com/virtual-kubelet/azure-aci/pkg/util"
utilvalidation "k8s.io/apimachinery/pkg/util/validation"

azaci "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2021-10-01/containerinstance"
Expand All @@ -25,103 +26,89 @@ import (

// DNS configuration settings
const (
maxDNSNameservers = 3
maxDNSSearchPaths = 6
maxDNSSearchListChars = 256
maxDNSNameservers = 3
maxDNSSearchPaths = 6
maxDNSSearchListChars = 256
subnetDelegationService = "Microsoft.ContainerInstance/containerGroups"
)

func (p *ACIProvider) setVNETConfig(ctx context.Context, azConfig *auth.Config) error {
type ProviderNetwork struct {
VnetSubscriptionID string
VnetName string
VnetResourceGroup string
SubnetName string
SubnetCIDR string
KubeDNSIP string
}

func (pn *ProviderNetwork) SetVNETConfig(ctx context.Context, azConfig *auth.Config) error {
// the VNET subscription ID by default is authentication subscription ID.
// We need to override when using cross subscription virtual network resource
p.vnetSubscriptionID = azConfig.AuthConfig.SubscriptionID
pn.VnetSubscriptionID = azConfig.AuthConfig.SubscriptionID
if vnetSubscriptionID := os.Getenv("ACI_VNET_SUBSCRIPTION_ID"); vnetSubscriptionID != "" {
p.vnetSubscriptionID = vnetSubscriptionID
pn.VnetSubscriptionID = vnetSubscriptionID
}

if vnetName := os.Getenv("ACI_VNET_NAME"); vnetName != "" {
p.vnetName = vnetName
} else if p.vnetName == "" {
pn.VnetName = vnetName
} else if pn.VnetName == "" {
return errors.New("vnet name can not be empty please set ACI_VNET_NAME")
}

if vnetResourceGroup := os.Getenv("ACI_VNET_RESOURCE_GROUP"); vnetResourceGroup != "" {
p.vnetResourceGroup = vnetResourceGroup
} else if p.vnetResourceGroup == "" {
pn.VnetResourceGroup = vnetResourceGroup
} else if pn.VnetResourceGroup == "" {
return errors.New("vnet resourceGroup can not be empty please set ACI_VNET_RESOURCE_GROUP")
}

// Set subnet properties.
if subnetName := os.Getenv("ACI_SUBNET_NAME"); p.vnetName != "" && subnetName != "" {
p.subnetName = subnetName
if subnetName := os.Getenv("ACI_SUBNET_NAME"); pn.VnetName != "" && subnetName != "" {
pn.SubnetName = subnetName
}

if subnetCIDR := os.Getenv("ACI_SUBNET_CIDR"); subnetCIDR != "" {
if p.subnetName == "" {
if pn.SubnetName == "" {
return fmt.Errorf("subnet CIDR defined but no subnet name, subnet name is required to set a subnet CIDR")
}
if _, _, err := net.ParseCIDR(subnetCIDR); err != nil {
return fmt.Errorf("error parsing provided subnet range: %v", err)
}
p.subnetCIDR = subnetCIDR
pn.SubnetCIDR = subnetCIDR
}

if p.subnetName != "" {
if err := p.setupNetwork(ctx, azConfig); err != nil {
if pn.SubnetName != "" {
if err := pn.setupNetwork(ctx, azConfig); err != nil {
return fmt.Errorf("error setting up network: %v", err)
}

masterURI := os.Getenv("MASTER_URI")
if masterURI == "" {
masterURI = "10.0.0.1"
}

clusterCIDR := os.Getenv("CLUSTER_CIDR")
if clusterCIDR == "" {
clusterCIDR = "10.240.0.0/16"
}

// setup aci extensions
kubeExtensions, err := client2.GetKubeProxyExtension(serviceAccountSecretMountPath, masterURI, clusterCIDR)
if err != nil {
return fmt.Errorf("error creating kube proxy extension: %v", err)
}

p.containerGroupExtensions = append(p.containerGroupExtensions, kubeExtensions)

enableRealTimeMetricsExtension := os.Getenv("ENABLE_REAL_TIME_METRICS")
if enableRealTimeMetricsExtension == "true" {
realtimeExtension := client2.GetRealtimeMetricsExtension()
p.containerGroupExtensions = append(p.containerGroupExtensions, realtimeExtension)
}

if kubeDNSIP := os.Getenv("KUBE_DNS_IP"); kubeDNSIP != "" {
p.kubeDNSIP = kubeDNSIP
pn.KubeDNSIP = kubeDNSIP
}
}
return nil
}

func (p *ACIProvider) setupNetwork(ctx context.Context, azConfig *auth.Config) error {
func (pn *ProviderNetwork) setupNetwork(ctx context.Context, azConfig *auth.Config) error {
c := aznetwork.NewSubnetsClient(azConfig.AuthConfig.SubscriptionID)
c.Authorizer = azConfig.Authorizer

createSubnet := true
subnet, err := c.Get(ctx, p.vnetResourceGroup, p.vnetName, p.subnetName, "")
subnet, err := c.Get(ctx, pn.VnetResourceGroup, pn.VnetName, pn.SubnetName, "")
if err != nil && !network.IsNotFound(err) {
return fmt.Errorf("error while looking up subnet: %v", err)
}
if network.IsNotFound(err) && p.subnetCIDR == "" {
return fmt.Errorf("subnet '%s' is not found in vnet '%s' in resource group '%s' and subscription '%s' and subnet CIDR is not specified", p.subnetName, p.vnetName, p.vnetResourceGroup, p.vnetSubscriptionID)
if network.IsNotFound(err) && pn.SubnetCIDR == "" {
return fmt.Errorf("subnet '%s' is not found in vnet '%s' in resource group '%s' and subscription '%s' and subnet CIDR is not specified", pn.SubnetName, pn.VnetName, pn.VnetResourceGroup, pn.VnetSubscriptionID)
}
if err == nil {
if p.subnetCIDR == "" {
p.subnetCIDR = *subnet.SubnetPropertiesFormat.AddressPrefix
if pn.SubnetCIDR == "" {
pn.SubnetCIDR = *subnet.SubnetPropertiesFormat.AddressPrefix
}
if p.subnetCIDR != *subnet.SubnetPropertiesFormat.AddressPrefix {
return fmt.Errorf("found subnet '%s' using different CIDR: '%s'. desired: '%s'", p.subnetName, *subnet.SubnetPropertiesFormat.AddressPrefix, p.subnetCIDR)
if pn.SubnetCIDR != *subnet.SubnetPropertiesFormat.AddressPrefix {
return fmt.Errorf("found subnet '%s' using different CIDR: '%s'. desired: '%s'", pn.SubnetName, *subnet.SubnetPropertiesFormat.AddressPrefix, pn.SubnetCIDR)
}
if subnet.SubnetPropertiesFormat.RouteTable != nil {
return fmt.Errorf("unable to delegate subnet '%s' to Azure Container Instance since it references the route table '%s'", p.subnetName, *subnet.SubnetPropertiesFormat.RouteTable.ID)
return fmt.Errorf("unable to delegate subnet '%s' to Azure Container Instance since it references the route table '%s'", pn.SubnetName, *subnet.SubnetPropertiesFormat.RouteTable.ID)
}
if subnet.SubnetPropertiesFormat.ServiceAssociationLinks != nil {
for _, l := range *subnet.SubnetPropertiesFormat.ServiceAssociationLinks {
Expand All @@ -130,7 +117,7 @@ func (p *ACIProvider) setupNetwork(ctx context.Context, azConfig *auth.Config) e
createSubnet = false
break
} else {
return fmt.Errorf("unable to delegate subnet '%s' to Azure Container Instance as it is used by other Azure resource: '%v'", p.subnetName, l)
return fmt.Errorf("unable to delegate subnet '%s' to Azure Container Instance as it is used by other Azure resource: '%v'", pn.SubnetName, l)
}
}
}
Expand All @@ -152,9 +139,9 @@ func (p *ACIProvider) setupNetwork(ctx context.Context, azConfig *auth.Config) e
)

subnet = aznetwork.Subnet{
Name: &p.subnetName,
Name: &pn.SubnetName,
SubnetPropertiesFormat: &aznetwork.SubnetPropertiesFormat{
AddressPrefix: &p.subnetCIDR,
AddressPrefix: &pn.SubnetCIDR,
Delegations: &[]aznetwork.Delegation{
{
Name: &delegationName,
Expand All @@ -166,40 +153,39 @@ func (p *ACIProvider) setupNetwork(ctx context.Context, azConfig *auth.Config) e
},
},
}
_, err = c.CreateOrUpdate(ctx, p.vnetResourceGroup, p.vnetName, p.subnetName, subnet)
_, err = c.CreateOrUpdate(ctx, pn.VnetResourceGroup, pn.VnetName, pn.SubnetName, subnet)
if err != nil {
return fmt.Errorf("error creating subnet: %v", err)
}
}
return nil
}

func (p *ACIProvider) amendVnetResources(ctx context.Context, cg client2.ContainerGroupWrapper, pod *v1.Pod) {
if p.subnetName == "" {
func (pn *ProviderNetwork) AmendVnetResources(ctx context.Context, cg client2.ContainerGroupWrapper, pod *v1.Pod, clusterDomain string) {
if pn.SubnetName == "" {
return
}

subnetID := "/subscriptions/" + p.vnetSubscriptionID + "/resourceGroups/" + p.vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + p.vnetName + "/subnets/" + p.subnetName
subnetID := "/subscriptions/" + pn.VnetSubscriptionID + "/resourceGroups/" + pn.VnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + pn.VnetName + "/subnets/" + pn.SubnetName
cgIDList := []azaci.ContainerGroupSubnetID{{ID: &subnetID}}
cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.SubnetIds = &cgIDList
cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.DNSConfig = p.getDNSConfig(ctx, pod)
cg.ContainerGroupPropertiesWrapper.Extensions = p.containerGroupExtensions
cg.ContainerGroupPropertiesWrapper.ContainerGroupProperties.DNSConfig = getDNSConfig(ctx, pod, pn.KubeDNSIP, clusterDomain)
}

func (p *ACIProvider) getDNSConfig(ctx context.Context, pod *v1.Pod) *azaci.DNSConfiguration {
func getDNSConfig(ctx context.Context, pod *v1.Pod, kubeDNSIP, clusterDomain string) *azaci.DNSConfiguration {
nameServers := make([]string, 0)
searchDomains := make([]string, 0)

if pod.Spec.DNSPolicy == v1.DNSClusterFirst || pod.Spec.DNSPolicy == v1.DNSClusterFirstWithHostNet {
nameServers = append(nameServers, p.kubeDNSIP)
searchDomains = p.generateSearchesForDNSClusterFirst(pod.Spec.DNSConfig, pod)
nameServers = append(nameServers, kubeDNSIP)
searchDomains = generateSearchesForDNSClusterFirst(pod.Spec.DNSConfig, pod, clusterDomain)
}

options := make([]string, 0)

if pod.Spec.DNSConfig != nil {
nameServers = omitDuplicates(append(nameServers, pod.Spec.DNSConfig.Nameservers...))
searchDomains = omitDuplicates(append(searchDomains, pod.Spec.DNSConfig.Searches...))
nameServers = util.OmitDuplicates(append(nameServers, pod.Spec.DNSConfig.Nameservers...))
searchDomains = util.OmitDuplicates(append(searchDomains, pod.Spec.DNSConfig.Searches...))

for _, option := range pod.Spec.DNSConfig.Options {
op := option.Name
Expand All @@ -226,22 +212,21 @@ func (p *ACIProvider) getDNSConfig(ctx context.Context, pod *v1.Pod) *azaci.DNSC
}

// This is taken from the kubelet equivalent - https://github.com/kubernetes/kubernetes/blob/d24fe8a801748953a5c34fd34faa8005c6ad1770/pkg/kubelet/network/dns/dns.go#L141-L151
func (p *ACIProvider) generateSearchesForDNSClusterFirst(dnsConfig *v1.PodDNSConfig, pod *v1.Pod) []string {

func generateSearchesForDNSClusterFirst(dnsConfig *v1.PodDNSConfig, pod *v1.Pod, clusterDomain string) []string {
hostSearch := make([]string, 0)

if dnsConfig != nil {
hostSearch = dnsConfig.Searches
}
if p.clusterDomain == "" {
if clusterDomain == "" {
return hostSearch
}

nsSvcDomain := fmt.Sprintf("%s.svc.%s", pod.Namespace, p.clusterDomain)
svcDomain := fmt.Sprintf("svc.%s", p.clusterDomain)
clusterSearch := []string{nsSvcDomain, svcDomain, p.clusterDomain}
nsSvcDomain := fmt.Sprintf("%s.svc.%s", pod.Namespace, clusterDomain)
svcDomain := fmt.Sprintf("svc.%s", clusterDomain)
clusterSearch := []string{nsSvcDomain, svcDomain, clusterDomain}

return omitDuplicates(append(clusterSearch, hostSearch...))
return util.OmitDuplicates(append(clusterSearch, hostSearch...))
}

// https://github.com/kubernetes/kubernetes/blob/4276ed36282405d026d8072e0ebed4f1da49070d/pkg/kubelet/network/dns/dns.go#L101-L149
Expand Down Expand Up @@ -298,12 +283,3 @@ func formDNSSearchFitsLimits(ctx context.Context, searches []string) string {

return strings.Join(searches, " ")
}

func getProtocol(pro v1.Protocol) azaci.ContainerNetworkProtocol {
switch pro {
case v1.ProtocolUDP:
return azaci.ContainerNetworkProtocolUDP
default:
return azaci.ContainerNetworkProtocolTCP
}
}
Loading