From fdf2e568ae9612f2638132950d7493d28e6cded7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Theresa=20Sch=C3=BCttig?= Date: Wed, 22 May 2024 17:33:40 +0200 Subject: [PATCH 1/2] Add port IDs and additional IPs to agent config These values are used by the OpenStackL3PortManager to create a VRRP setup that is compatible with new versions of the OVN ML2 plugin and need to be added to the configuration with yaook/k8s. Additionally, the openstack config was moved to the config package to avoid cyclic imports. --- .../ch-k8s-lbaas-controller.go | 5 +- internal/config/config.go | 21 ++--- internal/config/openstack_config.go | 76 +++++++++++++++++++ internal/openstack/client.go | 76 +------------------ internal/openstack/port_manager.go | 27 ++++--- 5 files changed, 111 insertions(+), 94 deletions(-) create mode 100644 internal/config/openstack_config.go diff --git a/cmd/ch-k8s-lbaas-controller/ch-k8s-lbaas-controller.go b/cmd/ch-k8s-lbaas-controller/ch-k8s-lbaas-controller.go index 42b98d8..c368c4f 100644 --- a/cmd/ch-k8s-lbaas-controller/ch-k8s-lbaas-controller.go +++ b/cmd/ch-k8s-lbaas-controller/ch-k8s-lbaas-controller.go @@ -20,11 +20,12 @@ package main import ( "flag" "fmt" - "github.com/cloudandheat/ch-k8s-lbaas/internal/static" "net" "net/http" "time" + "github.com/cloudandheat/ch-k8s-lbaas/internal/static" + kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" @@ -86,6 +87,8 @@ func main() { l3portmanager, err = osClient.NewOpenStackL3PortManager( &fileCfg.OpenStack.Networking, + fileCfg.Agents.Agents, + fileCfg.Agents.AdditionalIps, ) if err != nil { klog.Fatalf("Failed to create openstack L3 port manager: %s", err.Error()) diff --git a/internal/config/config.go b/internal/config/config.go index 008ec0a..f82d8ee 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -16,13 +16,12 @@ package config import ( "fmt" - "github.com/cloudandheat/ch-k8s-lbaas/internal/static" "io" "os" - "github.com/BurntSushi/toml" + "github.com/cloudandheat/ch-k8s-lbaas/internal/static" - "github.com/cloudandheat/ch-k8s-lbaas/internal/openstack" + "github.com/BurntSushi/toml" ) type BackendLayer string @@ -41,7 +40,8 @@ const ( ) type Agent struct { - URL string `toml:"url"` + URL string `toml:"url"` + PortId string `toml:"port-id"` } type ServiceConfig struct { @@ -83,9 +83,10 @@ type Nftables struct { } type Agents struct { - SharedSecret string `toml:"shared-secret"` - TokenLifetime int `toml:"token-lifetime"` - Agents []Agent `toml:"agent"` + SharedSecret string `toml:"shared-secret"` + TokenLifetime int `toml:"token-lifetime"` + AdditionalIps []string `toml:"additional-ips"` + Agents []Agent `toml:"agent"` } type ControllerConfig struct { @@ -95,9 +96,9 @@ type ControllerConfig struct { PortManager PortManager `toml:"port-manager"` BackendLayer BackendLayer `toml:"backend-layer"` - OpenStack openstack.Config `toml:"openstack"` - Static static.Config `toml:"static"` - Agents Agents `toml:"agents"` + OpenStack Config `toml:"openstack"` + Static static.Config `toml:"static"` + Agents Agents `toml:"agents"` } type AgentConfig struct { diff --git a/internal/config/openstack_config.go b/internal/config/openstack_config.go new file mode 100644 index 0000000..444305c --- /dev/null +++ b/internal/config/openstack_config.go @@ -0,0 +1,76 @@ +package config + +import ( + "github.com/gophercloud/gophercloud" + "github.com/gophercloud/utils/openstack/clientconfig" + "k8s.io/klog" +) + +type AuthOpts struct { + AuthURL string `toml:"auth-url"` + UserID string `toml:"user-id"` + Username string `toml:"username"` + Password string `toml:"password"` + ProjectID string `toml:"project-id"` + ProjectName string `toml:"project-name"` + TrustID string `toml:"trust-id"` + DomainID string `toml:"domain-id"` + DomainName string `toml:"domain-name"` + ProjectDomainID string `toml:"project-domain-id"` + ProjectDomainName string `toml:"project-domain-name"` + UserDomainID string `toml:"user-domain-id"` + UserDomainName string `toml:"user-domain-name"` + Region string `toml:"region"` + CAFile string `toml:"ca-file"` + TLSInsecure bool `toml:"tls-insecure"` + + ApplicationCredentialID string `toml:"application-credential-id"` + ApplicationCredentialName string `toml:"application-credential-name"` + ApplicationCredentialSecret string `toml:"application-credential-secret"` +} + +type NetworkingOpts struct { + UseFloatingIPs bool `toml:"use-floating-ips"` + FloatingIPNetworkID string `toml:"floating-ip-network-id"` + SubnetID string `toml:"subnet-id"` +} + +type Config struct { + Global AuthOpts `toml:"auth"` + Networking NetworkingOpts `toml:"network"` +} + +func (cfg AuthOpts) ToAuthOptions() gophercloud.AuthOptions { + opts := clientconfig.ClientOpts{ + // this is needed to disable the clientconfig.AuthOptions func env detection + EnvPrefix: "_", + AuthInfo: &clientconfig.AuthInfo{ + AuthURL: cfg.AuthURL, + UserID: cfg.UserID, + Username: cfg.Username, + Password: cfg.Password, + ProjectID: cfg.ProjectID, + ProjectName: cfg.ProjectName, + DomainID: cfg.DomainID, + DomainName: cfg.DomainName, + ProjectDomainID: cfg.ProjectDomainID, + ProjectDomainName: cfg.ProjectDomainName, + UserDomainID: cfg.UserDomainID, + UserDomainName: cfg.UserDomainName, + ApplicationCredentialID: cfg.ApplicationCredentialID, + ApplicationCredentialName: cfg.ApplicationCredentialName, + ApplicationCredentialSecret: cfg.ApplicationCredentialSecret, + }, + } + + ao, err := clientconfig.AuthOptions(&opts) + if err != nil { + klog.V(1).Infof("Error parsing auth: %s", err) + return gophercloud.AuthOptions{} + } + + // Persistent service, so we need to be able to renew tokens. + ao.AllowReauth = true + + return *ao +} diff --git a/internal/openstack/client.go b/internal/openstack/client.go index dc20320..a79c30b 100644 --- a/internal/openstack/client.go +++ b/internal/openstack/client.go @@ -20,92 +20,22 @@ import ( "fmt" "net/http" + "github.com/cloudandheat/ch-k8s-lbaas/internal/config" "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack" tokens3 "github.com/gophercloud/gophercloud/openstack/identity/v3/tokens" - "github.com/gophercloud/utils/openstack/clientconfig" netutil "k8s.io/apimachinery/pkg/util/net" certutil "k8s.io/client-go/util/cert" - "k8s.io/klog" ) -type AuthOpts struct { - AuthURL string `toml:"auth-url"` - UserID string `toml:"user-id"` - Username string `toml:"username"` - Password string `toml:"password"` - ProjectID string `toml:"project-id"` - ProjectName string `toml:"project-name"` - TrustID string `toml:"trust-id"` - DomainID string `toml:"domain-id"` - DomainName string `toml:"domain-name"` - ProjectDomainID string `toml:"project-domain-id"` - ProjectDomainName string `toml:"project-domain-name"` - UserDomainID string `toml:"user-domain-id"` - UserDomainName string `toml:"user-domain-name"` - Region string `toml:"region"` - CAFile string `toml:"ca-file"` - TLSInsecure bool `toml:"tls-insecure"` - - ApplicationCredentialID string `toml:"application-credential-id"` - ApplicationCredentialName string `toml:"application-credential-name"` - ApplicationCredentialSecret string `toml:"application-credential-secret"` -} - -type NetworkingOpts struct { - UseFloatingIPs bool `toml:"use-floating-ips"` - FloatingIPNetworkID string `toml:"floating-ip-network-id"` - SubnetID string `toml:"subnet-id"` -} - -type Config struct { - Global AuthOpts `toml:"auth"` - Networking NetworkingOpts `toml:"network"` -} - type OpenStackClient struct { provider *gophercloud.ProviderClient region string projectID string } -func (cfg AuthOpts) ToAuthOptions() gophercloud.AuthOptions { - opts := clientconfig.ClientOpts{ - // this is needed to disable the clientconfig.AuthOptions func env detection - EnvPrefix: "_", - AuthInfo: &clientconfig.AuthInfo{ - AuthURL: cfg.AuthURL, - UserID: cfg.UserID, - Username: cfg.Username, - Password: cfg.Password, - ProjectID: cfg.ProjectID, - ProjectName: cfg.ProjectName, - DomainID: cfg.DomainID, - DomainName: cfg.DomainName, - ProjectDomainID: cfg.ProjectDomainID, - ProjectDomainName: cfg.ProjectDomainName, - UserDomainID: cfg.UserDomainID, - UserDomainName: cfg.UserDomainName, - ApplicationCredentialID: cfg.ApplicationCredentialID, - ApplicationCredentialName: cfg.ApplicationCredentialName, - ApplicationCredentialSecret: cfg.ApplicationCredentialSecret, - }, - } - - ao, err := clientconfig.AuthOptions(&opts) - if err != nil { - klog.V(1).Infof("Error parsing auth: %s", err) - return gophercloud.AuthOptions{} - } - - // Persistent service, so we need to be able to renew tokens. - ao.AllowReauth = true - - return *ao -} - -func NewProviderClient(cfg *AuthOpts) (*gophercloud.ProviderClient, error) { +func NewProviderClient(cfg *config.AuthOpts) (*gophercloud.ProviderClient, error) { provider, err := openstack.NewClient(cfg.AuthURL) if err != nil { return nil, err @@ -139,7 +69,7 @@ func NewProviderClient(cfg *AuthOpts) (*gophercloud.ProviderClient, error) { return provider, err } -func NewClient(cfg *AuthOpts) (*OpenStackClient, error) { +func NewClient(cfg *config.AuthOpts) (*OpenStackClient, error) { provider, err := NewProviderClient(cfg) if err != nil { return nil, err diff --git a/internal/openstack/port_manager.go b/internal/openstack/port_manager.go index 53d394c..fb6055b 100644 --- a/internal/openstack/port_manager.go +++ b/internal/openstack/port_manager.go @@ -16,6 +16,8 @@ package openstack import ( "errors" + + "github.com/cloudandheat/ch-k8s-lbaas/internal/config" "github.com/gophercloud/gophercloud" tags "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/attributestags" floatingipsv2 "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" @@ -61,14 +63,17 @@ func (opts CustomCreateOpts) ToPortCreateMap() (map[string]interface{}, error) { } type OpenStackL3PortManager struct { - client *gophercloud.ServiceClient - networkID string - projectID string - cfg *NetworkingOpts - ports PortClient + client *gophercloud.ServiceClient + networkID string + projectID string + cfg *config.NetworkingOpts + additionalAddressPairs []string + agents []config.Agent + ports PortClient } -func (client *OpenStackClient) NewOpenStackL3PortManager(networkConfig *NetworkingOpts) (*OpenStackL3PortManager, error) { +func (client *OpenStackClient) NewOpenStackL3PortManager(networkConfig *config.NetworkingOpts, agents []config.Agent, additionalAddressPairs []string) (*OpenStackL3PortManager, error) { + networkingclient, err := client.NewNetworkV2() if err != nil { return nil, err @@ -82,10 +87,12 @@ func (client *OpenStackClient) NewOpenStackL3PortManager(networkConfig *Networki networkID := subnet.NetworkID return &OpenStackL3PortManager{ - client: networkingclient, - cfg: networkConfig, - networkID: networkID, - projectID: client.projectID, + client: networkingclient, + cfg: networkConfig, + networkID: networkID, + projectID: client.projectID, + additionalAddressPairs: additionalAddressPairs, + agents: agents, ports: NewPortClient( networkingclient, TagLBManagedPort, From 17b761eb06d95b031f165d8e7e4dc4db6c23fc51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Theresa=20Sch=C3=BCttig?= Date: Mon, 27 May 2024 17:36:38 +0200 Subject: [PATCH 2/2] Create VRRP setup for OVN-based OpenStack To setup VRRP with new versions of the OVN ML2 plugin, additional IP addresses of the agent configuration (at the moment of this commit only the IP address of the VIP port is used) and all fixed IP addresses of all L3 ports need to be added to the allowed address pairs of each agent node. This is done by overwriting the address pairs accordingly after the provisioning or deletion of a L3 port as well as periodically during the entire runtime of the k8s-lbaas using the EnsureAgentsStateJob. The periodic job is used to ensure the setup will still be created after an agent was unreachable. As a part of this commit functions provided by the package openstack/networking/v2/ports were wrapped into the PortClient to allow better testability. --- internal/config/config.go | 2 +- internal/config/openstack_config.go | 14 +++ internal/controller/controller.go | 6 ++ internal/controller/port_manager.go | 2 + internal/controller/worker.go | 14 +++ internal/openstack/port_manager.go | 86 +++++++++++++-- internal/openstack/port_manager_test.go | 137 ++++++++++++++++++++++++ internal/openstack/ports.go | 15 +++ internal/openstack/testing/mock.go | 38 +++++++ internal/static/port_manager.go | 7 +- 10 files changed, 312 insertions(+), 9 deletions(-) create mode 100644 internal/openstack/port_manager_test.go diff --git a/internal/config/config.go b/internal/config/config.go index f82d8ee..a43826b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -85,7 +85,7 @@ type Nftables struct { type Agents struct { SharedSecret string `toml:"shared-secret"` TokenLifetime int `toml:"token-lifetime"` - AdditionalIps []string `toml:"additional-ips"` + AdditionalIps []string `toml:"additional-address-pairs"` Agents []Agent `toml:"agent"` } diff --git a/internal/config/openstack_config.go b/internal/config/openstack_config.go index 444305c..0bbe37e 100644 --- a/internal/config/openstack_config.go +++ b/internal/config/openstack_config.go @@ -1,3 +1,17 @@ +/* Copyright 2020 CLOUD&HEAT Technologies GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package config import ( diff --git a/internal/controller/controller.go b/internal/controller/controller.go index e74ef1c..0a5f2c7 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -228,6 +228,8 @@ func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) error { // happens only after three sync intervals (900 seconds). go wait.Until(c.periodicCleanup, 907*time.Second, stopCh) + go wait.Until(c.ensureAgentsState, 300*time.Second, stopCh) + klog.Info("Started workers") <-stopCh klog.Info("Shutting down workers") @@ -253,6 +255,10 @@ func (c *Controller) periodicCleanup() { } } +func (c *Controller) ensureAgentsState() { + c.worker.EnqueueJob(&EnsureAgentsStateJob{}) +} + // handleObject will take any resource implementing metav1.Object and attempt // to find the Foo resource that 'owns' it. It does this by looking at the // objects metadata.ownerReferences field for an appropriate OwnerReference. diff --git a/internal/controller/port_manager.go b/internal/controller/port_manager.go index 02d5765..81e1d3e 100644 --- a/internal/controller/port_manager.go +++ b/internal/controller/port_manager.go @@ -5,6 +5,8 @@ type L3PortManager interface { ProvisionPort() (string, error) // CleanUnusedPorts deletes all L3 ports that are currently not used CleanUnusedPorts(usedPorts []string) error + // EnsureAgentsState ensures that all agents are configured correctly + EnsureAgentsState() error // GetAvailablePorts returns all L3 ports that are available GetAvailablePorts() ([]string, error) // GetExternalAddress returns the external address (floating IP) and hostname for a given portID diff --git a/internal/controller/worker.go b/internal/controller/worker.go index ab373df..5f6bc26 100644 --- a/internal/controller/worker.go +++ b/internal/controller/worker.go @@ -510,3 +510,17 @@ func (j *UpdateConfigJob) Run(w *Worker) (RequeueMode, error) { func (j *UpdateConfigJob) ToString() string { return "UpdateConfigJob" } + +type EnsureAgentsStateJob struct{} + +func (j *EnsureAgentsStateJob) Run(w *Worker) (RequeueMode, error) { + err := w.l3portmanager.EnsureAgentsState() + if err != nil { + return RequeueTail, err + } + return Drop, nil +} + +func (j *EnsureAgentsStateJob) ToString() string { + return "CheckAgentsJob" +} diff --git a/internal/openstack/port_manager.go b/internal/openstack/port_manager.go index fb6055b..99bc864 100644 --- a/internal/openstack/port_manager.go +++ b/internal/openstack/port_manager.go @@ -16,6 +16,7 @@ package openstack import ( "errors" + "sync" "github.com/cloudandheat/ch-k8s-lbaas/internal/config" "github.com/gophercloud/gophercloud" @@ -37,6 +38,7 @@ var ( ErrFixedIPMissing = errors.New("Port has no IP address assigned") ErrPortIsNil = errors.New("Port is nil") ErrNoFloatingIPCreated = errors.New("No floating IP was created by OpenStack") + ErrVRRPSetupFailed = errors.New("Failed to update address pairs of all agents") ) // We need options which are not included in the default gophercloud struct @@ -159,7 +161,7 @@ func (pm *OpenStackL3PortManager) CheckPortExists(portID string) (bool, error) { } func (pm *OpenStackL3PortManager) ProvisionPort() (string, error) { - port, err := portsv2.Create( + port, err := pm.ports.Create( pm.client, CustomCreateOpts{ NetworkID: pm.networkID, @@ -169,7 +171,7 @@ func (pm *OpenStackL3PortManager) ProvisionPort() (string, error) { }, PortSecurityEnabled: boolPtr(false), }, - ).Extract() + ) // XXX: this is meh because we can only set the tag after the port was // created. If we get killed between the previous line and setting the // tag, the port will linger, unusedly. @@ -180,8 +182,7 @@ func (pm *OpenStackL3PortManager) ProvisionPort() (string, error) { } cleanupPort := func() { - klog.Infof("Deleting port %v", port.ID) - deleteErr := portsv2.Delete(pm.client, port.ID).ExtractErr() + deleteErr := pm.deletePort(port.ID) if deleteErr != nil { klog.Warningf( "resource leak: could not delete dysfunctional port %q: %s:", @@ -200,7 +201,7 @@ func (pm *OpenStackL3PortManager) ProvisionPort() (string, error) { } if pm.cfg.UseFloatingIPs { - err = pm.provisionFloatingIP(port.ID) + err := pm.provisionFloatingIP(port.ID) if err != nil { klog.Warningf("Couldn't provide floating ip for port=%v: %s", port.ID, err) cleanupPort() @@ -208,6 +209,11 @@ func (pm *OpenStackL3PortManager) ProvisionPort() (string, error) { } } + err = pm.EnsureAgentsState() + if err != nil { + klog.Warningf("VRRP setup for port=%v failed during provisioning: %s", port.ID, err) + } + return port.ID, nil } @@ -269,9 +275,8 @@ func (pm *OpenStackL3PortManager) CleanUnusedPorts(usedPorts []string) error { continue } - klog.Infof("Trying to delete port %q", port.ID) // port not in use, issue deletion - err := portsv2.Delete(pm.client, port.ID).ExtractErr() + err := pm.deletePort(port.ID) if err != nil { klog.Warningf("Failed to delete unused port %q: %s. The operation will be retried later.", port.ID, err) } @@ -299,6 +304,61 @@ func (pm *OpenStackL3PortManager) GetAvailablePorts() ([]string, error) { return result, nil } +// Ensures that all fixed IPs of L3 ports as well as additional configured IPs +// are configured as allowed address pair of all agent nodes. Should be run periodically +// to ensure a correct setup in case an agent was unresponsive earlier +func (pm *OpenStackL3PortManager) EnsureAgentsState() error { + ports, err := pm.ports.GetPorts() + if err != nil { + klog.Warningf("Failed to get L3 ports during VRRP setup: %s", err) + return err + } + + addressPairs := []portsv2.AddressPair{} + + for _, ip := range pm.additionalAddressPairs { + addressPairs = append(addressPairs, portsv2.AddressPair{IPAddress: ip}) + } + + for _, port := range ports { + if len(port.FixedIPs) == 0 { + klog.Warningf("L3 port %v has no fixed IPs. Skipping this port during VRRP setup.", port.ID) + } + + for _, ip := range port.FixedIPs { + addressPairs = append(addressPairs, portsv2.AddressPair{IPAddress: ip.IPAddress}) + } + } + + allSucceeded := true + + var wg sync.WaitGroup + for i := range pm.agents { + wg.Add(1) + go func(agent *config.Agent) { + _, err := pm.ports.Update( + pm.client, + agent.PortId, + portsv2.UpdateOpts{ + AllowedAddressPairs: &addressPairs, + }, + ) + if err != nil { + klog.Warningf("Failed to configure VRRP address pairs for agent port %v : %s", agent.PortId, err) + allSucceeded = false + } + defer wg.Done() + }(&pm.agents[i]) + } + + wg.Wait() + if !allSucceeded { + return ErrVRRPSetupFailed + } + + return nil +} + func (pm *OpenStackL3PortManager) GetExternalAddress(portID string) (string, string, error) { port, fip, err := pm.ports.GetPortByID(portID) if err != nil { @@ -341,3 +401,15 @@ func (pm *OpenStackL3PortManager) GetInternalAddress(portID string) (string, err return port.FixedIPs[0].IPAddress, nil } + +func (pm *OpenStackL3PortManager) deletePort(portID string) error { + klog.Infof("Trying to delete port %q", portID) + + err := pm.ports.Delete(pm.client, portID).ExtractErr() + + if err == nil { + pm.EnsureAgentsState() + } + + return err +} diff --git a/internal/openstack/port_manager_test.go b/internal/openstack/port_manager_test.go new file mode 100644 index 0000000..1b9e481 --- /dev/null +++ b/internal/openstack/port_manager_test.go @@ -0,0 +1,137 @@ +package openstack + +import ( + "errors" + "fmt" + "testing" + + "github.com/cloudandheat/ch-k8s-lbaas/internal/config" + portsv2 "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + + ostesting "github.com/cloudandheat/ch-k8s-lbaas/internal/openstack/testing" +) + +type fixture struct { + t *testing.T + + pm *OpenStackL3PortManager + client *ostesting.MockPortClient + agents []config.Agent + expectedAddressPairs []portsv2.AddressPair + l3Ports []portsv2.Port +} + +func newFixture(t *testing.T) *fixture { + f := &fixture{ + t: t, + client: &ostesting.MockPortClient{}, + } + portIDs := []string{"gw-1-port-id", "gw-2-port-id", "gw-3-port-id"} + + f.agents = make([]config.Agent, len(portIDs)) + for i, portID := range portIDs { + f.agents[i] = config.Agent{PortId: portID} + } + + f.pm = &OpenStackL3PortManager{ + agents: f.agents, + ports: f.client, + cfg: &config.NetworkingOpts{}, + } + + // add sample setup for agents and L3 ports + l3Ip1 := "10.0.0.1" + l3Ip2 := "10.0.0.2" + l3Ip3 := "10.0.0.3" + additionalIp1 := "10.0.0.4" + additionalIp2 := "10.0.0.5" + + f.pm.additionalAddressPairs = []string{additionalIp1, additionalIp2} + + f.expectedAddressPairs = []portsv2.AddressPair{ + {IPAddress: l3Ip1}, + {IPAddress: l3Ip2}, + {IPAddress: l3Ip3}, + {IPAddress: additionalIp1}, + {IPAddress: additionalIp2}, + } + + f.l3Ports = []portsv2.Port{ + {FixedIPs: []portsv2.IP{{IPAddress: l3Ip1}}}, + {FixedIPs: []portsv2.IP{{IPAddress: l3Ip2}, {IPAddress: l3Ip3}}}, + } + + return f +} + +func getMatchIpFn(expectedAddressPairs []portsv2.AddressPair) func(opts portsv2.UpdateOpts) bool { + matchIpsFn := func(opts portsv2.UpdateOpts) bool { + addedIps := map[string]bool{} + for _, ip := range expectedAddressPairs { + addedIps[ip.IPAddress] = false + } + + for _, ap := range *opts.AllowedAddressPairs { + if _, expected := addedIps[ap.IPAddress]; !expected { + fmt.Println(fmt.Errorf("Update call was invoked with unexpected ip address %v in to allowed address pairs.", ap.IPAddress)) + return false + } + addedIps[ap.IPAddress] = true + } + + for ip, added := range addedIps { + if !added { + fmt.Println(fmt.Errorf("Did not add ip address to allowed address pairs %v", ip)) + return false + } + } + + return true + } + return matchIpsFn +} + +func TestEnsureAgentsStateUpdateAddressPairsCorrectly(t *testing.T) { + f := newFixture(t) + + f.client.On("GetPorts").Return(f.l3Ports, nil).Times(1) + + for _, agent := range f.agents { + f.client.On("Update", mock.Anything, agent.PortId, mock.MatchedBy(getMatchIpFn(f.expectedAddressPairs))).Return(&portsv2.Port{}, nil).Times(1) + } + + err := f.pm.EnsureAgentsState() + assert.Nil(t, err) + f.client.AssertExpectations(t) +} + +func TestEnsureAgentsStateReturnsErrorIfPortsCannotBeFetched(t *testing.T) { + f := newFixture(t) + + f.client.On("GetPorts").Return([]portsv2.Port{}, errors.New("")) + err := f.pm.EnsureAgentsState() + assert.NotNil(t, err) + + f.client.AssertExpectations(t) +} + +func TestEnsureAgentsStateReturnsErrorIfUpdateFails(t *testing.T) { + f := newFixture(t) + + f.client.On("GetPorts").Return(f.l3Ports, nil).Times(1) + + for i, agent := range f.agents { + var returnErr error = nil + if i == 0 { + returnErr = errors.New("") + } + + f.client.On("Update", mock.Anything, agent.PortId, mock.MatchedBy(getMatchIpFn(f.expectedAddressPairs))).Return(&portsv2.Port{}, returnErr).Times(1) + } + + err := f.pm.EnsureAgentsState() + assert.NotNil(t, err) + f.client.AssertExpectations(t) +} diff --git a/internal/openstack/ports.go b/internal/openstack/ports.go index 8d5b1d8..d8e11d9 100644 --- a/internal/openstack/ports.go +++ b/internal/openstack/ports.go @@ -36,8 +36,11 @@ type UncachedClient struct { } type PortClient interface { + Create(c *gophercloud.ServiceClient, opts portsv2.CreateOptsBuilder) (*portsv2.Port, error) GetPorts() ([]portsv2.Port, error) GetPortByID(ID string) (*portsv2.Port, *floatingipsv2.FloatingIP, error) + Update(c *gophercloud.ServiceClient, id string, opts portsv2.UpdateOptsBuilder) (*portsv2.Port, error) + Delete(c *gophercloud.ServiceClient, id string) portsv2.DeleteResult } func NewPortClient(networkingclient *gophercloud.ServiceClient, tag string, useFloatingIPs bool, projectID string) *UncachedClient { @@ -49,6 +52,10 @@ func NewPortClient(networkingclient *gophercloud.ServiceClient, tag string, useF } } +func (pc *UncachedClient) Create(c *gophercloud.ServiceClient, opts portsv2.CreateOptsBuilder) (*portsv2.Port, error) { + return portsv2.Create(c, opts).Extract() +} + func (pc *UncachedClient) GetPorts() (ports []portsv2.Port, err error) { err = portsv2.List( pc.client, @@ -102,3 +109,11 @@ func (pc *UncachedClient) GetPortByID(ID string) (port *portsv2.Port, fip *float } return port, fip, nil } + +func (pc *UncachedClient) Update(c *gophercloud.ServiceClient, id string, opts portsv2.UpdateOptsBuilder) (*portsv2.Port, error) { + return portsv2.Update(c, id, opts).Extract() +} + +func (pc *UncachedClient) Delete(c *gophercloud.ServiceClient, id string) (r portsv2.DeleteResult) { + return portsv2.Delete(c, id) +} diff --git a/internal/openstack/testing/mock.go b/internal/openstack/testing/mock.go index 8204d73..71018e2 100644 --- a/internal/openstack/testing/mock.go +++ b/internal/openstack/testing/mock.go @@ -15,7 +15,11 @@ package testing import ( + "github.com/gophercloud/gophercloud" "github.com/stretchr/testify/mock" + + floatingipsv2 "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" + portsv2 "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" ) // TODO: use mockery @@ -24,6 +28,10 @@ type MockL3PortManager struct { mock.Mock } +type MockPortClient struct { + mock.Mock +} + func NewMockL3PortManager() *MockL3PortManager { return new(MockL3PortManager) } @@ -43,6 +51,11 @@ func (m *MockL3PortManager) CleanUnusedPorts(usedPorts []string) error { return a.Error(0) } +func (m *MockL3PortManager) EnsureAgentsState() error { + a := m.Called() + return a.Error(0) +} + func (m *MockL3PortManager) GetAvailablePorts() ([]string, error) { a := m.Called() return a.Get(0).([]string), a.Error(1) @@ -57,3 +70,28 @@ func (m *MockL3PortManager) GetInternalAddress(portID string) (string, error) { a := m.Called(portID) return a.String(0), a.Error(1) } + +func (mpc *MockPortClient) Create(c *gophercloud.ServiceClient, opts portsv2.CreateOptsBuilder) (*portsv2.Port, error) { + a := mpc.Called(c, opts) + return a.Get(0).(*portsv2.Port), a.Error(1) +} + +func (mpc *MockPortClient) GetPorts() ([]portsv2.Port, error) { + a := mpc.Called() + return a.Get(0).([]portsv2.Port), a.Error(1) +} + +func (mpc *MockPortClient) GetPortByID(ID string) (*portsv2.Port, *floatingipsv2.FloatingIP, error) { + a := mpc.Called(ID) + return a.Get(0).(*portsv2.Port), a.Get(1).(*floatingipsv2.FloatingIP), a.Error(2) +} + +func (mpc *MockPortClient) Update(c *gophercloud.ServiceClient, id string, opts portsv2.UpdateOptsBuilder) (*portsv2.Port, error) { + a := mpc.Called(c, id, opts) + return a.Get(0).(*portsv2.Port), a.Error(1) +} + +func (mpc *MockPortClient) Delete(c *gophercloud.ServiceClient, id string) (r portsv2.DeleteResult) { + a := mpc.Called(c, id) + return a.Get(0).(portsv2.DeleteResult) +} diff --git a/internal/static/port_manager.go b/internal/static/port_manager.go index 3cc39f0..06ab161 100644 --- a/internal/static/port_manager.go +++ b/internal/static/port_manager.go @@ -2,8 +2,9 @@ package static import ( "fmt" - "golang.org/x/exp/slices" "net/netip" + + "golang.org/x/exp/slices" ) type Config struct { @@ -45,6 +46,10 @@ func (pm *StaticL3PortManager) CleanUnusedPorts(usedPorts []string) error { return nil } +func (pm *StaticL3PortManager) EnsureAgentsState() error { + return nil +} + func (pm *StaticL3PortManager) GetAvailablePorts() ([]string, error) { var ports []string