Merge pull request #54 from tschuettig/fix-ovn-based-clouds

Fix incompatibility with OVN-based clouds

cmd/ch-k8s-lbaas-controller/ch-k8s-lbaas-controller.go

@@ -20,11 +20,12 @@ package main
 import (
 	"flag"
 	"fmt"
-	"github.com/cloudandheat/ch-k8s-lbaas/internal/static"
 	"net"
 	"net/http"
 	"time"
 
+	"github.com/cloudandheat/ch-k8s-lbaas/internal/static"
+
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
@@ -86,6 +87,8 @@ func main() {
 
 		l3portmanager, err = osClient.NewOpenStackL3PortManager(
 			&fileCfg.OpenStack.Networking,
+			fileCfg.Agents.Agents,
+			fileCfg.Agents.AdditionalIps,
 		)
 		if err != nil {
 			klog.Fatalf("Failed to create openstack L3 port manager: %s", err.Error())

internal/config/config.go

@@ -16,13 +16,12 @@ package config
 
 import (
 	"fmt"
-	"github.com/cloudandheat/ch-k8s-lbaas/internal/static"
 	"io"
 	"os"
 
-	"github.com/BurntSushi/toml"
+	"github.com/cloudandheat/ch-k8s-lbaas/internal/static"
 
-	"github.com/cloudandheat/ch-k8s-lbaas/internal/openstack"
+	"github.com/BurntSushi/toml"
 )
 
 type BackendLayer string
@@ -41,7 +40,8 @@ const (
 )
 
 type Agent struct {
-	URL string `toml:"url"`
+	URL    string `toml:"url"`
+	PortId string `toml:"port-id"`
 }
 
 type ServiceConfig struct {
@@ -83,9 +83,10 @@ type Nftables struct {
 }
 
 type Agents struct {
-	SharedSecret  string  `toml:"shared-secret"`
-	TokenLifetime int     `toml:"token-lifetime"`
-	Agents        []Agent `toml:"agent"`
+	SharedSecret  string   `toml:"shared-secret"`
+	TokenLifetime int      `toml:"token-lifetime"`
+	AdditionalIps []string `toml:"additional-address-pairs"`
+	Agents        []Agent  `toml:"agent"`
 }
 
 type ControllerConfig struct {
@@ -95,9 +96,9 @@ type ControllerConfig struct {
 	PortManager  PortManager  `toml:"port-manager"`
 	BackendLayer BackendLayer `toml:"backend-layer"`
 
-	OpenStack openstack.Config `toml:"openstack"`
-	Static    static.Config    `toml:"static"`
-	Agents    Agents           `toml:"agents"`
+	OpenStack Config        `toml:"openstack"`
+	Static    static.Config `toml:"static"`
+	Agents    Agents        `toml:"agents"`
 }
 
 type AgentConfig struct {

internal/config/openstack_config.go

@@ -0,0 +1,90 @@
+/* Copyright 2020 CLOUD&HEAT Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package config
+
+import (
+	"github.com/gophercloud/gophercloud"
+	"github.com/gophercloud/utils/openstack/clientconfig"
+	"k8s.io/klog"
+)
+
+type AuthOpts struct {
+	AuthURL           string `toml:"auth-url"`
+	UserID            string `toml:"user-id"`
+	Username          string `toml:"username"`
+	Password          string `toml:"password"`
+	ProjectID         string `toml:"project-id"`
+	ProjectName       string `toml:"project-name"`
+	TrustID           string `toml:"trust-id"`
+	DomainID          string `toml:"domain-id"`
+	DomainName        string `toml:"domain-name"`
+	ProjectDomainID   string `toml:"project-domain-id"`
+	ProjectDomainName string `toml:"project-domain-name"`
+	UserDomainID      string `toml:"user-domain-id"`
+	UserDomainName    string `toml:"user-domain-name"`
+	Region            string `toml:"region"`
+	CAFile            string `toml:"ca-file"`
+	TLSInsecure       bool   `toml:"tls-insecure"`
+
+	ApplicationCredentialID     string `toml:"application-credential-id"`
+	ApplicationCredentialName   string `toml:"application-credential-name"`
+	ApplicationCredentialSecret string `toml:"application-credential-secret"`
+}
+
+type NetworkingOpts struct {
+	UseFloatingIPs      bool   `toml:"use-floating-ips"`
+	FloatingIPNetworkID string `toml:"floating-ip-network-id"`
+	SubnetID            string `toml:"subnet-id"`
+}
+
+type Config struct {
+	Global     AuthOpts       `toml:"auth"`
+	Networking NetworkingOpts `toml:"network"`
+}
+
+func (cfg AuthOpts) ToAuthOptions() gophercloud.AuthOptions {
+	opts := clientconfig.ClientOpts{
+		// this is needed to disable the clientconfig.AuthOptions func env detection
+		EnvPrefix: "_",
+		AuthInfo: &clientconfig.AuthInfo{
+			AuthURL:                     cfg.AuthURL,
+			UserID:                      cfg.UserID,
+			Username:                    cfg.Username,
+			Password:                    cfg.Password,
+			ProjectID:                   cfg.ProjectID,
+			ProjectName:                 cfg.ProjectName,
+			DomainID:                    cfg.DomainID,
+			DomainName:                  cfg.DomainName,
+			ProjectDomainID:             cfg.ProjectDomainID,
+			ProjectDomainName:           cfg.ProjectDomainName,
+			UserDomainID:                cfg.UserDomainID,
+			UserDomainName:              cfg.UserDomainName,
+			ApplicationCredentialID:     cfg.ApplicationCredentialID,
+			ApplicationCredentialName:   cfg.ApplicationCredentialName,
+			ApplicationCredentialSecret: cfg.ApplicationCredentialSecret,
+		},
+	}
+
+	ao, err := clientconfig.AuthOptions(&opts)
+	if err != nil {
+		klog.V(1).Infof("Error parsing auth: %s", err)
+		return gophercloud.AuthOptions{}
+	}
+
+	// Persistent service, so we need to be able to renew tokens.
+	ao.AllowReauth = true
+
+	return *ao
+}

internal/controller/controller.go

@@ -228,6 +228,8 @@ func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) error {
 	// happens only after three sync intervals (900 seconds).
 	go wait.Until(c.periodicCleanup, 907*time.Second, stopCh)
 
+	go wait.Until(c.ensureAgentsState, 300*time.Second, stopCh)
+
 	klog.Info("Started workers")
 	<-stopCh
 	klog.Info("Shutting down workers")
@@ -253,6 +255,10 @@ func (c *Controller) periodicCleanup() {
 	}
 }
 
+func (c *Controller) ensureAgentsState() {
+	c.worker.EnqueueJob(&EnsureAgentsStateJob{})
+}
+
 // handleObject will take any resource implementing metav1.Object and attempt
 // to find the Foo resource that 'owns' it. It does this by looking at the
 // objects metadata.ownerReferences field for an appropriate OwnerReference.

internal/controller/port_manager.go

@@ -5,6 +5,8 @@ type L3PortManager interface {
 	ProvisionPort() (string, error)
 	// CleanUnusedPorts deletes all L3 ports that are currently not used
 	CleanUnusedPorts(usedPorts []string) error
+	// EnsureAgentsState ensures that all agents are configured correctly
+	EnsureAgentsState() error
 	// GetAvailablePorts returns all L3 ports that are available
 	GetAvailablePorts() ([]string, error)
 	// GetExternalAddress returns the external address (floating IP) and hostname for a given portID

internal/controller/worker.go

@@ -510,3 +510,17 @@ func (j *UpdateConfigJob) Run(w *Worker) (RequeueMode, error) {
 func (j *UpdateConfigJob) ToString() string {
 	return "UpdateConfigJob"
 }
+
+type EnsureAgentsStateJob struct{}
+
+func (j *EnsureAgentsStateJob) Run(w *Worker) (RequeueMode, error) {
+	err := w.l3portmanager.EnsureAgentsState()
+	if err != nil {
+		return RequeueTail, err
+	}
+	return Drop, nil
+}
+
+func (j *EnsureAgentsStateJob) ToString() string {
+	return "CheckAgentsJob"
+}

internal/openstack/client.go

@@ -20,92 +20,22 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/cloudandheat/ch-k8s-lbaas/internal/config"
 	"github.com/gophercloud/gophercloud"
 	"github.com/gophercloud/gophercloud/openstack"
 	tokens3 "github.com/gophercloud/gophercloud/openstack/identity/v3/tokens"
-	"github.com/gophercloud/utils/openstack/clientconfig"
 
 	netutil "k8s.io/apimachinery/pkg/util/net"
 	certutil "k8s.io/client-go/util/cert"
-	"k8s.io/klog"
 )
 
-type AuthOpts struct {
-	AuthURL           string `toml:"auth-url"`
-	UserID            string `toml:"user-id"`
-	Username          string `toml:"username"`
-	Password          string `toml:"password"`
-	ProjectID         string `toml:"project-id"`
-	ProjectName       string `toml:"project-name"`
-	TrustID           string `toml:"trust-id"`
-	DomainID          string `toml:"domain-id"`
-	DomainName        string `toml:"domain-name"`
-	ProjectDomainID   string `toml:"project-domain-id"`
-	ProjectDomainName string `toml:"project-domain-name"`
-	UserDomainID      string `toml:"user-domain-id"`
-	UserDomainName    string `toml:"user-domain-name"`
-	Region            string `toml:"region"`
-	CAFile            string `toml:"ca-file"`
-	TLSInsecure       bool   `toml:"tls-insecure"`
-
-	ApplicationCredentialID     string `toml:"application-credential-id"`
-	ApplicationCredentialName   string `toml:"application-credential-name"`
-	ApplicationCredentialSecret string `toml:"application-credential-secret"`
-}
-
-type NetworkingOpts struct {
-	UseFloatingIPs      bool   `toml:"use-floating-ips"`
-	FloatingIPNetworkID string `toml:"floating-ip-network-id"`
-	SubnetID            string `toml:"subnet-id"`
-}
-
-type Config struct {
-	Global     AuthOpts       `toml:"auth"`
-	Networking NetworkingOpts `toml:"network"`
-}
-
 type OpenStackClient struct {
 	provider  *gophercloud.ProviderClient
 	region    string
 	projectID string
 }
 
-func (cfg AuthOpts) ToAuthOptions() gophercloud.AuthOptions {
-	opts := clientconfig.ClientOpts{
-		// this is needed to disable the clientconfig.AuthOptions func env detection
-		EnvPrefix: "_",
-		AuthInfo: &clientconfig.AuthInfo{
-			AuthURL:                     cfg.AuthURL,
-			UserID:                      cfg.UserID,
-			Username:                    cfg.Username,
-			Password:                    cfg.Password,
-			ProjectID:                   cfg.ProjectID,
-			ProjectName:                 cfg.ProjectName,
-			DomainID:                    cfg.DomainID,
-			DomainName:                  cfg.DomainName,
-			ProjectDomainID:             cfg.ProjectDomainID,
-			ProjectDomainName:           cfg.ProjectDomainName,
-			UserDomainID:                cfg.UserDomainID,
-			UserDomainName:              cfg.UserDomainName,
-			ApplicationCredentialID:     cfg.ApplicationCredentialID,
-			ApplicationCredentialName:   cfg.ApplicationCredentialName,
-			ApplicationCredentialSecret: cfg.ApplicationCredentialSecret,
-		},
-	}
-
-	ao, err := clientconfig.AuthOptions(&opts)
-	if err != nil {
-		klog.V(1).Infof("Error parsing auth: %s", err)
-		return gophercloud.AuthOptions{}
-	}
-
-	// Persistent service, so we need to be able to renew tokens.
-	ao.AllowReauth = true
-
-	return *ao
-}
-
-func NewProviderClient(cfg *AuthOpts) (*gophercloud.ProviderClient, error) {
+func NewProviderClient(cfg *config.AuthOpts) (*gophercloud.ProviderClient, error) {
 	provider, err := openstack.NewClient(cfg.AuthURL)
 	if err != nil {
 		return nil, err
@@ -139,7 +69,7 @@ func NewProviderClient(cfg *AuthOpts) (*gophercloud.ProviderClient, error) {
 	return provider, err
 }
 
-func NewClient(cfg *AuthOpts) (*OpenStackClient, error) {
+func NewClient(cfg *config.AuthOpts) (*OpenStackClient, error) {
 	provider, err := NewProviderClient(cfg)
 	if err != nil {
 		return nil, err